#!/usr/bin/env bash # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*- # SPDX-License-Identifier: AGPL-3.0-or-later # shellcheck disable=SC2034 main_cmd="$(basename "$0")" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh" # shellcheck source=utils/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh" # shellcheck source=utils/lib_sxng_data.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh" # shellcheck source=utils/lib_sxng_weblate.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_weblate.sh" # shellcheck source=utils/lib_sxng_static.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_static.sh" # shellcheck source=utils/lib_sxng_node.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_node.sh" # shellcheck source=utils/lib_sxng_themes.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh" # shellcheck source=utils/lib_sxng_test.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh" # shellcheck source=utils/lib_go.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh" # shellcheck source=utils/lib_redis.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh" # shellcheck source=utils/lib_sxng_vite.sh source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh" PATH="${REPO_ROOT}/node_modules/.bin:${PATH}" # config PYOBJECTS="searx" PY_SETUP_EXTRAS='[test]' GECKODRIVER_VERSION="v0.35.0" # SPHINXOPTS= BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization") BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests") _dev_redis_sock="/usr/local/searxng-redis/run/redis.sock" # set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists. if [ -S "${_dev_redis_sock}" ] && [ -z "${SEARXNG_REDIS_URL}" ]; then export SEARXNG_REDIS_URL="unix://${_dev_redis_sock}?db=0" fi YAMLLINT_FILES=() while IFS= read -r line; do if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then YAMLLINT_FILES+=("$line") fi done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')" RST_FILES=( 'README.rst' ) help() { nvm.help cat </dev/null; then container_engine="podman" elif command -v docker &>/dev/null; then container_engine="docker" else die 1 "podman/docker is not installed" fi build_msg CONTAINER "Engine: $container_engine" # Check if git is installed if ! command -v git &>/dev/null; then die 1 "git is not installed" fi ( set -e pyenv.activate # Check if it is a git repository if [ ! -d .git ]; then die 1 "This is not Git repository" fi if ! git remote get-url origin 2>/dev/null; then die 1 "there is no remote origin" fi # This is a git repository git update-index -q --refresh python -m searx.version freeze eval "$(python -m searx.version)" # Get the last git commit id version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) build_msg CONTAINER "Last commit: $version_gitcommit" if [ "$container_engine" = "docker" ]; then if [ "$1" = "buildx" ]; then docker_builder="buildx build" else docker_builder="build" warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback" fi params_build_builder="$docker_builder --platform=$platform --target=builder" params_build="$docker_builder --platform=$platform --squash" else params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false" params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false" fi # Define container image org/name # shellcheck disable=SC2001 container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" container_image_name="searxng" build_msg CONTAINER "Building..." # shellcheck disable=SC2086 "$container_engine" $params_build_builder \ --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ --tag="localhost/$container_image_organization/$container_image_name:builder" \ --file="./$dockerfile" # shellcheck disable=SC2086 "$container_engine" $params_build \ --build-arg="GIT_URL=$GIT_URL" \ --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ --build-arg="LABEL_VCS_URL=$GIT_URL" \ --tag="localhost/$container_image_organization/$container_image_name:latest" \ --tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \ --file="./$dockerfile" ) dump_return $? } # shellcheck disable=SC2119 gecko.driver() { pyenv.install build_msg INSTALL "gecko.driver" # run installation in a subprocess and activate pyenv ( set -e pyenv.activate INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" set +e if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then build_msg INSTALL "geckodriver already installed" return fi PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" case "$PLATFORM" in "linux 32bit" | "linux2 32bit") ARCH="linux32" ;; "linux 64bit" | "linux2 64bit") ARCH="linux64" ;; "windows 32 bit") ARCH="win32" ;; "windows 64 bit") ARCH="win64" ;; "mac 64bit") ARCH="macos" ;; esac GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz" build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" FILE="$(mktemp)" wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver rm -- "$FILE" chmod 755 -- "${PY_ENV_BIN}/geckodriver" ) dump_return $? } py.build() { build_msg BUILD "python package ${PYDIST}" pyenv.cmd python setup.py \ sdist -d "${PYDIST}" \ bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" } py.clean() { build_msg CLEAN pyenv ( set -e pyenv.drop [ "$VERBOSE" = "1" ] && set -x rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name __pycache__ -exec rm -rf {} + ) } pyenv.check() { cat < OK') EOF } pyenv.install() { if ! pyenv.OK; then py.clean >/dev/null fi if pyenv.install.OK >/dev/null; then return 0 fi ( set -e pyenv build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'" "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}" ) local exit_val=$? if [ ! $exit_val -eq 0 ]; then die 42 "error while pip install (${PY_ENV_BIN})" fi } pyenv.uninstall() { build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" pyenv.cmd python setup.py develop --uninstall 2>&1 | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " } format.python() { build_msg TEST "[format.python] black \$BLACK_TARGETS" pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}" dump_return $? } docs.prebuild() { build_msg DOCS "build ${DOCS_BUILD}/includes" ( set -e [ "$VERBOSE" = "1" ] && set -x mkdir -p "${DOCS_BUILD}/includes" ./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst" pyenv.cmd searxng_extra/docs_prebuild ) dump_return $? } ci.container.build() { if ! "$GITHUB_ACTIONS"; then die 1 "This command is intended to be run in GitHub Actions" fi pyenv.install local parch=${OVERRIDE_ARCH:-$(uname -m)} local dockerfile local arch local variant local platform # Setup arch specific case $parch in "X64" | "x86_64" | "amd64") dockerfile="Dockerfile" arch="amd64" variant="" platform="linux/$arch" ;; "ARM64" | "aarch64" | "arm64") dockerfile="Dockerfile" arch="arm64" variant="" platform="linux/$arch" ;; "ARMV7" | "armhf" | "armv7l" | "armv7") # TODO: Move ARMv7 to a separated Dockerfile dockerfile="Dockerfile" arch="arm" variant="v7" platform="linux/$arch/$variant" ;; *) err_msg "Unsupported architecture; (PARCH=\"$parch\")" exit 1 ;; esac build_msg CONTAINER "Selected platform: $platform" # Check if podman is installed if ! command -v podman &>/dev/null; then die 1 "podman is not installed" fi # Check if git is installed if ! command -v git &>/dev/null; then die 1 "git is not installed" fi ( set -eu pyenv.activate # Check if it is a git repository if [ ! -d .git ]; then die 1 "This is not Git repository" fi if ! git remote get-url origin 2>/dev/null; then die 1 "there is no remote origin" fi # This is a git repository git update-index -q --refresh python -m searx.version freeze eval "$(python -m searx.version)" # Get the last git commit id version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) build_msg CONTAINER "Last commit: $version_gitcommit" # Define container image org/name # shellcheck disable=SC2001 container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" container_image_name="searxng" build_msg CONTAINER "Building..." podman build --platform="$platform" --target=builder --layers --identity-label=false \ --cache-from="ghcr.io/$container_image_organization/cache" \ --cache-to="ghcr.io/$container_image_organization/cache" \ --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \ --file="./$dockerfile" podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \ --cache-from="ghcr.io/$container_image_organization/cache" \ --cache-to="ghcr.io/$container_image_organization/cache" \ --build-arg="GIT_URL=$GIT_URL" \ --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ --build-arg="LABEL_VCS_URL=$GIT_URL" \ --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \ --file="./$dockerfile" podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" ) dump_return $? } ci.container.push() { if ! "$GITHUB_ACTIONS"; then die 1 "This command is intended to be run in GitHub Actions" fi local parch=${OVERRIDE_ARCH:-$(uname -m)} local arch local variant local platform # Setup arch specific case $parch in "X64" | "x86_64" | "amd64") arch="amd64" variant="" platform="linux/$arch" ;; "ARM64" | "aarch64" | "arm64") arch="arm64" variant="" platform="linux/$arch" ;; "ARMV7" | "armhf" | "armv7l" | "armv7") arch="arm" variant="v7" platform="linux/$arch/$variant" ;; *) err_msg "Unsupported architecture; (PARCH=\"$parch\")" exit 1 ;; esac build_msg CONTAINER "Selected platform: $platform" # Check if podman is installed if ! command -v podman &>/dev/null; then die 1 "podman is not installed" fi ( set -eu # Define container image org/name # shellcheck disable=SC2001 container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" container_image_name="searxng" podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" # Get version tag container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \ "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant") # Recreate tags podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest" podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version" podman push "docker.io/$container_image_organization/$container_image_name:latest" podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version" ) dump_return $? } # shellcheck disable=SC2119 main() { local _type local cmd="$1" shift if [ "$cmd" == "" ]; then help err_msg "missing command" return 42 fi case "$cmd" in --getenv) var="$1" echo "${!var}" ;; --help) help ;; --*) help err_msg "unknown option $cmd" return 42 ;; *) _type="$(type -t "$cmd")" if [ "$_type" != 'function' ]; then err_msg "unknown command: $cmd / use --help" return 42 else "$cmd" "$@" fi ;; esac } main "$@"