searxng/manage

#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later

# shellcheck disable=SC2034
main_cmd="$(basename "$0")"

# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"

# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"

# shellcheck source=utils/lib_sxng_data.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh"

# shellcheck source=utils/lib_sxng_weblate.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_weblate.sh"

# shellcheck source=utils/lib_sxng_static.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_static.sh"

# shellcheck source=utils/lib_sxng_node.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_node.sh"

# shellcheck source=utils/lib_sxng_themes.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"

# shellcheck source=utils/lib_sxng_test.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"

# shellcheck source=utils/lib_go.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"

# shellcheck source=utils/lib_redis.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh"

# shellcheck source=utils/lib_sxng_vite.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh"

PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"

# config

PYOBJECTS="searx"
PY_SETUP_EXTRAS='[test]'
GECKODRIVER_VERSION="v0.35.0"
# SPHINXOPTS=
BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")

_dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
# set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists.
if [ -S "${_dev_redis_sock}" ] && [ -z "${SEARXNG_REDIS_URL}" ]; then
    export SEARXNG_REDIS_URL="unix://${_dev_redis_sock}?db=0"
fi

YAMLLINT_FILES=()
while IFS= read -r line; do
    if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
        YAMLLINT_FILES+=("$line")
    fi
done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"

RST_FILES=(
    'README.rst'
)

help() {
    nvm.help
    cat <<EOF
webapp.:
  run       : run developer instance
docs.:
  html      : build HTML documentation
  live      : autobuild HTML documentation while editing
  gh-pages  : deploy on gh-pages branch
  prebuild  : build reST include files (./${DOCS_BUILD}/includes)
  clean     : clean documentation build
container.  :
  build     : build container image
  buildx    : build container image with BuildKit
gecko.driver:
  download & install geckodriver if not already installed (required for
  robot_tests)
redis:
  build     : build redis binaries at $(redis._get_dist)
  install   : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
  help      : show more redis commands
py.:
  build     : Build python packages at ./${PYDIST}
  clean     : delete virtualenv and intermediate py files
pyenv.:
  install   : developer install of SearXNG into virtualenv
  uninstall : uninstall developer installation
  cmd ...   : run command ... in virtualenv
  OK        : test if virtualenv is OK
format.:
  python    : format Python code source using black
ci.:
  container.:
    build   : build container image
    release : push container images to remote registry
EOF
    go.help
    node.help
    weblate.help
    data.help
    test.help
    themes.help
    static.help
    vite.help
    cat <<EOF
environment ...
  SEARXNG_REDIS_URL : ${SEARXNG_REDIS_URL}
EOF
}

if [ "$VERBOSE" = "1" ]; then
    SPHINX_VERBOSE="-v"
    PYLINT_VERBOSE="-v"
fi

# needed by sphinx-docs
export DOCS_BUILD

webapp.run() {
    local parent_proc="$$"
    (
        if [ "${LIVE_THEME}" ]; then
            (themes.live "${LIVE_THEME}")
            kill $parent_proc
        fi
    ) &
    (
        sleep 3
        xdg-open http://127.0.0.1:8888/
    ) &
    SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
}

# Alias
docker.buildx() {
    container.buildx
}

# Alias
docker.build() {
    container.build
}

container.buildx() {
    container.build buildx
}

container.build() {
    pyenv.install

    local parch=${OVERRIDE_ARCH:-$(uname -m)}
    local container_engine
    local dockerfile
    local arch
    local variant
    local platform

    # Setup arch specific
    case $parch in
    "X64" | "x86_64" | "amd64")
        dockerfile="Dockerfile"
        arch="amd64"
        variant=""
        platform="linux/$arch"
        ;;
    "ARM64" | "aarch64" | "arm64")
        dockerfile="Dockerfile"
        arch="arm64"
        variant=""
        platform="linux/$arch"
        ;;
    "ARMV7" | "armhf" | "armv7l" | "armv7")
        # TODO: Move ARMv7 to a separated Dockerfile
        dockerfile="Dockerfile"
        arch="arm"
        variant="v7"
        platform="linux/$arch/$variant"
        ;;
    *)
        err_msg "Unsupported architecture; (PARCH=\"$parch\")"
        exit 1
        ;;
    esac
    build_msg CONTAINER "Selected platform: $platform"

    # Check if podman or docker is installed
    if command -v podman &>/dev/null; then
        container_engine="podman"
    elif command -v docker &>/dev/null; then
        container_engine="docker"
    else
        die 1 "podman/docker is not installed"
    fi
    build_msg CONTAINER "Engine: $container_engine"

    # Check if git is installed
    if ! command -v git &>/dev/null; then
        die 1 "git is not installed"
    fi

    (
        set -e
        pyenv.activate

        # Check if it is a git repository
        if [ ! -d .git ]; then
            die 1 "This is not Git repository"
        fi

        if ! git remote get-url origin 2>/dev/null; then
            die 1 "there is no remote origin"
        fi

        # This is a git repository
        git update-index -q --refresh
        python -m searx.version freeze
        eval "$(python -m searx.version)"

        # Get the last git commit id
        version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
        build_msg CONTAINER "Last commit: $version_gitcommit"

        if [ "$container_engine" = "docker" ]; then
            if [ "$1" = "buildx" ]; then
                docker_builder="buildx build"
            else
                docker_builder="build"
                warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback"
            fi

            params_build_builder="$docker_builder --platform=$platform --target=builder"
            params_build="$docker_builder --platform=$platform --squash"
        else
            params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false"
            params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false"
        fi

        # Define container image org/name
        # shellcheck disable=SC2001
        container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
        container_image_name="searxng"

        build_msg CONTAINER "Building..."

        # shellcheck disable=SC2086
        "$container_engine" $params_build_builder \
            --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
            --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
            --tag="localhost/$container_image_organization/$container_image_name:builder" \
            --file="./$dockerfile"

        # shellcheck disable=SC2086
        "$container_engine" $params_build \
            --build-arg="GIT_URL=$GIT_URL" \
            --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
            --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
            --build-arg="LABEL_VCS_URL=$GIT_URL" \
            --tag="localhost/$container_image_organization/$container_image_name:latest" \
            --tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \
            --file="./$dockerfile"
    )
    dump_return $?
}

# shellcheck disable=SC2119
gecko.driver() {
    pyenv.install

    build_msg INSTALL "gecko.driver"
    # run installation in a subprocess and activate pyenv
    (
        set -e
        pyenv.activate

        INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
        set +e
        if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
            build_msg INSTALL "geckodriver already installed"
            return
        fi
        PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
        case "$PLATFORM" in
        "linux 32bit" | "linux2 32bit") ARCH="linux32" ;;
        "linux 64bit" | "linux2 64bit") ARCH="linux64" ;;
        "windows 32 bit") ARCH="win32" ;;
        "windows 64 bit") ARCH="win64" ;;
        "mac 64bit") ARCH="macos" ;;
        esac
        GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"

        build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"

        FILE="$(mktemp)"
        wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
        rm -- "$FILE"
        chmod 755 -- "${PY_ENV_BIN}/geckodriver"
    )
    dump_return $?
}

py.build() {
    build_msg BUILD "python package ${PYDIST}"
    pyenv.cmd python setup.py \
        sdist -d "${PYDIST}" \
        bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
}

py.clean() {
    build_msg CLEAN pyenv
    (
        set -e
        pyenv.drop
        [ "$VERBOSE" = "1" ] && set -x
        rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
        find . -name '*.pyc' -exec rm -f {} +
        find . -name '*.pyo' -exec rm -f {} +
        find . -name __pycache__ -exec rm -rf {} +
    )
}

pyenv.check() {
    cat <<EOF
import yaml
print('import yaml --> OK')
EOF
}

pyenv.install() {
    if ! pyenv.OK; then
        py.clean >/dev/null
    fi
    if pyenv.install.OK >/dev/null; then
        return 0
    fi

    (
        set -e
        pyenv
        build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'"
        "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}"
    )
    local exit_val=$?
    if [ ! $exit_val -eq 0 ]; then
        die 42 "error while pip install (${PY_ENV_BIN})"
    fi
}

pyenv.uninstall() {
    build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
    pyenv.cmd python setup.py develop --uninstall 2>&1 |
        prefix_stdout "${_Blue}PYENV     ${_creset}[pyenv.uninstall] "

}

format.python() {
    build_msg TEST "[format.python] black \$BLACK_TARGETS"
    pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
    dump_return $?
}

docs.prebuild() {
    build_msg DOCS "build ${DOCS_BUILD}/includes"
    (
        set -e
        [ "$VERBOSE" = "1" ] && set -x
        mkdir -p "${DOCS_BUILD}/includes"
        ./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst"
        pyenv.cmd searxng_extra/docs_prebuild
    )
    dump_return $?
}

ci.container.build() {
    if ! "$GITHUB_ACTIONS"; then
        die 1 "This command is intended to be run in GitHub Actions"
    fi

    pyenv.install

    local parch=${OVERRIDE_ARCH:-$(uname -m)}
    local dockerfile
    local arch
    local variant
    local platform

    # Setup arch specific
    case $parch in
    "X64" | "x86_64" | "amd64")
        dockerfile="Dockerfile"
        arch="amd64"
        variant=""
        platform="linux/$arch"
        ;;
    "ARM64" | "aarch64" | "arm64")
        dockerfile="Dockerfile"
        arch="arm64"
        variant=""
        platform="linux/$arch"
        ;;
    "ARMV7" | "armhf" | "armv7l" | "armv7")
        # TODO: Move ARMv7 to a separated Dockerfile
        dockerfile="Dockerfile"
        arch="arm"
        variant="v7"
        platform="linux/$arch/$variant"
        ;;
    *)
        err_msg "Unsupported architecture; (PARCH=\"$parch\")"
        exit 1
        ;;
    esac
    build_msg CONTAINER "Selected platform: $platform"

    # Check if podman is installed
    if ! command -v podman &>/dev/null; then
        die 1 "podman is not installed"
    fi

    # Check if git is installed
    if ! command -v git &>/dev/null; then
        die 1 "git is not installed"
    fi

    (
        set -eu
        pyenv.activate

        # Check if it is a git repository
        if [ ! -d .git ]; then
            die 1 "This is not Git repository"
        fi

        if ! git remote get-url origin 2>/dev/null; then
            die 1 "there is no remote origin"
        fi

        # This is a git repository
        git update-index -q --refresh
        python -m searx.version freeze
        eval "$(python -m searx.version)"

        # Get the last git commit id
        version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
        build_msg CONTAINER "Last commit: $version_gitcommit"

        # Define container image org/name
        # shellcheck disable=SC2001
        container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
        container_image_name="searxng"

        build_msg CONTAINER "Building..."

        podman build --platform="$platform" --target=builder --layers --identity-label=false \
            --cache-from="ghcr.io/$container_image_organization/cache" \
            --cache-to="ghcr.io/$container_image_organization/cache" \
            --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
            --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
            --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \
            --file="./$dockerfile"

        podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \
            --cache-from="ghcr.io/$container_image_organization/cache" \
            --cache-to="ghcr.io/$container_image_organization/cache" \
            --build-arg="GIT_URL=$GIT_URL" \
            --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
            --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
            --build-arg="LABEL_VCS_URL=$GIT_URL" \
            --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \
            --file="./$dockerfile"

        podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant"
    )
    dump_return $?
}

ci.container.push() {
    if ! "$GITHUB_ACTIONS"; then
        die 1 "This command is intended to be run in GitHub Actions"
    fi

    local parch=${OVERRIDE_ARCH:-$(uname -m)}
    local arch
    local variant
    local platform

    # Setup arch specific
    case $parch in
    "X64" | "x86_64" | "amd64")
        arch="amd64"
        variant=""
        platform="linux/$arch"
        ;;
    "ARM64" | "aarch64" | "arm64")
        arch="arm64"
        variant=""
        platform="linux/$arch"
        ;;
    "ARMV7" | "armhf" | "armv7l" | "armv7")
        arch="arm"
        variant="v7"
        platform="linux/$arch/$variant"
        ;;
    *)
        err_msg "Unsupported architecture; (PARCH=\"$parch\")"
        exit 1
        ;;
    esac
    build_msg CONTAINER "Selected platform: $platform"

    # Check if podman is installed
    if ! command -v podman &>/dev/null; then
        die 1 "podman is not installed"
    fi

    (
        set -eu

        # Define container image org/name
        # shellcheck disable=SC2001
        container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
        container_image_name="searxng"

        podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant"

        # Get version tag
        container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \
            "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant")

        # Recreate tags
        podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest"
        podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"

        podman push "docker.io/$container_image_organization/$container_image_name:latest"
        podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
    )
    dump_return $?
}

# shellcheck disable=SC2119
main() {
    local _type
    local cmd="$1"
    shift

    if [ "$cmd" == "" ]; then
        help
        err_msg "missing command"
        return 42
    fi

    case "$cmd" in
    --getenv)
        var="$1"
        echo "${!var}"
        ;;
    --help) help ;;
    --*)
        help
        err_msg "unknown option $cmd"
        return 42
        ;;
    *)
        _type="$(type -t "$cmd")"
        if [ "$_type" != 'function' ]; then
            err_msg "unknown command: $cmd / use --help"
            return 42
        else
            "$cmd" "$@"
        fi
        ;;
    esac
}

main "$@"