searxng/manage
Ivan Gabaldon b1ffeb00ac
add test job
Needed to test the workflows in a separate fork so unrelated triggers appear in files.

Run tests before container release, if there are any problems the job fails and the images are not released. At the moment this is basic, but it should cover the most serious issues that may appear.

This will prevent fiascos like https://github.com/searxng/searxng/issues/4718 to ever happen again (hopefully)
2025-05-08 11:44:20 +02:00

662 lines
20 KiB
Bash
Executable File

#!/usr/bin/env bash
# -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
# SPDX-License-Identifier: AGPL-3.0-or-later
# shellcheck disable=SC2034
main_cmd="$(basename "$0")"
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
# shellcheck source=utils/lib.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"
# shellcheck source=utils/lib_sxng_data.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_data.sh"
# shellcheck source=utils/lib_sxng_weblate.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_weblate.sh"
# shellcheck source=utils/lib_sxng_static.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_static.sh"
# shellcheck source=utils/lib_sxng_node.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_node.sh"
# shellcheck source=utils/lib_sxng_themes.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_themes.sh"
# shellcheck source=utils/lib_sxng_test.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_test.sh"
# shellcheck source=utils/lib_go.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
# shellcheck source=utils/lib_redis.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_redis.sh"
# shellcheck source=utils/lib_sxng_vite.sh
source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_sxng_vite.sh"
PATH="${REPO_ROOT}/node_modules/.bin:${PATH}"
# config
PYOBJECTS="searx"
PY_SETUP_EXTRAS='[test]'
GECKODRIVER_VERSION="v0.35.0"
# SPHINXOPTS=
BLACK_OPTIONS=("--target-version" "py311" "--line-length" "120" "--skip-string-normalization")
BLACK_TARGETS=("--exclude" "(searx/static|searx/languages.py)" "--include" 'searxng.msg|\.pyi?$' "searx" "searxng_extra" "tests")
_dev_redis_sock="/usr/local/searxng-redis/run/redis.sock"
# set SEARXNG_REDIS_URL if it is not defined and "{_dev_redis_sock}" exists.
if [ -S "${_dev_redis_sock}" ] && [ -z "${SEARXNG_REDIS_URL}" ]; then
export SEARXNG_REDIS_URL="unix://${_dev_redis_sock}?db=0"
fi
YAMLLINT_FILES=()
while IFS= read -r line; do
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
YAMLLINT_FILES+=("$line")
fi
done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
RST_FILES=(
'README.rst'
)
help() {
nvm.help
cat <<EOF
webapp.:
run : run developer instance
docs.:
html : build HTML documentation
live : autobuild HTML documentation while editing
gh-pages : deploy on gh-pages branch
prebuild : build reST include files (./${DOCS_BUILD}/includes)
clean : clean documentation build
container. :
build : build container image
buildx : build container image with BuildKit
gecko.driver:
download & install geckodriver if not already installed (required for
robot_tests)
redis:
build : build redis binaries at $(redis._get_dist)
install : create user (${REDIS_USER}) and install systemd service (${REDIS_SERVICE_NAME})
help : show more redis commands
py.:
build : Build python packages at ./${PYDIST}
clean : delete virtualenv and intermediate py files
pyenv.:
install : developer install of SearXNG into virtualenv
uninstall : uninstall developer installation
cmd ... : run command ... in virtualenv
OK : test if virtualenv is OK
format.:
python : format Python code source using black
ci.:
container.:
build : build container image
release : push container images to remote registry
EOF
go.help
node.help
weblate.help
data.help
test.help
themes.help
static.help
vite.help
cat <<EOF
environment ...
SEARXNG_REDIS_URL : ${SEARXNG_REDIS_URL}
EOF
}
if [ "$VERBOSE" = "1" ]; then
SPHINX_VERBOSE="-v"
PYLINT_VERBOSE="-v"
fi
# needed by sphinx-docs
export DOCS_BUILD
webapp.run() {
local parent_proc="$$"
(
if [ "${LIVE_THEME}" ]; then
(themes.live "${LIVE_THEME}")
kill $parent_proc
fi
) &
(
sleep 3
xdg-open http://127.0.0.1:8888/
) &
SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
}
# Alias
docker.buildx() {
container.buildx
}
# Alias
docker.build() {
container.build
}
container.buildx() {
container.build buildx
}
container.build() {
pyenv.install
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local container_engine
local dockerfile
local arch
local variant
local platform
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
dockerfile="Dockerfile"
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
dockerfile="Dockerfile"
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
dockerfile="Dockerfile.compat"
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# Check if podman or docker is installed
if command -v podman &>/dev/null; then
container_engine="podman"
elif command -v docker &>/dev/null; then
container_engine="docker"
else
die 1 "podman/docker is not installed"
fi
build_msg CONTAINER "Engine: $container_engine"
# Check if git is installed
if ! command -v git &>/dev/null; then
die 1 "git is not installed"
fi
(
set -e
pyenv.activate
# Check if it is a git repository
if [ ! -d .git ]; then
die 1 "This is not Git repository"
fi
if ! git remote get-url origin 2>/dev/null; then
die 1 "there is no remote origin"
fi
# This is a git repository
git update-index -q --refresh
python -m searx.version freeze
eval "$(python -m searx.version)"
# Get the last git commit id
version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
build_msg CONTAINER "Last commit: $version_gitcommit"
if [ "$container_engine" = "docker" ]; then
if [ "$1" = "buildx" ]; then
docker_builder="buildx build"
else
docker_builder="build"
warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback"
fi
params_build_builder="$docker_builder --platform=$platform --target=builder"
params_build="$docker_builder --platform=$platform --squash"
else
params_build_builder="build --format=docker --platform=$platform --target=builder --layers --identity-label=false"
params_build="build --format=docker --platform=$platform --layers --squash-all --omit-history --identity-label=false"
fi
# Define container image org/name
# shellcheck disable=SC2001
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
build_msg CONTAINER "Building..."
# shellcheck disable=SC2086
"$container_engine" $params_build_builder \
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
--build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
--tag="localhost/$container_image_organization/$container_image_name:builder" \
--file="./$dockerfile"
# shellcheck disable=SC2086
"$container_engine" $params_build \
--build-arg="GIT_URL=$GIT_URL" \
--build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
--build-arg="LABEL_VCS_URL=$GIT_URL" \
--tag="localhost/$container_image_organization/$container_image_name:latest" \
--tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \
--file="./$dockerfile"
)
dump_return $?
}
# shellcheck disable=SC2119
gecko.driver() {
pyenv.install
build_msg INSTALL "gecko.driver"
# run installation in a subprocess and activate pyenv
(
set -e
pyenv.activate
INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
set +e
if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
build_msg INSTALL "geckodriver already installed"
return
fi
PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
case "$PLATFORM" in
"linux 32bit" | "linux2 32bit") ARCH="linux32" ;;
"linux 64bit" | "linux2 64bit") ARCH="linux64" ;;
"windows 32 bit") ARCH="win32" ;;
"windows 64 bit") ARCH="win64" ;;
"mac 64bit") ARCH="macos" ;;
esac
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"
build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
FILE="$(mktemp)"
wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
rm -- "$FILE"
chmod 755 -- "${PY_ENV_BIN}/geckodriver"
)
dump_return $?
}
py.build() {
build_msg BUILD "python package ${PYDIST}"
pyenv.cmd python setup.py \
sdist -d "${PYDIST}" \
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
}
py.clean() {
build_msg CLEAN pyenv
(
set -e
pyenv.drop
[ "$VERBOSE" = "1" ] && set -x
rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
find . -name '*.pyc' -exec rm -f {} +
find . -name '*.pyo' -exec rm -f {} +
find . -name __pycache__ -exec rm -rf {} +
)
}
pyenv.check() {
cat <<EOF
import yaml
print('import yaml --> OK')
EOF
}
pyenv.install() {
if ! pyenv.OK; then
py.clean >/dev/null
fi
if pyenv.install.OK >/dev/null; then
return 0
fi
(
set -e
pyenv
build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'"
"${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}"
)
local exit_val=$?
if [ ! $exit_val -eq 0 ]; then
die 42 "error while pip install (${PY_ENV_BIN})"
fi
}
pyenv.uninstall() {
build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
pyenv.cmd python setup.py develop --uninstall 2>&1 |
prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
}
format.python() {
build_msg TEST "[format.python] black \$BLACK_TARGETS"
pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
dump_return $?
}
docs.prebuild() {
build_msg DOCS "build ${DOCS_BUILD}/includes"
(
set -e
[ "$VERBOSE" = "1" ] && set -x
mkdir -p "${DOCS_BUILD}/includes"
./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst"
pyenv.cmd searxng_extra/docs_prebuild
)
dump_return $?
}
ci.container.build() {
if ! "$GITHUB_ACTIONS"; then
die 1 "This command is intended to be run in GitHub Actions"
fi
pyenv.install
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local dockerfile
local arch
local variant
local platform
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
dockerfile="Dockerfile"
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
dockerfile="Dockerfile"
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
dockerfile="Dockerfile.compat"
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# Check if podman is installed
if ! command -v podman &>/dev/null; then
die 1 "podman is not installed"
fi
# Check if git is installed
if ! command -v git &>/dev/null; then
die 1 "git is not installed"
fi
(
set -e
pyenv.activate
# Check if it is a git repository
if [ ! -d .git ]; then
die 1 "This is not Git repository"
fi
if ! git remote get-url origin 2>/dev/null; then
die 1 "there is no remote origin"
fi
# This is a git repository
git update-index -q --refresh
python -m searx.version freeze
eval "$(python -m searx.version)"
# TODO: Remove this
echo $VERSION_STRING
echo $VERSION_TAG
echo $DOCKER_TAG
echo $GIT_URL
echo $GIT_BRANCH
# Get the last git commit id
version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
build_msg CONTAINER "Last commit: $version_gitcommit"
# Define container image org/name
# shellcheck disable=SC2001
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
build_msg CONTAINER "Building..."
podman build --format=docker --platform="$platform" --target=builder --layers --identity-label=false \
--cache-from="ghcr.io/$container_image_organization/cache" \
--cache-to="ghcr.io/$container_image_organization/cache" \
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
--build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
--tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \
--file="./$dockerfile"
podman build --format=docker --platform="$platform" --layers --squash-all --omit-history --identity-label=false \
--cache-from="ghcr.io/$container_image_organization/cache" \
--cache-to="ghcr.io/$container_image_organization/cache" \
--build-arg="GIT_URL=$GIT_URL" \
--build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
--build-arg="LABEL_VCS_URL=$GIT_URL" \
--tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \
--file="./$dockerfile"
podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant"
)
dump_return $?
}
ci.container.test() {
if ! "$GITHUB_ACTIONS"; then
die 1 "This command is intended to be run in GitHub Actions"
fi
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local arch
local variant
local platform
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# Check if podman is installed
if ! command -v podman &>/dev/null; then
die 1 "podman is not installed"
fi
(
set -e
# Define container image org/name
git_url="$(git remote get-url origin)"
# shellcheck disable=SC2001
container_image_organization="$(echo "$git_url" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
podman pull "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant"
name="$container_image_name-$(date +%N)"
podman create --name="$name" --rm --timeout=60 --network="host" \
"ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" >/dev/null
podman start "$name" >/dev/null
podman logs -f "$name" &
pid_logs=$!
# Wait until container is ready
sleep 5
# TODO: Test failing
curl -vf --max-time 5 "http://localhost:8080/healthz"
kill $pid_logs &>/dev/null || true
podman stop "$name" >/dev/null
)
dump_return $?
}
ci.container.push() {
if ! "$GITHUB_ACTIONS"; then
die 1 "This command is intended to be run in GitHub Actions"
fi
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local arch
local variant
local platform
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# Check if podman is installed
if ! command -v podman &>/dev/null; then
die 1 "podman is not installed"
fi
(
set -e
# Define container image org/name
git_url="$(git remote get-url origin)"
# shellcheck disable=SC2001
container_image_organization="$(echo "$git_url" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant"
# Get version tag
container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \
"ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant")
# Recreate tags
podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest"
podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
podman push "docker.io/$container_image_organization/$container_image_name:latest"
podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
)
dump_return $?
}
# shellcheck disable=SC2119
main() {
local _type
local cmd="$1"
shift
if [ "$cmd" == "" ]; then
help
err_msg "missing command"
return 42
fi
case "$cmd" in
--getenv)
var="$1"
echo "${!var}"
;;
--help) help ;;
--*)
help
err_msg "unknown option $cmd"
return 42
;;
*)
_type="$(type -t "$cmd")"
if [ "$_type" != 'function' ]; then
err_msg "unknown command: $cmd / use --help"
return 42
else
"$cmd" "$@"
fi
;;
esac
}
main "$@"