Initial update

This commit is contained in:
Ivan Gabaldon 2025-05-02 13:39:47 +02:00
parent bc06b1aece
commit 9b8517ac8f
No known key found for this signature in database
GPG Key ID: 075587C93FA67582
8 changed files with 539 additions and 191 deletions

View File

@ -1,5 +1,7 @@
name: "Checker"
on: # yamllint disable-line rule:truthy
# yamllint disable-line rule:truthy
on:
schedule:
- cron: "0 4 * * 5"
workflow_dispatch:

127
.github/workflows/container.yml vendored Normal file
View File

@ -0,0 +1,127 @@
---
name: Container
# yamllint disable-line rule:truthy
on:
workflow_run:
workflows:
- Integration
types:
- completed
# TODO: Uncomment
# branches:
# - master
permissions:
contents: read
# Organization GHCR
packages: read
env:
PYTHON_VERSION: "3.13"
jobs:
build:
if: github.event.workflow_run.conclusion == 'success'
name: Build (${{ matrix.arch }})
runs-on: ${{ matrix.os }}
strategy:
matrix:
include:
- arch: amd64
os: ubuntu-24.04
- arch:
arm64
armv7
os: ubuntu-24.04-arm
permissions:
# Organization GHCR
packages: write
steps:
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@v4
with:
persist-credentials: "false"
- name: Setup cache Python
uses: actions/cache@v4
with:
key: "python-${{ env.PYTHON_VERSION }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: "python-${{ env.PYTHON_VERSION }}-"
path: "./local"
- name: Setup cache container mounts
uses: actions/cache@v4
with:
key: "container-mounts-${{ hashFiles('./Dockerfile') }}"
restore-keys: "container-mounts-"
path: |
/var/tmp/buildah-cache/
/var/tmp/buildah-cache-*/
- name: Setup QEMU
uses: docker/setup-qemu-action@v3
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Build
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
run: make -e GIT_URL=$(git remote get-url origin) ci.container.build
release:
# TODO: Uncomment before merge
# if: github.repository_owner == 'searxng'
if: false
name: Release (${{ matrix.arch }})
runs-on: ubuntu-24.04-arm
needs: build
strategy:
matrix:
arch:
- amd64
- arm64
- armv7
steps:
- if: env.DOCKERHUB_USERNAME != ''
name: Checkout
uses: actions/checkout@v4
with:
persist-credentials: "false"
# make sure "make ci.container.push" can get the git history
fetch-depth: "0"
- if: env.DOCKERHUB_USERNAME != ''
name: Login to GHCR
uses: docker/login-action@v3
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- if: env.DOCKERHUB_USERNAME != ''
name: Login to Docker Hub
uses: docker/login-action@v3
with:
registry: "docker.io"
username: "${{ env.DOCKERHUB_USERNAME }}"
password: "${{ secrets.DOCKERHUB_TOKEN }}"
- if: env.DOCKERHUB_USERNAME != ''
name: Release
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
run: make -e GIT_URL=$(git remote get-url origin) ci.container.push

View File

@ -1,8 +1,10 @@
name: "Update searx.data"
on: # yamllint disable-line rule:truthy
name: Update searx.data
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
schedule:
- cron: "59 23 28 * *"
workflow_dispatch:
jobs:
updateData:

View File

@ -1,70 +1,96 @@
name: Integration
on: # yamllint disable-line rule:truthy
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
push:
branches: ["master"]
branches:
- master
pull_request:
branches: ["master"]
branches:
- master
permissions:
contents: read
jobs:
python:
# TODO: Remove this
if: false
name: Python ${{ matrix.python-version }}
runs-on: ubuntu-24.04
strategy:
matrix:
os: [ubuntu-24.04]
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
python-version:
- "3.9"
- "3.10"
- "3.11"
- "3.12"
- "3.13"
steps:
- name: Checkout
uses: actions/checkout@v4
with:
persist-credentials: "false"
- name: Install Ubuntu packages
run: |
sudo ./utils/searxng.sh install packages
run: sudo ./utils/searxng.sh install packages
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
architecture: 'x64'
python-version: "${{ matrix.python-version }}"
- name: Run tests
run: make V=1 ci.test
themes:
# TODO: Remove this
if: false
name: Themes
runs-on: ubuntu-24.04
steps:
- name: Checkout
uses: actions/checkout@v4
with:
persist-credentials: "false"
- name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
architecture: 'x64'
python-version: "3.13"
- name: Build themes
run: make themes.all
documentation:
# TODO: Remove this
if: false
name: Documentation
runs-on: ubuntu-24.04
permissions:
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo
# for JamesIves/github-pages-deploy-action to push
contents: write
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: '0'
persist-credentials: false
persist-credentials: "false"
fetch-depth: "0"
- name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
architecture: 'x64'
python-version: "3.13"
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v4
@ -73,42 +99,50 @@ jobs:
./local
./.nvm
./node_modules
key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
key: python-ubuntu-24.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: Build documentation
run: |
make V=1 docs.clean docs.html
- name: Deploy
if: github.ref == 'refs/heads/master'
run: make V=1 docs.clean docs.html
- if: github.ref == 'refs/heads/master'
name: Deploy
uses: JamesIves/github-pages-deploy-action@3.7.1
with:
GITHUB_TOKEN: ${{ github.token }}
BRANCH: gh-pages
FOLDER: dist/docs
CLEAN: true # Automatically remove deleted files from the deploy branch
SINGLE_COMMIT: true
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
GITHUB_TOKEN: "${{ github.token }}"
BRANCH: "gh-pages"
FOLDER: "dist/docs"
# Automatically remove deleted files from the deploy branch
CLEAN: "true"
SINGLE_COMMIT: "true"
COMMIT_MESSAGE: "[doc] build from commit ${{ github.sha }}"
babel:
# TODO: Remove this
if: false
# if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master'
name: Update translations branch
runs-on: ubuntu-24.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
needs:
- python
- themes
- documentation
permissions:
contents: write # for make V=1 weblate.push.translations
# for make V=1 weblate.push.translations
contents: write
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: '0'
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
fetch-depth: "0"
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
architecture: 'x64'
python-version: "3.13"
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v4
@ -117,63 +151,17 @@ jobs:
./local
./.nvm
./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
key: python-ubuntu-20.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: weblate & git setup
env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
WEBLATE_CONFIG: "${{ secrets.WEBLATE_CONFIG }}"
run: |
mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot"
- name: Update transations
id: update
run: |
make V=1 weblate.push.translations
dockers:
name: Docker
if: github.ref == 'refs/heads/master'
needs:
- python
- themes
- documentation
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
runs-on: ubuntu-24.04
steps:
- name: Checkout
if: env.DOCKERHUB_USERNAME != null
uses: actions/checkout@v4
with:
# make sure "make docker.push" can get the git history
fetch-depth: '0'
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v4
with:
path: |
./local
./.nvm
./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: Set up QEMU
if: env.DOCKERHUB_USERNAME != null
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
if: env.DOCKERHUB_USERNAME != null
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
if: env.DOCKERHUB_USERNAME != null
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
if: env.DOCKERHUB_USERNAME != null
run: make -e GIT_URL=$(git remote get-url origin) docker.buildx
run: make V=1 weblate.push.translations

View File

@ -1,8 +1,10 @@
name: "Security checks"
on: # yamllint disable-line rule:truthy
name: Security checks
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
schedule:
- cron: "42 05 * * *"
workflow_dispatch:
jobs:
dockers:

View File

@ -1,8 +1,10 @@
name: "Update translations"
on: # yamllint disable-line rule:truthy
name: Update translations
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
schedule:
- cron: "05 07 * * 5"
workflow_dispatch:
jobs:
babel:

View File

@ -77,7 +77,8 @@ test.shell:
MANAGE += weblate.translations.commit weblate.push.translations
MANAGE += data.all data.traits data.useragents data.locales data.currencies
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
MANAGE += docker.build docker.push docker.buildx
MANAGE += docker.build docker.buildx
MANAGE += container.build container.buildx
MANAGE += gecko.driver
MANAGE += node.env node.env.dev node.clean
MANAGE += py.build py.clean
@ -87,6 +88,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover
MANAGE += themes.all themes.fix themes.test
MANAGE += static.build.commit static.build.drop static.build.restore
MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
MANAGE += ci.container.build ci.container.push
PHONY += $(MANAGE)

415
manage
View File

@ -60,7 +60,7 @@ while IFS= read -r line; do
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
YAMLLINT_FILES+=("$line")
fi
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
RST_FILES=(
'README.rst'
@ -77,9 +77,9 @@ docs.:
gh-pages : deploy on gh-pages branch
prebuild : build reST include files (./${DOCS_BUILD}/includes)
clean : clean documentation build
docker.:
build : build docker image
push : build and push docker image
container. :
build : build container image
buildx : build container image with BuildKit
gecko.driver:
download & install geckodriver if not already installed (required for
robot_tests)
@ -97,6 +97,10 @@ pyenv.:
OK : test if virtualenv is OK
format.:
python : format Python code source using black
ci.:
container.:
build : build container image
release : push container images to remote registry
EOF
go.help
node.help
@ -112,7 +116,6 @@ environment ...
EOF
}
if [ "$VERBOSE" = "1" ]; then
SPHINX_VERBOSE="-v"
PYLINT_VERBOSE="-v"
@ -125,53 +128,95 @@ webapp.run() {
local parent_proc="$$"
(
if [ "${LIVE_THEME}" ]; then
( themes.live "${LIVE_THEME}" )
(themes.live "${LIVE_THEME}")
kill $parent_proc
fi
)&
) &
(
sleep 3
xdg-open http://127.0.0.1:8888/
)&
) &
SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
}
docker.push() {
docker.build push
}
# Alias
docker.buildx() {
docker.build buildx
container.buildx
}
# shellcheck disable=SC2119
# Alias
docker.build() {
container.build
}
container.buildx() {
container.build buildx
}
container.build() {
pyenv.install
local SEARXNG_GIT_VERSION
local VERSION_GITCOMMIT
local GITHUB_USER
local SEARXNG_IMAGE_NAME
local BUILD
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local container_engine
local dockerfile
local arch
local variant
local platform
build_msg DOCKER build
# run installation in a subprocess and activate pyenv
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
dockerfile="Dockerfile"
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
dockerfile="Dockerfile"
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
# TODO: Move ARMv7 to a separated Dockerfile
dockerfile="Dockerfile"
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# See https://www.shellcheck.net/wiki/SC1001 and others ..
# shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
( set -e
# Check if podman or docker is installed
if command -v podman &>/dev/null; then
container_engine="podman"
elif command -v docker &>/dev/null; then
container_engine="docker"
else
die 1 "podman/docker is not installed"
fi
build_msg CONTAINER "Engine: $container_engine"
# Check if git is installed
if ! command -v git &>/dev/null; then
die 1 "git is not installed"
fi
(
set -e
pyenv.activate
# Check if it is a git repository
if [ ! -d .git ]; then
die 1 "This is not Git repository"
fi
if [ ! -x "$(which git)" ]; then
die 1 "git is not installed"
die 1 "This is not Git repository"
fi
if ! git remote get-url origin 2> /dev/null; then
die 1 "there is no remote origin"
if ! git remote get-url origin 2>/dev/null; then
die 1 "there is no remote origin"
fi
# This is a git repository
@ -180,42 +225,48 @@ docker.build() {
eval "$(python -m searx.version)"
# Get the last git commit id
VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2)
build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
build_msg CONTAINER "Last commit: $version_gitcommit"
# define the docker image name
GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
if [ "$container_engine" = "docker" ]; then
if [ "$1" = "buildx" ]; then
docker_builder="buildx build"
else
docker_builder="build"
warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback"
fi
BUILD="build"
if [ "$1" = "buildx" ]; then
# buildx includes the push option
CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache"
BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
shift
params_build_builder="$docker_builder --platform=$platform --target=builder"
params_build="$docker_builder --platform=$platform --squash"
else
params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false"
params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false"
fi
build_msg DOCKER "Build command: ${BUILD}"
# build Docker image
build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
# Define container image org/name
# shellcheck disable=SC2001
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
build_msg CONTAINER "Building..."
# shellcheck disable=SC2086
docker $BUILD \
--build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
--build-arg GIT_URL="${GIT_URL}" \
--build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \
--build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
--build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
--build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
--build-arg LABEL_VCS_URL="${GIT_URL}" \
--build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
--build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
-t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" .
"$container_engine" $params_build_builder \
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
--build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
--tag="localhost/$container_image_organization/$container_image_name:builder" \
--file="./$dockerfile"
if [ "$1" = "push" ]; then
docker push "${SEARXNG_IMAGE_NAME}:latest"
docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}"
fi
# shellcheck disable=SC2086
"$container_engine" $params_build \
--build-arg="GIT_URL=$GIT_URL" \
--build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
--build-arg="LABEL_VCS_URL=$GIT_URL" \
--tag="localhost/$container_image_organization/$container_image_name:latest" \
--tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \
--file="./$dockerfile"
)
dump_return $?
}
@ -226,10 +277,11 @@ gecko.driver() {
build_msg INSTALL "gecko.driver"
# run installation in a subprocess and activate pyenv
( set -e
(
set -e
pyenv.activate
INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
set +e
if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
build_msg INSTALL "geckodriver already installed"
@ -237,13 +289,13 @@ gecko.driver() {
fi
PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
case "$PLATFORM" in
"linux 32bit" | "linux2 32bit") ARCH="linux32";;
"linux 64bit" | "linux2 64bit") ARCH="linux64";;
"windows 32 bit") ARCH="win32";;
"windows 64 bit") ARCH="win64";;
"mac 64bit") ARCH="macos";;
"linux 32bit" | "linux2 32bit") ARCH="linux32" ;;
"linux 64bit" | "linux2 64bit") ARCH="linux64" ;;
"windows 32 bit") ARCH="win32" ;;
"windows 64 bit") ARCH="win64" ;;
"mac 64bit") ARCH="macos" ;;
esac
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"
build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
@ -258,13 +310,14 @@ gecko.driver() {
py.build() {
build_msg BUILD "python package ${PYDIST}"
pyenv.cmd python setup.py \
sdist -d "${PYDIST}" \
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
sdist -d "${PYDIST}" \
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
}
py.clean() {
build_msg CLEAN pyenv
( set -e
(
set -e
pyenv.drop
[ "$VERBOSE" = "1" ] && set -x
rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
@ -275,22 +328,22 @@ py.clean() {
}
pyenv.check() {
cat <<EOF
cat <<EOF
import yaml
print('import yaml --> OK')
EOF
}
pyenv.install() {
if ! pyenv.OK; then
py.clean > /dev/null
py.clean >/dev/null
fi
if pyenv.install.OK > /dev/null; then
if pyenv.install.OK >/dev/null; then
return 0
fi
( set -e
(
set -e
pyenv
build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'"
"${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}"
@ -303,8 +356,8 @@ pyenv.install() {
pyenv.uninstall() {
build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
pyenv.cmd python setup.py develop --uninstall 2>&1 \
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
pyenv.cmd python setup.py develop --uninstall 2>&1 |
prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
}
@ -320,17 +373,184 @@ docs.prebuild() {
set -e
[ "$VERBOSE" = "1" ] && set -x
mkdir -p "${DOCS_BUILD}/includes"
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst"
pyenv.cmd searxng_extra/docs_prebuild
)
dump_return $?
}
ci.container.build() {
if ! "$GITHUB_ACTIONS"; then
die 1 "This command is intended to be run in GitHub Actions"
fi
pyenv.install
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local dockerfile
local arch
local variant
local platform
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
dockerfile="Dockerfile"
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
dockerfile="Dockerfile"
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
# TODO: Move ARMv7 to a separated Dockerfile
dockerfile="Dockerfile"
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# Check if podman is installed
if ! command -v podman &>/dev/null; then
die 1 "podman is not installed"
fi
# Check if git is installed
if ! command -v git &>/dev/null; then
die 1 "git is not installed"
fi
(
set -eu
pyenv.activate
# Check if it is a git repository
if [ ! -d .git ]; then
die 1 "This is not Git repository"
fi
if ! git remote get-url origin 2>/dev/null; then
die 1 "there is no remote origin"
fi
# This is a git repository
git update-index -q --refresh
python -m searx.version freeze
eval "$(python -m searx.version)"
# Get the last git commit id
version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
build_msg CONTAINER "Last commit: $version_gitcommit"
# Define container image org/name
# shellcheck disable=SC2001
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
build_msg CONTAINER "Building..."
podman build --platform="$platform" --target=builder --layers --identity-label=false \
--cache-from="ghcr.io/$container_image_organization/cache" \
--cache-to="ghcr.io/$container_image_organization/cache" \
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
--build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
--tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \
--file="./$dockerfile"
podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \
--cache-from="ghcr.io/$container_image_organization/cache" \
--cache-to="ghcr.io/$container_image_organization/cache" \
--build-arg="GIT_URL=$GIT_URL" \
--build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
--build-arg="LABEL_VCS_URL=$GIT_URL" \
--tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \
--file="./$dockerfile"
podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant"
)
dump_return $?
}
ci.container.push() {
if ! "$GITHUB_ACTIONS"; then
die 1 "This command is intended to be run in GitHub Actions"
fi
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local arch
local variant
local platform
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# Check if podman is installed
if ! command -v podman &>/dev/null; then
die 1 "podman is not installed"
fi
(
set -eu
# Define container image org/name
# shellcheck disable=SC2001
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant"
# Get version tag
container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \
"ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant")
# Recreate tags
podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest"
podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
podman push "docker.io/$container_image_organization/$container_image_name:latest"
podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
)
dump_return $?
}
# shellcheck disable=SC2119
main() {
local _type
local cmd="$1"; shift
local cmd="$1"
shift
if [ "$cmd" == "" ]; then
help
@ -339,22 +559,25 @@ main() {
fi
case "$cmd" in
--getenv) var="$1"; echo "${!var}";;
--help) help;;
--*)
help
err_msg "unknown option $cmd"
--getenv)
var="$1"
echo "${!var}"
;;
--help) help ;;
--*)
help
err_msg "unknown option $cmd"
return 42
;;
*)
_type="$(type -t "$cmd")"
if [ "$_type" != 'function' ]; then
err_msg "unknown command: $cmd / use --help"
return 42
;;
*)
_type="$(type -t "$cmd")"
if [ "$_type" != 'function' ]; then
err_msg "unknown command: $cmd / use --help"
return 42
else
"$cmd" "$@"
fi
;;
else
"$cmd" "$@"
fi
;;
esac
}