Initial update

This commit is contained in:
Ivan Gabaldon 2025-05-02 13:39:47 +02:00
parent bc06b1aece
commit 9b8517ac8f
No known key found for this signature in database
GPG Key ID: 075587C93FA67582
8 changed files with 539 additions and 191 deletions

View File

@ -1,5 +1,7 @@
name: "Checker" name: "Checker"
on: # yamllint disable-line rule:truthy
# yamllint disable-line rule:truthy
on:
schedule: schedule:
- cron: "0 4 * * 5" - cron: "0 4 * * 5"
workflow_dispatch: workflow_dispatch:

127
.github/workflows/container.yml vendored Normal file
View File

@ -0,0 +1,127 @@
---
name: Container
# yamllint disable-line rule:truthy
on:
workflow_run:
workflows:
- Integration
types:
- completed
# TODO: Uncomment
# branches:
# - master
permissions:
contents: read
# Organization GHCR
packages: read
env:
PYTHON_VERSION: "3.13"
jobs:
build:
if: github.event.workflow_run.conclusion == 'success'
name: Build (${{ matrix.arch }})
runs-on: ${{ matrix.os }}
strategy:
matrix:
include:
- arch: amd64
os: ubuntu-24.04
- arch:
arm64
armv7
os: ubuntu-24.04-arm
permissions:
# Organization GHCR
packages: write
steps:
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "${{ env.PYTHON_VERSION }}"
- name: Checkout
uses: actions/checkout@v4
with:
persist-credentials: "false"
- name: Setup cache Python
uses: actions/cache@v4
with:
key: "python-${{ env.PYTHON_VERSION }}-${{ hashFiles('./requirements*.txt') }}"
restore-keys: "python-${{ env.PYTHON_VERSION }}-"
path: "./local"
- name: Setup cache container mounts
uses: actions/cache@v4
with:
key: "container-mounts-${{ hashFiles('./Dockerfile') }}"
restore-keys: "container-mounts-"
path: |
/var/tmp/buildah-cache/
/var/tmp/buildah-cache-*/
- name: Setup QEMU
uses: docker/setup-qemu-action@v3
- name: Login to GHCR
uses: docker/login-action@v3
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- name: Build
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
run: make -e GIT_URL=$(git remote get-url origin) ci.container.build
release:
# TODO: Uncomment before merge
# if: github.repository_owner == 'searxng'
if: false
name: Release (${{ matrix.arch }})
runs-on: ubuntu-24.04-arm
needs: build
strategy:
matrix:
arch:
- amd64
- arm64
- armv7
steps:
- if: env.DOCKERHUB_USERNAME != ''
name: Checkout
uses: actions/checkout@v4
with:
persist-credentials: "false"
# make sure "make ci.container.push" can get the git history
fetch-depth: "0"
- if: env.DOCKERHUB_USERNAME != ''
name: Login to GHCR
uses: docker/login-action@v3
with:
registry: "ghcr.io"
username: "${{ github.repository_owner }}"
password: "${{ secrets.GITHUB_TOKEN }}"
- if: env.DOCKERHUB_USERNAME != ''
name: Login to Docker Hub
uses: docker/login-action@v3
with:
registry: "docker.io"
username: "${{ env.DOCKERHUB_USERNAME }}"
password: "${{ secrets.DOCKERHUB_TOKEN }}"
- if: env.DOCKERHUB_USERNAME != ''
name: Release
env:
OVERRIDE_ARCH: "${{ matrix.arch }}"
run: make -e GIT_URL=$(git remote get-url origin) ci.container.push

View File

@ -1,8 +1,10 @@
name: "Update searx.data" name: Update searx.data
on: # yamllint disable-line rule:truthy
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
schedule: schedule:
- cron: "59 23 28 * *" - cron: "59 23 28 * *"
workflow_dispatch:
jobs: jobs:
updateData: updateData:

View File

@ -1,70 +1,96 @@
name: Integration name: Integration
on: # yamllint disable-line rule:truthy # yamllint disable-line rule:truthy
on:
workflow_dispatch:
push: push:
branches: ["master"] branches:
- master
pull_request: pull_request:
branches: ["master"] branches:
- master
permissions: permissions:
contents: read contents: read
jobs: jobs:
python: python:
# TODO: Remove this
if: false
name: Python ${{ matrix.python-version }} name: Python ${{ matrix.python-version }}
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
strategy: strategy:
matrix: matrix:
os: [ubuntu-24.04] python-version:
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - "3.9"
- "3.10"
- "3.11"
- "3.12"
- "3.13"
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with:
persist-credentials: "false"
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: | run: sudo ./utils/searxng.sh install packages
sudo ./utils/searxng.sh install packages
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: ${{ matrix.python-version }} python-version: "${{ matrix.python-version }}"
architecture: 'x64'
- name: Run tests - name: Run tests
run: make V=1 ci.test run: make V=1 ci.test
themes: themes:
# TODO: Remove this
if: false
name: Themes name: Themes
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with:
persist-credentials: "false"
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: "3.13"
architecture: 'x64'
- name: Build themes - name: Build themes
run: make themes.all run: make themes.all
documentation: documentation:
# TODO: Remove this
if: false
name: Documentation name: Documentation
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
permissions: permissions:
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo # for JamesIves/github-pages-deploy-action to push
contents: write
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
fetch-depth: '0' persist-credentials: "false"
persist-credentials: false fetch-depth: "0"
- name: Install Ubuntu packages - name: Install Ubuntu packages
run: sudo ./utils/searxng.sh install buildhost run: sudo ./utils/searxng.sh install buildhost
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: "3.13"
architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v4 uses: actions/cache@v4
@ -73,42 +99,50 @@ jobs:
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-24.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: Build documentation - name: Build documentation
run: | run: make V=1 docs.clean docs.html
make V=1 docs.clean docs.html
- name: Deploy - if: github.ref == 'refs/heads/master'
if: github.ref == 'refs/heads/master' name: Deploy
uses: JamesIves/github-pages-deploy-action@3.7.1 uses: JamesIves/github-pages-deploy-action@3.7.1
with: with:
GITHUB_TOKEN: ${{ github.token }} GITHUB_TOKEN: "${{ github.token }}"
BRANCH: gh-pages BRANCH: "gh-pages"
FOLDER: dist/docs FOLDER: "dist/docs"
CLEAN: true # Automatically remove deleted files from the deploy branch # Automatically remove deleted files from the deploy branch
SINGLE_COMMIT: true CLEAN: "true"
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}' SINGLE_COMMIT: "true"
COMMIT_MESSAGE: "[doc] build from commit ${{ github.sha }}"
babel: babel:
# TODO: Remove this
if: false
# if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master'
name: Update translations branch name: Update translations branch
runs-on: ubuntu-24.04 runs-on: ubuntu-24.04
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
needs: needs:
- python - python
- themes - themes
- documentation - documentation
permissions: permissions:
contents: write # for make V=1 weblate.push.translations # for make V=1 weblate.push.translations
contents: write
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
fetch-depth: '0' token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} fetch-depth: "0"
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: '3.12' python-version: "3.13"
architecture: 'x64'
- name: Cache Python dependencies - name: Cache Python dependencies
id: cache-python id: cache-python
uses: actions/cache@v4 uses: actions/cache@v4
@ -117,63 +151,17 @@ jobs:
./local ./local
./.nvm ./.nvm
./node_modules ./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} key: python-ubuntu-20.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: weblate & git setup - name: weblate & git setup
env: env:
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} WEBLATE_CONFIG: "${{ secrets.WEBLATE_CONFIG }}"
run: | run: |
mkdir -p ~/.config mkdir -p ~/.config
echo "${WEBLATE_CONFIG}" > ~/.config/weblate echo "${WEBLATE_CONFIG}" > ~/.config/weblate
git config --global user.email "searxng-bot@users.noreply.github.com" git config --global user.email "searxng-bot@users.noreply.github.com"
git config --global user.name "searxng-bot" git config --global user.name "searxng-bot"
- name: Update transations - name: Update transations
id: update id: update
run: | run: make V=1 weblate.push.translations
make V=1 weblate.push.translations
dockers:
name: Docker
if: github.ref == 'refs/heads/master'
needs:
- python
- themes
- documentation
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
runs-on: ubuntu-24.04
steps:
- name: Checkout
if: env.DOCKERHUB_USERNAME != null
uses: actions/checkout@v4
with:
# make sure "make docker.push" can get the git history
fetch-depth: '0'
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
architecture: 'x64'
- name: Cache Python dependencies
id: cache-python
uses: actions/cache@v4
with:
path: |
./local
./.nvm
./node_modules
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
- name: Set up QEMU
if: env.DOCKERHUB_USERNAME != null
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
if: env.DOCKERHUB_USERNAME != null
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
if: env.DOCKERHUB_USERNAME != null
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
if: env.DOCKERHUB_USERNAME != null
run: make -e GIT_URL=$(git remote get-url origin) docker.buildx

View File

@ -1,8 +1,10 @@
name: "Security checks" name: Security checks
on: # yamllint disable-line rule:truthy
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
schedule: schedule:
- cron: "42 05 * * *" - cron: "42 05 * * *"
workflow_dispatch:
jobs: jobs:
dockers: dockers:

View File

@ -1,8 +1,10 @@
name: "Update translations" name: Update translations
on: # yamllint disable-line rule:truthy
# yamllint disable-line rule:truthy
on:
workflow_dispatch:
schedule: schedule:
- cron: "05 07 * * 5" - cron: "05 07 * * 5"
workflow_dispatch:
jobs: jobs:
babel: babel:

View File

@ -77,7 +77,8 @@ test.shell:
MANAGE += weblate.translations.commit weblate.push.translations MANAGE += weblate.translations.commit weblate.push.translations
MANAGE += data.all data.traits data.useragents data.locales data.currencies MANAGE += data.all data.traits data.useragents data.locales data.currencies
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
MANAGE += docker.build docker.push docker.buildx MANAGE += docker.build docker.buildx
MANAGE += container.build container.buildx
MANAGE += gecko.driver MANAGE += gecko.driver
MANAGE += node.env node.env.dev node.clean MANAGE += node.env node.env.dev node.clean
MANAGE += py.build py.clean MANAGE += py.build py.clean
@ -87,6 +88,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover
MANAGE += themes.all themes.fix themes.test MANAGE += themes.all themes.fix themes.test
MANAGE += static.build.commit static.build.drop static.build.restore MANAGE += static.build.commit static.build.drop static.build.restore
MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
MANAGE += ci.container.build ci.container.push
PHONY += $(MANAGE) PHONY += $(MANAGE)

415
manage
View File

@ -60,7 +60,7 @@ while IFS= read -r line; do
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
YAMLLINT_FILES+=("$line") YAMLLINT_FILES+=("$line")
fi fi
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')" done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
RST_FILES=( RST_FILES=(
'README.rst' 'README.rst'
@ -77,9 +77,9 @@ docs.:
gh-pages : deploy on gh-pages branch gh-pages : deploy on gh-pages branch
prebuild : build reST include files (./${DOCS_BUILD}/includes) prebuild : build reST include files (./${DOCS_BUILD}/includes)
clean : clean documentation build clean : clean documentation build
docker.: container. :
build : build docker image build : build container image
push : build and push docker image buildx : build container image with BuildKit
gecko.driver: gecko.driver:
download & install geckodriver if not already installed (required for download & install geckodriver if not already installed (required for
robot_tests) robot_tests)
@ -97,6 +97,10 @@ pyenv.:
OK : test if virtualenv is OK OK : test if virtualenv is OK
format.: format.:
python : format Python code source using black python : format Python code source using black
ci.:
container.:
build : build container image
release : push container images to remote registry
EOF EOF
go.help go.help
node.help node.help
@ -112,7 +116,6 @@ environment ...
EOF EOF
} }
if [ "$VERBOSE" = "1" ]; then if [ "$VERBOSE" = "1" ]; then
SPHINX_VERBOSE="-v" SPHINX_VERBOSE="-v"
PYLINT_VERBOSE="-v" PYLINT_VERBOSE="-v"
@ -125,53 +128,95 @@ webapp.run() {
local parent_proc="$$" local parent_proc="$$"
( (
if [ "${LIVE_THEME}" ]; then if [ "${LIVE_THEME}" ]; then
( themes.live "${LIVE_THEME}" ) (themes.live "${LIVE_THEME}")
kill $parent_proc kill $parent_proc
fi fi
)& ) &
( (
sleep 3 sleep 3
xdg-open http://127.0.0.1:8888/ xdg-open http://127.0.0.1:8888/
)& ) &
SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
} }
docker.push() { # Alias
docker.build push
}
docker.buildx() { docker.buildx() {
docker.build buildx container.buildx
} }
# shellcheck disable=SC2119 # Alias
docker.build() { docker.build() {
container.build
}
container.buildx() {
container.build buildx
}
container.build() {
pyenv.install pyenv.install
local SEARXNG_GIT_VERSION local parch=${OVERRIDE_ARCH:-$(uname -m)}
local VERSION_GITCOMMIT local container_engine
local GITHUB_USER local dockerfile
local SEARXNG_IMAGE_NAME local arch
local BUILD local variant
local platform
build_msg DOCKER build # Setup arch specific
# run installation in a subprocess and activate pyenv case $parch in
"X64" | "x86_64" | "amd64")
dockerfile="Dockerfile"
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
dockerfile="Dockerfile"
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
# TODO: Move ARMv7 to a separated Dockerfile
dockerfile="Dockerfile"
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# See https://www.shellcheck.net/wiki/SC1001 and others .. # Check if podman or docker is installed
# shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 if command -v podman &>/dev/null; then
( set -e container_engine="podman"
elif command -v docker &>/dev/null; then
container_engine="docker"
else
die 1 "podman/docker is not installed"
fi
build_msg CONTAINER "Engine: $container_engine"
# Check if git is installed
if ! command -v git &>/dev/null; then
die 1 "git is not installed"
fi
(
set -e
pyenv.activate pyenv.activate
# Check if it is a git repository # Check if it is a git repository
if [ ! -d .git ]; then if [ ! -d .git ]; then
die 1 "This is not Git repository" die 1 "This is not Git repository"
fi
if [ ! -x "$(which git)" ]; then
die 1 "git is not installed"
fi fi
if ! git remote get-url origin 2> /dev/null; then if ! git remote get-url origin 2>/dev/null; then
die 1 "there is no remote origin" die 1 "there is no remote origin"
fi fi
# This is a git repository # This is a git repository
@ -180,42 +225,48 @@ docker.build() {
eval "$(python -m searx.version)" eval "$(python -m searx.version)"
# Get the last git commit id # Get the last git commit id
VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2) version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" build_msg CONTAINER "Last commit: $version_gitcommit"
# define the docker image name if [ "$container_engine" = "docker" ]; then
GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/') if [ "$1" = "buildx" ]; then
SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}" docker_builder="buildx build"
else
docker_builder="build"
warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback"
fi
BUILD="build" params_build_builder="$docker_builder --platform=$platform --target=builder"
if [ "$1" = "buildx" ]; then params_build="$docker_builder --platform=$platform --squash"
# buildx includes the push option else
CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache" params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false"
BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max" params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false"
shift
fi fi
build_msg DOCKER "Build command: ${BUILD}"
# build Docker image # Define container image org/name
build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" # shellcheck disable=SC2001
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
build_msg CONTAINER "Building..."
# shellcheck disable=SC2086 # shellcheck disable=SC2086
docker $BUILD \ "$container_engine" $params_build_builder \
--build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
--build-arg GIT_URL="${GIT_URL}" \ --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
--build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \ --tag="localhost/$container_image_organization/$container_image_name:builder" \
--build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ --file="./$dockerfile"
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
--build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
--build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
--build-arg LABEL_VCS_URL="${GIT_URL}" \
--build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
--build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
-t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" .
if [ "$1" = "push" ]; then # shellcheck disable=SC2086
docker push "${SEARXNG_IMAGE_NAME}:latest" "$container_engine" $params_build \
docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" --build-arg="GIT_URL=$GIT_URL" \
fi --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
--build-arg="LABEL_VCS_URL=$GIT_URL" \
--tag="localhost/$container_image_organization/$container_image_name:latest" \
--tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \
--file="./$dockerfile"
) )
dump_return $? dump_return $?
} }
@ -226,10 +277,11 @@ gecko.driver() {
build_msg INSTALL "gecko.driver" build_msg INSTALL "gecko.driver"
# run installation in a subprocess and activate pyenv # run installation in a subprocess and activate pyenv
( set -e (
set -e
pyenv.activate pyenv.activate
INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
set +e set +e
if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
build_msg INSTALL "geckodriver already installed" build_msg INSTALL "geckodriver already installed"
@ -237,13 +289,13 @@ gecko.driver() {
fi fi
PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
case "$PLATFORM" in case "$PLATFORM" in
"linux 32bit" | "linux2 32bit") ARCH="linux32";; "linux 32bit" | "linux2 32bit") ARCH="linux32" ;;
"linux 64bit" | "linux2 64bit") ARCH="linux64";; "linux 64bit" | "linux2 64bit") ARCH="linux64" ;;
"windows 32 bit") ARCH="win32";; "windows 32 bit") ARCH="win32" ;;
"windows 64 bit") ARCH="win64";; "windows 64 bit") ARCH="win64" ;;
"mac 64bit") ARCH="macos";; "mac 64bit") ARCH="macos" ;;
esac esac
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"
build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
@ -258,13 +310,14 @@ gecko.driver() {
py.build() { py.build() {
build_msg BUILD "python package ${PYDIST}" build_msg BUILD "python package ${PYDIST}"
pyenv.cmd python setup.py \ pyenv.cmd python setup.py \
sdist -d "${PYDIST}" \ sdist -d "${PYDIST}" \
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
} }
py.clean() { py.clean() {
build_msg CLEAN pyenv build_msg CLEAN pyenv
( set -e (
set -e
pyenv.drop pyenv.drop
[ "$VERBOSE" = "1" ] && set -x [ "$VERBOSE" = "1" ] && set -x
rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
@ -275,22 +328,22 @@ py.clean() {
} }
pyenv.check() { pyenv.check() {
cat <<EOF cat <<EOF
import yaml import yaml
print('import yaml --> OK') print('import yaml --> OK')
EOF EOF
} }
pyenv.install() { pyenv.install() {
if ! pyenv.OK; then if ! pyenv.OK; then
py.clean > /dev/null py.clean >/dev/null
fi fi
if pyenv.install.OK > /dev/null; then if pyenv.install.OK >/dev/null; then
return 0 return 0
fi fi
( set -e (
set -e
pyenv pyenv
build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'" build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'"
"${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}" "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}"
@ -303,8 +356,8 @@ pyenv.install() {
pyenv.uninstall() { pyenv.uninstall() {
build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
pyenv.cmd python setup.py develop --uninstall 2>&1 \ pyenv.cmd python setup.py develop --uninstall 2>&1 |
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
} }
@ -320,17 +373,184 @@ docs.prebuild() {
set -e set -e
[ "$VERBOSE" = "1" ] && set -x [ "$VERBOSE" = "1" ] && set -x
mkdir -p "${DOCS_BUILD}/includes" mkdir -p "${DOCS_BUILD}/includes"
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst" ./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst"
pyenv.cmd searxng_extra/docs_prebuild pyenv.cmd searxng_extra/docs_prebuild
) )
dump_return $? dump_return $?
} }
ci.container.build() {
if ! "$GITHUB_ACTIONS"; then
die 1 "This command is intended to be run in GitHub Actions"
fi
pyenv.install
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local dockerfile
local arch
local variant
local platform
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
dockerfile="Dockerfile"
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
dockerfile="Dockerfile"
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
# TODO: Move ARMv7 to a separated Dockerfile
dockerfile="Dockerfile"
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# Check if podman is installed
if ! command -v podman &>/dev/null; then
die 1 "podman is not installed"
fi
# Check if git is installed
if ! command -v git &>/dev/null; then
die 1 "git is not installed"
fi
(
set -eu
pyenv.activate
# Check if it is a git repository
if [ ! -d .git ]; then
die 1 "This is not Git repository"
fi
if ! git remote get-url origin 2>/dev/null; then
die 1 "there is no remote origin"
fi
# This is a git repository
git update-index -q --refresh
python -m searx.version freeze
eval "$(python -m searx.version)"
# Get the last git commit id
version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
build_msg CONTAINER "Last commit: $version_gitcommit"
# Define container image org/name
# shellcheck disable=SC2001
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
build_msg CONTAINER "Building..."
podman build --platform="$platform" --target=builder --layers --identity-label=false \
--cache-from="ghcr.io/$container_image_organization/cache" \
--cache-to="ghcr.io/$container_image_organization/cache" \
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
--build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
--tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \
--file="./$dockerfile"
podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \
--cache-from="ghcr.io/$container_image_organization/cache" \
--cache-to="ghcr.io/$container_image_organization/cache" \
--build-arg="GIT_URL=$GIT_URL" \
--build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
--build-arg="LABEL_VCS_URL=$GIT_URL" \
--tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \
--file="./$dockerfile"
podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant"
)
dump_return $?
}
ci.container.push() {
if ! "$GITHUB_ACTIONS"; then
die 1 "This command is intended to be run in GitHub Actions"
fi
local parch=${OVERRIDE_ARCH:-$(uname -m)}
local arch
local variant
local platform
# Setup arch specific
case $parch in
"X64" | "x86_64" | "amd64")
arch="amd64"
variant=""
platform="linux/$arch"
;;
"ARM64" | "aarch64" | "arm64")
arch="arm64"
variant=""
platform="linux/$arch"
;;
"ARMV7" | "armhf" | "armv7l" | "armv7")
arch="arm"
variant="v7"
platform="linux/$arch/$variant"
;;
*)
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
exit 1
;;
esac
build_msg CONTAINER "Selected platform: $platform"
# Check if podman is installed
if ! command -v podman &>/dev/null; then
die 1 "podman is not installed"
fi
(
set -eu
# Define container image org/name
# shellcheck disable=SC2001
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
container_image_name="searxng"
podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant"
# Get version tag
container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \
"ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant")
# Recreate tags
podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest"
podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
podman push "docker.io/$container_image_organization/$container_image_name:latest"
podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
)
dump_return $?
}
# shellcheck disable=SC2119 # shellcheck disable=SC2119
main() { main() {
local _type local _type
local cmd="$1"; shift local cmd="$1"
shift
if [ "$cmd" == "" ]; then if [ "$cmd" == "" ]; then
help help
@ -339,22 +559,25 @@ main() {
fi fi
case "$cmd" in case "$cmd" in
--getenv) var="$1"; echo "${!var}";; --getenv)
--help) help;; var="$1"
--*) echo "${!var}"
help ;;
err_msg "unknown option $cmd" --help) help ;;
--*)
help
err_msg "unknown option $cmd"
return 42
;;
*)
_type="$(type -t "$cmd")"
if [ "$_type" != 'function' ]; then
err_msg "unknown command: $cmd / use --help"
return 42 return 42
;; else
*) "$cmd" "$@"
_type="$(type -t "$cmd")" fi
if [ "$_type" != 'function' ]; then ;;
err_msg "unknown command: $cmd / use --help"
return 42
else
"$cmd" "$@"
fi
;;
esac esac
} }