diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index d0074712d..d308bf56d 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -1,5 +1,7 @@ name: "Checker" -on: # yamllint disable-line rule:truthy + +# yamllint disable-line rule:truthy +on: schedule: - cron: "0 4 * * 5" workflow_dispatch: diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml new file mode 100644 index 000000000..5c0cc09f7 --- /dev/null +++ b/.github/workflows/container.yml @@ -0,0 +1,127 @@ +--- +name: Container + +# yamllint disable-line rule:truthy +on: + workflow_run: + workflows: + - Integration + types: + - completed +# TODO: Uncomment +# branches: +# - master + +permissions: + contents: read + # Organization GHCR + packages: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + build: + if: github.event.workflow_run.conclusion == 'success' + name: Build (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + - arch: + arm64 + armv7 + os: ubuntu-24.04-arm + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-" + path: "./local" + + - name: Setup cache container mounts + uses: actions/cache@v4 + with: + key: "container-mounts-${{ hashFiles('./Dockerfile') }}" + restore-keys: "container-mounts-" + path: | + /var/tmp/buildah-cache/ + /var/tmp/buildah-cache-*/ + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Build + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make -e GIT_URL=$(git remote get-url origin) ci.container.build + + release: + # TODO: Uncomment before merge + # if: github.repository_owner == 'searxng' + if: false + name: Release (${{ matrix.arch }}) + runs-on: ubuntu-24.04-arm + needs: build + strategy: + matrix: + arch: + - amd64 + - arm64 + - armv7 + + steps: + - if: env.DOCKERHUB_USERNAME != '' + name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + # make sure "make ci.container.push" can get the git history + fetch-depth: "0" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: "docker.io" + username: "${{ env.DOCKERHUB_USERNAME }}" + password: "${{ secrets.DOCKERHUB_TOKEN }}" + + - if: env.DOCKERHUB_USERNAME != '' + name: Release + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make -e GIT_URL=$(git remote get-url origin) ci.container.push diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index 39893127d..ba7b6f800 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -1,8 +1,10 @@ -name: "Update searx.data" -on: # yamllint disable-line rule:truthy +name: Update searx.data + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "59 23 28 * *" - workflow_dispatch: jobs: updateData: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index ddc0498f7..b0170d794 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,70 +1,96 @@ name: Integration -on: # yamllint disable-line rule:truthy +# yamllint disable-line rule:truthy +on: + workflow_dispatch: push: - branches: ["master"] + branches: + - master pull_request: - branches: ["master"] + branches: + - master permissions: contents: read jobs: python: + # TODO: Remove this + if: false name: Python ${{ matrix.python-version }} runs-on: ubuntu-24.04 strategy: matrix: - os: [ubuntu-24.04] - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" + - name: Install Ubuntu packages - run: | - sudo ./utils/searxng.sh install packages + run: sudo ./utils/searxng.sh install packages + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' + python-version: "${{ matrix.python-version }}" + - name: Run tests run: make V=1 ci.test themes: + # TODO: Remove this + if: false name: Themes runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 + with: + persist-credentials: "false" + - name: Install Ubuntu packages run: sudo ./utils/searxng.sh install buildhost + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' + python-version: "3.13" + - name: Build themes run: make themes.all documentation: + # TODO: Remove this + if: false name: Documentation runs-on: ubuntu-24.04 permissions: - contents: write # for JamesIves/github-pages-deploy-action to push changes in repo + # for JamesIves/github-pages-deploy-action to push + contents: write + steps: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: '0' - persist-credentials: false + persist-credentials: "false" + fetch-depth: "0" + - name: Install Ubuntu packages run: sudo ./utils/searxng.sh install buildhost + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' + python-version: "3.13" + - name: Cache Python dependencies id: cache-python uses: actions/cache@v4 @@ -73,42 +99,50 @@ jobs: ./local ./.nvm ./node_modules - key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + key: python-ubuntu-24.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + - name: Build documentation - run: | - make V=1 docs.clean docs.html - - name: Deploy - if: github.ref == 'refs/heads/master' + run: make V=1 docs.clean docs.html + + - if: github.ref == 'refs/heads/master' + name: Deploy uses: JamesIves/github-pages-deploy-action@3.7.1 with: - GITHUB_TOKEN: ${{ github.token }} - BRANCH: gh-pages - FOLDER: dist/docs - CLEAN: true # Automatically remove deleted files from the deploy branch - SINGLE_COMMIT: true - COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}' + GITHUB_TOKEN: "${{ github.token }}" + BRANCH: "gh-pages" + FOLDER: "dist/docs" + # Automatically remove deleted files from the deploy branch + CLEAN: "true" + SINGLE_COMMIT: "true" + COMMIT_MESSAGE: "[doc] build from commit ${{ github.sha }}" babel: + # TODO: Remove this + if: false + # if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' name: Update translations branch runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} needs: - python - themes - documentation + permissions: - contents: write # for make V=1 weblate.push.translations + # for make V=1 weblate.push.translations + contents: write + steps: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: '0' - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' - architecture: 'x64' + python-version: "3.13" + - name: Cache Python dependencies id: cache-python uses: actions/cache@v4 @@ -117,63 +151,17 @@ jobs: ./local ./.nvm ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + key: python-ubuntu-20.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} + - name: weblate & git setup env: - WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} + WEBLATE_CONFIG: "${{ secrets.WEBLATE_CONFIG }}" run: | mkdir -p ~/.config echo "${WEBLATE_CONFIG}" > ~/.config/weblate git config --global user.email "searxng-bot@users.noreply.github.com" git config --global user.name "searxng-bot" + - name: Update transations id: update - run: | - make V=1 weblate.push.translations - - dockers: - name: Docker - if: github.ref == 'refs/heads/master' - needs: - - python - - themes - - documentation - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - runs-on: ubuntu-24.04 - steps: - - name: Checkout - if: env.DOCKERHUB_USERNAME != null - uses: actions/checkout@v4 - with: - # make sure "make docker.push" can get the git history - fetch-depth: '0' - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: Set up QEMU - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - if: env.DOCKERHUB_USERNAME != null - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - if: env.DOCKERHUB_USERNAME != null - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - if: env.DOCKERHUB_USERNAME != null - run: make -e GIT_URL=$(git remote get-url origin) docker.buildx + run: make V=1 weblate.push.translations diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 249db305b..f05522314 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -1,8 +1,10 @@ -name: "Security checks" -on: # yamllint disable-line rule:truthy +name: Security checks + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "42 05 * * *" - workflow_dispatch: jobs: dockers: diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml index 85e141e7f..a6f7fa9ce 100644 --- a/.github/workflows/translations-update.yml +++ b/.github/workflows/translations-update.yml @@ -1,8 +1,10 @@ -name: "Update translations" -on: # yamllint disable-line rule:truthy +name: Update translations + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: schedule: - cron: "05 07 * * 5" - workflow_dispatch: jobs: babel: diff --git a/Makefile b/Makefile index c1c067149..d7ac9eb09 100644 --- a/Makefile +++ b/Makefile @@ -77,7 +77,8 @@ test.shell: MANAGE += weblate.translations.commit weblate.push.translations MANAGE += data.all data.traits data.useragents data.locales data.currencies MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean -MANAGE += docker.build docker.push docker.buildx +MANAGE += docker.build docker.buildx +MANAGE += container.build container.buildx MANAGE += gecko.driver MANAGE += node.env node.env.dev node.clean MANAGE += py.build py.clean @@ -87,6 +88,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover MANAGE += themes.all themes.fix themes.test MANAGE += static.build.commit static.build.drop static.build.restore MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs +MANAGE += ci.container.build ci.container.push PHONY += $(MANAGE) diff --git a/manage b/manage index 61bc68b74..91f045d34 100755 --- a/manage +++ b/manage @@ -60,7 +60,7 @@ while IFS= read -r line; do if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then YAMLLINT_FILES+=("$line") fi -done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')" +done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')" RST_FILES=( 'README.rst' @@ -77,9 +77,9 @@ docs.: gh-pages : deploy on gh-pages branch prebuild : build reST include files (./${DOCS_BUILD}/includes) clean : clean documentation build -docker.: - build : build docker image - push : build and push docker image +container. : + build : build container image + buildx : build container image with BuildKit gecko.driver: download & install geckodriver if not already installed (required for robot_tests) @@ -97,6 +97,10 @@ pyenv.: OK : test if virtualenv is OK format.: python : format Python code source using black +ci.: + container.: + build : build container image + release : push container images to remote registry EOF go.help node.help @@ -112,7 +116,6 @@ environment ... EOF } - if [ "$VERBOSE" = "1" ]; then SPHINX_VERBOSE="-v" PYLINT_VERBOSE="-v" @@ -125,53 +128,95 @@ webapp.run() { local parent_proc="$$" ( if [ "${LIVE_THEME}" ]; then - ( themes.live "${LIVE_THEME}" ) + (themes.live "${LIVE_THEME}") kill $parent_proc fi - )& + ) & ( sleep 3 xdg-open http://127.0.0.1:8888/ - )& + ) & SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp } -docker.push() { - docker.build push -} - +# Alias docker.buildx() { - docker.build buildx + container.buildx } -# shellcheck disable=SC2119 +# Alias docker.build() { + container.build +} + +container.buildx() { + container.build buildx +} + +container.build() { pyenv.install - local SEARXNG_GIT_VERSION - local VERSION_GITCOMMIT - local GITHUB_USER - local SEARXNG_IMAGE_NAME - local BUILD + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local container_engine + local dockerfile + local arch + local variant + local platform - build_msg DOCKER build - # run installation in a subprocess and activate pyenv + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + dockerfile="Dockerfile" + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + dockerfile="Dockerfile" + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + # TODO: Move ARMv7 to a separated Dockerfile + dockerfile="Dockerfile" + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" - # See https://www.shellcheck.net/wiki/SC1001 and others .. - # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001 - ( set -e + # Check if podman or docker is installed + if command -v podman &>/dev/null; then + container_engine="podman" + elif command -v docker &>/dev/null; then + container_engine="docker" + else + die 1 "podman/docker is not installed" + fi + build_msg CONTAINER "Engine: $container_engine" + + # Check if git is installed + if ! command -v git &>/dev/null; then + die 1 "git is not installed" + fi + + ( + set -e pyenv.activate # Check if it is a git repository if [ ! -d .git ]; then - die 1 "This is not Git repository" - fi - if [ ! -x "$(which git)" ]; then - die 1 "git is not installed" + die 1 "This is not Git repository" fi - if ! git remote get-url origin 2> /dev/null; then - die 1 "there is no remote origin" + if ! git remote get-url origin 2>/dev/null; then + die 1 "there is no remote origin" fi # This is a git repository @@ -180,42 +225,48 @@ docker.build() { eval "$(python -m searx.version)" # Get the last git commit id - VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2) - build_msg DOCKER "Last commit : $VERSION_GITCOMMIT" + version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) + build_msg CONTAINER "Last commit: $version_gitcommit" - # define the docker image name - GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/') - SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}" + if [ "$container_engine" = "docker" ]; then + if [ "$1" = "buildx" ]; then + docker_builder="buildx build" + else + docker_builder="build" + warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback" + fi - BUILD="build" - if [ "$1" = "buildx" ]; then - # buildx includes the push option - CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache" - BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max" - shift + params_build_builder="$docker_builder --platform=$platform --target=builder" + params_build="$docker_builder --platform=$platform --squash" + else + params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false" + params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false" fi - build_msg DOCKER "Build command: ${BUILD}" - # build Docker image - build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}" + # Define container image org/name + # shellcheck disable=SC2001 + container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_name="searxng" + + build_msg CONTAINER "Building..." + # shellcheck disable=SC2086 - docker $BUILD \ - --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \ - --build-arg GIT_URL="${GIT_URL}" \ - --build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \ - --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \ - --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ - --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \ - --build-arg LABEL_VCS_URL="${GIT_URL}" \ - --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \ - --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \ - -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" . + "$container_engine" $params_build_builder \ + --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ + --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ + --tag="localhost/$container_image_organization/$container_image_name:builder" \ + --file="./$dockerfile" - if [ "$1" = "push" ]; then - docker push "${SEARXNG_IMAGE_NAME}:latest" - docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" - fi + # shellcheck disable=SC2086 + "$container_engine" $params_build \ + --build-arg="GIT_URL=$GIT_URL" \ + --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ + --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ + --build-arg="LABEL_VCS_URL=$GIT_URL" \ + --tag="localhost/$container_image_organization/$container_image_name:latest" \ + --tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \ + --file="./$dockerfile" ) dump_return $? } @@ -226,10 +277,11 @@ gecko.driver() { build_msg INSTALL "gecko.driver" # run installation in a subprocess and activate pyenv - ( set -e + ( + set -e pyenv.activate - INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" + INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION="" set +e if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then build_msg INSTALL "geckodriver already installed" @@ -237,13 +289,13 @@ gecko.driver() { fi PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')" case "$PLATFORM" in - "linux 32bit" | "linux2 32bit") ARCH="linux32";; - "linux 64bit" | "linux2 64bit") ARCH="linux64";; - "windows 32 bit") ARCH="win32";; - "windows 64 bit") ARCH="win64";; - "mac 64bit") ARCH="macos";; + "linux 32bit" | "linux2 32bit") ARCH="linux32" ;; + "linux 64bit" | "linux2 64bit") ARCH="linux64" ;; + "windows 32 bit") ARCH="win32" ;; + "windows 64 bit") ARCH="win64" ;; + "mac 64bit") ARCH="macos" ;; esac - GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"; + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz" build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL" @@ -258,13 +310,14 @@ gecko.driver() { py.build() { build_msg BUILD "python package ${PYDIST}" pyenv.cmd python setup.py \ - sdist -d "${PYDIST}" \ - bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" + sdist -d "${PYDIST}" \ + bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}" } py.clean() { build_msg CLEAN pyenv - ( set -e + ( + set -e pyenv.drop [ "$VERBOSE" = "1" ] && set -x rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info @@ -275,22 +328,22 @@ py.clean() { } pyenv.check() { - cat < OK') EOF } pyenv.install() { - if ! pyenv.OK; then - py.clean > /dev/null + py.clean >/dev/null fi - if pyenv.install.OK > /dev/null; then + if pyenv.install.OK >/dev/null; then return 0 fi - ( set -e + ( + set -e pyenv build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'" "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}" @@ -303,8 +356,8 @@ pyenv.install() { pyenv.uninstall() { build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}" - pyenv.cmd python setup.py develop --uninstall 2>&1 \ - | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " + pyenv.cmd python setup.py develop --uninstall 2>&1 | + prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] " } @@ -320,17 +373,184 @@ docs.prebuild() { set -e [ "$VERBOSE" = "1" ] && set -x mkdir -p "${DOCS_BUILD}/includes" - ./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst" + ./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst" pyenv.cmd searxng_extra/docs_prebuild ) dump_return $? } +ci.container.build() { + if ! "$GITHUB_ACTIONS"; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + pyenv.install + + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local dockerfile + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + dockerfile="Dockerfile" + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + dockerfile="Dockerfile" + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + # TODO: Move ARMv7 to a separated Dockerfile + dockerfile="Dockerfile" + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + # Check if git is installed + if ! command -v git &>/dev/null; then + die 1 "git is not installed" + fi + + ( + set -eu + pyenv.activate + + # Check if it is a git repository + if [ ! -d .git ]; then + die 1 "This is not Git repository" + fi + + if ! git remote get-url origin 2>/dev/null; then + die 1 "there is no remote origin" + fi + + # This is a git repository + git update-index -q --refresh + python -m searx.version freeze + eval "$(python -m searx.version)" + + # Get the last git commit id + version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) + build_msg CONTAINER "Last commit: $version_gitcommit" + + # Define container image org/name + # shellcheck disable=SC2001 + container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_name="searxng" + + build_msg CONTAINER "Building..." + + podman build --platform="$platform" --target=builder --layers --identity-label=false \ + --cache-from="ghcr.io/$container_image_organization/cache" \ + --cache-to="ghcr.io/$container_image_organization/cache" \ + --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ + --build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \ + --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \ + --file="./$dockerfile" + + podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \ + --cache-from="ghcr.io/$container_image_organization/cache" \ + --cache-to="ghcr.io/$container_image_organization/cache" \ + --build-arg="GIT_URL=$GIT_URL" \ + --build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \ + --build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \ + --build-arg="LABEL_VCS_URL=$GIT_URL" \ + --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \ + --file="./$dockerfile" + + podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" + ) + dump_return $? +} + +ci.container.push() { + if ! "$GITHUB_ACTIONS"; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + ( + set -eu + + # Define container image org/name + # shellcheck disable=SC2001 + container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_name="searxng" + + podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" + + # Get version tag + container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \ + "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant") + + # Recreate tags + podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest" + podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version" + + podman push "docker.io/$container_image_organization/$container_image_name:latest" + podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version" + ) + dump_return $? +} + # shellcheck disable=SC2119 main() { - local _type - local cmd="$1"; shift + local cmd="$1" + shift if [ "$cmd" == "" ]; then help @@ -339,22 +559,25 @@ main() { fi case "$cmd" in - --getenv) var="$1"; echo "${!var}";; - --help) help;; - --*) - help - err_msg "unknown option $cmd" + --getenv) + var="$1" + echo "${!var}" + ;; + --help) help ;; + --*) + help + err_msg "unknown option $cmd" + return 42 + ;; + *) + _type="$(type -t "$cmd")" + if [ "$_type" != 'function' ]; then + err_msg "unknown command: $cmd / use --help" return 42 - ;; - *) - _type="$(type -t "$cmd")" - if [ "$_type" != 'function' ]; then - err_msg "unknown command: $cmd / use --help" - return 42 - else - "$cmd" "$@" - fi - ;; + else + "$cmd" "$@" + fi + ;; esac }