diff --git a/.github/workflows/checker.yml b/.github/workflows/checker.yml index 04639726b..95e193d8f 100644 --- a/.github/workflows/checker.yml +++ b/.github/workflows/checker.yml @@ -14,8 +14,8 @@ env: PYTHON_VERSION: "3.13" jobs: - checker: - name: Checker + search: + name: Search runs-on: ubuntu-24.04-arm steps: - name: Setup Python @@ -38,6 +38,6 @@ jobs: - name: Setup venv run: make V=1 install - - name: Checker + - name: Search checker continue-on-error: true run: make search.checker diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 818d2bc6b..2d369d737 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -3,12 +3,12 @@ name: Container # yamllint disable-line rule:truthy on: + workflow_dispatch: workflow_run: workflows: - Integration types: - completed - # TODO: Test container.yml on merge https://github.com/searxng/searxng/pull/4699 branches: - master @@ -22,18 +22,22 @@ env: jobs: build: - if: github.event.workflow_run.conclusion == 'success' + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' name: Build (${{ matrix.arch }}) runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: include: - arch: amd64 os: ubuntu-24.04 - - arch: - arm64 - armv7 + emulation: false + - arch: arm64 os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true permissions: # Organization GHCR @@ -60,13 +64,14 @@ jobs: - name: Setup cache container mounts uses: actions/cache@v4 with: - key: "container-mounts-${{ hashFiles('./Dockerfile') }}" - restore-keys: "container-mounts-" + key: "container-mounts-${{ runner.arch }}-${{ hashFiles('./Dockerfile*') }}" + restore-keys: "container-mounts-${{ runner.arch }}-" path: | /var/tmp/buildah-cache/ /var/tmp/buildah-cache-*/ - - name: Setup QEMU + - if: ${{ matrix.emulation }} + name: Setup QEMU uses: docker/setup-qemu-action@v3 - name: Login to GHCR @@ -89,6 +94,7 @@ jobs: runs-on: ubuntu-24.04-arm needs: build strategy: + fail-fast: false matrix: arch: - amd64 @@ -124,4 +130,5 @@ jobs: name: Release env: OVERRIDE_ARCH: "${{ matrix.arch }}" + # TODO: Try this with GHCR or testing registry before merge run: make -e GIT_URL=$(git remote get-url origin) ci.container.push diff --git a/.github/workflows/data-update.yml b/.github/workflows/data-update.yml index 9665064ab..62428a8d6 100644 --- a/.github/workflows/data-update.yml +++ b/.github/workflows/data-update.yml @@ -14,7 +14,7 @@ env: PYTHON_VERSION: "3.13" jobs: - updateData: + data: if: github.repository_owner == 'searxng' name: ${{ matrix.fetch }} runs-on: ubuntu-24.04-arm @@ -54,24 +54,22 @@ jobs: - name: Fetch data run: V=1 ./manage pyenv.cmd python "./searxng_extra/update/${{ matrix.fetch }}" - - name: Create Pull Request + - name: Create PR id: cpr uses: peter-evans/create-pull-request@v7 with: author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>" - committer: "searxng-bot " + committer: "searxng-bot " title: "[data] update searx.data - ${{ matrix.fetch }}" commit-message: "[data] update searx.data - ${{ matrix.fetch }}" branch: "update_data_${{ matrix.fetch }}" delete-branch: "true" draft: "false" signoff: "false" - body: | - update searx.data - ${{ matrix.fetch }} labels: | data - - name: Check outputs + - name: Display information run: | echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 000000000..fbf73879b --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,65 @@ +--- +name: Documentation + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + release: + # TODO: Uncomment + # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + if: false + name: Release + runs-on: ubuntu-24.04-arm + permissions: + # for JamesIves/github-pages-deploy-action to push + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Build documentation + run: make V=1 docs.clean docs.html + + - name: Release + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: "dist/docs" + branch: "gh-pages" + commit-message: "[doc] build from commit ${{ github.sha }}" + # Automatically remove deleted files from the deploy branch + clean: "true" + single-commit: "true" diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d32a4fe96..7de57345b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -80,93 +80,3 @@ jobs: - name: Build run: make themes.all - - documentation: - # TODO: Remove this - if: false - name: Documentation - runs-on: ubuntu-24.04-arm - permissions: - # for JamesIves/github-pages-deploy-action to push - contents: write - - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "${{ env.PYTHON_VERSION }}" - - - name: Checkout - uses: actions/checkout@v4 - with: - persist-credentials: "false" - fetch-depth: "0" - - - name: Setup cache Python - uses: actions/cache@v4 - with: - key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" - restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" - path: "./local/" - - - name: Setup venv - run: make V=1 install - - - name: Build documentation - run: make V=1 docs.clean docs.html - - - if: github.ref == 'refs/heads/master' - name: Deploy - uses: JamesIves/github-pages-deploy-action@v4 - with: - folder: "dist/docs" - branch: "gh-pages" - commit-message: "[doc] build from commit ${{ github.sha }}" - # Automatically remove deleted files from the deploy branch - clean: "true" - single-commit: "true" - - babel: - if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' - name: Update translations branch - runs-on: ubuntu-24.04-arm - needs: - - test - - theme - - documentation - - permissions: - # For "make V=1 weblate.push.translations" - contents: write - - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "${{ env.PYTHON_VERSION }}" - - - name: Checkout - uses: actions/checkout@v4 - with: - token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" - fetch-depth: "0" - - - name: Setup cache Python - uses: actions/cache@v4 - with: - key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" - restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" - path: "./local/" - - - name: Setup venv - run: make V=1 install - - - name: weblate & git setup - run: | - mkdir -p ~/.config - echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate - git config --global user.email "searxng-bot@users.noreply.github.com" - git config --global user.name "searxng-bot" - - - name: Update transations - run: make V=1 weblate.push.translations diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml new file mode 100644 index 000000000..4a49cfbfa --- /dev/null +++ b/.github/workflows/l10n.yml @@ -0,0 +1,130 @@ +--- +name: Translation + +# yamllint disable-line rule:truthy +on: + workflow_dispatch: + workflow_run: + workflows: + - Integration + types: + - completed + branches: + - master + schedule: + - cron: "05 07 * * 5" + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.13" + +jobs: + update: + if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' + name: Update + runs-on: ubuntu-24.04-arm + permissions: + # For "make V=1 weblate.push.translations" + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Setup Weblate + run: | + mkdir -p ~/.config + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate + + - name: Setup Git + run: | + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + + - name: Update translations + run: make V=1 weblate.push.translations + + pr: + if: | + github.repository_owner == 'searxng' + && (github.event_name == 'workflow_dispatch' || github.event_name == 'schedule') + name: Pull Request + runs-on: ubuntu-24.04-arm + permissions: + # For "make V=1 weblate.translations.commit" + contents: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}" + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup venv + run: make V=1 install + + - name: Setup Weblate + run: | + mkdir -p ~/.config + echo "${{ secrets.WEBLATE_CONFIG }}" > ~/.config/weblate + + - name: Setup Git + run: | + git config --global user.email "searxng-bot@users.noreply.github.com" + git config --global user.name "searxng-bot" + + - name: Merge and push translation updates + run: make V=1 weblate.translations.commit + + - name: Create PR + id: cpr + uses: peter-evans/create-pull-request@v7 + with: + author: "${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>" + committer: "searxng-bot " + title: "[l10n] update translations from Weblate" + commit-message: "[l10n] update translations from Weblate" + branch: "translations_update" + delete-branch: "true" + draft: "false" + signoff: "false" + labels: | + translation + + - name: Display information + run: | + echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" + echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index a720e0df2..924d19fb8 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -6,6 +6,10 @@ on: workflow_dispatch: schedule: - cron: "42 05 * * *" + # TODO: Remove this + pull_request: + branches: + - container-gha permissions: contents: read diff --git a/.github/workflows/translations-update.yml b/.github/workflows/translations-update.yml deleted file mode 100644 index a6f7fa9ce..000000000 --- a/.github/workflows/translations-update.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Update translations - -# yamllint disable-line rule:truthy -on: - workflow_dispatch: - schedule: - - cron: "05 07 * * 5" - -jobs: - babel: - name: "create PR for additions from weblate" - runs-on: ubuntu-24.04 - if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: '0' - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - architecture: 'x64' - - name: Cache Python dependencies - id: cache-python - uses: actions/cache@v4 - with: - path: | - ./local - ./.nvm - ./node_modules - key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }} - - name: weblate & git setup - env: - WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }} - run: | - mkdir -p ~/.config - echo "${WEBLATE_CONFIG}" > ~/.config/weblate - git config --global user.email "searxng-bot@users.noreply.github.com" - git config --global user.name "searxng-bot" - - name: Merge and push transation updates - run: | - make V=1 weblate.translations.commit - - name: Create Pull Request - id: cpr - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.WEBLATE_GITHUB_TOKEN }} - commit-message: '[l10n] update translations from Weblate' - committer: searxng-bot - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: false - branch: translations_update - delete-branch: true - draft: false - title: '[l10n] update translations from Weblate' - body: | - update translations from Weblate - labels: | - translation diff --git a/Dockerfile b/Dockerfile index 9aeb28214..f05d2a1fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,6 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ brotli \ - # lxml - libxml2-dev \ - libxslt1-dev \ - zlib1g-dev \ # uwsgi libpcre3-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.compat b/Dockerfile.compat new file mode 100644 index 000000000..4b44d9537 --- /dev/null +++ b/Dockerfile.compat @@ -0,0 +1,104 @@ +FROM docker.io/library/python:3.13-slim AS builder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + brotli \ + # lxml + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev \ + # uwsgi + libpcre3-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /usr/local/searxng/ + +COPY ./requirements.txt ./requirements.txt + +RUN --mount=type=cache,id=pip,target=$HOME/.cache/pip python -m venv ./venv \ + && . ./venv/bin/activate \ + && pip install -r requirements.txt \ + && pip install "uwsgi~=2.0" + +COPY ./searx/ ./searx/ + +ARG TIMESTAMP_SETTINGS=0 +ARG TIMESTAMP_UWSGI=0 + +RUN python -m compileall -q searx \ + && touch -c --date=@$TIMESTAMP_SETTINGS ./searx/settings.yml \ + && touch -c --date=@$TIMESTAMP_UWSGI ./dockerfiles/uwsgi.ini \ + && find /usr/local/searxng/searx/static \ + \( -name '*.html' -o -name '*.css' -o -name '*.js' -o -name '*.svg' -o -name '*.ttf' -o -name '*.eot' \) \ + -type f -exec gzip -9 -k {} + -exec brotli --best {} + + +ARG SEARXNG_UID=977 +ARG SEARXNG_GID=977 + +RUN grep -m1 root /etc/group > /tmp/.searxng.group \ + && grep -m1 root /etc/passwd > /tmp/.searxng.passwd \ + && echo "searxng:x:$SEARXNG_GID:" >> /tmp/.searxng.group \ + && echo "searxng:x:$SEARXNG_UID:$SEARXNG_GID:searxng:/usr/local/searxng:/bin/bash" >> /tmp/.searxng.passwd + +FROM docker.io/library/python:3.13-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + # uwsgi + libpcre3 \ + libxml2 \ + mailcap \ + && rm -rf /var/lib/apt/lists/* + +COPY --chown=root:root --from=builder /tmp/.searxng.passwd /etc/passwd +COPY --chown=root:root --from=builder /tmp/.searxng.group /etc/group + +ARG LABEL_DATE="0001-01-01T00:00:00Z" +ARG GIT_URL="unspecified" +ARG SEARXNG_GIT_VERSION="unspecified" +ARG LABEL_VCS_REF="unspecified" +ARG LABEL_VCS_URL="unspecified" + +WORKDIR /usr/local/searxng/ + +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/venv/ ./venv/ +COPY --chown=searxng:searxng --from=builder /usr/local/searxng/searx/ ./searx/ +COPY --chown=searxng:searxng ./dockerfiles/ ./dockerfiles/ + +LABEL org.opencontainers.image.authors="searxng <$GIT_URL>" \ + org.opencontainers.image.created=$LABEL_DATE \ + org.opencontainers.image.description="A privacy-respecting, hackable metasearch engine" \ + org.opencontainers.image.documentation="https://github.com/searxng/searxng-docker" \ + org.opencontainers.image.licenses="AGPL-3.0-or-later" \ + org.opencontainers.image.revision=$LABEL_VCS_REF \ + org.opencontainers.image.source=$LABEL_VCS_URL \ + org.opencontainers.image.title="searxng" \ + org.opencontainers.image.url=$LABEL_VCS_URL \ + org.opencontainers.image.version=$SEARXNG_GIT_VERSION + +ENV CONFIG_PATH=/etc/searxng \ + DATA_PATH=/var/cache/searxng + +ENV SEARXNG_VERSION=$SEARXNG_GIT_VERSION \ + INSTANCE_NAME=searxng \ + AUTOCOMPLETE="" \ + BASE_URL="" \ + BIND_ADDRESS=[::]:8080 \ + MORTY_KEY="" \ + MORTY_URL="" \ + SEARXNG_SETTINGS_PATH=$CONFIG_PATH/settings.yml \ + UWSGI_SETTINGS_PATH=$CONFIG_PATH/uwsgi.ini \ + UWSGI_WORKERS=%k \ + UWSGI_THREADS=4 + +VOLUME $CONFIG_PATH +VOLUME $DATA_PATH + +EXPOSE 8080 + +USER searxng:searxng + +HEALTHCHECK CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1 + +ENTRYPOINT ["/usr/local/searxng/dockerfiles/docker-entrypoint.sh"] diff --git a/manage b/manage index 91f045d34..654a08ac0 100755 --- a/manage +++ b/manage @@ -178,8 +178,7 @@ container.build() { platform="linux/$arch" ;; "ARMV7" | "armhf" | "armv7l" | "armv7") - # TODO: Move ARMv7 to a separated Dockerfile - dockerfile="Dockerfile" + dockerfile="Dockerfile.compat" arch="arm" variant="v7" platform="linux/$arch/$variant" @@ -239,8 +238,8 @@ container.build() { params_build_builder="$docker_builder --platform=$platform --target=builder" params_build="$docker_builder --platform=$platform --squash" else - params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false" - params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false" + params_build_builder="build --format=docker --platform=$platform --target=builder --layers --identity-label=false" + params_build="build --format=docker --platform=$platform --layers --squash-all --omit-history --identity-label=false" fi # Define container image org/name @@ -407,8 +406,7 @@ ci.container.build() { platform="linux/$arch" ;; "ARMV7" | "armhf" | "armv7l" | "armv7") - # TODO: Move ARMv7 to a separated Dockerfile - dockerfile="Dockerfile" + dockerfile="Dockerfile.compat" arch="arm" variant="v7" platform="linux/$arch/$variant" @@ -459,7 +457,7 @@ ci.container.build() { build_msg CONTAINER "Building..." - podman build --platform="$platform" --target=builder --layers --identity-label=false \ + podman build --format=docker --platform="$platform" --target=builder --layers --identity-label=false \ --cache-from="ghcr.io/$container_image_organization/cache" \ --cache-to="ghcr.io/$container_image_organization/cache" \ --build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \ @@ -467,7 +465,7 @@ ci.container.build() { --tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \ --file="./$dockerfile" - podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \ + podman build --format=docker --platform="$platform" --layers --squash-all --omit-history --identity-label=false \ --cache-from="ghcr.io/$container_image_organization/cache" \ --cache-to="ghcr.io/$container_image_organization/cache" \ --build-arg="GIT_URL=$GIT_URL" \