diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 2d369d737..13ca7ffb2 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -22,7 +22,9 @@ env: jobs: build: - if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + # TODO: Uncomment + # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + if: false name: Build (${{ matrix.arch }}) runs-on: ${{ matrix.os }} strategy: @@ -53,6 +55,8 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: "false" + # make sure "make ci.container.build" can get the git branches + fetch-depth: "0" - name: Setup cache Python uses: actions/cache@v4 @@ -64,8 +68,8 @@ jobs: - name: Setup cache container mounts uses: actions/cache@v4 with: - key: "container-mounts-${{ runner.arch }}-${{ hashFiles('./Dockerfile*') }}" - restore-keys: "container-mounts-${{ runner.arch }}-" + key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./Dockerfile*') }}" + restore-keys: "container-mounts-${{ matrix.arch }}-" path: | /var/tmp/buildah-cache/ /var/tmp/buildah-cache-*/ @@ -84,7 +88,53 @@ jobs: - name: Build env: OVERRIDE_ARCH: "${{ matrix.arch }}" - run: make -e GIT_URL=$(git remote get-url origin) ci.container.build + run: make ci.container.build + + test: + name: Test (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + needs: build + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + # make sure "make ci.container.test" can get the git branches + fetch-depth: "0" + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Test + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make ci.container.test release: # TODO: Uncomment before merge @@ -92,7 +142,7 @@ jobs: if: false name: Release (${{ matrix.arch }}) runs-on: ubuntu-24.04-arm - needs: build + needs: test strategy: fail-fast: false matrix: @@ -130,5 +180,4 @@ jobs: name: Release env: OVERRIDE_ARCH: "${{ matrix.arch }}" - # TODO: Try this with GHCR or testing registry before merge - run: make -e GIT_URL=$(git remote get-url origin) ci.container.push + run: make ci.container.push diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7de57345b..25e9268b8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,7 +9,7 @@ on: - master pull_request: branches: - - master + - container-gha permissions: contents: read @@ -80,3 +80,119 @@ jobs: - name: Build run: make themes.all + + # TODO: Remove this + container-build: + # TODO: Uncomment + # if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + name: Build (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + # make sure "make ci.container.build" can get the git branches + fetch-depth: "0" + + - name: Setup cache Python + uses: actions/cache@v4 + with: + key: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-${{ hashFiles('./requirements*.txt') }}" + restore-keys: "python-${{ env.PYTHON_VERSION }}-${{ runner.arch }}-" + path: "./local/" + + - name: Setup cache container mounts + uses: actions/cache@v4 + with: + key: "container-mounts-${{ matrix.arch }}-${{ hashFiles('./Dockerfile*') }}" + restore-keys: "container-mounts-${{ matrix.arch }}-" + path: | + /var/tmp/buildah-cache/ + /var/tmp/buildah-cache-*/ + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Build + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make ci.container.build + + # TODO: Remove this + container-test: + name: Test (${{ matrix.arch }}) + runs-on: ${{ matrix.os }} + needs: container-build + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + os: ubuntu-24.04 + emulation: false + - arch: arm64 + os: ubuntu-24.04-arm + emulation: false + - arch: armv7 + os: ubuntu-24.04-arm + emulation: true + + permissions: + # Organization GHCR + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: "false" + # make sure "make ci.container.test" can get the git branches + # fetch-depth: "0" + + - if: ${{ matrix.emulation }} + name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: "ghcr.io" + username: "${{ github.repository_owner }}" + password: "${{ secrets.GITHUB_TOKEN }}" + + - name: Test + env: + OVERRIDE_ARCH: "${{ matrix.arch }}" + run: make ci.container.test diff --git a/.github/workflows/l10n.yml b/.github/workflows/l10n.yml index 4a49cfbfa..6975ea32d 100644 --- a/.github/workflows/l10n.yml +++ b/.github/workflows/l10n.yml @@ -22,7 +22,9 @@ env: jobs: update: - if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' + # TODO: Uncomment + # if: github.repository_owner == 'searxng' && github.event.workflow_run.conclusion == 'success' + if: false name: Update runs-on: ubuntu-24.04-arm permissions: diff --git a/Dockerfile b/Dockerfile index f05d2a1fd..f31a1183b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,7 @@ WORKDIR /usr/local/searxng/ COPY ./requirements.txt ./requirements.txt -# Readd on #4707 "--mount=type=cache,id=pip,target=/root/.cache/pip" -RUN python -m venv ./venv \ +RUN --mount=type=cache,id=pip,target=/root/.cache/pip python -m venv ./venv \ && . ./venv/bin/activate \ && pip install -r requirements.txt \ && pip install "uwsgi~=2.0" diff --git a/Makefile b/Makefile index 977143582..0b35621cc 100644 --- a/Makefile +++ b/Makefile @@ -88,7 +88,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover MANAGE += themes.all themes.fix themes.test MANAGE += static.build.commit static.build.drop static.build.restore MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs -MANAGE += ci.container.build ci.container.push +MANAGE += ci.container.build ci.container.test ci.container.push PHONY += $(MANAGE) diff --git a/manage b/manage index 654a08ac0..7bdefc312 100755 --- a/manage +++ b/manage @@ -429,7 +429,7 @@ ci.container.build() { fi ( - set -eu + set -e pyenv.activate # Check if it is a git repository @@ -446,6 +446,13 @@ ci.container.build() { python -m searx.version freeze eval "$(python -m searx.version)" + # TODO: Remove this + echo $VERSION_STRING + echo $VERSION_TAG + echo $DOCKER_TAG + echo $GIT_URL + echo $GIT_BRANCH + # Get the last git commit id version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2) build_msg CONTAINER "Last commit: $version_gitcommit" @@ -481,6 +488,77 @@ ci.container.build() { dump_return $? } +ci.container.test() { + if ! "$GITHUB_ACTIONS"; then + die 1 "This command is intended to be run in GitHub Actions" + fi + + local parch=${OVERRIDE_ARCH:-$(uname -m)} + local arch + local variant + local platform + + # Setup arch specific + case $parch in + "X64" | "x86_64" | "amd64") + arch="amd64" + variant="" + platform="linux/$arch" + ;; + "ARM64" | "aarch64" | "arm64") + arch="arm64" + variant="" + platform="linux/$arch" + ;; + "ARMV7" | "armhf" | "armv7l" | "armv7") + arch="arm" + variant="v7" + platform="linux/$arch/$variant" + ;; + *) + err_msg "Unsupported architecture; (PARCH=\"$parch\")" + exit 1 + ;; + esac + build_msg CONTAINER "Selected platform: $platform" + + # Check if podman is installed + if ! command -v podman &>/dev/null; then + die 1 "podman is not installed" + fi + + ( + set -e + + # Define container image org/name + git_url="$(git remote get-url origin)" + # shellcheck disable=SC2001 + container_image_organization="$(echo "$git_url" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_name="searxng" + + podman pull "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" + + name="$container_image_name-$(date +%N)" + + podman create --name="$name" --rm --timeout=60 --network="host" \ + "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" >/dev/null + + podman start "$name" >/dev/null + podman logs -f "$name" & + pid_logs=$! + + # Wait until container is ready + sleep 5 + + # TODO: Test failing + curl -vf --max-time 5 "http://localhost:8080/healthz" + + kill $pid_logs &>/dev/null || true + podman stop "$name" >/dev/null + ) + dump_return $? +} + ci.container.push() { if ! "$GITHUB_ACTIONS"; then die 1 "This command is intended to be run in GitHub Actions" @@ -521,11 +599,12 @@ ci.container.push() { fi ( - set -eu + set -e # Define container image org/name + git_url="$(git remote get-url origin)" # shellcheck disable=SC2001 - container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" + container_image_organization="$(echo "$git_url" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")" container_image_name="searxng" podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant"