Initial update
This commit is contained in:
parent
bc06b1aece
commit
9b8517ac8f
4
.github/workflows/checker.yml
vendored
4
.github/workflows/checker.yml
vendored
@ -1,5 +1,7 @@
|
||||
name: "Checker"
|
||||
on: # yamllint disable-line rule:truthy
|
||||
|
||||
# yamllint disable-line rule:truthy
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 4 * * 5"
|
||||
workflow_dispatch:
|
||||
|
127
.github/workflows/container.yml
vendored
Normal file
127
.github/workflows/container.yml
vendored
Normal file
@ -0,0 +1,127 @@
|
||||
---
|
||||
name: Container
|
||||
|
||||
# yamllint disable-line rule:truthy
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- Integration
|
||||
types:
|
||||
- completed
|
||||
# TODO: Uncomment
|
||||
# branches:
|
||||
# - master
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# Organization GHCR
|
||||
packages: read
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.13"
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.event.workflow_run.conclusion == 'success'
|
||||
name: Build (${{ matrix.arch }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
os: ubuntu-24.04
|
||||
- arch:
|
||||
arm64
|
||||
armv7
|
||||
os: ubuntu-24.04-arm
|
||||
|
||||
permissions:
|
||||
# Organization GHCR
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "${{ env.PYTHON_VERSION }}"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Setup cache Python
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: "python-${{ env.PYTHON_VERSION }}-${{ hashFiles('./requirements*.txt') }}"
|
||||
restore-keys: "python-${{ env.PYTHON_VERSION }}-"
|
||||
path: "./local"
|
||||
|
||||
- name: Setup cache container mounts
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
key: "container-mounts-${{ hashFiles('./Dockerfile') }}"
|
||||
restore-keys: "container-mounts-"
|
||||
path: |
|
||||
/var/tmp/buildah-cache/
|
||||
/var/tmp/buildah-cache-*/
|
||||
|
||||
- name: Setup QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: "ghcr.io"
|
||||
username: "${{ github.repository_owner }}"
|
||||
password: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
OVERRIDE_ARCH: "${{ matrix.arch }}"
|
||||
run: make -e GIT_URL=$(git remote get-url origin) ci.container.build
|
||||
|
||||
release:
|
||||
# TODO: Uncomment before merge
|
||||
# if: github.repository_owner == 'searxng'
|
||||
if: false
|
||||
name: Release (${{ matrix.arch }})
|
||||
runs-on: ubuntu-24.04-arm
|
||||
needs: build
|
||||
strategy:
|
||||
matrix:
|
||||
arch:
|
||||
- amd64
|
||||
- arm64
|
||||
- armv7
|
||||
|
||||
steps:
|
||||
- if: env.DOCKERHUB_USERNAME != ''
|
||||
name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
# make sure "make ci.container.push" can get the git history
|
||||
fetch-depth: "0"
|
||||
|
||||
- if: env.DOCKERHUB_USERNAME != ''
|
||||
name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: "ghcr.io"
|
||||
username: "${{ github.repository_owner }}"
|
||||
password: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- if: env.DOCKERHUB_USERNAME != ''
|
||||
name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: "docker.io"
|
||||
username: "${{ env.DOCKERHUB_USERNAME }}"
|
||||
password: "${{ secrets.DOCKERHUB_TOKEN }}"
|
||||
|
||||
- if: env.DOCKERHUB_USERNAME != ''
|
||||
name: Release
|
||||
env:
|
||||
OVERRIDE_ARCH: "${{ matrix.arch }}"
|
||||
run: make -e GIT_URL=$(git remote get-url origin) ci.container.push
|
8
.github/workflows/data-update.yml
vendored
8
.github/workflows/data-update.yml
vendored
@ -1,8 +1,10 @@
|
||||
name: "Update searx.data"
|
||||
on: # yamllint disable-line rule:truthy
|
||||
name: Update searx.data
|
||||
|
||||
# yamllint disable-line rule:truthy
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "59 23 28 * *"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
updateData:
|
||||
|
156
.github/workflows/integration.yml
vendored
156
.github/workflows/integration.yml
vendored
@ -1,70 +1,96 @@
|
||||
name: Integration
|
||||
|
||||
on: # yamllint disable-line rule:truthy
|
||||
# yamllint disable-line rule:truthy
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: ["master"]
|
||||
branches:
|
||||
- master
|
||||
pull_request:
|
||||
branches: ["master"]
|
||||
branches:
|
||||
- master
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
python:
|
||||
# TODO: Remove this
|
||||
if: false
|
||||
name: Python ${{ matrix.python-version }}
|
||||
runs-on: ubuntu-24.04
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-24.04]
|
||||
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
|
||||
python-version:
|
||||
- "3.9"
|
||||
- "3.10"
|
||||
- "3.11"
|
||||
- "3.12"
|
||||
- "3.13"
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Install Ubuntu packages
|
||||
run: |
|
||||
sudo ./utils/searxng.sh install packages
|
||||
run: sudo ./utils/searxng.sh install packages
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
architecture: 'x64'
|
||||
python-version: "${{ matrix.python-version }}"
|
||||
|
||||
- name: Run tests
|
||||
run: make V=1 ci.test
|
||||
|
||||
themes:
|
||||
# TODO: Remove this
|
||||
if: false
|
||||
name: Themes
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: "false"
|
||||
|
||||
- name: Install Ubuntu packages
|
||||
run: sudo ./utils/searxng.sh install buildhost
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
architecture: 'x64'
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Build themes
|
||||
run: make themes.all
|
||||
|
||||
documentation:
|
||||
# TODO: Remove this
|
||||
if: false
|
||||
name: Documentation
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write # for JamesIves/github-pages-deploy-action to push changes in repo
|
||||
# for JamesIves/github-pages-deploy-action to push
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: '0'
|
||||
persist-credentials: false
|
||||
persist-credentials: "false"
|
||||
fetch-depth: "0"
|
||||
|
||||
- name: Install Ubuntu packages
|
||||
run: sudo ./utils/searxng.sh install buildhost
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
architecture: 'x64'
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Cache Python dependencies
|
||||
id: cache-python
|
||||
uses: actions/cache@v4
|
||||
@ -73,42 +99,50 @@ jobs:
|
||||
./local
|
||||
./.nvm
|
||||
./node_modules
|
||||
key: python-ubuntu-24.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||
key: python-ubuntu-24.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||
|
||||
- name: Build documentation
|
||||
run: |
|
||||
make V=1 docs.clean docs.html
|
||||
- name: Deploy
|
||||
if: github.ref == 'refs/heads/master'
|
||||
run: make V=1 docs.clean docs.html
|
||||
|
||||
- if: github.ref == 'refs/heads/master'
|
||||
name: Deploy
|
||||
uses: JamesIves/github-pages-deploy-action@3.7.1
|
||||
with:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
BRANCH: gh-pages
|
||||
FOLDER: dist/docs
|
||||
CLEAN: true # Automatically remove deleted files from the deploy branch
|
||||
SINGLE_COMMIT: true
|
||||
COMMIT_MESSAGE: '[doc] build from commit ${{ github.sha }}'
|
||||
GITHUB_TOKEN: "${{ github.token }}"
|
||||
BRANCH: "gh-pages"
|
||||
FOLDER: "dist/docs"
|
||||
# Automatically remove deleted files from the deploy branch
|
||||
CLEAN: "true"
|
||||
SINGLE_COMMIT: "true"
|
||||
COMMIT_MESSAGE: "[doc] build from commit ${{ github.sha }}"
|
||||
|
||||
babel:
|
||||
# TODO: Remove this
|
||||
if: false
|
||||
# if: github.repository_owner == 'searxng' && github.ref == 'refs/heads/master'
|
||||
name: Update translations branch
|
||||
runs-on: ubuntu-24.04
|
||||
if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
|
||||
needs:
|
||||
- python
|
||||
- themes
|
||||
- documentation
|
||||
|
||||
permissions:
|
||||
contents: write # for make V=1 weblate.push.translations
|
||||
# for make V=1 weblate.push.translations
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: '0'
|
||||
token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
|
||||
token: "${{ secrets.WEBLATE_GITHUB_TOKEN }}"
|
||||
fetch-depth: "0"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
architecture: 'x64'
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Cache Python dependencies
|
||||
id: cache-python
|
||||
uses: actions/cache@v4
|
||||
@ -117,63 +151,17 @@ jobs:
|
||||
./local
|
||||
./.nvm
|
||||
./node_modules
|
||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||
key: python-ubuntu-20.04-3.13-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||
|
||||
- name: weblate & git setup
|
||||
env:
|
||||
WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
|
||||
WEBLATE_CONFIG: "${{ secrets.WEBLATE_CONFIG }}"
|
||||
run: |
|
||||
mkdir -p ~/.config
|
||||
echo "${WEBLATE_CONFIG}" > ~/.config/weblate
|
||||
git config --global user.email "searxng-bot@users.noreply.github.com"
|
||||
git config --global user.name "searxng-bot"
|
||||
|
||||
- name: Update transations
|
||||
id: update
|
||||
run: |
|
||||
make V=1 weblate.push.translations
|
||||
|
||||
dockers:
|
||||
name: Docker
|
||||
if: github.ref == 'refs/heads/master'
|
||||
needs:
|
||||
- python
|
||||
- themes
|
||||
- documentation
|
||||
env:
|
||||
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
if: env.DOCKERHUB_USERNAME != null
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# make sure "make docker.push" can get the git history
|
||||
fetch-depth: '0'
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
architecture: 'x64'
|
||||
- name: Cache Python dependencies
|
||||
id: cache-python
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
./local
|
||||
./.nvm
|
||||
./node_modules
|
||||
key: python-ubuntu-20.04-3.12-${{ hashFiles('requirements*.txt', 'setup.py','.nvmrc', 'package.json') }}
|
||||
- name: Set up QEMU
|
||||
if: env.DOCKERHUB_USERNAME != null
|
||||
uses: docker/setup-qemu-action@v1
|
||||
- name: Set up Docker Buildx
|
||||
if: env.DOCKERHUB_USERNAME != null
|
||||
uses: docker/setup-buildx-action@v1
|
||||
- name: Login to DockerHub
|
||||
if: env.DOCKERHUB_USERNAME != null
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Build and push
|
||||
if: env.DOCKERHUB_USERNAME != null
|
||||
run: make -e GIT_URL=$(git remote get-url origin) docker.buildx
|
||||
run: make V=1 weblate.push.translations
|
||||
|
8
.github/workflows/security.yml
vendored
8
.github/workflows/security.yml
vendored
@ -1,8 +1,10 @@
|
||||
name: "Security checks"
|
||||
on: # yamllint disable-line rule:truthy
|
||||
name: Security checks
|
||||
|
||||
# yamllint disable-line rule:truthy
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "42 05 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
dockers:
|
||||
|
8
.github/workflows/translations-update.yml
vendored
8
.github/workflows/translations-update.yml
vendored
@ -1,8 +1,10 @@
|
||||
name: "Update translations"
|
||||
on: # yamllint disable-line rule:truthy
|
||||
name: Update translations
|
||||
|
||||
# yamllint disable-line rule:truthy
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "05 07 * * 5"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
babel:
|
||||
|
4
Makefile
4
Makefile
@ -77,7 +77,8 @@ test.shell:
|
||||
MANAGE += weblate.translations.commit weblate.push.translations
|
||||
MANAGE += data.all data.traits data.useragents data.locales data.currencies
|
||||
MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
|
||||
MANAGE += docker.build docker.push docker.buildx
|
||||
MANAGE += docker.build docker.buildx
|
||||
MANAGE += container.build container.buildx
|
||||
MANAGE += gecko.driver
|
||||
MANAGE += node.env node.env.dev node.clean
|
||||
MANAGE += py.build py.clean
|
||||
@ -87,6 +88,7 @@ MANAGE += test.yamllint test.pylint test.black test.pybabel test.unit test.cover
|
||||
MANAGE += themes.all themes.fix themes.test
|
||||
MANAGE += static.build.commit static.build.drop static.build.restore
|
||||
MANAGE += nvm.install nvm.clean nvm.status nvm.nodejs
|
||||
MANAGE += ci.container.build ci.container.push
|
||||
|
||||
PHONY += $(MANAGE)
|
||||
|
||||
|
415
manage
415
manage
@ -60,7 +60,7 @@ while IFS= read -r line; do
|
||||
if [ "$line" != "tests/unit/settings/syntaxerror_settings.yml" ]; then
|
||||
YAMLLINT_FILES+=("$line")
|
||||
fi
|
||||
done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
|
||||
done <<<"$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml' '.github/*.yml' '.github/*/*.yml')"
|
||||
|
||||
RST_FILES=(
|
||||
'README.rst'
|
||||
@ -77,9 +77,9 @@ docs.:
|
||||
gh-pages : deploy on gh-pages branch
|
||||
prebuild : build reST include files (./${DOCS_BUILD}/includes)
|
||||
clean : clean documentation build
|
||||
docker.:
|
||||
build : build docker image
|
||||
push : build and push docker image
|
||||
container. :
|
||||
build : build container image
|
||||
buildx : build container image with BuildKit
|
||||
gecko.driver:
|
||||
download & install geckodriver if not already installed (required for
|
||||
robot_tests)
|
||||
@ -97,6 +97,10 @@ pyenv.:
|
||||
OK : test if virtualenv is OK
|
||||
format.:
|
||||
python : format Python code source using black
|
||||
ci.:
|
||||
container.:
|
||||
build : build container image
|
||||
release : push container images to remote registry
|
||||
EOF
|
||||
go.help
|
||||
node.help
|
||||
@ -112,7 +116,6 @@ environment ...
|
||||
EOF
|
||||
}
|
||||
|
||||
|
||||
if [ "$VERBOSE" = "1" ]; then
|
||||
SPHINX_VERBOSE="-v"
|
||||
PYLINT_VERBOSE="-v"
|
||||
@ -125,53 +128,95 @@ webapp.run() {
|
||||
local parent_proc="$$"
|
||||
(
|
||||
if [ "${LIVE_THEME}" ]; then
|
||||
( themes.live "${LIVE_THEME}" )
|
||||
(themes.live "${LIVE_THEME}")
|
||||
kill $parent_proc
|
||||
fi
|
||||
)&
|
||||
) &
|
||||
(
|
||||
sleep 3
|
||||
xdg-open http://127.0.0.1:8888/
|
||||
)&
|
||||
) &
|
||||
SEARXNG_DEBUG=1 pyenv.cmd python -m searx.webapp
|
||||
}
|
||||
|
||||
docker.push() {
|
||||
docker.build push
|
||||
}
|
||||
|
||||
# Alias
|
||||
docker.buildx() {
|
||||
docker.build buildx
|
||||
container.buildx
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2119
|
||||
# Alias
|
||||
docker.build() {
|
||||
container.build
|
||||
}
|
||||
|
||||
container.buildx() {
|
||||
container.build buildx
|
||||
}
|
||||
|
||||
container.build() {
|
||||
pyenv.install
|
||||
|
||||
local SEARXNG_GIT_VERSION
|
||||
local VERSION_GITCOMMIT
|
||||
local GITHUB_USER
|
||||
local SEARXNG_IMAGE_NAME
|
||||
local BUILD
|
||||
local parch=${OVERRIDE_ARCH:-$(uname -m)}
|
||||
local container_engine
|
||||
local dockerfile
|
||||
local arch
|
||||
local variant
|
||||
local platform
|
||||
|
||||
build_msg DOCKER build
|
||||
# run installation in a subprocess and activate pyenv
|
||||
# Setup arch specific
|
||||
case $parch in
|
||||
"X64" | "x86_64" | "amd64")
|
||||
dockerfile="Dockerfile"
|
||||
arch="amd64"
|
||||
variant=""
|
||||
platform="linux/$arch"
|
||||
;;
|
||||
"ARM64" | "aarch64" | "arm64")
|
||||
dockerfile="Dockerfile"
|
||||
arch="arm64"
|
||||
variant=""
|
||||
platform="linux/$arch"
|
||||
;;
|
||||
"ARMV7" | "armhf" | "armv7l" | "armv7")
|
||||
# TODO: Move ARMv7 to a separated Dockerfile
|
||||
dockerfile="Dockerfile"
|
||||
arch="arm"
|
||||
variant="v7"
|
||||
platform="linux/$arch/$variant"
|
||||
;;
|
||||
*)
|
||||
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
build_msg CONTAINER "Selected platform: $platform"
|
||||
|
||||
# See https://www.shellcheck.net/wiki/SC1001 and others ..
|
||||
# shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
|
||||
( set -e
|
||||
# Check if podman or docker is installed
|
||||
if command -v podman &>/dev/null; then
|
||||
container_engine="podman"
|
||||
elif command -v docker &>/dev/null; then
|
||||
container_engine="docker"
|
||||
else
|
||||
die 1 "podman/docker is not installed"
|
||||
fi
|
||||
build_msg CONTAINER "Engine: $container_engine"
|
||||
|
||||
# Check if git is installed
|
||||
if ! command -v git &>/dev/null; then
|
||||
die 1 "git is not installed"
|
||||
fi
|
||||
|
||||
(
|
||||
set -e
|
||||
pyenv.activate
|
||||
|
||||
# Check if it is a git repository
|
||||
if [ ! -d .git ]; then
|
||||
die 1 "This is not Git repository"
|
||||
fi
|
||||
if [ ! -x "$(which git)" ]; then
|
||||
die 1 "git is not installed"
|
||||
die 1 "This is not Git repository"
|
||||
fi
|
||||
|
||||
if ! git remote get-url origin 2> /dev/null; then
|
||||
die 1 "there is no remote origin"
|
||||
if ! git remote get-url origin 2>/dev/null; then
|
||||
die 1 "there is no remote origin"
|
||||
fi
|
||||
|
||||
# This is a git repository
|
||||
@ -180,42 +225,48 @@ docker.build() {
|
||||
eval "$(python -m searx.version)"
|
||||
|
||||
# Get the last git commit id
|
||||
VERSION_GITCOMMIT=$(echo "$VERSION_TAG" | cut -d+ -f2)
|
||||
build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
|
||||
version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
|
||||
build_msg CONTAINER "Last commit: $version_gitcommit"
|
||||
|
||||
# define the docker image name
|
||||
GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
|
||||
SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
|
||||
if [ "$container_engine" = "docker" ]; then
|
||||
if [ "$1" = "buildx" ]; then
|
||||
docker_builder="buildx build"
|
||||
else
|
||||
docker_builder="build"
|
||||
warn_msg "The legacy builder is deprecated and will be removed in a future release: https://docs.docker.com/engine/deprecated/#legacy-builder-fallback"
|
||||
fi
|
||||
|
||||
BUILD="build"
|
||||
if [ "$1" = "buildx" ]; then
|
||||
# buildx includes the push option
|
||||
CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache"
|
||||
BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
|
||||
shift
|
||||
params_build_builder="$docker_builder --platform=$platform --target=builder"
|
||||
params_build="$docker_builder --platform=$platform --squash"
|
||||
else
|
||||
params_build_builder="build --platform=$platform --target=builder --layers --identity-label=false"
|
||||
params_build="build --platform=$platform --layers --squash-all --omit-history --identity-label=false"
|
||||
fi
|
||||
build_msg DOCKER "Build command: ${BUILD}"
|
||||
|
||||
# build Docker image
|
||||
build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
|
||||
# Define container image org/name
|
||||
# shellcheck disable=SC2001
|
||||
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
|
||||
container_image_name="searxng"
|
||||
|
||||
build_msg CONTAINER "Building..."
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
docker $BUILD \
|
||||
--build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
|
||||
--build-arg GIT_URL="${GIT_URL}" \
|
||||
--build-arg SEARXNG_DOCKER_TAG="${DOCKER_TAG}" \
|
||||
--build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \
|
||||
--build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
|
||||
--build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
|
||||
--build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
|
||||
--build-arg LABEL_VCS_URL="${GIT_URL}" \
|
||||
--build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
|
||||
--build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
|
||||
-t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}" .
|
||||
"$container_engine" $params_build_builder \
|
||||
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
|
||||
--build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
|
||||
--tag="localhost/$container_image_organization/$container_image_name:builder" \
|
||||
--file="./$dockerfile"
|
||||
|
||||
if [ "$1" = "push" ]; then
|
||||
docker push "${SEARXNG_IMAGE_NAME}:latest"
|
||||
docker push "${SEARXNG_IMAGE_NAME}:${DOCKER_TAG}"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
"$container_engine" $params_build \
|
||||
--build-arg="GIT_URL=$GIT_URL" \
|
||||
--build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
|
||||
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
|
||||
--build-arg="LABEL_VCS_URL=$GIT_URL" \
|
||||
--tag="localhost/$container_image_organization/$container_image_name:latest" \
|
||||
--tag="localhost/$container_image_organization/$container_image_name:$DOCKER_TAG" \
|
||||
--file="./$dockerfile"
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
@ -226,10 +277,11 @@ gecko.driver() {
|
||||
|
||||
build_msg INSTALL "gecko.driver"
|
||||
# run installation in a subprocess and activate pyenv
|
||||
( set -e
|
||||
(
|
||||
set -e
|
||||
pyenv.activate
|
||||
|
||||
INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
|
||||
INSTALLED_VERSION=$(geckodriver -V 2>/dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
|
||||
set +e
|
||||
if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
|
||||
build_msg INSTALL "geckodriver already installed"
|
||||
@ -237,13 +289,13 @@ gecko.driver() {
|
||||
fi
|
||||
PLATFORM="$(python -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
|
||||
case "$PLATFORM" in
|
||||
"linux 32bit" | "linux2 32bit") ARCH="linux32";;
|
||||
"linux 64bit" | "linux2 64bit") ARCH="linux64";;
|
||||
"windows 32 bit") ARCH="win32";;
|
||||
"windows 64 bit") ARCH="win64";;
|
||||
"mac 64bit") ARCH="macos";;
|
||||
"linux 32bit" | "linux2 32bit") ARCH="linux32" ;;
|
||||
"linux 64bit" | "linux2 64bit") ARCH="linux64" ;;
|
||||
"windows 32 bit") ARCH="win32" ;;
|
||||
"windows 64 bit") ARCH="win64" ;;
|
||||
"mac 64bit") ARCH="macos" ;;
|
||||
esac
|
||||
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
|
||||
GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz"
|
||||
|
||||
build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
|
||||
|
||||
@ -258,13 +310,14 @@ gecko.driver() {
|
||||
py.build() {
|
||||
build_msg BUILD "python package ${PYDIST}"
|
||||
pyenv.cmd python setup.py \
|
||||
sdist -d "${PYDIST}" \
|
||||
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
|
||||
sdist -d "${PYDIST}" \
|
||||
bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
|
||||
}
|
||||
|
||||
py.clean() {
|
||||
build_msg CLEAN pyenv
|
||||
( set -e
|
||||
(
|
||||
set -e
|
||||
pyenv.drop
|
||||
[ "$VERBOSE" = "1" ] && set -x
|
||||
rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
|
||||
@ -275,22 +328,22 @@ py.clean() {
|
||||
}
|
||||
|
||||
pyenv.check() {
|
||||
cat <<EOF
|
||||
cat <<EOF
|
||||
import yaml
|
||||
print('import yaml --> OK')
|
||||
EOF
|
||||
}
|
||||
|
||||
pyenv.install() {
|
||||
|
||||
if ! pyenv.OK; then
|
||||
py.clean > /dev/null
|
||||
py.clean >/dev/null
|
||||
fi
|
||||
if pyenv.install.OK > /dev/null; then
|
||||
if pyenv.install.OK >/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
( set -e
|
||||
(
|
||||
set -e
|
||||
pyenv
|
||||
build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e 'searx${PY_SETUP_EXTRAS}'"
|
||||
"${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e ".${PY_SETUP_EXTRAS}"
|
||||
@ -303,8 +356,8 @@ pyenv.install() {
|
||||
|
||||
pyenv.uninstall() {
|
||||
build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
|
||||
pyenv.cmd python setup.py develop --uninstall 2>&1 \
|
||||
| prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
|
||||
pyenv.cmd python setup.py develop --uninstall 2>&1 |
|
||||
prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
|
||||
|
||||
}
|
||||
|
||||
@ -320,17 +373,184 @@ docs.prebuild() {
|
||||
set -e
|
||||
[ "$VERBOSE" = "1" ] && set -x
|
||||
mkdir -p "${DOCS_BUILD}/includes"
|
||||
./utils/searxng.sh searxng.doc.rst > "${DOCS_BUILD}/includes/searxng.rst"
|
||||
./utils/searxng.sh searxng.doc.rst >"${DOCS_BUILD}/includes/searxng.rst"
|
||||
pyenv.cmd searxng_extra/docs_prebuild
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
ci.container.build() {
|
||||
if ! "$GITHUB_ACTIONS"; then
|
||||
die 1 "This command is intended to be run in GitHub Actions"
|
||||
fi
|
||||
|
||||
pyenv.install
|
||||
|
||||
local parch=${OVERRIDE_ARCH:-$(uname -m)}
|
||||
local dockerfile
|
||||
local arch
|
||||
local variant
|
||||
local platform
|
||||
|
||||
# Setup arch specific
|
||||
case $parch in
|
||||
"X64" | "x86_64" | "amd64")
|
||||
dockerfile="Dockerfile"
|
||||
arch="amd64"
|
||||
variant=""
|
||||
platform="linux/$arch"
|
||||
;;
|
||||
"ARM64" | "aarch64" | "arm64")
|
||||
dockerfile="Dockerfile"
|
||||
arch="arm64"
|
||||
variant=""
|
||||
platform="linux/$arch"
|
||||
;;
|
||||
"ARMV7" | "armhf" | "armv7l" | "armv7")
|
||||
# TODO: Move ARMv7 to a separated Dockerfile
|
||||
dockerfile="Dockerfile"
|
||||
arch="arm"
|
||||
variant="v7"
|
||||
platform="linux/$arch/$variant"
|
||||
;;
|
||||
*)
|
||||
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
build_msg CONTAINER "Selected platform: $platform"
|
||||
|
||||
# Check if podman is installed
|
||||
if ! command -v podman &>/dev/null; then
|
||||
die 1 "podman is not installed"
|
||||
fi
|
||||
|
||||
# Check if git is installed
|
||||
if ! command -v git &>/dev/null; then
|
||||
die 1 "git is not installed"
|
||||
fi
|
||||
|
||||
(
|
||||
set -eu
|
||||
pyenv.activate
|
||||
|
||||
# Check if it is a git repository
|
||||
if [ ! -d .git ]; then
|
||||
die 1 "This is not Git repository"
|
||||
fi
|
||||
|
||||
if ! git remote get-url origin 2>/dev/null; then
|
||||
die 1 "there is no remote origin"
|
||||
fi
|
||||
|
||||
# This is a git repository
|
||||
git update-index -q --refresh
|
||||
python -m searx.version freeze
|
||||
eval "$(python -m searx.version)"
|
||||
|
||||
# Get the last git commit id
|
||||
version_gitcommit=$(echo "$VERSION_TAG" | cut -d+ -f2)
|
||||
build_msg CONTAINER "Last commit: $version_gitcommit"
|
||||
|
||||
# Define container image org/name
|
||||
# shellcheck disable=SC2001
|
||||
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
|
||||
container_image_name="searxng"
|
||||
|
||||
build_msg CONTAINER "Building..."
|
||||
|
||||
podman build --platform="$platform" --target=builder --layers --identity-label=false \
|
||||
--cache-from="ghcr.io/$container_image_organization/cache" \
|
||||
--cache-to="ghcr.io/$container_image_organization/cache" \
|
||||
--build-arg="TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- ./searx/settings.yml)" \
|
||||
--build-arg="TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- ./dockerfiles/uwsgi.ini)" \
|
||||
--tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant-builder" \
|
||||
--file="./$dockerfile"
|
||||
|
||||
podman build --platform="$platform" --layers --squash-all --omit-history --identity-label=false \
|
||||
--cache-from="ghcr.io/$container_image_organization/cache" \
|
||||
--cache-to="ghcr.io/$container_image_organization/cache" \
|
||||
--build-arg="GIT_URL=$GIT_URL" \
|
||||
--build-arg="SEARXNG_GIT_VERSION=$VERSION_STRING" \
|
||||
--build-arg="LABEL_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
--build-arg="LABEL_VCS_REF=$(git rev-parse HEAD)" \
|
||||
--build-arg="LABEL_VCS_URL=$GIT_URL" \
|
||||
--tag="ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant" \
|
||||
--file="./$dockerfile"
|
||||
|
||||
podman push "ghcr.io/$container_image_organization/cache:$container_image_name-$arch$variant"
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
ci.container.push() {
|
||||
if ! "$GITHUB_ACTIONS"; then
|
||||
die 1 "This command is intended to be run in GitHub Actions"
|
||||
fi
|
||||
|
||||
local parch=${OVERRIDE_ARCH:-$(uname -m)}
|
||||
local arch
|
||||
local variant
|
||||
local platform
|
||||
|
||||
# Setup arch specific
|
||||
case $parch in
|
||||
"X64" | "x86_64" | "amd64")
|
||||
arch="amd64"
|
||||
variant=""
|
||||
platform="linux/$arch"
|
||||
;;
|
||||
"ARM64" | "aarch64" | "arm64")
|
||||
arch="arm64"
|
||||
variant=""
|
||||
platform="linux/$arch"
|
||||
;;
|
||||
"ARMV7" | "armhf" | "armv7l" | "armv7")
|
||||
arch="arm"
|
||||
variant="v7"
|
||||
platform="linux/$arch/$variant"
|
||||
;;
|
||||
*)
|
||||
err_msg "Unsupported architecture; (PARCH=\"$parch\")"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
build_msg CONTAINER "Selected platform: $platform"
|
||||
|
||||
# Check if podman is installed
|
||||
if ! command -v podman &>/dev/null; then
|
||||
die 1 "podman is not installed"
|
||||
fi
|
||||
|
||||
(
|
||||
set -eu
|
||||
|
||||
# Define container image org/name
|
||||
# shellcheck disable=SC2001
|
||||
container_image_organization="$(echo "$GIT_URL" | sed 's|.*github\.com/\([^/]*\).*|\1|' || echo "searxng")"
|
||||
container_image_name="searxng"
|
||||
|
||||
podman pull "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant"
|
||||
|
||||
# Get version tag
|
||||
container_image_tag_version=$(podman inspect --format='{{index .Config.Labels "org.opencontainers.image.revision"}}' \
|
||||
"ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant")
|
||||
|
||||
# Recreate tags
|
||||
podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:latest"
|
||||
podman tag "ghcr.io/$container_image_organization/$container_image_name:cache-$arch$variant" "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
|
||||
|
||||
podman push "docker.io/$container_image_organization/$container_image_name:latest"
|
||||
podman push "docker.io/$container_image_organization/$container_image_name:$container_image_tag_version"
|
||||
)
|
||||
dump_return $?
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2119
|
||||
main() {
|
||||
|
||||
local _type
|
||||
local cmd="$1"; shift
|
||||
local cmd="$1"
|
||||
shift
|
||||
|
||||
if [ "$cmd" == "" ]; then
|
||||
help
|
||||
@ -339,22 +559,25 @@ main() {
|
||||
fi
|
||||
|
||||
case "$cmd" in
|
||||
--getenv) var="$1"; echo "${!var}";;
|
||||
--help) help;;
|
||||
--*)
|
||||
help
|
||||
err_msg "unknown option $cmd"
|
||||
--getenv)
|
||||
var="$1"
|
||||
echo "${!var}"
|
||||
;;
|
||||
--help) help ;;
|
||||
--*)
|
||||
help
|
||||
err_msg "unknown option $cmd"
|
||||
return 42
|
||||
;;
|
||||
*)
|
||||
_type="$(type -t "$cmd")"
|
||||
if [ "$_type" != 'function' ]; then
|
||||
err_msg "unknown command: $cmd / use --help"
|
||||
return 42
|
||||
;;
|
||||
*)
|
||||
_type="$(type -t "$cmd")"
|
||||
if [ "$_type" != 'function' ]; then
|
||||
err_msg "unknown command: $cmd / use --help"
|
||||
return 42
|
||||
else
|
||||
"$cmd" "$@"
|
||||
fi
|
||||
;;
|
||||
else
|
||||
"$cmd" "$@"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user