Merge pull request #250 from dalf/weblate_v2

[translations] weblate integration

.github/workflows/integration.yml

@@ -91,7 +91,7 @@ jobs:
         COMMIT_MESSAGE: build from commit ${{ github.sha }}
 
   babel:
-    name: Babel
+    name: Update translations branch
     runs-on: ubuntu-20.04
     if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
     needs:
@@ -102,32 +102,32 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v2
       with:
-        persist-credentials: false
+        fetch-depth: '0'
+        token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
         python-version: '3.9'
         architecture: 'x64'
+    - name: Cache Python dependencies
+      id: cache-python
+      uses: actions/cache@v2
+      with:
+        path: ./local
+        key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
+    - name: weblate & git setup
+      env:
+        WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
+      run: |
+        mkdir -p ~/.config
+        echo "${WEBLATE_CONFIG}" > ~/.config/weblate
+        git config --global user.email "searxng-bot@users.noreply.github.com"
+        git config --global user.name "searxng-bot"
     - name: Update transations
       id: update
-      continue-on-error: true
+      run: |
-      run: make V=1 ci.babel.update
+        git restore utils/brand.env
-    - name: Open pull request
+        make V=1 weblate.push.translations
-      if: steps.update.outcome == 'success'
-      uses: peter-evans/create-pull-request@v3
-      with:
-        commit-message: Update translations (pot, po)
-        committer: searx-bot <noreply@github.com>
-        author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
-        signoff: false
-        branch: update_translations_pot
-        delete-branch: true
-        draft: false
-        title: 'Update translations (pot, po)'
-        body: |
-          Update messages.pot and messages.po files
-        labels: |
-          translation
 
   dockers:
     name: Docker

.github/workflows/translations-update.yml

@@ -0,0 +1,56 @@
+name: "Update translations"
+on:
+  schedule:
+    - cron: "05 07 * * 5"
+  workflow_dispatch:
+
+jobs:
+  babel:
+    name: "create PR for additons from weblate"
+    runs-on: ubuntu-20.04
+    if: ${{ github.repository_owner == 'searxng' && github.ref == 'refs/heads/master' }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: '0'
+        token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+        architecture: 'x64'
+    - name: Cache Python dependencies
+      id: cache-python
+      uses: actions/cache@v2
+      with:
+        path: ./local
+        key: python-ubuntu-20.04-3.9-${{ hashFiles('requirements*.txt', 'setup.py') }}
+    - name: weblate & git setup
+      env:
+        WEBLATE_CONFIG: ${{ secrets.WEBLATE_CONFIG }}
+      run: |
+        mkdir -p ~/.config
+        echo "${WEBLATE_CONFIG}" > ~/.config/weblate
+        git config --global user.email "searxng-bot@users.noreply.github.com"
+        git config --global user.name "searxng-bot"
+    - name: Merge and push transation updates
+      run: |
+        make V=1 weblate.translations.commit
+    - name: Create Pull Request
+      id: cpr
+      uses: peter-evans/create-pull-request@v3
+      with:
+        token: ${{ secrets.WEBLATE_GITHUB_TOKEN }}
+        commit-message: Update translations
+        committer: searxng-bot <searxng-bot@users.noreply.github.com>
+        author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+        signoff: false
+        branch: translations_update
+        delete-branch: true
+        draft: false
+        title: 'Update translations'
+        body: |
+          Update translations
+        labels: |
+          translation

.weblate

@@ -0,0 +1,3 @@
+[weblate]
+url = https://weblate.bubu1.eu/api/
+translation = searxng/searxng

Makefile

@@ -77,7 +77,7 @@ test.shell:
 # wrap ./manage script
 
 MANAGE += buildenv
-MANAGE += ci.babel.update babel.extract babel.update babel.compile
+MANAGE += weblate.translations.commit weblate.push.translations
 MANAGE += data.all data.languages data.useragents data.osm_keys_tags
 MANAGE += docs.html docs.live docs.gh-pages docs.prebuild docs.clean
 MANAGE += docker.build docker.push docker.buildx

README.rst

@@ -32,6 +32,8 @@ Privacy-respecting, hackable `metasearch engine`_
 .. |commits| image:: https://img.shields.io/github/commit-activity/y/searxng/searxng?color=yellow&label=commits
    :target: https://github.com/searxng/searxng/commits/master
 
+.. |weblate| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
+   :target: https://weblate.bubu1.eu/projects/searxng/
 
 If you are looking for running instances, ready to use, then visit searx.space_.
 

docs/dev/translation.rst

@@ -4,69 +4,77 @@
 Translation
 ===========
 
-.. _searx@transifex: https://www.transifex.com/asciimoo/searx/
+.. _weblate.bubu1.eu: https://weblate.bubu1.eu/projects/searxng/
+.. _Weblate: https://docs.weblate.org
+.. _translations branch: https://github.com/searxng/searxng/tree/translations
+.. _orphan branch: https://git-scm.com/docs/git-checkout#Documentation/git-checkout.txt---orphanltnewbranchgt
+.. _Weblate repository: https://weblate.bubu1.eu/projects/searxng/searxng/#repository
+.. _wlc: https://docs.weblate.org/en/latest/wlc.html
 
-Translation currently takes place on `searx@transifex`_
+.. |translated| image:: https://weblate.bubu1.eu/widgets/searxng/-/searxng/svg-badge.svg
+   :target: https://weblate.bubu1.eu/projects/searxng/
 
-Requirements
+.. sidebar:: |translated|
-============
 
-* Transifex account
+   - Weblate_
+   - SearXNG `translations branch`_
+   - SearXNG `Weblate repository`_
+   - Weblate Client: wlc_
+   - Babel Command-Line: `pybabel <http://babel.pocoo.org/en/latest/cmdline.html>`_
+   - `weblate workflow <https://docs.weblate.org/en/latest/workflows.html>`_
 
-Init Transifex project
+Translation takes place on weblate.bubu1.eu_.
-======================
 
-After installing ``transifex`` using pip, run the following command to
+Translations which has been added by translators on the weblate.bubu1.eu_ UI are
-initialize the project.
+committed to Weblate's counterpart of the SearXNG *origin* repository which is
+located at ``https://weblate.bubu1.eu/git/searxng/searxng``.
 
-.. code:: sh
+There is no need to clone this repository, :ref:`SearXNG Weblate workflow` take
+care of the synchronization with the *origin*.  To avoid merging commits from
+the counterpart directly on the ``master`` branch of *SearXNG origin*, a *pull
+request* (PR) is created by this workflow.
 
-   ./manage pyenv.cmd tx init # Transifex instance: https://www.transifex.com/asciimoo/searx/
+Weblate monitors the `translations branch`_, not the ``master`` branch.  This
+branch is an `orphan branch`_, decoupled from the master branch (we already know
+orphan branches from the ``gh-pages``).  The `translations branch`_ contains
+only the
+
+- ``translation/messages.pot`` and the
+- ``translation/*/messages.po`` files, nothing else.
 
 
-After ``$HOME/.transifexrc`` is created, get a Transifex API key and insert it
+.. _SearXNG Weblate workflow:
-into the configuration file.
 
-Create a configuration file for ``tx`` named ``$HOME/.tx/config``.
+.. figure:: translation.svg
 
-.. code:: ini
+   SearXNG's PR workflow to be in sync with Weblate
 
-    [main]
+Sync from *origin* to *weblate*: using ``make weblate.push.translations``
-    host = https://www.transifex.com
+  For each commit on the ``master`` branch of SearXNG *origin* the GitHub job
-    [searx.messagespo]
+  :origin:`babel / Update translations branch
-    file_filter = searx/translations/<lang>/LC_MESSAGES/messages.po
+  <.github/workflows/integration.yml>` checks for updated translations.
-    source_file = messages.pot
-    source_lang = en
-    type = PO
 
+Sync from *weblate* to *origin*: using ``make weblate.translations.commit``
+  Every Friday, the GitHub workflow :origin:`babel / create PR for additons from
+  weblate <.github/workflows/translations-update.yml>` creates a PR with the
+  updated translation files:
 
-Then run ``tx set``:
+  - ``translation/messages.pot``,
+  - ``translation/*/messages.po`` and
+  - ``translation/*/messages.mo``
 
-.. code:: shell
+wlc
+===
 
-    ./manage pyenv.cmd tx set --auto-local \
+.. _wlc configuration: https://docs.weblate.org/en/latest/wlc.html#wlc-config
-        -r searx.messagespo 'searx/translations/<lang>/LC_MESSAGES/messages.po' \
+.. _API key: https://weblate.bubu1.eu/accounts/profile/#api
-        --source-lang en --type PO --source-file messages.pot --execute
 
+All weblate integration is done by GitHub workflows, but if you want to use wlc_,
+copy this content into `wlc configuration`_ in your HOME ``~/.config/weblate``
 
-Update translations
+.. code-block:: ini
-===================
 
-To retrieve the latest translations, pull it from Transifex.
+  [keys]
+  https://weblate.bubu1.eu/api/ = APIKEY
 
-.. code:: sh
+Replace ``APIKEY`` by your `API key`_.
-
-   ./manage pyenv.cmd tx pull -a
-   [?] Enter your api token: ....
-
-Then check the new languages.  If strings translated are not enough, delete those
-folders, because those should not be compiled.  Call the command below to compile
-the ``.po`` files.
-
-.. code:: shell
-
-   ./manage pyenv.cmd pybabel compile -d searx/translations
-
-
-After the compilation is finished commit the ``.po`` and ``.mo`` files and
-create a PR.

manage

@@ -45,9 +45,8 @@ help() {
 buildenv:
   rebuild ./utils/brand.env
 babel.:
-  extract   : extract messages from source files and generate POT file
+  master.to.translations: update the translations branch from the messages of the master branch.
-  update    : update existing message catalogs from POT file
+  translations.to.master: copy change from the translations branch to the master branch.
-  compile   : compile translation catalogs into binary MO files
 data.:
   all       : update searx/languages.py and ./data/*
   languages : update searx/data/engines_languages.json & searx/languages.py
@@ -122,46 +121,185 @@ buildenv() {
     return "${PIPESTATUS[0]}"
 }
 
-babel.sha256sum() {
+TRANSLATIONS_WORKTREE="$CACHE/translations"
-    grep "msgid" "searx/translations/messages.pot" | sort | sha256sum | cut -f1 -d ' '
+
+weblate.translations.worktree() {
+
+    # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
+    # 'translations' from Weblate's counterpart (weblate) of the SearXNG
+    # (origin).
+    #
+    #     remote weblate https://weblate.bubu1.eu/git/searxng/searxng/
+
+    (   set -e
+        if ! git remote get-url weblate 2> /dev/null; then
+            git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
+        fi
+        if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
+            pushd .
+            cd "${TRANSLATIONS_WORKTREE}"
+            git reset --hard HEAD
+            git pull origin translations
+            popd
+        else
+            mkdir -p "${TRANSLATIONS_WORKTREE}"
+            git worktree add "${TRANSLATIONS_WORKTREE}" translations
+        fi
+    )
 }
 
-ci.babel.update() {
+weblate.to.translations() {
-    local sha_before
+
+    # Update 'translations' branch of SearXNG (origin) with last additions from
+    # Weblate.
+
+    # 1. Check if Weblate is locked, if not die with error message
+    # 2. On Weblate's counterpart (weblate), pull master and translations branch
+    #    from SearXNG (origin).
+    # 3. Commit changes made in a Weblate object on Weblate's counterpart
+    #    (weblate).
+    # 4. In translations worktree, merge changes of branch 'translations' from
+    #    remote 'weblate' and push it on branch 'translations' of 'origin'
+
     (   set -e
-        sha_before="$(babel.sha256sum)"
+        if [ "$(pyenv.cmd wlc lock-status)" != "locked: True" ]; then
-        babel.extract
+            die 1 "weblate must be locked, currently: $(pyenv.cmd wlc lock-status)"
-        if [ "$(babel.sha256sum)" = "${sha_before}" ]; then
-            build_msg BABEL 'no changes detected, exiting'
-            return 1
         fi
-        babel.update
+        # weblate: commit pending changes
-        build_msg BABEL 'update done, edit .po files if required and run babel.compile'
+        pyenv.cmd wlc pull
+        pyenv.cmd wlc commit
+
+        # get the translations in a worktree
+        weblate.translations.worktree
+
+        pushd "${TRANSLATIONS_WORKTREE}"
+        git remote update weblate
+        git merge weblate/translations
+        git push
+        popd
     )
     dump_return $?
 }
 
-babel.extract() {
+weblate.translations.commit() {
-    build_msg BABEL 'extract messages from source files and generate POT file'
-    pyenv.cmd pybabel extract -F babel.cfg \
-            -o "searx/translations/messages.pot" \
-            "searx/"
-    dump_return $?
-}
 
-babel.update() {
+    # Update 'translations' branch of SearXNG (origin) with last additions from
-    build_msg BABEL 'update existing message catalogs from POT file'
+    # Weblate.  Copy the changes to the master branch, compile translations and
-    pyenv.cmd pybabel update -N \
+    # create a commit in the local branch (master)
-              -i "searx/translations/messages.pot" \
-              -d "searx/translations"
-    dump_return $?
-}
 
-babel.compile() {
+    local existing_commit_hash commit_body commit_message exitcode
+    (   set -e
+        # lock change on weblate
+        pyenv.cmd wlc lock
+
+        # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
+        weblate.translations.worktree
+        existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1  --pretty=format:'%h')
+
+        # pull weblate commits
+        weblate.to.translations
+
+        # copy the changes to the master branch
+        cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
+
+        # compile translations
         build_msg BABEL 'compile translation catalogs into binary MO files'
         pyenv.cmd pybabel compile --statistics \
                 -d "searx/translations"
-    dump_return $?
+        # git add/commit (no push)
+        commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
+        commit_message=$(echo -e "[translations] update\n${commit_body}")
+        git add searx/translations
+        git commit -m "${commit_message}"
+    )
+    exitcode=$?
+    (   # make sure to always unlock weblate
+        set -e
+        pyenv.cmd wlc unlock
+    )
+    dump_return $exitcode
+}
+
+weblate.push.translations() {
+
+    # Push *translation changes* from SearXNG (origin) to Weblate's counterpart
+    # (weblate).
+
+    # In branch master of SearXNG (origin) check for meaningful changes in
+    # folder 'searx/translations', commit changes on branch 'translations' and
+    # at least, pull updated branches on Weblate's counterpart (weblate).
+
+    # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
+    #    'translations' from remote 'weblate'.
+    # 2. Stop if there is no meaningful change in the 'master' branch (origin),
+    #    compared to the 'translations' branch (weblate), otherwise ...
+    # 3. Update 'translations' branch of SearXNG (origin) with last additions
+    #    from Weblate.
+    # 5. Notify Weblate to pull updated 'master' & 'translations' branch.
+
+    local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
+          last_commit_message exitcode
+    (   set -e
+        # lock change on weblate
+        pyenv.cmd wlc lock
+
+        # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
+        weblate.translations.worktree
+
+        # update messages.pot in the master branch
+        build_msg BABEL 'extract messages from source files and generate POT file'
+        messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
+        pyenv.cmd pybabel extract -F babel.cfg \
+                -o "${messages_pot}" \
+                "searx/"
+
+        # stop if there is no meaningful change in the master branch
+        diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
+                            git diff -- "searx/translations/messages.pot")
+        if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
+            build_msg BABEL 'no changes detected, exiting'
+            return 0
+        fi
+
+        # save messages.pot in the translations branch for later
+        pushd "${TRANSLATIONS_WORKTREE}"
+        git stash push
+        popd
+
+        # merge weblate commits into the translations branch
+        weblate.to.translations
+
+        # restore messages.pot in the translations branch
+        pushd "${TRANSLATIONS_WORKTREE}"
+        git stash pop
+        popd
+
+        # update messages.po files in the master branch
+        build_msg BABEL 'update existing message catalogs from POT file'
+        pyenv.cmd pybabel update -N \
+            -i "${messages_pot}" \
+            -d "${TRANSLATIONS_WORKTREE}/searx/translations"
+
+        # git add/commit/push
+        last_commit_hash=$(git log -n1  --pretty=format:'%h')
+        last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
+        last_commit_message="[translations] update messages.pot and messages.po files\nFrom ${last_commit_detail}"
+
+        pushd "${TRANSLATIONS_WORKTREE}"
+        git add searx/translations
+        git commit -m "${last_commit_message}"
+        git push
+        popd
+
+        # notify weblate to pull updated master & translations branch
+        pyenv.cmd wlc pull
+    )
+    exitcode=$?
+    (   # make sure to always unlock weblate
+        set -e
+        pyenv.cmd wlc unlock
+    )
+    dump_return $exitcode
 }
 
 data.all() {

requirements-dev.txt

@@ -17,3 +17,4 @@ sphinx-autobuild==2021.3.14
 linuxdoc==20210324
 aiounittest==1.4.0
 yamllint==1.26.2
+wlc==1.12