[enh] bundle js_dependencies of plugins

.gitignore

@@ -14,5 +14,6 @@ cache/
 build/
 dist/
 local/
+searx/static/bundles/
 gh-pages/
 *.egg-info/

searx/resource_bundler.py

@@ -0,0 +1,63 @@
+"""
+Bundles ``js_dependencies`` of plugins to remove the overhead of separate HTTP requests.
+"""
+import os
+import os.path
+import itertools
+import pathlib
+from typing import Iterable, Dict
+
+from flask import url_for
+
+from . import settings
+from .webutils import get_themes
+from .plugins import plugins, Plugin
+
+templates_path = settings['ui']['templates_path']
+themes = get_themes(templates_path)
+
+
+def build_bundles():
+    static_path = settings['ui']['static_path']
+    bundles_path = pathlib.Path(static_path).joinpath('bundles')
+    bundles_path.mkdir(exist_ok=True)
+
+    # delete all bundles
+    for js_file in bundles_path.glob('*.js'):
+        js_file.unlink()
+
+    # generate bundles
+    for theme in themes:
+        modules: Dict[str, str] = {}
+
+        for plugin in plugins:
+            js_deps = [dep.path for dep in plugin.js_dependencies if theme in dep.themes]
+            if js_deps:
+                js = ''
+                for path in js_deps:
+                    with open(os.path.join(static_path, path), encoding='utf-8') as s:
+                        # We wrap the code in a self-calling function to prevent
+                        # namespace collisions between scripts.
+                        js += f'/** {path} **/\n(function(){{\n{s.read()}\n}})();'
+
+                modules[plugin.id] = js
+
+        for i in range(1, len(modules) + 1):
+            for plugin_combination in itertools.combinations(modules, i):
+                with bundles_path.joinpath(_bundle_path(theme, plugin_combination)).open('w', encoding='utf-8') as f:
+                    js = ''
+                    for plugin in plugin_combination:
+                        js += f'/**** {plugin} ****/\n' + modules[plugin]
+                    f.write(js)
+
+
+def get_bundle_url(theme: str, plugins: Iterable[Plugin]):
+    plugin_ids = [p.id for p in plugins if any(dep for dep in p.js_dependencies if theme in dep.themes)]
+    if plugin_ids:
+        return url_for('static', filename='bundles/' + _bundle_path(theme, sorted(plugin_ids)))
+    else:
+        return None
+
+
+def _bundle_path(theme: str, sorted_plugin_ids: Iterable[str]):
+    return theme + '+' + '+'.join(sorted_plugin_ids) + '.js'

searx/templates/oscar/base.html

@@ -101,9 +101,9 @@
             data-method="{{ method or 'POST' }}"
             data-autocompleter="{% if autocomplete %}true{% else %}false{% endif %}"
             data-translations="{{ translations }}"></script>
-    {% for script in scripts %}
+    {% if plugin_scripts_bundle_url %}
-    {{""}}<script src="{{ url_for('static', filename=script) }}"></script>
+        <script src="{{ plugin_scripts_bundle_url }}"></script>
-    {% endfor %}
+    {% endif %}
     <noscript>
       <style>
         .glyphicon { display: none; }

searx/templates/simple/base.html

@@ -59,8 +59,8 @@
     </p>
   </footer>
   <script src="{{ url_for('static', filename='js/searxng.min.js') }}"></script>
-  {% for script in scripts %}
+  {% if plugin_scripts_bundle_url %}
-    <script src="{{ url_for('static', filename=script) }}"></script>
+    <script src="{{ plugin_scripts_bundle_url }}"></script>
-  {% endfor %}
+  {% endif %}
 </body>
 </html>

searx/webapp.py

@@ -55,6 +55,7 @@ from searx import (
     get_setting,
     settings,
     searx_debug,
+    resource_bundler,
 )
 from searx.data import ENGINE_DESCRIPTIONS
 from searx.results import Timing, UnresponsiveEngine
@@ -493,12 +494,7 @@ def render(template_name: str, override_theme: str = None, **kwargs):
         url_for('opensearch') + '?' + urlencode({'method': kwargs['method'], 'autocomplete': kwargs['autocomplete']})
     )
 
-    # scripts from plugins
+    kwargs['plugin_scripts_bundle_url'] = resource_bundler.get_bundle_url(current_theme, request.user_plugins)
-    kwargs['scripts'] = set()
-    for plugin in request.user_plugins:
-        for script in plugin.js_dependencies:
-            if current_theme in script.themes:
-                kwargs['scripts'].add(script.path)
 
     # styles from plugins
     kwargs['styles'] = set()
@@ -1361,6 +1357,7 @@ werkzeug_reloader = flask_run_development or (searx_debug and __name__ == "__mai
 if not werkzeug_reloader or (werkzeug_reloader and os.environ.get("WERKZEUG_RUN_MAIN") == "true"):
     plugin_initialize(app)
     search_initialize(enable_checker=True, check_network=True, enable_metrics=settings['general']['enable_metrics'])
+    resource_bundler.build_bundles()
 
 
 def run():