Lazy load search plugins

PR #20553.

src/searchengine/nova3/nova2.py

@@ -1,4 +1,4 @@
-#VERSION: 1.44
+#VERSION: 1.45
 
 # Author:
 #  Fabien Devaux <fab AT gnux DOT info>
@@ -58,26 +58,58 @@ CATEGORIES = {'all', 'movies', 'tv', 'music', 'games', 'anime', 'software', 'pic
 ################################################################################
 
 
-def initialize_engines():
+# global state
+engine_dict = dict()
+
+
+def list_engines():
+    """ List all engines,
+        including broken engines that fail on import
+
+        Faster than initialize_engines
+
+        Return list of all engines
+    """
+    found_engines = []
+
+    for engine_path in glob(path.join(path.dirname(__file__), 'engines', '*.py')):
+        engine_name = path.basename(engine_path).split('.')[0].strip()
+        if len(engine_name) == 0 or engine_name.startswith('_'):
+            continue
+        found_engines.append(engine_name)
+
+    return found_engines
+
+
+def get_engine(engine_name):
+    #global engine_dict
+    if engine_name in engine_dict:
+        return engine_dict[engine_name]
+    # when import fails, engine is None
+    engine = None
+    try:
+        # import engines.[engine]
+        engine_module = importlib.import_module("engines." + engine_name)
+        engine = getattr(engine_module, engine_name)
+    except Exception:
+        pass
+    engine_dict[engine_name] = engine
+    return engine
+
+
+def initialize_engines(found_engines):
     """ Import available engines
 
         Return list of available engines
     """
     supported_engines = []
 
-    engines = glob(path.join(path.dirname(__file__), 'engines', '*.py'))
+    for engine_name in found_engines:
-    for engine in engines:
+        # import engine
-        engi = path.basename(engine).split('.')[0].strip()
+        engine = get_engine(engine_name)
-        if len(engi) == 0 or engi.startswith('_'):
+        if engine == None:
             continue
-        try:
+        supported_engines.append(engine_name)
-            # import engines.[engine]
-            engine_module = importlib.import_module("engines." + engi)
-            # bind class name
-            globals()[engi] = getattr(engine_module, engi)
-            supported_engines.append(engi)
-        except Exception:
-            pass
 
     return supported_engines
 
@@ -86,8 +118,8 @@ def engines_to_xml(supported_engines):
     """ Generates xml for supported engines """
     tab = " " * 4
 
-    for short_name in supported_engines:
+    for engine_name in supported_engines:
-        search_engine = globals()[short_name]()
+        search_engine = get_engine(engine_name)
 
         supported_categories = ""
         if hasattr(search_engine, "supported_categories"):
@@ -95,11 +127,11 @@ def engines_to_xml(supported_engines):
                                              for key in search_engine.supported_categories.keys()
                                              if key != "all"))
 
-        yield "".join((tab, "<", short_name, ">\n",
+        yield "".join((tab, "<", engine_name, ">\n",
                        tab, tab, "<name>", search_engine.name, "</name>\n",
                        tab, tab, "<url>", search_engine.url, "</url>\n",
                        tab, tab, "<categories>", supported_categories, "</categories>\n",
-                       tab, "</", short_name, ">\n"))
+                       tab, "</", engine_name, ">\n"))
 
 
 def displayCapabilities(supported_engines):
@@ -148,47 +180,59 @@ def main(args):
     if current_path not in sys.path:
         sys.path.append(current_path)
 
-    supported_engines = initialize_engines()
+    found_engines = list_engines()
+
+    def show_usage():
+        print("./nova2.py all|engine1[,engine2]* <category> <keywords>", file=sys.stderr)
+        print("found engines: " + ','.join(found_engines), file=sys.stderr)
+        print("to list available engines: ./nova2.py --capabilities [--names]", file=sys.stderr)
 
     if not args:
-        raise SystemExit("./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n"
+        show_usage()
-                         "available engines: %s" % (','.join(supported_engines)))
+        sys.exit(1)
 
     elif args[0] == "--capabilities":
+        supported_engines = initialize_engines(found_engines)
+        if "--names" in args:
+            print(",".join(supported_engines))
+            return
         displayCapabilities(supported_engines)
         return
 
     elif len(args) < 3:
-        raise SystemExit("./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n"
+        show_usage()
-                         "available engines: %s" % (','.join(supported_engines)))
+        sys.exit(1)
+
+    cat = args[1].lower()
+
+    if cat not in CATEGORIES:
+        print(" - ".join(('Invalid category', cat)), file=sys.stderr)
+        sys.exit(1)
 
     # get only unique engines with set
     engines_list = set(e.lower() for e in args[0].strip().split(','))
 
-    if 'all' in engines_list:
-        engines_list = supported_engines
-    else:
-        # discard un-supported engines
-        engines_list = [engine for engine in engines_list
-                        if engine in supported_engines]
-
     if not engines_list:
         # engine list is empty. Nothing to do here
         return
 
-    cat = args[1].lower()
+    if 'all' in engines_list:
-
+        # use all supported engines
-    if cat not in CATEGORIES:
+        # note: this can be slower than passing a list of supported engines
-        raise SystemExit(" - ".join(('Invalid category', cat)))
+        # because initialize_engines will also try to import not-supported engines
+        engines_list = initialize_engines(found_engines)
+    else:
+        # discard not-found engines
+        engines_list = [engine for engine in engines_list if engine in found_engines]
 
     what = urllib.parse.quote(' '.join(args[2:]))
     if THREADED:
         # child process spawning is controlled min(number of searches, number of cpu)
         with Pool(min(len(engines_list), MAX_THREADS)) as pool:
-            pool.map(run_search, ([globals()[engine], what, cat] for engine in engines_list))
+            pool.map(run_search, ([get_engine(engine_name), what, cat] for engine_name in engines_list))
     else:
         # py3 note: map is needed to be evaluated for content to be executed
-        all(map(run_search, ([globals()[engine], what, cat] for engine in engines_list)))
+        all(map(run_search, ([get_engine(engine_name), what, cat] for engine_name in engines_list)))
 
 
 if __name__ == "__main__":