diff --git a/src/searchengine/nova3/nova2.py b/src/searchengine/nova3/nova2.py old mode 100644 new mode 100755 index 925481cd4..f02b324f4 --- a/src/searchengine/nova3/nova2.py +++ b/src/searchengine/nova3/nova2.py @@ -1,4 +1,4 @@ -#VERSION: 1.44 +#VERSION: 1.45 # Author: # Fabien Devaux @@ -58,26 +58,58 @@ CATEGORIES = {'all', 'movies', 'tv', 'music', 'games', 'anime', 'software', 'pic ################################################################################ -def initialize_engines(): +# global state +engine_dict = dict() + + +def list_engines(): + """ List all engines, + including broken engines that fail on import + + Faster than initialize_engines + + Return list of all engines + """ + found_engines = [] + + for engine_path in glob(path.join(path.dirname(__file__), 'engines', '*.py')): + engine_name = path.basename(engine_path).split('.')[0].strip() + if len(engine_name) == 0 or engine_name.startswith('_'): + continue + found_engines.append(engine_name) + + return found_engines + + +def get_engine(engine_name): + #global engine_dict + if engine_name in engine_dict: + return engine_dict[engine_name] + # when import fails, engine is None + engine = None + try: + # import engines.[engine] + engine_module = importlib.import_module("engines." + engine_name) + engine = getattr(engine_module, engine_name) + except Exception: + pass + engine_dict[engine_name] = engine + return engine + + +def initialize_engines(found_engines): """ Import available engines Return list of available engines """ supported_engines = [] - engines = glob(path.join(path.dirname(__file__), 'engines', '*.py')) - for engine in engines: - engi = path.basename(engine).split('.')[0].strip() - if len(engi) == 0 or engi.startswith('_'): + for engine_name in found_engines: + # import engine + engine = get_engine(engine_name) + if engine == None: continue - try: - # import engines.[engine] - engine_module = importlib.import_module("engines." + engi) - # bind class name - globals()[engi] = getattr(engine_module, engi) - supported_engines.append(engi) - except Exception: - pass + supported_engines.append(engine_name) return supported_engines @@ -86,8 +118,8 @@ def engines_to_xml(supported_engines): """ Generates xml for supported engines """ tab = " " * 4 - for short_name in supported_engines: - search_engine = globals()[short_name]() + for engine_name in supported_engines: + search_engine = get_engine(engine_name) supported_categories = "" if hasattr(search_engine, "supported_categories"): @@ -95,11 +127,11 @@ def engines_to_xml(supported_engines): for key in search_engine.supported_categories.keys() if key != "all")) - yield "".join((tab, "<", short_name, ">\n", + yield "".join((tab, "<", engine_name, ">\n", tab, tab, "", search_engine.name, "\n", tab, tab, "", search_engine.url, "\n", tab, tab, "", supported_categories, "\n", - tab, "\n")) + tab, "\n")) def displayCapabilities(supported_engines): @@ -148,47 +180,59 @@ def main(args): if current_path not in sys.path: sys.path.append(current_path) - supported_engines = initialize_engines() + found_engines = list_engines() + + def show_usage(): + print("./nova2.py all|engine1[,engine2]* ", file=sys.stderr) + print("found engines: " + ','.join(found_engines), file=sys.stderr) + print("to list available engines: ./nova2.py --capabilities [--names]", file=sys.stderr) if not args: - raise SystemExit("./nova2.py [all|engine1[,engine2]*] \n" - "available engines: %s" % (','.join(supported_engines))) + show_usage() + sys.exit(1) elif args[0] == "--capabilities": + supported_engines = initialize_engines(found_engines) + if "--names" in args: + print(",".join(supported_engines)) + return displayCapabilities(supported_engines) return elif len(args) < 3: - raise SystemExit("./nova2.py [all|engine1[,engine2]*] \n" - "available engines: %s" % (','.join(supported_engines))) + show_usage() + sys.exit(1) + + cat = args[1].lower() + + if cat not in CATEGORIES: + print(" - ".join(('Invalid category', cat)), file=sys.stderr) + sys.exit(1) # get only unique engines with set engines_list = set(e.lower() for e in args[0].strip().split(',')) - if 'all' in engines_list: - engines_list = supported_engines - else: - # discard un-supported engines - engines_list = [engine for engine in engines_list - if engine in supported_engines] - if not engines_list: # engine list is empty. Nothing to do here return - cat = args[1].lower() - - if cat not in CATEGORIES: - raise SystemExit(" - ".join(('Invalid category', cat))) + if 'all' in engines_list: + # use all supported engines + # note: this can be slower than passing a list of supported engines + # because initialize_engines will also try to import not-supported engines + engines_list = initialize_engines(found_engines) + else: + # discard not-found engines + engines_list = [engine for engine in engines_list if engine in found_engines] what = urllib.parse.quote(' '.join(args[2:])) if THREADED: # child process spawning is controlled min(number of searches, number of cpu) with Pool(min(len(engines_list), MAX_THREADS)) as pool: - pool.map(run_search, ([globals()[engine], what, cat] for engine in engines_list)) + pool.map(run_search, ([get_engine(engine_name), what, cat] for engine_name in engines_list)) else: # py3 note: map is needed to be evaluated for content to be executed - all(map(run_search, ([globals()[engine], what, cat] for engine in engines_list))) + all(map(run_search, ([get_engine(engine_name), what, cat] for engine_name in engines_list))) if __name__ == "__main__":