From 34d19e5714c69f243cec8128d3a35524d18e46ef Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Sun, 17 Mar 2024 14:56:26 +0800 Subject: [PATCH 1/3] GHA CI: check python scripts --- .github/workflows/ci_python.yaml | 79 ++++++++++++++++++++++++++++++++ .github/workflows/ci_webui.yaml | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci_python.yaml diff --git a/.github/workflows/ci_python.yaml b/.github/workflows/ci_python.yaml new file mode 100644 index 000000000..f08b382b4 --- /dev/null +++ b/.github/workflows/ci_python.yaml @@ -0,0 +1,79 @@ +name: CI - Python + +on: [pull_request, push] + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.head_ref != '' }} + +jobs: + ci: + name: Check + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup python (auxiliary scripts) + uses: actions/setup-python@v5 + with: + python-version: '3' # use default version + + - name: Install tools (auxiliary scripts) + run: pip install bandit pycodestyle pyflakes + + - name: Gather files (auxiliary scripts) + run: | + export "PY_FILES=$(find . -type f -name '*.py' ! -path '*searchengine*' -printf '%p ')" + echo $PY_FILES + echo "PY_FILES=$PY_FILES" >> "$GITHUB_ENV" + + - name: Lint code (auxiliary scripts) + run: | + pyflakes $PY_FILES + bandit --skip B314,B405 $PY_FILES + + - name: Format code (auxiliary scripts) + run: | + pycodestyle \ + --max-line-length=1000 \ + --statistics \ + $PY_FILES + + - name: Build code (auxiliary scripts) + run: | + python -m compileall $PY_FILES + + - name: Setup python (search engine) + uses: actions/setup-python@v5 + with: + python-version: '3.7' + + - name: Install tools (search engine) + run: pip install bandit pycodestyle pyflakes + + - name: Gather files (search engine) + run: | + export "PY_FILES=$(find . -type f -name '*.py' -path '*searchengine*' ! -name 'socks.py' -printf '%p ')" + echo $PY_FILES + echo "PY_FILES=$PY_FILES" >> "$GITHUB_ENV" + + - name: Lint code (search engine) + run: | + pyflakes $PY_FILES + bandit --skip B110,B310,B314,B405 $PY_FILES + + - name: Format code (search engine) + run: | + pycodestyle \ + --ignore=E265,E402 \ + --max-line-length=1000 \ + --statistics \ + $PY_FILES + + - name: Build code (search engine) + run: | + python -m compileall $PY_FILES diff --git a/.github/workflows/ci_webui.yaml b/.github/workflows/ci_webui.yaml index 01b4e67c6..fd6f7134d 100644 --- a/.github/workflows/ci_webui.yaml +++ b/.github/workflows/ci_webui.yaml @@ -41,7 +41,7 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: - config-file: ./.github/workflows/helper/codeql/js.yaml + config-file: .github/workflows/helper/codeql/js.yaml languages: javascript - name: Run CodeQL analysis From 47c38e8d9160b7af1a7f6c56ee1d7e437f7a1f52 Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Sun, 17 Mar 2024 15:39:31 +0800 Subject: [PATCH 2/3] Apply formatting --- .../pre-commit/check_translation_tag.py | 10 +++-- dist/windows/gather_qt_translations.py | 3 ++ src/searchengine/nova3/novaprinter.py | 6 +-- src/update_qrc_files.py | 34 +++++++-------- src/webui/www/split_merge_json.py | 5 +++ src/webui/www/translations_qt2i18next.py | 4 ++ src/webui/www/tstool.py | 42 ++++++++++--------- 7 files changed, 61 insertions(+), 43 deletions(-) diff --git a/.github/workflows/helper/pre-commit/check_translation_tag.py b/.github/workflows/helper/pre-commit/check_translation_tag.py index 34705f74d..f3d4457e2 100755 --- a/.github/workflows/helper/pre-commit/check_translation_tag.py +++ b/.github/workflows/helper/pre-commit/check_translation_tag.py @@ -30,6 +30,7 @@ from typing import Optional, Sequence import argparse import re + def main(argv: Optional[Sequence[str]] = None) -> int: parser = argparse.ArgumentParser() parser.add_argument('filenames', nargs='*', help='Filenames to check') @@ -47,12 +48,12 @@ def main(argv: Optional[Sequence[str]] = None) -> int: for line in file: if (match := regex.match(line)) is not None: error_buffer += str(f"Defect file: \"{filename}\"\n" - f"Line: {line_counter}\n" - f"Column span: {match.span()}\n" - f"Part: \"{match.group()}\"\n\n") + f"Line: {line_counter}\n" + f"Column span: {match.span()}\n" + f"Part: \"{match.group()}\"\n\n") line_counter += 1 - except UnicodeDecodeError as error: + except UnicodeDecodeError: # not a text file, skip continue @@ -64,5 +65,6 @@ def main(argv: Optional[Sequence[str]] = None) -> int: return 0 + if __name__ == '__main__': exit(main()) diff --git a/dist/windows/gather_qt_translations.py b/dist/windows/gather_qt_translations.py index d1ff44499..cfa212c8c 100644 --- a/dist/windows/gather_qt_translations.py +++ b/dist/windows/gather_qt_translations.py @@ -7,9 +7,11 @@ import shutil import sys from typing import List + def isNotStub(path: str) -> bool: return (os.path.getsize(path) >= (10 * 1024)) + def main() -> int: parser = argparse.ArgumentParser(description='Gather valid Qt translations for NSIS packaging.') parser.add_argument("qt_translations_folder", help="Qt's translations folder") @@ -27,5 +29,6 @@ def main() -> int: return 0 + if __name__ == '__main__': sys.exit(main()) diff --git a/src/searchengine/nova3/novaprinter.py b/src/searchengine/nova3/novaprinter.py index 09250dc85..fdd423ba0 100644 --- a/src/searchengine/nova3/novaprinter.py +++ b/src/searchengine/nova3/novaprinter.py @@ -1,4 +1,4 @@ -#VERSION: 1.46 +#VERSION: 1.47 # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: @@ -45,13 +45,13 @@ def anySizeToBytes(size_string): # separate integer from unit try: size, unit = size_string.split() - except: + except Exception: try: size = size_string.strip() unit = ''.join([c for c in size if c.isalpha()]) if len(unit) > 0: size = size[:-len(unit)] - except: + except Exception: return -1 if len(size) == 0: return -1 diff --git a/src/update_qrc_files.py b/src/update_qrc_files.py index 1767bc962..aa59fb7d4 100755 --- a/src/update_qrc_files.py +++ b/src/update_qrc_files.py @@ -36,8 +36,8 @@ output = ''' ''' for language in languages_list: - output += ' %s'%('lang'+os.sep+language) - output += os.linesep + output += ' %s' % ('lang' + os.sep + language) + output += os.linesep output += ''' ''' lang_file = open('lang.qrc', 'w') @@ -48,41 +48,41 @@ lang_file.close() os.chdir('gui/searchengine') search_list = [] for root, dirs, files in os.walk('nova3'): - for file in files: - if file.startswith("__"): - continue - if splitext(file)[-1] in ('.py', '.png'): - search_list.append(join(root, file)) + for file in files: + if file.startswith("__"): + continue + if splitext(file)[-1] in ('.py', '.png'): + search_list.append(join(root, file)) output = ''' ''' for file in search_list: - output += ' %s'%(file) - output += os.linesep + output += ' %s' % (file) + output += os.linesep output += ''' ''' search_file = open('search.qrc', 'w') search_file.write(output) search_file.close() -os.chdir('../..'); +os.chdir('../..') # update icons files directory icons_list = [] for root, dirs, files in os.walk('icons'): - if 'skin_unused' in dirs: - dirs.remove('skin_unused') - for file in files: - if splitext(file)[-1] in ('.png', '.jpg', '.gif'): - icons_list.append(join(root, file)) + if 'skin_unused' in dirs: + dirs.remove('skin_unused') + for file in files: + if splitext(file)[-1] in ('.png', '.jpg', '.gif'): + icons_list.append(join(root, file)) output = ''' ''' for icon in icons_list: - output += ' %s'%(icon) - output += os.linesep + output += ' %s' % (icon) + output += os.linesep output += ''' ''' icons_file = open('icons.qrc', 'w') diff --git a/src/webui/www/split_merge_json.py b/src/webui/www/split_merge_json.py index 4fb27742a..17cd12539 100755 --- a/src/webui/www/split_merge_json.py +++ b/src/webui/www/split_merge_json.py @@ -16,6 +16,7 @@ import os from pathlib import PurePath import sys + def updateJson(json_file: str, source: dict[str, str]) -> None: trimmed_path: str = json_file path_parts = PurePath(json_file).parts @@ -42,6 +43,7 @@ def updateJson(json_file: str, source: dict[str, str]) -> None: file.write("\n") file.truncate() + def splitJson(transifex_dir: str, json_public_dir: str, json_private_dir: str) -> None: locales: list[str] = glob.glob("*.json", root_dir=transifex_dir) locales = [x for x in locales if x != "en.json"] @@ -65,6 +67,7 @@ def splitJson(transifex_dir: str, json_public_dir: str, json_private_dir: str) - updateJson(public_file, transifex_json) updateJson(private_file, transifex_json) + def mergeJson(transifex_dir: str, json_public_dir: str, json_private_dir: str) -> None: transifex_en_file: str = f"{transifex_dir}/en.json" public_en_file: str = f"{json_public_dir}/en.json" @@ -92,6 +95,7 @@ def mergeJson(transifex_dir: str, json_public_dir: str, json_private_dir: str) - json.dump(transifex_en_json, file, ensure_ascii=False, indent=2, sort_keys=True) file.write("\n") + def main() -> int: script_path: str = os.path.dirname(os.path.realpath(__file__)) transifex_dir: str = f"{script_path}/transifex" @@ -124,5 +128,6 @@ def main() -> int: return 0 + if __name__ == '__main__': sys.exit(main()) diff --git a/src/webui/www/translations_qt2i18next.py b/src/webui/www/translations_qt2i18next.py index c685bd8ab..4c0f4a856 100755 --- a/src/webui/www/translations_qt2i18next.py +++ b/src/webui/www/translations_qt2i18next.py @@ -14,6 +14,7 @@ import os import sys import xml.etree.ElementTree as ET + def getTsStrings(ts_file: str, key: str) -> list[str]: tr_strings: list[str] = [] tree = ET.parse(ts_file) @@ -33,6 +34,7 @@ def getTsStrings(ts_file: str, key: str) -> list[str]: return tr_strings + def migrate2Json(ts_dir: str, json_dir: str, locale: str) -> None: ts_file: str = f"{ts_dir}/webui_{locale}.ts" js_file: str = f"{json_dir}/{locale}.json" @@ -71,6 +73,7 @@ def migrate2Json(ts_dir: str, json_dir: str, locale: str) -> None: print("\tFinished.") + def main() -> int: script_path: str = os.path.dirname(os.path.realpath(__file__)) ts_dir: str = f"{script_path}/translations" @@ -97,5 +100,6 @@ def main() -> int: return 0 + if __name__ == '__main__': sys.exit(main()) diff --git a/src/webui/www/tstool.py b/src/webui/www/tstool.py index 65f838765..2166815dc 100755 --- a/src/webui/www/tstool.py +++ b/src/webui/www/tstool.py @@ -41,9 +41,10 @@ no_obsolete = False www_folder = "." ts_folder = os.path.join(www_folder, "translations") + def parseSource(filename, sources): print("Parsing %s..." % (os.path.normpath(filename))) - with open(filename, encoding = 'utf-8', mode = 'r') as file: + with open(filename, encoding='utf-8', mode='r') as file: regex = re.compile( r"QBT_TR\((([^\)]|\)(?!QBT_TR))+)\)QBT_TR\[CONTEXT=([a-zA-Z_][a-zA-Z0-9_]*)\]") for match in regex.finditer(file.read()): @@ -54,11 +55,12 @@ def parseSource(filename, sources): sources[context] = set() sources[context].add(string) + def processTranslation(filename, sources): print('Processing %s...' % (os.path.normpath(filename))) try: - tree = ET.ElementTree(file = filename) + tree = ET.ElementTree(file=filename) except Exception: print('\tFailed to parse %s!' % (os.path.normpath(filename))) return @@ -82,7 +84,7 @@ def processTranslation(filename, sources): trtype = translation.attrib.get('type') if (trtype == 'obsolete') or (trtype == 'vanished'): - del translation.attrib['type'] # i.e. finished + del translation.attrib['type'] # i.e. finished else: if no_obsolete or (translation.attrib.get('type', '') == 'unfinished'): context.remove(message) @@ -117,13 +119,15 @@ def processTranslation(filename, sources): context.tail = '\n' context.find('./name').tail = '\n' + indent messages = context.findall('./message') - if len(messages) == 0: continue + if len(messages) == 0: + continue for message in messages: message.text = '\n' + (indent * 2) message.tail = '\n' + indent elems = message.findall('./') - if len(elems) == 0: continue + if len(elems) == 0: + continue for elem in elems: elem.tail = '\n' + (indent * 2) @@ -131,24 +135,25 @@ def processTranslation(filename, sources): messages[-1:][0].tail = '\n' try: - with open(filename, mode = 'wb') as file: + with open(filename, mode='wb') as file: file.write(b'\n' b'\n') - tree.write(file, encoding = 'utf-8') + tree.write(file, encoding='utf-8') except Exception: print('\tFailed to write %s!' % (os.path.normpath(filename))) + argp = argparse.ArgumentParser( - prog = 'tstool.py', description = 'Update qBittorrent WebUI translation files.') -argp.add_argument('--no-obsolete', dest = 'no_obsolete', action = 'store_true', - default = no_obsolete, - help = 'remove obsolete messages (default: mark them as obsolete)') -argp.add_argument('--www-folder', dest = 'www_folder', action = 'store', - default = www_folder, - help = 'folder with WebUI source files (default: "%s")' % (www_folder)) -argp.add_argument('--ts-folder', dest = 'ts_folder', action = 'store', - default = ts_folder, - help = 'folder with WebUI translation files (default: "%s")' % (ts_folder)) + prog='tstool.py', description='Update qBittorrent WebUI translation files.') +argp.add_argument('--no-obsolete', dest='no_obsolete', action='store_true', + default=no_obsolete, + help='remove obsolete messages (default: mark them as obsolete)') +argp.add_argument('--www-folder', dest='www_folder', action='store', + default=www_folder, + help='folder with WebUI source files (default: "%s")' % (www_folder)) +argp.add_argument('--ts-folder', dest='ts_folder', action='store', + default=ts_folder, + help='folder with WebUI translation files (default: "%s")' % (ts_folder)) args = argp.parse_args() no_obsolete = args.no_obsolete @@ -174,8 +179,7 @@ print("") print("Processing translation files...") for entry in os.scandir(ts_folder): - if (entry.is_file() and entry.name.startswith('webui_') - and entry.name.endswith(".ts")): + if (entry.is_file() and entry.name.startswith('webui_') and entry.name.endswith(".ts")): processTranslation(entry.path, copy.deepcopy(source_ts)) print("Done!") From abcf1e076e7a332863e7c83535bfbaacc55d2d83 Mon Sep 17 00:00:00 2001 From: Chocobo1 Date: Sat, 23 Mar 2024 02:11:08 +0800 Subject: [PATCH 3/3] Remove unused script --- src/update_qrc_files.py | 90 ----------------------------------------- 1 file changed, 90 deletions(-) delete mode 100755 src/update_qrc_files.py diff --git a/src/update_qrc_files.py b/src/update_qrc_files.py deleted file mode 100755 index aa59fb7d4..000000000 --- a/src/update_qrc_files.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python - -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of the author nor the names of its contributors may be -# used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -# Small script to update qrc files (lang.qrc, icons.qrc) -# by Christophe Dumez - -import os -from os.path import splitext, join - -# update languages files directory -languages_list = [x for x in os.listdir('lang') if x.endswith('.qm')] -output = ''' - -''' -for language in languages_list: - output += ' %s' % ('lang' + os.sep + language) - output += os.linesep -output += ''' -''' -lang_file = open('lang.qrc', 'w') -lang_file.write(output) -lang_file.close() - -# update search_engine directory -os.chdir('gui/searchengine') -search_list = [] -for root, dirs, files in os.walk('nova3'): - for file in files: - if file.startswith("__"): - continue - if splitext(file)[-1] in ('.py', '.png'): - search_list.append(join(root, file)) - -output = ''' - -''' -for file in search_list: - output += ' %s' % (file) - output += os.linesep -output += ''' -''' -search_file = open('search.qrc', 'w') -search_file.write(output) -search_file.close() - -os.chdir('../..') - -# update icons files directory -icons_list = [] -for root, dirs, files in os.walk('icons'): - if 'skin_unused' in dirs: - dirs.remove('skin_unused') - for file in files: - if splitext(file)[-1] in ('.png', '.jpg', '.gif'): - icons_list.append(join(root, file)) - -output = ''' - -''' -for icon in icons_list: - output += ' %s' % (icon) - output += os.linesep -output += ''' -''' -icons_file = open('icons.qrc', 'w') -icons_file.write(output) -icons_file.close()