From 2de1bc30dbcfab10038fe1dea9564b2b49ea6a5f Mon Sep 17 00:00:00 2001 From: "John E. Malmberg" Date: Sun, 29 Mar 2026 13:41:17 -0500 Subject: [PATCH] Improved pre-commit and GitHub Actions This is the start of getting python cross platform unit tests to be run on Ubuntu, Microsoft Windows, and Mac-OS so that we can try to prevent regression on platform that a developer does not have available. Also adding the unit tests to the changed Minor fixes not affecting function. - .github/workflows/codespell.yml - .github/workflows/xmllint.yml - .github/workflows/shellcheck.yml New files adding Unit Tests for GitHub actions and pre-commit checks - .github/workflows/python_unit_tests.yml - tests/pre-commit_d/python_unit_tests.sh Fixes needed to pass the new checking - d-rats.py - d_rats/dplatform.py - d_rats/dplatform_macos.py - d_rats/dplatform_unix.py - d_rats/dplatform_generic.py - d_rats/dplatform_win32.py - d_rats/utils.py --- .github/workflows/codespell.yml | 2 +- .github/workflows/python_unit_tests.yml | 75 ++++++++++++++++++++++ .github/workflows/shellcheck.yml | 2 +- .github/workflows/xmllint.yml | 2 +- d-rats.py | 6 +- d_rats/dplatform.py | 1 - d_rats/dplatform_generic.py | 82 ++++++++++++++++--------- d_rats/dplatform_macos.py | 2 + d_rats/dplatform_unix.py | 7 ++- d_rats/dplatform_win32.py | 16 ++++- d_rats/utils.py | 14 +++-- tests/pre-commit | 3 +- tests/pre-commit_d/python_unit_tests.sh | 44 +++++++++++++ workflow/Linux/requirements.txt | 22 +++++++ workflow/Windows/requirements.txt | 24 ++++++++ workflow/macOS/requirements.txt | 22 +++++++ 16 files changed, 280 insertions(+), 44 deletions(-) create mode 100755 .github/workflows/python_unit_tests.yml create mode 100755 tests/pre-commit_d/python_unit_tests.sh create mode 100755 workflow/Linux/requirements.txt create mode 100755 workflow/Windows/requirements.txt create mode 100755 workflow/macOS/requirements.txt diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 34f3d2b0..0921c8dd 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -6,7 +6,7 @@ on: # yamllint disable-line rule:truthy pull_request: jobs: - pylint: + codespell: name: Run Codespell runs-on: ubuntu-latest steps: diff --git a/.github/workflows/python_unit_tests.yml b/.github/workflows/python_unit_tests.yml new file mode 100755 index 00000000..3590cc23 --- /dev/null +++ b/.github/workflows/python_unit_tests.yml @@ -0,0 +1,75 @@ +--- +name: python_unit_tests + +# Always run on Pull Requests +on: # yamllint disable-line rule:truthy + pull_request: + +jobs: + get_files: + runs-on: ubuntu-latest + outputs: + changed_files_list: ${{ steps.get-changed-files.outputs.files }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch-depth 0 is required for diffing + + - name: Get changed Python files + id: get-changed-files + run: | + # Use git diff to find changed files and format them as a + # newline-separated string + changed="$(git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD |\ + grep '.*\.py$' | tr '\n' ' ')" + echo "changed files = $changed" + # Set the string as a step output + echo "files=$changed" >> $GITHUB_OUTPUT + + - name: Display changed files (in the same job) + run: | + echo "Changed files in this job:" + echo "${{ steps.get-changed-files.outputs.files }}" + shell: bash + + python_unit_tests: + needs: [get_files] + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + # Define additional variables for different target platforms or + # configurations + target: [x86_64, arm64] + # Optional: set fail-fast to false to allow all jobs to complete + # even if one fails + fail-fast: false + + runs-on: ${{ matrix.os }} + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python 3 + uses: actions/setup-python@v5 # Action to set up Python + with: + cache: 'pip' # Caches dependencies for faster subsequent runs + - name: Install dependencies + # The conditional check differs slightly between shells + # (Linux/macOS vs. Windows) + run: | + REQUIREMENTS_FILE="workflow/${{ runner.os }}/requirements.txt" + if [ -f "$REQUIREMENTS_FILE" ]; then + python -m pip install --upgrade pip + pip install -r "$REQUIREMENTS_FILE" + fi + shell: bash # Explicitly use bash for consistency on + # Linux/macOS/Windows runners + - name: Run unit test script + # GitHub VS-CODE plugin false positive + run: | + files="${{ needs.get_files.outputs.changed_files_list }}" + export CHANGED_FILES="$files" + ./tests/pre-commit_d/python_unit_tests.sh + shell: bash diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 130131e6..52a5b39b 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -6,7 +6,7 @@ on: # yamllint disable-line rule:truthy pull_request: jobs: - pylint: + shellcheck: name: Run ShellCheck runs-on: ubuntu-latest steps: diff --git a/.github/workflows/xmllint.yml b/.github/workflows/xmllint.yml index 22ff1e37..1947db19 100644 --- a/.github/workflows/xmllint.yml +++ b/.github/workflows/xmllint.yml @@ -6,7 +6,7 @@ on: # yamllint disable-line rule:truthy pull_request: jobs: - pylint: + xmllint: name: Run Xmllint # Oddly ubuntu-latest is older than 22.04 right now. runs-on: ubuntu-22.04 diff --git a/d-rats.py b/d-rats.py index 9b693474..65966435 100755 --- a/d-rats.py +++ b/d-rats.py @@ -30,6 +30,7 @@ import sys import traceback +# pylint: disable=import-error import gi gi.require_version("Gtk", "3.0") from gi.repository import Gtk @@ -59,8 +60,8 @@ sys.path.insert(0, os.path.join("/usr/share", "d-rats")) # import module to have spelling correction in chat and email applications -from d_rats import utils, spell +from d_rats import utils, spell spell.get_spell().test() IGNORE_ALL = False @@ -88,7 +89,8 @@ def handle_exception(except_type, value, trace_back): global IGNORE_ALL if except_type is KeyboardInterrupt or IGNORE_ALL: - return sys.__excepthook__(except_type, value, trace_back) + sys.__excepthook__(except_type, value, trace_back) + return # Gdk.pointer_ungrab(Gdk.CURRENT_TIME) # Gdk.keyboard_ungrab(Gdk.CURRENT_TIME) diff --git a/d_rats/dplatform.py b/d_rats/dplatform.py index 2ff532e8..f11f755b 100755 --- a/d_rats/dplatform.py +++ b/d_rats/dplatform.py @@ -22,7 +22,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - import logging import os import sys diff --git a/d_rats/dplatform_generic.py b/d_rats/dplatform_generic.py index ea6b33da..7deb3848 100755 --- a/d_rats/dplatform_generic.py +++ b/d_rats/dplatform_generic.py @@ -2,7 +2,7 @@ # # Copyright 2009 Dan Smith # review 2015 Maurizio Andreotti -# Copyright 2021-2023 John. E. Malmberg - Python3 Conversion +# Copyright 2021-2023,2026 John. E. Malmberg - Python3 Conversion # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,16 +37,39 @@ HAVE_AUDIO = True except ModuleNotFoundError: pass +else: + # This code is mainly to silence the linters when run on a + # platform with out sound support, it is actually unreachable. + class AudioSegment(): + '''Mocked class when sound is not available.''' -import urllib.request -import urllib.parse -import urllib.error + # pylint: disable=no-self-use + def from_wav(self, soundfile): + '''Mocked method when sound is not available''' + print(f'No support for playing {soundfile}') + + @staticmethod + def dummy(): + '''Needs two public methods''' + + def play(sound): + '''Mocked routine when sound is not available.''' + sound_type = type(sound) + print(f'Unable to play sound {sound_type} is unsupported') -import gi # type: ignore # Needed for pylance on Microsoft Windows -gi.require_version("Gtk", "3.0") -from gi.repository import Gtk # type: ignore -from gi.repository import Gio # type: ignore +import urllib.request + +# This version of D-Rats requires GTK 3.0, but not for doing unit +# tests. This allows the default unit tests to pass on a system with +# out GTK+ installed. +try: + import gi + gi.require_version("Gtk", "3.0") + from gi.repository import Gtk # type: ignore + from gi.repository import Gio # type: ignore +except (ImportError, ValueError): + pass if '_' not in locals(): import gettext @@ -176,14 +199,15 @@ def get_exe_path(name): :rtype: str ''' # Make trivial check from the normal paths - exe_path = shutil.which(name) + # known false positive in pylance + exe_path = shutil.which(name) # type: ignore if exe_path: return exe_path programfiles = os.getenv("PROGRAMFILES") if programfiles: for program in [programfiles, programfiles + " (x86)"]: test_path=os.path.join(program, name) - exe_path = shutil.which(name, path=test_path) + exe_path = shutil.which(name, path=test_path) # type: ignore if exe_path: return exe_path return None @@ -261,14 +285,14 @@ def open_html_file(path): gio_path = Gio.File.parse_name(path) appinfo.launch([gio_path], None) - @staticmethod - def list_serial_ports(): + def list_serial_ports(self): ''' List Serial Ports. :returns: empty list :rtype: list ''' + self.logger.info("list_serial_ports not available on this platform") return [] @staticmethod @@ -281,9 +305,8 @@ def default_dir(): ''' return "." - @staticmethod - # pylint: disable=unused-argument - def gui_open_file(mime_types=None, start_dir=None): + # pylint: disable=unused-argument, no-self-use + def gui_open_file(self, mime_types=None, start_dir=None): ''' GUI Open File. @@ -309,14 +332,14 @@ def gui_open_file(mime_types=None, start_dir=None): # for mime_type in mime_types: # exts = mimetypes.guess_all_extensions(mime_type, strict=True) # filter_mime.set_name=(exts) - # In this case it does not seem to matter what string was - # passed to the filter, the filter shows up as unnamed. - # This issue does not reproduce in a quick test program. + # In this case it does not seem to matter what string was + # passed to the filter, the filter shows up as unnamed. + # This issue does not reproduce in a quick test program. # filter_mime.add_mime_type=(mime_type) # dlg.add_filter(filter_mime) # filter_any = Gtk.FileFilter() # filter_any.set_name(_("All files")) - # In this case the filter name gets set prpoerly unlike above. + # In this case the filter name gets set properly unlike above. # filter_any.add_pattern=('*') # dlg.add_filter(filter_any) @@ -331,9 +354,9 @@ def gui_open_file(mime_types=None, start_dir=None): return fname return None - @staticmethod - # pylint: disable=unused-argument - def gui_save_file(mime_types=None, start_dir=None, default_name=None): + # pylint: disable=unused-argument, no-self-use + def gui_save_file(self, mime_types=None, start_dir=None, + default_name=None): ''' GUI Save File. @@ -376,8 +399,8 @@ def gui_save_file(mime_types=None, start_dir=None, default_name=None): return fname return None - @staticmethod - def gui_select_dir(start_dir=None): + # pylint: disable=no-self-use + def gui_select_dir(self, start_dir=None): ''' Gui Select Directory. @@ -405,14 +428,14 @@ def gui_select_dir(start_dir=None): return fname return None - @staticmethod - def os_version_string(): + def os_version_string(self): ''' OS Version String. :returns: "Unknown Operating System" :rtype: str ''' + self.logger.info("Unknown Operating System") return "Unknown Operating System" @staticmethod @@ -487,7 +510,7 @@ def have_sound(): Do we have sound support? :returns: Status of sound support - :rytpe: bool + :rtype: bool ''' return HAVE_AUDIO @@ -502,7 +525,8 @@ def play_sound(self, soundfile): if not HAVE_AUDIO: self.logger.info("play_sound: " "pydub and pyaudio not installed!") + return - sound = AudioSegment.from_wav(soundfile) + sound = AudioSegment.from_wav(soundfile) # type: ignore with suppress_stderr(): - play(sound) + play(sound) # type: ignore diff --git a/d_rats/dplatform_macos.py b/d_rats/dplatform_macos.py index 641145b6..8b11cbdf 100755 --- a/d_rats/dplatform_macos.py +++ b/d_rats/dplatform_macos.py @@ -108,6 +108,8 @@ def sys_data(self): @staticmethod def _unix_doublefork_run(*args): + # False positive when editing on some platforms. + # pylint: disable=no-member pid1 = os.fork() if pid1 == 0: pid2 = os.fork() diff --git a/d_rats/dplatform_unix.py b/d_rats/dplatform_unix.py index 0f0a8d0c..61e84cf2 100755 --- a/d_rats/dplatform_unix.py +++ b/d_rats/dplatform_unix.py @@ -57,7 +57,8 @@ def set_config_dir(self, basepath): os.makedirs(basepath, exist_ok=True) self._base = basepath - def default_dir(self): + @staticmethod + def default_dir(): ''' Default Directory. @@ -99,11 +100,15 @@ def os_version_string(self): issue = open("/etc/issue.net", "r") ver = issue.read().strip() issue.close() + # False positive when editing on some platforms + # pylint: disable=no-member ver = "%s - %s" % (os.uname()[0], ver) except IOError as err: if err.errno == 2: # No such file or directory # Linux use of this file seems to be deprecated. pass + # False positive when editing on some platforms + # pylint: disable=no-member ver = " ".join(os.uname()) return ver diff --git a/d_rats/dplatform_win32.py b/d_rats/dplatform_win32.py index a174ea7e..489179a5 100755 --- a/d_rats/dplatform_win32.py +++ b/d_rats/dplatform_win32.py @@ -95,7 +95,8 @@ def set_config_dir(self, basepath): os.makedirs(basepath, exist_ok=True) self._base = basepath - def default_dir(self): + @staticmethod + def default_dir(): ''' Default Directory. @@ -105,7 +106,8 @@ def default_dir(self): return os.path.abspath( os.path.join(os.getenv("USERPROFILE"), "Desktop")) - def filter_filename(self, filename): + @staticmethod + def filter_filename(filename): ''' Filter Filename. @@ -145,6 +147,8 @@ def list_serial_ports(self): win32file.CloseHandle(port) # type: ignore port = None + # On cross platform IDEs pylint may false positive. + # pylint: disable=no-member except pywintypes.error as err: # type: ignore # Error code 5 Apparently if the serial port is in use. # Error code 121 Apparently if timeout in operation. @@ -196,6 +200,8 @@ def gui_open_file(self, mime_types=None, start_dir=None): fname, _filter, __flags = \ win32gui.GetOpenFileNameW(Filter=win_filter) # type: ignore return str(fname) + # On cross platform IDEs pylint may false positive. + # pylint: disable=no-member except pywintypes.error as err: # type: ignore self.logger.info("gui_open_file: Failed to get filename: %s", err) except NameError: @@ -221,6 +227,8 @@ def gui_save_file(self, mime_types=None, start_dir=None, default_name=None): win32gui.GetSaveFileNameW(File=default_name, Filter=win_filter) # type: ignore return str(fname) + # On cross platform IDEs pylint may false positive. + # pylint: disable=no-member except pywintypes.error as err: # type: ignore self.logger.info("gui_save_file: Failed to get filename: %s", err) except NameError: @@ -243,6 +251,8 @@ def gui_select_dir(self, start_dir=None): err = "No error detected" pidl, _display_name, _ilmage_list = \ shell.SHBrowseForFolder() # type: ignore + # On cross platform IDEs pylint may false positive. + # pylint: disable=no-member except pywintypes.com_error: # type: ignore pass if not pidl: @@ -292,7 +302,7 @@ def have_sound(): Do we have sound support? :returns: Status of sound support - :rytpe: bool + :rtype: bool ''' return HAVE_AUDIO diff --git a/d_rats/utils.py b/d_rats/utils.py index 5bcb47aa..c0fcd96c 100755 --- a/d_rats/utils.py +++ b/d_rats/utils.py @@ -23,10 +23,16 @@ import urllib.request -import gi -gi.require_version("Gtk", "3.0") -from gi.repository import Gtk -from gi.repository import Gdk +# This version of D-Rats requires GTK 3.0, but not for doing unit +# tests. This allows the default unit tests to pass on a system with +# out GTK+ installed. +try: + import gi + gi.require_version("Gtk", "3.0") + from gi.repository import Gtk + from gi.repository import Gdk +except (ImportError, ValueError): + pass # This makes pylance happy with out overriding settings # from the invoker of the class diff --git a/tests/pre-commit b/tests/pre-commit index 366fed41..f28a18b6 100755 --- a/tests/pre-commit +++ b/tests/pre-commit @@ -47,9 +47,10 @@ CHANGED_FILES=$(git diff --name-only --cached --diff-filter=ACMR) export CHANGED_FILES rc=0 -for check_script in tests/pre-commit_d/*; do +for check_script in tests/pre-commit_d/*.sh; do if ! "${check_script}"; then (( rc=rc+PIPESTATUS[0] )) + echo "${check_script} returned $rc" fi done exit "$rc" diff --git a/tests/pre-commit_d/python_unit_tests.sh b/tests/pre-commit_d/python_unit_tests.sh new file mode 100755 index 00000000..fa8dee39 --- /dev/null +++ b/tests/pre-commit_d/python_unit_tests.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -uex + +# Needed for some platforms +export NO_AT_BRIDGE=1 + +# Default list of files to lint +: "${CHANGED_FILES:=}" +if [ -z "$CHANGED_FILES" ]; then + # Nothing to do + exit 0 +fi +rc=0 +#hack: Only way to put a newline into a variable +nl=' +' +delim="$nl" + +# If only one file, delimiter does not matter. +if [[ "$CHANGED_FILES" == *" "* ]]; then + delim=" " +fi + +test_scripts=() +test_scripts+=("d_rats/dplatform") +test_scripts+=("d_rats/ddt2") + +echo "CHANGED_FILES: $CHANGED_FILES" +for test_script in "${test_scripts[@]}"; do + # Hack is to add a leading delimiter for a quick check to + # see if filename is in a character delimited path + echo "test_script: $test_script" + if [[ "$delim$CHANGED_FILES" == *"$delim$test_script"* ]]; then + module="${test_scripts//\//.}" + python -m "${module}" || rc=1 + else + echo "No script file found." + module="${test_scripts//\//.}" + echo "module: $module" + fi +done +echo "final status = ${rc}" +exit "${rc}" diff --git a/workflow/Linux/requirements.txt b/workflow/Linux/requirements.txt new file mode 100755 index 00000000..e8b3f82e --- /dev/null +++ b/workflow/Linux/requirements.txt @@ -0,0 +1,22 @@ +# requirements for d-rats +# Packages needed for running +lxml +Pillow +# pycairo not available for Ubuntu GitHub workflows +# PyGObject not available for Ubuntu GitHub workflows +pyserial +pycountry +# packages recommended for running +geopy +feedparser +# pyaudio not available for Ubuntu GitHub workflows +pydub +# packages needed for development +build +pip +pylint +setuptools +Sphinx +towncrier +virtualenv +wheel diff --git a/workflow/Windows/requirements.txt b/workflow/Windows/requirements.txt new file mode 100755 index 00000000..1f2f5e37 --- /dev/null +++ b/workflow/Windows/requirements.txt @@ -0,0 +1,24 @@ +# requirements for d-rats +# Microsoft Windows needed +pywin32 +# Packages needed for running +lxml +Pillow +pycairo +# PyGObject not available for Windows GitHub workflows +pyserial +pycountry +# packages recommended for running +geopy +feedparser +pyaudio +pydub +# packages needed for development +build +pip +pylint +setuptools +Sphinx +towncrier +virtualenv +wheel diff --git a/workflow/macOS/requirements.txt b/workflow/macOS/requirements.txt new file mode 100755 index 00000000..da814695 --- /dev/null +++ b/workflow/macOS/requirements.txt @@ -0,0 +1,22 @@ +# requirements for d-rats +# Packages needed for running +lxml +Pillow +pycairo +PyGObject +pyserial +pycountry +# packages recommended for running +geopy +feedparser +# pyaudio not available on macos in GitHub Workflows. +pydub +# packages needed for development +build +pip +pylint +setuptools +Sphinx +towncrier +virtualenv +wheel