From 6e0c91429d6cc621f5760704de6e6bd235733aa5 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 13:34:40 -0300 Subject: [PATCH 01/11] ci: remove codacy analysis scan Motivation: - The job is failing due to the lack of multi-tools Sarif support on Github - We already have CodeQL running as well - It barely runs python linters like bandit, pylint, ruff - It runs some non related linters like: pmd, checov, jacksonlinter - Currently there is configuration for any python linters in the repository - It outputs too much issues that make it unfeasible - Nobody is checking the tool outputs - We have to settle on the tools as described in #597 - We can enable it again further --- .github/workflows/codacy-analysis.yml | 86 --------------------------- 1 file changed, 86 deletions(-) delete mode 100644 .github/workflows/codacy-analysis.yml diff --git a/.github/workflows/codacy-analysis.yml b/.github/workflows/codacy-analysis.yml deleted file mode 100644 index a132505c..00000000 --- a/.github/workflows/codacy-analysis.yml +++ /dev/null @@ -1,86 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -# This workflow checks out code, performs a Codacy security scan -# and integrates the results with the -# GitHub Advanced Security code scanning feature. For more information on -# the Codacy security scan action usage and parameters, see -# https://github.com/codacy/codacy-analysis-cli-action. -# For more information on Codacy Analysis CLI in general, see -# https://github.com/codacy/codacy-analysis-cli. - -name: Codacy Security Scan - -on: - schedule: - - cron: "59 11 27 * *" - push: - branches: ["master"] - merge_group: - branches: ["master"] - pull_request: - branches: ["master"] - types: [opened, reopened, synchronize, ready_for_review] - # workflow_run: - # workflows: [Test Changes] - # types: - # - completed - workflow_call: - workflow_dispatch: - inputs: - branch_commit_or_ref: - description: "Run this workflow in what branch/commit?" - required: true - type: string - default: master - -permissions: - contents: read - -jobs: - codacy-security-scan: - permissions: - contents: read # for actions/checkout to fetch code - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status - name: Codacy Security Scan - runs-on: ubuntu-latest - if: | - ( (!github.event.pull_request.draft) && - (github.event.action != 'labeled') && - (!contains( github.event.pull_request.labels.*.name, 'prevent-ci')) ) - || ((github.event.action != 'labeled') && contains( github.event.pull_request.labels.*.name, 'force-ci')) - || (github.event.label.name == 'force-ci') - steps: - # Checkout the repository to the GitHub Actions runner - - name: Checkout code at ${{ inputs.branch_commit_or_ref || github.ref }} - uses: actions/checkout@v6 - with: - ref: ${{ inputs.branch_commit_or_ref || github.ref }} - - # Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis - - name: Run Codacy Analysis CLI - uses: codacy/codacy-analysis-cli-action@v4 - with: - # Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository - # You can also omit the token and run the tools that support default configurations - project-token: ${{ secrets.CODACY_PROJECT_TOKEN }} - verbose: true - output: results.sarif - format: sarif - # Adjust severity of non-security issues - gh-code-scanning-compat: true - # Force 0 exit code to allow SARIF file generation - # This will handover control about PR rejection to the GitHub side - max-allowed-issues: 2147483647 - - # Upload the SARIF file generated in the previous step - - name: Upload SARIF results file - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: results.sarif - category: codacy - -## End of codacy-security-scan job ## From 77c720ff338da2699f1900a39d237fa2ced4b8f0 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 21:09:28 -0300 Subject: [PATCH 02/11] tests: add py314 to tox.ini --- tox.ini | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index 8139fe4b..3e4a41a2 100644 --- a/tox.ini +++ b/tox.ini @@ -11,14 +11,15 @@ #region Automated Testing -------------------------------------------------------------- [tox] -envlist = py27, py36, py37, py38, py39, py310, py311, py312, py313, {py36,py37,py38,py39,py310,py311,py312,py313}-docs +envlist = py27, py36, py37, py38, py39, py310, py311, py312, py313, py314, {py36,py37,py38,py39,py310,py311,py312,py313,py314}-docs [testenv] # get stable output for unordered types setenv = PYTHONHASHSEED = 42 + PYTHONTRACEMALLOC = 1 py27: PY_MAJOR_VERSION = py2 - py36,py37,py38,py39,py310,py311,py312,py313: PY_MAJOR_VERSION = py3 + py36,py37,py38,py39,py310,py311,py312,py313,py314: PY_MAJOR_VERSION = py3 commands = pytest --cov=petl --cov-report lcov:lcov.info petl coverage report -m @@ -26,15 +27,15 @@ deps = -rrequirements-tests.txt -rrequirements-formats.txt -[testenv:{py36,py37,py38,py39,py310,py311,py312,py313}-doctest] +[testenv:{py36,py37,py38,py39,py310,py311,py312,py313,py314}-doctest] commands = - py36,py37,py38,py39,py310,py311,py312,py313: pytest --doctest-modules --cov=petl --cov-report lcov:lcov.info petl + py36,py37,py38,py39,py310,py311,py312,py313,py314: pytest --doctest-modules --cov=petl --cov-report lcov:lcov.info petl #endregion ----------------------------------------------------------------------------- #region Documentation ------------------------------------------------------------------ -[testenv:{py36,py37,py38,py39,py310,py311,py312,py313}-docs] +[testenv:{py36,py37,py38,py39,py310,py311,py312,py313,py314}-docs] # build documentation under similar environment to readthedocs changedir = docs deps = @@ -42,14 +43,14 @@ deps = commands = sphinx-build -W -b html -d {envtmpdir}/doctrees . _build/html -[testenv:{py36,py37,py38,py39,py310,py311,py312,py313}-dochtml] +[testenv:{py36,py37,py38,py39,py310,py311,py312,py313,py314}-dochtml] changedir = docs deps = -rrequirements-docs.txt commands = sphinx-build -W -b singlehtml -d {envtmpdir}/doctrees . _build/singlehtml -[testenv:{py36,py37,py38,py39,py310,py311,py312,py313}-docpdf] +[testenv:{py36,py37,py38,py39,py310,py311,py312,py313,py314}-docpdf] changedir = docs deps = -rrequirements-docs.txt From bf656d178416964a19170c00412b23a98d569a3d Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 21:11:26 -0300 Subject: [PATCH 03/11] tests: fix python2.7 compatibility in requirements --- requirements-database.txt | 3 ++- requirements-docs.txt | 4 +++- requirements-formats.txt | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/requirements-database.txt b/requirements-database.txt index eace77cb..55605912 100644 --- a/requirements-database.txt +++ b/requirements-database.txt @@ -4,4 +4,5 @@ cryptography pymysql SQLAlchemy>=1.3.6,<2.0 psycopg2-binary -# PyMySQL==0.9.3 +mysqlclient ; python_version < '3' + diff --git a/requirements-docs.txt b/requirements-docs.txt index 67d536cd..65aa6963 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,7 +2,9 @@ sphinx sphinx-issues -rinohtype +rinohtype==0.4.2 ; python_version >= '3.5' and python_version <= '3.7.9' +rinohtype==0.5.5 ; python_version >= '3.7' and python_version <= '3.9.9' +rinohtype ; python_version >= '3.10' # Used for generating docs version from Git diff --git a/requirements-formats.txt b/requirements-formats.txt index b45b4a82..1354a10f 100644 --- a/requirements-formats.txt +++ b/requirements-formats.txt @@ -1,6 +1,7 @@ Cython numpy -numexpr +numexpr>=2.6.9,<2.8 ; python_version < '3' +numexpr ; python_version >= '3' intervaltree>=3.0.2 lxml>=4.6.5 openpyxl>=2.6.2 From 4ebe3c288eb62357a3e2fc398673f467155982a4 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 21:17:36 -0300 Subject: [PATCH 04/11] test: refactor test_db_server connection argument handling --- petl/test/io/test_db_create.py | 31 +++++----- petl/test/io/test_db_server.py | 104 ++++++++++++++++++++++----------- 2 files changed, 86 insertions(+), 49 deletions(-) diff --git a/petl/test/io/test_db_create.py b/petl/test/io/test_db_create.py index a4a6cc61..0f418b68 100644 --- a/petl/test/io/test_db_create.py +++ b/petl/test/io/test_db_create.py @@ -12,7 +12,7 @@ from petl.io.db_create import make_sqlalchemy_column from petl.test.helpers import ieq, eq_ from petl.util.vis import look -from petl.test.io.test_db_server import user, password, host, database +from petl.test.io.test_db_server import get_pg_args, get_mysql_args logger = logging.getLogger(__name__) @@ -110,7 +110,7 @@ def _setup_generic(dbapi_connection): try: - import sqlalchemy # noqa: F401 + import sqlalchemy except ImportError as e: pytest.skip('SKIP generic DB create tests: %s' % e, allow_module_level=True) else: @@ -149,10 +149,11 @@ def test_sqlite3_create(): try: import pymysql import sqlalchemy - pymysql.connect(host=host, - user=user, - password=password, - database=database) + myhost, myuser, mypassword, mydatabase = get_mysql_args() + pymysql.connect(host=myhost, + user=myuser, + password=mypassword, + database=mydatabase) except Exception as e: SKIP_PYMYSQL = 'SKIP pymysql create tests: %s' % e finally: @@ -160,13 +161,12 @@ def test_sqlite3_create(): def test_mysql_create(): import pymysql - connect = pymysql.connect # assume database already created - dbapi_connection = connect(host=host, - user=user, - password=password, - database=database) + dbapi_connection = pymysql.connect(host=myhost, + user=myuser, + password=mypassword, + database=mydatabase) # exercise using a dbapi_connection _setup_mysql(dbapi_connection) @@ -182,7 +182,7 @@ def test_mysql_create(): _setup_mysql(dbapi_connection) from sqlalchemy import create_engine sqlalchemy_engine = create_engine('mysql+pymysql://%s:%s@%s/%s' - % (user, password, host, database)) + % (myuser, mypassword, myhost, mydatabase)) sqlalchemy_connection = sqlalchemy_engine.connect() sqlalchemy_connection.execute('SET SQL_MODE=ANSI_QUOTES') _test_create(sqlalchemy_connection) @@ -200,10 +200,11 @@ def test_mysql_create(): SKIP_POSTGRES = False try: import psycopg2 - import sqlalchemy # noqa: F401 + import sqlalchemy + pghost, pguser, pgpassword, pgdatabase = get_pg_args() psycopg2.connect( 'host=%s dbname=%s user=%s password=%s' - % (host, database, user, password) + % (pghost, pgdatabase, pguser, pgpassword) ) except Exception as e: SKIP_POSTGRES = 'SKIP psycopg2 create tests: %s' % e @@ -218,7 +219,7 @@ def test_postgresql_create(): # assume database already created dbapi_connection = psycopg2.connect( 'host=%s dbname=%s user=%s password=%s' - % (host, database, user, password) + % (pghost, pgdatabase, pguser, pgpassword) ) dbapi_connection.autocommit = True diff --git a/petl/test/io/test_db_server.py b/petl/test/io/test_db_server.py index 75e2cd7d..8c16798c 100644 --- a/petl/test/io/test_db_server.py +++ b/petl/test/io/test_db_server.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, division import logging +import os +import warnings import pytest @@ -12,6 +14,36 @@ debug = logger.debug +def get_from_env(prefix, name, current, default): + if current != default: + return current + varname = "{}_{}".format(prefix.upper(), name.upper()) + return os.getenv(varname, default) + + +def get_db_args(*prefixes): + host = '127.0.0.1' + user = 'petl' + pwrd = 'test' + base = 'petl' + for prefix in prefixes: + host = get_from_env(prefix, 'HOST', host, '127.0.0.1') + user = get_from_env(prefix, 'USER', user, 'petl') + pwrd = get_from_env(prefix, 'PASSWORD', pwrd, 'test') + base = get_from_env(prefix, 'DATABASE', base, 'petl') + return host, user, pwrd, base + + +def get_mysql_args(*prefixes): + prefixes += ('PYMYSQL', 'MYSQL',) + return get_db_args(*prefixes) + + +def get_pg_args(*prefixes): + prefixes += ('PSYCOPG2', 'POSTGRESQL', 'POSTGRES', 'PG',) + return get_db_args(*prefixes) + + def _test_dbo(write_dbo, read_dbo=None): if read_dbo is None: read_dbo = write_dbo @@ -131,30 +163,29 @@ def _setup_sqlalchemy_quotes(dbapi_connection, connection_record): cursor.execute("SET sql_mode = 'ANSI_QUOTES'") -host, user, password, database = '127.0.0.1', 'petl', 'test', 'petl' - SKIP_PYMYSQL = False try: import pymysql import sqlalchemy - pymysql.connect(host=host, - user=user, - password=password, - database=database) + myhost, myuser, mypassword, mydatabase = get_mysql_args() + pymysql.connect(host=myhost, + user=myuser, + password=mypassword, + database=mydatabase) except Exception as e: SKIP_PYMYSQL = 'SKIP pymysql tests: %s' % e + warnings.warn(SKIP_PYMYSQL, UserWarning) finally: @pytest.mark.skipif(bool(SKIP_PYMYSQL), reason=str(SKIP_PYMYSQL)) def test_pymysql(): import pymysql - connect = pymysql.connect # assume database already created - dbapi_connection = connect(host=host, - user=user, - password=password, - database=database) + dbapi_connection = pymysql.connect(host=myhost, + user=myuser, + password=mypassword, + database=mydatabase) # exercise using a dbapi_connection _setup_mysql(dbapi_connection) @@ -170,7 +201,7 @@ def test_pymysql(): _setup_mysql(dbapi_connection) from sqlalchemy import create_engine sqlalchemy_engine = create_engine('mysql+pymysql://%s:%s@%s/%s' % - (user, password, host, database)) + (myuser, mypassword, myhost, mydatabase)) from sqlalchemy.event import listen listen(sqlalchemy_engine, "connect", _setup_sqlalchemy_quotes) sqlalchemy_connection = sqlalchemy_engine.connect() @@ -188,16 +219,17 @@ def test_pymysql(): # exercise sqlalchemy engine _setup_mysql(dbapi_connection) sqlalchemy_engine2 = create_engine('mysql+pymysql://%s:%s@%s/%s' % - (user, password, host, database), echo_pool='debug') + (myuser, mypassword, myhost, mydatabase), echo_pool='debug') listen(sqlalchemy_engine2, "connect", _setup_sqlalchemy_quotes) _test_dbo(sqlalchemy_engine2) sqlalchemy_engine2.dispose() # other exercises - _test_with_schema(dbapi_connection, database) - utf8_connection = connect(host=host, user=user, - password=password, - database=database, + _test_with_schema(dbapi_connection, mydatabase) + utf8_connection = pymysql.connect(host=myhost, + user=myuser, + password=mypassword, + database=mydatabase, charset='utf8') utf8_connection.cursor().execute('SET SQL_MODE=ANSI_QUOTES') _test_unicode(utf8_connection) @@ -208,24 +240,25 @@ def test_pymysql(): try: import MySQLdb import sqlalchemy - MySQLdb.connect(host=host, - user=user, - passwd=password, - db=database) + myhost, myuser, mypassword, mydatabase = get_db_args('MYSQLDB', 'MYSQL') + MySQLdb.connect(host=myhost, + user=myuser, + passwd=mypassword, + db=mydatabase) except Exception as e: SKIP_MYSQLDB = 'SKIP MySQLdb tests: %s' % e + warnings.warn(SKIP_MYSQLDB, UserWarning) finally: @pytest.mark.skipif(bool(SKIP_MYSQLDB), reason=str(SKIP_MYSQLDB)) def test_mysqldb(): import MySQLdb - connect = MySQLdb.connect # assume database already created - dbapi_connection = connect(host=host, - user=user, - passwd=password, - db=database) + dbapi_connection = MySQLdb.connect(host=myhost, + user=myuser, + passwd=mypassword, + db=mydatabase) # exercise using a dbapi_connection _setup_mysql(dbapi_connection) @@ -241,7 +274,7 @@ def test_mysqldb(): _setup_mysql(dbapi_connection) from sqlalchemy import create_engine sqlalchemy_engine = create_engine('mysql+mysqldb://%s:%s@%s/%s' % - (user, password, host, database)) + (myuser, mypassword, myhost, mydatabase)) from sqlalchemy.event import listen listen(sqlalchemy_engine, "connect", _setup_sqlalchemy_quotes) sqlalchemy_connection = sqlalchemy_engine.connect() @@ -258,10 +291,11 @@ def test_mysqldb(): sqlalchemy_session.close() # other exercises - _test_with_schema(dbapi_connection, database) - utf8_connection = connect(host=host, user=user, - passwd=password, - db=database, + _test_with_schema(dbapi_connection, mydatabase) + utf8_connection = MySQLdb.connect(host=myhost, + user=myuser, + passwd=mypassword, + db=mydatabase, charset='utf8') utf8_connection.cursor().execute('SET SQL_MODE=ANSI_QUOTES') _test_unicode(utf8_connection) @@ -270,12 +304,14 @@ def test_mysqldb(): try: import psycopg2 import sqlalchemy + pghost, pguser, pgpassword, pgdatabase = get_pg_args() psycopg2.connect( 'host=%s dbname=%s user=%s password=%s' - % (host, database, user, password) + % (pghost, pgdatabase, pguser, pgpassword) ) except Exception as e: SKIP_TEST_POSTGRES = 'SKIP psycopg2 tests: %s' % e + warnings.warn(SKIP_TEST_POSTGRES, UserWarning) finally: @pytest.mark.skipif(bool(SKIP_TEST_POSTGRES), reason=str(SKIP_TEST_POSTGRES)) def test_postgresql(): @@ -288,7 +324,7 @@ def test_postgresql(): # assume database already created dbapi_connection = psycopg2.connect( 'host=%s dbname=%s user=%s password=%s' - % (host, database, user, password) + % (pghost, pgdatabase, pguser, pgpassword) ) # exercise using a dbapi_connection @@ -305,7 +341,7 @@ def test_postgresql(): _setup_postgresql(dbapi_connection) from sqlalchemy import create_engine sqlalchemy_engine = create_engine('postgresql+psycopg2://%s:%s@%s/%s' % - (user, password, host, database)) + (pguser, pgpassword, pghost, pgdatabase)) sqlalchemy_connection = sqlalchemy_engine.connect() _test_dbo(sqlalchemy_connection) sqlalchemy_connection.close() From 8033846311acc271f2ab7f387ae43b1a8e1cf894 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 21:18:56 -0300 Subject: [PATCH 05/11] ci: put the workflow report after python setup --- .github/workflows/test-changes.yml | 31 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index 8ddbccf2..c180160a 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -87,19 +87,6 @@ jobs: with: ref: ${{ inputs.branch_commit_or_ref || github.ref }} - - name: Report Workflow Information - id: workflow_report - uses: ./.github/actions/workflow-info - with: - title: "${{ github.ref_name }} on ${{ matrix.os }} with python ${{ matrix.python }}" - parameters: "${{ toJson(inputs) }}" - content: | - - Ref: ${{ github.head_ref }} - - matrix.python: ${{ matrix.python }} - - matrix.os: ${{ matrix.os }} - - matrix.mode: ${{ matrix.mode }} - - matrix.install: ${{ matrix.install }} - - name: Install linux tools if: matrix.os == 'ubuntu-latest' run: | @@ -119,9 +106,25 @@ jobs: python-version: ${{ matrix.python }} cache: pip - - name: Install test dependencies + - name: Upgrade pip run: | python -m pip install --upgrade pip + + - name: Report Workflow Information + id: workflow_report + uses: ./.github/actions/workflow-info + with: + title: "${{ github.ref_name }} on ${{ matrix.os }} with python ${{ matrix.python }}" + parameters: "${{ toJson(inputs) }}" + content: | + - Ref: ${{ github.head_ref }} + - matrix.python: ${{ matrix.python }} + - matrix.os: ${{ matrix.os }} + - matrix.mode: ${{ matrix.mode }} + - matrix.install: ${{ matrix.install }} + + - name: Install test dependencies + run: | python -m pip install --prefer-binary -r requirements-tests.txt - name: Setup environment variables for remote filesystem testing with primary = ${{ matrix.primary }} From 1521d71b732fca4a2955509ac547d3b081912c50 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 21:19:50 -0300 Subject: [PATCH 06/11] tests: add isntalled packages to the job report --- .github/workflows/test-changes.yml | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index c180160a..e2cdb187 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -184,7 +184,13 @@ jobs: - name: List Installed Packages for Throubleshooting run: | echo "::group::List Installed Packages for Throubleshooting" - python -m pip list --format freeze + printf '\n## Installed Packages for python ${{ matrix.python }} with mode ${{ matrix.mode }}\n\n' >> "${GITHUB_STEP_SUMMARY}"; + echo "- Python Version: $(python --version)" + echo "- Pytest Version: $(pytest --version)" + echo "- Pip Version: $(pip --version | head -n 1 | cut -d' ' -f2)" + printf '```text' >> "${GITHUB_STEP_SUMMARY}"; + python -m pip list --format freeze >> "${GITHUB_STEP_SUMMARY}"; + printf '```' >> "${GITHUB_STEP_SUMMARY}"; echo "::endgroup::" - name: Test python source code for mode basic @@ -246,24 +252,3 @@ jobs: run: | cd docs sphinx-build -W -b singlehtml -d ../build/doctrees . ../build/singlehtml - - # TODO: 1 configuration not found: https://docs.github.com/en/actions/using-workflows/reusing-workflows#overview - # call-workflow-codeql: - # needs: test-source-code - # uses: ./.github/workflows/codeql-analysis.yml - # permissions: - # actions: read - # contents: read - # pull-requests: write - # security-events: write - # secrets: inherit # pass all secrets - - # call-workflow-codacity: - # needs: test-source-code - # uses: ./.github/workflows/codacy-analysis.yml - # permissions: - # actions: read - # contents: read - # pull-requests: write - # security-events: write - # secrets: inherit # pass all secrets From 5ee24527e8bba0e4daf9ee356813ccb9520ca5e6 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 21:20:41 -0300 Subject: [PATCH 07/11] tests: add local sandbox containers with python3.x and python2.7 --- docker-compose.yml | 126 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 24fa8875..1330f053 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,17 +2,26 @@ #region commands ----------------------------------------------------------------------------------- +# 1. Create the containers and the services in the background #$ docker compose --file docker-compose.yml up --detach -#$ docker compose --file docker-compose.yml down --remove-orphans -v --rmi local -#$ docker exec --tty --interactive --privileged petl-xxxxxx /bin/bash +# 2. Test database connections #$ docker exec -it petl-postgres psql -U petl --dbname=petl #$ docker exec -it petl-msyql mysql --user=petl --database=petl --password=test +# 3. Run a shell inside the python sandbox container + +#$ docker exec --interactive --tty --privileged petl-python27 /bin/bash +#$ docker exec -it --privileged petl-python3x /bin/bash + +# 5. Stop the services and remove the containers + +#$ docker compose --file docker-compose.yml down --remove-orphans -v --rmi local + #endregion ----------------------------------------------------------------------------------------- -#region docker composer ---------------------------------------------------------------------------- +#region file/db services --------------------------------------------------------------------------- --- services: @@ -31,11 +40,22 @@ services: stdin_open: true tty: true healthcheck: - test: ["CMD", "psql", "--host=localhost", "--username=petl", "--dbname=petl", "-c", "select 1 as ok"] + test: + [ + "CMD", + "psql", + "--host=localhost", + "--username=petl", + "--dbname=petl", + "-c", + "select 1 as ok", + ] interval: 20s timeout: 10s retries: 5 start_period: 2s + networks: + - petl_network mysql: container_name: petl-mysql @@ -54,11 +74,13 @@ services: stdin_open: true tty: true healthcheck: - test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"] + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] interval: 20s timeout: 10s retries: 5 start_period: 2s + networks: + - petl_network samba: container_name: petl-samba @@ -78,18 +100,108 @@ services: - /mnt:/mnt:z - /mnt2:/mnt2:z command: '-s "public;/mnt;yes;no;yes;all" -s "mount2;/mnt2" -u "petl;test" -p' + networks: + - petl_network sftp: container_name: petl-sftp hostname: petl_sftp image: atmoz/sftp ports: - - "22:22/tcp" + - "2244:22/tcp" tmpfs: - /tmp restart: unless-stopped stdin_open: true tty: true - command: 'petl:test:::public' + command: "petl:test:::public" + networks: + - petl_network + + #endregion ----------------------------------------------------------------------------------------- + + #region python sandbox ----------------------------------------------------------------------------- + + python3x: + container_name: petl-python3x + hostname: petl_py3x + # image: python:slim + environment: + - PYTHON3_IMAGE_TAG=${PYTHON3_IMAGE_TAG:-slim} + - PYTHONTRACEMALLOC=1 + - POSTGRES_HOST=petl_postgres + - MYSQL_HOST=petl_mysql + - PETL_TEST_SMB=smb://WORKGROUP;petl:test@petl_samba/public/ + - PETL_TEST_SFTP=sftp://petl:test@petl_sftp:2244/public/ + build: + context: . + dockerfile_inline: | + FROM python:${PYTHON3_IMAGE_TAG:-slim} + RUN apt-get update + RUN apt-get install -y postgresql-client default-mysql-client smbclient openssh-client + RUN apt-get install -y gcc git + RUN python -m pip install --upgrade pip + RUN mkdir -p /app + RUN mkdir -p /req + COPY requirements-*.txt /req/ + RUN python -m pip install -r req/requirements-tests.txt -r req/requirements-database.txt -r req/requirements-formats.txt -r req/requirements-remote.txt -r req/requirements-docs.txt -r req/requirements-linting.txt + # Keeps the container running forever + command: tail -f /dev/null + restart: unless-stopped + stdin_open: true + tty: true + # mounts the repository souce code into the container + volumes: + - .:/app + working_dir: /app + networks: + - petl_network + + python27: + container_name: petl-python27 + hostname: petl_py27 + # image: python:2.7.18-slim + environment: + - PYTHON2_IMAGE_TAG=${PYTHON2_IMAGE_TAG:-2.7.18-slim} + - PYTHONTRACEMALLOC=1 + - POSTGRES_HOST=petl_postgres + - MYSQL_HOST=petl_mysql + - PETL_TEST_SMB=smb://WORKGROUP;petl:test@petl_samba/public/ + - PETL_TEST_SFTP=sftp://petl:test@petl_sftp:2244/public/ + build: + context: . + dockerfile_inline: | + FROM python:${PYTHON2_IMAGE_TAG:-2.7.18-slim} + RUN sed -i 's/deb.debian.org/archive.debian.org/g' /etc/apt/sources.list \ + && sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list \ + && sed -i '/buster-updates/d' /etc/apt/sources.list + RUN apt-get update + RUN apt-get install -y postgresql-client default-mysql-client smbclient openssh-client + RUN apt-get install -y gcc git + RUN python -m pip install --upgrade pip + RUN mkdir -p /app + RUN apt-get install -y default-libmysqlclient-dev + RUN mkdir -p /req + COPY requirements-*.txt /req/ + RUN python -m pip install -r req/requirements-tests.txt -r req/requirements-database.txt -r req/requirements-formats.txt -r req/requirements-remote.txt -r req/requirements-docs.txt + # Keeps the container running forever + command: tail -f /dev/null + restart: unless-stopped + stdin_open: true + tty: true + # mounts the repository souce code into the container + volumes: + - .:/app + working_dir: /app + networks: + - petl_network + +#endregion ----------------------------------------------------------------------------------------- + +#region other settings ----------------------------------------------------------------------------- + +networks: + petl_network: + driver: bridge #endregion ----------------------------------------------------------------------------------------- From 089c87c42ad0b79848d6a386f7d3b33d5c5e3f3b Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 22:05:43 -0300 Subject: [PATCH 08/11] ci: print top 10 durations for pytest --- .github/workflows/test-changes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index e2cdb187..a7e39c85 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -205,7 +205,7 @@ jobs: PYTHONTRACEMALLOC: 1 run: | echo "::group::Perform doctest-modules execution with coverage" - pytest --doctest-modules --cov=petl petl + pytest --doctest-modules --cov=petl --durations=10 petl echo "::endgroup::" - name: Coveralls with primary = ${{ matrix.primary }} From c4a4bedd82e878c0b1c6044fe234f5dc3ad0708d Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 22:13:29 -0300 Subject: [PATCH 09/11] ci: fix some issues found after submitted the last PR --- .github/workflows/test-changes.yml | 7 ++++--- requirements-docs.txt | 2 +- tox.ini | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index a7e39c85..0ef1d773 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -112,6 +112,7 @@ jobs: - name: Report Workflow Information id: workflow_report + if: matrix.python >= '3' uses: ./.github/actions/workflow-info with: title: "${{ github.ref_name }} on ${{ matrix.os }} with python ${{ matrix.python }}" @@ -185,9 +186,9 @@ jobs: run: | echo "::group::List Installed Packages for Throubleshooting" printf '\n## Installed Packages for python ${{ matrix.python }} with mode ${{ matrix.mode }}\n\n' >> "${GITHUB_STEP_SUMMARY}"; - echo "- Python Version: $(python --version)" - echo "- Pytest Version: $(pytest --version)" - echo "- Pip Version: $(pip --version | head -n 1 | cut -d' ' -f2)" + echo "- Python Version: $(python --version)" >> "${GITHUB_STEP_SUMMARY}"; + echo "- Pytest Version: $(pytest --version)" >> "${GITHUB_STEP_SUMMARY}"; + echo "- Pip Version: $(pip --version | head -n 1 | cut -d' ' -f2)" >> "${GITHUB_STEP_SUMMARY}"; printf '```text' >> "${GITHUB_STEP_SUMMARY}"; python -m pip list --format freeze >> "${GITHUB_STEP_SUMMARY}"; printf '```' >> "${GITHUB_STEP_SUMMARY}"; diff --git a/requirements-docs.txt b/requirements-docs.txt index 65aa6963..b576cd15 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -2,7 +2,7 @@ sphinx sphinx-issues -rinohtype==0.4.2 ; python_version >= '3.5' and python_version <= '3.7.9' +rinohtype==0.4.2 ; python_version >= '3.5' and python_version <= '3.6.9' rinohtype==0.5.5 ; python_version >= '3.7' and python_version <= '3.9.9' rinohtype ; python_version >= '3.10' diff --git a/tox.ini b/tox.ini index 3e4a41a2..def203ce 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,7 @@ [tox] envlist = py27, py36, py37, py38, py39, py310, py311, py312, py313, py314, {py36,py37,py38,py39,py310,py311,py312,py313,py314}-docs +skip_missing_interpreters = true [testenv] # get stable output for unordered types @@ -72,7 +73,7 @@ commands = setenv = {[testenv]setenv} PETL_TEST_SMB=smb://WORKGROUP;petl:test@localhost/public/ - PETL_TEST_SFTP=sftp://petl:test@localhost/public/ + PETL_TEST_SFTP=sftp://petl:test@localhost:2244/public/ commands = pytest --cov=petl --cov-report lcov:lcov.info petl deps = From cefa877ec5b381471bce6c4e32f58cd5b2967558 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 22:55:56 -0300 Subject: [PATCH 10/11] ci: execute more tests with py2.7 --- .github/workflows/test-changes.yml | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index 0ef1d773..2a4fd6d2 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -51,6 +51,7 @@ jobs: include: # Default configuration for all jobs - mode: "full" + - scope: "full" - install: "no" - primary: "3.8" # Conditional configuration overrides @@ -60,14 +61,17 @@ jobs: install: "yes" - os: "windows-latest" mode: "basic" + scope: "basic" # Handle random test failures - os: "ubuntu-latest" python: "3.9" mode: "basic" + scope: "basic" # Additional Jobs not included in the matrix above - python: "2.7" os: "ubuntu-latest" mode: "basic" + scope: "full" install: "yes" exclude: # macos-latest == macos-15-arm64 @@ -93,6 +97,12 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends python3-h5py + - name: Install additional linux tools + if: matrix.os == 'ubuntu-latest' && matrix.mode == 'full' && matrix.scope == 'full' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends default-libmysqlclient-dev + - name: Set up Python ${{ matrix.python }} with install = ${{ matrix.install }} if: matrix.install == 'no' uses: actions/setup-python@v6 @@ -122,6 +132,7 @@ jobs: - matrix.python: ${{ matrix.python }} - matrix.os: ${{ matrix.os }} - matrix.mode: ${{ matrix.mode }} + - matrix.scope: ${{ matrix.scope }} - matrix.install: ${{ matrix.install }} - name: Install test dependencies @@ -162,8 +173,8 @@ jobs: docker run -it --name sftp -p 2244:22 -d atmoz/sftp petl:test:::public echo "::endgroup::" - - name: Install containers for remote database testing with mode = ${{ matrix.mode }} - if: matrix.os == 'ubuntu-latest' && matrix.mode == 'full' + - name: Install containers for remote database testing with scope = ${{ matrix.scope }} + if: matrix.os == 'ubuntu-latest' && matrix.scope == 'full' run: | echo "::group::Setup docker for MySQL" docker run -it --name mysql -p 3306:3306 -p 33060:33060 -e MYSQL_ROOT_PASSWORD=pass0 -e MYSQL_DATABASE=petl -e MYSQL_USER=petl -e MYSQL_PASSWORD=test -d mysql:latest @@ -178,14 +189,15 @@ jobs: - name: Setup petl package with mode = ${{ matrix.mode }} run: python setup.py sdist bdist_wheel - - name: Install extra packages dependencies for mode full - if: matrix.mode == 'full' + - name: Install extra packages dependencies for scope = ${{ matrix.scope }} + if: matrix.scope == 'full' run: python -m pip install --prefer-binary -r requirements-formats.txt - name: List Installed Packages for Throubleshooting + shell: bash run: | echo "::group::List Installed Packages for Throubleshooting" - printf '\n## Installed Packages for python ${{ matrix.python }} with mode ${{ matrix.mode }}\n\n' >> "${GITHUB_STEP_SUMMARY}"; + printf '\n## Installed Packages for python ${{ matrix.python }}\n\n' >> "${GITHUB_STEP_SUMMARY}"; echo "- Python Version: $(python --version)" >> "${GITHUB_STEP_SUMMARY}"; echo "- Pytest Version: $(pytest --version)" >> "${GITHUB_STEP_SUMMARY}"; echo "- Pip Version: $(pip --version | head -n 1 | cut -d' ' -f2)" >> "${GITHUB_STEP_SUMMARY}"; @@ -194,13 +206,13 @@ jobs: printf '```' >> "${GITHUB_STEP_SUMMARY}"; echo "::endgroup::" - - name: Test python source code for mode basic + - name: Test python source code for mode = basic if: matrix.mode == 'basic' env: PYTHONTRACEMALLOC: 1 run: pytest --cov=petl petl - - name: Test documentation inside source code for mode full + - name: Test including documentation inside source code for mode = full if: matrix.mode == 'full' env: PYTHONTRACEMALLOC: 1 @@ -209,7 +221,7 @@ jobs: pytest --doctest-modules --cov=petl --durations=10 petl echo "::endgroup::" - - name: Coveralls with primary = ${{ matrix.primary }} + - name: Coveralls with os = ${{ matrix.os }}, primary = ${{ matrix.primary }} if: matrix.os == 'ubuntu-latest' && matrix.python == matrix.primary env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -217,7 +229,7 @@ jobs: python -m pip install --upgrade coveralls coveralls --service=github - - name: Print source code coverage with mode = ${{ matrix.mode }} + - name: Print source code coverage run: | printf '\n## Coverage for python ${{ matrix.python }} with mode ${{ matrix.mode }}\n\n' >> "${GITHUB_STEP_SUMMARY}"; printf '```text' >> "${GITHUB_STEP_SUMMARY}"; From e4a6433bf48ee8fb5de7c9312aac8f5d1923dbc1 Mon Sep 17 00:00:00 2001 From: Juarez Rudsatz Date: Tue, 2 Jun 2026 23:07:59 -0300 Subject: [PATCH 11/11] ci: fix issue with workflow found by qodo.ai --- .github/workflows/test-changes.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-changes.yml b/.github/workflows/test-changes.yml index 2a4fd6d2..1be96ede 100644 --- a/.github/workflows/test-changes.yml +++ b/.github/workflows/test-changes.yml @@ -94,14 +94,17 @@ jobs: - name: Install linux tools if: matrix.os == 'ubuntu-latest' run: | + echo "::group::Install linux tools" sudo apt-get update sudo apt-get install -y --no-install-recommends python3-h5py + echo "::endgroup::" - name: Install additional linux tools - if: matrix.os == 'ubuntu-latest' && matrix.mode == 'full' && matrix.scope == 'full' + if: matrix.os == 'ubuntu-latest' && matrix.scope == 'full' run: | - sudo apt-get update + echo "::group::Install additional linux tools" sudo apt-get install -y --no-install-recommends default-libmysqlclient-dev + echo "::endgroup::" - name: Set up Python ${{ matrix.python }} with install = ${{ matrix.install }} if: matrix.install == 'no' @@ -118,7 +121,9 @@ jobs: - name: Upgrade pip run: | + echo "::group::Upgrading pip" python -m pip install --upgrade pip + echo "::endgroup::" - name: Report Workflow Information id: workflow_report @@ -137,7 +142,9 @@ jobs: - name: Install test dependencies run: | + echo "::group::Install test dependencies" python -m pip install --prefer-binary -r requirements-tests.txt + echo "::endgroup::" - name: Setup environment variables for remote filesystem testing with primary = ${{ matrix.primary }} if: matrix.os == 'ubuntu-latest' && matrix.python == matrix.primary @@ -191,7 +198,10 @@ jobs: - name: Install extra packages dependencies for scope = ${{ matrix.scope }} if: matrix.scope == 'full' - run: python -m pip install --prefer-binary -r requirements-formats.txt + run: | + echo "::group::Install extra packages dependencies" + python -m pip install --prefer-binary -r requirements-formats.txt + echo "::endgroup::" - name: List Installed Packages for Throubleshooting shell: bash @@ -210,7 +220,10 @@ jobs: if: matrix.mode == 'basic' env: PYTHONTRACEMALLOC: 1 - run: pytest --cov=petl petl + run: | + echo "::group::Perform basic test execution with coverage" + pytest --cov=petl petl + echo "::endgroup::" - name: Test including documentation inside source code for mode = full if: matrix.mode == 'full'