Improving the Dockerfile

.github/actions/setup-dev-python-env/action.yml

@@ -0,0 +1,15 @@
+name: Setup Dev Python Environment
+description: Set up Python 3.13 and install dev dependencies (requirements-dev.txt).
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+      with:
+        python-version: '3.13'
+        cache: 'pip'
+
+    - name: Install dev dependencies
+      shell: bash
+      run: pip install -r requirements-dev.txt

.github/copilot-instructions.md

@@ -53,5 +53,8 @@ Quality gates (run after changes, fix only if below threshold)
 - Run all quality gates at once: `make qa`
 - Once a quality gate passes, do not re-run it in different scenarios
 
+Dependencies
+- Two requirements files: `requirements.txt` (prod/Docker only) and `requirements-dev.txt` (extends prod with test and lint tools)
+
 Git workflow
 - Do NOT create git commits; committing is the developer's responsibility

.github/workflows/check_python.yml

@@ -39,9 +39,9 @@ jobs:
           if [[ "${{ github.event_name }}" == "pull_request" ]]; then
             CHANGED_FILES=$(gh api \
               "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
-              --jq '.[].filename | select(endswith(".py") or . == "requirements.txt")')
+              --jq '.[].filename | select(endswith(".py") or (startswith("requirements") and endswith(".txt")))')
           else
-            CHANGED_FILES=$(git diff --name-only "${{ github.sha }}~1" "${{ github.sha }}" -- '*.py' 'requirements.txt')
+            CHANGED_FILES=$(git diff --name-only "${{ github.sha }}~1" "${{ github.sha }}" -- '*.py' 'requirements*.txt')
           fi
 
           if [[ -n "$CHANGED_FILES" ]]; then
@@ -62,14 +62,8 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
-        with:
-          python-version: '3.13'
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: pip install -r requirements.txt
+      - name: Set up dev Python environment
+        uses: ./.github/actions/setup-dev-python-env
 
       - name: Analyze code with Pylint
         id: analyze-code
@@ -98,14 +92,8 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
-        with:
-          python-version: '3.13'
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: pip install -r requirements.txt
+      - name: Set up dev Python environment
+        uses: ./.github/actions/setup-dev-python-env
 
       - name: Check code format with Black
         id: check-format
@@ -123,14 +111,8 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
-        with:
-          python-version: '3.13'
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: pip install -r requirements.txt
+      - name: Set up dev Python environment
+        uses: ./.github/actions/setup-dev-python-env
 
       - name: Check types with Mypy
         id: check-types
@@ -148,14 +130,8 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
-        with:
-          python-version: '3.13'
-          cache: 'pip'
-
-      - name: Install Python dependencies
-        run: pip install -r requirements.txt
+      - name: Set up dev Python environment
+        uses: ./.github/actions/setup-dev-python-env
 
       - name: Check code coverage with Pytest
         run: pytest --cov=. -v tests/unit/ --cov-fail-under=80
@@ -173,14 +149,8 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
-        with:
-          python-version: '3.13'
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: pip install -r requirements.txt
+      - name: Set up dev Python environment
+        uses: ./.github/actions/setup-dev-python-env
 
       - name: Run integration tests
         run: pytest tests/integration/ -v --tb=short --log-cli-level=INFO

DEVELOPER.md

@@ -26,14 +26,13 @@ EventGate ships two Lambda functions:
 
 ## Prerequisites
 - Python 3.13 (current required runtime)
-- PostgreSQL client dev package
-  - For local development, you may temporarily switch to the commented `psycopg2-binary==2.9.10` in `requirements.txt`.
+- Docker (for local integration tests using testcontainers)
 
 ## Set Up Python Environment
 ```shell
 python3 -m venv .venv
 source .venv/bin/activate
-pip3 install -r requirements.txt
+pip3 install -r requirements-dev.txt
 ```
 
 ## Run Pylint Tool Locally

Dockerfile

@@ -25,6 +25,9 @@ ARG SASL_SSL_ARTIFACTS=./sasl_ssl_artifacts
 # Trusted certs
 COPY $TRUSTED_SSL_CERTS /opt/certs/
 
+# Production dependencies
+COPY requirements.txt ${LAMBDA_TASK_ROOT}/requirements.txt
+
 RUN \
   echo "######################################################" && \
   echo "### Import trusted certs before doing anything else ###" && \
@@ -36,33 +39,28 @@ RUN \
   echo "### -> Basics                                 ###" && \
   echo "### -> GCC (some makefiles require cmd which)###" && \
   echo "### -> dependencies for kerberos SASL_SSL   ###" && \
+  echo "### -> PostgreSQL dev headers (psycopg2)    ###" && \
   echo "##############################################" && \
     dnf install -y \
       wget tar xz bzip2-devel zlib-devel \
       which make gcc gcc-c++ \
-      libffi-devel cyrus-sasl-devel cyrus-sasl-gssapi openssl-devel krb5-workstation && \
+      libffi-devel cyrus-sasl-devel cyrus-sasl-gssapi openssl-devel krb5-workstation postgresql-devel && \
   echo "#################" && \
   echo "### librdkafka ###" && \
   echo "#################" && \
     mkdir -p /tmp/env-install-workdir/librdkafka && \
     cd /tmp/env-install-workdir/librdkafka && \
-    wget --ca-certificate=/etc/pki/tls/certs/ca-bundle.crt https://github.com/edenhill/librdkafka/archive/v2.4.0.tar.gz && \
-    tar -xf v2.4.0.tar.gz && \
-    cd /tmp/env-install-workdir/librdkafka/librdkafka-2.4.0 && \
+    wget --ca-certificate=/etc/pki/tls/certs/ca-bundle.crt https://github.com/confluentinc/librdkafka/archive/v2.14.0.tar.gz && \
+    tar -xf v2.14.0.tar.gz && \
+    cd /tmp/env-install-workdir/librdkafka/librdkafka-2.14.0 && \
     ./configure && make && make install && \
   echo "###################" && \
   echo "### pip installs ###" && \
   echo "###################" && \
-    pip install requests==2.31.0 urllib3==1.26.18 setuptools cryptography jsonschema PyJWT psycopg2-binary && \
-  echo "######################" && \
-  echo "### confluent-kafka ###" && \
-  echo "######################" && \
-    mkdir -p /tmp/env-install-workdir/confluent-kafka && \
-    cd /tmp/env-install-workdir/confluent-kafka && \
-    wget --ca-certificate=/etc/pki/tls/certs/ca-bundle.crt https://github.com/confluentinc/confluent-kafka-python/archive/v2.4.0.tar.gz && \
-    tar -xf v2.4.0.tar.gz && \
-    cd /tmp/env-install-workdir/confluent-kafka/confluent-kafka-python-2.4.0 && \
-    CPPFLAGS="-I/usr/local/include" LDFLAGS="-L/opt" python setup.py install && \
+    # requirements.txt pins the version of confluent-kafka.
+    # --no-binary confluent-kafka forces source compilation against the system librdkafka
+    # built above, which includes Kerberos/GSSAPI support. The PyPI compiles without GSSAPI.
+    pip install -r ${LAMBDA_TASK_ROOT}/requirements.txt --no-binary confluent-kafka && \
   echo "##############" && \
   echo "### cleanup ###" && \
   echo "##############" && \

requirements-dev.txt

@@ -0,0 +1,12 @@
+-r requirements.txt
+psycopg2-binary==2.9.12
+pytest==9.0.3
+pytest-cov==7.1.0
+pytest-mock==3.15.1
+pylint==4.0.5
+black==26.5.1
+mypy==2.1.0
+mypy-extensions==1.1.0
+moto[s3,secretsmanager,events]==5.2.1
+testcontainers==4.14.2
+docker==7.1.0

requirements.txt

@@ -1,21 +1,9 @@
-pytest==9.0.3
-pytest-cov==7.1.0
-pytest-mock==3.15.1
-pylint==4.0.5
-black==26.5.1
-mypy==2.1.0
-mypy-extensions==1.1.0
-urllib3==2.7.0
 cryptography==48.0.0
 jsonschema==4.26.0
 PyJWT==2.13.0
 requests==2.34.2
 boto3==1.43.14
-aiosql==15.0
 botocore==1.43.14
+aiosql==15.0
 confluent-kafka==2.14.0
-moto[s3,secretsmanager,events]==5.2.1
-testcontainers==4.14.2
-docker==7.1.0
-# psycopg2-binary==2.9.10 # Ideal for local development, but not for long-term production use
 psycopg2==2.9.12