From 57fcd19c667cb71334054d1e7465261266c31f85 Mon Sep 17 00:00:00 2001 From: David Christensen Date: Thu, 4 Jun 2026 16:53:25 +0000 Subject: [PATCH 01/11] ci: add PostgreSQL 19beta1 to build/test matrix, bump minor versions Bumps PG 16/17/18 to latest minors (16.14, 17.10, 18.4) and wires PG 19beta1 into the Dockerfile, CI matrices, install.sh, and docker docs/Taskfile. Code-level fixes for PG 19 are deliberately deferred; this commit just sets up the infrastructure so failures surface in CI. Every PG19 matrix leg is marked continue-on-error so failures surface in CI without blocking the PR's required checks. The build-and-install jobs also flip fail-fast: false so a PG19 build failure does not cancel the 16/17/18 legs. Drive-by: Oracle's /java/21/latest tarball bumped past jdk-21.0.10, which broke the debian image build with "cannot stat '/usr/lib/jvm/jdk-21.0.10'". Read the top-level directory from the tarball instead of hardcoding the patch version. Signed-off-by: David Christensen --- .github/workflows/pytest_all.yml | 23 +++++++++++----- docker/Dockerfile | 47 +++++++++++++++++++++++++++----- docker/LOCAL_DEV.md | 6 ++-- docker/README.md | 2 +- docker/TASKFILE.md | 7 +++-- docker/Taskfile.yml | 7 +++-- docs/building-from-source.md | 10 +++---- install.sh | 7 +++-- 8 files changed, 79 insertions(+), 30 deletions(-) diff --git a/.github/workflows/pytest_all.yml b/.github/workflows/pytest_all.yml index 578ce8250..7656bd7a0 100644 --- a/.github/workflows/pytest_all.yml +++ b/.github/workflows/pytest_all.yml @@ -83,6 +83,7 @@ jobs: build-and-install-almalinux: needs: build-image-almalinux runs-on: ubuntu-latest + continue-on-error: ${{ matrix.pg_version == 19 }} container: image: ghcr.io/snowflake-labs/pg-lake-ci-pg-almalinux:${{ needs.build-image-almalinux.outputs.tag }} @@ -92,8 +93,9 @@ jobs: password: ${{ secrets.github_token }} strategy: + fail-fast: false matrix: - pg_version: [16, 17, 18] + pg_version: [16, 17, 18, 19] steps: - uses: actions/checkout@v4 @@ -108,6 +110,7 @@ jobs: build-and-install-debian: needs: build-image-debian runs-on: ubuntu-latest + continue-on-error: ${{ matrix.pg_version == 19 }} container: image: ghcr.io/snowflake-labs/pg-lake-ci-pg-debian:${{ needs.build-image-debian.outputs.tag }} @@ -117,8 +120,9 @@ jobs: password: ${{ secrets.github_token }} strategy: + fail-fast: false matrix: - pg_version: [16, 17, 18] + pg_version: [16, 17, 18, 19] steps: - uses: actions/checkout@v4 @@ -141,6 +145,7 @@ jobs: test-check: needs: [build-image-almalinux, build-and-install-almalinux] runs-on: ubuntu-latest + continue-on-error: ${{ matrix.pg_version == 19 }} container: image: ghcr.io/snowflake-labs/pg-lake-ci-pg-almalinux:${{ needs.build-image-almalinux.outputs.tag }} options: --user 1001 --shm-size=128MB @@ -151,7 +156,7 @@ jobs: strategy: fail-fast: false matrix: - pg_version: [17, 18] + pg_version: [17, 18, 19] test_make_target: - 'check-duckdb_pglake' - 'check-avro' @@ -179,6 +184,7 @@ jobs: test-check-pg-lake-table: needs: [build-image-almalinux, build-and-install-almalinux] runs-on: ubuntu-latest + continue-on-error: ${{ matrix.pg_version == 19 }} container: image: ghcr.io/snowflake-labs/pg-lake-ci-pg-almalinux:${{ needs.build-image-almalinux.outputs.tag }} options: --user 1001 --shm-size=128MB @@ -189,7 +195,7 @@ jobs: strategy: fail-fast: false matrix: - pg_version: [17, 18] + pg_version: [17, 18, 19] test_make_target: - 'check-pg_lake_table' # helps splitting tests into 5 parallel jobs @@ -247,6 +253,7 @@ jobs: test-installcheck-postgres: needs: [build-image-debian, build-and-install-debian] runs-on: ubuntu-latest + continue-on-error: ${{ matrix.pg_version == 19 }} container: image: ghcr.io/snowflake-labs/pg-lake-ci-pg-debian:${{ needs.build-image-debian.outputs.tag }} options: --user 1001 --shm-size=128MB @@ -257,7 +264,7 @@ jobs: strategy: fail-fast: false matrix: - pg_version: [16, 17, 18] + pg_version: [16, 17, 18, 19] test_make_target: - 'installcheck-postgres' - 'installcheck-postgres-with_extensions_created' @@ -319,6 +326,7 @@ jobs: test-isolation: needs: [build-image-almalinux, build-and-install-almalinux] runs-on: ubuntu-latest + continue-on-error: ${{ matrix.pg_version == 19 }} container: image: ghcr.io/snowflake-labs/pg-lake-ci-pg-almalinux:${{ needs.build-image-almalinux.outputs.tag }} options: --user 1001 --shm-size=128MB @@ -329,7 +337,7 @@ jobs: strategy: fail-fast: false matrix: - pg_version: [17, 18] + pg_version: [17, 18, 19] test_make_target: - 'check-isolation_pg_lake_table' @@ -348,6 +356,7 @@ jobs: test-upgrade: needs: [build-image-almalinux, build-and-install-almalinux] runs-on: ubuntu-latest + continue-on-error: ${{ contains(matrix.pg_version_pairs, '19') }} container: image: ghcr.io/snowflake-labs/pg-lake-ci-pg-almalinux:${{ needs.build-image-almalinux.outputs.tag }} options: --user 1001 --shm-size=128MB @@ -358,7 +367,7 @@ jobs: strategy: fail-fast: false matrix: - pg_version_pairs: ["16-17", "17-18", "16-18"] + pg_version_pairs: ["16-17", "17-18", "16-18", "17-19", "18-19", "16-19"] test_make_target: - 'check-pg_lake_table-upgrade' diff --git a/docker/Dockerfile b/docker/Dockerfile index 94214a8e1..6912cb26f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,13 +9,15 @@ ARG BASE_IMAGE_TAG # Set environment variables for non-interactive installations and PostgreSQL/PostGIS versions ENV PG_LAKE_REF=main -ARG PG16_VERSION=16.11 -ARG PG17_VERSION=17.7 -ARG PG18_VERSION=18.1 -ARG POSTGIS_VERSION=3.6.0 +ARG PG16_VERSION=16.14 +ARG PG17_VERSION=17.10 +ARG PG18_VERSION=18.4 +ARG PG19_VERSION=19beta1 +ARG POSTGIS_VERSION=3.6.3 ARG PGAUDIT16_VERSION=REL_16_STABLE ARG PGAUDIT17_VERSION=REL_17_STABLE ARG PGAUDIT18_VERSION=REL_18_STABLE +ARG PGAUDIT19_VERSION=main # Install build dependencies RUN if [ "$BASE_IMAGE_OS" = "almalinux" ]; then \ @@ -28,6 +30,7 @@ RUN if [ "$BASE_IMAGE_OS" = "almalinux" ]; then \ ninja-build \ wget \ git \ + bzip2 \ readline-devel \ zlib-devel \ flex \ @@ -73,6 +76,7 @@ RUN if [ "$BASE_IMAGE_OS" = "debian" ]; then \ apt-get update \ && apt-get install -y \ build-essential \ + bzip2 \ cmake \ ninja-build \ libreadline-dev \ @@ -128,7 +132,8 @@ RUN if [ "$BASE_IMAGE_OS" = "debian" ]; then \ wget https://download.oracle.com/java/21/latest/jdk-21_linux-${ARCH}_bin.tar.gz && \ mkdir -p /usr/lib/jvm && \ tar -xzf jdk-21_linux-${ARCH}_bin.tar.gz -C /usr/lib/jvm/ && \ - mv /usr/lib/jvm/jdk-21.0.10 /usr/lib/jvm/java-21-openjdk && \ + JDK_DIR=$(tar -tzf jdk-21_linux-${ARCH}_bin.tar.gz | head -1 | cut -d/ -f1) && \ + mv "/usr/lib/jvm/${JDK_DIR}" /usr/lib/jvm/java-21-openjdk && \ rm jdk-21_linux-${ARCH}_bin.tar.gz; \ fi @@ -157,26 +162,31 @@ RUN echo "postgres ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/postgres USER 1001:1001 WORKDIR /home/postgres -# Download PostgreSQL 16, 17, 18, PostGIS +# Download PostgreSQL 16, 17, 18, 19, PostGIS RUN wget https://ftp.postgresql.org/pub/source/v$PG16_VERSION/postgresql-$PG16_VERSION.tar.gz && \ wget https://ftp.postgresql.org/pub/source/v$PG17_VERSION/postgresql-$PG17_VERSION.tar.gz && \ wget https://ftp.postgresql.org/pub/source/v$PG18_VERSION/postgresql-$PG18_VERSION.tar.gz && \ + wget https://ftp.postgresql.org/pub/source/v$PG19_VERSION/postgresql-$PG19_VERSION.tar.bz2 && \ wget https://download.osgeo.org/postgis/source/postgis-$POSTGIS_VERSION.tar.gz && \ wget https://github.com/pgaudit/pgaudit/archive/refs/heads/$PGAUDIT16_VERSION.tar.gz && \ wget https://github.com/pgaudit/pgaudit/archive/refs/heads/$PGAUDIT17_VERSION.tar.gz && \ wget https://github.com/pgaudit/pgaudit/archive/refs/heads/$PGAUDIT18_VERSION.tar.gz && \ + wget https://github.com/pgaudit/pgaudit/archive/refs/heads/$PGAUDIT19_VERSION.tar.gz && \ tar -xzf postgresql-$PG16_VERSION.tar.gz && rm postgresql-$PG16_VERSION.tar.gz && \ tar -xzf postgresql-$PG17_VERSION.tar.gz && rm postgresql-$PG17_VERSION.tar.gz && \ tar -xzf postgresql-$PG18_VERSION.tar.gz && rm postgresql-$PG18_VERSION.tar.gz && \ + tar -xjf postgresql-$PG19_VERSION.tar.bz2 && rm postgresql-$PG19_VERSION.tar.bz2 && \ tar -xzf $PGAUDIT16_VERSION.tar.gz && rm $PGAUDIT16_VERSION.tar.gz && \ tar -xzf $PGAUDIT17_VERSION.tar.gz && rm $PGAUDIT17_VERSION.tar.gz && \ tar -xzf $PGAUDIT18_VERSION.tar.gz && rm $PGAUDIT18_VERSION.tar.gz && \ + tar -xzf $PGAUDIT19_VERSION.tar.gz && rm $PGAUDIT19_VERSION.tar.gz && \ tar -xzf postgis-$POSTGIS_VERSION.tar.gz && rm postgis-$POSTGIS_VERSION.tar.gz ARG PG_COMPILE_FLAGS_16="--enable-debug --enable-cassert --enable-depend --with-openssl --with-libxml --with-libxslt --with-icu --with-uuid=ossp --with-lz4 --with-pam" # PG17 has asserts disabled ARG PG_COMPILE_FLAGS_17="--enable-debug --enable-depend --with-openssl --with-libxml --with-libxslt --with-icu --with-uuid=ossp --with-lz4 --enable-injection-points --with-pam" ARG PG_COMPILE_FLAGS_18="--enable-debug --enable-cassert --enable-depend --with-openssl --with-libxml --with-libxslt --with-icu --with-uuid=ossp --with-lz4 --enable-injection-points --with-pam" +ARG PG_COMPILE_FLAGS_19="--enable-debug --enable-cassert --enable-depend --with-openssl --with-libxml --with-libxslt --with-icu --with-uuid=ossp --with-lz4 --enable-injection-points --with-pam" ENV PGBASEDIR=/home/postgres # Compile and install PostgreSQL 16 with pgindent @@ -217,6 +227,19 @@ RUN cd postgresql-$PG18_VERSION && \ (cd contrib && make -j `nproc` install) && \ cd .. && rm -rf postgresql-$PG18_VERSION* +# Compile and install PostgreSQL 19 with pgindent +RUN cd postgresql-$PG19_VERSION && \ + ./configure --prefix=$PGBASEDIR/pgsql-19 $PG_COMPILE_FLAGS_19 && \ + make -j `nproc` && make install && \ + make -C src/test/isolation install && \ + make -C src/test/modules/injection_points install && \ + make -j `nproc` -C src/tools/pg_bsd_indent install && \ + cp src/tools/pgindent/pgindent $PGBASEDIR/pgsql-19/bin/ && \ + cp src/tools/pgindent/typedefs.list $PGBASEDIR/pgsql-19/share/ && \ + mkdir -pv $PGBASEDIR/pgsql-19/regress && cp -r src/test/regress/* $PGBASEDIR/pgsql-19/regress && \ + (cd contrib && make -j `nproc` install) && \ + cd .. && rm -rf postgresql-$PG19_VERSION* + # Compile and install PG AUDIT 16 RUN cd pgaudit-$PGAUDIT16_VERSION && \ make -j `nproc` install USE_PGXS=1 PG_CONFIG=$PGBASEDIR/pgsql-16/bin/pg_config @@ -229,19 +252,28 @@ RUN cd pgaudit-$PGAUDIT17_VERSION && \ RUN cd pgaudit-$PGAUDIT18_VERSION && \ make -j `nproc` install USE_PGXS=1 PG_CONFIG=$PGBASEDIR/pgsql-18/bin/pg_config +# Compile and install PG AUDIT 19 +RUN cd pgaudit-$PGAUDIT19_VERSION && \ + make -j `nproc` install USE_PGXS=1 PG_CONFIG=$PGBASEDIR/pgsql-19/bin/pg_config + # Compile and install PostGIS RUN cd postgis-$POSTGIS_VERSION && \ ./configure --prefix=$PGBASEDIR/pgsql-16/ --with-pgconfig=$PGBASEDIR/pgsql-16/bin/pg_config && make -j `nproc` && make install && make clean && \ ./configure --prefix=$PGBASEDIR/pgsql-17/ --with-pgconfig=$PGBASEDIR/pgsql-17/bin/pg_config && make -j `nproc` && make install && make clean && \ ./configure --prefix=$PGBASEDIR/pgsql-18/ --with-pgconfig=$PGBASEDIR/pgsql-18/bin/pg_config && make -j `nproc` && make install && make clean && \ + ./configure --prefix=$PGBASEDIR/pgsql-19/ --with-pgconfig=$PGBASEDIR/pgsql-19/bin/pg_config && make -j `nproc` && make install && make clean && \ cd .. && rm -rf postgis-$POSTGIS_VERSION* # Compile and install pg_cron -RUN git clone https://github.com/citusdata/pg_cron.git && \ +# Use CyberDem0n's postgresql-19 branch (PR citusdata/pg_cron#430) until the +# PG19 support patch is merged upstream. The patch is backward compatible +# with PG 16/17/18, so we can build all four from the same source. +RUN git clone -b postgresql-19 https://github.com/CyberDem0n/pg_cron.git && \ cd pg_cron && \ make install PG_CONFIG=$PGBASEDIR/pgsql-16/bin/pg_config && make clean PG_CONFIG=$PGBASEDIR/pgsql-16/bin/pg_config && \ make install PG_CONFIG=$PGBASEDIR/pgsql-17/bin/pg_config && make clean PG_CONFIG=$PGBASEDIR/pgsql-17/bin/pg_config && \ make install PG_CONFIG=$PGBASEDIR/pgsql-18/bin/pg_config && make clean PG_CONFIG=$PGBASEDIR/pgsql-18/bin/pg_config && \ + make install PG_CONFIG=$PGBASEDIR/pgsql-19/bin/pg_config && make clean PG_CONFIG=$PGBASEDIR/pgsql-19/bin/pg_config && \ cd .. && rm -rf pg_cron # Compile and install pg_incremental @@ -250,6 +282,7 @@ RUN git clone https://github.com/CrunchyData/pg_incremental.git && \ make install PG_CONFIG=$PGBASEDIR/pgsql-16/bin/pg_config && make clean PG_CONFIG=$PGBASEDIR/pgsql-16/bin/pg_config && \ make install PG_CONFIG=$PGBASEDIR/pgsql-17/bin/pg_config && make clean PG_CONFIG=$PGBASEDIR/pgsql-17/bin/pg_config && \ make install PG_CONFIG=$PGBASEDIR/pgsql-18/bin/pg_config && make clean PG_CONFIG=$PGBASEDIR/pgsql-18/bin/pg_config && \ + make install PG_CONFIG=$PGBASEDIR/pgsql-19/bin/pg_config && make clean PG_CONFIG=$PGBASEDIR/pgsql-19/bin/pg_config && \ cd .. && rm -rf pg_incremental # Install vcpkg as postgres user diff --git a/docker/LOCAL_DEV.md b/docker/LOCAL_DEV.md index 69c8815c5..5963a3e14 100644 --- a/docker/LOCAL_DEV.md +++ b/docker/LOCAL_DEV.md @@ -80,7 +80,7 @@ You can customize builds using these variables (can be set as environment variab | Variable | Default | Description | Example | |----------|---------|-------------|---------| -| `PG_MAJOR` | `18` | PostgreSQL major version (16, 17, or 18) | `PG_MAJOR=17` | +| `PG_MAJOR` | `18` | PostgreSQL major version (16, 17, 18, or 19) | `PG_MAJOR=17` | | `BASE_IMAGE_OS` | `almalinux` | Base OS (almalinux or debian) | `BASE_IMAGE_OS=debian` | | `BASE_IMAGE_TAG` | `9` | Base OS version tag | `BASE_IMAGE_TAG=12` | | `VERSION` | `latest` | Image version tag (for registry) | `VERSION=v1.0.0` | @@ -311,7 +311,7 @@ task compose:up PG_MAJOR=16 Create a `.env` file in the `docker` directory: ```env -# PostgreSQL Version (16, 17, or 18) +# PostgreSQL Version (16, 17, 18, or 19) PG_MAJOR=18 # AWS Profile for LocalStack (optional) @@ -559,6 +559,7 @@ Each PostgreSQL version uses its own buildx builder: - PG 16: `pg_lake_builder_pg16` - PG 17: `pg_lake_builder_pg17` - PG 18: `pg_lake_builder_pg18` +- PG 19: `pg_lake_builder_pg19` This prevents cache conflicts when switching between versions. @@ -590,6 +591,7 @@ This prevents cache conflicts when switching between versions. - PG 16: `pg_lake_builder_pg16` - PG 17: `pg_lake_builder_pg17` - PG 18: `pg_lake_builder_pg18` + - PG 19: `pg_lake_builder_pg19` --- diff --git a/docker/README.md b/docker/README.md index 6ed386e0a..6ac09aa1a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -103,7 +103,7 @@ docker-compose exec pgduck-server psql -p 5332 -h /home/postgres/pgduck_socket_d ### Key Optimizations -✅ **Single Runner PostgreSQL Version**: Builds only PG 16, 17, or 18 (not all 3) +✅ **Single Runner PostgreSQL Version**: Builds only PG 16, 17, 18, or 19 (not all 4) ✅ **Separate Build Stages**: Compilation happens in builder stages ✅ **Minimal Runtime**: Final images contain only binaries and libraries ✅ **Network Retry Logic**: Handles vcpkg download failures diff --git a/docker/TASKFILE.md b/docker/TASKFILE.md index c862efafb..070a0ebed 100644 --- a/docker/TASKFILE.md +++ b/docker/TASKFILE.md @@ -29,6 +29,7 @@ This Taskfile uses **per-version buildx builders** to prevent cache conflicts: - **PG 16**: Uses `pg_lake_builder_pg16` - **PG 17**: Uses `pg_lake_builder_pg17` - **PG 18**: Uses `pg_lake_builder_pg18` +- **PG 19**: Uses `pg_lake_builder_pg19` Each builder maintains its own isolated cache, allowing you to build and switch between PostgreSQL versions without cache conflicts. Builders are created automatically on first use. @@ -186,7 +187,7 @@ task compose:restart ### Cache Management -Each PostgreSQL version (16, 17, 18) uses its own isolated buildx builder to prevent cache conflicts: +Each PostgreSQL version (16, 17, 18, 19) uses its own isolated buildx builder to prevent cache conflicts: ```bash # Clear cache for specific PostgreSQL version @@ -239,7 +240,7 @@ You can customize builds using these variables: | `REGISTRY` | `docker.io` | Container registry (docker.io, ghcr.io, etc.) | | `IMAGE_OWNER` | `${DOCKER_HUB_USERNAME}` | Registry username/org | | `VERSION` | `latest` | Image version tag | -| `PG_MAJOR` | `18` | PostgreSQL major version (16, 17, or 18) | +| `PG_MAJOR` | `18` | PostgreSQL major version (16, 17, 18, or 19) | | `PLATFORMS` | `linux/amd64,linux/arm64` | Target platforms for registry builds | | `BASE_IMAGE_OS` | `almalinux` | Base OS (almalinux or debian) | | `BASE_IMAGE_TAG` | `9` | Base OS version | @@ -510,7 +511,7 @@ task push:all-pg-versions VERSION=v3.1.0 - `build:pg-lake-postgres` - Build pg_lake_postgres for registry (multi-platform, cache only unless PUSH=true) - `build:pgduck-server` - Build pgduck_server for registry (multi-platform, cache only unless PUSH=true) - `build:all` - Build all images (multi-platform, cache only unless PUSH=true) -- `build:all-pg-versions` - Build for PostgreSQL 16, 17, and 18 +- `build:all-pg-versions` - Build for PostgreSQL 16, 17, 18, and 19 ### Push Tasks diff --git a/docker/Taskfile.yml b/docker/Taskfile.yml index 2d1d3c3d6..645389f6f 100644 --- a/docker/Taskfile.yml +++ b/docker/Taskfile.yml @@ -139,7 +139,7 @@ tasks: } build:all-pg-versions: - desc: Build all images for PostgreSQL 16, 17, and 18 + desc: Build all images for PostgreSQL 16, 17, 18, and 19 silent: true cmds: - task: build:all @@ -148,9 +148,11 @@ tasks: vars: { PG_MAJOR: "17", VERSION: "{{.VERSION}}", PUSH: "{{.PUSH}}" } - task: build:all vars: { PG_MAJOR: "18", VERSION: "{{.VERSION}}", PUSH: "{{.PUSH}}" } + - task: build:all + vars: { PG_MAJOR: "19", VERSION: "{{.VERSION}}", PUSH: "{{.PUSH}}" } push:all-pg-versions: - desc: Build and push all images for PostgreSQL 16, 17, and 18 + desc: Build and push all images for PostgreSQL 16, 17, 18, and 19 silent: true cmds: - task: build:all-pg-versions @@ -170,6 +172,7 @@ tasks: - docker buildx rm pg_lake_builder_pg16 || true - docker buildx rm pg_lake_builder_pg17 || true - docker buildx rm pg_lake_builder_pg18 || true + - docker buildx rm pg_lake_builder_pg19 || true - docker buildx rm pg_lake_builder || true - echo "✅ All buildx builders removed" ignore_error: true diff --git a/docs/building-from-source.md b/docs/building-from-source.md index 654484ab6..597b6bd5c 100644 --- a/docs/building-from-source.md +++ b/docs/building-from-source.md @@ -4,7 +4,7 @@ This guide covers installing pg_lake from source. For Docker-based setup, see [d ## Add pg_lake to an Existing PostgreSQL Installation -If you already have PostgreSQL 16, 17, or 18 installed, you can add pg_lake extensions using the automated installation script: +If you already have PostgreSQL 16, 17, 18, or 19 installed, you can add pg_lake extensions using the automated installation script: ```bash # Clone the repository @@ -31,7 +31,7 @@ psql -c "CREATE EXTENSION pg_lake CASCADE;" **Note:** System build dependencies are skipped by default (assuming they're installed if PostgreSQL exists). Use `--with-system-deps` if needed. **Prerequisites:** -- PostgreSQL 16, 17, or 18 installed with `pg_config` in your PATH +- PostgreSQL 16, 17, 18, or 19 installed with `pg_config` in your PATH - `shared_preload_libraries = 'pg_extension_base'` in postgresql.conf ([see setup](#configure-postgresql)) For manual installation steps, see [Manual Installation to Existing PostgreSQL](#manual-installation-to-existing-postgresql). @@ -66,7 +66,7 @@ For more options and control, see [Development Environment Options](#development ## Prerequisites **For adding to existing PostgreSQL:** -- PostgreSQL 16, 17, or 18 with `pg_config` in PATH +- PostgreSQL 16, 17, 18, or 19 with `pg_config` in PATH - Git, C/C++ compiler, CMake, Ninja - Internet connection @@ -219,7 +219,7 @@ Run `./install.sh --help` to see all available options: ``` --build-postgres Build PostgreSQL from source and initialize database ---pg-version VERSION PostgreSQL version to build (16, 17, or 18) [default: 18] +--pg-version VERSION PostgreSQL version to build (16, 17, 18, or 19) [default: 18] --prefix DIR PostgreSQL installation prefix [default: auto-detect or $HOME/pgsql] --deps-dir DIR Directory for dependencies [default: $HOME/pg_lake-deps] --jobs N Number of parallel build jobs [default: nproc] @@ -376,7 +376,7 @@ export PKG_CONFIG_PATH="/opt/homebrew/opt/icu4c/lib/pkgconfig:/opt/homebrew/opt/ ### Build PostgreSQL from Source -pg_lake is supported with PostgreSQL 16, 17, and 18. For development, we recommend building PostgreSQL from source to get debug symbols and assertions. +pg_lake is supported with PostgreSQL 16, 17, 18, and 19. For development, we recommend building PostgreSQL from source to get debug symbols and assertions. ```bash # Clone PostgreSQL (version 18 example) diff --git a/install.sh b/install.sh index b8a887169..622a89c41 100755 --- a/install.sh +++ b/install.sh @@ -63,7 +63,7 @@ PostgreSQL installation. Use --build-postgres for full development setup. OPTIONS: --build-postgres Build PostgreSQL from source and initialize database - --pg-version VERSION PostgreSQL version to build (16, 17, or 18) [default: 18] + --pg-version VERSION PostgreSQL version to build (16, 17, 18, or 19) [default: 18] --prefix DIR PostgreSQL installation prefix [default: auto-detect or \$HOME/pgsql] --deps-dir DIR Directory for dependencies [default: \$HOME/pg_lake-deps] --jobs N Number of parallel build jobs [default: nproc] @@ -148,8 +148,8 @@ while [[ $# -gt 0 ]]; do done # Validate PostgreSQL version -if [[ ! "$PG_VERSION" =~ ^(16|17|18)$ ]]; then - print_error "Invalid PostgreSQL version: $PG_VERSION. Must be 16, 17, or 18." +if [[ ! "$PG_VERSION" =~ ^(16|17|18|19)$ ]]; then + print_error "Invalid PostgreSQL version: $PG_VERSION. Must be 16, 17, 18, or 19." exit 1 fi @@ -158,6 +158,7 @@ case $PG_VERSION in 16) PG_BRANCH="REL_16_STABLE" ;; 17) PG_BRANCH="REL_17_STABLE" ;; 18) PG_BRANCH="REL_18_STABLE" ;; + 19) PG_BRANCH="master" ;; esac # Determine PostgreSQL installation paths From 2cd00e3613c1058d59e61b822febe56e8772ead7 Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 5 Jun 2026 11:08:27 -0500 Subject: [PATCH 02/11] pg19: build cleanly against PostgreSQL 19beta1 (#385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PG19 dropped a number of transitive header includes and reshaped a few APIs. Add explicit includes wherever the build broke and shim the renamed APIs in pg_extension_base/pg_compat.h. Header reorg (PG19 dropped these from common transitive paths): - utils/tuplestore.h (was pulled in via nodes/execnodes.h) - access/htup_details.h (was pulled in via executor/tuptable.h) - utils/hsearch.h (HASHCTL/hash_create/hash_search) - catalog/pg_type_d.h (TEXTOID, INTERVALOID, etc.) - utils/wait_event.h (WAIT_EVENT_CLIENT_READ) - utils/timestamp.h (Datum<-->TimestampTz, IntervalPGetDatum) - storage/lock.h (LockAcquire, LockAcquireResult) - storage/fd.h (AllocateFile, FreeFile) API changes wrapped behind #if PG_VERSION_NUM >= 190000: - LWLockNewTrancheId() now requires a name argument; the separate LWLockRegisterTranche() helper is gone. - ShmemInitHash() collapsed init_size/max_size into a single nelems. - SIG_IGN -> PG_SIG_IGN (pqsigfunc-typed sentinel). - pg_plan_query() gained a trailing ExplainState* (pass NULL). - CopyFormatOptions.binary / .csv_mode were replaced by an enum .format field — wrapped behind CopyOptsIsBinary/CopyOptsIsCsvMode macros in pg_compat.h so callers keep the older names. - CopyHeaderChoice enum was removed; header_line is a plain int with COPY_HEADER_TRUE/FALSE/MATCH constants. Reintroduced the typedef locally so our exported API signature stays stable. - numeric_int4_opt_error() renamed to numeric_int4_safe() and switched from a bool* error flag to an ErrorSaveContext* (soft-error API). Wrapped via an inline shim that preserves the older bool* signature. pgduck_server is a frontend program; the unused #include "nodes/nodes.h" in numutils.h started failing because PG19 changed nodes.h to use Datum in extern declarations. Removed the vestigial include. Builds clean against PG19 locally for: pg_extension_base, pg_map, pg_lake_engine, pg_lake_copy, pg_lake. pg_lake_iceberg/pg_lake_table were not exercised locally because the avro library was not built; they're expected to need similar include fixes already applied here. Signed-off-by: David Christensen --- .../include/pg_extension_base/pg_compat.h | 49 +++++++++++++++++++ .../include/pg_extension_base/spi_helpers.h | 2 + pg_extension_base/src/attached_worker.c | 2 + pg_extension_base/src/base_worker_launcher.c | 34 ++++++++++++- pg_extension_base/src/library_preloader.c | 1 + .../src/pg_extension_updater.c | 1 + pg_lake_benchmark/src/tpcds.c | 1 + pg_lake_benchmark/src/tpch.c | 1 + pg_lake_copy/src/test/types.c | 2 + .../include/pg_lake/csv/csv_options.h | 9 ++++ .../include/pg_lake/pgduck/serialize.h | 1 + pg_lake_engine/src/cleanup/deletion_queue.c | 2 + .../src/cleanup/in_progress_files.c | 3 ++ pg_lake_engine/src/csv/csv_writer.c | 22 +++++---- .../src/data_file/data_file_stats.c | 1 + pg_lake_engine/src/json/json_reader.c | 1 + pg_lake_engine/src/parquet/field.c | 1 + pg_lake_engine/src/parsetree/const.c | 1 + pg_lake_engine/src/parsetree/expression.c | 1 + pg_lake_engine/src/pgduck/client.c | 1 + pg_lake_engine/src/pgduck/map.c | 2 + pg_lake_engine/src/pgduck/parse_struct.c | 1 + pg_lake_engine/src/pgduck/read_data.c | 1 + pg_lake_engine/src/pgduck/rewrite_query.c | 3 ++ .../src/pgduck/shippable_spatial_functions.c | 1 + pg_lake_engine/src/pgduck/type.c | 1 + pg_lake_engine/src/pgduck/write_data.c | 1 + pg_lake_engine/src/query/execute.c | 6 ++- pg_lake_engine/src/utils/compare_utils.c | 1 + pg_lake_engine/src/utils/operator_utils.c | 1 + pg_lake_engine/src/utils/plan_cache.c | 1 + .../src/iceberg/api/table_metadata.c | 1 + pg_lake_iceberg/src/iceberg/catalog.c | 1 + .../src/iceberg/iceberg_type_binary_serde.c | 4 ++ .../src/iceberg/iceberg_type_json_serde.c | 1 + .../src/iceberg/metadata_operations.c | 1 + .../operations/find_referenced_files.c | 1 + pg_lake_iceberg/src/iceberg/read_manifest.c | 3 ++ .../object_store_catalog.c | 2 + .../src/rest_catalog/rest_catalog.c | 1 + pg_lake_iceberg/src/test/test_http_client.c | 1 + .../src/test/test_iceberg_binary_serde.c | 2 + .../src/test/test_iceberg_field_ids.c | 1 + .../src/test/test_iceberg_manifest.c | 2 + pg_lake_iceberg/src/utils/truncate_utils.c | 1 + .../include/pg_lake/fdw/data_files_catalog.h | 2 + pg_lake_table/src/ddl/alter_table.c | 1 + pg_lake_table/src/ddl/create_table.c | 1 + pg_lake_table/src/describe/describe.c | 1 + .../src/duckdb/transform_query_to_duckdb.c | 1 + pg_lake_table/src/fdw/data_file_pruning.c | 3 ++ pg_lake_table/src/fdw/data_files_catalog.c | 2 + .../src/fdw/data_files_catalog_batch.c | 4 ++ pg_lake_table/src/fdw/deparse.c | 4 ++ pg_lake_table/src/fdw/deparse_ruleutils.c | 1 + pg_lake_table/src/fdw/partition_transform.c | 2 + .../fdw/partitioning/partition_spec_catalog.c | 2 + .../partitioning/partitioned_dest_receiver.c | 1 + pg_lake_table/src/fdw/pg_lake_table.c | 2 + pg_lake_table/src/fdw/position_delete_dest.c | 2 + .../field_id_mapping_catalog.c | 1 + pg_lake_table/src/fdw/update_tracking.c | 6 ++- pg_lake_table/src/fdw/writable_table.c | 3 ++ pg_lake_table/src/planner/insert_select.c | 1 + pg_lake_table/src/planner/query_pushdown.c | 20 ++++++-- .../src/planner/restriction_collector.c | 1 + pg_lake_table/src/recovery/recover.c | 1 + pg_lake_table/src/test/hide_lake_objects.c | 2 + pg_lake_table/src/test/test_partition_tuple.c | 3 ++ .../track_iceberg_metadata_changes.c | 1 + pg_map/src/map.c | 1 + pgduck_server/Makefile | 6 ++- pgduck_server/include/utils/numutils.h | 2 - pgduck_server/src/duckdb/type_conversion.c | 1 + pgduck_server/tests/ctests/Makefile | 6 ++- 75 files changed, 239 insertions(+), 22 deletions(-) diff --git a/pg_extension_base/include/pg_extension_base/pg_compat.h b/pg_extension_base/include/pg_extension_base/pg_compat.h index 75791c554..04f3a17f9 100644 --- a/pg_extension_base/include/pg_extension_base/pg_compat.h +++ b/pg_extension_base/include/pg_extension_base/pg_compat.h @@ -18,6 +18,55 @@ #pragma once #include "postgres.h" +#include "catalog/pg_type_d.h" +#include "commands/copy.h" +#include "nodes/miscnodes.h" +#include "utils/numeric.h" + +#if PG_VERSION_NUM >= 190000 + +/* + * PG19 reshaped CopyFormatOptions: the bool fields ".binary" and ".csv_mode" + * were replaced by a single CopyFormat enum field ".format". Provide + * accessor macros so callers keep the older names. + */ +#define CopyOptsIsBinary(opts) ((opts).format == COPY_FORMAT_BINARY) +#define CopyOptsIsCsvMode(opts) ((opts).format == COPY_FORMAT_CSV) + +/* + * PG19 renamed numeric_int4_opt_error -> numeric_int4_safe and switched the + * out-of-band error signal from a bool* to an ErrorSaveContext* (the standard + * "soft error" API). Wrap with the older name + bool* signature. + */ +static inline int32 +numeric_int4_opt_error(Numeric num, bool *have_error) +{ + ErrorSaveContext escontext = {T_ErrorSaveContext}; + int32 result = numeric_int4_safe(num, (Node *) &escontext); + + *have_error = escontext.error_occurred; + return result; +} + +/* + * Likewise for the binary numeric arithmetic helpers, which were also + * renamed from foo_opt_error(Numeric, Numeric, bool *) to + * foo_safe(Numeric, Numeric, Node *escontext) in PG19. Our callers always + * pass NULL for the error argument, so just forward to the new name. + */ +#define numeric_add_opt_error(num1, num2, _ignored) numeric_add_safe(num1, num2, NULL) +#define numeric_sub_opt_error(num1, num2, _ignored) numeric_sub_safe(num1, num2, NULL) +#define numeric_mul_opt_error(num1, num2, _ignored) numeric_mul_safe(num1, num2, NULL) +#define numeric_div_opt_error(num1, num2, _ignored) numeric_div_safe(num1, num2, NULL) +#define numeric_mod_opt_error(num1, num2, _ignored) numeric_mod_safe(num1, num2, NULL) + +#else + +#define CopyOptsIsBinary(opts) ((opts).binary) +#define CopyOptsIsCsvMode(opts) ((opts).csv_mode) + +#endif + #if PG_VERSION_NUM < 170000 #define foreach_ptr(type, var, lst) foreach_internal(type, *, var, lst, lfirst) diff --git a/pg_extension_base/include/pg_extension_base/spi_helpers.h b/pg_extension_base/include/pg_extension_base/spi_helpers.h index d8a15b875..7359b9b08 100644 --- a/pg_extension_base/include/pg_extension_base/spi_helpers.h +++ b/pg_extension_base/include/pg_extension_base/spi_helpers.h @@ -18,6 +18,7 @@ #ifndef SPI_UTILITIES_H #define SPI_UTILITIES_H +#include "catalog/pg_type_d.h" #include "miscadmin.h" #include "executor/spi.h" @@ -25,6 +26,7 @@ #include "utils/guc.h" #include "utils/jsonb.h" #include "utils/pg_lsn.h" +#include "utils/timestamp.h" /* SPI macros for setting parameters */ #define DATUMIZE_TEXTOID(Value) CStringGetTextDatum(Value) diff --git a/pg_extension_base/src/attached_worker.c b/pg_extension_base/src/attached_worker.c index f580a8e32..f6fa7a9f4 100644 --- a/pg_extension_base/src/attached_worker.c +++ b/pg_extension_base/src/attached_worker.c @@ -29,6 +29,7 @@ #include "miscadmin.h" #include "pgstat.h" +#include "access/htup_details.h" #include "access/xact.h" #include "catalog/pg_authid.h" #include "commands/dbcommands.h" @@ -61,6 +62,7 @@ #include "utils/syscache.h" #include "utils/timeout.h" #include "utils/typcache.h" +#include "utils/tuplestore.h" #define QUEUE_SIZE ((Size) 65536) #define ATTACHED_WORKER_MAGIC 0x52004040 diff --git a/pg_extension_base/src/base_worker_launcher.c b/pg_extension_base/src/base_worker_launcher.c index 393cac5c6..6eee40f2f 100644 --- a/pg_extension_base/src/base_worker_launcher.c +++ b/pg_extension_base/src/base_worker_launcher.c @@ -58,6 +58,9 @@ * should still exist, or whether to clean up their shared memory records. */ #include "postgres.h" +#include "utils/hsearch.h" +#include "access/htup_details.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" #include "miscadmin.h" @@ -94,6 +97,10 @@ #include "utils/memutils.h" #include "utils/snapmgr.h" #include "utils/syscache.h" +#include "utils/tuplestore.h" +#if PG_VERSION_NUM >= 190000 +#include "utils/wait_event.h" +#endif #include "tcop/utility.h" #include "pg_extension_base/base_workers.h" @@ -508,11 +515,20 @@ BaseWorkerSharedMemoryInit(void) if (!alreadyInitialized) { - BaseWorkerControl->trancheId = LWLockNewTrancheId(); BaseWorkerControl->lockTrancheName = "pg_extension_base server starter locks"; +#if PG_VERSION_NUM >= 190000 + /* + * PG19 folded the name into LWLockNewTrancheId and removed + * LWLockRegisterTranche entirely. + */ + BaseWorkerControl->trancheId = + LWLockNewTrancheId(BaseWorkerControl->lockTrancheName); +#else + BaseWorkerControl->trancheId = LWLockNewTrancheId(); LWLockRegisterTranche(BaseWorkerControl->trancheId, BaseWorkerControl->lockTrancheName); +#endif LWLockInitialize(&BaseWorkerControl->lock, BaseWorkerControl->trancheId); @@ -527,7 +543,11 @@ BaseWorkerSharedMemoryInit(void) int hashFlags = (HASH_ELEM | HASH_FUNCTION); DatabaseStarterHash = ShmemInitHash("pg_extension_base database starter hash", +#if PG_VERSION_NUM >= 190000 + 2 * max_worker_processes, +#else max_worker_processes, 2 * max_worker_processes, +#endif &hashInfo, hashFlags); memset(&hashInfo, 0, sizeof(hashInfo)); @@ -537,7 +557,11 @@ BaseWorkerSharedMemoryInit(void) hashFlags = (HASH_ELEM | HASH_FUNCTION); BaseWorkerHash = ShmemInitHash("pg_extension_base base worker hash", +#if PG_VERSION_NUM >= 190000 + 2 * max_worker_processes, +#else max_worker_processes, 2 * max_worker_processes, +#endif &hashInfo, hashFlags); LWLockRelease(AddinShmemInitLock); @@ -650,7 +674,11 @@ PgExtensionServerStarterMain(Datum arg) /* set up signal handlers */ pqsignal(SIGHUP, HandleSighup); pqsignal(SIGTERM, HandleSigterm); +#if PG_VERSION_NUM >= 190000 + pqsignal(SIGINT, PG_SIG_IGN); +#else pqsignal(SIGINT, SIG_IGN); +#endif before_shmem_exit(PgBaseExtensionServerStarterSharedMemoryExit, 0); @@ -1012,7 +1040,11 @@ PgExtensionBaseDatabaseStarterMain(Datum databaseIdDatum) /* Establish signal handlers before unblocking signals. */ pqsignal(SIGHUP, HandleSighup); +#if PG_VERSION_NUM >= 190000 + pqsignal(SIGINT, PG_SIG_IGN); +#else pqsignal(SIGINT, SIG_IGN); +#endif pqsignal(SIGTERM, HandleSigterm); /* Set our exit handler before any calls to proc_exit */ diff --git a/pg_extension_base/src/library_preloader.c b/pg_extension_base/src/library_preloader.c index fec185f62..9ab2c9651 100644 --- a/pg_extension_base/src/library_preloader.c +++ b/pg_extension_base/src/library_preloader.c @@ -36,6 +36,7 @@ #include "utils/builtins.h" #include "utils/conffiles.h" #include "utils/guc.h" +#include "utils/tuplestore.h" /* PreloadLibrary represents a library to preload */ typedef struct PreloadLibrary diff --git a/pg_extension_updater/src/pg_extension_updater.c b/pg_extension_updater/src/pg_extension_updater.c index 758590ea4..6e0eb2b32 100644 --- a/pg_extension_updater/src/pg_extension_updater.c +++ b/pg_extension_updater/src/pg_extension_updater.c @@ -22,6 +22,7 @@ *------------------------------------------------------------------------- */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "miscadmin.h" diff --git a/pg_lake_benchmark/src/tpcds.c b/pg_lake_benchmark/src/tpcds.c index f88ab28d5..8abf2e2d9 100644 --- a/pg_lake_benchmark/src/tpcds.c +++ b/pg_lake_benchmark/src/tpcds.c @@ -19,6 +19,7 @@ #include "fmgr.h" #include "funcapi.h" #include "utils/builtins.h" +#include "utils/tuplestore.h" #include "pg_lake/benchmark.h" #include "pg_lake/copy/copy_format.h" diff --git a/pg_lake_benchmark/src/tpch.c b/pg_lake_benchmark/src/tpch.c index b21ef35b1..338cdaa6c 100644 --- a/pg_lake_benchmark/src/tpch.c +++ b/pg_lake_benchmark/src/tpch.c @@ -19,6 +19,7 @@ #include "fmgr.h" #include "funcapi.h" #include "utils/builtins.h" +#include "utils/tuplestore.h" #include "pg_lake/benchmark.h" #include "pg_lake/copy/copy_format.h" diff --git a/pg_lake_copy/src/test/types.c b/pg_lake_copy/src/test/types.c index 00ee41b6c..51b535404 100644 --- a/pg_lake_copy/src/test/types.c +++ b/pg_lake_copy/src/test/types.c @@ -20,12 +20,14 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" #include "pg_lake/pgduck/type.h" #include "pg_lake/pgduck/parse_struct.h" #include "utils/builtins.h" +#include "utils/tuplestore.h" PG_FUNCTION_INFO_V1(duckdb_type_by_name); diff --git a/pg_lake_engine/include/pg_lake/csv/csv_options.h b/pg_lake_engine/include/pg_lake/csv/csv_options.h index d26afe1bb..dd16f2e48 100644 --- a/pg_lake_engine/include/pg_lake/csv/csv_options.h +++ b/pg_lake_engine/include/pg_lake/csv/csv_options.h @@ -20,6 +20,15 @@ #include "commands/copy.h" #include "nodes/pg_list.h" +#if PG_VERSION_NUM >= 190000 +/* + * PG19 dropped the CopyHeaderChoice enum and stores the header choice as a + * plain int holding COPY_HEADER_FALSE / COPY_HEADER_TRUE / COPY_HEADER_MATCH. + * Reintroduce the typedef here so we keep a stable signature. + */ +typedef int CopyHeaderChoice; +#endif + extern PGDLLEXPORT List *InternalCSVOptions(bool includeHeader); extern PGDLLEXPORT List *NormalizedExternalCSVOptions(List *inputOptions); extern PGDLLEXPORT CopyHeaderChoice GetCopyHeaderChoice(DefElem *def, bool is_from); diff --git a/pg_lake_engine/include/pg_lake/pgduck/serialize.h b/pg_lake_engine/include/pg_lake/pgduck/serialize.h index d76f4b0be..7c7cf7ae8 100644 --- a/pg_lake_engine/include/pg_lake/pgduck/serialize.h +++ b/pg_lake_engine/include/pg_lake/pgduck/serialize.h @@ -18,6 +18,7 @@ #pragma once #include "postgres.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "pg_lake/parquet/field.h" diff --git a/pg_lake_engine/src/cleanup/deletion_queue.c b/pg_lake_engine/src/cleanup/deletion_queue.c index a6d00640d..efbc258cd 100644 --- a/pg_lake_engine/src/cleanup/deletion_queue.c +++ b/pg_lake_engine/src/cleanup/deletion_queue.c @@ -19,6 +19,7 @@ * Functions for cleaning up orphaned files. */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "funcapi.h" #include "miscadmin.h" @@ -30,6 +31,7 @@ #include "pg_lake/util/string_utils.h" #include "datatype/timestamp.h" #include "storage/procarray.h" +#include "utils/tuplestore.h" #define DELETION_QUEUE_TABLE "lake_engine.deletion_queue" diff --git a/pg_lake_engine/src/cleanup/in_progress_files.c b/pg_lake_engine/src/cleanup/in_progress_files.c index 67b55644f..9d858ffce 100644 --- a/pg_lake_engine/src/cleanup/in_progress_files.c +++ b/pg_lake_engine/src/cleanup/in_progress_files.c @@ -24,6 +24,7 @@ * via VACUUM. */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "funcapi.h" #include "miscadmin.h" @@ -42,11 +43,13 @@ #include "pg_lake/util/plan_cache.h" #include "pg_lake/util/string_utils.h" #include "datatype/timestamp.h" +#include "storage/lock.h" #include "storage/procarray.h" #include "utils/fmgroids.h" #include "utils/memutils.h" #include "utils/lsyscache.h" #include "utils/snapmgr.h" +#include "utils/tuplestore.h" #define OPERATION_ID_SEQUENCE "operationid_seq" diff --git a/pg_lake_engine/src/csv/csv_writer.c b/pg_lake_engine/src/csv/csv_writer.c index 3b91870fe..4974f9d26 100644 --- a/pg_lake_engine/src/csv/csv_writer.c +++ b/pg_lake_engine/src/csv/csv_writer.c @@ -42,7 +42,9 @@ #include "mb/pg_wchar.h" #include "nodes/execnodes.h" #include "nodes/makefuncs.h" +#include "pg_extension_base/pg_compat.h" #include "port/pg_bswap.h" +#include "storage/fd.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/memutils.h" @@ -198,7 +200,7 @@ CopySendEndOfRow(CopyToState cstate) switch (cstate->copy_dest) { case COPY_FILE: - if (!cstate->opts.binary) + if (!CopyOptsIsBinary(cstate->opts)) { /* Default line termination depends on platform */ #ifndef WIN32 @@ -264,7 +266,7 @@ CopySendInt16(CopyToState cstate, int16 val) static void EndCopy(CopyToState cstate) { - if (cstate->opts.binary) + if (CopyOptsIsBinary(cstate->opts)) { /* Generate trailer for a binary copy */ CopySendInt16(cstate, -1); @@ -461,7 +463,7 @@ StartCopyTo(CopyToState cstate, TupleDesc tupDesc) bool isvarlena; Form_pg_attribute attr = TupleDescAttr(tupDesc, attnum - 1); - if (cstate->opts.binary) + if (CopyOptsIsBinary(cstate->opts)) getTypeBinaryOutputInfo(attr->atttypid, &out_func_oid, &isvarlena); @@ -482,7 +484,7 @@ StartCopyTo(CopyToState cstate, TupleDesc tupDesc) "COPY TO", ALLOCSET_DEFAULT_SIZES); - if (cstate->opts.binary) + if (CopyOptsIsBinary(cstate->opts)) { /* Generate header for a binary copy */ int32 tmp; @@ -523,7 +525,7 @@ StartCopyTo(CopyToState cstate, TupleDesc tupDesc) colname = NameStr(TupleDescAttr(tupDesc, attnum - 1)->attname); - if (cstate->opts.csv_mode) + if (CopyOptsIsCsvMode(cstate->opts)) CopyAttributeOutCSV(cstate, colname, false, list_length(cstate->attnumlist) == 1); else @@ -731,7 +733,7 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) MemoryContextReset(cstate->rowcontext); oldcontext = MemoryContextSwitchTo(cstate->rowcontext); - if (cstate->opts.binary) + if (CopyOptsIsBinary(cstate->opts)) { /* Binary per-tuple header */ CopySendInt16(cstate, list_length(cstate->attnumlist)); @@ -746,7 +748,7 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) Datum value = slot->tts_values[attnum - 1]; bool isnull = slot->tts_isnull[attnum - 1]; - if (!cstate->opts.binary) + if (!CopyOptsIsBinary(cstate->opts)) { if (need_delim) CopySendChar(cstate, cstate->opts.delim[0]); @@ -755,14 +757,14 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) if (isnull) { - if (!cstate->opts.binary) + if (!CopyOptsIsBinary(cstate->opts)) CopySendString(cstate, cstate->opts.null_print_client); else CopySendInt32(cstate, -1); } else { - if (!cstate->opts.binary) + if (!CopyOptsIsBinary(cstate->opts)) { /* * Lookup the underlying tuple's attribute so we can pass in @@ -802,7 +804,7 @@ CopyOneRowTo(CopyToState cstate, TupleTableSlot *slot) } - if (cstate->opts.csv_mode) + if (CopyOptsIsCsvMode(cstate->opts)) CopyAttributeOutCSV(cstate, string, cstate->opts.force_quote_flags[attnum - 1], list_length(cstate->attnumlist) == 1); diff --git a/pg_lake_engine/src/data_file/data_file_stats.c b/pg_lake_engine/src/data_file/data_file_stats.c index 09992d697..acd296432 100644 --- a/pg_lake_engine/src/data_file/data_file_stats.c +++ b/pg_lake_engine/src/data_file/data_file_stats.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "executor/executor.h" #include "pg_lake/data_file/data_files.h" diff --git a/pg_lake_engine/src/json/json_reader.c b/pg_lake_engine/src/json/json_reader.c index ff54c125f..37623c93f 100644 --- a/pg_lake_engine/src/json/json_reader.c +++ b/pg_lake_engine/src/json/json_reader.c @@ -18,6 +18,7 @@ #include "postgres.h" #include "common/fe_memutils.h" +#include "pg_extension_base/pg_compat.h" #include "pg_lake/json/json_reader.h" #include "utils/builtins.h" #include "utils/jsonb.h" diff --git a/pg_lake_engine/src/parquet/field.c b/pg_lake_engine/src/parquet/field.c index 8acdfb45f..b8f8e3299 100644 --- a/pg_lake_engine/src/parquet/field.c +++ b/pg_lake_engine/src/parquet/field.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "common/int.h" diff --git a/pg_lake_engine/src/parsetree/const.c b/pg_lake_engine/src/parsetree/const.c index 46276f348..861c34331 100644 --- a/pg_lake_engine/src/parsetree/const.c +++ b/pg_lake_engine/src/parsetree/const.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "pg_lake/parsetree/const.h" diff --git a/pg_lake_engine/src/parsetree/expression.c b/pg_lake_engine/src/parsetree/expression.c index c22279cb8..9ca195440 100644 --- a/pg_lake_engine/src/parsetree/expression.c +++ b/pg_lake_engine/src/parsetree/expression.c @@ -24,6 +24,7 @@ #include "pg_lake/parsetree/const.h" #include "pg_lake/parsetree/expression.h" #include "datatype/timestamp.h" +#include "utils/timestamp.h" #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/parsenodes.h" diff --git a/pg_lake_engine/src/pgduck/client.c b/pg_lake_engine/src/pgduck/client.c index e64e6c327..12023e76b 100644 --- a/pg_lake_engine/src/pgduck/client.c +++ b/pg_lake_engine/src/pgduck/client.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "utils/hsearch.h" #include "miscadmin.h" #include "libpq-fe.h" diff --git a/pg_lake_engine/src/pgduck/map.c b/pg_lake_engine/src/pgduck/map.c index 7a28fd0ed..b92c2a973 100644 --- a/pg_lake_engine/src/pgduck/map.c +++ b/pg_lake_engine/src/pgduck/map.c @@ -16,6 +16,8 @@ */ #include "postgres.h" +#include "access/htup_details.h" +#include "catalog/pg_type_d.h" #include "miscadmin.h" #include "libpq-fe.h" diff --git a/pg_lake_engine/src/pgduck/parse_struct.c b/pg_lake_engine/src/pgduck/parse_struct.c index 4c6a47306..3f47e9eeb 100644 --- a/pg_lake_engine/src/pgduck/parse_struct.c +++ b/pg_lake_engine/src/pgduck/parse_struct.c @@ -23,6 +23,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "miscadmin.h" #include "libpq-fe.h" diff --git a/pg_lake_engine/src/pgduck/read_data.c b/pg_lake_engine/src/pgduck/read_data.c index c0c4d7926..aecf5662d 100644 --- a/pg_lake_engine/src/pgduck/read_data.c +++ b/pg_lake_engine/src/pgduck/read_data.c @@ -19,6 +19,7 @@ * Functions for generating query for reading data from pgduck server. */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include diff --git a/pg_lake_engine/src/pgduck/rewrite_query.c b/pg_lake_engine/src/pgduck/rewrite_query.c index e37fff63d..5be822e80 100644 --- a/pg_lake_engine/src/pgduck/rewrite_query.c +++ b/pg_lake_engine/src/pgduck/rewrite_query.c @@ -16,6 +16,9 @@ */ #include "postgres.h" +#include "access/htup_details.h" +#include "catalog/pg_type_d.h" +#include "utils/hsearch.h" #include #include diff --git a/pg_lake_engine/src/pgduck/shippable_spatial_functions.c b/pg_lake_engine/src/pgduck/shippable_spatial_functions.c index 49a1e1c59..26d2c3da0 100644 --- a/pg_lake_engine/src/pgduck/shippable_spatial_functions.c +++ b/pg_lake_engine/src/pgduck/shippable_spatial_functions.c @@ -20,6 +20,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "nodes/nodeFuncs.h" #include "optimizer/optimizer.h" diff --git a/pg_lake_engine/src/pgduck/type.c b/pg_lake_engine/src/pgduck/type.c index eb40c5e38..6330b76b5 100644 --- a/pg_lake_engine/src/pgduck/type.c +++ b/pg_lake_engine/src/pgduck/type.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "miscadmin.h" #include "libpq-fe.h" diff --git a/pg_lake_engine/src/pgduck/write_data.c b/pg_lake_engine/src/pgduck/write_data.c index 2fb0d0798..120453fb9 100644 --- a/pg_lake_engine/src/pgduck/write_data.c +++ b/pg_lake_engine/src/pgduck/write_data.c @@ -19,6 +19,7 @@ * Functions for generating query for writing data via pgduck server. */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "access/tupdesc.h" #include "commands/defrem.h" diff --git a/pg_lake_engine/src/query/execute.c b/pg_lake_engine/src/query/execute.c index f4a6aa78e..3bd3730ee 100644 --- a/pg_lake_engine/src/query/execute.c +++ b/pg_lake_engine/src/query/execute.c @@ -75,7 +75,11 @@ uint64 ExecuteQueryToDestReceiver(Query *query, const char *queryString, ParamListInfo params, DestReceiver *dest) { - PlannedStmt *plan = pg_plan_query(query, queryString, CURSOR_OPT_PARALLEL_OK, params); + PlannedStmt *plan = pg_plan_query(query, queryString, CURSOR_OPT_PARALLEL_OK, params +#if PG_VERSION_NUM >= 190000 + ,NULL +#endif + ); return ExecutePlanToDestReceiver(plan, queryString, params, dest); } diff --git a/pg_lake_engine/src/utils/compare_utils.c b/pg_lake_engine/src/utils/compare_utils.c index 6c2b161f3..278e974b5 100644 --- a/pg_lake_engine/src/utils/compare_utils.c +++ b/pg_lake_engine/src/utils/compare_utils.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "catalog/pg_collation.h" #include "fmgr.h" #include "utils/builtins.h" diff --git a/pg_lake_engine/src/utils/operator_utils.c b/pg_lake_engine/src/utils/operator_utils.c index baa41aa93..a919dc961 100644 --- a/pg_lake_engine/src/utils/operator_utils.c +++ b/pg_lake_engine/src/utils/operator_utils.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "catalog/pg_operator.h" #include "pg_lake/util/operator_utils.h" diff --git a/pg_lake_engine/src/utils/plan_cache.c b/pg_lake_engine/src/utils/plan_cache.c index 166988dd0..efb9bfa7b 100644 --- a/pg_lake_engine/src/utils/plan_cache.c +++ b/pg_lake_engine/src/utils/plan_cache.c @@ -26,6 +26,7 @@ * consider the cache size and eviction policy. */ #include "postgres.h" +#include "utils/hsearch.h" #include "fmgr.h" #include "miscadmin.h" diff --git a/pg_lake_iceberg/src/iceberg/api/table_metadata.c b/pg_lake_iceberg/src/iceberg/api/table_metadata.c index 611b3781e..40766f9ac 100644 --- a/pg_lake_iceberg/src/iceberg/api/table_metadata.c +++ b/pg_lake_iceberg/src/iceberg/api/table_metadata.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "storage/fd.h" #include "fmgr.h" #include "funcapi.h" #include "libpq-fe.h" diff --git a/pg_lake_iceberg/src/iceberg/catalog.c b/pg_lake_iceberg/src/iceberg/catalog.c index 7e4cd64f9..e07994dc7 100644 --- a/pg_lake_iceberg/src/iceberg/catalog.c +++ b/pg_lake_iceberg/src/iceberg/catalog.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "miscadmin.h" #include "pg_lake/extensions/pg_lake_iceberg.h" diff --git a/pg_lake_iceberg/src/iceberg/iceberg_type_binary_serde.c b/pg_lake_iceberg/src/iceberg/iceberg_type_binary_serde.c index 67828a27e..463b4ec20 100644 --- a/pg_lake_iceberg/src/iceberg/iceberg_type_binary_serde.c +++ b/pg_lake_iceberg/src/iceberg/iceberg_type_binary_serde.c @@ -16,6 +16,10 @@ */ #include "postgres.h" + +#include + +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" #include "libpq-fe.h" diff --git a/pg_lake_iceberg/src/iceberg/iceberg_type_json_serde.c b/pg_lake_iceberg/src/iceberg/iceberg_type_json_serde.c index 5f0bbf0cd..b37a5aa67 100644 --- a/pg_lake_iceberg/src/iceberg/iceberg_type_json_serde.c +++ b/pg_lake_iceberg/src/iceberg/iceberg_type_json_serde.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" #include "libpq-fe.h" diff --git a/pg_lake_iceberg/src/iceberg/metadata_operations.c b/pg_lake_iceberg/src/iceberg/metadata_operations.c index 0938f5fe2..6aff58a6a 100644 --- a/pg_lake_iceberg/src/iceberg/metadata_operations.c +++ b/pg_lake_iceberg/src/iceberg/metadata_operations.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "utils/hsearch.h" #include "fmgr.h" #include "access/xact.h" #include "common/hashfn.h" diff --git a/pg_lake_iceberg/src/iceberg/operations/find_referenced_files.c b/pg_lake_iceberg/src/iceberg/operations/find_referenced_files.c index f42e8fdb7..3aadec87a 100644 --- a/pg_lake_iceberg/src/iceberg/operations/find_referenced_files.c +++ b/pg_lake_iceberg/src/iceberg/operations/find_referenced_files.c @@ -23,6 +23,7 @@ * referenced in the latest metadata.json file. */ #include "postgres.h" +#include "utils/hsearch.h" #include "miscadmin.h" #include "fmgr.h" #include "funcapi.h" diff --git a/pg_lake_iceberg/src/iceberg/read_manifest.c b/pg_lake_iceberg/src/iceberg/read_manifest.c index 3086eb3f9..6dc8d2a0c 100644 --- a/pg_lake_iceberg/src/iceberg/read_manifest.c +++ b/pg_lake_iceberg/src/iceberg/read_manifest.c @@ -16,11 +16,14 @@ */ #include "postgres.h" +#include "utils/hsearch.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" #include "libpq-fe.h" #include "miscadmin.h" +#include "storage/fd.h" #include "utils/builtins.h" #include "utils/snapmgr.h" #include "utils/wait_event.h" diff --git a/pg_lake_iceberg/src/object_store_catalog/object_store_catalog.c b/pg_lake_iceberg/src/object_store_catalog/object_store_catalog.c index 642674277..94808b9c2 100644 --- a/pg_lake_iceberg/src/object_store_catalog/object_store_catalog.c +++ b/pg_lake_iceberg/src/object_store_catalog/object_store_catalog.c @@ -1,4 +1,5 @@ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "funcapi.h" #include "miscadmin.h" @@ -7,6 +8,7 @@ #include "utils/inval.h" #include "utils/snapmgr.h" #include "utils/lsyscache.h" +#include "utils/tuplestore.h" #include "pg_lake/json/json_utils.h" #include "pg_lake/iceberg/catalog.h" diff --git a/pg_lake_iceberg/src/rest_catalog/rest_catalog.c b/pg_lake_iceberg/src/rest_catalog/rest_catalog.c index 8cf2b2ae3..cbbbdb5bd 100644 --- a/pg_lake_iceberg/src/rest_catalog/rest_catalog.c +++ b/pg_lake_iceberg/src/rest_catalog/rest_catalog.c @@ -18,6 +18,7 @@ #include #include "postgres.h" +#include "pg_extension_base/pg_compat.h" #include "miscadmin.h" #include "common/base64.h" diff --git a/pg_lake_iceberg/src/test/test_http_client.c b/pg_lake_iceberg/src/test/test_http_client.c index 185a77acd..539b2a2b4 100644 --- a/pg_lake_iceberg/src/test/test_http_client.c +++ b/pg_lake_iceberg/src/test/test_http_client.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "fmgr.h" #include "funcapi.h" #include "utils/array.h" diff --git a/pg_lake_iceberg/src/test/test_iceberg_binary_serde.c b/pg_lake_iceberg/src/test/test_iceberg_binary_serde.c index 685aee4a2..92d0a3fbb 100644 --- a/pg_lake_iceberg/src/test/test_iceberg_binary_serde.c +++ b/pg_lake_iceberg/src/test/test_iceberg_binary_serde.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" @@ -28,6 +29,7 @@ #include "utils/builtins.h" #include "utils/json.h" +#include "utils/tuplestore.h" static ColumnBound * FindColumnBoundByColumnId(ColumnBound * bounds, int numBounds, int columnId); diff --git a/pg_lake_iceberg/src/test/test_iceberg_field_ids.c b/pg_lake_iceberg/src/test/test_iceberg_field_ids.c index 1431697a3..f5689cc7a 100644 --- a/pg_lake_iceberg/src/test/test_iceberg_field_ids.c +++ b/pg_lake_iceberg/src/test/test_iceberg_field_ids.c @@ -24,6 +24,7 @@ #include "pg_lake/parquet/leaf_field.h" #include "utils/builtins.h" +#include "utils/tuplestore.h" PG_FUNCTION_INFO_V1(pg_lake_get_leaf_field_ids); diff --git a/pg_lake_iceberg/src/test/test_iceberg_manifest.c b/pg_lake_iceberg/src/test/test_iceberg_manifest.c index 62f4d6921..c0ee48f8a 100644 --- a/pg_lake_iceberg/src/test/test_iceberg_manifest.c +++ b/pg_lake_iceberg/src/test/test_iceberg_manifest.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" #include "libpq-fe.h" @@ -28,6 +29,7 @@ #include "pg_lake/iceberg/partitioning/partition.h" #include "utils/builtins.h" +#include "utils/tuplestore.h" static const char *FetchCurrentSnapshotManifestListPathFromTableMetadataUri(const char *tableMetadataPath); static List *FetchDataFilePathsFromTableMetadataUri(const char *tableMetadataPath, bool isDelete); diff --git a/pg_lake_iceberg/src/utils/truncate_utils.c b/pg_lake_iceberg/src/utils/truncate_utils.c index e136e4cab..73a333b5e 100644 --- a/pg_lake_iceberg/src/utils/truncate_utils.c +++ b/pg_lake_iceberg/src/utils/truncate_utils.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "pg_extension_base/pg_compat.h" #include "mb/pg_wchar.h" #include "utils/builtins.h" #include "utils/fmgrprotos.h" diff --git a/pg_lake_table/include/pg_lake/fdw/data_files_catalog.h b/pg_lake_table/include/pg_lake/fdw/data_files_catalog.h index 65e5a9c25..5db9e2d4a 100644 --- a/pg_lake_table/include/pg_lake/fdw/data_files_catalog.h +++ b/pg_lake_table/include/pg_lake/fdw/data_files_catalog.h @@ -22,7 +22,9 @@ #include "pg_lake/util/s3_reader_utils.h" #include "nodes/pg_list.h" +#if PG_VERSION_NUM < 190000 #include "utils/dynahash.h" +#endif #define DATA_FILES_TABLE_QUALIFIED \ PG_LAKE_TABLE_SCHEMA "." PG_LAKE_TABLE_FILES_TABLE_NAME diff --git a/pg_lake_table/src/ddl/alter_table.c b/pg_lake_table/src/ddl/alter_table.c index 1f3d08754..956f247be 100644 --- a/pg_lake_table/src/ddl/alter_table.c +++ b/pg_lake_table/src/ddl/alter_table.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "miscadmin.h" #include "access/heapam.h" diff --git a/pg_lake_table/src/ddl/create_table.c b/pg_lake_table/src/ddl/create_table.c index 510cd7d40..9f758554b 100644 --- a/pg_lake_table/src/ddl/create_table.c +++ b/pg_lake_table/src/ddl/create_table.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "miscadmin.h" #include "access/table.h" diff --git a/pg_lake_table/src/describe/describe.c b/pg_lake_table/src/describe/describe.c index 5a32d431f..8d4c78773 100644 --- a/pg_lake_table/src/describe/describe.c +++ b/pg_lake_table/src/describe/describe.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "libpq-fe.h" #include "commands/defrem.h" diff --git a/pg_lake_table/src/duckdb/transform_query_to_duckdb.c b/pg_lake_table/src/duckdb/transform_query_to_duckdb.c index f0fb4dfbd..f30b5d5d6 100644 --- a/pg_lake_table/src/duckdb/transform_query_to_duckdb.c +++ b/pg_lake_table/src/duckdb/transform_query_to_duckdb.c @@ -28,6 +28,7 @@ #include "access/relation.h" #include "access/xact.h" #include "utils/builtins.h" +#include "utils/timestamp.h" #include "catalog/pg_class_d.h" #include "catalog/pg_type_d.h" #include "pg_lake/iceberg/api/table_schema.h" diff --git a/pg_lake_table/src/fdw/data_file_pruning.c b/pg_lake_table/src/fdw/data_file_pruning.c index b2beca68a..b4cde17bc 100644 --- a/pg_lake_table/src/fdw/data_file_pruning.c +++ b/pg_lake_table/src/fdw/data_file_pruning.c @@ -21,6 +21,9 @@ * execution and statistics of the data files. */ #include "postgres.h" +#include "utils/hsearch.h" +#include "access/htup_details.h" +#include "catalog/pg_type_d.h" #include "catalog/pg_am_d.h" #include "catalog/pg_index.h" diff --git a/pg_lake_table/src/fdw/data_files_catalog.c b/pg_lake_table/src/fdw/data_files_catalog.c index 5890683b1..82d343813 100644 --- a/pg_lake_table/src/fdw/data_files_catalog.c +++ b/pg_lake_table/src/fdw/data_files_catalog.c @@ -16,6 +16,8 @@ */ #include "postgres.h" +#include "utils/hsearch.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" #include "miscadmin.h" diff --git a/pg_lake_table/src/fdw/data_files_catalog_batch.c b/pg_lake_table/src/fdw/data_files_catalog_batch.c index 750684a93..21d275529 100644 --- a/pg_lake_table/src/fdw/data_files_catalog_batch.c +++ b/pg_lake_table/src/fdw/data_files_catalog_batch.c @@ -25,6 +25,8 @@ */ #include "postgres.h" +#include "utils/hsearch.h" +#include "catalog/pg_type_d.h" #include "pg_extension_base/extension_ids.h" #include "pg_extension_base/spi_helpers.h" @@ -42,7 +44,9 @@ #include "executor/spi.h" #include "utils/builtins.h" +#if PG_VERSION_NUM < 190000 #include "utils/dynahash.h" +#endif /* path -> int64 file id, built from RETURNING output of ExecInsertDataFiles */ typedef struct FileIdHashEntry diff --git a/pg_lake_table/src/fdw/deparse.c b/pg_lake_table/src/fdw/deparse.c index 48c36da49..a26d3b5e1 100644 --- a/pg_lake_table/src/fdw/deparse.c +++ b/pg_lake_table/src/fdw/deparse.c @@ -1222,7 +1222,11 @@ is_foreign_pathkey(PlannerInfo *root, static char * deparse_type_name(Oid type_oid, int32 typemod) { +#if PG_VERSION_NUM >= 190000 + uint16 flags = FORMAT_TYPE_TYPEMOD_GIVEN; +#else bits16 flags = FORMAT_TYPE_TYPEMOD_GIVEN; +#endif if (!is_builtin(type_oid)) flags |= FORMAT_TYPE_FORCE_QUALIFY; diff --git a/pg_lake_table/src/fdw/deparse_ruleutils.c b/pg_lake_table/src/fdw/deparse_ruleutils.c index 9e3c565fe..6b7b4c3c0 100644 --- a/pg_lake_table/src/fdw/deparse_ruleutils.c +++ b/pg_lake_table/src/fdw/deparse_ruleutils.c @@ -25,6 +25,7 @@ *------------------------------------------------------------------------- */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "parser/parse_func.h" #include "parser/parsetree.h" diff --git a/pg_lake_table/src/fdw/partition_transform.c b/pg_lake_table/src/fdw/partition_transform.c index 077bf0655..6d385faae 100644 --- a/pg_lake_table/src/fdw/partition_transform.c +++ b/pg_lake_table/src/fdw/partition_transform.c @@ -16,6 +16,8 @@ */ #include "postgres.h" +#include "utils/hsearch.h" +#include "catalog/pg_type_d.h" #include "utils/builtins.h" #include "utils/date.h" #include "utils/lsyscache.h" diff --git a/pg_lake_table/src/fdw/partitioning/partition_spec_catalog.c b/pg_lake_table/src/fdw/partitioning/partition_spec_catalog.c index 05680c271..e3769a545 100644 --- a/pg_lake_table/src/fdw/partitioning/partition_spec_catalog.c +++ b/pg_lake_table/src/fdw/partitioning/partition_spec_catalog.c @@ -16,6 +16,8 @@ */ #include "postgres.h" +#include "utils/hsearch.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "pg_lake/extensions/pg_lake_table.h" diff --git a/pg_lake_table/src/fdw/partitioning/partitioned_dest_receiver.c b/pg_lake_table/src/fdw/partitioning/partitioned_dest_receiver.c index 8f34c30cc..45cb5a4ab 100644 --- a/pg_lake_table/src/fdw/partitioning/partitioned_dest_receiver.c +++ b/pg_lake_table/src/fdw/partitioning/partitioned_dest_receiver.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "utils/hsearch.h" #include "access/tupdesc.h" #include "commands/copy.h" diff --git a/pg_lake_table/src/fdw/pg_lake_table.c b/pg_lake_table/src/fdw/pg_lake_table.c index d41078945..70e0cf702 100644 --- a/pg_lake_table/src/fdw/pg_lake_table.c +++ b/pg_lake_table/src/fdw/pg_lake_table.c @@ -25,6 +25,8 @@ *------------------------------------------------------------------------- */ #include "postgres.h" +#include "utils/hsearch.h" +#include "catalog/pg_type_d.h" #include "funcapi.h" #include diff --git a/pg_lake_table/src/fdw/position_delete_dest.c b/pg_lake_table/src/fdw/position_delete_dest.c index 21b3fb564..66e204691 100644 --- a/pg_lake_table/src/fdw/position_delete_dest.c +++ b/pg_lake_table/src/fdw/position_delete_dest.c @@ -16,6 +16,8 @@ */ #include "postgres.h" +#include "access/htup_details.h" +#include "utils/hsearch.h" #include "access/tupdesc.h" #include "commands/copy.h" diff --git a/pg_lake_table/src/fdw/schema_operations/field_id_mapping_catalog.c b/pg_lake_table/src/fdw/schema_operations/field_id_mapping_catalog.c index 53dabb678..ce3d0ffe7 100644 --- a/pg_lake_table/src/fdw/schema_operations/field_id_mapping_catalog.c +++ b/pg_lake_table/src/fdw/schema_operations/field_id_mapping_catalog.c @@ -22,6 +22,7 @@ * from/to catalog lake_table.field_id_mappings. */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "miscadmin.h" #include "access/relation.h" diff --git a/pg_lake_table/src/fdw/update_tracking.c b/pg_lake_table/src/fdw/update_tracking.c index b206cd1f3..b46fe1b41 100644 --- a/pg_lake_table/src/fdw/update_tracking.c +++ b/pg_lake_table/src/fdw/update_tracking.c @@ -219,7 +219,11 @@ CreateUpdateTrackingTable(RangeVar *updateTableName) indexColumn->name = "rowid"; createPrimaryKey->indexParams = list_make1(indexColumn); - DefineIndex(updateTableAddress.objectId, + DefineIndex( +#if PG_VERSION_NUM >= 190000 + /* pstate */ NULL, +#endif + updateTableAddress.objectId, createPrimaryKey, /* indexRelationId */ InvalidOid, /* parentIndexId */ InvalidOid, diff --git a/pg_lake_table/src/fdw/writable_table.c b/pg_lake_table/src/fdw/writable_table.c index dcab98fed..1d8522e56 100644 --- a/pg_lake_table/src/fdw/writable_table.c +++ b/pg_lake_table/src/fdw/writable_table.c @@ -16,6 +16,9 @@ */ #include "postgres.h" +#include "storage/lock.h" +#include "utils/hsearch.h" +#include "catalog/pg_type_d.h" #include "fmgr.h" #include "funcapi.h" #include "miscadmin.h" diff --git a/pg_lake_table/src/planner/insert_select.c b/pg_lake_table/src/planner/insert_select.c index 0ba1bee5a..a00d19d9d 100644 --- a/pg_lake_table/src/planner/insert_select.c +++ b/pg_lake_table/src/planner/insert_select.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "catalog/pg_type_d.h" #include "access/table.h" #include "pg_lake/extensions/postgis.h" diff --git a/pg_lake_table/src/planner/query_pushdown.c b/pg_lake_table/src/planner/query_pushdown.c index 18c9fbe18..ffdd4d710 100644 --- a/pg_lake_table/src/planner/query_pushdown.c +++ b/pg_lake_table/src/planner/query_pushdown.c @@ -16,6 +16,8 @@ */ #include "postgres.h" +#include "utils/hsearch.h" +#include "catalog/pg_type_d.h" #include "funcapi.h" #include "miscadmin.h" @@ -95,7 +97,11 @@ typedef struct IsShippableContext static PlannedStmt *LakeTablePlanner(Query *parse, const char *queryString, - int cursorOptions, ParamListInfo boundParams); + int cursorOptions, ParamListInfo boundParams +#if PG_VERSION_NUM >= 190000 + ,ExplainState *es +#endif +); static bool AdjustParseTreeForPgLake(Node *node, void *context); static bool ProcessNotShippableExpressionWalker(Node *node, IsShippableContext * context); static bool AddMissingRTEAliasaes(Node *node, void *context); @@ -253,7 +259,11 @@ AppendPermInfos(PlannedStmt *pushdownPlan, PlannedStmt *localPlan) */ static PlannedStmt * LakeTablePlanner(Query *parse, const char *queryString, - int cursorOptions, ParamListInfo boundParams) + int cursorOptions, ParamListInfo boundParams +#if PG_VERSION_NUM >= 190000 + ,ExplainState *es +#endif +) { Query *originalQuery = NULL; bool hasLakeTable = false; @@ -293,7 +303,11 @@ LakeTablePlanner(Query *parse, const char *queryString, PG_TRY(); { - plan = PreviousPlannerHook(parse, queryString, cursorOptions, boundParams); + plan = PreviousPlannerHook(parse, queryString, cursorOptions, boundParams +#if PG_VERSION_NUM >= 190000 + ,es +#endif + ); if (EnableFullQueryPushdown && hasLakeTable && (cursorOptions & CURSOR_OPT_SCROLL) == 0) diff --git a/pg_lake_table/src/planner/restriction_collector.c b/pg_lake_table/src/planner/restriction_collector.c index 31087de91..385c964d3 100644 --- a/pg_lake_table/src/planner/restriction_collector.c +++ b/pg_lake_table/src/planner/restriction_collector.c @@ -24,6 +24,7 @@ * that Postgres planner knows about. */ #include "postgres.h" +#include "utils/hsearch.h" #include "funcapi.h" #include "miscadmin.h" diff --git a/pg_lake_table/src/recovery/recover.c b/pg_lake_table/src/recovery/recover.c index 53b3f22ab..e1f5bb038 100644 --- a/pg_lake_table/src/recovery/recover.c +++ b/pg_lake_table/src/recovery/recover.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "access/htup_details.h" #include "miscadmin.h" #include "fmgr.h" diff --git a/pg_lake_table/src/test/hide_lake_objects.c b/pg_lake_table/src/test/hide_lake_objects.c index d53835729..a4e70fe39 100644 --- a/pg_lake_table/src/test/hide_lake_objects.c +++ b/pg_lake_table/src/test/hide_lake_objects.c @@ -16,6 +16,8 @@ */ #include "postgres.h" +#include "utils/hsearch.h" +#include "access/htup_details.h" #include "miscadmin.h" #include "access/genam.h" diff --git a/pg_lake_table/src/test/test_partition_tuple.c b/pg_lake_table/src/test/test_partition_tuple.c index acd2644a8..2bef4fcc5 100644 --- a/pg_lake_table/src/test/test_partition_tuple.c +++ b/pg_lake_table/src/test/test_partition_tuple.c @@ -16,11 +16,14 @@ */ #include "postgres.h" +#include "access/htup_details.h" +#include "catalog/pg_type_d.h" #include "funcapi.h" #include "access/relation.h" #include "utils/builtins.h" #include "utils/lsyscache.h" #include "utils/rel.h" +#include "utils/tuplestore.h" #include "pg_lake/partitioning/partition_spec_catalog.h" #include "pg_lake/fdw/partition_transform.h" diff --git a/pg_lake_table/src/transaction/track_iceberg_metadata_changes.c b/pg_lake_table/src/transaction/track_iceberg_metadata_changes.c index 2c3e0655f..96104738f 100644 --- a/pg_lake_table/src/transaction/track_iceberg_metadata_changes.c +++ b/pg_lake_table/src/transaction/track_iceberg_metadata_changes.c @@ -16,6 +16,7 @@ */ #include "postgres.h" +#include "utils/hsearch.h" #include "access/xact.h" #include "common/int.h" #include "utils/memutils.h" diff --git a/pg_map/src/map.c b/pg_map/src/map.c index 15bf0daa3..fd0318e50 100644 --- a/pg_map/src/map.c +++ b/pg_map/src/map.c @@ -53,6 +53,7 @@ #include "utils/regproc.h" #include "utils/syscache.h" #include "utils/typcache.h" +#include "utils/tuplestore.h" /* Check PgSQL version */ diff --git a/pgduck_server/Makefile b/pgduck_server/Makefile index 5f6c956aa..3dd88b740 100644 --- a/pgduck_server/Makefile +++ b/pgduck_server/Makefile @@ -12,9 +12,11 @@ PGXS := $(shell $(PG_CONFIG) --pgxs) PG_LAKE_DELTA_SUPPORT ?= 0 -# compile with C11 option to use modern C +# compile with C11 option to use modern C; use GNU dialect so that +# typeof / static_assert resolve when including PG19's c.h, which +# uses both unconditionally # use duckdb.h from the duckdb submodule -PG_CPPFLAGS = -std=c11 -I$(PG_INCLUDEDIR) -Iinclude -DPG_LAKE_DELTA_SUPPORT=$(PG_LAKE_DELTA_SUPPORT) +PG_CPPFLAGS = -std=gnu11 -I$(PG_INCLUDEDIR) -Iinclude -DPG_LAKE_DELTA_SUPPORT=$(PG_LAKE_DELTA_SUPPORT) # Add dependencies SHLIB_LINK_INTERNAL = $(libpq) diff --git a/pgduck_server/include/utils/numutils.h b/pgduck_server/include/utils/numutils.h index daabe4dd8..eb349908c 100644 --- a/pgduck_server/include/utils/numutils.h +++ b/pgduck_server/include/utils/numutils.h @@ -25,8 +25,6 @@ #include "c.h" -#include "nodes/nodes.h" - extern int pg_itoa(int16 i, char *a); extern int pg_ulltoa_n(uint64 value, char *a); extern int pg_ultoa_n(uint32 value, char *a); diff --git a/pgduck_server/src/duckdb/type_conversion.c b/pgduck_server/src/duckdb/type_conversion.c index 6424abc43..4b7171288 100644 --- a/pgduck_server/src/duckdb/type_conversion.c +++ b/pgduck_server/src/duckdb/type_conversion.c @@ -22,6 +22,7 @@ * * Copyright (c) 2025 Snowflake Computing, Inc. All rights reserved. */ +#include "catalog/pg_type_d.h" #include "c.h" #include "postgres_fe.h" diff --git a/pgduck_server/tests/ctests/Makefile b/pgduck_server/tests/ctests/Makefile index 630245b61..2f1519908 100644 --- a/pgduck_server/tests/ctests/Makefile +++ b/pgduck_server/tests/ctests/Makefile @@ -12,8 +12,10 @@ PG_INCLUDEDIR = $(shell $(PG_CONFIG) --includedir) # PGXS: Locates PostgreSQL extension building infrastructure using 'pg_config'. PGXS := $(shell $(PG_CONFIG) --pgxs) -# compile with C11 option to use modern C -PG_CPPFLAGS = -std=c11 -I$(PG_INCLUDEDIR) -g +# compile with C11 option to use modern C; use GNU dialect so that +# typeof / static_assert resolve when including PG19's c.h, which +# uses both unconditionally +PG_CPPFLAGS = -std=gnu11 -I$(PG_INCLUDEDIR) -g PG_LDFLAGS = -L$(PG_LIBDIR) -lpq -lpgcommon -lduckdb -Wl,-rpath,$(PG_LIBDIR) From a599fe9deb954042b8fe152e3ef67c98e3f03ae8 Mon Sep 17 00:00:00 2001 From: David Christensen Date: Thu, 11 Jun 2026 23:49:51 +0000 Subject: [PATCH 03/11] pg_map: bump command counter after DefineDomain PG19's ProcedureCreate() now walks each new function's dependencies through get_object_namespace() (via find_temp_object), which does syscache lookups. Without a CommandCounterIncrement after DefineDomain, the freshly-created domain type isn't visible to syscache, so the very next map_create_function call ERRORs with "cache lookup failed for cache 90 oid ". Signed-off-by: David Christensen --- pg_map/src/map.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pg_map/src/map.c b/pg_map/src/map.c index fd0318e50..6c1c0c19d 100644 --- a/pg_map/src/map.c +++ b/pg_map/src/map.c @@ -565,6 +565,14 @@ map_create_type(char *typeName, Oid keyElementType, Oid valElementType) mapTypeAddr = DefineDomain(newDomain); #endif + /* + * Make the new domain visible to syscache before we use its OID as a + * function argument type below. PG19's ProcedureCreate() walks each + * dependency through get_object_namespace() (find_temp_object); without + * this CCI the pg_type row is invisible and the lookup ERRORs out. + */ + CommandCounterIncrement(); + /* map pair depend on map to make sure they are always dropped together */ recordDependencyOn(&keyValPairTypeAddr, &mapTypeAddr, DEPENDENCY_NORMAL); From ec06d9a4b138c83f8e664673414a16923a1e3ac6 Mon Sep 17 00:00:00 2001 From: David Christensen Date: Thu, 11 Jun 2026 23:56:37 +0000 Subject: [PATCH 04/11] pg_map: fix typo in CCI comment to satisfy typos linter Signed-off-by: David Christensen --- pg_map/src/map.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pg_map/src/map.c b/pg_map/src/map.c index 6c1c0c19d..90c8bd22a 100644 --- a/pg_map/src/map.c +++ b/pg_map/src/map.c @@ -569,7 +569,7 @@ map_create_type(char *typeName, Oid keyElementType, Oid valElementType) * Make the new domain visible to syscache before we use its OID as a * function argument type below. PG19's ProcedureCreate() walks each * dependency through get_object_namespace() (find_temp_object); without - * this CCI the pg_type row is invisible and the lookup ERRORs out. + * this CCI the pg_type row is invisible and the lookup raises an error. */ CommandCounterIncrement(); From 1483d6f6edbd9352928e5966e918126bda9f902c Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 12 Jun 2026 00:21:36 +0000 Subject: [PATCH 05/11] pgduck_server: support NegotiateProtocolVersion for PG19 clients PG19's libpq defaults max_pversion to PG_PROTOCOL_GREASE (3,9999) so it can probe whether a server speaks NegotiateProtocolVersion. pgduck_server previously slammed the connection on any protocol other than 3.0, which made every PG19 cache worker / pg_lake client fail to attach with "server closed the connection unexpectedly". When a client requests a 3.x version higher than 3.0, send a 'v' (NegotiateProtocolVersion) message advertising 3.0 and zero unrecognized options, mirroring what core PostgreSQL does in backend_startup.c. libpq accepts the downgrade transparently; older clients (PG <=18) that default to 3.0 are unaffected. Signed-off-by: David Christensen --- pgduck_server/src/pgsession/pgsession_io.c | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pgduck_server/src/pgsession/pgsession_io.c b/pgduck_server/src/pgsession/pgsession_io.c index 5e430f826..a82f3768c 100644 --- a/pgduck_server/src/pgsession/pgsession_io.c +++ b/pgduck_server/src/pgsession/pgsession_io.c @@ -281,12 +281,34 @@ pgsession_read_startup_packet(PGSession * pgSession) pg_free(startupPacketBuf); return EOF; } - else if (proto != PG_PROTOCOL(3, 0)) + else if (PG_PROTOCOL_MAJOR(proto) != 3) { PGDUCK_SERVER_ERROR("unexpected protocol message: %u", proto); pg_free(startupPacketBuf); return EOF; } + else if (proto != PG_PROTOCOL(3, 0)) + { + /* + * Client asked for a 3.x version higher than 3.0 (e.g. PG19's libpq + * defaults to PG_PROTOCOL_GREASE = 3.9999 to probe servers that + * support NegotiateProtocolVersion). Tell the client we only + * implement 3.0 by sending a 'v' (NegotiateProtocolVersion) message + * with our supported version and a zero count of unrecognized + * protocol options; libpq will then either accept the downgrade or + * disconnect on its own. After this, treat the connection as 3.0. + */ + StringInfoData buf; + + pq_beginmessage(&buf, 'v'); + pq_sendint32(&buf, PG_PROTOCOL(3, 0)); + pq_sendint32(&buf, 0); /* no unrecognized options */ + if (!IsOK(pq_endmessage(pgSession, &buf))) + { + pg_free(startupPacketBuf); + return EOF; + } + } pg_free(startupPacketBuf); return OK; From d67b9cd14c2faf877eacfa3449145ccdbc1d19d7 Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 12 Jun 2026 00:41:53 +0000 Subject: [PATCH 06/11] pgduck_server: echo unrecognized _pq_.* options in NegotiateProtocolVersion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PG19's libpq doesn't just probe protocol minor versions — when probing with PG_PROTOCOL_GREASE it also sends a deliberately-unknown protocol option named "_pq_.test_protocol_negotiation" and verifies the server echoes it back in the NegotiateProtocolVersion (`v`) message. Without that echo, libpq aborts with "server did not report the unsupported ... parameter in its protocol negotiation message". Scan the startup packet body for any name beginning with "_pq_." and include those names in the `v` reply, mirroring core's ProcessStartupPacket / SendNegotiateProtocolVersion behavior. Signed-off-by: David Christensen --- pgduck_server/src/pgsession/pgsession_io.c | 55 +++++++++++++++++++--- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/pgduck_server/src/pgsession/pgsession_io.c b/pgduck_server/src/pgsession/pgsession_io.c index a82f3768c..fc79ed320 100644 --- a/pgduck_server/src/pgsession/pgsession_io.c +++ b/pgduck_server/src/pgsession/pgsession_io.c @@ -292,17 +292,60 @@ pgsession_read_startup_packet(PGSession * pgSession) /* * Client asked for a 3.x version higher than 3.0 (e.g. PG19's libpq * defaults to PG_PROTOCOL_GREASE = 3.9999 to probe servers that - * support NegotiateProtocolVersion). Tell the client we only - * implement 3.0 by sending a 'v' (NegotiateProtocolVersion) message - * with our supported version and a zero count of unrecognized - * protocol options; libpq will then either accept the downgrade or - * disconnect on its own. After this, treat the connection as 3.0. + * support NegotiateProtocolVersion). Reply with a 'v' + * (NegotiateProtocolVersion) message advertising 3.0 plus the names + * of any "_pq_.*" protocol options we did not recognize, mirroring + * what core PostgreSQL does in backend_startup.c. libpq verifies + * that every _pq_.* option it sent is echoed back; otherwise it + * aborts with "server did not report the unsupported ... parameter + * in its protocol negotiation message". */ StringInfoData buf; + int32 offset = sizeof(ProtocolVersion); + int32 numUnrecognized = 0; + + /* + * First pass: count "_pq_.*" options. The startup body is a series + * of NUL-terminated name/value pairs followed by a final empty name. + * We rely on the trailing NUL appended in the malloc above so that + * unterminated strings can't run past the buffer. + */ + while (offset < startupPacketLen) + { + const char *name = startupPacketBuf + offset; + int32 valOffset; + + if (*name == '\0') + break; + valOffset = offset + (int32) strlen(name) + 1; + if (valOffset >= startupPacketLen) + break; + if (strncmp(name, "_pq_.", 5) == 0) + numUnrecognized++; + offset = valOffset + (int32) strlen(startupPacketBuf + valOffset) + 1; + } pq_beginmessage(&buf, 'v'); pq_sendint32(&buf, PG_PROTOCOL(3, 0)); - pq_sendint32(&buf, 0); /* no unrecognized options */ + pq_sendint32(&buf, numUnrecognized); + + /* Second pass: emit the names. */ + offset = sizeof(ProtocolVersion); + while (offset < startupPacketLen) + { + const char *name = startupPacketBuf + offset; + int32 valOffset; + + if (*name == '\0') + break; + valOffset = offset + (int32) strlen(name) + 1; + if (valOffset >= startupPacketLen) + break; + if (strncmp(name, "_pq_.", 5) == 0) + pq_sendstring(&buf, name); + offset = valOffset + (int32) strlen(startupPacketBuf + valOffset) + 1; + } + if (!IsOK(pq_endmessage(pgSession, &buf))) { pg_free(startupPacketBuf); From 5d81e23a3263775a67ad9167c31bfcb44235fbc9 Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 12 Jun 2026 01:01:10 +0000 Subject: [PATCH 07/11] pgduck_server: pgindent reflow for NegotiateProtocolVersion comments Signed-off-by: David Christensen --- pgduck_server/src/pgsession/pgsession_io.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pgduck_server/src/pgsession/pgsession_io.c b/pgduck_server/src/pgsession/pgsession_io.c index fc79ed320..75737cd90 100644 --- a/pgduck_server/src/pgsession/pgsession_io.c +++ b/pgduck_server/src/pgsession/pgsession_io.c @@ -295,19 +295,19 @@ pgsession_read_startup_packet(PGSession * pgSession) * support NegotiateProtocolVersion). Reply with a 'v' * (NegotiateProtocolVersion) message advertising 3.0 plus the names * of any "_pq_.*" protocol options we did not recognize, mirroring - * what core PostgreSQL does in backend_startup.c. libpq verifies - * that every _pq_.* option it sent is echoed back; otherwise it - * aborts with "server did not report the unsupported ... parameter - * in its protocol negotiation message". + * what core PostgreSQL does in backend_startup.c. libpq verifies that + * every _pq_.* option it sent is echoed back; otherwise it aborts + * with "server did not report the unsupported ... parameter in its + * protocol negotiation message". */ StringInfoData buf; int32 offset = sizeof(ProtocolVersion); int32 numUnrecognized = 0; /* - * First pass: count "_pq_.*" options. The startup body is a series - * of NUL-terminated name/value pairs followed by a final empty name. - * We rely on the trailing NUL appended in the malloc above so that + * First pass: count "_pq_.*" options. The startup body is a series of + * NUL-terminated name/value pairs followed by a final empty name. We + * rely on the trailing NUL appended in the malloc above so that * unterminated strings can't run past the buffer. */ while (offset < startupPacketLen) From ec96f6ac324a2e6dd9c35d86ad545ef0b44a10bd Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 12 Jun 2026 01:29:12 +0000 Subject: [PATCH 08/11] pg19: call TupleDescFinalize on manually-built TupleDescs PG19 added a firstNonCachedOffsetAttr cache to TupleDesc and asserts in BlessTupleDesc() that it has been populated. Manually-constructed descriptors (CreateTemplateTupleDesc + TupleDescInitEntry) now require an explicit TupleDescFinalize() call before they can be Blessed or used to form heap tuples; otherwise PG19 backends abort with "failed Assert(\"tupdesc->firstNonCachedOffsetAttr >= 0\")". Add a no-op TupleDescFinalize() shim in pg_compat.h for PG <=18 so callers can finalize unconditionally, and finalize at every site that builds a TupleDesc by hand. Drive-by: replace the C99 designated initializer for ErrorSaveContext in numeric_int4_opt_error() with explicit memset+field assignment, so pg_lake_copy (compiled with -Wmissing-field-initializers / -Werror) no longer rejects the new file. Signed-off-by: David Christensen --- .../include/pg_extension_base/pg_compat.h | 23 +++++++++++++++++-- pg_extension_base/src/attached_worker.c | 2 ++ pg_lake_copy/src/copy/copy.c | 6 +++++ pg_lake_copy/src/test/types.c | 2 ++ pg_lake_engine/src/parsetree/columns.c | 5 ++++ pg_lake_table/src/fdw/pg_lake_table.c | 2 ++ pg_lake_table/src/fdw/writable_table.c | 3 +++ 7 files changed, 41 insertions(+), 2 deletions(-) diff --git a/pg_extension_base/include/pg_extension_base/pg_compat.h b/pg_extension_base/include/pg_extension_base/pg_compat.h index 04f3a17f9..562c3a107 100644 --- a/pg_extension_base/include/pg_extension_base/pg_compat.h +++ b/pg_extension_base/include/pg_extension_base/pg_compat.h @@ -18,11 +18,26 @@ #pragma once #include "postgres.h" +#include "access/tupdesc.h" #include "catalog/pg_type_d.h" #include "commands/copy.h" #include "nodes/miscnodes.h" #include "utils/numeric.h" +/* + * PG19 added a firstNonCachedOffsetAttr cache to TupleDesc that must be + * populated via TupleDescFinalize() before BlessTupleDesc() or + * heap_form_tuple() are called on a manually-constructed descriptor; PG19's + * BlessTupleDesc Asserts on a missing finalize. Older releases don't expose + * the function, so define a no-op shim there and let callers always finalize. + */ +#if PG_VERSION_NUM < 190000 +static inline void +TupleDescFinalize(TupleDesc tupdesc) +{ +} +#endif + #if PG_VERSION_NUM >= 190000 /* @@ -41,8 +56,12 @@ static inline int32 numeric_int4_opt_error(Numeric num, bool *have_error) { - ErrorSaveContext escontext = {T_ErrorSaveContext}; - int32 result = numeric_int4_safe(num, (Node *) &escontext); + ErrorSaveContext escontext; + int32 result; + + memset(&escontext, 0, sizeof(escontext)); + escontext.type = T_ErrorSaveContext; + result = numeric_int4_safe(num, (Node *) &escontext); *have_error = escontext.error_occurred; return result; diff --git a/pg_extension_base/src/attached_worker.c b/pg_extension_base/src/attached_worker.c index f6fa7a9f4..097e2f785 100644 --- a/pg_extension_base/src/attached_worker.c +++ b/pg_extension_base/src/attached_worker.c @@ -517,6 +517,8 @@ ProcessProtocolMessages(shm_mq_handle *queue, bool nowait, typeMod, 0); } + TupleDescFinalize(tupleDesc); + *resultDesc = tupleDesc; } break; diff --git a/pg_lake_copy/src/copy/copy.c b/pg_lake_copy/src/copy/copy.c index c55bf4541..7aeb5e418 100644 --- a/pg_lake_copy/src/copy/copy.c +++ b/pg_lake_copy/src/copy/copy.c @@ -25,6 +25,8 @@ #include "miscadmin.h" #include "libpq-fe.h" +#include "pg_extension_base/pg_compat.h" + #include "access/table.h" #include "access/tupdesc.h" #include "access/xact.h" @@ -1362,6 +1364,8 @@ BuildTupleDescriptorForRelation(Relation relation, List *attributeList) attributeNumber++; } + TupleDescFinalize(attributeDescriptor); + return attributeDescriptor; } @@ -1407,6 +1411,8 @@ RemoveDroppedColumnsFromTupleDesc(TupleDesc tableDescriptor) attributeNumber++; } + TupleDescFinalize(cleanTupleDesc); + return cleanTupleDesc; } diff --git a/pg_lake_copy/src/test/types.c b/pg_lake_copy/src/test/types.c index 51b535404..a560737ca 100644 --- a/pg_lake_copy/src/test/types.c +++ b/pg_lake_copy/src/test/types.c @@ -24,6 +24,7 @@ #include "fmgr.h" #include "funcapi.h" +#include "pg_extension_base/pg_compat.h" #include "pg_lake/pgduck/type.h" #include "pg_lake/pgduck/parse_struct.h" #include "utils/builtins.h" @@ -82,6 +83,7 @@ duckdb_parse_struct(PG_FUNCTION_ARGS) TupleDescInitEntry(tupdesc, (AttrNumber) 4, "isarray", BOOLOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 5, "level", INT4OID, -1, 0); + TupleDescFinalize(tupdesc); tupdesc = BlessTupleDesc(tupdesc); CompositeType *parsedType = ParseStructString(structString); diff --git a/pg_lake_engine/src/parsetree/columns.c b/pg_lake_engine/src/parsetree/columns.c index 87cec1a80..10882ea56 100644 --- a/pg_lake_engine/src/parsetree/columns.c +++ b/pg_lake_engine/src/parsetree/columns.c @@ -16,6 +16,9 @@ */ #include "postgres.h" + +#include "pg_extension_base/pg_compat.h" + #include "nodes/makefuncs.h" #include "nodes/nodeFuncs.h" #include "nodes/primnodes.h" @@ -80,6 +83,8 @@ BuildTupleDescriptorForTargetList(List *targetList) attributeNumber++; } + TupleDescFinalize(tupleDesc); + return tupleDesc; } diff --git a/pg_lake_table/src/fdw/pg_lake_table.c b/pg_lake_table/src/fdw/pg_lake_table.c index 70e0cf702..28eaf1449 100644 --- a/pg_lake_table/src/fdw/pg_lake_table.c +++ b/pg_lake_table/src/fdw/pg_lake_table.c @@ -2770,6 +2770,8 @@ CreateRowIdTupleDesc(void) TupleDescInitEntry(tupleDescriptor, (AttrNumber) 2, "file_row_number", INT8OID, -1, 0); + TupleDescFinalize(tupleDescriptor); + return tupleDescriptor; } diff --git a/pg_lake_table/src/fdw/writable_table.c b/pg_lake_table/src/fdw/writable_table.c index 1d8522e56..0fc1b23b8 100644 --- a/pg_lake_table/src/fdw/writable_table.c +++ b/pg_lake_table/src/fdw/writable_table.c @@ -29,6 +29,7 @@ #include "catalog/pg_namespace.h" #include "commands/defrem.h" #include "common/hashfn.h" +#include "pg_extension_base/pg_compat.h" #include "pg_lake/cleanup/in_progress_files.h" #include "pg_lake/data_file/data_files.h" #include "pg_lake/data_file/data_file_stats.h" @@ -1611,6 +1612,8 @@ CreatePositionDeleteTupleDesc(void) TupleDescInitEntry(tupleDescriptor, (AttrNumber) 3, "row", RECORDOID, -1, 0); + TupleDescFinalize(tupleDescriptor); + return tupleDescriptor; } From 6f21bd5882c52f56da31559486db1a27f1f97f8e Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 12 Jun 2026 02:00:19 +0000 Subject: [PATCH 09/11] pg19: round 2 of code-level fixes Four independent regressions surfaced after the earlier batch: 1. pg_lake_copy: stop intercepting plain COPY ... TO STDOUT (FORMAT json) on PG18+. PG18 added native COPY-to-stdout JSON support and PG19 wired the regression suite to exercise force_array / on_error / reject_limit etc. Our hook was matching on format alone and rejecting these new options with "pg_lake_copy: invalid option ...". Now we only claim the JSON path when the destination is a URL or @STAGE; STDIN/STDOUT JSON is left to core. 2. pg_lake_table: skip system columns (varattno <= 0) in AddDeleteReturningTargetTableVars. PG19 added an Assert in TupleDescAttr() that fires on negative indexes; PG18 silently returned garbage. The pull_vars_of_level() walk over a DELETE RETURNING list can include system columns when the test triggers a re-DELETE after a previous "cannot be updated by multiple operations in a single command" error. 3. pg_lake_table: switch the synthetic-test-query pull_var_clause() in restriction_collector to PVC_RECURSE_AGGREGATES | PVC_RECURSE_PLACEHOLDERS so we descend into Aggref/PlaceHolderVar instead of elog'ing "Aggref found where not expected". PG19's planner leaves Aggrefs inside FDW-pushdown restriction clauses for some benchmark queries (TPC-DS Q10, TPC-H Q21). 4. Isolation tests: add `_1.out` alternative expected outputs for isolation_iceberg_serializable and isolation_iceberg_repeatable_read. PG19 split the previously-unified "could not serialize access due to concurrent update" message into "concurrent update" vs "concurrent delete" depending on TM_Updated vs TM_Deleted. The 5 affected lines are the cases where the conflicting transaction issued a DELETE. Signed-off-by: David Christensen --- pg_lake_copy/src/copy/copy.c | 16 ++ pg_lake_table/src/fdw/pg_lake_table.c | 9 + .../src/planner/restriction_collector.c | 11 +- .../isolation_iceberg_repeatable_read_1.out | 190 ++++++++++++++++++ .../isolation_iceberg_serializable_1.out | 84 ++++++++ 5 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 pg_lake_table/tests/isolation/expected/isolation_iceberg_repeatable_read_1.out create mode 100644 pg_lake_table/tests/isolation/expected/isolation_iceberg_serializable_1.out diff --git a/pg_lake_copy/src/copy/copy.c b/pg_lake_copy/src/copy/copy.c index 7aeb5e418..0d7784534 100644 --- a/pg_lake_copy/src/copy/copy.c +++ b/pg_lake_copy/src/copy/copy.c @@ -248,6 +248,22 @@ IsPgLakeCopy(CopyStmt *copyStmt) return false; } +#if PG_VERSION_NUM >= 180000 + + /* + * PostgreSQL 18 added native support for COPY ... TO STDOUT (FORMAT + * json), along with format-specific options like force_array, on_error, + * and reject_limit. Let core handle plain STDIN/STDOUT JSON COPY so we + * don't intercept upstream tests with our own option-validation errors. + * We still claim the JSON path when the destination is a URL or @STAGE. + */ + if (format == DATA_FORMAT_JSON && + (IsCopyFromStdin(copyStmt) || IsCopyToStdout(copyStmt))) + { + return false; + } +#endif + return true; } diff --git a/pg_lake_table/src/fdw/pg_lake_table.c b/pg_lake_table/src/fdw/pg_lake_table.c index 28eaf1449..d995f6e94 100644 --- a/pg_lake_table/src/fdw/pg_lake_table.c +++ b/pg_lake_table/src/fdw/pg_lake_table.c @@ -2103,6 +2103,15 @@ AddDeleteReturningTargetTableVars(PlannerInfo *root, if (returningVar->varno != root->parse->resultRelation) continue; + /* + * Skip whole-row references (handled above) and system columns; the + * latter have negative varattno and would index out of bounds in + * TupleDescAttr() below — PG19 added an Assert that fires on + * negative indexes where earlier majors silently returned garbage. + */ + if (returningVar->varattno <= 0) + continue; + #if PG_VERSION_NUM >= 180000 if (returningVar->varreturningtype == VAR_RETURNING_NEW || returningVar->varreturningtype == VAR_RETURNING_OLD) diff --git a/pg_lake_table/src/planner/restriction_collector.c b/pg_lake_table/src/planner/restriction_collector.c index 385c964d3..6a6f8df52 100644 --- a/pg_lake_table/src/planner/restriction_collector.c +++ b/pg_lake_table/src/planner/restriction_collector.c @@ -527,7 +527,16 @@ GenerateSyntheticTestQueryForRestrictInfo(int logLevel, RangeTblEntry *rte, * We'll only have a single RTE in the query, so we can set varno and * varnosyn to 1. */ - List *varList = pull_var_clause((Node *) copyOfRestrictClauses, 0); + + /* + * Recurse into aggregate arguments so we still rewrite the Vars inside + * them. PG19's planner can leave Aggrefs inside FDW-pushdown restriction + * clauses where earlier majors did not, and pull_var_clause(... 0) + * elog()s when it sees an Aggref. + */ + List *varList = pull_var_clause((Node *) copyOfRestrictClauses, + PVC_RECURSE_AGGREGATES | + PVC_RECURSE_PLACEHOLDERS); ListCell *cell = NULL; foreach(cell, varList) diff --git a/pg_lake_table/tests/isolation/expected/isolation_iceberg_repeatable_read_1.out b/pg_lake_table/tests/isolation/expected/isolation_iceberg_repeatable_read_1.out new file mode 100644 index 000000000..2dbbf5c69 --- /dev/null +++ b/pg_lake_table/tests/isolation/expected/isolation_iceberg_repeatable_read_1.out @@ -0,0 +1,190 @@ +Parsed test spec with 2 sessions + +starting permutation: s2-select-all s1-insert s1-commit s2-select-all s2-commit +step s2-select-all: + SELECT * FROM test_iceberg_rr ORDER BY key; + +key|value +---+----- + 1| 10 + 2| 20 + 3| 30 +(3 rows) + +step s1-insert: + INSERT INTO test_iceberg_rr VALUES (4, 40); + +step s1-commit: + COMMIT; + +step s2-select-all: + SELECT * FROM test_iceberg_rr ORDER BY key; + +key|value +---+----- + 1| 10 + 2| 20 + 3| 30 +(3 rows) + +step s2-commit: + COMMIT; + + +starting permutation: s2-select-all s1-update s1-commit s2-select-all s2-commit +step s2-select-all: + SELECT * FROM test_iceberg_rr ORDER BY key; + +key|value +---+----- + 1| 10 + 2| 20 + 3| 30 +(3 rows) + +step s1-update: + UPDATE test_iceberg_rr SET value = 99 WHERE key = 1; + +step s1-commit: + COMMIT; + +step s2-select-all: + SELECT * FROM test_iceberg_rr ORDER BY key; + +key|value +---+----- + 1| 10 + 2| 20 + 3| 30 +(3 rows) + +step s2-commit: + COMMIT; + + +starting permutation: s2-select-all s1-delete s1-commit s2-select-all s2-commit +step s2-select-all: + SELECT * FROM test_iceberg_rr ORDER BY key; + +key|value +---+----- + 1| 10 + 2| 20 + 3| 30 +(3 rows) + +step s1-delete: + DELETE FROM test_iceberg_rr WHERE key = 2; + +step s1-commit: + COMMIT; + +step s2-select-all: + SELECT * FROM test_iceberg_rr ORDER BY key; + +key|value +---+----- + 1| 10 + 2| 20 + 3| 30 +(3 rows) + +step s2-commit: + COMMIT; + + +starting permutation: s1-update s2-update s1-commit s2-rollback +step s1-update: + UPDATE test_iceberg_rr SET value = 99 WHERE key = 1; + +step s2-update: + UPDATE test_iceberg_rr SET value = 88 WHERE key = 1; + +step s1-commit: + COMMIT; + +step s2-update: <... completed> +ERROR: could not serialize access due to concurrent delete +step s2-rollback: + ROLLBACK; + + +starting permutation: s1-delete s2-delete s1-commit s2-rollback +step s1-delete: + DELETE FROM test_iceberg_rr WHERE key = 2; + +step s2-delete: + DELETE FROM test_iceberg_rr WHERE key = 2; + +step s1-commit: + COMMIT; + +step s2-delete: <... completed> +ERROR: could not serialize access due to concurrent delete +step s2-rollback: + ROLLBACK; + + +starting permutation: s1-insert s2-insert s1-commit s2-rollback +step s1-insert: + INSERT INTO test_iceberg_rr VALUES (4, 40); + +step s2-insert: + INSERT INTO test_iceberg_rr VALUES (5, 50); + +step s1-commit: + COMMIT; + +step s2-insert: <... completed> +ERROR: could not serialize access due to concurrent update +step s2-rollback: + ROLLBACK; + + +starting permutation: s1-insert s2-delete s1-commit s2-rollback +step s1-insert: + INSERT INTO test_iceberg_rr VALUES (4, 40); + +step s2-delete: + DELETE FROM test_iceberg_rr WHERE key = 2; + +step s1-commit: + COMMIT; + +step s2-delete: <... completed> +ERROR: could not serialize access due to concurrent update +step s2-rollback: + ROLLBACK; + + +starting permutation: s1-insert s2-update s1-commit s2-rollback +step s1-insert: + INSERT INTO test_iceberg_rr VALUES (4, 40); + +step s2-update: + UPDATE test_iceberg_rr SET value = 88 WHERE key = 1; + +step s1-commit: + COMMIT; + +step s2-update: <... completed> +ERROR: could not serialize access due to concurrent update +step s2-rollback: + ROLLBACK; + + +starting permutation: s1-update s2-delete-key-1 s1-commit s2-rollback +step s1-update: + UPDATE test_iceberg_rr SET value = 99 WHERE key = 1; + +step s2-delete-key-1: + DELETE FROM test_iceberg_rr WHERE key = 1; + +step s1-commit: + COMMIT; + +step s2-delete-key-1: <... completed> +ERROR: could not serialize access due to concurrent delete +step s2-rollback: + ROLLBACK; + diff --git a/pg_lake_table/tests/isolation/expected/isolation_iceberg_serializable_1.out b/pg_lake_table/tests/isolation/expected/isolation_iceberg_serializable_1.out new file mode 100644 index 000000000..600b2c946 --- /dev/null +++ b/pg_lake_table/tests/isolation/expected/isolation_iceberg_serializable_1.out @@ -0,0 +1,84 @@ +unused step name: s2-commit +Parsed test spec with 2 sessions + +starting permutation: s1-select-sum s2-select-sum s1-update-key-1 s2-update-key-2 s1-commit s2-rollback +step s1-select-sum: + SELECT SUM(value) FROM test_iceberg_ser; + +sum +--- +200 +(1 row) + +step s2-select-sum: + SELECT SUM(value) FROM test_iceberg_ser; + +sum +--- +200 +(1 row) + +step s1-update-key-1: + UPDATE test_iceberg_ser SET value = value - 50 WHERE key = 1; + +step s2-update-key-2: + UPDATE test_iceberg_ser SET value = value - 50 WHERE key = 2; + +step s1-commit: + COMMIT; + +step s2-update-key-2: <... completed> +ERROR: could not serialize access due to concurrent delete +step s2-rollback: + ROLLBACK; + + +starting permutation: s1-update-key-1 s2-update-key-1 s1-commit s2-rollback +step s1-update-key-1: + UPDATE test_iceberg_ser SET value = value - 50 WHERE key = 1; + +step s2-update-key-1: + UPDATE test_iceberg_ser SET value = value + 10 WHERE key = 1; + +step s1-commit: + COMMIT; + +step s2-update-key-1: <... completed> +ERROR: could not serialize access due to concurrent delete +step s2-rollback: + ROLLBACK; + + +starting permutation: s1-select-all s2-select-all s1-insert s2-insert s1-commit s2-rollback +step s1-select-all: + SELECT * FROM test_iceberg_ser ORDER BY key; + +key|value +---+----- + 1| 100 + 2| 100 +(2 rows) + +step s2-select-all: + SELECT * FROM test_iceberg_ser ORDER BY key; + +key|value +---+----- + 1| 100 + 2| 100 +(2 rows) + +step s1-insert: + INSERT INTO test_iceberg_ser VALUES (3, 30); + +step s2-insert: + INSERT INTO test_iceberg_ser VALUES (4, 40); + +step s1-commit: + COMMIT; + +step s2-insert: <... completed> +ERROR: could not serialize access due to concurrent update +step s2-rollback: + ROLLBACK; + From 382c9086aa0f74c5f63391f86e09c18673a7b22b Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 12 Jun 2026 02:26:32 +0000 Subject: [PATCH 10/11] pg19: round 3 of code-level fixes Three independent regressions surfaced after round 2: 1. pg_lake_table: skip Iceberg fields with no matching Postgres column in CreatePostgresColumnMappingsForIcebergTableFromExternalMetadata, regardless of catalog type. Previously only REST_CATALOG_READ_ONLY short-circuited; OBJECT_STORE_READ_ONLY (and the internal flow when the read-only schema check eventually rejects the table) fell through with attrNum=InvalidAttrNumber=0 and indexed TupleDesc with -1, which PG19 catches via Assert(i >= 0 && i < natts) inside TupleDescAttr(). The downstream caller (ErrorIfSchemasDoNotMatch) still produces the user-facing "Schema mismatch ..." error from the now-shorter list. 2. pg_lake_table: add PVC_RECURSE_AGGREGATES | PVC_RECURSE_PLACEHOLDERS to the second pull_var_clause() site (data_file_pruning.c ColumnsUsedInRestrictions) so it descends into Aggref / PlaceHolderVar instead of elog'ing "Aggref found where not expected" on PG19's TPC-H Q21 / TPC-DS Q10 plans. This mirrors the equivalent fix in restriction_collector.c from the previous batch. 3. pg_lake_copy: revert the JSON STDIN/STDOUT deferral added in round 2. The PG18 build does not recognize FORMAT json at the parser level, so handing JSON COPY off to core there breaks pg_lake_copy's own JSON tests. PG19's `installcheck-postgres` copy.out diff (driven by core's new force_array / on_error / encoding / reject_limit tests) is left unfixed in this PR; it is a known-failing leg until we either align pg_lake_copy's option validation with core or carry an alternative expected file for the upstream copy test. Signed-off-by: David Christensen --- pg_lake_copy/src/copy/copy.c | 16 ---------------- pg_lake_table/src/fdw/data_file_pruning.c | 10 +++++++++- .../fdw/schema_operations/register_field_ids.c | 15 ++++++++++----- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/pg_lake_copy/src/copy/copy.c b/pg_lake_copy/src/copy/copy.c index 0d7784534..7aeb5e418 100644 --- a/pg_lake_copy/src/copy/copy.c +++ b/pg_lake_copy/src/copy/copy.c @@ -248,22 +248,6 @@ IsPgLakeCopy(CopyStmt *copyStmt) return false; } -#if PG_VERSION_NUM >= 180000 - - /* - * PostgreSQL 18 added native support for COPY ... TO STDOUT (FORMAT - * json), along with format-specific options like force_array, on_error, - * and reject_limit. Let core handle plain STDIN/STDOUT JSON COPY so we - * don't intercept upstream tests with our own option-validation errors. - * We still claim the JSON path when the destination is a URL or @STAGE. - */ - if (format == DATA_FORMAT_JSON && - (IsCopyFromStdin(copyStmt) || IsCopyToStdout(copyStmt))) - { - return false; - } -#endif - return true; } diff --git a/pg_lake_table/src/fdw/data_file_pruning.c b/pg_lake_table/src/fdw/data_file_pruning.c index b4cde17bc..bd287d295 100644 --- a/pg_lake_table/src/fdw/data_file_pruning.c +++ b/pg_lake_table/src/fdw/data_file_pruning.c @@ -1568,7 +1568,15 @@ ColumnsUsedInRestrictions(Oid relationId, List *baseRestrictInfoList) RestrictInfo *restrictInfo = lfirst(restrictCell); Node *qual = (Node *) restrictInfo->clause; - List *varForQual = pull_var_clause(qual, PVC_INCLUDE_PLACEHOLDERS); + /* + * Recurse into Aggref/PlaceHolderVar so we still see the underlying + * Vars used in any pushed-up aggregate or PHV; PG19 can leave such + * nodes inside baserestrictinfo clauses where earlier majors did not, + * and pull_var_clause without a recurse flag elog()s on them. + */ + List *varForQual = pull_var_clause(qual, + PVC_RECURSE_AGGREGATES | + PVC_RECURSE_PLACEHOLDERS); ListCell *varCell = NULL; diff --git a/pg_lake_table/src/fdw/schema_operations/register_field_ids.c b/pg_lake_table/src/fdw/schema_operations/register_field_ids.c index ec6d00a51..86262216f 100644 --- a/pg_lake_table/src/fdw/schema_operations/register_field_ids.c +++ b/pg_lake_table/src/fdw/schema_operations/register_field_ids.c @@ -265,16 +265,21 @@ CreatePostgresColumnMappingsForIcebergTableFromExternalMetadata(Oid relationId) columnMapping->field = field; columnMapping->attrNum = get_attnum(relationId, field->name); - if (icebergCatalogType == REST_CATALOG_READ_ONLY && columnMapping->attrNum == InvalidAttrNumber) + + /* + * If no Postgres column exists for this Iceberg field, skip the + * mapping. The downstream caller (ErrorIfSchemasDoNotMatch) compares + * the resulting list length against the Iceberg schema and reports + * the mismatch with a clearer error than indexing into TupleDesc with + * an InvalidAttrNumber - 1 = -1, which on PG19 trips the new Assert(i + * >= 0 && i < natts) inside TupleDescAttr(). + */ + if (columnMapping->attrNum == InvalidAttrNumber) { - /* - * If no such column exists, skip. - */ continue; } columnMapping->attname = pstrdup(field->name); - columnMapping->attrNum = get_attnum(relationId, field->name); Form_pg_attribute attr = TupleDescAttr(tupDesc, columnMapping->attrNum - 1); columnMapping->pgType = MakePGType(attr->atttypid, attr->atttypmod); From fe47a9590cc3ad71e98041c3237f1d621b6410ac Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 12 Jun 2026 02:54:21 +0000 Subject: [PATCH 11/11] pg19: round 4 of code-level fixes Two regressions remained after round 3: 1. The schema-mismatch fix in CreatePostgresColumnMappingsForIcebergTableFromExternalMetadata skipped Iceberg fields with no Postgres counterpart, which made the downstream check fail with "field count 2 vs 1" instead of the expected "field ids 2 vs 2" message and broke test_object_store_catalog on PG16/17/18 too. Restore the original behavior of always pushing a PostgresColumnMapping but guard the TupleDescAttr() lookup so we only touch the descriptor when we have a valid attrNum. The mapping's pgType / attNotNull / attHasDef stay zero-initialised in the missing case, which still trips the per-field check in ErrorIfSchemasDoNotMatch() and surfaces the expected diagnostic without indexing the TupleDesc with -1 (which PG19 catches via the new Assert(i >= 0 && i < natts) inside TupleDescAttr()). 2. PG19's planner can leave Aggrefs inside the FDW's reltarget exprs and EPQ pathtarget exprs on TPC-H Q21 / TPC-DS Q10 plans, so add PVC_RECURSE_AGGREGATES to the three remaining pull_var_clause sites that previously only set PVC_INCLUDE/RECURSE_PLACEHOLDERS: - semijoin_target_ok (joinrel->reltarget->exprs) - postgresGetForeignSortPaths EPQ rewrite (target->exprs and fpinfo->local_conds) - build_tlist_to_deparse (foreignrel->reltarget->exprs and fpinfo->local_conds) Together with the data_file_pruning + restriction_collector fixes from round 2/3, this clears the "Aggref found where not expected" elog on benchmark queries. Signed-off-by: David Christensen --- pg_lake_table/src/fdw/deparse.c | 6 ++- pg_lake_table/src/fdw/pg_lake_table.c | 21 ++++++++-- .../schema_operations/register_field_ids.c | 40 ++++++++++++------- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/pg_lake_table/src/fdw/deparse.c b/pg_lake_table/src/fdw/deparse.c index a26d3b5e1..4f9325f6f 100644 --- a/pg_lake_table/src/fdw/deparse.c +++ b/pg_lake_table/src/fdw/deparse.c @@ -1262,14 +1262,16 @@ build_tlist_to_deparse(RelOptInfo *foreignrel) */ tlist = add_to_flat_tlist(tlist, pull_var_clause((Node *) foreignrel->reltarget->exprs, - PVC_RECURSE_PLACEHOLDERS)); + PVC_RECURSE_PLACEHOLDERS | + PVC_RECURSE_AGGREGATES)); foreach(lc, fpinfo->local_conds) { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); tlist = add_to_flat_tlist(tlist, pull_var_clause((Node *) rinfo->clause, - PVC_RECURSE_PLACEHOLDERS)); + PVC_RECURSE_PLACEHOLDERS | + PVC_RECURSE_AGGREGATES)); } return tlist; diff --git a/pg_lake_table/src/fdw/pg_lake_table.c b/pg_lake_table/src/fdw/pg_lake_table.c index d995f6e94..3182ed675 100644 --- a/pg_lake_table/src/fdw/pg_lake_table.c +++ b/pg_lake_table/src/fdw/pg_lake_table.c @@ -3925,7 +3925,14 @@ semijoin_target_ok(PlannerInfo *root, RelOptInfo *joinrel, RelOptInfo *outerrel, Assert(joinrel->reltarget); - vars = pull_var_clause((Node *) joinrel->reltarget->exprs, PVC_INCLUDE_PLACEHOLDERS); + /* + * Recurse into Aggref/PlaceHolderVar so we still see the underlying Vars + * and don't elog on the new PG19 join paths that can place an Aggref + * inside the target exprs of a SEMI join (e.g. TPC-H Q21). + */ + vars = pull_var_clause((Node *) joinrel->reltarget->exprs, + PVC_INCLUDE_PLACEHOLDERS | + PVC_RECURSE_AGGREGATES); foreach(lc, vars) { @@ -4314,10 +4321,15 @@ add_paths_with_pathkeys_for_rel(PlannerInfo *root, RelOptInfo *rel, PgLakeRelationInfo *fpinfo = (PgLakeRelationInfo *) rel->fdw_private; PathTarget *target = copy_pathtarget(epq_path->pathtarget); - /* Include columns required for evaluating PHVs in the tlist. */ + /* + * Include columns required for evaluating PHVs and (PG19) any Aggrefs + * left in the tlist; without PVC_RECURSE_AGGREGATES, pull_var_clause + * elog()s on TPC-H Q21 / TPC-DS Q10 EPQ paths. + */ add_new_columns_to_pathtarget(target, pull_var_clause((Node *) target->exprs, - PVC_RECURSE_PLACEHOLDERS)); + PVC_RECURSE_PLACEHOLDERS | + PVC_RECURSE_AGGREGATES)); /* Include columns required for evaluating the local conditions. */ foreach(lc, fpinfo->local_conds) @@ -4326,7 +4338,8 @@ add_paths_with_pathkeys_for_rel(PlannerInfo *root, RelOptInfo *rel, add_new_columns_to_pathtarget(target, pull_var_clause((Node *) rinfo->clause, - PVC_RECURSE_PLACEHOLDERS)); + PVC_RECURSE_PLACEHOLDERS | + PVC_RECURSE_AGGREGATES)); } /* diff --git a/pg_lake_table/src/fdw/schema_operations/register_field_ids.c b/pg_lake_table/src/fdw/schema_operations/register_field_ids.c index 86262216f..7c3bf2c03 100644 --- a/pg_lake_table/src/fdw/schema_operations/register_field_ids.c +++ b/pg_lake_table/src/fdw/schema_operations/register_field_ids.c @@ -265,26 +265,38 @@ CreatePostgresColumnMappingsForIcebergTableFromExternalMetadata(Oid relationId) columnMapping->field = field; columnMapping->attrNum = get_attnum(relationId, field->name); - - /* - * If no Postgres column exists for this Iceberg field, skip the - * mapping. The downstream caller (ErrorIfSchemasDoNotMatch) compares - * the resulting list length against the Iceberg schema and reports - * the mismatch with a clearer error than indexing into TupleDesc with - * an InvalidAttrNumber - 1 = -1, which on PG19 trips the new Assert(i - * >= 0 && i < natts) inside TupleDescAttr(). - */ - if (columnMapping->attrNum == InvalidAttrNumber) + if (icebergCatalogType == REST_CATALOG_READ_ONLY && columnMapping->attrNum == InvalidAttrNumber) { + /* + * If no such column exists, skip. + */ continue; } columnMapping->attname = pstrdup(field->name); - Form_pg_attribute attr = TupleDescAttr(tupDesc, columnMapping->attrNum - 1); + columnMapping->attrNum = get_attnum(relationId, field->name); + + if (columnMapping->attrNum != InvalidAttrNumber) + { + Form_pg_attribute attr = TupleDescAttr(tupDesc, columnMapping->attrNum - 1); - columnMapping->pgType = MakePGType(attr->atttypid, attr->atttypmod); - columnMapping->attNotNull = attr->attnotnull; - columnMapping->attHasDef = attr->atthasdef; + columnMapping->pgType = MakePGType(attr->atttypid, attr->atttypmod); + columnMapping->attNotNull = attr->attnotnull; + columnMapping->attHasDef = attr->atthasdef; + } + else + { + /* + * No Postgres column matches this Iceberg field name (e.g. an + * Iceberg-side ALTER added a column that the read-only Postgres + * mirror has not yet picked up). Leave the type/null/default + * fields zero-initialised so the downstream per-field mismatch + * check in ErrorIfSchemasDoNotMatch() fires with the precise + * "field ids X vs Y" diagnostic, instead of indexing TupleDesc + * with InvalidAttrNumber - 1 = -1, which on PG19 trips the new + * Assert(i >= 0 && i < natts) inside TupleDescAttr(). + */ + } pgColumnMappingList = lappend(pgColumnMappingList, columnMapping); }