diff --git a/.abi-compliance-history b/.abi-compliance-history new file mode 100644 index 0000000000000..e4a7770611a6e --- /dev/null +++ b/.abi-compliance-history @@ -0,0 +1,57 @@ +# Reference point for ABI compliance checks +# +# This file lists commits on the current branch that break ABI compatibility in +# ways that have been deemed acceptable (e.g., removing an extern function with +# no third-party uses). The primary intent of this file is to control the ABI +# compliance checks on the buildfarm, but it also serves as a central location +# to document the justification for each. +# +# In general, entries should be added reactively after an abi-compliance-check +# buildfarm failure. It is important to verify the details of the breakage +# match expectations, as the first entry listed will become the updated ABI +# baseline point. +# +# Add new entries by adding the output of the following to the top of the file: +# +# $ git log --pretty=format:"%H%n#%n# %s%n# %cd%n#%n# " $ABIBREAKGITHASH -1 --date=iso +# +# Be sure to replace "" with details of your change and +# why it is deemed acceptable. + +c5824536e7ca3c5ef5a9db59bd36c4e6beb23f51 +# +# Fix trigger transition table capture for MERGE in CTE queries. +# 2026-01-24 11:30:51 +0000 +# +# This commit changed the TransitionCaptureState structure, replacing +# the "tcs_private" field with 3 separate fields. This structure can +# only be built using MakeTransitionCaptureState(), and PGXN contained +# no calls to MakeTransitionCaptureState() or uses of the +# TransitionCaptureState structure. + +05d605b6c69f0276ae091ced1ed9082963052550 +# +# For inplace update, send nontransactional invalidations. +# 2025-12-16 16:13:55 -0800 +# +# This changed PrepareToInvalidateCacheTuple() parameters. PGXN contained no +# calls to that function. + +2393d374ae9c0bc8327adc80fe4490edb05be167 +# +# Check for CREATE privilege on the schema in CREATE STATISTICS. +# 2025-11-10 09:00:00 -0600 +# +# This commit added a parameter to CreateStatistics(). We are unaware of any +# impacted third-party code. + +fc0fb77c550fe8289d718c33c7aacf16023d9941 +# +# Fix re-distributing previously distributed invalidation messages during logical decoding. +# 2025-06-16 17:35:53 -0700 +# +# This is the original ABI baseline point for REL_15_STABLE. The first entry +# would ordinarily point to something just before the .0 release, but this file +# was first added in October 2025, and we're unlikely to act upon ABI breaks in +# released minor versions, so we've chosen to truncate the ABI history to start +# with the most recent ABI break documented in the git commit history. diff --git a/.cirrus.star b/.cirrus.star index d2d6ceca2075d..8e52e64cc5494 100644 --- a/.cirrus.star +++ b/.cirrus.star @@ -7,7 +7,7 @@ https://github.com/bazelbuild/starlark/blob/master/spec.md See also .cirrus.yml and src/tools/ci/README """ -load("cirrus", "env", "fs") +load("cirrus", "env", "fs", "re", "yaml") def main(): @@ -18,19 +18,36 @@ def main(): 1) the contents of .cirrus.yml - 2) if defined, the contents of the file referenced by the, repository + 2) computed environment variables + + 3) if defined, the contents of the file referenced by the, repository level, REPO_CI_CONFIG_GIT_URL variable (see https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted format) - 3) .cirrus.tasks.yml + 4) .cirrus.tasks.yml """ output = "" # 1) is evaluated implicitly + # Add 2) + additional_env = compute_environment_vars() + env_fmt = """ +### +# Computed environment variables start here +### +{0} +### +# Computed environment variables end here +### +""" + output += env_fmt.format(yaml.dumps({'env': additional_env})) + + + # Add 3) repo_config_url = env.get("REPO_CI_CONFIG_GIT_URL") if repo_config_url != None: print("loading additional configuration from \"{}\"".format(repo_config_url)) @@ -38,12 +55,65 @@ def main(): else: output += "\n# REPO_CI_CONFIG_URL was not set\n" - # Add 3) + + # Add 4) output += config_from(".cirrus.tasks.yml") + return output +def compute_environment_vars(): + cenv = {} + + ### + # Some tasks are manually triggered by default because they might use too + # many resources for users of free Cirrus credits, but they can be + # triggered automatically by naming them in an environment variable e.g. + # REPO_CI_AUTOMATIC_TRIGGER_TASKS="task_name other_task" under "Repository + # Settings" on Cirrus CI's website. + + default_manual_trigger_tasks = [] + + repo_ci_automatic_trigger_tasks = env.get('REPO_CI_AUTOMATIC_TRIGGER_TASKS', '') + for task in default_manual_trigger_tasks: + name = 'CI_TRIGGER_TYPE_' + task.upper() + if repo_ci_automatic_trigger_tasks.find(task) != -1: + value = 'automatic' + else: + value = 'manual' + cenv[name] = value + ### + + ### + # Parse "ci-os-only:" tag in commit message and set + # CI_{$OS}_ENABLED variable for each OS + + operating_systems = [ + 'freebsd', + 'linux', + 'macos', + 'windows', + ] + commit_message = env.get('CIRRUS_CHANGE_MESSAGE') + match_re = r"(^|.*\n)ci-os-only: ([^\n]+)($|\n.*)" + + # re.match() returns an array with a tuple of (matched-string, match_1, ...) + m = re.match(match_re, commit_message) + if m and len(m) > 0: + os_only = m[0][2] + os_only_list = re.split(r'[, ]+', os_only) + else: + os_only_list = operating_systems + + for os in operating_systems: + os_enabled = os in os_only_list + cenv['CI_{0}_ENABLED'.format(os.upper())] = os_enabled + ### + + return cenv + + def config_from(config_src): """return contents of config file `config_src`, surrounded by markers indicating start / end of the the included file diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 3d344327452df..ea11ad377ead1 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -32,7 +32,7 @@ on_failure: &on_failure type: text/plain task: - name: FreeBSD - 13 + name: FreeBSD env: # FreeBSD on GCP is slow when running with larger number of CPUS / @@ -40,14 +40,14 @@ task: CPUS: 2 BUILD_JOBS: 3 TEST_JOBS: 3 - IMAGE_FAMILY: pg-ci-freebsd-13 + IMAGE_FAMILY: pg-ci-freebsd DISK_SIZE: 50 CCACHE_DIR: /tmp/ccache_dir <<: *freebsd_task_template - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*' + only_if: $CI_FREEBSD_ENABLED sysinfo_script: | id @@ -76,6 +76,7 @@ task: # freebsd already takes longer than other platforms except for windows. configure_script: | su postgres <<-EOF + set -e ./configure \ --enable-cassert --enable-debug --enable-tap-tests \ --enable-nls \ @@ -138,13 +139,13 @@ LINUX_CONFIGURE_FEATURES: &LINUX_CONFIGURE_FEATURES >- task: - name: Linux - Debian Bookworm + name: Linux - Debian Trixie env: CPUS: 4 BUILD_JOBS: 4 TEST_JOBS: 8 # experimentally derived to be a decent choice - IMAGE_FAMILY: pg-ci-bookworm + IMAGE_FAMILY: pg-ci-trixie CCACHE_DIR: /tmp/ccache_dir DEBUGINFOD_URLS: "https://debuginfod.debian.net" @@ -153,7 +154,7 @@ task: <<: *linux_task_template - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + only_if: $CI_LINUX_ENABLED ccache_cache: folder: ${CCACHE_DIR} @@ -181,6 +182,7 @@ task: configure_script: | su postgres <<-EOF + set -e ./configure \ --enable-cassert --enable-debug --enable-tap-tests \ --enable-nls \ @@ -198,6 +200,7 @@ task: test_world_script: | su postgres <<-EOF + set -e ulimit -c unlimited # default is 0 make -s ${CHECK} ${CHECKFLAGS} -j${TEST_JOBS} EOF @@ -208,16 +211,16 @@ task: task: - name: macOS - Sonoma + name: macOS - Sequoia env: CPUS: 4 # always get that much for cirrusci macOS instances BUILD_JOBS: $CPUS - # Test performance regresses noticably when using all cores. 8 seems to + # Test performance regresses noticeably when using all cores. 8 seems to # work OK. See # https://postgr.es/m/20220927040208.l3shfcidovpzqxfh%40awork3.anarazel.de TEST_JOBS: 8 - IMAGE: ghcr.io/cirruslabs/macos-runner:sonoma + IMAGE: ghcr.io/cirruslabs/macos-runner:sequoia CIRRUS_WORKING_DIR: ${HOME}/pgsql/ CCACHE_DIR: ${HOME}/ccache @@ -239,7 +242,7 @@ task: <<: *macos_task_template - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*' + only_if: $CI_MACOS_ENABLED sysinfo_script: | id @@ -328,7 +331,7 @@ task: task: - name: Windows - Server 2019, VS 2019 + name: Windows - Server 2022, VS 2019 env: # Half the allowed per-user CPU cores @@ -389,7 +392,7 @@ task: <<: *windows_task_template - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*' + only_if: $CI_WINDOWS_ENABLED sysinfo_script: | chcp @@ -460,12 +463,12 @@ task: # To limit unnecessary work only run this once the normal linux test succeeds depends_on: - - Linux - Debian Bookworm + - Linux - Debian Trixie env: CPUS: 4 BUILD_JOBS: 4 - IMAGE_FAMILY: pg-ci-bookworm + IMAGE_FAMILY: pg-ci-trixie # Use larger ccache cache, as this task compiles with multiple compilers / # flag combinations @@ -476,7 +479,7 @@ task: # task that did not run, count as a success, so we need to recheck Linux' # condition here ... - only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*' + only_if: $CI_LINUX_ENABLED <<: *linux_task_template @@ -554,10 +557,10 @@ task: always: mingw_cross_warning_script: | time ./configure \ - --host=x86_64-w64-mingw32 \ + --host=x86_64-w64-mingw32ucrt \ --enable-cassert \ - CC="ccache x86_64-w64-mingw32-gcc" \ - CXX="ccache x86_64-w64-mingw32-g++" + CC="ccache x86_64-w64-mingw32ucrt-gcc" \ + CXX="ccache x86_64-w64-mingw32ucrt-g++" make -s -j${BUILD_JOBS} clean time make -s -j${BUILD_JOBS} world-bin @@ -581,8 +584,6 @@ task: # - Don't use ccache, the files are uncacheable, polluting ccache's # cache # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose - # - XXX have to disable ICU to avoid errors: - # https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de # - XXX: the -Wno-register avoids verbose warnings: # https://postgr.es/m/20220308181837.aun3tdtdvao4vb7o%40alap3.anarazel.de ### @@ -590,7 +591,6 @@ task: headers_headerscheck_script: | time ./configure \ ${LINUX_CONFIGURE_FEATURES} \ - --without-icu \ --quiet \ CC="gcc" CXX"=g++" CLANG="clang" make -s -j${BUILD_JOBS} clean diff --git a/.cirrus.yml b/.cirrus.yml index a83129ae46d1e..f270f61241f81 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -10,12 +10,20 @@ # # 1) the contents of this file # -# 2) if defined, the contents of the file referenced by the, repository +# 2) computed environment variables +# +# Used to enable/disable tasks based on the execution environment. See +# .cirrus.star: compute_environment_vars() +# +# 3) if defined, the contents of the file referenced by the, repository # level, REPO_CI_CONFIG_GIT_URL variable (see # https://cirrus-ci.org/guide/programming-tasks/#fs for the accepted # format) # -# 3) .cirrus.tasks.yml +# This allows running tasks in a different execution environment than the +# default, e.g. to have sufficient resources for cfbot. +# +# 4) .cirrus.tasks.yml # # This composition is done by .cirrus.star diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index f6c997ddf5d5f..b022c7b842dc0 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -14,6 +14,9 @@ # # $ git log --pretty=format:"%H # %cd%n# %s" $PGINDENTGITHASH -1 --date=iso +f9790ac5464da45eb44470ab831584356381597b # 2025-10-21 09:56:26 -0500 +# Re-pgindent brin.c. + 82d0ffae3219e4bc153a1306ce23013d168e04a2 # 2022-06-30 11:03:03 -0400 # pgindent run prior to branching v15. diff --git a/.gitattributes b/.gitattributes index 63178f9170020..5b8bc6a5e82e2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,8 +7,8 @@ *.x[ms]l whitespace=space-before-tab,trailing-space,tab-in-indent # Avoid confusing ASCII underlines with leftover merge conflict markers -README conflict-marker-size=32 -README.* conflict-marker-size=32 +README conflict-marker-size=48 +README.* conflict-marker-size=48 # Certain data files that contain special whitespace, and other special cases *.data -whitespace diff --git a/COPYRIGHT b/COPYRIGHT index be2d694b03854..0a397648dcd3c 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -1,7 +1,7 @@ PostgreSQL Database Management System -(formerly known as Postgres, then as Postgres95) +(also known as Postgres, formerly known as Postgres95) -Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group +Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California diff --git a/config/llvm.m4 b/config/llvm.m4 index 21d8cd4f90f97..93fa9e070e89d 100644 --- a/config/llvm.m4 +++ b/config/llvm.m4 @@ -4,7 +4,7 @@ # ----------------- # # Look for the LLVM installation, check that it's new enough, set the -# corresponding LLVM_{CFLAGS,CXXFLAGS,BINPATH} and LDFLAGS +# corresponding LLVM_{CFLAGS,CXXFLAGS,BINPATH,LIBS} # variables. Also verify that CLANG is available, to transform C # into bitcode. # @@ -55,7 +55,7 @@ AC_DEFUN([PGAC_LLVM_SUPPORT], for pgac_option in `$LLVM_CONFIG --ldflags`; do case $pgac_option in - -L*) LDFLAGS="$LDFLAGS $pgac_option";; + -L*) LLVM_LIBS="$LLVM_LIBS $pgac_option";; esac done diff --git a/configure b/configure index a52cef20f39ff..fb4fae74700b9 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for PostgreSQL 15.12. +# Generated by GNU Autoconf 2.69 for PostgreSQL 15.17. # # Report bugs to . # @@ -582,8 +582,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='PostgreSQL' PACKAGE_TARNAME='postgresql' -PACKAGE_VERSION='15.12' -PACKAGE_STRING='PostgreSQL 15.12' +PACKAGE_VERSION='15.17' +PACKAGE_STRING='PostgreSQL 15.17' PACKAGE_BUGREPORT='pgsql-bugs@lists.postgresql.org' PACKAGE_URL='https://www.postgresql.org/' @@ -1452,7 +1452,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures PostgreSQL 15.12 to adapt to many kinds of systems. +\`configure' configures PostgreSQL 15.17 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1517,7 +1517,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of PostgreSQL 15.12:";; + short | recursive ) echo "Configuration of PostgreSQL 15.17:";; esac cat <<\_ACEOF @@ -1691,7 +1691,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -PostgreSQL configure 15.12 +PostgreSQL configure 15.17 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2444,7 +2444,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by PostgreSQL $as_me 15.12, which was +It was created by PostgreSQL $as_me 15.17, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -5173,7 +5173,7 @@ fi for pgac_option in `$LLVM_CONFIG --ldflags`; do case $pgac_option in - -L*) LDFLAGS="$LDFLAGS $pgac_option";; + -L*) LLVM_LIBS="$LLVM_LIBS $pgac_option";; esac done @@ -9218,12 +9218,12 @@ fi # Note the user could also set XML2_CFLAGS/XML2_LIBS directly for pgac_option in $XML2_CFLAGS; do case $pgac_option in - -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";; + -I*|-D*) INCLUDES="$INCLUDES $pgac_option";; esac done for pgac_option in $XML2_LIBS; do case $pgac_option in - -L*) LDFLAGS="$LDFLAGS $pgac_option";; + -L*) LIBDIRS="$LIBDIRS $pgac_option";; esac done fi @@ -9448,12 +9448,12 @@ fi # note that -llz4 will be added by AC_CHECK_LIB below. for pgac_option in $LZ4_CFLAGS; do case $pgac_option in - -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";; + -I*|-D*) INCLUDES="$INCLUDES $pgac_option";; esac done for pgac_option in $LZ4_LIBS; do case $pgac_option in - -L*) LDFLAGS="$LDFLAGS $pgac_option";; + -L*) LIBDIRS="$LIBDIRS $pgac_option";; esac done fi @@ -9589,12 +9589,12 @@ fi # note that -lzstd will be added by AC_CHECK_LIB below. for pgac_option in $ZSTD_CFLAGS; do case $pgac_option in - -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";; + -I*|-D*) INCLUDES="$INCLUDES $pgac_option";; esac done for pgac_option in $ZSTD_LIBS; do case $pgac_option in - -L*) LDFLAGS="$LDFLAGS $pgac_option";; + -L*) LIBDIRS="$LIBDIRS $pgac_option";; esac done fi @@ -13448,13 +13448,15 @@ else fi fi - # Function introduced in OpenSSL 1.0.2. - for ac_func in X509_get_signature_nid + # Functions introduced in OpenSSL 1.0.2. LibreSSL does not have + # SSL_CTX_set_cert_cb(). + for ac_func in X509_get_signature_nid SSL_CTX_set_cert_cb do : - ac_fn_c_check_func "$LINENO" "X509_get_signature_nid" "ac_cv_func_X509_get_signature_nid" -if test "x$ac_cv_func_X509_get_signature_nid" = xyes; then : + as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` +ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" +if eval test \"x\$"$as_ac_var"\" = x"yes"; then : cat >>confdefs.h <<_ACEOF -#define HAVE_X509_GET_SIGNATURE_NID 1 +#define `$as_echo "HAVE_$ac_func" | $as_tr_cpp` 1 _ACEOF fi @@ -16308,7 +16310,7 @@ fi LIBS_including_readline="$LIBS" LIBS=`echo "$LIBS" | sed -e 's/-ledit//g' -e 's/-lreadline//g'` -for ac_func in backtrace_symbols clock_gettime copyfile fdatasync getifaddrs getpeerucred getrlimit inet_pton kqueue mbstowcs_l memset_s poll posix_fallocate ppoll pstat pthread_is_threaded_np readlink readv setproctitle setproctitle_fast setsid shm_open strsignal symlink syncfs sync_file_range uselocale wcstombs_l writev +for ac_func in backtrace_symbols clock_gettime copyfile fdatasync getifaddrs getpeerucred getrlimit inet_pton kqueue mbstowcs_l poll posix_fallocate ppoll pstat pthread_is_threaded_np readlink readv setproctitle setproctitle_fast setsid shm_open strsignal symlink syncfs sync_file_range uselocale wcstombs_l writev do : as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh` ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var" @@ -16942,6 +16944,19 @@ cat >>confdefs.h <<_ACEOF #define HAVE_DECL_STRCHRNUL $ac_have_decl _ACEOF +ac_fn_c_check_decl "$LINENO" "memset_s" "ac_cv_have_decl_memset_s" "#define __STDC_WANT_LIB_EXT1__ 1 +#include +" +if test "x$ac_cv_have_decl_memset_s" = xyes; then : + ac_have_decl=1 +else + ac_have_decl=0 +fi + +cat >>confdefs.h <<_ACEOF +#define HAVE_DECL_MEMSET_S $ac_have_decl +_ACEOF + # This is probably only present on macOS, but may as well check always ac_fn_c_check_decl "$LINENO" "F_FULLFSYNC" "ac_cv_have_decl_F_FULLFSYNC" "#include @@ -17639,7 +17654,7 @@ _ACEOF if test "$with_icu" = yes; then ac_save_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$ICU_CFLAGS $CPPFLAGS" + CPPFLAGS="$CPPFLAGS $ICU_CFLAGS" # Verify we have ICU's header files ac_fn_c_check_header_mongrel "$LINENO" "unicode/ucol.h" "ac_cv_header_unicode_ucol_h" "$ac_includes_default" @@ -18813,7 +18828,7 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi -# Check for x86 cpuid instruction +# Check for __get_cpuid() and __cpuid() { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __get_cpuid" >&5 $as_echo_n "checking for __get_cpuid... " >&6; } if ${pgac_cv__get_cpuid+:} false; then : @@ -18846,9 +18861,9 @@ if test x"$pgac_cv__get_cpuid" = x"yes"; then $as_echo "#define HAVE__GET_CPUID 1" >>confdefs.h -fi - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for __cpuid" >&5 +else + # __cpuid() + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __cpuid" >&5 $as_echo_n "checking for __cpuid... " >&6; } if ${pgac_cv__cpuid+:} false; then : $as_echo_n "(cached) " >&6 @@ -18860,7 +18875,7 @@ int main () { unsigned int exx[4] = {0, 0, 0, 0}; - __get_cpuid(exx[0], 1); + __cpuid(exx, 1); ; return 0; @@ -18876,10 +18891,11 @@ rm -f core conftest.err conftest.$ac_objext \ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv__cpuid" >&5 $as_echo "$pgac_cv__cpuid" >&6; } -if test x"$pgac_cv__cpuid" = x"yes"; then + if test x"$pgac_cv__cpuid" = x"yes"; then $as_echo "#define HAVE__CPUID 1" >>confdefs.h + fi fi # Check for Intel SSE 4.2 intrinsics to do CRC calculations. @@ -19792,7 +19808,7 @@ Use --without-tcl to disable building PL/Tcl." "$LINENO" 5 fi # now that we have TCL_INCLUDE_SPEC, we can check for ac_save_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$TCL_INCLUDE_SPEC $CPPFLAGS" + CPPFLAGS="$CPPFLAGS $TCL_INCLUDE_SPEC" ac_fn_c_check_header_mongrel "$LINENO" "tcl.h" "ac_cv_header_tcl_h" "$ac_includes_default" if test "x$ac_cv_header_tcl_h" = xyes; then : @@ -19861,7 +19877,7 @@ fi # check for if test "$with_python" = yes; then ac_save_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$python_includespec $CPPFLAGS" + CPPFLAGS="$CPPFLAGS $python_includespec" ac_fn_c_check_header_mongrel "$LINENO" "Python.h" "ac_cv_header_Python_h" "$ac_includes_default" if test "x$ac_cv_header_Python_h" = xyes; then : @@ -20916,7 +20932,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by PostgreSQL $as_me 15.12, which was +This file was extended by PostgreSQL $as_me 15.17, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -20987,7 +21003,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -PostgreSQL config.status 15.12 +PostgreSQL config.status 15.17 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 93dd05ddeba81..928495a8ff2c5 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ dnl Read the Autoconf manual for details. dnl m4_pattern_forbid(^PGAC_)dnl to catch undefined macros -AC_INIT([PostgreSQL], [15.12], [pgsql-bugs@lists.postgresql.org], [], [https://www.postgresql.org/]) +AC_INIT([PostgreSQL], [15.17], [pgsql-bugs@lists.postgresql.org], [], [https://www.postgresql.org/]) m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required. Untested combinations of 'autoconf' and PostgreSQL versions are not @@ -1052,12 +1052,12 @@ if test "$with_libxml" = yes ; then # Note the user could also set XML2_CFLAGS/XML2_LIBS directly for pgac_option in $XML2_CFLAGS; do case $pgac_option in - -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";; + -I*|-D*) INCLUDES="$INCLUDES $pgac_option";; esac done for pgac_option in $XML2_LIBS; do case $pgac_option in - -L*) LDFLAGS="$LDFLAGS $pgac_option";; + -L*) LIBDIRS="$LIBDIRS $pgac_option";; esac done fi @@ -1101,12 +1101,12 @@ if test "$with_lz4" = yes; then # note that -llz4 will be added by AC_CHECK_LIB below. for pgac_option in $LZ4_CFLAGS; do case $pgac_option in - -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";; + -I*|-D*) INCLUDES="$INCLUDES $pgac_option";; esac done for pgac_option in $LZ4_LIBS; do case $pgac_option in - -L*) LDFLAGS="$LDFLAGS $pgac_option";; + -L*) LIBDIRS="$LIBDIRS $pgac_option";; esac done fi @@ -1126,12 +1126,12 @@ if test "$with_zstd" = yes; then # note that -lzstd will be added by AC_CHECK_LIB below. for pgac_option in $ZSTD_CFLAGS; do case $pgac_option in - -I*|-D*) CPPFLAGS="$CPPFLAGS $pgac_option";; + -I*|-D*) INCLUDES="$INCLUDES $pgac_option";; esac done for pgac_option in $ZSTD_LIBS; do case $pgac_option in - -L*) LDFLAGS="$LDFLAGS $pgac_option";; + -L*) LIBDIRS="$LIBDIRS $pgac_option";; esac done fi @@ -1373,8 +1373,9 @@ if test "$with_ssl" = openssl ; then AC_SEARCH_LIBS(CRYPTO_new_ex_data, [eay32 crypto], [], [AC_MSG_ERROR([library 'eay32' or 'crypto' is required for OpenSSL])]) AC_SEARCH_LIBS(SSL_new, [ssleay32 ssl], [], [AC_MSG_ERROR([library 'ssleay32' or 'ssl' is required for OpenSSL])]) fi - # Function introduced in OpenSSL 1.0.2. - AC_CHECK_FUNCS([X509_get_signature_nid]) + # Functions introduced in OpenSSL 1.0.2. LibreSSL does not have + # SSL_CTX_set_cert_cb(). + AC_CHECK_FUNCS([X509_get_signature_nid SSL_CTX_set_cert_cb]) # Functions introduced in OpenSSL 1.1.0. We used to check for # OPENSSL_VERSION_NUMBER, but that didn't work with 1.1.0, because LibreSSL # defines OPENSSL_VERSION_NUMBER to claim version 2.0.0, even though it @@ -1842,7 +1843,6 @@ AC_CHECK_FUNCS(m4_normalize([ inet_pton kqueue mbstowcs_l - memset_s poll posix_fallocate ppoll @@ -1923,6 +1923,8 @@ AC_CHECK_DECLS([strlcat, strlcpy, strnlen]) AC_CHECK_DECLS([preadv], [], [AC_LIBOBJ(preadv)], [#include ]) AC_CHECK_DECLS([pwritev], [], [AC_LIBOBJ(pwritev)], [#include ]) AC_CHECK_DECLS([strchrnul], [], [], [#include ]) +AC_CHECK_DECLS([memset_s], [], [], [#define __STDC_WANT_LIB_EXT1__ 1 +#include ]) # This is probably only present on macOS, but may as well check always AC_CHECK_DECLS(F_FULLFSYNC, [], [], [#include ]) @@ -2087,7 +2089,7 @@ AC_CHECK_DECLS([strtoll, strtoull]) if test "$with_icu" = yes; then ac_save_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$ICU_CFLAGS $CPPFLAGS" + CPPFLAGS="$CPPFLAGS $ICU_CFLAGS" # Verify we have ICU's header files AC_CHECK_HEADER(unicode/ucol.h, [], @@ -2219,7 +2221,7 @@ PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS -# Check for x86 cpuid instruction +# Check for __get_cpuid() and __cpuid() AC_CACHE_CHECK([for __get_cpuid], [pgac_cv__get_cpuid], [AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], [[unsigned int exx[4] = {0, 0, 0, 0}; @@ -2229,17 +2231,18 @@ AC_CACHE_CHECK([for __get_cpuid], [pgac_cv__get_cpuid], [pgac_cv__get_cpuid="no"])]) if test x"$pgac_cv__get_cpuid" = x"yes"; then AC_DEFINE(HAVE__GET_CPUID, 1, [Define to 1 if you have __get_cpuid.]) -fi - -AC_CACHE_CHECK([for __cpuid], [pgac_cv__cpuid], -[AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], - [[unsigned int exx[4] = {0, 0, 0, 0}; - __get_cpuid(exx[0], 1); - ]])], - [pgac_cv__cpuid="yes"], - [pgac_cv__cpuid="no"])]) -if test x"$pgac_cv__cpuid" = x"yes"; then - AC_DEFINE(HAVE__CPUID, 1, [Define to 1 if you have __cpuid.]) +else + # __cpuid() + AC_CACHE_CHECK([for __cpuid], [pgac_cv__cpuid], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [[unsigned int exx[4] = {0, 0, 0, 0}; + __cpuid(exx, 1); + ]])], + [pgac_cv__cpuid="yes"], + [pgac_cv__cpuid="no"])]) + if test x"$pgac_cv__cpuid" = x"yes"; then + AC_DEFINE(HAVE__CPUID, 1, [Define to 1 if you have __cpuid.]) + fi fi # Check for Intel SSE 4.2 intrinsics to do CRC calculations. @@ -2437,7 +2440,7 @@ Use --without-tcl to disable building PL/Tcl.]) fi # now that we have TCL_INCLUDE_SPEC, we can check for ac_save_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$TCL_INCLUDE_SPEC $CPPFLAGS" + CPPFLAGS="$CPPFLAGS $TCL_INCLUDE_SPEC" AC_CHECK_HEADER(tcl.h, [], [AC_MSG_ERROR([header file is required for Tcl])]) CPPFLAGS=$ac_save_CPPFLAGS fi @@ -2474,7 +2477,7 @@ fi # check for if test "$with_python" = yes; then ac_save_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$python_includespec $CPPFLAGS" + CPPFLAGS="$CPPFLAGS $python_includespec" AC_CHECK_HEADER(Python.h, [], [AC_MSG_ERROR([header file is required for Python])]) CPPFLAGS=$ac_save_CPPFLAGS fi diff --git a/contrib/adminpack/expected/adminpack.out b/contrib/adminpack/expected/adminpack.out index 76aafe6316a66..6bcf2217f2525 100644 --- a/contrib/adminpack/expected/adminpack.out +++ b/contrib/adminpack/expected/adminpack.out @@ -36,10 +36,10 @@ SELECT pg_read_file('test_file1'); -- disallowed file paths for non-superusers and users who are -- not members of pg_write_server_files -CREATE ROLE regress_user1; -GRANT pg_read_all_settings TO regress_user1; -GRANT EXECUTE ON FUNCTION pg_file_write(text,text,bool) TO regress_user1; -SET ROLE regress_user1; +CREATE ROLE regress_adminpack_user1; +GRANT pg_read_all_settings TO regress_adminpack_user1; +GRANT EXECUTE ON FUNCTION pg_file_write(text,text,bool) TO regress_adminpack_user1; +SET ROLE regress_adminpack_user1; SELECT pg_file_write('../test_file0', 'test0', false); ERROR: path must be in or below the current directory SELECT pg_file_write('/tmp/test_file0', 'test0', false); @@ -53,9 +53,9 @@ SELECT pg_file_write(current_setting('data_directory') || '/test_file4', 'test4' SELECT pg_file_write(current_setting('data_directory') || '/../test_file4', 'test4', false); ERROR: absolute path not allowed RESET ROLE; -REVOKE EXECUTE ON FUNCTION pg_file_write(text,text,bool) FROM regress_user1; -REVOKE pg_read_all_settings FROM regress_user1; -DROP ROLE regress_user1; +REVOKE EXECUTE ON FUNCTION pg_file_write(text,text,bool) FROM regress_adminpack_user1; +REVOKE pg_read_all_settings FROM regress_adminpack_user1; +DROP ROLE regress_adminpack_user1; -- sync SELECT pg_file_sync('test_file1'); -- sync file pg_file_sync @@ -153,8 +153,8 @@ SELECT pg_file_unlink('test_file4'); (1 row) -- superuser checks -CREATE USER regress_user1; -SET ROLE regress_user1; +CREATE USER regress_adminpack_user1; +SET ROLE regress_adminpack_user1; SELECT pg_file_write('test_file0', 'test0', false); ERROR: permission denied for function pg_file_write SELECT pg_file_sync('test_file0'); @@ -167,6 +167,6 @@ ERROR: permission denied for function pg_file_unlink SELECT pg_logdir_ls(); ERROR: permission denied for function pg_logdir_ls RESET ROLE; -DROP USER regress_user1; +DROP USER regress_adminpack_user1; -- no further tests for pg_logdir_ls() because it depends on the -- server's logging setup diff --git a/contrib/adminpack/sql/adminpack.sql b/contrib/adminpack/sql/adminpack.sql index 918d0bdc65ee9..5776c9af0d135 100644 --- a/contrib/adminpack/sql/adminpack.sql +++ b/contrib/adminpack/sql/adminpack.sql @@ -14,20 +14,20 @@ SELECT pg_read_file('test_file1'); -- disallowed file paths for non-superusers and users who are -- not members of pg_write_server_files -CREATE ROLE regress_user1; +CREATE ROLE regress_adminpack_user1; -GRANT pg_read_all_settings TO regress_user1; -GRANT EXECUTE ON FUNCTION pg_file_write(text,text,bool) TO regress_user1; +GRANT pg_read_all_settings TO regress_adminpack_user1; +GRANT EXECUTE ON FUNCTION pg_file_write(text,text,bool) TO regress_adminpack_user1; -SET ROLE regress_user1; +SET ROLE regress_adminpack_user1; SELECT pg_file_write('../test_file0', 'test0', false); SELECT pg_file_write('/tmp/test_file0', 'test0', false); SELECT pg_file_write(current_setting('data_directory') || '/test_file4', 'test4', false); SELECT pg_file_write(current_setting('data_directory') || '/../test_file4', 'test4', false); RESET ROLE; -REVOKE EXECUTE ON FUNCTION pg_file_write(text,text,bool) FROM regress_user1; -REVOKE pg_read_all_settings FROM regress_user1; -DROP ROLE regress_user1; +REVOKE EXECUTE ON FUNCTION pg_file_write(text,text,bool) FROM regress_adminpack_user1; +REVOKE pg_read_all_settings FROM regress_adminpack_user1; +DROP ROLE regress_adminpack_user1; -- sync SELECT pg_file_sync('test_file1'); -- sync file @@ -59,8 +59,8 @@ SELECT pg_file_unlink('test_file4'); -- superuser checks -CREATE USER regress_user1; -SET ROLE regress_user1; +CREATE USER regress_adminpack_user1; +SET ROLE regress_adminpack_user1; SELECT pg_file_write('test_file0', 'test0', false); SELECT pg_file_sync('test_file0'); @@ -69,7 +69,7 @@ SELECT pg_file_unlink('test_file0'); SELECT pg_logdir_ls(); RESET ROLE; -DROP USER regress_user1; +DROP USER regress_adminpack_user1; -- no further tests for pg_logdir_ls() because it depends on the diff --git a/contrib/amcheck/t/002_cic.pl b/contrib/amcheck/t/002_cic.pl index 32e4e4abd844d..8304664f6e0d0 100644 --- a/contrib/amcheck/t/002_cic.pl +++ b/contrib/amcheck/t/002_cic.pl @@ -60,5 +60,29 @@ ) }); +# Test bt_index_parent_check() with indexes created with +# CREATE INDEX CONCURRENTLY. +$node->safe_psql('postgres', q(CREATE TABLE quebec(i int primary key))); +# Insert two rows into index +$node->safe_psql('postgres', + q(INSERT INTO quebec SELECT i FROM generate_series(1, 2) s(i);)); + +# start background transaction +my $in_progress_h = $node->background_psql('postgres'); +$in_progress_h->query_safe(q(BEGIN; SELECT pg_current_xact_id();)); + +# delete one row from table, while background transaction is in progress +$node->safe_psql('postgres', q(DELETE FROM quebec WHERE i = 1;)); +# create index concurrently, which will skip the deleted row +$node->safe_psql('postgres', + q(CREATE INDEX CONCURRENTLY oscar ON quebec(i);)); + +# check index using bt_index_parent_check +$result = $node->psql('postgres', + q(SELECT bt_index_parent_check('oscar', heapallindexed => true))); +is($result, '0', 'bt_index_parent_check for CIC after removed row'); + +$in_progress_h->quit; + $node->stop; done_testing(); diff --git a/contrib/amcheck/verify_nbtree.c b/contrib/amcheck/verify_nbtree.c index c08f19fa89496..073d259fd8848 100644 --- a/contrib/amcheck/verify_nbtree.c +++ b/contrib/amcheck/verify_nbtree.c @@ -463,11 +463,11 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace, bool readonly, bool heapallindexed, bool rootdescend) { BtreeCheckState *state; + Snapshot snapshot = InvalidSnapshot; Page metapage; BTMetaPageData *metad; uint32 previouslevel; BtreeLevel current; - Snapshot snapshot = SnapshotAny; if (!readonly) elog(DEBUG1, "verifying consistency of tree structure for index \"%s\"", @@ -516,37 +516,33 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace, state->heaptuplespresent = 0; /* - * Register our own snapshot in !readonly case, rather than asking + * Register our own snapshot for heapallindexed, rather than asking * table_index_build_scan() to do this for us later. This needs to * happen before index fingerprinting begins, so we can later be * certain that index fingerprinting should have reached all tuples * returned by table_index_build_scan(). */ - if (!state->readonly) - { - snapshot = RegisterSnapshot(GetTransactionSnapshot()); + snapshot = RegisterSnapshot(GetTransactionSnapshot()); - /* - * GetTransactionSnapshot() always acquires a new MVCC snapshot in - * READ COMMITTED mode. A new snapshot is guaranteed to have all - * the entries it requires in the index. - * - * We must defend against the possibility that an old xact - * snapshot was returned at higher isolation levels when that - * snapshot is not safe for index scans of the target index. This - * is possible when the snapshot sees tuples that are before the - * index's indcheckxmin horizon. Throwing an error here should be - * very rare. It doesn't seem worth using a secondary snapshot to - * avoid this. - */ - if (IsolationUsesXactSnapshot() && rel->rd_index->indcheckxmin && - !TransactionIdPrecedes(HeapTupleHeaderGetXmin(rel->rd_indextuple->t_data), - snapshot->xmin)) - ereport(ERROR, - (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), - errmsg("index \"%s\" cannot be verified using transaction snapshot", - RelationGetRelationName(rel)))); - } + /* + * GetTransactionSnapshot() always acquires a new MVCC snapshot in + * READ COMMITTED mode. A new snapshot is guaranteed to have all the + * entries it requires in the index. + * + * We must defend against the possibility that an old xact snapshot + * was returned at higher isolation levels when that snapshot is not + * safe for index scans of the target index. This is possible when + * the snapshot sees tuples that are before the index's indcheckxmin + * horizon. Throwing an error here should be very rare. It doesn't + * seem worth using a secondary snapshot to avoid this. + */ + if (IsolationUsesXactSnapshot() && rel->rd_index->indcheckxmin && + !TransactionIdPrecedes(HeapTupleHeaderGetXmin(rel->rd_indextuple->t_data), + snapshot->xmin)) + ereport(ERROR, + errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), + errmsg("index \"%s\" cannot be verified using transaction snapshot", + RelationGetRelationName(rel))); } Assert(!state->rootdescend || state->readonly); @@ -621,8 +617,7 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace, /* * Create our own scan for table_index_build_scan(), rather than * getting it to do so for us. This is required so that we can - * actually use the MVCC snapshot registered earlier in !readonly - * case. + * actually use the MVCC snapshot registered earlier. * * Note that table_index_build_scan() calls heap_endscan() for us. */ @@ -635,16 +630,15 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace, /* * Scan will behave as the first scan of a CREATE INDEX CONCURRENTLY - * behaves in !readonly case. + * behaves. * * It's okay that we don't actually use the same lock strength for the - * heap relation as any other ii_Concurrent caller would in !readonly - * case. We have no reason to care about a concurrent VACUUM - * operation, since there isn't going to be a second scan of the heap - * that needs to be sure that there was no concurrent recycling of - * TIDs. + * heap relation as any other ii_Concurrent caller would. We have no + * reason to care about a concurrent VACUUM operation, since there + * isn't going to be a second scan of the heap that needs to be sure + * that there was no concurrent recycling of TIDs. */ - indexinfo->ii_Concurrent = !state->readonly; + indexinfo->ii_Concurrent = true; /* * Don't wait for uncommitted tuple xact commit/abort when index is a @@ -668,13 +662,12 @@ bt_check_every_level(Relation rel, Relation heaprel, bool heapkeyspace, state->heaptuplespresent, RelationGetRelationName(heaprel), 100.0 * bloom_prop_bits_set(state->filter)))); - if (snapshot != SnapshotAny) - UnregisterSnapshot(snapshot); - bloom_free(state->filter); } /* Be tidy: */ + if (snapshot != InvalidSnapshot) + UnregisterSnapshot(snapshot); MemoryContextDelete(state->targetcontext); } @@ -785,7 +778,7 @@ bt_check_level_from_leftmost(BtreeCheckState *state, BtreeLevel level) errmsg("block %u is not leftmost in index \"%s\"", current, RelationGetRelationName(state->rel)))); - if (level.istruerootlevel && !P_ISROOT(opaque)) + if (level.istruerootlevel && (!P_ISROOT(opaque) && !P_INCOMPLETE_SPLIT(opaque))) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("block %u is not true root in index \"%s\"", @@ -2048,7 +2041,7 @@ bt_child_highkey_check(BtreeCheckState *state, * If we visit page with high key, check that it is equal to the * target key next to corresponding downlink. */ - if (!rightsplit && !P_RIGHTMOST(opaque)) + if (!rightsplit && !P_RIGHTMOST(opaque) && !P_ISHALFDEAD(opaque)) { BTPageOpaque topaque; IndexTuple highkey; diff --git a/contrib/auto_explain/Makefile b/contrib/auto_explain/Makefile index efd127d3cae64..94ab28e7c06b9 100644 --- a/contrib/auto_explain/Makefile +++ b/contrib/auto_explain/Makefile @@ -6,6 +6,8 @@ OBJS = \ auto_explain.o PGFILEDESC = "auto_explain - logging facility for execution plans" +REGRESS = alter_reset + TAP_TESTS = 1 ifdef USE_PGXS diff --git a/contrib/auto_explain/expected/alter_reset.out b/contrib/auto_explain/expected/alter_reset.out new file mode 100644 index 0000000000000..49317c765612a --- /dev/null +++ b/contrib/auto_explain/expected/alter_reset.out @@ -0,0 +1,17 @@ +-- +-- This tests resetting unknown custom GUCs with reserved prefixes. There's +-- nothing specific to auto_explain; this is just a convenient place to put +-- this test. +-- +SELECT current_database() AS datname \gset +CREATE ROLE regress_ae_role; +ALTER DATABASE :"datname" SET auto_explain.bogus = 1; +ALTER ROLE regress_ae_role SET auto_explain.bogus = 1; +ALTER ROLE regress_ae_role IN DATABASE :"datname" SET auto_explain.bogus = 1; +LOAD 'auto_explain'; +WARNING: invalid configuration parameter name "auto_explain.bogus", removing it +DETAIL: "auto_explain" is now a reserved prefix. +ALTER DATABASE :"datname" RESET auto_explain.bogus; +ALTER ROLE regress_ae_role RESET auto_explain.bogus; +ALTER ROLE regress_ae_role IN DATABASE :"datname" RESET auto_explain.bogus; +DROP ROLE regress_ae_role; diff --git a/contrib/auto_explain/sql/alter_reset.sql b/contrib/auto_explain/sql/alter_reset.sql new file mode 100644 index 0000000000000..3c472755d6a2e --- /dev/null +++ b/contrib/auto_explain/sql/alter_reset.sql @@ -0,0 +1,20 @@ +-- +-- This tests resetting unknown custom GUCs with reserved prefixes. There's +-- nothing specific to auto_explain; this is just a convenient place to put +-- this test. +-- + +SELECT current_database() AS datname \gset +CREATE ROLE regress_ae_role; + +ALTER DATABASE :"datname" SET auto_explain.bogus = 1; +ALTER ROLE regress_ae_role SET auto_explain.bogus = 1; +ALTER ROLE regress_ae_role IN DATABASE :"datname" SET auto_explain.bogus = 1; + +LOAD 'auto_explain'; + +ALTER DATABASE :"datname" RESET auto_explain.bogus; +ALTER ROLE regress_ae_role RESET auto_explain.bogus; +ALTER ROLE regress_ae_role IN DATABASE :"datname" RESET auto_explain.bogus; + +DROP ROLE regress_ae_role; diff --git a/contrib/btree_gist/btree_utils_var.c b/contrib/btree_gist/btree_utils_var.c index 2886c08b85e40..9d93b3c775e56 100644 --- a/contrib/btree_gist/btree_utils_var.c +++ b/contrib/btree_gist/btree_utils_var.c @@ -116,36 +116,47 @@ gbt_var_leaf2node(GBT_VARKEY *leaf, const gbtree_vinfo *tinfo, FmgrInfo *flinfo) /* * returns the common prefix length of a node key + * + * If the underlying type is character data, the prefix length may point in + * the middle of a multibyte character. */ static int32 gbt_var_node_cp_len(const GBT_VARKEY *node, const gbtree_vinfo *tinfo) { GBT_VARKEY_R r = gbt_var_key_readable(node); int32 i = 0; - int32 l = 0; + int32 l_left_to_match = 0; + int32 l_total = 0; int32 t1len = VARSIZE(r.lower) - VARHDRSZ; int32 t2len = VARSIZE(r.upper) - VARHDRSZ; int32 ml = Min(t1len, t2len); char *p1 = VARDATA(r.lower); char *p2 = VARDATA(r.upper); + const char *end1 = p1 + t1len; + const char *end2 = p2 + t2len; if (ml == 0) return 0; while (i < ml) { - if (tinfo->eml > 1 && l == 0) + if (tinfo->eml > 1 && l_left_to_match == 0) { - if ((l = pg_mblen(p1)) != pg_mblen(p2)) + l_total = pg_mblen_range(p1, end1); + if (l_total != pg_mblen_range(p2, end2)) { return i; } + l_left_to_match = l_total; } if (*p1 != *p2) { if (tinfo->eml > 1) { - return (i - l + 1); + int32 l_matched_subset = l_total - l_left_to_match; + + /* end common prefix at final byte of last matching char */ + return i - l_matched_subset; } else { @@ -155,7 +166,7 @@ gbt_var_node_cp_len(const GBT_VARKEY *node, const gbtree_vinfo *tinfo) p1++; p2++; - l--; + l_left_to_match--; i++; } return ml; /* lower == upper */ diff --git a/contrib/dblink/dblink.c b/contrib/dblink/dblink.c index 462db21592c9c..6f39df8577e45 100644 --- a/contrib/dblink/dblink.c +++ b/contrib/dblink/dblink.c @@ -48,6 +48,7 @@ #include "funcapi.h" #include "lib/stringinfo.h" #include "libpq-fe.h" +#include "libpq/libpq-be-fe-helpers.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "parser/scansup.h" @@ -59,6 +60,7 @@ #include "utils/memutils.h" #include "utils/rel.h" #include "utils/varlena.h" +#include "utils/wait_event.h" PG_MODULE_MAGIC; @@ -97,7 +99,7 @@ static PGresult *storeQueryResult(volatile storeInfo *sinfo, PGconn *conn, const static void storeRow(volatile storeInfo *sinfo, PGresult *res, bool first); static remoteConn *getConnectionByName(const char *name); static HTAB *createConnHash(void); -static void createNewConnection(const char *name, remoteConn *rconn); +static remoteConn *createNewConnection(const char *name); static void deleteConnection(const char *name); static char **get_pkey_attnames(Relation rel, int16 *indnkeyatts); static char **get_text_array_contents(ArrayType *array, int *numitems); @@ -110,7 +112,7 @@ static HeapTuple get_tuple_of_interest(Relation rel, int *pkattnums, int pknumat static Relation get_rel_from_relname(text *relname_text, LOCKMODE lockmode, AclMode aclmode); static char *generate_relation_name(Relation rel); static void dblink_connstr_check(const char *connstr); -static void dblink_security_check(PGconn *conn, remoteConn *rconn); +static void dblink_security_check(PGconn *conn, const char *connname); static void dblink_res_error(PGconn *conn, const char *conname, PGresult *res, bool fail, const char *fmt,...) pg_attribute_printf(5, 6); static char *get_connect_string(const char *servername); @@ -128,16 +130,22 @@ static remoteConn *pconn = NULL; static HTAB *remoteConnHash = NULL; /* - * Following is list that holds multiple remote connections. + * Following is hash that holds multiple remote connections. * Calling convention of each dblink function changes to accept - * connection name as the first parameter. The connection list is + * connection name as the first parameter. The connection hash is * much like ecpg e.g. a mapping between a name and a PGconn object. + * + * To avoid potentially leaking a PGconn object in case of out-of-memory + * errors, we first create the hash entry, then open the PGconn. + * Hence, a hash entry whose rconn.conn pointer is NULL must be + * understood as a leftover from a failed create; it should be ignored + * by lookup operations, and silently replaced by create operations. */ typedef struct remoteConnHashEnt { char name[NAMEDATALEN]; - remoteConn *rconn; + remoteConn rconn; } remoteConnHashEnt; /* initial number of connection hashes */ @@ -236,7 +244,7 @@ dblink_get_conn(char *conname_or_str, errmsg("could not establish connection"), errdetail_internal("%s", msg))); } - dblink_security_check(conn, rconn); + dblink_security_check(conn, NULL); if (PQclientEncoding(conn) != GetDatabaseEncoding()) PQsetClientEncoding(conn, GetDatabaseEncodingName()); freeconn = true; @@ -296,15 +304,6 @@ dblink_connect(PG_FUNCTION_ARGS) else if (PG_NARGS() == 1) conname_or_str = text_to_cstring(PG_GETARG_TEXT_PP(0)); - if (connname) - { - rconn = (remoteConn *) MemoryContextAlloc(TopMemoryContext, - sizeof(remoteConn)); - rconn->conn = NULL; - rconn->openCursorCount = 0; - rconn->newXactForCursor = false; - } - /* first check for valid foreign data server */ connstr = get_connect_string(conname_or_str); if (connstr == NULL) @@ -335,6 +334,13 @@ dblink_connect(PG_FUNCTION_ARGS) #endif } + /* if we need a hashtable entry, make that first, since it might fail */ + if (connname) + { + rconn = createNewConnection(connname); + Assert(rconn->conn == NULL); + } + /* OK to make connection */ conn = PQconnectdb(connstr); @@ -343,8 +349,8 @@ dblink_connect(PG_FUNCTION_ARGS) msg = pchomp(PQerrorMessage(conn)); PQfinish(conn); ReleaseExternalFD(); - if (rconn) - pfree(rconn); + if (connname) + deleteConnection(connname); ereport(ERROR, (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), @@ -353,16 +359,16 @@ dblink_connect(PG_FUNCTION_ARGS) } /* check password actually used if not superuser */ - dblink_security_check(conn, rconn); + dblink_security_check(conn, connname); /* attempt to set client encoding to match server encoding, if needed */ if (PQclientEncoding(conn) != GetDatabaseEncoding()) PQsetClientEncoding(conn, GetDatabaseEncodingName()); + /* all OK, save away the conn */ if (connname) { rconn->conn = conn; - createNewConnection(connname, rconn); } else { @@ -406,10 +412,7 @@ dblink_disconnect(PG_FUNCTION_ARGS) PQfinish(conn); ReleaseExternalFD(); if (rconn) - { deleteConnection(conname); - pfree(rconn); - } else pconn->conn = NULL; @@ -478,7 +481,7 @@ dblink_open(PG_FUNCTION_ARGS) /* If we are not in a transaction, start one */ if (PQtransactionStatus(conn) == PQTRANS_IDLE) { - res = PQexec(conn, "BEGIN"); + res = libpqsrv_exec(conn, "BEGIN", PG_WAIT_EXTENSION); if (PQresultStatus(res) != PGRES_COMMAND_OK) dblink_res_internalerror(conn, res, "begin error"); PQclear(res); @@ -497,7 +500,7 @@ dblink_open(PG_FUNCTION_ARGS) (rconn->openCursorCount)++; appendStringInfo(&buf, "DECLARE %s CURSOR FOR %s", curname, sql); - res = PQexec(conn, buf.data); + res = libpqsrv_exec(conn, buf.data, PG_WAIT_EXTENSION); if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) { dblink_res_error(conn, conname, res, fail, @@ -566,7 +569,7 @@ dblink_close(PG_FUNCTION_ARGS) appendStringInfo(&buf, "CLOSE %s", curname); /* close the cursor */ - res = PQexec(conn, buf.data); + res = libpqsrv_exec(conn, buf.data, PG_WAIT_EXTENSION); if (!res || PQresultStatus(res) != PGRES_COMMAND_OK) { dblink_res_error(conn, conname, res, fail, @@ -586,7 +589,7 @@ dblink_close(PG_FUNCTION_ARGS) { rconn->newXactForCursor = false; - res = PQexec(conn, "COMMIT"); + res = libpqsrv_exec(conn, "COMMIT", PG_WAIT_EXTENSION); if (PQresultStatus(res) != PGRES_COMMAND_OK) dblink_res_internalerror(conn, res, "commit error"); PQclear(res); @@ -668,7 +671,7 @@ dblink_fetch(PG_FUNCTION_ARGS) * PGresult will be long-lived even though we are still in a short-lived * memory context. */ - res = PQexec(conn, buf.data); + res = libpqsrv_exec(conn, buf.data, PG_WAIT_EXTENSION); if (!res || (PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)) @@ -816,7 +819,7 @@ dblink_record_internal(FunctionCallInfo fcinfo, bool is_async) else { /* async result retrieval, do it the old way */ - PGresult *res = PQgetResult(conn); + PGresult *res = libpqsrv_get_result(conn, PG_WAIT_EXTENSION); /* NULL means we're all done with the async results */ if (res) @@ -1127,7 +1130,8 @@ materializeQueryResult(FunctionCallInfo fcinfo, PQclear(sinfo.last_res); PQclear(sinfo.cur_res); /* and clear out any pending data in libpq */ - while ((res = PQgetResult(conn)) != NULL) + while ((res = libpqsrv_get_result(conn, PG_WAIT_EXTENSION)) != + NULL) PQclear(res); PG_RE_THROW(); } @@ -1154,7 +1158,7 @@ storeQueryResult(volatile storeInfo *sinfo, PGconn *conn, const char *sql) { CHECK_FOR_INTERRUPTS(); - sinfo->cur_res = PQgetResult(conn); + sinfo->cur_res = libpqsrv_get_result(conn, PG_WAIT_EXTENSION); if (!sinfo->cur_res) break; @@ -1329,6 +1333,9 @@ dblink_get_connections(PG_FUNCTION_ARGS) hash_seq_init(&status, remoteConnHash); while ((hentry = (remoteConnHashEnt *) hash_seq_search(&status)) != NULL) { + /* ignore it if it's not an open connection */ + if (hentry->rconn.conn == NULL) + continue; /* stash away current value */ astate = accumArrayResult(astate, CStringGetTextDatum(hentry->name), @@ -1482,7 +1489,7 @@ dblink_exec(PG_FUNCTION_ARGS) if (!conn) dblink_conn_not_avail(conname); - res = PQexec(conn, sql); + res = libpqsrv_exec(conn, sql, PG_WAIT_EXTENSION); if (!res || (PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)) @@ -2567,8 +2574,8 @@ getConnectionByName(const char *name) hentry = (remoteConnHashEnt *) hash_search(remoteConnHash, key, HASH_FIND, NULL); - if (hentry) - return hentry->rconn; + if (hentry && hentry->rconn.conn != NULL) + return &hentry->rconn; return NULL; } @@ -2585,8 +2592,8 @@ createConnHash(void) HASH_ELEM | HASH_STRINGS); } -static void -createNewConnection(const char *name, remoteConn *rconn) +static remoteConn * +createNewConnection(const char *name) { remoteConnHashEnt *hentry; bool found; @@ -2600,19 +2607,15 @@ createNewConnection(const char *name, remoteConn *rconn) hentry = (remoteConnHashEnt *) hash_search(remoteConnHash, key, HASH_ENTER, &found); - if (found) - { - PQfinish(rconn->conn); - ReleaseExternalFD(); - pfree(rconn); - + if (found && hentry->rconn.conn != NULL) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("duplicate connection name"))); - } - hentry->rconn = rconn; - strlcpy(hentry->name, name, sizeof(hentry->name)); + /* New, or reusable, so initialize the rconn struct to zeroes */ + memset(&hentry->rconn, 0, sizeof(remoteConn)); + + return &hentry->rconn; } static void @@ -2637,7 +2640,7 @@ deleteConnection(const char *name) } static void -dblink_security_check(PGconn *conn, remoteConn *rconn) +dblink_security_check(PGconn *conn, const char *connname) { if (!superuser()) { @@ -2645,8 +2648,8 @@ dblink_security_check(PGconn *conn, remoteConn *rconn) { PQfinish(conn); ReleaseExternalFD(); - if (rconn) - pfree(rconn); + if (connname) + deleteConnection(connname); ereport(ERROR, (errcode(ERRCODE_S_R_E_PROHIBITED_SQL_STATEMENT_ATTEMPTED), @@ -2744,8 +2747,8 @@ dblink_res_error(PGconn *conn, const char *conname, PGresult *res, /* * If we don't get a message from the PGresult, try the PGconn. This is - * needed because for connection-level failures, PQexec may just return - * NULL, not a PGresult at all. + * needed because for connection-level failures, PQgetResult may just + * return NULL, not a PGresult at all. */ if (message_primary == NULL) message_primary = pchomp(PQerrorMessage(conn)); diff --git a/contrib/dict_xsyn/dict_xsyn.c b/contrib/dict_xsyn/dict_xsyn.c index 584fe44753dbc..a4ced84dd8d9e 100644 --- a/contrib/dict_xsyn/dict_xsyn.c +++ b/contrib/dict_xsyn/dict_xsyn.c @@ -48,15 +48,15 @@ find_word(char *in, char **end) char *start; *end = NULL; - while (*in && t_isspace(in)) - in += pg_mblen(in); + while (*in && t_isspace_cstr(in)) + in += pg_mblen_cstr(in); if (!*in || *in == '#') return NULL; start = in; - while (*in && !t_isspace(in)) - in += pg_mblen(in); + while (*in && !t_isspace_cstr(in)) + in += pg_mblen_cstr(in); *end = in; diff --git a/contrib/file_fdw/expected/file_fdw.out b/contrib/file_fdw/expected/file_fdw.out index 0029f36b3593c..5ada148129f0c 100644 --- a/contrib/file_fdw/expected/file_fdw.out +++ b/contrib/file_fdw/expected/file_fdw.out @@ -48,6 +48,11 @@ SET ROLE regress_file_fdw_superuser; CREATE USER MAPPING FOR regress_file_fdw_superuser SERVER file_server; CREATE USER MAPPING FOR regress_no_priv_user SERVER file_server; -- validator tests +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (foo 'bar'); -- ERROR +ERROR: invalid option "foo" +HINT: Valid options in this context are: filename, program, format, header, delimiter, quote, escape, null, encoding +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS ("a=b" 'true'); -- ERROR +ERROR: invalid option name "a=b": must not contain "=" CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'xml'); -- ERROR ERROR: COPY format "xml" not recognized CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'text', quote ':'); -- ERROR @@ -411,6 +416,21 @@ SELECT tableoid::regclass, * FROM p2; p2 | 2 | xyzzy (3 rows) +-- Test DELETE/UPDATE/MERGE on a partitioned table when all partitions +-- are excluded and only the dummy root result relation remains. The +-- operation is a no-op but should not fail regardless of whether the +-- foreign child was processed (pruning off) or not (pruning on). +DROP TABLE p2; +SET enable_partition_pruning TO off; +DELETE FROM pt WHERE false; +UPDATE pt SET b = 'x' WHERE false; +MERGE INTO pt t USING (VALUES (1, 'x'::text)) AS s(a, b) + ON false WHEN MATCHED THEN UPDATE SET b = s.b; +SET enable_partition_pruning TO on; +DELETE FROM pt WHERE false; +UPDATE pt SET b = 'x' WHERE false; +MERGE INTO pt t USING (VALUES (1, 'x'::text)) AS s(a, b) + ON false WHEN MATCHED THEN UPDATE SET b = s.b; DROP TABLE pt; -- generated column tests \set filename :abs_srcdir '/data/list1.csv' diff --git a/contrib/file_fdw/sql/file_fdw.sql b/contrib/file_fdw/sql/file_fdw.sql index 563d824ccc8c8..0015b2b88e5b1 100644 --- a/contrib/file_fdw/sql/file_fdw.sql +++ b/contrib/file_fdw/sql/file_fdw.sql @@ -55,6 +55,8 @@ CREATE USER MAPPING FOR regress_file_fdw_superuser SERVER file_server; CREATE USER MAPPING FOR regress_no_priv_user SERVER file_server; -- validator tests +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (foo 'bar'); -- ERROR +CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS ("a=b" 'true'); -- ERROR CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'xml'); -- ERROR CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'text', quote ':'); -- ERROR CREATE FOREIGN TABLE tbl () SERVER file_server OPTIONS (format 'text', escape ':'); -- ERROR @@ -223,6 +225,24 @@ UPDATE pt set a = 1 where a = 2; -- ERROR SELECT tableoid::regclass, * FROM pt; SELECT tableoid::regclass, * FROM p1; SELECT tableoid::regclass, * FROM p2; + +-- Test DELETE/UPDATE/MERGE on a partitioned table when all partitions +-- are excluded and only the dummy root result relation remains. The +-- operation is a no-op but should not fail regardless of whether the +-- foreign child was processed (pruning off) or not (pruning on). +DROP TABLE p2; +SET enable_partition_pruning TO off; +DELETE FROM pt WHERE false; +UPDATE pt SET b = 'x' WHERE false; +MERGE INTO pt t USING (VALUES (1, 'x'::text)) AS s(a, b) + ON false WHEN MATCHED THEN UPDATE SET b = s.b; + +SET enable_partition_pruning TO on; +DELETE FROM pt WHERE false; +UPDATE pt SET b = 'x' WHERE false; +MERGE INTO pt t USING (VALUES (1, 'x'::text)) AS s(a, b) + ON false WHEN MATCHED THEN UPDATE SET b = s.b; + DROP TABLE pt; -- generated column tests diff --git a/contrib/hstore/hstore_io.c b/contrib/hstore/hstore_io.c index 03057f085d1db..da4d9d7fb952f 100644 --- a/contrib/hstore/hstore_io.c +++ b/contrib/hstore/hstore_io.c @@ -82,7 +82,7 @@ get_val(HSParser *state, bool ignoreeq, bool *escaped) else if (*(state->ptr) == '=' && !ignoreeq) { elog(ERROR, "Syntax error near \"%.*s\" at position %d", - pg_mblen(state->ptr), state->ptr, + pg_mblen_cstr(state->ptr), state->ptr, (int32) (state->ptr - state->begin)); } else if (*(state->ptr) == '\\') @@ -223,7 +223,7 @@ parse_hstore(HSParser *state) else if (!scanner_isspace((unsigned char) *(state->ptr))) { elog(ERROR, "Syntax error near \"%.*s\" at position %d", - pg_mblen(state->ptr), state->ptr, + pg_mblen_cstr(state->ptr), state->ptr, (int32) (state->ptr - state->begin)); } } @@ -240,7 +240,7 @@ parse_hstore(HSParser *state) else { elog(ERROR, "Syntax error near \"%.*s\" at position %d", - pg_mblen(state->ptr), state->ptr, + pg_mblen_cstr(state->ptr), state->ptr, (int32) (state->ptr - state->begin)); } } @@ -275,7 +275,7 @@ parse_hstore(HSParser *state) else if (!scanner_isspace((unsigned char) *(state->ptr))) { elog(ERROR, "Syntax error near \"%.*s\" at position %d", - pg_mblen(state->ptr), state->ptr, + pg_mblen_cstr(state->ptr), state->ptr, (int32) (state->ptr - state->begin)); } } @@ -346,7 +346,8 @@ hstoreUniquePairs(Pairs *a, int32 l, int32 *buflen) if (ptr->needfree) { pfree(ptr->key); - pfree(ptr->val); + if (ptr->val != NULL) + pfree(ptr->val); } } else diff --git a/contrib/intarray/_int_selfuncs.c b/contrib/intarray/_int_selfuncs.c index 3d8ff6781bc34..d2df83404ef07 100644 --- a/contrib/intarray/_int_selfuncs.c +++ b/contrib/intarray/_int_selfuncs.c @@ -19,6 +19,7 @@ #include "catalog/pg_operator.h" #include "catalog/pg_statistic.h" #include "catalog/pg_type.h" +#include "commands/extension.h" #include "miscadmin.h" #include "utils/builtins.h" #include "utils/lsyscache.h" @@ -171,14 +172,25 @@ _int_matchsel(PG_FUNCTION_ARGS) PG_RETURN_FLOAT8(0.0); } - /* The caller made sure the const is a query, so get it now */ + /* + * Verify that the Const is a query_int, else return a default estimate. + * (This could only fail if someone attached this estimator to the wrong + * operator.) + */ + if (((Const *) other)->consttype != + get_function_sibling_type(fcinfo->flinfo->fn_oid, "query_int")) + { + ReleaseVariableStats(vardata); + PG_RETURN_FLOAT8(DEFAULT_EQ_SEL); + } + query = DatumGetQueryTypeP(((Const *) other)->constvalue); /* Empty query matches nothing */ if (query->size == 0) { ReleaseVariableStats(vardata); - return (Selectivity) 0.0; + PG_RETURN_FLOAT8(0.0); } /* @@ -327,7 +339,12 @@ static int compare_val_int4(const void *a, const void *b) { int32 key = *(int32 *) a; - const Datum *t = (const Datum *) b; + int32 value = DatumGetInt32(*(const Datum *) b); - return key - DatumGetInt32(*t); + if (key < value) + return -1; + else if (key > value) + return 1; + else + return 0; } diff --git a/contrib/intarray/expected/_int.out b/contrib/intarray/expected/_int.out index 09ab23483f7bd..64d8878763283 100644 --- a/contrib/intarray/expected/_int.out +++ b/contrib/intarray/expected/_int.out @@ -473,6 +473,12 @@ SELECT count(*) from test__int WHERE a @@ '!20 & !21'; 6344 (1 row) +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; + count +------- + 12 +(1 row) + SET enable_seqscan = off; -- not all of these would use index by default CREATE INDEX text_idx on test__int using gist ( a gist__int_ops ); SELECT count(*) from test__int WHERE a && '{23,50}'; @@ -547,6 +553,12 @@ SELECT count(*) from test__int WHERE a @@ '!20 & !21'; 6344 (1 row) +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; + count +------- + 12 +(1 row) + INSERT INTO test__int SELECT array(SELECT x FROM generate_series(1, 1001) x); -- should fail ERROR: input array is too big (199 maximum allowed, 1001 current), use gist__intbig_ops opclass instead DROP INDEX text_idx; @@ -629,6 +641,12 @@ SELECT count(*) from test__int WHERE a @@ '!20 & !21'; 6344 (1 row) +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; + count +------- + 12 +(1 row) + DROP INDEX text_idx; CREATE INDEX text_idx on test__int using gist (a gist__intbig_ops(siglen = 0)); ERROR: value 0 out of bounds for option "siglen" @@ -709,6 +727,12 @@ SELECT count(*) from test__int WHERE a @@ '!20 & !21'; 6344 (1 row) +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; + count +------- + 12 +(1 row) + DROP INDEX text_idx; CREATE INDEX text_idx on test__int using gist ( a gist__intbig_ops ); SELECT count(*) from test__int WHERE a && '{23,50}'; @@ -783,6 +807,12 @@ SELECT count(*) from test__int WHERE a @@ '!20 & !21'; 6344 (1 row) +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; + count +------- + 12 +(1 row) + DROP INDEX text_idx; CREATE INDEX text_idx on test__int using gin ( a gin__int_ops ); SELECT count(*) from test__int WHERE a && '{23,50}'; @@ -857,6 +887,12 @@ SELECT count(*) from test__int WHERE a @@ '!20 & !21'; 6344 (1 row) +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; + count +------- + 12 +(1 row) + DROP INDEX text_idx; -- Repeat the same queries with an extended data set. The data set is the -- same that we used before, except that each element in the array is @@ -949,4 +985,10 @@ SELECT count(*) from more__int WHERE a @@ '!20 & !21'; 6344 (1 row) +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; + count +------- + 12 +(1 row) + RESET enable_seqscan; diff --git a/contrib/intarray/sql/_int.sql b/contrib/intarray/sql/_int.sql index 95eec96c14e81..ba4c298151a1c 100644 --- a/contrib/intarray/sql/_int.sql +++ b/contrib/intarray/sql/_int.sql @@ -92,6 +92,7 @@ SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}'; SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)'; SELECT count(*) from test__int WHERE a @@ '20 | !21'; SELECT count(*) from test__int WHERE a @@ '!20 & !21'; +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; SET enable_seqscan = off; -- not all of these would use index by default @@ -109,6 +110,7 @@ SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}'; SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)'; SELECT count(*) from test__int WHERE a @@ '20 | !21'; SELECT count(*) from test__int WHERE a @@ '!20 & !21'; +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; INSERT INTO test__int SELECT array(SELECT x FROM generate_series(1, 1001) x); -- should fail @@ -129,6 +131,7 @@ SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}'; SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)'; SELECT count(*) from test__int WHERE a @@ '20 | !21'; SELECT count(*) from test__int WHERE a @@ '!20 & !21'; +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; DROP INDEX text_idx; CREATE INDEX text_idx on test__int using gist (a gist__intbig_ops(siglen = 0)); @@ -147,6 +150,7 @@ SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}'; SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)'; SELECT count(*) from test__int WHERE a @@ '20 | !21'; SELECT count(*) from test__int WHERE a @@ '!20 & !21'; +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; DROP INDEX text_idx; CREATE INDEX text_idx on test__int using gist ( a gist__intbig_ops ); @@ -163,6 +167,7 @@ SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}'; SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)'; SELECT count(*) from test__int WHERE a @@ '20 | !21'; SELECT count(*) from test__int WHERE a @@ '!20 & !21'; +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; DROP INDEX text_idx; CREATE INDEX text_idx on test__int using gin ( a gin__int_ops ); @@ -179,6 +184,7 @@ SELECT count(*) from test__int WHERE a @> '{20,23}' or a @> '{50,68}'; SELECT count(*) from test__int WHERE a @@ '(20&23)|(50&68)'; SELECT count(*) from test__int WHERE a @@ '20 | !21'; SELECT count(*) from test__int WHERE a @@ '!20 & !21'; +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; DROP INDEX text_idx; @@ -214,6 +220,7 @@ SELECT count(*) from more__int WHERE a @> '{20,23}' or a @> '{50,68}'; SELECT count(*) from more__int WHERE a @@ '(20&23)|(50&68)'; SELECT count(*) from more__int WHERE a @@ '20 | !21'; SELECT count(*) from more__int WHERE a @@ '!20 & !21'; +SELECT count(*) from test__int WHERE a @@ '!2733 & (2738 | 254)'; RESET enable_seqscan; diff --git a/contrib/ltree/lquery_op.c b/contrib/ltree/lquery_op.c index ef86046fc4bc0..46019a0e83ab7 100644 --- a/contrib/ltree/lquery_op.c +++ b/contrib/ltree/lquery_op.c @@ -26,14 +26,14 @@ getlexeme(char *start, char *end, int *len) char *ptr; int charlen; - while (start < end && (charlen = pg_mblen(start)) == 1 && t_iseq(start, '_')) + while (start < end && (charlen = pg_mblen_range(start, end)) == 1 && t_iseq(start, '_')) start += charlen; ptr = start; if (ptr >= end) return NULL; - while (ptr < end && !((charlen = pg_mblen(ptr)) == 1 && t_iseq(ptr, '_'))) + while (ptr < end && !((charlen = pg_mblen_range(ptr, end)) == 1 && t_iseq(ptr, '_'))) ptr += charlen; *len = ptr - start; @@ -41,7 +41,8 @@ getlexeme(char *start, char *end, int *len) } bool -compare_subnode(ltree_level *t, char *qn, int len, int (*cmpptr) (const char *, const char *, size_t), bool anyend) +compare_subnode(ltree_level *t, char *qn, int len, + ltree_prefix_eq_func prefix_eq, bool anyend) { char *endt = t->name + t->len; char *endq = qn + len; @@ -57,7 +58,7 @@ compare_subnode(ltree_level *t, char *qn, int len, int (*cmpptr) (const char *, while ((tn = getlexeme(tn, endt, &lent)) != NULL) { if ((lent == lenq || (lent > lenq && anyend)) && - (*cmpptr) (qn, tn, lenq) == 0) + (*prefix_eq) (qn, lenq, tn, lent)) { isok = true; @@ -74,14 +75,29 @@ compare_subnode(ltree_level *t, char *qn, int len, int (*cmpptr) (const char *, return true; } -int -ltree_strncasecmp(const char *a, const char *b, size_t s) +/* + * Check if 'a' is a prefix of 'b'. + */ +bool +ltree_prefix_eq(const char *a, size_t a_sz, const char *b, size_t b_sz) +{ + if (a_sz > b_sz) + return false; + else + return (strncmp(a, b, a_sz) == 0); +} + +/* + * Case-insensitive check if 'a' is a prefix of 'b'. + */ +bool +ltree_prefix_eq_ci(const char *a, size_t a_sz, const char *b, size_t b_sz) { - char *al = str_tolower(a, s, DEFAULT_COLLATION_OID); - char *bl = str_tolower(b, s, DEFAULT_COLLATION_OID); - int res; + char *al = str_tolower(a, a_sz, DEFAULT_COLLATION_OID); + char *bl = str_tolower(b, b_sz, DEFAULT_COLLATION_OID); + bool res; - res = strncmp(al, bl, s); + res = (strncmp(al, bl, a_sz) == 0); pfree(al); pfree(bl); @@ -109,19 +125,19 @@ checkLevel(lquery_level *curq, ltree_level *curt) for (int i = 0; i < curq->numvar; i++) { - int (*cmpptr) (const char *, const char *, size_t); + ltree_prefix_eq_func prefix_eq; - cmpptr = (curvar->flag & LVAR_INCASE) ? ltree_strncasecmp : strncmp; + prefix_eq = (curvar->flag & LVAR_INCASE) ? ltree_prefix_eq_ci : ltree_prefix_eq; if (curvar->flag & LVAR_SUBLEXEME) { - if (compare_subnode(curt, curvar->name, curvar->len, cmpptr, + if (compare_subnode(curt, curvar->name, curvar->len, prefix_eq, (curvar->flag & LVAR_ANYEND))) return success; } else if ((curvar->len == curt->len || (curt->len > curvar->len && (curvar->flag & LVAR_ANYEND))) && - (*cmpptr) (curvar->name, curt->name, curvar->len) == 0) + (*prefix_eq) (curvar->name, curvar->len, curt->name, curt->len)) return success; curvar = LVAR_NEXT(curvar); diff --git a/contrib/ltree/ltree.h b/contrib/ltree/ltree.h index 564e4fa81b87b..a37ee6b2c679a 100644 --- a/contrib/ltree/ltree.h +++ b/contrib/ltree/ltree.h @@ -126,7 +126,8 @@ typedef struct #define LQUERY_HASNOT 0x01 -#define ISALNUM(x) ( t_isalpha(x) || t_isdigit(x) || ( pg_mblen(x) == 1 && t_iseq((x), '_') ) ) +/* Caller has already called mblen, so we can use _unbounded variants safely. */ +#define ISALNUM(x) ( t_isalpha_unbounded(x) || t_isdigit_unbounded(x) || ( pg_mblen_unbounded(x) == 1 && t_iseq((x), '_') ) ) /* full text query */ @@ -156,6 +157,8 @@ typedef struct char data[FLEXIBLE_ARRAY_MEMBER]; } ltxtquery; +typedef bool (*ltree_prefix_eq_func) (const char *, size_t, const char *, size_t); + #define HDRSIZEQT MAXALIGN(VARHDRSZ + sizeof(int32)) #define COMPUTESIZE(size,lenofoperand) ( HDRSIZEQT + (size) * sizeof(ITEM) + (lenofoperand) ) #define LTXTQUERY_TOO_BIG(size,lenofoperand) \ @@ -206,10 +209,11 @@ bool ltree_execute(ITEM *curitem, void *checkval, int ltree_compare(const ltree *a, const ltree *b); bool inner_isparent(const ltree *c, const ltree *p); -bool compare_subnode(ltree_level *t, char *q, int len, - int (*cmpptr) (const char *, const char *, size_t), bool anyend); +bool compare_subnode(ltree_level *t, char *qn, int len, + ltree_prefix_eq_func prefix_eq, bool anyend); ltree *lca_inner(ltree **a, int len); -int ltree_strncasecmp(const char *a, const char *b, size_t s); +bool ltree_prefix_eq(const char *a, size_t a_sz, const char *b, size_t b_sz); +bool ltree_prefix_eq_ci(const char *a, size_t a_sz, const char *b, size_t b_sz); /* fmgr macros for ltree objects */ #define DatumGetLtreeP(X) ((ltree *) PG_DETOAST_DATUM(X)) diff --git a/contrib/ltree/ltree_io.c b/contrib/ltree/ltree_io.c index 15115cb29f3c3..0a44a8c46915a 100644 --- a/contrib/ltree/ltree_io.c +++ b/contrib/ltree/ltree_io.c @@ -54,7 +54,7 @@ parse_ltree(const char *buf) ptr = buf; while (*ptr) { - charlen = pg_mblen(ptr); + charlen = pg_mblen_cstr(ptr); if (t_iseq(ptr, '.')) num++; ptr += charlen; @@ -69,7 +69,7 @@ parse_ltree(const char *buf) ptr = buf; while (*ptr) { - charlen = pg_mblen(ptr); + charlen = pg_mblen_cstr(ptr); switch (state) { @@ -285,7 +285,7 @@ parse_lquery(const char *buf) ptr = buf; while (*ptr) { - charlen = pg_mblen(ptr); + charlen = pg_mblen_cstr(ptr); if (t_iseq(ptr, '.')) num++; @@ -305,7 +305,7 @@ parse_lquery(const char *buf) ptr = buf; while (*ptr) { - charlen = pg_mblen(ptr); + charlen = pg_mblen_cstr(ptr); switch (state) { @@ -402,7 +402,7 @@ parse_lquery(const char *buf) case LQPRS_WAITFNUM: if (t_iseq(ptr, ',')) state = LQPRS_WAITSNUM; - else if (t_isdigit(ptr)) + else if (t_isdigit_cstr(ptr)) { int low = atoi(ptr); @@ -420,7 +420,7 @@ parse_lquery(const char *buf) UNCHAR; break; case LQPRS_WAITSNUM: - if (t_isdigit(ptr)) + if (t_isdigit_cstr(ptr)) { int high = atoi(ptr); @@ -451,7 +451,7 @@ parse_lquery(const char *buf) case LQPRS_WAITCLOSE: if (t_iseq(ptr, '}')) state = LQPRS_WAITEND; - else if (!t_isdigit(ptr)) + else if (!t_isdigit_cstr(ptr)) UNCHAR; break; case LQPRS_WAITND: @@ -462,7 +462,7 @@ parse_lquery(const char *buf) } else if (t_iseq(ptr, ',')) state = LQPRS_WAITSNUM; - else if (!t_isdigit(ptr)) + else if (!t_isdigit_cstr(ptr)) UNCHAR; break; case LQPRS_WAITEND: diff --git a/contrib/ltree/ltxtquery_io.c b/contrib/ltree/ltxtquery_io.c index 3eca5cb8ff303..bda2d97102179 100644 --- a/contrib/ltree/ltxtquery_io.c +++ b/contrib/ltree/ltxtquery_io.c @@ -59,7 +59,7 @@ gettoken_query(QPRS_STATE *state, int32 *val, int32 *lenval, char **strval, uint for (;;) { - charlen = pg_mblen(state->buf); + charlen = pg_mblen_cstr(state->buf); switch (state->state) { @@ -83,7 +83,7 @@ gettoken_query(QPRS_STATE *state, int32 *val, int32 *lenval, char **strval, uint *lenval = charlen; *flag = 0; } - else if (!t_isspace(state->buf)) + else if (!t_isspace_unbounded(state->buf)) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("operand syntax error"))); diff --git a/contrib/ltree/ltxtquery_op.c b/contrib/ltree/ltxtquery_op.c index 002102c9c75b6..3dcbab2c48460 100644 --- a/contrib/ltree/ltxtquery_op.c +++ b/contrib/ltree/ltxtquery_op.c @@ -58,19 +58,19 @@ checkcondition_str(void *checkval, ITEM *val) ltree_level *level = LTREE_FIRST(((CHKVAL *) checkval)->node); int tlen = ((CHKVAL *) checkval)->node->numlevel; char *op = ((CHKVAL *) checkval)->operand + val->distance; - int (*cmpptr) (const char *, const char *, size_t); + ltree_prefix_eq_func prefix_eq; - cmpptr = (val->flag & LVAR_INCASE) ? ltree_strncasecmp : strncmp; + prefix_eq = (val->flag & LVAR_INCASE) ? ltree_prefix_eq_ci : ltree_prefix_eq; while (tlen > 0) { if (val->flag & LVAR_SUBLEXEME) { - if (compare_subnode(level, op, val->length, cmpptr, (val->flag & LVAR_ANYEND))) + if (compare_subnode(level, op, val->length, prefix_eq, (val->flag & LVAR_ANYEND))) return true; } else if ((val->length == level->len || (level->len > val->length && (val->flag & LVAR_ANYEND))) && - (*cmpptr) (op, level->name, val->length) == 0) + (*prefix_eq) (op, val->length, level->name, level->len)) return true; tlen--; diff --git a/contrib/pageinspect/heapfuncs.c b/contrib/pageinspect/heapfuncs.c index 92a17900d61e5..ad3a0a4d97c5f 100644 --- a/contrib/pageinspect/heapfuncs.c +++ b/contrib/pageinspect/heapfuncs.c @@ -101,7 +101,7 @@ text_to_bits(char *str, int len) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid character \"%.*s\" in t_bits string", - pg_mblen(str + off), str + off))); + pg_mblen_cstr(str + off), str + off))); if (off % 8 == 7) bits[off / 8] = byte; @@ -212,11 +212,8 @@ heap_page_items(PG_FUNCTION_ARGS) lp_offset + lp_len <= raw_page_size) { HeapTupleHeader tuphdr; - bytea *tuple_data_bytea; - int tuple_data_len; /* Extract information from the tuple header */ - tuphdr = (HeapTupleHeader) PageGetItem(page, id); values[4] = UInt32GetDatum(HeapTupleHeaderGetRawXmin(tuphdr)); @@ -228,31 +225,32 @@ heap_page_items(PG_FUNCTION_ARGS) values[9] = UInt32GetDatum(tuphdr->t_infomask); values[10] = UInt8GetDatum(tuphdr->t_hoff); - /* Copy raw tuple data into bytea attribute */ - tuple_data_len = lp_len - tuphdr->t_hoff; - tuple_data_bytea = (bytea *) palloc(tuple_data_len + VARHDRSZ); - SET_VARSIZE(tuple_data_bytea, tuple_data_len + VARHDRSZ); - memcpy(VARDATA(tuple_data_bytea), (char *) tuphdr + tuphdr->t_hoff, - tuple_data_len); - values[13] = PointerGetDatum(tuple_data_bytea); - /* * We already checked that the item is completely within the raw * page passed to us, with the length given in the line pointer. - * Let's check that t_hoff doesn't point over lp_len, before using - * it to access t_bits and oid. + * But t_hoff could be out of range, so check it before relying on + * it to fetch additional info. */ if (tuphdr->t_hoff >= SizeofHeapTupleHeader && tuphdr->t_hoff <= lp_len && tuphdr->t_hoff == MAXALIGN(tuphdr->t_hoff)) { + int tuple_data_len; + bytea *tuple_data_bytea; + + /* Copy null bitmask and OID, if present */ if (tuphdr->t_infomask & HEAP_HASNULL) { - int bits_len; - - bits_len = - BITMAPLEN(HeapTupleHeaderGetNatts(tuphdr)) * BITS_PER_BYTE; - values[11] = CStringGetTextDatum(bits_to_text(tuphdr->t_bits, bits_len)); + int bitmaplen; + + bitmaplen = BITMAPLEN(HeapTupleHeaderGetNatts(tuphdr)); + /* better range-check the attribute count, too */ + if (bitmaplen <= tuphdr->t_hoff - SizeofHeapTupleHeader) + values[11] = + CStringGetTextDatum(bits_to_text(tuphdr->t_bits, + bitmaplen * BITS_PER_BYTE)); + else + nulls[11] = true; } else nulls[11] = true; @@ -261,11 +259,22 @@ heap_page_items(PG_FUNCTION_ARGS) values[12] = HeapTupleHeaderGetOidOld(tuphdr); else nulls[12] = true; + + /* Copy raw tuple data into bytea attribute */ + tuple_data_len = lp_len - tuphdr->t_hoff; + tuple_data_bytea = (bytea *) palloc(tuple_data_len + VARHDRSZ); + SET_VARSIZE(tuple_data_bytea, tuple_data_len + VARHDRSZ); + if (tuple_data_len > 0) + memcpy(VARDATA(tuple_data_bytea), + (char *) tuphdr + tuphdr->t_hoff, + tuple_data_len); + values[13] = PointerGetDatum(tuple_data_bytea); } else { nulls[11] = true; nulls[12] = true; + nulls[13] = true; } } else diff --git a/contrib/passwordcheck/expected/passwordcheck.out b/contrib/passwordcheck/expected/passwordcheck.out index e04cda6bd9566..2027681daf664 100644 --- a/contrib/passwordcheck/expected/passwordcheck.out +++ b/contrib/passwordcheck/expected/passwordcheck.out @@ -1,19 +1,19 @@ LOAD 'passwordcheck'; -CREATE USER regress_user1; +CREATE USER regress_passwordcheck_user1; -- ok -ALTER USER regress_user1 PASSWORD 'a_nice_long_password'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'a_nice_long_password'; -- error: too short -ALTER USER regress_user1 PASSWORD 'tooshrt'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'tooshrt'; ERROR: password is too short -- error: contains user name -ALTER USER regress_user1 PASSWORD 'xyzregress_user1'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'xyzregress_passwordcheck_user1'; ERROR: password must not contain user name -- error: contains only letters -ALTER USER regress_user1 PASSWORD 'alessnicelongpassword'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'alessnicelongpassword'; ERROR: password must contain both letters and nonletters -- encrypted ok (password is "secret") -ALTER USER regress_user1 PASSWORD 'md51a44d829a20a23eac686d9f0d258af13'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'md592350e12ac34e52dd598f90893bb3ae7'; -- error: password is user name -ALTER USER regress_user1 PASSWORD 'md5e589150ae7d28f93333afae92b36ef48'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'md507a112732ed9f2087fa90b192d44e358'; ERROR: password must not equal user name -DROP USER regress_user1; +DROP USER regress_passwordcheck_user1; diff --git a/contrib/passwordcheck/sql/passwordcheck.sql b/contrib/passwordcheck/sql/passwordcheck.sql index d98796ac494cf..1fbd6b0e96e08 100644 --- a/contrib/passwordcheck/sql/passwordcheck.sql +++ b/contrib/passwordcheck/sql/passwordcheck.sql @@ -1,23 +1,23 @@ LOAD 'passwordcheck'; -CREATE USER regress_user1; +CREATE USER regress_passwordcheck_user1; -- ok -ALTER USER regress_user1 PASSWORD 'a_nice_long_password'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'a_nice_long_password'; -- error: too short -ALTER USER regress_user1 PASSWORD 'tooshrt'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'tooshrt'; -- error: contains user name -ALTER USER regress_user1 PASSWORD 'xyzregress_user1'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'xyzregress_passwordcheck_user1'; -- error: contains only letters -ALTER USER regress_user1 PASSWORD 'alessnicelongpassword'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'alessnicelongpassword'; -- encrypted ok (password is "secret") -ALTER USER regress_user1 PASSWORD 'md51a44d829a20a23eac686d9f0d258af13'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'md592350e12ac34e52dd598f90893bb3ae7'; -- error: password is user name -ALTER USER regress_user1 PASSWORD 'md5e589150ae7d28f93333afae92b36ef48'; +ALTER USER regress_passwordcheck_user1 PASSWORD 'md507a112732ed9f2087fa90b192d44e358'; -DROP USER regress_user1; +DROP USER regress_passwordcheck_user1; diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index 1bd579fcbb0ca..206b809d04dbf 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -11,6 +11,7 @@ #include "access/htup_details.h" #include "catalog/pg_type.h" #include "funcapi.h" +#include "miscadmin.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" @@ -148,6 +149,8 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) BufferDesc *bufHdr; uint32 buf_state; + CHECK_FOR_INTERRUPTS(); + bufHdr = GetBufferDescriptor(i); /* Lock each buffer header before inspecting. */ buf_state = LockBufHdr(bufHdr); diff --git a/contrib/pg_freespacemap/pg_freespacemap.c b/contrib/pg_freespacemap/pg_freespacemap.c index b82cab2d97ef4..2de3c764e7940 100644 --- a/contrib/pg_freespacemap/pg_freespacemap.c +++ b/contrib/pg_freespacemap/pg_freespacemap.c @@ -11,6 +11,7 @@ #include "access/relation.h" #include "funcapi.h" #include "storage/freespace.h" +#include "utils/rel.h" PG_MODULE_MAGIC; @@ -30,6 +31,13 @@ pg_freespace(PG_FUNCTION_ARGS) rel = relation_open(relid, AccessShareLock); + if (!RELKIND_HAS_STORAGE(rel->rd_rel->relkind)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("relation \"%s\" does not have storage", + RelationGetRelationName(rel)), + errdetail_relkind_not_supported(rel->rd_rel->relkind))); + if (blkno < 0 || blkno > MaxBlockNumber) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), diff --git a/contrib/pg_prewarm/autoprewarm.c b/contrib/pg_prewarm/autoprewarm.c index c0c4f5d9ca7c9..c308286f94d14 100644 --- a/contrib/pg_prewarm/autoprewarm.c +++ b/contrib/pg_prewarm/autoprewarm.c @@ -609,8 +609,15 @@ apw_dump_now(bool is_bgworker, bool dump_unlogged) return 0; } - block_info_array = - (BlockInfoRecord *) palloc(sizeof(BlockInfoRecord) * NBuffers); + /* + * With sufficiently large shared_buffers, allocation will exceed 1GB, so + * allow for a huge allocation to prevent outright failure. + * + * (In the future, it might be a good idea to redesign this to use a more + * memory-efficient data structure.) + */ + block_info_array = (BlockInfoRecord *) + palloc_extended((sizeof(BlockInfoRecord) * NBuffers), MCXT_ALLOC_HUGE); for (num_blocks = 0, i = 0; i < NBuffers; i++) { diff --git a/contrib/pg_prewarm/pg_prewarm.c b/contrib/pg_prewarm/pg_prewarm.c index caff5c4a80f4e..40fb87bc5309c 100644 --- a/contrib/pg_prewarm/pg_prewarm.c +++ b/contrib/pg_prewarm/pg_prewarm.c @@ -16,9 +16,11 @@ #include #include "access/relation.h" +#include "catalog/index.h" #include "fmgr.h" #include "miscadmin.h" #include "storage/bufmgr.h" +#include "storage/lmgr.h" #include "storage/smgr.h" #include "utils/acl.h" #include "utils/builtins.h" @@ -67,6 +69,8 @@ pg_prewarm(PG_FUNCTION_ARGS) char *ttype; PrewarmType ptype; AclResult aclresult; + char relkind; + Oid privOid; /* Basic sanity checking. */ if (PG_ARGISNULL(0)) @@ -102,9 +106,43 @@ pg_prewarm(PG_FUNCTION_ARGS) forkString = text_to_cstring(forkName); forkNumber = forkname_to_number(forkString); - /* Open relation and check privileges. */ + /* + * Open relation and check privileges. If the relation is an index, we + * must check the privileges on its parent table instead. + */ + relkind = get_rel_relkind(relOid); + if (relkind == RELKIND_INDEX || + relkind == RELKIND_PARTITIONED_INDEX) + { + privOid = IndexGetRelation(relOid, true); + + /* Lock table before index to avoid deadlock. */ + if (OidIsValid(privOid)) + LockRelationOid(privOid, AccessShareLock); + } + else + privOid = relOid; + rel = relation_open(relOid, AccessShareLock); - aclresult = pg_class_aclcheck(relOid, GetUserId(), ACL_SELECT); + + /* + * It's possible that the relation with OID "privOid" was dropped and the + * OID was reused before we locked it. If that happens, we could be left + * with the wrong parent table OID, in which case we must ERROR. It's + * possible that such a race would change the outcome of + * get_rel_relkind(), too, but the worst case scenario there is that we'll + * check privileges on the index instead of its parent table, which isn't + * too terrible. + */ + if (!OidIsValid(privOid) || + (privOid != relOid && + privOid != IndexGetRelation(relOid, true))) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("could not find parent table of index \"%s\"", + RelationGetRelationName(rel)))); + + aclresult = pg_class_aclcheck(privOid, GetUserId(), ACL_SELECT); if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, get_relkind_objtype(rel->rd_rel->relkind), get_rel_name(relOid)); @@ -197,8 +235,11 @@ pg_prewarm(PG_FUNCTION_ARGS) } } - /* Close relation, release lock. */ + /* Close relation, release locks. */ relation_close(rel, AccessShareLock); + if (privOid != relOid) + UnlockRelationOid(privOid, AccessShareLock); + PG_RETURN_INT64(blocks_done); } diff --git a/contrib/pg_stat_statements/expected/pg_stat_statements.out b/contrib/pg_stat_statements/expected/pg_stat_statements.out index 49deebdfcacd6..89fde503ed647 100644 --- a/contrib/pg_stat_statements/expected/pg_stat_statements.out +++ b/contrib/pg_stat_statements/expected/pg_stat_statements.out @@ -462,6 +462,35 @@ SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C" | 0 | 0 (6 rows) +-- normalization of constants and parameters, with constant locations +-- recorded one or more times. +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +SELECT WHERE '1' IN ('1'::int, '3'::int::text); +-- +(1 row) + +SELECT WHERE (1, 2) IN ((1, 2), (2, 3)); +-- +(1 row) + +SELECT WHERE (3, 4) IN ((5, 6), (8, 7)); +-- +(0 rows) + +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + query | calls +------------------------------------------------------------------------+------- + SELECT WHERE $1 IN ($2::int, $3::int::text) | 1 + SELECT WHERE ($1, $2) IN (($3, $4), ($5, $6)) | 2 + SELECT pg_stat_statements_reset() IS NOT NULL AS t | 1 + SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C" | 0 +(4 rows) + -- -- queries with locking clauses -- diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index ae4f6e0a156d2..8d71ed71321cc 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -2674,6 +2674,7 @@ generate_normalized_query(JumbleState *jstate, const char *query, n_quer_loc = 0, /* Normalized query byte location */ last_off = 0, /* Offset from start for previous tok */ last_tok_len = 0; /* Length (in bytes) of that tok */ + int num_constants_replaced = 0; /* * Get constants' lengths (core system only gives us locations). Note @@ -2717,7 +2718,8 @@ generate_normalized_query(JumbleState *jstate, const char *query, /* And insert a param symbol in place of the constant token */ n_quer_loc += sprintf(norm_query + n_quer_loc, "$%d", - i + 1 + jstate->highest_extern_param_id); + num_constants_replaced + 1 + jstate->highest_extern_param_id); + num_constants_replaced++; quer_loc = off + tok_len; last_off = off; diff --git a/contrib/pg_stat_statements/sql/pg_stat_statements.sql b/contrib/pg_stat_statements/sql/pg_stat_statements.sql index 8828e74e8942b..7bf379a0a76df 100644 --- a/contrib/pg_stat_statements/sql/pg_stat_statements.sql +++ b/contrib/pg_stat_statements/sql/pg_stat_statements.sql @@ -223,6 +223,14 @@ SELECT PLUS_ONE(1); SELECT query, calls, rows FROM pg_stat_statements ORDER BY query COLLATE "C"; +-- normalization of constants and parameters, with constant locations +-- recorded one or more times. +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +SELECT WHERE '1' IN ('1'::int, '3'::int::text); +SELECT WHERE (1, 2) IN ((1, 2), (2, 3)); +SELECT WHERE (3, 4) IN ((5, 6), (8, 7)); +SELECT query, calls FROM pg_stat_statements ORDER BY query COLLATE "C"; + -- -- queries with locking clauses -- diff --git a/contrib/pg_trgm/Makefile b/contrib/pg_trgm/Makefile index 1fbdc9ec1ef43..c1756993ec7ba 100644 --- a/contrib/pg_trgm/Makefile +++ b/contrib/pg_trgm/Makefile @@ -14,7 +14,7 @@ DATA = pg_trgm--1.5--1.6.sql pg_trgm--1.4--1.5.sql pg_trgm--1.3--1.4.sql \ pg_trgm--1.0--1.1.sql PGFILEDESC = "pg_trgm - trigram matching" -REGRESS = pg_trgm pg_word_trgm pg_strict_word_trgm +REGRESS = pg_trgm pg_utf8_trgm pg_word_trgm pg_strict_word_trgm ifdef USE_PGXS PG_CONFIG = pg_config diff --git a/contrib/pg_trgm/data/trgm_utf8.data b/contrib/pg_trgm/data/trgm_utf8.data new file mode 100644 index 0000000000000..713856e76a625 --- /dev/null +++ b/contrib/pg_trgm/data/trgm_utf8.data @@ -0,0 +1,50 @@ +Mathematics +数学 +गणित +Matemáticas +رياضيات +Mathématiques +গণিত +Matemática +Математика +ریاضی +Matematika +Mathematik +数学 +Mathematics +गणित +గణితం +Matematik +கணிதம் +數學 +Toán học +Matematika +数学 +수학 +ریاضی +Lissafi +Hisabati +Matematika +Matematica +ریاضی +ಗಣಿತ +ગણિત +คณิตศาสตร์ +ሂሳብ +गणित +ਗਣਿਤ +數學 +数学 +Iṣiro +數學 +သင်္ချာ +Herrega +رياضي +गणित +Математика +Matematyka +ഗണിതം +Matematika +رياضي +Matematika +Matematică diff --git a/contrib/pg_trgm/expected/pg_trgm.out b/contrib/pg_trgm/expected/pg_trgm.out index 8b5dcf90d1864..250f12c346d0e 100644 --- a/contrib/pg_trgm/expected/pg_trgm.out +++ b/contrib/pg_trgm/expected/pg_trgm.out @@ -4693,6 +4693,23 @@ select count(*) from test_trgm where t like '%99%' and t like '%qw%'; 19 (1 row) +explain (costs off) +select count(*) from test_trgm where t %> '' and t %> '%qwerty%'; + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + -> Bitmap Heap Scan on test_trgm + Recheck Cond: ((t %> ''::text) AND (t %> '%qwerty%'::text)) + -> Bitmap Index Scan on trgm_idx + Index Cond: ((t %> ''::text) AND (t %> '%qwerty%'::text)) +(5 rows) + +select count(*) from test_trgm where t %> '' and t %> '%qwerty%'; + count +------- + 0 +(1 row) + -- ensure that pending-list items are handled correctly, too create temp table t_test_trgm(t text COLLATE "C"); create index t_trgm_idx on t_test_trgm using gin (t gin_trgm_ops); @@ -4731,6 +4748,23 @@ select count(*) from t_test_trgm where t like '%99%' and t like '%qw%'; 1 (1 row) +explain (costs off) +select count(*) from t_test_trgm where t %> '' and t %> '%qwerty%'; + QUERY PLAN +------------------------------------------------------------------------- + Aggregate + -> Bitmap Heap Scan on t_test_trgm + Recheck Cond: ((t %> ''::text) AND (t %> '%qwerty%'::text)) + -> Bitmap Index Scan on t_trgm_idx + Index Cond: ((t %> ''::text) AND (t %> '%qwerty%'::text)) +(5 rows) + +select count(*) from t_test_trgm where t %> '' and t %> '%qwerty%'; + count +------- + 0 +(1 row) + -- run the same queries with sequential scan to check the results set enable_bitmapscan=off; set enable_seqscan=on; @@ -4746,6 +4780,12 @@ select count(*) from test_trgm where t like '%99%' and t like '%qw%'; 19 (1 row) +select count(*) from test_trgm where t %> '' and t %> '%qwerty%'; + count +------- + 0 +(1 row) + select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%'; count ------- @@ -4758,6 +4798,12 @@ select count(*) from t_test_trgm where t like '%99%' and t like '%qw%'; 1 (1 row) +select count(*) from t_test_trgm where t %> '' and t %> '%qwerty%'; + count +------- + 0 +(1 row) + reset enable_bitmapscan; create table test2(t text COLLATE "C"); insert into test2 values ('abcdef'); diff --git a/contrib/pg_trgm/expected/pg_utf8_trgm.out b/contrib/pg_trgm/expected/pg_utf8_trgm.out new file mode 100644 index 0000000000000..0768e7d6a8320 --- /dev/null +++ b/contrib/pg_trgm/expected/pg_utf8_trgm.out @@ -0,0 +1,8 @@ +SELECT getdatabaseencoding() <> 'UTF8' AS skip_test \gset +\if :skip_test +\quit +\endif +-- Index 50 translations of the word "Mathematics" +CREATE TEMP TABLE mb (s text); +\copy mb from 'data/trgm_utf8.data' +CREATE INDEX ON mb USING gist(s gist_trgm_ops); diff --git a/contrib/pg_trgm/expected/pg_utf8_trgm_1.out b/contrib/pg_trgm/expected/pg_utf8_trgm_1.out new file mode 100644 index 0000000000000..8505c4fa55262 --- /dev/null +++ b/contrib/pg_trgm/expected/pg_utf8_trgm_1.out @@ -0,0 +1,3 @@ +SELECT getdatabaseencoding() <> 'UTF8' AS skip_test \gset +\if :skip_test +\quit diff --git a/contrib/pg_trgm/sql/pg_trgm.sql b/contrib/pg_trgm/sql/pg_trgm.sql index 340c9891899f0..44debced6d581 100644 --- a/contrib/pg_trgm/sql/pg_trgm.sql +++ b/contrib/pg_trgm/sql/pg_trgm.sql @@ -80,6 +80,9 @@ select count(*) from test_trgm where t like '%99%' and t like '%qwerty%'; explain (costs off) select count(*) from test_trgm where t like '%99%' and t like '%qw%'; select count(*) from test_trgm where t like '%99%' and t like '%qw%'; +explain (costs off) +select count(*) from test_trgm where t %> '' and t %> '%qwerty%'; +select count(*) from test_trgm where t %> '' and t %> '%qwerty%'; -- ensure that pending-list items are handled correctly, too create temp table t_test_trgm(t text COLLATE "C"); create index t_trgm_idx on t_test_trgm using gin (t gin_trgm_ops); @@ -90,14 +93,19 @@ select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%'; explain (costs off) select count(*) from t_test_trgm where t like '%99%' and t like '%qw%'; select count(*) from t_test_trgm where t like '%99%' and t like '%qw%'; +explain (costs off) +select count(*) from t_test_trgm where t %> '' and t %> '%qwerty%'; +select count(*) from t_test_trgm where t %> '' and t %> '%qwerty%'; -- run the same queries with sequential scan to check the results set enable_bitmapscan=off; set enable_seqscan=on; select count(*) from test_trgm where t like '%99%' and t like '%qwerty%'; select count(*) from test_trgm where t like '%99%' and t like '%qw%'; +select count(*) from test_trgm where t %> '' and t %> '%qwerty%'; select count(*) from t_test_trgm where t like '%99%' and t like '%qwerty%'; select count(*) from t_test_trgm where t like '%99%' and t like '%qw%'; +select count(*) from t_test_trgm where t %> '' and t %> '%qwerty%'; reset enable_bitmapscan; create table test2(t text COLLATE "C"); diff --git a/contrib/pg_trgm/sql/pg_utf8_trgm.sql b/contrib/pg_trgm/sql/pg_utf8_trgm.sql new file mode 100644 index 0000000000000..0dd962ced8310 --- /dev/null +++ b/contrib/pg_trgm/sql/pg_utf8_trgm.sql @@ -0,0 +1,9 @@ +SELECT getdatabaseencoding() <> 'UTF8' AS skip_test \gset +\if :skip_test +\quit +\endif + +-- Index 50 translations of the word "Mathematics" +CREATE TEMP TABLE mb (s text); +\copy mb from 'data/trgm_utf8.data' +CREATE INDEX ON mb USING gist(s gist_trgm_ops); diff --git a/contrib/pg_trgm/trgm.h b/contrib/pg_trgm/trgm.h index 405a1d95528da..06d3994e69277 100644 --- a/contrib/pg_trgm/trgm.h +++ b/contrib/pg_trgm/trgm.h @@ -52,10 +52,10 @@ typedef char trgm[3]; } while(0) #ifdef KEEPONLYALNUM -#define ISWORDCHR(c) (t_isalpha(c) || t_isdigit(c)) +#define ISWORDCHR(c, len) (t_isalpha_with_len(c, len) || t_isdigit_with_len(c, len)) #define ISPRINTABLECHAR(a) ( isascii( *(unsigned char*)(a) ) && (isalnum( *(unsigned char*)(a) ) || *(unsigned char*)(a)==' ') ) #else -#define ISWORDCHR(c) (!t_isspace(c)) +#define ISWORDCHR(c, len) (!t_isspace_with_len(c, len)) #define ISPRINTABLECHAR(a) ( isascii( *(unsigned char*)(a) ) && isprint( *(unsigned char*)(a) ) ) #endif #define ISPRINTABLETRGM(t) ( ISPRINTABLECHAR( ((char*)(t)) ) && ISPRINTABLECHAR( ((char*)(t))+1 ) && ISPRINTABLECHAR( ((char*)(t))+2 ) ) diff --git a/contrib/pg_trgm/trgm_op.c b/contrib/pg_trgm/trgm_op.c index e9b7981619fb6..fe7f1ca4412d1 100644 --- a/contrib/pg_trgm/trgm_op.c +++ b/contrib/pg_trgm/trgm_op.c @@ -173,18 +173,29 @@ static char * find_word(char *str, int lenstr, char **endword, int *charlen) { char *beginword = str; + const char *endstr = str + lenstr; - while (beginword - str < lenstr && !ISWORDCHR(beginword)) - beginword += pg_mblen(beginword); + while (beginword < endstr) + { + int clen = pg_mblen_range(beginword, endstr); - if (beginword - str >= lenstr) + if (ISWORDCHR(beginword, clen)) + break; + beginword += clen; + } + + if (beginword >= endstr) return NULL; *endword = beginword; *charlen = 0; - while (*endword - str < lenstr && ISWORDCHR(*endword)) + while (*endword < endstr) { - *endword += pg_mblen(*endword); + int clen = pg_mblen_range(*endword, endstr); + + if (!ISWORDCHR(*endword, clen)) + break; + *endword += clen; (*charlen)++; } @@ -232,9 +243,9 @@ make_trigrams(trgm *tptr, char *str, int bytelen, int charlen) if (bytelen > charlen) { /* Find multibyte character boundaries and apply compact_trigram */ - int lenfirst = pg_mblen(str), - lenmiddle = pg_mblen(str + lenfirst), - lenlast = pg_mblen(str + lenfirst + lenmiddle); + int lenfirst = pg_mblen_unbounded(str), + lenmiddle = pg_mblen_unbounded(str + lenfirst), + lenlast = pg_mblen_unbounded(str + lenfirst + lenmiddle); while ((ptr - str) + lenfirst + lenmiddle + lenlast <= bytelen) { @@ -245,7 +256,7 @@ make_trigrams(trgm *tptr, char *str, int bytelen, int charlen) lenfirst = lenmiddle; lenmiddle = lenlast; - lenlast = pg_mblen(ptr + lenfirst + lenmiddle); + lenlast = pg_mblen_unbounded(ptr + lenfirst + lenmiddle); } } else @@ -725,6 +736,7 @@ get_wildcard_part(const char *str, int lenstr, { const char *beginword = str; const char *endword; + const char *endstr = str + lenstr; char *s = buf; bool in_leading_wildcard_meta = false; bool in_trailing_wildcard_meta = false; @@ -737,11 +749,13 @@ get_wildcard_part(const char *str, int lenstr, * from this loop to the next one, since we may exit at a word character * that is in_escape. */ - while (beginword - str < lenstr) + while (beginword < endstr) { + clen = pg_mblen_range(beginword, endstr); + if (in_escape) { - if (ISWORDCHR(beginword)) + if (ISWORDCHR(beginword, clen)) break; in_escape = false; in_leading_wildcard_meta = false; @@ -752,12 +766,12 @@ get_wildcard_part(const char *str, int lenstr, in_escape = true; else if (ISWILDCARDCHAR(beginword)) in_leading_wildcard_meta = true; - else if (ISWORDCHR(beginword)) + else if (ISWORDCHR(beginword, clen)) break; else in_leading_wildcard_meta = false; } - beginword += pg_mblen(beginword); + beginword += clen; } /* @@ -790,12 +804,12 @@ get_wildcard_part(const char *str, int lenstr, * string boundary. Strip escapes during copy. */ endword = beginword; - while (endword - str < lenstr) + while (endword < endstr) { - clen = pg_mblen(endword); + clen = pg_mblen_range(endword, endstr); if (in_escape) { - if (ISWORDCHR(endword)) + if (ISWORDCHR(endword, clen)) { memcpy(s, endword, clen); (*charlen)++; @@ -823,7 +837,7 @@ get_wildcard_part(const char *str, int lenstr, in_trailing_wildcard_meta = true; break; } - else if (ISWORDCHR(endword)) + else if (ISWORDCHR(endword, clen)) { memcpy(s, endword, clen); (*charlen)++; diff --git a/contrib/pg_trgm/trgm_regexp.c b/contrib/pg_trgm/trgm_regexp.c index 3fc8a9ec6f0d6..f675bd0643743 100644 --- a/contrib/pg_trgm/trgm_regexp.c +++ b/contrib/pg_trgm/trgm_regexp.c @@ -480,7 +480,7 @@ static TRGM *createTrgmNFAInternal(regex_t *regex, TrgmPackedGraph **graph, static void RE_compile(regex_t *regex, text *text_re, int cflags, Oid collation); static void getColorInfo(regex_t *regex, TrgmNFA *trgmNFA); -static bool convertPgWchar(pg_wchar c, trgm_mb_char *result); +static int convertPgWchar(pg_wchar c, trgm_mb_char *result); static void transformGraph(TrgmNFA *trgmNFA); static void processState(TrgmNFA *trgmNFA, TrgmState *state); static void addKey(TrgmNFA *trgmNFA, TrgmState *state, TrgmStateKey *key); @@ -818,10 +818,11 @@ getColorInfo(regex_t *regex, TrgmNFA *trgmNFA) for (j = 0; j < charsCount; j++) { trgm_mb_char c; + int clen = convertPgWchar(chars[j], &c); - if (!convertPgWchar(chars[j], &c)) + if (!clen) continue; /* ok to ignore it altogether */ - if (ISWORDCHR(c.bytes)) + if (ISWORDCHR(c.bytes, clen)) colorInfo->wordChars[colorInfo->wordCharsCount++] = c; else colorInfo->containsNonWord = true; @@ -833,13 +834,15 @@ getColorInfo(regex_t *regex, TrgmNFA *trgmNFA) /* * Convert pg_wchar to multibyte format. - * Returns false if the character should be ignored completely. + * Returns 0 if the character should be ignored completely, else returns its + * byte length. */ -static bool +static int convertPgWchar(pg_wchar c, trgm_mb_char *result) { /* "s" has enough space for a multibyte character and a trailing NUL */ char s[MAX_MULTIBYTE_CHAR_LEN + 1]; + int clen; /* * We can ignore the NUL character, since it can never appear in a PG text @@ -847,11 +850,11 @@ convertPgWchar(pg_wchar c, trgm_mb_char *result) * reconstructing trigrams. */ if (c == 0) - return false; + return 0; /* Do the conversion, making sure the result is NUL-terminated */ memset(s, 0, sizeof(s)); - pg_wchar2mb_with_len(&c, s, 1); + clen = pg_wchar2mb_with_len(&c, s, 1); /* * In IGNORECASE mode, we can ignore uppercase characters. We assume that @@ -873,7 +876,7 @@ convertPgWchar(pg_wchar c, trgm_mb_char *result) if (strcmp(lowerCased, s) != 0) { pfree(lowerCased); - return false; + return 0; } pfree(lowerCased); } @@ -881,7 +884,7 @@ convertPgWchar(pg_wchar c, trgm_mb_char *result) /* Fill result with exactly MAX_MULTIBYTE_CHAR_LEN bytes */ memcpy(result->bytes, s, MAX_MULTIBYTE_CHAR_LEN); - return true; + return clen; } diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile index 7fb59f51b729f..52a5cefb03397 100644 --- a/contrib/pgcrypto/Makefile +++ b/contrib/pgcrypto/Makefile @@ -43,7 +43,8 @@ REGRESS = init md5 sha1 hmac-md5 hmac-sha1 blowfish rijndael \ sha2 des 3des cast5 \ crypt-des crypt-md5 crypt-blowfish crypt-xdes \ pgp-armor pgp-decrypt pgp-encrypt $(CF_PGP_TESTS) \ - pgp-pubkey-decrypt pgp-pubkey-encrypt pgp-info + pgp-pubkey-decrypt pgp-pubkey-encrypt pgp-pubkey-session \ + pgp-info EXTRA_CLEAN = gen-rtab diff --git a/contrib/pgcrypto/expected/pgp-decrypt.out b/contrib/pgcrypto/expected/pgp-decrypt.out index eb049ba9d4443..8ce6466f2e9a8 100644 --- a/contrib/pgcrypto/expected/pgp-decrypt.out +++ b/contrib/pgcrypto/expected/pgp-decrypt.out @@ -315,7 +315,7 @@ SaV9L04ky1qECNDx3XjnoKLC+H7IOQ== \xda39a3ee5e6b4b0d3255bfef95601890afd80709 (1 row) -select digest(pgp_sym_decrypt(dearmor(' +select digest(pgp_sym_decrypt_bytea(dearmor(' -----BEGIN PGP MESSAGE----- Comment: dat3.aes.sha1.mdc.s2k3.z0 @@ -387,6 +387,28 @@ ERROR: Wrong key or corrupt data select pgp_sym_decrypt(pgp_sym_encrypt_bytea('P', 'key'), 'key', 'debug=1'); NOTICE: dbg: parse_literal_data: data type=b ERROR: Not text data +-- NUL byte in text decrypt. Ciphertext source: +-- printf 'a\x00\xc' | gpg --homedir /nonexistent \ +-- --personal-compress-preferences uncompressed --textmode \ +-- --personal-cipher-preferences aes --no-emit-version --batch \ +-- --symmetric --passphrase key --armor +do $$ +begin + perform pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +jA0EBwMCXLc8pozB10Fg0jQBVUID59TLvWutJp0j6eh9ZgjqIRzdYaIymFB8y4XH +vu0YlJP5D5BX7yqZ+Pry7TlDmiFO +=rV7z +-----END PGP MESSAGE----- +'), 'key', 'debug=1'); +exception when others then + raise '%', + regexp_replace(sqlerrm, 'encoding "[^"]*"', 'encoding [REDACTED]'); +end +$$; +ERROR: invalid byte sequence for encoding [REDACTED]: 0x00 +CONTEXT: PL/pgSQL function inline_code_block line 12 at RAISE -- Decryption with a certain incorrect key yields an apparent BZip2-compressed -- plaintext. Ciphertext source: iterative pgp_sym_encrypt('secret', 'key') -- until the random prefix gave rise to that property. diff --git a/contrib/pgcrypto/expected/pgp-decrypt_1.out b/contrib/pgcrypto/expected/pgp-decrypt_1.out index 80a4c48613d00..ee57ad43cb75f 100644 --- a/contrib/pgcrypto/expected/pgp-decrypt_1.out +++ b/contrib/pgcrypto/expected/pgp-decrypt_1.out @@ -311,7 +311,7 @@ SaV9L04ky1qECNDx3XjnoKLC+H7IOQ== \xda39a3ee5e6b4b0d3255bfef95601890afd80709 (1 row) -select digest(pgp_sym_decrypt(dearmor(' +select digest(pgp_sym_decrypt_bytea(dearmor(' -----BEGIN PGP MESSAGE----- Comment: dat3.aes.sha1.mdc.s2k3.z0 @@ -383,6 +383,28 @@ ERROR: Wrong key or corrupt data select pgp_sym_decrypt(pgp_sym_encrypt_bytea('P', 'key'), 'key', 'debug=1'); NOTICE: dbg: parse_literal_data: data type=b ERROR: Not text data +-- NUL byte in text decrypt. Ciphertext source: +-- printf 'a\x00\xc' | gpg --homedir /nonexistent \ +-- --personal-compress-preferences uncompressed --textmode \ +-- --personal-cipher-preferences aes --no-emit-version --batch \ +-- --symmetric --passphrase key --armor +do $$ +begin + perform pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +jA0EBwMCXLc8pozB10Fg0jQBVUID59TLvWutJp0j6eh9ZgjqIRzdYaIymFB8y4XH +vu0YlJP5D5BX7yqZ+Pry7TlDmiFO +=rV7z +-----END PGP MESSAGE----- +'), 'key', 'debug=1'); +exception when others then + raise '%', + regexp_replace(sqlerrm, 'encoding "[^"]*"', 'encoding [REDACTED]'); +end +$$; +ERROR: invalid byte sequence for encoding [REDACTED]: 0x00 +CONTEXT: PL/pgSQL function inline_code_block line 12 at RAISE -- Decryption with a certain incorrect key yields an apparent BZip2-compressed -- plaintext. Ciphertext source: iterative pgp_sym_encrypt('secret', 'key') -- until the random prefix gave rise to that property. diff --git a/contrib/pgcrypto/expected/pgp-pubkey-session.out b/contrib/pgcrypto/expected/pgp-pubkey-session.out new file mode 100644 index 0000000000000..e57cb8fab99c3 --- /dev/null +++ b/contrib/pgcrypto/expected/pgp-pubkey-session.out @@ -0,0 +1,47 @@ +-- Test for overflow with session key at decrypt. +-- Data automatically generated by scripts/pgp_session_data.py. +-- See this file for details explaining how this data is generated. +SELECT pgp_pub_decrypt_bytea( +'\xc1c04c030000000000000000020800a46f5b9b1905b49457a6485474f71ed9b46c2527e1 +da08e1f7871e12c3d38828f2076b984a595bf60f616599ca5729d547de06a258bfbbcd30 +94a321e4668cd43010f0ca8ecf931e5d39bda1152c50c367b11c723f270729245d3ebdbd +0694d320c5a5aa6a405fb45182acb3d7973cbce398e0c5060af7603cfd9ed186ebadd616 +3b50ae42bea5f6d14dda24e6d4687b434c175084515d562e896742b0ba9a1c87d5642e10 +a5550379c71cc490a052ada483b5d96526c0a600fc51755052aa77fdf72f7b4989b920e7 +b90f4b30787a46482670d5caecc7a515a926055ad5509d135702ce51a0e4c1033f2d939d +8f0075ec3428e17310da37d3d2d7ad1ce99adcc91cd446c366c402ae1ee38250343a7fcc +0f8bc28020e603d7a4795ef0dcc1c04c030000000000000000020800a46f5b9b1905b494 +57a6485474f71ed9b46c2527e1da08e1f7871e12c3d38828f2076b984a595bf60f616599 +ca5729d547de06a258bfbbcd3094a321e4668cd43010f0ca8ecf931e5d39bda1152c50c3 +67b11c723f270729245d3ebdbd0694d320c5a5aa6a405fb45182acb3d7973cbce398e0c5 +060af7603cfd9ed186ebadd6163b50ae42bea5f6d14dda24e6d4687b434c175084515d56 +2e896742b0ba9a1c87d5642e10a5550379c71cc490a052ada483b5d96526c0a600fc5175 +5052aa77fdf72f7b4989b920e7b90f4b30787a46482670d5caecc7a515a926055ad5509d +135702ce51a0e4c1033f2d939d8f0075ec3428e17310da37d3d2d7ad1ce99adc'::bytea, +'\xc7c2d8046965d657020800eef8bf1515adb1a3ee7825f75c668ea8dd3e3f9d13e958f6ad +9c55adc0c931a4bb00abe1d52cf7bb0c95d537949d277a5292ede375c6b2a67a3bf7d19f +f975bb7e7be35c2d8300dacba360a0163567372f7dc24000cc7cb6170bedc8f3b1f98c12 +07a6cb4de870a4bc61319b139dcc0e20c368fd68f8fd346d2c0b69c5aed560504e2ec6f1 +23086fe3c5540dc4dd155c0c67257c4ada862f90fe172ace344089da8135e92aca5c2709 +f1c1bc521798bb8c0365841496e709bd184132d387e0c9d5f26dc00fd06c3a76ef66a75c +138285038684707a847b7bd33cfbefbf1d336be954a8048946af97a66352adef8e8b5ae4 +c4748c6f2510265b7a8267bc370dbb00110100010007ff7e72d4f95d2d39901ac12ca5c5 +18e767e719e72340c3fab51c8c5ab1c40f31db8eaffe43533fa61e2dbca2c3f4396c0847 +e5434756acbb1f68128f4136bb135710c89137d74538908dac77967de9e821c559700dd9 +de5a2727eec1f5d12d5d74869dd1de45ed369d94a8814d23861dd163f8c27744b26b98f0 +239c2e6dd1e3493b8cc976fdc8f9a5e250f715aa4c3d7d5f237f8ee15d242e8fa941d1a0 +ed9550ab632d992a97518d142802cb0a97b251319bf5742db8d9d8cbaa06cdfba2d75bc9 +9d77a51ff20bd5ba7f15d7af6e85b904de2855d19af08d45f39deb85403033c69c767a8e +74a343b1d6c8911d34ea441ac3850e57808ed3d885835cbe6c79d10400ef16256f3d5c4c +3341516a2d2aa888df81b603f48a27f3666b40f992a857c1d11ff639cd764a9b42d5a1f8 +58b4aeee36b85508bb5e8b91ef88a7737770b330224479d9b44eae8c631bc43628b69549 +507c0a1af0be0dd7696015abea722b571eb35eefc4ab95595378ec12814727443f625fcd +183bb9b3bccf53b54dd0e5e7a50400ffe08537b2d4e6074e4a1727b658cfccdec8962302 +25e300c05690de45f7065c3d40d86f544a64d51a3e94424f9851a16d1322ebdb41fa8a45 +3131f3e2dc94e858e6396722643df382680f815e53bcdcde5da622f50530a83b217f1103 +cdd6e5e9babe1e415bbff28d44bd18c95f43bbd04afeb2a2a99af38a571c7540de21df03 +ff62c0a33d9143dd3f639893f47732c11c5a12c6052d1935f4d507b7ae1f76ab0e9a69b8 +7305a7f7c19bd509daf4903bff614bc26d118f03e461469c72c12d3a2bb4f78e4d342ce8 +487723649a01ed2b9eb11c662134502c098d55dfcd361939d8370873422c3da75a515a75 +9ffedfe7df44fb3c20f81650801a30d43b5c90b98b3eee'::bytea); +ERROR: Session key too big diff --git a/contrib/pgcrypto/pgp-pgsql.c b/contrib/pgcrypto/pgp-pgsql.c index 0536bfb8921c9..cf315b126b748 100644 --- a/contrib/pgcrypto/pgp-pgsql.c +++ b/contrib/pgcrypto/pgp-pgsql.c @@ -631,6 +631,7 @@ pgp_sym_decrypt_text(PG_FUNCTION_ARGS) arg = PG_GETARG_BYTEA_PP(2); res = decrypt_internal(0, 1, data, key, NULL, arg); + pg_verifymbstr(VARDATA_ANY(res), VARSIZE_ANY_EXHDR(res), false); PG_FREE_IF_COPY(data, 0); PG_FREE_IF_COPY(key, 1); @@ -732,6 +733,7 @@ pgp_pub_decrypt_text(PG_FUNCTION_ARGS) arg = PG_GETARG_BYTEA_PP(3); res = decrypt_internal(1, 1, data, key, psw, arg); + pg_verifymbstr(VARDATA_ANY(res), VARSIZE_ANY_EXHDR(res), false); PG_FREE_IF_COPY(data, 0); PG_FREE_IF_COPY(key, 1); diff --git a/contrib/pgcrypto/pgp-pubdec.c b/contrib/pgcrypto/pgp-pubdec.c index a0a5738a40e89..2a13aa3e6adff 100644 --- a/contrib/pgcrypto/pgp-pubdec.c +++ b/contrib/pgcrypto/pgp-pubdec.c @@ -157,6 +157,7 @@ pgp_parse_pubenc_sesskey(PGP_Context *ctx, PullFilter *pkt) uint8 *msg; int msglen; PGP_MPI *m; + unsigned sess_key_len; pk = ctx->pub_key; if (pk == NULL) @@ -220,11 +221,19 @@ pgp_parse_pubenc_sesskey(PGP_Context *ctx, PullFilter *pkt) if (res < 0) goto out; + sess_key_len = msglen - 3; + if (sess_key_len > PGP_MAX_KEY) + { + px_debug("incorrect session key length=%u", sess_key_len); + res = PXE_PGP_KEY_TOO_BIG; + goto out; + } + /* * got sesskey */ ctx->cipher_algo = *msg; - ctx->sess_key_len = msglen - 3; + ctx->sess_key_len = sess_key_len; memcpy(ctx->sess_key, msg + 1, ctx->sess_key_len); out: diff --git a/contrib/pgcrypto/px.c b/contrib/pgcrypto/px.c index 3b098c61514a4..60248de48efc5 100644 --- a/contrib/pgcrypto/px.c +++ b/contrib/pgcrypto/px.c @@ -65,6 +65,7 @@ static const struct error_desc px_err_list[] = { {PXE_PGP_UNEXPECTED_PKT, "Unexpected packet in key data"}, {PXE_PGP_MATH_FAILED, "Math operation failed"}, {PXE_PGP_SHORT_ELGAMAL_KEY, "Elgamal keys must be at least 1024 bits long"}, + {PXE_PGP_KEY_TOO_BIG, "Session key too big"}, {PXE_PGP_UNKNOWN_PUBALGO, "Unknown public-key encryption algorithm"}, {PXE_PGP_WRONG_KEY, "Wrong key"}, {PXE_PGP_MULTIPLE_KEYS, diff --git a/contrib/pgcrypto/px.h b/contrib/pgcrypto/px.h index 4ef40f3f1c53f..be2431c1b145a 100644 --- a/contrib/pgcrypto/px.h +++ b/contrib/pgcrypto/px.h @@ -75,7 +75,7 @@ /* -108 is unused */ #define PXE_PGP_MATH_FAILED -109 #define PXE_PGP_SHORT_ELGAMAL_KEY -110 -/* -111 is unused */ +#define PXE_PGP_KEY_TOO_BIG -111 #define PXE_PGP_UNKNOWN_PUBALGO -112 #define PXE_PGP_WRONG_KEY -113 #define PXE_PGP_MULTIPLE_KEYS -114 diff --git a/contrib/pgcrypto/scripts/pgp_session_data.py b/contrib/pgcrypto/scripts/pgp_session_data.py new file mode 100644 index 0000000000000..999350bb2bc91 --- /dev/null +++ b/contrib/pgcrypto/scripts/pgp_session_data.py @@ -0,0 +1,491 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Generate PGP data to check the session key length of the input data provided +# to pgp_pub_decrypt_bytea(). +# +# First, the crafted data is generated from valid RSA data, freshly generated +# by this script each time it is run, see generate_rsa_keypair(). +# Second, the crafted PGP data is built, see build_message_data() and +# build_key_data(). Finally, the resulting SQL script is generated. +# +# This script generates in stdout the SQL file that is used in the regression +# tests of pgcrypto. The following command can be used to regenerate the file +# which should never be manually manipulated: +# python3 scripts/pgp_session_data.py > sql/pgp-pubkey-session.sql + +import os +import re +import struct +import secrets +import sys +import time + +# pwn for binary manipulation (p32, p64) +from pwn import * + +# Cryptographic libraries, to craft the PGP data. +from Crypto.Cipher import AES +from Crypto.PublicKey import RSA +from Crypto.Util.number import inverse + +# AES key used for session key encryption (16 bytes for AES-128) +AES_KEY = b'\x01' * 16 + +def generate_rsa_keypair(key_size: int = 2048) -> dict: + """ + Generate a fresh RSA key pair. + + The generated key includes all components needed for PGP operations: + - n: public modulus (p * q) + - e: public exponent (typically 65537) + - d: private exponent (e^-1 mod phi(n)) + - p, q: prime factors of n + - u: coefficient (p^-1 mod q) for CRT optimization + + The caller can pass the wanted key size in input, for a default of 2048 + bytes. This function returns the RSA key components, after performing + some validation on them. + """ + + start_time = time.time() + + # Generate RSA key + key = RSA.generate(key_size) + + # Extract all key components + rsa_components = { + 'n': key.n, # Public modulus (p * q) + 'e': key.e, # Public exponent (typically 65537) + 'd': key.d, # Private exponent (e^-1 mod phi(n)) + 'p': key.p, # First prime factor + 'q': key.q, # Second prime factor + 'u': inverse(key.p, key.q) # Coefficient for CRT: p^-1 mod q + } + + # Validate key components for correctness + validate_rsa_key(rsa_components) + + return rsa_components + +def validate_rsa_key(rsa: dict) -> None: + """ + Validate a generated RSA key. + + This function performs basic validation to ensure the RSA key is properly + constructed and all components are consistent, at least mathematically. + + Validations performed: + 1. n = p * q (modulus is product of primes) + 2. gcd(e, phi(n)) = 1 (public exponent is coprime to phi(n)) + 3. (d * e) mod(phi(n)) = 1 (private exponent is multiplicative inverse) + 4. (u * p) (mod q) = 1 (coefficient is correct for CRT) + """ + + n, e, d, p, q, u = rsa['n'], rsa['e'], rsa['d'], rsa['p'], rsa['q'], rsa['u'] + + # Check that n = p * q + if n != p * q: + raise ValueError("RSA validation failed: n <> p * q") + + # Check that p and q are different + if p == q: + raise ValueError("RSA validation failed: p = q (not allowed)") + + # Calculate phi(n) = (p-1)(q-1) + phi_n = (p - 1) * (q - 1) + + # Check that gcd(e, phi(n)) = 1 + def gcd(a, b): + while b: + a, b = b, a % b + return a + + if gcd(e, phi_n) != 1: + raise ValueError("RSA validation failed: gcd(e, phi(n)) <> 1") + + # Check that (d * e) mod(phi(n)) = 1 + if (d * e) % phi_n != 1: + raise ValueError("RSA validation failed: d * e <> 1 (mod phi(n))") + + # Check that (u * p) (mod q) = 1 + if (u * p) % q != 1: + raise ValueError("RSA validation failed: u * p <> 1 (mod q)") + +def mpi_encode(x: int) -> bytes: + """ + Encode an integer as an OpenPGP Multi-Precision Integer (MPI). + + Format (RFC 4880, Section 3.2): + - 2 bytes: bit length of the integer (big-endian) + - N bytes: the integer in big-endian format + + This is used to encode RSA key components (n, e, d, p, q, u) in PGP + packets. + + The integer to encode is given in input, returning an MPI-encoded + integer. + + For example: + mpi_encode(65537) -> b'\x00\x11\x01\x00\x01' + (17 bits, value 0x010001) + """ + if x < 0: + raise ValueError("MPI cannot encode negative integers") + + if x == 0: + # Special case: zero has 0 bits and empty magnitude + bits = 0 + mag = b"" + else: + # Calculate bit length and convert to bytes + bits = x.bit_length() + mag = x.to_bytes((bits + 7) // 8, 'big') + + # Pack: 2-byte bit length + magnitude bytes + return struct.pack('>H', bits) + mag + +def new_packet(tag: int, payload: bytes) -> bytes: + """ + Create a new OpenPGP packet with a proper header. + + OpenPGP packet format (RFC 4880, Section 4.2): + - New packet format: 0xC0 | tag + - Length encoding depends on payload size: + * 0-191: single byte + * 192-8383: two bytes (192 + ((length - 192) >> 8), (length - 192) & 0xFF) + * 8384+: five bytes (0xFF + 4-byte big-endian length) + + The packet is built from a "tag" (1-63) and some "payload" data. The + result generated is a complete OpenPGP packet. + + For example: + new_packet(1, b'data') -> b'\xC1\x04data' + (Tag 1, length 4, payload 'data') + """ + # New packet format: set bit 7 and 6, clear bit 5, tag in bits 0-5 + first = 0xC0 | (tag & 0x3F) + ln = len(payload) + + # Encode length according to OpenPGP specification + if ln <= 191: + # Single byte length for small packets + llen = bytes([ln]) + elif ln <= 8383: + # Two-byte length for medium packets + ln2 = ln - 192 + llen = bytes([192 + (ln2 >> 8), ln2 & 0xFF]) + else: + # Five-byte length for large packets + llen = bytes([255]) + struct.pack('>I', ln) + + return bytes([first]) + llen + payload + +def build_key_data(rsa: dict) -> bytes: + """ + Build the key data, containing an RSA private key. + + The RSA contents should have been generated previously. + + Format (see RFC 4880, Section 5.5.3): + - 1 byte: version (4) + - 4 bytes: creation time (current Unix timestamp) + - 1 byte: public key algorithm (2 = RSA encrypt) + - MPI: RSA public modulus n + - MPI: RSA public exponent e + - 1 byte: string-to-key usage (0 = no encryption) + - MPI: RSA private exponent d + - MPI: RSA prime p + - MPI: RSA prime q + - MPI: RSA coefficient u = p^-1 mod q + - 2 bytes: checksum of private key material + + This function takes a set of RSA key components in input (n, e, d, p, q, u) + and returns a secret key packet. + """ + + # Public key portion + ver = bytes([4]) # Version 4 key + ctime = struct.pack('>I', int(time.time())) # Current Unix timestamp + algo = bytes([2]) # RSA encrypt algorithm + n_mpi = mpi_encode(rsa['n']) # Public modulus + e_mpi = mpi_encode(rsa['e']) # Public exponent + pub = ver + ctime + algo + n_mpi + e_mpi + + # Private key portion + hide_type = bytes([0]) # No string-to-key encryption + d_mpi = mpi_encode(rsa['d']) # Private exponent + p_mpi = mpi_encode(rsa['p']) # Prime p + q_mpi = mpi_encode(rsa['q']) # Prime q + u_mpi = mpi_encode(rsa['u']) # Coefficient u = p^-1 mod q + + # Calculate checksum of private key material (simple sum mod 65536) + private_data = d_mpi + p_mpi + q_mpi + u_mpi + cksum = sum(private_data) & 0xFFFF + + secret = hide_type + private_data + struct.pack('>H', cksum) + payload = pub + secret + + return new_packet(7, payload) + +def pgp_cfb_encrypt_resync(key, plaintext): + """ + Implement OpenPGP CFB mode with resync. + + OpenPGP CFB mode is a variant of standard CFB with a resync operation + after the first two blocks. + + Algorithm (RFC 4880, Section 13.9): + 1. Block 1: FR=zeros, encrypt full block_size bytes + 2. Block 2: FR=block1, encrypt only 2 bytes + 3. Resync: FR = block1[2:] + block2 + 4. Remaining blocks: standard CFB mode + + This function uses the following arguments: + - key: AES encryption key (16 bytes for AES-128) + - plaintext: Data to encrypt + """ + block_size = 16 # AES block size + cipher = AES.new(key[:16], AES.MODE_ECB) # Use ECB for manual CFB + ciphertext = b'' + + # Block 1: FR=zeros, encrypt full 16 bytes + FR = b'\x00' * block_size + FRE = cipher.encrypt(FR) # Encrypt the feedback register + block1 = bytes(a ^ b for a, b in zip(FRE, plaintext[0:16])) + ciphertext += block1 + + # Block 2: FR=block1, encrypt only 2 bytes + FR = block1 + FRE = cipher.encrypt(FR) + block2 = bytes(a ^ b for a, b in zip(FRE[0:2], plaintext[16:18])) + ciphertext += block2 + + # Resync: FR = block1[2:16] + block2[0:2] + # This is the key difference from standard CFB mode + FR = block1[2:] + block2 + + # Block 3+: Continue with standard CFB mode + pos = 18 + while pos < len(plaintext): + FRE = cipher.encrypt(FR) + chunk_len = min(block_size, len(plaintext) - pos) + chunk = plaintext[pos:pos+chunk_len] + enc_chunk = bytes(a ^ b for a, b in zip(FRE[:chunk_len], chunk)) + ciphertext += enc_chunk + + # Update feedback register for next iteration + if chunk_len == block_size: + FR = enc_chunk + else: + # Partial block: pad with old FR bytes + FR = enc_chunk + FR[chunk_len:] + pos += chunk_len + + return ciphertext + +def build_literal_data_packet(data: bytes) -> bytes: + """ + Build a literal data packet containing a message. + + Format (RFC 4880, Section 5.9): + - 1 byte: data format ('b' = binary, 't' = text, 'u' = UTF-8 text) + - 1 byte: filename length (0 = no filename) + - N bytes: filename (empty in this case) + - 4 bytes: date (current Unix timestamp) + - M bytes: literal data + + The data used to build the packet is given in input, with the generated + result returned. + """ + body = bytes([ + ord('b'), # Binary data format + 0, # Filename length (0 = no filename) + ]) + struct.pack('>I', int(time.time())) + data # Current timestamp + data + + return new_packet(11, body) + +def build_symenc_data_packet(sess_key: bytes, cipher_algo: int, payload: bytes) -> bytes: + """ + Build a symmetrically-encrypted data packet using AES-128-CFB. + + This packet contains encrypted data using the session key. The format + includes a random prefix, for security (see RFC 4880, Section 5.7). + + Packet structure: + - Random prefix (block_size bytes) + - Prefix repeat (last 2 bytes of prefix repeated) + - Encrypted literal data packet + + This function uses the following set of arguments: + - sess_key: Session key for encryption + - cipher_algo: Cipher algorithm identifier (7 = AES-128) + - payload: Data to encrypt (wrapped in literal data packet) + """ + block_size = 16 # AES-128 block size + key = sess_key[:16] # Use first 16 bytes for AES-128 + + # Create random prefix + repeat last 2 bytes (total 18 bytes) + # This is required by OpenPGP for integrity checking + prefix_random = secrets.token_bytes(block_size) + prefix = prefix_random + prefix_random[-2:] # 18 bytes total + + # Wrap payload in literal data packet + literal_pkt = build_literal_data_packet(payload) + + # Plaintext = prefix + literal data packet + plaintext = prefix + literal_pkt + + # Encrypt using OpenPGP CFB mode with resync + ciphertext = pgp_cfb_encrypt_resync(key, plaintext) + + return new_packet(9, ciphertext) + +def build_tag1_packet(rsa: dict, sess_key: bytes) -> bytes: + """ + Build a public-key encrypted key. + + This is a very important function, as it is able to create the packet + triggering the overflow check. This function can also be used to create + "legit" packet data. + + Format (RFC 4880, Section 5.1): + - 1 byte: version (3) + - 8 bytes: key ID (0 = any key accepted) + - 1 byte: public key algorithm (2 = RSA encrypt) + - MPI: RSA-encrypted session key + + This uses in arguments the generated RSA key pair, and the session key + to encrypt. The latter is manipulated to trigger the overflow. + + This function returns a complete packet encrypted by a session key. + """ + + # Calculate RSA modulus size in bytes + n_bytes = (rsa['n'].bit_length() + 7) // 8 + + # Session key message format: + # - 1 byte: symmetric cipher algorithm (7 = AES-128) + # - N bytes: session key + # - 2 bytes: checksum (simple sum of session key bytes) + algo_byte = bytes([7]) # AES-128 algorithm identifier + cksum = sum(sess_key) & 0xFFFF # 16-bit checksum + M = algo_byte + sess_key + struct.pack('>H', cksum) + + # PKCS#1 v1.5 padding construction + # Format: 0x02 || PS || 0x00 || M + # Total padded message must be exactly n_bytes long. + total_len = n_bytes # Total length must equal modulus size in bytes + ps_len = total_len - len(M) - 2 # Subtract 2 for 0x02 and 0x00 bytes + + if ps_len < 8: + raise ValueError(f"Padding string too short ({ps_len} bytes); need at least 8 bytes. " + f"Message length: {len(M)}, Modulus size: {n_bytes} bytes") + + # Create padding string with *ALL* bytes being 0xFF (no zero separator!) + PS = bytes([0xFF]) * ps_len + + # Construct the complete padded message + # Normal PKCS#1 v1.5 padding: 0x02 || PS || 0x00 || M + padded = bytes([0x02]) + PS + bytes([0x00]) + M + + # Verify padding construction + if len(padded) != n_bytes: + raise ValueError(f"Padded message length ({len(padded)}) doesn't match RSA modulus size ({n_bytes})") + + # Convert padded message to integer and encrypt with RSA + m_int = int.from_bytes(padded, 'big') + + # Ensure message is smaller than modulus (required for RSA) + if m_int >= rsa['n']: + raise ValueError("Padded message is larger than RSA modulus") + + # RSA encryption: c = m^e mod n + c_int = pow(m_int, rsa['e'], rsa['n']) + + # Encode encrypted result as MPI + c_mpi = mpi_encode(c_int) + + # Build complete packet + ver = bytes([3]) # Version 3 packet + key_id = b"\x00" * 8 # Key ID (0 = any key accepted) + algo = bytes([2]) # RSA encrypt algorithm + payload = ver + key_id + algo + c_mpi + + return new_packet(1, payload) + +def build_message_data(rsa: dict) -> bytes: + """ + This function creates a crafted message, with a long session key + length. + + This takes in input the RSA key components generated previously, + returning a concatenated set of PGP packets crafted for the purpose + of this test. + """ + + # Base prefix for session key (AES key + padding + size). + # Note that the crafted size is the important part for this test. + prefix = AES_KEY + b"\x00" * 16 + p32(0x10) + + # Build encrypted data packet, legit. + sedata = build_symenc_data_packet(AES_KEY, cipher_algo=7, payload=b"\x0a\x00") + + # Build multiple packets + packets = [ + # First packet, legit. + build_tag1_packet(rsa, prefix), + + # Encrypted data packet, legit. + sedata, + + # Second packet: information payload. + # + # This packet contains a longer-crafted session key, able to trigger + # the overflow check in pgcrypto. This is the critical part, and + # and you are right to pay a lot of attention here if you are + # reading this code. + build_tag1_packet(rsa, prefix) + ] + + return b"".join(packets) + +def main(): + # Default key size. + # This number can be set to a higher number if wanted, like 4096. We + # just do not need to do that here. + key_size = 2048 + + # Generate fresh RSA key pair + rsa = generate_rsa_keypair(key_size) + + # Generate the message data. + print("### Building message data", file=sys.stderr) + message_data = build_message_data(rsa) + + # Build the key containing the RSA private key + print("### Building key data", file=sys.stderr) + key_data = build_key_data(rsa) + + # Convert to hexadecimal, for the bytea used in the SQL file. + message_data = message_data.hex() + key_data = key_data.hex() + + # Split each value into lines of 72 characters, for readability. + message_data = re.sub("(.{72})", "\\1\n", message_data, 0, re.DOTALL) + key_data = re.sub("(.{72})", "\\1\n", key_data, 0, re.DOTALL) + + # Get the script filename for documentation + file_basename = os.path.basename(__file__) + + # Output the SQL test case + print(f'''-- Test for overflow with session key at decrypt. +-- Data automatically generated by scripts/{file_basename}. +-- See this file for details explaining how this data is generated. +SELECT pgp_pub_decrypt_bytea( +'\\x{message_data}'::bytea, +'\\x{key_data}'::bytea);''', + file=sys.stdout) + +if __name__ == "__main__": + main() diff --git a/contrib/pgcrypto/sql/pgp-decrypt.sql b/contrib/pgcrypto/sql/pgp-decrypt.sql index 49a0267bbcbcc..b499bf757b0d4 100644 --- a/contrib/pgcrypto/sql/pgp-decrypt.sql +++ b/contrib/pgcrypto/sql/pgp-decrypt.sql @@ -228,7 +228,7 @@ SaV9L04ky1qECNDx3XjnoKLC+H7IOQ== -----END PGP MESSAGE----- '), '0123456789abcdefghij'), 'sha1'); -select digest(pgp_sym_decrypt(dearmor(' +select digest(pgp_sym_decrypt_bytea(dearmor(' -----BEGIN PGP MESSAGE----- Comment: dat3.aes.sha1.mdc.s2k3.z0 @@ -282,6 +282,27 @@ VsxxqLSPzNLAeIspJk5G -- Routine text/binary mismatch. select pgp_sym_decrypt(pgp_sym_encrypt_bytea('P', 'key'), 'key', 'debug=1'); +-- NUL byte in text decrypt. Ciphertext source: +-- printf 'a\x00\xc' | gpg --homedir /nonexistent \ +-- --personal-compress-preferences uncompressed --textmode \ +-- --personal-cipher-preferences aes --no-emit-version --batch \ +-- --symmetric --passphrase key --armor +do $$ +begin + perform pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +jA0EBwMCXLc8pozB10Fg0jQBVUID59TLvWutJp0j6eh9ZgjqIRzdYaIymFB8y4XH +vu0YlJP5D5BX7yqZ+Pry7TlDmiFO +=rV7z +-----END PGP MESSAGE----- +'), 'key', 'debug=1'); +exception when others then + raise '%', + regexp_replace(sqlerrm, 'encoding "[^"]*"', 'encoding [REDACTED]'); +end +$$; + -- Decryption with a certain incorrect key yields an apparent BZip2-compressed -- plaintext. Ciphertext source: iterative pgp_sym_encrypt('secret', 'key') -- until the random prefix gave rise to that property. diff --git a/contrib/pgcrypto/sql/pgp-pubkey-session.sql b/contrib/pgcrypto/sql/pgp-pubkey-session.sql new file mode 100644 index 0000000000000..51792f1f4d856 --- /dev/null +++ b/contrib/pgcrypto/sql/pgp-pubkey-session.sql @@ -0,0 +1,46 @@ +-- Test for overflow with session key at decrypt. +-- Data automatically generated by scripts/pgp_session_data.py. +-- See this file for details explaining how this data is generated. +SELECT pgp_pub_decrypt_bytea( +'\xc1c04c030000000000000000020800a46f5b9b1905b49457a6485474f71ed9b46c2527e1 +da08e1f7871e12c3d38828f2076b984a595bf60f616599ca5729d547de06a258bfbbcd30 +94a321e4668cd43010f0ca8ecf931e5d39bda1152c50c367b11c723f270729245d3ebdbd +0694d320c5a5aa6a405fb45182acb3d7973cbce398e0c5060af7603cfd9ed186ebadd616 +3b50ae42bea5f6d14dda24e6d4687b434c175084515d562e896742b0ba9a1c87d5642e10 +a5550379c71cc490a052ada483b5d96526c0a600fc51755052aa77fdf72f7b4989b920e7 +b90f4b30787a46482670d5caecc7a515a926055ad5509d135702ce51a0e4c1033f2d939d +8f0075ec3428e17310da37d3d2d7ad1ce99adcc91cd446c366c402ae1ee38250343a7fcc +0f8bc28020e603d7a4795ef0dcc1c04c030000000000000000020800a46f5b9b1905b494 +57a6485474f71ed9b46c2527e1da08e1f7871e12c3d38828f2076b984a595bf60f616599 +ca5729d547de06a258bfbbcd3094a321e4668cd43010f0ca8ecf931e5d39bda1152c50c3 +67b11c723f270729245d3ebdbd0694d320c5a5aa6a405fb45182acb3d7973cbce398e0c5 +060af7603cfd9ed186ebadd6163b50ae42bea5f6d14dda24e6d4687b434c175084515d56 +2e896742b0ba9a1c87d5642e10a5550379c71cc490a052ada483b5d96526c0a600fc5175 +5052aa77fdf72f7b4989b920e7b90f4b30787a46482670d5caecc7a515a926055ad5509d +135702ce51a0e4c1033f2d939d8f0075ec3428e17310da37d3d2d7ad1ce99adc'::bytea, +'\xc7c2d8046965d657020800eef8bf1515adb1a3ee7825f75c668ea8dd3e3f9d13e958f6ad +9c55adc0c931a4bb00abe1d52cf7bb0c95d537949d277a5292ede375c6b2a67a3bf7d19f +f975bb7e7be35c2d8300dacba360a0163567372f7dc24000cc7cb6170bedc8f3b1f98c12 +07a6cb4de870a4bc61319b139dcc0e20c368fd68f8fd346d2c0b69c5aed560504e2ec6f1 +23086fe3c5540dc4dd155c0c67257c4ada862f90fe172ace344089da8135e92aca5c2709 +f1c1bc521798bb8c0365841496e709bd184132d387e0c9d5f26dc00fd06c3a76ef66a75c +138285038684707a847b7bd33cfbefbf1d336be954a8048946af97a66352adef8e8b5ae4 +c4748c6f2510265b7a8267bc370dbb00110100010007ff7e72d4f95d2d39901ac12ca5c5 +18e767e719e72340c3fab51c8c5ab1c40f31db8eaffe43533fa61e2dbca2c3f4396c0847 +e5434756acbb1f68128f4136bb135710c89137d74538908dac77967de9e821c559700dd9 +de5a2727eec1f5d12d5d74869dd1de45ed369d94a8814d23861dd163f8c27744b26b98f0 +239c2e6dd1e3493b8cc976fdc8f9a5e250f715aa4c3d7d5f237f8ee15d242e8fa941d1a0 +ed9550ab632d992a97518d142802cb0a97b251319bf5742db8d9d8cbaa06cdfba2d75bc9 +9d77a51ff20bd5ba7f15d7af6e85b904de2855d19af08d45f39deb85403033c69c767a8e +74a343b1d6c8911d34ea441ac3850e57808ed3d885835cbe6c79d10400ef16256f3d5c4c +3341516a2d2aa888df81b603f48a27f3666b40f992a857c1d11ff639cd764a9b42d5a1f8 +58b4aeee36b85508bb5e8b91ef88a7737770b330224479d9b44eae8c631bc43628b69549 +507c0a1af0be0dd7696015abea722b571eb35eefc4ab95595378ec12814727443f625fcd +183bb9b3bccf53b54dd0e5e7a50400ffe08537b2d4e6074e4a1727b658cfccdec8962302 +25e300c05690de45f7065c3d40d86f544a64d51a3e94424f9851a16d1322ebdb41fa8a45 +3131f3e2dc94e858e6396722643df382680f815e53bcdcde5da622f50530a83b217f1103 +cdd6e5e9babe1e415bbff28d44bd18c95f43bbd04afeb2a2a99af38a571c7540de21df03 +ff62c0a33d9143dd3f639893f47732c11c5a12c6052d1935f4d507b7ae1f76ab0e9a69b8 +7305a7f7c19bd509daf4903bff614bc26d118f03e461469c72c12d3a2bb4f78e4d342ce8 +487723649a01ed2b9eb11c662134502c098d55dfcd361939d8370873422c3da75a515a75 +9ffedfe7df44fb3c20f81650801a30d43b5c90b98b3eee'::bytea); diff --git a/contrib/pgstattuple/pgstattuple.c b/contrib/pgstattuple/pgstattuple.c index 1d012233cbffe..a59fb42171ece 100644 --- a/contrib/pgstattuple/pgstattuple.c +++ b/contrib/pgstattuple/pgstattuple.c @@ -422,7 +422,7 @@ pgstat_btree_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno, /* fully empty page */ stat->free_space += BLCKSZ; } - else + else if (PageGetSpecialSize(page) == MAXALIGN(sizeof(BTPageOpaqueData))) { BTPageOpaque opaque; @@ -456,10 +456,16 @@ pgstat_hash_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno, Buffer buf; Page page; - buf = _hash_getbuf_with_strategy(rel, blkno, HASH_READ, 0, bstrategy); + buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, bstrategy); + LockBuffer(buf, HASH_READ); page = BufferGetPage(buf); - if (PageGetSpecialSize(page) == MAXALIGN(sizeof(HashPageOpaqueData))) + if (PageIsNew(page)) + { + /* fully empty page */ + stat->free_space += BLCKSZ; + } + else if (PageGetSpecialSize(page) == MAXALIGN(sizeof(HashPageOpaqueData))) { HashPageOpaque opaque; @@ -500,17 +506,23 @@ pgstat_gist_page(pgstattuple_type *stat, Relation rel, BlockNumber blkno, buf = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL, bstrategy); LockBuffer(buf, GIST_SHARE); - gistcheckpage(rel, buf); page = BufferGetPage(buf); - - if (GistPageIsLeaf(page)) + if (PageIsNew(page)) { - pgstat_index_page(stat, page, FirstOffsetNumber, - PageGetMaxOffsetNumber(page)); + /* fully empty page */ + stat->free_space += BLCKSZ; } - else + else if (PageGetSpecialSize(page) == MAXALIGN(sizeof(GISTPageOpaqueData))) { - /* root or node */ + if (GistPageIsLeaf(page)) + { + pgstat_index_page(stat, page, FirstOffsetNumber, + PageGetMaxOffsetNumber(page)); + } + else + { + /* root or node */ + } } UnlockReleaseBuffer(buf); diff --git a/contrib/postgres_fdw/.gitignore b/contrib/postgres_fdw/.gitignore index 5dcb3ff972350..b4903eba657fa 100644 --- a/contrib/postgres_fdw/.gitignore +++ b/contrib/postgres_fdw/.gitignore @@ -1,4 +1,6 @@ # Generated subdirectories /log/ /results/ +/output_iso/ /tmp_check/ +/tmp_check_iso/ diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile index c1b0cad453f25..c797f3e04d57f 100644 --- a/contrib/postgres_fdw/Makefile +++ b/contrib/postgres_fdw/Makefile @@ -17,6 +17,8 @@ EXTENSION = postgres_fdw DATA = postgres_fdw--1.0.sql postgres_fdw--1.0--1.1.sql REGRESS = postgres_fdw +ISOLATION = eval_plan_qual +ISOLATION_OPTS = --load-extension=postgres_fdw ifdef USE_PGXS PG_CONFIG = pg_config diff --git a/contrib/postgres_fdw/expected/eval_plan_qual.out b/contrib/postgres_fdw/expected/eval_plan_qual.out new file mode 100644 index 0000000000000..ec7236559b2fc --- /dev/null +++ b/contrib/postgres_fdw/expected/eval_plan_qual.out @@ -0,0 +1,131 @@ +Parsed test spec with 2 sessions + +starting permutation: s0_update_l s1_tuplock_l_0 s0_commit s1_commit +step s0_update_l: UPDATE l SET i = i + 1; +step s1_tuplock_l_0: + EXPLAIN (VERBOSE, COSTS OFF) + SELECT l.* FROM l, ft WHERE l.i = ft.i AND l.i = 123 FOR UPDATE OF l; + SELECT l.* FROM l, ft WHERE l.i = ft.i AND l.i = 123 FOR UPDATE OF l; + +step s0_commit: COMMIT; +step s1_tuplock_l_0: <... completed> +QUERY PLAN +--------------------------------------------------------------------- +LockRows + Output: l.i, l.v, l.ctid, ft.* + -> Nested Loop + Output: l.i, l.v, l.ctid, ft.* + -> Seq Scan on public.l + Output: l.i, l.v, l.ctid + Filter: (l.i = 123) + -> Foreign Scan on public.ft + Output: ft.*, ft.i + Remote SQL: SELECT i, v FROM public.t WHERE ((i = 123)) +(10 rows) + +i|v +-+- +(0 rows) + +step s1_commit: COMMIT; + +starting permutation: s0_update_l s1_tuplock_l_1 s0_commit s1_commit +step s0_update_l: UPDATE l SET i = i + 1; +step s1_tuplock_l_1: + EXPLAIN (VERBOSE, COSTS OFF) + SELECT l.* FROM l, ft WHERE l.i = ft.i AND l.v = 'foo' FOR UPDATE OF l; + SELECT l.* FROM l, ft WHERE l.i = ft.i AND l.v = 'foo' FOR UPDATE OF l; + +step s0_commit: COMMIT; +step s1_tuplock_l_1: <... completed> +QUERY PLAN +----------------------------------------------------------------------------- +LockRows + Output: l.i, l.v, l.ctid, ft.* + -> Nested Loop + Output: l.i, l.v, l.ctid, ft.* + -> Seq Scan on public.l + Output: l.i, l.v, l.ctid + Filter: (l.v = 'foo'::text) + -> Foreign Scan on public.ft + Output: ft.*, ft.i + Remote SQL: SELECT i, v FROM public.t WHERE (($1::integer = i)) +(10 rows) + +i|v +-+- +(0 rows) + +step s1_commit: COMMIT; + +starting permutation: s0_update_a s1_tuplock_a_0 s0_commit s1_commit +step s0_update_a: UPDATE a SET i = i + 1; +step s1_tuplock_a_0: + EXPLAIN (VERBOSE, COSTS OFF) + SELECT a.i FROM a, fb, fc WHERE a.i = fb.i AND fb.i = fc.i FOR UPDATE OF a; + SELECT a.i FROM a, fb, fc WHERE a.i = fb.i AND fb.i = fc.i FOR UPDATE OF a; + +step s0_commit: COMMIT; +step s1_tuplock_a_0: <... completed> +QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +LockRows + Output: a.i, a.ctid, fb.*, fc.* + -> Nested Loop + Output: a.i, a.ctid, fb.*, fc.* + Join Filter: (fb.i = a.i) + -> Foreign Scan + Output: fb.*, fb.i, fc.*, fc.i + Relations: (public.fb) INNER JOIN (public.fc) + Remote SQL: SELECT CASE WHEN (r2.*)::text IS NOT NULL THEN ROW(r2.i) END, r2.i, CASE WHEN (r3.*)::text IS NOT NULL THEN ROW(r3.i) END, r3.i FROM (public.b r2 INNER JOIN public.c r3 ON (((r2.i = r3.i)))) + -> Nested Loop + Output: fb.*, fb.i, fc.*, fc.i + Join Filter: (fb.i = fc.i) + -> Foreign Scan on public.fb + Output: fb.*, fb.i + Remote SQL: SELECT i FROM public.b ORDER BY i ASC NULLS LAST + -> Foreign Scan on public.fc + Output: fc.*, fc.i + Remote SQL: SELECT i FROM public.c + -> Seq Scan on public.a + Output: a.i, a.ctid +(20 rows) + +i +- +(0 rows) + +step s1_commit: COMMIT; + +starting permutation: s0_update_a s1_tuplock_a_1 s0_commit s1_commit +step s0_update_a: UPDATE a SET i = i + 1; +step s1_tuplock_a_1: + EXPLAIN (VERBOSE, COSTS OFF) + SELECT a.i, + (SELECT 1 FROM fb, fc WHERE a.i = fb.i AND fb.i = fc.i) + FROM a FOR UPDATE; + SELECT a.i, + (SELECT 1 FROM fb, fc WHERE a.i = fb.i AND fb.i = fc.i) + FROM a FOR UPDATE; + +step s0_commit: COMMIT; +step s1_tuplock_a_1: <... completed> +QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------- +LockRows + Output: a.i, ((SubPlan 1)), a.ctid + -> Seq Scan on public.a + Output: a.i, (SubPlan 1), a.ctid + SubPlan 1 + -> Foreign Scan + Output: 1 + Relations: (public.fb) INNER JOIN (public.fc) + Remote SQL: SELECT NULL FROM (public.b r1 INNER JOIN public.c r2 ON (((r2.i = $1::integer)) AND ((r1.i = $1::integer)))) +(9 rows) + +i|?column? +-+-------- +2| +(1 row) + +step s1_commit: COMMIT; diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 375769b828cfb..6ef1af29aaa7e 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -5934,20 +5934,31 @@ UPDATE ft2 d SET c2 = CASE WHEN random() >= 0 THEN d.c2 ELSE 0 END ALTER SERVER loopback OPTIONS (DROP extensions); INSERT INTO ft2 (c1,c2,c3) SELECT id, id % 10, to_char(id, 'FM00000') FROM generate_series(2001, 2010) id; +-- this will do a remote seqscan, causing unstable result order, so sort EXPLAIN (verbose, costs off) -UPDATE ft2 SET c3 = 'bar' WHERE postgres_fdw_abs(c1) > 2000 RETURNING *; -- can't be pushed down - QUERY PLAN ----------------------------------------------------------------------------------------------------------- - Update on public.ft2 - Output: c1, c2, c3, c4, c5, c6, c7, c8 - Remote SQL: UPDATE "S 1"."T 1" SET c3 = $2 WHERE ctid = $1 RETURNING "C 1", c2, c3, c4, c5, c6, c7, c8 - -> Foreign Scan on public.ft2 - Output: 'bar'::text, ctid, ft2.* - Filter: (postgres_fdw_abs(ft2.c1) > 2000) - Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8, ctid FROM "S 1"."T 1" FOR UPDATE -(7 rows) +WITH cte AS ( + UPDATE ft2 SET c3 = 'bar' WHERE postgres_fdw_abs(c1) > 2000 RETURNING * +) SELECT * FROM cte ORDER BY c1; -- can't be pushed down + QUERY PLAN +------------------------------------------------------------------------------------------------------------------ + Sort + Output: cte.c1, cte.c2, cte.c3, cte.c4, cte.c5, cte.c6, cte.c7, cte.c8 + Sort Key: cte.c1 + CTE cte + -> Update on public.ft2 + Output: ft2.c1, ft2.c2, ft2.c3, ft2.c4, ft2.c5, ft2.c6, ft2.c7, ft2.c8 + Remote SQL: UPDATE "S 1"."T 1" SET c3 = $2 WHERE ctid = $1 RETURNING "C 1", c2, c3, c4, c5, c6, c7, c8 + -> Foreign Scan on public.ft2 + Output: 'bar'::text, ft2.ctid, ft2.* + Filter: (postgres_fdw_abs(ft2.c1) > 2000) + Remote SQL: SELECT "C 1", c2, c3, c4, c5, c6, c7, c8, ctid FROM "S 1"."T 1" FOR UPDATE + -> CTE Scan on cte + Output: cte.c1, cte.c2, cte.c3, cte.c4, cte.c5, cte.c6, cte.c7, cte.c8 +(13 rows) -UPDATE ft2 SET c3 = 'bar' WHERE postgres_fdw_abs(c1) > 2000 RETURNING *; +WITH cte AS ( + UPDATE ft2 SET c3 = 'bar' WHERE postgres_fdw_abs(c1) > 2000 RETURNING * +) SELECT * FROM cte ORDER BY c1; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ------+----+-----+----+----+----+------------+---- 2001 | 1 | bar | | | | ft2 | @@ -7615,6 +7626,119 @@ DELETE FROM rem1; -- can't be pushed down (5 rows) DROP TRIGGER trig_row_after_delete ON rem1; +-- We are allowed to create transition-table triggers on both kinds of +-- inheritance even if they contain foreign tables as children, but currently +-- collecting transition tuples from such foreign tables is not supported. +CREATE TABLE local_tbl (a text, b int); +CREATE FOREIGN TABLE foreign_tbl (a text, b int) + SERVER loopback OPTIONS (table_name 'local_tbl'); +INSERT INTO foreign_tbl VALUES ('AAA', 42); +-- Test case for partition hierarchy +CREATE TABLE parent_tbl (a text, b int) PARTITION BY LIST (a); +ALTER TABLE parent_tbl ATTACH PARTITION foreign_tbl FOR VALUES IN ('AAA'); +CREATE TRIGGER parent_tbl_insert_trig + AFTER INSERT ON parent_tbl REFERENCING NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); +CREATE TRIGGER parent_tbl_update_trig + AFTER UPDATE ON parent_tbl REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); +CREATE TRIGGER parent_tbl_delete_trig + AFTER DELETE ON parent_tbl REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); +INSERT INTO parent_tbl VALUES ('AAA', 42); +ERROR: cannot collect transition tuples from child foreign tables +COPY parent_tbl (a, b) FROM stdin; +ERROR: cannot collect transition tuples from child foreign tables +CONTEXT: COPY parent_tbl, line 1: "AAA 42" +ALTER SERVER loopback OPTIONS (ADD batch_size '10'); +INSERT INTO parent_tbl VALUES ('AAA', 42); +ERROR: cannot collect transition tuples from child foreign tables +COPY parent_tbl (a, b) FROM stdin; +ERROR: cannot collect transition tuples from child foreign tables +CONTEXT: COPY parent_tbl, line 1: "AAA 42" +ALTER SERVER loopback OPTIONS (DROP batch_size); +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE parent_tbl SET b = b + 1; + QUERY PLAN +------------------------------------------------------------------------------------------------ + Update on public.parent_tbl + Foreign Update on public.foreign_tbl parent_tbl_1 + Remote SQL: UPDATE public.local_tbl SET b = $2 WHERE ctid = $1 + -> Foreign Scan on public.foreign_tbl parent_tbl_1 + Output: (parent_tbl_1.b + 1), parent_tbl_1.tableoid, parent_tbl_1.ctid, parent_tbl_1.* + Remote SQL: SELECT a, b, ctid FROM public.local_tbl FOR UPDATE +(6 rows) + +UPDATE parent_tbl SET b = b + 1; +ERROR: cannot collect transition tuples from child foreign tables +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM parent_tbl; + QUERY PLAN +------------------------------------------------------------------ + Delete on public.parent_tbl + Foreign Delete on public.foreign_tbl parent_tbl_1 + Remote SQL: DELETE FROM public.local_tbl WHERE ctid = $1 + -> Foreign Scan on public.foreign_tbl parent_tbl_1 + Output: parent_tbl_1.tableoid, parent_tbl_1.ctid + Remote SQL: SELECT ctid FROM public.local_tbl FOR UPDATE +(6 rows) + +DELETE FROM parent_tbl; +ERROR: cannot collect transition tuples from child foreign tables +ALTER TABLE parent_tbl DETACH PARTITION foreign_tbl; +DROP TABLE parent_tbl; +-- Test case for non-partition hierarchy +CREATE TABLE parent_tbl (a text, b int); +ALTER FOREIGN TABLE foreign_tbl INHERIT parent_tbl; +CREATE TRIGGER parent_tbl_update_trig + AFTER UPDATE ON parent_tbl REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); +CREATE TRIGGER parent_tbl_delete_trig + AFTER DELETE ON parent_tbl REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE parent_tbl SET b = b + 1; + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Update on public.parent_tbl + Update on public.parent_tbl parent_tbl_1 + Foreign Update on public.foreign_tbl parent_tbl_2 + Remote SQL: UPDATE public.local_tbl SET b = $2 WHERE ctid = $1 + -> Result + Output: (parent_tbl.b + 1), parent_tbl.tableoid, parent_tbl.ctid, (NULL::record) + -> Append + -> Seq Scan on public.parent_tbl parent_tbl_1 + Output: parent_tbl_1.b, parent_tbl_1.tableoid, parent_tbl_1.ctid, NULL::record + -> Foreign Scan on public.foreign_tbl parent_tbl_2 + Output: parent_tbl_2.b, parent_tbl_2.tableoid, parent_tbl_2.ctid, parent_tbl_2.* + Remote SQL: SELECT a, b, ctid FROM public.local_tbl FOR UPDATE +(12 rows) + +UPDATE parent_tbl SET b = b + 1; +ERROR: cannot collect transition tuples from child foreign tables +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM parent_tbl; + QUERY PLAN +------------------------------------------------------------------------ + Delete on public.parent_tbl + Delete on public.parent_tbl parent_tbl_1 + Foreign Delete on public.foreign_tbl parent_tbl_2 + Remote SQL: DELETE FROM public.local_tbl WHERE ctid = $1 + -> Append + -> Seq Scan on public.parent_tbl parent_tbl_1 + Output: parent_tbl_1.tableoid, parent_tbl_1.ctid + -> Foreign Scan on public.foreign_tbl parent_tbl_2 + Output: parent_tbl_2.tableoid, parent_tbl_2.ctid + Remote SQL: SELECT ctid FROM public.local_tbl FOR UPDATE +(10 rows) + +DELETE FROM parent_tbl; +ERROR: cannot collect transition tuples from child foreign tables +ALTER FOREIGN TABLE foreign_tbl NO INHERIT parent_tbl; +DROP TABLE parent_tbl; +-- Cleanup +DROP FOREIGN TABLE foreign_tbl; +DROP TABLE local_tbl; -- =================================================================== -- test inheritance features -- =================================================================== diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index 8aaa3a6a1daee..eb2c8feee86de 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -239,6 +239,7 @@ typedef struct PgFdwDirectModifyState PGresult *result; /* result for query */ int num_tuples; /* # of result tuples */ int next_tuple; /* index of next one to return */ + MemoryContextCallback result_cb; /* ensures result will get freed */ Relation resultRel; /* relcache entry for the target relation */ AttrNumber *attnoMap; /* array of attnums of input user columns */ AttrNumber ctidAttno; /* attnum of input ctid column */ @@ -2656,6 +2657,17 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags) dmstate = (PgFdwDirectModifyState *) palloc0(sizeof(PgFdwDirectModifyState)); node->fdw_state = (void *) dmstate; + /* + * We use a memory context callback to ensure that the dmstate's PGresult + * (if any) will be released, even if the query fails somewhere that's + * outside our control. The callback is always armed for the duration of + * the query; this relies on PQclear(NULL) being a no-op. + */ + dmstate->result_cb.func = (MemoryContextCallbackFunction) PQclear; + dmstate->result_cb.arg = NULL; + MemoryContextRegisterResetCallback(CurrentMemoryContext, + &dmstate->result_cb); + /* * Identify which user to do the remote access as. This should match what * ExecCheckRTEPerms() does. @@ -2805,7 +2817,12 @@ postgresEndDirectModify(ForeignScanState *node) /* Release PGresult */ if (dmstate->result) + { PQclear(dmstate->result); + dmstate->result = NULL; + /* ... and don't forget to disable the callback */ + dmstate->result_cb.arg = NULL; + } /* Release remote connection */ ReleaseConnection(dmstate->conn); @@ -4580,13 +4597,17 @@ execute_dml_stmt(ForeignScanState *node) /* * Get the result, and check for success. * - * We don't use a PG_TRY block here, so be careful not to throw error - * without releasing the PGresult. + * We use a memory context callback to ensure that the PGresult will be + * released, even if the query fails somewhere that's outside our control. + * The callback is already registered, just need to fill in its arg. */ + Assert(dmstate->result == NULL); dmstate->result = pgfdw_get_result(dmstate->conn, dmstate->query); + dmstate->result_cb.arg = dmstate->result; + if (PQresultStatus(dmstate->result) != (dmstate->has_returning ? PGRES_TUPLES_OK : PGRES_COMMAND_OK)) - pgfdw_report_error(ERROR, dmstate->result, dmstate->conn, true, + pgfdw_report_error(ERROR, dmstate->result, dmstate->conn, false, dmstate->query); /* Get the number of rows affected. */ @@ -4630,31 +4651,16 @@ get_returning_data(ForeignScanState *node) } else { - /* - * On error, be sure to release the PGresult on the way out. Callers - * do not have PG_TRY blocks to ensure this happens. - */ - PG_TRY(); - { - HeapTuple newtup; - - newtup = make_tuple_from_result_row(dmstate->result, - dmstate->next_tuple, - dmstate->rel, - dmstate->attinmeta, - dmstate->retrieved_attrs, - node, - dmstate->temp_cxt); - ExecStoreHeapTuple(newtup, slot, false); - } - PG_CATCH(); - { - if (dmstate->result) - PQclear(dmstate->result); - PG_RE_THROW(); - } - PG_END_TRY(); + HeapTuple newtup; + newtup = make_tuple_from_result_row(dmstate->result, + dmstate->next_tuple, + dmstate->rel, + dmstate->attinmeta, + dmstate->retrieved_attrs, + node, + dmstate->temp_cxt); + ExecStoreHeapTuple(newtup, slot, false); /* Get the updated/deleted tuple. */ if (dmstate->rel) resultSlot = slot; diff --git a/contrib/postgres_fdw/specs/eval_plan_qual.spec b/contrib/postgres_fdw/specs/eval_plan_qual.spec new file mode 100644 index 0000000000000..9f52270db6984 --- /dev/null +++ b/contrib/postgres_fdw/specs/eval_plan_qual.spec @@ -0,0 +1,102 @@ +# Tests for the EvalPlanQual mechanism involving foreign tables + +setup +{ + DO $d$ + BEGIN + EXECUTE $$CREATE SERVER loopback FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (dbname '$$||current_database()||$$', + port '$$||current_setting('port')||$$', + use_remote_estimate 'true' + )$$; + END; + $d$; + CREATE USER MAPPING FOR PUBLIC SERVER loopback; + + CREATE TABLE l (i int, v text); + CREATE TABLE t (i int, v text); + CREATE FOREIGN TABLE ft (i int, v text) SERVER loopback OPTIONS (table_name 't'); + + INSERT INTO l VALUES (123, 'foo'), (456, 'bar'), (789, 'baz'); + INSERT INTO t SELECT i, to_char(i, 'FM0000') FROM generate_series(1, 1000) i; + CREATE INDEX t_idx ON t (i); + ANALYZE l, t; + + CREATE TABLE a (i int); + CREATE TABLE b (i int); + CREATE TABLE c (i int); + CREATE FOREIGN TABLE fb (i int) SERVER loopback OPTIONS (table_name 'b'); + CREATE FOREIGN TABLE fc (i int) SERVER loopback OPTIONS (table_name 'c'); + + INSERT INTO a VALUES (1); + INSERT INTO b VALUES (1); + INSERT INTO c VALUES (1); + ANALYZE a, b, c; +} + +teardown +{ + DROP TABLE l; + DROP TABLE t; + DROP TABLE a; + DROP TABLE b; + DROP TABLE c; + DROP SERVER loopback CASCADE; +} + +session s0 +setup { BEGIN ISOLATION LEVEL READ COMMITTED; } +step s0_update_l { UPDATE l SET i = i + 1; } +step s0_update_a { UPDATE a SET i = i + 1; } +step s0_commit { COMMIT; } + +session s1 +setup { BEGIN ISOLATION LEVEL READ COMMITTED; } + +# Test for EPQ with a foreign scan pushing down a qual +step s1_tuplock_l_0 { + EXPLAIN (VERBOSE, COSTS OFF) + SELECT l.* FROM l, ft WHERE l.i = ft.i AND l.i = 123 FOR UPDATE OF l; + SELECT l.* FROM l, ft WHERE l.i = ft.i AND l.i = 123 FOR UPDATE OF l; +} + +# Same test, except that the qual is parameterized +step s1_tuplock_l_1 { + EXPLAIN (VERBOSE, COSTS OFF) + SELECT l.* FROM l, ft WHERE l.i = ft.i AND l.v = 'foo' FOR UPDATE OF l; + SELECT l.* FROM l, ft WHERE l.i = ft.i AND l.v = 'foo' FOR UPDATE OF l; +} + +# Test for EPQ with a foreign scan pushing down a join +step s1_tuplock_a_0 { + EXPLAIN (VERBOSE, COSTS OFF) + SELECT a.i FROM a, fb, fc WHERE a.i = fb.i AND fb.i = fc.i FOR UPDATE OF a; + SELECT a.i FROM a, fb, fc WHERE a.i = fb.i AND fb.i = fc.i FOR UPDATE OF a; +} + +# Same test, except that the join is contained in a SubLink sub-select, not +# in the main query +step s1_tuplock_a_1 { + EXPLAIN (VERBOSE, COSTS OFF) + SELECT a.i, + (SELECT 1 FROM fb, fc WHERE a.i = fb.i AND fb.i = fc.i) + FROM a FOR UPDATE; + SELECT a.i, + (SELECT 1 FROM fb, fc WHERE a.i = fb.i AND fb.i = fc.i) + FROM a FOR UPDATE; +} + +step s1_commit { COMMIT; } + +# This test checks the case of rechecking a pushed-down qual. +permutation s0_update_l s1_tuplock_l_0 s0_commit s1_commit + +# This test checks the same case, except that the qual is parameterized. +permutation s0_update_l s1_tuplock_l_1 s0_commit s1_commit + +# This test checks the case of rechecking a pushed-down join. +permutation s0_update_a s1_tuplock_a_0 s0_commit s1_commit + +# This test exercises EvalPlanQual with a SubLink sub-select (which should +# be unaffected by any EPQ recheck behavior in the outer query). +permutation s0_update_a s1_tuplock_a_1 s0_commit s1_commit diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 0512c633931b5..4f188c7da6bd5 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -1389,9 +1389,16 @@ UPDATE ft2 d SET c2 = CASE WHEN random() >= 0 THEN d.c2 ELSE 0 END ALTER SERVER loopback OPTIONS (DROP extensions); INSERT INTO ft2 (c1,c2,c3) SELECT id, id % 10, to_char(id, 'FM00000') FROM generate_series(2001, 2010) id; + +-- this will do a remote seqscan, causing unstable result order, so sort EXPLAIN (verbose, costs off) -UPDATE ft2 SET c3 = 'bar' WHERE postgres_fdw_abs(c1) > 2000 RETURNING *; -- can't be pushed down -UPDATE ft2 SET c3 = 'bar' WHERE postgres_fdw_abs(c1) > 2000 RETURNING *; +WITH cte AS ( + UPDATE ft2 SET c3 = 'bar' WHERE postgres_fdw_abs(c1) > 2000 RETURNING * +) SELECT * FROM cte ORDER BY c1; -- can't be pushed down +WITH cte AS ( + UPDATE ft2 SET c3 = 'bar' WHERE postgres_fdw_abs(c1) > 2000 RETURNING * +) SELECT * FROM cte ORDER BY c1; + EXPLAIN (verbose, costs off) UPDATE ft2 SET c3 = 'baz' FROM ft4 INNER JOIN ft5 ON (ft4.c1 = ft5.c1) @@ -2026,6 +2033,84 @@ EXPLAIN (verbose, costs off) DELETE FROM rem1; -- can't be pushed down DROP TRIGGER trig_row_after_delete ON rem1; + +-- We are allowed to create transition-table triggers on both kinds of +-- inheritance even if they contain foreign tables as children, but currently +-- collecting transition tuples from such foreign tables is not supported. + +CREATE TABLE local_tbl (a text, b int); +CREATE FOREIGN TABLE foreign_tbl (a text, b int) + SERVER loopback OPTIONS (table_name 'local_tbl'); + +INSERT INTO foreign_tbl VALUES ('AAA', 42); + +-- Test case for partition hierarchy +CREATE TABLE parent_tbl (a text, b int) PARTITION BY LIST (a); +ALTER TABLE parent_tbl ATTACH PARTITION foreign_tbl FOR VALUES IN ('AAA'); + +CREATE TRIGGER parent_tbl_insert_trig + AFTER INSERT ON parent_tbl REFERENCING NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); +CREATE TRIGGER parent_tbl_update_trig + AFTER UPDATE ON parent_tbl REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); +CREATE TRIGGER parent_tbl_delete_trig + AFTER DELETE ON parent_tbl REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); + +INSERT INTO parent_tbl VALUES ('AAA', 42); + +COPY parent_tbl (a, b) FROM stdin; +AAA 42 +\. + +ALTER SERVER loopback OPTIONS (ADD batch_size '10'); + +INSERT INTO parent_tbl VALUES ('AAA', 42); + +COPY parent_tbl (a, b) FROM stdin; +AAA 42 +\. + +ALTER SERVER loopback OPTIONS (DROP batch_size); + +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE parent_tbl SET b = b + 1; +UPDATE parent_tbl SET b = b + 1; + +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM parent_tbl; +DELETE FROM parent_tbl; + +ALTER TABLE parent_tbl DETACH PARTITION foreign_tbl; +DROP TABLE parent_tbl; + +-- Test case for non-partition hierarchy +CREATE TABLE parent_tbl (a text, b int); +ALTER FOREIGN TABLE foreign_tbl INHERIT parent_tbl; + +CREATE TRIGGER parent_tbl_update_trig + AFTER UPDATE ON parent_tbl REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); +CREATE TRIGGER parent_tbl_delete_trig + AFTER DELETE ON parent_tbl REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT EXECUTE PROCEDURE trigger_func(); + +EXPLAIN (VERBOSE, COSTS OFF) +UPDATE parent_tbl SET b = b + 1; +UPDATE parent_tbl SET b = b + 1; + +EXPLAIN (VERBOSE, COSTS OFF) +DELETE FROM parent_tbl; +DELETE FROM parent_tbl; + +ALTER FOREIGN TABLE foreign_tbl NO INHERIT parent_tbl; +DROP TABLE parent_tbl; + +-- Cleanup +DROP FOREIGN TABLE foreign_tbl; +DROP TABLE local_tbl; + -- =================================================================== -- test inheritance features -- =================================================================== diff --git a/contrib/test_decoding/Makefile b/contrib/test_decoding/Makefile index a4ba1a509aec2..02e961f4d3144 100644 --- a/contrib/test_decoding/Makefile +++ b/contrib/test_decoding/Makefile @@ -9,7 +9,7 @@ REGRESS = ddl xact rewrite toast permissions decoding_in_xact \ ISOLATION = mxact delayed_startup ondisk_startup concurrent_ddl_dml \ oldest_xmin snapshot_transfer subxact_without_top concurrent_stream \ twophase_snapshot slot_creation_error catalog_change_snapshot \ - skip_snapshot_restore + skip_snapshot_restore invalidation_distribution REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf ISOLATION_OPTS = --temp-config $(top_srcdir)/contrib/test_decoding/logical.conf diff --git a/contrib/test_decoding/expected/invalidation_distribution.out b/contrib/test_decoding/expected/invalidation_distribution.out new file mode 100644 index 0000000000000..49c1f60cc4fd9 --- /dev/null +++ b/contrib/test_decoding/expected/invalidation_distribution.out @@ -0,0 +1,41 @@ +Parsed test spec with 3 sessions + +starting permutation: s1_insert_tbl1 s1_begin s1_insert_tbl1 s2_alter_pub_add_tbl s1_commit s1_insert_tbl1 s2_get_binary_changes +step s1_insert_tbl1: INSERT INTO tbl1 (val1, val2) VALUES (1, 1); +step s1_begin: BEGIN; +step s1_insert_tbl1: INSERT INTO tbl1 (val1, val2) VALUES (1, 1); +step s2_alter_pub_add_tbl: ALTER PUBLICATION pub ADD TABLE tbl1; +step s1_commit: COMMIT; +step s1_insert_tbl1: INSERT INTO tbl1 (val1, val2) VALUES (1, 1); +step s2_get_binary_changes: SELECT count(data) FROM pg_logical_slot_get_binary_changes('isolation_slot', NULL, NULL, 'proto_version', '3', 'publication_names', 'pub') WHERE get_byte(data, 0) = 73; +count +----- + 1 +(1 row) + +?column? +-------- +stop +(1 row) + + +starting permutation: s1_begin s1_insert_tbl1 s3_begin s3_insert_tbl1 s2_alter_pub_add_tbl s1_insert_tbl1 s1_commit s3_commit s2_get_binary_changes +step s1_begin: BEGIN; +step s1_insert_tbl1: INSERT INTO tbl1 (val1, val2) VALUES (1, 1); +step s3_begin: BEGIN; +step s3_insert_tbl1: INSERT INTO tbl1 (val1, val2) VALUES (2, 2); +step s2_alter_pub_add_tbl: ALTER PUBLICATION pub ADD TABLE tbl1; +step s1_insert_tbl1: INSERT INTO tbl1 (val1, val2) VALUES (1, 1); +step s1_commit: COMMIT; +step s3_commit: COMMIT; +step s2_get_binary_changes: SELECT count(data) FROM pg_logical_slot_get_binary_changes('isolation_slot', NULL, NULL, 'proto_version', '3', 'publication_names', 'pub') WHERE get_byte(data, 0) = 73; +count +----- + 1 +(1 row) + +?column? +-------- +stop +(1 row) + diff --git a/contrib/test_decoding/expected/oldest_xmin.out b/contrib/test_decoding/expected/oldest_xmin.out index dd6053f9c1f4b..57268b38d3322 100644 --- a/contrib/test_decoding/expected/oldest_xmin.out +++ b/contrib/test_decoding/expected/oldest_xmin.out @@ -38,3 +38,44 @@ COMMIT stop (1 row) + +starting permutation: s0_begin s0_getxid s1_begin s1_insert s0_alter s0_commit s0_checkpoint s0_advance_slot s0_advance_slot s1_commit s0_vacuum s0_get_changes +step s0_begin: BEGIN; +step s0_getxid: SELECT pg_current_xact_id() IS NULL; +?column? +-------- +f +(1 row) + +step s1_begin: BEGIN; +step s1_insert: INSERT INTO harvest VALUES ((1, 2, 3)); +step s0_alter: ALTER TYPE basket DROP ATTRIBUTE mangos; +step s0_commit: COMMIT; +step s0_checkpoint: CHECKPOINT; +step s0_advance_slot: SELECT slot_name FROM pg_replication_slot_advance('isolation_slot', pg_current_wal_lsn()); +slot_name +-------------- +isolation_slot +(1 row) + +step s0_advance_slot: SELECT slot_name FROM pg_replication_slot_advance('isolation_slot', pg_current_wal_lsn()); +slot_name +-------------- +isolation_slot +(1 row) + +step s1_commit: COMMIT; +step s0_vacuum: VACUUM pg_attribute; +step s0_get_changes: SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); +data +------------------------------------------------------ +BEGIN +table public.harvest: INSERT: fruits[basket]:'(1,2,3)' +COMMIT +(3 rows) + +?column? +-------- +stop +(1 row) + diff --git a/contrib/test_decoding/specs/invalidation_distribution.spec b/contrib/test_decoding/specs/invalidation_distribution.spec new file mode 100644 index 0000000000000..c85f452938c81 --- /dev/null +++ b/contrib/test_decoding/specs/invalidation_distribution.spec @@ -0,0 +1,43 @@ +# Test that catalog cache invalidation messages are distributed to ongoing +# transactions, ensuring they can access the updated catalog content after +# processing these messages. +setup +{ + SELECT 'init' FROM pg_create_logical_replication_slot('isolation_slot', 'pgoutput'); + CREATE TABLE tbl1(val1 integer, val2 integer); + CREATE PUBLICATION pub; +} + +teardown +{ + DROP TABLE tbl1; + DROP PUBLICATION pub; + SELECT 'stop' FROM pg_drop_replication_slot('isolation_slot'); +} + +session "s1" +setup { SET synchronous_commit=on; } + +step "s1_begin" { BEGIN; } +step "s1_insert_tbl1" { INSERT INTO tbl1 (val1, val2) VALUES (1, 1); } +step "s1_commit" { COMMIT; } + +session "s2" +setup { SET synchronous_commit=on; } + +step "s2_alter_pub_add_tbl" { ALTER PUBLICATION pub ADD TABLE tbl1; } +step "s2_get_binary_changes" { SELECT count(data) FROM pg_logical_slot_get_binary_changes('isolation_slot', NULL, NULL, 'proto_version', '3', 'publication_names', 'pub') WHERE get_byte(data, 0) = 73; } + +session "s3" +setup { SET synchronous_commit=on; } +step "s3_begin" { BEGIN; } +step "s3_insert_tbl1" { INSERT INTO tbl1 (val1, val2) VALUES (2, 2); } +step "s3_commit" { COMMIT; } + +# Expect to get one insert change. LOGICAL_REP_MSG_INSERT = 'I' +permutation "s1_insert_tbl1" "s1_begin" "s1_insert_tbl1" "s2_alter_pub_add_tbl" "s1_commit" "s1_insert_tbl1" "s2_get_binary_changes" + +# Expect to get one insert change with LOGICAL_REP_MSG_INSERT = 'I' from +# the second "s1_insert_tbl1" executed after adding the table tbl1 to the +# publication in "s2_alter_pub_add_tbl". +permutation "s1_begin" "s1_insert_tbl1" "s3_begin" "s3_insert_tbl1" "s2_alter_pub_add_tbl" "s1_insert_tbl1" "s1_commit" "s3_commit" "s2_get_binary_changes" diff --git a/contrib/test_decoding/specs/oldest_xmin.spec b/contrib/test_decoding/specs/oldest_xmin.spec index 88bd30f5ff76c..7f2fe3d7ed776 100644 --- a/contrib/test_decoding/specs/oldest_xmin.spec +++ b/contrib/test_decoding/specs/oldest_xmin.spec @@ -25,6 +25,7 @@ step "s0_commit" { COMMIT; } step "s0_checkpoint" { CHECKPOINT; } step "s0_vacuum" { VACUUM pg_attribute; } step "s0_get_changes" { SELECT data FROM pg_logical_slot_get_changes('isolation_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1'); } +step "s0_advance_slot" { SELECT slot_name FROM pg_replication_slot_advance('isolation_slot', pg_current_wal_lsn()); } session "s1" setup { SET synchronous_commit=on; } @@ -40,3 +41,7 @@ step "s1_commit" { COMMIT; } # will be removed (xmax set) before T1 commits. That is, interlocking doesn't # forbid modifying catalog after someone read it (and didn't commit yet). permutation "s0_begin" "s0_getxid" "s1_begin" "s1_insert" "s0_alter" "s0_commit" "s0_checkpoint" "s0_get_changes" "s0_get_changes" "s1_commit" "s0_vacuum" "s0_get_changes" + +# Perform the same testing process as described above, but use advance_slot to +# forces xmin advancement during fast forward decoding. +permutation "s0_begin" "s0_getxid" "s1_begin" "s1_insert" "s0_alter" "s0_commit" "s0_checkpoint" "s0_advance_slot" "s0_advance_slot" "s1_commit" "s0_vacuum" "s0_get_changes" diff --git a/contrib/unaccent/unaccent.c b/contrib/unaccent/unaccent.c index 77ecd76528289..de20ac1ae8b47 100644 --- a/contrib/unaccent/unaccent.c +++ b/contrib/unaccent/unaccent.c @@ -149,9 +149,9 @@ initTrie(const char *filename) state = 0; for (ptr = line; *ptr; ptr += ptrlen) { - ptrlen = pg_mblen(ptr); + ptrlen = pg_mblen_cstr(ptr); /* ignore whitespace, but end src or trg */ - if (t_isspace(ptr)) + if (t_isspace_cstr(ptr)) { if (state == 1) state = 2; @@ -315,6 +315,7 @@ unaccent_lexize(PG_FUNCTION_ARGS) char *srcchar = (char *) PG_GETARG_POINTER(1); int32 len = PG_GETARG_INT32(2); char *srcstart = srcchar; + const char *srcend = srcstart + len; TSLexeme *res; StringInfoData buf; @@ -342,7 +343,7 @@ unaccent_lexize(PG_FUNCTION_ARGS) } else { - matchlen = pg_mblen(srcchar); + matchlen = pg_mblen_range(srcchar, srcend); if (buf.data != NULL) appendBinaryStringInfo(&buf, srcchar, matchlen); } diff --git a/contrib/xml2/xpath.c b/contrib/xml2/xpath.c index 212cb74aa220b..94bb31434c89b 100644 --- a/contrib/xml2/xpath.c +++ b/contrib/xml2/xpath.c @@ -174,7 +174,7 @@ pgxmlNodeSetToText(xmlNodeSetPtr nodeset, xmlBufferWriteCHAR(buf, toptagname); xmlBufferWriteChar(buf, ">"); } - result = xmlStrdup(buf->content); + result = xmlStrdup(xmlBufferContent(buf)); xmlBufferFree(buf); return result; } diff --git a/doc/src/sgml/amcheck.sgml b/doc/src/sgml/amcheck.sgml index 5d61a33936f1d..946bcfaa9c450 100644 --- a/doc/src/sgml/amcheck.sgml +++ b/doc/src/sgml/amcheck.sgml @@ -353,7 +353,7 @@ SET client_min_messages = DEBUG1; verification functions is true, an additional phase of verification is performed against the table associated with the target index relation. This consists of a dummy - CREATE INDEX operation, which checks for the + CREATE INDEX CONCURRENTLY operation, which checks for the presence of all hypothetical new index tuples against a temporary, in-memory summarizing structure (this is built when needed during the basic first phase of verification). The summarizing structure diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 88328ebb34bf1..27abc2ee107e2 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -1590,7 +1590,7 @@ rolpassword text - Password (possibly encrypted); null if none. The format depends + Encrypted password; null if none. The format depends on the form of encryption used. @@ -1626,11 +1626,6 @@ SCRAM-SHA-256$<iteration count>:&l ServerKey are in Base64 encoded format. This format is the same as that specified by RFC 5803. - - - A password that does not follow either of those formats is assumed to be - unencrypted. - diff --git a/doc/src/sgml/client-auth.sgml b/doc/src/sgml/client-auth.sgml index 02fa1ecc752c7..aa4c99723af6b 100644 --- a/doc/src/sgml/client-auth.sgml +++ b/doc/src/sgml/client-auth.sgml @@ -864,8 +864,9 @@ local db1,db2,@demodbs all md5 the remainder of the field is treated as a regular expression. (See for details of PostgreSQL's regular expression syntax.) The regular - expression can include a single capture, or parenthesized subexpression, - which can then be referenced in the database-username + expression can include a single capture, or parenthesized subexpression. + The portion of the system user name that matched the capture can then + be referenced in the database-username field as \1 (backslash-one). This allows the mapping of multiple user names in a single line, which is particularly useful for simple syntax substitutions. For example, these entries diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 9d398d8ee2019..4170676f1e278 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1587,7 +1587,7 @@ include_dir 'conf.d' This parameter determines whether the passphrase command set by ssl_passphrase_command will also be called during a configuration reload if a key file needs a passphrase. If this - parameter is off (the default), then + parameter is off (the default), then ssl_passphrase_command will be ignored during a reload and the SSL configuration will not be reloaded if a passphrase is needed. That setting is appropriate for a command that requires a @@ -1595,6 +1595,12 @@ include_dir 'conf.d' running. Setting this parameter to on might be appropriate if the passphrase is obtained from a file, for example. + + This parameter must be set to on when running on + Windows since all connections + will perform a configuration reload due to the different process model + of that platform. + This parameter can only be set in the postgresql.conf file or on the server command line. @@ -1667,7 +1673,8 @@ include_dir 'conf.d' Controls whether huge pages are requested for the main shared memory area. Valid values are try (the default), - on, and off. With + on, and off. + This parameter can only be set at server start. With huge_pages set to try, the server will try to request huge pages, but fall back to the default if that fails. With on, failure to request huge pages @@ -2021,6 +2028,7 @@ include_dir 'conf.d' platform, is generally discouraged because it typically requires non-default kernel settings to allow for large allocations (see ). + This parameter can only be set at server start. @@ -2048,6 +2056,7 @@ include_dir 'conf.d' however, it may be useful for debugging, when the pg_dynshmem directory is stored on a RAM disk, or when other shared memory facilities are not available. + This parameter can only be set at server start. @@ -3169,8 +3178,9 @@ include_dir 'conf.d' This parameter enables compression of WAL using the specified compression method. When enabled, the PostgreSQL - server compresses full page images written to WAL when - is on or during a base backup. + server compresses full page images written to WAL (e.g. when + is on, during a base backup, + etc.). A compressed page image will be decompressed during WAL replay. The supported methods are pglz, lz4 (if PostgreSQL @@ -3730,6 +3740,7 @@ include_dir 'conf.d' blocks to prefetch. If this value is specified without units, it is taken as bytes. The default is 512kB. + This parameter can only be set at server start. @@ -4333,9 +4344,9 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows - Record commit time of transactions. This parameter - can only be set in postgresql.conf file or on the server - command line. The default value is off. + Record commit time of transactions. + This parameter can only be set at server start. + The default value is off. @@ -5423,24 +5434,24 @@ ANY num_sync (