diff --git a/.github/workflows/build-push-image-private.yaml b/.github/workflows/build-push-image-private.yaml new file mode 100644 index 000000000..79c895e0c --- /dev/null +++ b/.github/workflows/build-push-image-private.yaml @@ -0,0 +1,86 @@ +name: Build and push Docker image to ECR (private) + +on: + push: + branches: + - release/v* + - feature/* + +permissions: + contents: read + +env: + AWS_REGION: eu-north-1 + ECR_REPOSITORY: 288509344804.dkr.ecr.eu-north-1.amazonaws.com + IMAGE_NAME: core.postgres-operator + +jobs: + ci: + name: Build and push Docker image to ECR + runs-on: ubuntu-22.04 + + steps: + - name: Checkout main repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "^1.26.4" + cache-dependency-path: go.sum + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.CORE_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.CORE_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Parse release version from Git commit message + run: | + readonly VERSION_REGEX="v[0-9]+\.[0-9]+\.[0-9]+" + + VERSION_PREV="$(git for-each-ref --sort=creatordate --format "%(refname:lstrip=2)" refs/tags | grep -E "^${VERSION_REGEX}$" | tail -1)" + + if [[ -z "${VERSION_PREV}" ]]; then + >&2 echo "At least one version tag is required in the repository." + >&2 echo "Tag a Git commit on the main branch manually before running the workflow." + exit 1 + fi + + readonly GIT_BRANCH="${GITHUB_REF#refs/heads/}" + + GIT_BRANCH_SANITIZED="${GIT_BRANCH}" + GIT_BRANCH_SANITIZED="${GIT_BRANCH_SANITIZED//\//-}" + GIT_BRANCH_SANITIZED="${GIT_BRANCH_SANITIZED//_/-}" + VERSION="${VERSION_PREV}-${GITHUB_SHA:0:7}-${GIT_BRANCH_SANITIZED}" + VERSION="${VERSION:0:128}" + VERSION="${VERSION%-}" + + echo "VERSION=${VERSION}" >> "${GITHUB_ENV}" + + - name: Login to ECR + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: private + + - name: Run unit tests + run: | + make mocks + make linux + go test -race ./... + + - name: Build and push Docker image + run: | + readonly IMAGE="${ECR_REPOSITORY}/${IMAGE_NAME}" + readonly TAG="${VERSION}" + + docker build --platform linux/amd64 \ + -f docker/Dockerfile \ + --build-arg VERSION="${VERSION}" \ + -t "${IMAGE}:${TAG}" \ + . + + docker push "${IMAGE}:${TAG}" diff --git a/.github/workflows/build-push-image-public.yaml b/.github/workflows/build-push-image-public.yaml new file mode 100644 index 000000000..72e2cb668 --- /dev/null +++ b/.github/workflows/build-push-image-public.yaml @@ -0,0 +1,99 @@ +name: Build and push Docker image to ECR (public) + +on: + push: + branches: + - master + +permissions: + contents: write + +env: + AWS_REGION: us-east-1 + ECR_REPOSITORY: public.ecr.aws/edenlabllc + GIT_USERNAME: github-actions + GIT_EMAIL: github-actions@github.com + IMAGE_NAME: core.postgres-operator + +jobs: + ci: + name: Build and push Docker image to ECR + runs-on: ubuntu-22.04 + + steps: + - name: Checkout main repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "^1.26.4" + cache-dependency-path: go.sum + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.CORE_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.CORE_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Parse release version from Git commit history + run: | + echo "Git commit message:" + GIT_COMMIT_MSG="$(git log -1 --pretty=format:"%s")" + echo "${GIT_COMMIT_MSG}" + + VERSION_REGEX="^Merge pull request #[0-9]+ from ${GITHUB_REPOSITORY_OWNER}/release/(v[0-9]+\.[0-9]+\.[0-9]+)$" + if [[ ! "${GIT_COMMIT_MSG}" =~ ${VERSION_REGEX} ]]; then + >&2 echo "Pushes to master should be done via merges of PR requests from release branches only." + >&2 echo " release branch - release/vN.N.N" + >&2 echo "The expected message format (will be used for parsing a release tag):" + >&2 echo "Merge pull request #N from ${GITHUB_REPOSITORY_OWNER}/release/vN.N.N" + exit 1 + fi + + VERSION="${BASH_REMATCH[1]}" + echo "VERSION=${VERSION}" >> "${GITHUB_ENV}" + + - name: Login to ECR + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Run unit tests + run: | + make mocks + make linux + go test -race ./... + + - name: Build and push Docker image + run: | + readonly IMAGE="${ECR_REPOSITORY}/${IMAGE_NAME}" + readonly TAG="${VERSION}" + + docker build --platform linux/amd64 \ + -f docker/Dockerfile \ + --build-arg VERSION="${VERSION}" \ + -t "${IMAGE}:${TAG}" \ + . + + docker push "${IMAGE}:${TAG}" + + - name: Create Git tag and GitHub release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + git config user.name "${GIT_USERNAME}" + git config user.email "${GIT_EMAIL}" + + if git rev-parse "${VERSION}" >/dev/null 2>&1; then + echo "Git tag ${VERSION} already exists" + else + git tag -m "${VERSION}" "${VERSION}" + git push origin "${VERSION}" + fi + + gh release view "${VERSION}" >/dev/null 2>&1 \ + || gh release create "${VERSION}" --title "${VERSION}" --generate-notes diff --git a/README.md b/README.md index 31e9c1748..b8a910da9 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,34 @@ clusters on Kubernetes (K8s) powered by [Patroni](https://github.com/zalando/pat It is configured only through Postgres manifests (CRDs) to ease integration into automated CI/CD pipelines with no access to Kubernetes API directly, promoting infrastructure as code vs manual operations. +## Changes by Edenlab LLC + +This fork by [Edenlab LLC](https://edenlab.io/) extends the `connectionPooler` section of the +Postgres CR with optional, backward-compatible pooler-specific pod template and deployment +settings. When omitted, behavior matches upstream Zalando Postgres Operator v1.15.1. + +New fields: + +* **inheritPodAnnotations** / **podAnnotations** — apply `spec.podAnnotations` to Spilo pods only + and set pooler annotations separately (e.g. keep Prometheus scrape on Postgres, not on PgBouncer). +* **inheritPodLabels** / **podLabels** — same pattern for labels. +* **nodeAffinity** / **podAntiAffinity** — pooler-specific scheduling overrides. +* **deploymentStrategy** — control pooler Deployment rollout (e.g. `maxSurge: 0` / + `maxUnavailable: 1` on small clusters with required pod anti-affinity). + +See [connection pooler parameters](docs/reference/cluster_manifest.md#connection-pooler) and +[examples](docs/user.md) for details. + +It also adds GitHub Actions [workflows](.github/workflows) to build and publish Docker images. + +Images are published to the public ECR gallery: +[public.ecr.aws/edenlabllc/core.postgres-operator](https://gallery.ecr.aws/edenlabllc/core.postgres-operator) + +Fork changes are based on upstream +[v1.15.1](https://github.com/zalando/postgres-operator/releases/tag/v1.15.1) and released under +Git tags such as [`v1.15.2`](https://github.com/edenlabllc/postgres-operator/releases/tag/v1.15.2) +(Gitflow: merge `release/v1.15.2` into `master`, CI tags and publishes the image). + ### Operator features * Rolling updates on Postgres cluster changes, incl. quick minor version updates diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index cea738dec..770dc2848 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -257,6 +257,42 @@ spec: type: string user: type: string + inheritPodAnnotations: + type: boolean + podAnnotations: + additionalProperties: + type: string + type: object + inheritPodLabels: + type: boolean + podLabels: + additionalProperties: + type: string + type: object + nodeAffinity: + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + properties: + enabled: + type: boolean + preferredDuringScheduling: + type: boolean + topologyKey: + type: string + type: object + deploymentStrategy: + properties: + type: + type: string + rollingUpdate: + properties: + maxSurge: + x-kubernetes-int-or-string: true + maxUnavailable: + x-kubernetes-int-or-string: true + type: object + type: object type: object databases: additionalProperties: diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 0717e411f..42393559d 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -659,6 +659,47 @@ for both master and replica pooler services (if `enableReplicaConnectionPooler` * **resources** Resource configuration for connection pooler deployment. +* **inheritPodAnnotations** + When `true` (default), `spec.podAnnotations` and operator `custom_pod_annotations` + are applied to connection pooler pods as well as Spilo pods. Set to `false` to + apply annotations to Spilo pods only and use `connectionPooler.podAnnotations` + for the pooler. + +* **podAnnotations** + Annotations applied to connection pooler pods only. Merged on top of inherited + annotations when `inheritPodAnnotations` is `true`. + +* **inheritPodLabels** + When `true` (default), labels from the Postgres CR metadata listed under operator + `inherited_labels` are applied to connection pooler pods. Set to `false` to + skip them. + +* **podLabels** + Extra labels applied to connection pooler pods only, merged on top of the base + pooler labels. + +* **nodeAffinity** + Node affinity for connection pooler pods. When omitted, `spec.nodeAffinity` is + used. + +* **podAntiAffinity** + Pod anti-affinity for connection pooler pods. When omitted, operator-level + `enable_pod_antiaffinity`, `pod_antiaffinity_preferred_during_scheduling`, and + `pod_antiaffinity_topology_key` apply. + + * **enabled** - enable or disable pod anti-affinity for the pooler. + * **preferredDuringScheduling** - use soft (`true`) or hard (`false`) anti-affinity. + * **topologyKey** - topology key for anti-affinity (default from operator config). + +* **deploymentStrategy** + Rollout strategy for the pooler Deployment. When omitted, the operator does + not set strategy and Kubernetes defaults apply (`RollingUpdate` with 25% + `maxSurge` / `maxUnavailable`). + + * **type** - deployment strategy type, typically `RollingUpdate`. + * **rollingUpdate.maxSurge** - maximum extra pods during rollout (e.g. `0`). + * **rollingUpdate.maxUnavailable** - maximum unavailable pods during rollout (e.g. `1`). + ## Custom TLS certificates Those parameters are grouped under the `tls` top-level key. Note, you have to diff --git a/docs/user.md b/docs/user.md index 293ddf8c2..c06dc5be9 100644 --- a/docs/user.md +++ b/docs/user.md @@ -1190,6 +1190,11 @@ each Postgres cluster, specify: ``` spec: + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8008" + prometheus.io/path: /metrics + connectionPooler: # how many instances of connection pooler to create numberOfInstances: 2 @@ -1212,6 +1217,22 @@ spec: limits: cpu: "1" memory: 100Mi + + # keep spec.podAnnotations on Spilo pods only; pooler gets podAnnotations below + inheritPodAnnotations: false + podAnnotations: + linkerd.io/inject: enabled + + # optional pooler-specific scheduling (omit to use spec/operator defaults) + podAntiAffinity: + enabled: true + preferredDuringScheduling: false + topologyKey: kubernetes.io/hostname + deploymentStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 ``` The `enableConnectionPooler` flag is not required when the `connectionPooler` diff --git a/manifests/complete-postgres-manifest.yaml b/manifests/complete-postgres-manifest.yaml index 93797e0e1..73b81c4a5 100644 --- a/manifests/complete-postgres-manifest.yaml +++ b/manifests/complete-postgres-manifest.yaml @@ -109,7 +109,11 @@ spec: # spiloRunAsGroup: 103 # spiloFSGroup: 103 # podAnnotations: -# annotation.key: value +# prometheus.io/scrape: "true" +# prometheus.io/port: "8008" +# prometheus.io/path: /metrics +# # podAnnotations apply to Spilo pods and, by default, to connection pooler pods too. +# # Use connectionPooler.inheritPodAnnotations: false to keep them on DB pods only. # serviceAnnotations: # annotation.key: value # podPriorityClassName: "spilo-pod-priority" @@ -184,6 +188,33 @@ spec: # limits: # cpu: "1" # memory: 100Mi +# # Pooler-specific pod annotations (optional). By default pooler inherits spec.podAnnotations. +# inheritPodAnnotations: false +# podAnnotations: +# linkerd.io/inject: enabled +# # Pooler-specific pod labels (optional). By default pooler inherits inherited_labels from CR metadata. +# inheritPodLabels: true +# podLabels: {} +# # Override spec.nodeAffinity for pooler pods only (optional). +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchExpressions: +# - key: db +# operator: In +# values: +# - postgres +# # Override operator-level pod anti-affinity for pooler pods (optional). +# podAntiAffinity: +# enabled: true +# preferredDuringScheduling: false +# topologyKey: kubernetes.io/hostname +# # Deployment rollout strategy for pooler (optional). When omitted, Kubernetes defaults apply. +# deploymentStrategy: +# type: RollingUpdate +# rollingUpdate: +# maxSurge: 0 +# maxUnavailable: 1 initContainers: - name: date diff --git a/manifests/connection-pooler-postgres-manifest.yaml b/manifests/connection-pooler-postgres-manifest.yaml new file mode 100644 index 000000000..89a33f1aa --- /dev/null +++ b/manifests/connection-pooler-postgres-manifest.yaml @@ -0,0 +1,50 @@ +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: acid-pooler-cluster + annotations: + linkerd.io/inject: enabled +spec: + teamId: "acid" + volume: + size: 1Gi + numberOfInstances: 2 + enableConnectionPooler: true + users: + zalando: + - superuser + - createdb + pooler: [] + databases: + foo: zalando + postgresql: + version: "17" + # Applied to Spilo pods. Also inherited by pooler unless inheritPodAnnotations is false. + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8008" + prometheus.io/path: /metrics + connectionPooler: + numberOfInstances: 2 + mode: session + schema: pooler + user: pooler + resources: + requests: + cpu: 300m + memory: 100Mi + limits: + cpu: "1" + memory: 100Mi + inheritPodAnnotations: false + podAnnotations: + linkerd.io/inject: enabled + podAntiAffinity: + enabled: true + preferredDuringScheduling: false + topologyKey: kubernetes.io/hostname + deploymentStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index cea738dec..770dc2848 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -257,6 +257,42 @@ spec: type: string user: type: string + inheritPodAnnotations: + type: boolean + podAnnotations: + additionalProperties: + type: string + type: object + inheritPodLabels: + type: boolean + podLabels: + additionalProperties: + type: string + type: object + nodeAffinity: + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + properties: + enabled: + type: boolean + preferredDuringScheduling: + type: boolean + topologyKey: + type: string + type: object + deploymentStrategy: + properties: + type: + type: string + rollingUpdate: + properties: + maxSurge: + x-kubernetes-int-or-string: true + maxUnavailable: + x-kubernetes-int-or-string: true + type: object + type: object type: object databases: additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml b/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml index cea738dec..770dc2848 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml +++ b/pkg/apis/acid.zalan.do/v1/postgresql.crd.yaml @@ -257,6 +257,42 @@ spec: type: string user: type: string + inheritPodAnnotations: + type: boolean + podAnnotations: + additionalProperties: + type: string + type: object + inheritPodLabels: + type: boolean + podLabels: + additionalProperties: + type: string + type: object + nodeAffinity: + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + properties: + enabled: + type: boolean + preferredDuringScheduling: + type: boolean + topologyKey: + type: string + type: object + deploymentStrategy: + properties: + type: + type: string + rollingUpdate: + properties: + maxSurge: + x-kubernetes-int-or-string: true + maxUnavailable: + x-kubernetes-int-or-string: true + type: object + type: object type: object databases: additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 85a73605c..09a2c9444 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -7,6 +7,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // +genclient @@ -329,6 +330,28 @@ type PostgresStatus struct { // TODO: figure out what other important parameters of the connection pooler it // makes sense to expose. E.g. pool size (min/max boundaries), max client // connections etc. + +// ConnectionPoolerPodAntiAffinity configures pod anti-affinity for connection pooler pods. +// When omitted, operator-level pod anti-affinity settings apply. +type ConnectionPoolerPodAntiAffinity struct { + Enabled *bool `json:"enabled,omitempty"` + PreferredDuringScheduling *bool `json:"preferredDuringScheduling,omitempty"` + TopologyKey string `json:"topologyKey,omitempty"` +} + +// ConnectionPoolerRollingUpdate configures rolling update parameters for the pooler Deployment. +type ConnectionPoolerRollingUpdate struct { + MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` +} + +// ConnectionPoolerDeploymentStrategy configures the pooler Deployment rollout strategy. +// When omitted, the operator does not set strategy and Kubernetes defaults apply. +type ConnectionPoolerDeploymentStrategy struct { + Type string `json:"type,omitempty"` + RollingUpdate *ConnectionPoolerRollingUpdate `json:"rollingUpdate,omitempty"` +} + type ConnectionPooler struct { // +kubebuilder:validation:Minimum=1 NumberOfInstances *int32 `json:"numberOfInstances,omitempty"` @@ -339,6 +362,21 @@ type ConnectionPooler struct { DockerImage string `json:"dockerImage,omitempty"` MaxDBConnections *int32 `json:"maxDBConnections,omitempty"` + // InheritPodAnnotations when true (default) merges spec.podAnnotations onto pooler pods. + InheritPodAnnotations *bool `json:"inheritPodAnnotations,omitempty"` + // PodAnnotations are applied to pooler pods only, merged on top when InheritPodAnnotations is true. + PodAnnotations map[string]string `json:"podAnnotations,omitempty"` + // InheritPodLabels when true (default) inherits cluster labels configured via inherited_labels. + InheritPodLabels *bool `json:"inheritPodLabels,omitempty"` + // PodLabels are applied to pooler pods only, merged on top of base pooler labels. + PodLabels map[string]string `json:"podLabels,omitempty"` + // NodeAffinity overrides spec.nodeAffinity for pooler pods when set. + NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"` + // PodAntiAffinity overrides operator-level pod anti-affinity for pooler pods when set. + PodAntiAffinity *ConnectionPoolerPodAntiAffinity `json:"podAntiAffinity,omitempty"` + // DeploymentStrategy configures pooler Deployment rollout; when omitted K8s defaults apply. + DeploymentStrategy *ConnectionPoolerDeploymentStrategy `json:"deploymentStrategy,omitempty"` + *Resources `json:"resources,omitempty"` } diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index ddccccf7f..8691c5956 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -32,6 +32,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -98,6 +99,79 @@ func (in *CloneDescription) DeepCopy() *CloneDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPoolerPodAntiAffinity) DeepCopyInto(out *ConnectionPoolerPodAntiAffinity) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.PreferredDuringScheduling != nil { + in, out := &in.PreferredDuringScheduling, &out.PreferredDuringScheduling + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolerPodAntiAffinity. +func (in *ConnectionPoolerPodAntiAffinity) DeepCopy() *ConnectionPoolerPodAntiAffinity { + if in == nil { + return nil + } + out := new(ConnectionPoolerPodAntiAffinity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPoolerRollingUpdate) DeepCopyInto(out *ConnectionPoolerRollingUpdate) { + *out = *in + if in.MaxSurge != nil { + in, out := &in.MaxSurge, &out.MaxSurge + *out = new(intstr.IntOrString) + **out = **in + } + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(intstr.IntOrString) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolerRollingUpdate. +func (in *ConnectionPoolerRollingUpdate) DeepCopy() *ConnectionPoolerRollingUpdate { + if in == nil { + return nil + } + out := new(ConnectionPoolerRollingUpdate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPoolerDeploymentStrategy) DeepCopyInto(out *ConnectionPoolerDeploymentStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(ConnectionPoolerRollingUpdate) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolerDeploymentStrategy. +func (in *ConnectionPoolerDeploymentStrategy) DeepCopy() *ConnectionPoolerDeploymentStrategy { + if in == nil { + return nil + } + out := new(ConnectionPoolerDeploymentStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) { *out = *in @@ -111,6 +185,45 @@ func (in *ConnectionPooler) DeepCopyInto(out *ConnectionPooler) { *out = new(int32) **out = **in } + if in.InheritPodAnnotations != nil { + in, out := &in.InheritPodAnnotations, &out.InheritPodAnnotations + *out = new(bool) + **out = **in + } + if in.PodAnnotations != nil { + in, out := &in.PodAnnotations, &out.PodAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.InheritPodLabels != nil { + in, out := &in.InheritPodLabels, &out.InheritPodLabels + *out = new(bool) + **out = **in + } + if in.PodLabels != nil { + in, out := &in.PodLabels, &out.PodLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NodeAffinity != nil { + in, out := &in.NodeAffinity, &out.NodeAffinity + *out = new(corev1.NodeAffinity) + (*in).DeepCopyInto(*out) + } + if in.PodAntiAffinity != nil { + in, out := &in.PodAntiAffinity, &out.PodAntiAffinity + *out = new(ConnectionPoolerPodAntiAffinity) + (*in).DeepCopyInto(*out) + } + if in.DeploymentStrategy != nil { + in, out := &in.DeploymentStrategy, &out.DeploymentStrategy + *out = new(ConnectionPoolerDeploymentStrategy) + (*in).DeepCopyInto(*out) + } if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = new(Resources) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index eed4ee933..55bf76d9d 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -836,6 +836,27 @@ func (c *Cluster) compareAnnotations(old, new map[string]string, removedList *[] return reason != "", reason } +func compareLabels(old, new map[string]string) (bool, string) { + reason := "" + + for key := range old { + if _, ok := new[key]; !ok { + reason += fmt.Sprintf(" Removed %q.", key) + } + } + + for key := range new { + v, ok := old[key] + if !ok { + reason += fmt.Sprintf(" Added %q with value %q.", key, new[key]) + } else if v != new[key] { + reason += fmt.Sprintf(" %q changed from %q to %q.", key, v, new[key]) + } + } + + return reason != "", reason +} + func (c *Cluster) compareServices(old, new *v1.Service) (bool, string) { if old.Spec.Type != new.Spec.Type { return false, fmt.Sprintf("new service's type %q does not match the current one %q", diff --git a/pkg/cluster/connection_pooler.go b/pkg/cluster/connection_pooler.go index 61cd9b041..c014c71fd 100644 --- a/pkg/cluster/connection_pooler.go +++ b/pkg/cluster/connection_pooler.go @@ -114,7 +114,11 @@ func (c *Cluster) poolerUser(spec *acidv1.PostgresSpec) string { // when listing pooler k8s objects func (c *Cluster) poolerLabelsSet(addExtraLabels bool) labels.Set { - poolerLabels := c.labelsSet(addExtraLabels) + return c.poolerLabelsSetWithInherit(addExtraLabels, true) +} + +func (c *Cluster) poolerLabelsSetWithInherit(addExtraLabels bool, inheritCRLabels bool) labels.Set { + poolerLabels := c.labelsSetWithInheritedLabels(addExtraLabels, inheritCRLabels) // TODO should be config values poolerLabels["application"] = "db-connection-pooler" @@ -122,13 +126,16 @@ func (c *Cluster) poolerLabelsSet(addExtraLabels bool) labels.Set { return poolerLabels } -// Return connection pooler labels selector, which should from one point of view -// inherit most of the labels from the cluster itself, but at the same time -// have e.g. different `application` label, so that recreatePod operation will -// not interfere with it (it lists all the pods via labels, and if there would -// be no difference, it will recreate also pooler pods). -func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) *metav1.LabelSelector { - poolerLabelsSet := c.poolerLabelsSet(addExtraLabels) +func (c *Cluster) connectionPoolerSpec() *acidv1.ConnectionPooler { + if c.Spec.ConnectionPooler != nil { + return c.Spec.ConnectionPooler + } + return &acidv1.ConnectionPooler{} +} + +func (c *Cluster) connectionPoolerLabelsForSpec(role PostgresRole, addExtraLabels bool, poolerSpec *acidv1.ConnectionPooler) *metav1.LabelSelector { + inheritLabels := poolerSpec.InheritPodLabels == nil || *poolerSpec.InheritPodLabels + poolerLabelsSet := c.poolerLabelsSetWithInherit(addExtraLabels, inheritLabels) // TODO should be config values poolerLabelsSet["connection-pooler"] = c.connectionPoolerName(role) @@ -140,12 +147,102 @@ func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) poolerLabelsSet = labels.Merge(poolerLabelsSet, extraLabels) } + if poolerSpec.PodLabels != nil { + poolerLabelsSet = labels.Merge(poolerLabelsSet, poolerSpec.PodLabels) + } + return &metav1.LabelSelector{ MatchLabels: poolerLabelsSet, MatchExpressions: nil, } } +func (c *Cluster) connectionPoolerLabels(role PostgresRole, addExtraLabels bool) *metav1.LabelSelector { + return c.connectionPoolerLabelsForSpec(role, addExtraLabels, c.connectionPoolerSpec()) +} + +func (c *Cluster) generateConnectionPoolerPodAnnotations(poolerSpec *acidv1.ConnectionPooler) map[string]string { + spec := &c.Spec + annotations := make(map[string]string) + + inherit := poolerSpec.InheritPodAnnotations == nil || *poolerSpec.InheritPodAnnotations + if inherit { + if podAnn := c.generatePodAnnotations(spec); podAnn != nil { + for k, v := range podAnn { + annotations[k] = v + } + } else { + for k, v := range c.OpConfig.CustomPodAnnotations { + annotations[k] = v + } + } + } else { + for k, v := range c.OpConfig.CustomPodAnnotations { + annotations[k] = v + } + } + + if poolerSpec.PodAnnotations != nil { + for k, v := range poolerSpec.PodAnnotations { + annotations[k] = v + } + } + + return c.annotationsSet(annotations) +} + +func (c *Cluster) connectionPoolerNodeAffinity(spec *acidv1.PostgresSpec, poolerSpec *acidv1.ConnectionPooler) *v1.NodeAffinity { + if poolerSpec.NodeAffinity != nil { + return poolerSpec.NodeAffinity + } + return spec.NodeAffinity +} + +func (c *Cluster) connectionPoolerPodAntiAffinitySettings(poolerSpec *acidv1.ConnectionPooler) (enabled bool, preferred bool, topologyKey string) { + enabled = c.OpConfig.EnablePodAntiAffinity + preferred = c.OpConfig.PodAntiAffinityPreferredDuringScheduling + topologyKey = c.OpConfig.PodAntiAffinityTopologyKey + + if poolerSpec.PodAntiAffinity != nil { + if poolerSpec.PodAntiAffinity.Enabled != nil { + enabled = *poolerSpec.PodAntiAffinity.Enabled + } + if poolerSpec.PodAntiAffinity.PreferredDuringScheduling != nil { + preferred = *poolerSpec.PodAntiAffinity.PreferredDuringScheduling + } + if poolerSpec.PodAntiAffinity.TopologyKey != "" { + topologyKey = poolerSpec.PodAntiAffinity.TopologyKey + } + } + + return enabled, preferred, topologyKey +} + +func (c *Cluster) connectionPoolerDeploymentStrategy(poolerSpec *acidv1.ConnectionPooler) *appsv1.DeploymentStrategy { + if poolerSpec.DeploymentStrategy == nil { + return nil + } + + ds := poolerSpec.DeploymentStrategy + strategy := &appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + } + if ds.Type != "" { + strategy.Type = appsv1.DeploymentStrategyType(ds.Type) + } + if ds.RollingUpdate != nil { + strategy.RollingUpdate = &appsv1.RollingUpdateDeployment{} + if ds.RollingUpdate.MaxSurge != nil { + strategy.RollingUpdate.MaxSurge = ds.RollingUpdate.MaxSurge + } + if ds.RollingUpdate.MaxUnavailable != nil { + strategy.RollingUpdate.MaxUnavailable = ds.RollingUpdate.MaxUnavailable + } + } + + return strategy +} + // Prepare the database for connection pooler to be used, i.e. install lookup // function (do it first, because it should be fast and if it didn't succeed, // it doesn't makes sense to create more K8S objects. At this moment we assume @@ -460,9 +557,9 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( podTemplate := &v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: c.connectionPoolerLabels(role, true).MatchLabels, + Labels: c.connectionPoolerLabelsForSpec(role, true, connectionPoolerSpec).MatchLabels, Namespace: c.Namespace, - Annotations: c.annotationsSet(c.generatePodAnnotations(spec)), + Annotations: c.generateConnectionPoolerPodAnnotations(connectionPoolerSpec), }, Spec: v1.PodSpec{ TerminationGracePeriodSeconds: &gracePeriod, @@ -474,14 +571,15 @@ func (c *Cluster) generateConnectionPoolerPodTemplate(role PostgresRole) ( }, } - nodeAffinity := c.nodeAffinity(c.OpConfig.NodeReadinessLabel, spec.NodeAffinity) - if c.OpConfig.EnablePodAntiAffinity { - labelsSet := labels.Set(c.connectionPoolerLabels(role, false).MatchLabels) + nodeAffinity := c.nodeAffinity(c.OpConfig.NodeReadinessLabel, c.connectionPoolerNodeAffinity(spec, connectionPoolerSpec)) + enableAntiAffinity, preferredAntiAffinity, antiAffinityTopologyKey := c.connectionPoolerPodAntiAffinitySettings(connectionPoolerSpec) + if enableAntiAffinity { + labelsSet := labels.Set(c.connectionPoolerLabelsForSpec(role, false, connectionPoolerSpec).MatchLabels) podTemplate.Spec.Affinity = podAffinity( labelsSet, - c.OpConfig.PodAntiAffinityTopologyKey, + antiAffinityTopologyKey, nodeAffinity, - c.OpConfig.PodAntiAffinityPreferredDuringScheduling, + preferredAntiAffinity, true, ) } else if nodeAffinity != nil { @@ -546,6 +644,10 @@ func (c *Cluster) generateConnectionPoolerDeployment(connectionPooler *Connectio }, } + if strategy := c.connectionPoolerDeploymentStrategy(connectionPoolerSpec); strategy != nil { + deployment.Spec.Strategy = *strategy + } + return deployment, nil } @@ -1183,7 +1285,7 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql syncReason = append(syncReason, specReason...) } - newPodAnnotations := c.annotationsSet(c.generatePodAnnotations(&c.Spec)) + newPodAnnotations := c.generateConnectionPoolerPodAnnotations(newConnectionPooler) deletedPodAnnotations := []string{} if changed, reason := c.compareAnnotations(deployment.Spec.Template.Annotations, newPodAnnotations, &deletedPodAnnotations); changed { specSync = true @@ -1208,6 +1310,28 @@ func (c *Cluster) syncConnectionPoolerWorker(oldSpec, newSpec *acidv1.Postgresql deployment.Spec.Template.Annotations = newPodAnnotations } + newPodLabels := c.connectionPoolerLabelsForSpec(role, true, newConnectionPooler).MatchLabels + if changed, reason := compareLabels(deployment.Spec.Template.Labels, newPodLabels); changed { + specSync = true + syncReason = append(syncReason, []string{"new connection pooler's pod template labels do not match the current ones: " + reason}...) + } + + expectedPodTemplate, err := c.generateConnectionPoolerPodTemplate(role) + if err != nil { + return nil, fmt.Errorf("could not generate pod template for %s connection pooler: %v", role, err) + } + if !reflect.DeepEqual(deployment.Spec.Template.Spec.Affinity, expectedPodTemplate.Spec.Affinity) { + specSync = true + syncReason = append(syncReason, "new connection pooler's pod template affinity does not match the current one") + } + + if desiredStrategy := c.connectionPoolerDeploymentStrategy(newConnectionPooler); desiredStrategy != nil { + if !reflect.DeepEqual(deployment.Spec.Strategy, *desiredStrategy) { + specSync = true + syncReason = append(syncReason, "new connection pooler deployment strategy does not match the current one") + } + } + defaultsSync, defaultsReason := c.needSyncConnectionPoolerDefaults(&c.Config, newConnectionPooler, deployment) syncReason = append(syncReason, defaultsReason...) diff --git a/pkg/cluster/connection_pooler_test.go b/pkg/cluster/connection_pooler_test.go index 1b41cbb02..93aecee14 100644 --- a/pkg/cluster/connection_pooler_test.go +++ b/pkg/cluster/connection_pooler_test.go @@ -17,6 +17,7 @@ import ( appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/fake" ) @@ -1332,3 +1333,143 @@ func TestConnectionPoolerServiceType(t *testing.T) { } } } + +func newPoolerTestCluster(t *testing.T, spec acidv1.PostgresSpec) *Cluster { + t.Helper() + client, _ := newFakeK8sPoolerTestClient() + pg := acidv1.Postgresql{ + ObjectMeta: metav1.ObjectMeta{ + Name: "acid-test-cluster", + Namespace: "default", + }, + Spec: spec, + } + cluster := New( + Config{ + OpConfig: config.Config{ + EnablePodAntiAffinity: true, + PodAntiAffinityPreferredDuringScheduling: false, + PodAntiAffinityTopologyKey: "kubernetes.io/hostname", + ConnectionPooler: config.ConnectionPooler{ + ConnectionPoolerDefaultCPURequest: "100m", + ConnectionPoolerDefaultCPULimit: "100m", + ConnectionPoolerDefaultMemoryRequest: "100Mi", + ConnectionPoolerDefaultMemoryLimit: "100Mi", + NumberOfInstances: k8sutil.Int32ToPointer(2), + }, + PodManagementPolicy: "ordered_ready", + Resources: config.Resources{ + ClusterLabels: map[string]string{"application": "spilo"}, + ClusterNameLabel: "cluster-name", + DefaultCPURequest: "300m", + DefaultCPULimit: "300m", + DefaultMemoryRequest: "300Mi", + DefaultMemoryLimit: "300Mi", + PodRoleLabel: "spilo-role", + }, + }, + }, client, pg, logger, eventRecorder) + cluster.Name = pg.Name + cluster.Namespace = pg.Namespace + return cluster +} + +func TestGenerateConnectionPoolerPodAnnotationsBackwardCompatible(t *testing.T) { + cluster := newPoolerTestCluster(t, acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + PodAnnotations: map[string]string{ + "prometheus.io/scrape": "true", + }, + ConnectionPooler: &acidv1.ConnectionPooler{}, + }) + + annotations := cluster.generateConnectionPoolerPodAnnotations(cluster.Spec.ConnectionPooler) + assert.Equal(t, "true", annotations["prometheus.io/scrape"]) +} + +func TestGenerateConnectionPoolerPodAnnotationsWithoutInheritance(t *testing.T) { + cluster := newPoolerTestCluster(t, acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + PodAnnotations: map[string]string{ + "prometheus.io/scrape": "true", + }, + ConnectionPooler: &acidv1.ConnectionPooler{ + InheritPodAnnotations: boolToPointer(false), + PodAnnotations: map[string]string{ + "linkerd.io/inject": "enabled", + }, + }, + }) + + annotations := cluster.generateConnectionPoolerPodAnnotations(cluster.Spec.ConnectionPooler) + _, hasPrometheus := annotations["prometheus.io/scrape"] + assert.False(t, hasPrometheus) + assert.Equal(t, "enabled", annotations["linkerd.io/inject"]) +} + +func TestConnectionPoolerDeploymentStrategy(t *testing.T) { + maxSurge := intstr.FromInt(0) + maxUnavailable := intstr.FromInt(1) + cluster := newPoolerTestCluster(t, acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{ + DeploymentStrategy: &acidv1.ConnectionPoolerDeploymentStrategy{ + Type: "RollingUpdate", + RollingUpdate: &acidv1.ConnectionPoolerRollingUpdate{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + }, + }) + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{ + Master: { + Name: cluster.connectionPoolerName(Master), + Namespace: cluster.Namespace, + ClusterName: cluster.Name, + Role: Master, + }, + } + + deployment, err := cluster.generateConnectionPoolerDeployment(cluster.ConnectionPooler[Master]) + assert.NoError(t, err) + assert.Equal(t, appsv1.RollingUpdateDeploymentStrategyType, deployment.Spec.Strategy.Type) + assert.Equal(t, int32(0), deployment.Spec.Strategy.RollingUpdate.MaxSurge.IntVal) + assert.Equal(t, int32(1), deployment.Spec.Strategy.RollingUpdate.MaxUnavailable.IntVal) +} + +func TestConnectionPoolerDeploymentStrategyOmittedByDefault(t *testing.T) { + cluster := newPoolerTestCluster(t, acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{}, + }) + cluster.ConnectionPooler = map[PostgresRole]*ConnectionPoolerObjects{ + Master: { + Name: cluster.connectionPoolerName(Master), + Namespace: cluster.Namespace, + ClusterName: cluster.Name, + Role: Master, + }, + } + + deployment, err := cluster.generateConnectionPoolerDeployment(cluster.ConnectionPooler[Master]) + assert.NoError(t, err) + assert.Equal(t, appsv1.DeploymentStrategy{}, deployment.Spec.Strategy) +} + +func TestConnectionPoolerPodAntiAffinitySettings(t *testing.T) { + cluster := newPoolerTestCluster(t, acidv1.PostgresSpec{ + EnableConnectionPooler: boolToPointer(true), + ConnectionPooler: &acidv1.ConnectionPooler{ + PodAntiAffinity: &acidv1.ConnectionPoolerPodAntiAffinity{ + Enabled: boolToPointer(true), + PreferredDuringScheduling: boolToPointer(false), + }, + }, + }) + + enabled, preferred, topologyKey := cluster.connectionPoolerPodAntiAffinitySettings(cluster.Spec.ConnectionPooler) + assert.True(t, enabled) + assert.False(t, preferred) + assert.Equal(t, "kubernetes.io/hostname", topologyKey) +} diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index c3a9dda31..7d035c695 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -481,6 +481,10 @@ func (c *Cluster) waitStatefulsetPodsReady() error { // For backward compatibility, shouldAddExtraLabels must be false // when listing k8s objects. See operator PR #252 func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set { + return c.labelsSetWithInheritedLabels(shouldAddExtraLabels, true) +} + +func (c *Cluster) labelsSetWithInheritedLabels(shouldAddExtraLabels bool, inheritCRLabels bool) labels.Set { lbls := make(map[string]string) for k, v := range c.OpConfig.ClusterLabels { lbls[k] = v @@ -491,17 +495,19 @@ func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set { // enables filtering resources owned by a team lbls["team"] = c.Postgresql.Spec.TeamID - // allow to inherit certain labels from the 'postgres' object - if spec, err := c.GetSpec(); err == nil { - for k, v := range spec.ObjectMeta.Labels { - for _, match := range c.OpConfig.InheritedLabels { - if k == match { - lbls[k] = v + if inheritCRLabels { + // allow to inherit certain labels from the 'postgres' object + if spec, err := c.GetSpec(); err == nil { + for k, v := range spec.ObjectMeta.Labels { + for _, match := range c.OpConfig.InheritedLabels { + if k == match { + lbls[k] = v + } } } + } else { + c.logger.Warningf("could not get the list of InheritedLabels for cluster %q: %v", c.Name, err) } - } else { - c.logger.Warningf("could not get the list of InheritedLabels for cluster %q: %v", c.Name, err) } }