diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ea00ab78..72732401 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,20 +8,24 @@ env: kind-version: 'v0.32.0' permissions: contents: read - statuses: write - checks: write - pull-requests: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: changes: runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read outputs: code: ${{ steps.filter.outputs.code }} steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Detect non-docs changes - uses: dorny/paths-filter@v4 + uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4 id: filter with: predicate-quantifier: 'every' @@ -36,19 +40,30 @@ jobs: lint: runs-on: ubuntu-latest + timeout-minutes: 15 needs: [ changes ] if: needs.changes.outputs.code == 'true' steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' - + cache: false + + - name: Go cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-lint-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: | + go-lint- - name: Go Tidy run: go mod tidy && git diff --exit-code @@ -60,17 +75,17 @@ jobs: run: make manifests && git diff --exit-code - name: Lint with golangci-lint - uses: golangci/golangci-lint-action@v9 + uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9 with: version: v2.12.2 - name: Lint with codespell - uses: codespell-project/actions-codespell@v2.2 + uses: codespell-project/actions-codespell@8f01853be192eb0f849a5c7d721450e7a467c579 # v2.2 with: config: ci/.codespellrc - name: actionlint - uses: raven-actions/actionlint@v2 + uses: raven-actions/actionlint@3d39aea434753780c3b3d4a1a31c854b4dbf49d7 # v2 with: flags: '-config-file ci/actionlint.yaml' @@ -84,14 +99,15 @@ jobs: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'crd-breaking-change') runs-on: ubuntu-latest + timeout-minutes: 15 steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' @@ -100,16 +116,17 @@ jobs: bundle: runs-on: ubuntu-latest + timeout-minutes: 20 needs: [ changes ] if: needs.changes.outputs.code == 'true' env: VERSION: 0.0.1 steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' @@ -117,7 +134,7 @@ jobs: run: make bundle - name: Create k8s Kind Cluster - uses: helm/kind-action@v1 + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1 with: cluster_name: 'test-bundle' version: ${{ env.kind-version }} @@ -136,16 +153,31 @@ jobs: build_and_test: runs-on: ubuntu-latest + timeout-minutes: 25 + permissions: + contents: read + checks: write needs: [ changes ] if: needs.changes.outputs.code == 'true' steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' + cache: false + + - name: Go cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-unit-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: | + go-unit- - name: Go Mod run: go mod download @@ -162,7 +194,7 @@ jobs: make test-ci - name: Test Report - uses: dorny/test-reporter@v3 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 if: ${{ !cancelled() }} with: name: Unit tests @@ -172,16 +204,28 @@ jobs: fuzz_specs: runs-on: ubuntu-latest + timeout-minutes: 20 needs: [ changes ] if: needs.changes.outputs.code == 'true' steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' + cache: false + + - name: Go cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-fuzz-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: | + go-fuzz- - name: Go Mod run: go mod download @@ -191,14 +235,15 @@ jobs: helm-test: runs-on: ubuntu-latest + timeout-minutes: 15 needs: [ changes ] if: needs.changes.outputs.code == 'true' steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' @@ -222,6 +267,7 @@ jobs: compat-e2e-test: runs-on: [ubuntu-latest] + timeout-minutes: 60 needs: [ changes ] if: needs.changes.outputs.code == 'true' strategy: @@ -230,40 +276,51 @@ jobs: include: - name: minimal-k8s-all-deploy-methods k8s_image: v1.28.15 - clickhouse_version: "26.5" + clickhouse_version: "26.6.2.81" deploy_target: test-compat-e2e - name: maximal-k8s-all-deploy-methods k8s_image: v1.36.1 - clickhouse_version: "26.5" + clickhouse_version: "26.6.2.81" deploy_target: test-compat-e2e - name: olm-deploy-method k8s_image: v1.28.15 - clickhouse_version: "26.5" + clickhouse_version: "26.6.2.81" deploy_target: test-compat-e2e-olm - name: supported-clickhouse-compatibility k8s_image: v1.30.13 - clickhouse_version: "26.5,26.5-distroless,26.4,26.3,25.8" + clickhouse_version: "26.6.2.81-distroless,26.5.5.8,26.4.5.143,26.3.17.56,25.8.28.1" deploy_target: test-compat-e2e-manifest - name: operator-upgrade k8s_image: v1.30.13 - clickhouse_version: "26.5" + clickhouse_version: "26.6.2.81" deploy_target: test-compat-e2e-upgrade steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-tags: 'true' - name: Free disk space - uses: jlumbroso/free-disk-space@v1.3.1 + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' + cache: false + + - name: Go cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-compat-e2e-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: | + go-compat-e2e- - name: Create k8s Kind Cluster - uses: helm/kind-action@v1 + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1 with: cluster_name: kind version: ${{ env.kind-version }} @@ -278,9 +335,16 @@ jobs: - name: Pre-pull ClickHouse images into Kind run: | + pull() { + for _ in 1 2 3; do + if docker pull "$1"; then return 0; fi + sleep 15 + done + return 1 + } for version in $(echo "${{ matrix.clickhouse_version }}" | tr ',' ' '); do for image in clickhouse/clickhouse-keeper clickhouse/clickhouse-server; do - docker pull "docker.io/${image}:${version}" + pull "docker.io/${image}:${version}" kind load docker-image "docker.io/${image}:${version}" done done @@ -291,7 +355,7 @@ jobs: CLICKHOUSE_VERSION: ${{ matrix.clickhouse_version }} - name: Upload test report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: ${{ !cancelled() }} with: name: e2e-report-${{ matrix.name }} @@ -307,12 +371,13 @@ jobs: matrix: shard: [ 1, 2, 3, 4 ] runs-on: [self-hosted, amd-medium] + timeout-minutes: 45 steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' @@ -320,7 +385,7 @@ jobs: run: go mod download - name: Create k8s Kind Cluster - uses: helm/kind-action@v1 + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1 with: cluster_name: kind version: ${{ env.kind-version }} @@ -334,7 +399,7 @@ jobs: run: make test-e2e - name: Upload test report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: ${{ !cancelled() }} with: name: e2e-report-shard-${{ matrix.shard }} @@ -345,13 +410,17 @@ jobs: e2e-report: needs: [ changes, compat-e2e-test, e2e-test ] runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + checks: write if: ${{ !cancelled() && needs.changes.outputs.code == 'true' }} steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Download all e2e test reports - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: e2e-report-* @@ -359,7 +428,7 @@ jobs: run: python3 ci/merge-junit-reports.py - name: Test Report - uses: dorny/test-reporter@v3 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 with: name: E2E tests badge-title: E2E tests @@ -380,9 +449,11 @@ jobs: permissions: contents: read packages: write + checks: write env: KUBECONFIG: /home/runner/okd-install/auth/kubeconfig VERSION: 0.0.99-test + PLATFORMS: linux/amd64 IMG: ghcr.io/clickhouse/clickhouse-operator:test BUNDLE_IMG: ghcr.io/clickhouse/clickhouse-operator-bundle:test CATALOG_IMG: ghcr.io/clickhouse/clickhouse-operator-catalog:test @@ -391,17 +462,17 @@ jobs: OPENSHIFT_CHANNEL: stable-v0 steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' - name: Log in to ghcr.io - uses: docker/login-action@v4 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -420,7 +491,7 @@ jobs: run: make test-compat-e2e-olm-openshift - name: Test Report - uses: dorny/test-reporter@v3 + uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3 with: name: E2E tests badge-title: E2E tests @@ -434,6 +505,7 @@ jobs: ci-success-check: name: All CI checks passed runs-on: ubuntu-latest + timeout-minutes: 5 needs: [ changes, lint, bundle, build_and_test, fuzz_specs, helm-test, compat-e2e-test, e2e-test, check-crd-compat ] if: always() steps: diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 00000000..3dcf4bf6 --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,47 @@ +name: CodeQL +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '30 4 * * 1' +permissions: + contents: read + +jobs: + analyze: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + include: + - language: go + build-mode: autobuild + - language: actions + build-mode: none + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Set up Go + if: matrix.language == 'go' + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 + with: + go-version-file: 'go.mod' + cache: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + + - name: Analyze + uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/docs-lint.yaml b/.github/workflows/docs-lint.yaml index 75670346..3fdcee42 100644 --- a/.github/workflows/docs-lint.yaml +++ b/.github/workflows/docs-lint.yaml @@ -7,17 +7,22 @@ on: branches: [ main ] permissions: contents: read - statuses: write - checks: write - pull-requests: write +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: vale: name: vale-linter runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + checks: write + pull-requests: read steps: - - uses: actions/checkout@v7 - - uses: errata-ai/vale-action@2.1.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: errata-ai/vale-action@85f9f7f2c5f449ac0ae5b66662961bae3f77ca6a # 2.1.2 with: fail_on_error: 'true' filter_mode: nofilter @@ -25,16 +30,20 @@ jobs: doc-links: name: Doc links runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + pull-requests: write steps: - - uses: actions/checkout@v7 - - uses: dorny/paths-filter@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4 id: changes with: filters: | docs: - 'docs/**' - if: steps.changes.outputs.docs == 'true' - uses: UmbrellaDocs/action-linkspector@v1 + uses: UmbrellaDocs/action-linkspector@6c637d70424624231467a4ca918be54fa3b792d0 # v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} reporter: github-pr-review @@ -45,11 +54,12 @@ jobs: api-reference-generated: name: API Reference Generated runs-on: ubuntu-latest + timeout-minutes: 15 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' @@ -62,6 +72,7 @@ jobs: ci-success-check: name: All Docs CI checks passed runs-on: ubuntu-latest + timeout-minutes: 5 needs: [ vale, doc-links, api-reference-generated ] if: always() steps: diff --git a/.github/workflows/release-main.yml b/.github/workflows/release-main.yml index a4a87006..541ef06f 100644 --- a/.github/workflows/release-main.yml +++ b/.github/workflows/release-main.yml @@ -6,34 +6,36 @@ env: REGISTRY: ghcr.io IMAGE_REPO: ghcr.io/clickhouse +permissions: + contents: read + jobs: release-main: permissions: + contents: read packages: write - attestations: write - id-token: write - contents: write runs-on: ubuntu-latest + timeout-minutes: 45 steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-tags: true fetch-depth: '0' - name: Free disk space - uses: jlumbroso/free-disk-space@v1.3.1 + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' - name: Log in to the Container registry - uses: docker/login-action@v4 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Install jq - uses: dcarbone/install-jq-action@v4.0.1 + uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 - name: Build operator image run: make docker-buildx-latest - name: Install Helm diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4045bfd3..362917d5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,22 +10,22 @@ env: VERSION: ${{ github.ref_name }} permissions: - packages: write - attestations: write - id-token: write - contents: write - pull-requests: read + contents: read jobs: release-operator-image: runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Free disk space - uses: jlumbroso/free-disk-space@v1.3.1 + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Log in to the Container registry - uses: docker/login-action@v4 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -37,19 +37,23 @@ jobs: release-operator-olm-images: runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Free disk space - uses: jlumbroso/free-disk-space@v1.3.1 + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Log in to the Container registry - uses: docker/login-action@v4 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Install jq - uses: dcarbone/install-jq-action@v4.0.1 + uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 - name: Set VERSION run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_ENV" - name: Build operator bundle image @@ -63,9 +67,13 @@ jobs: release-operator-helm-chart: runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: write steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Install Helm run: curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - name: Verify Helm installation @@ -83,15 +91,20 @@ jobs: create-release: runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + packages: read + pull-requests: read needs: - release-operator-image - release-operator-olm-images - release-operator-helm-chart steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' - name: Set VERSION @@ -117,7 +130,7 @@ jobs: run: make package-cluster-chart - name: Log in to the Container registry - uses: docker/login-action@v4 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -127,7 +140,7 @@ jobs: - name: Build Changelog id: build_changelog - uses: mikepenz/release-changelog-builder-action@v6.2.3 + uses: mikepenz/release-changelog-builder-action@c9bcd8238b6f41e05561348339429d360b1c0247 # v6.2.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -160,7 +173,7 @@ jobs: }] } - name: Create GitHub Release - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3 with: tag_name: ${{ github.ref_name }} name: ${{github.ref_name }} @@ -206,12 +219,15 @@ jobs: community-operators-pr: runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read needs: create-release steps: - name: Checkout code - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Multi-repo workflow authentication - uses: actions/create-github-app-token@v3 + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: generate-token with: app-id: ${{ secrets.WORKFLOW_AUTH_PUBLIC_APP_ID }} @@ -221,7 +237,7 @@ jobs: clickhouse-operator community-operators - name: Set up Go - uses: actions/setup-go@v7 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: 'go.mod' - name: Set VERSION @@ -229,7 +245,7 @@ jobs: - name: Generate operator bundle run: make bundle - name: Checkout community-operators fork - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: ${{ github.repository_owner }}/community-operators token: ${{ steps.generate-token.outputs.token }} diff --git a/Dockerfile b/Dockerfile index d78e6166..be0c9171 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY go.mod go.mod COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer -RUN go mod download +RUN --mount=type=cache,target=/go/pkg/mod go mod download # Copy the go source COPY cmd/main.go cmd/main.go @@ -26,7 +26,8 @@ COPY internal/ internal/ # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a \ +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ -ldflags "-X ${VERSION_PKG}.Version=${VERSION} -X ${VERSION_PKG}.GitCommitHash=${GIT_COMMIT} -X ${VERSION_PKG}.BuildTime=${BUILD_TIME}" -o manager cmd/main.go # Use distroless as minimal base image to package the manager binary diff --git a/Makefile b/Makefile index 6a497624..01142f1f 100644 --- a/Makefile +++ b/Makefile @@ -558,7 +558,7 @@ bundle-buildx: ## Build and push bundle docker image for cross-platform support .PHONY: scorecard scorecard: operator-sdk - $(OPERATOR_SDK) scorecard bundle + $(OPERATOR_SDK) scorecard bundle --wait-time=2m .PHONY: opm OPM = $(LOCALBIN)/opm diff --git a/dev.Dockerfile b/dev.Dockerfile index 44ff2291..f35dede5 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -1,6 +1,4 @@ -FROM ubuntu:latest - -RUN apt-get update && apt-get install -y ca-certificates +FROM alpine:3 ARG USER_ID=65532 USER $USER_ID:$USER_ID diff --git a/internal/controller/clickhouse/commands_test.go b/internal/controller/clickhouse/commands_test.go index a8d4f4e7..37ee67e4 100644 --- a/internal/controller/clickhouse/commands_test.go +++ b/internal/controller/clickhouse/commands_test.go @@ -32,8 +32,8 @@ const ( keeperHostname = "test-keeper" clickhouseHostnameFormat = "test-clickhouse-0-%d-0" testPassword = "test-password" - keeperImage = "clickhouse/clickhouse-keeper:26.5" - clickhouseImage = "clickhouse/clickhouse-server:26.5" + keeperImage = "clickhouse/clickhouse-keeper:26.6.2.81" + clickhouseImage = "clickhouse/clickhouse-server:26.6.2.81" testConfigRevision = "test-revision-v1" ) diff --git a/internal/controller/testutil/suite.go b/internal/controller/testutil/suite.go index e4d3ca0d..8cc716ec 100644 --- a/internal/controller/testutil/suite.go +++ b/internal/controller/testutil/suite.go @@ -214,11 +214,13 @@ func CompleteVersionProbeJob(ctx context.Context, suite TestSuit, namespace, spe ExpectWithOffset(1, suite.Client.Create(ctx, pod)).To(Succeed()) By("Setting version job pod status: " + pod.Name) pod.Status = corev1.PodStatus{ + Phase: corev1.PodSucceeded, ContainerStatuses: []corev1.ContainerStatus{{ Name: v1.VersionProbeContainerName, State: corev1.ContainerState{ Terminated: &corev1.ContainerStateTerminated{ - Message: "ClickHouse server version " + version, + ExitCode: 0, + Message: "ClickHouse server version " + version, }, }, }}, diff --git a/internal/controller/versionprobe.go b/internal/controller/versionprobe.go index 62472c8e..13b5aae3 100644 --- a/internal/controller/versionprobe.go +++ b/internal/controller/versionprobe.go @@ -28,6 +28,8 @@ const ( DefaultProbeMemoryRequest = "256Mi" versionProbeBinary = "/usr/bin/clickhouse" versionProbeQuery = "INSERT INTO FUNCTION file('/dev/termination-log', 'RawBLOB', 'version String') SELECT version()" + versionProbeBackoffLimit = int32(1) + versionProbeDeadline = int64(90) ) // VersionProbeConfig holds parameters for the version probe Job. @@ -244,20 +246,21 @@ func (rm *ResourceManager) buildVersionProbeJob(cfg VersionProbeConfig, revision Annotations: maps.Clone(cfg.Annotations), }, Spec: batchv1.JobSpec{ - BackoffLimit: new(int32(0)), + BackoffLimit: new(versionProbeBackoffLimit), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: maps.Clone(cfg.Labels), Annotations: maps.Clone(cfg.Annotations), }, Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, - ImagePullSecrets: cfg.PodTemplate.ImagePullSecrets, - SecurityContext: DefaultPodSecurityContext(), - NodeSelector: cfg.PodTemplate.NodeSelector, - Tolerations: cfg.PodTemplate.Tolerations, - ServiceAccountName: cfg.PodTemplate.ServiceAccountName, - SchedulerName: cfg.PodTemplate.SchedulerName, + ActiveDeadlineSeconds: new(versionProbeDeadline), + RestartPolicy: corev1.RestartPolicyNever, + ImagePullSecrets: cfg.PodTemplate.ImagePullSecrets, + SecurityContext: DefaultPodSecurityContext(), + NodeSelector: cfg.PodTemplate.NodeSelector, + Tolerations: cfg.PodTemplate.Tolerations, + ServiceAccountName: cfg.PodTemplate.ServiceAccountName, + SchedulerName: cfg.PodTemplate.SchedulerName, Containers: []corev1.Container{ { Name: v1.VersionProbeContainerName, @@ -344,12 +347,16 @@ func readVersionFromJob(ctx context.Context, log controllerutil.Logger, cli clie } if len(podList.Items) > 1 { - log.Warn("more than one pods found for version probe job") + log.Debug("multiple pods found for version probe job") } for _, pod := range podList.Items { + if pod.Status.Phase != corev1.PodSucceeded { + continue + } + for _, cs := range pod.Status.ContainerStatuses { - if cs.Name == v1.VersionProbeContainerName && cs.State.Terminated != nil { + if cs.Name == v1.VersionProbeContainerName && cs.State.Terminated != nil && cs.State.Terminated.ExitCode == 0 { version, err := controllerutil.ParseVersion(cs.State.Terminated.Message) if err != nil { return "", fmt.Errorf("parse version probe from job container output: %w", err) @@ -360,5 +367,5 @@ func readVersionFromJob(ctx context.Context, log controllerutil.Logger, cli clie } } - return "", errors.New("no termination message found in version probe job pods") + return "", errors.New("no successful version probe container found") } diff --git a/internal/controller/versionprobe_test.go b/internal/controller/versionprobe_test.go index 5b1118d4..5bb057ab 100644 --- a/internal/controller/versionprobe_test.go +++ b/internal/controller/versionprobe_test.go @@ -35,7 +35,7 @@ func baseJob() batchv1.Job { }, }, Spec: batchv1.JobSpec{ - BackoffLimit: new(int32(0)), + BackoffLimit: new(int32(1)), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -46,7 +46,8 @@ func baseJob() batchv1.Job { }, }, Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, + ActiveDeadlineSeconds: new(int64(90)), + RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{ FSGroup: new(int64(1000)), }, @@ -381,6 +382,10 @@ var _ = Describe("VersionProbe caching", func() { By("verifying the probe runs the shell-free distroless command") container := jobs.Items[0].Spec.Template.Spec.Containers[0] + Expect(jobs.Items[0].Spec.BackoffLimit).NotTo(BeNil()) + Expect(*jobs.Items[0].Spec.BackoffLimit).To(Equal(versionProbeBackoffLimit)) + Expect(jobs.Items[0].Spec.Template.Spec.ActiveDeadlineSeconds).NotTo(BeNil()) + Expect(*jobs.Items[0].Spec.Template.Spec.ActiveDeadlineSeconds).To(Equal(versionProbeDeadline)) Expect(container.Command).To(Equal([]string{"/usr/bin/clickhouse"})) Expect(container.Args).To(Equal([]string{"local", "--query", "INSERT INTO FUNCTION file('/dev/termination-log', 'RawBLOB', 'version String') SELECT version()"})) }) @@ -408,4 +413,82 @@ var _ = Describe("VersionProbe caching", func() { Expect(rm.ctrl.GetClient().List(ctx, &jobs, client.InNamespace("default"))).To(Succeed()) Expect(jobs.Items).To(HaveLen(1)) }) + + It("should retain a failed probe Job with the same spec", func(ctx context.Context) { + rm, log := setupProbeTest() + cfg := probeCfg("clickhouse/clickhouse-server", "", "") + + revision, err := imageRevision(cfg) + Expect(err).NotTo(HaveOccurred()) + + job, err := rm.buildVersionProbeJob(cfg, revision) + Expect(err).NotTo(HaveOccurred()) + + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobFailed, + Status: corev1.ConditionTrue, + Message: "deadline exceeded", + }} + Expect(rm.ctrl.GetClient().Create(ctx, &job)).To(Succeed()) + + result, err := rm.VersionProbe(ctx, log, cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Err).To(MatchError("deadline exceeded")) + + var jobs batchv1.JobList + Expect(rm.ctrl.GetClient().List(ctx, &jobs, client.InNamespace("default"))).To(Succeed()) + Expect(jobs.Items).To(HaveLen(1)) + }) + + It("should read version from the second Pod", func(ctx context.Context) { + rm, log := setupProbeTest() + cfg := probeCfg("clickhouse/clickhouse-server", "", "") + + result, err := rm.VersionProbe(ctx, log, cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Pending).To(BeTrue()) + + var jobs batchv1.JobList + Expect(rm.ctrl.GetClient().List(ctx, &jobs, client.InNamespace("default"))).To(Succeed()) + Expect(jobs.Items).To(HaveLen(1)) + + job := jobs.Items[0] + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobComplete, + Status: corev1.ConditionTrue, + }} + Expect(rm.ctrl.GetClient().Status().Update(ctx, &job)).To(Succeed()) + + createProbePod := func(name string, phase corev1.PodPhase, exitCode int32, message string) { + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: job.Namespace, + Name: name, + Labels: map[string]string{ + batchv1.ControllerUidLabel: string(job.UID), + batchv1.JobNameLabel: job.Name, + }, + }, + Status: corev1.PodStatus{ + Phase: phase, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: v1.VersionProbeContainerName, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: exitCode, + Message: message, + }}, + }}, + }, + } + + Expect(rm.ctrl.GetClient().Create(ctx, &pod)).To(Succeed()) + } + + createProbePod("failed", corev1.PodFailed, 1, "invalid version") + createProbePod("succeeded", corev1.PodSucceeded, 0, "ClickHouse server version 26.5.5.8") + + result, err = rm.VersionProbe(ctx, log, cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(result.Version).To(Equal("26.5.5.8")) + }) }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 13afffac..82f82009 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -45,12 +45,12 @@ const ( var releases = map[string][]upgrade.ClickHouseVersion{ upgrade.ChannelStable: { - {Major: 26, Minor: 2, Patch: 1, Build: 1}, - {Major: 26, Minor: 1, Patch: 1, Build: 1}, + {Major: 26, Minor: 6, Patch: 2, Build: 81}, + {Major: 26, Minor: 5, Patch: 5, Build: 8}, }, upgrade.ChannelLTS: { - {Major: 26, Minor: 3, Patch: 1, Build: 1}, - {Major: 25, Minor: 8, Patch: 1, Build: 1}, + {Major: 26, Minor: 3, Patch: 17, Build: 56}, + {Major: 25, Minor: 8, Patch: 28, Build: 1}, }, } @@ -215,11 +215,6 @@ var _ = JustAfterEach(func(ctx context.Context) { }) func WaitReplicaCount(ctx context.Context, k8sClient client.Client, namespace, app string, replicas int) { - deadline, ok := ctx.Deadline() - if !ok { - deadline = time.Now().Add(time.Minute) - } - Eventually(func() int { var pods corev1.PodList Expect(k8sClient.List(ctx, &pods, client.InNamespace(namespace), client.MatchingLabels{ @@ -227,7 +222,7 @@ func WaitReplicaCount(ctx context.Context, k8sClient client.Client, namespace, a })).To(Succeed()) return len(pods.Items) - }).WithTimeout(time.Until(deadline)).WithPolling(pollingInterval).Should(Equal(replicas)) + }).WithTimeout(time.Minute).WithPolling(pollingInterval).Should(Equal(replicas)) } func CheckPodReady(pod *corev1.Pod) bool { diff --git a/test/e2e/keeper_e2e_test.go b/test/e2e/keeper_e2e_test.go index 399a030d..dd794d1d 100644 --- a/test/e2e/keeper_e2e_test.go +++ b/test/e2e/keeper_e2e_test.go @@ -58,7 +58,7 @@ var _ = Describe("Keeper controller", Label("keeper"), func() { cr.Spec = specUpdate Expect(k8sClient.Update(ctx, &cr)).To(Succeed()) - WaitKeeperUpdatedAndReady(ctx, &cr, 3*time.Minute, true) + WaitKeeperUpdatedAndReady(ctx, &cr, 5*time.Minute, true) ExpectWithOffset(1, k8sClient.Get(ctx, cr.NamespacedName(), &cr)).To(Succeed()) Expect(cr.Status.Version).To(HavePrefix(cr.Spec.ContainerTemplate.Image.Tag)) KeeperRWChecks(ctx, &cr, &checks) @@ -291,9 +291,6 @@ var _ = Describe("Keeper controller", Label("keeper"), func() { }) func WaitKeeperUpdatedAndReady(ctx context.Context, cr *v1.KeeperCluster, timeout time.Duration, isUpdate bool) { - ctx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - By(fmt.Sprintf("waiting for cluster %s to be ready", cr.Name)) EventuallyWithOffset(1, func(g Gomega) { var cluster v1.KeeperCluster diff --git a/test/openshift/openshift_test.go b/test/openshift/openshift_test.go index 881da8f3..286d6fb5 100644 --- a/test/openshift/openshift_test.go +++ b/test/openshift/openshift_test.go @@ -94,6 +94,12 @@ var _ = BeforeSuite(func(ctx context.Context) { k8sClient, err = client.New(config, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) + By("waiting for project API service") + Expect(testutil.MustRun(ctx, "oc", "wait", + "--for=jsonpath={.status.conditions[?(@.type=='Available')].status}=True", + "apiservice", "v1.project.openshift.io", "--timeout=5m", + )).To(Succeed()) + By("creating project") Eventually(func(g Gomega) { if _, err := testutil.Run(exec.CommandContext(ctx, "oc", "get", "project", namespace)); err == nil { @@ -102,7 +108,7 @@ var _ = BeforeSuite(func(ctx context.Context) { out, err := testutil.Run(exec.CommandContext(ctx, "oc", "adm", "new-project", namespace)) g.Expect(err).NotTo(HaveOccurred(), string(out)) - }, "2m", "5s").Should(Succeed()) + }, "5m", "5s").Should(Succeed()) DeferCleanup(func(ctx context.Context) { _, _ = testutil.Run(exec.CommandContext(ctx, "oc", "delete", "project", namespace, @@ -115,7 +121,7 @@ var _ = BeforeSuite(func(ctx context.Context) { g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: namespace}, &ns)).To(Succeed()) _, ok := ns.Annotations["openshift.io/sa.scc.uid-range"] g.Expect(ok).To(BeTrue(), "waiting for SCC uid-range annotation") - }, "2m", "5s").Should(Succeed()) + }, "5m", "5s").Should(Succeed()) manifests := fmt.Sprintf(catalogManifests, namespace, diff --git a/test/testutil/utils.go b/test/testutil/utils.go index 4e3e2e7f..2dee4e54 100644 --- a/test/testutil/utils.go +++ b/test/testutil/utils.go @@ -41,8 +41,8 @@ const ( certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" logTailLines = 10 - BaseVersion = "26.3" - UpdateVersion = "26.5" + BaseVersion = "26.3.17.56" + UpdateVersion = "26.6.2.81" ) var ( @@ -334,16 +334,19 @@ func DumpNamespacePodLogs(ctx context.Context, config *rest.Config, namespace st return } + header := fmt.Sprintf("Container logs %s/%s/%s:\n", namespace, name, container) + full.WriteString(header) + short.WriteString(header) + if len(logs) == 0 { + full.WriteString("\n\n") + short.WriteString("\n\n") return } - header := fmt.Sprintf("Container logs %s/%s/%s:\n", namespace, name, container) - full.WriteString(header) full.WriteString(logs) full.WriteString("\n\n") - short.WriteString(header) short.WriteString(tailLines(logs, logTailLines)) short.WriteString("\n\n") }