Skip to content

CNTRLPLANE-3330: ci: improve unit test speed with sharding and parallelism #8330

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 2 commits into from
Apr 27, 2026
Merged

CNTRLPLANE-3330: ci: improve unit test speed with sharding and parallelism #8330

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd6bfdf
ci: shard unit tests and add Go build cache
bryan-cox Apr 24, 2026
a4b3600
test: add t.Parallel() to heaviest unit test files
bryan-cox Apr 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 55 additions & 11 deletions .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ permissions:
contents: read

jobs:
test:
name: Unit Tests
changes:
name: Detect Changes
runs-on: arc-runner-set
timeout-minutes: 30
timeout-minutes: 5
outputs:
run_tests: ${{ steps.changes.outputs.run_tests }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
Expand All @@ -27,23 +29,65 @@ jobs:
- name: Check for non-contrib changes
id: changes
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
echo "run_tests=true" >> "$GITHUB_OUTPUT"
exit 0
elif [ "${{ github.event_name }}" = "pull_request" ]; then
FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD)
else
FILES=$(git diff --name-only HEAD~1)
if [ "${{ github.event.before }}" = "0000000000000000000000000000000000000000" ]; then
echo "run_tests=true" >> "$GITHUB_OUTPUT"
exit 0
fi
FILES=$(git diff --name-only ${{ github.event.before }}..${{ github.sha }})
fi
if echo "$FILES" | grep -qvE '^(contrib|\.github)/'; then
if echo "$FILES" | grep -qvE '^(contrib|\.github|docs)/'; then
Comment thread
coderabbitai[bot] marked this conversation as resolved.
echo "run_tests=true" >> "$GITHUB_OUTPUT"
else
echo "run_tests=false" >> "$GITHUB_OUTPUT"
fi
- if: steps.changes.outputs.run_tests == 'true'
run: make test
- if: steps.changes.outputs.run_tests == 'true'
name: Upload to Codecov

test:
name: Unit Tests (${{ matrix.shard }})
needs: changes
if: needs.changes.outputs.run_tests == 'true'
runs-on: arc-runner-set
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- shard: cpo-hostedcontrolplane
packages: ./control-plane-operator/controllers/hostedcontrolplane/...
- shard: cpo-other
packages: ./control-plane-operator/controllers/awsprivatelink/... ./control-plane-operator/controllers/azureprivatelinkservice/... ./control-plane-operator/controllers/gcpprivateserviceconnect/... ./control-plane-operator/controllers/healthcheck/... ./control-plane-operator/controllers/openshiftmanager/... ./control-plane-operator/endpoint-resolver/... ./control-plane-operator/hostedclusterconfigoperator/... ./control-plane-operator/metrics-proxy/...
- shard: hypershift-operator
packages: ./hypershift-operator/...
- shard: cmd-support
packages: ./cmd/... ./support/...
- shard: other
packages: ./karpenter-operator/... ./control-plane-pki-operator/... ./contrib/... ./ignition-server/... ./pkg/... ./dnsresolver/... ./product-cli/... ./client/... ./test/integration/... ./test/e2e/util/... ./test/util/... ./availability-prober/... ./konnectivity-socks5-proxy/... ./konnectivity-https-proxy/... ./kubernetes-default-proxy/... ./kubevirtexternalinfra/... ./etcd-defrag/... ./etcd-backup/... ./etcd-recovery/... ./etcd-upload/... ./kas-bootstrap/... ./sharedingress-config-generator/... ./sync-fg-configmap/... ./sync-global-pullsecret/... ./token-minter/...
env:
GOCACHE: /tmp/go-build-cache
GOMODCACHE: /tmp/go-mod-cache
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
persist-credentials: false
- name: Restore Go build cache
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: /tmp/go-build-cache
key: go-build-${{ matrix.shard }}-${{ github.sha }}
restore-keys: go-build-${{ matrix.shard }}-
- name: Run tests
run: make test-shard TEST_PACKAGES="${{ matrix.packages }}" COVER_PROFILE="cover-${{ matrix.shard }}.out"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
- name: Upload to Codecov
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
env:
HOME: /tmp
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
with:
files: cover.out
files: cover-${{ matrix.shard }}.out
flags: ${{ matrix.shard }}
Comment thread
bryan-cox marked this conversation as resolved.
9 changes: 9 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,15 @@ test: generate
@echo "Running tests with $(NUM_CORES) parallel jobs..."
$(GO) test -race -parallel=$(NUM_CORES) -count=1 -timeout=30m ./... -coverprofile cover.out

# Run a subset of unit tests (used by CI sharding).
# Usage: make test-shard TEST_PACKAGES="./cmd/... ./support/..." COVER_PROFILE="cover-shard.out"
TEST_PACKAGES ?= ./...
COVER_PROFILE ?= cover.out
.PHONY: test-shard
test-shard: generate
@echo "Running shard tests for packages: $(TEST_PACKAGES)"
$(GO) test -race -parallel=$(NUM_CORES) -count=1 -timeout=30m $(TEST_PACKAGES) -coverprofile $(COVER_PROFILE)

# OCP envtest index for downstream kubebuilder assets
ENVTEST_OCP_INDEX := https://raw.githubusercontent.com/openshift/api/master/envtest-releases.yaml
# OCP version to Kubernetes version mapping (OCP 4.x -> K8s 1.(x+13))
Expand Down
8 changes: 8 additions & 0 deletions control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ import (
)

func TestReconcileKubeadminPassword(t *testing.T) {
t.Parallel()
targetNamespace := "test"

testsCases := []struct {
Expand Down Expand Up @@ -353,6 +354,7 @@ func TestReconcileIgnitionServer(t *testing.T) {
}

func TestEtcdRestoredCondition(t *testing.T) {
t.Parallel()
testsCases := []struct {
name string
sts *appsv1.StatefulSet
Expand Down Expand Up @@ -792,6 +794,7 @@ func (c *createTrackingWorkqueue) Add(item reconcile.Request) {
}

func TestNonReadyInfraTriggersRequeueAfter(t *testing.T) {
t.Parallel()
mockCtrl := gomock.NewController(t)
mockedProviderWithOpenshiftImageRegistryOverrides := releaseinfo.NewMockProviderWithOpenShiftImageRegistryOverrides(mockCtrl)
mockedProviderWithOpenshiftImageRegistryOverrides.EXPECT().
Expand Down Expand Up @@ -1313,6 +1316,7 @@ func TestControlPlaneComponents(t *testing.T) {
}

func TestAWSSecurityGroupTags(t *testing.T) {
t.Parallel()
tests := []struct {
name string
hcp *hyperv1.HostedControlPlane
Expand Down Expand Up @@ -1540,6 +1544,7 @@ func componentsFakeDependencies(componentName string, namespace string) []client
}

func TestControlPlaneComponentsAvailable(t *testing.T) {
t.Parallel()
testNamespace := "test-namespace"

testCases := []struct {
Expand Down Expand Up @@ -1774,6 +1779,7 @@ func TestControlPlaneComponentsAvailable(t *testing.T) {
}

func TestRemoveHCPIngressFromRoutes(t *testing.T) {
t.Parallel()
const namespace = "test-ns"

tests := []struct {
Expand Down Expand Up @@ -2202,6 +2208,7 @@ func TestRemoveHCPIngressFromRoutes(t *testing.T) {
}

func TestReconcileAvailabilityStatus(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
conditions []metav1.Condition
Expand Down Expand Up @@ -2472,6 +2479,7 @@ func TestReconcileAvailabilityStatus(t *testing.T) {
}

func TestEtcdStatefulSetCondition(t *testing.T) {
t.Parallel()
testNamespace := "test-namespace"

testCases := []struct {
Expand Down
7 changes: 7 additions & 0 deletions control-plane-operator/controllers/hostedcontrolplane/infra/infra_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -715,6 +715,7 @@ func simulateInfraProvisioning(ctx context.Context, c client.Client, hcp *hyperv
}

func TestReconcileInfrastructure_ErrorCases(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
hcp *hyperv1.HostedControlPlane
Expand Down Expand Up @@ -811,6 +812,7 @@ func TestReconcileInfrastructure_ErrorCases(t *testing.T) {
}

func TestReconcileInfrastructure_WhenTransitioningFromPublicToPrivate_ItShouldCleanUpPublicResources(t *testing.T) {
t.Parallel()
g := NewGomegaWithT(t)

// Start with Public configuration
Expand Down Expand Up @@ -865,6 +867,7 @@ func TestReconcileInfrastructure_WhenTransitioningFromPublicToPrivate_ItShouldCl
}

func TestReconcileInfrastructure_WhenTransitioningFromPrivateToPublic_ItShouldCleanUpPrivateResources(t *testing.T) {
t.Parallel()
g := NewGomegaWithT(t)

// Start with Private configuration
Expand Down Expand Up @@ -921,6 +924,7 @@ func TestReconcileInfrastructure_WhenTransitioningFromPrivateToPublic_ItShouldCl
// Tests moved from hostedcontrolplane_controller_test.go

func TestReconcileOAuthService(t *testing.T) {
t.Parallel()
targetNamespace := "test"
apiPort := int32(config.KASSVCPort)
hostname := "test.example.com"
Expand Down Expand Up @@ -1149,6 +1153,7 @@ func TestReconcileOAuthService(t *testing.T) {
}

func TestReconcileAPIServerService(t *testing.T) {
t.Parallel()
targetNamespace := "test"
apiPort := int32(config.KASSVCPort)
kasPort := "client"
Expand Down Expand Up @@ -1676,6 +1681,7 @@ func (c *fakeMessageCollector) ErrorMessages(resource client.Object) ([]string,
var _ events.MessageCollector = &fakeMessageCollector{}

func TestReconcileRouterServiceStatus(t *testing.T) {
t.Parallel()
const namespace = "test-ns"
const svcName = "test"
tests := []struct {
Expand Down Expand Up @@ -1826,6 +1832,7 @@ func TestReconcileInternalRouterServiceStatus(t *testing.T) {
}

func TestReconcileOAuthService_AzureLoadBalancer(t *testing.T) {
t.Parallel()
targetNamespace := "test"
apiPort := int32(config.KASSVCPort)
ipFamilyPolicy := corev1.IPFamilyPolicyPreferDualStack
Expand Down
18 changes: 18 additions & 0 deletions control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ func TestReconcileErrorHandling(t *testing.T) {
}

func TestReconcileOLM(t *testing.T) {
t.Parallel()
var errs []error
hcp := fakeHCP()
hcp.Namespace = "openshift-operator-lifecycle-manager"
Expand Down Expand Up @@ -475,6 +476,7 @@ func withICS(hcp *hyperv1.HostedControlPlane) *hyperv1.HostedControlPlane {
}

func TestReconcileKubeadminPasswordHashSecret(t *testing.T) {
t.Parallel()
testNamespace := "master-cluster1"
testHCPName := "cluster1"

Expand Down Expand Up @@ -548,6 +550,7 @@ func TestReconcileKubeadminPasswordHashSecret(t *testing.T) {
}

func TestReconcileUserCertCABundle(t *testing.T) {
t.Parallel()
testNamespace := "master-cluster1"
testHCPName := "cluster1"
tests := map[string]struct {
Expand Down Expand Up @@ -637,6 +640,7 @@ func TestReconcileUserCertCABundle(t *testing.T) {
var _ manifestReconciler = manifestAndReconcile[*rbacv1.ClusterRole]{}

func TestDestroyCloudResources(t *testing.T) {
t.Parallel()
originalConditionTime := time.Now().Add(-1 * time.Hour)
fakeHostedControlPlane := func() *hyperv1.HostedControlPlane {
return &hyperv1.HostedControlPlane{
Expand Down Expand Up @@ -950,6 +954,7 @@ func TestDestroyCloudResources(t *testing.T) {
}

func TestDestroyCloudResourcesWithKASUnavailable(t *testing.T) {
t.Parallel()
fakeHCP := &hyperv1.HostedControlPlane{
ObjectMeta: metav1.ObjectMeta{
Name: "test-hcp",
Expand Down Expand Up @@ -1038,6 +1043,7 @@ func (e *mockNetError) Timeout() bool { return e.timeout }
func (e *mockNetError) Temporary() bool { return e.temporary }

func TestConnectionErrorTracking(t *testing.T) {
t.Parallel()

tests := []struct {
name string
Expand Down Expand Up @@ -1107,6 +1113,7 @@ func TestConnectionErrorTracking(t *testing.T) {
}

func TestListAccessor(t *testing.T) {
t.Parallel()
pod := func(name string) corev1.Pod {
return corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -1130,6 +1137,7 @@ func TestListAccessor(t *testing.T) {
}

func TestReconcileClusterVersion(t *testing.T) {
t.Parallel()
hcp := &hyperv1.HostedControlPlane{
Spec: hyperv1.HostedControlPlaneSpec{
ClusterID: "test-cluster-id",
Expand Down Expand Up @@ -1205,6 +1213,7 @@ func TestReconcileClusterVersion(t *testing.T) {
}

func TestReconcileClusterVersionWithDisabledCapabilities(t *testing.T) {
t.Parallel()
hcp := &hyperv1.HostedControlPlane{
Spec: hyperv1.HostedControlPlaneSpec{
ClusterID: "test-cluster-id",
Expand Down Expand Up @@ -1282,6 +1291,7 @@ func TestReconcileClusterVersionWithDisabledCapabilities(t *testing.T) {
}

func TestReconcileClusterVersionWithEnabledCapabilities(t *testing.T) {
t.Parallel()
hcp := &hyperv1.HostedControlPlane{
Spec: hyperv1.HostedControlPlaneSpec{
ClusterID: "test-cluster-id",
Expand Down Expand Up @@ -1360,6 +1370,7 @@ func TestReconcileClusterVersionWithEnabledCapabilities(t *testing.T) {
}

func TestReconcileImageContentPolicyType(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
hcp *hyperv1.HostedControlPlane
Expand Down Expand Up @@ -1433,6 +1444,7 @@ func compareICSAndIDMS(g *WithT, ics []hyperv1.ImageContentSource, idms *configv
}

func TestReconcileKASEndpoints(t *testing.T) {
t.Parallel()

testCases := []struct {
name string
Expand Down Expand Up @@ -1490,6 +1502,7 @@ func TestReconcileKASEndpoints(t *testing.T) {
}

func TestReconcileKubeletConfig(t *testing.T) {
t.Parallel()
hcpNamespace := "hostedcontrolplane-namespace"
hcNamespace := "openshift-config-managed"
npName1 := "nodepool-test1"
Expand Down Expand Up @@ -1576,6 +1589,7 @@ func TestReconcileKubeletConfig(t *testing.T) {
}

func TestBuildAWSWebIdentityCredentials(t *testing.T) {
t.Parallel()
type args struct {
roleArn string
region string
Expand Down Expand Up @@ -2323,6 +2337,7 @@ func newCondition(conditionType string, status metav1.ConditionStatus, reason, m
}

func Test_reconciler_reconcileDataPlaneConnectionAvailable(t *testing.T) {
t.Parallel()
newKonnectivityAgentPod := func(name string, phase corev1.PodPhase) corev1.Pod {
return corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -2518,6 +2533,7 @@ func Test_reconciler_reconcileDataPlaneConnectionAvailable(t *testing.T) {
}

func Test_reconciler_reconcileControlPlaneConnectionAvailable(t *testing.T) {
t.Parallel()
newConnectivityConfigMap := func(data map[string]string) *corev1.ConfigMap {
return &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -2683,6 +2699,7 @@ func Test_reconciler_reconcileControlPlaneConnectionAvailable(t *testing.T) {
}

func Test_reconciler_reconcileKASConnectionCheckerDeployment(t *testing.T) {
t.Parallel()
const testCLIImage = "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:cli-test"

tests := []struct {
Expand Down Expand Up @@ -2969,6 +2986,7 @@ func Test_reconciler_reconcileKASConnectionCheckerDeployment(t *testing.T) {
}

func TestReconcileMetricsForwarder(t *testing.T) {
t.Parallel()
tests := []struct {
name string
annotations map[string]string
Expand Down
Loading