Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion GEMINI.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ Configurations are declared using custom resources like `RootSync` and `RepoSync
* Make
* Docker
* gcloud CLI
* gsutil

**Common Commands (from Makefile):**

Expand Down
1 change: 0 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,6 @@ DOCKER_RUN_ARGS = \
-v $(GO_DIR)/std/windows_amd64_static:/usr/local/go/pkg/windows_amd64_static \
-v $(TEMP_OUTPUT_DIR):/tmp \
-v $${HOME}/.config:/.config \
-v $${HOME}/.gsutil:/.gsutil \
-w /go/src/$(REPO) \
--rm \
$(BUILDENV_IMAGE) \
Expand Down
24 changes: 12 additions & 12 deletions Makefile.e2e.ci
Original file line number Diff line number Diff line change
Expand Up @@ -49,21 +49,21 @@ GCS_BINARIES := $(GCS_BUCKET)/binaries

.PHONY: publish-gcs
publish-gcs:
gsutil cp $(OSS_MANIFEST_STAGING_DIR)/* $(GCS_MANIFESTS)/oss/
gsutil cp $(NOMOS_MANIFEST_STAGING_DIR)/* $(GCS_MANIFESTS)/operator/
gsutil cp $(BIN_DIR)/darwin_amd64/nomos $(GCS_BINARIES)/darwin_amd64/nomos
gsutil cp $(BIN_DIR)/darwin_arm64/nomos $(GCS_BINARIES)/darwin_arm64/nomos
gsutil cp $(BIN_DIR)/linux_amd64/nomos $(GCS_BINARIES)/linux_amd64/nomos
gsutil cp $(BIN_DIR)/linux_arm64/nomos $(GCS_BINARIES)/linux_arm64/nomos
gcloud storage cp $(OSS_MANIFEST_STAGING_DIR)/* $(GCS_MANIFESTS)/oss/
gcloud storage cp $(NOMOS_MANIFEST_STAGING_DIR)/* $(GCS_MANIFESTS)/operator/
gcloud storage cp $(BIN_DIR)/darwin_amd64/nomos $(GCS_BINARIES)/darwin_amd64/nomos
gcloud storage cp $(BIN_DIR)/darwin_arm64/nomos $(GCS_BINARIES)/darwin_arm64/nomos
gcloud storage cp $(BIN_DIR)/linux_amd64/nomos $(GCS_BINARIES)/linux_amd64/nomos
gcloud storage cp $(BIN_DIR)/linux_arm64/nomos $(GCS_BINARIES)/linux_arm64/nomos

.PHONY: pull-gcs
pull-gcs: clean $(OUTPUT_DIR)
gsutil cp $(GCS_MANIFESTS)/oss/*.yaml $(OSS_MANIFEST_STAGING_DIR)/
gsutil cp $(GCS_MANIFESTS)/operator/*.yaml $(NOMOS_MANIFEST_STAGING_DIR)/
gsutil cp $(GCS_BINARIES)/darwin_amd64/nomos $(BIN_DIR)/darwin_amd64/nomos
gsutil cp $(GCS_BINARIES)/darwin_arm64/nomos $(BIN_DIR)/darwin_arm64/nomos
gsutil cp $(GCS_BINARIES)/linux_amd64/nomos $(BIN_DIR)/linux_amd64/nomos
gsutil cp $(GCS_BINARIES)/linux_arm64/nomos $(BIN_DIR)/linux_arm64/nomos
gcloud storage cp '$(GCS_MANIFESTS)/oss/*.yaml' $(OSS_MANIFEST_STAGING_DIR)/
gcloud storage cp '$(GCS_MANIFESTS)/operator/*.yaml' $(NOMOS_MANIFEST_STAGING_DIR)/
gcloud storage cp $(GCS_BINARIES)/darwin_amd64/nomos $(BIN_DIR)/darwin_amd64/nomos
gcloud storage cp $(GCS_BINARIES)/darwin_arm64/nomos $(BIN_DIR)/darwin_arm64/nomos
gcloud storage cp $(GCS_BINARIES)/linux_amd64/nomos $(BIN_DIR)/linux_amd64/nomos
gcloud storage cp $(GCS_BINARIES)/linux_arm64/nomos $(BIN_DIR)/linux_arm64/nomos
$(MAKE) copy-cli

.PHONY: pull-gcs-postsubmit
Expand Down
4 changes: 2 additions & 2 deletions build/buildenv/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ ENV GOCACHE=/go/cache

COPY --from=tools-base /go/bin/* /bin/

# Install gcloud & gsutil
# Requires /.config/ & /.gsutil/ volumes mounted for authentication
# Install gcloud & gcloud storage components
# Requires /.config/ volume mounted for authentication
COPY --from=gcloud-install /usr/lib/google-cloud-sdk /usr/lib/google-cloud-sdk
ENV PATH "$PATH:/usr/lib/google-cloud-sdk/bin"

Expand Down
4 changes: 1 addition & 3 deletions docs/development.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,10 @@ You must have the following tools:
* [make]
* [docker]
* [gcloud]
* [gsutil]

### Login to gcloud

In order to download some build artifacts and dependencies from Google Cloud Storage (GCS) with `gsutil`, you may need to be authenticated with `gcloud`.
In order to download some build artifacts and dependencies from Google Cloud Storage (GCS) with `gcloud storage`, you may need to be authenticated with `gcloud`.

One way to do this is with `gcloud auth login`. For other options, see https://cloud.google.com/docs/authentication/gcloud

Expand Down Expand Up @@ -157,5 +156,4 @@ make run-oss
[make]: https://www.gnu.org/software/make/
[docker]: https://www.docker.com/get-started
[gcloud]: https://cloud.google.com/sdk/docs/install
[gsutil]: https://cloud.google.com/storage/docs/gsutil_install
[create your own fork]: https://docs.github.com/en/get-started/quickstart/fork-a-repo
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ fi

kubectl delete deployment -n config-management-system config-management-operator --ignore-not-found --cascade=foreground

if kubectl get configmanagement config-management &> /dev/null ; then
if kubectl get configmanagement config-management &>/dev/null; then
kubectl patch configmanagement config-management --type="merge" -p '{"metadata":{"finalizers":[]}}'
kubectl delete configmanagement config-management --cascade=orphan --ignore-not-found
fi
Expand Down
6 changes: 3 additions & 3 deletions examples/post-sync/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ export SINK_NAME="sync-status-errors"
export BUCKET_NAME="sync-status-logs-${PROJECT_ID}"

# Create the storage bucket
gsutil mb -l ${REGION} gs://${BUCKET_NAME}
gcloud storage buckets create gs://${BUCKET_NAME} --location=${REGION}

# Create the log sink
gcloud logging sinks create ${SINK_NAME} storage.googleapis.com/${BUCKET_NAME} \
Expand All @@ -183,7 +183,7 @@ gcloud logging sinks create ${SINK_NAME} storage.googleapis.com/${BUCKET_NAME} \
export SINK_SA=$(gcloud logging sinks describe ${SINK_NAME} --format='value(writerIdentity)')

# Grant permissions to write to the bucket
gsutil iam ch ${SINK_SA}:roles/storage.objectCreator gs://${BUCKET_NAME}
gcloud storage buckets add-iam-policy-binding gs://${BUCKET_NAME} --member=${SINK_SA} --role=roles/storage.objectCreator
```

## Setting Up Alerting with Pub/Sub and Cloud Functions
Expand Down Expand Up @@ -502,7 +502,7 @@ gcloud iam service-accounts delete ${LOG_GSA_NAME}@${PROJECT_ID}.iam.gserviceacc
gcloud iam service-accounts delete ${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com

# Delete the storage bucket (if created)
gsutil rm -r gs://${BUCKET_NAME}
gcloud storage rm --recursive gs://${BUCKET_NAME}

# Delete GAR repository (optional)
gcloud artifacts repositories delete ${GAR_REPO_NAME} --location=${REGION}
Expand Down
6 changes: 3 additions & 3 deletions scripts/build-status.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,20 +29,20 @@ function pretty_print {

function local_image_exists {
image="$1"
docker image inspect "${image}" &> /dev/null
docker image inspect "${image}" &>/dev/null
}

function remote_image_exists {
image="$1"
flags=()
# must pass --insecure flag for local registry (e.g. localhost:5000)
[[ "${image}" == "localhost"* ]] && flags+=("--insecure")
docker manifest inspect "${flags[@]}" "${image}" &> /dev/null
docker manifest inspect "${flags[@]}" "${image}" &>/dev/null
}

pretty_print "Current commit" "$(git describe --tags --always --dirty --long)"

read -r -a images <<< "$(config_sync_images)"
read -r -a images <<<"$(config_sync_images)"
[[ ${#images[@]} -eq 0 ]] && exit 1
declare -A status_map
cs_tag=""
Expand Down
5 changes: 2 additions & 3 deletions scripts/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.


#
# golang e2e test launcher.
# This wraps the e2e test execution and creates a junit report.
Expand All @@ -27,15 +26,15 @@ start_time=$(date +%s)
go test ./e2e/testcases/... --p 1 --e2e --test.v "$@" | tee test_results.txt
exit_code=$?
end_time=$(date +%s)
echo "Tests took $(( end_time - start_time )) seconds"
echo "Tests took $((end_time - start_time)) seconds"

# Save test results to ARTIFACTS directory. The ARTIFACTS env var is set by prow.
# The containerized entry points mount the ARTIFACTS directory to a path inside
# the container, and pass the mounted path as ARTIFACTS. Using the env var directly
# enables running this script more flexibly, e.g. without docker in docker.
if [[ -n "${ARTIFACTS}" && -d "${ARTIFACTS}" ]]; then
echo "Creating junit xml report"
go-junit-report --subtest-mode=exclude-parents < test_results.txt > "${ARTIFACTS}/junit_report.xml"
go-junit-report --subtest-mode=exclude-parents <test_results.txt >"${ARTIFACTS}/junit_report.xml"
if [ "$exit_code" -eq 0 ]; then
echo "Running junit-report post processor"
# build our in-repo junit report post-processor binary
Expand Down
14 changes: 7 additions & 7 deletions scripts/generate-clientset.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,12 @@ source "${CODEGEN_PKG}/kube_codegen.sh"
GOMOD_NAME="$(grep "^module" "${SCRIPT_ROOT}/go.mod" | cut -d' ' -f2)"

kube::codegen::gen_helpers \
--boilerplate "${SCRIPT_ROOT}/hack/boilerplate.txt" \
"${SCRIPT_ROOT}/pkg/api"
--boilerplate "${SCRIPT_ROOT}/hack/boilerplate.txt" \
"${SCRIPT_ROOT}/pkg/api"

kube::codegen::gen_client \
--with-watch \
--output-dir "${SCRIPT_ROOT}/pkg/generated" \
--output-pkg "${GOMOD_NAME}/pkg/generated" \
--boilerplate "${SCRIPT_ROOT}/hack/boilerplate.txt" \
"${SCRIPT_ROOT}/pkg/api"
--with-watch \
--output-dir "${SCRIPT_ROOT}/pkg/generated" \
--output-pkg "${GOMOD_NAME}/pkg/generated" \
--boilerplate "${SCRIPT_ROOT}/hack/boilerplate.txt" \
"${SCRIPT_ROOT}/pkg/api"
26 changes: 13 additions & 13 deletions scripts/generate-test-loggers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ GO_MODULE="$(grep "^module" "go.mod" | cut -d' ' -f2)"
source_paths=("pkg" "cmd")

function render_main_test() {
cat << EOF
cat <<EOF
package ${PACKAGE_NAME}

import (
Expand All @@ -51,16 +51,16 @@ EOF
# find_test_dirs loops through all the directories under the specified path
# and prints the ones directly containing go tests.
function find_test_dirs() {
local parent_path="$1"
local test_dir_path
declare -A test_dir_paths
while IFS= read -r file_path; do
test_dir_path="$(dirname "${file_path}")"
test_dir_paths[$test_dir_path]="1"
done <<< "$(find "${parent_path}" -type f -name "*_test.go")"
for test_dir_path in "${!test_dir_paths[@]}"; do
echo "${test_dir_path}"
done
local parent_path="$1"
local test_dir_path
declare -A test_dir_paths
while IFS= read -r file_path; do
test_dir_path="$(dirname "${file_path}")"
test_dir_paths[$test_dir_path]="1"
done <<<"$(find "${parent_path}" -type f -name "*_test.go")"
for test_dir_path in "${!test_dir_paths[@]}"; do
echo "${test_dir_path}"
done
}

for source_path in "${source_paths[@]}"; do
Expand All @@ -69,7 +69,7 @@ for source_path in "${source_paths[@]}"; do
echo "Generating ${file_name}"
PACKAGE_NAME="$(basename "${test_dir_path}")"
PACKAGE_PATH="${test_dir_path}"
render_main_test > "${file_name}"
render_main_test >"${file_name}"
"addlicense" -c "Google LLC" -f LICENSE_TEMPLATE "${file_name}"
done <<< "$(find_test_dirs "${source_path}")"
done <<<"$(find_test_dirs "${source_path}")"
done
22 changes: 11 additions & 11 deletions scripts/install-helm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,17 @@ OUTPUT_DIR=${OUTPUT_DIR:-${REPO_ROOT}/.output}
STAGING_DIR=${STAGING_DIR:-${OUTPUT_DIR}/third_party/helm}

function helm_version_installed() {
local version
version=$("${INSTALL_DIR}/helm" version --short)
echo "${version%+*}" # remove commit suffix
local version
version=$("${INSTALL_DIR}/helm" version --short)
echo "${version%+*}" # remove commit suffix
}

# Check installed version
if [[ -f "${INSTALL_DIR}/helm" ]] && [[ -x "${INSTALL_DIR}/helm" ]]; then
if [[ "$(helm_version_installed)" == "${HELM_VERSION}" ]]; then
echo "helm version: ${HELM_VERSION} (already installed)"
exit 0
fi
if [[ "$(helm_version_installed)" == "${HELM_VERSION}" ]]; then
echo "helm version: ${HELM_VERSION} (already installed)"
exit 0
fi
fi

HELM_TARBALL_URL=gs://config-management-release/config-sync/helm/tag/${HELM_VERSION}/helm-${HELM_VERSION}-linux-amd64.tar.gz
Expand All @@ -49,14 +49,14 @@ HELM_TARBALL=${TMPDIR}/helm-${HELM_VERSION}-linux-amd64.tar.gz
HELM_CHECKSUM=${HELM_TARBALL}.sha256

function cleanup() {
rm -f "${HELM_TARBALL}"
rm -f "${HELM_CHECKSUM}"
rm -f "${HELM_TARBALL}"
rm -f "${HELM_CHECKSUM}"
}
trap cleanup EXIT

echo "Downloading helm ${HELM_VERSION}"
gsutil cp "${HELM_TARBALL_URL}" "${HELM_TARBALL}"
gsutil cp "${HELM_CHECKSUM_URL}" "${HELM_CHECKSUM}"
gcloud storage cp "${HELM_TARBALL_URL}" "${HELM_TARBALL}"
gcloud storage cp "${HELM_CHECKSUM_URL}" "${HELM_CHECKSUM}"

echo "Verifying helm checksum"
echo "$(cat "${HELM_CHECKSUM}") ${HELM_TARBALL}" | sha256sum -c
Expand Down
18 changes: 9 additions & 9 deletions scripts/install-kustomize.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ OUTPUT_DIR=${OUTPUT_DIR:-${REPO_ROOT}/.output}
STAGING_DIR=${STAGING_DIR:-${OUTPUT_DIR}/third_party/kustomize}

function helm_version_installed() {
"${INSTALL_DIR}/kustomize" version
"${INSTALL_DIR}/kustomize" version
}

# Check installed version
if [[ -f "${INSTALL_DIR}/kustomize" ]] && [[ -x "${INSTALL_DIR}/kustomize" ]]; then
if [[ "$(helm_version_installed)" == "${KUSTOMIZE_VERSION}" ]]; then
echo "kustomize version: ${KUSTOMIZE_VERSION} (already installed)"
exit 0
fi
if [[ "$(helm_version_installed)" == "${KUSTOMIZE_VERSION}" ]]; then
echo "kustomize version: ${KUSTOMIZE_VERSION} (already installed)"
exit 0
fi
fi

KUSTOMIZE_TARBALL_URL=gs://config-management-release/config-sync/kustomize/tag/${KUSTOMIZE_VERSION}/kustomize-${KUSTOMIZE_VERSION}-linux-amd64.tar.gz
Expand All @@ -47,14 +47,14 @@ KUSTOMIZE_TARBALL=${TMPDIR}/kustomize-${KUSTOMIZE_VERSION}-linux-amd64.tar.gz
KUSTOMIZE_CHECKSUM=${KUSTOMIZE_TARBALL}.sha256

function cleanup() {
rm -f "${KUSTOMIZE_TARBALL}"
rm -f "${KUSTOMIZE_CHECKSUM}"
rm -f "${KUSTOMIZE_TARBALL}"
rm -f "${KUSTOMIZE_CHECKSUM}"
}
trap cleanup EXIT

echo "Downloading kustomize ${KUSTOMIZE_VERSION}"
gsutil cp "${KUSTOMIZE_TARBALL_URL}" "${KUSTOMIZE_TARBALL}"
gsutil cp "${KUSTOMIZE_CHECKSUM_URL}" "${KUSTOMIZE_CHECKSUM}"
gcloud storage cp "${KUSTOMIZE_TARBALL_URL}" "${KUSTOMIZE_TARBALL}"
gcloud storage cp "${KUSTOMIZE_CHECKSUM_URL}" "${KUSTOMIZE_CHECKSUM}"

echo "Verifying kustomize checksum"
echo "$(cat "${KUSTOMIZE_CHECKSUM}") ${KUSTOMIZE_TARBALL}" | sha256sum -c
Expand Down
4 changes: 0 additions & 4 deletions scripts/lib/manifests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,3 @@ config_sync_images() {
echo "${images[@]}"
return 0
}




4 changes: 2 additions & 2 deletions scripts/package-test-helm-chart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ CHART_SRC_DIR="${REPO_ROOT}/e2e/testdata/helm-charts/${CHART_NAME}"
TMP_DIR=$(mktemp -d)

function cleanup() {
rm -rf -- "$TMP_DIR"
rm -rf -- "$TMP_DIR"
}

# Ensures the temporary directory is cleaned up on exit
Expand All @@ -42,4 +42,4 @@ helm repo index .

echo "Uploading charts and index to ${HELM_CHARTS_BUCKET}"
gcloud storage cp -n ./*.tgz "${HELM_CHARTS_BUCKET}/"
gcloud storage cp ./index.yaml "${HELM_CHARTS_BUCKET}/"
gcloud storage cp ./index.yaml "${HELM_CHARTS_BUCKET}/"
8 changes: 4 additions & 4 deletions scripts/pull-postsubmit-retry.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@ num_intervals=80
interval=15
SECONDS=0
until [[ "$n" -ge $num_intervals ]]; do
make pull-gcs-postsubmit && exit 0
echo "++++ Failed to pull postsubmit artifacts. Waiting ${interval} seconds to retry."
n=$((n+1))
sleep "${interval}"
make pull-gcs-postsubmit && exit 0
echo "++++ Failed to pull postsubmit artifacts. Waiting ${interval} seconds to retry."
n=$((n + 1))
sleep "${interval}"
done

echo "++++ Postsubmit artifacts not found after retrying for ${SECONDS} seconds"
Expand Down
2 changes: 1 addition & 1 deletion scripts/tag-release-candidate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ echo "+++ Incremented RC. NEXT_RC: $NEXT_RC"
if [[ "${branch}" == "main" ]]; then
git log --oneline --graph -30 "${remote_sha}"
else
git fetch "${REMOTE}" main > /dev/null
git fetch "${REMOTE}" main >/dev/null
main_sha=$(git rev-parse FETCH_HEAD)
git log --oneline --graph -30 "${main_sha}" "${remote_sha}"
fi
Expand Down
4 changes: 2 additions & 2 deletions scripts/test-kustomization.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ out=$(kustomize build --load-restrictor=LoadRestrictionsNone "${REPO_ROOT}/test/
expected_file="${REPO_ROOT}/test/kustomization/expected.yaml"

if [[ "${UPDATE_EXPECTED_OUTPUT:-}" == "true" ]]; then
echo "${out}" > "${expected_file}"
echo "${out}" >"${expected_file}"
exit 0
fi

diff "${expected_file}" <( echo "${out}" )
diff "${expected_file}" <(echo "${out}")
Loading