diff --git a/.tmuxinator/tmuxinator-k8.yaml b/.tmuxinator/tmuxinator-k8.yaml index 2a878d0..566b166 100644 --- a/.tmuxinator/tmuxinator-k8.yaml +++ b/.tmuxinator/tmuxinator-k8.yaml @@ -1,5 +1,5 @@ name: k8_dev -on_project_first_start: ./dev k8-init +# on_project_first_start: ./dev k8-init on_project_exit: minikube delete windows: - editor: diff --git a/config/config.go b/config/config.go index dd6d404..0dc48c6 100644 --- a/config/config.go +++ b/config/config.go @@ -8,11 +8,12 @@ import ( ) type ManagerConfig struct { - WokersCount int `mapstructure:"WORKERS_COUNT"` - ReqChannelSize int `mapstructure:"REQ_CHANNEL_SIZE"` - ReplicaCount int `mapstructure:"REPLICA_COUNT"` - WriteQuorum int `mapstructure:"WRITE_QUORUM"` - ReadQuorum int `mapstructure:"READ_QUORUM"` + WokersCount int `mapstructure:"WORKERS_COUNT"` + ThreadRequests bool `mapstructure:"THREAD_REQUESTS"` + ReqChannelSize int `mapstructure:"REQ_CHANNEL_SIZE"` + ReplicaCount int `mapstructure:"REPLICA_COUNT"` + WriteQuorum int `mapstructure:"WRITE_QUORUM"` + ReadQuorum int `mapstructure:"READ_QUORUM"` SaveInterval int DefaultTimeout int `mapstructure:"DEFAULT_TIMEOUT"` PartitionCount int `mapstructure:"PARTITION_COUNT"` diff --git a/config/config_test.go b/config/config_test.go index dd01c71..3520165 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -62,6 +62,7 @@ func TestConfigOverwrite(t *testing.T) { assert.EqualValues(t, 99999, config.Manager.PartitionBuckets, "PartitionBuckets wrong value") assert.EqualValues(t, "/tmp/store/data", config.Manager.DataPath, "DataPath wrong value") assert.EqualValues(t, 991, config.Manager.PartitionConcurrency, "PartitionConcurrency wrong value") + assert.EqualValues(t, true, config.Manager.ThreadRequests, "ThreadRequests wrong value") // consensus config assert.EqualValues(t, 9, config.Consensus.EpochTime, "EpochTime wrong value") diff --git a/config/default-config.yaml b/config/default-config.yaml index 3caa524..d8c9c8c 100644 --- a/config/default-config.yaml +++ b/config/default-config.yaml @@ -4,7 +4,8 @@ manager: write_quorum: 2 default_timeout: 10 req_channel_size: 20 - workers_count: 100 + workers_count: 2 + thread_requests: true partition_count: 15 partition_concurrency: 10 partition_buckets: 1000 diff --git a/config/test-config.yaml b/config/test-config.yaml index 8c05ef0..1d123b7 100644 --- a/config/test-config.yaml +++ b/config/test-config.yaml @@ -6,6 +6,7 @@ manager: partition_buckets: 99999 partition_concurrency: 991 data_path: "/tmp/store/data" + thread_requests: true consensus: epoch_time: 9 data_path: "/tmp/store/raft" diff --git a/deploy_k8.sh b/deploy_k8.sh index e3b4a9d..92e5000 100755 --- a/deploy_k8.sh +++ b/deploy_k8.sh @@ -3,15 +3,15 @@ # Exit on any error set -e +# eval $(minikube docker-env) + STATEFULSET_NAME="store" NAMESPACE="default" -eval $(minikube docker-env) TAG=$(date +%s) -NEW_IMAGE=docker.io/andrew-delph/main:$TAG +NEW_IMAGE=ghcr.io/andrew-delph/main:$TAG -bazel run --execution_log_json_file=events.json //main:store_image -docker tag docker.io/andrew-delph/main:store_image $NEW_IMAGE +bazel run --execution_log_json_file=events.json //main:image_push -- -dst=$NEW_IMAGE ROLL_OUT_FLAG=0 @@ -35,7 +35,6 @@ SECONDS=0 # Reset the SECONDS variable if [ $ROLL_OUT_FLAG -eq 0 ]; then echo "The 'rollout' flag is not set. Setting up." (cd operator && kustomize build config/crd | kubectl apply -f - || true) - bazel run //main:image_push kubectl apply -f ./operator/config/samples/ || true else echo "The 'rollout' flag is set." diff --git a/deploy_operator.sh b/deploy_operator.sh index 24efd3d..1743c10 100755 --- a/deploy_operator.sh +++ b/deploy_operator.sh @@ -1,12 +1,11 @@ #!/bin/bash +# eval $(minikube docker-env) -eval $(minikube docker-env) TAG=$(date +%s) -IMG=docker.io/andrew-delph/operator:$TAG -bazel run //operator:operator_image -docker tag docker.io/andrew-delph/operator:operator_image $IMG +IMG=ghcr.io/andrew-delph/operator:$TAG +bazel run //operator:image_push -- -dst=$IMG (cd operator && make deploy IMG=$IMG) (cd operator && kustomize build config/rbac | kubectl apply -f - || true) diff --git a/dev b/dev index 38da7b4..ebb9f02 100755 --- a/dev +++ b/dev @@ -23,10 +23,8 @@ case $1 in ;; k8-r) kubectl delete -f ./operator/config/samples/ || true - ./deploy_k8.sh ./deploy_operator.sh - kubectl apply -f ./resources/prom.yaml - kubectl create configmap prometheus-config --from-file=prometheus.yml || true + ./deploy_k8.sh ;; k8-operator) ./deploy_operator.sh @@ -38,7 +36,7 @@ case $1 in tmux kill-session -t k8_dev ;; k8-e2e) - k6 run -e ADDRESS="192.168.49.2:30000" e2e/test.js + ./e2e_k8.sh ;; dc-e2e) k6 run e2e/test.js diff --git a/docker-compose.yaml b/docker-compose.yaml index 463e37d..8551bf9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -38,7 +38,7 @@ services: store: # hostname: store - image: docker.io/andrew-delph/main:store_image + image: ghcr.io/andrew-delph/main:store_image scale: 5 depends_on: # - zookeeper @@ -57,7 +57,7 @@ services: # store-profile: # # hostname: store # # scale: 1 - # image: docker.io/andrew-delph/main:store_image + # image: ghcr.io/andrew-delph/main:store_image # environment: # - PROFILE_SERVER=true # - BOOTSTRAP_TIMEOUT=1 diff --git a/docs.md b/docs.md new file mode 100644 index 0000000..57c6997 --- /dev/null +++ b/docs.md @@ -0,0 +1,37 @@ +- summary +- layout + - hasring + - partitions + - epoch buckets + - indexs + lsm +- gossip + - client discovery + - non operator member management +- consensus + - epochs + - members + temp members +- hashring + - current members + - temp members +- replication + - on epoch + - on cluster change +- read/write quorum +- external/internal endpoints +- protobufs +- indexes +- storage + - indexdb + - badgerdb +- operator + - code rollout + - scale cluster +- development environment + - docker-compose + - minikube +- benchmarks +- future improvements + - scan query + - SAGA/2PC + - read repairs +- tradeoffs diff --git a/e2e/Makefile b/e2e/Makefile index d1718fe..1678001 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -6,12 +6,13 @@ build: init: @echo "init" curl https://raw.githubusercontent.com/grafana/k6-operator/main/bundle.yaml | kubectl apply -f - + make monitoring setup: - kubectl --context minikube create namespace k6 || echo "namespace already exists" - kubectl --context minikube delete configmap test-scripts --ignore-not-found -n k6 - kubectl --context minikube delete configmap prometheus-config --ignore-not-found -n k6 - kubectl --context minikube create configmap test-scripts \ + kubectl create namespace k6 || echo "namespace already exists" + kubectl delete configmap test-scripts --ignore-not-found -n k6 + kubectl delete configmap prometheus-config --ignore-not-found -n k6 + kubectl create configmap test-scripts \ --from-file=./test.js -n k6 run: @@ -21,14 +22,19 @@ run: logs: stern -n k6 .* -prod: - make setup - ./run-kube.sh resources/k6-output-grafana-cloud.yaml --external - watch: - watch -n 15 kubectl get pods -n k6 + watch kubectl get pods -n k6 -delete: - kubectl --context minikube delete -f resources/k6-output-grafana-cloud.yaml -n k6 - # cd dependencies/k6-operator && make delete +stop: + kubectl delete -f resources/k6-output-grafana-cloud.yaml -n k6 || true +delete: + kubectl delete -f resources/k6-output-grafana-cloud.yaml -n k6 || true + helm uninstall loki --namespace=monitoring --wait || true + +monitoring: + helm repo add grafana https://grafana.github.io/helm-charts + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml --wait + kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - + helm upgrade --install loki --namespace=monitoring grafana/loki-stack -f loki-stack.values.yaml + bash ./create_dashboards.sh diff --git a/e2e/create_dashboards.sh b/e2e/create_dashboards.sh new file mode 100755 index 0000000..3466cd5 --- /dev/null +++ b/e2e/create_dashboards.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# default dashboards: https://github.com/prometheus-community/helm-charts/tree/8f91606e01b85b9aa013ed472c5acb836c4e7ac8/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14 + +# delete existing dashboards +kubectl delete configmap --selector custom_dashboard='1' -n monitoring + +# Set the directory path +dir_path="./dashboards" + +# Loop through all files in the directory +for file_path in $dir_path/*; do + file_name=$(basename "${file_path%.*}") + + # # Create the ConfigMap using the file contents and name as the ConfigMap name + kubectl create configmap "$file_name" --from-file="$file_path" -n monitoring + + # # Add labels to the ConfigMap + kubectl label configmap "$file_name" grafana_dashboard='1' custom_dashboard='1' -n monitoring + +done \ No newline at end of file diff --git a/e2e/dashboards/dashboard-tests.json b/e2e/dashboards/dashboard-tests.json new file mode 100644 index 0000000..96f2ff1 --- /dev/null +++ b/e2e/dashboards/dashboard-tests.json @@ -0,0 +1,1984 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Visualize k6 OSS results stored in Prometheus", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Grafana k6 OSS Docs: Prometheus Remote Write", + "tooltip": "Open docs in a new tab", + "type": "link", + "url": "https://k6.io/docs/results-output/real-time/prometheus-remote-write/" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http_req_s_errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "reqps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_req_s" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "vus" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed" + } + }, + { + "id": "unit", + "value": "VUs" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_vus{testid=~\"$testid\"})", + "instant": false, + "legendFormat": "vus", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_duration_$quantile_stat", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_req_s", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(round(k6_http_req_failed_rate{testid=~\"$testid\"}, 0.1)*100)", + "hide": true, + "instant": false, + "legendFormat": "http_req_failed", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"false\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_req_s_errors", + "range": true, + "refId": "D" + } + ], + "title": "Performance Overview", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 1, + "panels": [], + "title": "Performance Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(k6_http_reqs_total{testid=~\"$testid\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "HTTP requests", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"false\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "HTTP request failures", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peak RPS", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Select a different Stat to change the query", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Request Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(irate(k6_data_sent_total{testid=~\"$testid\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "data_sent", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(irate(k6_data_received_total{testid=~\"$testid\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "data_received", + "range": true, + "refId": "B" + } + ], + "title": "Transfer Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "dropped_iterations" + }, + "properties": [ + { + "id": "unit", + "value": "none" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_iteration_duration_$quantile_stat{testid=~\"$testid\"})", + "instant": false, + "legendFormat": "iteration_duration_$quantile_stat", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(k6_dropped_iterations_total{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "dropped_iterations", + "range": true, + "refId": "B" + } + ], + "title": "Iterations", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 16, + "panels": [], + "title": "HTTP", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Select a different Stat to change the query\n\nHTTP-specific built-in metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_blocked_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_blocked_$quantile_stat", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_tls_handshaking_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_tls_handshaking_$quantile_stat", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_sending_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_sending_$quantile_stat", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_waiting_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_waiting_$quantile_stat", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_receiving_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_receiving_$quantile_stat", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_duration_$quantile_stat", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Latency Timings", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Select a different Stat to change the query", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "errors_http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "success_http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "http_req_duration_[a-zA-Z0-9_]+" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\"})", + "hide": false, + "instant": false, + "legendFormat": "http_req_duration_$quantile_stat", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\", expected_response=\"true\"})", + "instant": false, + "legendFormat": "success_http_req_duration_$quantile_stat", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(k6_http_req_duration_$quantile_stat{testid=~\"$testid\", expected_response=\"false\"})", + "hide": false, + "instant": false, + "legendFormat": "errors_http_req_duration_$quantile_stat", + "range": true, + "refId": "B" + } + ], + "title": "HTTP Latency Stats", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "http_req_s_errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_req_s" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "http_req_s_success" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "http_req_s", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"false\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_req_s_errors", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"true\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_req_s_success", + "range": true, + "refId": "C" + } + ], + "title": "HTTP Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "min/max/p95/p99 depends on the available Quantile Stats", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "method" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "min" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 2, + "showHeader": true + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(name, method, status) (k6_http_req_duration_min{testid=~\"$testid\"})", + "format": "table", + "hide": false, + "instant": false, + "legendFormat": "min", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(name, method, status) (k6_http_req_duration_max{testid=~\"$testid\"})", + "format": "table", + "hide": false, + "instant": false, + "legendFormat": "max", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(name, method, status) (k6_http_req_duration_p95{testid=~\"$testid\"})", + "format": "table", + "hide": false, + "instant": false, + "legendFormat": "p95", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(name, method, status) (k6_http_req_duration_p99{testid=~\"$testid\"})", + "format": "table", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "E" + } + ], + "title": "Requests by URL", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "Value #B": { + "aggregations": ["min"], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": ["max"], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": ["mean"], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": ["mean"], + "operation": "aggregate" + }, + "method": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "status": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": { + "Time": 0, + "Value #B": 4, + "Value #C": 5, + "Value #D": 6, + "Value #E": 7, + "method": 2, + "name": 1, + "status": 3 + }, + "renameByName": { + "Value #B": "min", + "Value #B (min)": "min", + "Value #C": "max", + "Value #C (max)": "max", + "Value #D": "p95", + "Value #D (mean)": "p95", + "Value #E": "p99", + "Value #E (mean)": "p99" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 11, + "panels": [], + "title": "Checks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Success Rate" + }, + "properties": [ + { + "id": "custom.hidden", + "value": false + }, + { + "id": "unit", + "value": "%" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value (mean)" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "check" + }, + "properties": [ + { + "id": "filterable", + "value": false + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 2, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value (count)" + } + ] + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "round(k6_checks_rate{testid=~\"$testid\"}, 0.1)", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Checks list", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "keepLabels": ["__name__", "check"], + "mode": "columns" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": ["mean"], + "operation": "aggregate" + }, + "check": { + "aggregations": [], + "operation": "groupby" + }, + "k6_checks_rate": { + "aggregations": ["sum", "count"], + "operation": "aggregate" + } + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Success Rate", + "binary": { + "left": "Value (mean)", + "operator": "*", + "reducer": "sum", + "right": "100" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [], + "fields": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Filter by check name to query a particular check", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 100, + "axisSoftMin": 0, + "barAlignment": -1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "%" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg(round(k6_checks_rate{testid=~\"$testid\"}, 0.1)*100)", + "instant": false, + "legendFormat": "k6_checks_rate", + "range": true, + "refId": "A" + } + ], + "title": "Checks Success Rate (aggregate individual checks)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 23, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "### Visualize other k6 results \n\nAt the top of the dashboard, click `Add` and select `Visualization` from the dropdown menu. Choose the visualization type and input the PromQL queries for the `k6_` metric(s).\n\nAlternatively, click on the `Explore` icon on the menu bar and input the queries for the `k6_` metric(s). From `Explore`, you can add new Panels to this dashboard. \n\nNote that all k6 metrics are prefixed with the `k6_` namespace when sent to Prometheus.", + "mode": "markdown" + }, + "pluginVersion": "10.1.2", + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": ["prometheus", "k6"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus", + "value": "P1809F7CD0C75ACF3" + }, + "description": "Choose a Prometheus Data Source", + "hide": 0, + "includeAll": false, + "label": "Prometheus DS", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(testid)", + "description": "Filter by \"testid\" tag. Define it by tagging: k6 run --tag testid=xyz", + "hide": 0, + "includeAll": true, + "label": "Test ID", + "multi": true, + "name": "testid", + "options": [], + "query": { + "query": "label_values(testid)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "metrics(k6_http_req_duration_)", + "description": "Statistic for Trend Metrics Queries. The available options depend on the values of the K6_PROMETHEUS_RW_TREND_STATS setting.", + "hide": 0, + "includeAll": false, + "label": "Trend Metrics Query", + "multi": false, + "name": "quantile_stat", + "options": [], + "query": { + "query": "metrics(k6_http_req_duration_)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "/http_req_duration_(min|max|count|sum|avg|med|p[0-9]+)/g", + "skipUrlSync": false, + "sort": 2, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Adhoc filters are applied to all panels. To enable it, go to Dashboard Settings / Variables / adhoc_filter and select the target Prometheus data source.", + "filters": [], + "hide": 0, + "label": "AdhocFilter", + "name": "adhoc_filter", + "skipUrlSync": false, + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "k6 Prometheus", + "uid": "ccbb2351-2ae2-462f-ae0e-f2c893ad1028", + "version": 1, + "weekStart": "" +} diff --git a/e2e/loki-stack.values.yaml b/e2e/loki-stack.values.yaml new file mode 100644 index 0000000..16fbd4f --- /dev/null +++ b/e2e/loki-stack.values.yaml @@ -0,0 +1,190 @@ +test_pod: + enabled: true + image: bats/bats:v1.1.0 + pullPolicy: IfNotPresent + +grafana: + enabled: true + adminUser: admin + adminPassword: password + image: + tag: "" + nodeSelector: + vke.vultr.com/node-pool: main + service: + annotations: {} + appProtocol: "" + enabled: true + labels: {} + port: 80 + portName: service + targetPort: 3000 + nodePort: 30007 + type: NodePort + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + labelValue: "1" + annotations: {} + multicluster: + global: + enabled: false + etcd: + enabled: false + provider: + foldersFromFilesStructure: true + updateIntervalSeconds: 30 + allowUiUpdates: false + datasources: + enabled: true + defaultDatasourceEnabled: true + isDefaultDatasource: true + uid: prometheus + annotations: {} + createPrometheusReplicasDatasources: false + label: grafana_datasource + labelValue: "1" + + exemplarTraceIdDestinations: {} + persistence: + enabled: false + existingClaim: storage-loki-0 + +prometheus: + enabled: true + server: + image: + tag: v2.42.0 + # priorityClassName: high-priority + service: + type: NodePort + nodePort: 30008 + + # global: + # scrape_interval: 3m + # extraArgs: + # log.level: debug + extraFlags: + - web.enable-lifecycle + - enable-feature=remote-write-receiver + # - storage.tsdb.max-block-duration=20m + # - storage.tsdb.min-block-duration=20m + # - storage.tsdb.retention.size=300MB + # - web.enable-remote-write-receiver + # - log.level=debug + # global: + # query_log_file: /prometheus/query.log + persistentVolume: + enabled: true + # existingClaim: "prometheus-pvc" + kubeStateMetrics: + enabled: false + alertmanager: + image: + tag: v0.25.0 + enabled: false + service: + type: NodePort + nodePort: 30009 + # priorityClassName: high-priority + persistentVolume: + enabled: false + configFromSecret: "loki-prometheus-alertmanager" + # alertmanagerFiles: + # alertmanager.yml: + # global: {} + # route: + # receiver: 'gmail-notifications' + # receivers: + # - name: 'gmail-notifications' + # email_configs: + # - to: example@gmail.ca + # from: example@gmail.com + # smarthost: smtp.gmail.com:587 + # auth_username: example@gmail.com + # auth_identity: example@gmail.com + # auth_password: example + # send_resolved: true + + # extraScrapeConfigs: | + # - job_name: rabbitmq + # scrape_interval: 3m + # scrape_timeout: 1m + # basic_auth: + # username: guest + # password: guest + # static_configs: + # - targets: + # - rabbitmq.default.svc.cluster.local:15692 + +loki: + enabled: true + isDefault: false + url: http://{{(include "loki.serviceName" .)}}:{{ .Values.loki.service.port }} + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + livenessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + datasource: + jsonData: "{}" + uid: "" + persistence: + enabled: true + size: 40Gi + +promtail: + enabled: true + # priorityClassName: high-priority + config: + serverPort: 3101 + clients: + - url: http://{{ .Release.Name }}:3100/loki/api/v1/push + snippets: + # addScrapeJobLabel: true + # extraLimitsConfig: | + # readline_rate: 1000 + # readline_burst: 2000 + # readline_rate_enabled: true + extraRelabelConfigs: + # keep all kubernetes labels + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [serving_knative_dev_service] + regex: "(.+)" + target_label: "app" + pipelineStages: + - drop: + source: "namespace" + expression: "(kube-public|kube-system)" + # - drop: + # older_than: 1h + # drop_counter_reason: 'line_too_old' + - cri: + - docker: + - json: + expressions: + level: level + - static_labels: + level: unknown + # - labels: + # level: stream + - labels: + level: + - output: + source: msg + +# proxy is currently only used by loki test pod +# Note: If http_proxy/https_proxy are set, then no_proxy should include the +# loki service name, so that tests are able to communicate with the loki +# service. +proxy: + http_proxy: "" + https_proxy: "" + no_proxy: "" diff --git a/e2e/resources/k6-output-grafana-cloud.yaml b/e2e/resources/k6-output-grafana-cloud.yaml index c669cdc..5ed6859 100644 --- a/e2e/resources/k6-output-grafana-cloud.yaml +++ b/e2e/resources/k6-output-grafana-cloud.yaml @@ -9,7 +9,7 @@ kind: K6 metadata: name: k6-output-grafana-cloud spec: - parallelism: 1 + parallelism: 3 script: configMap: # Remember...we created the 'test-scripts' ConfigMap with the available scripts inside. @@ -29,9 +29,9 @@ spec: - name: K6_PROMETHEUS_RW_TREND_STATS value: "avg,p(90),p(99),min,max" - name: ADDRESS - value: "store-external.default.svc.cluster.local:8080" + value: "store.default.svc.cluster.local:8080" - name: K6_PROMETHEUS_RW_SERVER_URL - value: "http://prometheus-service.default:9090/api/v1/write" + value: "http://loki-prometheus-server.monitoring:80/api/v1/write" # - name: K6_LOG_FORMAT # value: json # envFrom: diff --git a/e2e/test.js b/e2e/test.js index 1a35dcf..b7e1c7c 100644 --- a/e2e/test.js +++ b/e2e/test.js @@ -34,8 +34,8 @@ export let options = { // executor: "ramping-arrival-rate", // startRate: 50, // stages: [ - // { target: 75, duration: "2m" }, - // { target: 100, duration: "1h" }, + // { target: 30, duration: "2m" }, + // { target: 300, duration: "1h" }, // ], // // Start `rate` iterations per second // timeUnit: "1s", @@ -166,7 +166,7 @@ export function remove() { } // options = { duration: "2h", vus: 1 }; -// options = { iterations: 1, vus: 1 }; +options = { iterations: 10000, vus: 500 }; export default function () { const iterationNumber = exec.scenario.iterationInTest; @@ -186,6 +186,7 @@ export default function () { check(setRes, { "set status is 200": (r) => r.status === 200, }); + return; if (setRes.status != 200) { console.error("SET REQUEST FAILED:", setRes.status, setRes.body); diff --git a/e2e_k8.sh b/e2e_k8.sh new file mode 100755 index 0000000..47f1ae6 --- /dev/null +++ b/e2e_k8.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Define the service and namespace +SERVICE_NAME=store +NAMESPACE=default +LOCAL_PORT=8080 +REMOTE_PORT=8080 + +# Start kubectl port-forward in the background +kubectl port-forward -n $NAMESPACE svc/$SERVICE_NAME $LOCAL_PORT:$REMOTE_PORT > /dev/null & +# Get the PID of kubectl port-forward +PORT_FORWARD_PID=$! + +# Function to kill the port-forward process when the script exits +function cleanup { + echo "Cleaning up..." + kill -9 $PORT_FORWARD_PID +} +trap cleanup EXIT + +# Wait for the connection to be open +echo "Waiting for port-forward to be ready..." +sleep 5 + +# Run your k6 script +k6 run e2e/test.js \ No newline at end of file diff --git a/main/BUILD.bazel b/main/BUILD.bazel index 26ae873..3acba78 100644 --- a/main/BUILD.bazel +++ b/main/BUILD.bazel @@ -92,7 +92,7 @@ container_image( }, files = ["//config:default-config"], ports = ["80"], - repository = "docker.io/andrew-delph", + repository = "ghcr.io/andrew-delph", tags = ["latest"], # repository = "gcr.io/$PROJECT_ID" # repository = "us-central1-docker.pkg.dev/builder-project/repo1/greeter_server", diff --git a/main/manager.go b/main/manager.go index 0bb44f7..ed29686 100644 --- a/main/manager.go +++ b/main/manager.go @@ -188,7 +188,6 @@ func (m *Manager) startWorker(workerId int) { logrus.Debugf("starting worker %d", workerId) for { - workerLoop: select { case <-m.debugTick.C: // m.consensusCluster.Details() @@ -254,324 +253,341 @@ func (m *Manager) startWorker(workerId int) { switch task := data.(type) { case StopWorkerTask: + logrus.Warn("StopWorkerTask!") defer task.wg.Done() return + default: + if m.config.Manager.ThreadRequests { + go m.handleTask(data) + } else { + m.handleTask(data) + } + } - case rpc.PartitionsHealthCheckTask: - // logrus.Warn("PartitionsHealthCheckTask") - err := m.consistencyController.IsHealthy() - task.ResCh <- err + } + } +} - case rpc.UpdateMembersTask: - var err error +func (m *Manager) handleTask(data interface{}) { + switch task := data.(type) { - if m.consensusCluster.Isleader() == false { - task.ResCh <- true - continue - } - if m.ring.CompareMembers(task.Members, task.TempMembers) == true { - // logrus.Warn("members already changed") - task.ResCh <- true - continue - } + case StopWorkerTask: + defer task.wg.Done() + return - err = m.consensusCluster.UpdateFsm(m.GetCurrentEpoch(), task.Members, task.TempMembers) - if err != nil { - logrus.Error("UpdateFsm err = %v", err) - task.ResCh <- err - continue - } + case rpc.PartitionsHealthCheckTask: + // logrus.Warn("PartitionsHealthCheckTask") + err := m.consistencyController.IsHealthy() + task.ResCh <- err - task.ResCh <- true + case rpc.UpdateMembersTask: + var err error - case rpc.UpdateEpochTask: - var err error + if m.consensusCluster.Isleader() == false { + task.ResCh <- true + return + } + if m.ring.CompareMembers(task.Members, task.TempMembers) == true { + // logrus.Warn("members already changed") + task.ResCh <- true + return + } - if m.consensusCluster.Isleader() == false { - task.ResCh <- true - continue - } + err = m.consensusCluster.UpdateFsm(m.GetCurrentEpoch(), task.Members, task.TempMembers) + if err != nil { + logrus.Error("UpdateFsm err = %v", err) + task.ResCh <- err + return + } - if m.LastEpochUpdateId == task.UpdateId { - task.ResCh <- true - continue - } + task.ResCh <- true - logrus.Warnf("UpdateEpochTask %s", task.UpdateId) + case rpc.UpdateEpochTask: + var err error - err = m.consensusCluster.UpdateFsm(m.GetCurrentEpoch()+1, m.ring.GetMembersNames(false), m.ring.GetMembersNames(true)) - if err != nil { - logrus.Warnf("UpdateEpochTask UpdateMembers err = %v", err) - task.ResCh <- err - } + if m.consensusCluster.Isleader() == false { + task.ResCh <- true + return + } - err = m.consensusCluster.UpdateFsm(m.GetCurrentEpoch()+1, m.ring.GetMembersNames(false), m.ring.GetMembersNames(true)) - if err != nil { - logrus.Warnf("UpdateEpochTask UpdateMembers err = %v", err) - task.ResCh <- err - } + if m.LastEpochUpdateId == task.UpdateId { + task.ResCh <- true + return + } - m.LastEpochUpdateId = task.UpdateId + logrus.Warnf("UpdateEpochTask %s", task.UpdateId) - task.ResCh <- true + err = m.consensusCluster.UpdateFsm(m.GetCurrentEpoch()+1, m.ring.GetMembersNames(false), m.ring.GetMembersNames(true)) + if err != nil { + logrus.Warnf("UpdateEpochTask UpdateMembers err = %v", err) + task.ResCh <- err + } - case http.HealthTask: - err := m.consensusCluster.IsHealthy() - if err != nil { - logrus.Warnf("HealthTask err = %v", err) - task.ResCh <- err - continue - } + err = m.consensusCluster.UpdateFsm(m.GetCurrentEpoch()+1, m.ring.GetMembersNames(false), m.ring.GetMembersNames(true)) + if err != nil { + logrus.Warnf("UpdateEpochTask UpdateMembers err = %v", err) + task.ResCh <- err + } - task.ResCh <- true - case http.ReadyTask: - err := m.consensusCluster.IsHealthy() - if err != nil { - logrus.Debugf("IsHealthy err = %v", err) - task.ResCh <- err - continue - } + m.LastEpochUpdateId = task.UpdateId - // err = m.ring.IsHealthy() - // if err != nil { - // logrus.Warnf("ring.IsHealthy err = %v", err) - // task.ResCh <- err - // continue - // } - - // err = m.consistencyController.IsBusy() - // if err != nil { - // logrus.Warnf("HealthTask err = %v", err) - // task.ResCh <- err - // continue - // } - - task.ResCh <- true - - case http.SetTask: - logrus.Debugf("worker SetTask: %+v", task) - members, err := m.SetRequest(task.Key, task.Value) - errorStr := "" - if err != nil { - errorStr = err.Error() - } - task.ResCh <- http.SetResponse{Error: errorStr, Members: members} - - case http.GetTask: - logrus.Debugf("worker GetTask: %+v", task) - value, failed_members, err := m.GetRequest(task.Key) - var valueStr string - if value != nil { - valueStr = value.Value - } - errorStr := "" - if err != nil { - errorStr = err.Error() - } - task.ResCh <- http.GetResponse{Value: valueStr, Error: errorStr, Failed_members: failed_members} + task.ResCh <- true - case gossip.JoinTask: - // logrus.Warnf("worker JoinTask: %+v", task) - - err := m.consensusCluster.AddVoter(task.Name, task.IP) - if err != nil { - err = errors.Wrap(err, "gossip.JoinTask") - logrus.Error(err) - } else { - // logrus.Infof("AddVoter success") - } + case http.HealthTask: + err := m.consensusCluster.IsHealthy() + if err != nil { + logrus.Warnf("HealthTask err = %v", err) + task.ResCh <- err + return + } - _, rpcClient, err := m.rpcWrapper.CreateRpcClient(task.IP) - if err != nil { - err = errors.Wrap(err, "gossip.JoinTask") - logrus.Fatal(err) - continue - } - m.clientManager.AddClient(task.Name, rpcClient) + task.ResCh <- true + case http.ReadyTask: + err := m.consensusCluster.IsHealthy() + if err != nil { + logrus.Debugf("IsHealthy err = %v", err) + task.ResCh <- err + return + } - if m.config.Manager.Operator == false { - err = m.consensusCluster.UpdateFsm(m.GetCurrentEpoch(), m.gossipCluster.GetMembersNames(), m.gossipCluster.GetMembersNames()) - if err != nil { - logrus.Warnf("JoinTask UpdateMembers err = %v", err) - } - } + // err = m.ring.IsHealthy() + // if err != nil { + // logrus.Warnf("ring.IsHealthy err = %v", err) + // task.ResCh <- err + // continue + // } + + // err = m.consistencyController.IsBusy() + // if err != nil { + // logrus.Warnf("HealthTask err = %v", err) + // task.ResCh <- err + // continue + // } + + task.ResCh <- true + + case http.SetTask: + logrus.Debugf("worker SetTask: %+v", task) + members, err := m.SetRequest(task.Key, task.Value) + errorStr := "" + if err != nil { + errorStr = err.Error() + } + task.ResCh <- http.SetResponse{Error: errorStr, Members: members} + + case http.GetTask: + logrus.Debugf("worker GetTask: %+v", task) + value, failed_members, err := m.GetRequest(task.Key) + var valueStr string + if value != nil { + valueStr = value.Value + } + errorStr := "" + if err != nil { + errorStr = err.Error() + } + task.ResCh <- http.GetResponse{Value: valueStr, Error: errorStr, Failed_members: failed_members} - case gossip.LeaveTask: - // logrus.Warnf("worker LeaveTask: %+v", task) - m.clientManager.RemoveClient(task.Name) - m.consensusCluster.RemoveServer(task.Name) - if m.config.Manager.Operator == false { - err := m.consensusCluster.UpdateFsm(m.GetCurrentEpoch(), m.gossipCluster.GetMembersNames(), m.gossipCluster.GetMembersNames()) - if err != nil { - logrus.Warnf("JoinTask UpdateMembers err = %v", err) - } - } + case gossip.JoinTask: + // logrus.Warnf("worker JoinTask: %+v", task) - case consensus.FsmTask: - logrus.Warnf("FsmTask Epoch %v Members %v TempMembers %v", task.Epoch, len(task.Members), len(task.TempMembers)) - m.SetCurrentEpoch(task.Epoch) - m.consistencyController.PublishEpoch(task.Epoch) + err := m.consensusCluster.AddVoter(task.Name, task.IP) + if err != nil { + err = errors.Wrap(err, "gossip.JoinTask") + logrus.Error(err) + } else { + // logrus.Infof("AddVoter success") + } - m.ring.SetRingMembers(task.Members, task.TempMembers) - task.ResCh <- true + _, rpcClient, err := m.rpcWrapper.CreateRpcClient(task.IP) + if err != nil { + err = errors.Wrap(err, "gossip.JoinTask") + logrus.Fatal(err) + return + } + m.clientManager.AddClient(task.Name, rpcClient) - case rpc.SetValueTask: - logrus.Debugf("worker SetValueTask: %+v", task) + if m.config.Manager.Operator == false { + err = m.consensusCluster.UpdateFsm(m.GetCurrentEpoch(), m.gossipCluster.GetMembersNames(), m.gossipCluster.GetMembersNames()) + if err != nil { + logrus.Warnf("JoinTask UpdateMembers err = %v", err) + } + } - if task.Value.Epoch < m.GetCurrentEpoch()-1 { - task.ResCh <- errors.New("cannot set lagging epoch") - continue - } - err := m.SetValue(task.Value) - if err != nil { - logrus.Warnf("SetValue err = %v", err) - task.ResCh <- err - } else { - task.ResCh <- true - } + case gossip.LeaveTask: + // logrus.Warnf("worker LeaveTask: %+v", task) + m.clientManager.RemoveClient(task.Name) + m.consensusCluster.RemoveServer(task.Name) + if m.config.Manager.Operator == false { + err := m.consensusCluster.UpdateFsm(m.GetCurrentEpoch(), m.gossipCluster.GetMembersNames(), m.gossipCluster.GetMembersNames()) + if err != nil { + logrus.Warnf("JoinTask UpdateMembers err = %v", err) + } + } - case rpc.GetValueTask: - logrus.Debugf("worker GetValueTask: %+v", task) - // value, err := m.db.Get([]byte(task.Key)) - value, err := m.GetValue(task.Key) - if err == storage.KEY_NOT_FOUND { // TODO if the nodes partition is not up to date it should not count as response - task.ResCh <- nil - } else if err != nil { - task.ResCh <- err - } else { - task.ResCh <- value - } + case consensus.FsmTask: + logrus.Warnf("FsmTask Epoch %v Members %v TempMembers %v", task.Epoch, len(task.Members), len(task.TempMembers)) + m.SetCurrentEpoch(task.Epoch) + m.consistencyController.PublishEpoch(task.Epoch) - case rpc.StreamBucketsTask: // TODO test this is returning right values - logrus.Debugf("worker StreamBucketsTask: %+v", task) - var buckets []int32 = task.Buckets - if len(buckets) == 0 { - for i := 0; i < m.config.Manager.PartitionBuckets; i++ { - buckets = append(buckets, int32(i)) - } - } - for _, bucket := range buckets { - index1, err := BuildEpochIndex(int(task.PartitionId), uint64(bucket), task.LowerEpoch, "") - if err != nil { - logrus.Fatal(err) - continue - } - index2, err := BuildEpochIndex(int(task.PartitionId), uint64(bucket), task.UpperEpoch, "") - if err != nil { - logrus.Fatal(err) - continue - } - it := m.db.NewIterator( - []byte(index1), - []byte(index2), - false, - ) - for !it.IsDone() { - _, _, epoch, key, err := ParseEpochIndex(string(it.Key())) - if err != nil { - logrus.Fatal(err) - continue - } - timestamp, err := utils.DecodeBytesToInt64(it.Value()) - if err != nil { - logrus.Fatal(err) - continue - } - - task.ResCh <- &rpc.RpcValue{Key: key, Epoch: epoch, UnixTimestamp: timestamp} - it.Next() - } - it.Release() - } + m.ring.SetRingMembers(task.Members, task.TempMembers) + task.ResCh <- true - close(task.ResCh) + case rpc.SetValueTask: + logrus.Debugf("worker SetValueTask: %+v", task) - case VerifyPartitionEpochRequestTask: - logrus.Debugf("worker VerifyPartitionEpochRequestTask: %+v", task.PartitionId) - err := m.VerifyEpoch(task.PartitionId, task.Epoch) + if task.Value.Epoch < m.GetCurrentEpoch()-1 { + task.ResCh <- errors.New("cannot set lagging epoch") + return + } + err := m.SetValue(task.Value) + if err != nil { + logrus.Warnf("SetValue err = %v", err) + task.ResCh <- err + } else { + task.ResCh <- true + } - if err != nil { - task.ResCh <- err - } else { - task.ResCh <- VerifyPartitionEpochResponse{Valid: true} - } + case rpc.GetValueTask: + logrus.Debugf("worker GetValueTask: %+v", task) + // value, err := m.db.Get([]byte(task.Key)) + value, err := m.GetValue(task.Key) + if err == storage.KEY_NOT_FOUND { // TODO if the nodes partition is not up to date it should not count as response + task.ResCh <- nil + } else if err != nil { + task.ResCh <- err + } else { + task.ResCh <- value + } - case rpc.GetEpochTreeObjectTask: - logrus.Debugf("worker GetPartitionEpochObjectTask: %+v", task) - index, err := BuildEpochTreeObjectIndex(int(task.PartitionId), task.LowerEpoch) + case rpc.StreamBucketsTask: // TODO test this is returning right values + logrus.Debugf("worker StreamBucketsTask: %+v", task) + var buckets []int32 = task.Buckets + if len(buckets) == 0 { + for i := 0; i < m.config.Manager.PartitionBuckets; i++ { + buckets = append(buckets, int32(i)) + } + } + for _, bucket := range buckets { + index1, err := BuildEpochIndex(int(task.PartitionId), uint64(bucket), task.LowerEpoch, "") + if err != nil { + logrus.Fatal(err) + continue + } + index2, err := BuildEpochIndex(int(task.PartitionId), uint64(bucket), task.UpperEpoch, "") + if err != nil { + logrus.Fatal(err) + continue + } + it := m.db.NewIterator( + []byte(index1), + []byte(index2), + false, + ) + for !it.IsDone() { + _, _, epoch, key, err := ParseEpochIndex(string(it.Key())) if err != nil { - task.ResCh <- err + logrus.Fatal(err) continue } - - epochTreeObjectBytes, err := m.db.Get([]byte(index)) + timestamp, err := utils.DecodeBytesToInt64(it.Value()) if err != nil { - // logrus.Warnf("GetEpochTreeObjectTask err = %v index %v active: %v", err, index, m.consistencyController.IsPartitionActive(int(task.PartitionId))) - err = errors.Wrapf(err, "active: %v", m.consistencyController.IsPartitionActive(int(task.PartitionId))) - task.ResCh <- err + logrus.Fatal(err) continue } - epochTreeObject := &rpc.RpcEpochTreeObject{} - err = proto.Unmarshal(epochTreeObjectBytes, epochTreeObject) - if err != nil { - task.ResCh <- err - continue - } - task.ResCh <- epochTreeObject + task.ResCh <- &rpc.RpcValue{Key: key, Epoch: epoch, UnixTimestamp: timestamp} + it.Next() + } + it.Release() + } - case rpc.GetEpochTreeLastValidObjectTask: - logrus.Debugf("worker GetEpochTreeLastValidObjectTask: %+v", task) - epochTreeObjectLastValid, err := m.GetEpochTreeLastValid(task.PartitionId) - if err != nil { - task.ResCh <- err - } else if epochTreeObjectLastValid == nil { - task.ResCh <- errors.New("no valid EpochTreeObject for partition") - } else { - task.ResCh <- epochTreeObjectLastValid - } + close(task.ResCh) - case SyncPartitionTask: - logrus.Debugf("worker SyncPartitionTask: %+v", task.PartitionId) - epochTreeObjectLastValid, err := m.GetEpochTreeLastValid(task.PartitionId) - if err != nil { - task.ResCh <- errors.Wrap(err, "GetEpochTreeLastValid") - continue - } else if epochTreeObjectLastValid != nil && epochTreeObjectLastValid.LowerEpoch >= task.UpperEpoch { // TODO validate this is the correct compare - task.ResCh <- nil - continue - } + case VerifyPartitionEpochRequestTask: + logrus.Debugf("worker VerifyPartitionEpochRequestTask: %+v", task.PartitionId) + err := m.VerifyEpoch(task.PartitionId, task.Epoch) - lastValidEpoch := int64(0) - if epochTreeObjectLastValid != nil { - lastValidEpoch = epochTreeObjectLastValid.LowerEpoch - } + if err != nil { + task.ResCh <- err + } else { + task.ResCh <- VerifyPartitionEpochResponse{Valid: true} + } - logrus.Debugf("sync lastValidEpoch %d", lastValidEpoch) + case rpc.GetEpochTreeObjectTask: + logrus.Debugf("worker GetPartitionEpochObjectTask: %+v", task) + index, err := BuildEpochTreeObjectIndex(int(task.PartitionId), task.LowerEpoch) + if err != nil { + task.ResCh <- err + return + } - // find most healthy node - err = m.PoliteStreamRequest(int(task.PartitionId), lastValidEpoch, task.UpperEpoch+1, nil) + epochTreeObjectBytes, err := m.db.Get([]byte(index)) + if err != nil { + // logrus.Warnf("GetEpochTreeObjectTask err = %v index %v active: %v", err, index, m.consistencyController.IsPartitionActive(int(task.PartitionId))) + err = errors.Wrapf(err, "active: %v", m.consistencyController.IsPartitionActive(int(task.PartitionId))) + task.ResCh <- err + return + } - if err != nil { - logrus.Debug(err) - task.ResCh <- SyncPartitionResponse{Valid: false, LowerEpoch: lastValidEpoch, UpperEpoch: task.UpperEpoch + 1} - } else { - task.ResCh <- SyncPartitionResponse{Valid: true, LowerEpoch: lastValidEpoch, UpperEpoch: task.UpperEpoch + 1} - } + epochTreeObject := &rpc.RpcEpochTreeObject{} + err = proto.Unmarshal(epochTreeObjectBytes, epochTreeObject) + if err != nil { + task.ResCh <- err + return + } + task.ResCh <- epochTreeObject - case hashring.RingUpdateTask: - // logrus.Warnf("worker MembersUpdateTask #%+v", len(task.Partitions)) + case rpc.GetEpochTreeLastValidObjectTask: + logrus.Debugf("worker GetEpochTreeLastValidObjectTask: %+v", task) + epochTreeObjectLastValid, err := m.GetEpochTreeLastValid(task.PartitionId) + if err != nil { + task.ResCh <- err + } else if epochTreeObjectLastValid == nil { + task.ResCh <- errors.New("no valid EpochTreeObject for partition") + } else { + task.ResCh <- epochTreeObjectLastValid + } - currPartitions := utils.NewIntSet().From(task.Partitions) - m.consistencyController.HandleHashringChange(currPartitions) + case SyncPartitionTask: + logrus.Debugf("worker SyncPartitionTask: %+v", task.PartitionId) + epochTreeObjectLastValid, err := m.GetEpochTreeLastValid(task.PartitionId) + if err != nil { + task.ResCh <- errors.Wrap(err, "GetEpochTreeLastValid") + return + } else if epochTreeObjectLastValid != nil && epochTreeObjectLastValid.LowerEpoch >= task.UpperEpoch { // TODO validate this is the correct compare + task.ResCh <- nil + return + } - task.ResCh <- true + lastValidEpoch := int64(0) + if epochTreeObjectLastValid != nil { + lastValidEpoch = epochTreeObjectLastValid.LowerEpoch + } - default: - logrus.Panicf("worker unkown task type: %v", reflect.TypeOf(task)) - break workerLoop - } + logrus.Debugf("sync lastValidEpoch %d", lastValidEpoch) + + // find most healthy node + err = m.PoliteStreamRequest(int(task.PartitionId), lastValidEpoch, task.UpperEpoch+1, nil) + + if err != nil { + logrus.Debug(err) + task.ResCh <- SyncPartitionResponse{Valid: false, LowerEpoch: lastValidEpoch, UpperEpoch: task.UpperEpoch + 1} + } else { + task.ResCh <- SyncPartitionResponse{Valid: true, LowerEpoch: lastValidEpoch, UpperEpoch: task.UpperEpoch + 1} } + + case hashring.RingUpdateTask: + // logrus.Warnf("worker MembersUpdateTask #%+v", len(task.Partitions)) + + currPartitions := utils.NewIntSet().From(task.Partitions) + m.consistencyController.HandleHashringChange(currPartitions) + + task.ResCh <- true + + default: + logrus.Panicf("worker unkown task type: %v", reflect.TypeOf(task)) + } } diff --git a/operator/BUILD.bazel b/operator/BUILD.bazel index c585fd0..3ae0101 100644 --- a/operator/BUILD.bazel +++ b/operator/BUILD.bazel @@ -26,7 +26,7 @@ go_binary( name = "operator", embed = [":go_default_library"], env = { - # "MYKEYSTORE_IMAGE": "docker.io/andrew-delph/main:store_image", + # "MYKEYSTORE_IMAGE": "ghcr.io/andrew-delph/main:store_image", "MYKEYSTORE_IMAGE": "strm/helloworld-http:latest", }, visibility = ["//visibility:public"], @@ -51,16 +51,16 @@ container_image( # }, # files = ["//config:default-config"], ports = ["80"], - repository = "docker.io/andrew-delph", + repository = "ghcr.io/andrew-delph", tags = ["latest"], ) container_push( name = "image_push", format = "Docker", - image = ":store_image", + image = ":operator_image", registry = "ghcr.io", - repository = "andrew-delph/my-key-store", + repository = "andrew-delph/operator", tag = "latest", ) diff --git a/operator/PROJECT b/operator/PROJECT index 38d1f20..25dafc2 100644 --- a/operator/PROJECT +++ b/operator/PROJECT @@ -12,7 +12,7 @@ plugins: group: cache kind: MyKeyStore options: - image: docker.io/andrew-delph/main:store_image + image: ghcr.io/andrew-delph/main:store_image version: v1alpha1 manifests.sdk.operatorframework.io/v2: {} scorecard.sdk.operatorframework.io/v2: {} diff --git a/operator/config/manager/kustomization.yaml b/operator/config/manager/kustomization.yaml index 283df5d..21d4c39 100644 --- a/operator/config/manager/kustomization.yaml +++ b/operator/config/manager/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: controller - newName: docker.io/andrew-delph/operator - newTag: "1696944159" + newName: ghcr.io/andrew-delph/operator + newTag: "1696969023" diff --git a/operator/config/manager/manager.yaml b/operator/config/manager/manager.yaml index a99afdd..4d7639e 100644 --- a/operator/config/manager/manager.yaml +++ b/operator/config/manager/manager.yaml @@ -72,7 +72,7 @@ spec: name: manager env: - name: MYKEYSTORE_IMAGE - value: docker.io/andrew-delph/main:store_image + value: ghcr.io/andrew-delph/main:store_image securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/operator/config/samples/cache_v1alpha1_mykeystore.yaml b/operator/config/samples/cache_v1alpha1_mykeystore.yaml index 0a9b496..1ecc460 100644 --- a/operator/config/samples/cache_v1alpha1_mykeystore.yaml +++ b/operator/config/samples/cache_v1alpha1_mykeystore.yaml @@ -4,4 +4,4 @@ metadata: name: store spec: size: 5 - image: docker.io/andrew-delph/main:store_image + image: ghcr.io/andrew-delph/main:store_image diff --git a/resources/store.yaml b/resources/store.yaml index 461d236..54a959f 100644 --- a/resources/store.yaml +++ b/resources/store.yaml @@ -19,7 +19,7 @@ spec: app: store spec: containers: - - image: docker.io/andrew-delph/main:store_image + - image: ghcr.io/andrew-delph/main:store_image # imagePullPolicy: "Always" imagePullPolicy: Never name: store