Dev

.tmuxinator/tmuxinator-k8.yaml

@@ -1,5 +1,5 @@
 name: k8_dev
-on_project_first_start: ./dev k8-init
+# on_project_first_start: ./dev k8-init
 on_project_exit: minikube delete
 windows:
   - editor:

config/config.go

@@ -8,11 +8,12 @@ import (
 )
 
 type ManagerConfig struct {
-	WokersCount          int `mapstructure:"WORKERS_COUNT"`
-	ReqChannelSize       int `mapstructure:"REQ_CHANNEL_SIZE"`
-	ReplicaCount         int `mapstructure:"REPLICA_COUNT"`
-	WriteQuorum          int `mapstructure:"WRITE_QUORUM"`
-	ReadQuorum           int `mapstructure:"READ_QUORUM"`
+	WokersCount          int  `mapstructure:"WORKERS_COUNT"`
+	ThreadRequests       bool `mapstructure:"THREAD_REQUESTS"`
+	ReqChannelSize       int  `mapstructure:"REQ_CHANNEL_SIZE"`
+	ReplicaCount         int  `mapstructure:"REPLICA_COUNT"`
+	WriteQuorum          int  `mapstructure:"WRITE_QUORUM"`
+	ReadQuorum           int  `mapstructure:"READ_QUORUM"`
 	SaveInterval         int
 	DefaultTimeout       int     `mapstructure:"DEFAULT_TIMEOUT"`
 	PartitionCount       int     `mapstructure:"PARTITION_COUNT"`

config/config_test.go

@@ -62,6 +62,7 @@ func TestConfigOverwrite(t *testing.T) {
 	assert.EqualValues(t, 99999, config.Manager.PartitionBuckets, "PartitionBuckets wrong value")
 	assert.EqualValues(t, "/tmp/store/data", config.Manager.DataPath, "DataPath wrong value")
 	assert.EqualValues(t, 991, config.Manager.PartitionConcurrency, "PartitionConcurrency wrong value")
+	assert.EqualValues(t, true, config.Manager.ThreadRequests, "ThreadRequests wrong value")
 
 	// consensus config
 	assert.EqualValues(t, 9, config.Consensus.EpochTime, "EpochTime wrong value")

config/default-config.yaml

@@ -4,7 +4,8 @@ manager:
   write_quorum: 2
   default_timeout: 10
   req_channel_size: 20
-  workers_count: 100
+  workers_count: 2
+  thread_requests: true
   partition_count: 15
   partition_concurrency: 10
   partition_buckets: 1000

config/test-config.yaml

@@ -6,6 +6,7 @@ manager:
   partition_buckets: 99999
   partition_concurrency: 991
   data_path: "/tmp/store/data"
+  thread_requests: true
 consensus:
   epoch_time: 9
   data_path: "/tmp/store/raft"

deploy_k8.sh

@@ -3,15 +3,15 @@
 # Exit on any error
 set -e
 
+# eval $(minikube docker-env)
+
 STATEFULSET_NAME="store"
 NAMESPACE="default"
 
-eval $(minikube docker-env)
 TAG=$(date +%s)
-NEW_IMAGE=docker.io/andrew-delph/main:$TAG
+NEW_IMAGE=ghcr.io/andrew-delph/main:$TAG
 
-bazel run --execution_log_json_file=events.json //main:store_image
-docker tag docker.io/andrew-delph/main:store_image $NEW_IMAGE
+bazel run --execution_log_json_file=events.json //main:image_push -- -dst=$NEW_IMAGE
 
 
 ROLL_OUT_FLAG=0
@@ -35,7 +35,6 @@ SECONDS=0  # Reset the SECONDS variable
 if [ $ROLL_OUT_FLAG -eq 0 ]; then
     echo "The 'rollout' flag is not set. Setting up."
     (cd operator && kustomize build config/crd | kubectl apply -f - || true)
-    bazel run //main:image_push
     kubectl apply -f ./operator/config/samples/ || true
 else
     echo "The 'rollout' flag is set."

deploy_operator.sh

@@ -1,12 +1,11 @@
 #!/bin/bash
 
+# eval $(minikube docker-env)
 
 
-eval $(minikube docker-env)
 TAG=$(date +%s)
-IMG=docker.io/andrew-delph/operator:$TAG
-bazel run //operator:operator_image
-docker tag docker.io/andrew-delph/operator:operator_image $IMG
+IMG=ghcr.io/andrew-delph/operator:$TAG
+bazel run //operator:image_push -- -dst=$IMG
 (cd operator && make deploy IMG=$IMG)
 (cd operator && kustomize build config/rbac | kubectl apply -f -  || true)
 

dev

@@ -23,10 +23,8 @@ case $1 in
         ;;
     k8-r)
         kubectl delete -f ./operator/config/samples/ || true
-        ./deploy_k8.sh
         ./deploy_operator.sh
-        kubectl apply -f ./resources/prom.yaml
-        kubectl create configmap prometheus-config --from-file=prometheus.yml || true
+        ./deploy_k8.sh
         ;;
     k8-operator)
         ./deploy_operator.sh
@@ -38,7 +36,7 @@ case $1 in
         tmux kill-session -t k8_dev
         ;;
     k8-e2e)
-        k6 run -e ADDRESS="192.168.49.2:30000" e2e/test.js
+        ./e2e_k8.sh
         ;;
     dc-e2e)
         k6 run e2e/test.js

docker-compose.yaml

@@ -38,7 +38,7 @@ services:
 
   store:
     # hostname: store
-    image: docker.io/andrew-delph/main:store_image
+    image: ghcr.io/andrew-delph/main:store_image
     scale: 5
     depends_on:
       # - zookeeper
@@ -57,7 +57,7 @@ services:
   # store-profile:
   #   # hostname: store
   #   # scale: 1
-  #   image: docker.io/andrew-delph/main:store_image
+  #   image: ghcr.io/andrew-delph/main:store_image
   #   environment:
   #     - PROFILE_SERVER=true
   #     - BOOTSTRAP_TIMEOUT=1

docs.md

@@ -0,0 +1,37 @@
+- summary
+- layout
+  - hasring
+  - partitions
+  - epoch buckets
+  - indexs + lsm
+- gossip
+  - client discovery
+  - non operator member management
+- consensus
+  - epochs
+  - members + temp members
+- hashring
+  - current members
+  - temp members
+- replication
+  - on epoch
+  - on cluster change
+- read/write quorum
+- external/internal endpoints
+- protobufs
+- indexes
+- storage
+  - indexdb
+  - badgerdb
+- operator
+  - code rollout
+  - scale cluster
+- development environment
+  - docker-compose
+  - minikube
+- benchmarks
+- future improvements
+  - scan query
+  - SAGA/2PC
+  - read repairs
+- tradeoffs

e2e/Makefile

@@ -6,12 +6,13 @@ build:
 init:
 	@echo "init"
 	curl https://raw.githubusercontent.com/grafana/k6-operator/main/bundle.yaml | kubectl apply -f -
+	make monitoring
 
 setup:
-	kubectl --context minikube create namespace k6 || echo "namespace already exists"
-	kubectl --context minikube delete configmap test-scripts --ignore-not-found -n k6
-	kubectl --context minikube delete configmap prometheus-config --ignore-not-found -n k6
-	kubectl --context minikube create configmap test-scripts  \
+	kubectl create namespace k6 || echo "namespace already exists"
+	kubectl  delete configmap test-scripts --ignore-not-found -n k6
+	kubectl  delete configmap prometheus-config --ignore-not-found -n k6
+	kubectl  create configmap test-scripts  \
 		--from-file=./test.js -n k6
 
 run:
@@ -21,14 +22,19 @@ run:
 logs:
 	stern -n k6 .*
 
-prod:
-	make setup
-	./run-kube.sh resources/k6-output-grafana-cloud.yaml --external
-
 watch:
-	watch -n 15 kubectl get pods -n k6
+	watch kubectl get pods -n k6
 
-delete:
-	kubectl --context minikube delete  -f resources/k6-output-grafana-cloud.yaml -n k6
-	# cd dependencies/k6-operator && make delete
+stop:
+	kubectl  delete  -f resources/k6-output-grafana-cloud.yaml -n k6 || true
 
+delete:
+	kubectl  delete  -f resources/k6-output-grafana-cloud.yaml -n k6 || true
+	helm uninstall loki --namespace=monitoring --wait || true
+
+monitoring:
+	helm repo add grafana https://grafana.github.io/helm-charts
+	kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml --wait
+	kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f -
+	helm upgrade --install loki --namespace=monitoring grafana/loki-stack -f loki-stack.values.yaml
+	bash ./create_dashboards.sh

e2e/create_dashboards.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# default dashboards: https://github.com/prometheus-community/helm-charts/tree/8f91606e01b85b9aa013ed472c5acb836c4e7ac8/charts/kube-prometheus-stack/templates/grafana/dashboards-1.14
+
+# delete existing dashboards
+kubectl delete configmap --selector custom_dashboard='1' -n monitoring
+
+# Set the directory path
+dir_path="./dashboards"
+
+# Loop through all files in the directory
+for file_path in $dir_path/*; do
+  file_name=$(basename "${file_path%.*}")
+
+  # # Create the ConfigMap using the file contents and name as the ConfigMap name
+  kubectl create configmap "$file_name" --from-file="$file_path" -n monitoring
+
+  # # Add labels to the ConfigMap
+  kubectl label configmap "$file_name" grafana_dashboard='1' custom_dashboard='1' -n monitoring
+
+done