From e3fbfd5483d6531ade76f85d4152b0e794fb3e85 Mon Sep 17 00:00:00 2001 From: Moran Abilea Date: Mon, 22 Jun 2026 14:07:31 +0000 Subject: [PATCH] Add chaos test workflow also fixing review notes adding readme.md as well Signed-off-by: Moran Abilea --- .github/workflows/chaos-test.yml | 189 +++++++++++++++++++++++ test/chaos/README.md | 192 +++++++++++++++++++++++ test/chaos/chaos-runner.sh | 146 ++++++++++++++++++ test/chaos/chaos-test.sh | 178 ++++++++++++++++++++++ test/chaos/collect-results.sh | 253 +++++++++++++++++++++++++++++++ test/chaos/monitor-completion.sh | 171 +++++++++++++++++++++ test/chaos/start-arma-network.sh | 85 +++++++++++ 7 files changed, 1214 insertions(+) create mode 100644 .github/workflows/chaos-test.yml create mode 100644 test/chaos/README.md create mode 100755 test/chaos/chaos-runner.sh create mode 100755 test/chaos/chaos-test.sh create mode 100755 test/chaos/collect-results.sh create mode 100755 test/chaos/monitor-completion.sh create mode 100755 test/chaos/start-arma-network.sh diff --git a/.github/workflows/chaos-test.yml b/.github/workflows/chaos-test.yml new file mode 100644 index 000000000..e29773233 --- /dev/null +++ b/.github/workflows/chaos-test.yml @@ -0,0 +1,189 @@ +# Copyright the Hyperledger Fabric contributors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +name: Chaos Test + +on: + schedule: + # Sunday-Thursday: 2-hour chaos test + - cron: '0 3 * * 0-4' + # Friday: Extended 5.5-hour chaos test + - cron: '0 3 * * 5' + + workflow_dispatch: + inputs: + duration_minutes: + description: 'Test duration in minutes' + required: true + default: '120' + type: string + tx_rate: + description: 'Transactions per second' + required: true + default: '1000' + type: string + tx_size: + description: 'Transaction size in bytes' + required: true + default: '300' + type: string + num_parties: + description: 'Number of parties (4, 7, or 10)' + required: true + default: '4' + type: choice + options: + - '4' + - '7' + - '10' + num_shards: + description: 'Number of shards (1, 2, or 4)' + required: true + default: '2' + type: choice + options: + - '1' + - '2' + - '4' + chaos_enabled: + description: 'Enable chaos testing' + required: true + default: true + type: boolean + chaos_initial_wait: + description: 'Wait before starting chaos (seconds)' + required: true + default: '300' + type: string + chaos_stop_duration: + description: 'How long to keep component down (seconds)' + required: true + default: '60' + type: string + chaos_restart_wait: + description: 'Wait after component restart (seconds)' + required: true + default: '60' + type: string + +env: + GOPATH: /opt/go + PATH: /opt/go/bin:/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin + +permissions: + contents: read + +jobs: + chaos-test: + name: Chaos Test + runs-on: ubuntu-latest + timeout-minutes: 350 + + steps: + - name: Checkout Fabric-X Code + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Build arma and armageddon binaries + run: make binary + + - name: Determine test duration + id: duration + run: | + if [ -n "${{ inputs.duration_minutes }}" ]; then + echo "minutes=${{ inputs.duration_minutes }}" >> $GITHUB_OUTPUT + echo "Manual trigger: Using ${{ inputs.duration_minutes }} minutes" + else + DAY_OF_WEEK=$(date -u +%u) + if [ "$DAY_OF_WEEK" = "7" ]; then + DAY_OF_WEEK=0 + fi + + if [ "$DAY_OF_WEEK" = "5" ]; then + echo "minutes=330" >> $GITHUB_OUTPUT + echo "Friday: Using extended duration (330 minutes / 5.5 hours)" + else + echo "minutes=120" >> $GITHUB_OUTPUT + echo "Weekday: Using standard duration (120 minutes / 2 hours)" + fi + fi + + - name: Run Chaos Test + env: + DURATION_MINUTES: ${{ steps.duration.outputs.minutes }} + TX_RATE: ${{ inputs.tx_rate || '1000' }} + TX_SIZE: ${{ inputs.tx_size || '300' }} + NUM_PARTIES: ${{ inputs.num_parties || '4' }} + NUM_SHARDS: ${{ inputs.num_shards || '2' }} + CHAOS_ENABLED: ${{ inputs.chaos_enabled != false }} + CHAOS_INITIAL_WAIT: ${{ inputs.chaos_initial_wait || '300' }} + CHAOS_STOP_DURATION: ${{ inputs.chaos_stop_duration || '60' }} + CHAOS_RESTART_WAIT: ${{ inputs.chaos_restart_wait || '60' }} + run: | + chmod +x test/chaos/*.sh + test/chaos/chaos-test.sh + + - name: Upload test summary + if: always() + uses: actions/upload-artifact@v4 + with: + name: chaos-test-summary-${{ github.run_number }} + path: test-results/summary/ + retention-days: 30 + + - name: Upload statistics + if: always() + uses: actions/upload-artifact@v4 + with: + name: chaos-test-statistics-${{ github.run_number }} + path: test-results/statistics/ + retention-days: 30 + + - name: Upload error logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: chaos-test-errors-${{ github.run_number }} + path: test-results/logs/errors.log + retention-days: 30 + if-no-files-found: ignore + + - name: Upload loader and receiver logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: chaos-test-loader-receiver-logs-${{ github.run_number }} + path: | + loader.log + receiver*.log + retention-days: 30 + if-no-files-found: ignore + + - name: Upload component logs (short tests only) + if: always() && (inputs.duration_minutes == '' || inputs.duration_minutes <= 120) + uses: actions/upload-artifact@v4 + with: + name: chaos-test-component-logs-${{ github.run_number }} + path: | + consenter*.log + batcher*.log + assembler*.log + router*.log + retention-days: 30 + if-no-files-found: ignore + + - name: Upload full logs (short tests only) + if: always() && (inputs.duration_minutes == '' || inputs.duration_minutes <= 120) + uses: actions/upload-artifact@v4 + with: + name: chaos-test-logs-${{ github.run_number }} + path: test-results/logs/ + retention-days: 30 + if-no-files-found: ignore \ No newline at end of file diff --git a/test/chaos/README.md b/test/chaos/README.md new file mode 100644 index 000000000..a0b6e8ae3 --- /dev/null +++ b/test/chaos/README.md @@ -0,0 +1,192 @@ +# Chaos Test + +## Overview + +This directory contains the scripts used to run the ARMA chaos test. + +The test starts a local ARMA network, starts receivers and a loader, optionally runs a chaos runner that stops and restarts ARMA components, monitors the test until completion or timeout, and then collects logs, statistics, and a summary. + +The same scripts are used by the GitHub Actions workflow and can also be executed locally from the command line on a Linux machine. + +## Scripts + +```text +test/chaos/ +├── chaos-test.sh +├── start-arma-network.sh +├── chaos-runner.sh +├── monitor-completion.sh +├── collect-results.sh +└── README.md +``` + +### `chaos-test.sh` + +Main orchestration script. + +It reads the test parameters from environment variables, creates a temporary test directory, generates the Armageddon configuration, generates the ARMA configuration files, updates the generated FileStore locations, starts the ARMA network, starts receivers, starts the loader, optionally starts the chaos runner, monitors the test, and collects the results. + +### `start-arma-network.sh` + +Starts the ARMA network components. + +It starts consenters, batchers, assemblers, and routers, and stores their process IDs under the test directory. + +### `chaos-runner.sh` + +Stops and restarts ARMA components in sequence. + +It uses the PID files created by `start-arma-network.sh`, waits according to the chaos timing configuration, and exits when the stop signal file is created. + +### `monitor-completion.sh` + +Monitors the test execution. + +It waits until the configured duration is reached or until the loader and all receivers complete. When monitoring finishes, it creates the chaos stop signal and stops loader and receiver processes if they are still running. + +### `collect-results.sh` + +Collects test results. + +It creates the `test-results` directory, extracts loader and receiver statistics, copies or compresses logs depending on the test duration, collects receiver statistics CSV files, and creates a summary report. + +## Prerequisites + +Build the binaries before running the test: + +```bash +make binary +``` + +The scripts expect the following binaries to exist: + +```text +./bin/arma +./bin/armageddon +``` + +Run the test from the repository root. + +## Running Locally + +Execute the test from the repository root. + +The following command is an example configuration: + +```bash +chmod +x test/chaos/*.sh + +DURATION_MINUTES=10 \ +TX_RATE=1000 \ +TX_SIZE=300 \ +NUM_PARTIES=4 \ +NUM_SHARDS=2 \ +CHAOS_ENABLED=true \ +CHAOS_INITIAL_WAIT=30 \ +CHAOS_STOP_DURATION=10 \ +CHAOS_RESTART_WAIT=10 \ +test/chaos/chaos-test.sh +``` + +The values can be adjusted as needed for the desired test configuration. + +## Configuration + +The test is configured using environment variables. + +The values shown below are the defaults used by `chaos-test.sh`. + +| Variable | Description | Default | +| --------------------- | --------------------------------------------- | ------- | +| `DURATION_MINUTES` | Test duration in minutes | `120` | +| `TX_RATE` | Transactions per second | `1000` | +| `TX_SIZE` | Transaction size in bytes | `300` | +| `NUM_PARTIES` | Number of parties | `4` | +| `NUM_SHARDS` | Number of shards | `2` | +| `CHAOS_ENABLED` | Whether to run the chaos runner | `true` | +| `CHAOS_INITIAL_WAIT` | Wait before starting chaos, in seconds | `300` | +| `CHAOS_STOP_DURATION` | How long to keep a component down, in seconds | `60` | +| `CHAOS_RESTART_WAIT` | Wait after restarting a component, in seconds | `60` | + +## Test Flow + +When `chaos-test.sh` runs, it performs the following steps: + +1. Reads configuration from environment variables. +2. Calculates the total number of transactions. +3. Creates a temporary test directory using `mktemp`. +4. Generates a test configuration YAML file. +5. Runs `./bin/armageddon generate`. +6. Creates a data directory under the temporary test directory. +7. Updates the generated FileStore locations so each component uses its own data directory. +8. Starts the ARMA network using `start-arma-network.sh`. +9. Creates output directories for receivers. +10. Starts one receiver per party. +11. Starts the loader. +12. Starts `chaos-runner.sh` if `CHAOS_ENABLED=true`. +13. Runs `monitor-completion.sh`. +14. Waits briefly for the chaos runner to stop. +15. Runs `collect-results.sh`. +16. Cleans up remaining `arma` and `armageddon` processes. + +## Generated Files + +During execution, the test creates a temporary directory similar to: + +```text +/tmp/chaos-test-XXXXXX +``` + +That directory contains generated configuration, data directories, receiver output directories, PID files, and the chaos stop signal file. + +The result artifacts are written under: + +```text +test-results/ +├── logs/ +├── statistics/ +└── summary/ +``` + +## GitHub Actions Workflow + +The GitHub Actions workflow is defined under: + +```text +.github/workflows/chaos-test.yml +``` + +The workflow is responsible for invoking the chaos test in GitHub Actions. + +It performs the following steps: + +1. Checks out the repository. +2. Installs Go. +3. Builds the binaries using `make binary`. +4. Determines the test duration. +5. Sets the test configuration through environment variables. +6. Executes: + +```bash +test/chaos/chaos-test.sh +``` + +7. Uploads the generated artifacts from `test-results`. + +The workflow contains only the GitHub Actions orchestration. The chaos test implementation resides under `test/chaos`. + +## Manual Runs + +The workflow can also be triggered manually using `workflow_dispatch`. + +The following parameters can be configured: + +* Test duration +* Transaction rate +* Transaction size +* Number of parties +* Number of shards +* Whether chaos is enabled +* Chaos timing values + +These values are passed to `test/chaos/chaos-test.sh` through environment variables. \ No newline at end of file diff --git a/test/chaos/chaos-runner.sh b/test/chaos/chaos-runner.sh new file mode 100755 index 000000000..18e3007e7 --- /dev/null +++ b/test/chaos/chaos-runner.sh @@ -0,0 +1,146 @@ +#!/bin/bash +# Copyright the Hyperledger Fabric contributors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Chaos runner - stops and starts components in sequential order + +TEST_DIR=$1 +NUM_PARTIES=$2 +NUM_SHARDS=$3 + +# Read timing configuration from environment or use defaults +INITIAL_WAIT=${CHAOS_INITIAL_WAIT:-300} +STOP_WAIT=${CHAOS_STOP_DURATION:-60} +START_WAIT=${CHAOS_RESTART_WAIT:-60} + +# Get PID directory and working directory +PID_DIR="${TEST_DIR}/pids" +WORK_DIR=$(cat ${PID_DIR}/work_dir.txt 2>/dev/null || pwd) + +# Create a stop signal file that monitor-completion.sh will create when done +STOP_SIGNAL="${TEST_DIR}/chaos_stop_signal" + +echo "==========================================" +echo "Chaos Runner Started" +echo "==========================================" +echo "Configuration:" +echo " Initial wait: ${INITIAL_WAIT}s" +echo " Stop duration: ${STOP_WAIT}s" +echo " Restart wait: ${START_WAIT}s" +echo " PID directory: ${PID_DIR}" +echo " Working directory: ${WORK_DIR}" +echo " Stop signal file: ${STOP_SIGNAL}" +echo "==========================================" + +echo "[$(date '+%Y-%m-%d %H:%M:%S')] Waiting ${INITIAL_WAIT}s for network to stabilize..." +sleep $INITIAL_WAIT + +# Function to kill and restart a component +kill_and_restart() { + local component=$1 + local party=$2 + local shard=$3 # Optional, only for batchers + local config_file=$4 + local log_file=$5 + + # Determine PID file name + local pid_file + if [ -n "$shard" ]; then + pid_file="${PID_DIR}/${component}${party}-${shard}.pid" + else + pid_file="${PID_DIR}/${component}${party}.pid" + fi + + # Read PID from file + if [ ! -f "$pid_file" ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: ${component} (party ${party}${shard:+ shard ${shard}}) PID file not found: ${pid_file}" + return + fi + + local PID=$(cat ${pid_file} 2>/dev/null) + + # Check if process is still running + if [ -z "$PID" ] || ! kill -0 $PID 2>/dev/null; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] WARNING: ${component} (party ${party}${shard:+ shard ${shard}}) not running (PID: ${PID:-unknown})" + # Try to restart anyway + else + # Kill the process + kill $PID 2>/dev/null + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Stopping ${component} (party ${party}${shard:+ shard ${shard}}) - was PID ${PID}" + + # Wait for process to die + local wait_count=0 + while kill -0 $PID 2>/dev/null && [ $wait_count -lt 10 ]; do + sleep 1 + wait_count=$((wait_count + 1)) + done + + # Force kill if still running + if kill -0 $PID 2>/dev/null; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Force killing ${component} (party ${party}${shard:+ shard ${shard}})" + kill -9 $PID 2>/dev/null + fi + fi + + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ${component} (party ${party}${shard:+ shard ${shard}}) DOWN — waiting ${STOP_WAIT} seconds" + sleep $STOP_WAIT + + # Restart the component from the correct working directory + cd ${WORK_DIR} + ${WORK_DIR}/bin/arma ${component} --config=${config_file} >> ${log_file} 2>&1 & + local NEW_PID=$! + echo ${NEW_PID} > ${pid_file} + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting ${component} (party ${party}${shard:+ shard ${shard}}) - PID ${NEW_PID}" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ${component} (party ${party}${shard:+ shard ${shard}}) UP — waiting ${START_WAIT} seconds" + sleep $START_WAIT +} + +# Main chaos loop - run until stop signal is received +while true; do + # Check if we should stop (test completed) + if [ -f "${STOP_SIGNAL}" ]; then + echo "==========================================" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Stop signal received - chaos runner exiting" + echo "==========================================" + break + fi + + for party in $(seq 1 $NUM_PARTIES); do + # Check stop signal before each party + if [ -f "${STOP_SIGNAL}" ]; then + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Stop signal received - exiting chaos loop" + break 2 + fi + + echo "----------------------------------------------------" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] PARTY ${party} — starting chaos sequence" + echo "----------------------------------------------------" + + # 1. Assembler + kill_and_restart "assembler" ${party} "" \ + "${TEST_DIR}/config/party${party}/local_config_assembler.yaml" \ + "assembler${party}.log" + + # 2. Consenter + kill_and_restart "consensus" ${party} "" \ + "${TEST_DIR}/config/party${party}/local_config_consenter.yaml" \ + "consenter${party}.log" + + # 3. Router + kill_and_restart "router" ${party} "" \ + "${TEST_DIR}/config/party${party}/local_config_router.yaml" \ + "router${party}.log" + + # 4. Batchers (in order) + for shard in $(seq 1 $NUM_SHARDS); do + kill_and_restart "batcher" ${party} ${shard} \ + "${TEST_DIR}/config/party${party}/local_config_batcher${shard}.yaml" \ + "batcher${party}-${shard}.log" + done + + echo "[$(date '+%Y-%m-%d %H:%M:%S')] PARTY ${party} — chaos sequence DONE" + done +done + +echo "[$(date '+%Y-%m-%d %H:%M:%S')] Chaos runner finished" diff --git a/test/chaos/chaos-test.sh b/test/chaos/chaos-test.sh new file mode 100755 index 000000000..2e6305af2 --- /dev/null +++ b/test/chaos/chaos-test.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# Copyright the Hyperledger Fabric contributors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Main chaos test orchestration script + +# Exit on error +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +cd "${REPO_ROOT}" + +# Parse inputs from environment variables +DURATION=${DURATION_MINUTES:-120} +TX_RATE=${TX_RATE:-1000} +TX_SIZE=${TX_SIZE:-300} +NUM_PARTIES=${NUM_PARTIES:-4} +NUM_SHARDS=${NUM_SHARDS:-2} +CHAOS_ENABLED=${CHAOS_ENABLED:-true} + +# Export variables so child processes (like collect-results.sh) can access them +export DURATION TX_RATE TX_SIZE NUM_PARTIES NUM_SHARDS CHAOS_ENABLED + +# Calculate total transactions +TOTAL_TXS=$((DURATION * 60 * TX_RATE)) + +echo "==========================================" +echo "Chaos Test Configuration" +echo "==========================================" +echo "Duration: ${DURATION} minutes" +echo "TX Rate: ${TX_RATE} tx/s" +echo "TX Size: ${TX_SIZE} bytes" +echo "Total TXs: ${TOTAL_TXS}" +echo "Parties: ${NUM_PARTIES}" +echo "Shards: ${NUM_SHARDS}" +echo "Chaos Enabled: ${CHAOS_ENABLED}" +echo "==========================================" + +# Create temp directory for test +TEST_DIR=$(mktemp -d -t chaos-test-XXXXXX) +echo "Test directory: ${TEST_DIR}" + +# Generate config YAML +CONFIG_PATH="${TEST_DIR}/config.yaml" +echo "Generating config at ${CONFIG_PATH}..." + +cat > ${CONFIG_PATH} <> ${CONFIG_PATH} <> ${CONFIG_PATH} + done +done + +cat >> ${CONFIG_PATH} <> receiver${i}.log 2>&1 & + echo "Started receiver for party ${i} (PID: $!)" +done + +# Start loader (background) +echo "Starting loader..." +./bin/armageddon load \ + --config=${TEST_DIR}/config/party1/user_config.yaml \ + --transactions=${TOTAL_TXS} \ + --rate=${TX_RATE} \ + --txSize=${TX_SIZE} \ + >> loader.log 2>&1 & +LOADER_PID=$! +echo "Started loader (PID: ${LOADER_PID})" + +# Start chaos runner (if enabled) +if [ "$CHAOS_ENABLED" = "true" ]; then + echo "Starting chaos runner..." + "${SCRIPT_DIR}/chaos-runner.sh" "${TEST_DIR}" "${NUM_PARTIES}" "${NUM_SHARDS}" & + CHAOS_PID=$! + echo "Started chaos runner (PID: ${CHAOS_PID})" +fi + +# Monitor completion (with duration timeout) +echo "Monitoring test completion..." +"${SCRIPT_DIR}/monitor-completion.sh" "${NUM_PARTIES}" "${TOTAL_TXS}" "${TEST_DIR}" "${DURATION}" + +# Wait a bit for chaos runner to see the stop signal and exit gracefully +if [ "$CHAOS_ENABLED" = "true" ] && [ -n "$CHAOS_PID" ]; then + echo "Waiting for chaos runner to stop gracefully..." + sleep 5 + + # Check if it's still running and force kill if needed + if kill -0 ${CHAOS_PID} 2>/dev/null; then + echo "Force stopping chaos runner..." + kill ${CHAOS_PID} 2>/dev/null || true + else + echo "Chaos runner stopped gracefully" + fi +fi + +# Collect results +echo "Collecting results..." +"${SCRIPT_DIR}/collect-results.sh" "${TEST_DIR}" "${NUM_PARTIES}" "${DURATION}" + +# Cleanup processes +echo "Cleaning up processes..." +pkill -f "arma " || true +pkill -f "armageddon" || true + +echo "==========================================" +echo "✅ Chaos test completed successfully!" +echo "==========================================" \ No newline at end of file diff --git a/test/chaos/collect-results.sh b/test/chaos/collect-results.sh new file mode 100755 index 000000000..1c5c8b2bf --- /dev/null +++ b/test/chaos/collect-results.sh @@ -0,0 +1,253 @@ +#!/bin/bash +# Copyright the Hyperledger Fabric contributors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Collect test results and create summary + +TEST_DIR=$1 +NUM_PARTIES=$2 +DURATION=$3 + +echo "==========================================" +echo "Collecting Results" +echo "==========================================" + +# Create results directories +mkdir -p test-results/logs +mkdir -p test-results/statistics +mkdir -p test-results/summary + +# Determine log collection strategy based on test duration +if [ "$DURATION" -le 10 ]; then + LOG_MODE="full" + echo "Short test (${DURATION} min) - collecting all logs" +elif [ "$DURATION" -le 60 ]; then + LOG_MODE="standard" + echo "Medium test (${DURATION} min) - collecting compressed logs" +elif [ "$DURATION" -le 240 ]; then + LOG_MODE="errors_only" + echo "Long test (${DURATION} min) - collecting errors and summaries only" +else + LOG_MODE="minimal" + echo "Extended test (${DURATION} min) - collecting minimal artifacts" +fi + +# Extract statistics BEFORE compressing/moving logs +echo "Extracting statistics from logs..." + +# Extract loader statistics +if grep -q "Load command finished" loader.log 2>/dev/null; then + SENT=$(grep "Load command finished" loader.log 2>/dev/null | tail -1 | grep -oP 'sent \K[0-9]+') + LOADER_STATUS="completed" +else + SENT=$(grep -o "Sent [0-9]* transactions" loader.log 2>/dev/null | tail -1 | awk '{print $2}') + LOADER_STATUS="timeout" +fi + +# Extract receiver statistics +declare -a RECEIVER_STATS +declare -a RECEIVER_STATUS +for i in $(seq 1 $NUM_PARTIES); do + if grep -q "Receive command finished" receiver${i}.log 2>/dev/null; then + # Extract from "1800000 txs were expected and overall 1800186 were successfully received" example from the log + RECEIVED=$(grep "were successfully received" receiver${i}.log 2>/dev/null | tail -1 | grep -oP 'overall \K\d+(?= were successfully received)') + RECEIVER_STATS[$i]="${RECEIVED:-unknown}" + RECEIVER_STATUS[$i]="completed" + else + # For stopped receivers, check the statistics CSV file + if [ -f "${TEST_DIR}/output${i}/statistics.csv" ]; then + BLOCKS=$(tail -n +2 "${TEST_DIR}/output${i}/statistics.csv" 2>/dev/null | wc -l) + RECEIVED=$(tail -n +2 "${TEST_DIR}/output${i}/statistics.csv" 2>/dev/null | awk -F',' '{sum+=$3} END {print sum}') + RECEIVER_STATS[$i]="${RECEIVED:-0}" + RECEIVER_STATUS[$i]="timeout" + else + RECEIVER_STATS[$i]="0" + RECEIVER_STATUS[$i]="no_data" + fi + fi +done + +echo " Loader: ${SENT:-0} txs (${LOADER_STATUS})" +for i in $(seq 1 $NUM_PARTIES); do + echo " Party ${i}: ${RECEIVER_STATS[$i]} txs (${RECEIVER_STATUS[$i]})" +done + +# Collect component logs based on strategy +case "$LOG_MODE" in + full) + echo "Collecting all logs (uncompressed)..." + cp consenter*.log test-results/logs/ 2>/dev/null || true + cp batcher*.log test-results/logs/ 2>/dev/null || true + cp assembler*.log test-results/logs/ 2>/dev/null || true + cp router*.log test-results/logs/ 2>/dev/null || true + cp loader.log test-results/logs/ 2>/dev/null || true + cp receiver*.log test-results/logs/ 2>/dev/null || true + echo " Collected all logs without compression" + ;; + + standard) + echo "Collecting and compressing logs..." + cp consenter*.log test-results/logs/ 2>/dev/null || true + cp batcher*.log test-results/logs/ 2>/dev/null || true + cp assembler*.log test-results/logs/ 2>/dev/null || true + cp router*.log test-results/logs/ 2>/dev/null || true + cp loader.log test-results/logs/ 2>/dev/null || true + cp receiver*.log test-results/logs/ 2>/dev/null || true + + # Compress all logs + echo " Compressing logs..." + gzip test-results/logs/*.log 2>/dev/null || true + echo " Logs compressed with gzip" + ;; + + errors_only) + echo "Extracting errors and keeping essential logs..." + # Extract all errors/warnings to a single file (kept uncompressed for artifact upload) + grep -h -E "ERROR|WARN|PANIC|FATAL" *.log 2>/dev/null > test-results/logs/errors.log || true + + # Keep loader and receiver logs (they contain final statistics) + cp loader.log test-results/logs/ 2>/dev/null || true + cp receiver*.log test-results/logs/ 2>/dev/null || true + + # Compress loader/receiver logs, but keep errors.log uncompressed (workflow uploads errors.log) + gzip test-results/logs/loader.log test-results/logs/receiver*.log 2>/dev/null || true + echo " Collected errors and essential logs (compressed)" + ;; + + minimal) + echo "Collecting minimal artifacts..." + # Only critical errors (kept uncompressed for artifact upload) + grep -h -E "ERROR|PANIC|FATAL" *.log 2>/dev/null > test-results/logs/errors.log || true + + # First and last 100 lines of loader (shows start and completion) + head -n 100 loader.log 2>/dev/null > test-results/logs/loader_start.log || true + tail -n 100 loader.log 2>/dev/null > test-results/logs/loader_end.log || true + + # Last 50 lines of each receiver (shows completion status) + for i in $(seq 1 $NUM_PARTIES); do + if [ -f "receiver${i}.log" ]; then + tail -n 50 "receiver${i}.log" 2>/dev/null > "test-results/logs/receiver${i}_end.log" || true + fi + done + + # Compress everything except errors.log (workflow uploads errors.log as plain text) + for f in test-results/logs/*.log; do + [ -e "$f" ] || continue + [ "$(basename "$f")" = "errors.log" ] && continue + gzip "$f" 2>/dev/null || true + done + echo " Collected minimal artifacts (compressed)" + ;; +esac + +# Collect statistics from receivers +echo "Collecting statistics..." +for i in $(seq 1 $NUM_PARTIES); do + if [ -f "${TEST_DIR}/output${i}/statistics.csv" ]; then + cp "${TEST_DIR}/output${i}/statistics.csv" test-results/statistics/party${i}_statistics.csv + echo " Collected statistics for party ${i}" + fi +done + +# Create summary report +echo "Creating summary report..." +cat > test-results/summary/summary.txt <> test-results/summary/summary.txt + echo "Sent: ${SENT:-unknown} transactions" >> test-results/summary/summary.txt +else + echo "⏰ Loader stopped by timeout" >> test-results/summary/summary.txt + echo "Sent: ${SENT:-0} transactions (incomplete)" >> test-results/summary/summary.txt +fi + +cat >> test-results/summary/summary.txt <> test-results/summary/summary.txt + + RECEIVED="${RECEIVER_STATS[$i]}" + STATUS="${RECEIVER_STATUS[$i]}" + + if [ "$STATUS" = "completed" ]; then + echo " ✅ Completed - Received: ${RECEIVED} txs" >> test-results/summary/summary.txt + if [ -n "$RECEIVED" ] && [ "$RECEIVED" != "unknown" ] && [ "$RECEIVED" -gt 0 ] 2>/dev/null; then + TOTAL_RECEIVED=$((TOTAL_RECEIVED + RECEIVED)) + fi + elif [ "$STATUS" = "timeout" ]; then + # Get block count from CSV + if [ -f "${TEST_DIR}/output${i}/statistics.csv" ]; then + BLOCKS=$(tail -n +2 "${TEST_DIR}/output${i}/statistics.csv" 2>/dev/null | wc -l) + echo " ⏰ Stopped by timeout - Received: ${RECEIVED} txs in ${BLOCKS} blocks" >> test-results/summary/summary.txt + else + echo " ⏰ Stopped by timeout - Received: ${RECEIVED} txs" >> test-results/summary/summary.txt + fi + if [ -n "$RECEIVED" ] && [ "$RECEIVED" -gt 0 ] 2>/dev/null; then + TOTAL_RECEIVED=$((TOTAL_RECEIVED + RECEIVED)) + fi + else + echo " ❌ No statistics available" >> test-results/summary/summary.txt + fi +done + +cat >> test-results/summary/summary.txt <> test-results/summary/summary.txt +fi + +# Create a simple pass/fail indicator +echo "" >> test-results/summary/summary.txt +echo "========================================" >> test-results/summary/summary.txt +echo "Test Status" >> test-results/summary/summary.txt +echo "========================================" >> test-results/summary/summary.txt + +# For duration-based tests, we consider it successful if it ran for the full duration +# and processed transactions (even if not all completed) +if [ -n "$SENT" ] && [ "$SENT" -gt 0 ] && [ "$TOTAL_RECEIVED" -gt 0 ]; then + echo "✅ PASSED - Test ran for ${DURATION} minutes" >> test-results/summary/summary.txt + echo " Sent: ${SENT} txs, Received: ${TOTAL_RECEIVED} txs" >> test-results/summary/summary.txt +else + echo "❌ FAILED - No transactions processed" >> test-results/summary/summary.txt +fi + +echo "==========================================" +echo "✅ Results collected in test-results/" +echo "==========================================" + +# Display summary +cat test-results/summary/summary.txt diff --git a/test/chaos/monitor-completion.sh b/test/chaos/monitor-completion.sh new file mode 100755 index 000000000..6f868e8bc --- /dev/null +++ b/test/chaos/monitor-completion.sh @@ -0,0 +1,171 @@ +#!/bin/bash +# Copyright the Hyperledger Fabric contributors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Monitor test completion - wait for duration timeout OR all components finish + +NUM_PARTIES=$1 +TOTAL_TXS=$2 +TEST_DIR=$3 +DURATION_MINUTES=$4 + +# Calculate end time +START_TIME=$(date +%s) +END_TIME=$((START_TIME + DURATION_MINUTES * 60)) + +echo "==========================================" +echo "Monitoring Test Completion" +echo "==========================================" +echo "Test will run for ${DURATION_MINUTES} minutes" +echo "Expected TXs: ${TOTAL_TXS}" +echo "Start time: $(date -d @${START_TIME} '+%Y-%m-%d %H:%M:%S')" +echo "End time: $(date -d @${END_TIME} '+%Y-%m-%d %H:%M:%S')" +echo "==========================================" + +# Function to check if time limit reached +time_limit_reached() { + CURRENT_TIME=$(date +%s) + [ $CURRENT_TIME -ge $END_TIME ] +} + +# Function to get current stats +get_current_stats() { + echo "" + echo "==========================================" + echo "Current Status at $(date '+%Y-%m-%d %H:%M:%S')" + echo "==========================================" + + # Check loader + if grep -q "Load command finished" loader.log 2>/dev/null; then + SENT=$(grep "transactions were sent" loader.log 2>/dev/null | tail -1 | awk '{print $1}') + echo "Loader: ✅ Completed - Sent ${SENT:-unknown} txs" + else + SENT=$(grep -o "Sent [0-9]* transactions" loader.log 2>/dev/null | tail -1 | awk '{print $2}') + echo "Loader: 🔄 Running - Sent ${SENT:-0} txs so far" + fi + + # Check receivers + for i in $(seq 1 $NUM_PARTIES); do + if grep -q "Receive command finished" receiver${i}.log 2>/dev/null; then + # Extract from "X transactions were successfully received" + RECEIVED=$(grep "were successfully received" receiver${i}.log 2>/dev/null | tail -1 | grep -oP '^\d+') + echo "Party ${i}: ✅ Completed - Received ${RECEIVED:-unknown} txs" + else + # For running receivers, check CSV file for current count + if [ -f "${TEST_DIR}/output${i}/statistics.csv" ]; then + RECEIVED=$(tail -n +2 "${TEST_DIR}/output${i}/statistics.csv" 2>/dev/null | awk -F',' '{sum+=$3} END {print sum}') + echo "Party ${i}: 🔄 Running - Received ${RECEIVED:-0} txs so far" + else + echo "Party ${i}: 🔄 Running - Received 0 txs so far" + fi + fi + done + + CURRENT_TIME=$(date +%s) + ELAPSED=$((CURRENT_TIME - START_TIME)) + if [ $CURRENT_TIME -ge $END_TIME ]; then + REMAINING=0 + else + REMAINING=$((END_TIME - CURRENT_TIME)) + fi + echo "" + echo "Time elapsed: $((ELAPSED / 60)) minutes" + echo "Time remaining: $((REMAINING / 60)) minutes" + echo "==========================================" +} + +# Monitor with timeout +echo "Monitoring test progress..." +LAST_STATS_TIME=$START_TIME + +while true; do + CURRENT_TIME=$(date +%s) + + # Check if duration reached + if time_limit_reached; then + echo "" + echo "==========================================" + echo "⏰ Duration limit reached (${DURATION_MINUTES} minutes)" + echo "==========================================" + get_current_stats + break + fi + + # Check if all completed early + LOADER_DONE=false + ALL_RECEIVERS_DONE=true + + if grep -q "Load command finished" loader.log 2>/dev/null; then + LOADER_DONE=true + fi + + for i in $(seq 1 $NUM_PARTIES); do + if ! grep -q "Receive command finished" receiver${i}.log 2>/dev/null; then + ALL_RECEIVERS_DONE=false + break + fi + done + + if [ "$LOADER_DONE" = "true" ] && [ "$ALL_RECEIVERS_DONE" = "true" ]; then + echo "" + echo "==========================================" + echo "✅ All components completed before timeout!" + echo "==========================================" + get_current_stats + break + fi + + # Print stats every 5 minutes + if [ $((CURRENT_TIME - LAST_STATS_TIME)) -ge 300 ]; then + get_current_stats + LAST_STATS_TIME=$CURRENT_TIME + fi + + sleep 30 +done + +# Final statistics +echo "" +echo "==========================================" +echo "Final Test Statistics" +echo "==========================================" + +# Loader final stats +if grep -q "Load command finished" loader.log 2>/dev/null; then + SENT=$(grep "transactions were sent" loader.log 2>/dev/null | tail -1 | awk '{print $1}') + echo "Total Sent: ${SENT:-unknown} transactions" +else + SENT=$(grep -o "Sent [0-9]* transactions" loader.log 2>/dev/null | tail -1 | awk '{print $2}') + echo "Total Sent: ${SENT:-0} transactions (incomplete)" +fi + +echo "" +echo "Received per party:" +for i in $(seq 1 $NUM_PARTIES); do + if grep -q "Receive command finished" receiver${i}.log 2>/dev/null; then + RECEIVED=$(grep "were successfully received" receiver${i}.log 2>/dev/null | tail -1 | awk '{print $1}') + echo " Party ${i}: ${RECEIVED:-unknown} txs" + else + RECEIVED=$(grep -o "Received [0-9]* transactions" receiver${i}.log 2>/dev/null | tail -1 | awk '{print $2}') + echo " Party ${i}: ${RECEIVED:-0} txs (incomplete)" + fi +done + +echo "==========================================" + +# Signal chaos runner and other processes to stop +if [ -n "$TEST_DIR" ]; then + STOP_SIGNAL="${TEST_DIR}/chaos_stop_signal" + touch "${STOP_SIGNAL}" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] Created stop signal: ${STOP_SIGNAL}" +fi + +# Kill loader and receivers if still running +echo "Stopping loader and receivers..." +pkill -f "armageddon load" || true +pkill -f "armageddon receive" || true + +echo "==========================================" +echo "✅ Monitoring completed" +echo "==========================================" diff --git a/test/chaos/start-arma-network.sh b/test/chaos/start-arma-network.sh new file mode 100755 index 000000000..996d5f0bb --- /dev/null +++ b/test/chaos/start-arma-network.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright the Hyperledger Fabric contributors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Start all ARMA network components + +TEST_DIR=$1 +NUM_PARTIES=$2 +NUM_SHARDS=$3 + +echo "==========================================" +echo "Starting ARMA Network" +echo "==========================================" +echo "Parties: ${NUM_PARTIES}" +echo "Shards: ${NUM_SHARDS}" +echo "==========================================" + +# Create PID directory +PID_DIR="${TEST_DIR}/pids" +mkdir -p ${PID_DIR} +echo "PID directory: ${PID_DIR}" + +# Save current working directory for later use +WORK_DIR=$(pwd) +echo "${WORK_DIR}" > ${PID_DIR}/work_dir.txt +echo "Working directory: ${WORK_DIR}" + +# Start consenters first +echo "Starting consenters..." +for i in $(seq 1 $NUM_PARTIES); do + ./bin/arma consensus \ + --config=${TEST_DIR}/config/party${i}/local_config_consenter.yaml \ + >> consenter${i}.log 2>&1 & + PID=$! + echo ${PID} > ${PID_DIR}/consensus${i}.pid + echo " Started consenter ${i} (PID: ${PID})" +done + +# Wait for consenters to initialize +echo "Waiting 5 seconds for consenters to initialize..." +sleep 5 + +# Start batchers +echo "Starting batchers..." +for i in $(seq 1 $NUM_PARTIES); do + for j in $(seq 1 $NUM_SHARDS); do + ./bin/arma batcher \ + --config=${TEST_DIR}/config/party${i}/local_config_batcher${j}.yaml \ + >> batcher${i}-${j}.log 2>&1 & + PID=$! + echo ${PID} > ${PID_DIR}/batcher${i}-${j}.pid + echo " Started batcher ${i}-${j} (PID: ${PID})" + done +done + +# Start assemblers +echo "Starting assemblers..." +for i in $(seq 1 $NUM_PARTIES); do + ./bin/arma assembler \ + --config=${TEST_DIR}/config/party${i}/local_config_assembler.yaml \ + >> assembler${i}.log 2>&1 & + PID=$! + echo ${PID} > ${PID_DIR}/assembler${i}.pid + echo " Started assembler ${i} (PID: ${PID})" +done + +# Start routers +echo "Starting routers..." +for i in $(seq 1 $NUM_PARTIES); do + ./bin/arma router \ + --config=${TEST_DIR}/config/party${i}/local_config_router.yaml \ + >> router${i}.log 2>&1 & + PID=$! + echo ${PID} > ${PID_DIR}/router${i}.pid + echo " Started router ${i} (PID: ${PID})" +done + +# Wait for network to stabilize +echo "Waiting 10 seconds for network to stabilize..." +sleep 10 + +echo "==========================================" +echo "✅ ARMA network started successfully" +echo "=========================================="