Skip to content

Reorganize test execution from language-by-language to test-by-test with matrix output #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rijesha merged 6 commits into from
Nov 6, 2025
Merged

Reorganize test execution from language-by-language to test-by-test with matrix output #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b56365a
Initial plan
Copilot Nov 6, 2025
25e44d3
Refactor test execution to run test-by-test across all languages
Copilot Nov 6, 2025
cac718f
Reduce repetitive output in cross-platform pipe tests
Copilot Nov 6, 2025
95afbce
Update test documentation to reflect new test-by-test execution struc…
Copilot Nov 6, 2025
cc9475c
Address code review comments: use constants and improve formatting
Copilot Nov 6, 2025
a39227a
Simplify summary and clarify test section names
Copilot Nov 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions tests/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,30 @@ python tests/run_tests.py

## Test Output Format

Tests follow a consistent format across all languages:
The test runner provides a clean, organized output showing test results by test type across all languages:

```
[TEST START] <Language> <Test Type>
[TEST END] <Language> <Test Type>: PASS/FAIL
🔧 CODE GENERATION
C: ✅ PASS
TS: ✅ PASS
PY: ✅ PASS
CPP: ✅ PASS

🧪 Basic Types Tests
C: ✅ PASS
TS: ✅ PASS
PY: ✅ PASS
CPP: ✅ PASS
```

Tests are now organized by **test type** rather than by language, providing a clearer view of functionality across all languages. Each test type (basic types, arrays, serialization) runs for all languages before moving to the next test type.

Individual test programs follow this format:
- **Success**: Only start and end messages are printed
- **Failure**: Detailed failure information including expected values, actual values, and raw data hex dump

**Verbose Mode**: Use `--verbose` or `-v` flag to see detailed output for all operations, including successful ones.

## Test Types

### 1. Basic Types Test
Expand Down
79 changes: 32 additions & 47 deletions tests/cross_platform_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,8 @@ def test_cross_platform(self, encoder_lang: str, decoder_lang: str, framed: bool
test_name = f"{encoder_lang}→{decoder_lang} ({mode})"
self.total_tests += 1

self.log(f"Testing {test_name}...")
if self.verbose:
self.log(f"Testing {test_name}...")

# Encode
binary_data = self.run_encoder(encoder_lang, framed)
Expand All @@ -230,36 +231,38 @@ def test_cross_platform(self, encoder_lang: str, decoder_lang: str, framed: bool
if not success:
self.log(f"{test_name}: Decoder failed", "ERROR")
# Print debugging information on failure
print(f"\n 🔍 Failure Details:")
print(f" Encoded by: {encoder_lang}")
print(f" Decoded by: {decoder_lang}")
print(f" Raw data length: {len(binary_data)} bytes")
print(f" Raw data (hex): {binary_data.hex()}")
if binary_data:
print(f" Raw data (bytes): {list(binary_data)}")
if self.verbose:
print(f"\n 🔍 Failure Details:")
print(f" Encoded by: {encoder_lang}")
print(f" Decoded by: {decoder_lang}")
print(f" Raw data length: {len(binary_data)} bytes")
print(f" Raw data (hex): {binary_data.hex()}")
if binary_data:
print(f" Raw data (bytes): {list(binary_data)}")
self.failed_tests += 1
self.results[test_name] = False
return False

# Verify
if decoded_data and self.verify_decoded_data(decoded_data):
self.log(f"{test_name}: PASS", "SUCCESS")
if self.verbose:
self.log(f"{test_name}: PASS", "SUCCESS")
self.log(f"Decoded data: {decoded_data}", "INFO")
self.passed_tests += 1
self.results[test_name] = True
return True
else:
self.log(f"{test_name}: Verification failed", "ERROR")
# Print debugging information on verification failure
print(f"\n 🔍 Verification Failure Details:")
print(f" Encoded by: {encoder_lang}")
print(f" Decoded by: {decoder_lang}")
print(f" Decoded data: {decoded_data}")
print(f" Raw data length: {len(binary_data)} bytes")
print(f" Raw data (hex): {binary_data.hex()}")
if binary_data:
print(f" Raw data (bytes): {list(binary_data)}")
if self.verbose:
print(f"\n 🔍 Verification Failure Details:")
print(f" Encoded by: {encoder_lang}")
print(f" Decoded by: {decoder_lang}")
print(f" Decoded data: {decoded_data}")
print(f" Raw data length: {len(binary_data)} bytes")
print(f" Raw data (hex): {binary_data.hex()}")
if binary_data:
print(f" Raw data (bytes): {list(binary_data)}")
self.failed_tests += 1
self.results[test_name] = False
return False
Expand Down Expand Up @@ -297,10 +300,6 @@ def check_language_available(self, language: str, mode: str = "framed") -> bool:

def run_all_tests(self, test_struct=True, test_framed=True):
"""Run all cross-platform tests"""
print("="*60)
print("CROSS-PLATFORM PIPE TEST")
print("="*60)

# Detect available languages
available_languages = []
for lang in ["c", "python", "typescript"]:
Expand All @@ -309,56 +308,42 @@ def run_all_tests(self, test_struct=True, test_framed=True):
self.log(
f"{lang.capitalize()} encoder/decoder available", "SUCCESS")
else:
self.log(
f"{lang.capitalize()} encoder/decoder not available", "SKIP")
if self.verbose:
self.log(
f"{lang.capitalize()} encoder/decoder not available", "SKIP")

if len(available_languages) == 0:
self.log("No languages available for testing!", "ERROR")
return False

# Test struct mode (no framing)
# Test struct mode (no framing) - currently not implemented
if test_struct:
print("\n" + "="*60)
print("STRUCT-BASED TESTS (NO FRAMING)")
print("="*60)
self.log("Struct-based tests are currently NOT IMPLEMENTED", "WARNING")
self.log(
"This is due to encoder/decoder implementations not being complete", "WARNING")
self.log("for all languages in struct mode.", "WARNING")
self.log("TEST FAILED: Struct-based tests not implemented", "ERROR")
if self.verbose:
print("\nStruct-based tests: NOT IMPLEMENTED")
self.log("Struct-based tests are currently not implemented", "WARNING")
self.log(
"This is due to encoder/decoder implementations not being complete", "WARNING")
self.log("for all languages in struct mode.", "WARNING")

# Test framed mode
if test_framed:
print("\n" + "="*60)
print("FRAMED TESTS (WITH HEADERS/CHECKSUMS)")
print("="*60)

# Test each language encoding to all languages
for encoder_lang in available_languages:
for decoder_lang in available_languages:
self.test_cross_platform(
encoder_lang, decoder_lang, framed=True)

# Print summary
print("\n" + "="*60)
print("TEST RESULTS SUMMARY")
print("="*60)
print(f"Total tests: {self.total_tests}")
print(f"Passed: {self.passed_tests}")
print(f"Failed: {self.failed_tests}")

print()
if self.total_tests > 0:
pass_rate = (self.passed_tests / self.total_tests) * 100
print(f"Pass rate: {pass_rate:.1f}%")
print(f"Pipe tests: {self.passed_tests}/{self.total_tests} passed ({pass_rate:.1f}%)")

if pass_rate == 100:
print("ALL TESTS PASSED!")
return True
elif pass_rate >= 80:
print("MOST TESTS PASSED")
return True
else:
print("SOME TESTS FAILED")
return False
else:
print("NO TESTS RUN")
Expand Down
Loading