fix(compress): UTF-8 stdout, empty/identical guards, inline-code validation, frontmatter cleanup

Five user-contributed fixes consolidated:

* UTF-8 stdout (#289 by @ousamabenyounes) — reconfigure stdout/stderr at the
  top of cli.py so Windows cp1252 consoles don't crash on the ❌ glyph in
  error/validation branches and mask the real error.
* Empty / unchanged compression guards (#292 by @ousamabenyounes, closes #237)
  — refuse empty input, refuse empty/whitespace/identical Claude output,
  read back the backup before touching the input. Five regression tests in
  tests/test_compress_safety.py.
* Inline backtick validation (#309 by @hireblackout) — validate_inline_codes
  closes the silent-overwrite gap where `npm install` → `yarn install`
  passed validation. Wired into validate(); 11 unit tests in
  tests/test_validate_inline.py.
* Frontmatter angle-bracket fix (#268 by @Bortlesboat, closes #266) —
  caveman-compress/SKILL.md description now uses FILEPATH instead of
  <filepath>, plus verify_repo gains a new
  verify_skill_frontmatter_upload_compatibility check, UTF-8 hardening for
  Windows, and the activation-banner regex no longer requires a trailing
  period.
* Two test fixtures (claude-md-project.md, mixed-with-code.md) updated so
  the new inline-backtick validator passes — they were silently dropping
  `server/src/`, `type(scope): description`, and `status` references in
  compression. The fixture is documentation of "good" compression, so the
  fix is to preserve those references.

Co-Authored-By: Ben Younes <ousama.benyounes@oratelecom.net>
Co-Authored-By: hireblackout <hireblackout@users.noreply.github.com>
Co-Authored-By: Andrew Barnes <Bortlesboat@users.noreply.github.com>
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: 5 people

caveman-compress/SKILL.md

@@ -4,7 +4,7 @@ description: >
   Compress natural language memory files (CLAUDE.md, todos, preferences) into caveman format
   to save input tokens. Preserves all technical substance, code, URLs, and structure.
   Compressed version overwrites the original file. Human-readable backup saved as FILE.original.md.
-  Trigger: /caveman:compress <filepath> or "compress memory file"
+  Trigger: /caveman:compress FILEPATH or "compress memory file"
 ---
 
 # Caveman Compress

caveman-compress/scripts/cli.py

@@ -7,6 +7,18 @@
 """
 
 import sys
+
+# Force UTF-8 on stdout/stderr before any code can print. Windows consoles
+# default to cp1252 and crash on the ❌ glyphs in error/validation branches,
+# masking the real error and leaving the user with a half-compressed file.
+for _stream in (sys.stdout, sys.stderr):
+    reconfigure = getattr(_stream, "reconfigure", None)
+    if callable(reconfigure):
+        try:
+            reconfigure(encoding="utf-8", errors="replace")
+        except Exception:
+            pass
+
 from pathlib import Path
 
 from .compress import compress_file

caveman-compress/scripts/compress.py

@@ -182,6 +182,10 @@ def compress_file(filepath: Path) -> bool:
     original_text = filepath.read_text(errors="ignore")
     backup_path = filepath.with_name(filepath.stem + ".original.md")
 
+    if not original_text.strip():
+        print("❌ Refusing to compress: file is empty or whitespace-only.")
+        return False
+
     # Check if backup already exists to prevent accidental overwriting
     if backup_path.exists():
         print(f"⚠️ Backup file already exists: {backup_path}")
@@ -193,8 +197,31 @@ def compress_file(filepath: Path) -> bool:
     print("Compressing with Claude...")
     compressed = call_claude(build_compress_prompt(original_text))
 
-    # Save original as backup, write compressed to original path
+    if compressed is None or not compressed.strip():
+        print("❌ Compression aborted: Claude returned an empty response.")
+        print("   Original file is untouched (no backup created).")
+        return False
+
+    if compressed.strip() == original_text.strip():
+        print("❌ Compression aborted: output is identical to input.")
+        print("   Likely causes: Claude refused, returned the prompt verbatim, or the file is")
+        print("   already in caveman form. Original file is untouched (no backup created).")
+        return False
+
+    # Save original as backup, then verify the backup readback before
+    # touching the input file. If the filesystem dropped bytes (encoding,
+    # antivirus, disk full), unlink the bad backup and abort instead of
+    # leaving the user with a corrupt backup + compressed primary.
     backup_path.write_text(original_text)
+    backup_readback = backup_path.read_text(errors="ignore")
+    if backup_readback != original_text:
+        print(f"❌ Backup write verification failed: {backup_path}")
+        print("   In-memory original differs from on-disk backup. Aborting before touching the input file.")
+        try:
+            backup_path.unlink()
+        except OSError:
+            pass
+        return False
     filepath.write_text(compressed)
 
     # Step 2: Validate + Retry

caveman-compress/scripts/validate.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import re
+from collections import Counter
 from pathlib import Path
 
 URL_REGEX = re.compile(r"https?://[^\s)]+")
@@ -93,6 +94,12 @@ def count_bullets(text):
     return len(BULLET_REGEX.findall(text))
 
 
+def extract_inline_codes(text):
+    text_without_fences = re.sub(r"^```[\s\S]*?^```", "", text, flags=re.MULTILINE)
+    text_without_fences = re.sub(r"^~~~[\s\S]*?^~~~", "", text_without_fences, flags=re.MULTILINE)
+    return re.findall(r"`([^`]+)`", text_without_fences)
+
+
 # ---------- Validators ----------
 
 
@@ -144,6 +151,22 @@ def validate_bullets(orig, comp, result):
         result.add_warning(f"Bullet count changed too much: {b1} -> {b2}")
 
 
+def validate_inline_codes(orig, comp, result):
+    c1 = Counter(extract_inline_codes(orig))
+    c2 = Counter(extract_inline_codes(comp))
+
+    if c1 != c2:
+        lost = set(c1.keys()) - set(c2.keys())
+        added = set(c2.keys()) - set(c1.keys())
+        for code, count in c1.items():
+            if code in c2 and c2[code] < count:
+                lost.add(f"{code} (lost {count - c2[code]} of {count} occurrences)")
+        if lost:
+            result.add_error(f"Inline code lost: {lost}")
+        if added:
+            result.add_warning(f"Inline code added: {added}")
+
+
 # ---------- Main ----------
 
 
@@ -158,6 +181,7 @@ def validate(original_path: Path, compressed_path: Path) -> ValidationResult:
     validate_urls(orig, comp, result)
     validate_paths(orig, comp, result)
     validate_bullets(orig, comp, result)
+    validate_inline_codes(orig, comp, result)
 
     return result
 

plugins/caveman/skills/compress/scripts/cli.py

@@ -7,6 +7,18 @@
 """
 
 import sys
+
+# Force UTF-8 on stdout/stderr before any code can print. Windows consoles
+# default to cp1252 and crash on the ❌ glyphs in error/validation branches,
+# masking the real error and leaving the user with a half-compressed file.
+for _stream in (sys.stdout, sys.stderr):
+    reconfigure = getattr(_stream, "reconfigure", None)
+    if callable(reconfigure):
+        try:
+            reconfigure(encoding="utf-8", errors="replace")
+        except Exception:
+            pass
+
 from pathlib import Path
 
 from .compress import compress_file

plugins/caveman/skills/compress/scripts/compress.py

@@ -182,6 +182,10 @@ def compress_file(filepath: Path) -> bool:
     original_text = filepath.read_text(errors="ignore")
     backup_path = filepath.with_name(filepath.stem + ".original.md")
 
+    if not original_text.strip():
+        print("❌ Refusing to compress: file is empty or whitespace-only.")
+        return False
+
     # Check if backup already exists to prevent accidental overwriting
     if backup_path.exists():
         print(f"⚠️ Backup file already exists: {backup_path}")
@@ -193,8 +197,31 @@ def compress_file(filepath: Path) -> bool:
     print("Compressing with Claude...")
     compressed = call_claude(build_compress_prompt(original_text))
 
-    # Save original as backup, write compressed to original path
+    if compressed is None or not compressed.strip():
+        print("❌ Compression aborted: Claude returned an empty response.")
+        print("   Original file is untouched (no backup created).")
+        return False
+
+    if compressed.strip() == original_text.strip():
+        print("❌ Compression aborted: output is identical to input.")
+        print("   Likely causes: Claude refused, returned the prompt verbatim, or the file is")
+        print("   already in caveman form. Original file is untouched (no backup created).")
+        return False
+
+    # Save original as backup, then verify the backup readback before
+    # touching the input file. If the filesystem dropped bytes (encoding,
+    # antivirus, disk full), unlink the bad backup and abort instead of
+    # leaving the user with a corrupt backup + compressed primary.
     backup_path.write_text(original_text)
+    backup_readback = backup_path.read_text(errors="ignore")
+    if backup_readback != original_text:
+        print(f"❌ Backup write verification failed: {backup_path}")
+        print("   In-memory original differs from on-disk backup. Aborting before touching the input file.")
+        try:
+            backup_path.unlink()
+        except OSError:
+            pass
+        return False
     filepath.write_text(compressed)
 
     # Step 2: Validate + Retry

plugins/caveman/skills/compress/scripts/validate.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import re
+from collections import Counter
 from pathlib import Path
 
 URL_REGEX = re.compile(r"https?://[^\s)]+")
@@ -93,6 +94,12 @@ def count_bullets(text):
     return len(BULLET_REGEX.findall(text))
 
 
+def extract_inline_codes(text):
+    text_without_fences = re.sub(r"^```[\s\S]*?^```", "", text, flags=re.MULTILINE)
+    text_without_fences = re.sub(r"^~~~[\s\S]*?^~~~", "", text_without_fences, flags=re.MULTILINE)
+    return re.findall(r"`([^`]+)`", text_without_fences)
+
+
 # ---------- Validators ----------
 
 
@@ -144,6 +151,22 @@ def validate_bullets(orig, comp, result):
         result.add_warning(f"Bullet count changed too much: {b1} -> {b2}")
 
 
+def validate_inline_codes(orig, comp, result):
+    c1 = Counter(extract_inline_codes(orig))
+    c2 = Counter(extract_inline_codes(comp))
+
+    if c1 != c2:
+        lost = set(c1.keys()) - set(c2.keys())
+        added = set(c2.keys()) - set(c1.keys())
+        for code, count in c1.items():
+            if code in c2 and c2[code] < count:
+                lost.add(f"{code} (lost {count - c2[code]} of {count} occurrences)")
+        if lost:
+            result.add_error(f"Inline code lost: {lost}")
+        if added:
+            result.add_warning(f"Inline code added: {added}")
+
+
 # ---------- Main ----------
 
 
@@ -158,6 +181,7 @@ def validate(original_path: Path, compressed_path: Path) -> ValidationResult:
     validate_urls(orig, comp, result)
     validate_paths(orig, comp, result)
     validate_bullets(orig, comp, result)
+    validate_inline_codes(orig, comp, result)
 
     return result
 

skills/compress/scripts/cli.py

@@ -7,6 +7,18 @@
 """
 
 import sys
+
+# Force UTF-8 on stdout/stderr before any code can print. Windows consoles
+# default to cp1252 and crash on the ❌ glyphs in error/validation branches,
+# masking the real error and leaving the user with a half-compressed file.
+for _stream in (sys.stdout, sys.stderr):
+    reconfigure = getattr(_stream, "reconfigure", None)
+    if callable(reconfigure):
+        try:
+            reconfigure(encoding="utf-8", errors="replace")
+        except Exception:
+            pass
+
 from pathlib import Path
 
 from .compress import compress_file

skills/compress/scripts/compress.py

@@ -182,6 +182,10 @@ def compress_file(filepath: Path) -> bool:
     original_text = filepath.read_text(errors="ignore")
     backup_path = filepath.with_name(filepath.stem + ".original.md")
 
+    if not original_text.strip():
+        print("❌ Refusing to compress: file is empty or whitespace-only.")
+        return False
+
     # Check if backup already exists to prevent accidental overwriting
     if backup_path.exists():
         print(f"⚠️ Backup file already exists: {backup_path}")
@@ -193,8 +197,31 @@ def compress_file(filepath: Path) -> bool:
     print("Compressing with Claude...")
     compressed = call_claude(build_compress_prompt(original_text))
 
-    # Save original as backup, write compressed to original path
+    if compressed is None or not compressed.strip():
+        print("❌ Compression aborted: Claude returned an empty response.")
+        print("   Original file is untouched (no backup created).")
+        return False
+
+    if compressed.strip() == original_text.strip():
+        print("❌ Compression aborted: output is identical to input.")
+        print("   Likely causes: Claude refused, returned the prompt verbatim, or the file is")
+        print("   already in caveman form. Original file is untouched (no backup created).")
+        return False
+
+    # Save original as backup, then verify the backup readback before
+    # touching the input file. If the filesystem dropped bytes (encoding,
+    # antivirus, disk full), unlink the bad backup and abort instead of
+    # leaving the user with a corrupt backup + compressed primary.
     backup_path.write_text(original_text)
+    backup_readback = backup_path.read_text(errors="ignore")
+    if backup_readback != original_text:
+        print(f"❌ Backup write verification failed: {backup_path}")
+        print("   In-memory original differs from on-disk backup. Aborting before touching the input file.")
+        try:
+            backup_path.unlink()
+        except OSError:
+            pass
+        return False
     filepath.write_text(compressed)
 
     # Step 2: Validate + Retry

skills/compress/scripts/validate.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import re
+from collections import Counter
 from pathlib import Path
 
 URL_REGEX = re.compile(r"https?://[^\s)]+")
@@ -93,6 +94,12 @@ def count_bullets(text):
     return len(BULLET_REGEX.findall(text))
 
 
+def extract_inline_codes(text):
+    text_without_fences = re.sub(r"^```[\s\S]*?^```", "", text, flags=re.MULTILINE)
+    text_without_fences = re.sub(r"^~~~[\s\S]*?^~~~", "", text_without_fences, flags=re.MULTILINE)
+    return re.findall(r"`([^`]+)`", text_without_fences)
+
+
 # ---------- Validators ----------
 
 
@@ -144,6 +151,22 @@ def validate_bullets(orig, comp, result):
         result.add_warning(f"Bullet count changed too much: {b1} -> {b2}")
 
 
+def validate_inline_codes(orig, comp, result):
+    c1 = Counter(extract_inline_codes(orig))
+    c2 = Counter(extract_inline_codes(comp))
+
+    if c1 != c2:
+        lost = set(c1.keys()) - set(c2.keys())
+        added = set(c2.keys()) - set(c1.keys())
+        for code, count in c1.items():
+            if code in c2 and c2[code] < count:
+                lost.add(f"{code} (lost {count - c2[code]} of {count} occurrences)")
+        if lost:
+            result.add_error(f"Inline code lost: {lost}")
+        if added:
+            result.add_warning(f"Inline code added: {added}")
+
+
 # ---------- Main ----------
 
 
@@ -158,6 +181,7 @@ def validate(original_path: Path, compressed_path: Path) -> ValidationResult:
     validate_urls(orig, comp, result)
     validate_paths(orig, comp, result)
     validate_bullets(orig, comp, result)
+    validate_inline_codes(orig, comp, result)
 
     return result