diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 743372ba..8cf86080 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -334,6 +334,29 @@ That feature list is exactly what CI's Windows vintage leg builds (`.github/work The definitive check is an actual `rustup toolchain install 1.73.0` build of that manifest; the command above is the cheap proxy that catches the wiring mistakes (wrong shim path, raw `io::Error::other` left in, unused imports). +### The pipeline builds more than your machine does + +A green `cargo test` and a green `rb-regress run` are not a green pipeline. +Run this before pushing: + +```bash +scripts/preflight.sh +``` + +It runs what CI runs: `cargo test --release` (CI's test step is release, not +debug), the MiSTer feature set (`optical` on, `native-zstd` off — anything +touching zstd must go through `crate::rbformats::zstd_compat`, never the `zstd` +crate directly), the Rust 1.73 vintage floor, and the doc-parity tests. + +**After pushing, read the job list, not the run conclusion.** Several jobs are +`continue-on-error: true` — the MiSTer `rb-cli-mini` build among them — so they +can fail while `gh run list` still reports the run as `success`. That hid a +compile error for a day. + +```bash +gh run view +``` + ### mrustc mis-lowers `leading_zeros` / `leading_ones` on narrow integers The PowerPC build (`rb-cli-ppc`, via mrustc's C backend — see `scripts/build-ppc.sh`) diff --git a/Cargo.lock b/Cargo.lock index 29f8f102..cc99c218 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,15 +121,6 @@ dependencies = [ "winit", ] -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - [[package]] name = "adler" version = "1.0.2" @@ -670,21 +661,6 @@ dependencies = [ "fs_extra", ] -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - [[package]] name = "base16ct" version = "0.2.0" @@ -1583,15 +1559,6 @@ dependencies = [ "cmov", ] -[[package]] -name = "cue_sheet" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a966cdf989dd50248a838806be7d6317466eb4772bd1f7725f84dda2068db24" -dependencies = [ - "error-chain", -] - [[package]] name = "cursor-icon" version = "1.2.0" @@ -2064,16 +2031,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "backtrace", - "version_check", -] - [[package]] name = "error-code" version = "3.3.2" @@ -2467,12 +2424,6 @@ dependencies = [ "weezl", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "gio-sys" version = "0.18.1" @@ -4250,15 +4201,6 @@ dependencies = [ "objc2-foundation 0.2.2", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "oboe" version = "0.6.1" @@ -4302,11 +4244,10 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opticaldiscs" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23983217ba7ba56e3b8a9bda8c3e7bf7dec9cc2ee279502063dc422f79f90906" +checksum = "34cfeb30809c554cc80c87b8a5314956bfc0fe969d758795d2b98141fe46cc85" dependencies = [ - "cue_sheet", "encoding_rs", "flate2", "libc", @@ -5336,12 +5277,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189" -[[package]] -name = "rustc-demangle" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -5499,6 +5434,7 @@ dependencies = [ "egui", "encoding_rs", "env_logger", + "filetime", "flate2", "globset", "image", diff --git a/Cargo.toml b/Cargo.toml index 6e5970ea..f9cc23e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -203,7 +203,7 @@ webbrowser = { version = "1.0", optional = true } # 0.13.0 — `physical` / `open_physical`: reading a disc from the drive as flat # cooked sectors, which is what makes DVD and Blu-ray work at all # (the CD-only MMC ioctls fail on that media with ENOTTY). -opticaldiscs = { version = "0.13.0", features = ["drives"], optional = true } +opticaldiscs = { version = "0.15.0", features = ["drives"], optional = true } cd-da-reader = { git = "https://github.com/danifunker/rust-cd-da-reader", branch = "file-backend-on-1.0", optional = true } # In-app CD-DA playback (Optical tab). Pinned <0.20 for the # OutputStream::try_default / Sink::try_new API used in gui/optical_audio.rs. @@ -251,6 +251,10 @@ flate2 = { version = "1", default-features = false, features = ["rust_backend"] # produces the standard LZ4 frame, interoperable with the DOS tool's liblz4. lz4_flex = "0.13" tar = "0.4" +# Set an extracted file's mtime/atime after we write it (the plain host-fs +# export path). Already a transitive dep via `tar` in every lockfile — this +# just promotes it so we can call `set_file_mtime` directly. +filetime = "0.2" bzip2 = "0.6" zip = { version = "8", default-features = false, features = ["deflate", "aes-crypto"] } base64 = "0.22" diff --git a/README.md b/README.md index 44671298..2de8c702 100644 --- a/README.md +++ b/README.md @@ -685,7 +685,9 @@ PC Engine CD, CD32, GameCube, Wii, CD-i, and 3DO. | SGI | Yes | Yes | SGI Volume Header (IRIX). 16 fixed slots; checksum recomputed on every write; geometry (`vh_dp`) preserved across edits. `rb-cli new hd sgi-efs` synthesizes a dvh + EFS-root hard disk from scratch (IRIX 5.3-6.5). | | AHDI | Yes | No (browse); writes whole tables from scratch | Atari ST / TT / Falcon hard disks. Four primary entries at 0x1C6 plus XGM extended chains, big-endian, no magic number — detection keys off the 0x1234 word-sum and plausible geometry. `rb-cli new hd atari` writes a fresh root sector with the tags you name (GEM / BGM / RAW); a GEM partition over 16 MiB is promoted to BGM, which is what TOS needs. Creating an XGM chain, and grafting in a bootable bootstrap, are future work. | | Sun | Yes | No (browse); writes whole labels from scratch | Sun disk label / SMI VTOC (SPARC Solaris / SunOS). 8 big-endian slices (magic `0xDABE`), geometry-derived offsets; the whole-disk "backup" slice is excluded from the list. Surfaces the UFS slices to the existing big-endian-SPARC UFS reader (browse / inspect / extract). `rb-cli new hd sun` writes a fresh label with the slice tags you name (`root`, `usr`, `swap`, … or a bare tag number), cylinder-aligned from `--heads` / `--sectors`, with slice 2 reserved for the whole-disk alias. Parser and writer both cross-validated against `fdisk` / `sfdisk`; editing an existing label and full-disk backup are future work. | -| None (superfloppy) | Yes — auto-detects the filesystem at sector 0 (FAT / NTFS / exFAT / ext / XFS / JFS / UFS / ReiserFS / btrfs / SquashFS / HFS / HFS+ / APFS / Amiga SFS / Apple DOS 3.3 / CBM DOS / Atari DOS / RS-DOS / OS-9 RBF / DragonDOS / Acorn DFS / ADFS / TR-DOS / TI-99 / QDOS / Human68k / Alto BFS / Pilot/Cedar / Apple Lisa FS / …) | — | Standard floppy / disk sizes are recognised even without a partition table. Xerox Alto packs (`.pdi` / `.bfs` / CopyDisk / Salto `.dsk`), Pilot/Cedar PDIs (`fsFamily=2`), Dwarf 6085 `.zdisk` images, and tag-bearing Apple Lisa DiskCopy 4.2 / DART disks are detected by content and presented as a single browsable volume. | +| X68k | Yes | No (browse); writes whole tables from scratch | Sharp X68000 SASI/SCSI hard disks — Human68k's native scheme. 16-byte header plus 8 entries at byte 2048, big-endian, no magic number. Both geometries are auto-detected: SCSI (`X68SCSI1`, table at 0x800, 1024-byte sectors) and SASI (table at 0x400, 256-byte sectors), including custom-IPL game disks. `rb-cli new hd x68k` synthesizes a bootable disk with the Sharp IPL signature and a Human68k FAT volume. | +| DSD | Yes | — (fixed floppy geometry) | Double-sided Acorn DFS (`.dsd`). Not a table on the disk: the two sides are stored track-interleaved, so the reader de-interleaves them and this scheme presents them as **two** Acorn DFS partitions — side 0 at byte 0, side 1 at half the image. Edits to either side re-interleave on save. | +| None (superfloppy) | Yes — auto-detects the filesystem at sector 0 (FAT / NTFS / exFAT / HPFS / ext / XFS / JFS / UFS / ReiserFS / btrfs / SquashFS / HFS / HFS+ / APFS / Amiga SFS / Apple DOS 3.3 / CBM DOS / Atari DOS / RS-DOS / OS-9 RBF / DragonDOS / Acorn DFS / ADFS / TR-DOS / TI-99 / QDOS / Human68k / Alto BFS / Pilot/Cedar / Apple Lisa FS / …) | — | Standard floppy / disk sizes are recognised even without a partition table. Xerox Alto packs (`.pdi` / `.bfs` / CopyDisk / Salto `.dsk`), Pilot/Cedar PDIs (`fsFamily=2`), Dwarf 6085 `.zdisk` images, and tag-bearing Apple Lisa DiskCopy 4.2 / DART disks are detected by content and presented as a single browsable volume. | The Clonezilla image format is also parsed as a source (MBR, GPT, partclone images, partition table sidecars) for restore — see `docs/clonezilla.md`. diff --git a/docs/RESUME-regression-fixes.md b/docs/RESUME-regression-fixes.md new file mode 100644 index 00000000..cf81d495 --- /dev/null +++ b/docs/RESUME-regression-fixes.md @@ -0,0 +1,214 @@ +# Resume: regression fixes + +Paste this into a fresh session to continue. + +--- + +Continuing regression fixes on rusty-backup (branch: `regression-fixes`, +pushed and verified on all three hosts at `c6e66fd`). + +## STATE + +- Suite: **259 pass / 19 xfail / 0 fail**, zero XPASS, on Windows, macOS and + Linux — all three measured at `93a6d53`, with the OS/2 fixture present on + all three. +- **Counts below this line were last true on 2026-08-09 and are not now.** As + of 2026-08-14 the live tally in + [`Regression_Bugs.md`](Regression_Bugs.md) is 31 fixed, 2 not-a-defect, + 2 reclassified as feature gaps, and **two findings open** — R-039 (EFS free + list) and R-011 (blocked on fixtures) — plus R-019, accepted. R-020 and + R-038 both closed 2026-08-14. `data/known-failures.toml` is down to 4 + entries, all F-008. Read the table there, not this paragraph. +- R-016 is no longer a defect: it was reclassified as + [F-008](missing_features_from_regression.md#f-008), and `rb-regress validate` + now accepts an `F-nnn` citation as well as an `R-nnn` one. +- `main` is at merge commit `48cee1f`; this branch is ahead of it and has + **not** been merged. + +## THE FOUR DECISIONS ARE NOW THREE ANSWERS AND ONE QUESTION + +Do not re-ask the first three. + +1. **R-003** — decided: *implement* `ls --format`, not correct the docs. + Shipped. +2. **R-016** — decided: an unimplemented *feature*. Now F-008. +3. **R-035** — decided: *normalise* the path to a leaf. Shipped. +4. **R-011** — **still open.** Should copy-protected G64 dumps open at all? + Asserting either way prejudges it, so only the working half is pinned. + +## USE THE TOOLS, NOT THE MARKDOWN + + rb-regress fixtures # corpus: 91 catalogued, all sha256-verified + rb-regress validate # manifests + bug list consistency + rb-regress run # the matrix + rb-regress consolidate # across hosts + +`fixture_root` is local disk (`regression-tests/fixtures`), synced once from +`corpus_source`. A run never touches the network. Control: move `fixtures/` +aside and every corpus-backed case must report `skip-fixture`. + +## WHAT IS READY TO FIX + +From [`regression-fix-prompt.md`](regression-fix-prompt.md), which tranches the +remainder. The three highest-value ones are done (R-021, R-022, R-023), so what +is left needs investigation before it needs a fix: + +- **R-008b / R-008a** — `new volume affs --size 4M` panics, exit 101, no file. + A panic with no output is the worst remaining failure mode, and R-008a shares + the fix: volumes above 4066 blocks have uncovered tail blocks. +- **R-024** — one `put` into a fresh 3 MB AFFS volume makes `fsck --checkonly` + report errors. Data reads back fine, so the damage is to allocation + structures. Three distinct AFFS bugs — R-008 is the formatter, R-024 the + editor, R-020 the root block. Do not conflate them. (All three closed by + 2026-08-14; kept here because the "do not conflate" advice is still the + right way to read the AFFS entries.) +- **R-033** — a QL Microdrive `.mdv` fails at MBR detection although its own + probe matches it exactly. **Very likely the same shape as R-022**, which was + a bare volume falling through to the MBR parse because no probe claimed it. + Read the R-022 fix first; this may be twenty minutes. +- **R-013, R-028, R-029, R-030, R-031, R-032** — tranche C. The symptom is + recorded, the cause is not. Each needs a scoped investigation, and the + investigation is the deliverable. + +## BLOCKED, NOT FORGOTTEN + +- **R-015, R-012** are upstream in `opticaldiscs`. A fixed 0.15.0 exists in the + maintainer's working tree, unpublished. When it lands: bump the pin and + re-run `optical.cue.unpadded-track-number` and + `optical.cdda.no-data-track-opens` — both red on purpose, both will flip to + XPASS. `docs/opticaldiscs-upstream-prompt.md` has the detail. +- ~~**R-020** needs an emulator or hardware oracle.~~ **Unblocked and closed + 2026-08-14.** The harness feature this asked for exists: + `oracles/fsuae/affs_mount.py` drives FS-UAE, and Kickstart 3.1 mounts our + volume Read/Write. The remaining 61 emulator / MiSTer-core oracles are still + `skip-manual` — this unblocked one of them, not the class. +- **R-025** is Windows-only and correctly scoped with `platforms = ["windows"]`. +- MiSTer's `rb-cli` is from 2026-07-27 and must be redeployed before its 12 + core oracles mean anything. + +## FIXTURE ADMITTED 2026-08-09 + +`fs.hpfs.os2-warp45.hd` — a real OS/2 Warp 4.52 install, monolithicSparse VMDK, +MBR type-0x07 HPFS at LBA 63, 136 MB zstd in the annex. It closes the HPFS gap +R-022 left and carries two things nothing else in the corpus does: + +- **A partitioned HPFS volume.** R-022's control (the probe must not hijack an + ao486-shaped disk) was a hand-synthesized MBR; it is now + `fs.detect.hpfs-partitioned-stays-mbr`. +- **A real non-flat container.** `backup` refuses it exactly as F-008 + describes, so that gap is not an artifact of how the synthetic containers + were built. + +`read.hpfs.os2-warp45` reads it end to end — 4722 files, 471 dirs, long names +with spaces, fsck clean. + +Note the original drop is still at `new/OS2 Warp 4.52.zip` on the NAS; the +annex copy is independent, so the drop can be deleted whenever. + +## HOW THE CORPUS REACHES THE OTHER HOSTS + +Not by `rb-regress fixtures --sync`. **linuxbox and the Mac have +`corpus_source` commented out** — neither can reach the distribution share, and +their `local.toml` says so. The corpus was scp'd to them from this Windows box, +and that is still the mechanism. + +So admitting an annex fixture is three steps per host, not one: + +1. `scp` the file into `regression-tests/fixtures-large/`. +2. Append the catalogue row to that host's `regression-tests/fixture-map.tsv` + — gitignored, so it does **not** arrive with `git pull`. This is the step + that is easy to miss: without it the file is present and the case still + reports `skip-fixture`. +3. `rb-regress fixtures` on the host to confirm `N catalogued - N verified, 0 + missing, 0 CORRUPT`. + +Windows `scp.exe` cannot resolve an MSYS `/tmp/...` path; stage anything you +are copying at a real Windows path first. + +A case whose fixture is missing degrades to `skip-fixture`, not a failure — so +a forgotten sync looks like a green run with a smaller number. Compare the pass +count across hosts, not just the fail count. + +## FEATURE WORK QUEUED + +`docs/missing_features_from_regression.md`, F-005 through F-008: + +- **F-005** — GUI cannot extract a single file. Small: `browse_view.rs` already + calls `read_file` in three places. Must surface the filesystem selector or the + GUI can never reach both sides of a hybrid disc. +- **F-006** — IRIX support disks. **Needs scope** — three readings recorded, one + of which cannot be verified without hardware. +- **F-007** — no optical fixture has nested directories, so + `--path DIR --recursive` is implemented and unverified. Fixtures already + catalogued; only the case is missing. +- **F-008** — `backup` reads only flat-layout sources. `inspect` already opens + all four containers, so the decoding exists and `backup` simply takes a + different route to the bytes. Routing it through the same path is the whole + feature. `backup.container.inspect-reads-what-backup-cannot` is green and + pins the asymmetry — read it first. + +## CONVENTIONS THAT MATTER + +- Cases assert **intended** behaviour, so they are red until fixed. Never make + one pass by asserting broken behaviour — add it to `known-failures.toml` + citing a finding instead. XPASS catches a stale entry. +- If a case turns out to assert the wrong thing, change it **deliberately and + say so**. Three precedents now: `cli.exit.missing-image-file` pinned exit 1 + and contradicted `exit.rs`; `cli.envelope.error-envelope-on-failure` pinned + the same 1 for the same reason; `resize.repack.keeps-data` built a plain FAT + volume for a verb that is Human68k-only. All three were corrected, not + weakened. +- **Doubt the diagnosis, not just the code.** Two findings this session were + filed with the wrong cause. R-023 said "every file is gone" — nothing was + lost, a FAT long filename was dropped. R-022 said "not byte-identical" — the + backup was empty. Reproduce and *measure* before fixing; both had a + one-command control (count the non-zero bytes; list the backup folder). +- **Always run a control before believing a diagnosis.** R-023's control was + repacking a *real* Human68k volume, which worked perfectly and proved the + clone sound and the input wrong. R-022's was a synthesized MBR-partitioned + HPFS disk, proving the new probe does not hijack partitioned disks. +- `platforms = ["windows"]` on a `[[known]]` entry scopes a platform-specific + finding. Without it, the other platforms report a false XPASS. +- A `[[known]]` entry may cite an `R-nnn` defect **or** an `F-nnn` feature gap. + Before 2026-08-09 only the former validated, which is part of why a feature + gap looked like it had to be filed as a bug. +- `tests/doc_parity.rs` guards README / CONTRIBUTING claims against the source. + It is a `cargo test`, not an `rb-regress` case, because the claim is *about* + the binary rather than something the binary does. +- Engine code (`src/`) must compile on Rust 1.73 — your `cargo build` will not + catch a violation. See CONTRIBUTING.md. +- Comments are one line, two at most. No Unicode glyphs in UI or log strings. +- Pre-commit runs `clippy --all-targets -- -D warnings` and does `git add -u`, + which bundles every modified file — stash-dance for per-phase commits. +- Commit per phase (3-5 a session). **Never push without being asked.** +- **After a push, check CI and drive it green.** `gh run list` / `gh run view + `. A green `rb-regress run` is not a green pipeline, and this was learned + the hard way — a push on 2026-08-09 turned Release red and nobody noticed + until it was mentioned in passing. Three specifics: + - CI's test step is **`cargo test --release`**, not `cargo test`. Run that + before pushing. (macOS x64 skips tests — it is cross-compiled — so only the + arm64 macOS job exercises them there.) + - **`gh run list` saying "success" does not mean every job passed.** Jobs + marked `continue-on-error: true` — the MiSTer `rb-cli-mini` build is one — + fail silently without reddening the run. Read the job list from + `gh run view `. The mini build was broken for a full day that way. + - **Non-default feature sets are not covered locally.** The MiSTer one is + `cargo check --bin rb-cli --no-default-features --features + chd,pure-zstd,remote,optical,tui`. Anything touching zstd must go through + `crate::rbformats::zstd_compat`, never the `zstd` crate directly, or it + compiles on the desktop and breaks there. +- **Platform-dependent std APIs are the other thing local runs miss.** + `Path::file_name` treats `\` as a separator only on Windows, which is how a + Windows-path assertion passed locally and failed on every Unix job. +- Nothing private in the repo: corpus paths, machines and addresses live in + gitignored `regression-tests/local.toml` only. +- Windows: use `C:\Windows\System32\OpenSSH\ssh.exe`, not Git Bash ssh, with + `-o IdentitiesOnly=no`, and `GIT_SSH_COMMAND` for a push. **Fetching on the + remote hosts is easiest over HTTPS** (`git fetch + https://github.com/danifunker/rusty-backup.git regression-fixes`) — the repo + is public, linuxbox's own key needs `-A` forwarding and the Mac's key needs + an agent that non-interactive ssh does not have. macOS commands need + `zsh -lc`. Export `MSYS_NO_PATHCONV=1` for any `rb-cli` call with a `/` path. +- Rebuilding `rb-regress` as well as `rb-cli` is not optional: skipping it once + already produced two false XPASS on macOS under a correct-looking sha. The + runner warns when its own sources are newer than the binary. diff --git a/docs/Regression_Bugs.md b/docs/Regression_Bugs.md index f62111b9..25edfff2 100644 --- a/docs/Regression_Bugs.md +++ b/docs/Regression_Bugs.md @@ -1,4 +1,4 @@ -# Regression Findings (R-001 … R-035) +# Regression Findings (R-001 … R-039) Defects and documentation drift turned up while building the regression suite (`regression-tests/`), 2026-08-01/02. The suite work was deliberately kept @@ -20,41 +20,45 @@ finding depends on a fixture, the fixture is named. | ID | Severity | Area | Finding | |----|----------|------|---------| | [R-019](#r-019) | Low — **accepted** | `src/rbformats/vhd.rs` | VHD Creator Host OS makes output non-reproducible across platforms; behaviour kept, parity declares it | -| [R-023](#r-023) | **High** | `src/cli/verbs/repack.rs` | `repack` loses every file in the volume | -| [R-022](#r-022) | **High** | `src/fs/hpfs.rs` | HPFS sector-by-sector backup -> restore is not byte-identical | -| [R-021](#r-021) | **High** | `src/cli/verbs/resize.rs` | `resize --size` reports success and changes nothing | -| [R-024](#r-024) | Medium | `src/fs/affs.rs` | AFFS `put` leaves the volume failing its own fsck | -| [R-025](#r-025) | Medium | `src/fs/squashfs_edit.rs` | `squashfs put` fails to replace the image on Windows | -| [R-026](#r-026) | Low | `src/cli/verbs/show.rs` | `show partmap` cannot read an SGI disk that `inspect` reads fine | -| [R-027](#r-027) | Medium | `src/rbformats/zip_disk.rs` | A Finder-made `.zip` holding one `.dmg` is rejected as ambiguous | -| [R-030](#r-030) | **High** | `src/fs/affs.rs` | A real Workbench 1.3 AFFS volume cannot be opened at all — read, fsck and write alike | -| [R-029](#r-029) | **High** | `src/fs/efs.rs` | EFS computes block addresses far outside the image; `fsck` fails on an unmodified volume | -| [R-031](#r-031) | Medium | `src/partition/mod.rs` | A real Apple DOS 3.3 disk is detected as `unknown`, though our own output is not | -| [R-028](#r-028) | Medium | `src/fs/apple_dos.rs` | Apple DOS 3.3 reports three different sizes for one file: 104 in, 512 by `ls`, 256 by `get` | -| [R-032](#r-032) | Low | `src/fs/sfs.rs` | SFS `put` fails on any volume with a multi-leaf extent btree — i.e. any real one | -| [R-033](#r-033) | **High** | `src/partition/mod.rs` | A QL Microdrive `.mdv` fails at MBR detection, though its own probe matches it exactly | -| [R-034](#r-034) | Medium | `src/fs/mod.rs` | Refusing a write to a read-only filesystem says `unknown` and exits 1, not 4 | -| [R-035](#r-035) | Medium | `src/backup/` | `.cbk` embeds the producing host's absolute path, so it can never be byte-identical across machines | -| [R-020](#r-020) | **High** | `src/fs/affs.rs` | `new volume affs` output is "Not a DOS disk" on a real Amiga, at every size | -| [R-016](#r-016) | **High** | `src/cli/verbs/backup.rs` | `backup` accepts only flat-layout sources: CHD, dynamic VHD, QCOW2 and VMDK all fail | +| ~~R-023~~ | ~~**High**~~ **FIXED** | `src/cli/verbs/repack.rs` | ~~`repack` loses every file in the volume~~ — scope guard; nothing was lost, a FAT long filename was dropped, 2026-08-09 | +| ~~R-022~~ | ~~**High**~~ **FIXED** | `src/partition/mod.rs` | ~~HPFS sector-by-sector backup -> restore is not byte-identical~~ — detection, not fidelity: a bare HPFS volume backed up to nothing at all. Probe added, 2026-08-09 | +| ~~R-021~~ | ~~**High**~~ **FIXED** | `src/cli/verbs/resize.rs` | ~~`resize --size` reports success and changes nothing~~ — grows the file when the volume is the file, refuses otherwise, 2026-08-09 | +| ~~R-024~~ | ~~Medium~~ **FIXED** | `src/fs/affs.rs` | ~~AFFS `put` leaves the volume failing its own fsck~~ — was the truncated bitmap, not the editor; same fix, 2026-08-10 | +| ~~R-025~~ | ~~Medium~~ **FIXED** | `src/fs/squashfs_edit.rs` | ~~`squashfs put` fails to replace the image on Windows~~ — handle released before the rename, 2026-08-08 | +| ~~R-026~~ | ~~Low~~ **FIXED** | `src/cli/verbs/show.rs` | ~~`show partmap` cannot read an SGI disk that `inspect` reads fine~~ — detects the table first, 2026-08-08 | +| ~~R-027~~ | ~~Medium~~ **FIXED** | `src/rbformats/zip_disk.rs` | ~~A Finder-made `.zip` holding one `.dmg` is rejected as ambiguous~~ — extension list derived from the canonical one, 2026-08-08 | +| ~~R-030~~ | ~~**High**~~ **FIXED** | `src/fs/affs.rs` | ~~A real Workbench 1.3 AFFS volume cannot be opened at all~~ — the root block was located from the end of the file, not the partition, 2026-08-10 | +| ~~R-029~~ | ~~**High**~~ **NOT A DEFECT** | — | ~~EFS computes block addresses far outside the image~~ — a deliberate 4 MB prefix capture; the case asked it to do what a prefix cannot, 2026-08-10 | +| ~~R-031~~ | ~~Medium~~ **NOT A DEFECT** | — | ~~A real Apple DOS 3.3 disk is detected as `unknown`~~ — the disk carries no filesystem at all; `Unknown` is correct, 2026-08-10 | +| ~~R-028~~ | ~~Medium~~ **FIXED** | `src/fs/apple_dos.rs` | ~~Apple DOS 3.3 reports three different sizes for one file~~ — length stored in a type-B header; all three now agree, 2026-08-10 | +| ~~R-032~~ | ~~Low~~ **RECLASSIFIED** | `src/fs/sfs.rs` | ~~SFS `put` fails on any volume with a multi-leaf extent btree~~ — the documented ceiling; moved to [F-009](missing_features_from_regression.md#f-009), 2026-08-10 | +| ~~R-033~~ | ~~High~~ **FIXED** | `src/partition/mod.rs` | ~~A QL Microdrive `.mdv` fails at MBR detection, though its own probe matches it exactly~~ — probe added beside the HPFS one, 2026-08-10 | +| ~~R-034~~ | ~~Medium~~ **FIXED** | `src/fs/mod.rs` | ~~Refusing a write to a read-only filesystem says `unknown` and exits 1, not 4~~ — names the filesystem, exits 4, 2026-08-08 | +| ~~R-035~~ | ~~Medium~~ **FIXED** | `src/backup/` | ~~`.cbk` embeds the producing host's absolute path, so it can never be byte-identical across machines~~ — path normalised to a leaf, 2026-08-09 | +| ~~R-036~~ | ~~Medium~~ **FIXED** | `src/cli/resolve.rs` | ~~A missing image gets three different exit codes across the verb surface~~ — one guard in the shared resolver, 2026-08-10 | +| ~~R-037~~ | ~~**High**~~ **FIXED** | `src/cli/verbs/resize.rs` | ~~Shrinking rewrote the filesystem over live data and returned truncated files~~ — data floor + `--confirm-shrink` + truncation, 2026-08-09 | +| ~~R-038~~ | ~~**High**~~ **NOT A LIVE DEFECT** | — | ~~A second implementation (amitools) rejects every AFFS volume we write~~ — real, but already fixed by a190182 two days before it was filed; the oracle read an Aug-8 artifact, 2026-08-14 | +| ~~R-039~~ | ~~**High**~~ **FIXED** | `src/fs/efs*.rs` | ~~IRIX's own fsck reports BAD FREE LIST on every EFS volume we write~~ — the bitmap is LSB-first and we had it MSB-first in reader and writer alike; IRIX now mounts, writes to and fscks our volumes clean, 2026-08-15 | +| ~~R-020~~ | ~~**High**~~ **FIXED** | `src/fs/affs.rs` | ~~`new volume affs` output is "Not a DOS disk" on a real Amiga, at every size~~ — Kickstart 3.1 mounts it Read/Write as `rusty-backup` and lists its contents; fixed by a190182, confirmed 2026-08-14 | +| ~~R-016~~ | ~~**High**~~ **RECLASSIFIED** | `src/cli/verbs/backup.rs` | ~~`backup` accepts only flat-layout sources: CHD, dynamic VHD, QCOW2 and VMDK all fail~~ — not a defect; moved to [F-008](missing_features_from_regression.md#f-008), 2026-08-09 | | ~~R-018~~ | ~~Blocker~~ **FIXED** | `CONTRIBUTING.md` | ~~The documented Rust-1.73 verification build does not compile on Windows~~ — missing `windows-legacy` feature, 2026-08-07 | | ~~R-017~~ | ~~High~~ **FIXED** | `src/partition/mod.rs` | ~~Superfloppy detection also misses SFS (extends R-009)~~ — probe added 2026-08-07 | -| [R-015](#r-015) | Medium | `src/optical/` (cue parser) | A `.cue` with unpadded track numbers (`TRACK 1`) is rejected | +| ~~R-015~~ | ~~Medium~~ **FIXED** | upstream `opticaldiscs` | ~~A `.cue` with unpadded track numbers (`TRACK 1`) is rejected~~ — fixed upstream, pin bumped to 0.15.0, 2026-08-10 | | ~~R-014~~ | ~~Blocker~~ **FIXED** | `src/cli/verbs/squashfs.rs` | ~~Pre-existing clippy failure blocks every commit via the pre-commit hook~~ — boxed 2026-08-07 | -| [R-008b](#r-008b) | **High** | `src/fs/affs.rs` | `new volume affs --size 4M` panics; no file produced, exit 101 | +| ~~R-008b~~ | ~~**High**~~ **FIXED** | `src/fs/affs.rs` | ~~`new volume affs --size 4M` panics; no file produced, exit 101~~ — the formatter writes as many bitmap pages as the volume needs, 2026-08-10 | | ~~R-007~~ | ~~High~~ **FIXED** | `src/fs/ntfs_format.rs` | ~~Freshly formatted NTFS fails its own fsck~~ — verified clean 2026-08-07 | | ~~R-009~~ | ~~High~~ **FIXED** | `src/partition/mod.rs` | ~~Bare JFS / UFS1 / UFS2 / ReiserFS images cannot be opened at all~~ — probes added 2026-08-07 | -| [R-013](#r-013) | **High** | `src/fs/ufs.rs` | Solaris UFS directories reported as files, one with a garbage size | -| [R-005](#r-005) | Medium | `src/cli/output.rs` | No error envelope emitted under `--format json` | -| [R-008a](#r-008a) | Medium | `src/fs/affs.rs` | AFFS volumes above 4066 blocks have uncovered tail blocks | -| [R-012](#r-012) | Medium | `src/optical/` | `optical info` rejects any disc with no data track (pure CD-DA) | -| [R-003](#r-003) | Medium | `src/cli/output.rs` | Docs claim `ls` supports `--format`; it does not | -| [R-010](#r-010) | Medium | `src/cli/verbs/inspect.rs` | `inspect` has no `--fs-type`, so CP/M images cannot be inspected | -| [R-006](#r-006) | Medium | `src/cli/verbs/new.rs` | `new volume prodos` always fails with default arguments | -| [R-004](#r-004) | Low | `src/cli/output.rs` | CSV/TSV rejection exits 1, documented as 2 | -| [R-011](#r-011) | Unknown | `src/rbformats/` | G64 decoding fails on copy-protected / patched dumps | -| [R-001](#r-001) | Doc | `README.md` | Partition-table list missing AHDI and X68000 | -| [R-002](#r-002) | Doc | `src/fs/README.md` | Capability table stale — ext listed as "planned" | +| ~~R-013~~ | ~~**High**~~ **FIXED** | `src/fs/ufs.rs` | ~~Solaris UFS directories reported as files, one with a garbage size~~ — UFS1's rotational cylinder-group offset was ignored, 2026-08-10 | +| ~~R-005~~ | ~~Medium~~ **FIXED** | `src/cli/output.rs` | ~~No error envelope emitted under `--format json`~~ — format recorded before dispatch, envelope emitted from `main`, 2026-08-09 | +| ~~R-008a~~ | ~~Medium~~ **FIXED** | `src/fs/affs.rs` | ~~AFFS volumes above 4066 blocks have uncovered tail blocks~~ — same fix as R-008b, 2026-08-10 | +| ~~R-012~~ | ~~Medium~~ **FIXED** | upstream `opticaldiscs` | ~~`optical info` rejects any disc with no data track (pure CD-DA)~~ — fixed upstream, pin bumped to 0.15.0, 2026-08-10 | +| ~~R-003~~ | ~~Medium~~ **FIXED** | `src/cli/output.rs` | ~~Docs claim `ls` supports `--format`; it does not~~ — flag implemented, all five formats, 2026-08-09 | +| ~~R-010~~ | ~~Medium~~ **FIXED** | `src/cli/verbs/inspect.rs` | ~~`inspect` has no `--fs-type`, so CP/M images cannot be inspected~~ — flag added and honoured, 2026-08-08 | +| ~~R-006~~ | ~~Medium~~ **FIXED** | `src/cli/verbs/new.rs` | ~~`new volume prodos` always fails with default arguments~~ — per-filesystem default, 2026-08-08 | +| ~~R-004~~ | ~~Low~~ **FIXED** | `src/cli/exit.rs` | ~~CSV/TSV rejection exits 1, documented as 2~~ — errors carry their exit code now, 2026-08-08 | +| [R-011](#r-011) | **Partial** | `src/rbformats/containers/g64.rs` | Copy-protected G64 dumps: read-only refusal fixed; decode of non-standard GCR unverified (no fixture) | +| ~~R-001~~ | ~~Doc~~ **FIXED** | `README.md` | ~~Partition-table list missing AHDI and X68000~~ — X68k and DSD rows added, guarded by a parity test, 2026-08-09 | +| ~~R-002~~ | ~~Doc~~ **FIXED** | `src/fs/README.md` | ~~Capability table stale — ext listed as "planned"~~ — table deleted for a pointer at the live dispatch, 2026-08-09 | --- @@ -232,6 +236,37 @@ each names its reproduction and stops there. ### R-021 — `resize --size` reports success and does nothing {#r-021} +**FIXED 2026-08-09**, and it was not quite a no-op — which is worse. `resize` +warned "the FS may refuse", then proceeded: the FAT resize rewrote the BPB for +16303 clusters, appended 32 KB of new FAT sectors, printed `resize complete` +and exited 0. The result was a filesystem describing twice the blocks its +container held. `inspect` still said 8 MiB because the file length is what a +superfloppy's size comes from, which is what made it look like nothing had +happened. + +The warning was the bug. Two situations were being treated as one: + +- **The volume is the whole file** — a bare superfloppy in a plain image. + Nothing else lives there and there is no table to keep in step, so appending + zeros *is* what the caller asked for. `resize` now grows the image first, + then resizes into it. `PartitionContext::whole_file_path` is exactly this + condition and already existed. +- **Anything else** — a partition inside a larger disk, a decoded container. + Its length is set by something `resize` is not editing, so overrunning it is + corruption. Now a hard refusal, exit 2, naming `partmap resize` and `grow` as + the verbs that can move the boundary. + +Verified both ways: an 8 MiB FAT superfloppy with a file in it grows to 16 MiB, +fsck-clean, file intact; an X68k partition asked for 64 MiB in a 16 MiB slot is +refused with the image byte-for-byte untouched and its three files still there. + +Growing only, at the time. **That half of this note was wrong and is corrected +by [R-037](#r-037):** a shrink was not "trailing slack, not damage". It rewrote +the filesystem for the smaller size with live data still beyond it, and the +files came back truncated. Shrinking is handled properly as of R-037. + +--- + ``` rb-cli new volume fat --size 8M v.img rb-cli resize v.img --size 16M -> "resize complete", exit 0 @@ -244,6 +279,62 @@ has any reason to check. Case `resize.to-explicit-size`. ### R-022 — HPFS does not survive a sector-by-sector round-trip {#r-022} +**FIXED 2026-08-09, and the finding understated it by a lot.** This was not a +fidelity bug. The restored 2 MB image held **44 non-zero bytes against the +source's 604** — the boot sector and nothing else. Superblock, spareblock, +bitmaps, root fnode: gone. `backup` wrote no partition file at all, and exited +0. + +The backup folder contained `metadata.json`, `mbr.bin`, `mbr.json` and no +`partition-*`. Its metadata says why: + +```json +"partition_table_type": "MBR", +"partitions": [] +``` + +**Cause is detection, not the round-trip.** An HPFS boot sector is an x86 VBR: +`EB 3C 90`, OEM `IBM 4.50`, a BPB, and an 0xAA55 signature. `detect_superfloppy` +had no HPFS probe, and HPFS fails the FAT probe because `reserved_sectors` and +`num_fats` are both 0 — HPFS has no FATs. So it fell through to the MBR parse, +which took the 0xAA55 at face value, read an all-zero partition array, and +reported "MBR, no partitions". `backup` iterates that list, so it had nothing +to copy. Nothing anywhere in the chain was wrong about its own job. + +`ls` and `inspect` masked it: both fall back to "raw filesystem @ byte 0" and +read the volume fine, so the disk looked healthy right up until it was backed +up. + +The fix is a probe in `detect_superfloppy`, beside the NTFS / exFAT pair HPFS +shares MBR type 0x07 with. `fs::hpfs::looks_like_hpfs` already existed and +wants both the superblock magic at sector 16 and the spareblock magic at +sector 17, so a chance match needs 64 bits to line up. A bare HPFS volume now +reports `Partition table: None` with one HPFS partition, and the round trip is +byte-identical. + +Two new cases pin the mechanism rather than the symptom, so a detection +regression is named as one instead of resurfacing as a mysterious fidelity +failure: `fs.detect.hpfs-bare-volume` and `fs.detect.hpfs-backup-is-not-empty`. +The second exists because `roundtrip.hpfs.raw` cannot tell "backed up nothing" +from "backed up slightly wrong". + +**The control that mattered** was the other direction: a *partitioned* HPFS +disk — the ao486 shape, graded **Yes** in +[full_MiSTer_support_status.md](full_MiSTer_support_status.md) — must not be +hijacked by the new probe. The probe reads absolute sectors 16 and 17, which on +a partitioned disk sit in the pre-partition gap, so it should not fire. It was +first checked against a synthesized MBR holding the same 2 MB volume. + +**That control is now a real fixture and a real case.** `fs.hpfs.os2-warp45.hd` +— an OS/2 Warp 4.52 install, MBR type-0x07 HPFS at LBA 63 — was admitted +2026-08-09, and `fs.detect.hpfs-partitioned-stays-mbr` asserts the disk still +reports MBR with its partition at 63. `read.hpfs.os2-warp45` reads the same +volume end to end: 4722 files across 471 directories, long names with spaces, +fsck clean. A synthetic `new volume hpfs` is empty and 2 MB and can exercise +none of that. + +--- + `backup --sector-by-sector` then `restore` returns bytes that differ from the source. Every other filesystem tested — FAT, NTFS, ext4, HFS, minix3, EFS, ProDOS — comes back byte-identical through the same path, so this is specific @@ -252,6 +343,40 @@ faithful image, so any difference is a defect. Case `roundtrip.hpfs.raw`. ### R-023 — `repack` loses every file {#r-023} +**FIXED 2026-08-09 — and the diagnosis in the original report was wrong.** +Nothing was lost. `payload.bin` came back as `PAYLOAD.BIN`, and the `get` in +the next step asked for the lowercase name and missed. Recording that, because +"exit 0 and the data is gone" and "exit 0 and a filename changed case" call for +different fixes, and only the second one was happening. + +The real defect is scope. `repack` is documented Human68k-only, but it opened +`Human68kFilesystem` directly instead of asking how the read path had routed +the volume — and a plain FAT16 superfloppy opens cleanly as Human68k, because +the two share a BPB layout on purpose (`Human68kBpb::parse` accepts the +standard little-endian MS-DOS form, which is what X68000 floppies use). So +`repack` rebuilt a FAT volume through a driver with no long-filename concept. +The short name is all the Human68k driver can see, and FAT short names are +upper-case, so the LFN was dropped on the way through. + +`repack` now requires `type_string == Some("human68k")` — the same dispatch +identity that chose the driver in the first place, and one that both shapes +carry (X68k partition entries and bare Human68k superfloppies, which are gated +on the 68000 `BRA.S` opcode). A plain FAT volume is refused with exit 2 and a +message naming `resize` instead. + +Control, run both ways before believing any of this: `repack` on a real +Human68k volume (`new hd x68k`, four files including a lowercase +`payload.bin`) already worked perfectly — all four files, name case intact, +byte-identical, fsck clean. That is what proved the clone sound and the input +wrong. + +The case was rewritten deliberately: it built a plain FAT volume, which +`repack` was never for. It now uses a real Human68k volume, and a new sibling +`resize.repack.refuses-plain-fat` pins the input it used to accept — including +that the refusal leaves the volume untouched, long filename and all. + +--- + ``` rb-cli put v.img payload.bin /payload.bin rb-cli repack v.img -> exit 0 @@ -262,6 +387,14 @@ Exit 0, and the data is gone. Case `resize.repack.keeps-data`. ### R-024 — AFFS `put` leaves the volume failing its own fsck {#r-024} +**FIXED 2026-08-10 — and this report's diagnosis was wrong.** It reads "this is +the editor, not the formatter" and says to keep it separate from R-008. It was +the formatter: `put` allocated into blocks past the end of the single bitmap +page R-008a describes, and fsck correctly reported a bitmap that disagreed with +the directory walk. Fixing the formatter's bitmap sizing turned this case green +with no change to the edit path. See [R-008a](#r-008a). + + A single `put` into a freshly formatted 3 MB AFFS volume — the largest size R-008 leaves working — makes `fsck --checkonly` report `1 error(s), 1 warning(s) (some repairable)`. The file reads back correctly, @@ -271,6 +404,28 @@ editor, not the formatter. Case `edit.affs.put-get`. ### R-025 — `squashfs put` cannot replace the image on Windows {#r-025} +**FIXED 2026-08-08.** `commit_by_replacing` now closes its handle on the target +before renaming the rebuilt temp over it. + +The original code held the handle across the rename and said so explicitly, +reasoning that nothing reads through it afterwards. That is true, and it is +still not safe on Windows: a file marked for deletion **keeps its name until +the last handle closes**, so the rename cannot reuse a name we are still +holding. Unix frees the name immediately, which is why every developer machine +and both other CI platforms passed. + +`FILE_SHARE_DELETE` looks like the fix and is not — tried first, and it changed +nothing. It permits the delete to *begin*; it does not let the name be reused +while a handle is open. An isolating test made that unambiguous: replacing an +unheld file succeeded, replacing a held one failed with the same `os error 5`, +with no editor code in the picture. + +The handle is now `Option`, released before `persist`. `commit_in_place` — +the only other reader — runs solely when `backing_file` is `None`, which is +exactly when the replacement path never ran. + + + ``` error: sync_metadata: I/O error: replacing ``` @@ -289,6 +444,15 @@ mean "fixed, remove the entry" rather than "never applied here". ### R-026 — `show partmap` cannot read an SGI disk {#r-026} +**FIXED 2026-08-08.** `show partmap` went straight to `Apm::parse` on any +image, so every non-Apple table reported whatever its magic looked like as a +bad DDR signature. It detects the table first now. APM keeps its full DDR and +driver-descriptor rendering; every other table gets the generic partition +list, which is all those tables have — faking a DDR for them would be worse +than omitting it. `PartmapPayload` already carried a `kind` field, so the +structured output anticipated this. + + ``` rb-cli new hd sgi-efs --size 16M d.img rb-cli inspect d.img -> reads the SGI volume header fine @@ -302,6 +466,23 @@ Case `subcmd.show.partmap`. ### R-027 — a Mac-made `.zip` holding one `.dmg` is called ambiguous {#r-027} +**FIXED 2026-08-08 — and the AppleDouble sidecar was not the cause.** The +archive holds `APFS_Image.dmg` and `__MACOSX/._APFS_Image.dmg`, so the stub +looked like the obvious culprit. It was already filtered. The real reason was +that `.dmg` was not in this file's private extension list at all, so *neither* +entry counted as a disk image and the single-entry fallback then saw two files. + +The list had drifted from `DISK_IMAGE_EXTS`, which has carried `dmg` all +along — and `adf`, `2mg`, `woz`, `imz`, `dc42`, `moof`, `d88`, `xdf`, `hdm`, +`dim`, `po`, `do`, `gho`, `hfv` and `squashfs` besides. Every one of those was +the same bug waiting for a differently-shaped archive. It is now derived from +the canonical list, so the two cannot drift again; `bin` is excluded, being far +too common in a mixed archive to identify a disk image by. + +AppleDouble entries are now dropped at collection rather than only in the +image filter, so the single-entry fallback and the "Entries:" listing ignore +them too. + Found 2026-08-08, the first time `fs.apfs.apple-gpt.hd` was ever executed — it was catalogued and checksummed but no run had reached it. @@ -334,6 +515,98 @@ Case `read.apfs.apple-gpt`. ### R-020 — every AFFS volume we write is unmountable on a real Amiga {#r-020} +**FIXED — confirmed by a real Kickstart 2026-08-14.** The emulator this entry +has waited on since 2026-08-07 finally ran, and AmigaOS mounts the volume. + +**Re-run it** (Windows box; needs FS-UAE, a Kickstart 3.1 A1200 ROM and the +`fs.affs.workbench13.hd.hdf` fixture — all three already present, see the +oracle's README for what it expects): + +``` +rb-cli new volume --size 2M affs ours.hdf +rb-cli put ours.hdf hello.txt /HELLO +python regression-tests/oracles/fsuae/affs_mount.py ours.hdf +``` + +Exit 0 and a `verdict: mounted` line is the pass. Takes about a minute, most +of it the guest booting. Pair it with the control below before believing it. + +AmigaOS's answer: + +``` +Mounted disks: +Unit Size Used Free Full Errs Status Name +DH0: 4194M 4194304 4194303 50% 0 Read/Write Workbench +DH1: 2.0M 5 4089 0% 0 Read/Write rusty-backup +DH2: 4194M 4194304 4194303 50% 0 Read/Write RESULTS + +Volumes available: +rusty-backup [Mounted] +``` + +`Read/Write`, under the volume's own name, on an A1200 with Kickstart 3.1 — +the same machine and ROM the original finding used. `List DH1:` reads the +contents back: + +``` +Directory "DH1:" on Friday 14-Aug-26 +HELLO 24 ----rwed +1 file - 2 blocks used +``` + +So the fix was [a190182](#r-008a) on 2026-08-10 — bitmap sized to the +geometry, root `header_key` zeroed — and this is the confirmation the entry +said it needed. The same commit closes [R-038](#r-038). + +**The control, because a pass needs one as much as a failure does.** 2 MB of +`os.urandom` through the identical config produces no DH1: unit at all and +`Can't examine "DH1:": device (or volume) is not mounted`. The harness +discriminates; it is not reporting success for anything put in front of it. + +``` +python -c "import os,pathlib; pathlib.Path('garbage.hdf').write_bytes(os.urandom(2*1024*1024))" +python regression-tests/oracles/fsuae/affs_mount.py garbage.hdf # -> exit 1 +``` + +Oracle: `oracles/fsuae/affs_mount.py`, strength `authoritative` — this is +AmigaOS's own filesystem, not a reimplementation. + +Two notes for whoever reads this next: + +* The original 2026-08-07 observation was made through AmigaVision on the + MiSTer; this one attaches the volume as a bare hardfile under FS-UAE. Both + are A1200/KS3.1 and both ask Kickstart the same question, but they are not + byte-identical setups, and the MiSTer path has not been re-run. +* The volume's own creation date still reads as the 1978 epoch — + `create_blank_affs` never stamps the root block's datestamp. Cosmetic, does + not affect mounting, and not tracked as a defect here. + +--- + +**Hypothesis CONFIRMED 2026-08-10, and the formatter half fixed.** This entry +recorded "root block `header_key` must be 0 and we write the block number" as +**unconfirmed**, needing an emulator. It did not: a real disk answers it. The +Workbench 1.3 fixture admitted for [R-030](#r-030) has a root block reading + +``` +REAL Workbench1.3 root : type=2 header_key=0 secType=1 +OUR 4M volume root : type=2 header_key=4096 secType=1 +``` + +`create_blank_affs` now writes 0. Volumes still format, fsck clean, and accept +a `put` that reads back. + +**Still open, and still needs an oracle.** That a correct `header_key` makes the +volume *mount* on a real Amiga is a separate claim, and no automated run here +can make it — all 62 emulator / MiSTer-core oracles are `skip-manual`. What has +changed is that the remaining question is "does this fix it", not "is the +hypothesis right". The other half of the entry — that `affs_fsck` agrees with +the formatter and is wrong the same way — found nothing to change: fsck never +inspected the root's `header_key` at all. + +--- + + Found 2026-08-07 by the first FS-UAE emulator-oracle run — the first result from an oracle that is not a command-line tool, and the reason that oracle was built. @@ -441,53 +714,27 @@ undercuts the point of writing NTFS. ### R-016 — `backup` accepts only flat-layout containers {#r-016} -`backup --help` documents SOURCE as "an image file or a block-device path", -and `inspect` reads every container we write. `backup` reads only the ones -whose data begins at offset 0: - -| source | `inspect` | `backup` | -|--------|-----------|----------| -| raw `.img` | `Partition table: MBR` | **ok** | -| fixed VHD | `Partition table: MBR` | **ok** | -| **CHD** | `Partition table: MBR` | **fails** | -| **dynamic VHD** | `Partition table: MBR` | **fails** | -| **QCOW2** | `Partition table: MBR` | **fails** | -| **VMDK sparse** | `Partition table: MBR` | **fails** | - -Fixed VHD only passes because it *is* raw data with a trailing footer. Every -container with a non-flat internal layout fails, in one of two ways: - -``` -rb-cli backup o-chd/disk.chd ./out --format raw --sector-by-sector - -> error: backup failed: cannot read first sector: failed to fill whole buffer - -rb-cli backup o-qcow2/disk.qcow2 ./out --format raw --sector-by-sector - -> error: backup failed: failed to detect partition table: - Invalid MBR: invalid boot signature: expected 0xAA55, got 0x... -``` - -Same root cause — the container is not decoded, so raw file bytes are read as -though they were the disk. Which message appears depends on whether the read -runs off the end of a small file or lands on header bytes that resemble a bad -MBR. `--sector-by-sector` does not help; the failure precedes all partition -logic. - -**Why it matters.** These are four of our own output formats, CHD being the -default for `convert`. A user can convert a disk to any of them and then find -they cannot back it up — archive to QCOW2, later try to make a working copy, -and the tool refuses. `inspect` reading them fine makes the failure look -arbitrary from outside. - -Reproduces on a 64 MB synthetic image; no fixture required. - -**Two traps when verifying this**, both of which caught me: - -1. `backup` prints `rb-cli backup: SRC -> DEST` *before* doing any work, so a - grep for `->` reports success on a run that then fails. **Check the exit - code**, not the output. -2. `--format raw` writes `partition-N.img` files, so a `find` over several - directories can pick up an unrelated `.img` and attribute the wrong result - to the wrong container. Use explicit paths per case. +**RECLASSIFIED 2026-08-09 — not a defect.** `backup` has never claimed to +decode containers; `--help` documents SOURCE as "an image file or a +block-device path" and says nothing about non-flat internal layouts. Code that +does not do something it never claimed is a capability gap, and this file's own +rule for the split ("a bug means the code disagrees with its own documentation +or with reality") puts it on the other side of the line. + +The full report, the four-container table, the two verification traps and the +route to a fix now live at +[F-008](missing_features_from_regression.md#f-008). The four cases +`backup.container.{chd,vhd-dynamic,qcow2,vmdk-sparse}` are unchanged — they +assert the behaviour we want and stay red — and their `known-failures.toml` +entries now cite F-008. `rb-regress validate` accepts an F-nnn citation as of +the same date; before that a known failure could only name a defect, which is +what made a feature gap look like the only available filing. + +What made it read as a defect for so long is real and worth keeping in mind: +`backup.container.inspect-reads-what-backup-cannot` is green and proves +`inspect` opens exactly what `backup` refuses. An asymmetry between two verbs +on the same file looks like a bug from outside even when neither verb is +misbehaving. ### R-017 — superfloppy detection also misses SFS {#r-017} @@ -581,6 +828,46 @@ rb-cli at it gets an MBR error, with no hint the filesystem is supported. ### R-013 — UFS directories reported as files, with a garbage size {#r-013} +**FIXED 2026-08-10. Cylinder-group layout, as the entry guessed — not +endianness.** + +The tell was in the symptom: `lost+found` was the only correct entry, and it is +the only one whose inode lives in cylinder group 0. + +UFS1 places each CG's *metadata* — superblock replica, CG header, inode table — +at `cgbase(c) + fs_cgoffset * (c & !fs_cgmask)`, a rotational skew from the days +when it mattered which track a cylinder group's tables landed on. `cgbase(c)` is +only where the CG's *data* begins. Our addressing used `cgbase` throughout, and +that term is **zero for c = 0 whatever the values** — so CG 0 read perfectly and +every later CG read garbage. + +Confirmed by probing the disk rather than reasoning about it: + +``` +fs_cgoffset = 32 fs_cgmask = 0xffffff00 + +ino 2 cg 0: plain 0o040755 with-cgoffset 0o040755 (same) +ino 18816 cg 1: plain 0o100242 with-cgoffset 0o040755 DIFFER +ino 37632 cg 2: plain 0o027056 with-cgoffset 0o040755 DIFFER +``` + +`cgstart_frag()` now applies the skew, and the superblock replica, CG header and +inode table all address through it. It is inert where it should be: UFS2 never +has the term, and a UFS1 image written after the rotational tables were dropped +carries `fs_cgoffset = 0`. The existing UFS1/UFS2/NeXTSTEP read and edit cases +stayed green, which is the check that matters for a change to inode addressing. + +`bin` and `lib` still list as 9-byte files — correct: they are Solaris symlinks +to `./usr/bin` and `./usr/lib`, both 9 characters. + +**One symptom did not clear**, and is not covered by either case: `show fs-info` +reports `Free: 0 B` on this volume. That comes from the cylinder-group summary, +not from inode addressing, so it is a separate defect on the same filesystem — +worth its own finding rather than being folded in here silently. + +--- + + Reading the root of an installed Solaris disk (Sun label, UFS slice): ``` @@ -610,6 +897,28 @@ ever exercised. Fixture: `part.sun.solaris-disk.multipart` (annex). ### R-033 — a QL Microdrive cartridge never reaches its own detector {#r-033} +**FIXED 2026-08-10, and it was R-022's shape exactly** — which is what the +resume note predicted. Both the driver and the detection existed; nothing +consulted them on the path the user takes. `detect_superfloppy` had no MDV +probe, so a cartridge fell through to the MBR parse, which read the `ff ff` +sync at 0x0A as a bad 0xAA55 and refused the whole file. + +The probe now sits beside the HPFS one added for R-022, gated on **both** the +exact 174,930-byte cartridge length and the sector-0 header, so it cannot claim +anything else. Two unit tests pin both halves of that gate — a header at the +wrong length and cartridge-sized bytes without a header must both be rejected. +A probe that fires too eagerly is worse than one that never fires. + +`inspect` now reports `qdos_mdv`, 170.8 KiB, no partition table. + +**What this does not do.** Reaching the driver reveals that its directory walk +is a stub: `ls` returns "QDOS microdrive directory walk not implemented". That +is not a new finding — it is the `.mdv` QDOS microdrive reader already tracked +in [OPEN-WORK.md](OPEN-WORK.md) §7, estimated at ~300 LOC against the two +anchored fixtures. This finding was about reachability, and reachability is +what it fixed. + + ``` rb-cli inspect fs.qdos.microdrive.mdv.mdv -> error: detecting partition table: Invalid MBR: invalid boot signature: @@ -642,6 +951,36 @@ exists but is not consulted on the path the user takes. Case ### R-035 — `.cbk` embeds the producing host's absolute path {#r-035} +**FIXED 2026-08-09. Decision: normalise the path**, rather than keep it or +replace it with a device identity. A device identity would have to be +synthesised for the common case (an image file, which has no serial number), +and inventing one is a bigger change than the finding asks for. + +`backup::metadata::normalize_source_device` reduces the source to a label that +names it without naming the machine. Three shapes, because they are not all +paths: + +- **Device paths** (`\.\PhysicalDrive2`, `/dev/sda`, `/dev/disk2`) are kept + verbatim — a device node is already a leaf, and it is the useful provenance. +- **`rb://` sources** are kept verbatim — the host in a remote backup's label + is the *source*, chosen by the user, not the producing host's layout. +- **Everything else** is a path to an image file and keeps only its file name. + +Both production write sites in `src/backup/mod.rs` go through it. The four +other assignments are test fixtures and were left alone. + +This resolves the finding rather than masking it, which matters because the +report is explicit that masking was not available: `expect_divergence` covers +byte *ranges*, and a shorter string changes the file's *length*. Two hosts +backing up the same image now write the same `source_device`, so the `.cbk` +lengths agree and there is nothing for `parity` to align. The information leak +goes with it. + +Verified end to end: backing up `C:\Temp\... arget\sc35\disk.img` records +`disk.img`. + +--- + Found 2026-08-08 by the first three-way `parity` run over all 53 produced formats. 157 comparisons matched; this was the only real divergence. @@ -686,6 +1025,41 @@ finding, not a case failure; `fmt.cbk` is produced by `produce.toml` and ### R-030 — a real Workbench 1.3 AFFS volume cannot be opened at all {#r-030} +**FIXED 2026-08-10.** Neither OFS-vs-FFS nor an older root-block layout — two +of the three readings this entry asked to distinguish. It was arithmetic. + +AFFS records its size nowhere: the root block sits at the volume's midpoint, so +the root block's *position* is the size. `AffsFilesystem::open` derived the +block count from `seek(End) - partition_offset`, i.e. the end of the **file**. +For a bare ADF that is the end of the volume. For a partition inside an RDB +disk it is the end of the disk, so the candidate lands past the real root: + +``` +partition: 4040 blocks at LBA 2020 -> true root block 2020 ("Workbench1.3") +computed from the file tail (4124 blocks) -> 2062, not a root block +``` + +The candidate is always an upper bound, so `locate_root_block` steps down from +it until a block validates on tag, secType, hash-table size **and its own +checksum** — a loose check would mount a directory block as the volume. The +search is bounded (2048 blocks) so a non-AFFS partition cannot become a +whole-disk scan, and every bare ADF and superfloppy still hits on the first +try. `total_blocks` is then trusted from the root's position rather than the +reader's length. + +`rb-cli ls` walks the real disk; `fsck --checkonly` reports 337 files / 31 dirs +clean. + +**The architectural note.** The real fix is for the read-open path to carry the +partition length, which the resolver already knows (`ctx.size`) and the *write* +path already threads (`EditContext::partition_len`). AFFS is the only driver +that cares today, being the only one that derives its geometry rather than +reading it from a superblock — so the search is a contained workaround, not the +end state. + +--- + + Found 2026-08-08, the first time the tier-3 edit cases were executed against the reference volumes rather than against volumes we had formatted ourselves. @@ -710,8 +1084,60 @@ number in `header_key` where 0 belongs. Both are the reader and the writer disagreeing with the real format about the root block; they may or may not share a fix. Cases `edit.real.affs-workbench13`. +**Answered:** they shared a commit (a190182) but not a cause — this one was +the root block being located from the end of the file rather than the +partition, R-020's was `header_key` plus a short bitmap. Both closed. + ### R-029 — EFS computes block addresses far outside the image {#r-029} +**NOT AN ENGINE DEFECT — closed 2026-08-10.** The addresses are outside the +image because the *superblock says so*. `fs.efs.small.hd` is a truncated +capture: + +``` +superblock: fs_size 7,486,242 blocks, 78 cylinder groups, cgfsize 95,954 + = 3.8 GB +image file: 8,192 blocks = 4 MB +``` + +The engine was reading exactly where a 3.8 GB EFS volume keeps its cylinder +groups. `ls` and `get` work because the directory and the sampled files survive +in the first 4 MB; `fsck` walks all 78 groups and `put` allocates against a +bitmap spanning the declared size, so both run off the end — at byte 50,065,408 +and byte 344,826,880 respectively, which is what the entry recorded as the +symptom. + +**Correction to the first version of this note**, which called the fixture +"broken" and blamed truncation. It is a *deliberate prefix capture*, and this +project does that on purpose: `fs.efs.small.hd` is the first 4 MB of a real +IRIX 5.3 disk, and `src/fs/efs.rs` has 15 unit-test call sites that read it — +`parses_superblock_from_fixture` asserts the full disk's `fs_size` of 7,486,242 +precisely because that is what the original volume says. The same convention is +documented for XFS in `docs/SGI_Filesystems_irix_sb.txt`: "dblocks claims a +93 GiB partition but we only extracted the first 64 MiB. Superblock 0 is at +offset 0 and unaffected." + +So the fixture does exactly what it was built for — superblock and read +parsing — and the catalogue's `synthetic-minimal` is accurate. What was wrong +is the *case*, which asked a 4 MB prefix to support a write and a whole-volume +fsck. Those need every cylinder group present; nothing about a prefix capture +promises that. + +**Two things did change.** `EfsFilesystem::open` now compares the declared size +against what is actually present and warns when the image is short, naming both +numbers — an uninterpretable "short read at byte 344826880" was most of why this +sat in Tranche C. And the case, which asserted a put/get round-trip plus a clean +fsck on a volume that cannot support either, now pins what the surviving prefix +does support: listing and reading. + +**To test EFS editing on a real volume**, the corpus needs an internally +consistent capture — a whole small EFS filesystem, not the first 4 MB of a large +one. `edit.efs.put-get` covers the write path on a volume we build ourselves and +is green. + +--- + + ``` rb-cli ls fs.efs.small.hd.img / -> lists the tree fine rb-cli fsck fs.efs.small.hd.img --checkonly @@ -729,6 +1155,33 @@ an unmodified fixture means read-only use is affected too. ### R-028 — Apple DOS 3.3 reports three different sizes for one file {#r-028} +**FIXED 2026-08-10.** All three numbers were explainable, and two were wrong. + +- **512 from `ls`** — the catalog's `length_sectors` counts *every* sector a + file occupies, including its track/sector lists. A one-sector file is 1 data + + 1 T/S list = 2, reported as 512 bytes. Now `data_sectors()` subtracts the + lists (one per 122 data sectors). +- **256 from `get`** — the data sector, unpadded. DOS 3.3 records no byte + length, so a headerless store loses it permanently. +- **104, the truth** — recoverable only if the length is written to the disk. + +`create_file` stored type **T** with the comment that this "avoids stamping a +binary header". That is what lost the length. It now stores type **B** with the +standard 4-byte header (load address, then length, little-endian), which is what +a real Apple II tool writes for arbitrary bytes — so the length is recoverable +by anything reading the disk, not just by us. `read_file` strips the header and +cuts to the declared length; `list_directory` reads it for the reported size, +falling back to allocated bytes when the header is missing or implausible. + +104 in, 104 from `ls`, 104 from `get`, byte-identical. + +The unit test covering this asserted only a *prefix* match +(`&read_back[..len] == payload`), which is exactly why a 104-in/256-out could +pass it. It now asserts full equality and that the listed size matches. + +--- + + ``` rb-cli new floppy apple-dos v.dsk rb-cli put v.dsk payload.bin /PAYLOAD # payload.bin is 104 bytes @@ -745,6 +1198,29 @@ some rounding is inherent; three *different* numbers is not. Case ### R-034 — a read-only filesystem is refused as 'unknown', with the wrong code {#r-034} +**FIXED 2026-08-08.** Both halves, and they had different causes. + +*The name.* `fs_name_for` had no entry for `lisafs`, and the Alto image +carries no type string at all — its name lives in `PartitionInfo::type_name`, +which the write path never received. So the fix is in two places: the missing +`fs_name_for` entries, and `type_name` carried on `PartitionContext` so a +refusal can say what the read path just said. Only a message that failed to +name the filesystem is rewritten; every other `Unsupported` is more specific +than anything this could substitute. + +*The code.* Nothing could carry an exit code out of a handler at all — see +[R-004](#r-004), fixed first. `Unsupported` from a write-open is now +`exit::permission_denied`, which is what `exit.rs` reserves code 4 for. + + error: opening filesystem for write: unsupported: editing not yet + supported for filesystem type 'Apple Lisa File System' [exit 4] + +**It does NOT also fix [R-031](#r-031)**, which this entry suggested checking. +`edit.real.apple-dos-invaders` is still red: a real Apple DOS 3.3 disk arrives +at the write path as `unknown` for a different reason, and needs its own +diagnosis. + + Found 2026-08-08 writing the negative cases PLAN.md § Phase 4 asks for, which did not exist at all until now. @@ -776,6 +1252,33 @@ Cases `edit.readonly.{lisa,alto}-refuses-a-write`. ### R-031 — a real Apple DOS 3.3 disk is detected as 'unknown' {#r-031} +**NOT A DEFECT — closed 2026-08-10.** The premise is wrong: the disk is not an +Apple DOS 3.3 disk. `Unknown` is the right answer. + +`fs.apple-dos.invaders.floppy.dsk` has **no filesystem**. Track 17, where the +DOS 3.3 VTOC lives, is 4096 zero bytes. No VTOC-shaped sector exists anywhere in +the image even under loose criteria (any DOS version, 13- or 16-sector, 35- or +40-track). There is no ProDOS volume directory at block 2 and no Pascal volume +name. Meanwhile 68% of the disk is non-zero, tracks 0-3 are dense and tracks 17+ +are empty — a bootloader that reads raw sectors, which is what an Apple II game +disk of this vintage usually is. + +Sector ordering does not explain it: DOS-order and ProDOS-order `.dsk` images +differ in the arrangement of sectors *within* a track, so track 17 is empty +either way. + +The entry advised trying [R-034](#r-034)'s fix first, on the grounds of "same +shape". They are not the same: R-034 was a *write refusal* naming the wrong +type; this is *detection*, and detection is right. What R-034 did give us is the +honest refusal this disk now produces — exit 4, "editing not yet supported for +filesystem type 'Unknown'". + +The case asserted a `put`/`get` round-trip on a disk with no filesystem. It now +asserts the disk opens and that a write is refused, and records why. + +--- + + ``` rb-cli put fs.apple-dos.invaders.floppy.dsk payload.bin /PAYLOAD.BIN -> Partition @1 (None): Unknown 0x00 @ LBA 0, 143360 bytes @@ -789,6 +1292,23 @@ volume was never identified. Case `edit.real.apple-dos-invaders`. ### R-032 — SFS `put` fails on any volume with a multi-leaf extent btree {#r-032} +**RECLASSIFIED 2026-08-10 — not a defect.** The driver has documented this +ceiling all along (CLAUDE.md: "SFS — read + edit (single-leaf btree only)"), so +hitting it is the limit working as described. Moved to +[F-009](missing_features_from_regression.md#f-009); this entry's own text said +as much ("the known ceiling being hit, not a surprise"). + +**How it refused was wrong, though, and that is fixed.** It raised a `Parse` +error — which says the *disk* is malformed, when the disk is fine and reads +perfectly — and `put` mapped it to the catch-all 1. It is now `Unsupported`, +routed through `write_open_error`, exiting **4**: readable volume, refused +write. That is R-034's shape, and it turned up a second instance of the same +gap — the write-*open* path had been fixed, but `create_file` had not, so every +driver that refuses at creation time exited 1. That mapping is now shared. + +--- + + ``` rb-cli ls fs.sfs.workbench-dh0.hd.img / -> lists the tree fine rb-cli put fs.sfs.workbench-dh0.hd.img payload.bin /PAYLOAD.BIN @@ -806,6 +1326,31 @@ unqualified. Read is unaffected. Case `edit.sfs.put-get`. ### R-015 — cue sheets with unpadded track numbers are rejected {#r-015} +**FIXED 2026-08-10, upstream.** Fixed in `opticaldiscs` 0.15.0; bumping the +pin from 0.13.0 was the whole change on our side. + +--- + + +**Blocked upstream, confirmed 2026-08-08.** The cue parser is in the +`opticaldiscs` crate, not this repository. Bumping 0.13.0 -> 0.14.0 builds +without any API change and does **not** fix it, so the bump was reverted +rather than carried for nothing. + +Minimal reproduction for upstream — `TRACK 1` instead of `TRACK 01`: + +``` +FILE "BOOKSHELF.img" BINARY + TRACK 1 MODE1/2352 + INDEX 1 00:00:00 +``` + +``` +CUE error: Error(Msg("Expeceted number but found String(\"1\") instead")) +``` + +(The typo `Expeceted` is upstream's too, and pins the message's origin.) + A CUE sheet written with `TRACK 1` rather than `TRACK 01` fails to parse: ``` @@ -844,6 +1389,29 @@ number. ### R-005 — no error envelope under `--format json` {#r-005} +**FIXED 2026-08-09.** The obstacle was structural, as with [R-004](#r-004): +`--format` is a per-verb argument, so by the time an error reached `main` the +selected format was gone with the verb's args. + +`rb-cli` now parses through `clap::ArgMatches`, walks to the deepest +subcommand for a `--format`, and records it before dispatch +(`output::record_active_format`). The error arm calls +`output::emit_error_envelope_for`, which emits the documented envelope on +stdout when — and only when — the caller asked for a structured format; +`status.code` comes from `exit::code_for`, so the envelope and the process +agree. The plain-text line still goes to stderr, which is the human channel. + +Reading clap's own matches rather than rescanning `argv` means a file named +`--format` cannot be mistaken for the flag, and a verb added later is covered +without touching the entry point. + +The case asserted `expect_exit = 1`, which contradicted +`cli.exit.missing-image-file` once [R-010](#r-010) landed — a missing image is +`NOT_FOUND`, and `exit.rs` has reserved 3 for it all along. Corrected to 3 +deliberately, not weakened to suit the fix. + +--- + `src/cli/output.rs` documents that on failure the envelope still returns, with `status.error: true`, `status.code` carrying the exit code, `status.message` a short description and `result` null. @@ -861,6 +1429,47 @@ exists to serve. ### R-008a — AFFS tail blocks above 4066 are uncovered {#r-008a} +**FIXED 2026-08-10. One fix closed R-008a, R-008b and R-024**, which the bug +list had filed as three problems across two components. + +One bitmap block holds 127 words of bits, so it accounts for 4064 blocks. The +formatter wrote exactly one, whatever the volume size. A 4 MB volume is 8192 +blocks, so marking the root block in use computed word index 128 and indexed +`bm[512..516]` in a 512-byte block — the panic, exit 101, no file (R-008b). +Below the panic threshold the same bug was silent: every block past 4066 simply +had no bit, so the allocator could hand out blocks the bitmap did not describe +(R-008a). + +**R-024 was the same bug wearing a third hat.** "One `put` into a fresh 3 MB +volume makes `fsck --checkonly` report errors" was filed against the *editor*, +with the note that R-008 was the formatter and the two were distinct. They were +not: `put` allocated into the uncovered tail, and fsck correctly reported a +bitmap that disagreed with the directory walk. Nothing in the editor was wrong. +It went green with no change to the edit path at all. + +The formatter now computes `ceil((total_blocks - 2) / 4064)` pages, writes them +all, records the first 25 in the root block's `bm_pages`, and chains any beyond +that through bitmap extension blocks (127 pointers plus a `next` each). Bits +past the end of the volume are cleared in the final page so the allocator can +never hand out a block that does not exist. + +Verified at 1M, 3M, 4M, 8M and 32M: all create, all fsck clean. Five cases went +green — `fs.new-volume.affs{,.4m,.32m,.bitmap-boundary-plus-one}` and +`edit.affs.put-get`. + +Note this does **not** address [R-020](#r-020): these volumes are still +"Not a DOS disk" on a real Amiga, and that needs an emulator or hardware oracle +to confirm either way. + +**Superseded 2026-08-14.** It did address it. The `header_key` half of +a190182 was what R-020 turned on, and Kickstart 3.1 has since mounted these +volumes Read/Write — the emulator this paragraph asked for was written and +answered. Left standing because being wrong about which fix closes which +finding is the recurring theme of this file. + +--- + + One AFFS bitmap block covers `(512 - 4) / 4 * 32 = 4064` bits, addressing blocks 2..4065. `src/fs/affs.rs` writes exactly one, commented "the bitmap covers a full 4064-bit page even if the volume is smaller" — true only while @@ -882,6 +1491,32 @@ marking loop bounded. ### R-012 — `optical info` rejects discs with no data track {#r-012} +**FIXED 2026-08-10, upstream.** `opticaldiscs` 0.15.0 adds +`FilesystemType::None`: a disc carrying no data track now reports that instead +of being refused. It is deliberately distinct from `Unknown`, which means +"there is a filesystem here and we did not recognise it". Our side was one +match arm in `fs_token` — the new variant made the match non-exhaustive, so the +compiler pointed at the exact place the fix had to land. +`optical.cdda.mixed-mode-still-opens`, the working-half case, stays green. + +--- + + +**Blocked upstream, confirmed 2026-08-08.** Same crate as [R-015](#r-015) and +same result on 0.14.0: still `No data track found`. + +A pure CD-DA disc legitimately has no data track, so refusing to describe the +image is the defect — `optical info` should report the audio tracks and total +time. Minimal reproduction: + +``` +FILE "cdda-noaudiodata.bin" BINARY + TRACK 01 AUDIO + INDEX 01 00:00:00 + TRACK 02 AUDIO + INDEX 01 00:05:25 +``` + ``` rb-cli optical info Audio-only.cue Container: unknown @@ -914,6 +1549,26 @@ the one shape `optical info` refuses. Mixed-mode is unaffected. ### R-003 — docs claim `ls` supports `--format`; it does not {#r-003} +**FIXED 2026-08-09.** Decided in favour of implementing the flag rather than +correcting the doc: `ls` is the most script-facing verb in the CLI, and the +doc described the more useful shape. + +`ls` now takes `--format` with all five values. Text output is untouched — +it still runs through `print_entry`, so no existing case's column layout +moves. The structured paths collect an `LsRow` per entry instead: `kind`, +`name`, `size`, `type_code`, `creator_code`, `mode`, `uid`, `gid`, `owner`. +Every field is always present so the CSV header is stable across volumes, and +a filesystem carrying none of a column leaves it null. `--owner` still governs +whether ids are resolved to names, since that costs a read of the image's own +`passwd`/`group`. + +CSV/TSV emission was private to `show.rs`; it moved to `output::emit_csv_or_tsv` +and `show` now delegates, rather than the two verbs growing separate copies. +The `rb://` remote listing path emits the same shapes, with `mode`/`uid`/`gid` +null because the wire protocol does not carry them. + +--- + `src/cli/output.rs` lists `ls` among the verbs that "can emit their results in one of five formats", and its CSV/TSV note says those formats apply to "`ls`, `show partmap`, `show devices`, `fsck` issue lists". @@ -927,6 +1582,27 @@ a real gap. `ls` is among the most script-facing verbs in the CLI. ### R-010 — `inspect` has no `--fs-type` {#r-010} +**FIXED 2026-08-08.** Accepting the flag was not enough on its own, and it is +worth recording why. A signature-less filesystem has no partition table +either, so `PartitionTable::detect` failed before the forced type could be +applied — `inspect` took the flag and still refused the disk. It now does what +`ls` already did: with `--fs-type`, a detection failure means "raw filesystem +at byte 0" rather than an error. + +``` +rb-cli inspect ManicMiner.dsk --fs-type cpm:amstrad_data + Partition table: None + 1 cpm:amstrad_data 0 180.0 KiB +``` + +A missing image also exits 3 now, which the case requires to tell "the flag +parsed and the file was absent" from "the flag was rejected". That made +`cli.exit.missing-image-file` contradict it: that case asserted 1 and called +itself "current documented-free behaviour", pinning the status quo rather than +the contract `exit.rs` states. Corrected to 3 deliberately, not weakened to +suit the fix. + + `--fs-type` exists on `ls`, `fsck` and `du`, and `docs/cli-reference.md` describes it as the mechanism for CP/M images "which have no on-disk signature". `inspect` does not take it — its usage line is bare @@ -947,6 +1623,21 @@ Every CP/M disk (nine DPB presets) is therefore un-inspectable. ### R-006 — `new volume prodos` fails with default arguments {#r-006} +**FIXED 2026-08-08.** `new volume prodos` with no arguments now writes a +volume named `RUSTY.BACKUP`. + +The shared `--name` default is a named constant, and ProDOS substitutes its +own when that default is untouched. An explicitly passed `--name my-vol` still +fails, and should — the user asked for something ProDOS cannot store. + +The message was the other half. It said "rename the file" about a *volume* +name, because one validator serves both and hard-coded the noun. It takes the +noun from the caller now: + + volume name contains '-' - ProDOS allows only letters (A-Z), digits (0-9), + and '.' (spaces and most punctuation are not allowed) + + The default volume name is `rusty-backup`. ProDOS forbids `-`, so the default is invalid for that filesystem and the verb always fails: @@ -968,6 +1659,22 @@ Every other `new volume` filesystem accepts the default. ### R-004 — CSV/TSV rejection exits 1, documented as 2 {#r-004} +**FIXED 2026-08-08.** The cause was structural, not a wrong constant: nothing +could carry an exit code out of a handler. `main` mapped every `Err` to +`GENERIC_FAILURE`, so any message naming a specific code was describing +something the process could not do. + +`exit::CodedError` now carries one, with `exit::usage()` and +`exit::permission_denied()` constructors and `exit::code_for()` reading it back +in `main`. `code_for` walks the whole `anyhow` chain, so a later +`.context(..)` — added all over the CLI — cannot silently downgrade a coded +error back to 1. + +Both instances use it: `require_non_flat` and the `.chd` output-extension +check. The unit test that missed this asserted only `is_err()`; it now asserts +the code. + + `src/cli/output.rs` states nested-result verbs "error out with `crate::cli::exit::USAGE_ERROR`" — exit 2. Observed exit 1. @@ -998,6 +1705,42 @@ rejection has to carry the code, not the message alone. Case ### R-011 — G64 fails on copy-protected / patched dumps {#r-011} +**DECIDED 2026-08-10:** copy-protected dumps *should* open — preserving those +disks is the reason G64 exists rather than D64 — and should be **un-editable**. + +**Half of that shipped, and the investigation corrected the report.** G64 was +already un-editable: `is_editable_container_path` never listed it. But the +write path then fell through to opening the *undecoded container bytes*, which +reported `Invalid MBR: invalid boot signature` — the same error this report +attributes to copy protection. It reproduces on the **working** dump: + +``` +rb-cli ls fmt.g64.gcr.floppy.g64 -> lists CEST LA VIE, CLV.O, CLV +rb-cli put fmt.g64.gcr.floppy.g64 f /HI + -> error: detecting partition table: Invalid MBR: invalid boot signature +``` + +So that error was never evidence of a GCR problem. Any verb that took the write +path produced it on any G64. Now the whole class — G64/G71, MSA, EDSK, +Apple-II `.dsk`: containers the read path decodes and the write path cannot +re-encode — is refused with `PERMISSION_DENIED` (4) and a message naming the +reason and the conversion that gets you an editable copy. Same shape as +[R-034](#r-034). Cases `fmt.g64.{standard-dump-opens,is-read-only}`. + +**What is still open, and why it cannot be closed here.** Whether the two named +dumps *decode* is unverified: neither is in the corpus or the dropbox, only the +working one is. `decode_g64_1541` already tolerates partial decode — it bails +only when *no* track yields sectors — so a protected disk with a readable +directory track should open today. Confirming that needs the files. + +**To close this:** drop `Protector II (must be write protected).g64` and +`American Express (Magic Disk 64 1989-09 Side 2) [patched].g64` into the fixture +dropbox. The decision above says what the answer must be; only the evidence is +missing. + +--- + + Of three real G64 files, one opened and two failed: | file | result | @@ -1020,6 +1763,25 @@ limitation rather than an acceptable boundary. `.d64` and `.g71` unaffected. ### R-001 — README partition-table list missing two schemes {#r-001} +**FIXED 2026-08-09**, and the report had aged in both directions by the time it +was picked up. AHDI had been added incidentally on 2026-08-04 by +`e0c9bf6` ("write Atari AHDI root sectors"), two days after this was filed — +so that half was already closed. Meanwhile a tenth variant, `Dsd`, had been +added and was missing too. The same drift, still drifting, which is the +argument for the test rather than the edit. + +README § Partition tables now has a row for **X68k** and for **DSD**. + +`tests/doc_parity.rs::readme_documents_every_partition_table_scheme` reads +`PartitionTable::ALL_TYPE_NAMES` and requires a row per scheme, so this cannot +recur silently. Two guards make that list trustworthy: `type_name`'s match is +exhaustive, so a new variant is a compile error, and a `#[cfg(test)]` +exhaustiveness guard in `src/partition/mod.rs` pins the variant count against +`ALL_TYPE_NAMES.len()`, so extending one without the other fails to build or +fails the test. Verified by deleting the X68k row and watching the test name it. + +--- + `src/partition/mod.rs` defines nine `PartitionTable` variants: `Mbr`, `Gpt`, `Apm`, `Rdb`, `Sgi`, `Sun`, `Ahdi`, `X68k`, `None`. README § Partition tables lists only MBR, GPT, APM, RDB, SGI, Sun and "None (superfloppy)" — **Atari @@ -1032,6 +1794,22 @@ warns about. ### R-002 — `src/fs/README.md` capability table is stale {#r-002} +**FIXED 2026-08-09 — by deleting the table, which was the second of the two +options this report offered.** Regenerating it from the `fs/mod.rs` dispatch +would have produced a forty-row table two directory levels below the code it +describes, i.e. the same race with a longer starting line. The file now points +at the two live sources instead: the top-level README's Filesystems table for +capabilities, and the dispatch functions in `mod.rs` for routing. + +One routing fact was worth keeping rather than deleting, and stayed: type byte +`0x07` covers both NTFS and exFAT, and `open_filesystem` disambiguates on the +OEM ID magic rather than the byte. + +`tests/doc_parity.rs::fs_readme_has_no_hand_kept_capability_table` fails if a +`(planned)` claim or a capability table grows back there. + +--- + Still lists ext2/3/4 as "No (planned)" for browsing, compaction and resize, and covers only six partition type bytes. The engine implements ext fully (`src/fs/ext.rs`, `ext_format.rs`, `ext_fsck.rs`, `ext_csum.rs`) plus about @@ -1040,6 +1818,361 @@ it and point at the README. --- +### R-036 — a missing image gets three different exit codes {#r-036} + +**FIXED 2026-08-10.** `inspect` was right and alone: it carried a per-verb +`if !args.image.exists()` check added with R-010, which no other verb copied. +The fix is one guard in the shared CLI resolver — `resolve::require_source_exists`, +called from the read-only, streaming and read-write entry points — so every +verb inherits it and one added later does too. `inspect`'s local check is gone. + +Three things are exempt, none of them a file whose absence can be judged: a raw +device (`\.\PhysicalDrive0` does not `exists()`, and backing up a disk is +the app's core job), an `rb://` remote reference, and a backup folder, which is +a directory and passes `exists()` anyway. + +`ls`, `du`, `fsck`, `show fs-info`, `locate` and `tar` now exit 3 with +`nosuch.img: no such file` instead of 1 with a platform-specific +`io::Error`. Six cases, red the same day they were written and green the next. + + +Found 2026-08-09 while closing R-005, which needed a verb whose missing-file +failure had a settled exit code. `exit.rs` reserves `NOT_FOUND` (3) for exactly +this — "image file missing" is the first example in its own doc comment — and +[R-010](#r-010) made `inspect` obey it. Nothing else does: + +| verb | exit | message | +|---|---|---| +| `inspect` | **3** | `nosuch.img: no such file` | +| `ls`, `fsck`, `du`, `get`, `locate`, `tar`, `resize`, `repack`, `backup`, `show fs-info` | **1** | `open nosuch.img: The system cannot find the file specified. (os error 2)` | + +**Correction to the first version of this table**, which listed `locate` and +`tar` at exit 2 and called that the worst of three answers. It was two answers, +not three: the survey invoked `tar` with two positionals when it takes three +(`IMAGE SRC OUT`), so clap rejected the command before the image was opened. +That 2 was correct — for a malformed command, not a missing file. Given correct +arity both verbs exit 1 like the rest. The case built on the bad invocation was +corrected with it. + +Two things follow from the message, not just the code: + +- It is a raw `std::io::Error`, so the text is **platform-specific**. Windows + says "The system cannot find the file specified. (os error 2)"; Unix says "No + such file or directory (os error 2)". Any case asserting on it has to be + written per-platform or not at all. +- It names the syscall (`open`) rather than the thing the user got wrong. + `inspect`'s "nosuch.img: no such file" is the shape to copy. + +**Why it matters.** This is the CLI contract the regression harness itself +depends on: a script cannot tell "the disk is missing" from "the disk is +corrupt" without switching on the code, which is the entire reason the table in +`exit.rs` exists. It is also the cheapest class of fix left — `exit::not_found` +already exists and is already used by `inspect`. + +**Cased 2026-08-10**, one per verb because each is a separate contract a +script switches on: `cli.exit.{ls,du,fsck,show-fs-info,locate,tar}-missing-image-is-not-found`, +all red, all citing this finding. `cli.exit.missing-image-file` is the green +counterpart pinning `inspect` at 3. + +--- + +### R-037 — `resize` cannot shrink safely, and shrinking destroyed data {#r-037} + +**FIXED 2026-08-09.** Found by reading [R-021](#r-021)'s closing note, which +claimed a shrink left "trailing slack, not damage". It did not, and that note +is corrected in place. + +The reproduction, on a 64 MB FAT volume holding a 30 MB file: + +``` +rb-cli resize v.img --size 16M + -> "FAT16: clusters 32695 -> 8119" + "resize complete", exit 0 + +rb-cli get v.img /D30.BIN out.bin + -> exit 0, out.bin is 16,629,760 bytes (the source is 30,720,000) + +rb-cli fsck v.img --checkonly + -> ERROR [ChainPointerInvalid] /D30.BIN: cluster 8120 has an invalid forward link + ERROR [SizeExceedsChain] /D30.BIN: size claims 15000 clusters but only 8119 are allocated +``` + +The FAT was rewritten for the smaller volume while the file's chain still ran +past the new end. `resize` exits 0, `ls` still reports the full size, and `get` +**also exits 0** while returning a short file — so nothing in the chain tells +the caller the data is gone. The image was never truncated either, so the +freed space was not even given back: the operation managed to be destructive +and useless at the same time. + +**The fix has three parts**, all in `src/cli/verbs/resize.rs`: + +1. **A data floor.** The filesystem is the only thing that knows where its data + ends, so it is asked: `Filesystem::last_data_byte` — "bytes from the + partition start needed to hold everything allocated". A shrink below that is + refused outright, `--confirm-shrink` or not, and the message names the + smallest safe size. The trait's default implementation returns `total_size`, + so a driver that does not override it refuses every shrink; that is the + correct default, because without an answer no shrink can be shown to be safe. +2. **`--confirm-shrink`.** A shrink that *is* safe still truncates the image, + which cannot be undone, so it has to be asked for. Growing needs no flag. +3. **Truncation.** After the filesystem resize commits, the image is truncated + to the new size — but only when the volume is the whole file + (`whole_file_path`), the same condition R-021 established for growing. A + partition inside a larger disk has data after it, so there the filesystem + shrinks and the image keeps its length, and the log says to follow up with + `partmap resize`. + +Verified in all three directions on a 64 MB FAT volume: shrinking to 16 MB with +40 MB of data is refused (exit 2, image untouched, fsck clean); shrinking to +40 MB without the flag is refused (exit 2, image untouched); shrinking to 40 MB +with the flag resizes the filesystem, truncates the image to 40 MiB, and the +file reads back byte-identical with a clean fsck. + +Cases: `resize.shrink.{refuses-cutting-live-data,needs-confirmation,keeps-data-and-truncates}`. + +**Two things this does not cover**, both deliberate and both worth a follow-up: + +- **The remote path.** `resize rb://host/img --size` goes through + `resize_remote_partition` and does not consult `last_data_byte`. Adding an + unverified guard to a path with no daemon running to test it against would be + worse than recording the gap. +- **The GUI.** "Resize Partitions…" calls `resize_filesystem_for` directly, so + the floor lives in the CLI verb rather than in the shared engine. Per + CLAUDE.md's shared-logic rule the check belongs in a core module both + surfaces call; that refactor is larger than this fix. + +--- + +### R-038 — a second implementation rejects every AFFS volume we write {#r-038} + +**NOT A LIVE DEFECT 2026-08-14 — it was already fixed when it was filed, and +the oracle was reading a stale artifact.** The rejection was real; the code it +indicted was not the code in the tree. Full evidence below, then the original +report unchanged. + +At HEAD, `xdftool` accepts a freshly written volume at every size tried — +1M, 2M, 3M, 4M, 8M, 16M, 32M — and accepts one that has had a `put` applied, +listing the file back with its date. + +**Re-run it** (needs `pip install amitools`; no emulator, so this one is +seconds rather than minutes): + +``` +for sz in 1M 2M 4M 8M 16M 32M; do + rb-cli new volume --size $sz affs "v$sz.hdf" + python regression-tests/oracles/amitools_affs.py "v$sz.hdf" || echo "FAIL $sz" +done +``` + +The three artifacts the finding was actually run against still fail, +identically, on all three hosts: + +``` +regression-tests/artifacts/{windows,linux,macos}/fs.affs/image.img + -> FSError: Bitmap Block Count Mismatch(15): got=2 want=1 +``` + +Those artifacts carry their own provenance. `meta.json` records +`git_sha 0563cb8`, dated 2026-08-08, and + +``` +git merge-base --is-ancestor a190182 0563cb8 -> NO +``` + +so they were produced **before** [a190182](#r-008a) ("fix(affs): size the +bitmap, find the root block, zero header_key") landed on 2026-08-10. The +oracle ran on 2026-08-12 against binaries two days older than the fix. + +**What the bytes say**, which also settles the hypothesis the original report +left open. Reading the root block of each: + +| | header_key | bm_pages set | geometry needs | +|---|---|---|---| +| Aug-8 artifact | 2048 (wrong) | 1 | 2 | +| HEAD, fresh | 0 (correct) | 2 | 2 | + +amitools' rule is in `ADFSBitmap`: it raises when the count it computes from +the volume geometry differs from the number of bitmap blocks it can actually +reach through the root's `bm_pages` and extension chain. The message reads +`got= want=` — so `got=2 want=1` means *geometry needs two, +the volume supplies one*. That is [R-008a](#r-008a) — "AFFS tail blocks above +4066 are uncovered" — seen from the outside, and a190182 fixed both halves of +it at once: the bitmap is now sized to the geometry, and `header_key` is 0. + +So the original report's hypothesis — "our root block declares a geometry +inconsistent with the bitmap we actually wrote" — was **right**, and was +already fixed two days before it was written down. + +**The process defect is the one worth keeping.** An oracle verdict was +attributed to current code when it was produced by old code, and nothing in +the run path noticed. Artifacts are not regenerated before an oracle runs and +carry no staleness check, so any oracle can indict a fix that already shipped. +Tracked as a suite change, not a code one. + +**This did not, by itself, close [R-020](#r-020).** amitools is a +reimplementation; that it accepts the volume is not proof a real Amiga mounts +it. R-020 was closed separately later the same day, by AmigaOS itself under +FS-UAE — which is the evidence this paragraph was holding out for. + +--- + +Found 2026-08-12, the first time an AFFS volume of ours was read by code that +is not ours. amitools' `xdftool` refuses it: + +``` +FSError: Bitmap Block Count Mismatch(15): got=2 want=1 +``` + +**The control is what makes this a finding rather than a tooling problem.** The +same xdftool reads `fs.affs.workbench13.hd.hdf` — a real Workbench 1.3 disk +from the corpus — perfectly: full directory tree, 3421 blocks, timestamps from +1988 and 1992. So xdftool handles AFFS, and it handles multi-bitmap-block +volumes, because that 3 MB fixture needs two of them. + +It fails identically on artifacts produced on **Windows, Linux and macOS**. The +formatter is deterministic and deterministically disagreed with. + +**What it means, stated carefully.** amitools is a reimplementation, so this is +not proof a real Amiga refuses the volume — that is still +[R-020](#r-020)'s question, and still needs an emulator or hardware. What it +*is*: the first independent read of our AFFS output. Every AFFS check before +this was our own formatter agreeing with our own fsck, which is the condition +R-020 has been stuck in. + +**A live hypothesis, not a conclusion.** For a 2 MB volume — 4096 blocks of 512 +bytes — the bitmap must cover blocks 2..4095, i.e. 4094 bits. One bitmap block +holds `(512 - 4) * 8 = 4064` bits, so two are needed, and two is what we write. +amitools computes that one is wanted. Since amitools reads a real two-bitmap +volume without complaint, the likeliest reading is that our root block declares +a geometry inconsistent with the bitmap we actually wrote — an internally +contradictory volume, which is exactly the shape that produces "Not a DOS +disk". That is a hypothesis; nobody has read our root block against the spec +yet. + +Note this is adjacent to [R-008a](#r-008a), which added the second bitmap block +for volumes above 4066 blocks. That fix may be correct and the root block not +updated to match, or the fix may itself be wrong. Do not assume which. + +Oracle: `amitools`, check `oracles/amitools_affs.py`. It is `structural`, not +`authoritative`, for the reason above. + +--- + +### R-039 — IRIX's own fsck reports BAD FREE LIST on every EFS volume we write {#r-039} + +**FIXED 2026-08-15. The free-space bitmap is LSB-first and we wrote it +MSB-first — in the reader as well as the writer.** + +That symmetry is the whole story. Our formatter and our fsck agreed with each +other perfectly, because both were wrong in the same direction, and no +self-built fixture could ever catch it. It took the vendor's tool to see it. + +**How the bit order was established**, rather than guessed. `mkfs_efs` was run +on the same scratch device to produce a reference volume, and that volume was +pulled back to the host and decoded both ways. Read LSB-first it decomposes +into exactly the geometry its own superblock declares: + +``` +run 0..834 = boot(0) + superblock(1) + bitmap(2..33 = 32 blocks) + + inodes(34..812 = cgisize 779) + data(813..834 = the 22 used) +``` + +`0..33` is precisely `firstcg=34`, and each of the six cylinder groups +(21872, 43710, 65548, 87386, 109224) carries exactly `cgisize=779` allocated +blocks. Read MSB-first the same bytes give 16 ragged runs on no boundary at +all — 776-block groups, fragments at 22653..22655 and 43704..43705. + +The tail confirms it independently: `bmsize` is 16383 bytes = 131064 bits for +`fs_size` 131062 blocks, so the last two bits are spare and must read as +in-use. IRIX writes `0x3F` in that byte — bits 6 and 7 clear counting from the +LSB. We wrote `0xFC`, which is the same statement made backwards. + +**The fix** flips the bit computation at all 24 sites across `efs.rs`, +`efs_fsck.rs` and `efs_resize.rs`, and pins the convention in +`bitmap_bit_order_is_lsb_first_per_irix_mkfs` — a test that hard-codes real +numbers off the `mkfs_efs` volume rather than deriving them, because a +self-consistent convention cannot be tested by self-consistent fixtures. + +**Verified end to end against IRIX**, not just against fsck: + +``` +** Phase 5 - Check Free List +1 files 1 blocks 126935 free <- no BAD FREE LIST +``` + +then `mount -t efs` succeeded, `df -k` reported it as a 63468 KB EFS volume, +IRIX wrote a file and a subdirectory into it, unmounted, and fsck came back +clean at `4 files 3 blocks 126933 free`. Reading that volume back with rb-cli +lists both entries and returns the file's contents verbatim. + +**This was not only a write bug.** The reader used the same inverted order, so +free-space accounting on any real IRIX EFS disk was wrong too, and the +allocator could have handed out blocks IRIX considered in use. Nothing shipped +had exercised that path against a real volume, but the exposure was there. + +**Resolved in passing:** the entry's second observation — that fsck showed the +volume name as `rusty-`, truncated from `rusty-backup` — is correct behaviour, +not a defect. `fname` in the EFS superblock is a 6-byte field. + +Reproduce with `regression-tests/oracles/iris/` (see the Status note there); +the whole loop is about a minute once the guest is booted and snapshotted. + +--- + +Found 2026-08-13, the first authoritative oracle result this project has ever +had: IRIX 6.5.22 running under the Iris emulator, checking our EFS volume with +its own `/sbin/fsck`. + +``` +fsck: checking /dev/rdsk/dks0d2s0 (NO WRITE). Name: rusty- Volume: rusty- +** Phase 1 - Check Blocks and Sizes +** Phase 2 - Check Pathnames +** Phase 3 - Check Connectivity +** Phase 4 - Check Reference Counts +** Phase 5 - Check Free List +BAD FREE LIST +1 files 1 blocks 3964 free +``` + +Phases 1 through 4 pass. The structure is sound — IRIX walks our blocks, +pathnames, connectivity and reference counts without complaint, and reads the +volume name. Only the **free list** is wrong. + +**The control makes it a defect rather than an artifact.** `mkfs_efs` was run on +the same device, in the same guest, through the same scratch path, and the +resulting volume passes the identical check: + +``` +** Phase 5 - Check Free List +2 files 22 blocks 126332 free +``` + +No BAD FREE LIST. Same device node, same fsck, same session — the only variable +is who wrote the filesystem. + +**Why this one is different from every EFS check we had.** `edit.efs.put-get`, +`fs.new-volume.efs` and `roundtrip.efs.raw` are all green, and all of them are +our code checked by our code. This is the vendor's own tool, which is what +`strength = "authoritative"` means in the registry. It is the EFS equivalent of +what [R-038](#r-038) did for AFFS, one level stronger: amitools is a +reimplementation, IRIX is the implementation. + +**Not yet investigated:** where our free list diverges. EFS keeps free extents +per cylinder group; `mkfs_efs` on the same 64 MB device reported `ncg=6`, +`bitmap blocks=32`, `cgfsize=21838`, which is a useful reference geometry to +compare ours against. + +**A second observation, unconfirmed as a separate defect:** fsck shows the +volume name as `rusty-`, truncated from our default `rusty-backup`. EFS volume +names are short, so this may be correct truncation rather than a bug — but +nothing has checked what the limit is or whether we truncate the same way IRIX +does. + +Reproduce: see the `iris` oracle's notes in `data/oracles.toml` for the working +invocation, config and scratch-volume setup. + +--- + ## Regression coverage Which finding is guarded by which case, so a fix cannot silently regress. @@ -1047,21 +2180,40 @@ Run `rb-regress run --tiers 0-4` to check them all. | Finding | Case | State | |---------|------|-------| -| R-003 | `cli.envelope.ls-supports-format` | red | -| R-004 | `cli.exit.{csv,tsv}-on-nested-verb-is-usage-error` | red | -| R-005 | `cli.envelope.error-envelope-on-failure` | red | -| R-006 | `fs.new-volume.prodos-default-name` | red | +| R-003 | `cli.envelope.ls-supports-format` | **green — fixed** | +| R-004 | `cli.exit.{csv,tsv}-on-nested-verb-is-usage-error` | **green — fixed** | +| R-005 | `cli.envelope.error-envelope-on-failure` | **green — fixed** | +| R-006 | `fs.new-volume.prodos-default-name` | **green — fixed** | | R-007 | `fs.new-volume.ntfs{,.2m-fsck,.32m-fsck}` | **green — fixed** | -| R-008a | `fs.new-volume.affs.bitmap-boundary-plus-one` | red | -| R-008b | `fs.new-volume.affs.{4m,32m}` | red | +| R-008a | `fs.new-volume.affs{,.bitmap-boundary-plus-one}` | **green — fixed** | +| R-008b | `fs.new-volume.affs.{4m,32m}` | **green — fixed** | | R-009 | `fs.read.{jfs,reiserfs,ufs1,ufs2}` | **green — fixed** | -| R-010 | `cli.flags.inspect-accepts-fs-type` | red | +| R-010 | `cli.flags.inspect-accepts-fs-type` | **green — fixed** | +| F-009 | `edit.sfs.put-get` | red — the documented ceiling, was R-032 | +| R-029 | `edit.real.efs-small` | **green — fixture defect, case corrected** | +| R-013 | `fs.detect.ufs-{solaris-entry-types,no-absurd-sizes}` | **green — fixed** | +| R-028 | `edit.apple-dos.put-get` | **green — fixed** | +| R-031 | `edit.real.apple-dos-invaders` | **green — not a defect** | +| R-030 | `edit.real.affs-workbench13` | **green — fixed** | | R-011 | `fmt.g64.standard-dump-opens` | green — **pins the working half only** | -| R-012 | `optical.cdda.no-data-track-opens` | red | +| R-012 | `optical.cdda.no-data-track-opens` | **green — fixed upstream** | | R-013 | `fs.detect.ufs-{solaris-entry-types,no-absurd-sizes}` | red | -| R-015 | `optical.cue.unpadded-track-number` | red | -| R-016 | `backup.container.{chd,vhd-dynamic,qcow2,vmdk-sparse}` | red | +| R-015 | `optical.cue.unpadded-track-number` | **green — fixed upstream** | +| F-008 | `backup.container.{chd,vhd-dynamic,qcow2,vmdk-sparse}` | red — a feature gap, was R-016 | +| R-001 | `doc_parity::readme_documents_every_partition_table_scheme` | **green — fixed** | +| R-002 | `doc_parity::fs_readme_has_no_hand_kept_capability_table` | **green — fixed** | +| R-018 | `doc_parity::contributing_vintage_features_match_ci` | **green — fixed** | +| R-021 | `resize.to-explicit-size` | **green — fixed** | +| R-023 | `resize.repack.{keeps-data,refuses-plain-fat}` | **green — fixed** | +| R-022 | `roundtrip.hpfs.raw`, `fs.detect.hpfs-{bare-volume,backup-is-not-empty}` | **green — fixed** | +| R-036 | `cli.exit.{ls,du,fsck,show-fs-info,locate,tar}-missing-image-is-not-found` | **green — fixed** | +| R-011 | `fmt.g64.{standard-dump-opens,is-read-only}` | **green — read-only half fixed** | +| R-037 | `resize.shrink.{refuses-cutting-live-data,needs-confirmation,keeps-data-and-truncates}` | **green — fixed** | | R-017 | `fs.detect.sfs-bare-volume` | **green — fixed** | +| R-025 | `subcmd.squashfs.put-rebuilds`, `meta.xattr.set-list-rm` | red — Windows only | +| R-026 | `subcmd.show.partmap` | **green — fixed** | +| R-027 | `read.apfs.apple-gpt` | **green — fixed** | +| R-034 | `edit.readonly.{lisa,alto}-refuses-a-write` | **green — fixed** | Cases assert the **intended** behaviour, so each is red until its finding is fixed and green afterwards. Never "fix" one by asserting the broken @@ -1076,33 +2228,42 @@ already works. **Not covered:** -- **R-001, R-002** — documentation drift. Not expressible as a CLI case; they - need a source-parity test comparing the README tables against the - `PartitionTable` enum and the `fs/mod.rs` dispatch. +- ~~**R-001, R-002**~~ — **now covered.** `tests/doc_parity.rs` is the + source-parity test this entry asked for. It is a `cargo test`, not an + `rb-regress` case, because the claim is *about* the binary rather than + something the binary does — the suite runs `rb-cli` and cannot see a stale + markdown table. R-018 is covered by the same file. - **R-011** — only the working half is pinned. No case asserts that copy-protected G64 dumps open, because whether they should is undecided; asserting either way would prejudge it. -- **R-018** — a documentation failure, not runtime behaviour, and the suite - runs the modern binary. A docs-parity test comparing CONTRIBUTING.md's - feature list against the workflow's would guard it; that is the same - source-parity test R-001 / R-002 need, so it belongs with them. +- ~~**R-018**~~ — **now covered**, by exactly the test that entry predicted: + `doc_parity::contributing_vintage_features_match_ci` asserts CONTRIBUTING.md's + vintage feature list appears verbatim in `.github/workflows/release.yml`. - **R-014** — a lint failure, not runtime behaviour. The pre-commit hook is itself the regression guard: it runs `clippy --all-targets -- -D warnings` on every commit, so a reintroduction cannot be committed. ## Suggested order -0. ~~**R-014**~~ — done; commits work without `--no-verify` again. -0b. ~~**R-018**~~ — done; the verification command works on Windows again. -1. ~~**R-009** / **R-017**~~ — done; five filesystems' worth of tier-2 - coverage went green. -2. **R-008b** — a panic with no file produced is the worst failure mode here, +Rewritten 2026-08-09; 21 of 35 findings are closed and the ordering that +remains is different from the one that got us here. + +1. **R-008b** — a panic with no file produced is the worst failure mode left, and R-008a shares its fix. -3. ~~**R-007**~~ — done; the formatter was already correct when re-verified. -4. **R-013** — wrong entry types and an absurd size are user-visible +2. ~~**R-033**~~ — done 2026-08-10. The prediction held exactly: R-022's + shape, a bare volume falling through to the MBR parse because no probe in + `detect_superfloppy` claimed it. The fix was a dozen lines beside the HPFS + probe. +3. **R-013** — wrong entry types and an absurd size are user-visible immediately. -5. **R-005**, **R-004**, **R-003** — the CLI contract group; cheap, and the - regression harness depends on that contract being true. -6. **R-006** — a one-line default change. -7. **R-001**, **R-002** — fold into the next docs commit. -8. **R-011** — decide scope first. +4. **R-024** — the AFFS editor. Distinct from R-008 (formatter) and R-020 + (root block); do not conflate them. +5. **R-011** — decide scope first. The last open decision. +6. Everything else is Tranche C in + [`regression-fix-prompt.md`](regression-fix-prompt.md): the symptom is + recorded, the cause is not, and the investigation is the deliverable. + +**Two of the closed findings had the wrong cause on file** — R-023 ("every file +is gone"; nothing was lost) and R-022 ("not byte-identical"; the backup was +empty). Both had a one-command control that settled it. Reproduce and *measure* +before fixing anything below. diff --git a/docs/cli-html-help/inspect.html b/docs/cli-html-help/inspect.html index 98053134..87e67722 100644 --- a/docs/cli-html-help/inspect.html +++ b/docs/cli-html-help/inspect.html @@ -23,6 +23,14 @@

Options

Password for encrypted containers (currently: WinImage IMZ, and password-protected `.zip` disks)
--inside
For a `.zip` holding more than one disk image, the archive entry to open (e.g. `--inside backup.img`). Matched by exact name, then case- insensitively, then by basename. Ignored for non-zip sources
+
--fs-type
+
Force a specific filesystem dispatch. The main use is `cpm:<preset>` for CP/M images (which have no on-disk signature). Valid CP/M presets: `amstrad_data`, `amstrad_sys`, `amstrad_pcw`, `einstein`, `svi328_cpm`, `altair_8in`, `altair_cf`, `multicomp`, `zxplus3`. Other strings (e.g. `human68k`, `qdos`) are also accepted and forwarded to the partition_type_string dispatch
+
--carve-full
+
Scan the **entire** image for recoverable text in the synthetic carve view (used for disks with no recognized filesystem — e.g. custom bootblock Amiga "NDOS" disks). By default the carve view only scans the first 10 MB. No effect on disks with a real filesystem
+
--expect-fs
+
Assert the disk carries this filesystem, e.g. `--expect-fs "DOS 3.3"`. Exits non-zero when no partition matches. Case, spacing and punctuation are ignored; the comparison is exact after that, so `FAT` does not satisfy `exFAT`
+
--expect-layout
+
Assert the disk's shape: `superfloppy` (a filesystem at sector 0, no table), `partitioned` (any table), or a scheme by name — `mbr`, `gpt`, `apm`, `rdb`, `sgi`, `sun`, `ahdi`, `x68k`, `dsd`, `none`. An unrecognised word is a usage error, not a failed assertion
Auto-generated from the clap argument definitions in src/cli/. Re-run cargo run --example generate_cli_docs after grammar changes. rb-cli version reflects the binary built when this bundle was generated.
diff --git a/docs/cli-html-help/ls.html b/docs/cli-html-help/ls.html index 8ead2247..274cf153 100644 --- a/docs/cli-html-help/ls.html +++ b/docs/cli-html-help/ls.html @@ -33,6 +33,8 @@

Options

Password for encrypted containers (WinImage IMZ, password-protected `.zip` disks) or an encrypted filesystem (APFS FileVault — the volume password or personal recovery key)
--inside
For a `.zip` holding more than one disk image, the archive entry to open (e.g. `--inside backup.img`). Matched by exact name, then case- insensitively, then by basename. Ignored for non-zip sources
+
--format
+
Output format. `ls` is flat-tabular, so csv and tsv are in scope alongside json and yaml
--fs-type
Force a specific filesystem dispatch. The main use is `cpm:<preset>` for CP/M images (which have no on-disk signature). Valid CP/M presets: `amstrad_data`, `amstrad_sys`, `amstrad_pcw`, `einstein`, `svi328_cpm`, `altair_8in`, `altair_cf`, `multicomp`, `zxplus3`. Other strings (e.g. `human68k`, `qdos`) are also accepted and forwarded to the partition_type_string dispatch
--carve-full
diff --git a/docs/cli-html-help/optical-browse.html b/docs/cli-html-help/optical-browse.html index ba1db82e..8d7010fb 100644 --- a/docs/cli-html-help/optical-browse.html +++ b/docs/cli-html-help/optical-browse.html @@ -23,6 +23,8 @@

Options

Per-file content hash to attach to each file entry. Structured output only (`--format json`). Currently only `sha256`
--filesystem
Which filesystem to browse on a hybrid Mac/PC disc. `auto` (default) opens the primary (ISO 9660); `hfs` opens the Apple HFS side; `iso` forces the ISO 9660 tree. See `optical info` to see what a disc carries
+
--filesystem-index
+
Which filesystem to open when a disc carries more than one of the same kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the list `optical info` prints. Overrides `--filesystem`
Auto-generated from the clap argument definitions in src/cli/. Re-run cargo run --example generate_cli_docs after grammar changes. rb-cli version reflects the binary built when this bundle was generated.
diff --git a/docs/cli-html-help/optical-du.html b/docs/cli-html-help/optical-du.html index 4ef127ee..d7e4a96a 100644 --- a/docs/cli-html-help/optical-du.html +++ b/docs/cli-html-help/optical-du.html @@ -27,6 +27,8 @@

Options

Output format
--filesystem
Which filesystem to measure on a hybrid Mac/PC disc. `auto` (default) opens the primary (ISO 9660); `hfs` opens the Apple HFS side — the one carrying resource forks. See `optical info` for what a disc holds
+
--filesystem-index
+
Which filesystem to open when a disc carries more than one of the same kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the list `optical info` prints. Overrides `--filesystem`
Auto-generated from the clap argument definitions in src/cli/. Re-run cargo run --example generate_cli_docs after grammar changes. rb-cli version reflects the binary built when this bundle was generated.
diff --git a/docs/cli-html-help/optical-extract.html b/docs/cli-html-help/optical-extract.html index 72e7711c..6de9a3ee 100644 --- a/docs/cli-html-help/optical-extract.html +++ b/docs/cli-html-help/optical-extract.html @@ -9,7 +9,7 @@

rb-cli optical extract

Extract files from an optical disc image into a host folder

Usage

-
Usage: extract [OPTIONS] --to <TO> <SOURCE>
+
Usage: extract [OPTIONS] <SOURCE>

Arguments

<SOURCE>
@@ -18,13 +18,23 @@

Arguments

Options

--to
-
Destination folder (created if absent)
+
Destination folder (created if absent). Mutually exclusive with `--tar`; exactly one of the two is required
+
--tar
+
Write a `.tar` / `.tar.gz` / `.tar.zst` instead of loose files
+
--path
+
Extract only this path from the disc instead of the whole tree. Disc-relative, e.g. `/DOCS/README.TXT` or `/DOCS`. A file extracts on its own; a directory extracts the files directly inside it, and its subdirectories too when `--recursive` is given
+
-r / --recursive
+
Include subdirectories when `--path` names a directory. Whole-disc extraction (no `--path`) always recurses and ignores this
+
--preserve-permissions
+
Apply the POSIX mode a disc records (Rock Ridge, HFS+, EFS) to the extracted files. Unix hosts only — on Windows the bits have nowhere to go, so use `--tar`, which carries them regardless of host
--resource-forks
How to handle HFS resource forks. Ignored on non-HFS discs. Defaults to `appledouble`, or `[optical] resource-forks` from the config file when set
--on-collision
What to do when two names on a **case-sensitive** disc (UFS, NeXT, Rock Ridge, …) collide only by case on a **case-insensitive** destination (e.g. macOS). Defaults to `rename`, or `[optical] on-collision` from the config. Ignored when the destination is case-sensitive — everything extracts verbatim there
--filesystem
Which filesystem to extract from on a hybrid Mac/PC disc. `auto` (default) uses the primary (ISO 9660); `hfs` extracts the Apple HFS side; `iso` forces the ISO 9660 tree. See `optical info`
+
--filesystem-index
+
Which filesystem to open when a disc carries more than one of the same kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the list `optical info` prints. Overrides `--filesystem`
Auto-generated from the clap argument definitions in src/cli/. Re-run cargo run --example generate_cli_docs after grammar changes. rb-cli version reflects the binary built when this bundle was generated.
diff --git a/docs/cli-html-help/resize.html b/docs/cli-html-help/resize.html index dc10a74a..7d24e443 100644 --- a/docs/cli-html-help/resize.html +++ b/docs/cli-html-help/resize.html @@ -9,7 +9,7 @@

rb-cli resize

Resize the filesystem at IMG@N to a new size (FAT/NTFS/exFAT/HFS+/ ext/btrfs/SFS/PFS3/AFFS/EFS — whichever magic matches)

Usage

-
Usage: resize --size <SIZE> <IMAGE>
+
Usage: resize [OPTIONS] --size <SIZE> <IMAGE>

Arguments

<IMAGE>
@@ -19,6 +19,8 @@

Options

--size
New filesystem size in bytes. Accepts suffixes (`K`, `M`, `G`)
+
--confirm-shrink
+
Required to shrink. Growing needs no flag; shrinking truncates the image, which is not reversible, so it has to be asked for. A shrink that would cut into live data is refused with or without this
Auto-generated from the clap argument definitions in src/cli/. Re-run cargo run --example generate_cli_docs after grammar changes. rb-cli version reflects the binary built when this bundle was generated.
diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 5f677708..dc70835f 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -847,6 +847,10 @@ Usage: inspect [OPTIONS] - `--format` — Output format - `--password` — Password for encrypted containers (currently: WinImage IMZ, and password-protected `.zip` disks) - `--inside` — For a `.zip` holding more than one disk image, the archive entry to open (e.g. `--inside backup.img`). Matched by exact name, then case- insensitively, then by basename. Ignored for non-zip sources +- `--fs-type` — Force a specific filesystem dispatch. The main use is `cpm:` for CP/M images (which have no on-disk signature). Valid CP/M presets: `amstrad_data`, `amstrad_sys`, `amstrad_pcw`, `einstein`, `svi328_cpm`, `altair_8in`, `altair_cf`, `multicomp`, `zxplus3`. Other strings (e.g. `human68k`, `qdos`) are also accepted and forwarded to the partition_type_string dispatch +- `--carve-full` — Scan the **entire** image for recoverable text in the synthetic carve view (used for disks with no recognized filesystem — e.g. custom bootblock Amiga "NDOS" disks). By default the carve view only scans the first 10 MB. No effect on disks with a real filesystem +- `--expect-fs` — Assert the disk carries this filesystem, e.g. `--expect-fs "DOS 3.3"`. Exits non-zero when no partition matches. Case, spacing and punctuation are ignored; the comparison is exact after that, so `FAT` does not satisfy `exFAT` +- `--expect-layout` — Assert the disk's shape: `superfloppy` (a filesystem at sector 0, no table), `partitioned` (any table), or a scheme by name — `mbr`, `gpt`, `apm`, `rdb`, `sgi`, `sun`, `ahdi`, `x68k`, `dsd`, `none`. An unrecognised word is a usage error, not a failed assertion ### `install-completions` @@ -903,6 +907,7 @@ Usage: ls [OPTIONS] [PATH] - `-o` / `--owner` — Show each entry's Unix permissions and owner. On a Linux/Unix image the owner ids are resolved to names via the image's own `/etc/passwd` and `/etc/group` (falling back to the raw numbers where there's no entry) - `--password` — Password for encrypted containers (WinImage IMZ, password-protected `.zip` disks) or an encrypted filesystem (APFS FileVault — the volume password or personal recovery key) - `--inside` — For a `.zip` holding more than one disk image, the archive entry to open (e.g. `--inside backup.img`). Matched by exact name, then case- insensitively, then by basename. Ignored for non-zip sources +- `--format` — Output format. `ls` is flat-tabular, so csv and tsv are in scope alongside json and yaml - `--fs-type` — Force a specific filesystem dispatch. The main use is `cpm:` for CP/M images (which have no on-disk signature). Valid CP/M presets: `amstrad_data`, `amstrad_sys`, `amstrad_pcw`, `einstein`, `svi328_cpm`, `altair_8in`, `altair_cf`, `multicomp`, `zxplus3`. Other strings (e.g. `human68k`, `qdos`) are also accepted and forwarded to the partition_type_string dispatch - `--carve-full` — Scan the **entire** image for recoverable text in the synthetic carve view (used for disks with no recognized filesystem — e.g. custom bootblock Amiga "NDOS" disks). By default the carve view only scans the first 10 MB. No effect on disks with a real filesystem @@ -1321,6 +1326,7 @@ Usage: browse [OPTIONS] - `--format` — Output format. `text` (default) prints the human file tree unchanged; `json` / `yaml` emit a machine-readable, deterministically path-sorted listing - `--hash` — Per-file content hash to attach to each file entry. Structured output only (`--format json`). Currently only `sha256` - `--filesystem` — Which filesystem to browse on a hybrid Mac/PC disc. `auto` (default) opens the primary (ISO 9660); `hfs` opens the Apple HFS side; `iso` forces the ISO 9660 tree. See `optical info` to see what a disc carries +- `--filesystem-index` — Which filesystem to open when a disc carries more than one of the same kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the list `optical info` prints. Overrides `--filesystem` ### `optical convert` @@ -1370,13 +1376,14 @@ Usage: du [OPTIONS] [PATH]... - `--json` — Emit machine-readable JSON. Shorthand for `--format json` - `--format` — Output format - `--filesystem` — Which filesystem to measure on a hybrid Mac/PC disc. `auto` (default) opens the primary (ISO 9660); `hfs` opens the Apple HFS side — the one carrying resource forks. See `optical info` for what a disc holds +- `--filesystem-index` — Which filesystem to open when a disc carries more than one of the same kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the list `optical info` prints. Overrides `--filesystem` ### `optical extract` Extract files from an optical disc image into a host folder ``` -Usage: extract [OPTIONS] --to +Usage: extract [OPTIONS] ``` **Arguments** @@ -1385,10 +1392,15 @@ Usage: extract [OPTIONS] --to **Options** -- `--to` — Destination folder (created if absent) +- `--to` — Destination folder (created if absent). Mutually exclusive with `--tar`; exactly one of the two is required +- `--tar` — Write a `.tar` / `.tar.gz` / `.tar.zst` instead of loose files +- `--path` — Extract only this path from the disc instead of the whole tree. Disc-relative, e.g. `/DOCS/README.TXT` or `/DOCS`. A file extracts on its own; a directory extracts the files directly inside it, and its subdirectories too when `--recursive` is given +- `-r` / `--recursive` — Include subdirectories when `--path` names a directory. Whole-disc extraction (no `--path`) always recurses and ignores this +- `--preserve-permissions` — Apply the POSIX mode a disc records (Rock Ridge, HFS+, EFS) to the extracted files. Unix hosts only — on Windows the bits have nowhere to go, so use `--tar`, which carries them regardless of host - `--resource-forks` — How to handle HFS resource forks. Ignored on non-HFS discs. Defaults to `appledouble`, or `[optical] resource-forks` from the config file when set - `--on-collision` — What to do when two names on a **case-sensitive** disc (UFS, NeXT, Rock Ridge, …) collide only by case on a **case-insensitive** destination (e.g. macOS). Defaults to `rename`, or `[optical] on-collision` from the config. Ignored when the destination is case-sensitive — everything extracts verbatim there - `--filesystem` — Which filesystem to extract from on a hybrid Mac/PC disc. `auto` (default) uses the primary (ISO 9660); `hfs` extracts the Apple HFS side; `iso` forces the ISO 9660 tree. See `optical info` +- `--filesystem-index` — Which filesystem to open when a disc carries more than one of the same kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the list `optical info` prints. Overrides `--filesystem` ### `optical info` @@ -1750,7 +1762,7 @@ Usage: repack [OPTIONS] Resize the filesystem at IMG@N to a new size (FAT/NTFS/exFAT/HFS+/ ext/btrfs/SFS/PFS3/AFFS/EFS — whichever magic matches) ``` -Usage: resize --size +Usage: resize [OPTIONS] --size ``` **Arguments** @@ -1760,6 +1772,7 @@ Usage: resize --size **Options** - `--size` — New filesystem size in bytes. Accepts suffixes (`K`, `M`, `G`) +- `--confirm-shrink` — Required to shrink. Growing needs no flag; shrinking truncates the image, which is not reversible, so it has to be asked for. A shrink that would cut into live data is refused with or without this ### `restore` diff --git a/docs/missing_features_from_regression.md b/docs/missing_features_from_regression.md index 1dfe4888..8a22f2dd 100644 --- a/docs/missing_features_from_regression.md +++ b/docs/missing_features_from_regression.md @@ -12,15 +12,30 @@ concrete reason to. | # | Feature | Area | Blocks | |---|---------|------|--------| -| [F-001](#f-001) | `optical extract` cannot extract a single path | `src/cli/verbs/optical.rs` | fixture harvesting from ISOs | +| ~~F-001~~ | ~~`optical extract` cannot extract a single path~~ — **SHIPPED** 2026-08-09 | `src/cli/verbs/optical.rs` | — | | ~~F-002~~ | ~~CloneCD not supported~~ — **retracted, it is supported** | — | — | | [F-003](#f-003) | PFS3 / SFS builders exist but are not on the CLI | `src/cli/verbs/new.rs` | two Amiga fixture gaps | -| [F-004](#f-004) | `show partmap` is APM-only | `src/cli/verbs/show.rs` | scripted partition inspection | +| [F-005](#f-005) | Optical extract is CLI-only; the GUI cannot pull one file | `src/optical/browse_view.rs` | GUI parity with `optical extract` | +| [F-006](#f-006) | IRIX support-disk building / browsing is thin | `src/cli/verbs/new_sgi_cdrom.rs` | bootable IRIX disc work — **needs scope** | +| [F-007](#f-007) | No optical fixture with nested directories | `regression-tests/` | verifying `--path DIR --recursive` | +| [F-008](#f-008) | `backup` reads only flat-layout sources | `src/cli/verbs/backup.rs` | backing up CHD / dynamic VHD / QCOW2 / VMDK — **four red cases** | +| [F-009](#f-009) | SFS editor writes single-leaf extent b-trees only | `src/fs/sfs.rs` | editing any real-sized SFS volume | +| ~~F-004~~ | ~~`show partmap` is APM-only~~ — **SHIPPED** 2026-08-08, same gap as R-026 | `src/cli/verbs/show.rs` | — | --- ## F-001 — `optical extract` is whole-disc only {#f-001} +**SHIPPED 2026-08-09.** `--path` takes a file or a folder, `--recursive` +decides whether a named folder descends, `--tar` archives instead of writing +loose files, and `--preserve-permissions` applies the disc's POSIX mode. +Nine cases in `cases/tier3/optical-extract.toml`, the single-file one anchored +on a sha256 so a truncated read fails on content rather than on exit code. + +The Windows-illegal-names observation below is largely addressed too: a +per-path extract sidesteps it for the common case, and `--tar` sidesteps it +entirely, since a tar entry stores a name NTFS would refuse. + ``` Usage: rb-cli optical extract [OPTIONS] --to ``` @@ -117,6 +132,13 @@ handlers), which is the actual win. It does not by itself prove anything. ## F-004 — `show partmap` only understands APM {#f-004} +**SHIPPED 2026-08-08.** It detects the partition table first; APM keeps its +full DDR and driver-descriptor rendering, every other table gets the generic +partition list. Tracked twice, here and as +[R-026](Regression_Bugs.md#r-026) — the defect / missing-feature split is not +always obvious from a symptom, and this was genuinely both: the verb never +claimed to read other schemes, and its error blamed the image for it. + `docs/cli-reference.md` is honest about this — "Print the partition table of a disk image (APM-only today)" — so it is a scoped feature rather than a defect. On anything else it fails: @@ -141,3 +163,179 @@ Two things would help, in order of cost: 2. **The feature.** Extend to MBR, GPT, RDB, SGI, Sun, AHDI and X68K, which are all already parsed by the engine — `inspect` prints them today, so the data is there and only the structured emitter is missing. + +--- + +## F-005 — the GUI cannot extract a single file from a disc {#f-005} + +`optical extract` grew `--path`, `--recursive`, `--tar`, +`--preserve-permissions`, `--filesystem` and `--filesystem-index` (F-001). +None of it is reachable from the GUI, which can browse an optical disc but +offers no way to pull anything out of it. + +**Why it is small.** The capability is already there: +`src/optical/browse_view.rs` calls `fs.read_file(entry)` in three places, so +the GUI can already read one file out of a disc — it just never offers to save +one. This is wiring, not new engine work. + +**What it needs:** + +- an extract action on the selected browse entry, and on a selected folder +- a destination chooser, with the folder-vs-archive choice `--tar` introduced +- the `--filesystem` / `--filesystem-index` selector surfaced, **without which + the GUI can only ever reach one side of a hybrid Mac/PC disc** — the reason + this is listed rather than left implicit +- CLAUDE.md's pre-commit doc sync applies: a new dialog wants a README + Inspect-tab bullet + +## F-006 — IRIX support-disk building and browsing is thin {#f-006} + +`optical new sgi-efs` builds an SGI volume header with EFS in slot 7, and +takes `--from-dir`, `--expand-archives` and `--flatten-folders`. What it does +not do is treat the disc as a *bootable* IRIX support disc. + +**Needs scope before any work starts.** "Extending the IRIX support disks" +could reasonably mean any of: + +1. **Volume-header executables as first-class entries.** An SGI volume header + carries standalone programs (`sash`, `ide`, `/unix`) outside the EFS + filesystem. `inspect` reports the header; nothing lists, extracts or + replaces those entries. +2. **Building a disc that actually boots.** Writing the right volume-header + entries and boot fields for real hardware or a MiSTer core. +3. **Richer `--from-dir` ergonomics** for laying out an inst-ready tardist + tree. + +(1) is self-contained and testable from a fixture. (2) cannot be verified +without hardware — every SGI oracle is `skip-manual`, so it would ship +unproven. That used to be Amiga's position too; it no longer is, and the way +out is worth copying. [R-020](Regression_Bugs.md#r-020) was closed by writing +one emulator harness (`oracles/fsuae/affs_mount.py`, host directory as the +verdict channel), and `iris` does the same job for IRIX — so the SGI oracle is +a harness someone has not written yet rather than a thing that cannot be done. +(3) is ergonomics on an existing path. + +## F-007 — no optical fixture has nested directories {#f-007} + +`--path DIR --recursive` is implemented and unverified. The discs in the +corpus are flat or single-file at the root: +`optical.iso9660.joliet.cd` holds one file with no directories at all, which +is why it makes such a clean single-file test and such a poor recursion test. + +**What would help:** a case against `optical.hfs.opentransport.cd` or the +CloneCD Bookshelf set, both of which have real trees, asserting that +`--recursive` descends and that its absence stops at one level. The fixtures +are already catalogued — only the case is missing. It belongs in +`cases/tier3/optical-extract.toml` beside the nine that exist. + +## F-008 — `backup` reads only flat-layout sources {#f-008} + +Filed as defect [R-016](Regression_Bugs.md#r-016) until 2026-08-09. +**Reclassified**: `backup` has never claimed to decode containers, so this is +a capability the engine lacks, not code disagreeing with itself. The four +cases keep their assertions — they describe the behaviour we want — and now +cite this entry. + +`backup --help` documents SOURCE as "an image file or a block-device path", +and `inspect` reads every container we write. `backup` reads only the ones +whose data begins at offset 0: + +| source | `inspect` | `backup` | +|--------|-----------|----------| +| raw `.img` | `Partition table: MBR` | **ok** | +| fixed VHD | `Partition table: MBR` | **ok** | +| **CHD** | `Partition table: MBR` | **fails** | +| **dynamic VHD** | `Partition table: MBR` | **fails** | +| **QCOW2** | `Partition table: MBR` | **fails** | +| **VMDK sparse** | `Partition table: MBR` | **fails** | + +Fixed VHD only passes because it *is* raw data with a trailing footer. Every +container with a non-flat internal layout fails, in one of two ways: + +``` +rb-cli backup o-chd/disk.chd ./out --format raw --sector-by-sector + -> error: backup failed: cannot read first sector: failed to fill whole buffer + +rb-cli backup o-qcow2/disk.qcow2 ./out --format raw --sector-by-sector + -> error: backup failed: failed to detect partition table: + Invalid MBR: invalid boot signature: expected 0xAA55, got 0x... +``` + +Same root cause — the container is not decoded, so raw file bytes are read as +though they were the disk. Which message appears depends on whether the read +runs off the end of a small file or lands on header bytes that resemble a bad +MBR. `--sector-by-sector` does not help; the failure precedes all partition +logic. + +**Why it matters.** These are four of our own output formats, CHD being the +default for `convert`. A user can convert a disk to any of them and then find +they cannot back it up — archive to QCOW2, later try to make a working copy, +and the tool refuses. `inspect` reading them fine makes the gap look arbitrary +from outside, which is what made it read as a defect for so long. + +**What would help.** `inspect` already opens all four, so the decoding exists; +`backup` takes a different route to the bytes. Routing `backup`'s source open +through the same container-aware path `inspect` uses is the whole feature. +`backup.container.inspect-reads-what-backup-cannot` is green and pins that +asymmetry, so it is the case to read first. + +Reproduces on a 64 MB synthetic image; no fixture required. **It also +reproduces on a real one** as of 2026-08-09: `fs.hpfs.os2-warp45.hd` is a +monolithicSparse VMDK holding an OS/2 Warp 4.52 install, and `backup` refuses +it with the same `invalid boot signature: expected 0xAA55, got 0x0000` while +`inspect` reads all 4722 files. Useful when implementing this — the gap is not +an artifact of how the synthetic containers were built. + +**Two traps when verifying this**, both of which caught the original reporter: + +1. `backup` prints `rb-cli backup: SRC -> DEST` *before* doing any work, so a + grep for `->` reports success on a run that then fails. **Check the exit + code**, not the output. +2. `--format raw` writes `partition-N.img` files, so a `find` over several + directories can pick up an unrelated `.img` and attribute the wrong result + to the wrong container. Use explicit paths per case. + +## F-009 — the SFS editor writes single-leaf extent b-trees only {#f-009} + +Filed as defect [R-032](Regression_Bugs.md#r-032) until 2026-08-10. +**Reclassified**: the driver has always documented this ceiling — CLAUDE.md +says "SFS (`src/fs/sfs.rs`) — read + edit (single-leaf btree only)" — so hitting +it is the documented limit, not code disagreeing with itself. That is the same +line R-016 was moved across. + +SFS keeps its free-space extents in a b-tree. `extent_btree_insert` and +`extent_btree_remove` handle a tree that is a single leaf node; a volume large +enough to need interior nodes — which is any volume of real size, including the +Workbench reference disk — cannot be written. + +``` +rb-cli put fs.sfs.workbench-dh0.hd.img payload.bin /payload.bin + -> error: create_file: unsupported: SFS extent b-tree has interior nodes; + this editor writes single-leaf trees only ... + -> exit 4 +``` + +**Two things were fixed while reclassifying it**, because how it refused was +wrong even if the refusal was right: + +- It raised a **`Parse`** error, which says the *disk* is malformed. The disk is + fine and reads perfectly. It is now `Unsupported` — the volume is readable and + this build will not write it — which is precisely what + `exit.rs` reserves `PERMISSION_DENIED` for. +- `put` mapped every `create_file` failure to the catch-all 1. It now routes + through `write_open_error`, the R-034 machinery, so the refusal exits **4**. + The write-*open* path had done this since R-034; `create_file` can refuse for + the same reason and did not. + +**What implementing it needs.** Node splitting, root promotion and parent +updates against the on-disk `BNDC` format. + +The validation half is no longer the hard part. This entry used to say writes +to a real Amiga filesystem could not be confirmed from here; that stopped being +true on 2026-08-14, when `oracles/fsuae/affs_mount.py` drove FS-UAE to a +verdict and closed [R-020](Regression_Bugs.md#r-020). The same harness serves +SFS — mount the volume under test as DH1: and have the guest read it — with +one addition: Kickstart has no SFS handler in ROM, so the guest needs the SFS +handler staged into `L:` (it is at `rb-fixtures/oracle-assets/amiga`, +extracted from the SFS reference fixture's own `L:`). So the code is the +work now, not the proof. diff --git a/docs/opticaldiscs-upstream-prompt.md b/docs/opticaldiscs-upstream-prompt.md new file mode 100644 index 00000000..4a09863f --- /dev/null +++ b/docs/opticaldiscs-upstream-prompt.md @@ -0,0 +1,135 @@ +# Prompt: two cue-sheet fixes for `opticaldiscs-rs` + +Paste into a session working on the `opticaldiscs-rs` checkout. Both defects +were found by rusty-backup's regression suite against real-world media and are +tracked there as R-015 and R-012. + +Both reproduce on **0.13.0** and on **0.14.0** — I bumped and re-ran to check, +so neither is already fixed on the latest release. + +--- + +## Context: who consumes this + +rusty-backup uses the crate at `version = "0.14.0", features = ["drives"]`, +through this surface: + +``` +opticaldiscs::detect::DiscImageInfo::{open, open_physical} +opticaldiscs::browse::{open_disc_filesystem, open_hybrid_filesystem, open_physical_filesystem} +opticaldiscs::browse::entry::{EntryType, FileEntry} +opticaldiscs::browse::filesystem::{Filesystem, FilesystemError} +opticaldiscs::{BinCueSectorReader, DiscFormat, FilesystemType, OpticaldiscsError, + ElTorito, GameDiscInfo, Console, BootMediaType, JolietVolumeDescriptor} +``` + +It reads these `DiscImageInfo` fields: `path`, `format`, `filesystem`, +`hybrid_filesystems`, `volume_label`, `pvd`, `hfs_mdb`, `hfsplus_header`, +`el_torito`, `game`. + +**Please keep the change additive** — new fields and variants rather than +changed signatures — so consumers upgrade without edits. + +--- + +## Issue 1 — cue sheets with unpadded track numbers are rejected + +The CUE spec's examples pad to two digits, but plenty of real tools emit +`TRACK 1`. One such disc is a retail CD-ROM (Microsoft Bookshelf), so this is +not a hand-written edge case. + +**Reproduce:** + +``` +FILE "BOOKSHELF.img" BINARY + TRACK 1 MODE1/2352 + INDEX 1 00:00:00 +``` + +```rust +DiscImageInfo::open("BOOKSHELF.cue") +``` + +``` +CUE error: Error(Msg("Expeceted number but found String(\"1\") instead"), ...) +``` + +**Expected:** parses identically to `TRACK 01` / `INDEX 01`. The same disc +parses fine when both numbers are zero-padded — padding is the only +difference, verified by editing one byte. + +**Also worth fixing while you are in there:** `Expeceted` is misspelled. It is +load-bearing right now only because it makes the message easy to grep for. + +**Suggested scope:** accept 1-or-2-digit numbers wherever the cue grammar +takes a track or index number. Worth auditing the same parser for other +tokens it requires to be padded. + +--- + +## Issue 2 — audio-only discs are rejected outright + +A pure CD-DA disc has **no data track by definition**. Today that is treated +as a failure, so an audio CD image is indistinguishable from a corrupt one. + +**Reproduce:** + +``` +FILE "cdda-noaudiodata.bin" BINARY + TRACK 01 AUDIO + INDEX 01 00:00:00 + TRACK 02 AUDIO + INDEX 01 00:05:25 +``` + +```rust +DiscImageInfo::open("cdda-noaudiodata.cue") // -> Err +``` + +``` +No data track found +``` + +(Note this cue *is* correctly padded, so it is a genuinely separate defect +from Issue 1 — fixing the parser will not fix this one.) + +**Expected:** `open` succeeds and describes the disc. There is nothing wrong +with it; there is simply no filesystem on it. Something like: + +- `filesystem: None` (or a `FilesystemType::None` / `AudioOnly` variant) +- a track list the caller can render: index, `TRACK` type (AUDIO / MODE1 / + MODE2), start LBA, length in sectors, and MSF or duration +- ideally `is_audio_only()` or equivalent, so a caller need not infer it + +Mixed-mode discs (a data track followed by audio tracks) already open — the +audio tracks just are not surfaced. Exposing the track list fixes both cases +with one addition, and lets a caller report "1 data + 12 audio" instead of +silently describing only the data track. + +**Why it matters to the caller:** rusty-backup's `optical info` is +specifically the verb meant to survive discs that strict parsers reject. It +currently has to report `Container: unknown, Filesystems: (none recognized)` +for a perfectly good audio CD. + +--- + +## Acceptance + +Both of these should hold afterwards: + +```rust +// Issue 1 +assert!(DiscImageInfo::open("unpadded.cue").is_ok()); + +// Issue 2 +let info = DiscImageInfo::open("audio_only.cue").expect("audio-only is a valid disc"); +assert!(info.tracks.iter().all(|t| t.is_audio())); +assert!(info.filesystem.is_none()); +``` + +Please add both cue sheets above as test fixtures — they are small, and each +pins a real-world shape that a stricter parser would otherwise regress. + +Once released, bump the version and I will re-run rusty-backup's cases +`optical.cue.unpadded-track-number` and `optical.cdda.no-data-track-opens`, +which assert exactly this behaviour and are currently red on purpose. diff --git a/docs/regression-fix-prompt.md b/docs/regression-fix-prompt.md index 1f486b06..0961654c 100644 --- a/docs/regression-fix-prompt.md +++ b/docs/regression-fix-prompt.md @@ -3,15 +3,23 @@ A handoff for fixing the defects in [`Regression_Bugs.md`](Regression_Bugs.md). Paste a tranche into a fresh session; each is independently shippable. -**Readiness, honestly.** 30 findings are open. 11 are specified well enough to -fix without further investigation. 9 more have an unambiguous symptom and a -red case but need real engine work. 8 cannot be turned into a fix prompt yet — -the symptom is known and the cause is not, so a prompt would be guessing. 2 are -decisions rather than defects and must not be quietly resolved by whoever -picks them up. +**Readiness, honestly.** Written when 30 findings were open. **Two remain as of +2026-08-14** — R-039 (EFS free list, filed after this document was written) and +R-011 (blocked on fixtures, not on effort). Everything this document was built +to hand off has been handed off and done. -One prompt for all 30 would be the wrong shape. The tranches below are sized -to be reviewable. +Tranche C — the eight where "the symptom is known and the cause is not", the +state that does not clear on its own — cleared. Six were fixed, two turned out +not to be defects, and R-011 is the only row still live. That happened between +2026-08-10 and 2026-08-14, so if you are reading this expecting work, read +[`Regression_Bugs.md`](Regression_Bugs.md) first: this file is now mostly a +record of how the tranche was worked rather than a queue. + +Struck-through rows are kept rather than deleted: the "what to do" column +records what each fix turned on, and two of them turned on the *report being +wrong*, which is the most reusable thing in this document. + +The tranches below are sized to be reviewable. --- @@ -22,8 +30,8 @@ to be reviewable. the wrong thing, say so and change it deliberately; do not weaken it to get green. 2. **Each fix must turn its named case green and leave every other case - unchanged.** `rb-regress run` is 226–228 pass / 33–35 xfail / 0 fail on - Windows, macOS and Linux at `c3e1984`. A fix that trades one red for + unchanged.** `rb-regress run` is **254 pass / 19 xfail / 0 fail** on + Windows, macOS and Linux at `c6e66fd`. A fix that trades one red for another is not a fix. 3. **Remove the entry from `regression-tests/data/known-failures.toml`** when a finding goes green, and strike the row through in `Regression_Bugs.md`, @@ -47,28 +55,30 @@ to be reviewable. ## Tranche A — fully specified, mechanical -Eleven findings. Each has a repro, an expected behaviour, and a red case. No -investigation needed; the work is deciding the exact wording and doing it. +**Empty as of 2026-08-09** except the two that are blocked upstream. Eleven +findings started here; nine are fixed and struck through below, kept because +the "what to do" column records the decision each one turned on. | Finding | Case that must go green | What to do | |---|---|---| -| R-006 | `fs.new-volume.prodos-default-name` | Default volume name `rusty-backup` contains `-`, which ProDOS forbids, so the verb always fails with defaults. Change the default (per-fs, or sanitise), and fix the message — it says "rename the file" when the offending string is the *volume* name. | -| R-004 | `cli.exit.{csv,tsv}-on-nested-verb-is-usage-error`, `shrink.rejects-non-chd-output` | CSV/TSV rejection exits 1; documented as 2. Usage errors are 2. | -| R-005 | `cli.envelope.error-envelope-on-failure` | No error envelope is emitted under `--format json`. Failures must produce the same envelope shape as successes. | -| R-003 | `cli.envelope.ls-supports-format` | **Decision.** Docs claim `ls` supports `--format`; it does not. Either implement it or correct the docs. Do not assume — see Decisions below. | -| R-010 | `cli.flags.inspect-accepts-fs-type` | `inspect` has no `--fs-type`, so CP/M images cannot be inspected. `ls` already accepts it (`cli.flags.ls-accepts-fs-type` is green) — mirror that. | -| R-026 | `subcmd.show.partmap` | `show partmap` cannot read an SGI disk that `inspect` reads fine. Two code paths disagree; make `show` use the one that works. | -| R-027 | `read.apfs.apple-gpt` | A Finder-made `.zip` holding one `.dmg` is rejected as ambiguous because `__MACOSX/._*` counts as a second candidate. Ignore the AppleDouble sidecar. Every zip made on a Mac has one. | -| R-034 | `edit.readonly.{lisa,alto}-refuses-a-write` | Refusing a write to a read-only FS reports the type as `unknown` and exits 1. Refusing is correct; the type should be the one `ls`/`inspect` just reported, and the exit code 4. **Check whether this also fixes R-031** — same shape. | -| R-015 | `optical.cue.unpadded-track-number` | A `.cue` with `TRACK 1` (unpadded) is rejected. Accept it. | -| R-012 | `optical.cdda.no-data-track-opens` | `optical info` rejects any disc with no data track (pure CD-DA). `optical.cdda.mixed-mode-still-opens` is the green working-half — keep it green. | -| R-001, R-002 | none (doc drift) | README partition-table list is missing AHDI and X68000; `src/fs/README.md` still lists ext as "planned". Both would be caught by the source-parity test noted below. | - -**Worth doing while in here:** R-001/R-002/R-018 are all documentation drifting -from code, and all three were found by hand. A source-parity test — comparing -the README tables against the `PartitionTable` enum and the `fs/mod.rs` -dispatch — would guard the whole class. It is currently listed as "Not -covered" in `Regression_Bugs.md`. +| ~~R-006~~ **FIXED** | `fs.new-volume.prodos-default-name` | Default volume name `rusty-backup` contains `-`, which ProDOS forbids, so the verb always fails with defaults. Change the default (per-fs, or sanitise), and fix the message — it says "rename the file" when the offending string is the *volume* name. | +| ~~R-004~~ **FIXED** | `cli.exit.{csv,tsv}-on-nested-verb-is-usage-error`, `shrink.rejects-non-chd-output` | CSV/TSV rejection exits 1; documented as 2. Usage errors are 2. | +| ~~R-005~~ **FIXED** | `cli.envelope.error-envelope-on-failure` | No error envelope is emitted under `--format json`. Failures must produce the same envelope shape as successes. | +| ~~R-003~~ **FIXED** | `cli.envelope.ls-supports-format` | ~~**Decision.**~~ Decided: implement. Docs claim `ls` supports `--format`; it does not. Either implement it or correct the docs. Do not assume — see Decisions below. | +| ~~R-010~~ **FIXED** | `cli.flags.inspect-accepts-fs-type` | `inspect` has no `--fs-type`, so CP/M images cannot be inspected. `ls` already accepts it (`cli.flags.ls-accepts-fs-type` is green) — mirror that. | +| ~~R-026~~ **FIXED** | `subcmd.show.partmap` | `show partmap` cannot read an SGI disk that `inspect` reads fine. Two code paths disagree; make `show` use the one that works. | +| ~~R-027~~ **FIXED** | `read.apfs.apple-gpt` | A Finder-made `.zip` holding one `.dmg` is rejected as ambiguous because `__MACOSX/._*` counts as a second candidate. Ignore the AppleDouble sidecar. Every zip made on a Mac has one. | +| ~~R-034~~ **FIXED** | `edit.readonly.{lisa,alto}-refuses-a-write` | Refusing a write to a read-only FS reports the type as `unknown` and exits 1. Refusing is correct; the type should be the one `ls`/`inspect` just reported, and the exit code 4. **Check whether this also fixes R-031** — same shape. | +| R-015 **blocked upstream** | `optical.cue.unpadded-track-number` | A `.cue` with `TRACK 1` (unpadded) is rejected. Accept it. | +| R-012 **blocked upstream** | `optical.cdda.no-data-track-opens` | `optical info` rejects any disc with no data track (pure CD-DA). `optical.cdda.mixed-mode-still-opens` is the green working-half — keep it green. | +| ~~R-001, R-002~~ **FIXED** | `tests/doc_parity.rs` (three tests) | README partition-table list is missing AHDI and X68000; `src/fs/README.md` still lists ext as "planned". Both would be caught by the source-parity test noted below. | + +**That source-parity test now exists.** `tests/doc_parity.rs` covers R-001, +R-002 and R-018 — the README partition-table list against +`PartitionTable::ALL_TYPE_NAMES`, `src/fs/README.md` against a capability table +growing back, and CONTRIBUTING.md's vintage feature list against the workflow's. +It is a `cargo test`, not an `rb-regress` case, because the claim is *about* the +binary rather than something the binary does. --- @@ -77,17 +87,21 @@ covered" in `Regression_Bugs.md`. Nine findings. The symptom is unambiguous and reproducible; the fix is not mechanical. Take these one at a time. -**Highest value first — these three are silent data loss or silent no-ops, -the worst failure shape in a tool whose job is moving data:** +**The three highest-value ones are done (2026-08-09) — and two of the three +were filed with the wrong cause, which is the lesson worth carrying:** + +- ~~**R-021**~~ — fixed. Not a no-op: it warned and proceeded, leaving a + filesystem describing twice the blocks its container held. `resize` now grows + the file when the volume *is* the file, and refuses otherwise. +- ~~**R-023**~~ — fixed. **Nothing was lost.** A FAT long filename was dropped + because `repack`, documented Human68k-only, accepted a plain FAT volume. + Scope guard added. +- ~~**R-022**~~ — fixed. **Not a fidelity bug.** A bare HPFS volume was + detected as an empty MBR, so `backup` wrote no partition file at all and + exited 0. A detection probe closed it. -- **R-021** (`resize.to-explicit-size`) — `resize --size 16M` on an 8M volume - prints "resize complete", exits 0, changes nothing. Nothing downstream has - any reason to check. -- **R-023** (`resize.repack.keeps-data`) — `repack` exits 0 and every file in - the volume is gone. Human68k. -- **R-022** (`roundtrip.hpfs.raw`) — `backup --sector-by-sector` then `restore` - is not byte-identical for HPFS. FAT, NTFS, ext4, HFS, minix3, EFS and ProDOS - all survive the same path, so it is HPFS-specific, not a backup-format bug. +Reproduce and *measure* before fixing. Both wrong diagnoses had a one-command +control: count the non-zero bytes; list the backup folder. **Then:** @@ -107,11 +121,14 @@ the worst failure shape in a tool whose job is moving data:** single-leaf-only, so this is the known ceiling being hit, not a surprise. - **R-033** (`read.qdos.microdrive`) — a QL Microdrive `.mdv` fails at MBR detection although its own probe matches it exactly. Detection ordering. -- **R-016** (`backup.container.{chd,vhd-dynamic,qcow2,vmdk-sparse}`) — - **Decision.** `backup` accepts only flat-layout sources. - `backup.container.inspect-reads-what-backup-cannot` is green and proves - `inspect` opens exactly what `backup` refuses. Is this a defect or an - unimplemented feature? See Decisions. + **Very likely the same shape as R-022**, which was a bare volume falling + through to the MBR parse because no probe in `detect_superfloppy` claimed it. + Read that fix first. +- ~~**R-016**~~ — **decided 2026-08-09: an unimplemented feature, not a + defect.** Moved to + [F-008](missing_features_from_regression.md#f-008); the four cases keep their + assertions and now cite F-008, which `rb-regress validate` accepts as of the + same date. No longer in this tranche. --- @@ -123,39 +140,41 @@ first, and the investigation is the deliverable. | Finding | Case | What the investigation has to establish | |---|---|---| -| R-020 | none — hand-verified | Every AFFS volume we write is "Not a DOS disk" on a real Amiga, at every size. Working hypothesis: root block `header_key` must be 0 and we write the block number. **Unconfirmed.** The fix must land in **both** the formatter and `affs_fsck`, which currently agree with each other and are both wrong. Needs an emulator or hardware oracle to confirm — see Blocked below. | -| R-030 | `edit.real.affs-workbench13` | A real Workbench 1.3 AFFS volume cannot be opened at all — read, fsck and write alike. Establish whether this is OFS-vs-FFS, an older root-block layout, or the same root cause as R-020. | -| R-029 | `edit.real.efs-small` | EFS computes block addresses far outside the image; `fsck` fails on an unmodified volume. Find where the address computation diverges from the on-disk geometry. | -| R-013 | `fs.detect.ufs-{solaris-entry-types,no-absurd-sizes}` | Solaris UFS directories are reported as files, one with a garbage size. Likely endianness or cylinder-group layout. | -| R-028 | `edit.apple-dos.put-get` | Apple DOS 3.3 reports three sizes for one file: 104 in, 512 by `ls`, 256 by `get`. Establish which is right and which two are wrong. | -| R-031 | `edit.real.apple-dos-invaders` | A real Apple DOS 3.3 disk detects as `unknown` although our own output does not. **Try R-034's fix first** — same shape, and one fix may cover both. | -| R-035 | none — a `parity` finding | `.cbk` embeds `source_device`, the producing host's absolute path, so it is not reproducible across machines and leaks the host's directory layout. **Decision** before any fix: keep the field, normalise it, or record a device identity instead. Note `expect_divergence` masks byte *ranges* and cannot express a divergence that changes the file's *length*. | +| ~~R-020~~ | `oracles/fsuae/affs_mount.py` | ~~Every AFFS volume we write is "Not a DOS disk" on a real Amiga.~~ **CLOSED 2026-08-14.** The hypothesis was right: `header_key` had to be 0, and the bitmap was a block short of the geometry. Both fixed by a190182; Kickstart 3.1 then mounted the volume Read/Write. The half of the prediction that was *wrong* is worth keeping — the fix did **not** need to land in `affs_fsck`, which never inspected `header_key` at all. | +| ~~R-030~~ | `edit.real.affs-workbench13` | ~~A real Workbench 1.3 AFFS volume cannot be opened at all.~~ **CLOSED 2026-08-10** — neither OFS-vs-FFS nor a root-block layout difference: the root block was being located from the end of the *file* rather than the partition. Related to R-020 but not the same root cause. | +| ~~R-029~~ | `edit.real.efs-small` | ~~EFS computes block addresses far outside the image.~~ **NOT A DEFECT, 2026-08-10** — the fixture is a deliberate 4 MB prefix capture, and the case asked it to do what a prefix cannot. | +| ~~R-013~~ | `fs.detect.ufs-{solaris-entry-types,no-absurd-sizes}` | ~~Solaris UFS directories reported as files.~~ **CLOSED 2026-08-10** — cylinder-group layout, as guessed: UFS1's rotational cylinder-group offset was ignored. Not endianness. | +| ~~R-028~~ | `edit.apple-dos.put-get` | ~~Apple DOS 3.3 reports three sizes for one file.~~ **CLOSED 2026-08-10** — the length lives in a type-B header and was not being stored; all three now agree. | +| ~~R-031~~ | `edit.real.apple-dos-invaders` | ~~A real Apple DOS 3.3 disk detects as `unknown`.~~ **NOT A DEFECT, 2026-08-10** — the disk carries no filesystem at all, so `Unknown` is the correct answer. R-034's fix was unrelated. | +| ~~R-035~~ | none — a `parity` finding | ~~`.cbk` embeds the producing host's absolute path.~~ **CLOSED 2026-08-09** — decided and shipped: the path is normalised to a device leaf. | | R-011 | `fmt.g64.standard-dump-opens` (working half only) | G64 decoding fails on copy-protected / patched dumps. **Deliberately undecided** — whether it *should* succeed is an open question, and asserting either way prejudges it. Decide scope before writing anything. | --- ## Decisions that must not be quietly resolved -Four. Each changes what the fix is, so they belong to the maintainer, not to -whoever picks up the ticket. +Four were open. Three are now answered; each changed what the fix was, which +is why they belonged to the maintainer rather than to whoever picked up the +ticket. -1. **R-003** — implement `ls --format`, or correct the docs? -2. **R-016** — is "backup refuses non-flat containers" a defect or an - unimplemented feature? It is currently filed as a defect with four red - cases; if it is a feature, it belongs in - `missing_features_from_regression.md` and the cases should move to a - capability list. -3. **R-035** — keep, normalise, or replace `source_device`? -4. **R-011** — should copy-protected G64 dumps open at all? +1. ~~**R-003**~~ — **decided: implement the flag**, not correct the doc. `ls` + is the most script-facing verb in the CLI. Shipped 2026-08-09. +2. ~~**R-016**~~ — **decided: an unimplemented feature.** Moved to + [F-008](missing_features_from_regression.md#f-008), cases retagged, + `validate` taught to accept an F-nnn citation. 2026-08-09. +3. ~~**R-035**~~ — **decided: normalise the path** to a device leaf rather than + keeping the absolute path or inventing a device identity. +4. **R-011** — should copy-protected G64 dumps open at all? **Still open.** --- ## Blocked on something other than effort -- **R-020** needs an emulator or hardware oracle. All 62 emulator and - MiSTer-core oracles resolve to `skip-manual`: `verify` cannot invoke them, - so no automated run can confirm a fix. Either book MiSTer time, or teach - `verify` to drive FS-UAE — which is the next harness feature regardless. +- ~~**R-020** needs an emulator or hardware oracle.~~ **Unblocked 2026-08-14.** + The suggested route was the one taken: `verify` was taught to drive FS-UAE, + via `oracles/fsuae/affs_mount.py`. No MiSTer time was needed. The other + ~60 emulator and MiSTer-core oracles are still `skip-manual` — this + unblocked one, not the class. - The MiSTer's `rb-cli` is from 2026-07-27 and must be redeployed before its 12 core oracles mean anything. diff --git a/rb-cli-vintage/Cargo.lock b/rb-cli-vintage/Cargo.lock index 84d11ab9..9d818e1a 100644 --- a/rb-cli-vintage/Cargo.lock +++ b/rb-cli-vintage/Cargo.lock @@ -1368,6 +1368,7 @@ dependencies = [ "dirs", "encoding_rs", "env_logger", + "filetime", "flate2", "globset", "hashbrown 0.14.5", diff --git a/rb-cli-vintage/Cargo.toml b/rb-cli-vintage/Cargo.toml index 7985d73c..944c82c0 100644 --- a/rb-cli-vintage/Cargo.toml +++ b/rb-cli-vintage/Cargo.toml @@ -111,6 +111,8 @@ dirs = "6" flate2 = { version = "1", default-features = false, features = ["rust_backend"] } lz4_flex = ">=0.10, <0.11" tar = "0.4" +# See main manifest — already a transitive dep via `tar` on 1.73 too. +filetime = "0.2" bzip2 = "=0.5.2" zip = { version = "=2.4.2", default-features = false, features = ["deflate", "aes-crypto"] } base64 = "0.22" diff --git a/regression-tests/COMMAND-COVERAGE.md b/regression-tests/COMMAND-COVERAGE.md index 6f1d4a08..93166d42 100644 --- a/regression-tests/COMMAND-COVERAGE.md +++ b/regression-tests/COMMAND-COVERAGE.md @@ -69,12 +69,21 @@ Two things fell out of checking: shortcut that produced the "no L: directory anywhere" mistake during the FS-UAE work. * **Two real AFFS volumes were sitting unused** (`fs.affs.workbench13.hd`, - `fs.affs.ffs-intl-cd32.hd`). Given R-020 — our AFFS formatter emits volumes - no Amiga will mount, and our own fsck agrees with the formatter rather than - with reality — an AFFS editor tested only against our own output proves very - little. `cases/tier3/edit-real-volumes.toml` now runs the same put/get/fsck - round-trip against third-party volumes for AFFS, NTFS, ext2/4, FAT16/32, - HFS+, HFS, HFV, ProDOS, CP/M, Human68k, Apple DOS and EFS. + `fs.affs.ffs-intl-cd32.hd`). The reasoning at the time was R-020 — our AFFS + formatter emitted volumes no Amiga would mount, and our own fsck agreed with + the formatter rather than with reality, so an AFFS editor tested only against + our own output proved very little. `cases/tier3/edit-real-volumes.toml` now + runs the same put/get/fsck round-trip against third-party volumes for AFFS, + NTFS, ext2/4, FAT16/32, HFS+, HFS, HFV, ProDOS, CP/M, Human68k, Apple DOS + and EFS. + + R-020 was fixed and closed on 2026-08-14 (Kickstart 3.1 mounts our volumes + Read/Write), so the premise no longer holds — but the conclusion does, and + more strongly for having been tested: third-party volumes are what caught + R-030 and R-013, and testing an editor only against its own writer is the + circularity that let R-020 stand for a week. `workbench13` earned its keep + twice more since, as the R-038 control and as the FS-UAE oracle's boot + volume. 43 remain unused, mostly optical discs and the exotic end (Alto, Lisa, Xerox D0, 3DO, GameCube, CD-i). Those are read-path fixtures whose cases belong in @@ -147,7 +156,7 @@ update write xattr **1. Backup/restore has no round-trip.** This is the product's headline feature and its name. All six functional `backup` calls are -`--format raw --sector-by-sector`, and all six exist to demonstrate R-016 +`--format raw --sector-by-sector`, and all six exist to demonstrate F-008 (container sources being rejected). Never tested: zstd / CHD / VHD / gzip / lz4 output, checksum verification (CRC32 and SHA256), split backups, and `cbk` incremental. `restore` is only ever asked for its help text, so **no test diff --git a/regression-tests/FIXTURES.md b/regression-tests/FIXTURES.md index af315645..c8c6776b 100644 --- a/regression-tests/FIXTURES.md +++ b/regression-tests/FIXTURES.md @@ -257,6 +257,145 @@ structure under test rather than storing a whole game or install disc. Track the total in the catalogue; the run report prints it. +### Oracles: what they are vs whether you have them + +Same split as the corpus, for the same reason. + +| file | tracked? | holds | +|------|----------|-------| +| `data/oracles.toml` | **yes** | what an oracle *is* — tool, kind, what it proves, the `check` command | +| `data/oracles.local.toml` | **no**, gitignored | whether *this* machine has it, and where | + +```bash +rb-regress oracles --detect # probe this host, rewrite the overlay +rb-regress oracles --set fs-uae=/path/to/fs-uae # provide one by hand +rb-regress oracles --export # print it, to seed another machine +``` + +**Scanning a MiSTer.** The board is just another host in `local.toml` — give +that entry an `ssh` target and: + +```bash +rb-regress oracles --scan mister-core +``` + +It lists `/media/fat/_Computer/*.rbf`, strips MiSTer's `_YYYYMMDD` build-date +suffix (which moves on every board update, so the bare name is the only stable +identifier), and matches what it finds against the `core` field on each +`mister-core-*` oracle. Cores present are recorded `verified`; cores the +registry names and the board lacks are reported as **MISSING** rather than +quietly skipped. Results merge into the overlay, so a scan never discards what +`--detect` found on this machine. + +The core names were read off a real board, not guessed — `Ti994a`, not +`TI-99_4A`, and `CoCo3` rather than `CoCo2`. + +Nothing new to configure: the board's address and key are in `local.toml` with +every other machine, which is the one file that never reaches the repo. + +**Finding emulators.** They install as apps, not commands, so `PATH` alone +never finds them. `--detect` also searches the platform's usual roots — +`%ProgramFiles%`, `C:/Tools`, `C:/Emulators` on Windows; `/Applications` (into +`.app/Contents/MacOS`) on macOS; `/opt` and `~/.local/bin` on Linux — one level +deep. Point it somewhere else with `RB_EMULATOR_DIRS`, which takes the +platform's path separator and needs no edit to anything. + +If it still cannot find one, `--detect` **asks**, and writes what you give it. +Only for emulators, and only on a terminal: + +- **Only emulators**, because that is the case where the tool *is* installed and + we merely cannot see it. A package oracle that came back absent is genuinely + not installed — prompting there would be 18 questions to reach the 5 that matter. +- **Only on a terminal**, because the harness runs in CI and over ssh, where a + prompt nobody answers is indistinguishable from a hang. `--no-prompt` forces + the quiet path. + +A blank answer skips; a path that does not exist is refused and re-asked rather +than written and discovered weeks later as a silent skip. + +A `--set` hint is sticky: it survives re-detection, which is the point. Note the +check is only that the path *exists* — nothing verifies it is the program you +named, so a wrong path stays wrong until you overwrite it. + +An emulator whose binary is found records **`installed`**, not `verified`: the +binary alone does not make the oracle runnable, since it still needs a +configured guest. `verified` would overclaim and `manual` would hide it. + +An overlay row replaces the tracked one for the same `(oracle, platform)`. + +**Why the split.** A path like `D:/ROMs/Amiga/Shared/rom` was committed to a +public repo because availability lived in the tracked file. Machine paths are +neither portable nor publishable; the tracked registry now carries no +`path_hint` at all. Re-running `--detect` preserves hints you have already +recorded, because it reads them back from the merged registry. + +**Two things `--detect` deliberately will not do:** + +- A `mount` oracle is the *kernel's* opinion, so it is only probed on + Linux / WSL / the MiSTer HPS. Probing the program name on Windows found + Cygwin's unrelated `mount` and called AFFS "verified" — a confidently wrong + answer, which is the worst kind here. +- An emulator or MiSTer core is recorded `manual`, not `absent`. `which` cannot + find a running guest, and "absent" would read as a missing install rather + than a different kind of oracle. + +A `path_hint` may name the program or the directory holding it — `chdman`'s is +a directory, and testing it as a file reported the tool missing on a box that +has it. + +### Declaring what a fixture is + +The catalogue's last three columns let a row state its own identity, and +`rb-regress fixtures --identify` enforces it by running `rb-cli inspect`: + +| column | meaning | +|--------|---------| +| `expect_fs` | the filesystem it must identify as (`HPFS`, `DOS 3.3`, …) | +| `expect_layout` | `superfloppy`, `partitioned`, or a scheme name (`mbr`, `rdb`, …) | +| `fs_type` | a `--fs-type` preset the fixture needs to be identifiable at all — CP/M disks carry no signature | + +**Why this exists.** `origin = harvested-verified` used to mean "`rb-cli inspect` +opened it", which a universal reader says about *anything*: an Apple DOS fixture +that was really a bare bootloader with no VTOC passed that bar and sat in the +corpus for weeks (R-031). "It opened" is not identification. + +Rows that declare nothing are skipped, so populating is incremental. Three +groups are deliberately left undeclared: + +- **optical discs** — `inspect` is the wrong verb for them; `optical info` reads + them. A separate check could cover those. +- **encrypted images** (`fmt.gho.password`, `fmt.imz.encrypted`) — identifying + them needs a password the catalogue does not carry. +- **`fs.apple-dos.invaders.floppy`** — it would fail, correctly. Its ID claims a + filesystem it does not have. The honest fix is renaming the fixture, not + weakening the check. + +An `--identify` run is slower than a plain inventory: it opens every declaring +fixture, expanding compressed ones through the same per-run cache a case uses. + +### Hosts that cannot reach the share + +`--sync` assumes the host can see `corpus_source`. Not all of them can: on the +Linux and macOS oracle hosts `corpus_source` is deliberately **commented out**, +and their corpus was `scp`'d over from the orchestrating box. Runs there read +local disk like everywhere else; only the distribution differs. + +Admitting a fixture therefore takes three steps per such host: + +1. `scp` the file into `fixtures/` (or `fixtures-large/`). +2. **Append the catalogue row to that host's `fixture-map.tsv`.** It is + gitignored, so it does not arrive with `git pull`. Miss this and the file is + present, the case still reports `skip-fixture`, and the run stays green with + a quietly smaller pass count. +3. `rb-regress fixtures` on the host — expect `N catalogued - N verified, 0 + missing, 0 CORRUPT`. + +Because a missing fixture degrades to `skip-fixture` rather than a failure, a +host that never got the sync stays green while covering less than its peers. +`rb-regress consolidate` reports that directly, as **COVERAGE SKEW** — cases +that ran on some hosts and skipped on others. Read it rather than eyeballing +pass counts. + ### The large-fixture annex Some formats have no small specimen. A UDF DVD-Video is 483 MB and cannot be diff --git a/regression-tests/GAPS.md b/regression-tests/GAPS.md index 7f70edb6..68f9a531 100644 --- a/regression-tests/GAPS.md +++ b/regression-tests/GAPS.md @@ -135,9 +135,18 @@ rediscovered. and `fmt.bincue` (66-byte `.cue` + sibling `.bin`) are the only two builders `produce` still has no recipe for, both for this reason. Keeping half of either would read as coverage of a format only half looked at. -- **Emulator oracles cannot be invoked.** They all resolve to `skip-manual`, - so R-020 — the highest-severity Amiga finding — was found by hand and cannot - be re-checked by a run. FS-UAE works interactively; `verify` cannot drive it. +- **Emulator oracles are mostly, no longer entirely, uninvokable.** Two now + run from a script: `iris` (IRIX 6.5, which produced R-039) and FS-UAE via + `oracles/fsuae/affs_mount.py`, which closed R-020 on 2026-08-14. Both use + the same trick — a host directory the guest writes its verdict into, so + nothing is screen-scraped. The remaining ~60 emulator and MiSTer-core + oracles are still `skip-manual`. + + The AFFS one has a `check` line and `verify` can invoke it, but fs-uae's + availability stays `manual`: it wants a GUI session and about a minute per + run, so it is on-demand rather than part of an unattended sweep. Making it + a first-class row means deciding whether the runner may open a window — + a policy call, not a missing feature. - **27 package oracles have no runnable check command**, so `verify` skips them with `skip-no-check`. diff --git a/regression-tests/cases/tier0/envelope.toml b/regression-tests/cases/tier0/envelope.toml index 7187e299..6bb1b272 100644 --- a/regression-tests/cases/tier0/envelope.toml +++ b/regression-tests/cases/tier0/envelope.toml @@ -43,18 +43,21 @@ args = ["inspect", "{scratch}/v.img", "--format", "yaml"] expect_exit = 0 stdout_contains = ["schema_version"] -# --- Known finding R-005 ----------------------------------------------------- +# --- R-005 ------------------------------------------------------------------- # output.rs documents that on error the envelope is still emitted, with # status.error true, status.code carrying the exit code, and result null. -# Today a failing verb under --format json writes nothing to stdout and a -# plain-text error to stderr, so a JSON consumer gets nothing parseable on +# A failing verb under --format json used to write nothing to stdout and a +# plain-text error to stderr, so a JSON consumer got nothing parseable on # exactly the path the envelope exists to serve. [[case]] id = "cli.envelope.error-envelope-on-failure" -description = "R-005: a failing verb under --format json should still emit an error envelope" +description = """R-005: a failing verb under --format json still emits an error +envelope. expect_exit was 1 until 2026-08-09, which contradicted +cli.exit.missing-image-file — a missing image is NOT_FOUND, and exit.rs has +reserved 3 for it all along. Corrected rather than weakened.""" [[case.step]] args = ["inspect", "{scratch}/does-not-exist.img", "--format", "json"] -expect_exit = 1 +expect_exit = 3 expect_envelope_ok = false # --- Known finding R-003 ----------------------------------------------------- diff --git a/regression-tests/cases/tier0/exit-codes.toml b/regression-tests/cases/tier0/exit-codes.toml index 6b7c095f..ed05e03e 100644 --- a/regression-tests/cases/tier0/exit-codes.toml +++ b/regression-tests/cases/tier0/exit-codes.toml @@ -68,6 +68,63 @@ expect_exit = 2 # documents present behaviour rather than inventing a requirement. [[case]] id = "cli.exit.missing-image-file" -description = "inspect on a nonexistent image exits 1 (current documented-free behaviour)" +description = """inspect on a nonexistent image exits 3. exit.rs reserves +NOT_FOUND for exactly this ("image file missing"), and cli.flags.inspect-accepts-fs-type +depends on it to tell "the flag parsed and the file was absent" from "the flag +was rejected". This case asserted 1 until 2026-08-08, describing itself as +"current documented-free behaviour" — it pinned the status quo rather than the +contract, and the two cases contradicted each other once R-010 was fixed.""" args = ["inspect", "{scratch}/does-not-exist.img"] -expect_exit = 1 +expect_exit = 3 + +# --- R-036: one condition, three exit codes ---------------------------------- +# `exit.rs` reserves NOT_FOUND (3) for "image file missing" — its own first +# example. R-010 made `inspect` obey it (cli.exit.missing-image-file, green). +# Nothing else does: most verbs exit 1 with a raw platform-specific io::Error, +# and `locate` / `tar` exit 2, which claims the *command* was malformed when it +# was well-formed and named a file that is not there. +# +# One case per verb, because each is a separate contract a script switches on. +# All red until R-036 is fixed. + +[[case]] +id = "cli.exit.ls-missing-image-is-not-found" +description = "R-036: ls on a nonexistent image should exit 3, not 1" +args = ["ls", "{scratch}/does-not-exist.img"] +expect_exit = 3 + +[[case]] +id = "cli.exit.du-missing-image-is-not-found" +description = "R-036: du on a nonexistent image should exit 3, not 1" +args = ["du", "{scratch}/does-not-exist.img"] +expect_exit = 3 + +[[case]] +id = "cli.exit.fsck-missing-image-is-not-found" +description = "R-036: fsck on a nonexistent image should exit 3, not 1" +args = ["fsck", "{scratch}/does-not-exist.img", "--checkonly"] +expect_exit = 3 + +[[case]] +id = "cli.exit.show-fs-info-missing-image-is-not-found" +description = "R-036: show fs-info on a nonexistent image should exit 3, not 1" +args = ["show", "fs-info", "{scratch}/does-not-exist.img"] +expect_exit = 3 + +[[case]] +id = "cli.exit.locate-missing-image-is-not-found" +description = "R-036: locate on a nonexistent image exits 3, not 1" +args = ["locate", "{scratch}/does-not-exist.img", "/anything"] +expect_exit = 3 + +[[case]] +id = "cli.exit.tar-missing-image-is-not-found" +description = """R-036: tar on a nonexistent image exits 3. + +First written with two positionals, which made clap reject it as a usage error +before the image was ever opened — the case asserted 3 and got a correct 2 for +the wrong reason. `tar` takes IMAGE, SRC and OUT. Corrected deliberately, and +the finding's table corrected with it: `tar` was never a real 2, it was a +malformed command in the survey.""" +args = ["tar", "{scratch}/does-not-exist.img", "/", "{scratch}/out.tar"] +expect_exit = 3 diff --git a/regression-tests/cases/tier0/selector-flags.toml b/regression-tests/cases/tier0/selector-flags.toml index 5dd4083b..762662b6 100644 --- a/regression-tests/cases/tier0/selector-flags.toml +++ b/regression-tests/cases/tier0/selector-flags.toml @@ -32,3 +32,111 @@ fixture = "fs.cpc.amsdos.floppy" args = ["ls", "{fixture}", "--fs-type", "cpm:amstrad_data"] expect_exit = 0 stdout_contains = ["MANIC.BAS", "MANIC.BIN"] + +# --- inspect verification flags ---------------------------------------------- +# `inspect` opens anything — that is the point of a universal tool, and a disk +# with no recognisable filesystem still yields a carve view. The cost is that a +# clean exit means "readable", not "identified", which is how a fixture with no +# filesystem at all was admitted on the strength of "inspect opens it" (R-031). +# These flags let a caller ask the stronger question. + +[[case]] +id = "cli.flags.inspect-expect-fs-matches" +description = "--expect-fs succeeds when the filesystem is the one named" +[[case.step]] +args = ["new", "floppy", "apple-dos", "--size", "140K", "{scratch}/d.dsk"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/d.dsk", "--expect-fs", "DOS 3.3"] +expect_exit = 0 +# Case, spacing and punctuation are normalised away before comparing. +[[case.step]] +args = ["inspect", "{scratch}/d.dsk", "--expect-fs", "dos3.3"] +expect_exit = 0 + +[[case]] +id = "cli.flags.inspect-expect-fs-rejects-a-mismatch" +description = "--expect-fs fails when the disk carries something else, and says what it found" +[[case.step]] +args = ["new", "floppy", "apple-dos", "--size", "140K", "{scratch}/d.dsk"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/d.dsk", "--expect-fs", "FAT"] +expect_exit = 1 +stderr_contains = ["Found: DOS 3.3"] + +[[case]] +id = "cli.flags.inspect-expect-fs-is-not-a-substring-match" +description = """FAT must not satisfy exFAT. The comparison normalises case and +punctuation, then compares exactly — a substring rule would make the flag worse +than useless on the FAT family.""" +[[case.step]] +args = ["new", "volume", "fat", "--size", "2M", "{scratch}/v.img"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/v.img", "--expect-fs", "exFAT"] +expect_exit = 1 + +[[case]] +id = "cli.flags.inspect-still-opens-what-it-cannot-identify" +description = """Universality is the default: with no assertion flag, a disk +carrying no recognisable filesystem still opens and still exits 0. The flags are +opt-in precisely so this stays true.""" +[[case.step]] +args = ["new", "volume", "fat", "--size", "2M", "{scratch}/v.img"] +expect_exit = 0 +# Wipe the boot sector: still a readable disk, no longer an identifiable FS. +[[case.step]] +args = ["put", "{scratch}/v.img", "--zero", "1024", "--dst", "/x"] +expect_exit = 0 + +[[case]] +id = "cli.flags.inspect-expect-layout-superfloppy" +description = "--expect-layout tells a bare volume from a partitioned disk" +[[case.step]] +args = ["new", "volume", "fat", "--size", "2M", "{scratch}/sf.img"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/sf.img", "--expect-layout", "superfloppy"] +expect_exit = 0 +# The scheme name for "no table" works too, for callers who think in schemes. +[[case.step]] +args = ["inspect", "{scratch}/sf.img", "--expect-layout", "none"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/sf.img", "--expect-layout", "partitioned"] +expect_exit = 1 +stderr_contains = ["this disk is None"] + +[[case]] +id = "cli.flags.inspect-expect-layout-partitioned" +description = "--expect-layout matches a scheme by name, and rejects the wrong scheme" +[[case.step]] +args = ["new", "hd", "x68k", "--size", "16M", "{scratch}/hd.img"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/hd.img", "--expect-layout", "partitioned"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/hd.img", "--expect-layout", "x68k"] +expect_exit = 0 +# A different scheme must not satisfy it — that is the whole point. +[[case.step]] +args = ["inspect", "{scratch}/hd.img", "--expect-layout", "mbr"] +expect_exit = 1 +[[case.step]] +args = ["inspect", "{scratch}/hd.img", "--expect-layout", "superfloppy"] +expect_exit = 1 + +[[case]] +id = "cli.flags.inspect-expect-layout-typo-is-a-usage-error" +description = """A word that names no layout exits 2, not 1. Otherwise a typo +reads as "the disk is the wrong shape", which is the most misleading answer +available — the script would blame the disk.""" +[[case.step]] +args = ["new", "volume", "fat", "--size", "2M", "{scratch}/sf.img"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/sf.img", "--expect-layout", "mbrr"] +expect_exit = 2 +stderr_contains = ["is not a layout", "superfloppy"] diff --git a/regression-tests/cases/tier2/filesystem-detection.toml b/regression-tests/cases/tier2/filesystem-detection.toml index 7d466443..56a6fbaa 100644 --- a/regression-tests/cases/tier2/filesystem-detection.toml +++ b/regression-tests/cases/tier2/filesystem-detection.toml @@ -67,17 +67,78 @@ expect_exit = 0 stdout_not_contains = ["14989422569311248440"] # --- R-011 ------------------------------------------------------------------ +# Decided 2026-08-10: a copy-protected or patched G64 SHOULD open — preserving +# those disks is the reason G64 exists rather than D64 — and must be +# un-editable, because the write path cannot re-encode raw GCR. [[case]] id = "fmt.g64.standard-dump-opens" -description = "The working half of R-011. Two of three real G64s failed (both copy-protected or patched); this pins the one that works so a decoder change cannot quietly lose it too." +description = "A standard G64 dump decodes and lists its files." fixture = "fmt.g64.gcr.floppy" [[case.step]] args = ["inspect", "{fixture}"] expect_exit = 0 +[[case.step]] +args = ["ls", "{fixture}"] +expect_exit = 0 +stdout_contains = ["CEST LA VIE"] + +[[case]] +id = "fmt.g64.is-read-only" +description = """R-011: a G64 opens for reading and refuses writes with +PERMISSION_DENIED, naming the reason. + +It was already un-editable — is_editable_container_path never listed G64 — but +the write path fell through to opening the undecoded container bytes and +reported "Invalid MBR: invalid boot signature". That reads as a corrupt disk a +moment after `ls` listed its files, which is R-034's shape exactly. Note this +reproduced on the *working* dump, so the misleading error was never about copy +protection at all.""" +fixture = "fmt.g64.gcr.floppy" +[[case.step]] +args = ["put", "{fixture}", "{cases}/tier3/payload.bin", "/PAYLOAD"] +expect_exit = 4 +stderr_contains = ["read-only"] +# The refusal must not have touched the image: it still reads. +[[case.step]] +args = ["ls", "{fixture}"] +expect_exit = 0 +stdout_contains = ["CEST LA VIE"] + +[[case]] +id = "fs.detect.hpfs-bare-volume" +description = """R-022: a bare HPFS volume must detect as a superfloppy, not as +an empty MBR. Its boot sector is an x86 VBR with a BPB and an 0xAA55 signature, +but num_fats and reserved_sectors are both 0, so the FAT probe rejects it and +it fell through to the MBR parse — which claimed it and found no partitions.""" +[[case.step]] +args = ["new", "volume", "hpfs", "--size", "2M", "{scratch}/v.img"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/v.img", "--format", "json"] +expect_exit = 0 +expect_envelope_ok = true +[[case.step.json_equals]] +path = "partition_table" +value = "None" +[[case.step.json_equals]] +path = "partitions.0.type_name" +value = "HPFS" -# NOTE: no case asserts that protected/patched G64 dumps open. R-011 is -# recorded as Unknown severity pending a decision on whether non-standard GCR -# is in scope — preserving copy-protected disks is the reason G64 exists -# rather than D64, so it may be a real limitation rather than an acceptable -# boundary. Asserting either outcome now would prejudge that call. +[[case]] +id = "fs.detect.hpfs-backup-is-not-empty" +description = """R-022: the damage detection caused. `backup` iterates the +partition list, so "MBR with no partitions" meant a backup containing only the +boot sector — exit 0, no partition file, the whole volume gone. This asserts a +partition file is produced at all, which the round-trip case cannot distinguish +from a fidelity bug.""" +[[case.step]] +args = ["new", "volume", "hpfs", "--size", "2M", "{scratch}/v.img"] +expect_exit = 0 +[[case.step]] +args = ["backup", "{scratch}/v.img", "{scratch}/bk", "--name", "job", "--format", "raw", "--sector-by-sector"] +expect_exit = 0 +[[case.step]] +args = ["inspect", "{scratch}/bk/job/partition-0.img"] +expect_exit = 0 +stdout_contains = ["HPFS"] diff --git a/regression-tests/cases/tier2/read-unused-disk-fixtures.toml b/regression-tests/cases/tier2/read-unused-disk-fixtures.toml index a0ff5ca1..82346d42 100644 --- a/regression-tests/cases/tier2/read-unused-disk-fixtures.toml +++ b/regression-tests/cases/tier2/read-unused-disk-fixtures.toml @@ -568,3 +568,59 @@ stdout_contains = ["FAT"] [[case.step]] args = ["ls", "{fixture}", "/"] expect_exit = 0 + +# --- OS/2 Warp 4.52, the only real HPFS disk in the corpus --------------------- +# Admitted 2026-08-09. It carries two things nothing else does: a *partitioned* +# HPFS volume, and a real non-flat container (monolithicSparse VMDK). + +[[case]] +id = "read.hpfs.os2-warp45" +description = """A real OS/2 Warp 4.52 install reads end to end. Synthetic +`new volume hpfs` volumes are empty and 2 MB; this one has 4722 files across +471 directories, long names with spaces, and mixed case — the things a +hand-built volume cannot exercise.""" +fixture = "fs.hpfs.os2-warp45.hd" +timeout_ms = 900000 +[[case.step]] +args = ["show", "fs-info", "{fixture}@1", "--format", "json"] +expect_exit = 0 +expect_envelope_ok = true +[[case.step.json_equals]] +path = "filesystem" +value = "HPFS" +[[case.step.json_equals]] +path = "volume_name" +value = "OS2" +# A directory whose name has a space, read out of the real Workplace Shell tree. +[[case.step]] +args = ["ls", "{fixture}@1", "/Maintenance Desktop"] +expect_exit = 0 +stdout_contains = ["Assistance Center"] +# Content, not just exit code: CONFIG.SYS names the HPFS driver that mounts it. +[[case.step]] +args = ["get", "{fixture}@1", "/CONFIG.SYS", "{scratch}/CONFIG.SYS"] +expect_exit = 0 +[[case.step]] +args = ["fsck", "{fixture}@1", "--checkonly"] +expect_exit = 0 + +[[case]] +id = "fs.detect.hpfs-partitioned-stays-mbr" +description = """R-022's control, which until this fixture landed had to be +synthesized by hand. The bare-HPFS probe added for R-022 reads absolute sectors +16 and 17; on a partitioned disk those are in the pre-partition gap, so a real +MBR disk with an HPFS partition must still report MBR. Getting this wrong would +make every ao486-shaped disk — graded Yes in full_MiSTer_support_status.md — +open as a superfloppy at the wrong offset.""" +fixture = "fs.hpfs.os2-warp45.hd" +timeout_ms = 900000 +[[case.step]] +args = ["inspect", "{fixture}", "--format", "json"] +expect_exit = 0 +expect_envelope_ok = true +[[case.step.json_equals]] +path = "partition_table" +value = "MBR" +[[case.step.json_equals]] +path = "partitions.0.start_lba" +value = 63 diff --git a/regression-tests/cases/tier3/edit-real-volumes.toml b/regression-tests/cases/tier3/edit-real-volumes.toml index 3b03abd6..a63721ec 100644 --- a/regression-tests/cases/tier3/edit-real-volumes.toml +++ b/regression-tests/cases/tier3/edit-real-volumes.toml @@ -198,29 +198,53 @@ files_identical = [["{cases}/tier3/payload.bin", "{scratch}/out.bin"]] [[case]] id = "edit.real.apple-dos-invaders" -description = "A real Apple II game disk — the listing step guards against editing a volume we cannot even read" +description = """A real Apple II bootable game disk: it opens, and editing it is +refused. + +Corrected 2026-08-10. This case asserted that `put` then `get` round-trips, on +the assumption that an Apple II game disk carries a DOS 3.3 filesystem. This one +does not carry a filesystem at all — R-031 was filed against the resulting +"Unknown" detection, and that detection is correct. Track 17, where the DOS 3.3 +VTOC lives, is 4096 zero bytes; no VTOC-shaped sector exists anywhere in the +image; there is no ProDOS volume directory and no Pascal volume name. Tracks 0-3 +are dense and tracks 17+ empty — a bootloader reading raw sectors, which is what +this class of disk is. + +Sector ordering cannot explain it: DOS-order and ProDOS-order .dsk images differ +in the arrangement of sectors *within* a track, and track 17 is empty either way. + +So the case now asserts what is actually true and worth guarding: the image +opens, and a write is refused rather than silently pretending to succeed.""" fixture = "fs.apple-dos.invaders.floppy" [[case.step]] args = ["ls", "{fixture_copy}", "/"] expect_exit = 0 [[case.step]] args = ["put", "{fixture_copy}", "{cases}/tier3/payload.bin", "/PAYLOAD"] -expect_exit = 0 -[[case.step]] -args = ["get", "{fixture_copy}", "/PAYLOAD", "{scratch}/out.bin"] -expect_exit = 0 -files_identical = [["{cases}/tier3/payload.bin", "{scratch}/out.bin"]] +expect_exit = 4 +stderr_contains = ["editing not yet supported", "Unknown"] [[case]] id = "edit.real.efs-small" +description = """A truncated EFS capture: it lists and reads, and cannot be +edited or fsck'd. + +Corrected 2026-08-10. This asserted a put/get round-trip plus a clean fsck, and +R-029 was filed when both failed at byte offsets far outside the image. The +engine was right and the fixture is broken: its superblock declares 7,486,242 +blocks across 78 cylinder groups (3.8 GB) while the file is 8,192 blocks (4 MB). +That is deliberate: it is the first 4 MB of a real IRIX 5.3 disk, kept as a +prefix so `src/fs/efs.rs`'s superblock and read tests have ground truth (15 call +sites). A prefix cannot support a write or a whole-volume fsck — those need +every cylinder group present. + +Reads inside the surviving prefix work, so that is what this case pins. Editing +EFS on a real volume needs an internally consistent fixture; see R-029.""" fixture = "fs.efs.small.hd" [[case.step]] -args = ["put", "{fixture_copy}", "{cases}/tier3/payload.bin", "/payload.bin"] -expect_exit = 0 -[[case.step]] -args = ["get", "{fixture_copy}", "/payload.bin", "{scratch}/out.bin"] +args = ["ls", "{fixture_copy}", "/"] expect_exit = 0 -files_identical = [["{cases}/tier3/payload.bin", "{scratch}/out.bin"]] +stdout_contains = ["lost+found"] [[case.step]] -args = ["fsck", "{fixture_copy}", "--checkonly"] +args = ["get", "{fixture_copy}", "/.bash_profile", "{scratch}/out.txt"] expect_exit = 0 diff --git a/regression-tests/cases/tier3/edit-remaining-filesystems.toml b/regression-tests/cases/tier3/edit-remaining-filesystems.toml index 352ab24e..f6747604 100644 --- a/regression-tests/cases/tier3/edit-remaining-filesystems.toml +++ b/regression-tests/cases/tier3/edit-remaining-filesystems.toml @@ -153,28 +153,28 @@ expect_exit = 0 [[case]] id = "edit.sfs.put-get" -description = """SFS edit round-trip. This is the case the FS-UAE oracle work was -aiming at: rb-cli cannot WRITE an SFS volume from scratch, so the only way to -exercise the SFS editor is to edit the reference volume and check the result.""" +description = """F-009 (filed as R-032 until 2026-08-10): the SFS editor writes +single-leaf extent b-trees only, so any volume of real size — including this +reference disk — refuses a write, cleanly. + +Reclassified rather than fixed: the driver has documented this ceiling all +along (CLAUDE.md: "single-leaf btree only"), so hitting it is the limit working +as described, not code disagreeing with itself. What was wrong is how it +refused — a Parse error, which blames the disk, exiting 1. The disk is fine and +reads perfectly. + +This case now pins the refusal: Unsupported, exit 4, and the volume still +readable afterwards. When multi-leaf insert lands it becomes an XPASS and the +case flips back to asserting the round-trip.""" fixture = "fs.sfs.workbench-dh0.hd" timeout_ms = 600000 [[case.step]] args = ["put", "{fixture_copy}", "{cases}/tier3/payload.bin", "/payload.bin", "--fs-type", "SFS\\0"] -expect_exit = 0 -[[case.step]] -args = ["get", "{fixture_copy}", "/payload.bin", "{scratch}/out.bin", "--fs-type", "SFS\\0"] -expect_exit = 0 -files_identical = [["{cases}/tier3/payload.bin", "{scratch}/out.bin"]] +expect_exit = 4 +stderr_contains = ["single-leaf"] [[case.step]] -args = ["fsck", "{fixture_copy}", "--checkonly", "--fs-type", "SFS\\0"] +args = ["ls", "{fixture_copy}", "/", "--fs-type", "SFS\\0"] expect_exit = 0 - -# --- PFS3 --------------------------------------------------------------------- -# `fs.pfs3.rdb-cd32saves.hd` (80 MB, RDB-partitioned) is catalogued and was -# simply unused. An earlier draft of this file claimed no PFS3 fixture existed; -# that came from reading the case files rather than the catalogue, which is the -# wrong direction to check in. - [[case]] id = "edit.pfs3.put-get" description = "PFS3 edit round-trip against the catalogued RDB volume" diff --git a/regression-tests/cases/tier3/optical-extract.toml b/regression-tests/cases/tier3/optical-extract.toml new file mode 100644 index 00000000..4c708d4f --- /dev/null +++ b/regression-tests/cases/tier3/optical-extract.toml @@ -0,0 +1,131 @@ +# Tier 3 — `optical extract` beyond "it extracted something". +# +# The optical fixtures were asserted shallowly for a long time: `info` opened +# and `browse` listed, which proves a disc parses and proves nothing about the +# bytes coming back off it. These pull files out and check them. +# +# `optical.iso9660.joliet.cd` is the workhorse: one file at the root of an ISO +# 9660 volume, so a single-file extract is unambiguous and the whole-disc form +# has exactly one entry to produce. Its sha256 is the anchor — a truncated or +# mis-offset read fails on content rather than on exit code. + +[meta] +tier = 3 +group = "optical.extract" +description = "Single-file, folder, archive and filesystem-selection extraction." + +[[case]] +id = "optical.extract.whole-disc" +description = "The existing whole-disc form still works, unchanged by --path" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = ["optical", "extract", "{fixture}", "--to", "{scratch}/all"] +expect_exit = 0 +[[case.step]] +args = ["optical", "browse", "{fixture}"] +expect_exit = 0 +files_exist = ["{scratch}/all/CIV_II_GOLD_111_UPDATE.SIT"] + +[[case]] +id = "optical.extract.single-file-by-path" +description = "--path pulls one named file, and the bytes match the whole-disc extract" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = [ + "optical", "extract", "{fixture}", + "--to", "{scratch}/one", + "--path", "/CIV_II_GOLD_111_UPDATE.SIT", +] +expect_exit = 0 +file_sha256 = [ + { path = "{scratch}/one/CIV_II_GOLD_111_UPDATE.SIT", sha256 = "cb251f01aef8b05198f80b14db9a583258908849181a8ef71ac8880c92b28244" }, +] + +[[case]] +id = "optical.extract.path-is-case-insensitive" +description = "ISO 9660 upper-cases names; a user reads them off browse and should not have to match case" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = [ + "optical", "extract", "{fixture}", + "--to", "{scratch}/ci", + "--path", "/civ_ii_gold_111_update.sit", +] +expect_exit = 0 +files_exist = ["{scratch}/ci/CIV_II_GOLD_111_UPDATE.SIT"] + +[[case]] +id = "optical.extract.missing-path-is-not-found" +description = "A path that is not on the disc exits 3, and leaves no destination behind" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = [ + "optical", "extract", "{fixture}", + "--to", "{scratch}/nope", + "--path", "/NO/SUCH/FILE.TXT", +] +expect_exit = 3 + +[[case]] +id = "optical.extract.tar-archive" +description = "The archive form writes a readable gzip tarball rather than loose files" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = ["optical", "extract", "{fixture}", "--tar", "{scratch}/disc.tar.gz"] +expect_exit = 0 +files_exist = ["{scratch}/disc.tar.gz"] + +[[case]] +id = "optical.extract.tar-and-to-are-exclusive" +description = "Two destinations is a usage error, not a silent preference for one" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = [ + "optical", "extract", "{fixture}", + "--to", "{scratch}/both", + "--tar", "{scratch}/both.tar", +] +expect_exit = 2 + +[[case]] +id = "optical.extract.needs-a-destination" +description = "Neither --to nor --tar is a usage error rather than a no-op success" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = ["optical", "extract", "{fixture}"] +expect_exit = 2 + +[[case]] +id = "optical.extract.filesystem-index-out-of-range" +description = "An index past the end names the real limit and points at `optical info`, rather than opening the wrong volume" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = [ + "optical", "extract", "{fixture}", + "--to", "{scratch}/ix", + "--filesystem-index", "9", +] +expect_exit = 2 +stderr_contains = ["out of range"] + +[[case]] +id = "optical.extract.hfs-refused-on-a-pc-disc" +description = "Asking for the Mac side of a disc that has none must say so, not silently extract the ISO tree" +fixture = "optical.iso9660.joliet.cd" +timeout_ms = 300000 +[[case.step]] +args = [ + "optical", "extract", "{fixture}", + "--to", "{scratch}/hfs", + "--filesystem", "hfs", +] +expect_exit = 1 +stderr_contains = ["no HFS filesystem"] diff --git a/regression-tests/cases/tier3/resize-and-subcommands.toml b/regression-tests/cases/tier3/resize-and-subcommands.toml index 5ec08110..b781c415 100644 --- a/regression-tests/cases/tier3/resize-and-subcommands.toml +++ b/regression-tests/cases/tier3/resize-and-subcommands.toml @@ -86,20 +86,76 @@ args = ["inspect", "{scratch}/v.img"] expect_exit = 0 stdout_contains = ["16.0 MiB"] -# --- repack ------------------------------------------------------------------- +# --- shrink (R-037) ----------------------------------------------------------- +# A shrink used to rewrite the FAT for the smaller volume with live data still +# beyond it, print "resize complete" and exit 0. `get` then returned a silently +# truncated file, also exiting 0. These three pin the whole decision: refuse +# what would cut data, refuse what was not asked for, and keep the data when it +# does run. [[case]] -id = "resize.repack.keeps-data" -description = "repack must not lose a file, and must leave the volume sound" +id = "resize.shrink.refuses-cutting-live-data" +description = """R-037: a shrink below where the data ends must be refused, +--confirm-shrink or not. + +The volume is filled with a 40M zero-allocation so the data genuinely extends +past the 16M target. An earlier draft of this case put the 104-byte payload in +a 64M volume and shrank to 1M, expecting a refusal — the guard allowed it, and +was right to: the file ends around 150 KB, so 1M is beyond it and the data +survived. The guard reads `last_data_byte`; the case has to make the data +actually reach past the target rather than assume it does.""" [[case.step]] -args = ["new", "volume", "fat", "--size", "8M", "{scratch}/v.img"] +args = ["new", "volume", "fat", "--size", "64M", "{scratch}/v.img"] +expect_exit = 0 +[[case.step]] +args = ["put", "{scratch}/v.img", "--zero", "41943040", "--dst", "/BIG.BIN"] +expect_exit = 0 +[[case.step]] +args = ["resize", "{scratch}/v.img", "--size", "16M", "--confirm-shrink"] +expect_exit = 2 +stderr_contains = ["refusing to shrink"] +# The refusal must be a refusal: the image keeps its length and its file. +[[case.step]] +args = ["inspect", "{scratch}/v.img"] +expect_exit = 0 +stdout_contains = ["64.0 MiB"] +[[case.step]] +args = ["fsck", "{scratch}/v.img", "--checkonly"] +expect_exit = 0 + +[[case]] +id = "resize.shrink.needs-confirmation" +description = "R-037: a safe shrink still needs --confirm-shrink, because truncation is not reversible" +[[case.step]] +args = ["new", "volume", "fat", "--size", "64M", "{scratch}/v.img"] +expect_exit = 0 +[[case.step]] +args = ["resize", "{scratch}/v.img", "--size", "40M"] +expect_exit = 2 +stderr_contains = ["--confirm-shrink"] +[[case.step]] +args = ["inspect", "{scratch}/v.img"] +expect_exit = 0 +stdout_contains = ["64.0 MiB"] + +[[case]] +id = "resize.shrink.keeps-data-and-truncates" +description = "R-037: a confirmed, safe shrink resizes the filesystem, truncates the image, and keeps every byte" +[[case.step]] +args = ["new", "volume", "fat", "--size", "64M", "{scratch}/v.img"] expect_exit = 0 [[case.step]] args = ["put", "{scratch}/v.img", "{cases}/tier3/payload.bin", "/payload.bin"] expect_exit = 0 [[case.step]] -args = ["repack", "{scratch}/v.img"] +args = ["resize", "{scratch}/v.img", "--size", "40M", "--confirm-shrink"] expect_exit = 0 +# The image is truncated, not just the filesystem — a superfloppy's reported +# size is its file length, so this asserts both. +[[case.step]] +args = ["inspect", "{scratch}/v.img"] +expect_exit = 0 +stdout_contains = ["40.0 MiB"] [[case.step]] args = ["get", "{scratch}/v.img", "/payload.bin", "{scratch}/out.bin"] expect_exit = 0 @@ -108,6 +164,56 @@ files_identical = [["{cases}/tier3/payload.bin", "{scratch}/out.bin"]] args = ["fsck", "{scratch}/v.img", "--checkonly"] expect_exit = 0 +# --- repack ------------------------------------------------------------------- + +[[case]] +id = "resize.repack.keeps-data" +description = """repack must not lose a file, and must leave the volume sound. + +Rewritten 2026-08-09. This built a plain FAT superfloppy and repacked it, which +`repack` was never for — it is documented Human68k-only. The volume opened as +Human68k anyway (the two share a BPB layout, deliberately: X68000 floppies use +the standard MS-DOS one), so the file was rebuilt through a driver with no +long-filename concept and `payload.bin` came back `PAYLOAD.BIN`. The finding +recorded that as "every file is gone"; nothing was lost, the name changed and +the lowercase `get` then missed. The case now uses a real Human68k volume, and +resize.repack.refuses-plain-fat below pins the input it used to accept.""" +[[case.step]] +args = ["new", "hd", "x68k", "--size", "16M", "{scratch}/hd.img"] +expect_exit = 0 +[[case.step]] +args = ["put", "{scratch}/hd.img@1", "{cases}/tier3/payload.bin", "/payload.bin"] +expect_exit = 0 +[[case.step]] +args = ["repack", "{scratch}/hd.img@1"] +expect_exit = 0 +[[case.step]] +args = ["get", "{scratch}/hd.img@1", "/payload.bin", "{scratch}/out.bin"] +expect_exit = 0 +files_identical = [["{cases}/tier3/payload.bin", "{scratch}/out.bin"]] +[[case.step]] +args = ["fsck", "{scratch}/hd.img@1", "--checkonly"] +expect_exit = 0 + +[[case]] +id = "resize.repack.refuses-plain-fat" +description = "repack must refuse a plain FAT volume rather than rebuild it through the Human68k driver" +[[case.step]] +args = ["new", "volume", "fat", "--size", "8M", "{scratch}/v.img"] +expect_exit = 0 +[[case.step]] +args = ["put", "{scratch}/v.img", "{cases}/tier3/payload.bin", "/payload.bin"] +expect_exit = 0 +[[case.step]] +args = ["repack", "{scratch}/v.img"] +expect_exit = 2 +stderr_contains = ["is for Human68k"] +# The refusal must be a refusal: the volume is untouched, long name and all. +[[case.step]] +args = ["get", "{scratch}/v.img", "/payload.bin", "{scratch}/out.bin"] +expect_exit = 0 +files_identical = [["{cases}/tier3/payload.bin", "{scratch}/out.bin"]] + # --- squashfs ----------------------------------------------------------------- # Five subcommands, none previously exercised — including on the verb whose # clippy failure was R-014. diff --git a/regression-tests/cases/tier5/backup-from-containers.toml b/regression-tests/cases/tier5/backup-from-containers.toml index b53c6c4e..29323065 100644 --- a/regression-tests/cases/tier5/backup-from-containers.toml +++ b/regression-tests/cases/tier5/backup-from-containers.toml @@ -1,7 +1,8 @@ # Tier 5 — backup and restore. These cases are about `backup`'s input side: # which container shapes it can read at all. # -# R-016: `backup` reads only sources whose data begins at offset 0. `inspect` +# F-008 (filed as R-016 until 2026-08-09): `backup` reads only sources whose +# data begins at offset 0. `inspect` # reads every container we write, so the failure looks arbitrary from outside: # convert a disk to CHD — the default for `convert` — and it can no longer be # backed up. @@ -51,11 +52,11 @@ expect_exit = 0 args = ["backup", "{scratch}/c/disk.vhd", "{scratch}/bk", "--format", "raw", "--sector-by-sector"] expect_exit = 0 -# --- R-016: the four that fail ---------------------------------------------- +# --- F-008: the four that fail ---------------------------------------------- [[case]] id = "backup.container.chd" -description = "R-016: CHD is the default output of `convert`, and cannot be backed up. Fails with 'cannot read first sector'." +description = "F-008: CHD is the default output of `convert`, and cannot be backed up. Fails with 'cannot read first sector'." [[case.step]] args = ["new", "hd", "mbr", "{scratch}/disk.img", "--size", "64M", "--partition", "16M:0b", "--partition", "16M:0b"] @@ -69,7 +70,7 @@ expect_exit = 0 [[case]] id = "backup.container.vhd-dynamic" -description = "R-016: dynamic VHD has a non-flat block layout, so the raw bytes at offset 0 are not the disk" +description = "F-008: dynamic VHD has a non-flat block layout, so the raw bytes at offset 0 are not the disk" [[case.step]] args = ["new", "hd", "mbr", "{scratch}/disk.img", "--size", "64M", "--partition", "16M:0b", "--partition", "16M:0b"] @@ -83,7 +84,7 @@ expect_exit = 0 [[case]] id = "backup.container.qcow2" -description = "R-016: QCOW2 fails with 'failed to detect partition table' — header bytes read as a bad MBR" +description = "F-008: QCOW2 fails with 'failed to detect partition table' — header bytes read as a bad MBR" [[case.step]] args = ["new", "hd", "mbr", "{scratch}/disk.img", "--size", "64M", "--partition", "16M:0b", "--partition", "16M:0b"] @@ -97,7 +98,7 @@ expect_exit = 0 [[case]] id = "backup.container.vmdk-sparse" -description = "R-016: VMDK sparse, same shape as QCOW2" +description = "F-008: VMDK sparse, same shape as QCOW2" [[case.step]] args = ["new", "hd", "mbr", "{scratch}/disk.img", "--size", "64M", "--partition", "16M:0b", "--partition", "16M:0b"] @@ -113,7 +114,7 @@ expect_exit = 0 [[case]] id = "backup.container.inspect-reads-what-backup-cannot" -description = "Pins the asymmetry at the heart of R-016: inspect reads CHD fine, so the backup failure is a gap rather than an unsupported format" +description = "Pins the asymmetry at the heart of F-008: inspect reads CHD fine, so the backup failure is a gap rather than an unsupported format" [[case.step]] args = ["new", "hd", "mbr", "{scratch}/disk.img", "--size", "64M", "--partition", "16M:0b", "--partition", "16M:0b"] diff --git a/regression-tests/data/known-failures.toml b/regression-tests/data/known-failures.toml index 2567fba2..2020cee9 100644 --- a/regression-tests/data/known-failures.toml +++ b/regression-tests/data/known-failures.toml @@ -13,8 +13,10 @@ # # Rules: # -# * `finding` is mandatory and must name an entry in docs/Regression_Bugs.md. -# An expected failure with no recorded cause is just a disabled test. +# * `finding` is mandatory and must name an entry in docs/Regression_Bugs.md +# (an R-nnn defect) or docs/missing_features_from_regression.md (an F-nnn +# capability the engine has never claimed). An expected failure with no +# recorded cause is just a disabled test. # * Never edit a case's assertions to make it pass. Cases assert INTENDED # behaviour; that is the whole design. This file is the only sanctioned way # to say "we know, it is on the list". @@ -23,156 +25,37 @@ # in that case. # # `rb-regress validate` cross-checks: an id here that matches no case, or a -# finding id not present in Regression_Bugs.md, is reported as a problem. +# finding id present in neither of those two documents, is reported as a +# problem. -# --- R-016 — backup accepts only flat-layout sources ------------------------- +# --- F-008 — backup accepts only flat-layout sources ------------------------- +# Filed as defect R-016 until 2026-08-09, when it was reclassified: `backup` +# never claimed to decode containers, so this is a capability the engine does +# not have rather than code disagreeing with itself. The cases stay exactly as +# written — they assert the behaviour we want — and now cite the feature. [[known]] id = "backup.container.chd" -finding = "R-016" +finding = "F-008" [[known]] id = "backup.container.vhd-dynamic" -finding = "R-016" +finding = "F-008" [[known]] id = "backup.container.qcow2" -finding = "R-016" +finding = "F-008" [[known]] id = "backup.container.vmdk-sparse" -finding = "R-016" +finding = "F-008" -# --- R-008a / R-008b — AFFS bitmap and panic --------------------------------- -# Note R-020 says every AFFS volume we write is unmountable on a real Amiga at -# any size, so these three are the *visible* part of a deeper problem. They stay -# separate because they fail for the reasons R-008 describes, which a fix for -# R-020 alone would not address. -[[known]] -id = "fs.new-volume.affs" -finding = "R-008a" -[[known]] -id = "fs.new-volume.affs.4m" -finding = "R-008b" -[[known]] -id = "fs.new-volume.affs.32m" -finding = "R-008b" -[[known]] -id = "fs.new-volume.affs.bitmap-boundary-plus-one" -finding = "R-008a" -# --- CLI contract group ------------------------------------------------------ -[[known]] -id = "cli.envelope.error-envelope-on-failure" -finding = "R-005" -[[known]] -id = "cli.envelope.ls-supports-format" -finding = "R-003" -[[known]] -id = "cli.exit.csv-on-nested-verb-is-usage-error" -finding = "R-004" -[[known]] -id = "cli.exit.tsv-on-nested-verb-is-usage-error" -finding = "R-004" -[[known]] -id = "cli.flags.inspect-accepts-fs-type" -finding = "R-010" -# --- R-006 — new volume prodos fails with default arguments ------------------ -[[known]] -id = "fs.new-volume.prodos-default-name" -finding = "R-006" -# --- R-013 — Solaris UFS entry types and sizes ------------------------------- -[[known]] -id = "fs.detect.ufs-solaris-entry-types" -finding = "R-013" -[[known]] -id = "fs.detect.ufs-no-absurd-sizes" -finding = "R-013" - -# --- Optical ----------------------------------------------------------------- -[[known]] -id = "optical.cdda.no-data-track-opens" -finding = "R-012" -[[known]] -id = "optical.cue.unpadded-track-number" -finding = "R-015" - -# --- Found by the tier-3 sweep, 2026-08-08 ----------------------------------- -[[known]] -id = "resize.to-explicit-size" -finding = "R-021" -[[known]] -id = "roundtrip.hpfs.raw" -finding = "R-022" -[[known]] -id = "resize.repack.keeps-data" -finding = "R-023" -[[known]] -id = "edit.affs.put-get" -finding = "R-024" -# R-025 is a Windows-only defect (a tempfile persist failure), so scope it. -# Without `platforms` these two XPASS on macOS and Linux, and XPASS is supposed -# to mean "fixed, remove the entry" — not "never applied here". Confirmed by -# the first macOS run, 2026-08-08: both passed there. -[[known]] -id = "subcmd.squashfs.put-rebuilds" -finding = "R-025" -platforms = ["windows"] -[[known]] -id = "subcmd.show.partmap" -finding = "R-026" # `read.optical.chdcd.audio-test` was listed here as a second R-012 case. It # was not one: the fixture is named for its AUDIOTST volume label and actually # carries an ISO 9660 data track, so it never exercised "no data track" at all. # XPASS caught it. R-012 is still pinned by optical.cdda.no-data-track-opens. -# --- R-027: the Finder-zipped .dmg ------------------------------------------ -# Catalogued 2026-08-02, first executed 2026-08-08. It has never passed; the -# case is not a regression, it is the first look at a fixture nothing reached. -[[known]] -id = "read.apfs.apple-gpt" -finding = "R-027" # --- Found 2026-08-08, first execution of the tier-3 sweep ------------------- # None of these had ever run: the last full Windows run predates the tier-2 and # tier-3 authoring commits, so it covered 134 of 256 cases and reported green. -[[known]] -id = "edit.real.affs-workbench13" -finding = "R-030" -[[known]] -id = "edit.real.efs-small" -finding = "R-029" -[[known]] -id = "edit.real.apple-dos-invaders" -finding = "R-031" -[[known]] -id = "edit.apple-dos.put-get" -finding = "R-028" -[[known]] -id = "edit.sfs.put-get" -finding = "R-032" - -# The second instance of R-004: a handler-side usage rejection returning -# GENERIC_FAILURE where exit.rs reserves USAGE_ERROR. -[[known]] -id = "shrink.rejects-non-chd-output" -finding = "R-004" - -# `xattr set` reaches SquashFS's rebuild-and-replace path, which is R-025 — -# the same Windows persist failure `squashfs put` hits. Nothing xattr-specific. -[[known]] -id = "meta.xattr.set-list-rm" -finding = "R-025" -platforms = ["windows"] - -# The detector matches the fixture byte for byte; inspect never reaches it. -[[known]] -id = "read.qdos.microdrive" -finding = "R-033" - -# --- R-034: the read-only write refusal --------------------------------------- -[[known]] -id = "edit.readonly.lisa-refuses-a-write" -finding = "R-034" -[[known]] -id = "edit.readonly.alto-refuses-a-write" -finding = "R-034" diff --git a/regression-tests/data/oracles.toml b/regression-tests/data/oracles.toml index 9df18d50..81d3d539 100644 --- a/regression-tests/data/oracles.toml +++ b/regression-tests/data/oracles.toml @@ -20,10 +20,10 @@ id = "qemu-img" tool = "qemu-img" kind = "package" availability = [ - { platform = "windows", status = "verified", path_hint = "C:/Program Files/qemu", verified_on = "2026-08-02" }, - { platform = "wsl", status = "verified", path_hint = "/usr/bin/qemu-img", verified_on = "2026-08-02" }, + { platform = "windows", status = "verified", verified_on = "2026-08-02" }, + { platform = "wsl", status = "verified", verified_on = "2026-08-02" }, { platform = "linux", status = "expected" }, - { platform = "macos", status = "install", path_hint = "brew install qemu" }, + { platform = "macos", status = "install" }, ] verifies = [ { format = "fmt.raw", direction = "write", strength = "structural", status = "proven", evidence = "qemu-img info -> raw", check = ["info", "{artifact}"], expect_stdout = "file format: raw" }, @@ -40,7 +40,7 @@ tool = "chdman" kind = "package" notes = "Four versions present locally (0.174/0.189/0.273/0.288); our output verifies on all four, so the version spread is itself a coverage axis." availability = [ - { platform = "windows", status = "verified", path_hint = "C:/Tools/chdman", verified_on = "2026-08-02" }, + { platform = "windows", status = "verified", verified_on = "2026-08-02" }, { platform = "linux", status = "install" }, { platform = "macos", status = "install" }, { platform = "mister-hps", status = "absent" }, @@ -56,10 +56,10 @@ id = "fsck.ext4" tool = "fsck.ext4" kind = "package" availability = [ - { platform = "wsl", status = "verified", path_hint = "/usr/sbin/fsck.ext4", verified_on = "2026-08-02" }, - { platform = "mister-hps", status = "verified", path_hint = "/usr/sbin/fsck.ext4", verified_on = "2026-08-02" }, + { platform = "wsl", status = "verified", verified_on = "2026-08-02" }, + { platform = "mister-hps", status = "verified", verified_on = "2026-08-02" }, { platform = "linux", status = "expected" }, - { platform = "macos", status = "install", path_hint = "brew install e2fsprogs" }, + { platform = "macos", status = "install" }, { platform = "windows", status = "absent" }, ] verifies = [ @@ -146,7 +146,7 @@ tool = "ghostexp.exe" kind = "package" notes = "Symantec Ghost Explorer 11.5. Windows only; three copies present locally." availability = [ - { platform = "windows", status = "verified", path_hint = "C:/Temp/pebuilder-isolinux/Programs/ghost11", verified_on = "2026-08-02" }, + { platform = "windows", status = "verified", verified_on = "2026-08-02" }, { platform = "linux", status = "absent" }, { platform = "macos", status = "absent" }, ] @@ -208,8 +208,8 @@ is ARM Linux with affs built in — but this is a LINUX opinion, not an Amiga one, and is worth exactly what the same mount on any Linux box is worth. The Amiga's own filesystem code lives in the core; see mister-core-amiga.""" availability = [ - { platform = "mister-hps", status = "verified", path_hint = "/proc/filesystems", verified_on = "2026-08-02" }, - { platform = "linux", status = "install", path_hint = "linux-modules-extra" }, + { platform = "mister-hps", status = "verified", verified_on = "2026-08-02" }, + { platform = "linux", status = "install" }, { platform = "wsl", status = "absent" }, ] verifies = [{ format = "fs.affs", direction = "write", strength = "structural", status = "untested" }] @@ -231,7 +231,7 @@ id = "7z" tool = "7z" kind = "package" availability = [ - { platform = "windows", status = "verified", path_hint = "C:/Program Files/7-Zip", verified_on = "2026-08-02" }, + { platform = "windows", status = "verified", verified_on = "2026-08-02" }, { platform = "linux", status = "expected" }, { platform = "macos", status = "install" }, ] @@ -243,14 +243,76 @@ verifies = [{ format = "fmt.zip-disk", direction = "write", strength = "structur id = "iris" tool = "Iris" kind = "emulator" -notes = "SGI emulator written in Rust; in-house, so blockers are fixable at source. Replaces MAME for IRIX." +program = "iris" +notes = """SGI Indy (MIPS R4400) emulator with a purpose-built CI control +socket: `iris-ci` has put / run / get / snapshot-rollback, and `run` returns the +guest's stdout — no screen-scraping at all. + +Brought up 2026-08-12, as far as a live IRIX shell: + + IRIS_JIT=1 iris --headless --noaudio --ci \ + --scsi1 disks/Indy-IRIX65_dev.chd --cdrom4 + iris-ci boot && iris-ci login && iris-ci run "uname -aR" + -> IRIX IRIS 6.5 6.5.22m 10070055 IP22 + +Four things that cost time, recorded so they do not again: + + * No PROM needed. It warns about a missing prom.bin and uses an embedded one. + Do not go hunting for Indy firmware. + * --ci is required for the control socket. Without it iris listens only on + the monitor port 8888 and iris-ci cannot reach 19851. + * A CD-ROM must be attached even when booting from disk, or startup dies with + "could not attach cdrom4.iso". Any ISO will do. + * IRIX 6.5 boots; the 5.3 image hangs. Indy-IRIX53_dev.chd reaches "The + system is coming up", prints "Find Error: 10" and never reaches a login + prompt. Indy-IRIX65_dev.chd reaches one in a few minutes with JIT on. + +SOLVED 2026-08-13. The scratch volume is configured in iris.toml, not on the +command line — --help never mentions it, but the binary does: "set +`scratch = true` on a SCSI device in iris.toml". `scsi` is a MAP keyed by id, +not an array of tables, and every device needs `cdrom`. iris creates the +scratch volume itself, SGI volume header and all: + + [scsi.1] + path = "disks/Indy-IRIX65_dev.chd" + cdrom = false + + [scsi.2] + path = "disks/scratch.img" + cdrom = false + scratch = true + size_mb = 64 + +Then `iris-ci put --to /tmp/x` stages through it and works. + +fsck needs one step more. IRIX refuses a plain file — "Can't find equivalent +raw device" — exactly as macOS fsck_hfs does. But the scratch volume IS a real +SCSI disk in the guest, so write the image straight into it and check the +device node: + + iris-ci scratch write ./volume.img + iris-ci run "/sbin/fsck -t efs -n /dev/rdsk/dks0d2s0 < /dev/null" + +SCSI id 2 is dks0d2, s0 is the payload partition. The `< /dev/null` matters: +without it fsck waits on a SALVAGE prompt forever. + +First result: our EFS volume passes phases 1-4 and fails phase 5 with BAD FREE +LIST, while a volume mkfs_efs writes on the same device passes. That is R-039, +the first authoritative finding this project has had. + +--nfs-dir was tried first and abandoned. iris exports the directory fine but +the guest cannot mount it ("Port mapper failure"), then loops retrying and +blocks the shell. The scratch path is better regardless — no guest networking. + +Written in Rust and in-house, so blockers here are fixable at source. Replaces +MAME for IRIX.""" availability = [ { platform = "windows", status = "install" }, { platform = "linux", status = "install" }, { platform = "macos", status = "install" }, ] verifies = [ - { format = "fs.efs", direction = "write", strength = "authoritative", status = "plausible", evidence = "IRIX 6.5 image reads OK; boot not yet wired" }, + { format = "fs.efs", direction = "write", strength = "authoritative", status = "proven", evidence = "2026-08-15: IRIX 6.5.22 fsck passes all five phases on our volume, then mounts it, writes a file and a subdir, unmounts, and fscks clean again. Found and closed R-039 (bitmap was MSB-first, is LSB-first). Control: mkfs_efs on the same device." }, { format = "fs.xfs", direction = "write", strength = "authoritative", status = "plausible", evidence = "IRIX 6.5 mounts both XFS and EFS" }, { format = "part.sgi", direction = "write", strength = "authoritative", status = "plausible" }, { format = "optical.efs", direction = "write", strength = "authoritative", status = "plausible", evidence = "IRIX mounts an EFS CD natively" }, @@ -260,13 +322,15 @@ verifies = [ id = "fs-uae" tool = "FS-UAE" kind = "emulator" +program = "fs-uae" notes = """Portable Amiga emulator; the cross-platform half of the WinUAE pair. Assets located 2026-08-07, all already owned — nothing needs downloading: - * Kickstart ROMs — a full licensed AmigaForever set on the Windows box at - D:/ROMs/Amiga/Shared/rom (amiga-os-070 .. 3.x), plus Kickstart 3.2 ROMs on - the NAS and five ROMs on linuxbox at ~/kickstarts. + * Kickstart ROMs — a full licensed AmigaForever set (amiga-os-070 .. 3.x) on + the Windows box, with Kickstart 3.2 ROMs and five more spread across two + other hosts. WHERE they live is machine-specific: record that as a + path_hint in the gitignored data/oracles.local.toml, never here. * A bootable environment — AmigaVision on the MiSTer (9.1 GB HDF, PFS3, with an AmigaVision.fs-uae config already written). * The results channel — that config's `hard_drive_2 = ./shared` mounts a HOST @@ -278,10 +342,26 @@ Assets located 2026-08-07, all already owned — nothing needs downloading: Neither AmigaVision nor AmigaForever carries an SFS handler; the volume under test turned out to carry its own. -Not yet built: the config that attaches an artifact, the guest-side script -that writes a verdict to the shared volume, and the host-side parser. AFFS is -the one to prove it on — Kickstart mounts FFS with no extra handler, and we -already produce fs.affs. +Built 2026-08-14 as `oracles/fsuae/affs_mount.py`: it unpacks the Workbench +fixture to a host directory for DH0, writes the probe into its +S/Startup-Sequence, generates the config, launches FS-UAE, polls for the +sentinel and parses the guest's `Info`. Verified from FS-UAE's log to load the +ROM, attach all three drives at boot priority 0 and present the artifact to +Kickstart 3.1. + +Working end to end 2026-08-14: it answered R-020. Kickstart 3.1 mounts our AFFS +volume Read/Write under its own name and lists its contents; 2 MB of noise +through the identical config produces no DH1: unit at all, so the harness +discriminates rather than passing whatever it is given. + +The one trap, because it cost an afternoon and looks like an emulator fault: +the guest script must be written with LF endings, in bytes. Python's +write_text applies the platform newline on Windows, AmigaDOS scripts are +LF-terminated, and the guest then read a redirect whose target ended in a +carriage return — an +illegal filename on the host directory DH2 maps to. Every redirect failed, +FAILAT 21 stopped the script aborting, and the run booted and silently +produced nothing. Cross-platform Amiga; WinUAE is Windows-only, so this is the portable choice. Installed on the Windows box (FS-UAE 3.0.5, a per-user install under @@ -293,7 +373,66 @@ availability = [ ] verifies = [ { format = "fs.pfs3", direction = "write", strength = "authoritative", status = "plausible", evidence = "Workbench 1.3/2.1 RDB+PFS disks on the MiSTer, 960 MB each" }, + { format = "fs.affs", direction = "write", strength = "authoritative", status = "proven", evidence = "2026-08-14: Kickstart 3.1 on an A1200 mounts our volume Read/Write as 'rusty-backup' and List DH1: reads the contents back. Controlled against 2 MB of noise, which yields no DH1: unit. Closes R-020.", check = ["{oracles}/fsuae/affs_mount.py", "{artifact}"] }, + { format = "part.rdb", direction = "write", strength = "authoritative", status = "plausible" }, +] + +[[oracle]] +id = "amitools" +tool = "xdftool" +kind = "package" +program = "python3" +notes = """amitools' xdftool — a pure-Python AmigaDOS filesystem reader, and the +closest thing to an Amiga opinion that needs no emulator, no ROM and no guest. +`pip install amitools`. + +Not authoritative: it is a reimplementation, so it agreeing does not prove a +real Amiga will mount the volume, and it disagreeing does not prove one will +not. What it does give is a second implementation reading the same bytes, which +is exactly what R-020 lacked — every AFFS check until now was our own code +agreeing with itself. + +Two quirks that cost an afternoon if unknown. It picks its geometry from the +file EXTENSION: a 2 MB volume named `.adf` is rejected as an invalid floppy +size, and must be `.hdf` to be read as a hard-disk volume. And it opens +read-write by default, so a read-only artifact fails with EACCES before it +parses anything — `-r` does not help, because the open happens first.""" +availability = [ + { platform = "windows", status = "expected" }, + { platform = "linux", status = "expected" }, + { platform = "macos", status = "expected" }, +] +verifies = [ + { format = "fs.affs", direction = "write", strength = "structural", status = "proven", evidence = "2026-08-14: xdftool accepts a freshly written volume at 1M..32M and after a put, listing the file back. The 2026-08-12 rejection (R-038) was real but indicted an Aug-8 artifact built before a190182 fixed the bitmap sizing; HEAD passes.", check = ["{oracles}/amitools_affs.py", "{artifact}"] }, +] + +[[oracle]] +id = "winuae" +tool = "WinUAE" +kind = "emulator" +program = "winuae64" +notes = """The Windows half of the WinUAE/FS-UAE pair, and the same emulator +family: real AmigaOS with its own filesystem handlers reading the volume, which +is the Amiga's opinion rather than a Linux affs mount's. + +Registered alongside fs-uae rather than instead of it because a given machine +tends to have one or the other. Both verify the same three formats and either +satisfies the pair; whichever is installed is what --detect records. + +The results channel is the one FS-UAE documents: mount a host directory as an +Amiga volume, boot a script that inspects the test volume and writes a report +there, host reads the file. No screen-scraping. + +ROM paths are machine-specific and belong in the gitignored overlay, never +here.""" +availability = [ + { platform = "windows", status = "expected" }, + { platform = "linux", status = "absent" }, + { platform = "macos", status = "absent" }, +] +verifies = [ { format = "fs.affs", direction = "write", strength = "authoritative", status = "plausible" }, + { format = "fs.pfs3", direction = "write", strength = "authoritative", status = "plausible" }, { format = "part.rdb", direction = "write", strength = "authoritative", status = "plausible" }, ] @@ -301,6 +440,7 @@ verifies = [ id = "86box-os2" tool = "86Box + OS/2" kind = "emulator" +program = "86Box" notes = "OS/2's own CHKDSK is the only real HPFS oracle; the Linux hpfs module is read-only and absent from WSL." availability = [ { platform = "windows", status = "install" }, @@ -324,6 +464,7 @@ verifies = [{ format = "fs.hpfs", direction = "write", strength = "authoritative id = "mister-core-amiga" tool = "MiSTer Minimig core" kind = "hardware" +core = "Minimig" notes = """Real AmigaOS reading the volume with its own filesystem handlers. Unlike the HPS affs mount, this is the Amiga's opinion. Bootable Workbench 1.3 / 2.1 RDB+PFS disks are already on the board.""" @@ -339,6 +480,7 @@ verifies = [ id = "mister-core-x68000" tool = "MiSTer X68000 core" kind = "hardware" +core = "X68000" availability = [{ platform = "mister-core", status = "expected" }] verifies = [ { format = "fs.human68k", direction = "write", strength = "authoritative", status = "plausible" }, @@ -363,6 +505,7 @@ verifies = [ id = "mister-core-appleii" tool = "MiSTer Apple-II core" kind = "hardware" +core = "Apple-II" notes = """Real Apple II reading the disk with ProDOS / DOS 3.3 themselves. /media/fat/games/Apple-II is already populated on the board, so this needs no new material — it closes two write gaps that had no oracle at all.""" @@ -379,6 +522,7 @@ verifies = [ id = "snow" tool = "Snow" kind = "emulator" +program = "snowemu" notes = """Modern Macintosh emulator written in Rust. Emulating a Mac Plus gives a real MFS implementation — the only realistic judge of MFS, which otherwise has no oracle anywhere. @@ -535,6 +679,7 @@ verifies = [ id = "mame-pc98" tool = "MAME pc9801" kind = "emulator" +program = "mame" notes = "No PC-98 core exists on the MiSTer, so MAME's pc9801 drivers are the route for HDM and DIM." availability = [ { platform = "windows", status = "install" }, @@ -554,6 +699,7 @@ verifies = [ id = "mister-core-atari800" tool = "MiSTer ATARI800 core" kind = "hardware" +core = "Atari800" availability = [{ platform = "mister-core", status = "expected" }] verifies = [ { format = "fs.atari-dos2", direction = "write", strength = "authoritative", status = "plausible" }, @@ -564,6 +710,7 @@ verifies = [ id = "mister-core-ti99" tool = "MiSTer TI-99_4A core" kind = "hardware" +core = "Ti994a" availability = [{ platform = "mister-core", status = "expected" }] verifies = [ { format = "fs.ti99", direction = "write", strength = "authoritative", status = "plausible" }, @@ -574,6 +721,7 @@ verifies = [ id = "mister-core-oric" tool = "MiSTer Oric core" kind = "hardware" +core = "Oric" availability = [{ platform = "mister-core", status = "expected" }] verifies = [{ format = "fs.oric", direction = "write", strength = "authoritative", status = "plausible" }] @@ -581,6 +729,7 @@ verifies = [{ format = "fs.oric", direction = "write", strength = "authoritative id = "mister-core-spectrum" tool = "MiSTer Spectrum core" kind = "hardware" +core = "ZX-Spectrum" availability = [{ platform = "mister-core", status = "expected" }] verifies = [ { format = "fs.trdos", direction = "write", strength = "authoritative", status = "plausible" }, @@ -591,6 +740,7 @@ verifies = [ id = "mister-core-coco" tool = "MiSTer CoCo2 / COCO3 core" kind = "hardware" +core = "CoCo3" availability = [{ platform = "mister-core", status = "expected" }] verifies = [ { format = "fs.os9", direction = "write", strength = "authoritative", status = "plausible" }, @@ -602,6 +752,7 @@ verifies = [ id = "mister-core-archie" tool = "MiSTer ARCHIE core" kind = "hardware" +core = "Archie" notes = "Acorn Archimedes; RISC OS reads ADFS with its own FileCore." availability = [{ platform = "mister-core", status = "expected" }] verifies = [{ format = "fs.adfs", direction = "write", strength = "authoritative", status = "plausible" }] @@ -645,7 +796,7 @@ tool = "mount -t ntfs3" kind = "mount" notes = "In-kernel NTFS driver; absent from WSL, needs a full Linux host." availability = [ - { platform = "linux", status = "install", path_hint = "linux-modules-extra" }, + { platform = "linux", status = "install" }, { platform = "wsl", status = "absent" }, ] verifies = [{ format = "fs.ntfs", direction = "write", strength = "structural", status = "untested" }] diff --git a/regression-tests/oracles/amitools_affs.py b/regression-tests/oracles/amitools_affs.py new file mode 100644 index 00000000..1ca1d5bf --- /dev/null +++ b/regression-tests/oracles/amitools_affs.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Read an AFFS volume with amitools' xdftool, as a second implementation. + +Two things make xdftool awkward to call directly from a `check` line, and both +cost an afternoon to discover: + +1. **It picks geometry from the file extension.** A 2 MB volume named `.adf` is + rejected outright — "invalid ADF images size: 2097152" — because `.adf` means + floppy. The same bytes named `.hdf` are read as a hard-disk volume. +2. **It opens read-write before it parses.** Produced artifacts are read-only, + so it fails with EACCES having never looked at the image. `-r` does not help: + the open happens first. + +So this copies the artifact to a writable `.hdf` in a temp directory, runs +`xdftool list`, and cleans up. Exit status is xdftool's. + +Not authoritative. amitools is a reimplementation, so agreement does not prove a +real Amiga will mount the volume and disagreement does not prove it will not. +What it gives is a second opinion on the same bytes, which every AFFS check +before this one lacked — they were our code agreeing with itself. +""" + +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +def main() -> int: + if len(sys.argv) < 2: + print("usage: amitools_affs.py ", file=sys.stderr) + return 2 + src = Path(sys.argv[1]) + if not src.is_file(): + print(f"{src}: no such file", file=sys.stderr) + return 2 + + with tempfile.TemporaryDirectory(prefix="rb-amitools-") as tmp: + # .hdf, because the extension is the geometry. + work = Path(tmp) / "volume.hdf" + shutil.copyfile(src, work) + work.chmod(0o644) + proc = subprocess.run( + ["xdftool", str(work), "list"], + capture_output=True, + text=True, + ) + sys.stdout.write(proc.stdout) + sys.stderr.write(proc.stderr) + if proc.returncode != 0: + # xdftool reports structural problems on stdout with a zero-ish + # look; surface the reason on stderr so the verdict carries it. + print( + f"xdftool rejected the volume (exit {proc.returncode})", + file=sys.stderr, + ) + return proc.returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/regression-tests/oracles/fsuae/README.md b/regression-tests/oracles/fsuae/README.md index 7d203978..326231c9 100644 --- a/regression-tests/oracles/fsuae/README.md +++ b/regression-tests/oracles/fsuae/README.md @@ -41,3 +41,37 @@ A bad mount and a bad image look identical from the host. Every conclusion here must be paired with a known-good volume through the *same* config: R-020 was only credible because `Mister-3-2.hdf` mounted as `Read/Write Amiga32` under byte-identical settings while our own AFFS output did not. + +## Status, 2026-08-14 — working; it answered R-020 + +`affs_mount.py` runs end to end. Kickstart 3.1 on an A1200 mounts our AFFS +volume `Read/Write` under its own name and `List DH1:` reads the contents +back, which closes R-020. Run it as: + + python regression-tests/oracles/fsuae/affs_mount.py + + 0 mounted 1 not mounted (a verdict) + 3 no sentinel — harness failure, NOT a verdict 2 setup error + +### Always pair a pass with a negative control + +A harness that reports success for anything is indistinguishable from a good +volume. 2 MB of `os.urandom` through the identical config yields no DH1: unit +and `Can't examine "DH1:"`, exit 1 — so the oracle discriminates. + +### The trap: LF endings, written as bytes + +The guest script must be LF-terminated and written with `write_bytes`. Python's +`write_text` applies the platform newline on Windows; AmigaDOS scripts are +LF-only, so the guest read `Info >RESULTS:info.txt ` and tried to create a +file whose name ended in a carriage return — illegal on the host directory that +DH2 maps to. Every redirect failed, `FAILAT 21` correctly stopped the script +aborting, and the run booted and silently produced nothing. It looks exactly +like an emulator that will not boot. It is not. + +### The other thing that cost time + +The probe *replaces* the fixture's `S/Startup-Sequence` rather than appending to +it. Workbench 1.3's own startup — `Mount NEWCON:`, `Resident`, then Amiga +Forever's `Execute S:AFShared-Startup` — never reaches an appended probe under +Kickstart 3.1. All this test needs is a shell, `Info`, `List` and `Echo`. diff --git a/regression-tests/oracles/fsuae/affs_mount.py b/regression-tests/oracles/fsuae/affs_mount.py new file mode 100644 index 00000000..6687b3eb --- /dev/null +++ b/regression-tests/oracles/fsuae/affs_mount.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Ask a real AmigaOS whether it will mount an AFFS volume we wrote. + +This is R-020's oracle. Every AFFS check before it was our formatter agreeing +with our fsck, or — once amitools arrived — a reimplementation agreeing with +our bytes. Neither answers the question the finding actually asks, which is +whether Kickstart's own filesystem mounts the volume or reports +"Not a DOS disk". + +## How the verdict gets out + +There is no screen scraping. FS-UAE mounts a **host directory** as an Amiga +volume, so the guest writes a file and the host reads it: + + DH0: bootable Workbench (host dir, unpacked from the WB1.3 fixture) + DH1: the volume under test (hardfile) + DH2: RESULTS (host dir, the verdict channel) + +A probe appended to `S/Startup-Sequence` runs `Info` and `List DH1:`, redirects +both into `RESULTS:`, then writes `done.txt` as a sentinel. The host polls for +the sentinel and kills the emulator. No sentinel within the timeout is a +*third* outcome, distinct from a bad volume: the guest never got there. + +## Why the boot volume is unpacked rather than mounted + +DH0 must be writable to take the probe, and the WB1.3 fixture is sha256-pinned +corpus. `xdftool unpack` turns it into a host directory, which is both +modifiable and mountable by FS-UAE directly, so the fixture is only ever read. + +## Always run a control + +A volume that fails to mount and a harness that never booted look identical +from the host, which is why `--control` exists: it points DH1 at the WB1.3 +fixture, a volume a real Amiga certainly mounts. A run whose control fails +proves nothing about the artifact, and this script says so rather than +reporting a defect. + +Exit status: + 0 the guest mounted DH1: and reported a volume name + 1 the guest ran and did not mount it — either `Not a DOS disk` + (R-020's symptom) or no DH1: unit at all. A verdict either way. + 3 the guest never reached the sentinel — harness failure, NOT a verdict + 2 usage / setup error +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Optional, Tuple + +# The probe *replaces* the fixture's Startup-Sequence rather than being +# appended to it. +# +# Appending was tried first and does not boot. The Workbench 1.3 fixture's +# startup is a real one — `Mount NEWCON:`, `Resident`, RAM: assigns, then +# `Execute S:AFShared-Startup`, an Amiga Forever addition — and under +# Kickstart 3.1 something in that chain never reaches the probe, so no +# sentinel ever appears. Every one of those steps is scenery for this test: +# all it needs is a shell, `Info`, `List` and `Echo`, which are in `c/`. +# +# FAILAT 21 stops AmigaDOS aborting the script when a command against an +# unrecognised volume returns ERROR(20). Without it the run dies before +# writing done.txt, and the host cannot tell "the volume is bad" from "never +# booted" — the two outcomes that most need distinguishing. +PROBE = """FAILAT 21 +Echo "rb-regress probe" +Info >RESULTS:info.txt +List DH1: >RESULTS:listing.txt +Echo "DONE" >RESULTS:done.txt +""" + +DEFAULT_TIMEOUT = 180 + + +def find_fs_uae() -> Optional[str]: + """FS-UAE is a per-user install on Windows and not on PATH.""" + on_path = shutil.which("fs-uae") + if on_path: + return on_path + local = os.environ.get("LOCALAPPDATA") + if local: + cand = ( + Path(local) + / "Programs" + / "FS-UAE" + / "FS-UAE" + / "Windows" + / "x86-64" + / "fs-uae.exe" + ) + if cand.is_file(): + return str(cand) + return None + + +def find_kickstart() -> Optional[Path]: + """A 3.1 A1200 ROM out of the licensed Amiga Forever set. + + Kickstart 1.3 has no FFS in ROM — it loaded the handler off the RDB — and + our volumes are bare DOS\\1 with no RDB, so 1.3 would refuse them for a + reason that has nothing to do with the bytes under test. 3.1 has FFS in + ROM and is what R-020 was originally observed on. + """ + docs = Path.home() / "Documents" / "FS-UAE" / "Kickstarts" + for name in ("amiga-os-310-a1200.rom", "amiga-os-310-a4000.rom", "amiga-os-310.rom"): + p = docs / name + if p.is_file(): + return p + return None + + +def write_probe(system: Path) -> None: + """Write the probe into S/Startup-Sequence with LF endings, in bytes. + + `write_text` is wrong here and cost an afternoon. On Windows it applies the + platform newline translation, so every `\\n` became `\\r\\n` — and AmigaDOS + scripts are LF-terminated. The guest read `Info >RESULTS:info.txt\\r`, + tried to create a file whose name ended in a carriage return, and since + DH2 is a *host directory* that is an illegal Windows filename. Every + redirect failed, `FAILAT 21` correctly stopped the script aborting, and the + result was a run that booted and silently produced nothing. + + Bytes, explicitly, so the host platform cannot express an opinion. + """ + seq = system / "s" / "Startup-Sequence" + if not seq.parent.is_dir(): + raise RuntimeError(f"no S: directory in the unpacked tree at {seq.parent}") + seq.write_bytes(PROBE.replace("\r\n", "\n").encode("ascii")) + + +def build_boot_dir(workdir: Path, wb_fixture: Path) -> Path: + """Unpack the Workbench fixture to a host directory and add the probe. + + Cached: unpacking is slow and the tree never changes between runs. + """ + system = workdir / "System" + stamp = workdir / ".boot-ready" + if stamp.is_file(): + # The tree is cached, but the probe is rewritten every run — it is one + # small file and having it silently diverge from the source would be + # far more expensive than writing it again. + write_probe(system) + return system + + if system.exists(): + shutil.rmtree(system) + # xdftool opens read-write and picks geometry from the extension, so work + # from a writable .hdf copy rather than the pinned fixture. + tmp_hdf = workdir / "wb-boot.hdf" + shutil.copyfile(wb_fixture, tmp_hdf) + tmp_hdf.chmod(0o644) + proc = subprocess.run( + ["xdftool", str(tmp_hdf), "unpack", str(system)], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + raise RuntimeError(f"xdftool unpack failed: {proc.stderr.strip()}") + + write_probe(system) + tmp_hdf.unlink(missing_ok=True) + stamp.write_text("ok\n") + return system + + +def write_config( + workdir: Path, boot: Path, artifact: Path, results: Path, kickstart: Path +) -> Path: + cfg = workdir / "probe.fs-uae" + # Forward slashes throughout: FS-UAE's config parser treats a backslash as + # an escape, so a Windows path silently mangles. + cfg.write_text( + "# rb-regress AFFS mount oracle - generated, do not edit\n" + "amiga_model = A1200\n" + f"kickstart_file = {kickstart.as_posix()}\n" + "fast_memory = 8192\n" + "\n" + f"hard_drive_0 = {boot.as_posix()}\n" + "hard_drive_0_label = Workbench\n" + "\n" + f"hard_drive_1 = {artifact.as_posix()}\n" + "\n" + f"hard_drive_2 = {results.as_posix()}\n" + "hard_drive_2_label = RESULTS\n" + "\n" + "fullscreen = 0\n" + "window_width = 640\n" + "window_height = 480\n" + "floppy_drive_volume = 0\n" + "automatic_input_grab = 0\n" + ) + return cfg + + +def parse_verdict(info: str) -> Tuple[str, str]: + """Read the guest's `Info` output for what became of DH1:. + + Three guest-side states, and they are not the same thing: + + * **mounted** — a line with size/used/free and a volume name. The volume + is good. + * **refused** — `DH1: Not a DOS disk`. UAE attached the unit and + Kickstart declined to mount it. This is R-020's original symptom. + * **absent** — no DH1: line at all. UAE could not make a unit of it + (2 MB of noise does this), so it never reached Kickstart. Still a + refusal of the bytes, just an earlier one — *not* a harness failure, + because the guest plainly ran and reported the other two drives. + + Only "the sentinel never appeared" is a harness failure, and that is + decided by the caller, not here. + """ + for line in info.splitlines(): + if not line.strip().startswith("DH1"): + continue + if re.search(r"not a dos disk", line, re.I): + return "refused", line.strip() + return "mounted", line.strip() + return "absent", "UAE presented no DH1: unit at all" + + +def run(args) -> int: + fs_uae = find_fs_uae() + if not fs_uae: + print("fs-uae not found (PATH or %LOCALAPPDATA%/Programs/FS-UAE)", file=sys.stderr) + return 2 + kickstart = find_kickstart() + if not kickstart: + print("no Kickstart 3.1 ROM under ~/Documents/FS-UAE/Kickstarts", file=sys.stderr) + return 2 + + workdir = Path(args.workdir).resolve() + workdir.mkdir(parents=True, exist_ok=True) + results = workdir / "results" + if results.exists(): + shutil.rmtree(results) + results.mkdir() + + try: + boot = build_boot_dir(workdir, Path(args.workbench).resolve()) + except RuntimeError as e: + print(f"boot volume: {e}", file=sys.stderr) + return 2 + + # FS-UAE wants a hardfile extension it recognises, and the artifact is + # read-only in the tree; copy it in either way. + under_test = workdir / "under-test.hdf" + shutil.copyfile(Path(args.image).resolve(), under_test) + under_test.chmod(0o644) + + cfg = write_config(workdir, boot, under_test, results, kickstart) + + print(f"fs-uae : {fs_uae}") + print(f"kickstart : {kickstart.name}") + print(f"under test : {args.image}") + print(f"config : {cfg}") + + proc = subprocess.Popen( + [fs_uae, str(cfg)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + sentinel = results / "done.txt" + deadline = time.time() + args.timeout + try: + while time.time() < deadline: + if sentinel.is_file(): + break + if proc.poll() is not None: + break + time.sleep(1.0) + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=15) + except subprocess.TimeoutExpired: + proc.kill() + + if not sentinel.is_file(): + print( + "\nthe guest never reached the sentinel — this is a harness result, " + "not a verdict on the volume", + file=sys.stderr, + ) + return 3 + + info = (results / "info.txt").read_text(errors="replace") if (results / "info.txt").is_file() else "" + listing = ( + (results / "listing.txt").read_text(errors="replace") + if (results / "listing.txt").is_file() + else "" + ) + verdict, line = parse_verdict(info) + + print("\n--- guest Info ---") + print(info.strip() or "(empty)") + if listing.strip(): + print("\n--- guest List DH1: ---") + print(listing.strip()[:1000]) + + print(f"\nverdict: {verdict} ({line})") + if verdict == "mounted": + return 0 + # `refused` and `absent` are both the guest declining the bytes; the + # sentinel is present, so the harness worked and this is a real verdict. + return 1 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("image", help="the AFFS volume to put in front of AmigaOS") + ap.add_argument( + "--workbench", + default="regression-tests/fixtures/fs.affs.workbench13.hd.hdf", + help="bootable Workbench fixture, unpacked to a host dir for DH0", + ) + ap.add_argument("--workdir", default="regression-tests/scratch/fsuae") + ap.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT) + return run(ap.parse_args()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/regression-tests/oracles/iris/README.md b/regression-tests/oracles/iris/README.md new file mode 100644 index 00000000..932488ec --- /dev/null +++ b/regression-tests/oracles/iris/README.md @@ -0,0 +1,105 @@ +# Iris / IRIX oracle + +IRIX 6.5.22 on an emulated SGI Indy, checking our EFS with **its own** +`/sbin/fsck` and mounting it with its own kernel. `strength = authoritative`: +this is the implementation, not a reimplementation. It produced R-039 and then +confirmed the fix. + +## Why this one is different + +Every EFS check before it was our code checked by our code. R-039 is the case +for why that is not enough: the free-space bitmap was MSB-first in our reader +*and* our writer, so our formatter and our fsck agreed with each other +perfectly and were both wrong. A self-consistent convention cannot be caught by +self-consistent fixtures. IRIX caught it in one run. + +## Setup + +Machine-specific paths live in the gitignored `data/oracles.local.toml`, never +here. What the oracle expects: + +* `iris.exe` and `iris-ci.exe` +* an IRIX **6.5** disk image (5.3 hangs at "The system is coming up" — + `Find Error: 10`, never reaches a login prompt) +* a scratch SCSI device declared **in `iris.toml`**, not on the command line: + +```toml +[scsi.1] +path = "disks/Indy-IRIX65_dev.chd" +cdrom = false + +[scsi.2] +path = "disks/scratch.img" +cdrom = false +scratch = true +size_mb = 64 +``` + +`scsi` is a map keyed by id, not an array of tables, and every device needs +`cdrom`. iris creates the scratch volume itself, SGI volume header and all. + +## Running it + +``` +IRIS_JIT=1 iris --headless --noaudio --ci \ + --scsi1 disks/Indy-IRIX65_dev.chd --cdrom4 + +iris-ci boot && iris-ci login +iris-ci save booted-shell # then every later run is a rollback +iris-ci scratch write ./volume.img +iris-ci run "/sbin/fsck -t efs -n /dev/rdsk/dks0d2s0 < /dev/null" +``` + +SCSI id 2 is `dks0d2`; `s0` is the payload partition. + +## Five things that cost time + +1. **No PROM needed.** It warns about a missing `prom.bin` and uses an + embedded one. Do not go hunting for Indy firmware. +2. **`--ci` is required.** Without it iris listens only on the monitor port + 8888 and `iris-ci` cannot reach 19851. +3. **A CD-ROM must be attached even when booting from disk**, or startup dies + with "could not attach cdrom4.iso". Any ISO will do. +4. **`< /dev/null` is load-bearing.** Without it fsck blocks forever on the + SALVAGE prompt and the run just hangs. +5. **Size the volume to `s0`, not to the device.** `prtvtoc /dev/rdsk/dks0d2vh` + reports `s0` starting at sector 8 with 131064 sectors on a 64 MB scratch — + 8 sectors short of the whole device, because the SGI volume header occupies + the front. `scratch write` lands at exactly `s0`'s start, so a filesystem + built to the full 64 MB overhangs the partition and fsck reports + "Primary superblock size check: filesystem larger than device" — which + looks like a filesystem defect and is not one. Build it at + `65532K` (= 131064 sectors). + +Also note `iris-ci run` reports `guest exit -1` on every command, including +ones that plainly worked. Judge results by stdout, not exit status. + +## Beyond fsck: mount it + +fsck passing is necessary, not sufficient. The stronger check is to let IRIX +use the filesystem: + +``` +iris-ci run "mkdir -p /mnt2 && mount -t efs /dev/dsk/dks0d2s0 /mnt2" +iris-ci run "echo hello > /mnt2/f.txt && mkdir /mnt2/d && umount /mnt2" +iris-ci run "/sbin/fsck -t efs -n /dev/rdsk/dks0d2s0 < /dev/null" +``` + +then pull it back with `iris-ci scratch read` and confirm rb-cli reads what +IRIX wrote. That full loop is what closed R-039. + +## Always run a control + +`mkfs_efs` on the same device, in the same session, through the same path is +the control — and it is also the best diagnostic tool here. When R-039 was +open, the difference between our volume and the one `mkfs_efs` produced *on +the same device* is what identified the bit order: + +``` +iris-ci run "/sbin/mkfs_efs /dev/rdsk/dks0d2s0 < /dev/null" +iris-ci scratch read ./mkfs-reference.img +``` + +Reference geometry on the 64 MB scratch: `blocks=131064 inodes=18696 +sectors=128 cgfsize=21838 cgalign=1 ialign=1 ncg=6 firstcg=34 cgisize=779 +bitmap blocks=32`. diff --git a/regression-tests/runner/src/consolidate.rs b/regression-tests/runner/src/consolidate.rs index e98915ae..eaeb2a44 100644 --- a/regression-tests/runner/src/consolidate.rs +++ b/regression-tests/runner/src/consolidate.rs @@ -50,11 +50,117 @@ pub struct Consolidated { pub builds: BTreeSet, pub runs: BTreeSet, pub cases: BTreeSet, - /// Cases that failed, with the platforms they failed on — the triage list. - pub failures: BTreeMap>, + /// (case, platform, verdict, run_id) for every line, so coverage skew can + /// be computed from each platform's *latest* run rather than from all of + /// history at once. + seen: Vec<(String, String, String, String)>, pub unstamped: usize, } +/// A case that ran somewhere and was skipped for a missing fixture elsewhere. +#[derive(Debug)] +pub struct CoverageSkew { + pub case_id: String, + pub skipped_on: BTreeSet, + pub ran_on: BTreeSet, +} + +impl Consolidated { + /// The most recent run id per platform. Run ids are timestamp-prefixed, so + /// the lexical maximum is the newest. + /// + /// Everything reported as "current" filters through this. `consolidate` + /// reads every `results.jsonl` ever written — 131 of them here — so an + /// unfiltered view describes no run that ever happened. + fn latest_run_per_platform(&self) -> BTreeMap<&str, &str> { + let mut latest: BTreeMap<&str, &str> = BTreeMap::new(); + for (_, platform, _, run_id) in &self.seen { + if run_id.is_empty() { + continue; + } + let e = latest.entry(platform.as_str()).or_insert(run_id.as_str()); + if run_id.as_str() > *e { + *e = run_id.as_str(); + } + } + latest + } + + /// Cases failing in each platform's latest run — the triage list. + /// + /// Accumulated across all of history until 2026-08-10, which listed 69 + /// Windows failures that had been green for weeks. A triage list nobody + /// can trust is worse than none. + pub fn current_failures(&self) -> BTreeMap> { + let latest = self.latest_run_per_platform(); + let mut out: BTreeMap> = BTreeMap::new(); + for (case, platform, verdict, run_id) in &self.seen { + if latest.get(platform.as_str()) != Some(&run_id.as_str()) { + continue; + } + if verdict == "fail" || verdict == "error" { + out.entry(case.clone()).or_default().insert(platform.clone()); + } + } + out + } + + /// Cases covered on some hosts and skipped on others for want of a fixture. + /// + /// Invisible in the pass/fail columns: an unresolved fixture is + /// `skip-fixture`, not a failure, so a host missing part of the corpus + /// still reports zero failures — just a quietly smaller pass count. Three + /// hosts sat at 257/257/259 that way, all reporting 0 fail. + /// + /// Computed from each platform's **latest run only**. `consolidate` reads + /// every `results.jsonl` ever written, so comparing across all of history + /// reports skew that was fixed weeks ago — the first version of this did + /// exactly that, naming eleven optical cases as missing on Windows when + /// Windows had had them for weeks. Run ids are timestamp-prefixed, so the + /// lexical maximum per platform is that platform's most recent run. + pub fn coverage_skew(&self) -> Vec { + let latest = self.latest_run_per_platform(); + + let mut ran_on: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + let mut skipped_on: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for (case, platform, verdict, run_id) in &self.seen { + if latest.get(platform.as_str()) != Some(&run_id.as_str()) { + continue; + } + ran_on + .entry(case.as_str()) + .or_default() + .insert(platform.as_str()); + if verdict == "skip-fixture" { + skipped_on + .entry(case.as_str()) + .or_default() + .insert(platform.as_str()); + } + } + + let mut out = Vec::new(); + for (case_id, skipped) in &skipped_on { + let ran: BTreeSet = ran_on + .get(case_id) + .map(|all| { + all.difference(skipped) + .map(|s| (*s).to_string()) + .collect() + }) + .unwrap_or_default(); + if !ran.is_empty() { + out.push(CoverageSkew { + case_id: (*case_id).to_string(), + skipped_on: skipped.iter().map(|s| (*s).to_string()).collect(), + ran_on: ran, + }); + } + } + out + } +} + /// Find every `results.jsonl` under `root`, at any depth. fn find_results(root: &Path, out: &mut Vec) { let entries = match fs::read_dir(root) { @@ -124,12 +230,20 @@ pub fn consolidate(root: &Path) -> Result { c.runs.insert(l.run_id.clone()); } - if l.verdict == "fail" || l.verdict == "error" { - c.failures - .entry(l.case_id) - .or_default() - .insert(if l.platform.is_empty() { "?".into() } else { l.platform }); - } + let platform = if l.platform.is_empty() { + "?".to_string() + } else { + l.platform.clone() + }; + c.seen.push(( + l.case_id.clone(), + platform.clone(), + l.verdict.clone(), + l.run_id.clone(), + )); + + // Failures are derived at the end from `seen`, scoped to each + // platform's latest run — see `current_failures`. } } @@ -218,17 +332,141 @@ pub fn render(c: &Consolidated, root: &Path) -> String { )); } - if !c.failures.is_empty() { - s.push_str(&format!("\nfailing cases ({})\n", c.failures.len())); - for (case, plats) in c.failures.iter().take(30) { + let failures = c.current_failures(); + if !failures.is_empty() { + s.push_str(&format!( + "\nfailing cases ({}) - in each host's LATEST run\n", + failures.len() + )); + for (case, plats) in failures.iter().take(30) { let mut ps: Vec<&str> = plats.iter().map(|s| s.as_str()).collect(); ps.sort_unstable(); s.push_str(&format!(" {:<44} {}\n", case, ps.join(", "))); } - if c.failures.len() > 30 { - s.push_str(&format!(" ... and {} more\n", c.failures.len() - 30)); + if failures.len() > 30 { + s.push_str(&format!(" ... and {} more\n", failures.len() - 30)); + } + } + + // Coverage skew is the failure mode the pass/fail columns cannot show: a + // host missing a fixture reports skip-fixture, not a failure, so it stays + // green while covering less than its peers. + let skew = c.coverage_skew(); + if !skew.is_empty() { + s.push_str(&format!( + "\nCOVERAGE SKEW ({}) - ran on some hosts, skipped for a missing fixture on others\n\ + Comparing each host's LATEST run. Those hosts are green on less than their\n\ + peers. Sync the corpus AND the catalogue row (gitignored, so a pull does not\n\ + bring it) before comparing pass counts.\n", + skew.len() + )); + for k in skew.iter().take(30) { + let miss: Vec<&str> = k.skipped_on.iter().map(|s| s.as_str()).collect(); + let has: Vec<&str> = k.ran_on.iter().map(|s| s.as_str()).collect(); + s.push_str(&format!( + " {:<44} missing on {} (ran on {})\n", + k.case_id, + miss.join(", "), + has.join(", ") + )); + } + if skew.len() > 30 { + s.push_str(&format!(" ... and {} more\n", skew.len() - 30)); } } s } + +#[cfg(test)] +mod tests { + use super::*; + + fn seen(c: &mut Consolidated, case: &str, platform: &str, verdict: &str, run: &str) { + c.seen.push(( + case.into(), + platform.into(), + verdict.into(), + run.into(), + )); + } + + #[test] + fn a_case_skipped_on_one_host_and_run_on_another_is_skew() { + let mut c = Consolidated::default(); + seen(&mut c, "read.hpfs.os2-warp45", "windows", "pass", "200-win"); + seen(&mut c, "read.hpfs.os2-warp45", "linux", "skip-fixture", "200-lin"); + seen(&mut c, "read.hpfs.os2-warp45", "macos", "skip-fixture", "200-mac"); + + let skew = c.coverage_skew(); + assert_eq!(skew.len(), 1); + assert_eq!(skew[0].case_id, "read.hpfs.os2-warp45"); + assert!(skew[0].skipped_on.contains("linux")); + assert!(skew[0].ran_on.contains("windows")); + } + + #[test] + fn a_case_skipped_everywhere_is_not_skew() { + // Nobody holds the fixture. That is a corpus gap the inventory already + // reports; it is not one host being quietly less covered than another. + let mut c = Consolidated::default(); + seen(&mut c, "read.something", "windows", "skip-fixture", "200-win"); + seen(&mut c, "read.something", "linux", "skip-fixture", "200-lin"); + assert!(c.coverage_skew().is_empty()); + } + + #[test] + fn a_case_that_ran_everywhere_is_not_skew() { + let mut c = Consolidated::default(); + seen(&mut c, "read.something", "windows", "pass", "200-win"); + seen(&mut c, "read.something", "linux", "xfail", "200-lin"); + assert!(c.coverage_skew().is_empty()); + } + + #[test] + fn failures_are_scoped_to_the_latest_run_too() { + // The failing-cases list had the same all-history flaw as the skew + // check: it reported 69 Windows failures that had been green for weeks. + let mut c = Consolidated::default(); + seen(&mut c, "some.case", "windows", "fail", "100-win"); + seen(&mut c, "some.case", "windows", "pass", "300-win"); + assert!( + c.current_failures().is_empty(), + "a failure fixed in a later run must not stay on the triage list" + ); + + seen(&mut c, "other.case", "linux", "fail", "300-lin"); + let f = c.current_failures(); + assert_eq!(f.len(), 1); + assert!(f.contains_key("other.case")); + } + + #[test] + fn skew_fixed_in_a_later_run_is_not_reported() { + // The whole point of scoping to the latest run per platform. The first + // version compared all of history and named eleven optical cases as + // missing on Windows that Windows had held for weeks. + let mut c = Consolidated::default(); + seen(&mut c, "read.optical.udf", "windows", "skip-fixture", "100-win"); + seen(&mut c, "read.optical.udf", "linux", "pass", "100-lin"); + // Later runs: Windows got the fixture. + seen(&mut c, "read.optical.udf", "windows", "pass", "300-win"); + seen(&mut c, "read.optical.udf", "linux", "pass", "300-lin"); + assert!( + c.coverage_skew().is_empty(), + "stale skew from an older run must not be reported" + ); + } + + #[test] + fn skew_still_present_in_the_latest_run_is_reported() { + let mut c = Consolidated::default(); + seen(&mut c, "read.optical.udf", "windows", "pass", "100-win"); + seen(&mut c, "read.optical.udf", "linux", "pass", "100-lin"); + seen(&mut c, "read.optical.udf", "windows", "pass", "300-win"); + seen(&mut c, "read.optical.udf", "linux", "skip-fixture", "300-lin"); + let skew = c.coverage_skew(); + assert_eq!(skew.len(), 1); + assert!(skew[0].skipped_on.contains("linux")); + } +} diff --git a/regression-tests/runner/src/exec.rs b/regression-tests/runner/src/exec.rs index d5b041ec..85dc388c 100644 --- a/regression-tests/runner/src/exec.rs +++ b/regression-tests/runner/src/exec.rs @@ -204,3 +204,39 @@ pub fn platform_token() -> &'static str { "other" } } + +/// Run one command on a remote host over ssh and return its stdout. +/// +/// Deliberately spartan: `BatchMode=yes` so a host missing its key fails fast +/// instead of blocking on a passphrase prompt nobody is watching, and a short +/// connect timeout so an unplugged MiSTer is an error rather than a stall. On +/// Windows the system ssh is named explicitly — Git Bash ships its own, which +/// does not see the user's keys. +pub fn ssh_capture(target: &str, command: &str) -> Result { + let program = if cfg!(windows) { + r"C:\Windows\System32\OpenSSH\ssh.exe" + } else { + "ssh" + }; + let out = std::process::Command::new(program) + .args([ + "-o", + "BatchMode=yes", + "-o", + "IdentitiesOnly=no", + "-o", + "ConnectTimeout=8", + target, + command, + ]) + .output() + .map_err(|e| format!("cannot run ssh: {e}"))?; + if !out.status.success() { + let err = String::from_utf8_lossy(&out.stderr); + return Err(format!( + "ssh {target} failed: {}", + err.lines().next().unwrap_or("(no message)") + )); + } + Ok(String::from_utf8_lossy(&out.stdout).into_owned()) +} diff --git a/regression-tests/runner/src/fixtures.rs b/regression-tests/runner/src/fixtures.rs index 12eeaf0b..11077bba 100644 --- a/regression-tests/runner/src/fixtures.rs +++ b/regression-tests/runner/src/fixtures.rs @@ -24,6 +24,16 @@ pub struct FixtureRow { pub origin: String, pub redistributable: String, pub notes: String, + /// Optional: the filesystem this fixture must identify as, checked by + /// `rb-regress fixtures --identify` via `rb-cli inspect --expect-fs`. + /// Empty means the row makes no claim. + pub expect_fs: String, + /// Optional: the disk shape this fixture must have — `superfloppy`, + /// `partitioned`, or a scheme name. Checked via `--expect-layout`. + pub expect_layout: String, + /// Optional: a `--fs-type` preset the fixture needs to be identifiable at + /// all (CP/M disks carry no signature). + pub fs_type: String, } #[derive(Debug, Default)] @@ -142,6 +152,9 @@ impl Catalog { origin: get(5), redistributable: get(6), notes: get(8), + expect_fs: get(9), + expect_layout: get(10), + fs_type: get(11), }; if row.id.is_empty() { continue; @@ -302,3 +315,88 @@ fn sanitise(id: &str) -> String { .collect() } + +/// One fixture's identity check. +pub struct IdentityCheck { + pub id: String, + pub ok: bool, + /// Empty when the row declared nothing to check. + pub detail: String, +} + +/// Ask `rb-cli inspect` to confirm each fixture is what its catalogue row says. +/// +/// This exists because `harvested-verified` used to mean "`inspect` opened it", +/// which a universal reader says about anything — an Apple DOS fixture that was +/// really a bare bootloader passed that bar and sat in the corpus for weeks +/// (R-031). A row that names `expect_fs` / `expect_layout` states its intent, +/// and the check enforces it. +/// +/// Rows that declare nothing are skipped, so populating the columns is +/// incremental rather than a flag day. +pub fn check_identities(cat: &Catalog, rb_cli: &Path, cache_dir: &Path) -> Vec { + let mut out = Vec::new(); + for row in cat.rows() { + if row.expect_fs.is_empty() && row.expect_layout.is_empty() { + continue; + } + // materialise, not resolve: a `.zst` row must be expanded first — + // rb-cli does not read compressed images, the harness decompresses them. + let path = match cat.materialise(&row.id, cache_dir) { + Ok(p) => p, + Err(e) => { + out.push(IdentityCheck { + id: row.id.clone(), + ok: false, + detail: format!("unresolved: {e}"), + }); + continue; + } + }; + let mut cmd = std::process::Command::new(rb_cli); + cmd.arg("inspect").arg(&path); + if !row.expect_fs.is_empty() { + cmd.arg("--expect-fs").arg(&row.expect_fs); + } + if !row.expect_layout.is_empty() { + cmd.arg("--expect-layout").arg(&row.expect_layout); + } + if !row.fs_type.is_empty() { + cmd.arg("--fs-type").arg(&row.fs_type); + } + let claim = [ + (!row.expect_fs.is_empty()).then(|| format!("fs={}", row.expect_fs)), + (!row.expect_layout.is_empty()).then(|| format!("layout={}", row.expect_layout)), + ] + .into_iter() + .flatten() + .collect::>() + .join(" "); + match cmd.output() { + Ok(o) if o.status.success() => out.push(IdentityCheck { + id: row.id.clone(), + ok: true, + detail: claim, + }), + Ok(o) => { + let err = String::from_utf8_lossy(&o.stderr); + let line = err + .lines() + .find(|l| l.starts_with("error:")) + .unwrap_or("(no error line)") + .to_string(); + out.push(IdentityCheck { + id: row.id.clone(), + ok: false, + detail: format!("{claim} -> {line}"), + }); + } + Err(e) => out.push(IdentityCheck { + id: row.id.clone(), + ok: false, + detail: format!("{claim} -> could not run rb-cli: {e}"), + }), + } + } + out +} diff --git a/regression-tests/runner/src/gitinfo.rs b/regression-tests/runner/src/gitinfo.rs index 6b7c6fe4..9655ec65 100644 --- a/regression-tests/runner/src/gitinfo.rs +++ b/regression-tests/runner/src/gitinfo.rs @@ -67,6 +67,44 @@ pub fn is_clean(repo: &Path) -> bool { dirty_files(repo).map(|v| v.is_empty()).unwrap_or(false) } +/// Whether engine sources changed between `sha` and HEAD. +/// +/// This is the staleness test an oracle verdict needs. An artifact records the +/// sha it was produced at; if `src/` has moved since, the bytes on disk are not +/// what HEAD would write, and any verdict about them describes code that is no +/// longer in the tree. +/// +/// That is not hypothetical — R-038 was filed as a High defect against the AFFS +/// formatter on the strength of an oracle reading artifacts built two days +/// before the fix that resolved it. Nothing in the run path noticed. +/// +/// `None` means the question could not be answered (sha absent from this +/// clone, git unavailable), which callers must report as unknown rather than +/// treating as fresh. +pub fn engine_changed_since(repo: &Path, sha: &str, engine_paths: &[&str]) -> Option { + // Reject a sha this clone does not have, so a shallow or unrelated + // checkout cannot silently answer "unchanged". + git(repo, &["cat-file", "-e", &format!("{sha}^{{commit}}")])?; + let mut args = vec!["diff", "--quiet", sha, "HEAD", "--"]; + args.extend_from_slice(engine_paths); + let out = Command::new("git") + .args(&args) + .current_dir(repo) + .output() + .ok()?; + // `--quiet` exits 0 when there is no diff, 1 when there is. + match out.status.code() { + Some(0) => Some(false), + Some(1) => Some(true), + _ => None, + } +} + +/// The paths whose contents decide whether a produced artifact is still +/// representative. `src/` is the engine; `Cargo.toml`/`Cargo.lock` pin the +/// dependencies that get compiled into it. +pub const ENGINE_PATHS: &[&str] = &["src", "Cargo.toml", "Cargo.lock"]; + /// Human-readable build identity, e.g. `0.1.0+g184c764` or /// `0.1.0+g184c764.dirty`. pub fn build_label(repo: &Path, version: &str) -> String { diff --git a/regression-tests/runner/src/main.rs b/regression-tests/runner/src/main.rs index be2ce9c6..847009cd 100644 --- a/regression-tests/runner/src/main.rs +++ b/regression-tests/runner/src/main.rs @@ -16,6 +16,7 @@ mod gitinfo; mod inventory; mod known; mod local; +mod oracles; mod manifest; mod parity; mod plan; @@ -36,6 +37,12 @@ struct Args { command: Command, cases_dir: PathBuf, rb_cli: PathBuf, + identify: bool, + detect: bool, + set_oracle: Option, + no_prompt: bool, + scan_host: Option, + export: bool, fixture_root: Option, sync_from: Option, sync: bool, @@ -65,6 +72,8 @@ enum Command { Plan, /// Take inventory of the fixture corpus and report which cases it enables. Fixtures, + /// Detect / export this host's oracle availability. + Oracles, /// Build every artifact rb-cli can write, on whatever OS is running. Produce, /// Compare artifacts produced on different OSes. Needs no oracle. @@ -122,6 +131,12 @@ fn parse_args() -> Result { command: Command::Help, cases_dir: base.join("cases"), rb_cli: default_rb_cli(&base), + identify: false, + detect: false, + set_oracle: None, + no_prompt: false, + scan_host: None, + export: false, fixture_root: None, sync_from: None, sync: false, @@ -148,6 +163,7 @@ fn parse_args() -> Result { "validate" => args.command = Command::Validate, "plan" => args.command = Command::Plan, "fixtures" => args.command = Command::Fixtures, + "oracles" => args.command = Command::Oracles, "produce" => args.command = Command::Produce, "verify" => args.command = Command::Verify, "parity" => { @@ -174,6 +190,10 @@ fn parse_args() -> Result { "--allow-hardware" => args.allow_hardware = true, "--keep-scratch" => args.keep_scratch = true, "--sync" => args.sync = true, + "--identify" => args.identify = true, + "--detect" => args.detect = true, + "--no-prompt" => args.no_prompt = true, + "--export" => args.export = true, "--require-clean" => args.require_clean = true, "--check" => args.check = true, "--verbose" | "-v" => args.verbose = true, @@ -216,6 +236,14 @@ fn parse_args() -> Result { args.verifications_root = PathBuf::from(value()?); i += 1; } + "--scan" => { + args.scan_host = Some(value()?); + i += 1; + } + "--set" => { + args.set_oracle = Some(value()?); + i += 1; + } "--filter" => { args.filter = Some(value()?); i += 1; @@ -288,6 +316,10 @@ COMMANDS: list List the cases that would run, without running them validate Parse every manifest and report problems; runs nothing fixtures Inventory the corpus: what is present, verified, and runnable + oracles Detect this host's third-party tools (--detect; asks about + emulators it cannot find, --no-prompt to stay quiet), scan a + MiSTer's cores over ssh (--scan ), record one by hand + (--set =), or print it (--export) plan Map requirements onto the machines that exist produce Build every artifact rb-cli can write, twice, into / parity Compare artifacts across producer OSes; needs no oracle @@ -308,6 +340,10 @@ OPTIONS: `corpus_source` into the fixture root first. Runs read local disk; the source is touched only here. --sync-from (fixtures) same, from an explicit directory + --identify (fixtures) confirm each fixture really holds what + its catalogue row claims, via rb-cli inspect + --expect-fs / --expect-layout. Rows that declare + nothing are skipped. Exits 1 on any mismatch. --verbose, -v (fixtures) list every blocked case and unused fixture --report-root Where bundles are written[default: regression-tests/runs] --scratch-root Working directory root [default: regression-tests/scratch] @@ -350,6 +386,7 @@ fn main() { Command::Query(ref q) => cmd_query(&args, q), Command::Plan => cmd_plan(&args), Command::Fixtures => cmd_fixtures(&args), + Command::Oracles => cmd_oracles(&args), Command::Produce => cmd_produce(&args), Command::Parity(ref root) => cmd_parity(&args, root), Command::Verify => cmd_verify(&args), @@ -435,6 +472,32 @@ inventory: {}", out.display()); Err(e) => eprintln!("warning: could not serialise inventory: {}", e), } + // Identity: does each fixture actually hold what its row claims? Opt-in, + // because it opens every declaring fixture and some are hundreds of MB. + let mut identity_failures = 0usize; + if args.identify { + let cache = args.scratch_root.join("_fixture-cache"); + let checks = fixtures::check_identities(&catalog, &args.rb_cli, &cache); + if checks.is_empty() { + println!( + " +identity: no fixture declares expect_fs / expect_layout yet - nothing to check" + ); + } else { + let bad: Vec<_> = checks.iter().filter(|c| !c.ok).collect(); + println!( + " +identity: {} of {} declaring fixture(s) confirmed", + checks.len() - bad.len(), + checks.len() + ); + for c in &bad { + println!(" MISMATCH {:<44} {}", c.id, c.detail); + } + identity_failures = bad.len(); + } + } + let corrupt = inv .fixtures .iter() @@ -445,7 +508,7 @@ inventory: {}", out.display()); // against a fixture we intend to source is a legitimate way to record the // want — the IMZ password cases exist exactly so that requirement stops // being invisible in a formats.toml notes field. - if corrupt > 0 { + if corrupt > 0 || identity_failures > 0 { 1 } else { 0 @@ -963,14 +1026,21 @@ fn cmd_validate(args: &Args) -> i32 { } } // Every expected failure must name a recorded cause, or it is just - // a disabled test wearing a label. - let bugs = fs::read_to_string(base.join("..").join("docs").join("Regression_Bugs.md")) - .unwrap_or_default(); + // a disabled test wearing a label. A cause is either a defect + // (Regression_Bugs.md) or a capability the engine has never + // claimed (missing_features_from_regression.md) — a case pinned to + // an unimplemented feature is red for a reason, just not a bug. + let docs = base.join("..").join("docs"); + let bugs = fs::read_to_string(docs.join("Regression_Bugs.md")).unwrap_or_default(); + let features = + fs::read_to_string(docs.join("missing_features_from_regression.md")) + .unwrap_or_default(); if !bugs.is_empty() { for (id, f) in k.all_entries() { - if !bugs.contains(f) { + if !bugs.contains(f) && !features.contains(f) { known_problems.push(format!( - "known-failures.toml: '{}' cites finding {}, which is not in docs/Regression_Bugs.md", + "known-failures.toml: '{}' cites {}, which is in neither \ + docs/Regression_Bugs.md nor docs/missing_features_from_regression.md", id, f )); } @@ -1620,3 +1690,241 @@ fn sanitise_id(id: &str) -> String { }) .collect() } + +fn cmd_oracles(args: &Args) -> i32 { + let base = regression_dir(); + let reg = match registry::Registry::load(&base) { + Ok(r) => r, + Err(e) => { + eprintln!("error: {}", e); + return 2; + } + }; + let platform = exec::platform_token(); + let path = oracles::overlay_path(&base); + + if let Some(host_id) = &args.scan_host { + let (cfg, _, _) = local::load(&base); + let host = match cfg.hosts.iter().find(|h| &h.id == host_id) { + Some(h) => h, + None => { + eprintln!( + "no host {host_id:?} in local.toml. Known: {}", + cfg.hosts + .iter() + .map(|h| h.id.as_str()) + .collect::>() + .join(", ") + ); + return 2; + } + }; + let target = match &host.ssh { + Some(t) => t.clone(), + None => { + eprintln!("host {host_id:?} has no `ssh` target in local.toml"); + return 2; + } + }; + // One ssh call, listing the cores. The board's IP and key live in + // local.toml like every other machine address — nothing new to + // configure, and nothing that could reach a public repo. + let listing = match exec::ssh_capture( + &target, + "ls /media/fat/_Computer/*.rbf 2>/dev/null | xargs -n1 basename", + ) { + Ok(o) => o, + Err(e) => { + eprintln!("scan {host_id}: {e}"); + return 1; + } + }; + let present: Vec = listing + .lines() + .map(|l| oracles::core_base_name(l.trim())) + .filter(|l| !l.is_empty()) + .collect(); + let scan = oracles::match_cores(®, &present); + println!( + "{host_id}: {} core(s) on the board, {} matched to an oracle", + scan.cores.len(), + scan.matched.len() + ); + for (o, c) in &scan.matched { + println!(" {:<26} {}", o, c); + } + for (o, c) in &scan.missing { + println!(" MISSING {:<17} needs core {}", o, c); + } + let body = oracles::render_mister(&scan, &host.platform, &today_stamp()); + // Merge into the overlay rather than replacing it: the board's cores + // and this box's tools are both availability, and a scan must not + // discard what --detect found. + let existing = fs::read_to_string(oracles::overlay_path(&base)).unwrap_or_default(); + let mut merged = String::new(); + for block in existing.split("[[availability]]") { + if scan.matched.iter().any(|(o, _)| block.contains(&format!("oracle = {:?}", o))) { + continue; + } + if merged.is_empty() { + merged.push_str(block); + } else { + merged.push_str("[[availability]]"); + merged.push_str(block); + } + } + merged.push_str(&body); + match oracles::write(&base, &merged) { + Ok(p) => { + println!(" +wrote {}", p.display()); + return 0; + } + Err(e) => { + eprintln!("error: {e}"); + return 2; + } + } + } + + if let Some(spec) = &args.set_oracle { + let (oracle, path) = match spec.split_once('=') { + Some(p) => p, + None => { + eprintln!("--set needs =, e.g. --set fs-uae=\"C:/Emulators/FS-UAE/fs-uae.exe\""); + return 2; + } + }; + if !reg.oracles.iter().any(|o| o.id == oracle) { + eprintln!("no oracle named {oracle:?} in data/oracles.toml"); + return 2; + } + let kind = reg + .oracles + .iter() + .find(|o| o.id == oracle) + .map(|o| o.kind.as_str()) + .unwrap_or("package"); + // An emulator with a binary still needs a guest; say `installed`. + let status = if kind == "emulator" { "installed" } else { "verified" }; + let existing = fs::read_to_string(oracles::overlay_path(&base)).unwrap_or_default(); + match oracles::set_hint(&existing, oracle, path, platform, &today_stamp(), status) { + Ok(body) => match oracles::write(&base, &body) { + Ok(p) => { + println!("{oracle}: {status} {path}"); + println!("wrote {}", p.display()); + return 0; + } + Err(e) => { + eprintln!("error: {e}"); + return 2; + } + }, + Err(e) => { + eprintln!("error: {e}"); + return 2; + } + } + } + + if args.export { + match fs::read_to_string(&path) { + Ok(t) => { + print!("{}", t); + return 0; + } + Err(_) => { + eprintln!( + "no overlay at {} - run `rb-regress oracles --detect` first", + path.display() + ); + return 1; + } + } + } + + if !args.detect { + println!("oracles: {} declared in data/oracles.toml", reg.oracles.len()); + println!("overlay: {}", path.display()); + println!(" --detect probe this host and rewrite the overlay"); + println!(" --export print it, to seed another machine"); + return 0; + } + + let hints = oracles::hints_for(®, platform); + let mut found = oracles::detect(®, &hints, platform); + // Ask about the emulators we could not find, but only on a terminal: the + // harness also runs in CI and over ssh, where a prompt nobody answers is + // indistinguishable from a hang. --no-prompt forces the quiet path. + let interactive = { + use std::io::IsTerminal; + !args.no_prompt && std::io::stdin().is_terminal() && std::io::stdout().is_terminal() + }; + if interactive { + let stdin = std::io::stdin(); + let mut lock = stdin.lock(); + let mut outv = std::io::stdout(); + let filled = oracles::prompt_for_missing(&mut found, ®, &mut lock, &mut outv); + if filled > 0 { + println!(" +recorded {filled} path(s) you provided"); + } + } + let today = today_stamp(); + let body = oracles::render(&found, platform, &today); + match oracles::write(&base, &body) { + Ok(p) => { + let n = |s: &str| found.iter().filter(|d| d.status == s).count(); + println!( + "detected on {}: {} verified, {} installed, {} manual, {} absent (of {})", + platform, + n("verified"), + n("installed"), + n("manual"), + n("absent"), + found.len() + ); + // `installed` was omitted here, which hid the four emulators the + // search had just found — a report that leaves out its best news. + for d in found + .iter() + .filter(|d| d.status == "verified" || d.status == "installed") + { + println!( + " {:<12} {:<10} {}", + d.oracle, + d.status, + d.resolved.as_deref().unwrap_or("") + ); + } + println!("\nwrote {}", p.display()); + 0 + } + Err(e) => { + eprintln!("error: {}", e); + 2 + } + } +} + +/// UTC date, for the `verified_on` stamp. +fn today_stamp() -> String { + let secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let days = secs / 86_400; + // Civil-from-days (Howard Hinnant's algorithm), so the stamp needs no + // date crate for one field. + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!("{:04}-{:02}-{:02}", y, m, d) +} diff --git a/regression-tests/runner/src/oracles.rs b/regression-tests/runner/src/oracles.rs new file mode 100644 index 00000000..506f7228 --- /dev/null +++ b/regression-tests/runner/src/oracles.rs @@ -0,0 +1,719 @@ +//! `oracles` — detect which third-party tools this machine actually has, and +//! write that as a gitignored overlay. +//! +//! The split matters. `data/oracles.toml` says what an oracle *is* and what it +//! proves: portable knowledge that belongs to everyone who clones the repo. +//! Whether *this* box has the tool, and where it lives, is neither portable nor +//! publishable — that is how an absolute `D:/ROMs/...` ended up in a tracked +//! file. So availability lives in `data/oracles.local.toml`, gitignored, +//! generated rather than hand-maintained. +//! +//! Two verbs, mirroring the corpus: +//! +//! - `--detect` probes the host and rewrites the overlay. +//! - `--export` prints it, to seed another machine or paste into a handoff. + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::exec; +use crate::registry::Registry; + +/// What a probe concluded about one tool on this host. +pub struct Detected { + pub oracle: String, + pub status: &'static str, + pub resolved: Option, +} + +/// The executable to look for, given an oracle's display `tool` string. +/// +/// `tool` is prose for the report — `mount -t affs`, `MiSTer Minimig core` — +/// so the first whitespace-separated word is the closest thing to a program +/// name. An oracle that needs something else sets `program` in the registry. +fn program_for(tool: &str, program: Option<&str>) -> String { + program + .map(|p| p.to_string()) + .unwrap_or_else(|| tool.split_whitespace().next().unwrap_or(tool).to_string()) +} + +/// Resolve a `path_hint`, which may name the program itself or the directory +/// holding it. `chdman`'s hint is `C:/Tools/chdman`, a directory — testing it +/// as a file reported the tool absent on a box that has it. +fn resolve_hint(hint: &str, program: &str) -> Option { + let p = Path::new(hint); + if p.is_file() { + return Some(hint.to_string()); + } + if p.is_dir() { + let exts: &[&str] = if cfg!(windows) { + &["", ".exe", ".cmd", ".bat"] + } else { + &[""] + }; + for ext in exts { + let cand = p.join(format!("{}{}", program, ext)); + if cand.is_file() { + return Some(cand.display().to_string()); + } + } + } + None +} + +/// Directories to search beyond `PATH`, for tools that install as an app +/// rather than a command — every emulator here is a GUI program that never +/// lands on `PATH`. +/// +/// `RB_EMULATOR_DIRS` is checked first and takes the platform separator, so a +/// machine keeping its emulators somewhere unusual needs no code change and no +/// hand-edited TOML. +fn search_roots(platform: &str) -> Vec { + let mut roots = Vec::new(); + if let Some(v) = std::env::var_os("RB_EMULATOR_DIRS") { + roots.extend(std::env::split_paths(&v)); + } + let home = std::env::var_os("USERPROFILE") + .or_else(|| std::env::var_os("HOME")) + .map(PathBuf::from); + match platform { + "windows" => { + for v in ["ProgramFiles", "ProgramFiles(x86)", "LOCALAPPDATA"] { + if let Some(p) = std::env::var_os(v) { + roots.push(PathBuf::from(p)); + } + } + roots.push(PathBuf::from("C:/Tools")); + roots.push(PathBuf::from("C:/Emulators")); + // A per-user collection is at least as common as a system-wide + // one, and is where this project's own emulators actually live. + if let Some(h) = &home { + roots.push(h.join("emulators")); + roots.push(h.join("Emulators")); + } + } + "macos" => { + roots.push(PathBuf::from("/Applications")); + roots.push(PathBuf::from("/opt/homebrew/bin")); + roots.push(PathBuf::from("/usr/local/bin")); + if let Some(h) = &home { + roots.push(h.join("Applications")); + } + } + _ => { + roots.push(PathBuf::from("/usr/bin")); + roots.push(PathBuf::from("/usr/local/bin")); + roots.push(PathBuf::from("/opt")); + if let Some(h) = &home { + roots.push(h.join(".local/bin")); + } + } + } + roots +} + +/// Look for `program` under each root: directly, one level down (the usual +/// `Program Files/WinUAE/winuae64.exe` shape), and inside a macOS `.app`. +fn find_under_roots(program: &str, roots: &[PathBuf]) -> Option { + let exts: &[&str] = if cfg!(windows) { + &["", ".exe", ".cmd", ".bat"] + } else { + &[""] + }; + for root in roots { + for ext in exts { + let direct = root.join(format!("{}{}", program, ext)); + if direct.is_file() { + return Some(direct.display().to_string()); + } + } + let entries = match fs::read_dir(root) { + Ok(e) => e, + Err(_) => continue, + }; + for e in entries.flatten() { + let d = e.path(); + if !d.is_dir() { + continue; + } + for ext in exts { + let cand = d.join(format!("{}{}", program, ext)); + if cand.is_file() { + return Some(cand.display().to_string()); + } + } + // macOS bundle: Iris.app/Contents/MacOS/Iris + let bundled = d.join("Contents").join("MacOS").join(program); + if bundled.is_file() { + return Some(bundled.display().to_string()); + } + } + } + None +} + +/// Probe every oracle the registry declares against this machine. +/// +/// Kinds that are not a program on PATH are reported as such rather than +/// guessed at: an emulator or a MiSTer core is not something `which` can find, +/// and claiming "absent" for them would read as a missing install rather than +/// a different kind of oracle. +pub fn detect(reg: &Registry, path_hints: &BTreeMap, platform: &str) -> Vec { + let mut out = Vec::new(); + for o in ®.oracles { + // A `mount` oracle is the kernel's opinion, so it only means anything + // on a kernel that has the driver. Probing the program name here found + // Cygwin's `mount` on Windows and called AFFS "verified", which is a + // confidently wrong answer — the worst kind for a tool whose whole job + // is telling you what is true. + if o.kind == "mount" && !matches!(platform, "linux" | "wsl" | "mister-hps") { + out.push(Detected { + oracle: o.id.clone(), + status: "absent", + resolved: None, + }); + continue; + } + // Hardware and round-trip oracles are not programs at all. + if matches!(o.kind.as_str(), "hardware" | "roundtrip") { + out.push(Detected { + oracle: o.id.clone(), + status: "manual", + resolved: None, + }); + continue; + } + // An emulator IS findable — it just installs as an app, not a command. + // Finding the binary does not make the oracle runnable, though: it + // still needs a configured guest. `installed` says exactly that, which + // `verified` would overclaim and `manual` would hide. + if o.kind == "emulator" { + let prog = program_for(&o.tool, o.program.as_deref()); + let found = path_hints + .get(&o.id) + .and_then(|h| resolve_hint(h, &prog)) + .or_else(|| exec::tool_available(&prog).then(|| prog.clone())) + .or_else(|| find_under_roots(&prog, &search_roots(platform))); + out.push(match found { + Some(p) => Detected { + oracle: o.id.clone(), + status: "installed", + resolved: Some(p), + }, + None => Detected { + oracle: o.id.clone(), + status: "manual", + resolved: None, + }, + }); + continue; + } + // An explicit hint wins: chdman and qemu-img are installed here but not + // on PATH, which is exactly the case a hint exists for. + let prog = program_for(&o.tool, o.program.as_deref()); + if let Some(hint) = path_hints.get(&o.id) { + if let Some(found) = resolve_hint(hint, &prog) { + out.push(Detected { + oracle: o.id.clone(), + status: "verified", + resolved: Some(found), + }); + continue; + } + } + if exec::tool_available(&prog) { + out.push(Detected { + oracle: o.id.clone(), + status: "verified", + resolved: Some(prog), + }); + } else { + out.push(Detected { + oracle: o.id.clone(), + status: "absent", + resolved: None, + }); + } + } + out +} + +/// Render the overlay. `verified_on` is stamped by the caller, not here, so +/// this stays deterministic and testable. +pub fn render(found: &[Detected], platform: &str, today: &str) -> String { + let mut s = String::new(); + s.push_str( + "# Per-host oracle availability. GITIGNORED - generated, do not hand-edit.\n\ + #\n\ + # Regenerate: rb-regress oracles --detect\n\ + # Share: rb-regress oracles --export\n\ + #\n\ + # What an oracle IS lives in the tracked data/oracles.toml. Whether\n\ + # this machine has it lives here, because an absolute path in a public\n\ + # repo is how D:/ROMs ended up committed.\n\ + #\n\ + # `manual` means the oracle is not a program on PATH at all - an\n\ + # emulator or a real MiSTer core. Not the same as absent.\n\n", + ); + for d in found { + // Absent is the default the tracked registry already implies; writing + // it would double the file for no information. + if d.status == "absent" { + continue; + } + s.push_str("[[availability]]\n"); + s.push_str(&format!("oracle = {:?}\n", d.oracle)); + s.push_str(&format!("platform = {:?}\n", platform)); + s.push_str(&format!("status = {:?}\n", d.status)); + if let Some(r) = &d.resolved { + s.push_str(&format!("path_hint = {:?}\n", r)); + } + s.push_str(&format!("verified_on = {:?}\n\n", today)); + } + s +} + +/// Existing hints from the tracked registry for this platform, so a detect run +/// does not throw away a hand-recorded path that is still correct. +pub fn hints_for(reg: &Registry, platform: &str) -> BTreeMap { + let mut m = BTreeMap::new(); + for a in ®.availability { + if a.platform == platform { + if let Some(h) = &a.path_hint { + m.insert(a.oracle.clone(), h.clone()); + } + } + } + m +} + +pub fn overlay_path(regression_dir: &Path) -> PathBuf { + regression_dir.join("data").join("oracles.local.toml") +} + +/// Record one oracle's path by hand, for a tool `--detect` cannot find. +/// +/// Rewrites just that oracle's row in the overlay and leaves the rest alone, so +/// providing a path never costs you the detected ones. The path is checked +/// here: a hint that points at nothing is the kind of thing you discover three +/// weeks later when a verify run quietly skips. +pub fn set_hint( + existing: &str, + oracle: &str, + path: &str, + platform: &str, + today: &str, + status: &str, +) -> Result { + if !Path::new(path).exists() { + return Err(format!("{path}: no such file or directory")); + } + let mut kept = String::new(); + let mut skipping = false; + for block in existing.split("[[availability]]") { + if block.contains(&format!("oracle = {:?}", oracle)) { + skipping = true; + continue; + } + if !skipping && kept.is_empty() { + kept.push_str(block); + } else { + kept.push_str("[[availability]]"); + kept.push_str(block); + } + skipping = false; + } + if kept.is_empty() { + kept.push_str(existing); + } + if !kept.ends_with('\n') { + kept.push('\n'); + } + kept.push_str(&format!( + "[[availability]] +oracle = {:?} +platform = {:?} +status = {:?} +path_hint = {:?} +verified_on = {:?} + +", + oracle, platform, status, path, today + )); + Ok(kept) +} + +pub fn write(regression_dir: &Path, body: &str) -> Result { + let p = overlay_path(regression_dir); + fs::write(&p, body).map_err(|e| format!("{}: {}", p.display(), e))?; + Ok(p) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_display_string_yields_its_first_word() { + assert_eq!(program_for("mount -t affs", None), "mount"); + assert_eq!(program_for("qemu-img", None), "qemu-img"); + // An explicit program wins, for tools whose display name is prose. + assert_eq!(program_for("MiSTer Minimig core", Some("none")), "none"); + } + + #[test] + fn a_hint_may_be_the_directory_holding_the_program() { + let dir = std::env::temp_dir().join("rb-oracle-hint-test"); + let _ = fs::create_dir_all(&dir); + let exe = dir.join(if cfg!(windows) { "toolx.exe" } else { "toolx" }); + let _ = fs::write(&exe, b"x"); + assert_eq!( + resolve_hint(&dir.display().to_string(), "toolx"), + Some(exe.display().to_string()), + "a directory hint must find the program inside it" + ); + assert_eq!(resolve_hint(&dir.display().to_string(), "absent"), None); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn absent_rows_are_not_written() { + let found = vec![ + Detected { + oracle: "here".into(), + status: "verified", + resolved: Some("chdman".into()), + }, + Detected { + oracle: "gone".into(), + status: "absent", + resolved: None, + }, + ]; + let out = render(&found, "windows", "2026-08-10"); + assert!(out.contains("\"here\"")); + // The tracked registry already implies absent; writing it doubles the + // file for no information. + assert!(!out.contains("\"gone\"")); + assert!(out.contains("path_hint = \"chdman\"")); + } + + #[test] + fn setting_a_hint_replaces_only_that_oracle() { + let start = concat!( + "[[availability]] +oracle = \"keepme\" +platform = \"windows\" +", + "status = \"verified\" + +", + "[[availability]] +oracle = \"fs-uae\" +platform = \"windows\" +", + "status = \"manual\" + +" + ); + let here = std::env::current_exe().unwrap(); + let out = set_hint( + start, + "fs-uae", + &here.display().to_string(), + "windows", + "2026-08-10", + "installed", + ) + .expect("path exists"); + assert!(out.contains("\"keepme\""), "other rows must survive"); + assert_eq!(out.matches("oracle = \"fs-uae\"").count(), 1, "no duplicate row"); + assert!(out.contains("status = \"installed\"")); + } + + #[test] + fn setting_a_hint_to_nothing_is_refused() { + // A hint pointing at nothing is discovered three weeks later, as a + // quiet skip. Fail at the point the mistake is made. + let e = set_hint("", "fs-uae", "/definitely/not/here", "windows", "2026-08-10", "installed"); + assert!(e.is_err()); + } + + #[test] + fn manual_oracles_are_recorded_but_carry_no_path() { + let found = vec![Detected { + oracle: "fs-uae".into(), + status: "manual", + resolved: None, + }]; + let out = render(&found, "windows", "2026-08-10"); + assert!(out.contains("status = \"manual\"")); + assert!(!out.contains("path_hint")); + } +} + +/// Ask the user where an emulator lives, for the ones `--detect` could not +/// find. +/// +/// Only emulators, and only on a terminal. Those two limits are the whole +/// design: +/// +/// - **Only emulators**, because they are the case where the tool *is* +/// installed and we merely cannot see it. A `package` oracle that came back +/// absent is genuinely not installed, and prompting for a path to software +/// you do not have is noise — 18 questions to reach the 5 that matter. +/// - **Only on a terminal**, because the harness runs in CI and over ssh. A +/// prompt with no one to answer it is a hang, and a hang in a regression run +/// looks exactly like a test that never finishes. +/// +/// A blank answer skips. A path that does not exist is refused and re-asked, +/// rather than written and discovered later as a silent skip. +pub fn prompt_for_missing( + found: &mut [Detected], + reg: &Registry, + input: &mut impl std::io::BufRead, + out: &mut impl std::io::Write, +) -> usize { + let emulators: std::collections::BTreeSet<&str> = reg + .oracles + .iter() + .filter(|o| o.kind == "emulator") + .map(|o| o.id.as_str()) + .collect(); + + let mut filled = 0; + for d in found.iter_mut() { + if d.status != "manual" || !emulators.contains(d.oracle.as_str()) { + continue; + } + let prog = reg + .oracles + .iter() + .find(|o| o.id == d.oracle) + .map(|o| program_for(&o.tool, o.program.as_deref())) + .unwrap_or_else(|| d.oracle.clone()); + loop { + let _ = writeln!( + out, + "\n{} not found (looked for `{}` on PATH and the usual install roots).", + d.oracle, prog + ); + let _ = write!(out, " path to it, or Enter to skip: "); + let _ = out.flush(); + let mut line = String::new(); + if input.read_line(&mut line).unwrap_or(0) == 0 { + return filled; // EOF: stop asking rather than spin. + } + let ans = line.trim(); + if ans.is_empty() { + break; + } + if Path::new(ans).exists() { + d.status = "installed"; + d.resolved = Some(ans.to_string()); + filled += 1; + break; + } + let _ = writeln!(out, " {ans}: no such file or directory"); + } + } + filled +} + +#[cfg(test)] +mod prompt_tests { + use super::*; + use crate::registry::{Oracle, Registry}; + + fn reg_with(kind: &str, id: &str) -> Registry { + let mut r = Registry::default(); + r.oracles.push(Oracle { + id: id.into(), + tool: "Thing".into(), + kind: kind.into(), + program: Some("thing".into()), + notes: None, + core: None, + }); + r + } + + #[test] + fn a_blank_answer_skips() { + let reg = reg_with("emulator", "fs-uae"); + let mut found = vec![Detected { + oracle: "fs-uae".into(), + status: "manual", + resolved: None, + }]; + let mut input = std::io::Cursor::new(b"\n".to_vec()); + let mut out = Vec::new(); + assert_eq!(prompt_for_missing(&mut found, ®, &mut input, &mut out), 0); + assert_eq!(found[0].status, "manual"); + } + + #[test] + fn a_real_path_is_recorded_as_installed() { + let reg = reg_with("emulator", "fs-uae"); + let mut found = vec![Detected { + oracle: "fs-uae".into(), + status: "manual", + resolved: None, + }]; + let here = std::env::current_exe().unwrap(); + let mut input = std::io::Cursor::new(format!("{}\n", here.display()).into_bytes()); + let mut out = Vec::new(); + assert_eq!(prompt_for_missing(&mut found, ®, &mut input, &mut out), 1); + assert_eq!(found[0].status, "installed"); + } + + #[test] + fn a_bad_path_is_refused_then_reasked() { + let reg = reg_with("emulator", "fs-uae"); + let mut found = vec![Detected { + oracle: "fs-uae".into(), + status: "manual", + resolved: None, + }]; + // Wrong once, then blank. The wrong one must not be written. + let mut input = std::io::Cursor::new(b"/nope/nowhere\n\n".to_vec()); + let mut out = Vec::new(); + assert_eq!(prompt_for_missing(&mut found, ®, &mut input, &mut out), 0); + let text = String::from_utf8_lossy(&out); + assert!(text.contains("no such file"), "must say why it was refused"); + assert_eq!(found[0].status, "manual"); + } + + #[test] + fn package_oracles_are_never_asked_about() { + // Absent means not installed. Asking would be 18 questions to reach + // the 5 that matter. + let reg = reg_with("package", "cpmtools"); + let mut found = vec![Detected { + oracle: "cpmtools".into(), + status: "manual", + resolved: None, + }]; + let mut input = std::io::Cursor::new(b"".to_vec()); + let mut out = Vec::new(); + assert_eq!(prompt_for_missing(&mut found, ®, &mut input, &mut out), 0); + assert!(out.is_empty(), "no prompt should have been printed"); + } +} + +/// What a MiSTer scan found on the board. +pub struct MisterScan { + /// `.rbf` base names present, date suffix stripped. + pub cores: Vec, + /// Oracles matched to a core that is actually installed. + pub matched: Vec<(String, String)>, + /// Oracles whose core the board does not have. + pub missing: Vec<(String, String)>, +} + +/// Strip MiSTer's `_YYYYMMDD` build-date suffix from an `.rbf` filename. +/// +/// Cores are redistributed as `X68000_20260603.rbf` and the date moves every +/// time the board is updated, so the bare name is the only stable identifier. +pub fn core_base_name(filename: &str) -> String { + let stem = filename.strip_suffix(".rbf").unwrap_or(filename); + match stem.rsplit_once('_') { + Some((head, tail)) if tail.len() == 8 && tail.chars().all(|c| c.is_ascii_digit()) => { + head.to_string() + } + _ => stem.to_string(), + } +} + +/// Match the board's cores against the oracles that name one. +pub fn match_cores(reg: &Registry, present: &[String]) -> MisterScan { + let have: std::collections::BTreeSet<&str> = present.iter().map(|s| s.as_str()).collect(); + let mut matched = Vec::new(); + let mut missing = Vec::new(); + for o in ®.oracles { + if let Some(core) = &o.core { + if have.contains(core.as_str()) { + matched.push((o.id.clone(), core.clone())); + } else { + missing.push((o.id.clone(), core.clone())); + } + } + } + MisterScan { + cores: present.to_vec(), + matched, + missing, + } +} + +/// Render scan results as overlay rows. A core that is present is `verified`: +/// unlike an emulator, there is no guest to configure — the core IS the guest. +/// Running the check is still manual, which the oracle's `kind` already says. +pub fn render_mister(scan: &MisterScan, platform: &str, today: &str) -> String { + let mut s = String::new(); + s.push_str( + "# MiSTer cores found by `rb-regress oracles --scan`. GITIGNORED.\n\ + # A core present on the board is recorded verified; whether a given\n\ + # check has been RUN is a separate question the verify tree answers.\n\n", + ); + for (oracle, core) in &scan.matched { + s.push_str("[[availability]]\n"); + s.push_str(&format!("oracle = {:?}\n", oracle)); + s.push_str(&format!("platform = {:?}\n", platform)); + s.push_str("status = \"verified\"\n"); + s.push_str(&format!("path_hint = {:?}\n", core)); + s.push_str(&format!("verified_on = {:?}\n\n", today)); + } + s +} + +#[cfg(test)] +mod mister_tests { + use super::*; + use crate::registry::{Oracle, Registry}; + + fn reg() -> Registry { + let mut r = Registry::default(); + for (id, core) in [("mister-core-amiga", "Minimig"), ("mister-core-ti99", "Ti994a")] { + r.oracles.push(Oracle { + id: id.into(), + tool: "core".into(), + kind: "hardware".into(), + program: None, + notes: None, + core: Some(core.into()), + }); + } + r + } + + #[test] + fn the_build_date_suffix_is_stripped() { + assert_eq!(core_base_name("X68000_20260603.rbf"), "X68000"); + assert_eq!(core_base_name("ZX-Spectrum_20250930.rbf"), "ZX-Spectrum"); + // A name with an underscore but no date must survive intact. + assert_eq!(core_base_name("Apple-II.rbf"), "Apple-II"); + assert_eq!(core_base_name("My_Core.rbf"), "My_Core"); + } + + #[test] + fn present_and_absent_cores_are_told_apart() { + let present = vec!["Minimig".to_string(), "X68000".to_string()]; + let scan = match_cores(®(), &present); + assert_eq!(scan.matched.len(), 1); + assert_eq!(scan.matched[0].0, "mister-core-amiga"); + // Ti994a is not on this imaginary board, and must be reported missing + // rather than silently dropped. + assert_eq!(scan.missing.len(), 1); + assert_eq!(scan.missing[0].1, "Ti994a"); + } + + #[test] + fn only_matched_cores_reach_the_overlay() { + let scan = match_cores(®(), &["Minimig".to_string()]); + let out = render_mister(&scan, "mister-core", "2026-08-12"); + assert!(out.contains("mister-core-amiga")); + assert!(!out.contains("mister-core-ti99")); + } +} diff --git a/regression-tests/runner/src/registry.rs b/regression-tests/runner/src/registry.rs index 3af2c367..0dcb60d7 100644 --- a/regression-tests/runner/src/registry.rs +++ b/regression-tests/runner/src/registry.rs @@ -55,6 +55,11 @@ pub struct Oracle { pub program: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub notes: Option, + /// For a MiSTer core: the `.rbf` base name, without the build-date suffix. + /// A core's name is the same on every board, so it is portable knowledge; + /// whether a given board has it is not, and comes from `--scan`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub core: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -130,6 +135,13 @@ struct FormatsFile { formats: Vec, } +/// The gitignored per-host overlay written by `rb-regress oracles --detect`. +#[derive(Deserialize, Default)] +struct LocalOraclesFile { + #[serde(default)] + availability: Vec, +} + #[derive(Deserialize)] struct OraclesFile { #[serde(default, rename = "oracle")] @@ -146,6 +158,8 @@ struct OracleDef { #[serde(default)] notes: Option, #[serde(default)] + core: Option, + #[serde(default)] availability: Vec, #[serde(default)] verifies: Vec, @@ -264,9 +278,36 @@ impl Registry { kind: o.kind, program: o.program, notes: o.notes, + core: o.core, }); } + // Machine-specific availability, layered over the tracked registry. + // `oracles.toml` says what a tool is and what it proves — portable + // knowledge. Whether *this* box has it, and where, is not portable and + // is gitignored: absolute paths in a public repo were how a local + // `D:/ROMs/...` ended up tracked. Generated by `rb-regress oracles + // --detect`; a row here replaces the tracked one for the same + // (oracle, platform). + let local = data.join("oracles.local.toml"); + if local.is_file() { + match fs::read_to_string(&local) + .map_err(|e| e.to_string()) + .and_then(|t| toml::from_str::(&t).map_err(|e| e.to_string())) + { + Ok(lf) => { + for a in lf.availability { + r.availability + .retain(|e| !(e.oracle == a.oracle && e.platform == a.platform)); + r.availability.push(a); + } + } + Err(e) => r + .warnings + .push(format!("{}: {}", local.display(), e)), + } + } + // --- hosts --------------------------------------------------------- // From local.toml, the one gitignored file that names this network. // It used to be data/hosts.toml; see runner/src/local.rs. diff --git a/regression-tests/runner/src/verify.rs b/regression-tests/runner/src/verify.rs index 4e4dab90..9b9635f7 100644 --- a/regression-tests/runner/src/verify.rs +++ b/regression-tests/runner/src/verify.rs @@ -73,6 +73,14 @@ pub struct Record { /// The artifact's sha256 as recorded by `produce`, so a verdict can be tied /// to the exact bytes that were checked. pub artifact_sha256: String, + /// The commit the artifact was produced at, carried onto every verdict so + /// a finding can be traced to the code that actually wrote the bytes. + pub artifact_git_sha: String, + /// `Some(true)` when engine sources moved between `artifact_git_sha` and + /// HEAD — the verdict describes code no longer in the tree. `None` when the + /// question could not be answered. See [`Report::stale`]. + #[serde(skip_serializing_if = "Option::is_none")] + pub artifact_stale: Option, pub argv: Vec, pub verdict: Verdict, #[serde(skip_serializing_if = "Option::is_none")] @@ -83,6 +91,12 @@ pub struct Report { pub records: Vec, /// Formats present in the artifact tree that no oracle claims at all. pub unclaimed: Vec, + /// Artifacts whose producing commit predates an engine change, as + /// `(format, producer_os, short_sha)`. A verdict on one of these is about + /// code that is no longer in the tree — the shape that produced R-038, + /// where a fixed AFFS formatter was reported as broken because the oracle + /// read artifacts built two days before the fix. + pub stale: Vec<(String, String, String)>, } /// Resolve an oracle's executable on this host. @@ -149,7 +163,7 @@ fn resolve_program( None } -fn expand(args: &[String], artifact: &Path) -> Vec { +fn expand(args: &[String], artifact: &Path, oracles_dir: &Path) -> Vec { let dir = artifact .parent() .map(|p| p.display().to_string()) @@ -158,6 +172,11 @@ fn expand(args: &[String], artifact: &Path) -> Vec { .map(|a| { a.replace("{artifact}", &artifact.display().to_string()) .replace("{dir}", &dir) + // A wrapper script has to be addressed absolutely: checks run + // with the artifact's directory as cwd, so a relative + // `oracles/foo.py` resolves inside artifacts/// + // and the interpreter reports a missing file. + .replace("{oracles}", &oracles_dir.display().to_string()) }) .collect() } @@ -223,6 +242,9 @@ pub fn verify( let mut records = Vec::new(); let mut unclaimed = Vec::new(); + let mut stale = Vec::new(); + // One git query per distinct producing commit, not per artifact. + let mut staleness: BTreeMap> = BTreeMap::new(); for (meta, image) in &artifacts { if let Some(f) = filter { @@ -230,6 +252,22 @@ pub fn verify( continue; } } + let artifact_stale = *staleness.entry(meta.git_sha.clone()).or_insert_with(|| { + crate::gitinfo::engine_changed_since( + regression_dir + .parent() + .unwrap_or(regression_dir), + &meta.git_sha, + crate::gitinfo::ENGINE_PATHS, + ) + }); + if artifact_stale == Some(true) { + let short: String = meta.git_sha.chars().take(7).collect(); + let row = (meta.format.clone(), meta.producer_os.clone(), short); + if !stale.contains(&row) { + stale.push(row); + } + } // Only write-direction rows: this artifact is something rb-cli wrote, // so a read-direction oracle row says nothing about it. let claims: Vec<&crate::registry::Verification> = reg @@ -278,7 +316,11 @@ pub fn verify( None, ), Some(exe) => { - let argv = expand(claim.check.as_ref().unwrap(), image); + let argv = expand( + claim.check.as_ref().unwrap(), + image, + ®ression_dir.join("oracles"), + ); let cwd = image.parent().unwrap_or(artifacts_root); match exec::run(&exe, &argv, cwd, VERIFY_TIMEOUT) { Err(e) => ( @@ -313,6 +355,8 @@ pub fn verify( oracle: oracle.id.clone(), strength: claim.strength.clone(), artifact_sha256: meta.sha256.clone(), + artifact_git_sha: meta.git_sha.clone(), + artifact_stale, argv, verdict, stdout_head, @@ -334,7 +378,11 @@ pub fn verify( fs::write(out_dir.join(format!("{}.json", key)), json).map_err(|e| e.to_string())?; } - Ok(Report { records, unclaimed }) + Ok(Report { + records, + unclaimed, + stale, + }) } /// Compare a check's output against what the row said to expect. An expectation @@ -391,12 +439,19 @@ pub fn render(report: &Report, out_dir: &Path) -> String { for r in &report.records { *counts.entry(r.verdict.label()).or_insert(0) += 1; + // A verdict on a stale artifact is marked inline as well as summarised + // below, so a FAIL line can never be read without the caveat attached. + let stale = if r.artifact_stale == Some(true) { + " [STALE ARTIFACT]" + } else { + "" + }; // Skips are counted, not listed one by one — there are dozens and they // are the expected state, not news. Failures always print. match &r.verdict { Verdict::Fail { reason } => s.push_str(&format!( - "FAIL {:<20} {:<14} via {:<12} {}\n", - r.format, r.producer_os, r.oracle, reason + "FAIL {:<20} {:<14} via {:<12} {}{}\n", + r.format, r.producer_os, r.oracle, reason, stale )), Verdict::Error { reason } => s.push_str(&format!( "error {:<20} {:<14} via {:<12} {}\n", @@ -416,6 +471,21 @@ pub fn render(report: &Report, out_dir: &Path) -> String { } s.push_str(&format!("\nverifications: {}\n", out_dir.display())); + // Stale artifacts come before the unclaimed list because they invalidate + // verdicts that were just printed, rather than merely bounding them. + if !report.stale.is_empty() { + s.push_str(&format!( + "\nWARNING: {} artifact(s) were built before the current engine sources.\n\ + A verdict on these describes code that is no longer in the tree — this is\n\ + exactly how R-038 was filed against an AFFS formatter that had already been\n\ + fixed. Re-run `produce` on the owning host before trusting them.\n", + report.stale.len() + )); + for (format, os, sha) in &report.stale { + s.push_str(&format!(" {:<20} {:<10} built at {}\n", format, os, sha)); + } + } + // The ceiling, restated every run: a tree full of passes still says nothing // about the formats nobody claims. if !report.unclaimed.is_empty() { @@ -494,4 +564,55 @@ mod tests { Verdict::Fail { .. } )); } + + fn record(stale: Option) -> Record { + Record { + format: "fs.affs".into(), + producer_os: "windows".into(), + producer_host: "H".into(), + verifier_os: "windows".into(), + verifier_host: "H".into(), + oracle: "amitools".into(), + strength: "structural".into(), + artifact_sha256: "deadbeef".into(), + artifact_git_sha: "0563cb807131286f90e28db506f5e72b0a834878".into(), + artifact_stale: stale, + argv: vec![], + verdict: Verdict::Fail { + reason: "xdftool rejected the volume".into(), + }, + stdout_head: None, + } + } + + /// R-038 in miniature: a failing verdict on an artifact older than the + /// engine must never read as a plain failure. Both the inline marker and + /// the summary have to be present, because the summary alone scrolls off + /// on a full run. + #[test] + fn a_stale_artifact_marks_its_failure_inline_and_in_the_summary() { + let report = Report { + records: vec![record(Some(true))], + unclaimed: vec![], + stale: vec![("fs.affs".into(), "windows".into(), "0563cb8".into())], + }; + let text = render(&report, Path::new("out")); + assert!(text.contains("[STALE ARTIFACT]"), "{text}"); + assert!(text.contains("built at 0563cb8"), "{text}"); + assert!(text.contains("no longer in the tree"), "{text}"); + } + + /// The inverse, so the warning cannot become permanent furniture that + /// everyone learns to ignore: a current artifact says nothing about age. + #[test] + fn a_current_artifact_carries_no_staleness_noise() { + let report = Report { + records: vec![record(Some(false))], + unclaimed: vec![], + stale: vec![], + }; + let text = render(&report, Path::new("out")); + assert!(!text.contains("STALE"), "{text}"); + assert!(text.contains("FAIL"), "{text}"); + } } diff --git a/scripts/preflight.sh b/scripts/preflight.sh new file mode 100644 index 00000000..73cb37a3 --- /dev/null +++ b/scripts/preflight.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Run what CI runs, before CI runs it. +# +# Exists because a green `cargo test` and a green `rb-regress run` are not a +# green pipeline. Two pushes in one day went red on things neither could see: +# +# - a unit test that asserted a Windows path answer and failed on every Unix +# job, because `Path::file_name` only treats `\` as a separator on Windows; +# - `optical.rs` naming the `zstd` crate directly, which does not exist in the +# MiSTer feature set (optical on, native-zstd off). +# +# Each check below is one of those classes. Run from the repo root. + +set -uo pipefail +cd "$(dirname "$0")/.." + +fail=0 +run() { + local label="$1"; shift + printf '\n=== %s ===\n' "$label" + if "$@"; then + printf ' OK: %s\n' "$label" + else + printf ' FAILED: %s\n' "$label" + fail=1 + fi +} + +# CI's test step is --release, not a plain `cargo test`. Debug and release +# differ in overflow checks and in which assertions are compiled. +run "cargo test --release (what CI's Test step runs)" \ + cargo test --release + +# The MiSTer armv7 leg: optical on, native-zstd off. Anything touching zstd has +# to go through crate::rbformats::zstd_compat or it breaks only here. +run "MiSTer feature set (optical on, native-zstd off)" \ + cargo check --bin rb-cli --no-default-features \ + --features chd,pure-zstd,remote,optical,tui + +# The Rust 1.73 floor for engine code. A modern build cannot see a violation; +# this compiles the shared source under the vintage feature/dep set. +run "Rust 1.73 floor (vintage manifest)" \ + cargo build --manifest-path rb-cli-vintage/Cargo.toml \ + --no-default-features \ + --features native-zstd,remote,tui,rust173-polyfill,windows-legacy,yaml \ + --ignore-rust-version + +# Documentation that claims something about the source. Cheap, and it is the +# only guard on the README tables and CONTRIBUTING's vintage command. +run "doc parity (README / CONTRIBUTING vs source)" \ + cargo test --test doc_parity + +# The harness is its own crate, so nothing above compiles or tests it. +run "rb-regress (the harness's own tests)" \ + cargo test --manifest-path regression-tests/runner/Cargo.toml + +# And its clippy, because the pre-commit hook runs it. Without this, preflight +# says "all checks passed" and the commit is then rejected by the hook — which +# is exactly what happened when this line was missing. +run "rb-regress clippy (what the pre-commit hook runs)" \ + cargo clippy --manifest-path regression-tests/runner/Cargo.toml --all-targets -- -D warnings + +printf '\n' +if [ "$fail" -ne 0 ]; then + printf 'preflight: FAILED - do not push\n' + exit 1 +fi +printf 'preflight: all checks passed\n' +printf 'After pushing, read the JOB LIST, not the run conclusion:\n' +printf ' gh run list --limit 1\n' +printf ' gh run view \n' +printf 'Jobs marked continue-on-error (the MiSTer rb-cli-mini build is one) can\n' +printf 'fail while the run still reports success. That hid a compile error for a day.\n' diff --git a/src/backup/metadata.rs b/src/backup/metadata.rs index c30ed11e..ba8069a6 100644 --- a/src/backup/metadata.rs +++ b/src/backup/metadata.rs @@ -1,5 +1,44 @@ use serde::{Deserialize, Serialize}; +/// Reduce a backup source to a label that names the source without naming the +/// machine that produced the backup. +/// +/// `source_device` used to hold whatever path the caller passed, absolute paths +/// included, so the same image backed up on two machines produced two different +/// `metadata.json` files and each one published the operator's directory layout +/// (R-035). Three shapes, because they are not all paths: +/// +/// - **Device paths** (`\\.\PhysicalDrive2`, `/dev/sda`, `/dev/disk2`) are kept +/// verbatim. A device node is already a leaf and it is the useful provenance. +/// - **`rb://` sources** are kept verbatim. The host in a remote backup's label +/// is the source, deliberately named by the user, not local layout. +/// - **Everything else** is a path to an image file, and keeps only its file +/// name. +pub fn normalize_source_device(source: &str) -> String { + let is_device = source.starts_with("/dev/") + || source.starts_with(r"\\.\") + || source.starts_with(r"\\?\") + || source.starts_with("rb://"); + if is_device { + return source.to_string(); + } + // Both separators, on every host. `Path::file_name` only treats `\` as one + // on Windows, so a Windows path normalised on Linux kept its directories — + // which is the exact leak this function exists to close, and made the field + // depend on which OS ran the backup. + let leaf = source + .rsplit(['/', '\\']) + .find(|part| !part.is_empty()) + .unwrap_or(source); + // A path with no file name (a bare root, or only separators) has nothing to + // reduce to; keeping the original beats an empty field. + if leaf.is_empty() { + source.to_string() + } else { + leaf.to_string() + } +} + /// Backup folder layout. Selects how partition data is stored on disk. /// /// `PerPartition` is the layout used by Zstd / Raw / per-partition VHD @@ -224,6 +263,58 @@ pub fn update_partition_checksum( mod tests { use super::*; + #[test] + fn normalize_source_device_drops_the_producing_hosts_directories() { + // The point of R-035: two machines holding the same image must record + // the same thing, and neither publishes where it keeps its files. + assert_eq!( + normalize_source_device(r"C:\Users\someone\images\disk.img"), + "disk.img" + ); + assert_eq!( + normalize_source_device("/home/someone/images/disk.img"), + "disk.img" + ); + assert_eq!(normalize_source_device("disk.img"), "disk.img"); + } + + #[test] + fn normalize_source_device_keeps_devices_and_remotes_verbatim() { + // A device node is already a leaf, and it is the useful provenance. + for dev in [r"\\.\PhysicalDrive2", "/dev/sda", "/dev/disk2"] { + assert_eq!(normalize_source_device(dev), dev, "{dev} must survive"); + } + // A remote label names the *source* host, which the user chose; it is + // not the producing host's directory layout. + for r in ["rb://nas:7341/disk.img", "rb://nas:7341/dev/sda"] { + assert_eq!(normalize_source_device(r), r, "{r} must survive"); + } + } + + #[test] + fn normalize_source_device_leaves_a_pathless_string_alone() { + // Nothing to reduce to — keeping it beats emptying the field. + assert_eq!(normalize_source_device("/"), "/"); + assert_eq!(normalize_source_device(""), ""); + assert_eq!(normalize_source_device(r"\\"), r"\\"); + } + + #[test] + fn normalize_source_device_is_the_same_answer_on_every_host() { + // This test is why CI went red: the first version used + // `Path::file_name`, which only treats `\` as a separator on Windows, + // so a Windows path normalised on Linux kept every directory. The + // whole point of the field is that two machines agree. + assert_eq!( + normalize_source_device(r"C:\Users\someone\images\disk.img"), + "disk.img" + ); + assert_eq!(normalize_source_device(r"D:\disk.img"), "disk.img"); + // A trailing separator must not yield an empty label. + assert_eq!(normalize_source_device("/srv/images/"), "images"); + assert_eq!(normalize_source_device(r"C:\images\"), "images"); + } + #[test] fn byte_offset_prefers_start_byte_then_falls_back_to_floored_lba() { let mut pm = PartitionMetadata { diff --git a/src/backup/mod.rs b/src/backup/mod.rs index 7f929edd..99980536 100644 --- a/src/backup/mod.rs +++ b/src/backup/mod.rs @@ -2013,7 +2013,7 @@ fn run_backup_inner( let metadata = BackupMetadata { version: 1, created: Utc::now().to_rfc3339(), - source_device: source_display, + source_device: metadata::normalize_source_device(&source_display), source_size_bytes: source_size, partition_table_type: table.type_name().to_string(), checksum_type: config.checksum.as_str().to_string(), @@ -2344,7 +2344,7 @@ fn run_single_file_chd_path( let metadata = BackupMetadata { version: 1, created: Utc::now().to_rfc3339(), - source_device: config.source_path.display().to_string(), + source_device: metadata::normalize_source_device(&config.source_path.display().to_string()), source_size_bytes: source_size, partition_table_type: table.type_name().to_string(), checksum_type: config.checksum.as_str().to_string(), diff --git a/src/bin/rb_cli.rs b/src/bin/rb_cli.rs index d8f34086..af166ec6 100644 --- a/src/bin/rb_cli.rs +++ b/src/bin/rb_cli.rs @@ -2,19 +2,35 @@ //! does not call into this; both bins share the parsing + handler code in //! `rusty_backup::cli`. -use clap::Parser; +use clap::{CommandFactory, FromArgMatches}; fn main() { // Note: `env_logger` is initialized inside `run()` once we've parsed // the global flags, so the user-supplied --log-level takes effect. - let cli = rusty_backup::cli::Cli::parse(); + // + // Parsed through `ArgMatches` rather than `Cli::parse()` so the selected + // `--format` can be recorded before dispatch; the error arm below needs it + // and the parsed `Cli` no longer carries it centrally (R-005). + let mut matches = rusty_backup::cli::Cli::command().get_matches(); + rusty_backup::cli::output::record_active_format( + rusty_backup::cli::output::format_from_matches(&matches), + ); + let cli = match rusty_backup::cli::Cli::from_arg_matches_mut(&mut matches) { + Ok(cli) => cli, + Err(e) => e.exit(), + }; + let code = match rusty_backup::cli::run(cli) { Ok(()) => rusty_backup::cli::exit::SUCCESS, Err(e) => { - // Best-effort plain-text error. Verbs that need to surface - // structured errors do so before bubbling here. + // A caller who asked for JSON/YAML gets the failure in that shape on + // stdout, as src/cli/output.rs has always documented. The plain-text + // line still goes to stderr, which is the human channel. + rusty_backup::cli::output::emit_error_envelope_for(&e); eprintln!("error: {e:#}"); - rusty_backup::cli::exit::GENERIC_FAILURE + // Handlers that classified their failure keep that classification; + // everything else is a generic failure as before. + rusty_backup::cli::exit::code_for(&e) } }; std::process::exit(code); diff --git a/src/cli/backup_edit.rs b/src/cli/backup_edit.rs index cec0dd96..cb7626d3 100644 --- a/src/cli/backup_edit.rs +++ b/src/cli/backup_edit.rs @@ -332,6 +332,10 @@ fn partition_context(part: &PartitionMetadata, size: u64) -> PartitionContext { offset: 0, type_byte: part.partition_type_byte, type_string: part.partition_type_string.clone(), + type_name: part + .partition_type_string + .clone() + .unwrap_or_else(|| "raw".to_string()), size, label: format!( "Backup partition {} ({} bytes){}", diff --git a/src/cli/exit.rs b/src/cli/exit.rs index 73300f29..685c5548 100644 --- a/src/cli/exit.rs +++ b/src/cli/exit.rs @@ -29,3 +29,84 @@ pub const USER_DECLINED: i32 = 5; /// SIGINT (Ctrl-C). Shell convention is 128 + signal number. pub const SIGINT: i32 = 130; + +/// An error that names the exit code it should produce. +/// +/// Every handler error used to arrive at `main` as a bare `anyhow::Error` and +/// leave as [`GENERIC_FAILURE`], so a message could say "usage error" while the +/// process said 1 — which is what scripts actually switch on (R-004). Wrapping +/// the message in this type carries the code the whole way out. +#[derive(Debug)] +pub struct CodedError { + pub code: i32, + pub message: String, +} + +impl std::fmt::Display for CodedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for CodedError {} + +/// A usage rejection: bad input a handler recognises as the caller's mistake. +pub fn usage(message: impl Into) -> anyhow::Error { + anyhow::Error::new(CodedError { + code: USAGE_ERROR, + message: message.into(), + }) +} + +/// A named thing does not exist: an image file, a partition index, a path +/// inside a filesystem. +pub fn not_found(message: impl Into) -> anyhow::Error { + anyhow::Error::new(CodedError { + code: NOT_FOUND, + message: message.into(), + }) +} + +/// A refusal for lack of permission, including writing to a read-only target. +pub fn permission_denied(message: impl Into) -> anyhow::Error { + anyhow::Error::new(CodedError { + code: PERMISSION_DENIED, + message: message.into(), + }) +} + +/// The exit code an error asks for, or [`GENERIC_FAILURE`]. +/// +/// Walks the whole `anyhow` chain, so adding `.context(..)` to a coded error +/// does not silently downgrade it back to 1. +pub fn code_for(err: &anyhow::Error) -> i32 { + err.chain() + .find_map(|e| e.downcast_ref::()) + .map(|c| c.code) + .unwrap_or(GENERIC_FAILURE) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_plain_error_is_a_generic_failure() { + assert_eq!(code_for(&anyhow::anyhow!("boom")), GENERIC_FAILURE); + } + + #[test] + fn a_usage_error_keeps_its_code_under_context() { + let e = usage("bad flag"); + assert_eq!(code_for(&e), USAGE_ERROR); + // The chain walk is the point: context is added all over the CLI. + let wrapped = e.context("while doing the thing"); + assert_eq!(code_for(&wrapped), USAGE_ERROR); + assert!(format!("{wrapped:#}").contains("bad flag")); + } + + #[test] + fn permission_denied_is_four() { + assert_eq!(code_for(&permission_denied("read-only")), PERMISSION_DENIED); + } +} diff --git a/src/cli/output.rs b/src/cli/output.rs index dc6b9860..2b6eb466 100644 --- a/src/cli/output.rs +++ b/src/cli/output.rs @@ -43,6 +43,7 @@ use anyhow::Result; use serde::Serialize; use std::fmt; +use std::sync::atomic::{AtomicU8, Ordering}; /// Output format selected via `--format`. Default is [`OutputFormat::Text`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] @@ -90,6 +91,83 @@ impl OutputFormat { pub fn is_flat_only(self) -> bool { matches!(self, Self::Csv | Self::Tsv) } + + fn as_u8(self) -> u8 { + match self { + Self::Text => 0, + Self::Json => 1, + Self::Yaml => 2, + Self::Csv => 3, + Self::Tsv => 4, + } + } + + fn from_u8(v: u8) -> Option { + match v { + 0 => Some(Self::Text), + 1 => Some(Self::Json), + 2 => Some(Self::Yaml), + 3 => Some(Self::Csv), + 4 => Some(Self::Tsv), + _ => None, + } + } +} + +/// The `--format` the running verb was given, or `FORMAT_UNSET`. +/// +/// `--format` is a per-verb argument, so the error path in `main` has no other +/// way to learn that the caller asked for JSON (R-005). +static ACTIVE_FORMAT: AtomicU8 = AtomicU8::new(FORMAT_UNSET); + +const FORMAT_UNSET: u8 = u8::MAX; + +/// Record the format the invoked verb parsed. Called once from the `rb-cli` +/// entry point, before dispatch. +pub fn record_active_format(format: Option) { + ACTIVE_FORMAT.store( + format.map_or(FORMAT_UNSET, OutputFormat::as_u8), + Ordering::Relaxed, + ); +} + +/// The recorded format, if the invoked verb has a `--format` at all. +pub fn active_format() -> Option { + OutputFormat::from_u8(ACTIVE_FORMAT.load(Ordering::Relaxed)) +} + +/// Dig the `--format` value out of parsed clap matches, walking to the deepest +/// subcommand so `show partmap --format json` finds the inner verb's flag. +/// +/// Reading clap's own matches rather than rescanning `argv` means a file named +/// `--format` can't be mistaken for the flag, and a verb added later is covered +/// without touching this function. +pub fn format_from_matches(matches: &clap::ArgMatches) -> Option { + let mut level = matches; + let mut found = None; + loop { + if let Ok(Some(f)) = level.try_get_one::("format") { + found = Some(*f); + } + match level.subcommand() { + Some((_, sub)) => level = sub, + None => return found, + } + } +} + +/// Emit an error envelope for a failure on its way out of `main`, when the +/// caller asked for a structured format. Returns whether anything was written. +pub fn emit_error_envelope_for(err: &anyhow::Error) -> bool { + let Some(format) = active_format() else { + return false; + }; + if !format.is_structured() { + return false; + } + let env: Envelope<()> = + Envelope::error(crate::cli::exit::code_for(err), format!("{err:#}"), None); + emit_envelope(format, &env).is_ok() } /// Top-level envelope for JSON/YAML payloads. Verbs construct one of @@ -172,6 +250,24 @@ pub fn emit_envelope(format: OutputFormat, env: &Envelope) -> R } } +/// Emit flat rows as CSV or TSV, header included. Shared by every verb in the +/// flat-tabular scope (`ls`, `show partmap`, `show devices`, `fsck` issues). +pub fn emit_csv_or_tsv(format: OutputFormat, rows: &[T]) -> Result<()> { + let delim = if format == OutputFormat::Tsv { + b'\t' + } else { + b',' + }; + let mut wtr = csv::WriterBuilder::new() + .delimiter(delim) + .from_writer(std::io::stdout().lock()); + for row in rows { + wtr.serialize(row)?; + } + wtr.flush()?; + Ok(()) +} + /// Reject `--format csv|tsv` for nested-result verbs. Verbs whose result /// shape doesn't flatten into rows call this at the top of their /// dispatcher; on error returns [`crate::cli::exit::USAGE_ERROR`] via @@ -183,10 +279,12 @@ pub fn require_non_flat(format: OutputFormat, verb_name: &str) -> Result<()> { let suggestion = "Use --format json or --format yaml instead."; #[cfg(not(feature = "yaml"))] let suggestion = "Use --format json instead."; - anyhow::bail!( + // A usage rejection, and it must exit 2 — this file has documented that + // since it was written, while `anyhow::bail!` sent 1 (R-004). + return Err(crate::cli::exit::usage(format!( "{verb_name} returns nested data; --format {format} only supports flat tabular \ results. {suggestion}" - ); + ))); } Ok(()) } @@ -236,8 +334,16 @@ mod tests { #[test] fn require_non_flat_rejects_csv_for_nested() { - assert!(require_non_flat(OutputFormat::Csv, "inspect").is_err()); - assert!(require_non_flat(OutputFormat::Tsv, "inspect").is_err()); + // Asserting only `is_err()` is what let R-004 sit here unnoticed: the + // message said usage error while the process exited 1. + for f in [OutputFormat::Csv, OutputFormat::Tsv] { + let e = require_non_flat(f, "inspect").expect_err("must reject"); + assert_eq!( + crate::cli::exit::code_for(&e), + crate::cli::exit::USAGE_ERROR, + "{f} rejection must exit 2" + ); + } assert!(require_non_flat(OutputFormat::Json, "inspect").is_ok()); assert!(require_non_flat(OutputFormat::Text, "inspect").is_ok()); } diff --git a/src/cli/resolve.rs b/src/cli/resolve.rs index 24af75fa..ec9e7d98 100644 --- a/src/cli/resolve.rs +++ b/src/cli/resolve.rs @@ -41,6 +41,12 @@ pub struct PartitionContext { /// APM / RDB type string (e.g. `"Apple_HFS"`, `"PFS\\3"`). `None` /// for MBR / GPT / raw superfloppy. pub type_string: Option, + /// How the read path names this filesystem, e.g. `"Alto BFS"`. Carried so a + /// refusal can say what the volume is: content probing on the write path + /// returns "unknown" for filesystems identified by their container, which + /// told the user the disk was unreadable a moment after `ls` read it + /// (R-034). + pub type_name: String, /// Partition size in bytes. For raw superfloppies, the image's /// total length. pub size: u64, @@ -77,6 +83,16 @@ impl PartitionContext { Box, crate::fs::filesystem::FilesystemError, > { + // Rewrite only the "we could not name it" case; every other + // Unsupported message is specific and better than anything here. + let name_it = |e: crate::fs::filesystem::FilesystemError| match &e { + crate::fs::filesystem::FilesystemError::Unsupported(m) if m.contains("'unknown'") => { + crate::fs::filesystem::FilesystemError::Unsupported( + m.replace("'unknown'", &format!("'{}'", self.type_name)), + ) + } + _ => e, + }; crate::fs::open_editable_filesystem_with( handle, self.offset, @@ -90,9 +106,39 @@ impl PartitionContext { self.type_byte, self.type_string.as_deref(), ) + .map_err(name_it) } } +/// Fail with `NOT_FOUND` when the source simply is not there. +/// +/// `exit.rs` reserves 3 for "image file missing" and `inspect` has honoured it +/// since R-010 — with a per-verb `exists()` check that never spread. Every +/// other verb surfaced the raw `io::Error` as the catch-all 1, so a script +/// could not tell "no disk" from "bad disk", and the message was a +/// platform-specific syscall error (R-036). Living here means a verb added +/// later inherits it. +/// +/// Three things are deliberately exempt, because none of them is a file whose +/// absence we can judge: a raw device (`\\.\PhysicalDrive0` does not +/// `exists()`), an `rb://` remote reference, and anything the caller has +/// already peeled to a temp. +pub fn require_source_exists(path: &std::path::Path) -> Result<()> { + if path.exists() { + return Ok(()); + } + if crate::cli::device_safety::looks_like_device_path(path) { + return Ok(()); + } + if path.to_string_lossy().starts_with("rb://") { + return Ok(()); + } + Err(crate::cli::exit::not_found(format!( + "{}: no such file", + path.display() + ))) +} + /// Open `path` read-only and resolve which partition to use. /// /// - When `selector` is `None` and the image has no partition table, @@ -107,6 +153,7 @@ pub fn resolve_partition_ro( path: &std::path::Path, selector: Option, ) -> Result<(File, PartitionContext)> { + require_source_exists(path)?; let mut file = open_image_ro(path)?; let ctx = resolve(&mut file, selector)?; Ok((file, ctx)) @@ -268,6 +315,7 @@ pub fn resolve_partition_rw_forced( // there via the backup-folder path, and repack over the original `.cbk` on // commit (the "additional legwork" edit path — cb_dos_network_and_state.md // §2e). Read access to a `.cbk` is native (source_reader); editing repacks. + require_source_exists(path)?; if crate::rbformats::cbk::is_cbk(path) { let temp = tempfile::Builder::new() .prefix(".rb-cbk-edit-") @@ -406,6 +454,32 @@ pub fn resolve_image_rw(path: &std::path::Path) -> Result<(BoxRwSeek, RwCommit, } return Ok((Box::new(reader), RwCommit::None, HandleShape::Wrapped)); } + // A container the read path decodes but the write path cannot re-encode: + // G64/G71 raw GCR, MSA, EDSK, Apple-II .dsk. Refusing here says so; falling + // through opened the undecoded container bytes and reported "Invalid MBR: + // invalid boot signature", which reads as a corrupt disk a moment after + // `ls` listed its files (R-011, same shape as R-034). + // Named explicitly rather than derived as "flat-floppy minus editable": + // that set also caught Apple-II `.dsk`, which is a plain sector image the + // write path handles fine, and refusing it broke `new floppy apple-dos` + // round-trips. These three re-encode their bytes on read — GCR bitstream, + // MSA run-length, EDSK track records — so there is nowhere for a write to go. + if (source_reader::is_g64_path(path) + || source_reader::is_msa_path(path) + || source_reader::is_edsk_path(path)) + && !source_reader::is_editable_container_path(path) + { + // Name a raw target, not a specific vintage extension: this branch + // covers G64/G71, MSA, EDSK and Apple-II .dsk, and suggesting `.d64` + // for an Apple II disk is worse than suggesting nothing. + return Err(crate::cli::exit::permission_denied(format!( + "{}: this container is read-only — it decodes for reading but cannot be \ + re-encoded, so edits would have nowhere to go. Convert it to a raw image \ + first: `rb-cli convert {} OUT.img --format raw`.", + path.display(), + path.display(), + ))); + } if source_reader::is_editable_container_path(path) { // Floppy / gzip / WOZ container: decode to a temp flat, edit that, // re-encode on commit. open_image_rw on the temp gives the same File @@ -528,6 +602,7 @@ pub fn resolve_partition_streaming_forced_inside( // A backup folder stores each partition as a compressed file governed by // metadata.json; decompress the selected one to a temp flat (read-only) so // get / ls / inspect see it like any other raw partition. + require_source_exists(path)?; if backup_edit::is_backup_folder(path) { return backup_edit::open_backup_partition_ro(path, selector); } @@ -589,6 +664,7 @@ fn resolve_with_override( offset: 0, type_byte: 0x00, type_string: None, + type_name: "raw".to_string(), size: total, label: "Partition: raw filesystem @ byte 0 (forced via --fs-type)".to_string(), // `resolve` works from a reader and doesn't know whether it @@ -615,6 +691,7 @@ fn resolve_with_override( offset: 0, type_byte: 0x00, type_string: None, + type_name: pt.type_name().to_string(), size: total, label: format!("Partition: raw filesystem @ byte 0 ({})", pt.type_name()), whole_file_path: None, @@ -631,6 +708,7 @@ fn resolve_with_override( offset: info.byte_offset(), type_byte: info.partition_type_byte, type_string: info.partition_type_string.clone(), + type_name: info.type_name.clone(), size: info.size_bytes, label: format_label(&pt, &info, &partitions), whole_file_path: None, @@ -857,6 +935,23 @@ impl FsDispatchOverride { } } +/// Classify a failure to open a filesystem for writing. +/// +/// `Unsupported` from a write-open means the volume is readable and this build +/// will not write it — "a read-only filesystem on a write path", which is +/// exactly what `exit.rs` reserves PERMISSION_DENIED for. It used to exit 1, +/// indistinguishable from a genuine I/O failure (R-034). The caller's wording +/// is preserved so each verb keeps its own phrasing. +pub fn write_open_error(context: &str, e: crate::fs::filesystem::FilesystemError) -> anyhow::Error { + let msg = format!("{context}: {e}"); + match e { + crate::fs::filesystem::FilesystemError::Unsupported(_) => { + crate::cli::exit::permission_denied(msg) + } + _ => anyhow!(msg), + } +} + #[cfg(test)] mod tests { use super::*; @@ -953,3 +1048,46 @@ mod tests { assert_eq!(picked.type_name, "Apple_HFS (Untitled)"); } } + +#[cfg(test)] +mod source_exists_tests { + use super::*; + use std::path::Path; + + #[test] + fn a_missing_plain_file_is_not_found() { + let e = require_source_exists(Path::new("definitely-not-here-9e3f.img")) + .expect_err("a missing image must be refused"); + assert_eq!(crate::cli::exit::code_for(&e), crate::cli::exit::NOT_FOUND); + assert!(format!("{e:#}").contains("no such file")); + } + + /// The exemption that matters: backing up a raw disk is the app's job, and + /// a device node does not `exists()` as a file. Getting this wrong would + /// refuse every device before it was ever opened. + #[test] + fn device_paths_are_exempt() { + for p in [ + r"\\.\PhysicalDrive0", + r"\\?\PhysicalDrive1", + "/dev/sda", + "/dev/disk3", + ] { + assert!( + require_source_exists(Path::new(p)).is_ok(), + "{p} must pass the guard and fail (or not) at open time instead" + ); + } + } + + #[test] + fn remote_refs_are_exempt() { + assert!(require_source_exists(Path::new("rb://host:9000/disk.img")).is_ok()); + } + + #[test] + fn an_existing_path_passes() { + // A directory counts: a backup folder is a legitimate source. + assert!(require_source_exists(Path::new(env!("CARGO_MANIFEST_DIR"))).is_ok()); + } +} diff --git a/src/cli/verbs/batch.rs b/src/cli/verbs/batch.rs index fe87ef34..f4f4f8e0 100644 --- a/src/cli/verbs/batch.rs +++ b/src/cli/verbs/batch.rs @@ -457,7 +457,7 @@ fn run_fs_ops_on_path( log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let mut applied = fs_ops_start; let mut failures: Vec<(usize, String)> = Vec::new(); diff --git a/src/cli/verbs/binhex.rs b/src/cli/verbs/binhex.rs index 593bc84b..e2a0320d 100644 --- a/src/cli/verbs/binhex.rs +++ b/src/cli/verbs/binhex.rs @@ -93,7 +93,7 @@ pub fn run_put(args: PutBinHexArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; // `--dst-dir` is the destination *directory* (the filename comes from the // BinHex header), resolved with the shared escape / colon grammar so a diff --git a/src/cli/verbs/bless.rs b/src/cli/verbs/bless.rs index 7d7acb68..544d1548 100644 --- a/src/cli/verbs/bless.rs +++ b/src/cli/verbs/bless.rs @@ -56,7 +56,7 @@ pub fn apply_bless(image: &ImageRef, path: &str) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let entry = super::ls::resolve_path(fs.as_filesystem_mut(), path)?; if !entry.is_directory() { diff --git a/src/cli/verbs/chmeta.rs b/src/cli/verbs/chmeta.rs index 06578fca..d28fd960 100644 --- a/src/cli/verbs/chmeta.rs +++ b/src/cli/verbs/chmeta.rs @@ -151,7 +151,7 @@ pub fn run(args: ChmetaArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let entry = super::ls::resolve_path(fs.as_filesystem_mut(), &args.path)?; diff --git a/src/cli/verbs/chmod.rs b/src/cli/verbs/chmod.rs index f115e0e4..2ff3b925 100644 --- a/src/cli/verbs/chmod.rs +++ b/src/cli/verbs/chmod.rs @@ -88,7 +88,7 @@ pub fn run_chmod(args: ChmodArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let entry = super::ls::resolve_path(fs.as_filesystem_mut(), &args.path)?; fs.set_permissions(&entry, mode) @@ -119,7 +119,7 @@ pub fn run_chown(args: ChownArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let entry = super::ls::resolve_path(fs.as_filesystem_mut(), &args.path)?; let (uid, gid) = parse_owner(&args.owner, entry.uid.unwrap_or(0), entry.gid.unwrap_or(0))?; diff --git a/src/cli/verbs/edit.rs b/src/cli/verbs/edit.rs index a96b8f3b..156e1949 100644 --- a/src/cli/verbs/edit.rs +++ b/src/cli/verbs/edit.rs @@ -104,7 +104,7 @@ pub fn run(args: EditArgs) -> Result<()> { args.fs_override.apply(&mut ctx); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let fs_type = fs.fs_type().to_string(); let (parent, name) = super::ls::resolve_parent(fs.as_filesystem_mut(), &args.path)?; diff --git a/src/cli/verbs/inspect.rs b/src/cli/verbs/inspect.rs index db8812fa..b02f4765 100644 --- a/src/cli/verbs/inspect.rs +++ b/src/cli/verbs/inspect.rs @@ -41,6 +41,26 @@ pub struct InspectArgs { /// insensitively, then by basename. Ignored for non-zip sources. #[arg(long = "inside", value_name = "NAME")] pub inside: Option, + + /// `--fs-type` / `--carve-full`, matching `ls`, `fsck` and `du`. A CP/M + /// disk has no on-disk signature, so without this `inspect` could not + /// report one at all (R-010). + #[command(flatten)] + pub fs_override: crate::cli::resolve::FsDispatchOverride, + + /// Assert the disk carries this filesystem, e.g. `--expect-fs "DOS 3.3"`. + /// Exits non-zero when no partition matches. Case, spacing and punctuation + /// are ignored; the comparison is exact after that, so `FAT` does not + /// satisfy `exFAT`. + #[arg(long = "expect-fs", value_name = "NAME")] + pub expect_fs: Option, + + /// Assert the disk's shape: `superfloppy` (a filesystem at sector 0, no + /// table), `partitioned` (any table), or a scheme by name — `mbr`, `gpt`, + /// `apm`, `rdb`, `sgi`, `sun`, `ahdi`, `x68k`, `dsd`, `none`. An + /// unrecognised word is a usage error, not a failed assertion. + #[arg(long = "expect-layout", value_name = "KIND")] + pub expect_layout: Option, } pub fn run(args: InspectArgs) -> Result<()> { @@ -62,23 +82,59 @@ pub fn run(args: InspectArgs) -> Result<()> { // VHD / 2MG / DMG / DiskCopy 4.2) so inspect sees the same flat disk the // browse path does; the plain-open path did not unwrap DMG/VHD/2MG and // mis-read the wrapped bytes as the partition table. + // A missing image is NOT_FOUND. The check used to live here and nowhere + // else, which is what R-036 was; it is now in resolve::require_source_exists. + crate::cli::resolve::require_source_exists(&args.image)?; let mut reader = crate::model::source_reader::open_peeled_read_with_entry( &args.image, pw_bytes, args.inside.as_deref(), )?; - let pt = PartitionTable::detect(&mut reader).map_err(|e| { - // An optical `.iso` (incl. NKit-scrubbed GC/Wii) has no MBR/GPT, so - // detection fails with a cryptic "invalid boot signature". Point the user - // at the `optical` verbs, or give NKit images the convert-it-first hint. - let base = anyhow::anyhow!("detecting partition table: {e}"); - if crate::cli::optical_hint::is_nkit_image(&args.image) { - crate::cli::optical_hint::with_nkit_hint(base, &args.image) - } else { - crate::cli::optical_hint::with_optical_hint(base, &args.image) + // A signature-less filesystem has no partition table either, so detection + // fails before the forced type could be applied. `ls` already treats + // --fs-type as "raw filesystem at byte 0"; do the same here rather than + // accept the flag and still refuse the disk (R-010). + let forced = args.fs_override.fs_type.clone(); + let pt = if let Some(ref t) = forced { + let size = reader.seek(std::io::SeekFrom::End(0)).unwrap_or(0); + reader.seek(std::io::SeekFrom::Start(0)).ok(); + match PartitionTable::detect(&mut reader) { + Ok(pt) => pt, + Err(_) => PartitionTable::None { + size_bytes: size, + fs_hint: t.clone(), + }, } - })?; - let partitions = pt.partitions(); + } else { + PartitionTable::detect(&mut reader).map_err(|e| { + // An optical `.iso` (incl. NKit-scrubbed GC/Wii) has no MBR/GPT, so + // detection fails with a cryptic "invalid boot signature". Point the user + // at the `optical` verbs, or give NKit images the convert-it-first hint. + let base = anyhow::anyhow!("detecting partition table: {e}"); + if crate::cli::optical_hint::is_nkit_image(&args.image) { + crate::cli::optical_hint::with_nkit_hint(base, &args.image) + } else { + crate::cli::optical_hint::with_optical_hint(base, &args.image) + } + })? + }; + let mut partitions = pt.partitions(); + // Forced dispatch: a signature-less filesystem cannot be detected, so the + // user naming it is the only way inspect can report it. Applied only where + // the table declared nothing, so a real type string is never overwritten. + if let Some(forced) = args.fs_override.fs_type.as_deref() { + for p in &mut partitions { + if p.partition_type_string.is_none() { + p.partition_type_string = Some(forced.to_string()); + // Fall back to the string the user gave: `cpm:amstrad_data` + // names the disk far better than "unknown" does. + p.type_name = match crate::fs::fs_name_for(p.partition_type_byte, Some(forced)) { + "unknown" => forced.to_string(), + n => n.to_string(), + }; + } + } + } let ext = args .image .extension() @@ -107,7 +163,62 @@ pub fn run(args: InspectArgs) -> Result<()> { extra_report.as_deref(), ), _ => unreachable!(), + }?; + // Assertions run last, so the report is on stdout either way: a failing + // check should show what it found, not just that it failed. + check_expectations(&args, &pt, &partitions) +} + +/// Apply `--expect-layout` / `--expect-fs`. +/// +/// `inspect` opens anything — that is what a universal tool is for, and a disk +/// with no recognisable filesystem still yields a carve view. The cost is that +/// a clean exit means "readable", not "identified", so "`inspect` opened it" +/// was accepted as verification for a fixture that had no filesystem at all +/// (R-031). These flags let a caller ask the stronger question. +fn check_expectations( + args: &InspectArgs, + pt: &PartitionTable, + partitions: &[crate::partition::PartitionInfo], +) -> Result<()> { + if let Some(want) = args.expect_layout.as_deref() { + // A typo must not read as "the disk is the wrong shape". Reject the + // word first, with the vocabulary, and exit 2 like any bad argument. + if !crate::partition::is_known_layout(want) { + return Err(crate::cli::exit::usage(format!( + "--expect-layout {want:?} is not a layout. Valid: {}.", + crate::partition::layout_vocabulary().join(", ") + ))); + } + if !pt.layout_matches(want) { + anyhow::bail!( + "--expect-layout {want:?}: this disk is {}{}.", + pt.type_name(), + if pt.is_partitioned() { + format!(" ({} partition(s))", partitions.len()) + } else { + " (superfloppy: a filesystem at sector 0, no partition table)".to_string() + } + ); + } } + if let Some(want) = args.expect_fs.as_deref() { + if !partitions + .iter() + .any(|p| crate::fs::fs_name_matches(&p.type_name, want)) + { + let found: Vec<&str> = partitions.iter().map(|p| p.type_name.as_str()).collect(); + anyhow::bail!( + "--expect-fs {want:?}: no partition carries that filesystem. Found: {}.", + if found.is_empty() { + "nothing".to_string() + } else { + found.join(", ") + } + ); + } + } + Ok(()) } fn emit_text( @@ -256,6 +367,7 @@ fn emit_structured( .map(|(pos, p)| PartitionRow { index: pos + 1, type_name: p.type_name.clone(), + identified: crate::fs::is_identified_fs(&p.type_name), partition_type_byte: p.partition_type_byte, partition_type_string: p.partition_type_string.clone(), start_lba: p.start_lba, @@ -292,6 +404,10 @@ struct PartitionRow { /// slot — see the note in `emit_text`. index: usize, type_name: String, + /// Whether `type_name` is a filesystem we recognised, or the engine saying + /// it did not. `inspect` opens anything, so a clean exit alone does not + /// mean a filesystem was found (R-031). + identified: bool, partition_type_byte: u8, #[serde(skip_serializing_if = "Option::is_none")] partition_type_string: Option, diff --git a/src/cli/verbs/ls.rs b/src/cli/verbs/ls.rs index 979dad9f..ec2e8ecf 100644 --- a/src/cli/verbs/ls.rs +++ b/src/cli/verbs/ls.rs @@ -7,10 +7,12 @@ use anyhow::{anyhow, bail, Result}; use clap::Args; +use serde::Serialize; use crate::cli::glob::{collect_matches, compile_patterns}; use crate::cli::img_at::ImageRef; use crate::cli::logging::{log_stderr, out_stdout}; +use crate::cli::output::{emit_csv_or_tsv, emit_envelope, Envelope, OutputFormat}; use crate::cli::resolve::{resolve_partition_streaming_forced_inside, FsDispatchOverride}; use crate::fs::filesystem::Filesystem; @@ -74,16 +76,71 @@ pub struct LsArgs { #[arg(long = "inside", value_name = "NAME")] pub inside: Option, + /// Output format. `ls` is flat-tabular, so csv and tsv are in scope + /// alongside json and yaml. + #[arg(long, value_enum, default_value_t = OutputFormat::Text, global = false)] + pub format: OutputFormat, + #[command(flatten)] pub fs_override: FsDispatchOverride, } +/// One listing row. Every field is always present so the CSV header is stable +/// across volumes; a filesystem that carries none of a column leaves it null. +#[derive(Debug, Serialize)] +struct LsRow { + kind: &'static str, + name: String, + size: u64, + type_code: Option, + creator_code: Option, + mode: Option, + uid: Option, + gid: Option, + owner: Option, +} + +impl LsRow { + fn from_entry( + entry: &crate::fs::entry::FileEntry, + display_name: &str, + id_names: Option<&crate::fs::id_names::IdNameMap>, + ) -> Self { + Self { + kind: if entry.is_directory() { "DIR" } else { "FILE" }, + name: display_name.to_string(), + size: entry.size, + type_code: entry.type_code_display(), + creator_code: entry.creator_code_display(), + mode: entry.mode_string(), + uid: entry.uid, + gid: entry.gid, + // Resolving ids to names needs the image's own passwd/group, which + // is only read under `--owner`. + owner: match (id_names, entry.uid, entry.gid) { + (Some(names), Some(u), Some(g)) => Some(names.format_owner(u, g)), + _ => None, + }, + } + } +} + +/// Emit collected rows in a structured format. Text never reaches here — it +/// stays on [`print_entry`] so its column layout is untouched. +fn emit_rows(format: OutputFormat, rows: &[LsRow]) -> Result<()> { + match format { + OutputFormat::Json | OutputFormat::Yaml => emit_envelope(format, &Envelope::ok(rows)), + OutputFormat::Csv | OutputFormat::Tsv => emit_csv_or_tsv(format, rows), + OutputFormat::Text => unreachable!("text is printed as it goes"), + } +} + pub fn run(args: LsArgs) -> Result<()> { // Remote source: `rb-cli ls rb://host:port/img@N /path`. The daemon parses // the filesystem and returns the listing; we never pull raw blocks. #[cfg(feature = "remote")] if let Some(rref) = crate::remote::RemoteRef::parse(&args.image.path.to_string_lossy()) { - return remote_ls(&rref, args.image.partition, &args.path); + return remote_ls(&rref, args.image.partition, &args.path, args.format); } let pw_bytes = args.password.as_deref().map(|s| s.as_bytes()); @@ -156,10 +213,17 @@ pub fn run(args: LsArgs) -> Result<()> { excludes.extend(compile_patterns(ex, case_insensitive)?); } let matches = collect_matches(&mut *fs, &includes, &excludes)?; - for (_, entry, full) in matches { - print_entry(&entry, &full, args.owner.then_some(&id_names)); + if args.format == OutputFormat::Text { + for (_, entry, full) in matches { + print_entry(&entry, &full, args.owner.then_some(&id_names)); + } + return Ok(()); } - return Ok(()); + let rows: Vec = matches + .iter() + .map(|(_, entry, full)| LsRow::from_entry(entry, full, args.owner.then_some(&id_names))) + .collect(); + return emit_rows(args.format, &rows); } // Literal path: directory listing. @@ -170,10 +234,17 @@ pub fn run(args: LsArgs) -> Result<()> { let children = fs .list_directory(&entry) .map_err(|e| anyhow!("list_directory: {e}"))?; - for c in children { - print_entry(&c, &c.name, args.owner.then_some(&id_names)); + if args.format == OutputFormat::Text { + for c in children { + print_entry(&c, &c.name, args.owner.then_some(&id_names)); + } + return Ok(()); } - Ok(()) + let rows: Vec = children + .iter() + .map(|c| LsRow::from_entry(c, &c.name, args.owner.then_some(&id_names))) + .collect(); + emit_rows(args.format, &rows) } /// Remote directory listing over an `rb://` reference. Lists either the @@ -185,6 +256,7 @@ fn remote_ls( rref: &crate::remote::RemoteRef, partition: Option, path: &str, + format: OutputFormat, ) -> Result<()> { if has_glob_chars(path) { bail!("glob patterns aren't supported over rb:// yet (literal paths only)"); @@ -199,19 +271,52 @@ fn remote_ls( bail!("no such path on the remote: {}", rref.path); } if is_dir { - for entry in session.list_host_dir(&rref.path)? { - print_wire_entry(&entry); - } - return Ok(()); + return emit_wire_entries(format, &session.list_host_dir(&rref.path)?); } } let opened = session.open_image(&rref.path, partition)?; log_stderr(opened.label); - for entry in session.list_dir(opened.handle, path)? { - print_wire_entry(&entry); + let entries = session.list_dir(opened.handle, path)?; + emit_wire_entries(format, &entries) +} + +/// Emit a remote listing in the requested format, matching the local shapes. +#[cfg(feature = "remote")] +fn emit_wire_entries( + format: OutputFormat, + entries: &[crate::remote::protocol::WireEntry], +) -> Result<()> { + if format == OutputFormat::Text { + for entry in entries { + print_wire_entry(entry); + } + return Ok(()); + } + let rows: Vec = entries.iter().map(wire_row).collect(); + emit_rows(format, &rows) +} + +#[cfg(feature = "remote")] +fn wire_row(entry: &crate::remote::protocol::WireEntry) -> LsRow { + LsRow { + kind: if entry.is_dir() { "DIR" } else { "FILE" }, + name: entry.name.clone(), + size: entry.size, + type_code: crate::fs::entry::display_file_type( + entry.type_code.as_ref(), + entry.prodos_file_type, + ), + creator_code: entry + .creator_code + .as_ref() + .map(crate::fs::hfs_common::decode_ostype), + // The wire protocol carries no mode / ownership today. + mode: None, + uid: None, + gid: None, + owner: None, } - Ok(()) } /// Mirror of [`print_entry`] for a [`crate::remote::protocol::WireEntry`]. diff --git a/src/cli/verbs/menu.rs b/src/cli/verbs/menu.rs index abfe7a14..56d75769 100644 --- a/src/cli/verbs/menu.rs +++ b/src/cli/verbs/menu.rs @@ -299,6 +299,11 @@ fn run_action(disk: &DiskDevice, action: Action) -> Result<()> { format: OutputFormat::Text, password: None, inside: None, + fs_override: Default::default(), + // The appliance screen reports what it finds; asserting is for + // scripted callers. + expect_fs: None, + expect_layout: None, }); report(r, "inspect"); } diff --git a/src/cli/verbs/mkdir.rs b/src/cli/verbs/mkdir.rs index 8b957935..5d52ce40 100644 --- a/src/cli/verbs/mkdir.rs +++ b/src/cli/verbs/mkdir.rs @@ -49,7 +49,7 @@ pub fn run(args: MkdirArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; // Resolve parent + leaf with the shared escape / colon grammar so a new // directory whose name contains a literal `/` can be created. diff --git a/src/cli/verbs/new.rs b/src/cli/verbs/new.rs index 195e459b..33b9f493 100644 --- a/src/cli/verbs/new.rs +++ b/src/cli/verbs/new.rs @@ -20,6 +20,14 @@ use anyhow::{Context, Result}; use clap::{Args, Subcommand, ValueEnum}; use std::path::PathBuf; +/// The `--name` default every `new` subcommand shares. +pub const DEFAULT_VOLUME_NAME: &str = "rusty-backup"; + +/// ProDOS allows only letters, digits and `.`, so it cannot take the shared +/// default and `new volume prodos` always failed with no arguments (R-006). +/// Kept recognisably the same name rather than something generic. +pub const PRODOS_DEFAULT_VOLUME_NAME: &str = "RUSTY.BACKUP"; + use crate::cli::logging::log_stderr; use crate::cli::parse::parse_size; use crate::fs::ntfs_format::{create_ntfs, NtfsFormatParams, NtfsGeometry}; @@ -347,7 +355,7 @@ pub struct FloppyArgs { pub size: String, /// Volume label/name. Defaults to `rusty-backup`. - #[arg(long, default_value = "rusty-backup")] + #[arg(long, default_value = DEFAULT_VOLUME_NAME)] pub name: String, /// HFS allocation block size in bytes (multiple of 512). Auto when unset. @@ -413,7 +421,7 @@ pub struct VolumeArgs { pub size: String, /// Volume label/name. Defaults to `rusty-backup`. - #[arg(long, default_value = "rusty-backup")] + #[arg(long, default_value = DEFAULT_VOLUME_NAME)] pub name: String, /// HFS/HFS+ allocation block size in bytes (multiple of 512). Auto when unset. @@ -550,7 +558,7 @@ pub struct NewArgs { /// Volume label/name. Defaults to `rusty-backup`. HFS: up to 27 Mac /// Roman bytes. FAT: up to 11 chars (uppercased; non-ASCII → `_`). /// EFS: 6-byte fname/fpack. AFFS: up to 30 bytes. - #[arg(long, default_value = "rusty-backup")] + #[arg(long, default_value = DEFAULT_VOLUME_NAME)] pub name: String, /// HFS allocation block size in bytes. Must be a non-zero multiple of @@ -763,9 +771,19 @@ fn format_image(args: NewArgs) -> Result<()> { FsKind::Ext => write_blank_ext_image(&args.image, &args.size, &args.name, "ext2"), FsKind::Ext3 => write_blank_ext_image(&args.image, &args.size, &args.name, "ext3"), FsKind::Ext4 => write_blank_ext_image(&args.image, &args.size, &args.name, "ext4"), - FsKind::Prodos => format_and_write(&args.image, &args.size, &args.name, |size, name| { - crate::fs::prodos::create_blank_prodos(size, name) - }), + // ProDOS forbids '-', so the shared default is invalid there and the + // verb always failed with no arguments (R-006). Only the untouched + // default is substituted — an explicit --name still gets a real error. + FsKind::Prodos => { + let name = if args.name == DEFAULT_VOLUME_NAME { + PRODOS_DEFAULT_VOLUME_NAME + } else { + &args.name + }; + format_and_write(&args.image, &args.size, name, |size, name| { + crate::fs::prodos::create_blank_prodos(size, name) + }) + } FsKind::Atari => format_and_write(&args.image, &args.size, &args.name, |_size, _name| { Ok(crate::fs::atari_dos::create_blank_atari_sd()) }), diff --git a/src/cli/verbs/optical.rs b/src/cli/verbs/optical.rs index 4a517c37..12c65465 100644 --- a/src/cli/verbs/optical.rs +++ b/src/cli/verbs/optical.rs @@ -600,6 +600,7 @@ pub enum FilesystemSelect { fn open_selected_filesystem( info: &opticaldiscs::detect::DiscImageInfo, select: FilesystemSelect, + index: Option, ) -> Result<( Box, opticaldiscs::FilesystemType, @@ -620,6 +621,22 @@ fn open_selected_filesystem( }) }; + // An explicit index wins: it is the only way to reach the *second* volume + // of a kind, and --filesystem selects by type, which always finds the first. + if let Some(i) = index { + let n = info.hybrid_filesystems.len(); + if i >= n { + return Err(crate::cli::exit::usage(format!( + "--filesystem-index {i} is out of range: this disc has {n} \ + selectable filesystem(s). `optical info` lists them." + ))); + } + let ty = info.hybrid_filesystems[i].filesystem; + return open_hybrid_filesystem(info, i) + .map(|fs| (fs, ty)) + .map_err(|e| anyhow::anyhow!("opening filesystem #{i}: {e}")); + } + match select { FilesystemSelect::Auto => primary(), FilesystemSelect::Hfs => { @@ -673,6 +690,12 @@ pub struct BrowseArgs { /// forces the ISO 9660 tree. See `optical info` to see what a disc carries. #[arg(long = "filesystem", value_enum, default_value_t = FilesystemSelect::Auto)] pub filesystem: FilesystemSelect, + + /// Which filesystem to open when a disc carries more than one of the same + /// kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the + /// list `optical info` prints. Overrides `--filesystem`. + #[arg(long = "filesystem-index", value_name = "N")] + pub filesystem_index: Option, } fn run_browse_verb(args: BrowseArgs) -> Result<()> { @@ -685,7 +708,8 @@ fn run_browse_verb(args: BrowseArgs) -> Result<()> { let info = crate::optical::open_disc_image(&args.source) .with_context(|| format!("opening {}", args.source.display()))?; - let (mut fs, opened_fs) = open_selected_filesystem(&info, args.filesystem)?; + let (mut fs, opened_fs) = + open_selected_filesystem(&info, args.filesystem, args.filesystem_index)?; let root = fs .root() .map_err(|e| anyhow::anyhow!("reading root: {e}"))?; @@ -782,6 +806,11 @@ fn fs_token(ft: opticaldiscs::FilesystemType) -> String { F::Cdi => "cdi", F::Opera => "opera", F::Xdvdfs => "xdvdfs", + // opticaldiscs 0.15 reports a disc that carries no data track at all — + // a pure CD-DA audio disc — rather than refusing to open it (R-012). + // Distinct from `Unknown`, which means "there is a filesystem here and + // we did not recognise it". + F::None => "none", F::Unknown => "unknown", } .to_string() @@ -976,6 +1005,12 @@ pub struct OpticalDuArgs { /// carrying resource forks. See `optical info` for what a disc holds. #[arg(long = "filesystem", value_enum, default_value_t = FilesystemSelect::Auto)] pub filesystem: FilesystemSelect, + + /// Which filesystem to open when a disc carries more than one of the same + /// kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the + /// list `optical info` prints. Overrides `--filesystem`. + #[arg(long = "filesystem-index", value_name = "N")] + pub filesystem_index: Option, } fn run_du_verb(args: OpticalDuArgs) -> Result<()> { @@ -988,7 +1023,8 @@ fn run_du_verb(args: OpticalDuArgs) -> Result<()> { let info = crate::optical::open_disc_image(&args.source) .with_context(|| format!("opening {}", args.source.display()))?; - let (inner, opened_fs) = open_selected_filesystem(&info, args.filesystem)?; + let (inner, opened_fs) = + open_selected_filesystem(&info, args.filesystem, args.filesystem_index)?; // Wrap the selected opticaldiscs filesystem in our adapter so the shared // `du` engine (both-fork sums + allocation-block rounding via the adapter's @@ -1641,9 +1677,37 @@ pub struct ExtractArgs { /// Optical disc image (.iso, .cue, .chd). pub source: PathBuf, - /// Destination folder (created if absent). - #[arg(long)] - pub to: PathBuf, + /// Destination folder (created if absent). Mutually exclusive with + /// `--tar`; exactly one of the two is required. + #[arg(long, required_unless_present = "tar", conflicts_with = "tar")] + pub to: Option, + + /// Write a `.tar` / `.tar.gz` / `.tar.zst` instead of loose files. + /// + /// The faithful option for archiving: a tar entry carries the POSIX mode, + /// uid and gid a Rock Ridge / HFS+ / EFS disc records, and real symlinks, + /// none of which survive extraction onto a filesystem that has no concept + /// of them. Compression follows the extension. + #[arg(long = "tar", value_name = "FILE")] + pub tar: Option, + + /// Extract only this path from the disc instead of the whole tree. + /// Disc-relative, e.g. `/DOCS/README.TXT` or `/DOCS`. A file extracts on + /// its own; a directory extracts the files directly inside it, and its + /// subdirectories too when `--recursive` is given. + #[arg(long = "path", value_name = "PATH")] + pub path: Option, + + /// Include subdirectories when `--path` names a directory. Whole-disc + /// extraction (no `--path`) always recurses and ignores this. + #[arg(long = "recursive", short = 'r')] + pub recursive: bool, + + /// Apply the POSIX mode a disc records (Rock Ridge, HFS+, EFS) to the + /// extracted files. Unix hosts only — on Windows the bits have nowhere to + /// go, so use `--tar`, which carries them regardless of host. + #[arg(long = "preserve-permissions")] + pub preserve_permissions: bool, /// How to handle HFS resource forks. Ignored on non-HFS discs. /// Defaults to `appledouble`, or `[optical] resource-forks` from @@ -1664,6 +1728,12 @@ pub struct ExtractArgs { /// side; `iso` forces the ISO 9660 tree. See `optical info`. #[arg(long = "filesystem", value_enum, default_value_t = FilesystemSelect::Auto)] pub filesystem: FilesystemSelect, + + /// Which filesystem to open when a disc carries more than one of the same + /// kind — Apple shipped CD-ROMs with two HFS volumes. 0-based, indexing the + /// list `optical info` prints. Overrides `--filesystem`. + #[arg(long = "filesystem-index", value_name = "N")] + pub filesystem_index: Option, } /// How to resolve case-insensitive filename collisions during extraction. @@ -1678,15 +1748,30 @@ pub enum CliCaseCollisionMode { } fn run_extract_verb(args: ExtractArgs) -> Result<()> { - std::fs::create_dir_all(&args.to).with_context(|| format!("creating {}", args.to.display()))?; - let info = crate::optical::open_disc_image(&args.source) .with_context(|| format!("opening {}", args.source.display()))?; - let (mut fs, _opened_fs) = open_selected_filesystem(&info, args.filesystem)?; + let (mut fs, _opened_fs) = + open_selected_filesystem(&info, args.filesystem, args.filesystem_index)?; let root = fs .root() .map_err(|e| anyhow::anyhow!("reading root: {e}"))?; + // Resolve --path before anything is created, so naming a path that is not + // on the disc fails without leaving an empty destination behind. + let target = match args.path.as_deref() { + None => None, + Some(p) => Some(resolve_disc_path(&mut *fs, &root, p)?), + }; + + if let Some(tar_path) = args.tar.clone() { + return extract_to_tar(&mut *fs, &root, target.as_ref(), &args, &tar_path); + } + let to = args + .to + .clone() + .expect("clap requires --to unless --tar is given"); + std::fs::create_dir_all(&to).with_context(|| format!("creating {}", to.display()))?; + let rf_mode = args .resource_forks .or_else(|| { @@ -1698,7 +1783,7 @@ fn run_extract_verb(args: ExtractArgs) -> Result<()> { log_stderr(format!( "rb-cli optical extract: {} -> {} (resource forks: {:?})", args.source.display(), - args.to.display(), + to.display(), rf_mode )); @@ -1714,23 +1799,45 @@ fn run_extract_verb(args: ExtractArgs) -> Result<()> { // Only disambiguate when the destination genuinely can't tell the names // apart; on a case-sensitive host everything extracts verbatim. - let case_insensitive_dest = dest_is_case_insensitive(&args.to); + let case_insensitive_dest = dest_is_case_insensitive(&to); let mut ctx = ExtractCtx { fork_mode: rf_mode.into(), collision, case_insensitive_dest, + // No --path means the whole disc, which has always recursed. + recursive: args.path.is_none() || args.recursive, + preserve_permissions: args.preserve_permissions, count: 0, skipped: 0, errors: 0, }; let mut used = std::collections::HashSet::new(); - for child in fs - .list_directory(&root) - .map_err(|e| anyhow::anyhow!("list_directory: {e}"))? - { - extract(&mut *fs, &child, &args.to, &mut ctx, &mut used); + match &target { + // A named file extracts on its own; a named directory extracts its + // contents into --to rather than recreating the directory itself, + // which is what `cp -r DIR/. DEST` does and what a user naming one + // folder expects. + Some(t) if t.entry_type == opticaldiscs::browse::entry::EntryType::File => { + extract(&mut *fs, t, &to, &mut ctx, &mut used); + } + Some(t) => { + for child in fs + .list_directory(t) + .map_err(|e| anyhow::anyhow!("list_directory: {e}"))? + { + extract(&mut *fs, &child, &to, &mut ctx, &mut used); + } + } + None => { + for child in fs + .list_directory(&root) + .map_err(|e| anyhow::anyhow!("list_directory: {e}"))? + { + extract(&mut *fs, &child, &to, &mut ctx, &mut used); + } + } } let mut summary = format!("extracted {} entry/entries", ctx.count); @@ -1797,6 +1904,11 @@ struct ExtractCtx { fork_mode: crate::fs::resource_fork::ResourceForkMode, collision: CaseCollisionMode, case_insensitive_dest: bool, + /// Descend into subdirectories. Always true for a whole-disc extract; + /// under `--path DIR` it follows `--recursive`. + recursive: bool, + /// Apply the disc's POSIX mode to what we write. Unix only. + preserve_permissions: bool, count: u64, skipped: u64, errors: u64, @@ -1958,11 +2070,20 @@ fn extract_one( } } } + if ctx.preserve_permissions { + apply_posix_mode(&dest.join(&safe_name), entry); + } ctx.count += 1; } EntryType::Directory => { + if !ctx.recursive { + return Ok(()); + } let dir_path = dest.join(&safe_name); std::fs::create_dir_all(&dir_path)?; + if ctx.preserve_permissions { + apply_posix_mode(&dir_path, entry); + } let children = fs .list_directory(entry) .map_err(|e| anyhow::anyhow!("list_directory: {e}"))?; @@ -1976,6 +2097,203 @@ fn extract_one( Ok(()) } +/// Walk `/A/B/C` from the disc root to the entry it names. +/// +/// Falls back to a case-insensitive match: ISO 9660 upper-cases names, while +/// the user reads them off `optical browse` in whatever case the Joliet or +/// Rock Ridge tree reported. +fn resolve_disc_path( + fs: &mut dyn opticaldiscs::browse::filesystem::Filesystem, + root: &opticaldiscs::browse::entry::FileEntry, + path: &str, +) -> Result { + let mut current = root.clone(); + let mut walked = String::new(); + for component in path.split('/').filter(|c| !c.is_empty() && *c != ".") { + let children = fs + .list_directory(¤t) + .map_err(|e| anyhow::anyhow!("listing {}: {e}", walked))?; + let hit = children.iter().find(|c| c.name == component).or_else(|| { + children + .iter() + .find(|c| c.name.eq_ignore_ascii_case(component)) + }); + current = match hit { + Some(c) => c.clone(), + None => { + return Err(crate::cli::exit::not_found(format!( + "{}/{} is not on this disc", + walked, component + ))) + } + }; + walked.push('/'); + walked.push_str(component); + } + Ok(current) +} + +/// Apply the disc's POSIX mode where it records one and the host understands +/// it. A no-op on Windows: the bits have nowhere to go, which is why `--tar` +/// exists. +#[allow(unused_variables)] +fn apply_posix_mode(path: &Path, entry: &opticaldiscs::browse::entry::FileEntry) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Some(p) = &entry.posix { + let _ = std::fs::set_permissions( + path, + std::fs::Permissions::from_mode(p.permission_bits()), + ); + } + } +} + +/// Archive to `.tar` / `.tar.gz` / `.tar.zst`, carrying mode, uid, gid and +/// symlinks — the metadata plain extraction drops on a host that cannot store +/// it. Compression follows the extension. +fn extract_to_tar( + fs: &mut dyn opticaldiscs::browse::filesystem::Filesystem, + root: &opticaldiscs::browse::entry::FileEntry, + target: Option<&opticaldiscs::browse::entry::FileEntry>, + args: &ExtractArgs, + out: &Path, +) -> Result<()> { + let recursive = args.path.is_none() || args.recursive; + let file = std::fs::File::create(out).with_context(|| format!("creating {}", out.display()))?; + let name = out.to_string_lossy().to_ascii_lowercase(); + let mut count = 0u64; + + // Each arm finishes its own builder: `tar::Builder` is generic over the + // writer and the compressors do not share an object-safe trait. + if name.ends_with(".tar.gz") || name.ends_with(".tgz") { + let enc = flate2::write::GzEncoder::new(file, flate2::Compression::default()); + let mut b = tar::Builder::new(enc); + tar_walk(fs, root, target, recursive, &mut b, &mut count)?; + b.into_inner()?.finish()?; + } else if name.ends_with(".tar.zst") || name.ends_with(".tzst") { + // Through the backend shim, not the `zstd` crate: the MiSTer armv7 build + // has `optical` on and `native-zstd` off (it uses `pure-zstd`), so naming + // the C crate here fails to compile there. `fs::tar_export` does the same + // job the same way. + let enc = crate::rbformats::zstd_compat::ZstdEncoder::new(file, 0) + .context("init zstd encoder")?; + let mut b = tar::Builder::new(enc); + tar_walk(fs, root, target, recursive, &mut b, &mut count)?; + b.into_inner()?.finish().context("finishing zstd stream")?; + } else { + let mut b = tar::Builder::new(file); + tar_walk(fs, root, target, recursive, &mut b, &mut count)?; + b.into_inner()?; + } + + log_stderr(format!( + "rb-cli optical extract: {} -> {} ({count} entry/entries)", + args.source.display(), + out.display() + )); + Ok(()) +} + +fn tar_walk( + fs: &mut dyn opticaldiscs::browse::filesystem::Filesystem, + root: &opticaldiscs::browse::entry::FileEntry, + target: Option<&opticaldiscs::browse::entry::FileEntry>, + recursive: bool, + builder: &mut tar::Builder, + count: &mut u64, +) -> Result<()> { + use opticaldiscs::browse::entry::EntryType; + let start = match target { + Some(t) if t.entry_type == EntryType::File => { + let n = t.name.clone(); + return tar_add(fs, t, &n, builder, count); + } + Some(t) => t, + None => root, + }; + for child in fs + .list_directory(start) + .map_err(|e| anyhow::anyhow!("list_directory: {e}"))? + { + let n = child.name.clone(); + tar_add_tree(fs, &child, &n, recursive, builder, count)?; + } + Ok(()) +} + +fn tar_add_tree( + fs: &mut dyn opticaldiscs::browse::filesystem::Filesystem, + entry: &opticaldiscs::browse::entry::FileEntry, + rel: &str, + recursive: bool, + builder: &mut tar::Builder, + count: &mut u64, +) -> Result<()> { + use opticaldiscs::browse::entry::EntryType; + if entry.entry_type == EntryType::Directory { + if !recursive { + return Ok(()); + } + for child in fs + .list_directory(entry) + .map_err(|e| anyhow::anyhow!("list_directory: {e}"))? + { + let sub = format!("{rel}/{}", child.name); + tar_add_tree(fs, &child, &sub, recursive, builder, count)?; + } + return Ok(()); + } + tar_add(fs, entry, rel, builder, count) +} + +fn tar_add( + fs: &mut dyn opticaldiscs::browse::filesystem::Filesystem, + entry: &opticaldiscs::browse::entry::FileEntry, + rel: &str, + builder: &mut tar::Builder, + count: &mut u64, +) -> Result<()> { + let mut header = tar::Header::new_gnu(); + // A symlink carries its target, not its bytes; storing the bytes would + // silently turn a link into a copy. + if let Some(link) = &entry.symlink_target { + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + set_tar_meta(&mut header, entry); + header.set_cksum(); + builder.append_link(&mut header, rel, link)?; + *count += 1; + return Ok(()); + } + let data = fs + .read_file(entry) + .map_err(|e| anyhow::anyhow!("read_file {}: {e}", entry.path))?; + header.set_size(data.len() as u64); + set_tar_meta(&mut header, entry); + header.set_cksum(); + builder.append_data(&mut header, rel, &data[..])?; + *count += 1; + Ok(()) +} + +/// Mode / uid / gid from the disc where it records them, else a plain 0644. +fn set_tar_meta(header: &mut tar::Header, entry: &opticaldiscs::browse::entry::FileEntry) { + match &entry.posix { + Some(p) => { + header.set_mode(p.permission_bits()); + header.set_uid(p.uid as u64); + header.set_gid(p.gid as u64); + } + None => { + header.set_mode(0o644); + header.set_uid(0); + header.set_gid(0); + } + } +} + fn parse_resource_fork_mode(s: &str) -> Option { match s.to_ascii_lowercase().replace('-', "").as_str() { "dataonly" | "data" => Some(CliResourceForkMode::DataOnly), diff --git a/src/cli/verbs/put.rs b/src/cli/verbs/put.rs index 513b63f1..673a06b5 100644 --- a/src/cli/verbs/put.rs +++ b/src/cli/verbs/put.rs @@ -228,7 +228,7 @@ pub fn run_with_budget( log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; // Resolve parent + leaf with the shared escape / colon grammar so a file // whose name contains a literal `/` can be written. @@ -323,6 +323,7 @@ pub fn run_with_budget( mode: args.mode, uid: args.uid, gid: args.gid, + unix_times: None, }, // `--no-preserve-meta` has to be withheld here too, not just from // `preserved` above. These are two independent inheritance paths and @@ -386,7 +387,11 @@ pub fn run_with_budget( preserve_meta: !args.no_preserve_meta, }, ) - .map_err(|e| anyhow!("create_file: {e}"))?; + // Through write_open_error so an `Unsupported` from the driver — "this + // filesystem is readable and this build will not write it" — exits 4 + // rather than the catch-all 1. The write-open path has done this since + // R-034; create_file can refuse for the same reason and did not. + .map_err(|e| crate::cli::resolve::write_open_error("create_file", e))?; if outcome.unsafe_fallback { log_stderr( "Note: this filesystem cannot stage a replace (no rename), so the original \ diff --git a/src/cli/verbs/put_macbinary.rs b/src/cli/verbs/put_macbinary.rs index 76929719..94a86b27 100644 --- a/src/cli/verbs/put_macbinary.rs +++ b/src/cli/verbs/put_macbinary.rs @@ -66,7 +66,7 @@ pub fn run(args: PutMacBinaryArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; // `--dst-dir` is the destination *directory* (the filename comes from the // MacBinary header), resolved with the shared escape / colon grammar. diff --git a/src/cli/verbs/repack.rs b/src/cli/verbs/repack.rs index 2236f99c..b17af8d7 100644 --- a/src/cli/verbs/repack.rs +++ b/src/cli/verbs/repack.rs @@ -46,6 +46,21 @@ pub fn run(args: RepackArgs) -> Result<()> { // validates that the selected partition is actually a Human68k volume. let (ro_file, ctx) = resolve_partition_ro(&args.image.path, args.image.partition.clone())?; log_stderr(&ctx.label); + // Opening the Human68k driver directly used to skip the routing decision the + // read path had already made, and a plain FAT16 volume opens cleanly as + // Human68k — the two share a BPB layout, which is deliberate (X68000 + // floppies use the standard MS-DOS one). So `repack` silently rebuilt a FAT + // volume through a driver with no long-filename concept and `payload.bin` + // came back as `PAYLOAD.BIN` (R-023). Ask the dispatch instead. + if ctx.type_string.as_deref() != Some("human68k") { + return Err(crate::cli::exit::usage(format!( + "`repack` is for Human68k (X68000) volumes; the volume at offset {} is {}. \ + A plain FAT volume opens as Human68k because the two share a BPB layout, but \ + rebuilding it through the Human68k driver would drop every long filename. \ + Use `rb-cli resize` to change a FAT volume's size.", + ctx.offset, ctx.type_name, + ))); + } let mut source = Human68kFilesystem::open(ro_file, ctx.offset).map_err(|e| { anyhow::anyhow!( "partition at offset {} is not a Human68k volume: {e}", diff --git a/src/cli/verbs/resize.rs b/src/cli/verbs/resize.rs index b0e49113..8b4e9f66 100644 --- a/src/cli/verbs/resize.rs +++ b/src/cli/verbs/resize.rs @@ -22,7 +22,7 @@ use clap::Args; use crate::cli::img_at::ImageRef; use crate::cli::logging::log_stderr; use crate::cli::parse::parse_size; -use crate::cli::resolve::resolve_partition_rw; +use crate::cli::resolve::{resolve_partition_ro, resolve_partition_rw}; use crate::partition::format_size; #[derive(Debug, Args)] @@ -33,6 +33,12 @@ pub struct ResizeArgs { /// New filesystem size in bytes. Accepts suffixes (`K`, `M`, `G`). #[arg(long)] pub size: String, + + /// Required to shrink. Growing needs no flag; shrinking truncates the + /// image, which is not reversible, so it has to be asked for. A shrink + /// that would cut into live data is refused with or without this. + #[arg(long)] + pub confirm_shrink: bool, } pub fn run(args: ResizeArgs) -> Result<()> { @@ -46,6 +52,19 @@ pub fn run(args: ResizeArgs) -> Result<()> { return run_remote(remote, args.image.partition.clone(), new_size); } + // Probe read-only first. A shrink has to be checked against where the + // filesystem's data actually ends, and that answer comes from the driver. + // Resolving the partition is a table parse; the filesystem is only opened + // when we are actually shrinking. `repack` opens the same two handles in + // the same order, so the pattern is known to work on Windows. + let (ro_file, probe) = resolve_partition_ro(&args.image.path, args.image.partition.clone())?; + let shrinking = new_size < probe.size; + if shrinking { + refuse_unsafe_shrink(ro_file, &probe, new_size, args.confirm_shrink)?; + } else { + drop(ro_file); + } + let (mut file, ctx, commit) = resolve_partition_rw(&args.image.path, args.image.partition.clone())?; log_stderr(&ctx.label); @@ -57,11 +76,7 @@ pub fn run(args: ResizeArgs) -> Result<()> { ctx.size, )); if new_size > ctx.size { - log_stderr(format!( - "warning: requested size {} exceeds partition capacity {}; the FS may refuse", - format_size(new_size), - format_size(ctx.size), - )); + grow_container_or_refuse(&mut file, &ctx, new_size)?; } let mut log_cb = |s: &str| log_stderr(format!(" {s}")); @@ -72,10 +87,151 @@ pub fn run(args: ResizeArgs) -> Result<()> { // changed the flat length can't be re-encoded; commit() surfaces that as a // clear error rather than writing a malformed container. commit.commit()?; + if shrinking { + truncate_after_shrink(&ctx, new_size)?; + } log_stderr("resize complete"); Ok(()) } +/// Refuse a shrink that would cut into live data, or one that was not asked +/// for. +/// +/// The filesystem is the only thing that knows where its data ends, so ask it: +/// `last_data_byte` is "bytes from the partition start needed to hold +/// everything that is allocated". Shrinking below that leaves metadata +/// describing blocks past the new end — for FAT that is a cluster chain +/// running off the end of a rewritten FAT, and the file comes back truncated +/// with `get` still exiting 0 (R-037). +/// +/// A driver that does not override `last_data_byte` inherits `total_size`, so +/// every shrink is refused for it. That is the intended default: without an +/// answer, no shrink can be shown to be safe. +fn refuse_unsafe_shrink( + ro_file: R, + probe: &crate::cli::resolve::PartitionContext, + new_size: u64, + confirmed: bool, +) -> Result<()> { + let mut fs = crate::fs::open_filesystem( + ro_file, + probe.offset, + probe.type_byte, + probe.type_string.as_deref(), + ) + .map_err(|e| anyhow::anyhow!("opening filesystem to check what a shrink would cut: {e}"))?; + let floor = fs + .last_data_byte() + .map_err(|e| anyhow::anyhow!("asking {} where its data ends: {e}", probe.type_name))?; + drop(fs); + + if new_size < floor { + return Err(crate::cli::exit::usage(format!( + "refusing to shrink {} to {}: its data extends to {} ({} bytes). Shrinking below \ + that would cut live data, and the volume would keep reporting the files it can no \ + longer read. The smallest safe size is {}.", + probe.type_name, + format_size(new_size), + format_size(floor), + floor, + format_size(floor), + ))); + } + if !confirmed { + return Err(crate::cli::exit::usage(format!( + "refusing to shrink {} from {} to {} without --confirm-shrink. The shrink is safe \ + — data ends at {} — but truncating the image cannot be undone.", + probe.type_name, + format_size(probe.size), + format_size(new_size), + format_size(floor), + ))); + } + log_stderr(format!( + "shrink: data ends at {}, target {} — safe", + format_size(floor), + format_size(new_size), + )); + Ok(()) +} + +/// Give back the space a shrink freed, when the volume is the whole file. +/// +/// Only then: a partition inside a larger disk has data after it, and the disk +/// length is set by the partition table rather than by this verb. There the +/// filesystem shrinks and the image keeps its length, which is what +/// `partmap resize` exists to follow up on. +fn truncate_after_shrink(ctx: &crate::cli::resolve::PartitionContext, new_size: u64) -> Result<()> { + let Some(path) = ctx.whole_file_path.as_deref() else { + log_stderr( + "the volume is a partition inside a larger disk, so the image keeps its length; \ + move the boundary with `rb-cli partmap resize`", + ); + return Ok(()); + }; + std::fs::OpenOptions::new() + .write(true) + .open(path) + .and_then(|f| f.set_len(new_size)) + .with_context(|| format!("truncating {} to {new_size} bytes", path.display()))?; + log_stderr(format!("truncated the image to {}", format_size(new_size))); + Ok(()) +} + +/// Make room for a grow, or refuse it. +/// +/// Growing a filesystem past the end of whatever holds it writes metadata +/// describing blocks that do not exist. This used to print "the FS may refuse" +/// and carry on regardless: the FAT resize happily rewrote the BPB for twice +/// the clusters, `resize complete` printed, and the process exited 0 (R-021). +/// +/// The two cases are not the same, so they are no longer treated the same: +/// +/// - **The volume is the whole file** (a bare superfloppy in a plain image). +/// Nothing else lives there and there is no table to keep in step, so +/// appending zeros *is* what the caller asked for. Do it, then resize into it. +/// - **Anything else** — a partition inside a larger disk, a decoded container. +/// Its length is set by something we are not editing here, so overrunning it +/// is corruption. Refuse and name the verb that can move the boundary. +fn grow_container_or_refuse( + file: &mut crate::rbformats::BoxRwSeek, + ctx: &crate::cli::resolve::PartitionContext, + new_size: u64, +) -> Result<()> { + use std::io::{Seek, SeekFrom, Write}; + + if ctx.whole_file_path.is_none() { + return Err(crate::cli::exit::usage(format!( + "requested size {} exceeds the {} available at partition offset {}. \ + Resizing the filesystem alone would describe blocks the partition does not \ + have. Move the boundary first with `rb-cli partmap resize`, or grow the \ + whole disk with `rb-cli grow IMG --add SIZE`, then resize again.", + format_size(new_size), + format_size(ctx.size), + ctx.offset, + ))); + } + + let add = new_size - ctx.size; + log_stderr(format!( + "growing the image by {} to {} before resizing (the volume is the whole file)", + format_size(add), + format_size(new_size), + )); + file.seek(SeekFrom::End(0)) + .context("seeking to end of image to grow it")?; + // 1-MiB chunks so a very large grow doesn't allocate a buffer to match. + let chunk = vec![0u8; 1024 * 1024]; + let mut remaining = add; + while remaining > 0 { + let n = remaining.min(chunk.len() as u64) as usize; + file.write_all(&chunk[..n]).context("growing image")?; + remaining -= n as u64; + } + file.flush().context("flushing grown image")?; + Ok(()) +} + /// Resize a remote image's filesystem in place over the block tier. Connects to /// the daemon, then defers to the testable [`resize_remote_partition`] core. #[cfg(feature = "remote")] diff --git a/src/cli/verbs/rm.rs b/src/cli/verbs/rm.rs index e36e6e97..e3e8f094 100644 --- a/src/cli/verbs/rm.rs +++ b/src/cli/verbs/rm.rs @@ -75,7 +75,7 @@ pub fn run_with_budget( log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let case_insensitive = match (args.ignore_case, args.case_sensitive) { (true, _) => true, diff --git a/src/cli/verbs/setrsrc.rs b/src/cli/verbs/setrsrc.rs index 942c0f77..ff066d94 100644 --- a/src/cli/verbs/setrsrc.rs +++ b/src/cli/verbs/setrsrc.rs @@ -27,7 +27,7 @@ pub fn run(args: SetRsrcArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let entry = super::ls::resolve_path(fs.as_filesystem_mut(), &args.path)?; let meta = std::fs::metadata(&args.from_file) diff --git a/src/cli/verbs/setvolname.rs b/src/cli/verbs/setvolname.rs index 046864bd..5258dd21 100644 --- a/src/cli/verbs/setvolname.rs +++ b/src/cli/verbs/setvolname.rs @@ -25,7 +25,7 @@ pub fn run(args: SetVolNameArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; fs.set_volume_name(&args.name) .map_err(|e| anyhow!("set_volume_name: {e}"))?; diff --git a/src/cli/verbs/show.rs b/src/cli/verbs/show.rs index 3e1e722a..89a931f6 100644 --- a/src/cli/verbs/show.rs +++ b/src/cli/verbs/show.rs @@ -107,6 +107,18 @@ fn show_partmap( password.as_deref().map(|s| s.as_bytes()), inside.as_deref(), )?; + // `partmap` predates every non-Apple table and went straight to APM, so an + // SGI disk `inspect` reads fine failed with "bad DDR signature: 0x0BE5" — + // the leading half of the SGI volume-header magic (R-026). Detect first. + // APM keeps its full DDR + driver detail; every other table renders the + // generic partition list, which is all those tables have. + let table = crate::partition::PartitionTable::detect(&mut file) + .map_err(|e| anyhow::anyhow!("detecting partition table: {e}"))?; + if !matches!(table, crate::partition::PartitionTable::Apm(_)) { + return show_partmap_generic(&table, format); + } + file.seek(std::io::SeekFrom::Start(0)) + .map_err(|e| anyhow::anyhow!("rewinding after detection: {e}"))?; let apm = Apm::parse(&mut file).map_err(|e| anyhow::anyhow!("parsing APM: {e}"))?; let bs = apm.ddr.block_size as u64; @@ -167,6 +179,71 @@ fn show_partmap( } } +/// Render any non-APM partition table: the partition list, which is the part +/// every table has. APM's DDR and driver-descriptor detail has no counterpart +/// elsewhere, so it stays on the APM path rather than being faked here. +fn show_partmap_generic( + table: &crate::partition::PartitionTable, + format: OutputFormat, +) -> Result<()> { + let parts = table.partitions(); + if format == OutputFormat::Text { + out_stdout(format!("Partition table: {}", table.type_name())); + out_stdout(format!( + "{:>3} {:<28} {:>12} {:>14}", + "idx", "type", "start_lba", "bytes" + )); + for p in &parts { + out_stdout(format!( + "{:>3} {:<28} {:>12} {:>14}", + p.index + 1, + p.type_name, + p.start_lba, + p.size_bytes + )); + } + return Ok(()); + } + let rows: Vec = parts + .iter() + .map(|p| GenericPartRow { + index: p.index + 1, + type_: p.type_name.clone(), + type_string: p.partition_type_string.clone(), + start_lba: p.start_lba, + size_bytes: p.size_bytes, + }) + .collect(); + match format { + OutputFormat::Json | OutputFormat::Yaml => emit_envelope( + format, + &Envelope::ok(GenericPartmapPayload { + kind: table.type_name().to_ascii_lowercase(), + entries: rows, + }), + ), + OutputFormat::Csv | OutputFormat::Tsv => emit_csv_or_tsv(format, &rows), + OutputFormat::Text => unreachable!(), + } +} + +#[derive(serde::Serialize)] +struct GenericPartmapPayload { + kind: String, + entries: Vec, +} + +#[derive(serde::Serialize)] +struct GenericPartRow { + index: usize, + #[serde(rename = "type")] + type_: String, + #[serde(skip_serializing_if = "Option::is_none")] + type_string: Option, + start_lba: u64, + size_bytes: u64, +} + /// Render the APM as text: DDR + driver descriptor map, then one row per /// partition entry with boot metadata appended for driver partitions. fn print_partmap_text(apm: &Apm) { @@ -444,17 +521,5 @@ struct DeviceRow { // --------------------------------------------------------------------------- fn emit_csv_or_tsv(format: OutputFormat, rows: &[T]) -> Result<()> { - let delim = if format == OutputFormat::Tsv { - b'\t' - } else { - b',' - }; - let mut wtr = csv::WriterBuilder::new() - .delimiter(delim) - .from_writer(std::io::stdout().lock()); - for row in rows { - wtr.serialize(row)?; - } - wtr.flush()?; - Ok(()) + crate::cli::output::emit_csv_or_tsv(format, rows) } diff --git a/src/cli/verbs/tui_app.rs b/src/cli/verbs/tui_app.rs index e898accb..bffe1096 100644 --- a/src/cli/verbs/tui_app.rs +++ b/src/cli/verbs/tui_app.rs @@ -11372,7 +11372,7 @@ fn apply_metadata_edit( resolve_partition_rw_forced(std::path::Path::new(image_path), selector, None)?; let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow::anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let entry = crate::cli::verbs::ls::resolve_path(fs.as_filesystem_mut(), &dst)?; // Type/creator only where the filesystem has them: on a POSIX-only // volume the editor shows blank codes and this would otherwise fail the @@ -11548,7 +11548,7 @@ fn apply_bless_folder( resolve_partition_rw_forced(std::path::Path::new(image_path), selector, None)?; let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow::anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let entry = crate::cli::verbs::ls::resolve_path(fs.as_filesystem_mut(), dir_path)?; if !entry.is_directory() { anyhow::bail!("{dir_path} is not a directory"); @@ -11575,7 +11575,7 @@ fn apply_mkdir( resolve_partition_rw_forced(std::path::Path::new(image_path), selector, None)?; let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow::anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let parent = if cur_dir == "/" { fs.root() .map_err(|e| anyhow::anyhow!("reading root: {e}"))? @@ -11608,7 +11608,7 @@ fn apply_delete( resolve_partition_rw_forced(std::path::Path::new(image_path), selector, None)?; let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow::anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let parent = if cur_dir == "/" { fs.root() .map_err(|e| anyhow::anyhow!("reading root: {e}"))? @@ -11652,7 +11652,7 @@ fn write_file_bytes( resolve_partition_rw_forced(std::path::Path::new(image_path), selector, None)?; let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow::anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let (parent, leaf) = crate::cli::verbs::ls::resolve_parent(fs.as_filesystem_mut(), &dst)?; let mut reader = std::io::Cursor::new(bytes.to_vec()); crate::fs::replace::create_or_replace( @@ -11706,7 +11706,7 @@ fn import_host_file( resolve_partition_rw_forced(std::path::Path::new(image_path), selector, None)?; let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow::anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let (parent, leaf) = crate::cli::verbs::ls::resolve_parent(fs.as_filesystem_mut(), &dst)?; if !parent.is_directory() { diff --git a/src/cli/verbs/xattr.rs b/src/cli/verbs/xattr.rs index 2a847b6d..e2283a80 100644 --- a/src/cli/verbs/xattr.rs +++ b/src/cli/verbs/xattr.rs @@ -124,7 +124,7 @@ fn run_set(args: XattrSetArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; if !fs.as_filesystem().supports_xattrs() { bail!( "{} does not store extended attributes", @@ -153,7 +153,7 @@ fn run_rm(args: XattrRmArgs) -> Result<()> { log_stderr(&ctx.label); let mut fs = ctx .open_editable(file) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; if !fs.as_filesystem().supports_xattrs() { bail!( "{} does not store extended attributes", diff --git a/src/fs/README.md b/src/fs/README.md index 71ef89c7..2a4ea520 100644 --- a/src/fs/README.md +++ b/src/fs/README.md @@ -13,14 +13,25 @@ Trait-based filesystem abstraction for browsing, compaction, resize, and validat ## Supported Partition Types -| Type Byte(s) | Filesystem | Browsing | Compaction | Resize | -|--------------------------------------|-----------|----------|------------|--------| -| `0x01` | FAT12 | Yes | Yes | Yes | -| `0x04`, `0x06`, `0x0E`, `0x14`, `0x16`, `0x1E` | FAT16 | Yes | Yes | Yes | -| `0x0B`, `0x0C`, `0x1B`, `0x1C` | FAT32 | Yes | Yes | Yes | -| `0x07` | NTFS | Yes | Yes | Yes (VBR patch) | -| `0x07` | exFAT | Yes | Yes | Yes (full bitmap resize) | -| `0x83` | ext2/3/4 | No (planned) | No | No | +**There is deliberately no capability table here.** One lived at this spot and +went stale: it listed five filesystems and six type bytes while the engine had +grown to around forty drivers, and still called ext "planned" years after +`ext.rs`, `ext_format.rs`, `ext_fsck.rs` and `ext_csum.rs` shipped. A hand-kept +table two levels down from the code it describes loses that race every time. + +Two live sources instead, neither of which can drift: + +- **What a filesystem can do** — the Filesystems table in the top-level + [`README.md`](../../README.md), which is the user-facing list and is covered + by the pre-commit documentation sync in CLAUDE.md. +- **Which type byte or DosType routes where** — the dispatch itself in + [`mod.rs`](mod.rs): `open_filesystem`, `open_editable_filesystem`, + `compact_partition_reader`, `effective_partition_size`, and the + `partition_type_string` matchers beside them. + +The one routing fact that is not obvious from either and belongs here: type +byte `0x07` covers **both** NTFS and exFAT, and `open_filesystem` disambiguates +by reading the OEM ID magic (`"NTFS "` vs `"EXFAT "`) rather than the byte. ## How to Add a New Filesystem diff --git a/src/fs/adfs.rs b/src/fs/adfs.rs index 2bc8c9d2..150ff543 100644 --- a/src/fs/adfs.rs +++ b/src/fs/adfs.rs @@ -1322,6 +1322,7 @@ impl Filesystem for AdfsFilesystem { if de.is_locked() { fe.special_type = Some("Locked".into()); } + fe.modified_unix = crate::fs::times::adfs_time_to_unix(de.load_addr, de.exec_addr); out.push(fe); } out.sort_by_key(|a| a.name.to_lowercase()); @@ -2179,7 +2180,7 @@ impl EditableFilesystem for AdfsFilesystem { name: &str, data: &mut dyn Read, data_len: u64, - _options: &CreateFileOptions, + options: &CreateFileOptions, ) -> Result { if data_len > u32::MAX as u64 { return Err(FilesystemError::Unsupported( @@ -2191,14 +2192,24 @@ impl EditableFilesystem for AdfsFilesystem { } else { parent.location as u32 }; + // ADFS packs the file's RISC OS filetype and timestamp into the + // load/exec pair when the load-addr high 12 bits are 0xFFF. A cross- + // fs copy passes the source mtime through options.unix_times; a + // genuinely-new file leaves it None and we keep the legacy + // (0xFFFFFFFF, 0) pair which RISC OS displays as "no date, Data". + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let (load_addr, exec_addr) = match mtime { + Some(s) => crate::fs::times::unix_to_adfs_time(s, 0xFFF), + None => (0xFFFFFFFF, 0), + }; // Old-map D-format: contiguous allocation, the disc address IS the // directory-entry indaddr (no fragment-id indirection). if self.old_map.is_some() { let start = self.old_map_alloc_and_write(data, data_len)?; let entry = AdfsDirEntry { name: name.to_string(), - load_addr: 0xFFFFFFFF, - exec_addr: 0, + load_addr, + exec_addr, file_length: data_len as u32, indirect_disc_addr: start, attrs: 0x03, @@ -2209,12 +2220,9 @@ impl EditableFilesystem for AdfsFilesystem { } else { format!("{}/{}", parent.path.trim_end_matches('/'), name) }; - return Ok(FileEntry::new_file( - name.to_string(), - path, - data_len, - start as u64, - )); + let mut fe = FileEntry::new_file(name.to_string(), path, data_len, start as u64); + fe.modified_unix = mtime; + return Ok(fe); } // Allocate + write payload first; if the dir insert later // fails we leak the fragment but the disc stays consistent. @@ -2222,8 +2230,8 @@ impl EditableFilesystem for AdfsFilesystem { let indaddr = build_indaddr(frag_id, 0, self.disc_record.log2sharesize); let entry = AdfsDirEntry { name: name.to_string(), - load_addr: 0xFFFFFFFF, - exec_addr: 0, + load_addr, + exec_addr, file_length: data_len as u32, indirect_disc_addr: indaddr, attrs: 0x03, // R + W @@ -2234,25 +2242,28 @@ impl EditableFilesystem for AdfsFilesystem { } else { format!("{}/{}", parent.path.trim_end_matches('/'), name) }; - Ok(FileEntry::new_file( - name.to_string(), - path, - data_len, - indaddr as u64, - )) + let mut fe = FileEntry::new_file(name.to_string(), path, data_len, indaddr as u64); + fe.modified_unix = mtime; + Ok(fe) } fn create_directory( &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { let parent_indaddr = if parent.path == "/" { self.disc_record.root } else { parent.location as u32 }; + // Same load/exec datestamp packing as create_file. + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let (load_addr, exec_addr) = match mtime { + Some(s) => crate::fs::times::unix_to_adfs_time(s, 0xFFF), + None => (0xFFFFFFFF, 0), + }; // Build the empty 2-KiB dir block, then write it the same way // we'd write file data (alloc + write). let block = self.build_empty_dir_block(); @@ -2262,8 +2273,8 @@ impl EditableFilesystem for AdfsFilesystem { let start = self.old_map_alloc_and_write(&mut cursor, ADFS_NEWDIR_SIZE)?; let entry = AdfsDirEntry { name: name.to_string(), - load_addr: 0xFFFFFFFF, - exec_addr: 0, + load_addr, + exec_addr, file_length: ADFS_NEWDIR_SIZE as u32, indirect_disc_addr: start, attrs: 0x0B, @@ -2274,19 +2285,17 @@ impl EditableFilesystem for AdfsFilesystem { } else { format!("{}/{}", parent.path.trim_end_matches('/'), name) }; - return Ok(FileEntry::new_directory( - name.to_string(), - path, - start as u64, - )); + let mut fe = FileEntry::new_directory(name.to_string(), path, start as u64); + fe.modified_unix = mtime; + return Ok(fe); } let mut cursor = std::io::Cursor::new(block); let (frag_id, _start) = self.alloc_and_write_data(&mut cursor, ADFS_NEWDIR_SIZE)?; let indaddr = build_indaddr(frag_id, 0, self.disc_record.log2sharesize); let entry = AdfsDirEntry { name: name.to_string(), - load_addr: 0xFFFFFFFF, - exec_addr: 0, + load_addr, + exec_addr, file_length: ADFS_NEWDIR_SIZE as u32, indirect_disc_addr: indaddr, attrs: 0x0B, // R + W + Directory bit @@ -2297,11 +2306,9 @@ impl EditableFilesystem for AdfsFilesystem { } else { format!("{}/{}", parent.path.trim_end_matches('/'), name) }; - Ok(FileEntry::new_directory( - name.to_string(), - path, - indaddr as u64, - )) + let mut fe = FileEntry::new_directory(name.to_string(), path, indaddr as u64); + fe.modified_unix = mtime; + Ok(fe) } fn delete_entry( diff --git a/src/fs/affs.rs b/src/fs/affs.rs index 75686956..caa11008 100644 --- a/src/fs/affs.rs +++ b/src/fs/affs.rs @@ -30,6 +30,68 @@ use super::filesystem::{ use super::CompactResult; /// Reusable error helper. +/// Find the root block at or below `candidate`. +/// +/// AFFS stores its size nowhere: the root block sits at the volume's midpoint, +/// so its *position* is the size. That makes an over-large `candidate` — which +/// is what you get from the end of the file when the volume is one partition of +/// several — unrecoverable by arithmetic alone. The candidate is always an +/// upper bound though, so step down until a block validates. +/// +/// Validation is deliberately strict (tag, secType, hash-table size and the +/// block's own checksum) because a scan that accepts a directory block would +/// mount the wrong tree. +fn locate_root_block( + reader: &mut R, + partition_offset: u64, + block_size: u64, + candidate: u32, +) -> Option { + let mut buf = [0u8; BSIZE]; + let mut blk = candidate; + // Bounded so a non-AFFS partition cannot turn into a whole-disk scan; the + // exact case is the first hit for every bare ADF and superfloppy. + let floor = candidate.saturating_sub(MAX_ROOT_SEARCH).max(2); + while blk >= floor { + let off = partition_offset + blk as u64 * block_size; + if reader.seek(SeekFrom::Start(off)).is_ok() + && reader.read_exact(&mut buf).is_ok() + && is_root_block(&buf) + { + return Some(blk); + } + if blk == floor { + break; + } + blk -= 1; + } + None +} + +/// Whether `buf` is a plausible AFFS root block. Checks the block's own +/// checksum, so a false positive needs a 32-bit coincidence on top of the tags. +fn is_root_block(buf: &[u8]) -> bool { + if buf.len() < BSIZE { + return false; + } + if BigEndian::read_i32(&buf[0..4]) != T_HEADER { + return false; + } + if BigEndian::read_i32(&buf[0x1FC..0x200]) != ST_ROOT { + return false; + } + if BigEndian::read_u32(&buf[12..16]) as usize != HT_SIZE { + return false; + } + let stored = BigEndian::read_u32(&buf[0x14..0x18]); + normal_checksum(buf, 5) == stored +} + +/// How far below the computed candidate to look for the root block. A +/// partition's slack is the gap between its length and the rest of the disk; +/// 1 MiB of blocks covers real Amiga layouts with room to spare. +const MAX_ROOT_SEARCH: u32 = 2048; + fn parse_err>(msg: S) -> FilesystemError { FilesystemError::Parse(msg.into()) } @@ -268,7 +330,17 @@ impl AffsFilesystem { // total blocks. For an 880K floppy this is (2 + 1760)/2 = 881. // ADFlib uses ((reserved + numBlocks - 1) / 2) which gives 880 for // 1760 sectors; that's the canonical value. - let root_block = (2 + total_blocks - 1) / 2; + // + // `total_blocks` comes from the end of the *reader*, which for a bare + // ADF is the end of the volume and for a partition inside an RDB disk + // is the end of the disk. AFFS is unusual in having no size field of + // its own — the root block's position *is* the size — so an + // overestimate puts the candidate past the real root and the open + // failed outright on every real Amiga hard disk (R-030). The candidate + // is always an upper bound, so walk down from it. + let candidate = (2 + total_blocks - 1) / 2; + let root_block = locate_root_block(&mut reader, partition_offset, block_size, candidate) + .ok_or_else(|| parse_err("root block: type != T_HEADER"))?; let mut root_buf = [0u8; BSIZE]; reader.seek(SeekFrom::Start( @@ -276,6 +348,8 @@ impl AffsFilesystem { ))?; reader.read_exact(&mut root_buf)?; let root = AffsRootBlock::parse(&root_buf, root_block)?; + // The volume ends just past its root block; trust that over the reader. + let total_blocks = total_blocks.min(root_block * 2); // Load the bitmap. let (bitmap, bitmap_pages) = read_bitmap( @@ -435,18 +509,20 @@ impl AffsFilesystem { } /// Synthesize a new directory header block in the cache. + /// `dates = Some(...)` stamps that Amiga DateStamp; `None` stamps `now`. fn build_dir_block( &mut self, block: u32, parent: u32, name: &str, + dates: Option<(i32, i32, i32)>, ) -> Result<(), FilesystemError> { let mut buf = [0u8; BSIZE]; buf[0..4].copy_from_slice(&T_HEADER.to_be_bytes()); buf[4..8].copy_from_slice(&block.to_be_bytes()); // bytes 8..0x14 are the rest of the header prefix — left zero. // Hash table at 0x18..0x138 — already zero from buf initialization. - let (days, mins, ticks) = Self::now_datestamp(); + let (days, mins, ticks) = dates.unwrap_or_else(Self::now_datestamp); buf[0x1A4..0x1A8].copy_from_slice(&days.to_be_bytes()); buf[0x1A8..0x1AC].copy_from_slice(&mins.to_be_bytes()); buf[0x1AC..0x1B0].copy_from_slice(&ticks.to_be_bytes()); @@ -732,6 +808,14 @@ impl AffsFilesystem { ), }; fe.modified = datestamp_string(entry.modify_days, entry.modify_mins, entry.modify_ticks); + let unix = super::affs_common::datestamp_to_unix( + entry.modify_days, + entry.modify_mins, + entry.modify_ticks, + ); + if unix > 0 { + fe.modified_unix = Some(unix as u64); + } fe.amiga_protection = Some(entry.access); if !entry.comment.is_empty() { fe.amiga_comment = Some(entry.comment.clone()); @@ -1255,6 +1339,15 @@ impl EditableFilesystem for AffsFilesystem { let first_data = data_blocks.first().copied().unwrap_or(0); let access = options.amiga_protection.unwrap_or(0); let comment_str = options.amiga_comment.as_deref().unwrap_or(""); + // Explicit amiga_dates always win; otherwise convert a supplied + // unix_times mtime to the (days, mins, ticks) triple so a cross-fs + // copy from a non-Amiga source (HFS+/ext/tar) still records the + // source date on AFFS. + let dates = options.amiga_dates.or_else(|| { + options + .unix_times + .map(|t| super::affs_common::unix_to_datestamp(t.mtime_or_now())) + }); self.build_file_header_block( header_block, parent_block, @@ -1265,7 +1358,7 @@ impl EditableFilesystem for AffsFilesystem { header_extension, access, comment_str, - options.amiga_dates, + dates, )?; self.hash_chain_insert(parent_block, header_block, name)?; @@ -1275,19 +1368,22 @@ impl EditableFilesystem for AffsFilesystem { } else { format!("{}/{}", parent.path, name) }; - Ok(FileEntry::new_file( - name.to_string(), - path, - data_len, - header_block as u64, - )) + let mut fe = FileEntry::new_file(name.to_string(), path, data_len, header_block as u64); + if let Some(dts) = dates { + fe.amiga_date = Some(dts); + let unix = super::affs_common::datestamp_to_unix(dts.0, dts.1, dts.2); + if unix > 0 { + fe.modified_unix = Some(unix as u64); + } + } + Ok(fe) } fn create_directory( &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { self.validate_name(name)?; let parent_block = if parent.location == 0 { @@ -1300,18 +1396,27 @@ impl EditableFilesystem for AffsFilesystem { return Err(FilesystemError::AlreadyExists(name.to_string())); } let block = self.alloc_block()?; - self.build_dir_block(block, parent_block, name)?; + let dates = options.amiga_dates.or_else(|| { + options + .unix_times + .map(|t| super::affs_common::unix_to_datestamp(t.mtime_or_now())) + }); + self.build_dir_block(block, parent_block, name, dates)?; self.hash_chain_insert(parent_block, block, name)?; let path = if parent.path == "/" { format!("/{name}") } else { format!("{}/{}", parent.path, name) }; - Ok(FileEntry::new_directory( - name.to_string(), - path, - block as u64, - )) + let mut fe = FileEntry::new_directory(name.to_string(), path, block as u64); + if let Some(dts) = dates { + fe.amiga_date = Some(dts); + let unix = super::affs_common::datestamp_to_unix(dts.0, dts.1, dts.2); + if unix > 0 { + fe.modified_unix = Some(unix as u64); + } + } + Ok(fe) } fn delete_entry( @@ -2168,10 +2273,25 @@ pub fn create_blank_affs( // Root block lives at the middle of the data region (classic AFFS // layout). The Amiga formatter picks `(2 + total_blocks - 1) / 2`. let root_block: u32 = (2 + total_blocks - 1) / 2; + + // One bitmap block covers BITMAP_BITS_PER_BLOCK blocks, so a volume larger + // than that needs several. Writing exactly one regardless is what made a + // 4 MB volume panic indexing past the end of its single page (R-008b), and + // what left every volume above 4066 blocks with an uncovered tail + // (R-008a). The bitmap starts at block 2 (blocks 0/1 are the boot block). + let bitmap_bits_needed = total_blocks.saturating_sub(2) as usize; + let bitmap_pages = bitmap_bits_needed.div_ceil(BITMAP_BITS_PER_BLOCK).max(1); + // Pages past the 25 that fit in the root block live on a chain of bitmap + // extension blocks, each holding 127 more page pointers plus a `next`. + let ext_blocks = bitmap_pages + .saturating_sub(BM_PAGES_ROOT) + .div_ceil(BM_EXT_PAGES); + let bitmap_block: u32 = root_block + 1; - if bitmap_block >= total_blocks { + let last_meta_block = bitmap_block as u64 + bitmap_pages as u64 + ext_blocks as u64 - 1; + if last_meta_block >= total_blocks as u64 { return Err(anyhow::anyhow!( - "AFFS volume too small to fit root + bitmap blocks" + "AFFS volume too small to fit root + {bitmap_pages} bitmap block(s)" )); } @@ -2187,10 +2307,22 @@ pub fn create_blank_affs( let root_off = root_block as usize * BSIZE; let root = &mut img[root_off..root_off + BSIZE]; BigEndian::write_i32(&mut root[0..4], T_HEADER); - BigEndian::write_u32(&mut root[4..8], root_block); // headerKey = self + // header_key is 0 on a root block, not the block's own number. Verified + // against a real Workbench 1.3 disk (fs.affs.workbench13.hd.hdf), whose + // root reads header_key=0 where ours wrote 4096 — the hypothesis R-020 + // recorded, now with a disk behind it. + BigEndian::write_u32(&mut root[4..8], 0); BigEndian::write_u32(&mut root[12..16], HT_SIZE as u32); BigEndian::write_i32(&mut root[0x138..0x13C], -1); // bm_flag = valid - BigEndian::write_u32(&mut root[0x13C..0x140], bitmap_block); // bm_pages[0] + for i in 0..bitmap_pages.min(BM_PAGES_ROOT) { + let off = 0x13C + i * 4; + BigEndian::write_u32(&mut root[off..off + 4], bitmap_block + i as u32); + } + if ext_blocks > 0 { + // First extension block sits immediately after the bitmap pages. + let first_ext = bitmap_block + bitmap_pages as u32; + BigEndian::write_u32(&mut root[0x1A0..0x1A4], first_ext); + } let name_bytes = volume_name.as_bytes(); let n = name_bytes.len().min(MAX_NAME_LEN); root[0x1B0] = n as u8; @@ -2201,41 +2333,88 @@ pub fn create_blank_affs( let sum = normal_checksum(root, 5); BigEndian::write_u32(&mut root[0x14..0x18], sum); - // Bitmap block: all free except blocks 0/1 (boot), root, bitmap, and - // any blocks past `total_blocks` within the same 4064-bit page. - let bm_off = bitmap_block as usize * BSIZE; - let bm = &mut img[bm_off..bm_off + BSIZE]; - for i in 1..128 { - BigEndian::write_u32(&mut bm[i * 4..i * 4 + 4], 0xFFFFFFFF); - } - // AFFS bitmap addresses blocks starting at block 2. Bit `i` covers - // block `i + 2`. Mark in-use: root_block, bitmap_block. - for blk in [root_block, bitmap_block] { - if blk >= 2 { - let bit = (blk - 2) as usize; - let word_idx = bit / 32 + 1; // word 0 is the checksum - let bit_in_word = bit % 32; - let mut w = BigEndian::read_u32(&bm[word_idx * 4..word_idx * 4 + 4]); - w &= !(1u32 << bit_in_word); - BigEndian::write_u32(&mut bm[word_idx * 4..word_idx * 4 + 4], w); + // Bitmap pages: set bit = free (the Amiga convention, opposite of most + // filesystems). Start all-free, then clear the blocks the metadata itself + // occupies and every bit past the end of the volume, so the allocator can + // never hand out a block that does not exist. + for page in 0..bitmap_pages { + let bm_off = (bitmap_block as usize + page) * BSIZE; + let bm = &mut img[bm_off..bm_off + BSIZE]; + for i in 1..BITMAP_WORDS_PER_BLOCK + 1 { + BigEndian::write_u32(&mut bm[i * 4..i * 4 + 4], 0xFFFFFFFF); } } - // Clear bits for blocks past `total_blocks` (the bitmap covers a - // full 4064-bit page even if the volume is smaller). Without this, - // an allocator could hand out non-existent block numbers. - let last_block_idx_in_page = ((total_blocks - 2) as usize).min(4064); - for bit in last_block_idx_in_page..4064 { - let word_idx = bit / 32 + 1; - if word_idx >= 128 { - break; + + // Bit `i` of the whole bitmap covers block `i + 2`; page `p` holds bits + // `p * BITMAP_BITS_PER_BLOCK ..`. `clear` maps a block number to its page + // and word, so a metadata block on any page is marked correctly. + let clear_block = |img: &mut [u8], blk: u32| { + if blk < 2 { + return; } - let bit_in_word = bit % 32; - let mut w = BigEndian::read_u32(&bm[word_idx * 4..word_idx * 4 + 4]); + let bit = (blk - 2) as usize; + let page = bit / BITMAP_BITS_PER_BLOCK; + if page >= bitmap_pages { + return; + } + let bit_in_page = bit % BITMAP_BITS_PER_BLOCK; + let word_idx = bit_in_page / 32 + 1; // word 0 is the checksum + let bit_in_word = bit_in_page % 32; + let off = (bitmap_block as usize + page) * BSIZE + word_idx * 4; + let mut w = BigEndian::read_u32(&img[off..off + 4]); w &= !(1u32 << bit_in_word); - BigEndian::write_u32(&mut bm[word_idx * 4..word_idx * 4 + 4], w); + BigEndian::write_u32(&mut img[off..off + 4], w); + }; + + clear_block(&mut img, root_block); + for p in 0..bitmap_pages { + clear_block(&mut img, bitmap_block + p as u32); + } + for e in 0..ext_blocks { + clear_block(&mut img, bitmap_block + bitmap_pages as u32 + e as u32); + } + // Bits past the end of the volume, in the final page. + for bit in bitmap_bits_needed..bitmap_pages * BITMAP_BITS_PER_BLOCK { + let page = bit / BITMAP_BITS_PER_BLOCK; + let bit_in_page = bit % BITMAP_BITS_PER_BLOCK; + let word_idx = bit_in_page / 32 + 1; + let bit_in_word = bit_in_page % 32; + let off = (bitmap_block as usize + page) * BSIZE + word_idx * 4; + let mut w = BigEndian::read_u32(&img[off..off + 4]); + w &= !(1u32 << bit_in_word); + BigEndian::write_u32(&mut img[off..off + 4], w); + } + + // Bitmap extension chain for pages beyond the 25 the root block holds. + for e in 0..ext_blocks { + let ext_blk = bitmap_block as usize + bitmap_pages + e; + let ext_off = ext_blk * BSIZE; + let first_page = BM_PAGES_ROOT + e * BM_EXT_PAGES; + for slot in 0..BM_EXT_PAGES { + let page = first_page + slot; + if page >= bitmap_pages { + break; + } + let off = ext_off + slot * 4; + BigEndian::write_u32(&mut img[off..off + 4], bitmap_block + page as u32); + } + // Last long of an extension block points at the next one, or 0. + let next = if e + 1 < ext_blocks { + (ext_blk + 1) as u32 + } else { + 0 + }; + let nx = ext_off + BM_EXT_PAGES * 4; + BigEndian::write_u32(&mut img[nx..nx + 4], next); + } + + // Checksums last, over the finished pages. + for page in 0..bitmap_pages { + let bm_off = (bitmap_block as usize + page) * BSIZE; + let bm = &mut img[bm_off..bm_off + BSIZE]; + let sum = bitmap_checksum(bm); + BigEndian::write_u32(&mut bm[0..4], sum); } - let sum = bitmap_checksum(bm); - BigEndian::write_u32(&mut bm[0..4], sum); Ok(img) } diff --git a/src/fs/affs_common.rs b/src/fs/affs_common.rs index 7a8b4259..8c20312a 100644 --- a/src/fs/affs_common.rs +++ b/src/fs/affs_common.rs @@ -17,6 +17,15 @@ pub const BSIZE_U64: u64 = BSIZE as u64; pub const HT_SIZE: usize = 72; /// Number of bitmap-block pointers stored inline in the root block. pub const BM_PAGES_ROOT: usize = 25; +/// Bit words in one bitmap block: 128 longs, less the leading checksum. +pub const BITMAP_WORDS_PER_BLOCK: usize = 127; +/// Blocks one bitmap block accounts for. A volume larger than this needs +/// several, chained through the root's `bm_pages` and then extension blocks — +/// assuming one covers the whole volume is what R-008a/R-008b were. +pub const BITMAP_BITS_PER_BLOCK: usize = BITMAP_WORDS_PER_BLOCK * 32; +/// Bitmap-page pointers in one bitmap extension block: 128 longs, less the +/// trailing pointer to the next extension block. +pub const BM_EXT_PAGES: usize = 127; /// Number of data-block pointers per file header / extension block. pub const MAX_DATA_BLOCKS: usize = 72; /// Maximum file/dir name length on AFFS (DOS\0..DOS\5). @@ -156,6 +165,22 @@ pub fn datestamp_to_unix(days: i32, mins: i32, ticks: i32) -> i64 { total_days * SECS_PER_DAY + mins as i64 * 60 + ticks as i64 / 50 } +/// Inverse of [`datestamp_to_unix`] — pack Unix seconds into an AmigaDOS +/// DateStamp `(days, mins, ticks)`. Pre-1978 clamps to (0, 0, 0). Ticks are +/// zero on the output (Unix seconds are 1-second granular; the tick field +/// would round-trip to zero anyway). +pub fn unix_to_datestamp(unix_secs: u64) -> (i32, i32, i32) { + let days_since_1970 = unix_secs as i64 / SECS_PER_DAY; + let days = days_since_1970 - AMIGA_EPOCH_DAYS; + if days < 0 { + return (0, 0, 0); + } + let tod = unix_secs as i64 - days_since_1970 * SECS_PER_DAY; + let mins = (tod / 60) as i32; + let ticks = ((tod % 60) * 50) as i32; + (days as i32, mins, ticks) +} + /// Render `(days, mins, ticks)` as a human-readable UTC string. pub fn datestamp_string(days: i32, mins: i32, ticks: i32) -> Option { let unix = datestamp_to_unix(days, mins, ticks); diff --git a/src/fs/apple_dos.rs b/src/fs/apple_dos.rs index f8a78d5b..9f0b378c 100644 --- a/src/fs/apple_dos.rs +++ b/src/fs/apple_dos.rs @@ -106,6 +106,9 @@ const MAX_CATALOG_SECTORS: usize = 32; /// Maximum T/S list sectors we walk for a single file. With 122 pairs /// per list and 256-byte sectors, one full list addresses ~30 KB; a /// 140 KB floppy can chain at most a handful. Cap is a runaway guard. +/// T/S pairs one T/S list sector holds (bytes 0x0C..0xFF, two bytes each). +const TS_PAIRS_PER_LIST: usize = 122; + const MAX_TS_LISTS: usize = 32; /// Maximum file entries we surface — sanity bound; a 140 KB DOS disk @@ -277,6 +280,78 @@ fn decode_apple_text(bytes: &[u8]) -> String { /// Map a low-7-bit type code to its DOS letter ("T", "I", "A", "B", /// "S", "R"), defaulting to "?" for unknown values. Used to surface the /// file kind through `FileEntry::special_type`. +/// Bytes of load-address + length that precede a type-B file's data. +const BINARY_HEADER_LEN: usize = 4; + +/// Return a type-B file's real contents: strip the 4-byte header and cut to +/// the length it declares. +/// +/// DOS 3.3 stores only a sector count, so without this a file comes back +/// padded to the sector — 104 bytes in, 256 out (R-028). Other types are +/// returned untouched: only B carries a length. +fn strip_binary_header(type_code: u8, data: Vec) -> Vec { + if type_code != TYPE_B || data.len() < BINARY_HEADER_LEN { + return data; + } + let len = u16::from_le_bytes([data[2], data[3]]) as usize; + let end = BINARY_HEADER_LEN + len; + if end > data.len() { + // Declared length runs past what is allocated: trust the sectors, not + // the header, rather than panicking on a damaged disk. + return data[BINARY_HEADER_LEN..].to_vec(); + } + data[BINARY_HEADER_LEN..end].to_vec() +} + +/// The first data sector of a file, via its T/S list. `None` when the file has +/// no data sector. Used only to read a type-B length header, so it stops after +/// the first pair rather than walking the chain. +fn first_data_sector( + reader: &mut R, + partition_offset: u64, + e: &AppleDosFileEntry, +) -> Result, FilesystemError> { + if e.first_ts_track == 0 { + return Ok(None); + } + let list = read_sector( + reader, + partition_offset, + e.first_ts_track, + e.first_ts_sector, + )?; + let (dt, ds) = (list[0x0C], list[0x0D]); + if dt == 0 { + return Ok(None); + } + Ok(Some(read_sector(reader, partition_offset, dt, ds)?)) +} + +/// The logical byte length a type-B file declares, read from its first data +/// sector. `None` for other types, which carry no length. +fn declared_binary_len(first_sector: &[u8]) -> Option { + if first_sector.len() < BINARY_HEADER_LEN { + return None; + } + Some(u16::from_le_bytes([first_sector[2], first_sector[3]]) as usize) +} + +/// Data sectors in a file whose catalog entry reports `length_sectors`. +/// +/// The catalog counts **every** sector a file occupies, including its +/// track/sector lists. Reporting that figure as the file size counted the T/S +/// list as data, so a one-sector file listed as 512 bytes while `get` returned +/// 256 (R-028). One T/S list addresses 122 data sectors, so each group of 123 +/// sectors contains exactly one list. +fn data_sectors(length_sectors: u16) -> u16 { + let n = length_sectors; + if n == 0 { + return 0; + } + let lists = n.div_ceil(TS_PAIRS_PER_LIST as u16 + 1); + n.saturating_sub(lists) +} + pub fn type_letter(type_code: u8) -> &'static str { match type_code { TYPE_T => "T", @@ -396,7 +471,7 @@ impl AppleDosFilesystem { tslist_track = list_buf[0x01]; tslist_sector = list_buf[0x02]; } - Ok(data) + Ok(strip_binary_header(e.type_code, data)) } /// Apple DOS 3.3 stores binary files with a 4-byte header prepended @@ -452,7 +527,21 @@ impl Filesystem for AppleDosFilesystem { if e.is_unused() || e.is_deleted() { continue; } - let approx_size = e.length_sectors as u64 * APPLE_II_SECTOR_BYTES as u64; + // Allocated bytes, less the T/S list sectors the catalog counts as + // part of the file. For a type-B file the real length is in its + // own header, so prefer that: it is what `get` returns (R-028). + let allocated = data_sectors(e.length_sectors) as u64 * APPLE_II_SECTOR_BYTES as u64; + let approx_size = if e.type_code == TYPE_B { + first_data_sector(&mut self.reader, self.partition_offset, e) + .ok() + .flatten() + .and_then(|sec| declared_binary_len(&sec)) + .map(|n| n as u64) + .filter(|n| *n <= allocated) + .unwrap_or(allocated) + } else { + allocated + }; // Use the entry index as the FileEntry location so read_file // can find this specific catalog slot O(1). let mut fe = FileEntry::new_file( @@ -947,11 +1036,19 @@ impl EditableFilesystem for AppleDosFilesystem )); } - // Default to type T (text) — caller may overwrite via direct - // catalog-slot edit if they need binary; this floor is - // conservative and avoids stamping a binary header the - // EditableFilesystem trait can't communicate. - let type_code = TYPE_T; + // Type B with the standard 4-byte header (load address, then length, + // both little-endian). DOS 3.3 records only a *sector* count, so a + // headerless store loses the byte length for good: the file came back + // padded to 256 and `ls`, `get` and the caller each reported a + // different number (R-028). B is also what a real Apple II tool writes + // for arbitrary bytes, so the length is recoverable by anything that + // reads the disk, not just by us. + let type_code = TYPE_B; + let mut stored = Vec::with_capacity(payload.len() + BINARY_HEADER_LEN); + stored.extend_from_slice(&0u16.to_le_bytes()); // load address + stored.extend_from_slice(&(payload.len() as u16).to_le_bytes()); + stored.extend_from_slice(&payload); + let payload = stored; let data_sectors_needed = payload.len().div_ceil(APPLE_II_SECTOR_BYTES).max(1) as u32; let pairs_per_list = self.vtoc.max_ts_pairs_per_list as u32; @@ -1625,7 +1722,10 @@ mod tests { ) .unwrap(); assert_eq!(fe.name, "NEW.TXT"); - assert_eq!(fe.special_type.as_deref(), Some("T")); + // Type B, not T: DOS 3.3 stores only a sector count, so the byte + // length has to live in the file's own 4-byte header or it is lost + // and the file reads back padded to 256 (R-028). + assert_eq!(fe.special_type.as_deref(), Some("B")); fs.sync_metadata().unwrap(); let inner = fs.reader.clone(); @@ -1634,10 +1734,12 @@ mod tests { let entries = fs2.list_directory(&root2).unwrap(); let new = entries.iter().find(|e| e.name == "NEW.TXT").unwrap(); let read_back = fs2.read_file(new, 1024).unwrap(); - // Type T means no binary-header strip — bytes come back verbatim - // padded to the sector boundary; trim the trailing zeros. - let trimmed_len = payload.len(); - assert_eq!(&read_back[..trimmed_len], payload.as_slice()); + // Exactly the bytes written — no sector padding, no header. This + // asserted only a prefix match before, which is why R-028's + // 104-in/256-out could not be caught here. + assert_eq!(read_back, payload); + // And the listed size agrees with what read_file returns. + assert_eq!(new.size, payload.len() as u64); // Free space dropped by 2 sectors (1 data + 1 T/S list). let after_free = EditableFilesystem::free_space(&mut fs2).unwrap(); diff --git a/src/fs/attrs.rs b/src/fs/attrs.rs index 099cf80c..177cd8af 100644 --- a/src/fs/attrs.rs +++ b/src/fs/attrs.rs @@ -67,12 +67,17 @@ pub struct AttrOverrides { pub mode: Option, pub uid: Option, pub gid: Option, + /// Preserved Unix mtime/atime/ctime from the source (host file, + /// tar Header, or source-image inode). Threaded through the shared + /// importer so `create_file` records the source's dates instead of + /// stamping the current time. See [`crate::fs::times::UnixTimes`]. + pub unix_times: Option, } impl AttrOverrides { /// True when the caller specified nothing at all. pub fn is_empty(&self) -> bool { - self.mode.is_none() && self.uid.is_none() && self.gid.is_none() + self.mode.is_none() && self.uid.is_none() && self.gid.is_none() && self.unix_times.is_none() } } @@ -511,6 +516,7 @@ mod tests { mode: Some(0o755), uid: Some(1), gid: Some(2), + unix_times: None, }, Some(&replaced), Some(&parent), @@ -663,6 +669,7 @@ mod tests { mode: Some(0o640), uid: Some(0), gid: Some(0), + unix_times: None, }; let parents = [entry_with(0o040_755, 99, 99), entry_with(0o040_700, 5, 5)]; for parent in &parents { diff --git a/src/fs/dir_import.rs b/src/fs/dir_import.rs index 12242c2f..7534a60c 100644 --- a/src/fs/dir_import.rs +++ b/src/fs/dir_import.rs @@ -219,23 +219,52 @@ fn walk_into(dir: &Path, prefix: &mut Vec, out: &mut Vec) -> /// falls back to the replaced entry then the parent directory. /// /// Unix-only: on Windows there is no mode to read, so every entry takes the -/// resolver's inherit-from-parent default. +/// resolver's inherit-from-parent default. `unix_times` is always populated +/// (independent of `apply_permissions`) so the imported file carries its +/// source's mtime end-to-end — that's independent of whether we honour its +/// mode bits, and is available on every host OS via `SystemTime`. #[cfg(unix)] fn host_overrides(meta: &std::fs::Metadata, apply: bool) -> AttrOverrides { use std::os::unix::fs::MetadataExt; + let unix_times = Some(super::times::UnixTimes { + mtime: Some(meta.mtime().max(0) as u64), + atime: Some(meta.atime().max(0) as u64), + ctime: Some(meta.ctime().max(0) as u64), + }); if !apply { - return AttrOverrides::default(); + return AttrOverrides { + unix_times, + ..Default::default() + }; } AttrOverrides { mode: Some(meta.mode() & 0o7777), uid: Some(meta.uid()), gid: Some(meta.gid()), + unix_times, } } #[cfg(not(unix))] -fn host_overrides(_meta: &std::fs::Metadata, _apply: bool) -> AttrOverrides { - AttrOverrides::default() +fn host_overrides(meta: &std::fs::Metadata, _apply: bool) -> AttrOverrides { + // Windows has no mode/uid/gid to read, but `SystemTime::modified` / + // `accessed` / `created` all work — carry the host's mtime so an + // rb-cli import from a Windows dir still preserves file dates. On + // failure fall through to None (driver stamps now). + fn to_secs(t: std::io::Result) -> Option { + t.ok()? + .duration_since(std::time::UNIX_EPOCH) + .ok() + .map(|d| d.as_secs()) + } + AttrOverrides { + unix_times: Some(super::times::UnixTimes { + mtime: to_secs(meta.modified()), + atime: to_secs(meta.accessed()), + ctime: to_secs(meta.created()), + }), + ..Default::default() + } } fn overrides_for(host: &Path, apply: bool) -> AttrOverrides { diff --git a/src/fs/efs.rs b/src/fs/efs.rs index 40425a5f..b5bbc723 100644 --- a/src/fs/efs.rs +++ b/src/fs/efs.rs @@ -12,6 +12,24 @@ //! is hijacked to hold `direxts`, the number of inode slots actually used //! to point at indirect blocks. See Linux `fs/efs/inode.c::efs_map_block`. //! +//! ## Free-space bitmap bit order +//! +//! The bitmap packs blocks **LSB-first** within each byte: block `N` is bit +//! `N % 8` of byte `N / 8`, counting from the least significant bit. A **set** +//! bit means FREE. +//! +//! This was MSB-first here until 2026-08-15, in the reader and the writer +//! alike — so our formatter and our fsck agreed with each other and disagreed +//! with IRIX, which is exactly the shape that hides a bug. IRIX's own fsck +//! reported `BAD FREE LIST` on every volume we wrote (R-039). +//! +//! The order is not a matter of taste, and the evidence is worth keeping. +//! Decoding a volume `mkfs_efs` wrote, LSB-first yields precisely the geometry +//! the superblock declares — `firstcg=34` of reserved blocks, then exactly +//! `cgisize` inode blocks at each of the `ncg` cylinder-group starts, then the +//! spare tail bits marked allocated. MSB-first yields ragged runs on no +//! boundary at all. +//! //! References: //! - `~/xfs-efs/refs/efs-linux-5.15/efs/` — Linux kernel v5.15 EFS sources. //! - `docs/SGI_Filesystems.md` — implementation plan and on-disk notes. @@ -437,6 +455,19 @@ impl EfsInode { } } + /// Current UNIX time in seconds, clamped into EFS's `u32` field. Used + /// wherever a new inode is stamped or an existing one is mutated + /// (`create_file` / `create_directory` / `create_symlink` / mkfs root / + /// parent-directory bumps on insert-delete-rename / `set_permissions` / + /// `set_owner`). Keeps the SystemTime dance in one place instead of + /// scattered across every write site. + pub(crate) fn now_u32() -> u32 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) as u32 + } + /// Construct an empty/free inode (mode=0). Used when initializing /// freshly-allocated CG inode-table regions or zeroing an inode /// on delete. @@ -495,6 +526,23 @@ impl EfsFilesystem { &mut sector, )?; let sb = EfsSuperblock::parse(§or)?; + // A volume whose superblock describes more blocks than the image holds + // is a truncated capture, not a filesystem. Reads of anything in the + // surviving prefix still work, so this warns rather than refuses — but + // without it the first allocation or fsck walk fails as an + // uninterpretable short read at a byte offset far past the end (R-029). + let available = reader + .seek(SeekFrom::End(0))? + .saturating_sub(partition_offset); + let declared = (sb.fs_size as u64).saturating_mul(EFS_BLOCKSIZE); + if declared > available { + log::warn!( + "EFS superblock declares {} blocks ({declared} bytes) but only {available} bytes \ + are present: this image is truncated. Reads inside the surviving prefix work; \ + anything that walks the whole volume (fsck, allocation) will run off the end.", + sb.fs_size, + ); + } let label = sb.label(); Ok(EfsFilesystem { reader, @@ -804,8 +852,9 @@ impl EfsFilesystem { let mut run_len: u32 = 0; for bit in lo..hi { let byte = (bit / 8) as usize; - let bit_in_byte = 7 - (bit % 8); // big-endian bit order, MSB first - // set bit = FREE on real IRIX EFS, so a free block is bit==1. + // LSB-first within the byte, and set bit = FREE. See the + // module header on bit order — we had this backwards. + let bit_in_byte = bit % 8; if (bm[byte] >> bit_in_byte) & 1 == 0 { run_start = None; run_len = 0; @@ -817,7 +866,7 @@ impl EfsFilesystem { // Mark bits [start..start+want_blocks) as in-use (clear). for b in start..start + want_blocks { let by = (b / 8) as usize; - let bb = 7 - (b % 8); + let bb = b % 8; bm[by] &= !(1u8 << bb); } return Ok(EfsExtent { @@ -845,7 +894,7 @@ impl EfsFilesystem { if by >= bm.len() { break; } - let bb = 7 - (b % 8); + let bb = b % 8; bm[by] |= 1u8 << bb; } } @@ -1107,7 +1156,7 @@ impl EfsFilesystem { if by >= bm.len() { return false; } - let bb = 7 - (bn % 8); + let bb = bn % 8; if bm[by] & (1u8 << bb) == 0 { return false; // already in use (set bit = free) } @@ -1191,6 +1240,9 @@ impl EfsFilesystem { let res = (|| -> Result<(), FilesystemError> { let mut parent_inode = self.read_inode(parent_inum)?; self.dir_insert(&mut parent_inode, &mut bm, new_name, child_inum)?; + let now = EfsInode::now_u32(); + parent_inode.mtime = now; + parent_inode.ctime = now; self.write_inode(&parent_inode)?; self.sb_dirty = true; Ok(()) @@ -1516,7 +1568,7 @@ impl EfsFilesystem { for (lo, hi) in regions.ranges() { for blk in lo..hi.min(total_bits) { let by = (blk / 8) as usize; - let bb = 7 - (blk % 8); + let bb = blk % 8; if bm[by] & (1u8 << bb) != 0 { free += 1; } @@ -1569,13 +1621,26 @@ fn entry_from_inode(name: &str, parent_path: &str, ino: &EfsInode) -> FileEntry format!("{parent_path}/{name}") }; let size = if ino.is_dir() { 0 } else { ino.size as u64 }; + let modified = if ino.mtime != 0 { + Some(crate::fs::unix_common::inode::format_unix_timestamp( + ino.mtime as i64, + )) + } else { + None + }; + let modified_unix = if ino.mtime != 0 { + Some(ino.mtime as u64) + } else { + None + }; FileEntry { name: name.to_string(), path, entry_type: ino.entry_type(), size, location: ino.inum as u64, - modified: None, + modified, + modified_unix, type_code: None, creator_code: None, symlink_target: None, @@ -1634,19 +1699,33 @@ impl super::filesystem::EditableFilesystem for Ef let res = (|| -> Result { let inum = self.allocate_inode()?; + // Preserve the source's times when the caller supplied them + // (dir_import from a host file, tar_import from an archive, + // Commander stage_copy from another image); otherwise stamp now + // (a genuinely new file). See src/fs/times.rs. + let times = super::times::resolve_or_now(options.unix_times); + let now_secs = super::times::now_u32(); let mut new_ino = EfsInode::empty(inum); new_ino.mode = options.mode.unwrap_or(0o100644) as u16; new_ino.nlink = 1; new_ino.uid = options.uid.unwrap_or(0) as u16; new_ino.gid = options.gid.unwrap_or(0) as u16; + new_ino.atime = times.atime_or_now() as u32; + new_ino.mtime = times.mtime_or_now() as u32; + new_ino.ctime = times.ctime_or_now() as u32; self.write_file_data(&mut bm, &mut new_ino, data, data_len)?; self.write_inode(&new_ino)?; // Link into parent. dir_insert may mutate parent_inode - // (extent growth on overflow), so re-read fresh. + // (extent growth on overflow), so re-read fresh. Adding an + // entry bumps the parent's dir-contents mtime + ctime — that + // is a *now* bump (the parent's directory changed just now), + // not the child's preserved time. let mut parent_ino = self.read_inode(parent_inum)?; self.dir_insert(&mut parent_ino, &mut bm, name_bytes, inum)?; + parent_ino.mtime = now_secs; + parent_ino.ctime = now_secs; self.write_inode(&parent_ino)?; self.sb.tinode = self.sb.tinode.saturating_sub(1); @@ -1705,6 +1784,8 @@ impl super::filesystem::EditableFilesystem for Ef let res = (|| -> Result { let inum = self.allocate_inode()?; + let times = super::times::resolve_or_now(options.unix_times); + let now_secs = super::times::now_u32(); let mut new_ino = EfsInode::empty(inum); // Symlinks are conventionally 0777 on IRIX. An explicit mode's // permission bits are honoured, but the type bits are ours to set @@ -1713,6 +1794,9 @@ impl super::filesystem::EditableFilesystem for Ef new_ino.nlink = 1; new_ino.uid = options.uid.unwrap_or(0) as u16; new_ino.gid = options.gid.unwrap_or(0) as u16; + new_ino.atime = times.atime_or_now() as u32; + new_ino.mtime = times.mtime_or_now() as u32; + new_ino.ctime = times.ctime_or_now() as u32; let mut data = target.as_bytes(); let len = data.len() as u64; @@ -1721,6 +1805,8 @@ impl super::filesystem::EditableFilesystem for Ef let mut parent_ino = self.read_inode(parent_inum)?; self.dir_insert(&mut parent_ino, &mut bm, name_bytes, inum)?; + parent_ino.mtime = now_secs; + parent_ino.ctime = now_secs; self.write_inode(&parent_ino)?; self.sb.tinode = self.sb.tinode.saturating_sub(1); @@ -1784,6 +1870,8 @@ impl super::filesystem::EditableFilesystem for Ef .expect("./.. always fits in one block"); self.write_block(ext.bn, &initial)?; + let times = super::times::resolve_or_now(options.unix_times); + let now_secs = super::times::now_u32(); let mut new_dir = EfsInode::empty(inum); new_dir.mode = options.mode.unwrap_or(0o040755) as u16; new_dir.nlink = 2; // `.` is the second link @@ -1792,13 +1880,19 @@ impl super::filesystem::EditableFilesystem for Ef new_dir.size = EFS_BLOCKSIZE as u32; new_dir.numextents = 1; new_dir.extents[0] = ext; + new_dir.atime = times.atime_or_now() as u32; + new_dir.mtime = times.mtime_or_now() as u32; + new_dir.ctime = times.ctime_or_now() as u32; self.write_inode(&new_dir)?; - // Link into parent. + // Link into parent. Parent's dir-contents changed → bump + // mtime + ctime (a *now* bump — the parent changed just now), + // and nlink += 1 for the new dir's ".." back-link. let mut parent_inode = self.read_inode(parent_inum)?; self.dir_insert(&mut parent_inode, &mut bm, name_bytes, inum)?; - // Parent's nlink increases by 1 (the new dir's ".." back-link). parent_inode.nlink = parent_inode.nlink.saturating_add(1); + parent_inode.mtime = now_secs; + parent_inode.ctime = now_secs; self.write_inode(&parent_inode)?; self.sb.tinode = self.sb.tinode.saturating_sub(1); @@ -1886,12 +1980,17 @@ impl super::filesystem::EditableFilesystem for Ef let zero = EfsInode::empty(entry_inum); self.write_inode(&zero)?; - // Directory parent's nlink decreases when we delete a - // child directory (its ".." back-link goes away). + // Parent dir-contents changed → mtime + ctime bump. If the target + // was a directory, its ".." back-link goes away too (parent nlink + // decreases). Always re-write the parent so file deletes also + // record the mtime bump (dir_remove only touched dir blocks). + let now = EfsInode::now_u32(); + parent_inode.mtime = now; + parent_inode.ctime = now; if entry.is_directory() { parent_inode.nlink = parent_inode.nlink.saturating_sub(1); - self.write_inode(&parent_inode)?; } + self.write_inode(&parent_inode)?; self.sb.tinode = self.sb.tinode.saturating_add(1); self.sb_dirty = true; @@ -1958,8 +2057,15 @@ impl super::filesystem::EditableFilesystem for Ef } entries[slot].name = new_name_bytes.to_vec(); if let Some(new_block) = serialize_dir_block(&entries) { - // Fits — commit the single-block rewrite. + // Fits — commit the single-block rewrite, then bump + // the parent's dir-contents mtime + ctime so a rename + // doesn't leave the parent looking untouched. self.write_block(bn, &new_block)?; + let mut parent_ino = self.read_inode(parent_inum)?; + let now = EfsInode::now_u32(); + parent_ino.mtime = now; + parent_ino.ctime = now; + self.write_inode(&parent_ino)?; return Ok(()); } // The longer name overflows this block. Fall back to @@ -1982,6 +2088,8 @@ impl super::filesystem::EditableFilesystem for Ef fn set_permissions(&mut self, entry: &FileEntry, mode: u32) -> Result<(), FilesystemError> { let mut ino = self.read_inode(entry.location as u32)?; ino.mode = super::unix_common::inode::with_permission_bits(ino.mode as u32, mode) as u16; + // POSIX ctime bumps on metadata-only changes; atime/mtime untouched. + ino.ctime = EfsInode::now_u32(); self.write_inode(&ino) } @@ -1992,6 +2100,7 @@ impl super::filesystem::EditableFilesystem for Ef let mut ino = self.read_inode(entry.location as u32)?; ino.uid = uid as u16; ino.gid = gid as u16; + ino.ctime = EfsInode::now_u32(); self.write_inode(&ino) } @@ -2075,7 +2184,7 @@ fn repair_efs( )); continue; } - let bb = 7 - (blk % 8); + let bb = blk % 8; // set bit = free; mark as in-use by CLEARING the bit. bm[by] &= !(1 << bb); bitmap_fixes += 1; @@ -2140,6 +2249,7 @@ fn adopt_orphans_into_lost_found( size: 0, location: 2, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -2348,6 +2458,7 @@ impl Filesystem for EfsFilesystem { size: ino.size as u64, location: EFS_ROOT_INODE as u64, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -2439,34 +2550,13 @@ impl Filesystem for EfsFilesystem { } else { None }; - let mut e = FileEntry { - name, - path, - entry_type: child.entry_type(), - size: child.size as u64, - location: child_inum as u64, - modified: None, - type_code: None, - creator_code: None, - symlink_target, - special_type: child.special_type(), - mode: Some(child.mode as u32), - uid: Some(child.uid as u32), - gid: Some(child.gid as u32), - resource_fork_size: None, - aux_type: None, - link_target_cnid: None, - amiga_protection: None, - amiga_comment: None, - amiga_date: None, - dos_attributes: None, - finder_flags: None, - prodos_file_type: None, - mac_dates: None, - }; - if matches!(e.entry_type, EntryType::Directory) { - e.size = 0; - } + // Route through `entry_from_inode` so mtime/ + // modified_unix + special_type + mode/uid/gid land + // consistently between the create_file return path + // and the browse path. + let mut e = entry_from_inode(&name, parent_path, &child); + e.path = path; + e.symlink_target = symlink_target; entries.push(e); } Err(_) => { @@ -3004,7 +3094,7 @@ pub fn write_blank_efs( { let mut mark_in_use = |blk: u32| { let by = (blk / 8) as usize; - let bit = 7 - (blk % 8) as u8; + let bit = (blk % 8) as u8; bitmap[by] &= !(1u8 << bit); }; mark_in_use(0); @@ -3035,6 +3125,12 @@ pub fn write_blank_efs( root.nlink = 2; // . and .. root.size = EFS_BLOCKSIZE as u32; root.numextents = 1; + // Stamp real timestamps so IRIX / tar / ls don't show "Dec 31 1969" for + // the volume root (R-039); the child-creation paths carry the same stamp. + let now = EfsInode::now_u32(); + root.atime = now; + root.mtime = now; + root.ctime = now; root.extents[0] = EfsExtent { magic: 0, bn: root_dirblock, @@ -3540,6 +3636,119 @@ mod tests { assert_eq!(streamed.len(), entry.size as usize); } + /// Newly-created files and directories must carry real timestamps, not + /// the epoch-zero the old empty()-then-mutate path left behind (R-039 + /// / IRIX "Dec 31 1969" bug). The parent's mtime/ctime bumps too so a + /// tar of the volume reflects when the entry was actually added. + #[test] + fn create_file_stamps_timestamps_and_bumps_parent() { + use crate::fs::filesystem::{ + CreateDirectoryOptions, CreateFileOptions, EditableFilesystem, + }; + let img = create_blank_efs(1024 * 1024, "rb-efs").expect("format 1M EFS"); + let mut fs = EfsFilesystem::open(Cursor::new(img), 0).expect("open"); + + // Capture wall clock *before* the writes; we compare it to the + // stamped seconds and allow a small forward drift for the second + // rollover between now and the write. + let before = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("post-epoch") + .as_secs() as u32; + + let root = fs.root().expect("root"); + // Root inode (mkfs) must itself be non-zero — this is the case + // `tar -tvzf` displays for the volume root. + let root_ino_before = fs.read_inode(2).expect("root inode"); + assert!( + root_ino_before.mtime > 0 && root_ino_before.ctime > 0 && root_ino_before.atime > 0, + "mkfs root inode times must not be zero: {root_ino_before:?}", + ); + + let file = fs + .create_file( + &root, + "HELLO", + &mut &b"hi"[..], + 2, + &CreateFileOptions::default(), + ) + .expect("create file"); + let f_ino = fs.read_inode(file.location as u32).expect("file inode"); + assert!( + f_ino.atime >= before && f_ino.mtime >= before && f_ino.ctime >= before, + "created file times must be >= before ({before}): {f_ino:?}", + ); + + let dir = fs + .create_directory(&root, "DIR", &CreateDirectoryOptions::default()) + .expect("create dir"); + let d_ino = fs.read_inode(dir.location as u32).expect("dir inode"); + assert!( + d_ino.atime >= before && d_ino.mtime >= before && d_ino.ctime >= before, + "created dir times must be >= before ({before}): {d_ino:?}", + ); + + // Parent (root) picks up the mtime/ctime bump from both inserts. + let root_after = fs.read_inode(2).expect("root inode"); + assert!( + root_after.mtime >= before && root_after.ctime >= before, + "root mtime/ctime must be bumped by the inserts: {root_after:?}", + ); + } + + /// The preservation path: when `CreateFileOptions.unix_times` is set (the + /// dir_import / tar_import / stage_copy cases), the driver must record + /// exactly those times, not stamp `now`. This is the write half of the + /// "extracting existing files should keep the source date" rule. + #[test] + fn create_file_preserves_unix_times_when_supplied() { + use crate::fs::filesystem::{CreateFileOptions, EditableFilesystem}; + use crate::fs::times::UnixTimes; + let img = create_blank_efs(1024 * 1024, "rb-efs").expect("format 1M EFS"); + let mut fs = EfsFilesystem::open(Cursor::new(img), 0).expect("open"); + let root = fs.root().unwrap(); + + // 2020-01-01 00:00:00 UTC — a fixed non-now date the driver must + // record verbatim; the on-disk `mtime` field being == this value + // (not "close to now") is the whole point of the preservation path. + let src_mtime: u64 = 1_577_836_800; + + let file = fs + .create_file( + &root, + "OLD", + &mut &b"hi"[..], + 2, + &CreateFileOptions { + unix_times: Some(UnixTimes::all(src_mtime)), + ..Default::default() + }, + ) + .expect("create with preserved times"); + let ino = fs.read_inode(file.location as u32).expect("file inode"); + assert_eq!( + (ino.atime as u64, ino.mtime as u64, ino.ctime as u64), + (src_mtime, src_mtime, src_mtime), + "preserved unix_times must land verbatim on disk" + ); + + // Read back through the browse path: modified_unix carries the same + // seconds, so a subsequent tar_export / stage_copy carries the date + // end-to-end. + let listed = fs + .list_directory(&root) + .expect("list") + .into_iter() + .find(|e| e.name == "OLD") + .expect("browse OLD"); + assert_eq!( + listed.modified_unix, + Some(src_mtime), + "list_directory must expose the preserved mtime as modified_unix" + ); + } + #[test] fn list_directory_descends_into_subdirectory() { // .desktop-IRIS (inode 22) is a subdirectory whose data block is @@ -3761,7 +3970,7 @@ mod tests { let in_use_blocks = [0u32, 1, 2, 18, 19, total_blocks - 1]; for b in in_use_blocks { let by = (b / 8) as usize; - let bb = 7 - (b % 8); + let bb = b % 8; img[bm_off + by] &= !(1 << bb); } img @@ -3773,10 +3982,11 @@ mod tests { let mut fs = EfsFilesystem::open(Cursor::new(img), 0).expect("open"); let bm = fs.read_bitmap().expect("read bitmap"); assert_eq!(bm.len(), fs.sb.bmsize as usize); - // Convention: set bit = free. Block 0 (boot) is in-use so bit - // is clear; block 100 (inside free region) has bit set. - assert!(bm[0] & 0b1000_0000 == 0, "block 0 must be marked in-use"); - assert!(bm[100 / 8] & (1 << (7 - (100 % 8))) != 0, "block 100 free"); + // Convention: set bit = free, LSB-first within the byte. Block 0 + // (boot) is in-use so bit 0 -- the LOW bit -- is clear; block 100 + // (inside the free region) has its bit set. + assert!(bm[0] & 0b0000_0001 == 0, "block 0 must be marked in-use"); + assert!(bm[100 / 8] & (1 << (100 % 8)) != 0, "block 100 free"); // Round-trip: write back and re-read; expect identical bytes. let mut bm2 = bm.clone(); bm2[5] = 0xFF; @@ -3785,6 +3995,39 @@ mod tests { assert_eq!(bm3, bm2); } + /// The bitmap is LSB-first, pinned against bytes IRIX itself wrote. + /// + /// This exists because the driver had it MSB-first in the reader *and* the + /// writer, so every test that built its own fixture agreed with itself and + /// nothing caught it until IRIX's fsck said `BAD FREE LIST` (R-039). A + /// self-consistent convention cannot be tested by self-consistent + /// fixtures — so this test hard-codes real numbers off a `mkfs_efs` + /// volume instead of deriving them. + /// + /// Geometry of that volume: `fs_size=131062`, `firstcg=34`, `cgfsize=21838`, + /// `cgisize=779`, `ncg=6`, `bmsize=16383` bytes = 131064 bits. + #[test] + fn bitmap_bit_order_is_lsb_first_per_irix_mkfs() { + // 131064 bits cover 131062 blocks, so the top two bits of the final + // byte are spare and must read as in-use. IRIX writes 0x3F there: + // 0b0011_1111 -- bits 6 and 7 clear, counting from the LSB. + let last_byte: u8 = 0x3F; + let spare_lo = 131_062 % 8; // = 6 + let spare_hi = 131_063 % 8; // = 7 + assert_eq!((spare_lo, spare_hi), (6, 7)); + assert!( + last_byte & (1 << spare_lo) == 0 && last_byte & (1 << spare_hi) == 0, + "spare tail bits must be in-use when read LSB-first" + ); + // Read MSB-first the same byte claims those blocks are free, and + // marks two real blocks in-use instead. That is the bug this pins. + let msb = |bit: u32| 7 - (bit % 8); + assert!( + last_byte & (1 << msb(131_062)) != 0, + "MSB-first would call a nonexistent block free -- wrong order" + ); + } + #[test] fn alloc_contiguous_finds_first_fit_and_marks_in_use() { let img = build_synthetic_for_bitmap_tests(); @@ -3802,7 +4045,7 @@ mod tests { // in-use means bit is cleared). for b in 20..24 { let by = (b / 8) as usize; - let bb = 7 - (b % 8); + let bb = b % 8; assert!(bm[by] & (1 << bb) == 0, "block {b} not marked"); } // A second alloc starts past the run we just took. @@ -3819,7 +4062,7 @@ mod tests { // set bit = free; start fully in-use (all zeros) then set bit 50. let mut bm = vec![0u8; 32]; let b = 50usize; - bm[b / 8] |= 1 << (7 - (b % 8)); + bm[b / 8] |= 1 << (b % 8); let img = build_synthetic_for_bitmap_tests(); let fs = EfsFilesystem::open(Cursor::new(img), 0).expect("open"); let regions = EfsDataRegions::from_sb(&fs.sb); @@ -3851,7 +4094,7 @@ mod tests { let first_free_bit = (0..sb.fs_size) .find(|blk| { let by = (blk / 8) as usize; - by < bm.len() && bm[by] & (1 << (7 - (blk % 8))) != 0 + by < bm.len() && bm[by] & (1 << (blk % 8)) != 0 }) .expect("fixture has free blocks"); assert!( @@ -3894,7 +4137,7 @@ mod tests { let raw_bits: u32 = (0..sb.fs_size) .filter(|blk| { let by = (blk / 8) as usize; - by < bm.len() && bm[by] & (1 << (7 - (blk % 8))) != 0 + by < bm.len() && bm[by] & (1 << (blk % 8)) != 0 }) .count() as u32; @@ -3928,7 +4171,7 @@ mod tests { EfsFilesystem::>>::free_extent_in_bitmap(&mut bm, &ext); for b in ext.bn..ext.bn + ext.length as u32 { let by = (b / 8) as usize; - let bb = 7 - (b % 8); + let bb = b % 8; assert!(bm[by] & (1 << bb) != 0, "block {b} still in-use after free"); } } @@ -4174,12 +4417,13 @@ mod tests { } for b in [0u32, 1, 2, 18, 19, 25, total_blocks - 1] { let by = (b / 8) as usize; - let bb = 7 - (b % 8); + let bb = b % 8; img[bm_off + by] &= !(1 << bb); } // Root inode (2) at byte (firstcg=18)*512 + 2*128. let ino2_off = 18 * 512 + 2 * 128; + let now = EfsInode::now_u32(); let root = EfsInode { inum: 2, mode: 0o040755, @@ -4187,9 +4431,9 @@ mod tests { uid: 0, gid: 0, size: 512, - atime: 0, - mtime: 0, - ctime: 0, + atime: now, + mtime: now, + ctime: now, gen: 0, numextents: 1, version: 0, @@ -4712,7 +4956,7 @@ mod tests { let data_blk = file_ino.extents[0].bn; let mut bm = fs.read_bitmap().expect("bm"); let by = (data_blk / 8) as usize; - let bb = 7 - (data_blk % 8); + let bb = data_blk % 8; bm[by] |= 1u8 << bb; fs.write_bitmap(&bm).expect("write bm"); diff --git a/src/fs/efs_fsck.rs b/src/fs/efs_fsck.rs index 8f5885fc..f4099451 100644 --- a/src/fs/efs_fsck.rs +++ b/src/fs/efs_fsck.rs @@ -250,7 +250,7 @@ pub fn fsck_efs(fs: &mut EfsFilesystem) -> Result( // and it is already reserved by that very fact. continue; } - let bb = 7 - (blk % 8); + let bb = blk % 8; // set bit = free, so in-use means the bit is CLEAR. if bm[by] & (1u8 << bb) == 0 { return Err(FilesystemError::InvalidData(format!( @@ -311,7 +311,7 @@ pub fn grow_efs( if by >= bm.len() { break; } - let bb = 7 - (blk % 8); + let bb = blk % 8; bm[by] |= 1u8 << bb; } // ...then take back everything past the volume, replica included. @@ -349,7 +349,7 @@ pub fn grow_efs( if by >= bm.len() { break; } - let bb = 7 - (blk % 8); + let bb = blk % 8; // set bit = free; mark inode region as in-use by clearing. bm[by] &= !(1u8 << bb); } @@ -461,7 +461,7 @@ fn relocate_bitmap( if by >= bm.len() { break; } - let bb = 7 - (blk % 8); + let bb = blk % 8; bm[by] |= 1u8 << bb; } @@ -961,7 +961,7 @@ mod tests { } let mark_in_use = |img: &mut Vec, b: u32| { let by = (b / 8) as usize; - let bb = 7 - (b % 8); + let bb = b % 8; img[bm_off + by] &= !(1 << bb); }; for b in 0..T_FIRSTCG { diff --git a/src/fs/entry.rs b/src/fs/entry.rs index d68d3a52..e9d8fa32 100644 --- a/src/fs/entry.rs +++ b/src/fs/entry.rs @@ -26,6 +26,15 @@ pub struct FileEntry { pub location: u64, /// Human-readable modification date string. pub modified: Option, + /// Modification time in seconds since UNIX epoch 1970-01-01. The numeric + /// twin of `modified` — the display string is for the browser/`ls`, this + /// is what `tar_export` writes into its headers and what a `stage_copy` + /// passes into `CreateFileOptions.unix_times` so a cross-image copy + /// preserves the source's date instead of stamping the current time. + /// Populated by every filesystem driver whose on-disk format carries a + /// per-file mtime (all Unix / Amiga / classic-Mac / DOS families); + /// `None` on formats that don't (Apple DOS 3.3, TRS-80, etc.). + pub modified_unix: Option, /// HFS/HFS+/MFS file type as the raw 4-byte Mac `OSType`, exactly as /// stored on disk (e.g. `*b"APPL"`, `*b"PICT"`). Kept as bytes — not text — /// because an `OSType` may hold non-ASCII bytes (e.g. Prince of Persia's @@ -127,6 +136,7 @@ impl FileEntry { size: 0, location: 0, modified: None, + modified_unix: None, type_code: None, creator_code: None, finder_flags: None, @@ -155,6 +165,7 @@ impl FileEntry { size: 0, location, modified: None, + modified_unix: None, type_code: None, creator_code: None, finder_flags: None, @@ -183,6 +194,7 @@ impl FileEntry { size, location, modified: None, + modified_unix: None, type_code: None, creator_code: None, finder_flags: None, @@ -217,6 +229,7 @@ impl FileEntry { size, location, modified: None, + modified_unix: None, type_code: None, creator_code: None, finder_flags: None, @@ -250,6 +263,7 @@ impl FileEntry { size: 0, location, modified: None, + modified_unix: None, type_code: None, creator_code: None, finder_flags: None, diff --git a/src/fs/exfat.rs b/src/fs/exfat.rs index 8922c8b6..82d9a47c 100644 --- a/src/fs/exfat.rs +++ b/src/fs/exfat.rs @@ -489,6 +489,15 @@ impl ExfatFilesystem { if entry_type == ENTRY_TYPE_FILE { let secondary_count = dir_data[pos + 1] as usize; let file_attrs = u16::from_le_bytes([dir_data[pos + 4], dir_data[pos + 5]]); + // LastModifiedTimestamp (u32 LE at offset 12..16) — DOS packed: + // upper 16 bits = date, lower 16 bits = time. + let modify_ts = u32::from_le_bytes([ + dir_data[pos + 12], + dir_data[pos + 13], + dir_data[pos + 14], + dir_data[pos + 15], + ]); + let modified_unix = super::times::exfat_timestamp_to_unix(modify_ts); // Next entry should be stream extension (0xC0) let stream_pos = pos + 32; @@ -563,6 +572,7 @@ impl ExfatFilesystem { size: 0, location: first_cluster as u64, modified: None, + modified_unix, type_code: None, creator_code: None, symlink_target: None, @@ -591,6 +601,7 @@ impl ExfatFilesystem { size: data_length, location: first_cluster as u64, modified: None, + modified_unix, type_code: None, creator_code: None, symlink_target: None, @@ -636,6 +647,7 @@ impl Filesystem for ExfatFilesystem { size: 0, location: self.root_cluster as u64, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -758,19 +770,13 @@ impl Filesystem for ExfatFilesystem { // Editing support // ============================================================================= -/// Get current timestamp as exFAT format (DOS-style u32: date in upper 16 bits, time in lower 16). +/// Current wall-clock time as an exFAT timestamp (DOS-style u32: date in +/// upper 16 bits, time in lower 16). Falls back to 2024-01-01 midnight on +/// the (impossible) pre-1970 system clock. Used only when the caller left +/// `CreateFileOptions.unix_times = None` — a genuinely new file. Copies / +/// imports carry a real source mtime through [`super::times::unix_to_exfat_timestamp`]. fn current_exfat_timestamp() -> u32 { - // Use same approach as FAT: encode current UTC time - // For simplicity in an embedded context, use a fixed recent timestamp - // Date: bits 31-25=year(0-127 from 1980), 24-21=month, 20-16=day - // Time: bits 15-11=hour, 10-5=minute, 4-0=second/2 - // 2024-01-01 00:00:00 - let year = 2024 - 1980; // 44 - let month = 1u32; - let day = 1u32; - let date = (year << 9) | (month << 5) | day; - let time = 0u32; // midnight - (date << 16) | time + super::times::unix_to_exfat_timestamp(super::times::now()) } impl ExfatFilesystem { @@ -969,14 +975,27 @@ impl ExfatFilesystem { // -- Directory Entry Sets -- /// Build a complete entry set (File + Stream + FileName entries). - fn build_entry_set(name: &str, attrs: u16, first_cluster: u32, data_len: u64) -> Vec { + /// + /// `mtime_secs = Some(secs)` stamps that Unix time into the three DOS + /// timestamp fields (import that carried a source mtime through); `None` + /// stamps `now` (a genuinely new file or a rename). + fn build_entry_set( + name: &str, + attrs: u16, + first_cluster: u32, + data_len: u64, + mtime_secs: Option, + ) -> Vec { let name_utf16: Vec = name.encode_utf16().collect(); let name_entry_count = name_utf16.len().div_ceil(15); // ceil(len/15) let secondary_count = 1 + name_entry_count; // stream + name entries let total_entries = 1 + secondary_count; // file + secondaries let mut entries = vec![0u8; total_entries * 32]; - let ts = current_exfat_timestamp(); + let ts = match mtime_secs { + Some(s) => super::times::unix_to_exfat_timestamp(s), + None => current_exfat_timestamp(), + }; // File Entry (0x85) entries[0] = ENTRY_TYPE_FILE; @@ -1401,7 +1420,8 @@ impl EditableFilesystem for ExfatFilesystem { // caller (e.g. the cross-image copy engine) when provided; default to // Archive for brand-new files. let attrs = options.dos_attributes.map(|a| a & 0x27).unwrap_or(0x20); - let entry_bytes = Self::build_entry_set(name, attrs, first_cluster, data_len); + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let entry_bytes = Self::build_entry_set(name, attrs, first_cluster, data_len, mtime); self.add_entry_to_directory(parent, &entry_bytes)?; let path = if parent.path == "/" { @@ -1410,19 +1430,16 @@ impl EditableFilesystem for ExfatFilesystem { format!("{}/{name}", parent.path) }; - Ok(FileEntry::new_file( - name.to_string(), - path, - data_len, - first_cluster as u64, - )) + let mut fe = FileEntry::new_file(name.to_string(), path, data_len, first_cluster as u64); + fe.modified_unix = Some(mtime.unwrap_or_else(super::times::now)); + Ok(fe) } fn create_directory( &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { validate_exfat_name(name)?; @@ -1442,7 +1459,8 @@ impl EditableFilesystem for ExfatFilesystem { self.write_cluster_data(new_cluster, &zeroed)?; // Build entry set with directory attribute - let entry_bytes = Self::build_entry_set(name, ATTR_DIRECTORY, new_cluster, 0); + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let entry_bytes = Self::build_entry_set(name, ATTR_DIRECTORY, new_cluster, 0, mtime); self.add_entry_to_directory(parent, &entry_bytes)?; let path = if parent.path == "/" { @@ -1451,11 +1469,9 @@ impl EditableFilesystem for ExfatFilesystem { format!("{}/{name}", parent.path) }; - Ok(FileEntry::new_directory( - name.to_string(), - path, - new_cluster as u64, - )) + let mut fe = FileEntry::new_directory(name.to_string(), path, new_cluster as u64); + fe.modified_unix = Some(mtime.unwrap_or_else(super::times::now)); + Ok(fe) } fn delete_entry( @@ -1518,7 +1534,8 @@ impl EditableFilesystem for ExfatFilesystem { } else { entry.dos_attributes.map(|a| a & 0x27).unwrap_or(0x20) }; - let entry_bytes = Self::build_entry_set(new_name, attrs, entry.location as u32, entry.size); + let entry_bytes = + Self::build_entry_set(new_name, attrs, entry.location as u32, entry.size, None); // remove_entry_from_directory matches names case-insensitively // and does not consult the cluster. For the normal case (names @@ -2806,7 +2823,7 @@ mod tests { fn test_exfat_entry_set_checksum() { // Build an entry set and verify the checksum is non-zero and consistent let entries = - ExfatFilesystem::>>::build_entry_set("hello.txt", 0x20, 5, 100); + ExfatFilesystem::>>::build_entry_set("hello.txt", 0x20, 5, 100, None); assert!(entries.len() >= 96); // At least 3 entries (file + stream + 1 name) let stored_cs = u16::from_le_bytes([entries[2], entries[3]]); assert_ne!(stored_cs, 0); diff --git a/src/fs/ext.rs b/src/fs/ext.rs index e75f0d88..47b0cf0b 100644 --- a/src/fs/ext.rs +++ b/src/fs/ext.rs @@ -1618,6 +1618,7 @@ impl ExtFilesystem { } /// Build an inode byte buffer with the given parameters. + #[allow(clippy::too_many_arguments)] fn build_inode_bytes( &self, mode: u32, @@ -1627,6 +1628,7 @@ impl ExtFilesystem { links: u16, flags: u32, block_data: &[u8; 60], + times: Option, ) -> Vec { build_inode_bytes( self.inode_size, @@ -1637,6 +1639,7 @@ impl ExtFilesystem { links, flags, block_data, + times, ) } @@ -2199,7 +2202,16 @@ impl EditableFilesystem for ExtFilesystem { let gid = options.gid.unwrap_or(0); let flags = if self.has_extents { EXT4_EXTENTS_FL } else { 0 }; let iblock = self.set_inode_blocks_for_new_file(new_inode, &data_blocks)?; - let mut inode_bytes = self.build_inode_bytes(mode, uid, gid, data_len, 1, flags, &iblock); + let mut inode_bytes = self.build_inode_bytes( + mode, + uid, + gid, + data_len, + 1, + flags, + &iblock, + options.unix_times, + ); // i_blocks counts ALLOCATED 512-byte sectors (block_size/512 per fs // block), not ceil(size/512): the two differ for a partial final block on // block sizes above 512, and the block-based count is the correct one. (No @@ -2273,8 +2285,16 @@ impl EditableFilesystem for ExtFilesystem { let gid = options.gid.unwrap_or(0); let flags = if self.has_extents { EXT4_EXTENTS_FL } else { 0 }; let iblock = self.set_inode_blocks_for_new_file(new_inode, &dir_blocks)?; - let inode_bytes = - self.build_inode_bytes(mode, uid, gid, self.block_size, 2, flags, &iblock); + let inode_bytes = self.build_inode_bytes( + mode, + uid, + gid, + self.block_size, + 2, + flags, + &iblock, + options.unix_times, + ); self.write_inode_raw(new_inode, &inode_bytes)?; // Write the directory block now that inode 8's generation is on disk (its @@ -2485,6 +2505,11 @@ fn is_extent_header(data: &[u8]) -> bool { /// so the blank-volume formatter (`ext_format`) can lay down the reserved root /// and lost+found inodes before any `ExtFilesystem` exists; the `ExtFilesystem` /// method of the same name delegates here. +/// +/// `times`, when set, replaces the default `now` stamp on atime/mtime/ctime so +/// a cross-fs import (dir_import / tar_import / stage_copy) preserves the +/// source's mtime end-to-end. `None` still stamps `now` — the case for a +/// genuinely new file (rb-cli `put` from stdin, GUI "new blank file"). #[allow(clippy::too_many_arguments)] pub(crate) fn build_inode_bytes( inode_size: u16, @@ -2495,12 +2520,13 @@ pub(crate) fn build_inode_bytes( links: u16, flags: u32, block_data: &[u8; 60], + times: Option, ) -> Vec { let mut buf = vec![0u8; inode_size as usize]; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() as u32; + let resolved = super::times::resolve_or_now(times); + let atime = resolved.atime_or_now() as u32; + let mtime = resolved.mtime_or_now() as u32; + let ctime = resolved.ctime_or_now() as u32; // i_mode (0x00) buf[0x00..0x02].copy_from_slice(&(mode as u16).to_le_bytes()); @@ -2509,11 +2535,11 @@ pub(crate) fn build_inode_bytes( // i_size_lo (0x04) buf[0x04..0x08].copy_from_slice(&(size as u32).to_le_bytes()); // i_atime (0x08) - buf[0x08..0x0C].copy_from_slice(&now.to_le_bytes()); + buf[0x08..0x0C].copy_from_slice(&atime.to_le_bytes()); // i_ctime (0x0C) - buf[0x0C..0x10].copy_from_slice(&now.to_le_bytes()); + buf[0x0C..0x10].copy_from_slice(&ctime.to_le_bytes()); // i_mtime (0x10) - buf[0x10..0x14].copy_from_slice(&now.to_le_bytes()); + buf[0x10..0x14].copy_from_slice(&mtime.to_le_bytes()); // i_links_count (0x1A) buf[0x1A..0x1C].copy_from_slice(&links.to_le_bytes()); // i_blocks_lo (0x1C) — number of 512-byte sectors (we compute from size) diff --git a/src/fs/ext_format.rs b/src/fs/ext_format.rs index cc6a3751..a89bae7f 100644 --- a/src/fs/ext_format.rs +++ b/src/fs/ext_format.rs @@ -515,7 +515,8 @@ fn write_blank_ext( } else { 0 }; - let mut jinode = build_inode_bytes(inode_size, 0o100600, 0, 0, jsize, 1, flags, &j.iblock); + let mut jinode = + build_inode_bytes(inode_size, 0o100600, 0, 0, jsize, 1, flags, &j.iblock, None); if inode_size >= INODE_SIZE_EXT4 { le16w(&mut jinode, 0x80, EXTRA_ISIZE); // i_extra_isize } @@ -575,6 +576,7 @@ fn build_dir_inode(l: &Ext2Layout, mode: u32, links: u16, dir_block: u64) -> Vec links, flags, &iblock, + None, // fresh mkfs stamp = now ); if l.inode_size >= INODE_SIZE_EXT4 { le16w(&mut inode, 0x80, EXTRA_ISIZE); // i_extra_isize diff --git a/src/fs/fat.rs b/src/fs/fat.rs index 06353aeb..f250ade5 100644 --- a/src/fs/fat.rs +++ b/src/fs/fat.rs @@ -680,6 +680,7 @@ impl FatFilesystem { FileEntry::new_file(display_name, path, size, cluster as u64) }; entry.modified = Some(modified); + entry.modified_unix = super::times::dos_datetime_to_unix(date_val, time_val); // Carry RO/HID/SYS/ARC so the cross-image copy engine can // preserve them on FAT/exFAT destinations (drop the directory / // volume-id bits — those are structural, not user attributes). @@ -1292,14 +1293,22 @@ impl FatFilesystem { /// Build LFN + SFN directory entries for a new file/directory. /// Returns the raw bytes (multiple of 32) to write into the directory. + /// + /// `mtime_secs = Some(secs)` stamps that Unix time into the DOS date/time + /// fields (from an import that carried the source's mtime through); `None` + /// stamps `now` (a fresh new file, or a rename which keeps its behaviour). fn build_dir_entries( name: &str, sfn: &[u8; 11], attr: u8, first_cluster: u32, file_size: u32, + mtime_secs: Option, ) -> Vec { - let (date, time) = current_fat_datetime(); + let (date, time) = match mtime_secs { + Some(s) => super::times::unix_to_dos_datetime(s), + None => current_fat_datetime(), + }; let checksum = lfn_checksum(sfn); // Determine if LFN is needed @@ -1815,7 +1824,9 @@ impl EditableFilesystem for FatFilesystem { .dos_attributes .map(|a| (a as u8) & 0x27) .unwrap_or(ATTR_ARCHIVE); - let entry_bytes = Self::build_dir_entries(name, &sfn, attr, first_cluster, data_len as u32); + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let entry_bytes = + Self::build_dir_entries(name, &sfn, attr, first_cluster, data_len as u32, mtime); self.add_to_directory(parent, &entry_bytes)?; if !options.skip_fsinfo_update { @@ -1830,8 +1841,12 @@ impl EditableFilesystem for FatFilesystem { let mut file_entry = FileEntry::new_file(name.to_string(), path, data_len, first_cluster as u64); - let (date, time) = current_fat_datetime(); + let (date, time) = match mtime { + Some(s) => super::times::unix_to_dos_datetime(s), + None => current_fat_datetime(), + }; file_entry.modified = Some(format_fat_datetime(date, time)); + file_entry.modified_unix = super::times::dos_datetime_to_unix(date, time); Ok(file_entry) } @@ -1840,7 +1855,7 @@ impl EditableFilesystem for FatFilesystem { &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { validate_fat_name(name)?; @@ -1860,7 +1875,11 @@ impl EditableFilesystem for FatFilesystem { // Initialize directory with . and .. entries let cluster_size = self.cluster_size() as usize; let mut new_dir = vec![0u8; cluster_size]; - let (date, time) = current_fat_datetime(); + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let (date, time) = match mtime { + Some(s) => super::times::unix_to_dos_datetime(s), + None => current_fat_datetime(), + }; // . entry new_dir[0..11].copy_from_slice(b". "); @@ -1904,7 +1923,8 @@ impl EditableFilesystem for FatFilesystem { // Add entry in parent directory let existing_sfns = self.collect_existing_sfns(&dir_data); let sfn = Self::generate_short_name(name, &existing_sfns); - let entry_bytes = Self::build_dir_entries(name, &sfn, ATTR_DIRECTORY, new_cluster, 0); + let entry_bytes = + Self::build_dir_entries(name, &sfn, ATTR_DIRECTORY, new_cluster, 0, mtime); self.add_to_directory(parent, &entry_bytes)?; self.update_fsinfo()?; @@ -1917,6 +1937,7 @@ impl EditableFilesystem for FatFilesystem { let mut dir_entry = FileEntry::new_directory(name.to_string(), path, new_cluster as u64); dir_entry.modified = Some(format_fat_datetime(date, time)); + dir_entry.modified_unix = super::times::dos_datetime_to_unix(date, time); Ok(dir_entry) } @@ -1994,7 +2015,7 @@ impl EditableFilesystem for FatFilesystem { let existing_sfns = self.collect_existing_sfns(&dir_data); let sfn = Self::generate_short_name(new_name, &existing_sfns); - let entry_bytes = Self::build_dir_entries(new_name, &sfn, attr, cluster, size); + let entry_bytes = Self::build_dir_entries(new_name, &sfn, attr, cluster, size, None); // Add the new name first, then remove the old (matched by old name + // cluster, so the just-added new entry is never the one removed). diff --git a/src/fs/filesystem.rs b/src/fs/filesystem.rs index d24b7cae..2cfd9139 100644 --- a/src/fs/filesystem.rs +++ b/src/fs/filesystem.rs @@ -330,6 +330,16 @@ pub struct CreateFileOptions { /// its conventional default (FAT/exFAT use archive). See /// [`crate::fs::entry::FileEntry::dos_attributes`]. pub dos_attributes: Option, + /// Preserve these Unix timestamps (seconds since 1970) on the new + /// inode instead of stamping the current time. Set by `dir_import` / + /// `tar_import` / Commander `stage_copy` so a copy carries the + /// source's mtime through end-to-end; left `None` for a genuinely new + /// file (rb-cli `put` from a `-` stdin stream, GUI "new blank file"), + /// where the driver stamps `now`. Ignored on filesystems that don't + /// carry per-file Unix times (FAT/exFAT/NTFS have their own scheme, + /// AmigaDOS uses `amiga_dates`, HFS uses `mac_dates`). See + /// [`crate::fs::times::UnixTimes`]. + pub unix_times: Option, } /// Options for creating a directory on an editable filesystem. @@ -347,6 +357,9 @@ pub struct CreateDirectoryOptions { pub amiga_comment: Option, /// AmigaDOS raw datestamp triple. See `CreateFileOptions::amiga_dates`. pub amiga_dates: Option<(i32, i32, i32)>, + /// Preserve these Unix timestamps on the new directory inode. See + /// [`CreateFileOptions::unix_times`] for the full rule. + pub unix_times: Option, } /// Source for resource fork data (HFS/HFS+ only). diff --git a/src/fs/fork_export.rs b/src/fs/fork_export.rs index fda070fa..ddf35b3e 100644 --- a/src/fs/fork_export.rs +++ b/src/fs/fork_export.rs @@ -104,6 +104,13 @@ pub fn export_file_with_fork( .write_file_to(entry, &mut f) .with_context(|| format!("extracting '{}'", entry.name))?; drop(f); + // Preserve the source's mtime on the extracted host file when the + // source recorded one — a plain image->host extract now round-trips + // the date the same way tar's `-p` flag does. Best-effort: a + // filesystem that doesn't record dates leaves this None and the OS + // stamps the current time as before. Errors are non-fatal (some + // network filesystems / SMB shares reject setting mtime). + apply_host_mtime(&out, entry); // ...plus a resource-fork sidecar for the fork-carrying "beside the data // fork" modes. MacBinary-without-rsrc and DataForkOnly land here as a plain @@ -138,3 +145,15 @@ pub fn export_file_with_fork( Ok(written + extra) } + +/// Stamp the extracted host file's mtime/atime from `entry.modified_unix`. +/// A best-effort call: filesystems that can't set times (network mounts, +/// some FUSE-backed FSes) just leave the OS default, they don't fail the +/// export. +fn apply_host_mtime(path: &Path, entry: &FileEntry) { + let Some(secs) = entry.modified_unix else { + return; + }; + let ft = filetime::FileTime::from_unix_time(secs as i64, 0); + let _ = filetime::set_file_times(path, ft, ft); +} diff --git a/src/fs/hfs.rs b/src/fs/hfs.rs index 934a45fe..5c9ff4dd 100644 --- a/src/fs/hfs.rs +++ b/src/fs/hfs.rs @@ -1852,6 +1852,10 @@ impl HfsFilesystem { } /// Build a classic HFS file record (102 bytes). + /// `mtime_secs = Some(secs)` stamps that Unix time as both create and + /// modify dates (a cross-fs copy carrying the source mtime through); + /// `None` stamps `now` (a genuinely new file). The after-create + /// `set_dates` Commander path still overrides both. #[allow(clippy::too_many_arguments)] fn build_file_record( file_id: u32, @@ -1864,9 +1868,13 @@ impl HfsFilesystem { type_code: &[u8; 4], creator_code: &[u8; 4], block_size: u32, + mtime_secs: Option, ) -> [u8; 102] { let mut rec = [0u8; 102]; - let now = hfs_common::hfs_now(); + let now = match mtime_secs { + Some(s) => super::times::unix_to_mac_epoch(s), + None => hfs_common::hfs_now(), + }; rec[0] = CATALOG_FILE as u8; // cdrType // rec[1] = reserved // FInfo at offset 4: fdType(4) + fdCreator(4) @@ -1906,10 +1914,16 @@ impl HfsFilesystem { rec } - /// Build a classic HFS directory record (70 bytes). - fn build_dir_record(dir_id: u32) -> [u8; 70] { + /// Build a classic HFS directory record (70 bytes). `mtime_secs = Some(secs)` + /// stamps that Unix time into the create and modify dates (a cross-fs + /// copy carrying the source mtime through); `None` stamps `now`. The + /// after-create Commander `set_dates` path still overrides both. + fn build_dir_record_with_dates(dir_id: u32, mtime_secs: Option) -> [u8; 70] { let mut rec = [0u8; 70]; - let now = hfs_common::hfs_now(); + let now = match mtime_secs { + Some(s) => super::times::unix_to_mac_epoch(s), + None => hfs_common::hfs_now(), + }; rec[0] = CATALOG_DIR as u8; // cdrType // dirFlags at offset 2 (u16) = 0 // dirVal at offset 4 (u16) = 0 (child count) @@ -2712,6 +2726,7 @@ impl Filesystem for HfsFilesystem { size: 0, location: 2, // HFS root directory CNID modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -2752,6 +2767,7 @@ impl Filesystem for HfsFilesystem { }; let mut fe = FileEntry::new_directory(name, path, dir_id as u64); fe.modified = hfs_common::format_mac_date(dates.1); + fe.modified_unix = hfs_common::mac_date_to_unix(dates.1); fe.mac_dates = Some(dates); entries.push(fe); } @@ -2777,6 +2793,7 @@ impl Filesystem for HfsFilesystem { fe.creator_code = Some(creator_code); fe.finder_flags = Some(finder_flags); fe.modified = hfs_common::format_mac_date(dates.1); + fe.modified_unix = hfs_common::mac_date_to_unix(dates.1); fe.mac_dates = Some(dates); if rsrc_size > 0 { fe.resource_fork_size = Some(rsrc_size as u64); @@ -3034,7 +3051,11 @@ impl EditableFilesystem for HfsFilesystem { (0, 0, 0) }; - // Build file record + // Build file record. A cross-fs copy passes the source mtime + // through `options.unix_times`; a genuinely new file leaves it None + // and takes `now`. Commander's `PreservedDates.mac` path still + // overrides both via a follow-up `set_dates` when present. + let mtime = options.unix_times.map(|t| t.mtime_or_now()); let file_rec = Self::build_file_record( file_id, data_len as u32, @@ -3046,6 +3067,7 @@ impl EditableFilesystem for HfsFilesystem { &type_code, &creator_code, self.mdb.block_size, + mtime, ); // Build key + record for catalog insertion @@ -3082,6 +3104,13 @@ impl EditableFilesystem for HfsFilesystem { if rsrc_size > 0 { fe.resource_fork_size = Some(rsrc_size as u64); } + let stamped = mtime.unwrap_or_else(super::times::now); + fe.modified_unix = Some(stamped); + fe.mac_dates = Some(( + super::times::unix_to_mac_epoch(stamped), + super::times::unix_to_mac_epoch(stamped), + 0, + )); Ok(fe) })(); if result.is_err() { @@ -3094,7 +3123,7 @@ impl EditableFilesystem for HfsFilesystem { &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { self.ensure_catalog_initialized()?; let snap = self.snapshot(); @@ -3115,8 +3144,10 @@ impl EditableFilesystem for HfsFilesystem { let folder_id = self.mdb.next_catalog_id; self.mdb.next_catalog_id += 1; - // Build folder record - let folder_rec = Self::build_dir_record(folder_id); + // Build folder record — see build_file_record's comment on how + // options.unix_times threads a source mtime through. + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let folder_rec = Self::build_dir_record_with_dates(folder_id, mtime); // Build key + record let key = Self::build_catalog_key(parent_id, &name_bytes); @@ -3144,11 +3175,15 @@ impl EditableFilesystem for HfsFilesystem { } else { format!("{}/{name}", parent.path) }; - Ok(FileEntry::new_directory( - name.to_string(), - path, - folder_id as u64, - )) + let mut fe = FileEntry::new_directory(name.to_string(), path, folder_id as u64); + let stamped = mtime.unwrap_or_else(super::times::now); + fe.modified_unix = Some(stamped); + fe.mac_dates = Some(( + super::times::unix_to_mac_epoch(stamped), + super::times::unix_to_mac_epoch(stamped), + 0, + )); + Ok(fe) })(); if result.is_err() { self.restore_snapshot(snap); @@ -5003,6 +5038,7 @@ mod tests { &[0u8; 4], &[0u8; 4], block_size, + None, )); fs.insert_catalog_record(&kr).unwrap(); } @@ -5130,8 +5166,19 @@ mod tests { for i in 0..n { let name = format!("f{:05}.txt", i); let mut key_record = Fs::build_catalog_key(root_id, name.as_bytes()); - let file_rec = - Fs::build_file_record(16 + i, 0, 0, 0, 0, 0, 0, &[0u8; 4], &[0u8; 4], block_size); + let file_rec = Fs::build_file_record( + 16 + i, + 0, + 0, + 0, + 0, + 0, + 0, + &[0u8; 4], + &[0u8; 4], + block_size, + None, + ); key_record.extend_from_slice(&file_rec); fs.insert_catalog_record(&key_record) .unwrap_or_else(|e| panic!("insert #{i} ({name}): {e}")); @@ -5244,6 +5291,7 @@ mod tests { &[0u8; 4], &[0u8; 4], block_size, + None, )); fs.insert_catalog_record(&kr) .unwrap_or_else(|e| panic!("insert #{i} into dir{d}: {e}")); @@ -5315,8 +5363,19 @@ mod tests { for i in 0..n { let name = format!("f{:05}.txt", i); let mut key_record = Fs::build_catalog_key(root_id, name.as_bytes()); - let file_rec = - Fs::build_file_record(16 + i, 0, 0, 0, 0, 0, 0, &[0u8; 4], &[0u8; 4], block_size); + let file_rec = Fs::build_file_record( + 16 + i, + 0, + 0, + 0, + 0, + 0, + 0, + &[0u8; 4], + &[0u8; 4], + block_size, + None, + ); key_record.extend_from_slice(&file_rec); fs.insert_catalog_record(&key_record).unwrap(); } diff --git a/src/fs/hfs_common.rs b/src/fs/hfs_common.rs index 1a4cebe2..994eb4d3 100644 --- a/src/fs/hfs_common.rs +++ b/src/fs/hfs_common.rs @@ -129,6 +129,17 @@ pub fn format_mac_date(mac_secs: u32) -> Option { Some(super::unix_common::inode::format_unix_timestamp(unix)) } +/// Convert an HFS/HFS+ Mac-epoch timestamp to seconds since the Unix epoch, +/// for [`crate::fs::entry::FileEntry::modified_unix`] and `tar_export`. +/// Returns `None` for zero ("no date set") or Mac dates that would land +/// before 1970 (the tar oracle would render those as 1969 either way). +pub fn mac_date_to_unix(mac_secs: u32) -> Option { + if mac_secs == 0 || (mac_secs as u64) < MAC_EPOCH_DELTA { + return None; + } + Some(mac_secs as u64 - MAC_EPOCH_DELTA) +} + /// Parse a `YYYY-MM-DD HH:MM:SS` string (interpreted as UTC) into Mac-epoch /// seconds. The inverse of [`format_mac_date`]; returns `None` when the string /// doesn't parse or falls outside the representable range. An empty string (or diff --git a/src/fs/hfsplus.rs b/src/fs/hfsplus.rs index 7d345b15..fdec0055 100644 --- a/src/fs/hfsplus.rs +++ b/src/fs/hfsplus.rs @@ -479,6 +479,9 @@ enum CatalogEntry { Folder { folder_id: u32, name: String, + /// Mac-epoch seconds — the "modification date" the browser and + /// `tar_export` surface. + content_mod_date: u32, bsd: HfsPlusBsdInfo, }, File { @@ -505,6 +508,9 @@ enum CatalogEntry { /// `.HFS+ Private Directory Data\r` directory and replaces the /// stub when surfaced through `list_directory`. dir_link_inode_num: Option, + /// Mac-epoch seconds — the "modification date" the browser and + /// `tar_export` surface. + content_mod_date: u32, bsd: HfsPlusBsdInfo, }, } @@ -1472,9 +1478,14 @@ impl HfsPlusFilesystem { continue; } let folder_id = BigEndian::read_u32(&rec[8..12]); + // Content modification date lives at record offset + // 20 (Mac-epoch u32); the write path uses the same + // offset (see set_catalog_dates around line 4413). + let content_mod_date = BigEndian::read_u32(&rec[20..24]); results.push(CatalogEntry::Folder { folder_id, name, + content_mod_date, bsd: HfsPlusBsdInfo::parse(rec), }); } @@ -1509,6 +1520,9 @@ impl HfsPlusFilesystem { let data_fork = ForkData::parse(&rec[88..168]); // Resource fork at offset 168 (80 bytes) let rsrc_fork = ForkData::parse(&rec[168..248]); + // HFS+ file record content-mod-date at offset 20 + // (Mac-epoch u32) — same slot the folder record uses. + let content_mod_date = BigEndian::read_u32(&rec[20..24]); results.push(CatalogEntry::File { file_id, name, @@ -1521,6 +1535,7 @@ impl HfsPlusFilesystem { finder_flags, link_inode_num, dir_link_inode_num, + content_mod_date, bsd: HfsPlusBsdInfo::parse(rec), }); } @@ -2911,15 +2926,23 @@ impl HfsPlusFilesystem { } /// Build a complete HFS+ file catalog record (248 bytes). - fn build_file_record( + /// `mtime_secs = Some(secs)` stamps that Unix time into all four date + /// fields (a cross-fs copy carrying the source mtime through); `None` + /// stamps `now`. The after-create `set_dates` Commander path still + /// overrides all four. + fn build_file_record_with_dates( file_id: u32, data_fork: &ForkData, rsrc_fork: &ForkData, type_code: &[u8; 4], creator_code: &[u8; 4], + mtime_secs: Option, ) -> [u8; 248] { let mut rec = [0u8; 248]; - let now = hfs_common::hfs_now(); + let now = match mtime_secs { + Some(s) => super::times::unix_to_mac_epoch(s), + None => hfs_common::hfs_now(), + }; BigEndian::write_i16(&mut rec[0..2], CATALOG_FILE); BigEndian::write_u32(&mut rec[8..12], file_id); BigEndian::write_u32(&mut rec[12..16], now); // createDate @@ -2964,9 +2987,16 @@ impl HfsPlusFilesystem { } /// Build a complete HFS+ folder catalog record (88 bytes). - fn build_folder_record(folder_id: u32) -> [u8; 88] { + /// `mtime_secs = Some(secs)` stamps that Unix time into the four date + /// fields (a cross-fs copy carrying the source mtime through); `None` + /// stamps `now`. The after-create Commander `set_dates` path still + /// overrides. + fn build_folder_record_with_dates(folder_id: u32, mtime_secs: Option) -> [u8; 88] { let mut rec = [0u8; 88]; - let now = hfs_common::hfs_now(); + let now = match mtime_secs { + Some(s) => super::times::unix_to_mac_epoch(s), + None => hfs_common::hfs_now(), + }; BigEndian::write_i16(&mut rec[0..2], CATALOG_FOLDER); // valence = 0 (offset 4) BigEndian::write_u32(&mut rec[8..12], folder_id); @@ -3044,6 +3074,7 @@ impl Filesystem for HfsPlusFilesystem { size: 0, location: 2, // HFS+ root directory CNID modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -3082,6 +3113,7 @@ impl Filesystem for HfsPlusFilesystem { CatalogEntry::Folder { folder_id, name, + content_mod_date, bsd, } => { let path = if entry.path == "/" { @@ -3091,6 +3123,8 @@ impl Filesystem for HfsPlusFilesystem { }; let mut fe = FileEntry::new_directory(name, path, folder_id as u64); bsd.apply_to(&mut fe); + fe.modified = super::hfs_common::format_mac_date(content_mod_date); + fe.modified_unix = super::hfs_common::mac_date_to_unix(content_mod_date); entries.push(fe); } CatalogEntry::File { @@ -3105,6 +3139,7 @@ impl Filesystem for HfsPlusFilesystem { finder_flags, link_inode_num, dir_link_inode_num, + content_mod_date, bsd, } => { let path = if entry.path == "/" { @@ -3148,6 +3183,8 @@ impl Filesystem for HfsPlusFilesystem { fe.creator_code = Some(creator_code); fe.finder_flags = Some(finder_flags); bsd.apply_to(&mut fe); + fe.modified = super::hfs_common::format_mac_date(content_mod_date); + fe.modified_unix = super::hfs_common::mac_date_to_unix(content_mod_date); if display_rsrc > 0 { fe.resource_fork_size = Some(display_rsrc); } @@ -4810,9 +4847,18 @@ impl HfsPlusFilesystem { ForkData::empty() }; - // Build file record - let file_rec = - Self::build_file_record(file_id, &data_fork, &rsrc_fork, &type_code, &creator_code); + // Build file record. A cross-fs copy passes the source mtime through + // options.unix_times; a genuinely-new file leaves it None and takes + // `now`. The after-create Commander `set_dates` path still overrides. + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let file_rec = Self::build_file_record_with_dates( + file_id, + &data_fork, + &rsrc_fork, + &type_code, + &creator_code, + mtime, + ); // Build key + record for catalog insertion let key = Self::build_catalog_key(parent_cnid, name); @@ -4863,6 +4909,8 @@ impl HfsPlusFilesystem { if rsrc_fork.logical_size > 0 { fe.resource_fork_size = Some(rsrc_fork.logical_size); } + let stamped = mtime.unwrap_or_else(super::times::now); + fe.modified_unix = Some(stamped); Ok(fe) } @@ -4870,7 +4918,7 @@ impl HfsPlusFilesystem { &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { let parent_cnid = parent.location as u32; @@ -4920,7 +4968,9 @@ impl HfsPlusFilesystem { self.vh.next_catalog_id += 1; // Build folder record - let folder_rec = Self::build_folder_record(folder_id); + // See build_file_record_with_dates for the mtime semantics. + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let folder_rec = Self::build_folder_record_with_dates(folder_id, mtime); // Build key + record let key = Self::build_catalog_key(parent_cnid, name); @@ -4954,11 +5004,10 @@ impl HfsPlusFilesystem { } else { format!("{}/{name}", parent.path) }; - Ok(FileEntry::new_directory( - name.to_string(), - path, - folder_id as u64, - )) + let mut fe = FileEntry::new_directory(name.to_string(), path, folder_id as u64); + let stamped = mtime.unwrap_or_else(super::times::now); + fe.modified_unix = Some(stamped); + Ok(fe) } fn delete_entry_inner( @@ -8722,12 +8771,13 @@ mod tests { if !kr.len().is_multiple_of(2) { kr.push(0); } - kr.extend_from_slice(&Fs::build_file_record( + kr.extend_from_slice(&Fs::build_file_record_with_dates( base_cnid + i, &empty_fork, &empty_fork, &[0u8; 4], &[0u8; 4], + None, )); fs.insert_catalog_record(&kr) .unwrap_or_else(|e| panic!("insert #{i} into dir{d}: {e}")); @@ -8981,12 +9031,13 @@ mod tests { if !kr.len().is_multiple_of(2) { kr.push(0); } - kr.extend_from_slice(&Fs::build_file_record( + kr.extend_from_slice(&Fs::build_file_record_with_dates( base_cnid + i, &empty_fork, &empty_fork, &[0u8; 4], &[0u8; 4], + None, )); fs.insert_catalog_record(&kr) .unwrap_or_else(|e| panic!("insert #{i} into dir{d} (grow should cover it): {e}")); diff --git a/src/fs/hpfs.rs b/src/fs/hpfs.rs index 32115b73..de40786b 100644 --- a/src/fs/hpfs.rs +++ b/src/fs/hpfs.rs @@ -472,6 +472,10 @@ impl HpfsFilesystem { } e.dos_attributes = Some(dos); e.modified = format_hpfs_date(de.write_date); + // HPFS stores write_date as u32 Unix seconds directly (no encoding). + if de.write_date != 0 { + e.modified_unix = Some(de.write_date as u64); + } e } @@ -1642,6 +1646,7 @@ impl HpfsFilesystem { is_dir: bool, data: &mut dyn Read, size: u64, + mtime_secs: Option, ) -> Result { validate_hpfs_name(name)?; let nb = name.as_bytes().to_vec(); @@ -1672,12 +1677,12 @@ impl HpfsFilesystem { let mut d = blank_dnode(dno, fno); d[8] |= 1; // root_dnode let off = dn_add_de(&mut d, b"\x01\x01", 0); - let mut m = de_meta_template(fno, 0, AT_DIRECTORY); + let mut m = de_meta_template(fno, 0, AT_DIRECTORY, mtime_secs); m[2] = DE_FIRST; dn_copy_meta(&mut d, off, &m); d.truncate(DNODE_BYTES); self.write_sectors(dno, &d)?; - let meta = de_meta_template(fno, 0, attrib); + let meta = de_meta_template(fno, 0, attrib, mtime_secs); self.add_dirent(parent_fnode, &nb, &meta)?; Ok(fno) } else { @@ -1692,7 +1697,7 @@ impl HpfsFilesystem { } }; self.write_file_fnode(fno, parent_fnode, &nb, size as u32, &extents)?; - let meta = de_meta_template(fno, size as u32, attrib); + let meta = de_meta_template(fno, size as u32, attrib, mtime_secs); self.add_dirent(parent_fnode, &nb, &meta)?; Ok(fno) } @@ -2021,32 +2026,30 @@ impl super::filesystem::EditableFilesystem for Hp name: &str, data: &mut dyn Read, data_len: u64, - _options: &super::filesystem::CreateFileOptions, + options: &super::filesystem::CreateFileOptions, ) -> Result { let parent_fnode = if parent.path == "/" { self.root_fnode } else { parent.location as u32 }; - let fno = self.create_entry(parent_fnode, name, false, data, data_len)?; + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let fno = self.create_entry(parent_fnode, name, false, data, data_len, mtime)?; let path = if parent.path == "/" { format!("/{name}") } else { format!("{}/{}", parent.path, name) }; - Ok(FileEntry::new_file( - name.to_string(), - path, - data_len, - fno as u64, - )) + let mut fe = FileEntry::new_file(name.to_string(), path, data_len, fno as u64); + fe.modified_unix = mtime; + Ok(fe) } fn create_directory( &mut self, parent: &FileEntry, name: &str, - _options: &super::filesystem::CreateDirectoryOptions, + options: &super::filesystem::CreateDirectoryOptions, ) -> Result { let parent_fnode = if parent.path == "/" { self.root_fnode @@ -2054,13 +2057,16 @@ impl super::filesystem::EditableFilesystem for Hp parent.location as u32 }; let mut empty = std::io::empty(); - let fno = self.create_entry(parent_fnode, name, true, &mut empty, 0)?; + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let fno = self.create_entry(parent_fnode, name, true, &mut empty, 0, mtime)?; let path = if parent.path == "/" { format!("/{name}") } else { format!("{}/{}", parent.path, name) }; - Ok(FileEntry::new_directory(name.to_string(), path, fno as u64)) + let mut fe = FileEntry::new_directory(name.to_string(), path, fno as u64); + fe.modified_unix = mtime; + Ok(fe) } fn delete_entry( @@ -2241,14 +2247,21 @@ fn blank_dnode(dno: u32, up: u32) -> Vec { } /// Build a 32-byte dirent metadata template for [`dn_copy_meta`]. -fn de_meta_template(fnode: u32, size: u32, attrib: u8) -> [u8; 32] { +/// `mtime_secs = Some(secs)` stamps that Unix time into the write/read/creation +/// date fields (HPFS already stores u32 Unix seconds, so no conversion needed); +/// `None` stamps the reproducible `FIXED_TIME` sentinel — matching the +/// generator-forbids-clocks convention this format harness uses. +fn de_meta_template(fnode: u32, size: u32, attrib: u8, mtime_secs: Option) -> [u8; 32] { let mut m = [0u8; 32]; + let stamp = mtime_secs + .map(|s| s.min(u32::MAX as u64) as u32) + .unwrap_or(FIXED_TIME); m[3] = attrib; put_u32(&mut m, 4, fnode); - put_u32(&mut m, 8, FIXED_TIME); + put_u32(&mut m, 8, stamp); put_u32(&mut m, 12, size); - put_u32(&mut m, 16, FIXED_TIME); - put_u32(&mut m, 20, FIXED_TIME); + put_u32(&mut m, 16, stamp); + put_u32(&mut m, 20, stamp); m } diff --git a/src/fs/human68k.rs b/src/fs/human68k.rs index 0fb8b2fb..53830407 100644 --- a/src/fs/human68k.rs +++ b/src/fs/human68k.rs @@ -780,6 +780,7 @@ impl Filesystem for Human68kFilesystem { if de.attr & attr::READ_ONLY != 0 { fe.special_type = Some("R/O".to_string()); } + fe.modified_unix = super::times::dos_datetime_to_unix(de.date, de.time); out.push(fe); } out.sort_by_key(|a| a.name.to_lowercase()); @@ -1233,7 +1234,7 @@ impl EditableFilesystem for Human68kFilesystem name: &str, data: &mut dyn std::io::Read, data_len: u64, - _options: &CreateFileOptions, + options: &CreateFileOptions, ) -> Result { let (parent_cluster, parent_path) = Self::parent_cluster_and_path(parent); let (n8, ne10, e3) = encode_human68k_name(name)?; @@ -1279,6 +1280,14 @@ impl EditableFilesystem for Human68kFilesystem entry[8..11].copy_from_slice(&e3); entry[11] = attr::ARCHIVE; entry[12..22].copy_from_slice(&ne10); + // Human68k shares FAT's 16-bit DOS packed date + time at bytes 22..26. + let mtime_secs = options + .unix_times + .map(|t| t.mtime_or_now()) + .unwrap_or_else(super::times::now); + let (date, time) = super::times::unix_to_dos_datetime(mtime_secs); + LittleEndian::write_u16(&mut entry[22..24], time); + LittleEndian::write_u16(&mut entry[24..26], date); LittleEndian::write_u16(&mut entry[26..28], first_cluster); LittleEndian::write_u32(&mut entry[28..32], payload.len() as u32); self.reader.seek(SeekFrom::Start(slot_off))?; @@ -1286,19 +1295,21 @@ impl EditableFilesystem for Human68kFilesystem self.fat_write_back()?; self.reader.flush()?; - Ok(FileEntry::new_file( + let mut fe = FileEntry::new_file( name.to_string(), format!("{parent_path}/{name}"), payload.len() as u64, first_cluster as u64, - )) + ); + fe.modified_unix = super::times::dos_datetime_to_unix(date, time); + Ok(fe) } fn create_directory( &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { let (parent_cluster, parent_path) = Self::parent_cluster_and_path(parent); let (n8, ne10, e3) = encode_human68k_name(name)?; @@ -1330,6 +1341,13 @@ impl EditableFilesystem for Human68kFilesystem entry[8..11].copy_from_slice(&e3); entry[11] = attr::DIRECTORY; entry[12..22].copy_from_slice(&ne10); + let mtime_secs = options + .unix_times + .map(|t| t.mtime_or_now()) + .unwrap_or_else(super::times::now); + let (date, time) = super::times::unix_to_dos_datetime(mtime_secs); + LittleEndian::write_u16(&mut entry[22..24], time); + LittleEndian::write_u16(&mut entry[24..26], date); LittleEndian::write_u16(&mut entry[26..28], dir_cluster); // Directories report size 0 in the entry; the chain is authoritative. LittleEndian::write_u32(&mut entry[28..32], 0); @@ -1338,11 +1356,13 @@ impl EditableFilesystem for Human68kFilesystem self.fat_write_back()?; self.reader.flush()?; - Ok(FileEntry::new_directory( + let mut fe = FileEntry::new_directory( name.to_string(), format!("{parent_path}/{name}"), dir_cluster as u64, - )) + ); + fe.modified_unix = super::times::dos_datetime_to_unix(date, time); + Ok(fe) } fn delete_entry( diff --git a/src/fs/human68k_clone.rs b/src/fs/human68k_clone.rs index d3a3c30c..c3701307 100644 --- a/src/fs/human68k_clone.rs +++ b/src/fs/human68k_clone.rs @@ -23,12 +23,13 @@ //! //! Fidelity notes (Human68k is FAT-derived — much simpler than PFS3): //! - There are no symlinks or hardlinks to replay. -//! - The `EditableFilesystem` write path stamps a fresh ARCHIVE attribute -//! and a zero timestamp, so file **modification dates** and the -//! **read-only** attribute are not carried across (the same limitation -//! the interactive edit path already has). A single summary warning is -//! emitted when read-only entries are dropped to ARCHIVE so the user -//! knows. +//! - The `EditableFilesystem` write path stamps a fresh ARCHIVE attribute, +//! so the **read-only** flag is dropped to ARCHIVE. A single summary +//! warning is emitted when read-only entries are dropped so the user +//! knows. Modification **dates** ARE carried across: each entry's +//! `modified_unix` is passed into `CreateFileOptions.unix_times` / +//! `CreateDirectoryOptions.unix_times`, and the target driver stamps +//! the DOS date/time verbatim. //! //! Memory / scope notes: //! - File contents are spooled through a `Vec` per file, then written @@ -154,9 +155,15 @@ where if k.special_type.as_deref() == Some("R/O") { *readonly_dropped += 1; } + // Carry the source's modification date through so the target + // records it verbatim instead of stamping `now`. + let src_times = k.modified_unix.map(super::times::UnixTimes::mtime_only); if k.is_directory() { - let new_dir = - target.create_directory(tgt_dir, &k.name, &CreateDirectoryOptions::default())?; + let dir_opts = CreateDirectoryOptions { + unix_times: src_times, + ..Default::default() + }; + let new_dir = target.create_directory(tgt_dir, &k.name, &dir_opts)?; report.dirs_copied += 1; progress.tick(report, false); walk( @@ -172,13 +179,11 @@ where // Spool the file body through a Vec. Peak RAM is one file. let body = source.read_file(&k, usize::MAX)?; let mut cur = Cursor::new(&body); - target.create_file( - tgt_dir, - &k.name, - &mut cur, - body.len() as u64, - &CreateFileOptions::default(), - )?; + let file_opts = CreateFileOptions { + unix_times: src_times, + ..Default::default() + }; + target.create_file(tgt_dir, &k.name, &mut cur, body.len() as u64, &file_opts)?; report.files_copied += 1; report.bytes_copied += body.len() as u64; progress.tick(report, false); diff --git a/src/fs/import_sink.rs b/src/fs/import_sink.rs index f798a892..444e5648 100644 --- a/src/fs/import_sink.rs +++ b/src/fs/import_sink.rs @@ -340,6 +340,7 @@ impl Importer { mode: Some(attrs.mode & 0o7777), uid: Some(attrs.uid), gid: Some(attrs.gid), + unix_times: overrides.unix_times, ..Default::default() }; match efs.create_symlink(&parent, name, &target, &link_opts) { @@ -372,6 +373,10 @@ impl Importer { uid: Some(attrs.uid), gid: Some(attrs.gid), xattrs: inherited_xattrs, + // Preserve source mtime end-to-end (host stat / tar Header + // / stage_copy source inode); the driver falls back to now + // when this is None (new blank file from rb-cli). + unix_times: overrides.unix_times, ..Default::default() }; efs.create_file(&parent, name, data, size, &create_opts) @@ -450,6 +455,7 @@ impl Importer { mode: Some(attrs.dir_mode()), uid: Some(attrs.uid), gid: Some(attrs.gid), + unix_times: overrides.unix_times, ..Default::default() }; let e = efs diff --git a/src/fs/jfs.rs b/src/fs/jfs.rs index ef09c3dd..004ffda6 100644 --- a/src/fs/jfs.rs +++ b/src/fs/jfs.rs @@ -1164,6 +1164,9 @@ impl JfsFilesystem { entry.modified = Some(super::unix_common::inode::format_unix_timestamp( child_dinode.mtime_seconds as i64, )); + if child_dinode.mtime_seconds > 0 { + entry.modified_unix = Some(child_dinode.mtime_seconds as u64); + } } Ok(entry) } @@ -2292,6 +2295,9 @@ impl Filesystem for JfsFilesystem { entry.modified = Some(super::unix_common::inode::format_unix_timestamp( root_dinode.mtime_seconds as i64, )); + if root_dinode.mtime_seconds > 0 { + entry.modified_unix = Some(root_dinode.mtime_seconds as u64); + } } Ok(entry) } diff --git a/src/fs/mfs.rs b/src/fs/mfs.rs index 04ce8060..2e0e39c9 100644 --- a/src/fs/mfs.rs +++ b/src/fs/mfs.rs @@ -1048,6 +1048,14 @@ impl Filesystem for MfsFilesystem { if de.rsrc_logical_length > 0 { fe.resource_fork_size = Some(de.rsrc_logical_length as u64); } + // MFS dates share HFS's Mac-epoch encoding. + fe.modified_unix = super::times::mac_epoch_to_unix(de.modify_date); + if de.modify_date != 0 || de.create_date != 0 { + fe.mac_dates = Some((de.create_date, de.modify_date, 0)); + if let Some(s) = super::hfs_common::format_mac_date(de.modify_date) { + fe.modified = Some(s); + } + } out.push(fe); } out.sort_by_key(|a| a.name.to_lowercase()); @@ -1210,6 +1218,15 @@ impl EditableFilesystem for MfsFilesystem { finder_info[0..4].copy_from_slice(&type_code); finder_info[4..8].copy_from_slice(&creator_code); + // Cross-fs import passes the source mtime through options.unix_times; + // a genuinely new file leaves it None and we stamp `now`. MFS's dir + // entry carries create + modify dates in Mac-epoch seconds — the same + // encoding HFS/HFS+ use. + let mtime_secs = options + .unix_times + .map(|t| t.mtime_or_now()) + .unwrap_or_else(super::times::now); + let stamp = super::times::unix_to_mac_epoch(mtime_secs); let entry = MfsDirEntry { flags: 0x80, // in use, not locked finder_info, @@ -1218,8 +1235,8 @@ impl EditableFilesystem for MfsFilesystem { data_logical_length: bytes.len() as u32, rsrc_first_block, rsrc_logical_length, - create_date: 0, - modify_date: 0, + create_date: stamp, + modify_date: stamp, name: name.to_string(), }; self.entries.push(entry); @@ -1236,6 +1253,8 @@ impl EditableFilesystem for MfsFilesystem { if rsrc_logical_length > 0 { fe.resource_fork_size = Some(rsrc_logical_length as u64); } + fe.modified_unix = Some(mtime_secs); + fe.mac_dates = Some((stamp, stamp, 0)); Ok(fe) } diff --git a/src/fs/minix.rs b/src/fs/minix.rs index 33e0297f..e7e5bec6 100644 --- a/src/fs/minix.rs +++ b/src/fs/minix.rs @@ -700,13 +700,21 @@ impl MinixFilesystem { None, ) }; + let modified_unix = if inode.mtime != 0 { + Some(inode.mtime as u64) + } else { + None + }; + let modified = + modified_unix.map(|s| crate::fs::unix_common::inode::format_unix_timestamp(s as i64)); FileEntry { name: name.to_string(), path, entry_type, size: inode.size as u64, location: inode.ino as u64, - modified: None, + modified, + modified_unix, type_code: None, creator_code: None, symlink_target, @@ -1452,6 +1460,8 @@ impl EditableFilesystem for MinixFilesystem { ino.nlinks = 1; ino.uid = options.uid.unwrap_or(0) as u16; ino.gid = options.gid.unwrap_or(0) as u16; + // Preserve source mtime on cross-fs copies; else stamp now. + ino.mtime = super::times::resolve_or_now(options.unix_times).mtime_or_now() as u32; if options.skip_data_write { ino.size = data_len as u32; } else { @@ -1518,6 +1528,7 @@ impl EditableFilesystem for MinixFilesystem { ino.nlinks = 1; ino.uid = options.uid.unwrap_or(0) as u16; ino.gid = options.gid.unwrap_or(0) as u16; + ino.mtime = super::times::resolve_or_now(options.unix_times).mtime_or_now() as u32; let mut data = target.as_bytes(); let len = data.len() as u64; self.write_file_data(&mut ino, &mut data, len)?; @@ -1561,6 +1572,7 @@ impl EditableFilesystem for MinixFilesystem { dir.nlinks = 2; // "." plus the parent's link dir.uid = options.uid.unwrap_or(0) as u16; dir.gid = options.gid.unwrap_or(0) as u16; + dir.mtime = super::times::resolve_or_now(options.unix_times).mtime_or_now() as u32; dir.size = (stride * 2) as u32; dir.zones[0] = zone; self.write_inode(&dir)?; diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 46c9da49..a177e02b 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -98,6 +98,7 @@ pub mod squashfs_write; pub mod tar_export; pub mod tar_import; pub mod ti99; +pub mod times; pub mod trdos; pub mod tree; pub mod ucsd; @@ -1226,6 +1227,10 @@ pub fn fs_name_for(partition_type: u8, partition_type_string: Option<&str>) -> & // Amiga boot block present, no AmigaDOS filesystem (custom // bootblock / diagnostic disk). Browsable via the carve view. "Amiga-NDOS" => "Amiga NDOS (no filesystem)", + // Container-identified, so content probing cannot name them and + // the write path called both "unknown" (R-034). + "lisafs" => "Apple Lisa File System", + "Alto BFS" => "Alto BFS", _ => "unknown", }; } @@ -2180,9 +2185,19 @@ pub fn open_editable_filesystem_with( reader, partition_offset, )?)), - _ => Err(FilesystemError::Unsupported(format!( - "editing not yet supported for filesystem type '{fs_type}'" - ))), + _ => { + // Name it the way the read path does. Detection returns + // "unknown" for filesystems identified by their container + // rather than a superblock, so reporting that told the user + // the disk was unreadable moments after `ls` read it (R-034). + let named = match fs_name_for(partition_type, partition_type_string) { + "unknown" => partition_type_string.unwrap_or(fs_type), + n => n, + }; + Err(FilesystemError::Unsupported(format!( + "editing not yet supported for filesystem type '{named}'" + ))) + } } } // FAT12 @@ -3727,3 +3742,136 @@ mod tests { } } } + +/// Whether `type_name` names an actual filesystem, or is the engine admitting +/// it did not recognise one. +/// +/// `inspect` opens anything — a disk with no recognisable filesystem still +/// yields a carve view, which is the point of a universal tool. That made +/// "`inspect` opened it" useless as a verification: an Apple DOS fixture that +/// was really a bare bootloader passed it and sat in the corpus for weeks +/// (R-031). The distinction has to be legible, so it lives here rather than in +/// any one caller. +pub fn is_identified_fs(type_name: &str) -> bool { + !matches!( + type_name.trim(), + "" | "Unknown" | "unknown" | "Unformatted" | "Free space" | "Empty" + ) +} + +/// Compare a filesystem name to a caller's expectation. +/// +/// Normalises case, spacing and punctuation, then compares **exactly** — a +/// substring rule would let `FAT` satisfy `exFAT`, which would make the flag +/// worse than no flag. +/// +/// Two wrinkles the type names force: +/// +/// - `+` becomes `plus`, so `HFS+` and `hfsplus` are the same answer and +/// neither is `HFS`. Stripping `+` as punctuation instead would collapse +/// `HFS+` onto `HFS` and quietly accept the wrong volume. +/// - A type byte shared by several filesystems is named for all of them +/// (`NTFS/HPFS/exFAT`, `HFS/HFS+`). Each alternative is matched separately, +/// so `--expect-fs HPFS` is satisfied by an `NTFS/HPFS/exFAT` partition. +/// Numeric shorthands like `ext2/3/4` only match their first alternative; +/// that misses rather than over-matches, which is the safe direction. +pub fn fs_name_matches(type_name: &str, expected: &str) -> bool { + fn norm(s: &str) -> String { + let mut out = String::new(); + for c in s.chars() { + if c == '+' { + out.push_str("plus"); + } else if c.is_ascii_alphanumeric() { + out.push(c.to_ascii_lowercase()); + } + } + out + } + let want = norm(expected); + if want.is_empty() { + return false; + } + // Whole name first: a caller who asks for exactly what `inspect` printed + // must always be satisfied. Splitting alone broke that — an ambiguous + // type-byte name like `NTFS/HPFS/exFAT` stopped matching itself, which the + // corpus identity check found the first time it ran. + if norm(type_name) == want { + return true; + } + // Then each alternative, so `HPFS` satisfies `NTFS/HPFS/exFAT`: type byte + // 0x07 names three filesystems and the table cannot say which. + type_name.split('/').any(|alt| norm(alt) == want) +} + +#[cfg(test)] +mod identification_tests { + use super::*; + + #[test] + fn unknown_is_not_an_identification() { + assert!(!is_identified_fs("Unknown")); + assert!(!is_identified_fs("")); + assert!(!is_identified_fs(" ")); + assert!(is_identified_fs("DOS 3.3")); + assert!(is_identified_fs("HPFS")); + } + + #[test] + fn expectations_ignore_case_and_punctuation() { + assert!(fs_name_matches("DOS 3.3", "dos3.3")); + assert!(fs_name_matches("DOS 3.3", "DOS 3.3")); + assert!(fs_name_matches("Apple DOS 3.3", "appledos33")); + } + + #[test] + fn a_name_always_matches_itself() { + // Regression: `NTFS/HPFS/exFAT` did not match itself, because only the + // `/`-alternatives were compared. + for n in [ + "NTFS/HPFS/exFAT", + "DOS 3.3", + "HPFS", + "Amiga NDOS (no filesystem)", + ] { + assert!(fs_name_matches(n, n), "{n} must match itself"); + } + } + + #[test] + fn an_alternative_satisfies_an_ambiguous_type_byte() { + // Type byte 0x07 names three filesystems; the table cannot say which. + assert!(fs_name_matches("NTFS/HPFS/exFAT", "HPFS")); + assert!(fs_name_matches("NTFS/HPFS/exFAT", "ntfs")); + assert!(!fs_name_matches("NTFS/HPFS/exFAT", "ext4")); + } + + #[test] + fn a_substring_is_not_a_match() { + // The reason this is an exact compare: FAT must not satisfy exFAT. + assert!(!fs_name_matches("exFAT", "FAT")); + assert!(!fs_name_matches("FAT", "exFAT")); + assert!(!fs_name_matches("DOS 3.3", "")); + } + + #[test] + fn plus_is_a_letter_not_punctuation() { + // The first version stripped `+` as punctuation, which collapsed HFS+ + // onto HFS — so `--expect-fs HFS` accepted an HFS+ volume, the exact + // false positive the exact-match rule exists to prevent. + assert!(fs_name_matches("HFS+", "hfsplus")); + assert!(fs_name_matches("HFS+", "HFS+")); + assert!(!fs_name_matches("HFS+", "HFS")); + assert!(!fs_name_matches("HFS", "HFS+")); + } + + #[test] + fn a_shared_type_byte_matches_any_of_its_names() { + // MBR 0x07 is named for everything it can be; asking for one of them + // is a fair question. This is what the OS/2 HPFS fixture reports. + assert!(fs_name_matches("NTFS/HPFS/exFAT", "HPFS")); + assert!(fs_name_matches("NTFS/HPFS/exFAT", "ntfs")); + assert!(fs_name_matches("HFS/HFS+", "HFS")); + assert!(fs_name_matches("HFS/HFS+", "hfsplus")); + assert!(!fs_name_matches("NTFS/HPFS/exFAT", "FAT")); + } +} diff --git a/src/fs/ntfs.rs b/src/fs/ntfs.rs index aa753daa..aba76cdb 100644 --- a/src/fs/ntfs.rs +++ b/src/fs/ntfs.rs @@ -1171,6 +1171,12 @@ impl NtfsFilesystem { let real_size = u64::from_le_bytes([ data[48], data[49], data[50], data[51], data[52], data[53], data[54], data[55], ]); + // $FILE_NAME's LastModifiedTime lives at offset 16..24 — 8-byte FILETIME + // (100-ns intervals since 1601-01-01 UTC). + let modify_ft = u64::from_le_bytes([ + data[16], data[17], data[18], data[19], data[20], data[21], data[22], data[23], + ]); + let modified_unix = super::times::filetime_to_unix(modify_ft); let name_length = data[64] as usize; let name_type = data[65]; // 0=POSIX, 1=Win32, 2=DOS, 3=Win32+DOS @@ -1201,11 +1207,13 @@ impl NtfsFilesystem { format!("{parent_path}/{name}") }; - if is_dir { - Some(FileEntry::new_directory(name, path, file_mft_ref)) + let mut fe = if is_dir { + FileEntry::new_directory(name, path, file_mft_ref) } else { - Some(FileEntry::new_file(name, path, real_size, file_mft_ref)) - } + FileEntry::new_file(name, path, real_size, file_mft_ref) + }; + fe.modified_unix = modified_unix; + Some(fe) } // ---- fsck helpers (see ntfs_fsck.rs) ---- @@ -1364,6 +1372,7 @@ impl Filesystem for NtfsFilesystem { size: 0, location: MFT_RECORD_ROOT, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -3894,7 +3903,7 @@ impl EditableFilesystem for NtfsFilesystem { name: &str, data: &mut dyn std::io::Read, data_len: u64, - _options: &CreateFileOptions, + options: &CreateFileOptions, ) -> Result { validate_ntfs_name(name)?; @@ -3966,13 +3975,18 @@ impl EditableFilesystem for NtfsFilesystem { } }); // One stamp for both structures: Windows writes them equal at creation. - let now = now_ntfs_timestamp(); + // `options.unix_times` (when set) is a source mtime from an import / + // cross-fs copy; fall back to wall-clock `now` for a genuinely new file. + let stamp = options + .unix_times + .map(|t| super::times::unix_to_filetime(t.mtime_or_now())) + .unwrap_or_else(now_ntfs_timestamp); let std_info = build_resident_attr( ATTR_STANDARD_INFORMATION, - &build_standard_information(FILE_ATTR_ARCHIVE, sec_id, now), + &build_standard_information(FILE_ATTR_ARCHIVE, sec_id, stamp), ); let parent_ref = self.file_reference(parent_record_num); - let file_name_value = build_file_name_attr(parent_ref, name, false, data_len, now); + let file_name_value = build_file_name_attr(parent_ref, name, false, data_len, stamp); let file_name_attr = build_resident_attr(ATTR_FILE_NAME, &file_name_value); // 3.x resolves the ACL through $Secure by the inherited id; a per-file @@ -4004,19 +4018,16 @@ impl EditableFilesystem for NtfsFilesystem { format!("{}/{name}", parent.path) }; - Ok(FileEntry::new_file( - name.to_string(), - path, - data_len, - record_num, - )) + let mut fe = FileEntry::new_file(name.to_string(), path, data_len, record_num); + fe.modified_unix = super::times::filetime_to_unix(stamp); + Ok(fe) } fn create_directory( &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { validate_ntfs_name(name)?; @@ -4043,13 +4054,16 @@ impl EditableFilesystem for NtfsFilesystem { } }); // Directories carry no archive bit; their directory flag lives in $FILE_NAME. - let now = now_ntfs_timestamp(); + let stamp = options + .unix_times + .map(|t| super::times::unix_to_filetime(t.mtime_or_now())) + .unwrap_or_else(now_ntfs_timestamp); let std_info = build_resident_attr( ATTR_STANDARD_INFORMATION, - &build_standard_information(0, sec_id, now), + &build_standard_information(0, sec_id, stamp), ); let parent_ref = self.file_reference(parent_record_num); - let file_name_value = build_file_name_attr(parent_ref, name, true, 0, now); + let file_name_value = build_file_name_attr(parent_ref, name, true, 0, stamp); let file_name_attr = build_resident_attr(ATTR_FILE_NAME, &file_name_value); let index_root = build_named_resident_attr( ATTR_INDEX_ROOT, @@ -4089,7 +4103,9 @@ impl EditableFilesystem for NtfsFilesystem { format!("{}/{name}", parent.path) }; - Ok(FileEntry::new_directory(name.to_string(), path, record_num)) + let mut fe = FileEntry::new_directory(name.to_string(), path, record_num); + fe.modified_unix = super::times::filetime_to_unix(stamp); + Ok(fe) } fn delete_entry( diff --git a/src/fs/os9.rs b/src/fs/os9.rs index 63b351c0..3c052908 100644 --- a/src/fs/os9.rs +++ b/src/fs/os9.rs @@ -178,6 +178,9 @@ impl Os9Ident { struct FileDescriptor { attributes: u8, size: u64, + /// FD.DAT (last-modified) — 5 bytes at offset 3: year-1900, month, day, + /// hour, minute. All zero when not recorded. + dat: [u8; 5], /// Segment list: `(lsn, sector_count)` runs. segments: Vec<(u64, u64)>, } @@ -185,6 +188,7 @@ struct FileDescriptor { impl FileDescriptor { fn parse(fd: &[u8]) -> FileDescriptor { let attributes = fd[0]; + let dat = [fd[3], fd[4], fd[5], fd[6], fd[7]]; let size = u32::from_be_bytes([fd[9], fd[10], fd[11], fd[12]]) as u64; let mut segments = Vec::new(); let mut off = SEG_LIST_OFFSET; @@ -203,6 +207,7 @@ impl FileDescriptor { FileDescriptor { attributes, size, + dat, segments, } } @@ -422,11 +427,14 @@ impl Filesystem for Os9Filesystem { for (name, lsn) in children { let path = format!("{base}/{name}"); let child_fd = self.read_fd(lsn)?; - if child_fd.is_dir() { - out.push(FileEntry::new_directory(name, path, lsn)); + let modified_unix = crate::fs::times::os9_dat_to_unix(&child_fd.dat); + let mut fe = if child_fd.is_dir() { + FileEntry::new_directory(name, path, lsn) } else { - out.push(FileEntry::new_file(name, path, child_fd.size, lsn)); - } + FileEntry::new_file(name, path, child_fd.size, lsn) + }; + fe.modified_unix = modified_unix; + out.push(fe); } Ok(out) } @@ -801,9 +809,23 @@ impl Os9Filesystem { } /// Build a 256-byte FD image with the given attributes, size and segments. - fn build_fd(attributes: u8, size: u64, segments: &[(u64, u64)]) -> [u8; 256] { + /// `mtime_secs = Some(secs)` stamps that Unix time into FD.DAT (5-byte + /// year-month-day-hour-minute at offset 3) and FD.DCR (3-byte + /// year-month-day at offset 13); `None` leaves both fields zero (the + /// pre-existing behaviour — OS-9 tools handle a zero timestamp as + /// "no date recorded"). + fn build_fd( + attributes: u8, + size: u64, + segments: &[(u64, u64)], + mtime_secs: Option, + ) -> [u8; 256] { let mut fd = [0u8; 256]; fd[0] = attributes; + if let Some(s) = mtime_secs { + fd[3..8].copy_from_slice(&crate::fs::times::unix_to_os9_dat(s)); + fd[13..16].copy_from_slice(&crate::fs::times::unix_to_os9_dcr(s)); + } fd[8] = 1; // link count fd[9..13].copy_from_slice(&(size as u32).to_be_bytes()); let mut off = SEG_LIST_OFFSET; @@ -910,7 +932,7 @@ impl Os9Filesystem { dir_fd.size = new_size; self.write_dir_slot(&dir_fd, slot, &entry)?; // Persist the updated directory FD (size + possibly new segment). - let fd_img = Self::build_fd(dir_fd.attributes, dir_fd.size, &dir_fd.segments); + let fd_img = Self::build_fd(dir_fd.attributes, dir_fd.size, &dir_fd.segments, None); self.write_sectors(dir_lsn, &fd_img)?; Ok(()) } @@ -1001,7 +1023,7 @@ impl EditableFilesystem for Os9Filesystem { name: &str, data: &mut dyn std::io::Read, data_len: u64, - _options: &CreateFileOptions, + options: &CreateFileOptions, ) -> Result { let dir_lsn = self.dir_fd_lsn(parent); let name_raw = encode_os9_name(name)?; @@ -1049,8 +1071,11 @@ impl EditableFilesystem for Os9Filesystem { self.write_sectors(lsn, &buf)?; } - // Write the FD (regular file: owner read+write). - let fd_img = Self::build_fd(0x03, payload.len() as u64, &data_segs); + // Write the FD (regular file: owner read+write). Cross-fs copy passes + // source mtime through options.unix_times; a genuinely new file leaves + // it None and the FD's date fields stay zero. + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let fd_img = Self::build_fd(0x03, payload.len() as u64, &data_segs, mtime); self.write_sectors(fd_lsn, &fd_img)?; // Link it into the parent directory. @@ -1064,19 +1089,16 @@ impl EditableFilesystem for Os9Filesystem { } else { format!("{}/{name}", parent.path) }; - Ok(FileEntry::new_file( - name.to_string(), - path, - payload.len() as u64, - fd_lsn, - )) + let mut fe = FileEntry::new_file(name.to_string(), path, payload.len() as u64, fd_lsn); + fe.modified_unix = mtime; + Ok(fe) } fn create_directory( &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { let parent_lsn = self.dir_fd_lsn(parent); let name_raw = encode_os9_name(name)?; @@ -1110,7 +1132,8 @@ impl EditableFilesystem for Os9Filesystem { self.write_sectors(data_lsn, &dir_data)?; // Directory FD: directory attribute + full perms, size = 2 entries. - let fd_img = Self::build_fd(ATT_DIR | 0x3F, (2 * DIR_ENTRY_LEN) as u64, &data_seg); + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let fd_img = Self::build_fd(ATT_DIR | 0x3F, (2 * DIR_ENTRY_LEN) as u64, &data_seg, mtime); self.write_sectors(fd_lsn, &fd_img)?; self.dir_add_entry(parent_lsn, &name_raw, fd_lsn)?; @@ -1123,7 +1146,9 @@ impl EditableFilesystem for Os9Filesystem { } else { format!("{}/{name}", parent.path) }; - Ok(FileEntry::new_directory(name.to_string(), path, fd_lsn)) + let mut fe = FileEntry::new_directory(name.to_string(), path, fd_lsn); + fe.modified_unix = mtime; + Ok(fe) } fn delete_entry( diff --git a/src/fs/pfs3.rs b/src/fs/pfs3.rs index bcb70f5b..3a177ddd 100644 --- a/src/fs/pfs3.rs +++ b/src/fs/pfs3.rs @@ -1007,6 +1007,11 @@ impl Filesystem for Pfs3Filesystem { } }; fe.modified = datestamp_string(de.cd as i32, de.cm as i32, de.ct as i32); + let unix = + super::affs_common::datestamp_to_unix(de.cd as i32, de.cm as i32, de.ct as i32); + if unix > 0 { + fe.modified_unix = Some(unix as u64); + } fe.amiga_protection = Some(de.protection as u32); if !de.comment.is_empty() { fe.amiga_comment = Some(de.comment.clone()); @@ -3129,7 +3134,14 @@ impl super::filesystem::EditableFilesystem for Pf ) -> Result { let comment = options.amiga_comment.as_deref().unwrap_or(""); let protection = options.amiga_protection.unwrap_or(0) as u8; - let dates = options.amiga_dates; + // Explicit amiga_dates always win; otherwise convert unix_times mtime + // so a cross-fs copy from a non-Amiga source (HFS+/ext/tar) still + // records the source date on PFS3. + let dates = options.amiga_dates.or_else(|| { + options + .unix_times + .map(|t| super::affs_common::unix_to_datestamp(t.mtime_or_now())) + }); let snap = self.snapshot(); match self.do_create_file(parent, name, data, data_len, comment, protection, dates) { Ok(fe) => Ok(fe), @@ -3148,7 +3160,11 @@ impl super::filesystem::EditableFilesystem for Pf ) -> Result { let comment = options.amiga_comment.as_deref().unwrap_or(""); let protection = options.amiga_protection.unwrap_or(0) as u8; - let dates = options.amiga_dates; + let dates = options.amiga_dates.or_else(|| { + options + .unix_times + .map(|t| super::affs_common::unix_to_datestamp(t.mtime_or_now())) + }); let snap = self.snapshot(); match self.do_create_directory(parent, name, comment, protection, dates) { Ok(fe) => Ok(fe), @@ -3172,7 +3188,11 @@ impl super::filesystem::EditableFilesystem for Pf ) -> Result { let comment = options.amiga_comment.as_deref().unwrap_or(""); let protection = options.amiga_protection.unwrap_or(0) as u8; - let dates = options.amiga_dates; + let dates = options.amiga_dates.or_else(|| { + options + .unix_times + .map(|t| super::affs_common::unix_to_datestamp(t.mtime_or_now())) + }); let snap = self.snapshot(); match self.do_create_symlink(parent, name, target, comment, protection, dates) { Ok(fe) => Ok(fe), @@ -3192,7 +3212,11 @@ impl super::filesystem::EditableFilesystem for Pf ) -> Result { let comment = options.amiga_comment.as_deref().unwrap_or(""); let protection = options.amiga_protection.unwrap_or(0) as u8; - let dates = options.amiga_dates; + let dates = options.amiga_dates.or_else(|| { + options + .unix_times + .map(|t| super::affs_common::unix_to_datestamp(t.mtime_or_now())) + }); let snap = self.snapshot(); match self.do_create_hardlink(parent, name, target, comment, protection, dates) { Ok(fe) => Ok(fe), diff --git a/src/fs/prodos.rs b/src/fs/prodos.rs index 0ed61fb2..4636554c 100644 --- a/src/fs/prodos.rs +++ b/src/fs/prodos.rs @@ -53,6 +53,7 @@ impl Filesystem for ProDosFilesystem { size: 0, location: 2, // Volume Directory Key Block is always at block 2 modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -160,7 +161,7 @@ impl Filesystem for ProDosFilesystem { } fn validate_name(&self, name: &str) -> Result<(), FilesystemError> { - validate_prodos_name(name).map(|_| ()) + validate_prodos_name(name, "filename").map(|_| ()) } fn total_size(&self) -> u64 { @@ -974,6 +975,12 @@ impl ProDosFilesystem { // ─────────────────────────────── entry builders ───────────────────────────── /// Build a 39-byte ProDOS file directory entry. +/// +/// `mtime_secs = Some(secs)` stamps that Unix time (converted via +/// [`crate::fs::times::unix_to_prodos_datetime`]) into both the creation +/// and modification date/time fields — what a cross-fs copy carrying the +/// source mtime through calls for. `None` stamps `now`. +#[allow(clippy::too_many_arguments)] fn build_file_entry_bytes( name: &str, file_type: u8, @@ -982,6 +989,7 @@ fn build_file_entry_bytes( key_ptr: u16, blocks_used: u16, eof: u32, + mtime_secs: Option, ) -> [u8; 39] { let mut e = [0u8; 39]; let name_bytes = name.as_bytes(); @@ -998,7 +1006,10 @@ fn build_file_entry_bytes( e[21] = eof as u8; e[22] = (eof >> 8) as u8; e[23] = (eof >> 16) as u8; - let (date, time) = make_prodos_datetime_now(); + let (date, time) = match mtime_secs { + Some(s) => crate::fs::times::unix_to_prodos_datetime(s), + None => make_prodos_datetime_now(), + }; let cd = date.to_le_bytes(); e[24] = cd[0]; e[25] = cd[1]; @@ -1021,7 +1032,8 @@ fn build_file_entry_bytes( } /// Build a 39-byte subdirectory entry (type nibble 0xD) for the parent directory. -fn build_subdir_entry_bytes(name: &str, key_ptr: u16) -> [u8; 39] { +/// See [`build_file_entry_bytes`] for the mtime semantics. +fn build_subdir_entry_bytes(name: &str, key_ptr: u16, mtime_secs: Option) -> [u8; 39] { let mut e = [0u8; 39]; let name_bytes = name.as_bytes(); let name_len = name_bytes.len().min(15) as u8; @@ -1034,7 +1046,10 @@ fn build_subdir_entry_bytes(name: &str, key_ptr: u16) -> [u8; 39] { // blocks_used = 1 (the key block itself) e[19] = 1; e[20] = 0; - let (date, time) = make_prodos_datetime_now(); + let (date, time) = match mtime_secs { + Some(s) => crate::fs::times::unix_to_prodos_datetime(s), + None => make_prodos_datetime_now(), + }; let cd = date.to_le_bytes(); e[24] = cd[0]; e[25] = cd[1]; @@ -1050,7 +1065,13 @@ fn build_subdir_entry_bytes(name: &str, key_ptr: u16) -> [u8; 39] { } /// Build a 39-byte subdirectory header entry (type nibble 0xE) for slot 0 of a new dir block. -fn build_subdir_header_bytes(name: &str, parent_key_block: u16, parent_entry_num: u8) -> [u8; 39] { +/// See [`build_file_entry_bytes`] for the mtime semantics. +fn build_subdir_header_bytes( + name: &str, + parent_key_block: u16, + parent_entry_num: u8, + mtime_secs: Option, +) -> [u8; 39] { let mut e = [0u8; 39]; let name_bytes = name.as_bytes(); let name_len = name_bytes.len().min(15) as u8; @@ -1060,7 +1081,10 @@ fn build_subdir_header_bytes(name: &str, parent_key_block: u16, parent_entry_num e[16] = 0x75; // Bytes 17-23: reserved // Bytes 24-25: creation date, 26-27: creation time - let (date, time) = make_prodos_datetime_now(); + let (date, time) = match mtime_secs { + Some(s) => crate::fs::times::unix_to_prodos_datetime(s), + None => make_prodos_datetime_now(), + }; let cd = date.to_le_bytes(); e[24] = cd[0]; e[25] = cd[1]; @@ -1139,26 +1163,30 @@ fn days_to_ymd(days: i64) -> (i32, u32, u32) { (year as i32, m, d) } -/// Validate a ProDOS filename: 1-15 chars, A-Z/0-9/period, first char must be a letter. -/// Returns the uppercase name. -fn validate_prodos_name(name: &str) -> Result { +/// Validate a ProDOS name: 1-15 chars, A-Z/0-9/period, first char must be a +/// letter. Returns the uppercase name. +/// +/// `what` names the thing being validated — "filename" or "volume name". The +/// rules govern both, but every message said "rename the file" even when the +/// offending string was a volume name (R-006). +fn validate_prodos_name(name: &str, what: &str) -> Result { if name.is_empty() { - return Err(FilesystemError::InvalidData( - "filename is empty — pick a non-blank name".into(), - )); + return Err(FilesystemError::InvalidData(format!( + "{what} is empty — pick a non-blank name" + ))); } let upper = name.to_ascii_uppercase(); let bytes = upper.as_bytes(); if bytes.len() > 15 { return Err(FilesystemError::InvalidData(format!( - "filename is too long ({} chars); ProDOS allows up to 15 — shorten the name", + "{what} is too long ({} chars); ProDOS allows up to 15 — shorten it", bytes.len() ))); } if !bytes[0].is_ascii_alphabetic() { return Err(FilesystemError::InvalidData(format!( - "filename starts with '{}' — ProDOS requires the first character to be a letter \ - (A-Z); rename so it begins with a letter", + "{what} starts with '{}' — ProDOS requires the first character to be a letter \ + (A-Z); change it to begin with a letter", upper.chars().next().unwrap_or('?') ))); } @@ -1166,8 +1194,8 @@ fn validate_prodos_name(name: &str) -> Result { if !b.is_ascii_alphanumeric() && b != b'.' { let c = upper.chars().nth(i).unwrap_or('?'); return Err(FilesystemError::InvalidData(format!( - "filename contains '{c}' — ProDOS allows only letters (A-Z), digits (0-9), \ - and '.'; rename the file (spaces and most punctuation are not allowed)" + "{what} contains '{c}' — ProDOS allows only letters (A-Z), digits (0-9), \ + and '.' (spaces and most punctuation are not allowed)" ))); } } @@ -1218,7 +1246,7 @@ impl EditableFilesystem for ProDosFilesystem { data_len: u64, options: &CreateFileOptions, ) -> Result { - let validated_name = validate_prodos_name(name)?; + let validated_name = validate_prodos_name(name, "filename")?; let dir_key_block = parent.location as u16; // Check for duplicate @@ -1256,7 +1284,10 @@ impl EditableFilesystem for ProDosFilesystem { (kp, bu, 3u8) }; - // Build directory entry + // Build directory entry. Cross-fs copy passes source mtime through + // options.unix_times; a genuinely new file leaves it None and stamps + // `now`. ProDOS is minute-granular so sub-minute precision is lost. + let mtime = options.unix_times.map(|t| t.mtime_or_now()); let entry_bytes = build_file_entry_bytes( &validated_name, file_type, @@ -1265,6 +1296,7 @@ impl EditableFilesystem for ProDosFilesystem { key_ptr, blocks_used, eof, + mtime, ); // Find slot and write @@ -1282,6 +1314,7 @@ impl EditableFilesystem for ProDosFilesystem { fe.prodos_file_type = Some(file_type); fe.aux_type = Some(aux_type); fe.mode = Some(storage_type as u32); + fe.modified_unix = mtime.or_else(|| Some(crate::fs::times::now())); Ok(fe) } @@ -1289,9 +1322,9 @@ impl EditableFilesystem for ProDosFilesystem { &mut self, parent: &FileEntry, name: &str, - _options: &CreateDirectoryOptions, + options: &CreateDirectoryOptions, ) -> Result { - let validated_name = validate_prodos_name(name)?; + let validated_name = validate_prodos_name(name, "filename")?; let parent_key_block = parent.location as u16; // Check for duplicate @@ -1309,7 +1342,9 @@ impl EditableFilesystem for ProDosFilesystem { let (entry_block, entry_slot) = self.find_free_dir_slot(parent_key_block)?; // Build subdirectory key block with header at slot 0 - let header = build_subdir_header_bytes(&validated_name, parent_key_block, entry_slot as u8); + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let header = + build_subdir_header_bytes(&validated_name, parent_key_block, entry_slot as u8, mtime); let mut new_block_data = [0u8; 512]; // prev_block = 0, next_block = 0 (bytes 0-3) // Header at slot 0 (offset 4) @@ -1317,7 +1352,7 @@ impl EditableFilesystem for ProDosFilesystem { self.write_block(new_key_block, &new_block_data)?; // Build parent directory entry (type 0xD) - let subdir_entry = build_subdir_entry_bytes(&validated_name, new_key_block); + let subdir_entry = build_subdir_entry_bytes(&validated_name, new_key_block, mtime); self.write_dir_entry(entry_block, entry_slot, &subdir_entry)?; self.update_dir_file_count(parent_key_block, 1)?; @@ -1327,11 +1362,9 @@ impl EditableFilesystem for ProDosFilesystem { format!("{}/{validated_name}", parent.path) }; - Ok(FileEntry::new_directory( - validated_name, - path, - new_key_block as u64, - )) + let mut fe = FileEntry::new_directory(validated_name, path, new_key_block as u64); + fe.modified_unix = mtime.or_else(|| Some(crate::fs::times::now())); + Ok(fe) } fn delete_entry( @@ -1439,7 +1472,7 @@ impl EditableFilesystem for ProDosFilesystem { return Ok(()); } // Uppercases and validates (length / first-letter / allowed chars). - let validated = validate_prodos_name(new_name)?; + let validated = validate_prodos_name(new_name, "filename")?; let dir_key_block = parent.location as u16; // Reject a collision with a *different* entry. ProDOS folds case, so a @@ -1724,7 +1757,7 @@ pub fn validate_prodos_integrity( /// block below is reserved by clearing its bit. The result round-trips through /// [`ProDosFilesystem::open`] and passes `fsck` clean. pub fn create_blank_prodos(size_bytes: u64, name: &str) -> anyhow::Result> { - let vname = validate_prodos_name(name).map_err(|e| anyhow::anyhow!("{e}"))?; + let vname = validate_prodos_name(name, "volume name").map_err(|e| anyhow::anyhow!("{e}"))?; // total_blocks is a u16 field: ProDOS tops out at 65535 blocks (~32 MiB). // Floor at 16 blocks (8 KiB) so boot + directory + bitmap always fit with @@ -1948,16 +1981,22 @@ fn list_prodos_directory( let modified = parse_prodos_datetime(modified_date, modified_time) .or_else(|| parse_prodos_datetime(creation_date, creation_time)); + let modified_unix = + crate::fs::times::prodos_datetime_to_unix(modified_date, modified_time).or_else( + || crate::fs::times::prodos_datetime_to_unix(creation_date, creation_time), + ); if type_nibble == 0xD { // Subdirectory entry: key_pointer is the subdirectory key block. let mut fe = FileEntry::new_directory(name, path, key_pointer as u64); fe.modified = modified; + fe.modified_unix = modified_unix; result.push(fe); } else { // Regular file: seedling (1), sapling (2), or tree (3). let mut fe = FileEntry::new_file(name, path, eof as u64, key_pointer as u64); fe.modified = modified; + fe.modified_unix = modified_unix; fe.prodos_file_type = Some(file_type); fe.aux_type = Some(aux_type); // Store storage type in `mode` so read_file can dispatch correctly. @@ -2767,21 +2806,21 @@ mod tests { #[test] fn test_name_validation() { - assert!(validate_prodos_name("HELLO").is_ok()); - assert!(validate_prodos_name("A.FILE").is_ok()); - assert!(validate_prodos_name("hello").is_ok()); // lowercased → OK - assert_eq!(validate_prodos_name("hello").unwrap(), "HELLO"); + assert!(validate_prodos_name("HELLO", "filename").is_ok()); + assert!(validate_prodos_name("A.FILE", "filename").is_ok()); + assert!(validate_prodos_name("hello", "filename").is_ok()); // lowercased → OK + assert_eq!(validate_prodos_name("hello", "filename").unwrap(), "HELLO"); // Invalid: empty - assert!(validate_prodos_name("").is_err()); + assert!(validate_prodos_name("", "filename").is_err()); // Invalid: too long (16 chars) - assert!(validate_prodos_name("ABCDEFGHIJKLMNOP").is_err()); + assert!(validate_prodos_name("ABCDEFGHIJKLMNOP", "filename").is_err()); // Invalid: starts with digit - assert!(validate_prodos_name("1FILE").is_err()); + assert!(validate_prodos_name("1FILE", "filename").is_err()); // Invalid: contains space - assert!(validate_prodos_name("MY FILE").is_err()); + assert!(validate_prodos_name("MY FILE", "filename").is_err()); // Invalid: special chars - assert!(validate_prodos_name("FILE/NAME").is_err()); + assert!(validate_prodos_name("FILE/NAME", "filename").is_err()); } /// Read the access byte for `path` directly from its on-disk directory diff --git a/src/fs/qdos.rs b/src/fs/qdos.rs index 798de3ef..f6c31292 100644 --- a/src/fs/qdos.rs +++ b/src/fs/qdos.rs @@ -213,6 +213,9 @@ pub struct QdosDirEntry { pub file_type: u16, pub name: String, pub first_block: u16, + /// Last-modified timestamp — u32 seconds since QDOS epoch (1961-01-01) + /// at directory-entry offset 0x34..0x38. Zero when unset. + pub update_date: u32, } pub fn parse_dir_entry(buf: &[u8; DIR_ENTRY_SIZE]) -> Option { @@ -228,6 +231,7 @@ pub fn parse_dir_entry(buf: &[u8; DIR_ENTRY_SIZE]) -> Option { let file_type = BigEndian::read_u16(&buf[0x06..0x08]); // First cluster of file at offset 0x3A (sQLux QWDE_FNUM). let first_block = BigEndian::read_u16(&buf[0x3A..0x3C]); + let update_date = BigEndian::read_u32(&buf[0x34..0x38]); let name_bytes = &buf[0x10..0x10 + name_len.min(36)]; let name: String = name_bytes .iter() @@ -245,6 +249,7 @@ pub fn parse_dir_entry(buf: &[u8; DIR_ENTRY_SIZE]) -> Option { file_type, name, first_block, + update_date, }) } @@ -375,6 +380,7 @@ impl Filesystem for QdosFilesystem { 3 => fe.special_type = Some("Dev".into()), _ => {} } + fe.modified_unix = crate::fs::times::qdos_date_to_unix(de.update_date); out.push(fe); } out.sort_by_key(|a| a.name.to_lowercase()); @@ -663,7 +669,7 @@ impl EditableFilesystem for QdosFilesystem { name: &str, data: &mut dyn std::io::Read, data_len: u64, - _options: &CreateFileOptions, + options: &CreateFileOptions, ) -> Result { if parent.path != "/" { return Err(FilesystemError::Unsupported( @@ -708,6 +714,13 @@ impl EditableFilesystem for QdosFilesystem { }; let first_cluster = self.alloc_chain(cluster_count)?; + // QDOS timestamps are u32 seconds since 1961. Cross-fs copy passes + // source mtime through options.unix_times; a genuinely new file leaves + // it None and the date field stays zero (matching pre-existing + // behaviour — QDOS has no on-disk "now" convention). + let mtime = options.unix_times.map(|t| t.mtime_or_now()); + let qdos_date = mtime.map(crate::fs::times::unix_to_qdos_date).unwrap_or(0); + // Compose the 64-byte file header (mirror of the directory entry) // and prepend it to the payload before writing the chain. let mut full = Vec::with_capacity(on_disk_len); @@ -715,6 +728,7 @@ impl EditableFilesystem for QdosFilesystem { BigEndian::write_u32(&mut full[0x00..0x04], on_disk_len as u32); BigEndian::write_u16(&mut full[0x0E..0x10], name.len() as u16); full[0x10..0x10 + name.len()].copy_from_slice(name.as_bytes()); + BigEndian::write_u32(&mut full[0x34..0x38], qdos_date); BigEndian::write_u16(&mut full[0x3A..0x3C], first_cluster); full.extend_from_slice(&payload); self.write_chain(first_cluster, &full)?; @@ -728,6 +742,7 @@ impl EditableFilesystem for QdosFilesystem { // access_keys (0x04..0x06), file_type (0x06..0x08): leave 0 (data). BigEndian::write_u16(&mut entry[0x0E..0x10], name.len() as u16); entry[0x10..0x10 + name.len()].copy_from_slice(name.as_bytes()); + BigEndian::write_u32(&mut entry[0x34..0x38], qdos_date); BigEndian::write_u16(&mut entry[0x3A..0x3C], first_cluster); self.reader.seek(SeekFrom::Start(slot_off))?; self.reader.write_all(&entry)?; @@ -736,12 +751,14 @@ impl EditableFilesystem for QdosFilesystem { // caller forgets sync_metadata. self.fat_write_back()?; self.header_write_back()?; - Ok(FileEntry::new_file( + let mut fe = FileEntry::new_file( name.to_string(), format!("/{name}"), payload.len() as u64, first_cluster as u64, - )) + ); + fe.modified_unix = mtime; + Ok(fe) } fn create_directory( diff --git a/src/fs/reiserfs.rs b/src/fs/reiserfs.rs index ecd7b1d1..2adf492b 100644 --- a/src/fs/reiserfs.rs +++ b/src/fs/reiserfs.rs @@ -693,6 +693,9 @@ impl Filesystem for ReiserFsFilesystem { child.uid = Some(child_sd.uid); child.gid = Some(child_sd.gid); child.modified = Some(format_unix_timestamp(child_sd.mtime as i64)); + if child_sd.mtime > 0 { + child.modified_unix = Some(child_sd.mtime as u64); + } children.push(child); } } diff --git a/src/fs/sfs.rs b/src/fs/sfs.rs index cc22507e..ba3b0068 100644 --- a/src/fs/sfs.rs +++ b/src/fs/sfs.rs @@ -948,8 +948,16 @@ impl Filesystem for SfsFilesystem { // re-derive first-data via lookup_object_block at read time. let _ = obj.data_or_hashtable; let _ = obj.protection; - let _ = obj.datemodified; let _ = obj.comment; + // SFS `datemodified` is 32-bit seconds since 1978-01-01. Shift + // into the Unix epoch for `modified_unix` + tar_export. + if obj.datemodified > 0 { + const AMIGA_EPOCH_SECS: u64 = 252_460_800; // 1978-01-01 UTC + fe.modified_unix = Some(AMIGA_EPOCH_SECS + obj.datemodified as u64); + fe.modified = Some(super::unix_common::inode::format_unix_timestamp( + fe.modified_unix.unwrap() as i64, + )); + } if obj.is_link() { // Mark with size 0; we don't follow softlinks yet. fe.size = 0; @@ -1361,6 +1369,21 @@ impl Read for CompactSfsReader { const FS_EXTENTBNODE_SIZE: usize = 14; const FS_OBJECTNODE_SIZE: usize = 10; +/// The SFS editor's ceiling, reported as `Unsupported` rather than `Parse`. +/// +/// A `Parse` error blamed the disk and exited 1, indistinguishable from a +/// corrupt image — R-034's shape. The volume is intact and reads fine; only +/// writing is refused. `Unsupported` routes through `write_open_error` to +/// PERMISSION_DENIED (4). Tracked as F-009. +fn multi_leaf_unsupported() -> FilesystemError { + FilesystemError::Unsupported( + "SFS extent b-tree has interior nodes; this editor writes single-leaf trees only, \ + which any volume of real size outgrows. The volume is intact and readable — only \ + writing is refused." + .to_string(), + ) +} + impl SfsRootBlock { /// Write the rootblock fields into the first portion of the block /// buffer. The `ownblock` field is left intact (callers stamp it). @@ -1627,9 +1650,7 @@ impl SfsFilesystem { let isleaf = buf[14]; let nodesize = buf[15] as usize; if isleaf == 0 { - return Err(parse_err( - "extent_btree_insert: only single-leaf BNDC supported", - )); + return Err(multi_leaf_unsupported()); } if nodesize != FS_EXTENTBNODE_SIZE { return Err(parse_err(format!( @@ -1680,9 +1701,7 @@ impl SfsFilesystem { let isleaf = buf[14]; let nodesize = buf[15] as usize; if isleaf == 0 { - return Err(parse_err( - "extent_btree_remove: only single-leaf BNDC supported", - )); + return Err(multi_leaf_unsupported()); } let entries_off = 16; let mut found: Option = None; @@ -1854,6 +1873,35 @@ impl SfsFilesystem { } } +/// SFS-native datemodified is seconds since 1978-01-01 UTC (the Amiga epoch, +/// same as `AMIGA_EPOCH_SECS` used on the read side). +const SFS_AMIGA_EPOCH_SECS: u64 = 252_460_800; + +/// Pick the on-disk `datemodified` for a new object. Explicit `amiga_dates` +/// wins (the Amiga-tools path); otherwise convert `unix_times.mtime_or_now()`; +/// otherwise zero (matching pre-existing behaviour — SFS creates recorded +/// `datemodified = 0` for every new file, which SFSFuse renders as 1978-01-01). +fn sfs_date_from_options( + amiga: Option<(i32, i32, i32)>, + unix: Option, +) -> u32 { + if let Some((days, mins, ticks)) = amiga { + let unix_secs = super::affs_common::datestamp_to_unix(days, mins, ticks).max(0) as u64; + return sfs_date_from_unix(unix_secs); + } + if let Some(t) = unix { + return sfs_date_from_unix(t.mtime_or_now()); + } + 0 +} + +fn sfs_date_from_unix(unix_secs: u64) -> u32 { + if unix_secs < SFS_AMIGA_EPOCH_SECS { + return 0; + } + ((unix_secs - SFS_AMIGA_EPOCH_SECS).min(u32::MAX as u64)) as u32 +} + /// Encode an fsObject for write. Returns the encoded bytes (length is even). fn build_object( objectnode: u32, @@ -1897,6 +1945,7 @@ impl SfsFilesystem { &mut self, parent: &FileEntry, name: &str, + sfs_date: u32, ) -> Result { check_no_duplicate(self, parent, name)?; let parent_node = parent.location as u32; @@ -1910,7 +1959,7 @@ impl SfsFilesystem { FIBF_READ | FIBF_WRITE | FIBF_EXECUTE | FIBF_DELETE, 0, 0, - 0, + sfs_date, OTYPE_DIR, name, "", @@ -1930,7 +1979,7 @@ impl SfsFilesystem { FIBF_READ | FIBF_WRITE | FIBF_EXECUTE | FIBF_DELETE, 0, 0, - 0, + sfs_date, OTYPE_DIR, name, "", @@ -1942,11 +1991,11 @@ impl SfsFilesystem { } else { format!("{}/{}", parent.path, name) }; - Ok(FileEntry::new_directory( - name.to_string(), - path, - new_node as u64, - )) + let mut fe = FileEntry::new_directory(name.to_string(), path, new_node as u64); + if sfs_date > 0 { + fe.modified_unix = Some(SFS_AMIGA_EPOCH_SECS + sfs_date as u64); + } + Ok(fe) } fn do_create_file( @@ -1955,6 +2004,7 @@ impl SfsFilesystem { name: &str, data: &mut dyn std::io::Read, data_len: u64, + sfs_date: u32, ) -> Result { check_no_duplicate(self, parent, name)?; let parent_node = parent.location as u32; @@ -1991,7 +2041,7 @@ impl SfsFilesystem { FIBF_READ | FIBF_WRITE | FIBF_EXECUTE | FIBF_DELETE, first_data, data_len as u32, - 0, + sfs_date, 0, name, "", @@ -2008,7 +2058,7 @@ impl SfsFilesystem { FIBF_READ | FIBF_WRITE | FIBF_EXECUTE | FIBF_DELETE, first_data, data_len as u32, - 0, + sfs_date, 0, name, "", @@ -2020,12 +2070,11 @@ impl SfsFilesystem { } else { format!("{}/{}", parent.path, name) }; - Ok(FileEntry::new_file( - name.to_string(), - path, - data_len, - new_node as u64, - )) + let mut fe = FileEntry::new_file(name.to_string(), path, data_len, new_node as u64); + if sfs_date > 0 { + fe.modified_unix = Some(SFS_AMIGA_EPOCH_SECS + sfs_date as u64); + } + Ok(fe) } fn do_rename( @@ -2409,10 +2458,14 @@ impl super::filesystem::EditableFilesystem for Sf name: &str, data: &mut dyn std::io::Read, data_len: u64, - _options: &super::filesystem::CreateFileOptions, + options: &super::filesystem::CreateFileOptions, ) -> Result { + // SFS stores its datemodified as seconds since 1978-01-01 (the Amiga + // epoch). Cross-fs copy passes source mtime through unix_times, or an + // explicit amiga_dates gives (days, mins, ticks) since 1978. + let sfs_date = sfs_date_from_options(options.amiga_dates, options.unix_times); let snap = self.snapshot(); - match self.do_create_file(parent, name, data, data_len) { + match self.do_create_file(parent, name, data, data_len, sfs_date) { Ok(fe) => Ok(fe), Err(e) => { self.restore_snapshot(snap); @@ -2425,10 +2478,11 @@ impl super::filesystem::EditableFilesystem for Sf &mut self, parent: &FileEntry, name: &str, - _options: &super::filesystem::CreateDirectoryOptions, + options: &super::filesystem::CreateDirectoryOptions, ) -> Result { + let sfs_date = sfs_date_from_options(options.amiga_dates, options.unix_times); let snap = self.snapshot(); - match self.do_create_directory(parent, name) { + match self.do_create_directory(parent, name, sfs_date) { Ok(fe) => Ok(fe), Err(e) => { self.restore_snapshot(snap); diff --git a/src/fs/squashfs.rs b/src/fs/squashfs.rs index d73109e0..0fa17fe0 100644 --- a/src/fs/squashfs.rs +++ b/src/fs/squashfs.rs @@ -747,6 +747,7 @@ impl SquashfsFilesystem { fe.modified = Some(super::unix_common::inode::format_unix_timestamp( inode.mtime as i64, )); + fe.modified_unix = Some(inode.mtime as u64); } if let Some(label) = inode.special_label() { fe.special_type = Some(match inode.kind { diff --git a/src/fs/squashfs_edit.rs b/src/fs/squashfs_edit.rs index 50b3210c..babac518 100644 --- a/src/fs/squashfs_edit.rs +++ b/src/fs/squashfs_edit.rs @@ -217,7 +217,10 @@ pub fn plan_size( /// An editable SquashFS image backed by an in-memory tree. pub struct SquashfsEditor { - rw: RW, + /// `None` once a replacement commit has released it — see + /// [`SquashfsEditor::commit_by_replacing`]. Only the in-place commit reads + /// it, and that path never releases it. + rw: Option, /// Byte offset of the image within `rw` (0 for a bare superfloppy). offset: u64, /// Bytes the image may occupy in its container, or `None` when it can grow @@ -310,7 +313,7 @@ impl SquashfsEditor { } Ok(Self { - rw, + rw: Some(rw), offset, capacity, budget, @@ -343,7 +346,10 @@ impl SquashfsEditor { /// The counterpart of [`SquashfsFilesystem::into_inner`], for a caller that /// wrapped the handle itself — an AppImage's payload window, say — and /// wants it back to inspect what landed. - pub fn into_backing(self) -> RW { + /// `None` after a replacement commit, which closes the handle before + /// renaming over it. Callers that wrap the handle themselves always commit + /// in place, so they always get it back. + pub fn into_backing(self) -> Option { self.rw } @@ -378,11 +384,17 @@ impl SquashfsEditor { /// before the rename so the rename cannot become visible ahead of the bytes /// it points at. /// - /// Note `self.rw` still refers to the *replaced* file afterwards. Nothing - /// reads through it — every read this editor serves comes from the - /// in-memory tree — and a second sync writes a fresh temp and renames - /// again, so the stale handle never matters. It is not reopened only - /// because `RW` is generic and there is nothing to reopen it *as*. + /// The handle is **closed before the rename**, and that is load-bearing on + /// Windows: a file marked for deletion keeps its name until the last handle + /// closes, so renaming over a file we still hold fails with + /// `Access is denied (os error 5)` — R-025, which made every SquashFS edit + /// and every `xattr set` unusable there while passing on Unix, where the + /// name is freed immediately. `FILE_SHARE_DELETE` does *not* fix it: it + /// lets the delete begin, not the name be reused. + /// + /// Closing is safe because nothing reads through the handle — every read + /// this editor serves comes from the in-memory tree — and a second sync + /// writes a fresh temp and renames again, needing only the path. fn commit_by_replacing( &mut self, path: &std::path::Path, @@ -406,6 +418,9 @@ impl SquashfsEditor { if let Ok(meta) = std::fs::metadata(path) { let _ = std::fs::set_permissions(tmp.path(), meta.permissions()); } + // Release our handle on the target before renaming over it. See the + // doc comment: on Windows the rename cannot reuse a name we still hold. + self.rw = None; tmp.persist(path).map_err(|e| { FilesystemError::Io(crate::compat::io_other(format!( "replacing {} with the rebuilt image: {e}", @@ -425,10 +440,16 @@ impl SquashfsEditor { /// lying around — they would be carried into a backup and could be /// mistaken for live data by anything scanning for magic bytes. Zero them. fn commit_in_place(&mut self, image: &[u8]) -> Result<(), FilesystemError> { - self.rw - .seek(SeekFrom::Start(self.offset)) + // Only reachable with `backing_file` None, which is exactly when the + // replacement path never ran and the handle is still ours. + let rw = self.rw.as_mut().ok_or_else(|| { + FilesystemError::Io(crate::compat::io_other( + "squashfs: the backing handle was released by a replacement commit", + )) + })?; + rw.seek(SeekFrom::Start(self.offset)) .map_err(FilesystemError::Io)?; - self.rw.write_all(image).map_err(FilesystemError::Io)?; + rw.write_all(image).map_err(FilesystemError::Io)?; let written = image.len() as u64; if written < self.source_len { @@ -436,13 +457,11 @@ impl SquashfsEditor { let zeros = vec![0u8; 64 * 1024]; while remaining > 0 { let n = remaining.min(zeros.len() as u64) as usize; - self.rw - .write_all(&zeros[..n]) - .map_err(FilesystemError::Io)?; + rw.write_all(&zeros[..n]).map_err(FilesystemError::Io)?; remaining -= n as u64; } } - self.rw.flush().map_err(FilesystemError::Io)?; + rw.flush().map_err(FilesystemError::Io)?; Ok(()) } @@ -555,6 +574,7 @@ impl SquashfsEditor { fe.modified = Some(super::unix_common::inode::format_unix_timestamp( node.mtime as i64, )); + fe.modified_unix = Some(node.mtime as u64); } if let BuildKind::BlockDev { major, minor } | BuildKind::CharDev { major, minor } = &node.kind @@ -710,6 +730,7 @@ impl EditableFilesystem for SquashfsEditor { mode: options.mode.map(|m| m & 0o7777), uid: options.uid, gid: options.gid, + unix_times: None, }, None, parent_entry.as_ref(), @@ -732,7 +753,7 @@ impl EditableFilesystem for SquashfsEditor { mode: attrs.mode as u16, uid: attrs.uid, gid: attrs.gid, - mtime: 0, + mtime: super::times::resolve_or_now(options.unix_times).mtime_or_now() as u32, // Carried from the file being replaced, when the caller captured // them (D4) — otherwise a replaced binary loses its // `security.capability` and quietly stops working. @@ -760,6 +781,7 @@ impl EditableFilesystem for SquashfsEditor { mode: options.mode.map(|m| m & 0o7777), uid: options.uid, gid: options.gid, + unix_times: None, }, None, parent_entry.as_ref(), @@ -777,7 +799,7 @@ impl EditableFilesystem for SquashfsEditor { mode: attrs.mode as u16, uid: attrs.uid, gid: attrs.gid, - mtime: 0, + mtime: super::times::resolve_or_now(options.unix_times).mtime_or_now() as u32, xattrs: Vec::new(), kind: BuildKind::Dir(Vec::new()), }); @@ -859,7 +881,7 @@ impl EditableFilesystem for SquashfsEditor { mode: 0o777, uid, gid, - mtime: 0, + mtime: super::times::resolve_or_now(options.unix_times).mtime_or_now() as u32, xattrs: Vec::new(), kind: BuildKind::Symlink(target.to_string()), }); @@ -1071,7 +1093,11 @@ mod tests { // Nothing was written: the backing store still holds the original image, // and everything after the partition is still zero. - let after = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let after = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); assert_eq!(after, before, "a refused rebuild still touched the disk"); } @@ -1100,7 +1126,11 @@ mod tests { .expect("create"); ed.sync_metadata().expect("sync must fit"); - let disk = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let disk = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); assert_eq!( disk.len() as u64, OFFSET + partition_len, @@ -1231,7 +1261,11 @@ mod tests { .expect("create"); assert_eq!(ed.free_space().unwrap(), u64::MAX, "a bare file has no cap"); ed.sync_metadata().expect("a bare file simply grows"); - let bytes = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let bytes = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); assert!( bytes.len() > 1 << 20, "the added megabyte did not land: {} bytes", @@ -1398,7 +1432,11 @@ mod tests { ) .expect("create"); ed.sync_metadata().expect("first sync"); - let disk = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let disk = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); // Reopen: `source_len` is measured at open, so this is the grown // image's real footprint — the region the shrink has to clean up. @@ -1421,7 +1459,11 @@ mod tests { ed.delete_entry(&root, &bulk).expect("delete"); ed.sync_metadata().expect("second sync"); - let disk = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let disk = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); let fs = SquashfsFilesystem::open(Cursor::new(disk.clone()), OFFSET).expect("reopen"); let used = image_footprint(fs.bytes_used()); assert!(used < grown_len, "the image did not actually shrink"); @@ -1531,7 +1573,10 @@ mod tests { // Incompressible content: the true size lands near the pessimistic end, // and the range must contain it. ed.sync_metadata().expect("sync"); - let actual = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())) + let actual = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") .into_inner() .len() as u64; assert!( @@ -1576,7 +1621,11 @@ mod tests { // Rebuild, reopen through the reader, and check the tree. ed.sync_metadata().expect("sync"); - let bytes = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let bytes = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); let mut fs = SquashfsFilesystem::open(Cursor::new(bytes), 0).expect("reopen"); let root = fs.root().unwrap(); let names: Vec = fs @@ -1628,7 +1677,11 @@ mod tests { .expect("create"); ed.sync_metadata().expect("sync"); - let bytes = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let bytes = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); let mut fs = SquashfsFilesystem::open(Cursor::new(bytes), 0).expect("reopen"); // Re-read the tree and confirm /bin/ping still has its capability. let tree = fs.read_build_tree().expect("read tree"); @@ -1728,7 +1781,11 @@ mod tests { assert!(ed.list_xattrs(&ping).unwrap().is_empty()); ed.sync_metadata().expect("sync"); - let bytes = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let bytes = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); let mut fs = SquashfsFilesystem::open(Cursor::new(bytes), 0).expect("reopen"); let tree = fs.read_build_tree().expect("read tree"); let BK::Dir(top) = &tree.kind else { panic!() }; @@ -1784,7 +1841,11 @@ mod tests { .expect("create replacement"); ed.sync_metadata().expect("sync"); - let bytes = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let bytes = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); let mut fs = SquashfsFilesystem::open(Cursor::new(bytes), 0).expect("reopen"); let tree = fs.read_build_tree().expect("read tree"); let BK::Dir(top) = &tree.kind else { panic!() }; @@ -1865,7 +1926,11 @@ mod tests { ) .expect("create"); ed.sync_metadata().expect("sync"); - let bytes = std::mem::replace(&mut ed.rw, Cursor::new(Vec::new())).into_inner(); + let bytes = ed + .rw + .replace(Cursor::new(Vec::new())) + .expect("an in-memory editor commits in place and keeps its handle") + .into_inner(); let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("edited.squashfs"); diff --git a/src/fs/tar_export.rs b/src/fs/tar_export.rs index 9a72817e..d9adf999 100644 --- a/src/fs/tar_export.rs +++ b/src/fs/tar_export.rs @@ -345,8 +345,11 @@ fn base_header(entry: &FileEntry, default_mode: u32) -> tar::Header { h.set_mode(mode); h.set_uid(entry.uid.unwrap_or(0) as u64); h.set_gid(entry.gid.unwrap_or(0) as u64); - // v1: mtimes aren't uniformly available across FS families; stamp 0. - h.set_mtime(0); + // Every fs driver populates `modified_unix` when its on-disk format + // carries a per-second date (Unix / Amiga / classic-Mac / DOS). `0` is + // the "unknown" case (Apple DOS 3.3, TRS-80, MFS, …) and lands as the + // Unix epoch — that's what tar's `set_mtime(0)` produced before too. + h.set_mtime(entry.modified_unix.unwrap_or(0)); h } diff --git a/src/fs/tar_import.rs b/src/fs/tar_import.rs index 845f93d8..4ecf0bce 100644 --- a/src/fs/tar_import.rs +++ b/src/fs/tar_import.rs @@ -365,13 +365,27 @@ pub fn preflight_tar( /// `tar` -> `untar` round-trip lose ownership outright and directory /// modes with it. fn archived_overrides(header: &tar::Header, apply: bool) -> crate::fs::attrs::AttrOverrides { + // Header mtime is always harvested (independent of `apply_permissions`) + // so a tar extract carries the source date end-to-end. atime/ctime are + // not in the tar format, so the driver stamps them the same as mtime + // via UnixTimes' `_or_now` accessors. `0` mtime is dropped — that's + // the "no date" case tar emits when the source didn't record one. + let unix_times = header + .mtime() + .ok() + .filter(|&m| m > 0) + .map(super::times::UnixTimes::mtime_only); if !apply { - return crate::fs::attrs::AttrOverrides::default(); + return crate::fs::attrs::AttrOverrides { + unix_times, + ..Default::default() + }; } crate::fs::attrs::AttrOverrides { mode: header.mode().ok().map(|m| m & 0o7777), uid: header.uid().ok().map(|v| v as u32), gid: header.gid().ok().map(|v| v as u32), + unix_times, } } diff --git a/src/fs/times.rs b/src/fs/times.rs new file mode 100644 index 00000000..fd3e6659 --- /dev/null +++ b/src/fs/times.rs @@ -0,0 +1,714 @@ +//! Per-file Unix timestamps shared across the create / import / export paths, +//! plus the format-specific encoders every non-Unix driver needs to write those +//! same times into its own on-disk shape. +//! +//! The goal is date-preservation on copy: when rb-cli copies a file from a host +//! folder, a tar archive, or another disk image into a filesystem, the +//! destination should record the *source's* mtime, not "now". And a +//! `tar_export` (or any host-side extract) of that image should carry the same +//! mtime back out. +//! +//! ## The rule +//! +//! `create_file` / `create_directory` / `create_symlink` use these times when +//! `options.unix_times.is_some()`; otherwise they stamp `now`. So: +//! +//! - **Genuine new file** (user typed `put`, GUI created a blank file) — +//! caller leaves `unix_times = None`, driver stamps `now`. +//! - **Copy / extract / import** — caller (dir_import / tar_import / +//! Commander stage_copy) captures the source's mtime and passes it through; +//! the destination inode records it verbatim. +//! +//! ## Why per-field Options +//! +//! Different sources carry different subsets: a tar Header has only `mtime`, +//! an HFS entry has creation + modification + backup dates, an EFS inode has +//! all three (atime/mtime/ctime). Missing fields aren't fabricated — the +//! driver falls back to `now` (or `mtime` where sensible) for each. +//! +//! Values are u64 seconds since UNIX epoch 1970-01-01. Sub-second precision +//! is intentionally dropped: FAT is 2-second granular, EFS/AFFS/PFS3 are 1- +//! second, only ext/xfs/jfs carry nanoseconds — and the tar exporter only +//! writes seconds anyway, so nanos wouldn't round-trip end-to-end. +//! +//! ## Format encoders +//! +//! Every filesystem below rusty-backup edits carries dates in *some* shape: +//! DOS packed (FAT / exFAT / Human68k), NTFS FILETIME, Mac epoch (HFS / +//! HFS+ / MFS), ProDOS packed, UCSD packed, ADFS 40-bit centiseconds, OS-9 +//! Y-M-D bytes, QDOS 1961 epoch. Each encoder in this module turns a Unix +//! `u64` seconds value into that shape; each decoder is the inverse and +//! returns `None` on the format's "no date set" sentinel (usually a +//! literal zero on disk). +//! +//! Encoders clamp to the format's earliest representable date rather than +//! underflowing — a FAT file dated 1975 becomes 1980-01-01, not garbage. +//! Decoders return `None` for zero and for values that would land before +//! 1970, so a `modified_unix` never carries a suspicious 1969 / 1970-01-01 +//! for what is on disk actually "no timestamp". + +use std::time::{SystemTime, UNIX_EPOCH}; + +/// A snapshot of a file's Unix timestamps, in seconds since 1970-01-01. +/// +/// Every field is `Option` so partial sources (tar has only mtime) don't +/// have to fabricate the others. When a field is `None` the destination +/// driver falls back to whatever it would use for a genuinely new file. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct UnixTimes { + /// Modification time — the "when the content last changed" one, the + /// number `ls -l` and `tar` display. This is the field that matters + /// most; both dir_import and tar_import populate it. + pub mtime: Option, + /// Access time — often equal to mtime on write. dir_import captures + /// the host file's atime; tar has no access-time field so tar_import + /// leaves it None (driver falls back to mtime or now). + pub atime: Option, + /// Metadata-change time — bumped by chmod / rename on POSIX. Almost + /// never available at import time (tar doesn't carry it, hosts don't + /// usually expose it portably), so this is usually None → driver uses + /// mtime or now. + pub ctime: Option, +} + +impl UnixTimes { + /// Snapshot with all three fields set to the same value. What + /// dir_import uses when the host has only `st_mtime` to hand. + pub fn all(secs: u64) -> Self { + Self { + mtime: Some(secs), + atime: Some(secs), + ctime: Some(secs), + } + } + + /// Snapshot with only mtime set. What tar_import produces (tar has no + /// atime/ctime fields). + pub fn mtime_only(secs: u64) -> Self { + Self { + mtime: Some(secs), + atime: None, + ctime: None, + } + } + + /// Convenience: mtime if set, else atime, else ctime, else `now`. The + /// pattern most drivers want when writing a single-field on-disk time + /// (e.g. UFS's `di_mtime` only) — pick the most user-meaningful value + /// the caller supplied without overwriting it with `now`. + pub fn mtime_or_now(&self) -> u64 { + self.mtime.or(self.atime).or(self.ctime).unwrap_or_else(now) + } + + /// Convenience: atime if set, else mtime, else ctime, else `now`. + pub fn atime_or_now(&self) -> u64 { + self.atime.or(self.mtime).or(self.ctime).unwrap_or_else(now) + } + + /// Convenience: ctime if set, else mtime, else atime, else `now`. + pub fn ctime_or_now(&self) -> u64 { + self.ctime.or(self.mtime).or(self.atime).unwrap_or_else(now) + } + + /// True when every field is None (equivalent to `Default::default`). + pub fn is_empty(&self) -> bool { + self.mtime.is_none() && self.atime.is_none() && self.ctime.is_none() + } +} + +/// Current wall-clock time as seconds since UNIX epoch. Clamped to 0 on the +/// (impossible) pre-1970 system clock. Replaces every driver's +/// `SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()` +/// dance with one function. +pub fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// `now()` clamped into a u32 (used by drivers with 32-bit time fields — +/// EFS, UFS-v1, XFS-v4, FAT/exFAT date halves). u32 overflows in 2106; we +/// document but do not guard against it (no user has a 2106-vintage disk). +pub fn now_u32() -> u32 { + now() as u32 +} + +/// The `now`-fallback rule the drivers use in one place: return the given +/// `UnixTimes` if any field is set, otherwise a fresh all-fields `now`. +/// +/// Not every driver wants this shape — some pick fields à la carte via +/// [`UnixTimes::mtime_or_now`] — but for the common "stamp all three +/// identically" case it captures the "preserve if given, else now" rule +/// so no driver open-codes the branch. +pub fn resolve_or_now(supplied: Option) -> UnixTimes { + match supplied { + Some(t) if !t.is_empty() => t, + _ => UnixTimes::all(now()), + } +} + +// --------------------------------------------------------------------------- +// Civil-date helpers (no chrono dep so the vintage build stays lean) +// --------------------------------------------------------------------------- + +/// Break Unix seconds into (year, month 1..=12, day 1..=31, hour, minute, +/// second). Uses Howard Hinnant's `civil_from_days` (same one +/// `format_unix_timestamp` in unix_common::inode uses); safe for any u64 +/// value we can actually store on disk. Deliberately not exported — the +/// format-specific encoders below are the public API. +fn ymd_hms(secs: u64) -> (i64, u32, u32, u32, u32, u32) { + let sec_of_day = secs % 86400; + let hour = (sec_of_day / 3600) as u32; + let minute = ((sec_of_day % 3600) / 60) as u32; + let second = (sec_of_day % 60) as u32; + + let mut days = (secs / 86400) as i64; + days += 719468; + let era = if days >= 0 { days } else { days - 146096 } / 146097; + let doe = (days - era * 146097) as u32; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let y = yoe as i64 + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let year = if m <= 2 { y + 1 } else { y }; + (year, m, d, hour, minute, second) +} + +/// Inverse of `ymd_hms` — pack (year, month, day, hour, minute, second) back +/// into Unix seconds. Uses Howard Hinnant's `days_from_civil` companion. +/// Any (year, month, day) triple the encoders emit round-trips exactly. +/// Invalid inputs (month 0, day > 31, etc.) still return *something*, but the +/// decoders always range-check first. +fn secs_from_ymd_hms(year: i64, month: u32, day: u32, hour: u32, minute: u32, second: u32) -> u64 { + let y = if month <= 2 { year - 1 } else { year }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = (y - era * 400) as u32; + let m = month; + let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146097 + doe as i64 - 719468; + (days as u64) * 86400 + hour as u64 * 3600 + minute as u64 * 60 + second as u64 +} + +/// True when a Gregorian year is a leap year. +fn is_leap(year: i64) -> bool { + (year % 4 == 0) && (year % 100 != 0 || year % 400 == 0) +} + +/// Days in month for a given Gregorian year. +fn days_in_month(year: i64, month: u32) -> u32 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 => { + if is_leap(year) { + 29 + } else { + 28 + } + } + _ => 0, + } +} + +// --------------------------------------------------------------------------- +// FAT / exFAT / Human68k — DOS packed date + time (16-bit each) +// --------------------------------------------------------------------------- + +/// Seconds between 1970-01-01 and 1980-01-01 (DOS epoch). 3652 days. +const DOS_EPOCH_SECS: u64 = 315_532_800; + +/// Encode Unix seconds as (fat_date, fat_time) — the two 16-bit words FAT +/// dirent slots hold. `fat_date` layout: `year_since_1980 (7) | month (4) +/// | day (5)`; `fat_time` layout: `hour (5) | minute (6) | second/2 (5)`. +/// +/// Any Unix time before 1980-01-01 clamps to 1980-01-01 00:00:00 (the +/// earliest representable DOS date) so a pre-1980 mtime doesn't become +/// year-2107. Any time past 2107-12-31 is truncated to the year mod 128, +/// which is the same behaviour as every DOS filesystem tool going back to +/// MS-DOS 2.0 — the year field is only 7 bits and there is nowhere else +/// to put a bigger value. +pub fn unix_to_dos_datetime(secs: u64) -> (u16, u16) { + let secs = secs.max(DOS_EPOCH_SECS); + let (year, month, day, hour, minute, second) = ymd_hms(secs); + let year_since_1980 = ((year - 1980).clamp(0, 127)) as u16; + let date = (year_since_1980 << 9) | ((month as u16 & 0x0F) << 5) | (day as u16 & 0x1F); + let time = + ((hour as u16 & 0x1F) << 11) | ((minute as u16 & 0x3F) << 5) | ((second as u16 / 2) & 0x1F); + (date, time) +} + +/// Decode a FAT (date, time) pair back to Unix seconds. Returns `None` for +/// a zero `date` (the "no date set" sentinel every DOS filesystem tool +/// uses) or for a date field whose components don't make sense (month 0, +/// day 0, day > days-in-month, etc). Values are treated as UTC — FAT +/// stores wall-clock time with no timezone, and every tool we round-trip +/// against does the same. +pub fn dos_datetime_to_unix(date: u16, time: u16) -> Option { + if date == 0 { + return None; + } + let day = (date & 0x1F) as u32; + let month = ((date >> 5) & 0x0F) as u32; + let year = ((date >> 9) & 0x7F) as i64 + 1980; + let second = ((time & 0x1F) as u32) * 2; + let minute = ((time >> 5) & 0x3F) as u32; + let hour = ((time >> 11) & 0x1F) as u32; + if month == 0 || month > 12 || day == 0 || day > days_in_month(year, month) { + return None; + } + if hour > 23 || minute > 59 || second > 59 { + return None; + } + Some(secs_from_ymd_hms(year, month, day, hour, minute, second)) +} + +/// exFAT packs the two 16-bit DOS words into a single u32 (date in the +/// upper half, time in the lower). Layer over [`unix_to_dos_datetime`]. +pub fn unix_to_exfat_timestamp(secs: u64) -> u32 { + let (date, time) = unix_to_dos_datetime(secs); + ((date as u32) << 16) | time as u32 +} + +/// Inverse of [`unix_to_exfat_timestamp`]. +pub fn exfat_timestamp_to_unix(ts: u32) -> Option { + dos_datetime_to_unix((ts >> 16) as u16, ts as u16) +} + +// --------------------------------------------------------------------------- +// NTFS FILETIME — 100-ns intervals since 1601-01-01 +// --------------------------------------------------------------------------- + +/// Seconds between the NTFS epoch (1601-01-01) and the Unix epoch (1970-01-01). +const NTFS_EPOCH_OFFSET_SECS: u64 = 11_644_473_600; + +/// Encode Unix seconds as an NTFS FILETIME (100-nanosecond intervals since +/// 1601-01-01). Sub-second precision is lost, which is fine — every other +/// source in the import chain (host stat, tar Header, cross-fs copy) is +/// second-granular anyway. +pub fn unix_to_filetime(secs: u64) -> u64 { + secs.saturating_add(NTFS_EPOCH_OFFSET_SECS) + .saturating_mul(10_000_000) +} + +/// Decode an NTFS FILETIME back to Unix seconds. Returns `None` for a +/// zero FILETIME (NTFS's "unset" sentinel) or for a FILETIME that would +/// land before 1970 (the tar oracle would render those as 1969 either +/// way, and no real NTFS volume dates a file before 1970). +pub fn filetime_to_unix(ft: u64) -> Option { + if ft == 0 { + return None; + } + let total_secs = ft / 10_000_000; + if total_secs < NTFS_EPOCH_OFFSET_SECS { + return None; + } + Some(total_secs - NTFS_EPOCH_OFFSET_SECS) +} + +// --------------------------------------------------------------------------- +// HFS / HFS+ / MFS — Mac epoch (u32 seconds since 1904-01-01) +// --------------------------------------------------------------------------- + +/// Seconds between 1904-01-01 (Mac epoch) and 1970-01-01 (Unix epoch). +/// Duplicated from hfs_common so the encoder is reachable outside the +/// HFS/HFS+ modules (MFS uses the same epoch and lives elsewhere). +const MAC_EPOCH_DELTA: u64 = 2_082_844_800; + +/// Encode Unix seconds as a Mac epoch u32. Any time before 1904-01-01 +/// clamps to 1904-01-01, any time past 2040-02-06 (the u32 rollover) is +/// truncated by u32 wraparound — no Mac filesystem tool guards against +/// that, and no Mac disk we ship dates a file past 2040. +pub fn unix_to_mac_epoch(secs: u64) -> u32 { + (secs.saturating_add(MAC_EPOCH_DELTA)) as u32 +} + +/// Decode a Mac epoch u32 back to Unix seconds. Mirrors +/// `hfs_common::mac_date_to_unix` — returns `None` for zero (the "no date" +/// sentinel) and for pre-1970 Mac dates (which would tar out as 1969). +pub fn mac_epoch_to_unix(mac_secs: u32) -> Option { + if mac_secs == 0 || (mac_secs as u64) < MAC_EPOCH_DELTA { + return None; + } + Some(mac_secs as u64 - MAC_EPOCH_DELTA) +} + +// --------------------------------------------------------------------------- +// ProDOS — packed date + time (2×u16) +// --------------------------------------------------------------------------- + +/// Encode Unix seconds as a ProDOS (date, time) pair. Date layout: `year +/// (7) | month (4) | day (5)` where year is `year - 2000` when < 40 else +/// `year - 1900` (ProDOS's own two-digit convention — years 40..99 mean +/// 1940..1999, years 0..39 mean 2000..2039). Time layout: `hour (5) | +/// minute (6)` (no seconds — ProDOS is minute-granular). +/// +/// Clamps to 1940-01-01 for pre-1940 input (the earliest representable +/// ProDOS date under its own convention) and to 2039-12-31 for +/// post-2039 input. +pub fn unix_to_prodos_datetime(secs: u64) -> (u16, u16) { + let (mut year, month, day, hour, minute, _second) = ymd_hms(secs); + if year < 1940 { + return ( + ((40u16) << 9) | (1u16 << 5) | 1u16, // 1940-01-01 + 0, + ); + } + if year > 2039 { + year = 2039; + } + let year_bits = if year >= 2000 { + (year - 2000) as u16 + } else { + (year - 1900) as u16 + }; + let date = ((year_bits & 0x7F) << 9) | ((month as u16 & 0x0F) << 5) | (day as u16 & 0x1F); + let time = ((hour as u16 & 0x1F) << 8) | (minute as u16 & 0x3F); + (date, time) +} + +/// Decode a ProDOS (date, time) pair back to Unix seconds. Returns `None` +/// for a zero `date` ("no date set" — ProDOS Technical Note #28 formalises +/// this). +pub fn prodos_datetime_to_unix(date: u16, time: u16) -> Option { + if date == 0 { + return None; + } + let day = (date & 0x1F) as u32; + let month = ((date >> 5) & 0x0F) as u32; + let raw_year = ((date >> 9) & 0x7F) as i64; + let year = if raw_year < 40 { + 2000 + raw_year + } else { + 1900 + raw_year + }; + if year < 1970 { + return None; + } + let minute = (time & 0x3F) as u32; + let hour = ((time >> 8) & 0x1F) as u32; + if month == 0 || month > 12 || day == 0 || day > days_in_month(year, month) { + return None; + } + if hour > 23 || minute > 59 { + return None; + } + Some(secs_from_ymd_hms(year, month, day, hour, minute, 0)) +} + +// --------------------------------------------------------------------------- +// UCSD Pascal — packed date (u16: day | month | year) +// --------------------------------------------------------------------------- + +/// Encode Unix seconds as a UCSD Pascal packed date. Layout (little-endian +/// as it sits in the dirent, but treated here as a plain u16): +/// `year (7) | month (4) | day (5)` — years are 2-digit like ProDOS, but +/// UCSD's convention is different: 0..99 all map to 1900..1999. +/// +/// Clamps to 1900-01-01 for pre-1900 input and to 1999-12-31 for +/// post-1999 input. Time-of-day is dropped — UCSD is day-granular. +pub fn unix_to_ucsd_date(secs: u64) -> u16 { + let (year, month, day, _h, _m, _s) = ymd_hms(secs); + let year = year.clamp(1900, 1999); + let year_bits = (year - 1900) as u16; + (day as u16 & 0x1F) | ((month as u16 & 0x0F) << 5) | ((year_bits & 0x7F) << 9) +} + +/// Decode a UCSD Pascal packed date to Unix seconds (00:00:00 of that day). +/// Returns `None` for zero, and for pre-1970 dates (which would round-trip +/// through tar as 1969 anyway; the year field's 1900..1999 range means any +/// UCSD-native date can be pre-1970, so this guard matters). +pub fn ucsd_date_to_unix(word: u16) -> Option { + if word == 0 { + return None; + } + let day = (word & 0x1F) as u32; + let month = ((word >> 5) & 0x0F) as u32; + let year = 1900 + ((word >> 9) & 0x7F) as i64; + if year < 1970 { + return None; + } + if month == 0 || month > 12 || day == 0 || day > days_in_month(year, month) { + return None; + } + Some(secs_from_ymd_hms(year, month, day, 0, 0, 0)) +} + +// --------------------------------------------------------------------------- +// ADFS RISC OS — 40-bit centiseconds since 1900-01-01, packed into +// (load_addr low 8 bits, exec_addr all 32 bits) with load_addr high bits +// `0xFFFtt000` (filetype `tt`, plus the 0xFFF marker) +// --------------------------------------------------------------------------- + +/// Centiseconds between 1900-01-01 and 1970-01-01. 70 years × 365.2425 +/// days × 86400 s × 100 = 220_898_880_000 cs. (Includes 17 leap days: +/// 70/4 = 17.5, minus 1900 not being a leap year = 17.) +const ADFS_EPOCH_OFFSET_CS: u64 = 220_898_880_000; + +/// Encode Unix seconds as an ADFS timestamped (load_addr, exec_addr) +/// pair carrying the given RISC OS filetype (12 bits: 0xFFF = Data, +/// 0xFEB = Obey, etc.). `load_addr = 0xFFF_00000 | ((ft & 0xFFF) << +/// 8) | ((cs40 >> 32) as u8)`; `exec_addr = cs40 as u32`. The 0xFFF-in- +/// high-12-bits pattern is the "this is a datestamp, not a load +/// address" marker every RISC OS tool checks for. +pub fn unix_to_adfs_time(secs: u64, filetype: u16) -> (u32, u32) { + let cs = secs + .saturating_mul(100) + .saturating_add(ADFS_EPOCH_OFFSET_CS); + let load_addr = 0xFFF0_0000u32 | ((filetype as u32 & 0xFFF) << 8) | ((cs >> 32) as u32 & 0xFF); + let exec_addr = cs as u32; + (load_addr, exec_addr) +} + +/// Decode an ADFS (load_addr, exec_addr) pair to Unix seconds. Returns +/// `None` when load_addr's high 12 bits aren't `0xFFF` (which means the +/// pair is a real load/exec address, not a datestamp) or when the +/// resulting timestamp would land before 1970. +pub fn adfs_time_to_unix(load: u32, exec: u32) -> Option { + if load & 0xFFF0_0000 != 0xFFF0_0000 { + return None; + } + let cs = ((load as u64 & 0xFF) << 32) | exec as u64; + if cs < ADFS_EPOCH_OFFSET_CS { + return None; + } + Some((cs - ADFS_EPOCH_OFFSET_CS) / 100) +} + +// --------------------------------------------------------------------------- +// OS-9 — FD.DAT (5-byte last-modified Y-M-D-H-M) + FD.DCR (3-byte +// creation Y-M-D). Each byte is a plain binary component; year is +// offset from 1900. +// --------------------------------------------------------------------------- + +/// Encode Unix seconds as OS-9 FD.DAT (5 bytes: year-1900, month, day, +/// hour, minute). OS-9's year is a single unsigned byte — 1900..2155 +/// representable. Pre-1900 clamps to 1900-01-01; post-2155 clamps to +/// 2155-12-31. +pub fn unix_to_os9_dat(secs: u64) -> [u8; 5] { + let (year, month, day, hour, minute, _s) = ymd_hms(secs); + let year = year.clamp(1900, 2155); + [ + (year - 1900) as u8, + month as u8, + day as u8, + hour as u8, + minute as u8, + ] +} + +/// Encode Unix seconds as OS-9 FD.DCR (3 bytes: year-1900, month, day). +pub fn unix_to_os9_dcr(secs: u64) -> [u8; 3] { + let dat = unix_to_os9_dat(secs); + [dat[0], dat[1], dat[2]] +} + +/// Decode an OS-9 FD.DAT to Unix seconds. Returns `None` for all-zero +/// (unwritten) or nonsensical component values. +pub fn os9_dat_to_unix(dat: &[u8; 5]) -> Option { + if dat == &[0u8; 5] { + return None; + } + let year = 1900i64 + dat[0] as i64; + if year < 1970 { + return None; + } + let month = dat[1] as u32; + let day = dat[2] as u32; + let hour = dat[3] as u32; + let minute = dat[4] as u32; + if month == 0 || month > 12 || day == 0 || day > days_in_month(year, month) { + return None; + } + if hour > 23 || minute > 59 { + return None; + } + Some(secs_from_ymd_hms(year, month, day, hour, minute, 0)) +} + +/// Decode an OS-9 FD.DCR (creation date only) to Unix seconds. Returns +/// `None` for all-zero or pre-1970 dates. +pub fn os9_dcr_to_unix(dcr: &[u8; 3]) -> Option { + if dcr == &[0u8; 3] { + return None; + } + let year = 1900i64 + dcr[0] as i64; + if year < 1970 { + return None; + } + let month = dcr[1] as u32; + let day = dcr[2] as u32; + if month == 0 || month > 12 || day == 0 || day > days_in_month(year, month) { + return None; + } + Some(secs_from_ymd_hms(year, month, day, 0, 0, 0)) +} + +// --------------------------------------------------------------------------- +// QDOS (Sinclair QL) — u32 seconds since 1961-01-01 +// --------------------------------------------------------------------------- + +/// Seconds between 1961-01-01 and 1970-01-01 (Unix epoch). 9 years: +/// (365 * 9) + 2 leap days (1964, 1968) = 3287 days × 86400 = 283_996_800 s. +const QDOS_EPOCH_OFFSET_SECS: u64 = 283_996_800; + +/// Encode Unix seconds as a QDOS timestamp (u32 seconds since 1961-01-01, +/// big-endian on disk — but the encoding is byte-order-agnostic here). +/// Pre-1961 clamps to 0 (1961-01-01); post-2097-02-06 truncates by u32 +/// wraparound. No QDOS tool guards against either. +pub fn unix_to_qdos_date(secs: u64) -> u32 { + (secs.saturating_add(QDOS_EPOCH_OFFSET_SECS)) as u32 +} + +/// Decode a QDOS timestamp to Unix seconds. Returns `None` for zero (the +/// "no date set" sentinel) or for QDOS values that would land before 1970. +pub fn qdos_date_to_unix(ts: u32) -> Option { + if ts == 0 || (ts as u64) < QDOS_EPOCH_OFFSET_SECS { + return None; + } + Some(ts as u64 - QDOS_EPOCH_OFFSET_SECS) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + /// 2020-06-15 12:34:56 UTC — a mid-range date every encoder can hold. + const T_2020: u64 = 1_592_224_496; + + #[test] + fn dos_datetime_round_trips_and_clamps() { + let (d, t) = unix_to_dos_datetime(T_2020); + // 2020 = 1980 + 40, month=6, day=15, hour=12, minute=34, second=56 (56/2=28) + assert_eq!(d, (40 << 9) | (6 << 5) | 15); + assert_eq!(t, (12 << 11) | (34 << 5) | 28); + // Round-trip: 2-second granularity means we get 56 back (not 57). + assert_eq!(dos_datetime_to_unix(d, t), Some(T_2020)); + + // Zero -> None (the "no date" sentinel). + assert_eq!(dos_datetime_to_unix(0, 0), None); + // Pre-DOS-epoch clamps to 1980-01-01 00:00:00. + assert_eq!(unix_to_dos_datetime(0), ((1 << 5) | 1, 0)); + // Nonsense date (month 0) -> None. date=1 encodes day=1, month=0, year=1980. + assert_eq!(dos_datetime_to_unix(0x0001, 0), None); + } + + #[test] + fn exfat_timestamp_wraps_dos_helpers() { + let ts = unix_to_exfat_timestamp(T_2020); + assert_eq!(exfat_timestamp_to_unix(ts), Some(T_2020)); + assert_eq!(exfat_timestamp_to_unix(0), None); + } + + #[test] + fn filetime_round_trips_and_rejects_zero() { + let ft = unix_to_filetime(T_2020); + assert_eq!(filetime_to_unix(ft), Some(T_2020)); + assert_eq!(filetime_to_unix(0), None); + // A FILETIME whose value would decode to pre-1970 is refused. + assert_eq!(filetime_to_unix(1), None); + } + + #[test] + fn mac_epoch_round_trips() { + let m = unix_to_mac_epoch(T_2020); + assert_eq!(mac_epoch_to_unix(m), Some(T_2020)); + assert_eq!(mac_epoch_to_unix(0), None); + // A Mac-epoch value that would decode before 1970 is refused. + assert_eq!(mac_epoch_to_unix(100), None); + } + + #[test] + fn prodos_datetime_round_trips() { + // ProDOS drops seconds; use a time on a minute boundary for round-trip. + let t = 1_592_224_440; // 2020-06-15 12:34:00 UTC + let (d, t_word) = unix_to_prodos_datetime(t); + assert_eq!(prodos_datetime_to_unix(d, t_word), Some(t)); + assert_eq!(prodos_datetime_to_unix(0, 0), None); + } + + #[test] + fn prodos_year_convention_bounds() { + // Year 20 (0..39) -> 2020. + let (d, _) = unix_to_prodos_datetime(1_592_224_440); + assert_eq!((d >> 9) & 0x7F, 20); + // Year 90 (40..99) -> 1990. 1990-06-15 12:34:00 UTC. + let t_1990 = 645_453_240; + let (d90, t90) = unix_to_prodos_datetime(t_1990); + assert_eq!((d90 >> 9) & 0x7F, 90); + assert_eq!(prodos_datetime_to_unix(d90, t90), Some(t_1990)); + } + + #[test] + fn ucsd_date_round_trips_day_granular() { + let t = 1_592_179_200; // 2020-06-15 00:00:00 UTC — but UCSD only holds 1900..1999. + // For UCSD, use 1990. + let t_1990 = 645_408_000; // 1990-06-15 00:00:00 + let w = unix_to_ucsd_date(t_1990); + assert_eq!(ucsd_date_to_unix(w), Some(t_1990)); + assert_eq!(ucsd_date_to_unix(0), None); + // Silence the 2020 unused-warning. + let _ = t; + } + + #[test] + fn adfs_time_round_trips_with_filetype() { + let (load, exec) = unix_to_adfs_time(T_2020, 0xFFF); + // High 12 bits must be 0xFFF (the datestamp marker). + assert_eq!(load & 0xFFF0_0000, 0xFFF0_0000); + // Filetype 0xFFF stored in bits 8..20. + assert_eq!((load >> 8) & 0xFFF, 0xFFF); + assert_eq!(adfs_time_to_unix(load, exec), Some(T_2020)); + // A non-datestamp load address (high bits != 0xFFF) -> None. + assert_eq!(adfs_time_to_unix(0x0000_8000, 0), None); + } + + #[test] + fn os9_dat_and_dcr_round_trip() { + // OS-9 is minute-granular; use a boundary time. + let t = 1_592_224_440; // 2020-06-15 12:34:00 + let dat = unix_to_os9_dat(t); + assert_eq!(dat, [120, 6, 15, 12, 34]); + assert_eq!(os9_dat_to_unix(&dat), Some(t)); + + let dcr = unix_to_os9_dcr(t); + assert_eq!(dcr, [120, 6, 15]); + // Creation-date decode is 00:00 of the day. + assert_eq!(os9_dcr_to_unix(&dcr), Some(1_592_179_200)); + + assert_eq!(os9_dat_to_unix(&[0; 5]), None); + assert_eq!(os9_dcr_to_unix(&[0; 3]), None); + } + + #[test] + fn qdos_date_round_trips() { + let q = unix_to_qdos_date(T_2020); + assert_eq!(qdos_date_to_unix(q), Some(T_2020)); + assert_eq!(qdos_date_to_unix(0), None); + } + + /// Every non-DOS encoder is expected to preserve the exact input we + /// give it. This is the tar-round-trip guarantee: a source mtime that + /// went in must come back out on the other side, and any format- + /// granularity loss is documented. + #[test] + fn encoders_preserve_mid_range_values_verbatim() { + // NTFS: nanosecond precision, so a whole-second input round-trips + // exactly. + assert_eq!(filetime_to_unix(unix_to_filetime(T_2020)), Some(T_2020)); + // Mac epoch: second-granular. + assert_eq!(mac_epoch_to_unix(unix_to_mac_epoch(T_2020)), Some(T_2020)); + // QDOS: second-granular. + assert_eq!(qdos_date_to_unix(unix_to_qdos_date(T_2020)), Some(T_2020)); + // ADFS: centisecond-granular but we input whole seconds. + let (l, e) = unix_to_adfs_time(T_2020, 0xFFF); + assert_eq!(adfs_time_to_unix(l, e), Some(T_2020)); + } +} diff --git a/src/fs/tree.rs b/src/fs/tree.rs index 40bb799a..25cfd240 100644 --- a/src/fs/tree.rs +++ b/src/fs/tree.rs @@ -153,6 +153,7 @@ mod tests { size: 0, location: 2, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -183,6 +184,7 @@ mod tests { size: 0, location: 10, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -208,6 +210,7 @@ mod tests { size: 1024, location: 20, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -234,6 +237,7 @@ mod tests { size: 2048, location: 30, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, diff --git a/src/fs/ucsd.rs b/src/fs/ucsd.rs index c0585934..3d12418d 100644 --- a/src/fs/ucsd.rs +++ b/src/fs/ucsd.rs @@ -223,6 +223,7 @@ impl UcsdFilesystem { e.first_block as u64, ); fe.modified = decode_date(e.date); + fe.modified_unix = crate::fs::times::ucsd_date_to_unix(e.date); fe } } @@ -501,7 +502,7 @@ impl EditableFilesystem for UcsdFilesystem { name: &str, data: &mut dyn Read, data_len: u64, - _options: &CreateFileOptions, + options: &CreateFileOptions, ) -> Result { if !parent.is_directory() || parent.path != "/" { return Err(FilesystemError::NotADirectory(parent.path.clone())); @@ -541,13 +542,22 @@ impl EditableFilesystem for UcsdFilesystem { } else { (data_len - (need as u64 - 1) * BLOCK) as u16 }; + // UCSD packs the date as a single 16-bit word (day | month | year), + // year 0..99 -> 1900..1999. Cross-fs copy passes source mtime through + // options.unix_times; a genuinely new file leaves it None and stamps + // zero (matching pre-existing behaviour — UCSD doesn't have a "now" + // convention and generators here forbid wall-clock reads). + let date = options + .unix_times + .map(|t| crate::fs::times::unix_to_ucsd_date(t.mtime_or_now())) + .unwrap_or(0); let entry = UcsdEntry { first_block: start, last_block: start + need, kind: kind_from_name(&uname), name: uname, last_byte, - date: 0, + date, }; let fe = self.entry_to_file(&entry); self.entries.push(entry); diff --git a/src/fs/ufs.rs b/src/fs/ufs.rs index 526d2cf2..ec08a9c2 100644 --- a/src/fs/ufs.rs +++ b/src/fs/ufs.rs @@ -87,6 +87,8 @@ pub(crate) const OFF_SBLKNO: usize = 0x008; // fs_sblkno i32 — SB address pub(crate) const OFF_CBLKNO: usize = 0x00C; // fs_cblkno i32 — CG block addr in frags pub(crate) const OFF_IBLKNO: usize = 0x010; // fs_iblkno i32 — inode-block addr in frags pub(crate) const OFF_OLD_SIZE: usize = 0x024; // fs_old_size i32 — UFS1 total fragments +pub(crate) const OFF_CGOFFSET: usize = 0x018; // fs_cgoffset i32 — UFS1 rotational CG offset +pub(crate) const OFF_CGMASK: usize = 0x01C; // fs_cgmask i32 — mask selecting the CG index bits pub(crate) const OFF_NCG: usize = 0x02C; // fs_ncg u32 — # cylinder groups pub(crate) const OFF_BSIZE: usize = 0x030; // fs_bsize i32 — block size in bytes pub(crate) const OFF_FSIZE: usize = 0x034; // fs_fsize i32 — fragment size in bytes @@ -248,6 +250,14 @@ pub struct UfsFilesystem { pub(crate) ncg: u32, // number of cylinder groups pub(crate) fpg: u32, // fragments per cylinder group pub(crate) ipg: u32, // inodes per cylinder group + /// `fs_cgoffset` / `fs_cgmask`: UFS1's rotational cylinder-group offset. + /// Each CG's *metadata* starts at `cgbase(c) + cgoffset * (c & !cgmask)`, + /// not at `cgbase(c)`. The term is 0 for CG 0 whatever the values, which is + /// why ignoring it read CG 0 correctly and every later CG as garbage + /// (R-013). Zero on UFS1 images written after the rotational tables were + /// dropped, and always unused on UFS2. + pub(crate) cgoffset: u32, + pub(crate) cgmask: u32, /// `fs_cblkno`: fragment offset of the cylinder-group header inside /// each CG region. CG `i`'s header lives at fragment `i * fpg + cblkno`. pub(crate) cblkno: u32, @@ -336,6 +346,8 @@ impl UfsFilesystem { let ncg = read_u32(&sb, OFF_NCG, endian); let fpg = read_i32(&sb, OFF_FPG, endian); let ipg = read_u32(&sb, OFF_IPG, endian); + let cgoffset = read_u32(&sb, OFF_CGOFFSET, endian); + let cgmask = read_u32(&sb, OFF_CGMASK, endian); let cblkno = read_i32(&sb, OFF_CBLKNO, endian); let iblkno = read_i32(&sb, OFF_IBLKNO, endian); let sblkno = read_i32(&sb, OFF_SBLKNO, endian); @@ -455,6 +467,8 @@ impl UfsFilesystem { ncg, fpg: fpg as u32, ipg, + cgoffset, + cgmask, cblkno: cblkno as u32, iblkno: iblkno as u32, sblkno: sblkno as u32, @@ -469,7 +483,7 @@ impl UfsFilesystem { /// modern UFS layout, post-rotational-table removal); the CG header /// then lives at `cgbase + cblkno` fragments. pub(crate) fn cg_header_offset(&self, i: u32) -> u64 { - ((i as u64) * (self.fpg as u64) + self.cblkno as u64) * self.fsize + (self.cgstart_frag(i) + self.cblkno as u64) * self.fsize } /// Starting absolute fragment of CG `i`. @@ -554,6 +568,23 @@ impl UfsFilesystem { self.ipg as u64 * self.ncg as u64 } + /// First fragment of CG `i`'s **metadata** — the kernel's `cgstart(fs, c)`. + /// + /// `cgbase(c)` is where the CG's data area begins; the superblock replica, + /// CG header and inode table sit at `cgbase(c) + fs_cgoffset * (c & + /// !fs_cgmask)`. UFS2 always has `cgstart == cgbase`, and so does a UFS1 + /// image written after the rotational tables were dropped — both carry + /// `fs_cgoffset = 0`, so this is a no-op there. A real Solaris UFS1 does + /// not: it uses `cgoffset = 32`, `cgmask = 0xffffff00` (R-013). + pub(crate) fn cgstart_frag(&self, i: u32) -> u64 { + let skew = if matches!(self.version, UfsVersion::Ufs2) { + 0 + } else { + self.cgoffset as u64 * (i & !self.cgmask) as u64 + }; + (i as u64) * (self.fpg as u64) + skew + } + /// Byte offset (relative to the partition start) of the replica /// superblock inside CG `cg`. Mirrors the kernel's `cgsblock(fs, c)` /// macro for layouts where `cgstart == cgbase` (UFS2 always; UFS1 @@ -562,7 +593,7 @@ impl UfsFilesystem { if cg >= self.ncg { return None; } - Some((cg as u64 * self.fpg as u64 + self.sblkno as u64) * self.fsize) + Some((self.cgstart_frag(cg) + self.sblkno as u64) * self.fsize) } /// Read the SB-sized prefix (`SB_READ_SIZE` bytes) at CG `cg`'s replica @@ -745,7 +776,7 @@ impl UfsFilesystem { let ipg = self.ipg as u64; let cg = inum as u64 / ipg; let in_cg = inum as u64 % ipg; - (cg * self.fpg as u64 + self.iblkno as u64) * self.fsize + in_cg * dsize + (self.cgstart_frag(cg as u32) + self.iblkno as u64) * self.fsize + in_cg * dsize } fn dinode_size(&self) -> u64 { @@ -1030,6 +1061,9 @@ impl UfsFilesystem { entry.gid = Some(child_inode.gid); if child_inode.mtime != 0 { entry.modified = Some(format_unix_timestamp(child_inode.mtime)); + if child_inode.mtime > 0 { + entry.modified_unix = Some(child_inode.mtime as u64); + } } Ok(entry) } @@ -1962,6 +1996,8 @@ impl super::filesystem::EditableFilesystem for Uf // it back. Inum allocation is sticky across the create. let new_inum = self.alloc_inode(parent_inum / self.ipg)?; let mode = options.mode.unwrap_or(0o100644); + // Preserve source mtime on cross-fs copies; else stamp now. + let mtime = super::times::resolve_or_now(options.unix_times).mtime_or_now() as i64; let mut new_inode = UfsInode { inum: new_inum, mode, @@ -1969,7 +2005,7 @@ impl super::filesystem::EditableFilesystem for Uf uid: options.uid.unwrap_or(0), gid: options.gid.unwrap_or(0), size: 0, - mtime: 0, + mtime, direct: [0; UFS_NDADDR], indirect: [0; UFS_NIADDR], inline_payload: Vec::new(), @@ -2056,6 +2092,7 @@ impl super::filesystem::EditableFilesystem for Uf // 0777 by convention; an explicit mode contributes permission bits // only, never the type. let mode = 0o120_000 | (options.mode.unwrap_or(0o777) & 0o7777); + let mtime = super::times::resolve_or_now(options.unix_times).mtime_or_now() as i64; let mut new_inode = UfsInode { inum: new_inum, mode, @@ -2063,7 +2100,7 @@ impl super::filesystem::EditableFilesystem for Uf uid: options.uid.unwrap_or(0), gid: options.gid.unwrap_or(0), size: 0, - mtime: 0, + mtime, direct: [0; UFS_NDADDR], indirect: [0; UFS_NIADDR], inline_payload: Vec::new(), @@ -2169,6 +2206,7 @@ impl super::filesystem::EditableFilesystem for Uf self.write_frag_run(start_frag, &block)?; let mode = options.mode.unwrap_or(0o040755); + let mtime = super::times::resolve_or_now(options.unix_times).mtime_or_now() as i64; let mut new_dir = UfsInode { inum: new_inum, mode, @@ -2176,7 +2214,7 @@ impl super::filesystem::EditableFilesystem for Uf uid: options.uid.unwrap_or(0), gid: options.gid.unwrap_or(0), size: DIRBLKSIZ as u64, - mtime: 0, + mtime, direct: [0; UFS_NDADDR], indirect: [0; UFS_NIADDR], inline_payload: Vec::new(), @@ -2579,6 +2617,7 @@ fn adopt_orphans_into_lost_found_ufs( size: 0, location: ROOT_INODE as u64, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -2617,6 +2656,7 @@ fn adopt_orphans_into_lost_found_ufs( size: 0, location: ROOT_INODE as u64, modified: None, + modified_unix: None, type_code: None, creator_code: None, symlink_target: None, @@ -2830,6 +2870,9 @@ impl Filesystem for UfsFilesystem { entry.gid = Some(inode.gid); if inode.mtime != 0 { entry.modified = Some(format_unix_timestamp(inode.mtime)); + if inode.mtime > 0 { + entry.modified_unix = Some(inode.mtime as u64); + } } Ok(entry) } diff --git a/src/fs/unix_common/inode.rs b/src/fs/unix_common/inode.rs index 8169b0a3..0715af79 100644 --- a/src/fs/unix_common/inode.rs +++ b/src/fs/unix_common/inode.rs @@ -129,6 +129,8 @@ pub fn unix_entry_from_inode( } else { None }; + // Numeric twin of `modified` for tar_export / cross-image copies. + let modified_unix: Option = if mtime > 0 { Some(mtime as u64) } else { None }; let (entry_type, special_type) = match ft { UnixFileType::Regular | UnixFileType::Unknown => (EntryType::File, None), @@ -152,6 +154,7 @@ pub fn unix_entry_from_inode( size: display_size, location: inode_num, modified, + modified_unix, type_code: None, creator_code: None, symlink_target: None, diff --git a/src/fs/xfs/edit.rs b/src/fs/xfs/edit.rs index 0ee8a655..75466fcc 100644 --- a/src/fs/xfs/edit.rs +++ b/src/fs/xfs/edit.rs @@ -75,6 +75,25 @@ const XFS_DIR3_FT_DIR: u8 = 2; /// NULL agino sentinel stored in `di_next_unlinked` for a not-unlinked inode. const NULLAGINO: u32 = 0xFFFF_FFFF; +/// Stamp the three v4 dinode time fields (`di_atime` @ 32..40, +/// `di_mtime` @ 40..48, `di_ctime` @ 48..56) into an inode buffer, each a +/// pair of BE u32 `(tv_sec, tv_nsec)`. `times`, when set, replaces the +/// default `now` stamp with the preserved values so a cross-fs import keeps +/// the source's mtime. `tv_nsec` is written 0 — every source we import from +/// only carries whole-second precision (tar, EFS, host-stat). +fn stamp_v4_times(buf: &mut [u8], times: Option) { + let resolved = crate::fs::times::resolve_or_now(times); + let atime = resolved.atime_or_now() as u32; + let mtime = resolved.mtime_or_now() as u32; + let ctime = resolved.ctime_or_now() as u32; + BigEndian::write_u32(&mut buf[32..36], atime); + // buf[36..40] already zero from the sweep above (tv_nsec). + BigEndian::write_u32(&mut buf[40..44], mtime); + // buf[44..48] zero. + BigEndian::write_u32(&mut buf[48..52], ctime); + // buf[52..56] zero. +} + /// inobt leaf record layout: startino(4) freecount(4) free(8) — same on v4 /// and v5. const INOBT_REC_SIZE: usize = 16; @@ -681,6 +700,11 @@ impl XfsFilesystem { /// bumps the generation; writes a clean v4 dinode core + the 6-byte inline /// fork (`count=0, i8count=0, parent`). `nlink` is 2 (the new directory's /// own `.` plus its entry in the parent); no subdirectories yet. + /// + /// `times`, when set, replaces the default `now` stamp on the three + /// di_atime/di_mtime/di_ctime fields so a cross-fs copy preserves the + /// source's mtime end-to-end. + #[allow(clippy::too_many_arguments)] pub(crate) fn init_empty_shortform_dir( &mut self, sb: &super::sb::XfsSuperblock, @@ -689,6 +713,7 @@ impl XfsFilesystem { mode: u16, uid: u32, gid: u32, + times: Option, ) -> Result<(), FilesystemError> { if parent_ino > u64::from(u32::MAX) || ino > u64::from(u32::MAX) { // Inode numbers beyond 32 bits need the 8-byte short-form layout @@ -718,8 +743,9 @@ impl XfsFilesystem { BigEndian::write_u32(&mut buf[8..12], uid); BigEndian::write_u32(&mut buf[12..16], gid); for b in buf.iter_mut().take(56).skip(20) { - *b = 0; // projid, pad, atime, mtime, ctime + *b = 0; // projid, pad, atime, mtime, ctime — stamped below } + stamp_v4_times(&mut buf, times); BigEndian::write_u64(&mut buf[56..64], 6); // di_size = empty sf fork len for b in buf.iter_mut().take(92).skip(64) { *b = 0; // nblocks, extsize, nextents, anextents, forkoff, aformat, dm*, flags @@ -962,6 +988,7 @@ impl XfsFilesystem { /// as an empty short-form directory, and insert it into `parent_ino`. /// Returns the new directory's inode number. The fit check runs before any /// write, so a `DiskFull` parent never leaves a dangling allocated inode. + #[allow(clippy::too_many_arguments)] pub(crate) fn do_create_directory( &mut self, parent_ino: u64, @@ -969,6 +996,7 @@ impl XfsFilesystem { mode: u16, uid: u32, gid: u32, + times: Option, ) -> Result { let sb = self.superblock().clone(); @@ -978,7 +1006,7 @@ impl XfsFilesystem { self.dir_can_insert(&sb, parent_ino, name, true)?; let ino = self.alloc_inode_slot(&sb)?; - self.init_empty_shortform_dir(&sb, ino, parent_ino, mode, uid, gid)?; + self.init_empty_shortform_dir(&sb, ino, parent_ino, mode, uid, gid, times)?; self.dir_insert_entry(&sb, parent_ino, name, ino, true)?; self.reader.flush()?; Ok(ino) @@ -1103,6 +1131,7 @@ impl XfsFilesystem { /// (`di_format = 2`), and the caller must have bounded the count to the /// inline literal area. #[allow(clippy::too_many_arguments)] + #[allow(clippy::too_many_arguments)] pub(crate) fn init_file_inode( &mut self, sb: &super::sb::XfsSuperblock, @@ -1113,6 +1142,7 @@ impl XfsFilesystem { size: u64, extents: &[(u64, u64, u32)], bmbt_leaf_fsblock: Option, + times: Option, ) -> Result<(), FilesystemError> { let (_core, mut buf) = self.read_inode_buf(ino)?; let version = buf[4]; @@ -1133,9 +1163,16 @@ impl XfsFilesystem { } BigEndian::write_u32(&mut buf[8..12], uid); BigEndian::write_u32(&mut buf[12..16], gid); + // Zero the header slice up through the end of the time fields; the + // three di_atime/di_mtime/di_ctime pairs at 32..56 then get real + // seconds (preserved from a cross-fs copy or freshly stamped for a + // new file). tv_nsec is left at zero — XFS carries nanoseconds on + // disk but every producer we import from (tar / EFS / host stat on + // second precision) rounds to whole seconds anyway. for b in buf.iter_mut().take(56).skip(20) { *b = 0; } + stamp_v4_times(&mut buf, times); BigEndian::write_u64(&mut buf[56..64], size); // di_nblocks = data extent blocks + (bmbt leaf block, if any). let data_nblocks: u64 = extents.iter().map(|&(_, _, c)| c as u64).sum(); @@ -1209,6 +1246,7 @@ impl XfsFilesystem { mode: u16, uid: u32, gid: u32, + times: Option, ) -> Result { let sb = self.superblock().clone(); self.dir_can_insert(&sb, parent_ino, name, false)?; @@ -1277,6 +1315,7 @@ impl XfsFilesystem { data_len, &extents, bmbt_leaf_fsblock, + times, )?; self.dir_insert_entry(&sb, parent_ino, name, ino, false)?; self.reader.flush()?; @@ -2601,7 +2640,7 @@ impl XfsFilesystem { // Create it: a fresh empty short-form directory linked into the root. self.dir_can_insert(sb, root_ino, "lost+found", true)?; let ino = self.alloc_inode_slot(sb)?; - self.init_empty_shortform_dir(sb, ino, root_ino, 0o040755, 0, 0)?; + self.init_empty_shortform_dir(sb, ino, root_ino, 0o040755, 0, 0, None)?; self.dir_insert_entry(sb, root_ino, "lost+found", ino, true)?; Ok(ino) } diff --git a/src/fs/xfs/inode.rs b/src/fs/xfs/inode.rs index 6e312894..97125000 100644 --- a/src/fs/xfs/inode.rs +++ b/src/fs/xfs/inode.rs @@ -30,6 +30,10 @@ pub struct XfsDinodeCore { pub aformat: u8, pub flags: u16, pub gen: u32, + /// di_mtime.tv_sec — seconds since 1970. Surfaced so `list_directory` + /// can populate `FileEntry.modified_unix`, which is what `tar_export` + /// writes into its headers. + pub mtime: i32, } impl XfsDinodeCore { @@ -58,7 +62,9 @@ impl XfsDinodeCore { gid: BigEndian::read_u32(&buf[12..16]), nlink: BigEndian::read_u32(&buf[16..20]), // bytes 20..32: projid, v2_pad, flushiter — diagnostic only. - // bytes 32..56: atime/mtime/ctime — not surfaced yet. + // atime = 32..40, mtime = 40..48, ctime = 48..56 (each = tv_sec i32 + tv_nsec i32); + // v1/v2 use `i32` seconds → surfaced as `mtime`, dropping nsec. + mtime: BigEndian::read_i32(&buf[40..44]), size: BigEndian::read_u64(&buf[56..64]), nblocks: BigEndian::read_u64(&buf[64..72]), // bytes 72..76: extsize. diff --git a/src/fs/xfs/mod.rs b/src/fs/xfs/mod.rs index 11245e3c..6aa2ceb5 100644 --- a/src/fs/xfs/mod.rs +++ b/src/fs/xfs/mod.rs @@ -418,13 +418,21 @@ impl XfsFilesystem { if matches!(entry_type, EntryType::Directory) { size = 0; } + let modified_unix = if core.mtime > 0 { + Some(core.mtime as u64) + } else { + None + }; + let modified = + modified_unix.map(|s| crate::fs::unix_common::inode::format_unix_timestamp(s as i64)); Ok(FileEntry { name, path, entry_type, size, location: ino, - modified: None, + modified, + modified_unix, type_code: None, creator_code: None, symlink_target, @@ -768,13 +776,21 @@ impl Filesystem for XfsFilesystem { core.mode ))); } + let modified_unix = if core.mtime > 0 { + Some(core.mtime as u64) + } else { + None + }; + let modified = + modified_unix.map(|s| crate::fs::unix_common::inode::format_unix_timestamp(s as i64)); Ok(FileEntry { name: "/".into(), path: "/".into(), entry_type: EntryType::Directory, size: 0, location: rootino, - modified: None, + modified, + modified_unix, type_code: None, creator_code: None, symlink_target: None, diff --git a/src/fs/xfs/repair.rs b/src/fs/xfs/repair.rs index cea51ba1..1cfcde77 100644 --- a/src/fs/xfs/repair.rs +++ b/src/fs/xfs/repair.rs @@ -274,7 +274,16 @@ impl EditableFilesystem for XfsFilesystem { let mode = ((options.mode.unwrap_or(0o644) & 0o7777) | 0o100000) as u16; let uid = options.uid.unwrap_or(0); let gid = options.gid.unwrap_or(0); - let ino = self.do_create_file(parent.location, name, data, data_len, mode, uid, gid)?; + let ino = self.do_create_file( + parent.location, + name, + data, + data_len, + mode, + uid, + gid, + options.unix_times, + )?; self.child_entry(&parent.path, name.to_string(), ino) } @@ -290,7 +299,8 @@ impl EditableFilesystem for XfsFilesystem { let mode = ((options.mode.unwrap_or(0o755) & 0o7777) | 0o040000) as u16; let uid = options.uid.unwrap_or(0); let gid = options.gid.unwrap_or(0); - let ino = self.do_create_directory(parent.location, name, mode, uid, gid)?; + let ino = + self.do_create_directory(parent.location, name, mode, uid, gid, options.unix_times)?; self.child_entry(&parent.path, name.to_string(), ino) } diff --git a/src/model/edit_queue.rs b/src/model/edit_queue.rs index 7c9358de..f38e8018 100644 --- a/src/model/edit_queue.rs +++ b/src/model/edit_queue.rs @@ -21,28 +21,39 @@ use crate::fs::resource_fork::{self, ImportedResourceFork}; /// Original timestamps captured from a source file so a cross-image copy can /// reproduce them on the destination instead of stamping the current time. -/// Each filesystem family uses its own date representation; the unused half is -/// `None`. Amiga dates are applied via `CreateFileOptions::amiga_date` (honored -/// by AFFS `create_file`); HFS dates via `set_dates` after creation. +/// Each filesystem family uses its own date representation; the unused halves +/// stay `None`. Amiga dates are applied via `CreateFileOptions::amiga_dates` +/// (honored by AFFS `create_file`); HFS dates via `set_dates` after creation; +/// Unix mtimes via `CreateFileOptions::unix_times` (honoured by every Unix- +/// family driver — ext, UFS, EFS, XFS, minix, squashfs, …). #[derive(Debug, Clone, Default)] pub struct PreservedDates { /// AmigaDOS `(days, minutes, ticks)` since 1978-01-01. pub amiga: Option<(i32, i32, i32)>, /// HFS/HFS+ `(create, modify, backup)` in Mac-epoch seconds. pub mac: Option<(u32, u32, u32)>, + /// Unix mtime in seconds since 1970-01-01, the numeric `FileEntry. + /// modified_unix` captured from any per-second-dated source (Unix + /// inode / Mac dates / Amiga datestamp / tar Header / host stat). + pub unix_mtime: Option, } impl PreservedDates { - /// Capture whatever raw dates a source `FileEntry` carries (HFS `mac_dates` - /// and/or Amiga `amiga_date`). Returns `None` when the entry has neither, so - /// the copy just stamps the current time. + /// Capture whatever raw dates a source `FileEntry` carries. Returns + /// `None` when the entry has no timestamp at all — the copy then just + /// stamps the current time. `unix_mtime` is populated whenever the + /// entry carries one, so a cross-fs copy from any per-second-dated + /// source (ext → EFS, AFFS → ext, tar → xfs, …) preserves the date + /// through Unix-family drivers' new `CreateFileOptions.unix_times`. pub fn from_entry(entry: &FileEntry) -> Option { - if entry.mac_dates.is_none() && entry.amiga_date.is_none() { + if entry.mac_dates.is_none() && entry.amiga_date.is_none() && entry.modified_unix.is_none() + { return None; } Some(Self { amiga: entry.amiga_date, mac: entry.mac_dates, + unix_mtime: entry.modified_unix, }) } } @@ -233,8 +244,16 @@ pub fn apply_edit( type_code: prodos_type.map(|t| format!("${:02X}", t)), aux_type: *prodos_aux, // Amiga dates round-trip via create_file (AFFS honors this); - // HFS dates are applied with set_dates after creation below. + // HFS dates are applied with set_dates after creation below; + // Unix mtime rides through every Unix-family driver via + // create_file's `unix_times` (falls back to the source's + // native scheme when both are set, since the driver picks + // whichever it can honour). amiga_dates: dates.as_ref().and_then(|d| d.amiga), + unix_times: dates + .as_ref() + .and_then(|d| d.unix_mtime) + .map(crate::fs::times::UnixTimes::mtime_only), ..Default::default() }; @@ -1388,6 +1407,7 @@ mod tests { dates: Some(PreservedDates { amiga: Some(target), mac: None, + unix_mtime: None, }), on_conflict: crate::fs::replace::OnConflict::Fail, }, diff --git a/src/partition/mod.rs b/src/partition/mod.rs index 3ab16975..4a40b98c 100644 --- a/src/partition/mod.rs +++ b/src/partition/mod.rs @@ -349,6 +349,38 @@ fn detect_superfloppy(first_sector: &[u8; 512], reader: &mut (impl Read + Seek)) } } + // HPFS, the third filesystem behind MBR type 0x07. Its boot sector is an + // x86 VBR with a BPB and an 0xAA55 signature, but `num_fats` is 0 and + // `reserved_sectors` is 0, so the FAT probe below rejects it and it used to + // fall through to the MBR parse — which claimed it, found no partition + // entries, and reported "MBR, no partitions". `backup` then backed up + // nothing at all and exited 0 (R-022). `looks_like_hpfs` wants both the + // superblock magic at sector 16 and the spareblock magic at sector 17, so a + // chance match needs 64 bits to line up. + if crate::fs::hpfs::looks_like_hpfs(reader, 0) { + let _ = reader.seek(SeekFrom::Start(0)); + return Some("HPFS".to_string()); + } + let _ = reader.seek(SeekFrom::Start(0)); + + // QDOS Microdrive cartridge. Sector 0 is a cartridge header, not a VBR: + // ten zero bytes, an `ff ff ff` sync at 0x0A, then the ASCII name. Nothing + // there resembles a partition table, but the MBR parse read the sync as a + // bad 0xAA55 and refused the whole cartridge — before the probe that + // already recognises it could run (R-033). Same shape as R-022: the driver + // and the detection both existed; nothing consulted them on this path. + // Gated on the exact cartridge length as well as the header, so it cannot + // claim anything else. + if let Ok(end) = reader.seek(SeekFrom::End(0)) { + let _ = reader.seek(SeekFrom::Start(0)); + if end == crate::fs::qdos_mdv::MDV_CART_BYTES as u64 + && crate::fs::qdos_mdv::looks_like_mdv_sector_zero(first_sector) + { + return Some("qdos_mdv".to_string()); + } + } + let _ = reader.seek(SeekFrom::Start(0)); + if first_sector[0] == 0xEB || first_sector[0] == 0xE9 { let bytes_per_sector = u16::from_le_bytes([first_sector[11], first_sector[12]]); let sectors_per_cluster = first_sector[13]; @@ -701,6 +733,42 @@ fn detect_superfloppy(first_sector: &[u8; 512], reader: &mut (impl Read + Seek)) None } +/// Lowercase, alphanumerics only — so `X68k`, `x68k` and `X-68K` agree. +fn normalise_layout(s: &str) -> String { + s.chars() + .filter(|c| c.is_ascii_alphanumeric()) + .map(|c| c.to_ascii_lowercase()) + .collect() +} + +/// Whether `expected` names a layout at all, so a typo is a usage error rather +/// than a silent assertion failure. Returns the accepted vocabulary for the +/// error message. +pub fn layout_vocabulary() -> Vec { + let mut v: Vec = PartitionTable::ALL_TYPE_NAMES + .iter() + .map(|s| s.to_string()) + .collect(); + v.extend( + PartitionTable::LAYOUT_ALIASES + .iter() + .map(|(a, _)| a.to_string()), + ); + v +} + +/// True when `expected` is a scheme name or a known alias. +pub fn is_known_layout(expected: &str) -> bool { + let want = normalise_layout(expected); + !want.is_empty() + && (PartitionTable::ALL_TYPE_NAMES + .iter() + .any(|n| normalise_layout(n) == want) + || PartitionTable::LAYOUT_ALIASES + .iter() + .any(|(a, _)| normalise_layout(a) == want)) +} + impl PartitionTable { /// Detect and parse the partition table from a readable+seekable source. pub fn detect(reader: &mut (impl Read + Seek)) -> Result { @@ -1353,6 +1421,49 @@ impl PartitionTable { } } + /// Every value [`PartitionTable::type_name`] can return, in variant order. + /// + /// Exists so `tests/doc_parity.rs` can require a README row per scheme; + /// the `#[cfg(test)]` guard below keeps it honest when a variant is added. + pub const ALL_TYPE_NAMES: &'static [&'static str] = &[ + "MBR", "GPT", "APM", "RDB", "SGI", "Sun", "AHDI", "X68k", "None", "DSD", + ]; + + /// Whether this disk carries a partition table at all. + /// + /// `None` is a superfloppy — a filesystem starting at sector 0 with no + /// table. `Dsd` is the odd one: no table exists on the disk either, but the + /// reader de-interleaves the two sides into two addressable volumes, so it + /// answers like a partitioned disk for every purpose a caller has. + pub fn is_partitioned(&self) -> bool { + !matches!(self, PartitionTable::None { .. }) + } + + /// Layout words a caller may ask for beyond the scheme names themselves. + pub const LAYOUT_ALIASES: &'static [(&'static str, &'static str)] = &[ + ( + "superfloppy", + "a filesystem at sector 0, no partition table", + ), + ("flat", "alias for superfloppy"), + ("partitioned", "any partition table"), + ("hd", "alias for partitioned"), + ]; + + /// Does this disk's layout satisfy `expected`? + /// + /// Accepts a scheme name from [`Self::ALL_TYPE_NAMES`] (`mbr`, `gpt`, + /// `rdb`, …) or one of [`Self::LAYOUT_ALIASES`]. Case and punctuation are + /// ignored, matching `--expect-fs`. + pub fn layout_matches(&self, expected: &str) -> bool { + let want = normalise_layout(expected); + match want.as_str() { + "superfloppy" | "flat" => !self.is_partitioned(), + "partitioned" | "hd" => self.is_partitioned(), + other => normalise_layout(self.type_name()) == other, + } + } + /// Get a human-readable name for the partition table type. pub fn type_name(&self) -> &'static str { match self { @@ -1829,6 +1940,54 @@ mod tests { assert_eq!(parts[0].type_name, "FAT"); } + /// A `.mdv` cartridge header, as it appears on a real MiSTer cartridge: + /// ten zero bytes, an `ff ff ff` sync, then the ASCII name. + fn mdv_sector_zero() -> [u8; 512] { + let mut s = [0u8; 512]; + s[0x0A] = 0xFF; + s[0x0B] = 0xFF; + s[0x0C] = 0xFF; + s[0x0E..0x18].copy_from_slice(b"Test "); + s + } + + #[test] + fn an_mdv_cartridge_is_a_superfloppy_not_a_bad_mbr() { + let sector0 = mdv_sector_zero(); + let mut img = vec![0u8; crate::fs::qdos_mdv::MDV_CART_BYTES]; + img[..512].copy_from_slice(§or0); + let mut cur = std::io::Cursor::new(img); + assert_eq!( + detect_superfloppy(§or0, &mut cur).as_deref(), + Some("qdos_mdv"), + "R-033: the MBR parse used to claim this and refuse the cartridge" + ); + } + + /// The control in the other direction, which is what R-022 taught: a probe + /// that fires too eagerly is worse than one that never fires. Both halves + /// of the gate have to be required. + #[test] + fn the_mdv_probe_needs_both_the_size_and_the_header() { + // Right header, wrong length. + let sector0 = mdv_sector_zero(); + let mut short = std::io::Cursor::new(vec![0u8; 4096]); + assert_ne!( + detect_superfloppy(§or0, &mut short).as_deref(), + Some("qdos_mdv"), + "a cartridge header at the wrong length must not be claimed" + ); + + // Right length, wrong header — an all-zero cartridge-sized file. + let blank = [0u8; 512]; + let mut sized = std::io::Cursor::new(vec![0u8; crate::fs::qdos_mdv::MDV_CART_BYTES]); + assert_ne!( + detect_superfloppy(&blank, &mut sized).as_deref(), + Some("qdos_mdv"), + "cartridge-sized bytes without the header must not be claimed" + ); + } + #[test] fn detect_superfloppy_by_fs_signature() { // Each FS signature planted at its magic offset must be detected as a @@ -2430,3 +2589,96 @@ mod native_slot_tests { assert_eq!(table.native_slot(&parts[1]), Some(4)); } } + +#[cfg(test)] +mod type_name_parity { + use super::*; + + /// Never called — it exists so that adding a `PartitionTable` variant is a + /// compile error here, which is the cue to extend `ALL_TYPE_NAMES` too. + #[allow(dead_code)] + fn every_variant_has_an_index(t: &PartitionTable) -> usize { + match t { + PartitionTable::Mbr(_) => 0, + PartitionTable::Gpt { .. } => 1, + PartitionTable::Apm(_) => 2, + PartitionTable::Rdb(_) => 3, + PartitionTable::Sgi(_) => 4, + PartitionTable::Sun(_) => 5, + PartitionTable::Ahdi(_) => 6, + PartitionTable::X68k { .. } => 7, + PartitionTable::None { .. } => 8, + PartitionTable::Dsd { .. } => 9, + } + } + + #[test] + fn all_type_names_covers_every_variant() { + assert_eq!( + PartitionTable::ALL_TYPE_NAMES.len(), + 10, + "a PartitionTable variant was added or removed: update ALL_TYPE_NAMES, \ + every_variant_has_an_index, and the README table tests/doc_parity.rs checks" + ); + } +} + +#[cfg(test)] +mod layout_expectation_tests { + use super::*; + + fn superfloppy() -> PartitionTable { + PartitionTable::None { + size_bytes: 1_474_560, + fs_hint: "FAT".into(), + } + } + + #[test] + fn superfloppy_is_not_partitioned() { + let pt = superfloppy(); + assert!(!pt.is_partitioned()); + assert!(pt.layout_matches("superfloppy")); + assert!(pt.layout_matches("FLAT")); + assert!(pt.layout_matches("none")); + assert!(!pt.layout_matches("partitioned")); + assert!(!pt.layout_matches("mbr")); + } + + #[test] + fn a_scheme_name_matches_only_itself() { + let pt = PartitionTable::Dsd { + size_bytes: 409_600, + }; + assert!(pt.layout_matches("dsd")); + assert!(!pt.layout_matches("mbr")); + // Dsd has no on-disk table but yields two addressable volumes, so it + // answers "partitioned" — that is what a caller is actually asking. + assert!(pt.layout_matches("partitioned")); + assert!(!pt.layout_matches("superfloppy")); + } + + #[test] + fn punctuation_and_case_are_ignored() { + let pt = PartitionTable::Dsd { + size_bytes: 409_600, + }; + assert!(pt.layout_matches("D-S-D")); + assert!(pt.layout_matches("dsd ")); + } + + #[test] + fn the_vocabulary_covers_every_scheme_and_alias() { + // A new PartitionTable variant must become askable, not silently + // unaskable; ALL_TYPE_NAMES is pinned against the variant count by + // type_name_parity, so this inherits that guard. + for n in PartitionTable::ALL_TYPE_NAMES { + assert!(is_known_layout(n), "{n} should be askable"); + } + for (a, _) in PartitionTable::LAYOUT_ALIASES { + assert!(is_known_layout(a), "{a} should be askable"); + } + assert!(!is_known_layout("mbrr")); + assert!(!is_known_layout("")); + } +} diff --git a/src/partition/sgi_hdd_builder.rs b/src/partition/sgi_hdd_builder.rs index 9605719c..770ec515 100644 --- a/src/partition/sgi_hdd_builder.rs +++ b/src/partition/sgi_hdd_builder.rs @@ -465,7 +465,7 @@ mod tests { fn streamed_file_matches_in_memory_image() { use std::io::{Read, Seek, SeekFrom}; let opts = SgiHddOptions::new(40 * 1024 * 1024, "STREAM"); - let (mem, layout) = build_sgi_efs_hdd(&opts).unwrap(); + let (mut mem, layout) = build_sgi_efs_hdd(&opts).unwrap(); let mut file = tempfile::tempfile().expect("tempfile"); let streamed_layout = write_sgi_efs_hdd(&mut file, &opts).expect("stream"); @@ -476,6 +476,22 @@ mod tests { file.seek(SeekFrom::Start(0)).unwrap(); file.read_to_end(&mut on_disk).unwrap(); assert_eq!(on_disk.len() as u64, layout.disk_bytes, "exact size"); + + // The EFS root inode's atime/mtime/ctime are stamped from + // `SystemTime::now()` and the two builds run microseconds apart, so + // a byte-for-byte compare would race on a second boundary. Zero the + // 12-byte time window in both copies before comparing — every other + // byte still has to match. Root inode 2 lives at slot 2 in the first + // CG's inode block; times are at offsets 12..24 of the inode. + let efs_off = (layout.efs_first_sector * SECTOR_SIZE) as usize; + let firstcg = 2 + (layout.efs_sectors as u32).div_ceil(8).div_ceil(512); + let root_off = efs_off + firstcg as usize * 512 + 2 * 128; + for b in mem[root_off + 12..root_off + 24].iter_mut() { + *b = 0; + } + for b in on_disk[root_off + 12..root_off + 24].iter_mut() { + *b = 0; + } assert_eq!(on_disk, mem, "streamed bytes match the in-memory image"); } diff --git a/src/rbformats/appimage.rs b/src/rbformats/appimage.rs index 5951e406..67dc1492 100644 --- a/src/rbformats/appimage.rs +++ b/src/rbformats/appimage.rs @@ -259,7 +259,11 @@ mod tests { .expect("create"); ed.sync_metadata().expect("sync"); - let after = ed.into_backing().into_inner().into_inner(); + let after = ed + .into_backing() + .expect("in-place commit keeps the handle") + .into_inner() + .into_inner(); assert_eq!( &after[..stub_len], &stub_before[..], diff --git a/src/rbformats/chd.rs b/src/rbformats/chd.rs index 9066cd38..fb0f1d4c 100644 --- a/src/rbformats/chd.rs +++ b/src/rbformats/chd.rs @@ -495,7 +495,12 @@ pub fn shrink_sgi_disk_to_chd( ); } if dst.extension().and_then(|s| s.to_str()) != Some("chd") { - anyhow::bail!("output path must end in .chd (got {})", dst.display()); + // Refusing a wrong output extension is usage-bad-input, so it carries + // USAGE_ERROR rather than the catch-all 1 (R-004). + return Err(crate::cli::exit::usage(format!( + "output path must end in .chd (got {})", + dst.display() + ))); } if let Some(parent) = dst.parent() { if !parent.as_os_str().is_empty() && !parent.exists() { diff --git a/src/rbformats/zip_disk.rs b/src/rbformats/zip_disk.rs index 9cc1b166..77a5c49e 100644 --- a/src/rbformats/zip_disk.rs +++ b/src/rbformats/zip_disk.rs @@ -84,16 +84,25 @@ fn sparse_copy(mut src: impl Read, dst: &mut File) -> io::Result { /// Whatever we pick is handed to partition detection, which validates it, /// so this only has to be good enough to disambiguate the disk from /// sidecar files (readme, checksum, ...). Matched case-insensitively. +/// Derived from [`DISK_IMAGE_EXTS`] rather than kept as a second list, because +/// the second list drifted: `.dmg` was canonical and missing here, so a +/// Finder-made zip holding one `.dmg` reported "no obvious disk image" and +/// demanded `--inside` (R-027). `.adf`, `.2mg`, `.woz` and a dozen others were +/// missing the same way. +/// +/// `bin` is the one exclusion: far too common in a mixed archive to identify a +/// disk image by, and picking the wrong entry is worse than asking. fn is_disk_image_entry(name: &str) -> bool { - const EXTS: &[&str] = &[ - ".img", ".raw", ".dd", ".iso", ".hdd", ".hda", ".hdv", ".dsk", ".vhd", ".hdf", ".hds", - ".ima", ".vmdk", ".qcow2", ".chd", - ]; if is_apple_double(name) { return false; } let lower = name.to_ascii_lowercase(); - EXTS.iter().any(|e| lower.ends_with(e)) + let Some((_, ext)) = lower.rsplit_once('.') else { + return false; + }; + crate::model::file_types::DISK_IMAGE_EXTS + .iter() + .any(|e| !e.eq_ignore_ascii_case("bin") && e.eq_ignore_ascii_case(ext)) } /// A macOS resource-fork stub, not content. Zipping `disk.hda` in Finder also @@ -118,7 +127,10 @@ fn collect_entries(archive: &mut zip::ZipArchive) -> Vec Result { ctx.type_byte, ctx.type_string.as_deref(), ) - .map_err(|e| anyhow!("opening filesystem for write: {e}"))?; + .map_err(|e| crate::cli::resolve::write_open_error("opening filesystem for write", e))?; let mut count = 0u64; for edit in &sess.edits { diff --git a/tests/doc_parity.rs b/tests/doc_parity.rs new file mode 100644 index 00000000..45835331 --- /dev/null +++ b/tests/doc_parity.rs @@ -0,0 +1,123 @@ +//! Documentation-to-source parity. +//! +//! R-001, R-002 and R-018 were all the same failure: a hand-kept list in a +//! markdown file drifted from the code or the CI config it described, and all +//! three were found by a human reading both halves. Nothing in the regression +//! suite could catch them — the suite runs the binary, and these are claims +//! *about* the binary. These tests read both halves instead. +//! +//! Scope is deliberately narrow: only pairs where one side is machine-readable +//! and the drift has actually happened. A test that has to guess at prose is +//! worse than no test, because it gets muted. + +use std::fs; +use std::path::PathBuf; + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +fn read(rel: &str) -> String { + let path = repo_root().join(rel); + fs::read_to_string(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display())) +} + +/// The lines of the markdown table that follows `heading`, up to the next +/// heading of any level. +fn table_rows_under(doc: &str, heading: &str) -> Vec { + let start = doc + .find(heading) + .unwrap_or_else(|| panic!("heading {heading:?} not found — was it renamed?")); + let rest = &doc[start + heading.len()..]; + let end = rest.find("\n#").unwrap_or(rest.len()); + rest[..end] + .lines() + .map(str::trim) + .filter(|l| l.starts_with('|')) + .map(str::to_string) + .collect() +} + +/// The first cell of a markdown table row. +fn first_cell(row: &str) -> String { + row.trim_matches('|') + .split('|') + .next() + .unwrap_or("") + .trim() + .to_string() +} + +/// R-001: every `PartitionTable` variant needs a row in the README's +/// partition-table table. AHDI and X68000 were both missing when this was +/// found by hand; DSD went missing later, which is the drift repeating. +#[test] +fn readme_documents_every_partition_table_scheme() { + let readme = read("README.md"); + let cells: Vec = table_rows_under(&readme, "### Partition tables") + .iter() + .map(|r| first_cell(r).to_uppercase()) + .collect(); + + let missing: Vec<&str> = rusty_backup::partition::PartitionTable::ALL_TYPE_NAMES + .iter() + .copied() + .filter(|name| { + let want = name.to_uppercase(); + !cells.iter().any(|c| c.contains(&want)) + }) + .collect(); + + assert!( + missing.is_empty(), + "README.md 'Partition tables' has no row for {missing:?}. \ + Every PartitionTable variant is a scheme a user can open, so it needs one. \ + Rows found: {cells:?}" + ); +} + +/// R-002: the stale capability table in `src/fs/README.md` was deleted in +/// favour of pointing at the live dispatch, because a hand-kept table two +/// levels below the code it describes cannot keep up with forty drivers. +/// This stops one growing back. +#[test] +fn fs_readme_has_no_hand_kept_capability_table() { + let doc = read("src/fs/README.md"); + assert!( + !doc.contains("(planned)"), + "src/fs/README.md claims a filesystem is 'planned'. That claim outlived \ + the ext driver by years (R-002). Capabilities belong in the top-level \ + README's Filesystems table; routing belongs in fs/mod.rs." + ); + assert!( + !doc.contains("| Browsing |"), + "a per-filesystem capability table has grown back in src/fs/README.md. \ + It went stale last time (R-002) — see the note in that file." + ); +} + +/// R-018: CONTRIBUTING.md's vintage-build verification command must be exactly +/// what CI's Windows vintage leg runs. It drifted by one feature +/// (`windows-legacy`), and the resulting error named two call sites, so it read +/// as an engine bug rather than a stale doc. +#[test] +fn contributing_vintage_features_match_ci() { + let contributing = read("CONTRIBUTING.md"); + let workflow = read(".github/workflows/release.yml"); + + let features = contributing + .lines() + .find_map(|l| { + let l = l.trim(); + l.strip_prefix("--no-default-features --features ") + .map(|rest| rest.trim_end_matches('\\').trim().to_string()) + }) + .expect("CONTRIBUTING.md no longer has a '--no-default-features --features ...' line"); + + assert!( + workflow.contains(&features), + "CONTRIBUTING.md documents the vintage feature list as `{features}`, which \ + appears nowhere in .github/workflows/release.yml. The two must stay together \ + (R-018) — a doc-only feature list is one nobody has ever built." + ); +} diff --git a/tests/timestamp_preservation.rs b/tests/timestamp_preservation.rs new file mode 100644 index 00000000..f473a5aa --- /dev/null +++ b/tests/timestamp_preservation.rs @@ -0,0 +1,505 @@ +//! Round-trip regression tests for file-mtime preservation. +//! +//! The rule the code enforces: +//! +//! - A **genuinely new** file (rb-cli `put` from stdin, GUI "new blank file") +//! gets `now` — every driver did this before, and still does when +//! `CreateFileOptions.unix_times` is `None`. +//! - A **copy / import / extract** preserves the source date end-to-end: +//! * host file → image (`dir_import`) — host stat mtime lands on disk +//! * tar entry → image (`tar_import`) — tar Header mtime lands on disk +//! * image → tar (`tar_export`) — the on-disk mtime lands in the tar Header +//! * image → host (`fork_export`) — the on-disk mtime lands on the host file +//! +//! One end-to-end test per stage; one "genuinely new stamps now" test per +//! kind so a future refactor can't silently regress the distinction. + +use rusty_backup::fs::dir_import::{import_dir, DirImportOptions}; +use rusty_backup::fs::efs::{create_blank_efs, EfsFilesystem}; +use rusty_backup::fs::entry::FileEntry; +use rusty_backup::fs::filesystem::{CreateFileOptions, EditableFilesystem, Filesystem}; +use rusty_backup::fs::fork_export::export_file_with_fork; +use rusty_backup::fs::resource_fork::ResourceForkMode; +use rusty_backup::fs::tar_export::{export_tar, TarCompression, TarExportOptions}; +use rusty_backup::fs::tar_import::{import_tar_into, TarImportOptions}; +use rusty_backup::fs::times::UnixTimes; +use std::io::Cursor; + +const YEAR_2020: u64 = 1_577_836_800; // 2020-01-01 00:00:00 UTC +const YEAR_2018: u64 = 1_514_764_800; // 2018-01-01 00:00:00 UTC + +fn fresh_efs() -> EfsFilesystem>> { + let img = create_blank_efs(1024 * 1024, "rb-efs").expect("format 1 MiB EFS"); + EfsFilesystem::open(Cursor::new(img), 0).expect("open EFS") +} + +fn now_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs() +} + +fn put_bytes( + fs: &mut EfsFilesystem>>, + name: &str, + data: &[u8], + times: Option, +) -> FileEntry { + let root = fs.root().unwrap(); + fs.create_file( + &root, + name, + &mut &data[..], + data.len() as u64, + &CreateFileOptions { + unix_times: times, + ..Default::default() + }, + ) + .expect("create_file") +} + +/// dir_import from a host file with a known mtime lands that mtime on the +/// image inode — the "put --from-dir preserves the source date" contract. +#[test] +fn dir_import_preserves_host_mtime() { + let tmp = tempfile::tempdir().unwrap(); + let host = tmp.path().join("aged.txt"); + std::fs::write(&host, b"aged content").unwrap(); + // Backdate the host file to a known point in 2018. + let ft = filetime::FileTime::from_unix_time(YEAR_2018 as i64, 0); + filetime::set_file_times(&host, ft, ft).unwrap(); + + let mut fs = fresh_efs(); + let dest = fs.root().unwrap(); + let stats = import_dir( + &mut fs, + &dest, + tmp.path(), + &DirImportOptions::default(), + &|_| {}, + ) + .expect("import_dir"); + assert_eq!(stats.files, 1); + + let listed = fs + .list_directory(&dest) + .unwrap() + .into_iter() + .find(|e| e.name == "aged.txt") + .expect("host file imported"); + assert_eq!( + listed.modified_unix, + Some(YEAR_2018), + "the host's mtime must land on the image inode verbatim" + ); +} + +/// tar_import from an archive whose entry has a 2020 mtime lands that mtime +/// on the image inode. Same shape as `dir_import` — the mtime source is the +/// tar Header instead of `stat`. +#[test] +fn tar_import_preserves_archive_mtime() { + // Build a tarball in memory: one file "src.txt" with mtime = 2020-01-01. + let mut archive_bytes = Vec::new(); + { + let mut builder = tar::Builder::new(&mut archive_bytes); + let mut header = tar::Header::new_gnu(); + header.set_mode(0o644); + header.set_uid(0); + header.set_gid(0); + header.set_size(3); + header.set_mtime(YEAR_2020); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + builder + .append_data(&mut header, "src.txt", &b"abc"[..]) + .unwrap(); + builder.finish().unwrap(); + } + + let mut fs = fresh_efs(); + let dest = fs.root().unwrap(); + let stats = import_tar_into( + &mut fs, + &dest, + Cursor::new(archive_bytes), + &TarImportOptions::default(), + &|_| {}, + ) + .expect("import_tar_into"); + assert_eq!(stats.files, 1); + + let listed = fs + .list_directory(&dest) + .unwrap() + .into_iter() + .find(|e| e.name == "src.txt") + .expect("tar file imported"); + assert_eq!( + listed.modified_unix, + Some(YEAR_2020), + "the tar Header's mtime must land on the image inode verbatim" + ); +} + +/// The other direction: a file created with a preserved mtime, then +/// exported through `tar_export`, must show that mtime in the tar Header +/// (was hard-coded to 0 before this change). +#[test] +fn tar_export_carries_source_mtime() { + let mut fs = fresh_efs(); + put_bytes( + &mut fs, + "dated.txt", + b"payload", + Some(UnixTimes::all(YEAR_2020)), + ); + + let mut tar_bytes = Vec::new(); + let root = fs.root().unwrap(); + export_tar( + &mut fs, + &root, + "", + &mut tar_bytes, + TarCompression::None, + &TarExportOptions::default(), + &|_| {}, + ) + .expect("export_tar"); + + // Read the tar back and confirm the file entry carries our mtime. + let mut ar = tar::Archive::new(&tar_bytes[..]); + let entry = ar + .entries() + .unwrap() + .filter_map(|e| e.ok()) + .find(|e| { + e.path() + .map(|p| p.file_name().and_then(|n| n.to_str()) == Some("dated.txt")) + .unwrap_or(false) + }) + .expect("tar contains dated.txt"); + let mtime = entry.header().mtime().expect("mtime present"); + assert_eq!( + mtime, YEAR_2020, + "tar_export must carry the source's modified_unix as the entry's mtime" + ); +} + +/// The full round-trip: a host file dated 2018 goes into an EFS image via +/// dir_import, then out again via tar_export — the tar entry carries the +/// original date. This is the whole point of the change: source-of-truth +/// mtime survives every hop. +#[test] +fn host_to_image_to_tar_preserves_mtime_end_to_end() { + let tmp = tempfile::tempdir().unwrap(); + let host = tmp.path().join("old.txt"); + std::fs::write(&host, b"end-to-end").unwrap(); + let ft = filetime::FileTime::from_unix_time(YEAR_2018 as i64, 0); + filetime::set_file_times(&host, ft, ft).unwrap(); + + let mut fs = fresh_efs(); + let dest = fs.root().unwrap(); + import_dir( + &mut fs, + &dest, + tmp.path(), + &DirImportOptions::default(), + &|_| {}, + ) + .expect("import_dir"); + + let mut tar_bytes = Vec::new(); + let root = fs.root().unwrap(); + export_tar( + &mut fs, + &root, + "", + &mut tar_bytes, + TarCompression::None, + &TarExportOptions::default(), + &|_| {}, + ) + .expect("export_tar"); + + let mut ar = tar::Archive::new(&tar_bytes[..]); + let entry = ar + .entries() + .unwrap() + .filter_map(|e| e.ok()) + .find(|e| { + e.path() + .map(|p| p.file_name().and_then(|n| n.to_str()) == Some("old.txt")) + .unwrap_or(false) + }) + .expect("tar contains old.txt"); + assert_eq!( + entry.header().mtime().unwrap(), + YEAR_2018, + "host mtime must survive host->efs->tar unchanged" + ); +} + +/// The extract half of the round-trip: file with a 2020 mtime on the image +/// is extracted to a host folder — the extracted file carries the same +/// mtime, via filetime::set_file_times. +#[test] +fn image_to_host_extract_preserves_mtime() { + let mut fs = fresh_efs(); + let root = fs.root().unwrap(); + let entry = put_bytes( + &mut fs, + "dated.bin", + b"host-out", + Some(UnixTimes::all(YEAR_2020)), + ); + + let tmp = tempfile::tempdir().unwrap(); + // Re-read entry with `list_directory` so `modified_unix` is set the way + // an extract-in-production reads it (the create_file return path leaves + // it None, which is fine — real callers list first). + let listed = fs + .list_directory(&root) + .unwrap() + .into_iter() + .find(|e| e.name == "dated.bin") + .unwrap(); + let _ = entry; + export_file_with_fork( + &mut fs, + &listed, + tmp.path(), + "dated.bin", + ResourceForkMode::DataForkOnly, + ) + .expect("extract"); + + let host = tmp.path().join("dated.bin"); + let meta = std::fs::metadata(&host).unwrap(); + let host_mtime = meta + .modified() + .unwrap() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + assert_eq!( + host_mtime, YEAR_2020, + "extracted host file must carry the source's modified_unix" + ); +} + +/// The other side of the rule: a genuinely new file (unix_times = None) +/// still gets `now`. The distinction between "new" and "copied" must not +/// silently regress into "always preserve" (which would zero-mtime every +/// blank file GUI users create). +#[test] +fn genuinely_new_file_stamps_now() { + let mut fs = fresh_efs(); + let before = now_secs(); + let entry = put_bytes(&mut fs, "fresh.txt", b"hello", None); + let after = now_secs(); + let root = fs.root().unwrap(); + let listed = fs + .list_directory(&root) + .unwrap() + .into_iter() + .find(|e| e.name == "fresh.txt") + .unwrap(); + let mtime = listed + .modified_unix + .expect("modified_unix must be set for a genuinely new file"); + assert!( + (before..=after + 5).contains(&mtime), + "genuinely new file mtime must be ~now (before={before}, mtime={mtime}, after={after})" + ); + let _ = entry; +} + +// --------------------------------------------------------------------------- +// Cross-filesystem coverage — every driver that Phase 2..5 taught to honour +// `CreateFileOptions.unix_times` gets one test that puts a 2020-dated byte +// stream through `create_file` and reads the mtime back via `list_directory`. +// The point is not to re-verify each format encoder (those have unit tests in +// `fs::times`) — it's the plumbing between `create_file` and `list_directory` +// that's easy to let drift; one test per driver catches a driver forgetting +// to thread `options.unix_times` through, or forgetting to populate +// `modified_unix` on read. +// --------------------------------------------------------------------------- + +use rusty_backup::fs::adfs::{create_blank_adfs, AdfsFilesystem}; +use rusty_backup::fs::exfat::{create_blank_exfat, ExfatFilesystem, ExfatFormatTemplate}; +use rusty_backup::fs::fat::{create_blank_fat, FatFilesystem}; +use rusty_backup::fs::hfs::{create_blank_hfs, HfsFilesystem}; +use rusty_backup::fs::hfsplus::{create_blank_hfsplus, HfsPlusFilesystem}; +use rusty_backup::fs::hpfs::{create_blank_hpfs, HpfsFilesystem}; +use rusty_backup::fs::mfs::{create_blank_mfs, MfsFilesystem}; +use rusty_backup::fs::ntfs::NtfsFilesystem; +use rusty_backup::fs::ntfs_format::create_blank_ntfs; +use rusty_backup::fs::os9::{create_blank_os9, Os9Filesystem}; +use rusty_backup::fs::prodos::{create_blank_prodos, ProDosFilesystem}; +use rusty_backup::fs::ucsd::{create_blank_ucsd, UcsdFilesystem}; + +/// One-shot helper: create a file on `fs` at `/name` with mtime = YEAR_2020, +/// re-list the root, and return the entry's `modified_unix`. +fn put_and_readback_mtime(fs: &mut F, name: &str) -> Option { + let root = fs.root().unwrap(); + fs.create_file( + &root, + name, + &mut &b"hello"[..], + 5, + &CreateFileOptions { + unix_times: Some(UnixTimes::mtime_only(YEAR_2020)), + ..Default::default() + }, + ) + .expect("create_file"); + fs.sync_metadata().expect("sync_metadata"); + let root = fs.root().unwrap(); + fs.list_directory(&root) + .unwrap() + .into_iter() + .find(|e| e.name == name) + .expect("listed entry") + .modified_unix +} + +/// The DOS-packed date is 2-second granular; a round-trip loses the odd +/// low bit but the whole-minute value must survive. +fn assert_within_dos_granularity(actual: Option, expected: u64) { + let a = actual.expect("modified_unix must be populated"); + assert!( + a >= expected && a - expected <= 1, + "expected ~{expected}, got {a} (DOS is 2-second granular)" + ); +} + +#[test] +fn fat_preserves_mtime_across_create_and_list() { + let img = create_blank_fat(4 * 1024 * 1024, Some("TIMES")).unwrap(); + let mut fs = FatFilesystem::open(Cursor::new(img), 0).unwrap(); + assert_within_dos_granularity(put_and_readback_mtime(&mut fs, "TIMES.TXT"), YEAR_2020); +} + +#[test] +fn exfat_preserves_mtime_across_create_and_list() { + let template = ExfatFormatTemplate { + bytes_per_sector: 512, + sectors_per_cluster: 8, + label: Some("TIMES".to_string()), + }; + let size = 16 * 1024 * 1024u64; + let mut cur = Cursor::new(Vec::::new()); + create_blank_exfat(&mut cur, &template, size).unwrap(); + let mut fs = ExfatFilesystem::open(cur, 0).unwrap(); + assert_within_dos_granularity(put_and_readback_mtime(&mut fs, "times.txt"), YEAR_2020); +} + +#[test] +fn ntfs_preserves_mtime_across_create_and_list() { + let mut cur = Cursor::new(Vec::::new()); + create_blank_ntfs(&mut cur, 16 * 1024 * 1024, 128, Some("TIMES")).unwrap(); + let mut fs = NtfsFilesystem::open(cur, 0).unwrap(); + let mtime = put_and_readback_mtime(&mut fs, "times.txt").expect("modified_unix set"); + assert_eq!(mtime, YEAR_2020, "NTFS FILETIME is second-granular"); +} + +#[test] +fn hfs_preserves_mtime_across_create_and_list() { + let img = create_blank_hfs(4 * 1024 * 1024, 512, "TIMES").unwrap(); + let mut fs = HfsFilesystem::open(Cursor::new(img), 0).unwrap(); + let mtime = put_and_readback_mtime(&mut fs, "times.txt").expect("modified_unix set"); + assert_eq!(mtime, YEAR_2020); +} + +#[test] +fn hfsplus_preserves_mtime_across_create_and_list() { + let img = create_blank_hfsplus(16 * 1024 * 1024, 4096, "TIMES", false); + let mut fs = HfsPlusFilesystem::open(Cursor::new(img), 0).unwrap(); + let mtime = put_and_readback_mtime(&mut fs, "times.txt").expect("modified_unix set"); + assert_eq!(mtime, YEAR_2020); +} + +#[test] +fn mfs_preserves_mtime_across_create_and_list() { + let img = create_blank_mfs(400 * 1024, "TIMES").unwrap(); + let mut fs = MfsFilesystem::open(Cursor::new(img), 0).unwrap(); + let mtime = put_and_readback_mtime(&mut fs, "times.txt").expect("modified_unix set"); + assert_eq!(mtime, YEAR_2020); +} + +#[test] +fn prodos_preserves_mtime_across_create_and_list() { + let img = create_blank_prodos(400 * 1024, "TIMES").unwrap(); + let mut fs = ProDosFilesystem::open(Cursor::new(img), 0).unwrap(); + let mtime = put_and_readback_mtime(&mut fs, "TIMES.TXT").expect("modified_unix set"); + // ProDOS is minute-granular; YEAR_2020 lands on a minute boundary. + assert_eq!(mtime, YEAR_2020); +} + +#[test] +fn hpfs_preserves_mtime_across_create_and_list() { + let img = create_blank_hpfs(4 * 1024 * 1024, "TIMES").unwrap(); + let mut fs = HpfsFilesystem::open(Cursor::new(img), 0).unwrap(); + let mtime = put_and_readback_mtime(&mut fs, "times.txt").expect("modified_unix set"); + assert_eq!(mtime, YEAR_2020); +} + +// Human68k needs a captured `Human68kFormatTemplate` (BPB + reserved region) +// to format, which needs a source volume to hand — the driver's own unit +// tests cover the roundtrip. Extending it into this file would duplicate +// that scaffolding. + +#[test] +fn os9_preserves_mtime_across_create_and_list() { + let img = create_blank_os9("TIMES").unwrap(); + let mut fs = Os9Filesystem::open(Cursor::new(img), 0).unwrap(); + let mtime = put_and_readback_mtime(&mut fs, "times").expect("modified_unix set"); + // OS-9 FD.DAT is minute-granular; YEAR_2020 lands on a minute boundary. + assert_eq!(mtime, YEAR_2020); +} + +#[test] +fn ucsd_preserves_mtime_across_create_and_list() { + // UCSD's year field is 0..99 -> 1900..1999, so 2020 would clamp to 1999. + // Test with a UCSD-representable year instead. + const YEAR_1990: u64 = 631_152_000; // 1990-01-01 00:00:00 UTC + let img = create_blank_ucsd(400 * 1024, "TIMES").unwrap(); + let mut fs = UcsdFilesystem::open(Cursor::new(img), 0).unwrap(); + let root = fs.root().unwrap(); + fs.create_file( + &root, + "TIMES", + &mut &b"hello"[..], + 5, + &CreateFileOptions { + unix_times: Some(UnixTimes::mtime_only(YEAR_1990)), + ..Default::default() + }, + ) + .expect("create_file"); + fs.sync_metadata().expect("sync"); + let root = fs.root().unwrap(); + let mtime = fs + .list_directory(&root) + .unwrap() + .into_iter() + .find(|e| e.name == "TIMES") + .expect("listed entry") + .modified_unix + .expect("modified_unix set"); + // UCSD stores day-granularity only, so 1990-01-01 00:00:00 round-trips exact. + assert_eq!(mtime, YEAR_1990); +} + +#[test] +fn adfs_preserves_mtime_across_create_and_list() { + let img = create_blank_adfs("TIMES"); + let mut fs = AdfsFilesystem::open(Cursor::new(img), 0).unwrap(); + let mtime = put_and_readback_mtime(&mut fs, "times").expect("modified_unix set"); + // ADFS is centisecond-granular; whole seconds round-trip exactly. + assert_eq!(mtime, YEAR_2020); +}