Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ The following commands were merged into the core commands:
- `mm cat file -m accurate` — LLM-generated caption/description (image/video/audio/PDF); passthrough for code/text/docx/pptx
- `mm cat video.mp4 -m accurate` — auto-generates keyframe mosaic → LLM description
- `mm cat photo.png -p resize` — encode with named encoder
- `mm cat ./my-folder` — recursively expand a folder into its files (gitignore-aware; pass `--no-ignore` to include ignored entries)
- `mm cat photo.png -m accurate -p my-pipeline.yaml` — custom pipeline YAML

### Schema and SQL
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,8 @@ mm cat bench.jpg # short VLM capt
mm cat bench.jpg -m accurate # full LLM caption + tags + objects
mm cat Timelapse.mp4 -m accurate # mosaic → LLM description
mm cat bench.jpg -p image-tile # use named encoder
mm cat ./my-folder # cat every file in the folder (gitignore-aware)
mm cat ./my-folder --no-ignore # include files normally excluded by .gitignore
mm cat bench.jpg -m accurate -p my-pipeline.yaml # custom pipeline YAML
mm cat Timelapse.mp4 -m accurate --no-cache # force fresh LLM call
mm cat bench.jpg -m accurate --no-generate # snapshot encoder output (no LLM)
Expand Down
1 change: 1 addition & 0 deletions docs/spec/cat.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ Unified content extraction. Behaviour driven by **file type × mode × pipeline*

- **Multimodal**: auto-detects kind from extension → image, video, audio, document, text
- **Multi-file**: `mm cat a.jpg b.pdf c.mp4` — processes each sequentially
- **Directories**: `mm cat ./my-folder` — recursively expands the folder to its files (gitignore-aware); mix with files. Pass `--no-ignore` to include files normally excluded by `.gitignore`.
- **Large batches**: if the path count is **≥ 9** (i.e. more than 8 files; override with `MM_CAT_BATCH_CONFIRM_THRESHOLD`), `cat` asks for confirmation in a TTY; in non-interactive use it **exits with an error** unless you pass **`--yes` / `-y`**
- **Stdin**: `find . -name '*.pdf' | mm cat` — reads newline-delimited paths from stdin
- **Head/tail**: `-n 20` (first 20 lines), `-n -20` (last 20 lines)
Expand Down
70 changes: 67 additions & 3 deletions python/mm/commands/bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,37 @@ def _is_filesystem_noise(name: str) -> bool:
return [f for f in files if not _is_filesystem_noise(Path(f.path).name)]


def _resolve_bench_target(target: Path) -> tuple[Path, frozenset[str] | None]:
"""Normalize a CLI ``mm bench`` target into ``(directory, files_filter)``.

Accepts either a directory (legacy behaviour: bench the whole tree) or
a single regular file (bench only that file, using its parent as
``{dir}``). The returned ``files_filter`` is a frozenset of relative
paths (as emitted by ``Scanner``) that the harness should keep; ``None``
means "no filter".

Args:
target: Path passed on the command line.

Returns:
``(directory, files_filter)`` tuple suitable for ``_run_benchmarks``
and ``_run_stdout_snapshot``.

Raises:
typer.Exit: If ``target`` doesn't exist or is neither a file nor a
directory.
"""
if not target.exists():
typer.echo(f"Error: {target} not found.", err=True)
raise typer.Exit(code=1)
if target.is_file():
return target.parent.resolve(), frozenset({target.name})
if target.is_dir():
return target, None
typer.echo(f"Error: {target} is neither a file nor a directory.", err=True)
raise typer.Exit(code=1)


# ── Timing harness ──────────────────────────────────────────────────


Expand Down Expand Up @@ -280,6 +311,7 @@ def _run_benchmarks(
on_progress: Callable[[str, str], None] | None = None,
commands: list | None = None,
dry_run: bool = False,
files_filter: frozenset[str] | None = None,
) -> tuple[list[BenchResult], dict[str, Any]]:
"""Run benchmark commands, return (results, target_info).

Expand All @@ -288,6 +320,9 @@ def _run_benchmarks(
are populated), but does not invoke ``_time_cmd``. Each row is marked
``is_dry_run=True`` so the renderer / JSON encoder can show ``-``
placeholders instead of zero metrics.

When ``files_filter`` is provided the pre-scan is narrowed to only
those relative paths — used when the CLI target is a single file.
"""
from mm.context import Context

Expand All @@ -300,6 +335,8 @@ def _run_benchmarks(
# Pre-scan to get target info and pick representative files.
ctx = Context(directory)
files = _sanitize_files(ctx.files)
if files_filter is not None:
files = [f for f in files if f.path in files_filter]
num_files = len(files)
total_bytes = sum(f.size for f in files)

Expand Down Expand Up @@ -1359,6 +1396,7 @@ def _run_stdout_snapshot(
command_filter: str | None,
timeout_s: float,
with_generate: bool,
files_filter: frozenset[str] | None = None,
) -> None:
"""Run each filtered ``mm cat`` encoder variant once and emit its stdout.

Expand Down Expand Up @@ -1399,6 +1437,8 @@ def _run_stdout_snapshot(

ctx = Context(directory)
files = _sanitize_files(ctx.files)
if files_filter is not None:
files = [f for f in files if f.path in files_filter]
if not files:
typer.echo(f"Error: no files found in {directory}", err=True)
raise typer.Exit(code=1)
Expand Down Expand Up @@ -1564,7 +1604,17 @@ def _load_benchfile(


def bench_cmd(
directory: Annotated[Path, typer.Argument(help="Directory to benchmark")] = Path("."),
target: Annotated[
Path,
typer.Argument(
help=(
"File or directory to benchmark. A directory is scanned for "
"representative files; a single file benchmarks just that "
"file (with its parent directory used for `{dir}` "
"substitutions)."
),
),
] = Path("."),
rounds: Annotated[int, typer.Option("--rounds", "-r", help="Measurement rounds")] = 3,
warmup: Annotated[int, typer.Option("--warmup", "-w", help="Warmup rounds")] = 1,
mode: Annotated[
Expand Down Expand Up @@ -1749,13 +1799,16 @@ def bench_cmd(
render_host_info(collect_host_info(), fmt=fmt)
return

directory, files_filter = _resolve_bench_target(target)

if fmt == "stdout":
_run_stdout_snapshot(
directory=directory,
mode=mode or "fast",
command_filter=command,
timeout_s=timeout,
with_generate=with_generate,
files_filter=files_filter,
)
return

Expand Down Expand Up @@ -1863,14 +1916,20 @@ def on_progress(group: str, name: str) -> None:
on_progress,
commands,
dry_run=dry_run,
files_filter=files_filter,
)
finally:
status.stop()

_render_table(results, target_info)
elif fmt == "json":
results, target_info = _run_benchmarks(
directory, rounds, warmup, commands=commands, dry_run=dry_run
directory,
rounds,
warmup,
commands=commands,
dry_run=dry_run,
files_filter=files_filter,
)

from mm.display import json_dumps
Expand All @@ -1883,7 +1942,12 @@ def on_progress(group: str, name: str) -> None:
else:
# tsv/csv fallback
results, target_info = _run_benchmarks(
directory, rounds, warmup, commands=commands, dry_run=dry_run
directory,
rounds,
warmup,
commands=commands,
dry_run=dry_run,
files_filter=files_filter,
)

from mm.display import emit_tsv
Expand Down
15 changes: 15 additions & 0 deletions python/mm/commands/bench_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,21 @@ def resolve_command(
]

FAST_COMMANDS: list[BenchCommand] = [
# Folder-expansion path: ``mm cat <folder>`` walks the directory via the
# gitignore-aware Scanner and dispatches each file through fast mode.
# ``--no-generate`` keeps timings focused on the expansion + encoder
# dispatch cost (no per-file LLM calls during bench rounds); ``-y``
# bypasses the >=9-paths confirmation prompt for non-interactive runs.
BenchCommand(
"mm cat <folder>",
"fast",
"mm cat {dir} --mode fast --no-cache --no-generate --format json -y",
),
BenchCommand(
"mm cat <folder> --no-ignore",
"fast",
"mm cat {dir} --mode fast --no-cache --no-generate --format json --no-ignore -y",
),
BenchCommand(
"mm cat <code> (x20)",
"fast",
Expand Down
58 changes: 53 additions & 5 deletions python/mm/commands/cat.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from mm.cat_utils.extract_meta import extract_meta
from mm.common.audio._base import BackendLabel
from mm.pipe import read_paths_from_stdin
from mm.utils import Format, file_kind
from mm.utils import Format, expand_path_arg, file_kind

if TYPE_CHECKING:
from mm.constants import BinaryFileKind
Expand Down Expand Up @@ -102,7 +102,16 @@ def cat_cmd(
),
] = None,
# -- Positional argument --
files: Annotated[Optional[list[Path]], typer.Argument(help="Files to display")] = None,
files: Annotated[
Optional[list[Path]],
typer.Argument(
help=(
"Files and/or directories to display. Directories are "
"expanded recursively (gitignore-aware; pass --no-ignore "
"to bypass)."
),
),
] = None,
n: Annotated[
Optional[int],
typer.Option("-n", help="Line limit: +N = head, -N = tail"),
Expand Down Expand Up @@ -221,13 +230,25 @@ def cat_cmd(
help="Confirm when path count ≥ threshold (default 9; env MM_CAT_BATCH_CONFIRM_THRESHOLD)",
),
] = False,
no_ignore: Annotated[
bool,
typer.Option(
"--no-ignore",
help="Include files excluded by .gitignore when expanding directory arguments",
),
] = False,
) -> None:
"""Extract and describe file content.

\b
Behavior auto-detects from file type. Default mode is 'fast'. For raw
file metadata (dimensions / EXIF / codec / mime / hash), use ``mm peek``.

\b
Directory arguments are expanded recursively into all files inside,
respecting ``.gitignore`` by default. Pass ``--no-ignore`` to include
ignored entries.

\b
fast (default) accurate
Images: short VLM caption full VLM caption + tags
Expand All @@ -248,6 +269,8 @@ def cat_cmd(
mm cat photo.png # short VLM caption (fast pipeline)
mm cat photo.png -m accurate # full VLM description
mm cat video.mp4 -m accurate # mosaic → VLM
mm cat ./my-folder # cat every file in the folder (gitignore-aware)
mm cat ./my-folder --no-ignore # include files normally excluded by .gitignore
mm cat photo.png -p tile # use named encoder
mm cat photo.png -m accurate -p my-pipeline.yaml
# custom pipeline YAML
Expand Down Expand Up @@ -307,13 +330,38 @@ def cat_cmd(
do_print_pipeline(print_pipeline)
return

paths: list[str] = []
raw_paths: list[str] = []

stdin_paths = read_paths_from_stdin()
if stdin_paths:
paths.extend(stdin_paths)
raw_paths.extend(stdin_paths)
if files:
paths.extend(str(f) for f in files)
raw_paths.extend(str(f) for f in files)

if not raw_paths:
typer.echo("Error: No files specified.", err=True)
raise typer.Exit(1)

paths: list[str] = []
seen: set[str] = set()
for entry in raw_paths:
try:
expanded = expand_path_arg(entry, no_ignore=no_ignore)
except FileNotFoundError:
# Defer "not found" reporting to the downstream processing loop
# so the user sees the canonical "Error: <path> not found."
# message (and the prune-from-cache side-effect) instead of a
# bare traceback. Keep the original entry so the error message
# echoes the user's input form.
if entry not in seen:
seen.add(entry)
paths.append(entry)
continue
for f in expanded:
key = str(f)
if key not in seen:
seen.add(key)
paths.append(key)

if not paths:
typer.echo("Error: No files specified.", err=True)
Expand Down
77 changes: 77 additions & 0 deletions python/mm/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,83 @@ def file_kind_with_code(path: Path) -> str:
return "text"


def expand_path_arg(path: Path | str, *, no_ignore: bool = False) -> list[Path]:
"""Expand a single CLI path argument into the file list it represents.

Files are returned as a one-element list. Directories are walked
recursively via the Rust ``Scanner`` (gitignore-aware by default; pass
``no_ignore=True`` to include ignored entries) and the resulting paths
are returned sorted by their relative path within the directory so the
output is deterministic across runs.

Returned paths are always absolute (``Path.resolve()`` applied) so the
helper has a single, predictable contract regardless of whether the
caller passed a file or a directory. This is what lets the de-dup
logic in :func:`expand_path_args` and :mod:`mm.commands.cat` use the
string form as a reliable set key.

Args:
path: A filesystem path. May be a file or a directory.
no_ignore: When True, bypass ``.gitignore`` while walking
directories. Has no effect on file inputs.

Returns:
Ordered list of resolved ``Path`` objects. Empty if ``path`` is a
directory with no scannable files.

Raises:
FileNotFoundError: If ``path`` does not exist on disk.
"""
import json as _json

p = Path(path)
if not p.exists():
raise FileNotFoundError(str(p))
if p.is_file():
return [p.resolve()]

from mm._mm import Scanner

root = p.resolve()
scanner = Scanner(str(root), None, no_ignore=no_ignore)
scanner.scan()
rows = _json.loads(scanner.to_json_fast(sort_by="path"))
return [root / row["path"] for row in rows]


def expand_path_args(
paths: list[Path] | list[str],
*,
no_ignore: bool = False,
) -> list[Path]:
"""Expand a list of CLI path arguments, flattening directories into files.

Equivalent to calling :func:`expand_path_arg` on each entry and
concatenating the results, with duplicates removed while preserving
first-seen order.

Args:
paths: Mix of file and directory paths.
no_ignore: Forwarded to :func:`expand_path_arg`.

Returns:
De-duplicated, order-preserving list of file ``Path`` objects.

Raises:
FileNotFoundError: If any entry does not exist.
"""
seen: set[str] = set()
out: list[Path] = []
for entry in paths:
for f in expand_path_arg(entry, no_ignore=no_ignore):
key = str(f)
if key in seen:
continue
seen.add(key)
out.append(f)
return out


def is_binary_content(*, kind: str, content: str | None = None) -> bool:
"""Heuristic to determine if content is binary based on kind and content."""
return kind in ("image", "document", "video", "audio") or bool(
Expand Down
Loading
Loading