From 02ef96120131df7dbafd4dcac3fde63d4bc710f5 Mon Sep 17 00:00:00 2001 From: "github.com/ib-bsb-br/ib-bsb-br.github.io" Date: Sun, 22 Mar 2026 20:05:37 -0300 Subject: [PATCH] Split srch flat collector into builder and mover --- README.md | 138 ++++++++++++++++- usr/local/bin/srch_flat_collect | 253 ++++++++++++++++++++++++++++++++ usr/local/libexec/srch_move_one | 124 ++++++++++++++++ 3 files changed, 514 insertions(+), 1 deletion(-) create mode 100755 usr/local/bin/srch_flat_collect create mode 100755 usr/local/libexec/srch_move_one diff --git a/README.md b/README.md index 6c6af6a..80bc239 100644 --- a/README.md +++ b/README.md @@ -1 +1,137 @@ -`` +[[ + + Provide a formal, comprehensive specification for a two-script refactor that separates an interactive srch command builder from a reusable per-file mover, while preserving flat-destination moves, deterministic collision handling, and auditable logging. + + + + + root (invoked via sudo by user linaro) + + /usr/local/bin/srch_flat_collect + /usr/local/libexec/srch_move_one + + /home/linaro/ + operation_log.txt inside the destination folder + + + + All matched files are moved into a single flat directory under /home/linaro. + srch -r must invoke the per-file mover script for each matched path. + Collision handling must use deterministic numeric suffixes before extensions. + Moves must use copy+delete semantics to support cross-filesystem operations. + Ownership must be linaro:linaro and permissions must allow read/write for owner, group, and others (a+rwX). + No archive or zip operation is permitted. + Interactive prompts must cover feasible srch flags and filters. + + + + Three draft approaches were provided and must be merged into a single, reusable two-script design. + Reducing heredocs is required to improve versioning and reuse of the per-file mover. + Logging must capture original paths, normalized paths, renames, and errors. + + + + + + Refactor the srch-based collection workflow into a builder and a reusable mover with consistent logging and collision handling. + + + /usr/local/bin/srch_flat_collect + + + /usr/local/libexec/srch_move_one + + + Single-level folder under /home/linaro used as the collection target. + + + + + 1. Restate the goal: split the workflow into a builder script and a reusable mover while keeping flat moves, collision-safe naming, and audit logging. + + 2. Implement /usr/local/bin/srch_flat_collect as an interactive srch option builder that: + (a) collects srch start points; + (b) prompts for name matching (-N/-n/-i with optional negation); + (c) prompts for path matching (-a), exclusions (-e/-E/-Z), depth (-m), and filesystem boundary (-x); + (d) prompts for size and age filters (-s, -o/-O/-P, -y/-Y/-W); + (e) prompts for ownership filters (-u/-U/-g/-G); + (f) prompts for performance options (-t/-I/-q/-Q/-X) and progress/stats (-v/-S/-T/-C); + (g) always enforces -f for regular files. + + + 3. Implement /usr/local/libexec/srch_move_one as a standalone mover that: + (a) normalizes paths to absolute form; + (b) allocates collision-free names using numeric suffixes; + (c) performs copy+delete moves into the destination; + (d) logs every action to a .txt file in the destination. + + + 4. Ensure the builder exports DESTDIR, LOGFILE, LOCKFILE/LOCKDIR, and DRYRUN so the mover can operate consistently across runs. + + 5. Preserve auditability by rotating existing logs and recording the full srch invocation context. + + + + Plain text + + Provide both scripts as versionable files rather than large heredocs embedded in one script. + Maintain formal, impersonal language and the structure of this template. + Include a complete example of each script. + + + + + + + Provide the builder and mover scripts as two separate files for reuse and testing. + + + Builder: /usr/local/bin/srch_flat_collect + Mover: /usr/local/libexec/srch_move_one + + The builder prompts for srch options, sets DESTDIR/LOGFILE/LOCKFILE/DRYRUN, and invokes: + srch -r /usr/local/libexec/srch_move_one + + The mover validates each match, resolves collisions with suffixes, copies to the destination, + deletes the source, and writes audit lines to operation_log.txt. + + + + + + + The builder and mover are separated into /usr/local/bin and /usr/local/libexec. + srch -r invokes the mover without embedding a large heredoc in the builder. + Collision handling uses deterministic numeric suffixes before extensions. + Moves use copy+delete semantics and are logged in operation_log.txt. + Destination ownership and permissions are set for linaro with a+rwX. + + + + + + Multiple identical basenames across different source paths. + Parallel srch -r runs with lock-guarded name allocation. + Files disappearing between traversal and move. + Paths containing spaces or special characters. + + All matched files land in the flat destination with correct naming, permissions, and complete audit logs. + + + + + Install /usr/local/bin/srch_flat_collect and /usr/local/libexec/srch_move_one, then run the builder with sudo. + Follow prompts to build the srch command, confirm the operation, and review operation_log.txt. + + + Large traversals can be I/O intensive; filters should be tuned before execution. + Locking uses flock when available and a mkdir fallback otherwise; behavior may vary on some filesystems. + + + + + Drafts differed on whether to allow existing destinations; this version prompts for confirmation if the destination exists. + Drafts used both three- and four-digit suffixes; this version standardizes on three-digit suffixes for collisions. + Drafts varied between embedded mover heredocs and external helpers; this version externalizes the mover for reuse. + +]] diff --git a/usr/local/bin/srch_flat_collect b/usr/local/bin/srch_flat_collect new file mode 100755 index 0000000..781c0d5 --- /dev/null +++ b/usr/local/bin/srch_flat_collect @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# srch flat-file collector: interactive builder + srch invocation. + +set -u +set -o pipefail + +if [[ "${EUID:-$(id -u)}" -ne 0 ]]; then + echo "ERROR: must run as root (sudo)." >&2 + exit 1 +fi + +if ! command -v srch >/dev/null 2>&1; then + echo "ERROR: srch not found in PATH." >&2 + exit 1 +fi + +if ! command -v fold >/dev/null 2>&1; then + echo "ERROR: fold not found in PATH." >&2 + exit 1 +fi + +umask 000 + +ts() { date -Is; } +fold_line() { fold -w 1200 -s; } + +prompt() { + local label="$1" def="${2-}" ans + if [[ -n "$def" ]]; then + read -r -p "$label [$def]: " ans + echo "${ans:-$def}" + else + read -r -p "$label: " ans + echo "$ans" + fi +} + +prompt_yesno() { + local q="$1" def="${2:-n}" ans + while :; do + read -r -p "$q (y/n) [$def]: " ans + ans="${ans:-$def}" + case "${ans,,}" in + y|yes) echo "y"; return 0 ;; + n|no) echo "n"; return 0 ;; + *) echo "Please answer y or n." >&2 ;; + esac + done +} + +sanitize_folder_name() { + local s="$1" + s="${s//\//_}" + s="${s//\\/_}" + s="${s//\0/}" + printf '%s' "$s" +} + +DEST_BASE="/home/linaro" +DEFAULT_DEST_NAME="srch_flat_$(date +%Y%m%d_%H%M%S)" +DEST_NAME_RAW="$(prompt "Destination folder name under ${DEST_BASE} (single level; no slashes)" "$DEFAULT_DEST_NAME")" +DEST_NAME="$(sanitize_folder_name "$DEST_NAME_RAW")" + +if [[ -z "$DEST_NAME" ]]; then + echo "ERROR: destination folder name must not be empty." >&2 + exit 1 +fi + +DESTDIR="${DEST_BASE}/${DEST_NAME}" +LOGFILE="${DESTDIR}/operation_log.txt" +LOCKFILE="${DESTDIR}/.rename.lock" +LOCKDIR="${DESTDIR}/.rename.lockdir" +HELPER="/usr/local/libexec/srch_move_one" + +if [[ ! -x "$HELPER" ]]; then + echo "ERROR: helper not found or not executable: $HELPER" >&2 + exit 1 +fi + +if [[ -e "$DESTDIR" ]]; then + use_existing="$(prompt_yesno "Destination exists. Use it anyway?" "n")" + if [[ "$use_existing" != "y" ]]; then + echo "ERROR: destination already exists: $DESTDIR" >&2 + exit 1 + fi +fi + +mkdir -p -- "$DESTDIR" || { echo "ERROR: cannot create destination: $DESTDIR" >&2; exit 1; } +chmod 0777 -- "$DESTDIR" 2>/dev/null || true + +if [[ -e "$LOGFILE" ]]; then + mv -f -- "$LOGFILE" "${LOGFILE}.bak.$(date +%Y%m%d%H%M%S)" 2>/dev/null || true +fi +: >"$LOGFILE" || { echo "ERROR: cannot write log: $LOGFILE" >&2; exit 1; } +chmod 0666 -- "$LOGFILE" 2>/dev/null || true + +log() { + printf '%s %s\n' "$(ts)" "$*" | fold_line | tee -a "$LOGFILE" >&2 +} + +log "BEGIN collection" +log "effective_uid=$(id -u) effective_user=$(id -un 2>/dev/null || true)" +log "destination=$DESTDIR" +log "logfile=$LOGFILE" + +SRCH_ARGS=() +SRCH_ROOTS=() + +roots_raw="$(prompt "srch start directories (space-separated)" "/home/linaro /mnt /userdata /usr/share/doc")" +# shellcheck disable=SC2206 +SRCH_ROOTS=( $roots_raw ) + +threads="$(prompt "Threads (-t): number, or '*' for all cores, or blank for default" "")" +[[ -n "$threads" ]] && SRCH_ARGS+=("-t" "$threads") + +name_mode="$(prompt "Name matching mode: none | -N (substring) | -n (regex) | -i (regex, case-insensitive)" "-n")" +name_mode="${name_mode// /}" +if [[ "$name_mode" != "none" && -n "$name_mode" ]]; then + name_neg="$(prompt_yesno "Negate the name match? (prefix '!' to the pattern)" "n")" + name_pat="$(prompt "Pattern for ${name_mode} (empty disables name filter)" "")" + if [[ -n "$name_pat" ]]; then + [[ "$name_neg" == "y" ]] && name_pat="!$name_pat" + case "$name_mode" in + -N|-n|-i) SRCH_ARGS+=("$name_mode" "$name_pat") ;; + *) log "WARN: Unrecognized name mode '$name_mode'; skipping name filter." ;; + esac + fi +fi + +use_full="$(prompt_yesno "Apply pattern to full path (-a)?" "n")" +[[ "$use_full" == "y" ]] && SRCH_ARGS+=("-a") + +add_e="$(prompt_yesno "Add directory exclude regex rules (-e)?" "n")" +if [[ "$add_e" == "y" ]]; then + while :; do + ex="$(prompt "-e (blank to stop)" "")" + [[ -z "$ex" ]] && break + SRCH_ARGS+=("-e" "$ex") + done +fi + +add_E="$(prompt_yesno "Add exact directory excludes (-E)?" "n")" +if [[ "$add_E" == "y" ]]; then + while :; do + ex2="$(prompt "-E (blank to stop)" "")" + [[ -z "$ex2" ]] && break + SRCH_ARGS+=("-E" "$ex2") + done +fi + +use_Z="$(prompt_yesno "Exclude .snapshot directories (-Z)?" "y")" +[[ "$use_Z" == "y" ]] && SRCH_ARGS+=("-Z") + +depth="$(prompt "Depth control (-m): e.g., '4', '2-5', '2-' (blank for unlimited)" "")" +[[ -n "$depth" ]] && SRCH_ARGS+=("-m" "$depth") + +xdev="$(prompt_yesno "Stay on same filesystem for each start point (-x)?" "n")" +[[ "$xdev" == "y" ]] && SRCH_ARGS+=("-x") + +ftype="$(prompt "Type filter (for completeness): f/d/l/b/c/p/k/none" "f")" +if [[ "${ftype,,}" != "f" ]]; then + log "WARN: Operation moves regular files only; forcing '-f' regardless of selection ('$ftype')." +fi +SRCH_ARGS+=("-f") + +size_spec="$(prompt "Size constraint (-s): e.g., +1k, -10m, +1k:-10m (blank for none)" "")" +[[ -n "$size_spec" ]] && SRCH_ARGS+=("-s" "$size_spec") + +older_days="$(prompt "Older than days (-o) (blank for none)" "")" +[[ -n "$older_days" ]] && SRCH_ARGS+=("-o" "$older_days") + +older_mins="$(prompt "Older than minutes (-O) (blank for none)" "")" +[[ -n "$older_mins" ]] && SRCH_ARGS+=("-O" "$older_mins") + +older_tstamp="$(prompt "Older than timestamp file (-P ) (blank for none)" "")" +[[ -n "$older_tstamp" ]] && SRCH_ARGS+=("-P" "$older_tstamp") + +younger_days="$(prompt "Younger than days (-y) (blank for none)" "")" +[[ -n "$younger_days" ]] && SRCH_ARGS+=("-y" "$younger_days") + +younger_mins="$(prompt "Younger than minutes (-Y) (blank for none)" "")" +[[ -n "$younger_mins" ]] && SRCH_ARGS+=("-Y" "$younger_mins") + +younger_tstamp="$(prompt "Younger than timestamp file (-W ) (blank for none)" "")" +[[ -n "$younger_tstamp" ]] && SRCH_ARGS+=("-W" "$younger_tstamp") + +inc_user="$(prompt "Owned by user (-u) (blank for none)" "")" +[[ -n "$inc_user" ]] && SRCH_ARGS+=("-u" "$inc_user") + +exc_user="$(prompt "Not owned by user (-U) (blank for none)" "")" +[[ -n "$exc_user" ]] && SRCH_ARGS+=("-U" "$exc_user") + +inc_group="$(prompt "Owned by group (-g) (blank for none)" "")" +[[ -n "$inc_group" ]] && SRCH_ARGS+=("-g" "$inc_group") + +exc_group="$(prompt "Not owned by group (-G) (blank for none)" "")" +[[ -n "$exc_group" ]] && SRCH_ARGS+=("-G" "$exc_group") + +inline_dirs="$(prompt "Inline subdir threshold (-I ) (blank for none)" "")" +[[ -n "$inline_dirs" ]] && SRCH_ARGS+=("-I" "$inline_dirs") + +use_q="$(prompt_yesno "Use FIFO queue (-q)?" "n")"; [[ "$use_q" == "y" ]] && SRCH_ARGS+=("-q") +use_Q="$(prompt_yesno "Queue sorted by inode (-Q)?" "n")"; [[ "$use_Q" == "y" ]] && SRCH_ARGS+=("-Q") +use_X="$(prompt_yesno "Enable extremely-large-dir optimization (-X)?" "n")"; [[ "$use_X" == "y" ]] && SRCH_ARGS+=("-X") + +prog_v="$(prompt "Progress interval (-v ) (blank for none)" "")" +[[ -n "$prog_v" ]] && SRCH_ARGS+=("-v" "$prog_v") + +use_S="$(prompt_yesno "Print summary stats (-S)?" "y")"; [[ "$use_S" == "y" ]] && SRCH_ARGS+=("-S") +use_T="$(prompt_yesno "Print elapsed time (-T)?" "y")"; [[ "$use_T" == "y" ]] && SRCH_ARGS+=("-T") +use_C="$(prompt_yesno "Suppress missing/disappearing entry errors (-C)?" "y")"; [[ "$use_C" == "y" ]] && SRCH_ARGS+=("-C") + +dryrun="$(prompt_yesno "Dry-run (log only; do not copy/delete)?" "y")" + +extra_raw="$(prompt "Additional srch arguments (space-separated; blank for none)" "")" +if [[ -n "$extra_raw" ]]; then + # shellcheck disable=SC2206 + extra_arr=( $extra_raw ) + for a in "${extra_arr[@]}"; do SRCH_ARGS+=("$a"); done +fi + +log "srch_roots=${SRCH_ROOTS[*]}" +log "srch_args=${SRCH_ARGS[*]}" +log "dryrun=$dryrun" + +confirm="$(prompt_yesno "Proceed now? (Matches will be processed by srch -r)" "n")" +if [[ "$confirm" != "y" ]]; then + log "ABORTED by user" + exit 0 +fi + +export DESTDIR LOGFILE LOCKFILE LOCKDIR +export DRYRUN="$dryrun" + +set +e +srch "${SRCH_ARGS[@]}" -r "$HELPER" "${SRCH_ROOTS[@]}" 2>&1 | fold -w 1500 -s | tee -a "$LOGFILE" +rc=${PIPESTATUS[0]} +set -e 2>/dev/null || true + +log "srch_exit_code=$rc" + +chown -R linaro:linaro -- "$DESTDIR" 2>/dev/null || true +chmod -R a+rwX -- "$DESTDIR" 2>/dev/null || true + +moved_count="$(grep -c " MOVED " "$LOGFILE" 2>/dev/null || true)" +skip_count="$(grep -c " SKIP " "$LOGFILE" 2>/dev/null || true)" +dry_count="$(grep -c " DRYRUN " "$LOGFILE" 2>/dev/null || true)" +err_count="$(grep -c " ERROR " "$LOGFILE" 2>/dev/null || true)" + +log "SUMMARY moved=$moved_count skipped=$skip_count dryrun=$dry_count errors=$err_count" +log "DONE destination=$DESTDIR" + +exit 0 diff --git a/usr/local/libexec/srch_move_one b/usr/local/libexec/srch_move_one new file mode 100755 index 0000000..b4d8156 --- /dev/null +++ b/usr/local/libexec/srch_move_one @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# srch per-file mover: flat destination, collision-safe renames, auditable log. + +set -u +set -o pipefail + +DESTDIR="${DESTDIR:?}" +LOGFILE="${LOGFILE:?}" +LOCKFILE="${LOCKFILE:?}" +LOCKDIR="${LOCKDIR:-}" +DRYRUN="${DRYRUN:-n}" + +_ts() { date -Is; } +_fold() { fold -w 1200 -s; } +_log() { printf '%s %s\n' "$(_ts)" "$*" | _fold >>"$LOGFILE"; } + +_lock_acquire() { + if command -v flock >/dev/null 2>&1; then + exec 9>>"$LOCKFILE" + flock -x 9 + return 0 + fi + if [[ -z "$LOCKDIR" ]]; then + LOCKDIR="${LOCKFILE}.d" + fi + while ! mkdir -- "$LOCKDIR" 2>/dev/null; do + sleep 0.05 + done +} + +_lock_release() { + if command -v flock >/dev/null 2>&1; then + return 0 + fi + if [[ -n "$LOCKDIR" ]]; then + rmdir -- "$LOCKDIR" 2>/dev/null || true + fi +} + +orig="${1-}" +if [[ -z "$orig" ]]; then + _log "ERROR event=invoke_missing_path" + exit 0 +fi + +norm="$orig" +if [[ "$norm" == ./* ]]; then + norm="$(pwd -P)/${norm#./}" +elif [[ "$norm" != /* ]]; then + norm="$(pwd -P)/$norm" +fi + +if [[ ! -f "$norm" ]]; then + _log "SKIP event=not_regular_file orig=\"$orig\" norm=\"$norm\"" + exit 0 +fi + +base="$(basename -- "$norm")" + +stem="$base" +ext="" +if [[ "$base" == .* ]]; then + if [[ "$base" == .*.* ]]; then + ext=".${base##*.}" + stem="${base%$ext}" + fi +else + if [[ "$base" == *.* ]]; then + ext=".${base##*.}" + stem="${base%$ext}" + fi +fi + +_lock_acquire + +cand="$DESTDIR/$base" +if [[ -e "$cand" ]]; then + i=1 + while :; do + suffix="_$(printf '%03d' "$i")" + cand="$DESTDIR/${stem}${suffix}${ext}" + [[ -e "$cand" ]] || break + i=$((i + 1)) + done +fi + +if ! : >"$cand" 2>/dev/null; then + _log "ERROR event=reserve_failed orig=\"$orig\" norm=\"$norm\" dest=\"$cand\"" + _lock_release + exit 0 +fi + +_lock_release + +if [[ "$DRYRUN" == "y" ]]; then + rm -f -- "$cand" 2>/dev/null || true + _log "DRYRUN event=would_move orig=\"$orig\" norm=\"$norm\" dest=\"$cand\"" + exit 0 +fi + +tmp="$DESTDIR/.tmp.$(basename -- "$cand").$$.$RANDOM" +if cp -a -- "$norm" "$tmp" 2>/dev/null; then + if mv -f -- "$tmp" "$cand" 2>/dev/null; then + if rm -f -- "$norm" 2>/dev/null; then + chmod 0666 -- "$cand" 2>/dev/null || true + chown linaro:linaro -- "$cand" 2>/dev/null || true + if [[ "$cand" == "$DESTDIR/$base" ]]; then + _log "MOVED event=direct orig=\"$orig\" norm=\"$norm\" dest=\"$cand\"" + else + _log "MOVED event=collision_renamed orig=\"$orig\" norm=\"$norm\" dest=\"$cand\"" + fi + else + _log "ERROR event=delete_failed orig=\"$orig\" norm=\"$norm\" dest=\"$cand\"" + fi + else + rm -f -- "$tmp" 2>/dev/null || true + rm -f -- "$cand" 2>/dev/null || true + _log "ERROR event=commit_failed orig=\"$orig\" norm=\"$norm\" dest=\"$cand\"" + fi +else + rm -f -- "$tmp" 2>/dev/null || true + rm -f -- "$cand" 2>/dev/null || true + _log "ERROR event=copy_failed orig=\"$orig\" norm=\"$norm\" dest=\"$cand\"" +fi