diff --git a/README.md b/README.md index 945a805..a550e55 100644 --- a/README.md +++ b/README.md @@ -115,10 +115,12 @@ Buildtree now includes a powerful feature that allows you to export your entire ### Key Features - **Complete project context**: Export all files with their paths and contents in a structured format -- **Smart filtering**: Include only relevant files by extension with `--filter` +- **Smart filtering**: Include only text files by default (go, js, py, md, json, yaml, xml, txt, etc.) - **Size control**: Limit file size with `--max-size` (default: 100kb) - **Ignore patterns**: Skip directories like `.git` and `node_modules` by default - **Hidden files**: Control inclusion of hidden files with `--include-hidden` +- **File ignore patterns**: Ignore specific files using glob patterns with `--ignore` +- **Minification**: Reduce token usage with `--minify` (strips whitespace and empty lines) ### Usage @@ -126,9 +128,65 @@ Buildtree now includes a powerful feature that allows you to export your entire buildtree -s OUTPUT_FILE [DIRECTORY] [OPTIONS] ``` -This example will collect the contents of all files with the *.go, *.txt, *.md extension and combine them into a single file with relative paths. +Options: +- `-f, --filter EXTS` - Comma-separated list of file extensions or glob patterns (default: common text formats like .go, .js, .txt, .md, .json, etc.) +- `-I, --ignore-dir DIRS` - Comma-separated list of directories to ignore (in addition to `.git` and `node_modules`) +- `--ignore PATTERNS` - Comma-separated glob patterns for files to ignore (e.g., `*_test.go,*.log`) +- `--max-size SIZE` - Maximum file size to include (default: 100kb, e.g., 100kb, 1mb) +- `-H, --include-hidden` - Include hidden files and directories (starting with `.`) +- `--minify` - Strip whitespace and empty lines to reduce token usage for LLM + +Flags can be placed before or after the directory argument: +```bash +# Flag after directory +buildtree -s export.txt ./project-dir -f "go,md" + +# Flag before directory +buildtree -s export.txt -f "go,md" ./project-dir +``` + +Examples: +```bash +# Export project structure (exports text files by default) +buildtree -s export-structure.txt ./project-dir -I vendor + +# Export with glob patterns +buildtree -s all.txt . -f "*.go,*.md" --ignore="*_test.go,*.log" + +# Export all files (don't filter by extension) +buildtree -s all.txt . -f "*" + +# Export with multiple filters and ignore patterns +buildtree -s context.txt ./myapp -f "go,js,ts,md,json" --ignore="*_test.go,*.log,*.tmp" -I dist +``` + +### Advanced Filtering and Ignoring + +**Glob pattern support**: Both `--filter` and `--ignore` support glob patterns like `*`, `?`, and `[...]`. + +By default, only text file extensions are included: go, js, ts, py, rb, php, java, c, cpp, h, cs, rs, html, css, json, yaml, yml, xml, md, txt, csv, toml, ini, cfg, and more. + ```bash -buildtree -s export-structure.txt ./project-dir -f go,txt,md -I vendor +# Include only Go files and files starting with config +buildtree -s export.txt . -f "go,config.*" + +# Ignore test files and log files +buildtree -s export.txt . --ignore="*_test.go,*.log" + +# Multiple ignore patterns +buildtree -s export.txt . --ignore="*_test.go,*.log,*.tmp" + +# Export all files without filtering by extension +buildtree -s all.txt . -f "*" + +# Export with maximum file size limit +buildtree -s context.txt . --max-size 50kb + +# Export including hidden files +buildtree -s all.txt . -f "*" -H + +# Minify output to reduce token usage +buildtree -s context.txt . --minify ``` ## Use Cases diff --git a/cmd/buildtree/main.go b/cmd/buildtree/main.go index 0f4bbea..e2d86de 100644 --- a/cmd/buildtree/main.go +++ b/cmd/buildtree/main.go @@ -1,7 +1,6 @@ package main import ( - "flag" "fmt" "io" "os" @@ -42,129 +41,200 @@ func (r *realBuilder) BuildTree(root *parser.Node, maxDepth int) error { // Let's put the main logic in a separate function for testing func run(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer, p parserInterface, b builderInterface) int { - // First, we look for the -s or --structure flag in the arguments + // Parse flags manually from args (before -s handling) to handle flags that come after arguments var saveStructureValue string - var newArgs []string - skipNext := false + var filterValue string + var ignoreDirsValue string + var ignorePatternsValue string + var maxSizeValue string + var inputFilePath string + var maxDepthValue string + var includeHiddenValue string + var minifyValue bool + var helpFlagValue bool + var versionFlagValue bool - // We go through all the arguments to find the -s flag + skipNext := false for i, arg := range args { if skipNext { skipNext = false continue } - // Checking the long shape of the flag + // Handle -s flag if arg == "-s" || arg == "--structure" { if i+1 < len(args) { saveStructureValue = args[i+1] skipNext = true - continue } - // Checking the form with equality } else if strings.HasPrefix(arg, "-s=") || strings.HasPrefix(arg, "--structure=") { parts := strings.SplitN(arg, "=", 2) if len(parts) == 2 { saveStructureValue = parts[1] - continue } + } else if arg == "-f" || arg == "--filter" { + if i+1 < len(args) { + filterValue = args[i+1] + skipNext = true + } + } else if strings.HasPrefix(arg, "-f=") || strings.HasPrefix(arg, "--filter=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 { + filterValue = parts[1] + } + } else if arg == "-I" || arg == "--ignore-dir" { + if i+1 < len(args) { + ignoreDirsValue = args[i+1] + skipNext = true + } + } else if strings.HasPrefix(arg, "-I=") || strings.HasPrefix(arg, "--ignore-dir=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 { + ignoreDirsValue = parts[1] + } + } else if arg == "--ignore-pattern" || arg == "--ignore" { + if i+1 < len(args) { + ignorePatternsValue = args[i+1] + skipNext = true + } + } else if strings.HasPrefix(arg, "--ignore-pattern=") || strings.HasPrefix(arg, "--ignore=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 { + ignorePatternsValue = parts[1] + } + } else if arg == "--max-size" { + if i+1 < len(args) { + maxSizeValue = args[i+1] + skipNext = true + } + } else if strings.HasPrefix(arg, "--max-size=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 { + maxSizeValue = parts[1] + } + } else if arg == "--input-file" || arg == "-i" { + if i+1 < len(args) { + inputFilePath = args[i+1] + skipNext = true + } + } else if strings.HasPrefix(arg, "--input-file=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 { + inputFilePath = parts[1] + } + } else if arg == "--max-depth" || arg == "-d" { + if i+1 < len(args) { + maxDepthValue = args[i+1] + skipNext = true + } + } else if strings.HasPrefix(arg, "--max-depth=") || strings.HasPrefix(arg, "-d=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 { + maxDepthValue = parts[1] + } + } else if arg == "--include-hidden" || arg == "-H" { + includeHiddenValue = "true" + } else if strings.HasPrefix(arg, "--include-hidden=") || strings.HasPrefix(arg, "-H=") { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 { + includeHiddenValue = parts[1] + } + } else if arg == "--minify" { + minifyValue = true + } else if arg == "--help" || arg == "-h" { + helpFlagValue = true + } else if arg == "--version" || arg == "-v" { + versionFlagValue = true } - - // Saving arguments, except for the found flag - newArgs = append(newArgs, arg) - } - - // Creating a new set of flags - flags := flag.NewFlagSet("buildtree", flag.ContinueOnError) - flags.SetOutput(stderr) - - // Defining all flags - filePath := flags.String("input-file", "", "Path to file containing directory structure") - helpFlag := flags.Bool("help", false, "Show help") - maxDepth := flags.Int("max-depth", 20, "Maximum nesting depth allowed (0 = no limit)") - versionFlag := flags.Bool("version", false, "Show version information") - - // A flag for exporting the structure (needed for parsing other flags) - saveStructure := flags.String("s", "", "Save current directory structure with file contents to specified file") - - // Additional flags for export - structureFilter := flags.String("filter", "", "Comma-separated list of file extensions to include (e.g., go,js,txt)") - ignoreDirs := flags.String("ignore-dir", "", "Comma-separated list of directories to ignore (in addition to .git and node_modules)") - maxSize := flags.String("max-size", "100kb", "Maximum file size to include (e.g., 100kb, 1mb)") - includeHidden := flags.Bool("include-hidden", false, "Include hidden files and directories (starting with .)") - - // Adding Aliases - flags.StringVar(filePath, "i", "", "Alias for --input-file") - flags.IntVar(maxDepth, "d", 20, "Alias for --max-depth") - flags.BoolVar(versionFlag, "v", false, "Alias for --version") - flags.BoolVar(helpFlag, "h", false, "Alias for --help") - flags.StringVar(saveStructure, "structure", "", "Alias for --save-structure") - flags.StringVar(structureFilter, "f", "", "Alias for --filter") - flags.StringVar(ignoreDirs, "I", "", "Alias for --ignore-dir (uppercase i)") - - // Parse the remaining arguments - if err := flags.Parse(newArgs); err != nil { - return 1 - } - - // If we found the -s flag manually, we use its value - if saveStructureValue != "" { - *saveStructure = saveStructureValue } // Processing of standard flags - if *helpFlag { + if helpFlagValue { printHelp(stdout) return 0 } - if *versionFlag { + if versionFlagValue { fmt.Fprintf(stdout, "buildtree v%s\nCommit: %s\nBuilt: %s\n", version, commit, date) return 0 } // FLAG HANDLING -s - if *saveStructure != "" { + if saveStructureValue != "" { // Checking for conflicts - if *filePath != "" { + if inputFilePath != "" { fmt.Fprintln(stderr, "Error: -s/--structure cannot be used with --input-file") return 1 } // Defining the root directory rootDir := "." - args := flags.Args() - if len(args) > 0 { - rootDir = args[0] + // Find the first non-flag argument after -s value + skipNext := false + for _, arg := range args { + if skipNext { + skipNext = false + continue + } + if strings.HasPrefix(arg, "-") { + // Skip the value if this is -s/--structure + if arg == "-s" || arg == "--structure" || strings.HasPrefix(arg, "-s=") || strings.HasPrefix(arg, "--structure=") { + skipNext = true + } + continue + } + rootDir = arg + break } - // Processing filters + // Processing filters - default to text file extensions if not specified var filters []string - if *structureFilter != "" { - filters = strings.Split(*structureFilter, ",") + if filterValue != "" { + filters = strings.Split(filterValue, ",") for i := range filters { filters[i] = strings.TrimSpace(filters[i]) } + } else { + // Default text file extensions for export + filters = []string{"go", "js", "ts", "jsx", "tsx", "py", "rb", "php", "java", "c", "cpp", "h", "hpp", "cs", "rs", "swift", "kt", "scala", "html", "css", "scss", "sass", "json", "yaml", "yml", "xml", "md", "txt", "csv", "toml", "ini", "cfg", "conf"} } // Processing ignored directories var ignoreList []string - if *ignoreDirs != "" { - ignoreList = strings.Split(*ignoreDirs, ",") + if ignoreDirsValue != "" { + ignoreList = strings.Split(ignoreDirsValue, ",") for i := range ignoreList { ignoreList[i] = strings.TrimSpace(ignoreList[i]) } } - // Parse the maximum size - maxSizeBytes, err := utils.ParseSize(*maxSize) - if err != nil { - fmt.Fprintf(stderr, "Error parsing max-size: %v\n", err) - return 1 + // Processing ignore patterns + var ignorePatternsList []string + if ignorePatternsValue != "" { + ignorePatternsList = strings.Split(ignorePatternsValue, ",") + for i := range ignorePatternsList { + ignorePatternsList[i] = strings.TrimSpace(ignorePatternsList[i]) + } } + // Default max size if empty (use 100kb as default) + maxSizeBytes, err := utils.ParseSize("100kb") + if maxSizeValue != "" { + maxSizeBytes, err = utils.ParseSize(maxSizeValue) + if err != nil { + fmt.Fprintf(stderr, "Error parsing max-size: %v\n", err) + return 1 + } + } + + // Check include hidden + includeHidden := includeHiddenValue != "" && includeHiddenValue != "false" + + // Check minify + minify := minifyValue + // Export the structure - if err := exporter.ExportStructure(*saveStructure, rootDir, filters, ignoreList, maxSizeBytes, *includeHidden); err != nil { + if err := exporter.ExportStructure(saveStructureValue, rootDir, filters, ignoreList, ignorePatternsList, maxSizeBytes, includeHidden, minify); err != nil { fmt.Fprintf(stderr, "Error exporting structure: %v\n", err) return 1 } @@ -172,11 +242,17 @@ func run(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer, p p } // Normal processing (creating a structure) - input := getInput(*filePath, stdin, flags, stderr) + input := getInput(inputFilePath, stdin, args, stderr) if input == "" { return 1 } + // Parse max depth + maxDepth := 20 + if maxDepthValue != "" { + fmt.Sscanf(maxDepthValue, "%d", &maxDepth) + } + // Parsing the structure root, err := p.ParseInput(input) if err != nil { @@ -185,7 +261,7 @@ func run(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer, p p } // Building a tree - if err := b.BuildTree(root, *maxDepth); err != nil { + if err := b.BuildTree(root, maxDepth); err != nil { fmt.Fprintf(stderr, "Error building tree: %v\n", err) return 1 } @@ -193,7 +269,7 @@ func run(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer, p p return 0 } -func getInput(filePath string, stdin io.Reader, flags *flag.FlagSet, stderr io.Writer) string { +func getInput(filePath string, stdin io.Reader, args []string, stderr io.Writer) string { if filePath != "" { content, err := os.ReadFile(filePath) if err != nil { @@ -203,13 +279,17 @@ func getInput(filePath string, stdin io.Reader, flags *flag.FlagSet, stderr io.W return string(content) } - args := flags.Args() - if len(args) < 1 { - printHelp(stderr) - fmt.Fprintln(stderr, "Error: No input structure provided") - return "" + // Find the first non-flag argument + for _, arg := range args { + if strings.HasPrefix(arg, "-") { + continue + } + return arg } - return args[0] + + printHelp(stderr) + fmt.Fprintln(stderr, "Error: No input structure provided") + return "" } func printHelp(w io.Writer) { @@ -221,10 +301,12 @@ func printHelp(w io.Writer) { fmt.Fprintln(w, " -h, --help Show this help message") fmt.Fprintln(w, " -v, --version Show version information") fmt.Fprintln(w, " -s, --structure FILE Save current directory structure with file contents to FILE") - fmt.Fprintln(w, " -f, --filter EXTS Comma-separated list of file extensions to include (e.g., go,js,txt)") + fmt.Fprintln(w, " -f, --filter EXTS Comma-separated list of file extensions or glob patterns (default: common text formats)") fmt.Fprintln(w, " -I, --ignore-dir DIRS Comma-separated list of directories to ignore (in addition to .git and node_modules)") + fmt.Fprintln(w, " --ignore PATTERNS Comma-separated glob patterns for files to ignore (e.g., *_test.go,*.log)") fmt.Fprintln(w, " --max-size SIZE Maximum file size to include (default: 100kb, e.g., 100kb, 1mb)") fmt.Fprintln(w, " -H, --include-hidden Include hidden files and directories (starting with .)") + fmt.Fprintln(w, " --minify Strip whitespace and empty lines to reduce token usage for LLM") fmt.Fprintln(w, "\nExamples:") fmt.Fprintln(w, " buildtree \"project/\n├── src/\n│ └── main.go\"") fmt.Fprintln(w, " buildtree --input-file structure.txt") diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index e95f7b9..f90eed5 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -6,11 +6,16 @@ import ( "os" "path/filepath" "strings" + ) // ExportStructure saves the current directory structure with file contents to the specified file // with filtering options -func ExportStructure(outputFile, rootDir string, filters []string, ignoreDirs []string, maxSize int64, includeHidden bool) error { +// filters: list of file extensions or glob patterns (e.g., "go", "*.go", "*.md") +// ignoreDirs: list of directory names to skip entirely +// ignorePatterns: list of glob patterns for files to ignore (e.g., "*_test.go", "*.log") +// minify: remove empty lines and strip whitespace from file contents +func ExportStructure(outputFile, rootDir string, filters []string, ignoreDirs []string, ignorePatterns []string, maxSize int64, includeHidden bool, minify bool) error { // Convert relative path to absolute rootDirAbs, err := filepath.Abs(rootDir) if err != nil { @@ -101,14 +106,35 @@ func ExportStructure(outputFile, rootDir string, filters []string, ignoreDirs [] return nil } + // Check ignore patterns (glob patterns for files to skip) + for _, pattern := range ignorePatterns { + if strings.TrimSpace(pattern) == "" { + continue + } + if match, _ := filepath.Match(pattern, fileName); match { + return nil + } + } + // Apply filters if specified if len(filters) > 0 { - ext := strings.TrimPrefix(filepath.Ext(d.Name()), ".") matched := false + ext := strings.TrimPrefix(filepath.Ext(fileName), ".") for _, filter := range filters { - if strings.EqualFold(ext, filter) { - matched = true - break + // Check if filter contains glob pattern (*, ?, []) + if strings.ContainsAny(filter, "*?[]") { + // Use filepath.Match for glob patterns + match, err := filepath.Match(filter, fileName) + if err == nil && match { + matched = true + break + } + } else { + // Simple extension match + if strings.EqualFold(ext, filter) { + matched = true + break + } } } if !matched { @@ -131,11 +157,19 @@ func ExportStructure(outputFile, rootDir string, filters []string, ignoreDirs [] return fmt.Errorf("failed to read file %s: %w", path, err) } - // Write header and content + // Write header if _, err := fmt.Fprintf(f, "# %s\n", relPath); err != nil { return err } - if _, err := f.Write(content); err != nil { + + // Write minified or original content + var contentToWrite string + if minify { + contentToWrite = minifyContent(string(content)) + } else { + contentToWrite = string(content) + } + if _, err := f.WriteString(contentToWrite); err != nil { return err } if _, err := f.WriteString("\n\n"); err != nil { @@ -145,3 +179,30 @@ func ExportStructure(outputFile, rootDir string, filters []string, ignoreDirs [] return nil }) } + +// minifyContent removes empty lines and strips whitespace from content +// to reduce token usage for LLM contexts +func minifyContent(content string) string { + lines := strings.Split(content, "\n") + result := make([]string, 0, len(lines)) + emptyLines := 0 + + for _, line := range lines { + stripped := strings.TrimSpace(line) + + if stripped == "" { + emptyLines++ + continue + } + + // Allow at most 1 empty line between content + if emptyLines > 0 { + result = append(result, "") + emptyLines = 0 + } + + result = append(result, stripped) + } + + return strings.Join(result, "\n") +} diff --git a/internal/exporter/exporter_test.go b/internal/exporter/exporter_test.go index bb7bf9a..e4baba4 100644 --- a/internal/exporter/exporter_test.go +++ b/internal/exporter/exporter_test.go @@ -18,7 +18,7 @@ func TestExportStructure_BasicExport(t *testing.T) { // Creating the output file outputFile := filepath.Join(tempDir, "export.txt") // Exporting the structure - err := ExportStructure(outputFile, tempDir, nil, nil, 0, false) + err := ExportStructure(outputFile, tempDir, nil, nil, nil, 0, false, false) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -53,7 +53,7 @@ func TestExportStructure_WithExtensionFilter(t *testing.T) { // Creating the output file outputFile := filepath.Join(tempDir, "export.txt") // Exporting only .go files - err := ExportStructure(outputFile, tempDir, []string{"go"}, nil, 0, false) + err := ExportStructure(outputFile, tempDir, []string{"go"}, nil, nil, 0, false, false) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -94,7 +94,7 @@ func TestExportStructure_WithIgnoreDirs(t *testing.T) { // Creating the output file outputFile := filepath.Join(tempDir, "export.txt") // Exporting, ignoring dir2 and node_modules - err := ExportStructure(outputFile, tempDir, nil, []string{"dir2"}, 0, false) + err := ExportStructure(outputFile, tempDir, nil, []string{"dir2"}, nil, 0, false, false) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -139,7 +139,7 @@ func TestExportStructure_WithMaxSize(t *testing.T) { // Creating the output file outputFile := filepath.Join(tempDir, "export.txt") // Exporting with a limit of 100KB - err := ExportStructure(outputFile, tempDir, nil, nil, 100*1024, false) + err := ExportStructure(outputFile, tempDir, nil, nil, nil, 100*1024, false, false) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -177,7 +177,7 @@ func TestExportStructure_WithHiddenFiles(t *testing.T) { outputFile := filepath.Join(tempDir, "export.txt") // First we check without including hidden files - err := ExportStructure(outputFile, tempDir, nil, nil, 0, false) + err := ExportStructure(outputFile, tempDir, nil, nil, nil, 0, false, false) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -197,7 +197,7 @@ func TestExportStructure_WithHiddenFiles(t *testing.T) { } // Now we check with the inclusion of hidden files - err = ExportStructure(outputFile, tempDir, nil, nil, 0, true) + err = ExportStructure(outputFile, tempDir, nil, nil, nil, 0, true, false) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -226,7 +226,7 @@ func TestExportStructure_SkipOutputFile(t *testing.T) { // Creating the output file INSIDE the exported directory outputFile := filepath.Join(tempDir, "export.txt") // Exporting the structure - err := ExportStructure(outputFile, tempDir, nil, nil, 0, false) + err := ExportStructure(outputFile, tempDir, nil, nil, nil, 0, false, false) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -257,7 +257,7 @@ func TestExportStructure_EmptyDirectory(t *testing.T) { tempDir := t.TempDir() outputFile := filepath.Join(tempDir, "export.txt") // Exporting the structure - err := ExportStructure(outputFile, tempDir, nil, nil, 0, false) + err := ExportStructure(outputFile, tempDir, nil, nil, nil, 0, false, false) if err != nil { t.Fatalf("Unexpected error: %v", err) } @@ -279,7 +279,7 @@ func TestExportStructure_InvalidRootDir(t *testing.T) { // Using a non-existent directory invalidDir := filepath.Join(tempDir, "nonexistent") // Trying to export - err := ExportStructure(outputFile, invalidDir, nil, nil, 0, false) + err := ExportStructure(outputFile, invalidDir, nil, nil, nil, 0, false, false) if err == nil { t.Fatal("Expected error for invalid root directory, but got none") } @@ -298,7 +298,7 @@ func TestExportStructure_InvalidOutputFile(t *testing.T) { // Trying to use an invalid path for the output file invalidOutput := string([]rune{0}) // null character // Trying to export - err := ExportStructure(invalidOutput, tempDir, nil, nil, 0, false) + err := ExportStructure(invalidOutput, tempDir, nil, nil, nil, 0, false, false) if err == nil { t.Fatal("Expected error for invalid output file path, but got none") } @@ -315,7 +315,7 @@ func TestExportStructure_PermissionError(t *testing.T) { // Creating the output file outputFile := filepath.Join(tempDir, "export.txt") // Exporting the structure - err := ExportStructure(outputFile, tempDir, nil, nil, 0, false) + err := ExportStructure(outputFile, tempDir, nil, nil, nil, 0, false, false) if err != nil { t.Fatalf("Unexpected error: %v", err) }