Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .github/workflows/pylint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
name: Lint (pylint)

on:
pull_request:
types: [opened, synchronize, reopened]

jobs:
pylint:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
cache: 'pip'

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements-dev.txt

- name: Run pylint
run: |
pylint rglob setup.py features
11 changes: 11 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
.PHONY: lint

lint:
@echo "Running pylint on package, setup, and tests..."
pylint rglob setup.py features

.PHONY: dev-setup

dev-setup:
@echo "Installing dev dependencies (pylint, behave)..."
pip install -r requirements-dev.txt
8 changes: 6 additions & 2 deletions features/steps/steps.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,9 @@ def count_lines_in_files(context, file_type: str) -> None:

@then("I should find {expected_line_count:d} lines")
def check_line_count(context, expected_line_count: int) -> None:
assert context.line_count == expected_line_count, f"Expected {expected_line_count} lines, but found {context.line_count}"
assert context.line_count == expected_line_count, (
f"Expected {expected_line_count} lines, but found {context.line_count}"
)

@when("I change the current working directory to the root directory")
def change_cwd_to_root(context) -> None:
Expand All @@ -129,4 +131,6 @@ def use_rglob_(context, file_type: str) -> None:

@then("I should find {expected_num_files:d} files")
def check_found_files(context, expected_num_files: int) -> None:
assert len(context.found_files) == expected_num_files, f"Expected {expected_num_files} files, but found {len(context.found_files)}"
assert len(context.found_files) == expected_num_files, (
f"Expected {expected_num_files} files, but found {len(context.found_files)}"
)
9 changes: 9 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[tool.pylint.main]
recursive = true

[tool.pylint.messages_control]
disable = []

[tool.pylint.format]
max-line-length = 100

3 changes: 3 additions & 0 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
pylint
behave

16 changes: 15 additions & 1 deletion rglob/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1,15 @@
from rglob.rglob import *
"""Public API for rglob package."""

from rglob.rglob import * # noqa: F401,F403 re-export convenience

__all__ = [
# re-exported from rglob.rglob
"rglob",
"rglob_",
"lcount",
"tsize",
"kilobytes",
"megabytes",
"gigabytes",
"terabytes",
]
61 changes: 49 additions & 12 deletions rglob/cli.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,20 @@
"""Command-line interface for rglob helpers."""

import argparse
import os
from rglob import rglob, lcount, tsize, kilobytes, megabytes, gigabytes, terabytes
from rglob import (
rglob,
lcount,
tsize,
kilobytes,
megabytes,
gigabytes,
terabytes,
)

def main() -> None:
"""Parse arguments and execute rglob CLI commands."""

def main():
parser = argparse.ArgumentParser(
description="Recursive file operations.",
epilog=(
Expand All @@ -16,20 +28,41 @@ def main():
# Find command
find_parser = subparsers.add_parser("find", help="Find files recursively.")
find_parser.add_argument("pattern", help="The glob pattern to match.")
find_parser.add_argument("--base", default=os.getcwd(), help="The base directory to search in.")
find_parser.add_argument(
"--base", default=os.getcwd(), help="The base directory to search in."
)

# Line count command
lcount_parser = subparsers.add_parser("lcount", help="Count lines in files recursively.")
lcount_parser = subparsers.add_parser(
"lcount", help="Count lines in files recursively."
)
lcount_parser.add_argument("pattern", help="The glob pattern to match.")
lcount_parser.add_argument("--base", default=os.getcwd(), help="The base directory to search in.")
lcount_parser.add_argument("--no-empty", action="store_true", help="Exclude empty lines.")
lcount_parser.add_argument("--no-comments", action="store_true", help="Exclude comment lines (starting with #).")
lcount_parser.add_argument(
"--base", default=os.getcwd(), help="The base directory to search in."
)
lcount_parser.add_argument(
"--no-empty", action="store_true", help="Exclude empty lines."
)
lcount_parser.add_argument(
"--no-comments",
action="store_true",
help="Exclude comment lines (starting with #).",
)

# Total size command
tsize_parser = subparsers.add_parser("tsize", help="Calculate the total size of files recursively.")
tsize_parser = subparsers.add_parser(
"tsize", help="Calculate the total size of files recursively."
)
tsize_parser.add_argument("pattern", help="The glob pattern to match.")
tsize_parser.add_argument("--base", default=os.getcwd(), help="The base directory to search in.")
tsize_parser.add_argument("--unit", default="mb", choices=["kb", "mb", "gb", "tb"], help="The unit to display the size in.")
tsize_parser.add_argument(
"--base", default=os.getcwd(), help="The base directory to search in."
)
tsize_parser.add_argument(
"--unit",
default="mb",
choices=["kb", "mb", "gb", "tb"],
help="The unit to display the size in.",
)

args = parser.parse_args()

Expand All @@ -43,11 +76,15 @@ def main():
filter_funcs.append(lambda line: line.strip())
if args.no_comments:
filter_funcs.append(lambda line: not line.strip().startswith("#"))

def combined_filter(line):
return all(f(line) for f in filter_funcs)

count = lcount(args.base, args.pattern, func=combined_filter if filter_funcs else lambda x: True)
count = lcount(
args.base,
args.pattern,
func=combined_filter if filter_funcs else (lambda _line: True),
)
print(f"Total lines: {count}")
elif args.command == "tsize":
unit_funcs = {
Expand Down
201 changes: 122 additions & 79 deletions rglob/rglob.py
Original file line number Diff line number Diff line change
@@ -1,84 +1,127 @@
"""
Recursive Glob Module
Methods:
rglob(base, pattern)
rglob_(pattern)
lcount(base, pattern, func=lambda x : True)
tsize(base, pattern, func=lambda x : (x / math.pow(2.0, 20))
"""
import math
"""Lightweight recursive glob helpers for files, line counts, and sizes."""

from __future__ import annotations

import glob
import math
import os
from typing import Callable, Iterable, List


def kilobytes(value: float) -> float:
"""Convert bytes to kilobytes (KiB)."""

return value / math.pow(2.0, 10.0)


def megabytes(value: float) -> float:
"""Convert bytes to megabytes (MiB)."""

return value / math.pow(2.0, 20.0)


def gigabytes(value: float) -> float:
"""Convert bytes to gigabytes (GiB)."""

return value / math.pow(2.0, 30.0)


def terabytes(value: float) -> float:
"""Convert bytes to terabytes (TiB)."""

return value / math.pow(2.0, 40.0)


def _get_dirs(base: str) -> List[str]:
"""Return immediate sub-paths under base that are directories."""

return [
path for path in glob.iglob(os.path.join(base, "*")) if os.path.isdir(path)
]


def _count(files: Iterable[str], func: Callable[[str], bool]) -> int:
"""Count lines across files that satisfy predicate func."""

total = 0
for path in files:
with open(path, "r", encoding="utf-8", errors="ignore") as handle:
total += sum(1 for line in handle if func(line))
return total


def _sum(files: Iterable[str]) -> int:
"""Sum sizes of file paths (ignores directories)."""

total_size = 0
for path in files:
if not os.path.isdir(path):
total_size += os.path.getsize(path)
return total_size


def rglob(base: str, pattern: str) -> list[str]:
"""Recursive glob starting in specified directory."""

matches: list[str] = []
matches.extend(glob.glob(os.path.join(base, pattern)))
dirs = _get_dirs(base)
if dirs:
for dpath in dirs:
matches.extend(rglob(dpath, pattern))
return matches


def rglob_(pattern: str) -> list[str]:
"""Recursive glob using the current working directory as base."""

return rglob(os.getcwd(), pattern)


def lcount(
base: str, pattern: str, func: Callable[[str], bool] = lambda _line: True
) -> int:
"""Count number of lines across files found matching pattern.

Params:
base: root directory to start the search
pattern: pattern for glob to match (e.g. "*.py")
func: boolean filter function applied per line
example: lambda line: bool(line.strip()) # don't count empty lines
default: lambda line: True
"""

all_files = rglob(base, pattern)
return _count(all_files, func)


def tsize(base: str, pattern: str, func: Callable[[float], float] = megabytes) -> float:
"""Sum and return the total size of every file found from glob(base, pattern).

Params:
base: root directory to start the search
pattern: pattern for glob to match (e.g. "*.py")
func: unit prefix conversion function
example: lambda x: x / math.pow(2.0, 20.0) # megabytes
default: megabytes
"""

all_files = rglob(base, pattern)
total_size = _sum(all_files)
return func(total_size)

kilobytes = lambda x : (x / math.pow(2.0, 10.0))
megabytes = lambda x : (x / math.pow(2.0, 20.0))
gigabytes = lambda x : (x / math.pow(2.0, 30.0))
terabytes = lambda x : (x / math.pow(2.0, 40.0))

def _getDirs(base):
return [x for x in glob.iglob(os.path.join( base, '*')) if os.path.isdir(x) ]

def _count(files, func):
lines = 0
for f in files:
lines += sum([1 for l in open(f) if func(l)])
return lines

def _sum(files):
total_size = 0
for f in files:
if not os.path.isdir(f):
total_size += os.path.getsize(f)
return total_size


def rglob(base, pattern):
""" Recursive glob starting in specified directory """
flist = []
flist.extend(glob.glob(os.path.join(base,pattern)))
dirs = _getDirs(base)
if len(dirs):
for d in dirs:
flist.extend(rglob(os.path.join(base,d), pattern))
return flist

def rglob_(pattern):
""" Performs a recursive glob in the current working directory """
return rglob(os.getcwd(), pattern)

def lcount(base, pattern, func = lambda x : True):
""" Counts the number of lines in each file found matching pattern.
Params:
base - root directory to start the search
pattern - pattern for glob to match (i.e '*.py')
func - boolean filter function
example: lambda x : True if len(x.strip()) else False #don't count empty lines
default: lambda x : True
"""
all_files = rglob(base, pattern)
return _count(all_files, func)

def tsize(base, pattern, func = megabytes ):
""" Sums and returns the total size of every file found from glob(base,pattern)
Params:
base - root directory to start the search
pattern - pattern for glob to match (i.e '*.py')
func - unit prefix conversion function
example: lambda x : (x / math.pow(2.0,20.0)) # megabytes
default: func = megabytes
"""
all_files = rglob(base, pattern)
total_size = _sum(all_files)
return func(total_size)

if __name__ == "__main__":
#filter out empty lines and comments
filterFunc = lambda x : True if (len(x.strip()) and x.strip()[0] != '#') else False

pyFiles = rglob(os.path.dirname(__file__), "*.py")
print(" {} total lines".format(_count(pyFiles, filterFunc)))

pyFiles_ = rglob_("*.py")
print(" {} total lines".format(_count(pyFiles_, filterFunc)))
print(" {} total lines".format(lcount(os.path.dirname(__file__), "*.py", filterFunc)))

# Filter out empty lines and comments.
def filter_func(line: str) -> bool:
"""Return True for non-empty, non-comment lines."""
return bool(line.strip()) and not line.strip().startswith("#")

py_files = rglob(os.path.dirname(__file__), "*.py")
print(f"{_count(py_files, filter_func)} total lines")

py_files_cwd = rglob_("*.py")
print(f"{_count(py_files_cwd, filter_func)} total lines")
print(
f"{lcount(os.path.dirname(__file__), '*.py', filter_func)} total lines"
)
3 changes: 2 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env python
"""Package configuration for rglob."""

from setuptools import setup, Extension
from setuptools import setup

setup(name='rglob',
version='1.7',
Expand Down