diff --git a/README.md b/README.md index 0f8fbb2..1a332b1 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ Add these in **Settings → Secrets → Actions**: | `MODEL_API_KEY` | API key for the model endpoint (leave empty if not required) | | `MODEL_NAME` | Model name to use (e.g., `meta-llama/Llama-3.1-8B-Instruct`, `gemini-2.0-flash`) | | `DOCS_REPO_URL` | Docs repository URL (e.g., `https://github.com/org/docs`) | -| `GH_PAT` | GitHub token with `repo` + `pull_requests:write` permissions | +| `GH_PAT` | GitHub token with `repo` + `pull_requests:write` permissions. Used for creating docs PRs, posting review comments, and submitting index update PRs | | `DOCS_SUBFOLDER` | _(Optional)_ Docs subfolder path (e.g., `docs`) | | `DOCS_BASE_BRANCH` | _(Optional)_ Base branch for docs PRs (default: `main`) | | `JIRA_URL` | _(Optional, for `[review-feature]`)_ Jira instance URL (e.g., `https://your-company.atlassian.net`) | @@ -208,4 +208,4 @@ The action builds semantic indexes stored in `.doc-index/`: - **Folder Indexes** — AI-generated summaries of each documentation folder, including per-file descriptions. The LLM selects relevant files directly from these descriptions without loading the actual file content, reducing API calls and runtime. -Indexes are automatically committed to your main branch and shared across all PRs, reducing runtime from ~20 minutes to ~4 minutes on large projects. +Indexes are automatically submitted via a PR to your main branch and shared across all PRs once merged, reducing runtime from ~20 minutes to ~4 minutes on large projects. diff --git a/src/doc_index.py b/src/doc_index.py index 9923eb7..958bbba 100644 --- a/src/doc_index.py +++ b/src/doc_index.py @@ -41,6 +41,7 @@ SUMMARIES_DIR = "summaries" SUMMARIES_MANIFEST = "summaries_manifest.json" INDEX_VERSION = "1.0" +INDEX_BRANCH = "code-to-docs/update-indexes" MAX_WORKERS_INDEX = 5 # Parallel threads for index generation MAX_WORKERS_API = 10 # Parallel threads for API calls @@ -642,26 +643,27 @@ def update_indexes_if_needed(): return updated_folders + def commit_indexes_to_repo(content_type="indexes"): """ - Commit the .doc-index folder to the repository. + Commit the .doc-index folder and open a PR to the base branch. - This persists the indexes/summaries so they don't need to be rebuilt on every run. + Instead of pushing directly to main (which bypasses branch protection), + this pushes to a persistent branch and creates/updates a PR. Args: - content_type: What's being committed - "indexes" or "summaries" (for clearer logs) + content_type: What's being committed - "indexes", "summaries", or both Returns: - bool: True if content was committed successfully, False otherwise + bool: True if content was committed and PR created/updated, False otherwise """ - docs_root = get_docs_root().resolve() # Resolve to absolute path + docs_root = get_docs_root().resolve() index_path = docs_root / INDEX_DIR if not index_path.exists(): print(f"No {content_type} to commit") return False - # Determine target directory and relative path for git operations docs_subfolder = os.environ.get("DOCS_SUBFOLDER", "") if docs_subfolder: repo_root = docs_root.parent @@ -674,12 +676,6 @@ def commit_indexes_to_repo(content_type="indexes"): try: with working_directory(target_dir): - if docs_subfolder: - print("Changed to repo root for git operations") - else: - print("Using docs root for git operations") - - # Check if there are any changes to commit status_result = run_command_safe( ["git", "status", "--porcelain", index_relative_path], check=False @@ -689,157 +685,95 @@ def commit_indexes_to_repo(content_type="indexes"): print(f"No {content_type} changes to commit") return False - # Check if the current branch is up-to-date with main before pushing - # This prevents older branches from overwriting newer indexes base_branch = os.environ.get("DOCS_BASE_BRANCH", "main") + gh_token = os.environ.get("GH_TOKEN") + if not gh_token: + print("Warning: GH_TOKEN not set, cannot create index PR") + return False - # Fetch latest main to get accurate comparison - run_command_safe(["git", "fetch", "origin", base_branch], check=False) - - # Get the merge-base between current HEAD and origin/main - merge_base_result = run_command_safe( - ["git", "merge-base", "HEAD", f"origin/{base_branch}"], - check=False + current_branch_result = run_command_safe( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + check=True ) + current_branch = current_branch_result.stdout.strip() - # Get the latest commit on origin/main - main_head_result = run_command_safe( - ["git", "rev-parse", f"origin/{base_branch}"], - check=False - ) + temp_dir = tempfile.mkdtemp() + index_full_path = Path(repo_root) / index_relative_path + temp_index_path = Path(temp_dir) / ".doc-index-backup" + if index_full_path.exists(): + shutil.copytree(index_full_path, temp_index_path) - branch_up_to_date = True - if merge_base_result.returncode == 0 and main_head_result.returncode == 0: - merge_base = merge_base_result.stdout.strip() - main_head = main_head_result.stdout.strip() - branch_up_to_date = (merge_base == main_head) + run_command_safe(["git", "stash", "--include-untracked"], check=False) - # Track what files to add (used both here and after branch switch) - files_to_add = [] - add_all = False + try: + run_command_safe(["git", "fetch", "origin", base_branch], check=False) + run_command_safe( + ["git", "checkout", "-B", INDEX_BRANCH, f"origin/{base_branch}"], + check=True + ) + + if temp_index_path.exists(): + if index_full_path.exists(): + shutil.rmtree(index_full_path) + shutil.copytree(temp_index_path, index_full_path) - if branch_up_to_date: - # Branch is up-to-date, safe to push everything - add_all = True run_command_safe(["git", "add", index_relative_path], check=True) - else: - # Branch is not up-to-date - be selective about what we push - print(f"⚠️ Branch is not up-to-date with {base_branch}") - - if "indexes" in content_type: - # For folder indexes: skip entirely (could overwrite newer indexes on main) - print(f" Skipping index push to avoid overwriting newer indexes on {base_branch}") - print(f" Indexes will be used locally but not committed") - if "summaries" not in content_type: - return False - # Combined push — indexes can't be pushed but try summaries below - content_type = "summaries" # update label to reflect what's actually being pushed - - # For summaries: only push if doc content matches main (safe) - print(f" Checking which summaries are safe to push...") - - safe_summaries = get_safe_summaries_to_push(base_branch) - - if safe_summaries: - print(f" Found {len(safe_summaries)} summaries safe to push (doc content matches main)") - # Track the files we're adding (for use after branch switch) - files_to_add = list(safe_summaries) - files_to_add.append(f"{index_relative_path}/{SUMMARIES_MANIFEST}") - - # Add only the safe summaries and the summaries manifest - for summary_path in safe_summaries: - run_command_safe(["git", "add", summary_path], check=False) - # Also add the summaries manifest (stored at .doc-index/summaries_manifest.json) - summaries_manifest_path = f"{index_relative_path}/{SUMMARIES_MANIFEST}" - run_command_safe(["git", "add", summaries_manifest_path], check=False) - else: - print(f" No summaries safe to push, skipping commit") - return False - # Check if there's actually anything staged - staged_result = run_command_safe( - ["git", "diff", "--cached", "--name-only"], - check=False - ) - if not staged_result.stdout.strip(): - print("No changes staged for commit") - return False + staged_result = run_command_safe( + ["git", "diff", "--cached", "--name-only"], + check=False + ) + if not staged_result.stdout.strip(): + print(f"No {content_type} changes to commit (already up to date)") + return False - # Commit the indexes - commit_msg = "chore: Update documentation semantic indexes\n\nAuto-generated by code-to-docs action" - run_command_safe( - ["git", "commit", "-m", commit_msg], - check=True - ) + commit_msg = f"chore: Update documentation semantic {content_type}\n\nAuto-generated by code-to-docs action" + run_command_safe(["git", "commit", "-m", commit_msg], check=True) - # Push to the base/main branch so indexes are reusable across all PRs + print(f"Pushing {content_type} to branch {INDEX_BRANCH}...") + run_command_safe( + ["git", "push", "--set-upstream", "origin", INDEX_BRANCH, "--force-with-lease"], + check=True, + ) - # Get current branch to restore later - current_branch_result = run_command_safe( - ["git", "rev-parse", "--abbrev-ref", "HEAD"], - check=True - ) - current_branch = current_branch_result.stdout.strip() + check_pr = run_command_safe( + ["gh", "pr", "list", "--head", INDEX_BRANCH, "--state", "open", "--json", "number"], + check=False, + env={**os.environ, "GH_TOKEN": gh_token}, + ) + existing_pr = check_pr.stdout.strip() if check_pr.returncode == 0 else "[]" - if current_branch != base_branch: - print(f"Switching to {base_branch} to push {content_type}...") - - # Save the .doc-index directory content BEFORE switching branches - # (because cherry-pick often fails and stash doesn't help since we already committed) - temp_dir = tempfile.mkdtemp() - index_full_path = Path(repo_root) / index_relative_path - temp_index_path = Path(temp_dir) / ".doc-index-backup" - if index_full_path.exists(): - shutil.copytree(index_full_path, temp_index_path) - - # Stash any uncommitted changes - run_command_safe(["git", "stash", "--include-untracked"], check=False) - - try: - # Checkout base branch (create or reset local branch from origin) - run_command_safe(["git", "fetch", "origin", base_branch], check=False) - run_command_safe(["git", "checkout", "-B", base_branch, f"origin/{base_branch}"], check=True) - - # Restore our saved .doc-index directory (overwrites main's version with our updated version) - if temp_index_path.exists(): - if index_full_path.exists(): - shutil.rmtree(index_full_path) - shutil.copytree(temp_index_path, index_full_path) - - # Add files - respect the same selective logic we used on PR branch - if add_all: - run_command_safe(["git", "add", index_relative_path], check=True) - else: - # Add only the specific files we determined were safe - for file_path in files_to_add: - run_command_safe(["git", "add", file_path], check=False) - run_command_safe( - ["git", "commit", "-m", commit_msg], - check=False # May fail if no changes + if existing_pr and existing_pr != "[]": + print(f"✅ Updated existing index PR (branch {INDEX_BRANCH})") + else: + pr_body = ( + f"This PR updates documentation semantic {content_type}.\n\n" + "These are auto-generated indexes and file summaries used by the " + "code-to-docs action to speed up documentation file discovery.\n\n" + "*Auto-generated by code-to-docs action*" ) - - # Push to base branch - print(f"Pushing {content_type} to {base_branch}...") run_command_safe( - ["git", "push", "origin", base_branch], - check=True + [ + "gh", "pr", "create", + "--title", f"chore: Update documentation semantic {content_type}", + "--body", pr_body, + "--base", base_branch, + "--head", INDEX_BRANCH, + ], + check=True, + env={**os.environ, "GH_TOKEN": gh_token}, ) + print(f"✅ Created PR for {content_type} (branch {INDEX_BRANCH})") - print(f"✅ {content_type.capitalize()} committed and pushed to {base_branch}") - finally: - # Always return to original branch and restore stash - run_command_safe(["git", "checkout", current_branch], check=False) - run_command_safe(["git", "stash", "pop"], check=False) - shutil.rmtree(temp_dir, ignore_errors=True) - else: - # Already on base branch, just push - print(f"Pushing {content_type} to {base_branch}...") - run_command_safe( - ["git", "push", "origin", base_branch], - check=True - ) - print(f"✅ {content_type.capitalize()} committed and pushed to {base_branch}") - return True + return True + + finally: + run_command_safe(["git", "checkout", current_branch], check=False) + stash_result = run_command_safe(["git", "stash", "pop"], check=False) + if stash_result.returncode != 0 and stash_result.stderr: + print(f"Warning: git stash pop failed — stashed changes may need manual recovery. " + f"Run 'git stash list' to find them.") + shutil.rmtree(temp_dir, ignore_errors=True) except subprocess.CalledProcessError as e: print(f"Warning: Failed to commit {content_type}: {sanitize_output(str(e))}") @@ -1305,127 +1239,6 @@ def summaries_exist(docs_root=None): return len(summary_files) > 0 -def doc_matches_main(doc_file_path, base_branch="main"): - """ - Check if a local doc file matches the same file on origin/main. - - This determines if a summary generated from this doc is safe to push to main. - - Args: - doc_file_path: Path to the doc file (relative to docs root) - base_branch: The base branch to compare against - - Returns: - bool: True if file matches main (safe to push summary), False otherwise - """ - try: - docs_root = get_docs_root() - - # Get local file hash - local_path = Path(docs_root) / doc_file_path - if not local_path.exists(): - local_path = Path(doc_file_path) - - if not local_path.exists(): - return False - - local_hash = hash_file(local_path) - - # Get file content from origin/main - docs_subfolder = os.environ.get("DOCS_SUBFOLDER", "") - if docs_subfolder: - main_file_path = f"{docs_subfolder}/{doc_file_path}" - else: - main_file_path = doc_file_path - - result = run_command_safe( - ["git", "show", f"origin/{base_branch}:{main_file_path}"], - check=False - ) - - if result.returncode != 0: - # File doesn't exist on main - check if it's a new file - # New files are safe to push - return True - - # Hash main's content - main_hash = hashlib.sha256(result.stdout.encode()).hexdigest() - - return local_hash == main_hash - except Exception as e: - print(f"Warning: Could not compare {doc_file_path} with main: {sanitize_output(str(e))}") - return False - - -def load_manifest_from_main(base_branch="main"): - """Load the summaries manifest from the main branch (via git show).""" - docs_root = get_docs_root() - - # Get the relative path to manifest from repo root - # We need to construct the path as git sees it - docs_subfolder = os.environ.get("DOCS_SUBFOLDER", "") - if docs_subfolder: - manifest_git_path = f"{docs_subfolder}/{INDEX_DIR}/{SUMMARIES_MANIFEST}" - else: - manifest_git_path = f"{INDEX_DIR}/{SUMMARIES_MANIFEST}" - - result = run_command_safe( - ["git", "show", f"origin/{base_branch}:{manifest_git_path}"], - check=False - ) - - if result.returncode == 0 and result.stdout.strip(): - try: - return json.loads(result.stdout) - except json.JSONDecodeError: - return {"version": "1.0", "files": {}} - - return {"version": "1.0", "files": {}} - - -def get_safe_summaries_to_push(base_branch="main"): - """ - Get list of summary files that are safe to push to main. - - Only returns summaries that are: - 1. NEW (not already on main), AND - 2. Safe (doc content matches main - prevents pushing summaries for modified docs) - - Args: - base_branch: The base branch to compare against - - Returns: - list: List of summary file paths that are safe to push - """ - docs_root = get_docs_root() - local_manifest = load_summaries_manifest(docs_root) - main_manifest = load_manifest_from_main(base_branch) - - main_files = main_manifest.get("files", {}) - - safe_summaries = [] - - for doc_path, info in local_manifest.get("files", {}).items(): - summary_file = info.get("summary_file") - if not summary_file: - continue - - # Skip if already on main with same hash (no need to re-push) - if doc_path in main_files: - main_hash = main_files[doc_path].get("hash") - local_hash = info.get("hash") - if main_hash == local_hash: - continue # Already on main, skip - - # Check if doc matches main (safe to push) - if doc_matches_main(doc_path, base_branch): - summary_path = get_summaries_dir(docs_root) / summary_file - if summary_path.exists(): - safe_summaries.append(str(summary_path)) - - return safe_summaries - - # CLI interface for testing if __name__ == "__main__": import argparse