allow sync-podcasts to choose embedding model and give grant

projects/grant/.flox/env/manifest.toml

@@ -47,8 +47,8 @@ on-activate = '''
   fi
 
   uv python install 3.13
-  uv venv
-  source "${FLOX_ENV_PROJECT}/.venv/bin/activate"
+  #uv venv
+  #source "${FLOX_ENV_PROJECT}/.venv/bin/activate"
   uv sync
 
   echo "Grant RAG environment activated"

projects/grant/grant/cli.py

@@ -216,6 +216,47 @@ def stats_command(args):
     print(f"  Chunk size: {stats['chunk_size']} tokens")
 
 
+def check_unindexed_command(args):
+    """Handle the check-unindexed command."""
+    client = OllamaClient(base_url=args.base_url)
+
+    # Check if database exists
+    if not Path(args.db).exists():
+        print(f"Error: Database not found: {args.db}", file=sys.stderr)
+        sys.exit(1)
+
+    # Initialize RAG pipeline
+    rag = RAGPipeline(
+        ollama_client=client,
+        vector_store_path=args.vector_store,
+        db_path=args.db,
+    )
+
+    # Get unindexed transcriptions
+    unindexed = rag.get_unindexed_transcriptions()
+
+    if not unindexed:
+        print("All transcriptions have been indexed! ✓")
+        sys.exit(0)
+
+    # Display summary
+    print(f"Found {len(unindexed)} unindexed transcription(s)")
+    print("=" * 80)
+
+    # Display details
+    for trans in unindexed:
+        print(f"\nID: {trans['id']}")
+        print(f"Title: {trans['title']}")
+        print(f"Feed: {trans['feed_url']}")
+        print(f"Published: {trans['published']}")
+        print(f"Filename: {trans['filename']}")
+
+    print("\n" + "=" * 80)
+    print(f"Total unindexed: {len(unindexed)}")
+    print("\nTo index these transcriptions, run:")
+    print(f"  grant index --db {args.db} --vector-store {args.vector_store}")
+
+
 def main():
     parser = argparse.ArgumentParser(description="Grant - RAG tool using Ollama")
     parser.add_argument(
@@ -324,6 +365,21 @@ def main():
         help="Path to vector store directory",
     )
 
+    # Check-unindexed command
+    check_parser = subparsers.add_parser(
+        "check-unindexed", help="Check for transcriptions not yet indexed"
+    )
+    check_parser.add_argument(
+        "--db",
+        default=DEFAULT_FC_DB,
+        help="Path to transcriptions database",
+    )
+    check_parser.add_argument(
+        "--vector-store",
+        default=DEFAULT_CHROMA_DB,
+        help="Path to vector store directory",
+    )
+
     args = parser.parse_args()
 
     # Setup logging
@@ -348,6 +404,8 @@ def main():
         ask_command(args)
     elif args.command == "stats":
         stats_command(args)
+    elif args.command == "check-unindexed":
+        check_unindexed_command(args)
 
 
 if __name__ == "__main__":

projects/grant/grant/rag.py

@@ -363,3 +363,26 @@ def get_stats(self) -> Dict[str, Any]:
             stats["indexed_transcriptions"] = "N/A (no database)"
 
         return stats
+
+    def get_unindexed_transcriptions(self) -> List[Dict[str, Any]]:
+        """Get list of transcriptions that haven't been indexed yet."""
+        if not self.db:
+            return []
+
+        unindexed = []
+        transcriptions = self.db.get_all_transcriptions()
+
+        for trans in transcriptions:
+            if trans.id and not self._is_transcription_indexed(trans.id):
+                unindexed.append({
+                    "id": trans.id,
+                    "title": trans.feed_item_title or "Untitled",
+                    "feed_url": trans.feed_url or "Unknown Feed",
+                    "published": trans.feed_item_published.isoformat() if trans.feed_item_published else "Unknown",
+                    "filename": trans.filename,
+                })
+
+        # Sort by published date (newest first)
+        unindexed.sort(key=lambda x: x["published"], reverse=True)
+
+        return unindexed

projects/sync-podcasts/.flox/env/manifest.toml

@@ -28,6 +28,7 @@ ollama.pkg-path = "ollama"
 ## -------------------------------------------------------------------
 [vars]
 # INTRO_MESSAGE = "It's gettin' Flox in here"
+PYTORCH_CUDA_ALLOC_CONF="expandable_segments:True"
 
 
 ## Activation Hook ---------------------------------------------------

projects/sync-podcasts/SYNC.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+sync-podcasts --since 2025-02-28 --daemon --verbose --round-robin --model large --db ../../resources/fc/original-fc.db --feeds ../../resources/fc/feeds.toml --vector-store ../../resources/fc/grant_chroma_db --embedding-model nomic-embed-text:v1.5
+
+#  -h, --help            show this help message and exit
+#  --since, --date-threshold DATE_THRESHOLD
+#                        Only process episodes published after this date (ISO8601 format)
+#  --days DAYS           Process episodes from the last N days (alternative to --since)
+#  --feeds FEEDS         Path to TOML file containing RSS feed URLs (default: ./resources/fc/feeds.toml)
+#  --db DB               Path to beige-book database (default: ./resources/fc/fc.db)
+#  --vector-store VECTOR_STORE
+#                        Path to Grant vector store directory (default: ./projects/grant/grant_chroma_db)
+#  --model {tiny,base,small,medium,large}
+#                        Whisper model to use for transcription (default: tiny)
+#  --round-robin         Process feeds in round-robin mode (newest episode from each feed before moving to next)
+#  --daemon              Run in daemon mode (continuous processing with exponential backoff)
+#  --ollama-base-url OLLAMA_BASE_URL
+#                        Ollama API base URL (default: http://localhost:11434)
+#  --verbose, -v         Enable verbose logging

projects/sync-podcasts/sync_podcasts/cli.py

@@ -71,6 +71,11 @@ def main():
         default="http://localhost:11434",
         help="Ollama API base URL (default: http://localhost:11434)",
     )
+    parser.add_argument(
+        "--embedding-model",
+        default="nomic-embed-text",
+        help="Embedding model to use for indexing (default: nomic-embed-text)",
+    )
     parser.add_argument(
         "--verbose",
         "-v",
@@ -99,6 +104,7 @@ def main():
         date_threshold=args.date_threshold,
         days_back=args.days,
         ollama_base_url=args.ollama_base_url,
+        embedding_model=args.embedding_model,
         verbose=args.verbose,
     )
 

projects/sync-podcasts/sync_podcasts/sync.py

@@ -19,7 +19,7 @@
     DatabaseConfig,
 )
 from beige_book.feed_parser import FeedParser
-from grant import RAGPipeline, OllamaClient
+from grant import RAGPipeline, OllamaClient, RAGConfig
 from pinkhaus_models import TranscriptionDatabase
 
 from .validate_feed import validate_feeds_toml
@@ -40,6 +40,7 @@ class SyncConfig:
     date_threshold: Optional[str] = None
     days_back: Optional[int] = None
     ollama_base_url: str = "http://localhost:11434"
+    embedding_model: str = "nomic-embed-text"
     verbose: bool = False
     max_failures: int = 3  # Maximum retry attempts before marking as permanently failed
     stale_minutes: int = 30  # Minutes before considering a processing task stale
@@ -320,8 +321,13 @@ def process_one_podcast(self) -> bool:
 
                 # Index the new transcription immediately
                 try:
+                    rag_config = RAGConfig(
+                        embedding_model=self.config.embedding_model
+                    )
+
                     rag_pipeline = RAGPipeline(
                         ollama_client=self.ollama_client,
+                        config=rag_config,
                         db_path=self.config.db_path,
                         vector_store_path=self.config.vector_store_path,
                     )

resources/fc/feeds.toml

@@ -14,4 +14,13 @@ rss = [
 	"https://feeds.acast.com/public/shows/681ccb505acb8b715f1f1330",
 	"https://feeds.acast.com/public/shows/6819aed5eb146d8e3506a450",
 	"https://feeds.acast.com/public/shows/681ccc398b1f3232bc1e68ae",
+	"https://feeds.acast.com/public/shows/6818cd43eb146d8e35d312c7",
+	"https://feeds.acast.com/public/shows/d2a98018-244b-4999-9a83-9c2b9d43a1e3",
+	"https://feeds.megaphone.fm/GLT8847082992",
+	"https://feeds.acast.com/public/shows/8af0c347-5e6e-48e2-8fae-3f4e765c84ab",
+	"https://feeds.acast.com/public/shows/681cccd63e6644d7a3b3065c",
+	"https://feeds.acast.com/public/shows/62e90af8d40557001270a30e",
+	"https://feeds.megaphone.fm/wrightys-house",
+	"https://feeds.acast.com/public/shows/6242e9f71c8beb00139400a6",
+	"https://feeds.acast.com/public/shows/deafe434-0a5c-4762-bc95-b11e2ce6be4e",
 ]