diff --git a/.gitignore b/.gitignore index 206a9d5..23a63eb 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .DS_Store .lite_project.lua *.md +.vogte/ /bin/ config.yaml diff --git a/analyst.go b/analyst.go index 1932fb5..be2d0eb 100644 --- a/analyst.go +++ b/analyst.go @@ -73,7 +73,7 @@ func collectArticlesForAnalysis(cfg *Config, store *Storage, fp *gofeed.Parser) articles = append(articles, articleText) if !seenToday { - if err := store.MarkAsSeen(articleLink, summary); err != nil { + if err := store.MarkAsSeen(articleLink, summary, item.Title, feed.Title, feed.Link); err != nil { continue } } diff --git a/config.go b/config.go index a7c6443..8251492 100644 --- a/config.go +++ b/config.go @@ -66,6 +66,7 @@ type Config struct { DatabaseFilePath string NotificationTrigger string NotificationWebhookURL string + GenerateAll bool } type RSS struct { @@ -81,6 +82,8 @@ func LoadConfig() (*Config, error) { configFile := flag.String("c", "", "Config file path (if you want to override the current directory config.yaml)") opmlFile := flag.String("o", "", "OPML file path to append feeds from opml files") build := flag.Bool("build", false, "Dev: Build matcha binaries in the bin directory") + generateAll := flag.Bool("generate-all", false, "Generate markdown files for all days in the database") + flag.Parse() if *build { @@ -125,6 +128,7 @@ func LoadConfig() (*Config, error) { DatabaseFilePath: viper.GetString("database_file_path"), NotificationTrigger: viper.GetString("notification_trigger"), NotificationWebhookURL: viper.GetString("notification_webhook_url"), + GenerateAll: *generateAll, } if cfg.MarkdownDirPath == "" { diff --git a/main.go b/main.go index 4b14887..aeb65a9 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "log" "os" + "time" "github.com/mmcdole/gofeed" _ "modernc.org/sqlite" @@ -20,13 +21,18 @@ func main() { } defer store.Close() + if cfg.GenerateAll { + runGenerateAll(cfg, store) + return + } + llm := NewLLMClient(cfg) var writer Writer if cfg.TerminalMode { writer = TerminalWriter{} } else { - mw := NewMarkdownWriter(cfg) + mw := NewMarkdownWriter(cfg, time.Now().Format("2006-01-02")) os.Remove(mw.FilePath) writer = mw } diff --git a/markdown_writer.go b/markdown_writer.go index b7ad0ae..049f7c3 100644 --- a/markdown_writer.go +++ b/markdown_writer.go @@ -16,14 +16,36 @@ type MarkdownWriter struct { FilePath string } -func NewMarkdownWriter(cfg *Config) *MarkdownWriter { - date := time.Now().Format("2006-01-02") - fname := cfg.MarkdownFilePrefix + date + cfg.MarkdownFileSuffix + ".md" +func NewMarkdownWriter(cfg *Config, dateStr string) *MarkdownWriter { + if dateStr == "" { + dateStr = time.Now().Format("2006-01-02") + } + fname := cfg.MarkdownFilePrefix + dateStr + cfg.MarkdownFileSuffix + ".md" return &MarkdownWriter{ FilePath: filepath.Join(cfg.MarkdownDirPath, fname), } } +func (w MarkdownWriter) WriteFeedHeaderRaw(title, articleURL string) string { + // Logic to reconstruct the Favicon HTML without a gofeed object + var src string + if strings.Contains(title, "Hacker News") { + src = "https://news.ycombinator.com/favicon.ico" + } else if articleURL == "" { + return fmt.Sprintf("\n### 🍵 %s\n", title) + } else { + u, err := url.Parse(articleURL) + if err != nil { + return fmt.Sprintf("\n### 🍵 %s\n", title) + } + // Google S2 Favicon service + src = "https://www.google.com/s2/favicons?sz=32&domain=" + u.Hostname() + } + + faviconImg := fmt.Sprintf(``, src) + return fmt.Sprintf("\n### %s %s\n", faviconImg, title) +} + func (w MarkdownWriter) Write(body string) { f, err := os.OpenFile(w.FilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { @@ -42,9 +64,8 @@ func (w MarkdownWriter) WriteLink(title string, url string, newLine bool, readin if readingTime != "" { content += fmt.Sprintf(" (%s)", readingTime) } - if newLine { - content += " \n" + content += " \n" } return content } @@ -53,16 +74,15 @@ func (w MarkdownWriter) WriteSummary(content string, newLine bool) string { if content == "" { return "" } - if newLine { - content += " \n\n" + content += " \n\n" } return content } func (w MarkdownWriter) WriteHeader(feed *gofeed.Feed) string { favicon := w.getFaviconHTML(feed) - return fmt.Sprintf("\n### %s %s\n", favicon, feed.Title) + return fmt.Sprintf("\n### %s %s\n", favicon, feed.Title) } // Helper method specifically for MarkdownWriter diff --git a/migrations.go b/migrations.go index 771ccfe..992a81d 100644 --- a/migrations.go +++ b/migrations.go @@ -2,6 +2,7 @@ package main import ( "database/sql" + "fmt" "time" ) @@ -9,6 +10,15 @@ type Storage struct { db *sql.DB } +type ArchivedItem struct { + URL string + Date string + Summary string + Title string + FeedTitle string + FeedURL string +} + func NewStorage(dbPath string) (*Storage, error) { db, err := sql.Open("sqlite", dbPath) if err != nil { @@ -19,91 +29,87 @@ func NewStorage(dbPath string) (*Storage, error) { if err := s.applyMigrations(); err != nil { return nil, err } + return s, nil } func (s *Storage) applyMigrations() error { - // create new table on database var err error _, err = s.db.Exec("CREATE TABLE IF NOT EXISTS seen (url TEXT, date TEXT, summary TEXT)") - if err != nil { return err } - err = s.addSummaryColumnIfNotExists() - if err != nil { + + if err := s.addTextColumnIfNotExists("seen", "summary"); err != nil { return err } - if err := s.addNotificationsTableIfNotExists(); err != nil { + if err := s.addTextColumnIfNotExists("seen", "title"); err != nil { + return err + } + if err := s.addTextColumnIfNotExists("seen", "feed_title"); err != nil { return err } + if err := s.addNotificationsTableIfNotExists(); err != nil { + return err + } return nil } -func (s *Storage) addNotificationsTableIfNotExists() error { - _, err := s.db.Exec(` - CREATE TABLE IF NOT EXISTS notifications ( - feed TEXT, - date TEXT, - notified INTEGER DEFAULT 0, - PRIMARY KEY (feed, date) - ) - `) - return err -} - -func (s *Storage) addSummaryColumnIfNotExists() error { +// Generic helper to add columns +func (s *Storage) addTextColumnIfNotExists(table, colName string) error { tx, err := s.db.Begin() if err != nil { return err } defer tx.Rollback() - // Check if the 'summary' column already exists in the 'seen' table - rows, err := tx.Query("PRAGMA table_info(seen)") + rows, err := tx.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) if err != nil { return err } defer rows.Close() - var columnExists bool + exists := false for rows.Next() { var cid int - var name string - var dataType string - var notNull int - var dfltValue interface{} - var pk int - err := rows.Scan(&cid, &name, &dataType, ¬Null, &dfltValue, &pk) - if err != nil { + var name, ctype string + var notNull, pk int + var dflt interface{} + if err := rows.Scan(&cid, &name, &ctype, ¬Null, &dflt, &pk); err != nil { return err } - if name == "summary" { - columnExists = true + if name == colName { + exists = true break } } - // If the 'summary' column doesn't exist, add it to the table - if !columnExists { - _, err = tx.Exec("ALTER TABLE seen ADD COLUMN summary TEXT") + if !exists { + _, err = tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s TEXT", table, colName)) if err != nil { return err } } + return tx.Commit() +} - // Commit the transaction - err = tx.Commit() - if err != nil { - return err - } - return nil +func (s *Storage) addNotificationsTableIfNotExists() error { + _, err := s.db.Exec(` + CREATE TABLE IF NOT EXISTS notifications ( + feed TEXT, + date TEXT, + notified INTEGER DEFAULT 0, + PRIMARY KEY (feed, date) + ) + `) + return err } -func (s *Storage) MarkAsSeen(url, summary string) error { +func (s *Storage) MarkAsSeen(url, summary, title, feedTitle, feedURL string) error { today := time.Now().Format("2006-01-02") - _, err := s.db.Exec("INSERT INTO seen(url, date, summary) values(?,?,?)", url, today, summary) + _, err := s.db.Exec("INSERT INTO seen(url, date, summary, title, feed_title) values(?,?,?,?,?)", + url, today, summary, title, feedTitle) return err } @@ -123,6 +129,44 @@ func (s *Storage) IsSeen(link string) (bool, bool, string) { return isSeen, isSeenToday, summary.String } +func (s *Storage) GetAllArticles() (map[string][]ArchivedItem, error) { + // Order by Date DESC, then by Feed Title so they group nicely + rows, err := s.db.Query("SELECT url, date, summary, IFNULL(title, ''), IFNULL(feed_title, '') FROM seen ORDER BY date DESC, feed_title ASC") + if err != nil { + return nil, err + } + defer rows.Close() + + grouped := make(map[string][]ArchivedItem) + + for rows.Next() { + var item ArchivedItem + var date, summary, title, feedTitle sql.NullString + if err := rows.Scan(&item.URL, &date, &summary, &title, &feedTitle); err != nil { + return nil, err + } + + item.Date = date.String + item.Summary = summary.String + item.Title = title.String + item.FeedTitle = feedTitle.String + + // Fallback for old records where title might be empty + if item.Title == "" { + item.Title = item.URL + } + if item.FeedTitle == "" { + item.FeedTitle = "Unknown Feed" + } + if item.Date == "" { + item.Date = "Unknown Date" + } + + grouped[item.Date] = append(grouped[item.Date], item) + } + return grouped, nil +} + func (s *Storage) Close() { s.db.Close() } @@ -132,9 +176,9 @@ func (s *Storage) MarkFeedNotified(feed string) error { today := time.Now().Format("2006-01-02") // Upsert: insert or replace into notifications _, err := s.db.Exec(` - INSERT INTO notifications(feed, date, notified) VALUES(?, ?, 1) - ON CONFLICT(feed, date) DO UPDATE SET notified = 1 - `, feed, today) + INSERT INTO notifications(feed, date, notified) VALUES(?, ?, 1) + ON CONFLICT(feed, date) DO UPDATE SET notified = 1 + `, feed, today) return err } @@ -149,3 +193,49 @@ func (s *Storage) WasFeedNotifiedToday(feed string) bool { } return notified == 1 } + +func (s *Storage) GetAllDays() ([]string, error) { + rows, err := s.db.Query(` + SELECT DISTINCT date + FROM seen + ORDER BY date ASC + `) + if err != nil { + return nil, err + } + defer rows.Close() + + var days []string + for rows.Next() { + var d string + if err := rows.Scan(&d); err == nil { + days = append(days, d) + } + } + return days, nil +} + +func (s *Storage) GetArticlesForDay(day string) ([]SeenArticle, error) { + rows, err := s.db.Query(` + SELECT url, summary + FROM seen + WHERE date = ? + `, day) + if err != nil { + return nil, err + } + defer rows.Close() + + var res []SeenArticle + for rows.Next() { + var a SeenArticle + rows.Scan(&a.URL, &a.Summary) + res = append(res, a) + } + return res, nil +} + +type SeenArticle struct { + URL string + Summary string +} diff --git a/processor.go b/processor.go index 0828f33..3114ccc 100644 --- a/processor.go +++ b/processor.go @@ -3,7 +3,9 @@ package main import ( "fmt" "log" + "os" "regexp" + "sort" "strconv" "strings" "time" @@ -37,14 +39,12 @@ func ProcessFeed(rss RSS, cfg *Config, store *Storage, llm *LLMClient, w Writer, } itemsFound = true - title := item.Title if title == "" { title = stripHtmlRegex(item.Description) } summary := prevSummary - if summary == "" && rss.summarize { summary = getSummary(llm, item, cfg) } @@ -76,7 +76,7 @@ func ProcessFeed(rss RSS, cfg *Config, store *Storage, llm *LLMClient, w Writer, } if !seenToday { - store.MarkAsSeen(item.Link, summary) + store.MarkAsSeen(item.Link, summary, item.Title, feed.Title, feed.FeedLink) } } @@ -94,12 +94,9 @@ func getSummary(llm *LLMClient, item *gofeed.Item, cfg *Config) string { if err == nil { content = scrapedText.TextContent } - fmt.Println("we have crawled") - return llm.Summarize(content) } - return item.Description } @@ -142,13 +139,16 @@ func extractImageTagFromHTML(htmlText string) string { if width != "" && height != "" { wInt, _ := strconv.Atoi(width) hInt, _ := strconv.Atoi(height) + if wInt > 0 && hInt > 0 { aspectRatio := float64(wInt) / float64(hInt) const maxWidth = 400 + if wInt > maxWidth { wInt = maxWidth hInt = int(float64(wInt) / aspectRatio) } + firstImgTag.SetAttr("width", fmt.Sprintf("%d", wInt)) firstImgTag.SetAttr("height", fmt.Sprintf("%d", hInt)) } @@ -158,13 +158,14 @@ func extractImageTagFromHTML(htmlText string) string { if err != nil { return "" } + return html } func formatHackerNewsLinks(w Writer, item *gofeed.Item) string { desc := item.Description - commentsURL := "" + if start := strings.Index(desc, "Comments URL"); start != -1 { safeStart := start + 23 if safeStart+45 < len(desc) { @@ -203,3 +204,66 @@ func stripHtmlRegex(s string) string { r := regexp.MustCompile(regex) return r.ReplaceAllString(s, "") } + +func runGenerateAll(cfg *Config, store *Storage) { + fmt.Println("Regenerating all daily digests from database...") + + allData, err := store.GetAllArticles() + if err != nil { + log.Fatalf("Error querying database: %v", err) + } + + var dates []string + for d := range allData { + dates = append(dates, d) + } + sort.Strings(dates) // 2023-01-01, 2023-01-02... + + for _, date := range dates { + items := allData[date] + fmt.Printf("Processing %s (%d articles)... \n", date, len(items)) + + mw := NewMarkdownWriter(cfg, date) + os.Remove(mw.FilePath) + + // Group items by Feed Title so we can create sections + feeds := make(map[string][]ArchivedItem) + var feedOrder []string // To keep consistent order + + for _, item := range items { + if _, exists := feeds[item.FeedTitle]; !exists { + feedOrder = append(feedOrder, item.FeedTitle) + } + feeds[item.FeedTitle] = append(feeds[item.FeedTitle], item) + } + sort.Strings(feedOrder) + + for _, feedTitle := range feedOrder { + feedItems := feeds[feedTitle] + + // Write Feed Header (using the first item's feed URL for favicon) + firstItem := feedItems[0] + mw.Write(mw.WriteFeedHeaderRaw(feedTitle, firstItem.URL)) + + for _, item := range feedItems { + var line string + + // Instapaper Icon (if enabled) + if cfg.Instapaper { + instapaperURL := fmt.Sprintf("https://www.instapaper.com/hello2?url=%s", item.URL) + line += fmt.Sprintf(`[](%s)`, instapaperURL) + } + + // Article Title Link + line += fmt.Sprintf("[%s](%s) \n", item.Title, item.URL) + mw.Write(line) + + // Summary (if exists) + if item.Summary != "" { + mw.Write(mw.WriteSummary(item.Summary, true)) + } + } + } + } + fmt.Println("Done.") +}