diff --git a/.gitignore b/.gitignore
index 206a9d5..23a63eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,6 +3,7 @@
.DS_Store
.lite_project.lua
*.md
+.vogte/
/bin/
config.yaml
diff --git a/analyst.go b/analyst.go
index 1932fb5..be2d0eb 100644
--- a/analyst.go
+++ b/analyst.go
@@ -73,7 +73,7 @@ func collectArticlesForAnalysis(cfg *Config, store *Storage, fp *gofeed.Parser)
articles = append(articles, articleText)
if !seenToday {
- if err := store.MarkAsSeen(articleLink, summary); err != nil {
+ if err := store.MarkAsSeen(articleLink, summary, item.Title, feed.Title, feed.Link); err != nil {
continue
}
}
diff --git a/config.go b/config.go
index a7c6443..8251492 100644
--- a/config.go
+++ b/config.go
@@ -66,6 +66,7 @@ type Config struct {
DatabaseFilePath string
NotificationTrigger string
NotificationWebhookURL string
+ GenerateAll bool
}
type RSS struct {
@@ -81,6 +82,8 @@ func LoadConfig() (*Config, error) {
configFile := flag.String("c", "", "Config file path (if you want to override the current directory config.yaml)")
opmlFile := flag.String("o", "", "OPML file path to append feeds from opml files")
build := flag.Bool("build", false, "Dev: Build matcha binaries in the bin directory")
+ generateAll := flag.Bool("generate-all", false, "Generate markdown files for all days in the database")
+
flag.Parse()
if *build {
@@ -125,6 +128,7 @@ func LoadConfig() (*Config, error) {
DatabaseFilePath: viper.GetString("database_file_path"),
NotificationTrigger: viper.GetString("notification_trigger"),
NotificationWebhookURL: viper.GetString("notification_webhook_url"),
+ GenerateAll: *generateAll,
}
if cfg.MarkdownDirPath == "" {
diff --git a/main.go b/main.go
index 4b14887..aeb65a9 100644
--- a/main.go
+++ b/main.go
@@ -3,6 +3,7 @@ package main
import (
"log"
"os"
+ "time"
"github.com/mmcdole/gofeed"
_ "modernc.org/sqlite"
@@ -20,13 +21,18 @@ func main() {
}
defer store.Close()
+ if cfg.GenerateAll {
+ runGenerateAll(cfg, store)
+ return
+ }
+
llm := NewLLMClient(cfg)
var writer Writer
if cfg.TerminalMode {
writer = TerminalWriter{}
} else {
- mw := NewMarkdownWriter(cfg)
+ mw := NewMarkdownWriter(cfg, time.Now().Format("2006-01-02"))
os.Remove(mw.FilePath)
writer = mw
}
diff --git a/markdown_writer.go b/markdown_writer.go
index b7ad0ae..049f7c3 100644
--- a/markdown_writer.go
+++ b/markdown_writer.go
@@ -16,14 +16,36 @@ type MarkdownWriter struct {
FilePath string
}
-func NewMarkdownWriter(cfg *Config) *MarkdownWriter {
- date := time.Now().Format("2006-01-02")
- fname := cfg.MarkdownFilePrefix + date + cfg.MarkdownFileSuffix + ".md"
+func NewMarkdownWriter(cfg *Config, dateStr string) *MarkdownWriter {
+ if dateStr == "" {
+ dateStr = time.Now().Format("2006-01-02")
+ }
+ fname := cfg.MarkdownFilePrefix + dateStr + cfg.MarkdownFileSuffix + ".md"
return &MarkdownWriter{
FilePath: filepath.Join(cfg.MarkdownDirPath, fname),
}
}
+func (w MarkdownWriter) WriteFeedHeaderRaw(title, articleURL string) string {
+ // Logic to reconstruct the Favicon HTML without a gofeed object
+ var src string
+ if strings.Contains(title, "Hacker News") {
+ src = "https://news.ycombinator.com/favicon.ico"
+ } else if articleURL == "" {
+ return fmt.Sprintf("\n### 🍵 %s\n", title)
+ } else {
+ u, err := url.Parse(articleURL)
+ if err != nil {
+ return fmt.Sprintf("\n### 🍵 %s\n", title)
+ }
+ // Google S2 Favicon service
+ src = "https://www.google.com/s2/favicons?sz=32&domain=" + u.Hostname()
+ }
+
+ faviconImg := fmt.Sprintf(`
`, src)
+ return fmt.Sprintf("\n### %s %s\n", faviconImg, title)
+}
+
func (w MarkdownWriter) Write(body string) {
f, err := os.OpenFile(w.FilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
@@ -42,9 +64,8 @@ func (w MarkdownWriter) WriteLink(title string, url string, newLine bool, readin
if readingTime != "" {
content += fmt.Sprintf(" (%s)", readingTime)
}
-
if newLine {
- content += " \n"
+ content += " \n"
}
return content
}
@@ -53,16 +74,15 @@ func (w MarkdownWriter) WriteSummary(content string, newLine bool) string {
if content == "" {
return ""
}
-
if newLine {
- content += " \n\n"
+ content += " \n\n"
}
return content
}
func (w MarkdownWriter) WriteHeader(feed *gofeed.Feed) string {
favicon := w.getFaviconHTML(feed)
- return fmt.Sprintf("\n### %s %s\n", favicon, feed.Title)
+ return fmt.Sprintf("\n### %s %s\n", favicon, feed.Title)
}
// Helper method specifically for MarkdownWriter
diff --git a/migrations.go b/migrations.go
index 771ccfe..992a81d 100644
--- a/migrations.go
+++ b/migrations.go
@@ -2,6 +2,7 @@ package main
import (
"database/sql"
+ "fmt"
"time"
)
@@ -9,6 +10,15 @@ type Storage struct {
db *sql.DB
}
+type ArchivedItem struct {
+ URL string
+ Date string
+ Summary string
+ Title string
+ FeedTitle string
+ FeedURL string
+}
+
func NewStorage(dbPath string) (*Storage, error) {
db, err := sql.Open("sqlite", dbPath)
if err != nil {
@@ -19,91 +29,87 @@ func NewStorage(dbPath string) (*Storage, error) {
if err := s.applyMigrations(); err != nil {
return nil, err
}
+
return s, nil
}
func (s *Storage) applyMigrations() error {
- // create new table on database
var err error
_, err = s.db.Exec("CREATE TABLE IF NOT EXISTS seen (url TEXT, date TEXT, summary TEXT)")
-
if err != nil {
return err
}
- err = s.addSummaryColumnIfNotExists()
- if err != nil {
+
+ if err := s.addTextColumnIfNotExists("seen", "summary"); err != nil {
return err
}
- if err := s.addNotificationsTableIfNotExists(); err != nil {
+ if err := s.addTextColumnIfNotExists("seen", "title"); err != nil {
+ return err
+ }
+ if err := s.addTextColumnIfNotExists("seen", "feed_title"); err != nil {
return err
}
+ if err := s.addNotificationsTableIfNotExists(); err != nil {
+ return err
+ }
return nil
}
-func (s *Storage) addNotificationsTableIfNotExists() error {
- _, err := s.db.Exec(`
- CREATE TABLE IF NOT EXISTS notifications (
- feed TEXT,
- date TEXT,
- notified INTEGER DEFAULT 0,
- PRIMARY KEY (feed, date)
- )
- `)
- return err
-}
-
-func (s *Storage) addSummaryColumnIfNotExists() error {
+// Generic helper to add columns
+func (s *Storage) addTextColumnIfNotExists(table, colName string) error {
tx, err := s.db.Begin()
if err != nil {
return err
}
defer tx.Rollback()
- // Check if the 'summary' column already exists in the 'seen' table
- rows, err := tx.Query("PRAGMA table_info(seen)")
+ rows, err := tx.Query(fmt.Sprintf("PRAGMA table_info(%s)", table))
if err != nil {
return err
}
defer rows.Close()
- var columnExists bool
+ exists := false
for rows.Next() {
var cid int
- var name string
- var dataType string
- var notNull int
- var dfltValue interface{}
- var pk int
- err := rows.Scan(&cid, &name, &dataType, ¬Null, &dfltValue, &pk)
- if err != nil {
+ var name, ctype string
+ var notNull, pk int
+ var dflt interface{}
+ if err := rows.Scan(&cid, &name, &ctype, ¬Null, &dflt, &pk); err != nil {
return err
}
- if name == "summary" {
- columnExists = true
+ if name == colName {
+ exists = true
break
}
}
- // If the 'summary' column doesn't exist, add it to the table
- if !columnExists {
- _, err = tx.Exec("ALTER TABLE seen ADD COLUMN summary TEXT")
+ if !exists {
+ _, err = tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s TEXT", table, colName))
if err != nil {
return err
}
}
+ return tx.Commit()
+}
- // Commit the transaction
- err = tx.Commit()
- if err != nil {
- return err
- }
- return nil
+func (s *Storage) addNotificationsTableIfNotExists() error {
+ _, err := s.db.Exec(`
+ CREATE TABLE IF NOT EXISTS notifications (
+ feed TEXT,
+ date TEXT,
+ notified INTEGER DEFAULT 0,
+ PRIMARY KEY (feed, date)
+ )
+ `)
+ return err
}
-func (s *Storage) MarkAsSeen(url, summary string) error {
+func (s *Storage) MarkAsSeen(url, summary, title, feedTitle, feedURL string) error {
today := time.Now().Format("2006-01-02")
- _, err := s.db.Exec("INSERT INTO seen(url, date, summary) values(?,?,?)", url, today, summary)
+ _, err := s.db.Exec("INSERT INTO seen(url, date, summary, title, feed_title) values(?,?,?,?,?)",
+ url, today, summary, title, feedTitle)
return err
}
@@ -123,6 +129,44 @@ func (s *Storage) IsSeen(link string) (bool, bool, string) {
return isSeen, isSeenToday, summary.String
}
+func (s *Storage) GetAllArticles() (map[string][]ArchivedItem, error) {
+ // Order by Date DESC, then by Feed Title so they group nicely
+ rows, err := s.db.Query("SELECT url, date, summary, IFNULL(title, ''), IFNULL(feed_title, '') FROM seen ORDER BY date DESC, feed_title ASC")
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ grouped := make(map[string][]ArchivedItem)
+
+ for rows.Next() {
+ var item ArchivedItem
+ var date, summary, title, feedTitle sql.NullString
+ if err := rows.Scan(&item.URL, &date, &summary, &title, &feedTitle); err != nil {
+ return nil, err
+ }
+
+ item.Date = date.String
+ item.Summary = summary.String
+ item.Title = title.String
+ item.FeedTitle = feedTitle.String
+
+ // Fallback for old records where title might be empty
+ if item.Title == "" {
+ item.Title = item.URL
+ }
+ if item.FeedTitle == "" {
+ item.FeedTitle = "Unknown Feed"
+ }
+ if item.Date == "" {
+ item.Date = "Unknown Date"
+ }
+
+ grouped[item.Date] = append(grouped[item.Date], item)
+ }
+ return grouped, nil
+}
+
func (s *Storage) Close() {
s.db.Close()
}
@@ -132,9 +176,9 @@ func (s *Storage) MarkFeedNotified(feed string) error {
today := time.Now().Format("2006-01-02")
// Upsert: insert or replace into notifications
_, err := s.db.Exec(`
- INSERT INTO notifications(feed, date, notified) VALUES(?, ?, 1)
- ON CONFLICT(feed, date) DO UPDATE SET notified = 1
- `, feed, today)
+ INSERT INTO notifications(feed, date, notified) VALUES(?, ?, 1)
+ ON CONFLICT(feed, date) DO UPDATE SET notified = 1
+ `, feed, today)
return err
}
@@ -149,3 +193,49 @@ func (s *Storage) WasFeedNotifiedToday(feed string) bool {
}
return notified == 1
}
+
+func (s *Storage) GetAllDays() ([]string, error) {
+ rows, err := s.db.Query(`
+ SELECT DISTINCT date
+ FROM seen
+ ORDER BY date ASC
+ `)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ var days []string
+ for rows.Next() {
+ var d string
+ if err := rows.Scan(&d); err == nil {
+ days = append(days, d)
+ }
+ }
+ return days, nil
+}
+
+func (s *Storage) GetArticlesForDay(day string) ([]SeenArticle, error) {
+ rows, err := s.db.Query(`
+ SELECT url, summary
+ FROM seen
+ WHERE date = ?
+ `, day)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ var res []SeenArticle
+ for rows.Next() {
+ var a SeenArticle
+ rows.Scan(&a.URL, &a.Summary)
+ res = append(res, a)
+ }
+ return res, nil
+}
+
+type SeenArticle struct {
+ URL string
+ Summary string
+}
diff --git a/processor.go b/processor.go
index 0828f33..3114ccc 100644
--- a/processor.go
+++ b/processor.go
@@ -3,7 +3,9 @@ package main
import (
"fmt"
"log"
+ "os"
"regexp"
+ "sort"
"strconv"
"strings"
"time"
@@ -37,14 +39,12 @@ func ProcessFeed(rss RSS, cfg *Config, store *Storage, llm *LLMClient, w Writer,
}
itemsFound = true
-
title := item.Title
if title == "" {
title = stripHtmlRegex(item.Description)
}
summary := prevSummary
-
if summary == "" && rss.summarize {
summary = getSummary(llm, item, cfg)
}
@@ -76,7 +76,7 @@ func ProcessFeed(rss RSS, cfg *Config, store *Storage, llm *LLMClient, w Writer,
}
if !seenToday {
- store.MarkAsSeen(item.Link, summary)
+ store.MarkAsSeen(item.Link, summary, item.Title, feed.Title, feed.FeedLink)
}
}
@@ -94,12 +94,9 @@ func getSummary(llm *LLMClient, item *gofeed.Item, cfg *Config) string {
if err == nil {
content = scrapedText.TextContent
}
-
fmt.Println("we have crawled")
-
return llm.Summarize(content)
}
-
return item.Description
}
@@ -142,13 +139,16 @@ func extractImageTagFromHTML(htmlText string) string {
if width != "" && height != "" {
wInt, _ := strconv.Atoi(width)
hInt, _ := strconv.Atoi(height)
+
if wInt > 0 && hInt > 0 {
aspectRatio := float64(wInt) / float64(hInt)
const maxWidth = 400
+
if wInt > maxWidth {
wInt = maxWidth
hInt = int(float64(wInt) / aspectRatio)
}
+
firstImgTag.SetAttr("width", fmt.Sprintf("%d", wInt))
firstImgTag.SetAttr("height", fmt.Sprintf("%d", hInt))
}
@@ -158,13 +158,14 @@ func extractImageTagFromHTML(htmlText string) string {
if err != nil {
return ""
}
+
return html
}
func formatHackerNewsLinks(w Writer, item *gofeed.Item) string {
desc := item.Description
-
commentsURL := ""
+
if start := strings.Index(desc, "Comments URL"); start != -1 {
safeStart := start + 23
if safeStart+45 < len(desc) {
@@ -203,3 +204,66 @@ func stripHtmlRegex(s string) string {
r := regexp.MustCompile(regex)
return r.ReplaceAllString(s, "")
}
+
+func runGenerateAll(cfg *Config, store *Storage) {
+ fmt.Println("Regenerating all daily digests from database...")
+
+ allData, err := store.GetAllArticles()
+ if err != nil {
+ log.Fatalf("Error querying database: %v", err)
+ }
+
+ var dates []string
+ for d := range allData {
+ dates = append(dates, d)
+ }
+ sort.Strings(dates) // 2023-01-01, 2023-01-02...
+
+ for _, date := range dates {
+ items := allData[date]
+ fmt.Printf("Processing %s (%d articles)... \n", date, len(items))
+
+ mw := NewMarkdownWriter(cfg, date)
+ os.Remove(mw.FilePath)
+
+ // Group items by Feed Title so we can create sections
+ feeds := make(map[string][]ArchivedItem)
+ var feedOrder []string // To keep consistent order
+
+ for _, item := range items {
+ if _, exists := feeds[item.FeedTitle]; !exists {
+ feedOrder = append(feedOrder, item.FeedTitle)
+ }
+ feeds[item.FeedTitle] = append(feeds[item.FeedTitle], item)
+ }
+ sort.Strings(feedOrder)
+
+ for _, feedTitle := range feedOrder {
+ feedItems := feeds[feedTitle]
+
+ // Write Feed Header (using the first item's feed URL for favicon)
+ firstItem := feedItems[0]
+ mw.Write(mw.WriteFeedHeaderRaw(feedTitle, firstItem.URL))
+
+ for _, item := range feedItems {
+ var line string
+
+ // Instapaper Icon (if enabled)
+ if cfg.Instapaper {
+ instapaperURL := fmt.Sprintf("https://www.instapaper.com/hello2?url=%s", item.URL)
+ line += fmt.Sprintf(`[
](%s)`, instapaperURL)
+ }
+
+ // Article Title Link
+ line += fmt.Sprintf("[%s](%s) \n", item.Title, item.URL)
+ mw.Write(line)
+
+ // Summary (if exists)
+ if item.Summary != "" {
+ mw.Write(mw.WriteSummary(item.Summary, true))
+ }
+ }
+ }
+ }
+ fmt.Println("Done.")
+}