diff --git a/database.go b/database.go index 5500c66..ea6bbde 100644 --- a/database.go +++ b/database.go @@ -111,6 +111,30 @@ func (ndb newsDatabase) initFrontpageDB() error { ); `, ` + CREATE TABLE IF NOT EXISTS latest_story_stats ( + id INTEGER PRIMARY KEY, + cumulativeUpvotes INTEGER NOT NULL, + cumulativeExpectedUpvotes REAL NOT NULL, + score INTEGER NOT NULL, + highestTopRank INTEGER, + highestBestRank INTEGER, + highestAskRank INTEGER, + highestShowRank INTEGER, + highestQNRank INTEGER, + flagged BOOLEAN NOT NULL DEFAULT FALSE, + dupe BOOLEAN NOT NULL DEFAULT FALSE + ); + `, + ` + CREATE INDEX IF NOT EXISTS latest_story_stats_score_idx ON latest_story_stats(score); + `, + ` + CREATE INDEX IF NOT EXISTS latest_story_stats_upvotes_idx ON latest_story_stats(cumulativeUpvotes); + `, + ` + CREATE INDEX IF NOT EXISTS latest_story_stats_expected_upvotes_idx ON latest_story_stats(cumulativeExpectedUpvotes); + `, + ` CREATE INDEX IF NOT EXISTS dataset_sampletime_id ON dataset(sampletime, id); `, @@ -141,7 +165,6 @@ func (ndb newsDatabase) initFrontpageDB() error { `alter table stories add column archived boolean default false not null`, `DROP INDEX if exists archived`, `CREATE INDEX IF NOT EXISTS dataset_sampletime on dataset(sampletime)`, - `update dataset set upvoteRate = ( cumulativeUpvotes + 2.3 ) / ( cumulativeExpectedUpvotes + 2.3) where upvoteRate = 0`, } @@ -420,10 +443,10 @@ func (ndb newsDatabase) purgeStory(ctx context.Context, storyID int) error { } // Delete story record - _, err = tx.ExecContext(ctx, `DELETE FROM stories WHERE id = ?`, storyID) - if err != nil { - return errors.Wrap(err, "delete from stories") - } + // _, err = tx.ExecContext(ctx, `DELETE FROM stories WHERE id = ?`, storyID) + // if err != nil { + // return errors.Wrap(err, "delete from stories") + // } return nil } @@ -442,24 +465,54 @@ func (ndb newsDatabase) selectStoryDetails(id int) (Story, error) { , unixepoch() - sampleTime + coalesce(ageApprox, sampleTime - submissionTime) , score , descendants - , cumulativeUpvotes - , cumulativeExpectedUpvotes + , lst.cumulativeUpvotes + , lst.cumulativeExpectedUpvotes , topRank , qnRank , rawRank - , flagged - , dupe + , lst.highestTopRank + , lst.highestBestRank + , lst.highestAskRank + , lst.highestShowRank + , lst.highestQNRank + , lst.flagged + , lst.dupe , job , archived from stories JOIN dataset USING (id) + JOIN latest_story_stats lst + USING (id) WHERE id = ? ORDER BY sampleTime DESC LIMIT 1 ` - err := ndb.db.QueryRow(sqlStatement, id).Scan(&s.ID, &s.By, &s.Title, &s.URL, &s.SubmissionTime, &s.OriginalSubmissionTime, &s.AgeApprox, &s.Score, &s.Comments, &s.CumulativeUpvotes, &s.CumulativeExpectedUpvotes, &s.TopRank, &s.QNRank, &s.RawRank, &s.Flagged, &s.Dupe, &s.Job, &s.Archived) + err := ndb.db.QueryRow(sqlStatement, id).Scan( + &s.ID, + &s.By, + &s.Title, + &s.URL, + &s.SubmissionTime, + &s.OriginalSubmissionTime, + &s.AgeApprox, + &s.Score, + &s.Comments, + &s.CumulativeUpvotes, + &s.CumulativeExpectedUpvotes, + &s.TopRank, + &s.QNRank, + &s.RawRank, + &s.HighestTopRank, + &s.HighestBestRank, + &s.HighestAskRank, + &s.HighestShowRank, + &s.HighestQNRank, + &s.Flagged, + &s.Dupe, + &s.Job, + &s.Archived) if err != nil { return s, err } @@ -567,19 +620,39 @@ func (ndb newsDatabase) getDatabaseStats() (size int64, freelist int64, fragment func (ndb newsDatabase) deleteOldData(ctx context.Context) (int64, error) { // Delete data older than one month. Stories whose first datapoint is more than 21 days old with score greater than 2 are already being archived. So this // should delete what's left -- as long as archiving is working! - sqlStatement := ` - delete from dataset where sampleTime <= unixepoch()-30*24*60*60 - ` + var rowsAffected int64 = 0 + { + sqlStatement := ` + delete from dataset where sampleTime <= unixepoch()-30*24*60*60 + ` - result, err := ndb.db.ExecContext(ctx, sqlStatement) - if err != nil { - return 0, errors.Wrap(err, "executing delete old data query") - } + result, err := ndb.db.ExecContext(ctx, sqlStatement) + if err != nil { + return 0, errors.Wrap(err, "executing delete old data query") + } - rowsAffected, err := result.RowsAffected() - if err != nil { - return 0, errors.Wrap(err, "getting rows affected") + rowsAffected, err = result.RowsAffected() + if err != nil { + return 0, errors.Wrap(err, "getting number of rows of data deleted") + } } + // { + // sqlStatement := ` + // delete from stories where archived = 0 and timestamp < unixepoch()-30*24*60*60 + // ` + + // result, err := ndb.db.ExecContext(ctx, sqlStatement) + // if err != nil { + // return 0, errors.Wrap(err, "executing delete old stories query") + // } + + // rowsAffectedStories, err := result.RowsAffected() + // if err != nil { + // return 0, errors.Wrap(err, "getting number of stories deleted") + // } + // rowsAffected += rowsAffectedStories + // } + return rowsAffected, nil } diff --git a/postprocessing.go b/postprocessing.go index e04c486..8a24f0f 100644 --- a/postprocessing.go +++ b/postprocessing.go @@ -45,6 +45,7 @@ func (app app) crawlPostprocess(ctx context.Context, tx *sql.Tx) error { "previous-crawl.sql", "resubmissions.sql", "raw-ranks.sql", + "latest-story-stats.sql", } { app.logger.Info("Processing SQL file", slog.String("filename", filename)) err = executeSQLFile(ctx, tx, filename) diff --git a/story-details.go b/story-details.go index b7279cf..90fbf01 100644 --- a/story-details.go +++ b/story-details.go @@ -28,6 +28,11 @@ type Story struct { TopRank sql.NullInt32 QNRank sql.NullInt32 RawRank sql.NullInt32 + HighestTopRank sql.NullInt32 + HighestBestRank sql.NullInt32 + HighestAskRank sql.NullInt32 + HighestShowRank sql.NullInt32 + HighestQNRank sql.NullInt32 Job bool Flagged bool Dupe bool