feat(backend): implement session coverage, resumable ingestion, deep year backfill, backoff rate-limit, and dynamic cache TTLs

internal/store/coverage.go

@@ -0,0 +1,117 @@
+package store
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+// CoverageEntry represents the coverage status for a single dataset of a session.
+type CoverageEntry struct {
+	Status    string
+	ErrorMsg  string
+	RowCount  int
+	UpdatedAt string
+}
+
+// SessionCoverageRow represents a joined session coverage record for reporting.
+type SessionCoverageRow struct {
+	MeetingKey  int
+	MeetingName string
+	SessionKey  int
+	SessionName string
+	Dataset     string
+	Status      string
+	ErrorMsg    string
+	RowCount    int
+	UpdatedAt   string
+}
+
+// UpsertCoverage inserts or updates a session coverage record.
+func (s *Store) UpsertCoverage(sessionKey int, dataset string, status string, rowCount int, errMsg string) error {
+	_, err := s.db.Exec(`
+		INSERT INTO session_coverage (session_key, dataset, status, row_count, error_msg, updated_at)
+		VALUES (?, ?, ?, ?, ?, datetime('now'))
+		ON CONFLICT(session_key, dataset) DO UPDATE SET
+			status = excluded.status,
+			row_count = excluded.row_count,
+			error_msg = excluded.error_msg,
+			updated_at = excluded.updated_at
+	`, sessionKey, dataset, status, rowCount, nullString(errMsg))
+	if err != nil {
+		return fmt.Errorf("upsert coverage: %w", err)
+	}
+	return nil
+}
+
+// GetSessionCoverage fetches the coverage statuses for all datasets of a given session.
+func (s *Store) GetSessionCoverage(sessionKey int) (map[string]CoverageEntry, error) {
+	rows, err := s.db.Query(`
+		SELECT dataset, status, row_count, error_msg, updated_at
+		FROM session_coverage
+		WHERE session_key = ?
+	`, sessionKey)
+	if err != nil {
+		return nil, fmt.Errorf("get session coverage: %w", err)
+	}
+	defer rows.Close()
+
+	coverage := make(map[string]CoverageEntry)
+	for rows.Next() {
+		var dataset string
+		var entry CoverageEntry
+		var errMsg sql.NullString
+		if err := rows.Scan(&dataset, &entry.Status, &entry.RowCount, &errMsg, &entry.UpdatedAt); err != nil {
+			return nil, fmt.Errorf("scan session coverage: %w", err)
+		}
+		entry.ErrorMsg = errMsg.String
+		coverage[dataset] = entry
+	}
+	return coverage, rows.Err()
+}
+
+// GetSeasonCoverage returns coverage records for all sessions of a given year.
+func (s *Store) GetSeasonCoverage(year int) ([]SessionCoverageRow, error) {
+	rows, err := s.db.Query(`
+		SELECT
+			m.meeting_key,
+			m.meeting_name,
+			s.session_key,
+			s.session_name,
+			COALESCE(c.dataset, '') as dataset,
+			COALESCE(c.status, 'pending') as status,
+			COALESCE(c.error_msg, '') as error_msg,
+			COALESCE(c.row_count, 0) as row_count,
+			COALESCE(c.updated_at, '') as updated_at
+		FROM sessions s
+		JOIN meetings m ON s.meeting_key = m.meeting_key
+		LEFT JOIN session_coverage c ON s.session_key = c.session_key
+		WHERE m.year = ?
+		ORDER BY m.date_start ASC, m.meeting_key ASC, s.date_start ASC, s.session_key ASC
+	`, year)
+	if err != nil {
+		return nil, fmt.Errorf("get season coverage query: %w", err)
+	}
+	defer rows.Close()
+
+	var coverageRows []SessionCoverageRow
+	for rows.Next() {
+		var r SessionCoverageRow
+		var errMsg sql.NullString
+		if err := rows.Scan(
+			&r.MeetingKey,
+			&r.MeetingName,
+			&r.SessionKey,
+			&r.SessionName,
+			&r.Dataset,
+			&r.Status,
+			&r.ErrorMsg,
+			&r.RowCount,
+			&r.UpdatedAt,
+		); err != nil {
+			return nil, fmt.Errorf("scan season coverage row: %w", err)
+		}
+		r.ErrorMsg = errMsg.String
+		coverageRows = append(coverageRows, r)
+	}
+	return coverageRows, rows.Err()
+}

internal/store/coverage_test.go

@@ -0,0 +1,134 @@
+package store
+
+import (
+	"testing"
+)
+
+func TestCoverageCRUD(t *testing.T) {
+	s := openTestStore(t)
+
+	// Verify schema migration version is 4 (since we added 004_coverage.sql)
+	version, err := s.SchemaVersion()
+	if err != nil {
+		t.Fatalf("SchemaVersion() error = %v", err)
+	}
+	if version != 4 {
+		t.Fatalf("SchemaVersion() = %d, want 4", version)
+	}
+
+	// Verify session_coverage table exists
+	var tableName string
+	err = s.db.QueryRow(
+		`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session_coverage'`,
+	).Scan(&tableName)
+	if err != nil {
+		t.Fatalf("session_coverage table is missing: %v", err)
+	}
+
+	// Insert meeting and session to test GetSeasonCoverage joins
+	meetingKey := 1234
+	sessionKey := 5678
+	year := 2024
+
+	if err := s.UpsertMeeting(Meeting{
+		MeetingKey:  meetingKey,
+		MeetingName: "Test Grand Prix",
+		Year:        year,
+		DateStart:   "2024-03-01T12:00:00Z",
+	}); err != nil {
+		t.Fatalf("UpsertMeeting() error = %v", err)
+	}
+
+	if err := s.UpsertSession(Session{
+		SessionKey:  sessionKey,
+		MeetingKey:  meetingKey,
+		SessionName: "Qualifying",
+		SessionType: "Qualifying",
+		DateStart:   "2024-03-02T14:00:00Z",
+	}); err != nil {
+		t.Fatalf("UpsertSession() error = %v", err)
+	}
+
+	// 1. Test empty session coverage
+	cov, err := s.GetSessionCoverage(sessionKey)
+	if err != nil {
+		t.Fatalf("GetSessionCoverage() error = %v", err)
+	}
+	if len(cov) != 0 {
+		t.Fatalf("Expected empty coverage, got: %v", cov)
+	}
+
+	// 2. Upsert multiple datasets
+	err = s.UpsertCoverage(sessionKey, "drivers", "complete", 20, "")
+	if err != nil {
+		t.Fatalf("UpsertCoverage() drivers error = %v", err)
+	}
+	err = s.UpsertCoverage(sessionKey, "laps", "failed", 0, "429 Rate Limit")
+	if err != nil {
+		t.Fatalf("UpsertCoverage() laps error = %v", err)
+	}
+
+	// 3. Retrieve session coverage
+	cov, err = s.GetSessionCoverage(sessionKey)
+	if err != nil {
+		t.Fatalf("GetSessionCoverage() error = %v", err)
+	}
+	if len(cov) != 2 {
+		t.Fatalf("Expected coverage size 2, got: %d", len(cov))
+	}
+
+	drv, ok := cov["drivers"]
+	if !ok {
+		t.Fatalf("Expected 'drivers' key to exist in coverage map")
+	}
+	if drv.Status != "complete" || drv.RowCount != 20 || drv.ErrorMsg != "" {
+		t.Fatalf("Unexpected drivers status: %+v", drv)
+	}
+
+	laps, ok := cov["laps"]
+	if !ok {
+		t.Fatalf("Expected 'laps' key to exist in coverage map")
+	}
+	if laps.Status != "failed" || laps.RowCount != 0 || laps.ErrorMsg != "429 Rate Limit" {
+		t.Fatalf("Unexpected laps status: %+v", laps)
+	}
+
+	// 4. Test updates (idempotency/upsert)
+	err = s.UpsertCoverage(sessionKey, "laps", "complete", 150, "")
+	if err != nil {
+		t.Fatalf("UpsertCoverage() second laps error = %v", err)
+	}
+	cov, err = s.GetSessionCoverage(sessionKey)
+	if err != nil {
+		t.Fatalf("GetSessionCoverage() error = %v", err)
+	}
+	laps = cov["laps"]
+	if laps.Status != "complete" || laps.RowCount != 150 || laps.ErrorMsg != "" {
+		t.Fatalf("Expected updated laps status to be complete with 150 rows, got: %+v", laps)
+	}
+
+	// 5. Test GetSeasonCoverage joins
+	seasonRows, err := s.GetSeasonCoverage(year)
+	if err != nil {
+		t.Fatalf("GetSeasonCoverage() error = %v", err)
+	}
+	if len(seasonRows) != 2 {
+		t.Fatalf("Expected 2 season coverage rows, got: %d", len(seasonRows))
+	}
+
+	// The two rows should correspond to drivers and laps datasets
+	for _, row := range seasonRows {
+		if row.MeetingKey != meetingKey || row.MeetingName != "Test Grand Prix" {
+			t.Errorf("Unexpected meeting info: %+v", row)
+		}
+		if row.SessionKey != sessionKey || row.SessionName != "Qualifying" {
+			t.Errorf("Unexpected session info: %+v", row)
+		}
+		if row.Dataset != "drivers" && row.Dataset != "laps" {
+			t.Errorf("Unexpected dataset: %q", row.Dataset)
+		}
+		if row.Status != "complete" {
+			t.Errorf("Expected status to be 'complete', got %q", row.Status)
+		}
+	}
+}

internal/store/migrations/004_coverage.sql

@@ -0,0 +1,9 @@
+CREATE TABLE IF NOT EXISTS session_coverage (
+    session_key  INTEGER NOT NULL,
+    dataset      TEXT    NOT NULL,
+    status       TEXT    NOT NULL DEFAULT 'pending',
+    row_count    INTEGER NOT NULL DEFAULT 0,
+    updated_at   TEXT    NOT NULL DEFAULT (datetime('now')),
+    error_msg    TEXT,
+    PRIMARY KEY (session_key, dataset)
+);

internal/store/store_test.go

@@ -28,8 +28,8 @@ func TestOpenAppliesMigrations(t *testing.T) {
 	if err != nil {
 		t.Fatalf("SchemaVersion() error = %v", err)
 	}
-	if version != 3 {
-		t.Fatalf("SchemaVersion() = %d, want 3", version)
+	if version != 4 {
+		t.Fatalf("SchemaVersion() = %d, want 4", version)
 	}
 
 	tables := []string{
@@ -50,6 +50,7 @@ func TestOpenAppliesMigrations(t *testing.T) {
 		"laps",
 		"news_sources",
 		"news_items",
+		"session_coverage",
 	}
 	for _, table := range tables {
 		var name string
@@ -92,6 +93,12 @@ func TestMigrationsAreIdempotent(t *testing.T) {
 	if count != 1 {
 		t.Fatalf("schema_migrations v3 count = %d, want 1", count)
 	}
+	if err := s.db.QueryRow(`SELECT COUNT(*) FROM schema_migrations WHERE version = 4`).Scan(&count); err != nil {
+		t.Fatalf("count schema_migrations v4: %v", err)
+	}
+	if count != 1 {
+		t.Fatalf("schema_migrations v4 count = %d, want 1", count)
+	}
 }
 
 func TestRawPayloadInsertAndRead(t *testing.T) {