Add paddock briefing feed ingestion

2026-08-07 11:54:59 -04:00 · 2026-05-25 12:30:21 -04:00
parent 84a8827244
commit ee88a07aa1
5 changed files with 686 additions and 1 deletions
--- a/cmd/main.go
+++ b/cmd/main.go
@@ -1,14 +1,17 @@
 package main

 import (
+	"context"
 	"flag"
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"time"

 	"github.com/AmanTahiliani/box-box/internal/api"
 	"github.com/AmanTahiliani/box-box/internal/ingest"
+	"github.com/AmanTahiliani/box-box/internal/news"
 	"github.com/AmanTahiliani/box-box/internal/store"
 	"github.com/AmanTahiliani/box-box/internal/ui"
 	"github.com/AmanTahiliani/box-box/internal/web"
@@ -21,6 +24,7 @@ func main() {
 	ingestYear := flag.Int("ingest-year", 0, "Ingest OpenF1 meetings for a season year")
 	ingestMeeting := flag.Int("ingest-meeting", 0, "Ingest meeting metadata and Race Hub datasets for all sessions")
 	ingestSession := flag.Int("ingest-session", 0, "Ingest Race Hub datasets for a session key")
+	ingestNews := flag.Bool("ingest-news", false, "Refresh RSS/Atom paddock briefing feeds")
 	dryRun := flag.Bool("dry-run", false, "Preview ingestion without writing domain rows")
 	dbPath := flag.String("db", "", "Domain database path (default: ~/.local/share/box-box/boxbox.db)")
 	flag.Parse()
@@ -46,11 +50,21 @@ func main() {
 	if *ingestSession != 0 {
 		ingestFlags++
 	}
+	if *ingestNews {
+		ingestFlags++
+	}
 	if ingestFlags > 0 {
 		if ingestFlags > 1 {
-			fmt.Fprintln(os.Stderr, "box-box: only one of --ingest-year, --ingest-meeting, or --ingest-session may be set")
+			fmt.Fprintln(os.Stderr, "box-box: only one of --ingest-year, --ingest-meeting, --ingest-session, or --ingest-news may be set")
 			os.Exit(1)
 		}
+		if *ingestNews {
+			if err := runNewsIngestion(*dryRun, *dbPath); err != nil {
+				fmt.Fprintf(os.Stderr, "box-box ingest error: %v\n", err)
+				os.Exit(1)
+			}
+			return
+		}
 		if err := runIngestion(client, *ingestYear, *ingestMeeting, *ingestSession, *dryRun, *dbPath); err != nil {
 			fmt.Fprintf(os.Stderr, "box-box ingest error: %v\n", err)
 			os.Exit(1)
@@ -129,3 +143,45 @@ func runIngestion(client *api.OpenF1Client, year, meetingKey, sessionKey int, dr
 	}
 	return err
 }
+
+func runNewsIngestion(dryRun bool, dbPath string) error {
+	log.SetOutput(os.Stderr)
+
+	path := dbPath
+	if path == "" {
+		path = store.DefaultDBPath()
+	}
+
+	var st *store.Store
+	if !dryRun {
+		var err error
+		st, err = store.Open(path)
+		if err != nil {
+			return fmt.Errorf("open domain database: %w", err)
+		}
+		defer st.Close()
+	}
+
+	if dryRun {
+		fmt.Fprintf(os.Stderr, "news: dry run, not writing to %s\n", path)
+	} else {
+		fmt.Fprintf(os.Stderr, "news: refreshing feeds into %s\n", path)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	result, err := news.Refresh(ctx, st, news.RefreshOptions{
+		Client:   &http.Client{Timeout: 10 * time.Second},
+		DryRun:   dryRun,
+		Progress: os.Stderr,
+	})
+	fmt.Fprintf(
+		os.Stderr,
+		"news: %d source(s) fetched, %d failed, %d item(s) fetched, %d upserted\n",
+		result.SourcesFetched,
+		result.SourcesFailed,
+		result.ItemsFetched,
+		result.ItemsUpserted,
+	)
+	return err
+}
--- a/documentations/refactor/28-orchestrator-handoff.md
+++ b/documentations/refactor/28-orchestrator-handoff.md
@@ -0,0 +1,292 @@
+# Orchestrator Handoff
+
+You are taking over as the primary orchestration/coding agent for the box-box refactor.
+
+Repo:
+
+```text
+/Users/aman/HomeBase/Programming/Projects/box-box
+```
+
+Branch:
+
+```text
+box-refactor
+```
+
+## Role
+
+You are the engineering manager/orchestrator. Inspect before acting, keep changes scoped, review agent work before committing, prune stale docs after phases, and commit cleanly after each accepted phase. The user prefers Cursor for backend/test/hardening work and Claude for major frontend/product/design work, but you may implement directly when appropriate.
+
+## Operating Rules
+
+- Do not rush into implementation if the user wants to discuss.
+- If implementing, keep phases small and commit-ready.
+- Commit after each completed/reviewed phase.
+- Never revert user/other-agent changes without explicit permission.
+- Use `rg` for searches.
+- Use `apply_patch` for manual edits.
+- For frontend work, run browser or Playwright verification where practical.
+- For review requests, lead with findings and file/line references.
+- `frontend/dist` is ignored and should not be committed.
+- Preserve TUI live mode and official F1 SignalR live behavior carefully.
+
+## Project Direction
+
+box-box started as a Go Bubble Tea F1 TUI backed mostly by OpenF1. The refactor direction is now:
+
+- Web UI is the primary product surface.
+- React + TypeScript frontend is the production Web UI stack.
+- Go backend remains the API/server.
+- Historical/completed-session data should be local-first from SQLite.
+- OpenF1 ingestion is explicit via CLI, not fetched live on every page load.
+- Official F1 SignalR remains the live source.
+- Desired product feel: clean, dense, technical F1 operations room. Avoid card-heavy AI-slop.
+
+## Important Commands
+
+```bash
+git status --short
+git log --oneline -10
+
+go test ./...
+npm --prefix frontend test -- --run
+npm --prefix frontend run build
+npm run test:e2e
+npm run test:e2e:prod
+npm run test:visual
+npm run test:visual:prod
+```
+
+Known test note: do not run Playwright suites that share the same seeded SQLite DB in parallel. Run prod E2E and prod visual sequentially, or they may hit `database is locked`.
+
+## Recent Commits
+
+- `571edb9 Add visual regression coverage`
+- `e539abc Add command center screen`
+- `9532206 Polish live timing UI`
+- `a0f135a Update operator documentation`
+- `79b0b9f Rework command center for race weekends`
+- `84a8827 Add paddock briefing RSS backend spike`
+
+## Current Uncommitted Work
+
+Two phases are currently uncommitted and need review/integration. Review and
+commit Phase 19 first, then Phase 20, or stage hunks carefully if separating the
+shared README/refactor README edits.
+
+### Phase 19: Weekend Workspace / Race Hub Flow V1
+
+Claude has completed Phase 19. It is currently uncommitted and needs review.
+
+Claude reported these Phase 19 changes:
+
+- New:
+  - `frontend/src/components/OverviewView.tsx`
+  - `frontend/src/components/WeekendSwitcher.tsx`
+  - `frontend/src/test/RaceHubPage.test.tsx`
+  - `documentations/refactor/26-phase-19-weekend-workspace.md`
+- Modified:
+  - `frontend/src/pages/RaceHubPage.tsx`
+  - `frontend/src/components/TabBar.tsx`
+  - `frontend/src/components/DatasetStatusView.tsx`
+  - `frontend/src/styles/app.css`
+  - tests for TabBar, DatasetStatusView, race-hub, command-center, data-library, production-smoke
+  - `tests/visual/helpers.ts`
+  - race-hub visual snapshots
+  - root `README.md`
+  - `documentations/refactor/README.md`
+
+Claude reported these UX changes:
+
+- Race Hub is now a Weekend Workspace.
+- Compact GP identity band with country decal/accent strip.
+- Horizontal session rail replaces “big table then analysis below.”
+- Active session context stays visible above tabs.
+- Tabs regrouped into Overview, Race Story, Strategy, Lap Data, Conditions, Race Control, Data Status.
+- New Overview tab with winner/pole/fastest/podium cards, condition chips, latest race control, and local coverage meter.
+- Inline Switch Weekend panel replaces legacy LocalDataNavigator on Race Hub.
+- Data Status links to `/admin`; no CLI/admin text on fan surface.
+- Mobile/iPad improved with wrapping identity band, horizontal session rail, single-column stats.
+- `/race-hub?session_key=9472` still works and loads Bahrain GP 2024 seeded session.
+- Bare `/race-hub` now resolves to a focus weekend/session via `pickFocusMeeting` and navigation replace.
+
+Claude reported these tests:
+
+- `npm --prefix frontend test -- --run` passed, 98 tests.
+- `npm --prefix frontend run build` passed.
+- `npm run test:e2e` passed, 18 tests.
+- `npm run test:e2e:prod` passed, 6 tests.
+- `npm run test:visual` passed, 12 screenshots after regenerating race-hub baselines.
+- `npm run test:visual:prod` passed, 12 screenshots.
+
+### Phase 20: Paddock Briefing Ingestion CLI
+
+A backend subagent implemented Phase 20 after the RSS backend spike. It is also
+currently uncommitted and needs review.
+
+Reported Phase 20 changes:
+
+- Modified:
+  - `cmd/main.go`
+  - `README.md`
+  - `documentations/refactor/README.md`
+- New:
+  - `internal/news/refresh.go`
+  - `internal/news/refresh_test.go`
+  - `documentations/refactor/29-phase-20-paddock-briefing-ingestion.md`
+
+Implemented behavior:
+
+- Adds `--ingest-news` as a CLI mode.
+- Keeps it mutually exclusive with `--ingest-year`, `--ingest-meeting`, and
+  `--ingest-session`.
+- Reuses `--db` for the domain SQLite path.
+- Reuses `--dry-run` to fetch and report feed counts without opening or writing
+  the domain database.
+- Uses `internal/news.Refresh`, which fetches `DefaultSources`, upserts
+  `news_sources`, upserts URL-deduped `news_items`, records `fetched_at` and
+  `expires_at`, and continues through individual feed failures before returning
+  a summary error.
+- Web requests still do not fetch feeds; `/api/v1/news` remains read-only
+  against SQLite.
+
+Commands added:
+
+```bash
+go run ./cmd/main.go --ingest-news
+go run ./cmd/main.go --dry-run --ingest-news
+go run ./cmd/main.go --ingest-news --db /tmp/boxbox.db
+```
+
+Phase 20 verification already run by Codex:
+
+```bash
+go test ./cmd/... ./internal/news ./internal/store
+go test ./internal/web ./internal/query
+go test ./...
+git diff --check
+```
+
+## Immediate Task
+
+Start by reviewing and committing Phase 19. Then review and commit Phase 20.
+Do not start new implementation until both are accepted and committed.
+
+1. Inspect working tree:
+
+```bash
+git status --short
+git diff --stat
+git diff --name-only
+```
+
+2. Review Claude’s Phase 19 work quickly but responsibly:
+
+- Check `RaceHubPage.tsx`, `OverviewView.tsx`, `WeekendSwitcher.tsx`, `TabBar.tsx`, `DatasetStatusView.tsx`, `app.css`, route/test updates, docs.
+- Make sure no admin/CLI guidance leaked back into Race Hub.
+- Make sure `/race-hub?session_key=9472` compatibility is preserved.
+- Make sure `/admin` remains the admin/data-health surface.
+- Confirm visual tests and docs match the changed UX.
+
+3. Run a focused verification pass. At minimum:
+
+```bash
+npm --prefix frontend test -- --run
+npm --prefix frontend run build
+npm run test:e2e
+npm run test:visual
+```
+
+If time allows or if production behavior changed:
+
+```bash
+npm run test:e2e:prod
+npm run test:visual:prod
+```
+
+4. Patch only small issues if found.
+5. Stage only Phase 19 files.
+6. Commit with a message like:
+
+```bash
+git commit -m "Rework Race Hub as weekend workspace"
+```
+
+Then review Phase 20:
+
+1. Check `cmd/main.go`, `internal/news/refresh.go`,
+   `internal/news/refresh_test.go`,
+   `documentations/refactor/29-phase-20-paddock-briefing-ingestion.md`, and the
+   README/refactor README hunks.
+2. Confirm the CLI mode does not interfere with OpenF1 ingestion modes or web/TUI
+   startup.
+3. Confirm no live internet tests were added.
+4. Re-run targeted backend tests if needed:
+
+```bash
+go test ./cmd/... ./internal/news ./internal/store
+go test ./...
+```
+
+5. Stage Phase 20 files/hunks and commit with a message like:
+
+```bash
+git commit -m "Add paddock briefing feed ingestion"
+```
+
+## RSS / Paddock Briefing Context
+
+Cursor completed and Codex committed a backend spike as `84a8827 Add paddock briefing RSS backend spike`.
+
+Implemented:
+
+- `internal/news`: RSS/Atom parser and fetch helper.
+- SQLite tables:
+  - `news_sources`
+  - `news_items`
+- Store/query methods for cached news.
+- Read-only API:
+  - `GET /api/v1/news?limit=25&source=racefans-f1`
+- No request-time network fetching.
+- Unit tests use local XML fixtures.
+
+Recommended feed sources:
+
+- FIA official RSS
+- BBC Sport F1
+- Autosport F1
+- RaceFans F1
+- Guardian Formula One
+
+Optional:
+
+- Motorsport.com
+- RACER
+- Formula 1 YouTube Atom
+
+Avoid:
+
+- Formula1.com scraping/hidden endpoints
+- X/Twitter scraping
+- Reddit as primary source
+- RSS.app/Feedspot as primary source
+
+## Likely Next Phases After Phase 19 And 20
+
+1. Phase 21: Paddock Briefing UI
+   - Claude/frontend.
+   - Add fan-facing briefing module, likely on Command Center first.
+   - Query `/api/v1/news`.
+   - Show source, title, age, category, short feed-provided snippet, external link.
+   - Keep publisher attribution visible.
+   - Avoid full article storage or scraping.
+
+2. Phase 22: Race Story Deepening
+   - Claude/frontend or mixed.
+   - Collapse legacy classification/grid/position components into a more fluid Race Story canvas.
+   - Improve mobile scanning and session narrative.
+
+3. Phase 23: Full Season Backfill / ingest hardening
+   - Cursor/backend.
+   - Safer season workflows, resumability, rate-limit controls, coverage reporting.
--- a/documentations/refactor/29-phase-20-paddock-briefing-ingestion.md
+++ b/documentations/refactor/29-phase-20-paddock-briefing-ingestion.md
@@ -0,0 +1,47 @@
+# Phase 20 Paddock Briefing Ingestion CLI
+
+## Goal
+
+Turn the Phase 19B RSS backend spike into an explicit local refresh command for
+the Paddock Briefing cache. Feed fetching remains a CLI-only operation; web
+requests continue to read SQLite only.
+
+## Implemented
+
+- Added `--ingest-news` as a CLI ingestion mode on `cmd/main.go`.
+- Reused `--db` path behavior from the existing OpenF1 ingestion flows.
+- Reused `--dry-run` to fetch and report feed counts without opening or writing
+  the domain database.
+- Added `internal/news.Refresh`, which:
+  - fetches `internal/news.DefaultSources` unless tests provide a custom list;
+  - uses `internal/news.Fetch` with a 10-second HTTP client timeout;
+  - upserts `news_sources` with `fetched_at` and `expires_at`;
+  - upserts URL-deduped `news_items`;
+  - continues after individual source failures and returns a summary error after
+    successful sources are stored.
+
+## CLI
+
+```bash
+go run ./cmd/main.go --ingest-news
+go run ./cmd/main.go --dry-run --ingest-news
+go run ./cmd/main.go --ingest-news --db /tmp/boxbox.db
+```
+
+`--ingest-news` is mutually exclusive with `--ingest-year`, `--ingest-meeting`,
+and `--ingest-session`.
+
+## Verification
+
+Tests use local `httptest.Server` feeds only. No live internet test is required
+for the refresh logic.
+
+```bash
+go test ./internal/news ./internal/store
+```
+
+## Notes
+
+The refresh TTL is currently 30 minutes for all sources. A future phase can add
+source-specific TTLs, retention/pruning, or admin UI controls without changing
+the read-only `/api/v1/news` contract.
--- a/internal/news/refresh.go
+++ b/internal/news/refresh.go
@@ -0,0 +1,141 @@
+package news
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/AmanTahiliani/box-box/internal/store"
+)
+
+const DefaultTTL = 30 * time.Minute
+
+// Store is the storage surface needed by feed refreshes.
+type Store interface {
+	UpsertNewsSource(store.NewsSource) error
+	UpsertNewsItem(store.NewsItem) error
+}
+
+// RefreshOptions configures one local news refresh run.
+type RefreshOptions struct {
+	Sources  []Source
+	Client   *http.Client
+	TTL      time.Duration
+	DryRun   bool
+	Now      func() time.Time
+	Progress io.Writer
+}
+
+// RefreshResult summarizes one local news refresh run.
+type RefreshResult struct {
+	SourcesFetched int
+	SourcesFailed  int
+	ItemsFetched   int
+	ItemsUpserted  int
+}
+
+// Refresh fetches RSS/Atom sources and stores normalized, URL-deduped items.
+func Refresh(ctx context.Context, st Store, opts RefreshOptions) (RefreshResult, error) {
+	if st == nil && !opts.DryRun {
+		return RefreshResult{}, errors.New("news refresh: store is required")
+	}
+	if len(opts.Sources) == 0 {
+		opts.Sources = DefaultSources
+	}
+	if opts.Client == nil {
+		opts.Client = &http.Client{Timeout: 10 * time.Second}
+	}
+	if opts.TTL <= 0 {
+		opts.TTL = DefaultTTL
+	}
+	now := func() time.Time { return time.Now().UTC() }
+	if opts.Now != nil {
+		now = func() time.Time { return opts.Now().UTC() }
+	}
+
+	var result RefreshResult
+	var failures []string
+	for _, source := range opts.Sources {
+		fetchedAt := now()
+		expiresAt := fetchedAt.Add(opts.TTL)
+		if !opts.DryRun {
+			if err := st.UpsertNewsSource(store.NewsSource{
+				Source:    source.ID,
+				Name:      source.Name,
+				FeedURL:   source.URL,
+				Category:  source.Category,
+				Enabled:   true,
+				UpdatedAt: fetchedAt,
+			}); err != nil {
+				return result, err
+			}
+		}
+		items, err := Fetch(ctx, opts.Client, source)
+		if err != nil {
+			result.SourcesFailed++
+			failures = append(failures, fmt.Sprintf("%s: %v", source.ID, err))
+			progressf(opts.Progress, "news: %s failed: %v\n", source.ID, err)
+			continue
+		}
+
+		result.SourcesFetched++
+		result.ItemsFetched += len(items)
+		progressf(opts.Progress, "news: %s fetched %d items\n", source.ID, len(items))
+		if opts.DryRun {
+			continue
+		}
+
+		if err := st.UpsertNewsSource(store.NewsSource{
+			Source:    source.ID,
+			Name:      source.Name,
+			FeedURL:   source.URL,
+			Category:  source.Category,
+			Enabled:   true,
+			FetchedAt: &fetchedAt,
+			ExpiresAt: &expiresAt,
+			UpdatedAt: fetchedAt,
+		}); err != nil {
+			return result, err
+		}
+		for _, item := range items {
+			item.FetchedAt = fetchedAt
+			publishedAt := timePtr(item.PublishedAt)
+			if err := st.UpsertNewsItem(store.NewsItem{
+				URL:         item.URL,
+				Source:      item.Source,
+				Title:       item.Title,
+				PublishedAt: publishedAt,
+				Summary:     item.Summary,
+				Category:    item.Category,
+				FetchedAt:   item.FetchedAt,
+			}); err != nil {
+				return result, err
+			}
+			result.ItemsUpserted++
+		}
+	}
+
+	if len(failures) > 0 {
+		return result, fmt.Errorf("news refresh completed with %d source failure(s): %s", len(failures), strings.Join(failures, "; "))
+	}
+	return result, nil
+}
+
+func progressf(w io.Writer, format string, args ...any) {
+	if w == nil {
+		return
+	}
+	fmt.Fprintf(w, format, args...)
+}
+
+func timePtr(v time.Time) *time.Time {
+	if v.IsZero() {
+		return nil
+	}
+	t := v.UTC()
+	return &t
+}
--- a/internal/news/refresh_test.go
+++ b/internal/news/refresh_test.go
@@ -0,0 +1,149 @@
+package news
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/AmanTahiliani/box-box/internal/store"
+)
+
+func TestRefreshStoresSourcesAndItems(t *testing.T) {
+	feed := `<?xml version="1.0"?>
+<rss version="2.0">
+  <channel>
+    <item>
+      <title>Briefing one</title>
+      <link>https://example.com/f1/one?utm_source=rss</link>
+      <pubDate>Mon, 25 May 2026 10:00:00 GMT</pubDate>
+      <description>Morning note</description>
+    </item>
+    <item>
+      <title>Briefing duplicate newer</title>
+      <link>https://example.com/f1/one</link>
+      <pubDate>Mon, 25 May 2026 11:00:00 GMT</pubDate>
+      <description>Updated note</description>
+    </item>
+  </channel>
+</rss>`
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if got := r.Header.Get("User-Agent"); got != UserAgent {
+			t.Fatalf("User-Agent = %q, want %q", got, UserAgent)
+		}
+		w.Header().Set("Content-Type", "application/rss+xml")
+		_, _ = w.Write([]byte(feed))
+	}))
+	defer server.Close()
+
+	st := openNewsTestStore(t)
+	now := time.Unix(1800000000, 0).UTC()
+	result, err := Refresh(context.Background(), st, RefreshOptions{
+		Sources: []Source{{
+			ID:       "example",
+			Name:     "Example F1",
+			URL:      server.URL + "/feed.xml",
+			Category: "news",
+		}},
+		Client: server.Client(),
+		TTL:    time.Hour,
+		Now:    func() time.Time { return now },
+	})
+	if err != nil {
+		t.Fatalf("Refresh() error = %v", err)
+	}
+	if result.SourcesFetched != 1 || result.ItemsFetched != 1 || result.ItemsUpserted != 1 || result.SourcesFailed != 0 {
+		t.Fatalf("result = %+v, want one fetched/upserted item and no failures", result)
+	}
+
+	var fetchedAt, expiresAt int64
+	if err := st.DB().QueryRow(`
+		SELECT fetched_at, expires_at
+		FROM news_sources
+		WHERE source = 'example'
+	`).Scan(&fetchedAt, &expiresAt); err != nil {
+		t.Fatalf("query source metadata: %v", err)
+	}
+	if fetchedAt != now.Unix() || expiresAt != now.Add(time.Hour).Unix() {
+		t.Fatalf("source times = %d/%d, want %d/%d", fetchedAt, expiresAt, now.Unix(), now.Add(time.Hour).Unix())
+	}
+
+	items, err := st.ListNewsItems(10, "example")
+	if err != nil {
+		t.Fatalf("ListNewsItems() error = %v", err)
+	}
+	if len(items) != 1 {
+		t.Fatalf("items len = %d, want 1", len(items))
+	}
+	if items[0].URL != "https://example.com/f1/one" || items[0].Title != "Briefing duplicate newer" {
+		t.Fatalf("stored item = %+v, want canonical newer duplicate", items[0])
+	}
+	if !items[0].FetchedAt.Equal(now) {
+		t.Fatalf("item fetched_at = %v, want %v", items[0].FetchedAt, now)
+	}
+}
+
+func TestRefreshDryRunDoesNotRequireStore(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte(`<?xml version="1.0"?><rss version="2.0"><channel><item><title>Dry</title><link>https://example.com/dry</link></item></channel></rss>`))
+	}))
+	defer server.Close()
+
+	result, err := Refresh(context.Background(), nil, RefreshOptions{
+		Sources: []Source{{ID: "dry", Name: "Dry", URL: server.URL, Category: "news"}},
+		Client:  server.Client(),
+		DryRun:  true,
+	})
+	if err != nil {
+		t.Fatalf("Refresh() dry run error = %v", err)
+	}
+	if result.ItemsFetched != 1 || result.ItemsUpserted != 0 {
+		t.Fatalf("result = %+v, want fetched item with no upsert", result)
+	}
+}
+
+func TestRefreshContinuesAfterSourceFailure(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.Contains(r.URL.Path, "bad") {
+			http.Error(w, "nope", http.StatusBadGateway)
+			return
+		}
+		_, _ = w.Write([]byte(`<?xml version="1.0"?><rss version="2.0"><channel><item><title>Good</title><link>https://example.com/good</link></item></channel></rss>`))
+	}))
+	defer server.Close()
+
+	st := openNewsTestStore(t)
+	result, err := Refresh(context.Background(), st, RefreshOptions{
+		Sources: []Source{
+			{ID: "bad", Name: "Bad", URL: server.URL + "/bad", Category: "news"},
+			{ID: "good", Name: "Good", URL: server.URL + "/good", Category: "news"},
+		},
+		Client: server.Client(),
+	})
+	if err == nil {
+		t.Fatal("Refresh() error = nil, want source failure")
+	}
+	if result.SourcesFetched != 1 || result.SourcesFailed != 1 || result.ItemsUpserted != 1 {
+		t.Fatalf("result = %+v, want one failure and one stored item", result)
+	}
+	items, err := st.ListNewsItems(10, "")
+	if err != nil {
+		t.Fatalf("ListNewsItems() error = %v", err)
+	}
+	if len(items) != 1 || items[0].Source != "good" {
+		t.Fatalf("items = %+v, want good source item stored", items)
+	}
+}
+
+func openNewsTestStore(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.Open(filepath.Join(t.TempDir(), "news.db"))
+	if err != nil {
+		t.Fatalf("store.Open() error = %v", err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+	return st
+}