diff --git a/cmd/main.go b/cmd/main.go index 50a3771..5c07232 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,14 +1,17 @@ package main import ( + "context" "flag" "fmt" "log" + "net/http" "os" "time" "github.com/AmanTahiliani/box-box/internal/api" "github.com/AmanTahiliani/box-box/internal/ingest" + "github.com/AmanTahiliani/box-box/internal/news" "github.com/AmanTahiliani/box-box/internal/store" "github.com/AmanTahiliani/box-box/internal/ui" "github.com/AmanTahiliani/box-box/internal/web" @@ -21,6 +24,7 @@ func main() { ingestYear := flag.Int("ingest-year", 0, "Ingest OpenF1 meetings for a season year") ingestMeeting := flag.Int("ingest-meeting", 0, "Ingest meeting metadata and Race Hub datasets for all sessions") ingestSession := flag.Int("ingest-session", 0, "Ingest Race Hub datasets for a session key") + ingestNews := flag.Bool("ingest-news", false, "Refresh RSS/Atom paddock briefing feeds") dryRun := flag.Bool("dry-run", false, "Preview ingestion without writing domain rows") dbPath := flag.String("db", "", "Domain database path (default: ~/.local/share/box-box/boxbox.db)") flag.Parse() @@ -46,11 +50,21 @@ func main() { if *ingestSession != 0 { ingestFlags++ } + if *ingestNews { + ingestFlags++ + } if ingestFlags > 0 { if ingestFlags > 1 { - fmt.Fprintln(os.Stderr, "box-box: only one of --ingest-year, --ingest-meeting, or --ingest-session may be set") + fmt.Fprintln(os.Stderr, "box-box: only one of --ingest-year, --ingest-meeting, --ingest-session, or --ingest-news may be set") os.Exit(1) } + if *ingestNews { + if err := runNewsIngestion(*dryRun, *dbPath); err != nil { + fmt.Fprintf(os.Stderr, "box-box ingest error: %v\n", err) + os.Exit(1) + } + return + } if err := runIngestion(client, *ingestYear, *ingestMeeting, *ingestSession, *dryRun, *dbPath); err != nil { fmt.Fprintf(os.Stderr, "box-box ingest error: %v\n", err) os.Exit(1) @@ -129,3 +143,45 @@ func runIngestion(client *api.OpenF1Client, year, meetingKey, sessionKey int, dr } return err } + +func runNewsIngestion(dryRun bool, dbPath string) error { + log.SetOutput(os.Stderr) + + path := dbPath + if path == "" { + path = store.DefaultDBPath() + } + + var st *store.Store + if !dryRun { + var err error + st, err = store.Open(path) + if err != nil { + return fmt.Errorf("open domain database: %w", err) + } + defer st.Close() + } + + if dryRun { + fmt.Fprintf(os.Stderr, "news: dry run, not writing to %s\n", path) + } else { + fmt.Fprintf(os.Stderr, "news: refreshing feeds into %s\n", path) + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + result, err := news.Refresh(ctx, st, news.RefreshOptions{ + Client: &http.Client{Timeout: 10 * time.Second}, + DryRun: dryRun, + Progress: os.Stderr, + }) + fmt.Fprintf( + os.Stderr, + "news: %d source(s) fetched, %d failed, %d item(s) fetched, %d upserted\n", + result.SourcesFetched, + result.SourcesFailed, + result.ItemsFetched, + result.ItemsUpserted, + ) + return err +} diff --git a/documentations/refactor/28-orchestrator-handoff.md b/documentations/refactor/28-orchestrator-handoff.md new file mode 100644 index 0000000..fc5f94d --- /dev/null +++ b/documentations/refactor/28-orchestrator-handoff.md @@ -0,0 +1,292 @@ +# Orchestrator Handoff + +You are taking over as the primary orchestration/coding agent for the box-box refactor. + +Repo: + +```text +/Users/aman/HomeBase/Programming/Projects/box-box +``` + +Branch: + +```text +box-refactor +``` + +## Role + +You are the engineering manager/orchestrator. Inspect before acting, keep changes scoped, review agent work before committing, prune stale docs after phases, and commit cleanly after each accepted phase. The user prefers Cursor for backend/test/hardening work and Claude for major frontend/product/design work, but you may implement directly when appropriate. + +## Operating Rules + +- Do not rush into implementation if the user wants to discuss. +- If implementing, keep phases small and commit-ready. +- Commit after each completed/reviewed phase. +- Never revert user/other-agent changes without explicit permission. +- Use `rg` for searches. +- Use `apply_patch` for manual edits. +- For frontend work, run browser or Playwright verification where practical. +- For review requests, lead with findings and file/line references. +- `frontend/dist` is ignored and should not be committed. +- Preserve TUI live mode and official F1 SignalR live behavior carefully. + +## Project Direction + +box-box started as a Go Bubble Tea F1 TUI backed mostly by OpenF1. The refactor direction is now: + +- Web UI is the primary product surface. +- React + TypeScript frontend is the production Web UI stack. +- Go backend remains the API/server. +- Historical/completed-session data should be local-first from SQLite. +- OpenF1 ingestion is explicit via CLI, not fetched live on every page load. +- Official F1 SignalR remains the live source. +- Desired product feel: clean, dense, technical F1 operations room. Avoid card-heavy AI-slop. + +## Important Commands + +```bash +git status --short +git log --oneline -10 + +go test ./... +npm --prefix frontend test -- --run +npm --prefix frontend run build +npm run test:e2e +npm run test:e2e:prod +npm run test:visual +npm run test:visual:prod +``` + +Known test note: do not run Playwright suites that share the same seeded SQLite DB in parallel. Run prod E2E and prod visual sequentially, or they may hit `database is locked`. + +## Recent Commits + +- `571edb9 Add visual regression coverage` +- `e539abc Add command center screen` +- `9532206 Polish live timing UI` +- `a0f135a Update operator documentation` +- `79b0b9f Rework command center for race weekends` +- `84a8827 Add paddock briefing RSS backend spike` + +## Current Uncommitted Work + +Two phases are currently uncommitted and need review/integration. Review and +commit Phase 19 first, then Phase 20, or stage hunks carefully if separating the +shared README/refactor README edits. + +### Phase 19: Weekend Workspace / Race Hub Flow V1 + +Claude has completed Phase 19. It is currently uncommitted and needs review. + +Claude reported these Phase 19 changes: + +- New: + - `frontend/src/components/OverviewView.tsx` + - `frontend/src/components/WeekendSwitcher.tsx` + - `frontend/src/test/RaceHubPage.test.tsx` + - `documentations/refactor/26-phase-19-weekend-workspace.md` +- Modified: + - `frontend/src/pages/RaceHubPage.tsx` + - `frontend/src/components/TabBar.tsx` + - `frontend/src/components/DatasetStatusView.tsx` + - `frontend/src/styles/app.css` + - tests for TabBar, DatasetStatusView, race-hub, command-center, data-library, production-smoke + - `tests/visual/helpers.ts` + - race-hub visual snapshots + - root `README.md` + - `documentations/refactor/README.md` + +Claude reported these UX changes: + +- Race Hub is now a Weekend Workspace. +- Compact GP identity band with country decal/accent strip. +- Horizontal session rail replaces “big table then analysis below.” +- Active session context stays visible above tabs. +- Tabs regrouped into Overview, Race Story, Strategy, Lap Data, Conditions, Race Control, Data Status. +- New Overview tab with winner/pole/fastest/podium cards, condition chips, latest race control, and local coverage meter. +- Inline Switch Weekend panel replaces legacy LocalDataNavigator on Race Hub. +- Data Status links to `/admin`; no CLI/admin text on fan surface. +- Mobile/iPad improved with wrapping identity band, horizontal session rail, single-column stats. +- `/race-hub?session_key=9472` still works and loads Bahrain GP 2024 seeded session. +- Bare `/race-hub` now resolves to a focus weekend/session via `pickFocusMeeting` and navigation replace. + +Claude reported these tests: + +- `npm --prefix frontend test -- --run` passed, 98 tests. +- `npm --prefix frontend run build` passed. +- `npm run test:e2e` passed, 18 tests. +- `npm run test:e2e:prod` passed, 6 tests. +- `npm run test:visual` passed, 12 screenshots after regenerating race-hub baselines. +- `npm run test:visual:prod` passed, 12 screenshots. + +### Phase 20: Paddock Briefing Ingestion CLI + +A backend subagent implemented Phase 20 after the RSS backend spike. It is also +currently uncommitted and needs review. + +Reported Phase 20 changes: + +- Modified: + - `cmd/main.go` + - `README.md` + - `documentations/refactor/README.md` +- New: + - `internal/news/refresh.go` + - `internal/news/refresh_test.go` + - `documentations/refactor/29-phase-20-paddock-briefing-ingestion.md` + +Implemented behavior: + +- Adds `--ingest-news` as a CLI mode. +- Keeps it mutually exclusive with `--ingest-year`, `--ingest-meeting`, and + `--ingest-session`. +- Reuses `--db` for the domain SQLite path. +- Reuses `--dry-run` to fetch and report feed counts without opening or writing + the domain database. +- Uses `internal/news.Refresh`, which fetches `DefaultSources`, upserts + `news_sources`, upserts URL-deduped `news_items`, records `fetched_at` and + `expires_at`, and continues through individual feed failures before returning + a summary error. +- Web requests still do not fetch feeds; `/api/v1/news` remains read-only + against SQLite. + +Commands added: + +```bash +go run ./cmd/main.go --ingest-news +go run ./cmd/main.go --dry-run --ingest-news +go run ./cmd/main.go --ingest-news --db /tmp/boxbox.db +``` + +Phase 20 verification already run by Codex: + +```bash +go test ./cmd/... ./internal/news ./internal/store +go test ./internal/web ./internal/query +go test ./... +git diff --check +``` + +## Immediate Task + +Start by reviewing and committing Phase 19. Then review and commit Phase 20. +Do not start new implementation until both are accepted and committed. + +1. Inspect working tree: + +```bash +git status --short +git diff --stat +git diff --name-only +``` + +2. Review Claude’s Phase 19 work quickly but responsibly: + +- Check `RaceHubPage.tsx`, `OverviewView.tsx`, `WeekendSwitcher.tsx`, `TabBar.tsx`, `DatasetStatusView.tsx`, `app.css`, route/test updates, docs. +- Make sure no admin/CLI guidance leaked back into Race Hub. +- Make sure `/race-hub?session_key=9472` compatibility is preserved. +- Make sure `/admin` remains the admin/data-health surface. +- Confirm visual tests and docs match the changed UX. + +3. Run a focused verification pass. At minimum: + +```bash +npm --prefix frontend test -- --run +npm --prefix frontend run build +npm run test:e2e +npm run test:visual +``` + +If time allows or if production behavior changed: + +```bash +npm run test:e2e:prod +npm run test:visual:prod +``` + +4. Patch only small issues if found. +5. Stage only Phase 19 files. +6. Commit with a message like: + +```bash +git commit -m "Rework Race Hub as weekend workspace" +``` + +Then review Phase 20: + +1. Check `cmd/main.go`, `internal/news/refresh.go`, + `internal/news/refresh_test.go`, + `documentations/refactor/29-phase-20-paddock-briefing-ingestion.md`, and the + README/refactor README hunks. +2. Confirm the CLI mode does not interfere with OpenF1 ingestion modes or web/TUI + startup. +3. Confirm no live internet tests were added. +4. Re-run targeted backend tests if needed: + +```bash +go test ./cmd/... ./internal/news ./internal/store +go test ./... +``` + +5. Stage Phase 20 files/hunks and commit with a message like: + +```bash +git commit -m "Add paddock briefing feed ingestion" +``` + +## RSS / Paddock Briefing Context + +Cursor completed and Codex committed a backend spike as `84a8827 Add paddock briefing RSS backend spike`. + +Implemented: + +- `internal/news`: RSS/Atom parser and fetch helper. +- SQLite tables: + - `news_sources` + - `news_items` +- Store/query methods for cached news. +- Read-only API: + - `GET /api/v1/news?limit=25&source=racefans-f1` +- No request-time network fetching. +- Unit tests use local XML fixtures. + +Recommended feed sources: + +- FIA official RSS +- BBC Sport F1 +- Autosport F1 +- RaceFans F1 +- Guardian Formula One + +Optional: + +- Motorsport.com +- RACER +- Formula 1 YouTube Atom + +Avoid: + +- Formula1.com scraping/hidden endpoints +- X/Twitter scraping +- Reddit as primary source +- RSS.app/Feedspot as primary source + +## Likely Next Phases After Phase 19 And 20 + +1. Phase 21: Paddock Briefing UI + - Claude/frontend. + - Add fan-facing briefing module, likely on Command Center first. + - Query `/api/v1/news`. + - Show source, title, age, category, short feed-provided snippet, external link. + - Keep publisher attribution visible. + - Avoid full article storage or scraping. + +2. Phase 22: Race Story Deepening + - Claude/frontend or mixed. + - Collapse legacy classification/grid/position components into a more fluid Race Story canvas. + - Improve mobile scanning and session narrative. + +3. Phase 23: Full Season Backfill / ingest hardening + - Cursor/backend. + - Safer season workflows, resumability, rate-limit controls, coverage reporting. diff --git a/documentations/refactor/29-phase-20-paddock-briefing-ingestion.md b/documentations/refactor/29-phase-20-paddock-briefing-ingestion.md new file mode 100644 index 0000000..35fc955 --- /dev/null +++ b/documentations/refactor/29-phase-20-paddock-briefing-ingestion.md @@ -0,0 +1,47 @@ +# Phase 20 Paddock Briefing Ingestion CLI + +## Goal + +Turn the Phase 19B RSS backend spike into an explicit local refresh command for +the Paddock Briefing cache. Feed fetching remains a CLI-only operation; web +requests continue to read SQLite only. + +## Implemented + +- Added `--ingest-news` as a CLI ingestion mode on `cmd/main.go`. +- Reused `--db` path behavior from the existing OpenF1 ingestion flows. +- Reused `--dry-run` to fetch and report feed counts without opening or writing + the domain database. +- Added `internal/news.Refresh`, which: + - fetches `internal/news.DefaultSources` unless tests provide a custom list; + - uses `internal/news.Fetch` with a 10-second HTTP client timeout; + - upserts `news_sources` with `fetched_at` and `expires_at`; + - upserts URL-deduped `news_items`; + - continues after individual source failures and returns a summary error after + successful sources are stored. + +## CLI + +```bash +go run ./cmd/main.go --ingest-news +go run ./cmd/main.go --dry-run --ingest-news +go run ./cmd/main.go --ingest-news --db /tmp/boxbox.db +``` + +`--ingest-news` is mutually exclusive with `--ingest-year`, `--ingest-meeting`, +and `--ingest-session`. + +## Verification + +Tests use local `httptest.Server` feeds only. No live internet test is required +for the refresh logic. + +```bash +go test ./internal/news ./internal/store +``` + +## Notes + +The refresh TTL is currently 30 minutes for all sources. A future phase can add +source-specific TTLs, retention/pruning, or admin UI controls without changing +the read-only `/api/v1/news` contract. diff --git a/internal/news/refresh.go b/internal/news/refresh.go new file mode 100644 index 0000000..b572564 --- /dev/null +++ b/internal/news/refresh.go @@ -0,0 +1,141 @@ +package news + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/AmanTahiliani/box-box/internal/store" +) + +const DefaultTTL = 30 * time.Minute + +// Store is the storage surface needed by feed refreshes. +type Store interface { + UpsertNewsSource(store.NewsSource) error + UpsertNewsItem(store.NewsItem) error +} + +// RefreshOptions configures one local news refresh run. +type RefreshOptions struct { + Sources []Source + Client *http.Client + TTL time.Duration + DryRun bool + Now func() time.Time + Progress io.Writer +} + +// RefreshResult summarizes one local news refresh run. +type RefreshResult struct { + SourcesFetched int + SourcesFailed int + ItemsFetched int + ItemsUpserted int +} + +// Refresh fetches RSS/Atom sources and stores normalized, URL-deduped items. +func Refresh(ctx context.Context, st Store, opts RefreshOptions) (RefreshResult, error) { + if st == nil && !opts.DryRun { + return RefreshResult{}, errors.New("news refresh: store is required") + } + if len(opts.Sources) == 0 { + opts.Sources = DefaultSources + } + if opts.Client == nil { + opts.Client = &http.Client{Timeout: 10 * time.Second} + } + if opts.TTL <= 0 { + opts.TTL = DefaultTTL + } + now := func() time.Time { return time.Now().UTC() } + if opts.Now != nil { + now = func() time.Time { return opts.Now().UTC() } + } + + var result RefreshResult + var failures []string + for _, source := range opts.Sources { + fetchedAt := now() + expiresAt := fetchedAt.Add(opts.TTL) + if !opts.DryRun { + if err := st.UpsertNewsSource(store.NewsSource{ + Source: source.ID, + Name: source.Name, + FeedURL: source.URL, + Category: source.Category, + Enabled: true, + UpdatedAt: fetchedAt, + }); err != nil { + return result, err + } + } + items, err := Fetch(ctx, opts.Client, source) + if err != nil { + result.SourcesFailed++ + failures = append(failures, fmt.Sprintf("%s: %v", source.ID, err)) + progressf(opts.Progress, "news: %s failed: %v\n", source.ID, err) + continue + } + + result.SourcesFetched++ + result.ItemsFetched += len(items) + progressf(opts.Progress, "news: %s fetched %d items\n", source.ID, len(items)) + if opts.DryRun { + continue + } + + if err := st.UpsertNewsSource(store.NewsSource{ + Source: source.ID, + Name: source.Name, + FeedURL: source.URL, + Category: source.Category, + Enabled: true, + FetchedAt: &fetchedAt, + ExpiresAt: &expiresAt, + UpdatedAt: fetchedAt, + }); err != nil { + return result, err + } + for _, item := range items { + item.FetchedAt = fetchedAt + publishedAt := timePtr(item.PublishedAt) + if err := st.UpsertNewsItem(store.NewsItem{ + URL: item.URL, + Source: item.Source, + Title: item.Title, + PublishedAt: publishedAt, + Summary: item.Summary, + Category: item.Category, + FetchedAt: item.FetchedAt, + }); err != nil { + return result, err + } + result.ItemsUpserted++ + } + } + + if len(failures) > 0 { + return result, fmt.Errorf("news refresh completed with %d source failure(s): %s", len(failures), strings.Join(failures, "; ")) + } + return result, nil +} + +func progressf(w io.Writer, format string, args ...any) { + if w == nil { + return + } + fmt.Fprintf(w, format, args...) +} + +func timePtr(v time.Time) *time.Time { + if v.IsZero() { + return nil + } + t := v.UTC() + return &t +} diff --git a/internal/news/refresh_test.go b/internal/news/refresh_test.go new file mode 100644 index 0000000..386481f --- /dev/null +++ b/internal/news/refresh_test.go @@ -0,0 +1,149 @@ +package news + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/AmanTahiliani/box-box/internal/store" +) + +func TestRefreshStoresSourcesAndItems(t *testing.T) { + feed := ` + + + + Briefing one + https://example.com/f1/one?utm_source=rss + Mon, 25 May 2026 10:00:00 GMT + Morning note + + + Briefing duplicate newer + https://example.com/f1/one + Mon, 25 May 2026 11:00:00 GMT + Updated note + + +` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("User-Agent"); got != UserAgent { + t.Fatalf("User-Agent = %q, want %q", got, UserAgent) + } + w.Header().Set("Content-Type", "application/rss+xml") + _, _ = w.Write([]byte(feed)) + })) + defer server.Close() + + st := openNewsTestStore(t) + now := time.Unix(1800000000, 0).UTC() + result, err := Refresh(context.Background(), st, RefreshOptions{ + Sources: []Source{{ + ID: "example", + Name: "Example F1", + URL: server.URL + "/feed.xml", + Category: "news", + }}, + Client: server.Client(), + TTL: time.Hour, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("Refresh() error = %v", err) + } + if result.SourcesFetched != 1 || result.ItemsFetched != 1 || result.ItemsUpserted != 1 || result.SourcesFailed != 0 { + t.Fatalf("result = %+v, want one fetched/upserted item and no failures", result) + } + + var fetchedAt, expiresAt int64 + if err := st.DB().QueryRow(` + SELECT fetched_at, expires_at + FROM news_sources + WHERE source = 'example' + `).Scan(&fetchedAt, &expiresAt); err != nil { + t.Fatalf("query source metadata: %v", err) + } + if fetchedAt != now.Unix() || expiresAt != now.Add(time.Hour).Unix() { + t.Fatalf("source times = %d/%d, want %d/%d", fetchedAt, expiresAt, now.Unix(), now.Add(time.Hour).Unix()) + } + + items, err := st.ListNewsItems(10, "example") + if err != nil { + t.Fatalf("ListNewsItems() error = %v", err) + } + if len(items) != 1 { + t.Fatalf("items len = %d, want 1", len(items)) + } + if items[0].URL != "https://example.com/f1/one" || items[0].Title != "Briefing duplicate newer" { + t.Fatalf("stored item = %+v, want canonical newer duplicate", items[0]) + } + if !items[0].FetchedAt.Equal(now) { + t.Fatalf("item fetched_at = %v, want %v", items[0].FetchedAt, now) + } +} + +func TestRefreshDryRunDoesNotRequireStore(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`Dryhttps://example.com/dry`)) + })) + defer server.Close() + + result, err := Refresh(context.Background(), nil, RefreshOptions{ + Sources: []Source{{ID: "dry", Name: "Dry", URL: server.URL, Category: "news"}}, + Client: server.Client(), + DryRun: true, + }) + if err != nil { + t.Fatalf("Refresh() dry run error = %v", err) + } + if result.ItemsFetched != 1 || result.ItemsUpserted != 0 { + t.Fatalf("result = %+v, want fetched item with no upsert", result) + } +} + +func TestRefreshContinuesAfterSourceFailure(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.Path, "bad") { + http.Error(w, "nope", http.StatusBadGateway) + return + } + _, _ = w.Write([]byte(`Goodhttps://example.com/good`)) + })) + defer server.Close() + + st := openNewsTestStore(t) + result, err := Refresh(context.Background(), st, RefreshOptions{ + Sources: []Source{ + {ID: "bad", Name: "Bad", URL: server.URL + "/bad", Category: "news"}, + {ID: "good", Name: "Good", URL: server.URL + "/good", Category: "news"}, + }, + Client: server.Client(), + }) + if err == nil { + t.Fatal("Refresh() error = nil, want source failure") + } + if result.SourcesFetched != 1 || result.SourcesFailed != 1 || result.ItemsUpserted != 1 { + t.Fatalf("result = %+v, want one failure and one stored item", result) + } + items, err := st.ListNewsItems(10, "") + if err != nil { + t.Fatalf("ListNewsItems() error = %v", err) + } + if len(items) != 1 || items[0].Source != "good" { + t.Fatalf("items = %+v, want good source item stored", items) + } +} + +func openNewsTestStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "news.db")) + if err != nil { + t.Fatalf("store.Open() error = %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +}