Add local-first refactor foundation

internal/ingest/ingest.go

@@ -0,0 +1,471 @@
+package ingest
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/AmanTahiliani/box-box/internal/api"
+	"github.com/AmanTahiliani/box-box/internal/models"
+	"github.com/AmanTahiliani/box-box/internal/store"
+)
+
+// Options configures ingestion behavior.
+type Options struct {
+	DryRun       bool
+	RequestDelay time.Duration
+	MaxRetries   int
+	RetryBackoff time.Duration
+	Progress     *Progress
+}
+
+// DefaultOptions returns conservative ingestion defaults.
+func DefaultOptions() Options {
+	return Options{
+		RequestDelay: 300 * time.Millisecond,
+		MaxRetries:   3,
+		RetryBackoff: 500 * time.Millisecond,
+		Progress:     NewProgress(nil),
+	}
+}
+
+// Summary captures the outcome of an ingestion run.
+type Summary struct {
+	ScopeType      string   `json:"scope_type"`
+	ScopeKey       string   `json:"scope_key"`
+	Status         string   `json:"status"`
+	DryRun         bool     `json:"dry_run"`
+	Meetings       int      `json:"meetings"`
+	Sessions       int      `json:"sessions"`
+	Drivers        int      `json:"drivers"`
+	SessionResults int      `json:"session_results"`
+	StartingGrid   int      `json:"starting_grid"`
+	RawPayloads    int      `json:"raw_payloads"`
+	RawInserted    int      `json:"raw_inserted"`
+	Errors         []string `json:"errors,omitempty"`
+}
+
+// Service orchestrates OpenF1-to-store ingestion workflows.
+type Service struct {
+	store  *store.Store
+	source Source
+	opts   Options
+}
+
+// NewService creates an ingestion service.
+func NewService(st *store.Store, source Source, opts Options) *Service {
+	if opts.MaxRetries <= 0 {
+		opts.MaxRetries = 3
+	}
+	if opts.RetryBackoff <= 0 {
+		opts.RetryBackoff = 500 * time.Millisecond
+	}
+	if opts.RequestDelay <= 0 {
+		opts.RequestDelay = 300 * time.Millisecond
+	}
+	if opts.Progress == nil {
+		opts.Progress = NewProgress(nil)
+	}
+	return &Service{store: st, source: source, opts: opts}
+}
+
+// IngestYear fetches and stores all meetings for a season year.
+func (s *Service) IngestYear(year int) (Summary, error) {
+	summary := Summary{
+		ScopeType: "year",
+		ScopeKey:  fmt.Sprintf("%d", year),
+		DryRun:    s.opts.DryRun,
+	}
+	if year < 2023 {
+		return summary, fmt.Errorf("invalid year %d: must be 2023 or later", year)
+	}
+
+	runID, err := s.beginRun(summary.ScopeType, summary.ScopeKey)
+	if err != nil {
+		return summary, err
+	}
+
+	s.opts.Progress.Step("fetching meetings for %d", year)
+	fetch, meetings, err := fetchWithRetry(s, func() (FetchResult, []models.Meeting, error) {
+		return s.source.FetchMeetingsForYear(year)
+	})
+	if err != nil {
+		return s.finishFailed(runID, summary, err)
+	}
+
+	summary.RawPayloads++
+	if !s.opts.DryRun {
+		inserted, err := s.storeRaw(fetch, nil, nil)
+		if err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		if inserted {
+			summary.RawInserted++
+		}
+	}
+	s.delay()
+
+	for _, m := range meetings {
+		if s.opts.DryRun {
+			summary.Meetings++
+			continue
+		}
+		if err := s.store.UpsertMeeting(meetingToStore(m)); err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		summary.Meetings++
+	}
+
+	summary.Status = statusForDryRun(s.opts.DryRun)
+	s.finishRun(runID, summary)
+	s.opts.Progress.Summary(summary)
+	return summary, nil
+}
+
+// IngestMeeting fetches meeting metadata and all sessions for a meeting key.
+func (s *Service) IngestMeeting(meetingKey int) (Summary, error) {
+	summary := Summary{
+		ScopeType: "meeting",
+		ScopeKey:  fmt.Sprintf("%d", meetingKey),
+		DryRun:    s.opts.DryRun,
+	}
+
+	runID, err := s.beginRun(summary.ScopeType, summary.ScopeKey)
+	if err != nil {
+		return summary, err
+	}
+
+	s.opts.Progress.Step("fetching meeting %d", meetingKey)
+	meetingFetch, meetings, err := fetchWithRetry(s, func() (FetchResult, []models.Meeting, error) {
+		return s.source.FetchMeetingsForMeetingKey(meetingKey)
+	})
+	if err != nil {
+		return s.finishFailed(runID, summary, err)
+	}
+	summary.RawPayloads++
+	if !s.opts.DryRun {
+		mk := meetingKey
+		inserted, err := s.storeRaw(meetingFetch, &mk, nil)
+		if err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		if inserted {
+			summary.RawInserted++
+		}
+	}
+	s.delay()
+
+	for _, m := range meetings {
+		if s.opts.DryRun {
+			summary.Meetings++
+			continue
+		}
+		if err := s.store.UpsertMeeting(meetingToStore(m)); err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		summary.Meetings++
+	}
+
+	s.opts.Progress.Step("fetching sessions for meeting %d", meetingKey)
+	sessionFetch, sessions, err := fetchWithRetry(s, func() (FetchResult, []models.Session, error) {
+		return s.source.FetchSessionsForMeeting(meetingKey)
+	})
+	if err != nil {
+		return s.finishFailed(runID, summary, err)
+	}
+	summary.RawPayloads++
+	if !s.opts.DryRun {
+		mk := meetingKey
+		inserted, err := s.storeRaw(sessionFetch, &mk, nil)
+		if err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		if inserted {
+			summary.RawInserted++
+		}
+	}
+	s.delay()
+
+	for _, sess := range sessions {
+		if s.opts.DryRun {
+			summary.Sessions++
+			continue
+		}
+		if err := s.store.UpsertSession(sessionToStore(sess)); err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		summary.Sessions++
+	}
+
+	summary.Status = statusForDryRun(s.opts.DryRun)
+	s.finishRun(runID, summary)
+	s.opts.Progress.Summary(summary)
+	return summary, nil
+}
+
+// IngestSession ingests Race Hub v1 datasets for a single session.
+func (s *Service) IngestSession(sessionKey int) (Summary, error) {
+	summary := Summary{
+		ScopeType: "session",
+		ScopeKey:  fmt.Sprintf("%d", sessionKey),
+		DryRun:    s.opts.DryRun,
+	}
+
+	runID, err := s.beginRun(summary.ScopeType, summary.ScopeKey)
+	if err != nil {
+		return summary, err
+	}
+
+	s.opts.Progress.Step("fetching session %d", sessionKey)
+	sessionFetch, sessions, err := fetchWithRetry(s, func() (FetchResult, []models.Session, error) {
+		return s.source.FetchSessionsForSessionKey(sessionKey)
+	})
+	if err != nil {
+		return s.finishFailed(runID, summary, err)
+	}
+	if len(sessions) == 0 {
+		err := fmt.Errorf("session %d not found", sessionKey)
+		return s.finishFailed(runID, summary, err)
+	}
+	sess := sessions[0]
+	meetingKey := sess.MeetingKey
+	sk := sessionKey
+
+	summary.RawPayloads++
+	if !s.opts.DryRun {
+		inserted, err := s.storeRaw(sessionFetch, &meetingKey, &sk)
+		if err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		if inserted {
+			summary.RawInserted++
+		}
+	}
+	s.delay()
+
+	s.opts.Progress.Step("fetching meeting %d for session context", meetingKey)
+	meetingFetch, meetings, err := fetchWithRetry(s, func() (FetchResult, []models.Meeting, error) {
+		return s.source.FetchMeetingsForMeetingKey(meetingKey)
+	})
+	if err != nil {
+		return s.finishFailed(runID, summary, err)
+	}
+	summary.RawPayloads++
+	if !s.opts.DryRun {
+		inserted, err := s.storeRaw(meetingFetch, &meetingKey, &sk)
+		if err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		if inserted {
+			summary.RawInserted++
+		}
+		for _, m := range meetings {
+			if err := s.store.UpsertMeeting(meetingToStore(m)); err != nil {
+				return s.finishFailed(runID, summary, err)
+			}
+			summary.Meetings++
+		}
+	} else {
+		summary.Meetings = len(meetings)
+	}
+	s.delay()
+
+	if s.opts.DryRun {
+		summary.Sessions++
+	} else {
+		if err := s.store.UpsertSession(sessionToStore(sess)); err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		summary.Sessions++
+	}
+
+	s.opts.Progress.Step("fetching drivers for session %d", sessionKey)
+	driverFetch, drivers, err := fetchWithRetry(s, func() (FetchResult, []models.Driver, error) {
+		return s.source.FetchDriversForSession(sessionKey)
+	})
+	if err != nil {
+		return s.finishFailed(runID, summary, err)
+	}
+	summary.RawPayloads++
+	if !s.opts.DryRun {
+		inserted, err := s.storeRaw(driverFetch, &meetingKey, &sk)
+		if err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		if inserted {
+			summary.RawInserted++
+		}
+		for _, d := range drivers {
+			if err := s.store.UpsertDriver(driverToStore(d)); err != nil {
+				return s.finishFailed(runID, summary, err)
+			}
+			if err := s.store.UpsertSessionDriver(sessionDriverToStore(d)); err != nil {
+				return s.finishFailed(runID, summary, err)
+			}
+			summary.Drivers++
+		}
+	} else {
+		summary.Drivers = len(drivers)
+	}
+	s.delay()
+
+	s.opts.Progress.Step("fetching session results for session %d", sessionKey)
+	resultFetch, results, err := fetchWithRetry(s, func() (FetchResult, []models.SessionResult, error) {
+		return s.source.FetchSessionResult(sessionKey)
+	})
+	if err != nil {
+		return s.finishFailed(runID, summary, err)
+	}
+	summary.RawPayloads++
+	if !s.opts.DryRun {
+		inserted, err := s.storeRaw(resultFetch, &meetingKey, &sk)
+		if err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		if inserted {
+			summary.RawInserted++
+		}
+		for _, r := range results {
+			if err := s.store.UpsertSessionResult(sessionResultToStore(r)); err != nil {
+				return s.finishFailed(runID, summary, err)
+			}
+			summary.SessionResults++
+		}
+	} else {
+		summary.SessionResults = len(results)
+	}
+	s.delay()
+
+	s.opts.Progress.Step("fetching starting grid for session %d", sessionKey)
+	gridFetch, grid, err := fetchWithRetry(s, func() (FetchResult, []models.StartingGrid, error) {
+		return s.source.FetchStartingGrid(sessionKey)
+	})
+	if err != nil {
+		return s.finishFailed(runID, summary, err)
+	}
+	summary.RawPayloads++
+	if !s.opts.DryRun {
+		inserted, err := s.storeRaw(gridFetch, &meetingKey, &sk)
+		if err != nil {
+			return s.finishFailed(runID, summary, err)
+		}
+		if inserted {
+			summary.RawInserted++
+		}
+		for _, g := range grid {
+			if err := s.store.UpsertStartingGridEntry(startingGridToStore(g)); err != nil {
+				return s.finishFailed(runID, summary, err)
+			}
+			summary.StartingGrid++
+		}
+	} else {
+		summary.StartingGrid = len(grid)
+	}
+
+	summary.Status = statusForDryRun(s.opts.DryRun)
+	s.finishRun(runID, summary)
+	s.opts.Progress.Summary(summary)
+	return summary, nil
+}
+
+func (s *Service) beginRun(scopeType, scopeKey string) (int64, error) {
+	if s.opts.DryRun {
+		return 0, nil
+	}
+	return s.store.CreateIngestionRun(scopeType, scopeKey, false)
+}
+
+func (s *Service) finishRun(runID int64, summary Summary) {
+	if s.opts.DryRun || runID == 0 {
+		return
+	}
+	b, _ := json.Marshal(summary)
+	_ = s.store.FinishIngestionRun(runID, summary.Status, string(b))
+}
+
+func (s *Service) finishFailed(runID int64, summary Summary, err error) (Summary, error) {
+	summary.Status = "failed"
+	summary.Errors = append(summary.Errors, err.Error())
+	if runID != 0 && !s.opts.DryRun {
+		b, _ := json.Marshal(summary)
+		_ = s.store.FinishIngestionRun(runID, summary.Status, string(b))
+	}
+	s.opts.Progress.Summary(summary)
+	return summary, err
+}
+
+func (s *Service) storeRaw(fetch FetchResult, meetingKey, sessionKey *int) (bool, error) {
+	_, inserted, err := s.store.InsertRawPayload(store.RawPayload{
+		Source:         sourceOpenF1,
+		Endpoint:       fetch.Endpoint,
+		RequestKey:     fetch.RequestKey,
+		MeetingKey:     meetingKey,
+		SessionKey:     sessionKey,
+		Payload:        string(fetch.Body),
+		FetchedAt:      fetch.FetchedAt,
+		ProvenanceJSON: provenanceJSON(fetch),
+	})
+	return inserted, err
+}
+
+func (s *Service) delay() {
+	if s.opts.RequestDelay > 0 {
+		time.Sleep(s.opts.RequestDelay)
+	}
+}
+
+func statusForDryRun(dryRun bool) string {
+	if dryRun {
+		return "dry_run"
+	}
+	return "completed"
+}
+
+type fetchFunc[T any] func() (FetchResult, T, error)
+
+func fetchWithRetry[T any](s *Service, fn fetchFunc[T]) (FetchResult, T, error) {
+	var zero T
+	var lastErr error
+
+	for attempt := 0; attempt < s.opts.MaxRetries; attempt++ {
+		if attempt > 0 {
+			time.Sleep(s.opts.RetryBackoff * time.Duration(attempt))
+		}
+
+		fetch, data, err := fn()
+		if err == nil {
+			return fetch, data, nil
+		}
+		lastErr = err
+
+		if api.IsLiveSessionError(err) {
+			return FetchResult{}, zero, err
+		}
+		if !isRetryable(err) {
+			return FetchResult{}, zero, err
+		}
+	}
+
+	return FetchResult{}, zero, lastErr
+}
+
+func isRetryable(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	if strings.Contains(msg, "status 429") ||
+		strings.Contains(msg, "status 5") ||
+		strings.Contains(msg, "timeout") ||
+		strings.Contains(msg, "connection reset") ||
+		strings.Contains(msg, "temporary") {
+		return true
+	}
+	var netErr interface{ Timeout() bool }
+	if errors.As(err, &netErr) && netErr.Timeout() {
+		return true
+	}
+	return false
+}

internal/ingest/ingest_test.go

@@ -0,0 +1,381 @@
+package ingest
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/AmanTahiliani/box-box/internal/api"
+	"github.com/AmanTahiliani/box-box/internal/models"
+	"github.com/AmanTahiliani/box-box/internal/store"
+)
+
+type fakeSource struct {
+	meetingsByYear    map[int][]models.Meeting
+	meetingsByKey     map[int][]models.Meeting
+	sessionsByMeeting map[int][]models.Session
+	sessionsByKey     map[int][]models.Session
+	drivers           map[int][]models.Driver
+	results           map[int][]models.SessionResult
+	grid              map[int][]models.StartingGrid
+	failOn            string
+	liveLockout       bool
+}
+
+func newFakeSource() *fakeSource {
+	return &fakeSource{
+		meetingsByYear:    make(map[int][]models.Meeting),
+		meetingsByKey:     make(map[int][]models.Meeting),
+		sessionsByMeeting: make(map[int][]models.Session),
+		sessionsByKey:     make(map[int][]models.Session),
+		drivers:           make(map[int][]models.Driver),
+		results:           make(map[int][]models.SessionResult),
+		grid:              make(map[int][]models.StartingGrid),
+	}
+}
+
+func (f *fakeSource) maybeFail(endpoint string) error {
+	if f.liveLockout {
+		return fmt.Errorf("%w", api.ErrLiveSessionLocked)
+	}
+	if f.failOn == endpoint {
+		return errors.New("simulated fetch failure")
+	}
+	return nil
+}
+
+func (f *fakeSource) wrap(endpoint, requestKey string, payload any) FetchResult {
+	body, _ := json.Marshal(payload)
+	return FetchResult{
+		Endpoint:   endpoint,
+		RequestKey: requestKey,
+		URL:        "fake://" + endpoint + "?" + requestKey,
+		Body:       body,
+		FetchedAt:  time.Now(),
+	}
+}
+
+func (f *fakeSource) FetchMeetingsForYear(year int) (FetchResult, []models.Meeting, error) {
+	if err := f.maybeFail("meetings"); err != nil {
+		return FetchResult{}, nil, err
+	}
+	data := f.meetingsByYear[year]
+	return f.wrap("meetings", fmt.Sprintf("year=%d", year), data), data, nil
+}
+
+func (f *fakeSource) FetchMeetingsForMeetingKey(meetingKey int) (FetchResult, []models.Meeting, error) {
+	if err := f.maybeFail("meetings"); err != nil {
+		return FetchResult{}, nil, err
+	}
+	data := f.meetingsByKey[meetingKey]
+	return f.wrap("meetings", fmt.Sprintf("meeting_key=%d", meetingKey), data), data, nil
+}
+
+func (f *fakeSource) FetchSessionsForMeeting(meetingKey int) (FetchResult, []models.Session, error) {
+	if err := f.maybeFail("sessions"); err != nil {
+		return FetchResult{}, nil, err
+	}
+	data := f.sessionsByMeeting[meetingKey]
+	return f.wrap("sessions", fmt.Sprintf("meeting_key=%d", meetingKey), data), data, nil
+}
+
+func (f *fakeSource) FetchSessionsForSessionKey(sessionKey int) (FetchResult, []models.Session, error) {
+	if err := f.maybeFail("sessions"); err != nil {
+		return FetchResult{}, nil, err
+	}
+	data := f.sessionsByKey[sessionKey]
+	return f.wrap("sessions", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
+}
+
+func (f *fakeSource) FetchDriversForSession(sessionKey int) (FetchResult, []models.Driver, error) {
+	if err := f.maybeFail("drivers"); err != nil {
+		return FetchResult{}, nil, err
+	}
+	data := f.drivers[sessionKey]
+	return f.wrap("drivers", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
+}
+
+func (f *fakeSource) FetchSessionResult(sessionKey int) (FetchResult, []models.SessionResult, error) {
+	if err := f.maybeFail("session_result"); err != nil {
+		return FetchResult{}, nil, err
+	}
+	data := f.results[sessionKey]
+	return f.wrap("session_result", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
+}
+
+func (f *fakeSource) FetchStartingGrid(sessionKey int) (FetchResult, []models.StartingGrid, error) {
+	if err := f.maybeFail("starting_grid"); err != nil {
+		return FetchResult{}, nil, err
+	}
+	data := f.grid[sessionKey]
+	return f.wrap("starting_grid", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
+}
+
+func openTestStore(t *testing.T) *store.Store {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "ingest.db")
+	s, err := store.Open(path)
+	if err != nil {
+		t.Fatalf("store.Open() error = %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}
+
+func testSessionFixtures() (int, int, *fakeSource) {
+	const meetingKey = 1229
+	const sessionKey = 9472
+
+	src := newFakeSource()
+	src.meetingsByKey[meetingKey] = []models.Meeting{{
+		MeetingKey:          meetingKey,
+		MeetingName:         "Monaco",
+		MeetingOfficialName: "FORMULA 1 GRAND PRIX DE MONACO 2025",
+		Location:            "Monaco",
+		CountryCode:         "MON",
+		CountryName:         "Monaco",
+		Circuit: models.Circuit{
+			CircuitKey:       10,
+			CircuitShortName: "Monte Carlo",
+		},
+		Year: 2025,
+	}}
+	src.sessionsByKey[sessionKey] = []models.Session{{
+		SessionKey:  sessionKey,
+		MeetingKey:  meetingKey,
+		SessionName: "Race",
+		SessionType: "Race",
+		CircuitKey:  10,
+	}}
+	src.drivers[sessionKey] = []models.Driver{
+		{
+			DriverNumber: 1,
+			FullName:     "Max Verstappen",
+			SessionKey:   sessionKey,
+			MeetingKey:   meetingKey,
+			TeamName:     "Red Bull Racing",
+			TeamColour:   "3671C6",
+		},
+		{
+			DriverNumber: 44,
+			FullName:     "Lewis Hamilton",
+			SessionKey:   sessionKey,
+			MeetingKey:   meetingKey,
+			TeamName:     "Ferrari",
+			TeamColour:   "E8002D",
+		},
+	}
+	src.results[sessionKey] = []models.SessionResult{
+		{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 1, Position: 1, Points: 25, NumberOfLaps: 78},
+		{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 44, Position: 2, Points: 18, NumberOfLaps: 78, GapToLeader: 1.5},
+	}
+	src.grid[sessionKey] = []models.StartingGrid{
+		{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 1, Position: 1, LapDuration: 71.234},
+		{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 44, Position: 2, LapDuration: 71.456},
+	}
+
+	return meetingKey, sessionKey, src
+}
+
+func TestIngestSessionWritesDomainAndRawRows(t *testing.T) {
+	_, sessionKey, src := testSessionFixtures()
+	st := openTestStore(t)
+
+	opts := DefaultOptions()
+	opts.RequestDelay = 0
+	svc := NewService(st, src, opts)
+
+	summary, err := svc.IngestSession(sessionKey)
+	if err != nil {
+		t.Fatalf("IngestSession() error = %v", err)
+	}
+	if summary.Status != "completed" {
+		t.Fatalf("summary.Status = %q, want completed", summary.Status)
+	}
+	if summary.Drivers != 2 || summary.SessionResults != 2 || summary.StartingGrid != 2 {
+		t.Fatalf("summary counts = %+v, want 2 drivers/results/grid", summary)
+	}
+	if summary.RawPayloads != 5 {
+		t.Fatalf("summary.RawPayloads = %d, want 5", summary.RawPayloads)
+	}
+
+	drivers, err := st.ListSessionDrivers(sessionKey)
+	if err != nil {
+		t.Fatalf("ListSessionDrivers() error = %v", err)
+	}
+	if len(drivers) != 2 {
+		t.Fatalf("session drivers = %d, want 2", len(drivers))
+	}
+
+	results, err := st.ListSessionResults(sessionKey)
+	if err != nil {
+		t.Fatalf("ListSessionResults() error = %v", err)
+	}
+	if len(results) != 2 {
+		t.Fatalf("session results = %d, want 2", len(results))
+	}
+
+	grid, err := st.ListStartingGrid(sessionKey)
+	if err != nil {
+		t.Fatalf("ListStartingGrid() error = %v", err)
+	}
+	if len(grid) != 2 {
+		t.Fatalf("starting grid = %d, want 2", len(grid))
+	}
+
+	raw, err := st.ListRawPayloadsBySession(sessionKey)
+	if err != nil {
+		t.Fatalf("ListRawPayloadsBySession() error = %v", err)
+	}
+	if len(raw) != 5 {
+		t.Fatalf("raw payloads = %d, want 5", len(raw))
+	}
+}
+
+func TestIngestSessionIsIdempotent(t *testing.T) {
+	_, sessionKey, src := testSessionFixtures()
+	st := openTestStore(t)
+
+	opts := DefaultOptions()
+	opts.RequestDelay = 0
+	svc := NewService(st, src, opts)
+
+	if _, err := svc.IngestSession(sessionKey); err != nil {
+		t.Fatalf("first IngestSession() error = %v", err)
+	}
+	if _, err := svc.IngestSession(sessionKey); err != nil {
+		t.Fatalf("second IngestSession() error = %v", err)
+	}
+
+	count := func(query string) int {
+		var n int
+		if err := st.DB().QueryRow(query, sessionKey).Scan(&n); err != nil {
+			t.Fatalf("count query failed: %v", err)
+		}
+		return n
+	}
+
+	if got := count(`SELECT COUNT(*) FROM session_drivers WHERE session_key = ?`); got != 2 {
+		t.Fatalf("session_drivers count = %d, want 2", got)
+	}
+	if got := count(`SELECT COUNT(*) FROM session_results WHERE session_key = ?`); got != 2 {
+		t.Fatalf("session_results count = %d, want 2", got)
+	}
+	if got := count(`SELECT COUNT(*) FROM starting_grid WHERE session_key = ?`); got != 2 {
+		t.Fatalf("starting_grid count = %d, want 2", got)
+	}
+}
+
+func TestDryRunDoesNotWriteDomainRows(t *testing.T) {
+	_, sessionKey, src := testSessionFixtures()
+	st := openTestStore(t)
+
+	opts := DefaultOptions()
+	opts.DryRun = true
+	opts.RequestDelay = 0
+	svc := NewService(st, src, opts)
+
+	summary, err := svc.IngestSession(sessionKey)
+	if err != nil {
+		t.Fatalf("IngestSession() error = %v", err)
+	}
+	if summary.Status != "dry_run" {
+		t.Fatalf("summary.Status = %q, want dry_run", summary.Status)
+	}
+
+	var driverCount int
+	if err := st.DB().QueryRow(`SELECT COUNT(*) FROM drivers`).Scan(&driverCount); err != nil {
+		t.Fatalf("count drivers: %v", err)
+	}
+	if driverCount != 0 {
+		t.Fatalf("drivers written during dry-run = %d, want 0", driverCount)
+	}
+
+	var rawCount int
+	if err := st.DB().QueryRow(`SELECT COUNT(*) FROM raw_payloads`).Scan(&rawCount); err != nil {
+		t.Fatalf("count raw payloads: %v", err)
+	}
+	if rawCount != 0 {
+		t.Fatalf("raw payloads written during dry-run = %d, want 0", rawCount)
+	}
+}
+
+func TestSourceErrorStopsRun(t *testing.T) {
+	_, sessionKey, src := testSessionFixtures()
+	src.failOn = "session_result"
+	st := openTestStore(t)
+
+	opts := DefaultOptions()
+	opts.RequestDelay = 0
+	svc := NewService(st, src, opts)
+
+	summary, err := svc.IngestSession(sessionKey)
+	if err == nil {
+		t.Fatal("IngestSession() expected error, got nil")
+	}
+	if summary.Status != "failed" {
+		t.Fatalf("summary.Status = %q, want failed", summary.Status)
+	}
+
+	var resultCount int
+	if err := st.DB().QueryRow(`SELECT COUNT(*) FROM session_results WHERE session_key = ?`, sessionKey).Scan(&resultCount); err != nil {
+		t.Fatalf("count session_results: %v", err)
+	}
+	if resultCount != 0 {
+		t.Fatalf("session_results after failure = %d, want 0", resultCount)
+	}
+}
+
+func TestLiveSessionLockoutSurfacesControlledFailure(t *testing.T) {
+	_, sessionKey, src := testSessionFixtures()
+	src.liveLockout = true
+	st := openTestStore(t)
+
+	opts := DefaultOptions()
+	opts.RequestDelay = 0
+	svc := NewService(st, src, opts)
+
+	_, err := svc.IngestSession(sessionKey)
+	if err == nil {
+		t.Fatal("IngestSession() expected live lockout error, got nil")
+	}
+	if !api.IsLiveSessionError(err) {
+		t.Fatalf("error = %v, want live session lockout", err)
+	}
+}
+
+func TestIngestYearAndMeeting(t *testing.T) {
+	src := newFakeSource()
+	src.meetingsByYear[2025] = []models.Meeting{
+		{MeetingKey: 100, MeetingName: "Bahrain", Year: 2025},
+		{MeetingKey: 101, MeetingName: "Saudi Arabia", Year: 2025},
+	}
+	src.meetingsByKey[100] = src.meetingsByYear[2025][:1]
+	src.sessionsByMeeting[100] = []models.Session{
+		{SessionKey: 9001, MeetingKey: 100, SessionName: "Race", SessionType: "Race"},
+	}
+
+	st := openTestStore(t)
+	opts := DefaultOptions()
+	opts.RequestDelay = 0
+	svc := NewService(st, src, opts)
+
+	yearSummary, err := svc.IngestYear(2025)
+	if err != nil {
+		t.Fatalf("IngestYear() error = %v", err)
+	}
+	if yearSummary.Meetings != 2 {
+		t.Fatalf("year meetings = %d, want 2", yearSummary.Meetings)
+	}
+
+	meetingSummary, err := svc.IngestMeeting(100)
+	if err != nil {
+		t.Fatalf("IngestMeeting() error = %v", err)
+	}
+	if meetingSummary.Sessions != 1 {
+		t.Fatalf("meeting sessions = %d, want 1", meetingSummary.Sessions)
+	}
+}

internal/ingest/openf1.go

@@ -0,0 +1,270 @@
+package ingest
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/AmanTahiliani/box-box/internal/api"
+	"github.com/AmanTahiliani/box-box/internal/models"
+	"github.com/AmanTahiliani/box-box/internal/store"
+)
+
+const sourceOpenF1 = "openf1"
+
+// FetchResult holds a fetched OpenF1 endpoint response with provenance metadata.
+type FetchResult struct {
+	Endpoint   string
+	RequestKey string
+	URL        string
+	Body       []byte
+	FetchedAt  time.Time
+}
+
+// Source fetches OpenF1 REST data for ingestion workflows.
+type Source interface {
+	FetchMeetingsForYear(year int) (FetchResult, []models.Meeting, error)
+	FetchMeetingsForMeetingKey(meetingKey int) (FetchResult, []models.Meeting, error)
+	FetchSessionsForMeeting(meetingKey int) (FetchResult, []models.Session, error)
+	FetchSessionsForSessionKey(sessionKey int) (FetchResult, []models.Session, error)
+	FetchDriversForSession(sessionKey int) (FetchResult, []models.Driver, error)
+	FetchSessionResult(sessionKey int) (FetchResult, []models.SessionResult, error)
+	FetchStartingGrid(sessionKey int) (FetchResult, []models.StartingGrid, error)
+}
+
+// OpenF1Source adapts OpenF1Client for ingestion using strict fetches.
+type OpenF1Source struct {
+	client *api.OpenF1Client
+}
+
+// NewOpenF1Source returns a Source backed by the OpenF1 API client.
+func NewOpenF1Source(client *api.OpenF1Client) *OpenF1Source {
+	return &OpenF1Source{client: client}
+}
+
+func (s *OpenF1Source) FetchMeetingsForYear(year int) (FetchResult, []models.Meeting, error) {
+	url := fmt.Sprintf("%s/v1/meetings?year=%d", s.client.BaseURL(), year)
+	return s.fetchMeetings(url, "meetings", fmt.Sprintf("year=%d", year))
+}
+
+func (s *OpenF1Source) FetchMeetingsForMeetingKey(meetingKey int) (FetchResult, []models.Meeting, error) {
+	url := fmt.Sprintf("%s/v1/meetings?meeting_key=%d", s.client.BaseURL(), meetingKey)
+	return s.fetchMeetings(url, "meetings", fmt.Sprintf("meeting_key=%d", meetingKey))
+}
+
+func (s *OpenF1Source) FetchSessionsForMeeting(meetingKey int) (FetchResult, []models.Session, error) {
+	url := fmt.Sprintf("%s/v1/sessions?meeting_key=%d", s.client.BaseURL(), meetingKey)
+	return s.fetchSessions(url, "sessions", fmt.Sprintf("meeting_key=%d", meetingKey))
+}
+
+func (s *OpenF1Source) FetchSessionsForSessionKey(sessionKey int) (FetchResult, []models.Session, error) {
+	url := fmt.Sprintf("%s/v1/sessions?session_key=%d", s.client.BaseURL(), sessionKey)
+	return s.fetchSessions(url, "sessions", fmt.Sprintf("session_key=%d", sessionKey))
+}
+
+func (s *OpenF1Source) FetchDriversForSession(sessionKey int) (FetchResult, []models.Driver, error) {
+	url := fmt.Sprintf("%s/v1/drivers?session_key=%d", s.client.BaseURL(), sessionKey)
+	return s.fetchDrivers(url, "drivers", fmt.Sprintf("session_key=%d", sessionKey))
+}
+
+func (s *OpenF1Source) FetchSessionResult(sessionKey int) (FetchResult, []models.SessionResult, error) {
+	url := fmt.Sprintf("%s/v1/session_result?session_key=%d", s.client.BaseURL(), sessionKey)
+	return s.fetchSessionResults(url, "session_result", fmt.Sprintf("session_key=%d", sessionKey))
+}
+
+func (s *OpenF1Source) FetchStartingGrid(sessionKey int) (FetchResult, []models.StartingGrid, error) {
+	url := fmt.Sprintf("%s/v1/starting_grid?session_key=%d", s.client.BaseURL(), sessionKey)
+	return s.fetchStartingGrid(url, "starting_grid", fmt.Sprintf("session_key=%d", sessionKey))
+}
+
+func (s *OpenF1Source) fetchMeetings(url, endpoint, requestKey string) (FetchResult, []models.Meeting, error) {
+	body, err := s.client.FetchStrict(url)
+	if err != nil {
+		return FetchResult{}, nil, err
+	}
+	var result []models.Meeting
+	if err := json.Unmarshal(body, &result); err != nil {
+		return FetchResult{}, nil, err
+	}
+	return FetchResult{
+		Endpoint:   endpoint,
+		RequestKey: requestKey,
+		URL:        url,
+		Body:       body,
+		FetchedAt:  time.Now(),
+	}, result, nil
+}
+
+func (s *OpenF1Source) fetchSessions(url, endpoint, requestKey string) (FetchResult, []models.Session, error) {
+	body, err := s.client.FetchStrict(url)
+	if err != nil {
+		return FetchResult{}, nil, err
+	}
+	var result []models.Session
+	if err := json.Unmarshal(body, &result); err != nil {
+		return FetchResult{}, nil, err
+	}
+	return FetchResult{
+		Endpoint:   endpoint,
+		RequestKey: requestKey,
+		URL:        url,
+		Body:       body,
+		FetchedAt:  time.Now(),
+	}, result, nil
+}
+
+func (s *OpenF1Source) fetchDrivers(url, endpoint, requestKey string) (FetchResult, []models.Driver, error) {
+	body, err := s.client.FetchStrict(url)
+	if err != nil {
+		return FetchResult{}, nil, err
+	}
+	var result []models.Driver
+	if err := json.Unmarshal(body, &result); err != nil {
+		return FetchResult{}, nil, err
+	}
+	return FetchResult{
+		Endpoint:   endpoint,
+		RequestKey: requestKey,
+		URL:        url,
+		Body:       body,
+		FetchedAt:  time.Now(),
+	}, result, nil
+}
+
+func (s *OpenF1Source) fetchSessionResults(url, endpoint, requestKey string) (FetchResult, []models.SessionResult, error) {
+	body, err := s.client.FetchStrict(url)
+	if err != nil {
+		return FetchResult{}, nil, err
+	}
+	var result []models.SessionResult
+	if err := json.Unmarshal(body, &result); err != nil {
+		return FetchResult{}, nil, err
+	}
+	return FetchResult{
+		Endpoint:   endpoint,
+		RequestKey: requestKey,
+		URL:        url,
+		Body:       body,
+		FetchedAt:  time.Now(),
+	}, result, nil
+}
+
+func (s *OpenF1Source) fetchStartingGrid(url, endpoint, requestKey string) (FetchResult, []models.StartingGrid, error) {
+	body, err := s.client.FetchStrict(url)
+	if err != nil {
+		return FetchResult{}, nil, err
+	}
+	var result []models.StartingGrid
+	if err := json.Unmarshal(body, &result); err != nil {
+		return FetchResult{}, nil, err
+	}
+	return FetchResult{
+		Endpoint:   endpoint,
+		RequestKey: requestKey,
+		URL:        url,
+		Body:       body,
+		FetchedAt:  time.Now(),
+	}, result, nil
+}
+
+func meetingToStore(m models.Meeting) store.Meeting {
+	return store.Meeting{
+		MeetingKey:          int(m.MeetingKey),
+		MeetingName:         m.MeetingName,
+		MeetingOfficialName: m.MeetingOfficialName,
+		Location:            m.Location,
+		CountryCode:         m.CountryCode,
+		CountryName:         m.CountryName,
+		CircuitKey:          m.CircuitKey,
+		CircuitShortName:    m.CircuitShortName,
+		GMTOffset:           m.GMTOffset,
+		DateStart:           m.DateStart,
+		DateEnd:             m.DateEnd,
+		Year:                m.Year,
+	}
+}
+
+func sessionToStore(s models.Session) store.Session {
+	return store.Session{
+		SessionKey:  s.SessionKey,
+		MeetingKey:  s.MeetingKey,
+		SessionName: s.SessionName,
+		SessionType: s.SessionType,
+		CircuitKey:  s.CircuitKey,
+		DateStart:   s.DateStart,
+		DateEnd:     s.DateEnd,
+		GMTOffset:   s.GMTOffset,
+	}
+}
+
+func driverToStore(d models.Driver) store.Driver {
+	return store.Driver{
+		DriverNumber:  d.DriverNumber,
+		BroadcastName: d.BroadcastName,
+		FirstName:     d.FirstName,
+		FullName:      d.FullName,
+		LastName:      d.LastName,
+		NameAcronym:   d.NameAcronym,
+		HeadshotURL:   d.HeadshotURL,
+		TeamName:      d.TeamName,
+		TeamColour:    d.TeamColour,
+	}
+}
+
+func sessionDriverToStore(d models.Driver) store.SessionDriver {
+	return store.SessionDriver{
+		SessionKey:   d.SessionKey,
+		DriverNumber: d.DriverNumber,
+		MeetingKey:   d.MeetingKey,
+		TeamName:     d.TeamName,
+		TeamColour:   d.TeamColour,
+	}
+}
+
+func sessionResultToStore(r models.SessionResult) store.SessionResult {
+	return store.SessionResult{
+		SessionKey:      r.SessionKey,
+		DriverNumber:    r.DriverNumber,
+		MeetingKey:      r.MeetingKey,
+		Position:        r.Position,
+		Points:          r.Points,
+		NumberOfLaps:    r.NumberOfLaps,
+		DurationJSON:    jsonField(r.Duration),
+		GapToLeaderJSON: jsonField(r.GapToLeader),
+		DNF:             r.DNF,
+		DNS:             r.DNS,
+		DSQ:             r.DSQ,
+	}
+}
+
+func startingGridToStore(g models.StartingGrid) store.StartingGridEntry {
+	return store.StartingGridEntry{
+		SessionKey:   g.SessionKey,
+		DriverNumber: g.DriverNumber,
+		MeetingKey:   g.MeetingKey,
+		Position:     g.Position,
+		LapDuration:  g.LapDuration,
+	}
+}
+
+func jsonField(v any) string {
+	if v == nil {
+		return ""
+	}
+	b, err := json.Marshal(v)
+	if err != nil {
+		return ""
+	}
+	return string(b)
+}
+
+func provenanceJSON(fetch FetchResult) string {
+	meta := map[string]string{
+		"url": fetch.URL,
+	}
+	b, err := json.Marshal(meta)
+	if err != nil {
+		return ""
+	}
+	return string(b)
+}

internal/ingest/progress.go

@@ -0,0 +1,49 @@
+package ingest
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+// Progress reports ingestion progress to a writer.
+type Progress struct {
+	w io.Writer
+}
+
+// NewProgress returns a progress helper writing to w, or stderr when w is nil.
+func NewProgress(w io.Writer) *Progress {
+	if w == nil {
+		w = os.Stderr
+	}
+	return &Progress{w: w}
+}
+
+func (p *Progress) Step(format string, args ...any) {
+	fmt.Fprintf(p.w, "ingest: "+format+"\n", args...)
+}
+
+func (p *Progress) Summary(summary Summary) {
+	fmt.Fprintf(p.w, "\ningest summary (%s %s): status=%s\n", summary.ScopeType, summary.ScopeKey, summary.Status)
+	if summary.Meetings > 0 {
+		fmt.Fprintf(p.w, "  meetings: %d\n", summary.Meetings)
+	}
+	if summary.Sessions > 0 {
+		fmt.Fprintf(p.w, "  sessions: %d\n", summary.Sessions)
+	}
+	if summary.Drivers > 0 {
+		fmt.Fprintf(p.w, "  drivers: %d\n", summary.Drivers)
+	}
+	if summary.SessionResults > 0 {
+		fmt.Fprintf(p.w, "  session results: %d\n", summary.SessionResults)
+	}
+	if summary.StartingGrid > 0 {
+		fmt.Fprintf(p.w, "  starting grid: %d\n", summary.StartingGrid)
+	}
+	if summary.RawPayloads > 0 {
+		fmt.Fprintf(p.w, "  raw payloads fetched: %d (inserted: %d)\n", summary.RawPayloads, summary.RawInserted)
+	}
+	for _, errMsg := range summary.Errors {
+		fmt.Fprintf(p.w, "  error: %s\n", errMsg)
+	}
+}