Add analytics data foundation

This commit is contained in:
2026-05-25 02:04:16 -04:00
parent 5470b0df38
commit c7438f0ed8
23 changed files with 2167 additions and 94 deletions

View File

@@ -42,6 +42,12 @@ type Summary struct {
Drivers int `json:"drivers"`
SessionResults int `json:"session_results"`
StartingGrid int `json:"starting_grid"`
Stints int `json:"stints"`
PitStops int `json:"pit_stops"`
Positions int `json:"positions"`
RaceControl int `json:"race_control"`
Weather int `json:"weather"`
Laps int `json:"laps"`
RawPayloads int `json:"raw_payloads"`
RawInserted int `json:"raw_inserted"`
Errors []string `json:"errors,omitempty"`
@@ -363,6 +369,26 @@ func (s *Service) IngestSession(sessionKey int) (Summary, error) {
} else {
summary.StartingGrid = len(grid)
}
s.delay()
if err := s.ingestStints(&summary, meetingKey, sk); err != nil {
return s.finishFailed(runID, summary, err)
}
if err := s.ingestPitStops(&summary, meetingKey, sk); err != nil {
return s.finishFailed(runID, summary, err)
}
if err := s.ingestPositions(&summary, meetingKey, sk); err != nil {
return s.finishFailed(runID, summary, err)
}
if err := s.ingestRaceControl(&summary, meetingKey, sk); err != nil {
return s.finishFailed(runID, summary, err)
}
if err := s.ingestWeather(&summary, meetingKey, sk); err != nil {
return s.finishFailed(runID, summary, err)
}
if err := s.ingestLaps(&summary, meetingKey, sk); err != nil {
return s.finishFailed(runID, summary, err)
}
summary.Status = statusForDryRun(s.opts.DryRun)
s.finishRun(runID, summary)
@@ -416,6 +442,150 @@ func (s *Service) delay() {
}
}
func (s *Service) ingestStints(summary *Summary, meetingKey, sessionKey int) error {
s.opts.Progress.Step("fetching stints for session %d", sessionKey)
fetch, stints, err := fetchWithRetry(s, func() (FetchResult, []models.Stint, error) {
return s.source.FetchStintsForSession(sessionKey)
})
if err != nil {
return err
}
return s.storeAnalyticsFetch(summary, fetch, meetingKey, sessionKey, func() error {
for _, st := range stints {
if err := s.store.UpsertStint(stintToStore(st)); err != nil {
return err
}
summary.Stints++
}
return nil
}, func() { summary.Stints = len(stints) })
}
func (s *Service) ingestPitStops(summary *Summary, meetingKey, sessionKey int) error {
s.opts.Progress.Step("fetching pit stops for session %d", sessionKey)
fetch, pits, err := fetchWithRetry(s, func() (FetchResult, []models.Pit, error) {
return s.source.FetchPitStopsForSession(sessionKey)
})
if err != nil {
return err
}
return s.storeAnalyticsFetch(summary, fetch, meetingKey, sessionKey, func() error {
for _, p := range pits {
if err := s.store.UpsertPitStop(pitStopToStore(p)); err != nil {
return err
}
summary.PitStops++
}
return nil
}, func() { summary.PitStops = len(pits) })
}
func (s *Service) ingestPositions(summary *Summary, meetingKey, sessionKey int) error {
s.opts.Progress.Step("fetching positions for session %d", sessionKey)
fetch, positions, err := fetchWithRetry(s, func() (FetchResult, []models.Position, error) {
return s.source.FetchPositionsForSession(sessionKey)
})
if err != nil {
return err
}
return s.storeAnalyticsFetch(summary, fetch, meetingKey, sessionKey, func() error {
for _, p := range positions {
if err := s.store.UpsertPositionSample(positionToStore(p)); err != nil {
return err
}
summary.Positions++
}
return nil
}, func() { summary.Positions = len(positions) })
}
func (s *Service) ingestRaceControl(summary *Summary, meetingKey, sessionKey int) error {
s.opts.Progress.Step("fetching race control for session %d", sessionKey)
fetch, messages, err := fetchWithRetry(s, func() (FetchResult, []models.RaceControl, error) {
return s.source.FetchRaceControlForSession(sessionKey)
})
if err != nil {
return err
}
return s.storeAnalyticsFetch(summary, fetch, meetingKey, sessionKey, func() error {
for _, rc := range messages {
if err := s.store.UpsertRaceControlMessage(raceControlToStore(rc)); err != nil {
return err
}
summary.RaceControl++
}
return nil
}, func() { summary.RaceControl = len(messages) })
}
func (s *Service) ingestWeather(summary *Summary, meetingKey, sessionKey int) error {
s.opts.Progress.Step("fetching weather for session %d", sessionKey)
fetch, samples, err := fetchWithRetry(s, func() (FetchResult, []models.Weather, error) {
return s.source.FetchWeatherForSession(sessionKey)
})
if err != nil {
return err
}
return s.storeAnalyticsFetch(summary, fetch, meetingKey, sessionKey, func() error {
for _, w := range samples {
if err := s.store.UpsertWeatherSample(weatherToStore(w)); err != nil {
return err
}
summary.Weather++
}
return nil
}, func() { summary.Weather = len(samples) })
}
func (s *Service) ingestLaps(summary *Summary, meetingKey, sessionKey int) error {
s.opts.Progress.Step("fetching laps for session %d", sessionKey)
fetch, laps, err := fetchWithRetry(s, func() (FetchResult, []models.Lap, error) {
return s.source.FetchLapsForSession(sessionKey)
})
if err != nil {
return err
}
return s.storeAnalyticsFetch(summary, fetch, meetingKey, sessionKey, func() error {
for _, l := range laps {
if err := s.store.UpsertLap(lapToStore(l)); err != nil {
return err
}
summary.Laps++
}
return nil
}, func() { summary.Laps = len(laps) })
}
func (s *Service) storeAnalyticsFetch(
summary *Summary,
fetch FetchResult,
meetingKey, sessionKey int,
storeRows func() error,
setDryRunCount func(),
) error {
mk := meetingKey
sk := sessionKey
summary.RawPayloads++
if s.opts.DryRun {
setDryRunCount()
s.delay()
return nil
}
inserted, err := s.storeRaw(fetch, &mk, &sk)
if err != nil {
return err
}
if inserted {
summary.RawInserted++
}
if err := storeRows(); err != nil {
return err
}
s.delay()
return nil
}
func statusForDryRun(dryRun bool) string {
if dryRun {
return "dry_run"

View File

@@ -21,6 +21,12 @@ type fakeSource struct {
drivers map[int][]models.Driver
results map[int][]models.SessionResult
grid map[int][]models.StartingGrid
stints map[int][]models.Stint
pitStops map[int][]models.Pit
positions map[int][]models.Position
raceControl map[int][]models.RaceControl
weather map[int][]models.Weather
laps map[int][]models.Lap
failOn string
liveLockout bool
}
@@ -34,6 +40,12 @@ func newFakeSource() *fakeSource {
drivers: make(map[int][]models.Driver),
results: make(map[int][]models.SessionResult),
grid: make(map[int][]models.StartingGrid),
stints: make(map[int][]models.Stint),
pitStops: make(map[int][]models.Pit),
positions: make(map[int][]models.Position),
raceControl: make(map[int][]models.RaceControl),
weather: make(map[int][]models.Weather),
laps: make(map[int][]models.Lap),
}
}
@@ -114,6 +126,54 @@ func (f *fakeSource) FetchStartingGrid(sessionKey int) (FetchResult, []models.St
return f.wrap("starting_grid", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
}
func (f *fakeSource) FetchStintsForSession(sessionKey int) (FetchResult, []models.Stint, error) {
if err := f.maybeFail("stints"); err != nil {
return FetchResult{}, nil, err
}
data := f.stints[sessionKey]
return f.wrap("stints", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
}
func (f *fakeSource) FetchPitStopsForSession(sessionKey int) (FetchResult, []models.Pit, error) {
if err := f.maybeFail("pit"); err != nil {
return FetchResult{}, nil, err
}
data := f.pitStops[sessionKey]
return f.wrap("pit", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
}
func (f *fakeSource) FetchPositionsForSession(sessionKey int) (FetchResult, []models.Position, error) {
if err := f.maybeFail("position"); err != nil {
return FetchResult{}, nil, err
}
data := f.positions[sessionKey]
return f.wrap("position", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
}
func (f *fakeSource) FetchRaceControlForSession(sessionKey int) (FetchResult, []models.RaceControl, error) {
if err := f.maybeFail("race_control"); err != nil {
return FetchResult{}, nil, err
}
data := f.raceControl[sessionKey]
return f.wrap("race_control", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
}
func (f *fakeSource) FetchWeatherForSession(sessionKey int) (FetchResult, []models.Weather, error) {
if err := f.maybeFail("weather"); err != nil {
return FetchResult{}, nil, err
}
data := f.weather[sessionKey]
return f.wrap("weather", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
}
func (f *fakeSource) FetchLapsForSession(sessionKey int) (FetchResult, []models.Lap, error) {
if err := f.maybeFail("laps"); err != nil {
return FetchResult{}, nil, err
}
data := f.laps[sessionKey]
return f.wrap("laps", fmt.Sprintf("session_key=%d", sessionKey), data), data, nil
}
func openTestStore(t *testing.T) *store.Store {
t.Helper()
path := filepath.Join(t.TempDir(), "ingest.db")
@@ -176,10 +236,35 @@ func testSessionFixtures() (int, int, *fakeSource) {
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 1, Position: 1, LapDuration: 71.234},
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 44, Position: 2, LapDuration: 71.456},
}
src.stints[sessionKey] = []models.Stint{
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 1, StintNumber: 1, Compound: models.CompoundMedium, LapStart: 1, LapEnd: 30},
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 44, StintNumber: 1, Compound: models.CompoundSoft, LapStart: 1, LapEnd: 18},
}
src.pitStops[sessionKey] = []models.Pit{
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 44, LapNumber: 19, Date: "2025-05-25T14:00:00+00:00", StopDuration: 2.4},
}
src.positions[sessionKey] = []models.Position{
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 1, Date: "2025-05-25T13:05:00+00:00", Position: 1},
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 1, Date: "2025-05-25T13:10:00+00:00", Position: 1},
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 44, Date: "2025-05-25T13:05:00+00:00", Position: 2},
}
src.raceControl[sessionKey] = []models.RaceControl{
{SessionKey: sessionKey, MeetingKey: meetingKey, Date: "2025-05-25T13:01:00+00:00", Category: models.CategoryFlag, Flag: models.FlagGreen, Message: "Green light"},
}
src.weather[sessionKey] = []models.Weather{
{SessionKey: sessionKey, MeetingKey: meetingKey, Date: "2025-05-25T13:00:00+00:00", AirTemperature: 22.0, TrackTemperature: 34.0},
}
src.laps[sessionKey] = []models.Lap{
{SessionKey: sessionKey, MeetingKey: meetingKey, DriverNumber: 1, LapNumber: 1, LapDuration: ptrFloat64(75.1)},
}
return meetingKey, sessionKey, src
}
func ptrFloat64(v float64) *float64 {
return &v
}
func TestIngestSessionWritesDomainAndRawRows(t *testing.T) {
_, sessionKey, src := testSessionFixtures()
st := openTestStore(t)
@@ -198,8 +283,11 @@ func TestIngestSessionWritesDomainAndRawRows(t *testing.T) {
if summary.Drivers != 2 || summary.SessionResults != 2 || summary.StartingGrid != 2 {
t.Fatalf("summary counts = %+v, want 2 drivers/results/grid", summary)
}
if summary.RawPayloads != 5 {
t.Fatalf("summary.RawPayloads = %d, want 5", summary.RawPayloads)
if summary.Stints != 2 || summary.PitStops != 1 || summary.Positions != 3 {
t.Fatalf("summary analytics counts = %+v, want stints=2 pits=1 positions=3", summary)
}
if summary.RawPayloads != 11 {
t.Fatalf("summary.RawPayloads = %d, want 11", summary.RawPayloads)
}
drivers, err := st.ListSessionDrivers(sessionKey)
@@ -230,8 +318,17 @@ func TestIngestSessionWritesDomainAndRawRows(t *testing.T) {
if err != nil {
t.Fatalf("ListRawPayloadsBySession() error = %v", err)
}
if len(raw) != 5 {
t.Fatalf("raw payloads = %d, want 5", len(raw))
if len(raw) != 11 {
t.Fatalf("raw payloads = %d, want 11", len(raw))
}
stints, err := st.ListStints(sessionKey)
if err != nil || len(stints) != 2 {
t.Fatalf("stints = %+v, err = %v, want 2", stints, err)
}
positions, err := st.ListPositionSamples(sessionKey)
if err != nil || len(positions) != 3 {
t.Fatalf("positions = %+v, err = %v, want 3", positions, err)
}
}

View File

@@ -30,6 +30,12 @@ type Source interface {
FetchDriversForSession(sessionKey int) (FetchResult, []models.Driver, error)
FetchSessionResult(sessionKey int) (FetchResult, []models.SessionResult, error)
FetchStartingGrid(sessionKey int) (FetchResult, []models.StartingGrid, error)
FetchStintsForSession(sessionKey int) (FetchResult, []models.Stint, error)
FetchPitStopsForSession(sessionKey int) (FetchResult, []models.Pit, error)
FetchPositionsForSession(sessionKey int) (FetchResult, []models.Position, error)
FetchRaceControlForSession(sessionKey int) (FetchResult, []models.RaceControl, error)
FetchWeatherForSession(sessionKey int) (FetchResult, []models.Weather, error)
FetchLapsForSession(sessionKey int) (FetchResult, []models.Lap, error)
}
// OpenF1Source adapts OpenF1Client for ingestion using strict fetches.
@@ -77,6 +83,36 @@ func (s *OpenF1Source) FetchStartingGrid(sessionKey int) (FetchResult, []models.
return s.fetchStartingGrid(url, "starting_grid", fmt.Sprintf("session_key=%d", sessionKey))
}
func (s *OpenF1Source) FetchStintsForSession(sessionKey int) (FetchResult, []models.Stint, error) {
url := fmt.Sprintf("%s/v1/stints?session_key=%d", s.client.BaseURL(), sessionKey)
return s.fetchStints(url, "stints", fmt.Sprintf("session_key=%d", sessionKey))
}
func (s *OpenF1Source) FetchPitStopsForSession(sessionKey int) (FetchResult, []models.Pit, error) {
url := fmt.Sprintf("%s/v1/pit?session_key=%d", s.client.BaseURL(), sessionKey)
return s.fetchPitStops(url, "pit", fmt.Sprintf("session_key=%d", sessionKey))
}
func (s *OpenF1Source) FetchPositionsForSession(sessionKey int) (FetchResult, []models.Position, error) {
url := fmt.Sprintf("%s/v1/position?session_key=%d", s.client.BaseURL(), sessionKey)
return s.fetchPositions(url, "position", fmt.Sprintf("session_key=%d", sessionKey))
}
func (s *OpenF1Source) FetchRaceControlForSession(sessionKey int) (FetchResult, []models.RaceControl, error) {
url := fmt.Sprintf("%s/v1/race_control?session_key=%d", s.client.BaseURL(), sessionKey)
return s.fetchRaceControl(url, "race_control", fmt.Sprintf("session_key=%d", sessionKey))
}
func (s *OpenF1Source) FetchWeatherForSession(sessionKey int) (FetchResult, []models.Weather, error) {
url := fmt.Sprintf("%s/v1/weather?session_key=%d", s.client.BaseURL(), sessionKey)
return s.fetchWeather(url, "weather", fmt.Sprintf("session_key=%d", sessionKey))
}
func (s *OpenF1Source) FetchLapsForSession(sessionKey int) (FetchResult, []models.Lap, error) {
url := fmt.Sprintf("%s/v1/laps?session_key=%d", s.client.BaseURL(), sessionKey)
return s.fetchLaps(url, "laps", fmt.Sprintf("session_key=%d", sessionKey))
}
func (s *OpenF1Source) fetchMeetings(url, endpoint, requestKey string) (FetchResult, []models.Meeting, error) {
body, err := s.client.FetchStrict(url)
if err != nil {
@@ -167,6 +203,114 @@ func (s *OpenF1Source) fetchStartingGrid(url, endpoint, requestKey string) (Fetc
}, result, nil
}
func (s *OpenF1Source) fetchStints(url, endpoint, requestKey string) (FetchResult, []models.Stint, error) {
body, err := s.client.FetchStrict(url)
if err != nil {
return FetchResult{}, nil, err
}
var result []models.Stint
if err := json.Unmarshal(body, &result); err != nil {
return FetchResult{}, nil, err
}
return FetchResult{
Endpoint: endpoint,
RequestKey: requestKey,
URL: url,
Body: body,
FetchedAt: time.Now(),
}, result, nil
}
func (s *OpenF1Source) fetchPitStops(url, endpoint, requestKey string) (FetchResult, []models.Pit, error) {
body, err := s.client.FetchStrict(url)
if err != nil {
return FetchResult{}, nil, err
}
var result []models.Pit
if err := json.Unmarshal(body, &result); err != nil {
return FetchResult{}, nil, err
}
return FetchResult{
Endpoint: endpoint,
RequestKey: requestKey,
URL: url,
Body: body,
FetchedAt: time.Now(),
}, result, nil
}
func (s *OpenF1Source) fetchPositions(url, endpoint, requestKey string) (FetchResult, []models.Position, error) {
body, err := s.client.FetchStrict(url)
if err != nil {
return FetchResult{}, nil, err
}
var result []models.Position
if err := json.Unmarshal(body, &result); err != nil {
return FetchResult{}, nil, err
}
return FetchResult{
Endpoint: endpoint,
RequestKey: requestKey,
URL: url,
Body: body,
FetchedAt: time.Now(),
}, result, nil
}
func (s *OpenF1Source) fetchRaceControl(url, endpoint, requestKey string) (FetchResult, []models.RaceControl, error) {
body, err := s.client.FetchStrict(url)
if err != nil {
return FetchResult{}, nil, err
}
var result []models.RaceControl
if err := json.Unmarshal(body, &result); err != nil {
return FetchResult{}, nil, err
}
return FetchResult{
Endpoint: endpoint,
RequestKey: requestKey,
URL: url,
Body: body,
FetchedAt: time.Now(),
}, result, nil
}
func (s *OpenF1Source) fetchWeather(url, endpoint, requestKey string) (FetchResult, []models.Weather, error) {
body, err := s.client.FetchStrict(url)
if err != nil {
return FetchResult{}, nil, err
}
var result []models.Weather
if err := json.Unmarshal(body, &result); err != nil {
return FetchResult{}, nil, err
}
return FetchResult{
Endpoint: endpoint,
RequestKey: requestKey,
URL: url,
Body: body,
FetchedAt: time.Now(),
}, result, nil
}
func (s *OpenF1Source) fetchLaps(url, endpoint, requestKey string) (FetchResult, []models.Lap, error) {
body, err := s.client.FetchStrict(url)
if err != nil {
return FetchResult{}, nil, err
}
var result []models.Lap
if err := json.Unmarshal(body, &result); err != nil {
return FetchResult{}, nil, err
}
return FetchResult{
Endpoint: endpoint,
RequestKey: requestKey,
URL: url,
Body: body,
FetchedAt: time.Now(),
}, result, nil
}
func meetingToStore(m models.Meeting) store.Meeting {
return store.Meeting{
MeetingKey: int(m.MeetingKey),
@@ -247,6 +391,101 @@ func startingGridToStore(g models.StartingGrid) store.StartingGridEntry {
}
}
func stintToStore(st models.Stint) store.Stint {
return store.Stint{
SessionKey: st.SessionKey,
DriverNumber: st.DriverNumber,
MeetingKey: st.MeetingKey,
StintNumber: st.StintNumber,
Compound: string(st.Compound),
LapStart: st.LapStart,
LapEnd: st.LapEnd,
TyreAgeAtStart: st.TyreAgeAtStart,
}
}
func pitStopToStore(p models.Pit) store.PitStop {
stopDuration := p.StopDuration
if stopDuration == 0 {
stopDuration = p.PitDuration
}
return store.PitStop{
SessionKey: p.SessionKey,
DriverNumber: p.DriverNumber,
MeetingKey: p.MeetingKey,
LapNumber: p.LapNumber,
Date: p.Date,
PitDuration: p.PitDuration,
LaneDuration: p.LaneDuration,
StopDuration: stopDuration,
}
}
func positionToStore(p models.Position) store.PositionSample {
return store.PositionSample{
SessionKey: p.SessionKey,
DriverNumber: p.DriverNumber,
MeetingKey: p.MeetingKey,
Date: p.Date,
Position: p.Position,
}
}
func raceControlToStore(rc models.RaceControl) store.RaceControlMessage {
return store.RaceControlMessage{
SessionKey: rc.SessionKey,
MeetingKey: rc.MeetingKey,
Date: rc.Date,
Category: string(rc.Category),
Flag: string(rc.Flag),
Message: rc.Message,
Scope: rc.Scope,
DriverNumber: rc.DriverNumber,
LapNumber: rc.LapNumber,
Sector: rc.Sector,
QualifyingPhase: rc.QualifyingPhase,
}
}
func weatherToStore(w models.Weather) store.WeatherSample {
return store.WeatherSample{
SessionKey: w.SessionKey,
MeetingKey: w.MeetingKey,
Date: w.Date,
AirTemperature: w.AirTemperature,
TrackTemperature: w.TrackTemperature,
Humidity: w.Humidity,
Pressure: w.Pressure,
Rainfall: w.Rainfall,
WindDirection: w.WindDirection,
WindSpeed: w.WindSpeed,
}
}
func lapToStore(l models.Lap) store.Lap {
lap := store.Lap{
SessionKey: l.SessionKey,
DriverNumber: l.DriverNumber,
MeetingKey: l.MeetingKey,
LapNumber: l.LapNumber,
DateStart: l.DateStart,
IsPitOutLap: l.IsPitOutLap,
}
if l.LapDuration != nil {
lap.LapDuration = *l.LapDuration
}
if l.DurationSector1 != nil {
lap.DurationSector1 = *l.DurationSector1
}
if l.DurationSector2 != nil {
lap.DurationSector2 = *l.DurationSector2
}
if l.DurationSector3 != nil {
lap.DurationSector3 = *l.DurationSector3
}
return lap
}
func jsonField(v any) string {
if v == nil {
return ""