feat(paddock): initial paddock briefing pages and store changes

internal/news/refresh.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/AmanTahiliani/box-box/internal/store"
@@ -22,12 +23,14 @@ type Store interface {
 
 // RefreshOptions configures one local news refresh run.
 type RefreshOptions struct {
-	Sources  []Source
-	Client   *http.Client
-	TTL      time.Duration
-	DryRun   bool
-	Now      func() time.Time
-	Progress io.Writer
+	Sources    []Source
+	Client     *http.Client
+	TTL        time.Duration
+	DryRun     bool
+	Now        func() time.Time
+	Progress   io.Writer
+	EnrichOG   bool // fetch og:image and og:description for each new item
+	OGParallel int  // max concurrent OG fetches (default 5)
 }
 
 // RefreshResult summarizes one local news refresh run.
@@ -52,6 +55,9 @@ func Refresh(ctx context.Context, st Store, opts RefreshOptions) (RefreshResult,
 	if opts.TTL <= 0 {
 		opts.TTL = DefaultTTL
 	}
+	if opts.OGParallel <= 0 {
+		opts.OGParallel = 5
+	}
 	now := func() time.Time { return time.Now().UTC() }
 	if opts.Now != nil {
 		now = func() time.Time { return opts.Now().UTC() }
@@ -59,6 +65,10 @@ func Refresh(ctx context.Context, st Store, opts RefreshOptions) (RefreshResult,
 
 	var result RefreshResult
 	var failures []string
+
+	// Collect all items across sources for OG enrichment.
+	var allItems []Item
+
 	for _, source := range opts.Sources {
 		fetchedAt := now()
 		expiresAt := fetchedAt.Add(opts.TTL)
@@ -86,6 +96,7 @@ func Refresh(ctx context.Context, st Store, opts RefreshOptions) (RefreshResult,
 		result.ItemsFetched += len(items)
 		progressf(opts.Progress, "news: %s fetched %d items\n", source.ID, len(items))
 		if opts.DryRun {
+			allItems = append(allItems, items...)
 			continue
 		}
 
@@ -101,22 +112,40 @@ func Refresh(ctx context.Context, st Store, opts RefreshOptions) (RefreshResult,
 		}); err != nil {
 			return result, err
 		}
-		for _, item := range items {
-			item.FetchedAt = fetchedAt
-			publishedAt := timePtr(item.PublishedAt)
-			if err := st.UpsertNewsItem(store.NewsItem{
-				URL:         item.URL,
-				Source:      item.Source,
-				Title:       item.Title,
-				PublishedAt: publishedAt,
-				Summary:     item.Summary,
-				Category:    item.Category,
-				FetchedAt:   item.FetchedAt,
-			}); err != nil {
-				return result, err
-			}
-			result.ItemsUpserted++
+
+		for i := range items {
+			items[i].FetchedAt = fetchedAt
 		}
+		allItems = append(allItems, items...)
+	}
+
+	// OG enrichment pass: fetch og:image + og:description for each item.
+	if opts.EnrichOG && len(allItems) > 0 {
+		progressf(opts.Progress, "news: enriching %d items with OG metadata\n", len(allItems))
+		enrichOG(ctx, opts.Client, allItems, opts.OGParallel, opts.Progress)
+	}
+
+	if opts.DryRun {
+		return result, nil
+	}
+
+	// Store all items (with OG data already populated).
+	for _, item := range allItems {
+		publishedAt := timePtr(item.PublishedAt)
+		if err := st.UpsertNewsItem(store.NewsItem{
+			URL:           item.URL,
+			Source:        item.Source,
+			Title:         item.Title,
+			PublishedAt:   publishedAt,
+			Summary:       item.Summary,
+			Category:      item.Category,
+			FetchedAt:     item.FetchedAt,
+			OGImageURL:    item.OGImageURL,
+			OGDescription: item.OGDescription,
+		}); err != nil {
+			return result, err
+		}
+		result.ItemsUpserted++
 	}
 
 	if len(failures) > 0 {
@@ -125,6 +154,38 @@ func Refresh(ctx context.Context, st Store, opts RefreshOptions) (RefreshResult,
 	return result, nil
 }
 
+// enrichOG fetches og:image and og:description for each item concurrently.
+func enrichOG(ctx context.Context, client *http.Client, items []Item, parallel int, progress io.Writer) {
+	sem := make(chan struct{}, parallel)
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+
+	ogClient := &http.Client{Timeout: 8 * time.Second}
+	if client != nil {
+		ogClient = &http.Client{Timeout: 8 * time.Second, Transport: client.Transport}
+	}
+
+	for i := range items {
+		wg.Add(1)
+		go func(idx int) {
+			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
+
+			imgURL, desc, err := FetchOGMeta(ctx, ogClient, items[idx].URL)
+			if err != nil {
+				progressf(progress, "news: og fetch failed %s: %v\n", items[idx].URL, err)
+				return
+			}
+			mu.Lock()
+			items[idx].OGImageURL = imgURL
+			items[idx].OGDescription = desc
+			mu.Unlock()
+		}(i)
+	}
+	wg.Wait()
+}
+
 func progressf(w io.Writer, format string, args ...any) {
 	if w == nil {
 		return