mirror of
https://github.com/AmanTahiliani/box-box.git
synced 2026-08-08 04:06:18 -04:00
Add paddock briefing RSS backend spike
This commit is contained in:
260
internal/news/news.go
Normal file
260
internal/news/news.go
Normal file
@@ -0,0 +1,260 @@
|
||||
package news
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const UserAgent = "box-box/phase-19b-rss-spike"
|
||||
|
||||
// Source describes a feed that can be fetched and normalized.
|
||||
type Source struct {
|
||||
ID string
|
||||
Name string
|
||||
URL string
|
||||
Category string
|
||||
}
|
||||
|
||||
// Item is the normalized shape consumed by storage and API layers.
|
||||
type Item struct {
|
||||
Source string
|
||||
Title string
|
||||
URL string
|
||||
PublishedAt time.Time
|
||||
Summary string
|
||||
Category string
|
||||
FetchedAt time.Time
|
||||
}
|
||||
|
||||
// DefaultSources are free RSS/Atom feeds worth using for the Paddock Briefing spike.
|
||||
var DefaultSources = []Source{
|
||||
{ID: "fia", Name: "FIA", URL: "https://www.fia.com/rss/news", Category: "official"},
|
||||
{ID: "bbc-f1", Name: "BBC Sport F1", URL: "https://feeds.bbci.co.uk/sport/formula1", Category: "news"},
|
||||
{ID: "autosport-f1", Name: "Autosport F1", URL: "https://www.autosport.com/rss/f1/news/", Category: "news"},
|
||||
{ID: "racefans-f1", Name: "RaceFans F1", URL: "https://www.racefans.net/category/f1-news/feed/", Category: "news"},
|
||||
{ID: "guardian-f1", Name: "Guardian Formula One", URL: "https://www.theguardian.com/sport/formulaone/rss", Category: "news"},
|
||||
{ID: "racer-f1", Name: "RACER F1", URL: "https://racer.com/f1/feed", Category: "news"},
|
||||
{ID: "f1-youtube", Name: "Formula 1 YouTube", URL: "https://www.youtube.com/feeds/videos.xml?channel_id=UCB_qr75-ydFVKSF9Dmo6izg", Category: "video"},
|
||||
}
|
||||
|
||||
// Fetch retrieves and parses one RSS or Atom feed with the provided HTTP client.
|
||||
func Fetch(ctx context.Context, client *http.Client, source Source) ([]Item, error) {
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 10 * time.Second}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, source.URL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", UserAgent)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("fetch %s: status %d", source.ID, resp.StatusCode)
|
||||
}
|
||||
return Parse(source, resp.Body, time.Now().UTC())
|
||||
}
|
||||
|
||||
// Parse normalizes RSS 2.0 or Atom XML into Items.
|
||||
func Parse(source Source, r io.Reader, fetchedAt time.Time) ([]Item, error) {
|
||||
payload, err := io.ReadAll(io.LimitReader(r, 2<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var rss rssFeed
|
||||
if err := xml.Unmarshal(payload, &rss); err == nil && len(rss.Channel.Items) > 0 {
|
||||
return normalizeRSS(source, rss.Channel.Items, fetchedAt), nil
|
||||
}
|
||||
|
||||
var atom atomFeed
|
||||
if err := xml.Unmarshal(payload, &atom); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(atom.Entries) == 0 {
|
||||
return nil, fmt.Errorf("parse %s: no RSS items or Atom entries found", source.ID)
|
||||
}
|
||||
return normalizeAtom(source, atom.Entries, fetchedAt), nil
|
||||
}
|
||||
|
||||
// DeduplicateByURL keeps the newest instance of each canonical URL.
|
||||
func DeduplicateByURL(items []Item) []Item {
|
||||
byURL := make(map[string]Item, len(items))
|
||||
for _, item := range items {
|
||||
key := canonicalURL(item.URL)
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
item.URL = key
|
||||
if existing, ok := byURL[key]; !ok || item.PublishedAt.After(existing.PublishedAt) {
|
||||
byURL[key] = item
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]Item, 0, len(byURL))
|
||||
for _, item := range byURL {
|
||||
out = append(out, item)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return out[i].PublishedAt.After(out[j].PublishedAt)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
type rssFeed struct {
|
||||
Channel struct {
|
||||
Items []rssItem `xml:"item"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
|
||||
type rssItem struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
GUID string `xml:"guid"`
|
||||
PubDate string `xml:"pubDate"`
|
||||
Description string `xml:"description"`
|
||||
Categories []string `xml:"category"`
|
||||
}
|
||||
|
||||
type atomFeed struct {
|
||||
Entries []atomEntry `xml:"entry"`
|
||||
}
|
||||
|
||||
type atomEntry struct {
|
||||
Title string `xml:"title"`
|
||||
ID string `xml:"id"`
|
||||
Updated string `xml:"updated"`
|
||||
Published string `xml:"published"`
|
||||
Summary string `xml:"summary"`
|
||||
Content string `xml:"content"`
|
||||
Links []atomLink `xml:"link"`
|
||||
Categories []struct {
|
||||
Term string `xml:"term,attr"`
|
||||
Label string `xml:"label,attr"`
|
||||
} `xml:"category"`
|
||||
}
|
||||
|
||||
type atomLink struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Rel string `xml:"rel,attr"`
|
||||
}
|
||||
|
||||
func normalizeRSS(source Source, raw []rssItem, fetchedAt time.Time) []Item {
|
||||
items := make([]Item, 0, len(raw))
|
||||
for _, entry := range raw {
|
||||
link := strings.TrimSpace(entry.Link)
|
||||
if link == "" {
|
||||
link = strings.TrimSpace(entry.GUID)
|
||||
}
|
||||
items = append(items, Item{
|
||||
Source: source.ID,
|
||||
Title: cleanText(entry.Title),
|
||||
URL: link,
|
||||
PublishedAt: parseFeedTime(entry.PubDate),
|
||||
Summary: cleanText(entry.Description),
|
||||
Category: firstNonEmpty(entry.Categories, source.Category),
|
||||
FetchedAt: fetchedAt,
|
||||
})
|
||||
}
|
||||
return DeduplicateByURL(items)
|
||||
}
|
||||
|
||||
func normalizeAtom(source Source, raw []atomEntry, fetchedAt time.Time) []Item {
|
||||
items := make([]Item, 0, len(raw))
|
||||
for _, entry := range raw {
|
||||
category := source.Category
|
||||
if len(entry.Categories) > 0 {
|
||||
category = firstNonEmpty([]string{entry.Categories[0].Label, entry.Categories[0].Term}, source.Category)
|
||||
}
|
||||
items = append(items, Item{
|
||||
Source: source.ID,
|
||||
Title: cleanText(entry.Title),
|
||||
URL: atomEntryURL(entry),
|
||||
PublishedAt: parseFeedTime(firstNonEmpty([]string{entry.Published, entry.Updated}, "")),
|
||||
Summary: cleanText(firstNonEmpty([]string{entry.Summary, entry.Content}, "")),
|
||||
Category: category,
|
||||
FetchedAt: fetchedAt,
|
||||
})
|
||||
}
|
||||
return DeduplicateByURL(items)
|
||||
}
|
||||
|
||||
func atomEntryURL(entry atomEntry) string {
|
||||
for _, link := range entry.Links {
|
||||
if link.Rel == "" || link.Rel == "alternate" {
|
||||
return strings.TrimSpace(link.Href)
|
||||
}
|
||||
}
|
||||
if len(entry.Links) > 0 {
|
||||
return strings.TrimSpace(entry.Links[0].Href)
|
||||
}
|
||||
return strings.TrimSpace(entry.ID)
|
||||
}
|
||||
|
||||
func parseFeedTime(value string) time.Time {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
layouts := []string{
|
||||
time.RFC1123Z,
|
||||
time.RFC1123,
|
||||
time.RFC3339,
|
||||
time.RFC3339Nano,
|
||||
"Mon, 02 Jan 2006 15:04:05 -0700",
|
||||
"Mon, 2 Jan 2006 15:04:05 -0700",
|
||||
}
|
||||
for _, layout := range layouts {
|
||||
if ts, err := time.Parse(layout, value); err == nil {
|
||||
return ts.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func cleanText(value string) string {
|
||||
value = strings.TrimSpace(value)
|
||||
value = strings.ReplaceAll(value, "\n", " ")
|
||||
value = strings.ReplaceAll(value, "\t", " ")
|
||||
return strings.Join(strings.Fields(value), " ")
|
||||
}
|
||||
|
||||
func firstNonEmpty(values []string, fallback string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func canonicalURL(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return raw
|
||||
}
|
||||
parsed.Fragment = ""
|
||||
q := parsed.Query()
|
||||
for key := range q {
|
||||
if strings.HasPrefix(strings.ToLower(key), "utm_") {
|
||||
q.Del(key)
|
||||
}
|
||||
}
|
||||
parsed.RawQuery = q.Encode()
|
||||
return parsed.String()
|
||||
}
|
||||
Reference in New Issue
Block a user