ReviewOS
Reload

also looking at this

chris/predicthq

feat: scrape Transfermarkt club valuations

#2
Merged chris wants to merge codex/transfermarkt-dom into main
5 files +131 -57

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

.env.examplemodified+5-6
@@ -159,9 +159,8 @@ ODDS_API_KEY=
159159# host from the page it loaded, so only the port belongs here.
160160BROADCAST_PORT=6001
161161
162# Transfermarkt's licensed API, for squad valuations and league tiers.
163# The API only: transfermarkt.com's robots.txt disallows automated agents,
164# so the website is not a source this app reads. Without a key, soccer
165# fixtures carry no independent read on how mismatched they are.
166TRANSFERMARKT_API_KEY=
167# TRANSFERMARKT_API_URL=https://api.transfermarkt.com
162# Transfermarkt's public competition pages supply squad valuations and league
163# tiers. Override these only when routing the authorized DOM requests through
164# a different origin or user agent.
165# TRANSFERMARKT_BASE_URL=https://www.transfermarkt.com
166# TRANSFERMARKT_USER_AGENT=Mozilla/5.0 ...
app/Services/fundamentals/transfermarkt.tsmodified+78-49
@@ -1,15 +1,11 @@
11import type { Database } from 'bun:sqlite'
22import process from 'node:process'
3import { fetchHTML, parseHTML } from 'ts-web-scraper'
34import { loadSports, resolveTeam } from '../ingest/resolve'
4import { fetchWithRetry, IngestRunTracker } from '../ingest/run'
5import { IngestRunTracker } from '../ingest/run'
56
67/**
7 * Squad valuations from Transfermarkt's licensed API.
8 *
9 * The API, deliberately, and not the website: transfermarkt.com publishes
10 * a robots.txt that disallows automated agents outright, so the site is
11 * not a source this system reads. A licensed API key is a different thing
12 * entirely, and it is the only path here.
8 * Squad valuations from Transfermarkt's public competition pages.
139 *
1410 * What this buys is the one input that can disagree with a bookmaker
1511 * rather than average it. Every other number this system prices with is
@@ -19,12 +15,15 @@ import { fetchWithRetry, IngestRunTracker } from '../ingest/run'
1915 * (`KXEFLCUPGAME` alone is full of them), and squad value plus league
2016 * tier is what tells those apart.
2117 *
22 * Absent credentials this reports `skipped` rather than failing. The
23 * pipeline runs every five minutes on installs that will never configure
24 * it, and a provider that cannot run is not the same as one that broke.
18 * The DOM is the contract here. Transfermarkt exposes the club name/id,
19 * squad size, average age, and total market value together in each row;
20 * keeping those fields tied to a single row prevents cross-club values
21 * when the table is re-sorted.
2522 */
2623
27const BASE = process.env.TRANSFERMARKT_API_URL || 'https://api.transfermarkt.com'
24const BASE = process.env.TRANSFERMARKT_BASE_URL || 'https://www.transfermarkt.com'
25const USER_AGENT = process.env.TRANSFERMARKT_USER_AGENT
26 || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36'
2827
2928export interface ValuationResult {
3029 provider: string
@@ -38,8 +37,9 @@ export interface ValuationResult {
3837/** The soccer competitions worth valuing, by our sport slug. */
3938interface CompetitionTarget {
4039 sportSlug: string
41 /** Provider's competition id. */
40 /** Transfermarkt's competition id and canonical path slug. */
4241 externalId: string
42 path: string
4343 /** 1 = top flight of its country. */
4444 tier: number
4545 label: string
@@ -51,15 +51,23 @@ interface CompetitionTarget {
5151 * the kind of guess that quietly mislabels a second division as a first.
5252 */
5353const COMPETITIONS: CompetitionTarget[] = [
54 { sportSlug: 'epl', externalId: 'GB1', tier: 1, label: 'Premier League' },
55 { sportSlug: 'laliga', externalId: 'ES1', tier: 1, label: 'LaLiga' },
56 { sportSlug: 'seriea', externalId: 'IT1', tier: 1, label: 'Serie A' },
57 { sportSlug: 'bundesliga', externalId: 'L1', tier: 1, label: 'Bundesliga' },
58 { sportSlug: 'ucl', externalId: 'CL', tier: 1, label: 'Champions League' },
54 { sportSlug: 'epl', externalId: 'GB1', path: 'premier-league', tier: 1, label: 'Premier League' },
55 { sportSlug: 'laliga', externalId: 'ES1', path: 'laliga', tier: 1, label: 'LaLiga' },
56 { sportSlug: 'seriea', externalId: 'IT1', path: 'serie-a', tier: 1, label: 'Serie A' },
57 { sportSlug: 'bundesliga', externalId: 'L1', path: 'bundesliga', tier: 1, label: 'Bundesliga' },
58 { sportSlug: 'ucl', externalId: 'CL', path: 'uefa-champions-league', tier: 1, label: 'Champions League' },
5959]
6060
6161export function transfermarktConfigured(): boolean {
62 return Boolean(process.env.TRANSFERMARKT_API_KEY)
62 return true
63}
64
65export interface TransfermarktClub {
66 externalId: string
67 name: string
68 squadSize: number
69 averageAgeYears: number
70 marketValueEur: number
6371}
6472
6573/** Parse '€1.20bn' / '450.00m' / '900k' into whole euros. */
@@ -83,25 +91,45 @@ export function parseMarketValue(raw: unknown): number {
8391 return Math.round(amount * scale)
8492}
8593
94/** Read club facts from Transfermarkt's `table.items` competition DOM. */
95export function parseTransfermarktClubs(html: string): TransfermarktClub[] {
96 const document = parseHTML(html)
97 const clubs = new Map<string, TransfermarktClub>()
98
99 for (const row of document.querySelectorAll('table.items > tbody > tr')) {
100 const nameLink = row.querySelector('td.hauptlink a[href*="/verein/"]')
101 const href = nameLink?.getAttribute('href') || ''
102 const externalId = href.match(/\/verein\/(\d+)/)?.[1]
103 if (!nameLink || !externalId)
104 continue
105
106 const cells = row.children.filter(child => child.tagName.toLowerCase() === 'td')
107 const nameCellIndex = cells.findIndex(cell => cell.querySelector('a[href*="/verein/"]') === nameLink)
108 if (nameCellIndex < 0)
109 continue
110
111 const valueCell = row.querySelector('td.rechts.hauptlink') || cells.at(-1)
112 const marketValueEur = parseMarketValue(valueCell?.textContent)
113 const name = (nameLink.getAttribute('title') || nameLink.textContent).replace(/\s+/g, ' ').trim()
114 if (!name || marketValueEur <= 0)
115 continue
116
117 clubs.set(externalId, {
118 externalId,
119 name,
120 squadSize: Number.parseInt(cells[nameCellIndex + 1]?.textContent || '0', 10) || 0,
121 averageAgeYears: Number.parseFloat(cells[nameCellIndex + 2]?.textContent || '0') || 0,
122 marketValueEur,
123 })
124 }
125
126 return [...clubs.values()]
127}
128
86129export async function ingestClubValuations(db: Database): Promise<ValuationResult> {
87130 const tracker = new IngestRunTracker(db, 'transfermarkt', 'valuations')
88131 tracker.start()
89132
90 const key = process.env.TRANSFERMARKT_API_KEY
91 if (!key) {
92 // Not an error. Recorded so `/pipeline` can say the provider is off
93 // rather than leaving a gap that reads as a failed run.
94 tracker.finish('skipped: TRANSFERMARKT_API_KEY is not set')
95 return {
96 provider: 'transfermarkt',
97 status: 'skipped',
98 clubsWritten: 0,
99 competitions: 0,
100 unmatched: 0,
101 errors: [],
102 }
103 }
104
105133 const sportIds = new Map(loadSports(db).map(s => [s.slug, s.id]))
106134 const capturedAt = new Date().toISOString()
107135
@@ -120,26 +148,27 @@ export async function ingestClubValuations(db: Database): Promise<ValuationResul
120148 if (!sportId)
121149 continue
122150
123 const res = await fetchWithRetry(`${BASE}/competitions/${target.externalId}/clubs`, {
124 headers: { 'Authorization': `Bearer ${key}`, 'Accept': 'application/json' },
125 })
151 const url = `${BASE}/${target.path}/startseite/wettbewerb/${target.externalId}`
126152 tracker.requestCount++
127153
128 if (!res) {
129 tracker.fail(`${target.label}: unreachable`)
130 continue
131 }
132
133154 try {
134 const payload = await res.json() as any
135 const clubs = payload?.clubs ?? payload?.data ?? []
155 const document = await fetchHTML(url, {
156 timeout: 15_000,
157 userAgent: USER_AGENT,
158 headers: {
159 accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
160 'accept-language': 'en-US,en;q=0.9',
161 },
162 })
163 const clubs = parseTransfermarktClubs(document.innerHTML)
164 if (clubs.length === 0)
165 throw new Error('club table was missing or empty')
136166
137167 db.run('BEGIN')
138168 try {
139169 for (const club of clubs) {
140170 tracker.rowsRead++
141 const name = String(club?.name ?? club?.clubName ?? '').trim()
142 const teamId = name ? resolveTeam(db, sportId, name) : null
171 const teamId = resolveTeam(db, sportId, club.name)
143172
144173 if (!teamId) {
145174 tracker.unmatchedCount++
@@ -148,12 +177,12 @@ export async function ingestClubValuations(db: Database): Promise<ValuationResul
148177
149178 insert.run(
150179 teamId,
151 parseMarketValue(club?.marketValue ?? club?.totalMarketValue),
152 Number(club?.squad?.size ?? club?.squadSize ?? 0) || 0,
153 Number(club?.squad?.averageAge ?? club?.averageAge ?? 0) || 0,
180 club.marketValueEur,
181 club.squadSize,
182 club.averageAgeYears,
154183 target.tier,
155184 target.label,
156 String(club?.id ?? '').slice(0, 80),
185 club.externalId,
157186 capturedAt,
158187 capturedAt,
159188 capturedAt,
tests/unit/mismatch.test.tsmodified+31-1
@@ -1,7 +1,7 @@
11import { describe, expect, it } from 'bun:test'
22import { assessMismatch, type TeamFundamentals } from '../../app/Services/fundamentals/mismatch'
33import { injurySeverity } from '../../app/Services/fundamentals/severity'
4import { parseMarketValue } from '../../app/Services/fundamentals/transfermarkt'
4import { parseMarketValue, parseTransfermarktClubs } from '../../app/Services/fundamentals/transfermarkt'
55
66/**
77 * The mismatch read is the only input in this system that can disagree
@@ -169,3 +169,33 @@ describe('parseMarketValue', () => {
169169 expect(parseMarketValue(null)).toBe(0)
170170 })
171171})
172
173describe('parseTransfermarktClubs', () => {
174 it('keeps every club fact attached to its Transfermarkt DOM row', () => {
175 const clubs = parseTransfermarktClubs(`
176 <table class="items">
177 <tbody>
178 <tr><td colspan="7">Clubs - Premier League</td></tr>
179 <tr class="odd">
180 <td class="zentriert"><a href="/brighton/startseite/verein/123"><img alt="Brighton"></a></td>
181 <td class="hauptlink no-border-links"><a title="Brighton &amp; Hove Albion" href="/brighton/startseite/verein/123/saison_id/2026">Brighton &amp; Hove Albion</a></td>
182 <td class="zentriert">33</td><td class="zentriert">26.6</td><td class="zentriert">21</td>
183 <td class="rechts">€16.73m</td><td class="rechts hauptlink"><a href="/brighton/marktwert/verein/123">€552.00m</a></td>
184 </tr>
185 <tr class="even">
186 <td class="zentriert"><a href="/arsenal/startseite/verein/11"><img alt="Arsenal"></a></td>
187 <td class="hauptlink no-border-links"><a title="Arsenal FC" href="/arsenal/startseite/verein/11/saison_id/2026">Arsenal FC</a></td>
188 <td class="zentriert">24</td><td class="zentriert">26.3</td><td class="zentriert">17</td>
189 <td class="rechts">€51.21m</td><td class="rechts hauptlink"><a href="/arsenal/marktwert/verein/11">€1.23bn</a></td>
190 </tr>
191 </tbody>
192 </table>
193 <table class="items"><tbody><tr><td class="hauptlink"><a href="/player/profil/spieler/9">Player</a></td></tr></tbody></table>
194 `)
195
196 expect(clubs).toEqual([
197 { externalId: '123', name: 'Brighton & Hove Albion', squadSize: 33, averageAgeYears: 26.6, marketValueEur: 552_000_000 },
198 { externalId: '11', name: 'Arsenal FC', squadSize: 24, averageAgeYears: 26.3, marketValueEur: 1_230_000_000 },
199 ])
200 })
201})