also looking at this
feat: scrape Transfermarkt club valuations
#2
5 files
+131
-57
| @@ -1,15 +1,11 @@ | ||
| 1 | 1 | import type { Database } from 'bun:sqlite' |
| 2 | 2 | import process from 'node:process' |
| 3 | import { fetchHTML, parseHTML } from 'ts-web-scraper' | |
| 3 | 4 | import { loadSports, resolveTeam } from '../ingest/resolve' |
| 4 | import { fetchWithRetry, IngestRunTracker } from '../ingest/run' | |
| 5 | import { IngestRunTracker } from '../ingest/run' | |
| 5 | 6 | |
| 6 | 7 | /** |
| 7 | * Squad valuations from Transfermarkt's licensed API. | |
| 8 | * | |
| 9 | * The API, deliberately, and not the website: transfermarkt.com publishes | |
| 10 | * a robots.txt that disallows automated agents outright, so the site is | |
| 11 | * not a source this system reads. A licensed API key is a different thing | |
| 12 | * entirely, and it is the only path here. | |
| 8 | * Squad valuations from Transfermarkt's public competition pages. | |
| 13 | 9 | * |
| 14 | 10 | * What this buys is the one input that can disagree with a bookmaker |
| 15 | 11 | * rather than average it. Every other number this system prices with is |
| @@ -19,12 +15,15 @@ import { fetchWithRetry, IngestRunTracker } from '../ingest/run' | ||
| 19 | 15 | * (`KXEFLCUPGAME` alone is full of them), and squad value plus league |
| 20 | 16 | * tier is what tells those apart. |
| 21 | 17 | * |
| 22 | * Absent credentials this reports `skipped` rather than failing. The | |
| 23 | * pipeline runs every five minutes on installs that will never configure | |
| 24 | * it, and a provider that cannot run is not the same as one that broke. | |
| 18 | * The DOM is the contract here. Transfermarkt exposes the club name/id, | |
| 19 | * squad size, average age, and total market value together in each row; | |
| 20 | * keeping those fields tied to a single row prevents cross-club values | |
| 21 | * when the table is re-sorted. | |
| 25 | 22 | */ |
| 26 | 23 | |
| 27 | const BASE = process.env.TRANSFERMARKT_API_URL || 'https://api.transfermarkt.com' | |
| 24 | const BASE = process.env.TRANSFERMARKT_BASE_URL || 'https://www.transfermarkt.com' | |
| 25 | const USER_AGENT = process.env.TRANSFERMARKT_USER_AGENT | |
| 26 | || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36' | |
| 28 | 27 | |
| 29 | 28 | export interface ValuationResult { |
| 30 | 29 | provider: string |
| @@ -38,8 +37,9 @@ export interface ValuationResult { | ||
| 38 | 37 | /** The soccer competitions worth valuing, by our sport slug. */ |
| 39 | 38 | interface CompetitionTarget { |
| 40 | 39 | sportSlug: string |
| 41 | /** Provider's competition id. */ | |
| 40 | /** Transfermarkt's competition id and canonical path slug. */ | |
| 42 | 41 | externalId: string |
| 42 | path: string | |
| 43 | 43 | /** 1 = top flight of its country. */ |
| 44 | 44 | tier: number |
| 45 | 45 | label: string |
| @@ -51,15 +51,23 @@ interface CompetitionTarget { | ||
| 51 | 51 | * the kind of guess that quietly mislabels a second division as a first. |
| 52 | 52 | */ |
| 53 | 53 | const COMPETITIONS: CompetitionTarget[] = [ |
| 54 | { sportSlug: 'epl', externalId: 'GB1', tier: 1, label: 'Premier League' }, | |
| 55 | { sportSlug: 'laliga', externalId: 'ES1', tier: 1, label: 'LaLiga' }, | |
| 56 | { sportSlug: 'seriea', externalId: 'IT1', tier: 1, label: 'Serie A' }, | |
| 57 | { sportSlug: 'bundesliga', externalId: 'L1', tier: 1, label: 'Bundesliga' }, | |
| 58 | { sportSlug: 'ucl', externalId: 'CL', tier: 1, label: 'Champions League' }, | |
| 54 | { sportSlug: 'epl', externalId: 'GB1', path: 'premier-league', tier: 1, label: 'Premier League' }, | |
| 55 | { sportSlug: 'laliga', externalId: 'ES1', path: 'laliga', tier: 1, label: 'LaLiga' }, | |
| 56 | { sportSlug: 'seriea', externalId: 'IT1', path: 'serie-a', tier: 1, label: 'Serie A' }, | |
| 57 | { sportSlug: 'bundesliga', externalId: 'L1', path: 'bundesliga', tier: 1, label: 'Bundesliga' }, | |
| 58 | { sportSlug: 'ucl', externalId: 'CL', path: 'uefa-champions-league', tier: 1, label: 'Champions League' }, | |
| 59 | 59 | ] |
| 60 | 60 | |
| 61 | 61 | export function transfermarktConfigured(): boolean { |
| 62 | return Boolean(process.env.TRANSFERMARKT_API_KEY) | |
| 62 | return true | |
| 63 | } | |
| 64 | ||
| 65 | export interface TransfermarktClub { | |
| 66 | externalId: string | |
| 67 | name: string | |
| 68 | squadSize: number | |
| 69 | averageAgeYears: number | |
| 70 | marketValueEur: number | |
| 63 | 71 | } |
| 64 | 72 | |
| 65 | 73 | /** Parse '€1.20bn' / '450.00m' / '900k' into whole euros. */ |
| @@ -83,25 +91,45 @@ export function parseMarketValue(raw: unknown): number { | ||
| 83 | 91 | return Math.round(amount * scale) |
| 84 | 92 | } |
| 85 | 93 | |
| 94 | /** Read club facts from Transfermarkt's `table.items` competition DOM. */ | |
| 95 | export function parseTransfermarktClubs(html: string): TransfermarktClub[] { | |
| 96 | const document = parseHTML(html) | |
| 97 | const clubs = new Map<string, TransfermarktClub>() | |
| 98 | ||
| 99 | for (const row of document.querySelectorAll('table.items > tbody > tr')) { | |
| 100 | const nameLink = row.querySelector('td.hauptlink a[href*="/verein/"]') | |
| 101 | const href = nameLink?.getAttribute('href') || '' | |
| 102 | const externalId = href.match(/\/verein\/(\d+)/)?.[1] | |
| 103 | if (!nameLink || !externalId) | |
| 104 | continue | |
| 105 | ||
| 106 | const cells = row.children.filter(child => child.tagName.toLowerCase() === 'td') | |
| 107 | const nameCellIndex = cells.findIndex(cell => cell.querySelector('a[href*="/verein/"]') === nameLink) | |
| 108 | if (nameCellIndex < 0) | |
| 109 | continue | |
| 110 | ||
| 111 | const valueCell = row.querySelector('td.rechts.hauptlink') || cells.at(-1) | |
| 112 | const marketValueEur = parseMarketValue(valueCell?.textContent) | |
| 113 | const name = (nameLink.getAttribute('title') || nameLink.textContent).replace(/\s+/g, ' ').trim() | |
| 114 | if (!name || marketValueEur <= 0) | |
| 115 | continue | |
| 116 | ||
| 117 | clubs.set(externalId, { | |
| 118 | externalId, | |
| 119 | name, | |
| 120 | squadSize: Number.parseInt(cells[nameCellIndex + 1]?.textContent || '0', 10) || 0, | |
| 121 | averageAgeYears: Number.parseFloat(cells[nameCellIndex + 2]?.textContent || '0') || 0, | |
| 122 | marketValueEur, | |
| 123 | }) | |
| 124 | } | |
| 125 | ||
| 126 | return [...clubs.values()] | |
| 127 | } | |
| 128 | ||
| 86 | 129 | export async function ingestClubValuations(db: Database): Promise<ValuationResult> { |
| 87 | 130 | const tracker = new IngestRunTracker(db, 'transfermarkt', 'valuations') |
| 88 | 131 | tracker.start() |
| 89 | 132 | |
| 90 | const key = process.env.TRANSFERMARKT_API_KEY | |
| 91 | if (!key) { | |
| 92 | // Not an error. Recorded so `/pipeline` can say the provider is off | |
| 93 | // rather than leaving a gap that reads as a failed run. | |
| 94 | tracker.finish('skipped: TRANSFERMARKT_API_KEY is not set') | |
| 95 | return { | |
| 96 | provider: 'transfermarkt', | |
| 97 | status: 'skipped', | |
| 98 | clubsWritten: 0, | |
| 99 | competitions: 0, | |
| 100 | unmatched: 0, | |
| 101 | errors: [], | |
| 102 | } | |
| 103 | } | |
| 104 | ||
| 105 | 133 | const sportIds = new Map(loadSports(db).map(s => [s.slug, s.id])) |
| 106 | 134 | const capturedAt = new Date().toISOString() |
| 107 | 135 | |
| @@ -120,26 +148,27 @@ export async function ingestClubValuations(db: Database): Promise<ValuationResul | ||
| 120 | 148 | if (!sportId) |
| 121 | 149 | continue |
| 122 | 150 | |
| 123 | const res = await fetchWithRetry(`${BASE}/competitions/${target.externalId}/clubs`, { | |
| 124 | headers: { 'Authorization': `Bearer ${key}`, 'Accept': 'application/json' }, | |
| 125 | }) | |
| 151 | const url = `${BASE}/${target.path}/startseite/wettbewerb/${target.externalId}` | |
| 126 | 152 | tracker.requestCount++ |
| 127 | 153 | |
| 128 | if (!res) { | |
| 129 | tracker.fail(`${target.label}: unreachable`) | |
| 130 | continue | |
| 131 | } | |
| 132 | ||
| 133 | 154 | try { |
| 134 | const payload = await res.json() as any | |
| 135 | const clubs = payload?.clubs ?? payload?.data ?? [] | |
| 155 | const document = await fetchHTML(url, { | |
| 156 | timeout: 15_000, | |
| 157 | userAgent: USER_AGENT, | |
| 158 | headers: { | |
| 159 | accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', | |
| 160 | 'accept-language': 'en-US,en;q=0.9', | |
| 161 | }, | |
| 162 | }) | |
| 163 | const clubs = parseTransfermarktClubs(document.innerHTML) | |
| 164 | if (clubs.length === 0) | |
| 165 | throw new Error('club table was missing or empty') | |
| 136 | 166 | |
| 137 | 167 | db.run('BEGIN') |
| 138 | 168 | try { |
| 139 | 169 | for (const club of clubs) { |
| 140 | 170 | tracker.rowsRead++ |
| 141 | const name = String(club?.name ?? club?.clubName ?? '').trim() | |
| 142 | const teamId = name ? resolveTeam(db, sportId, name) : null | |
| 171 | const teamId = resolveTeam(db, sportId, club.name) | |
| 143 | 172 | |
| 144 | 173 | if (!teamId) { |
| 145 | 174 | tracker.unmatchedCount++ |
| @@ -148,12 +177,12 @@ export async function ingestClubValuations(db: Database): Promise<ValuationResul | ||
| 148 | 177 | |
| 149 | 178 | insert.run( |
| 150 | 179 | teamId, |
| 151 | parseMarketValue(club?.marketValue ?? club?.totalMarketValue), | |
| 152 | Number(club?.squad?.size ?? club?.squadSize ?? 0) || 0, | |
| 153 | Number(club?.squad?.averageAge ?? club?.averageAge ?? 0) || 0, | |
| 180 | club.marketValueEur, | |
| 181 | club.squadSize, | |
| 182 | club.averageAgeYears, | |
| 154 | 183 | target.tier, |
| 155 | 184 | target.label, |
| 156 | String(club?.id ?? '').slice(0, 80), | |
| 185 | club.externalId, | |
| 157 | 186 | capturedAt, |
| 158 | 187 | capturedAt, |
| 159 | 188 | capturedAt, |