also looking at this
fix: support structured DOM selectors
#847
6 files
+225
-57
Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.
| @@ -119,10 +119,10 @@ export async function fetchHTML(url: string, options: FetchHTMLOptions = {}): Pr | ||
| 119 | 119 | * Extract text content from HTML, stripping all tags |
| 120 | 120 | */ |
| 121 | 121 | export function extractText(html: string): string { |
| 122 | return html | |
| 122 | return decodeHTMLEntities(html | |
| 123 | 123 | .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') |
| 124 | 124 | .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') |
| 125 | .replace(/<[^>]+>/g, '') | |
| 125 | .replace(/<[^>]+>/g, ' ')) | |
| 126 | 126 | .replace(/\s+/g, ' ') |
| 127 | 127 | .trim() |
| 128 | 128 | } |
| @@ -329,7 +329,7 @@ function parseElement(html: string, startIndex: number, parent: ParsedElement | | ||
| 329 | 329 | parseChildren(innerHtml, element) |
| 330 | 330 | } |
| 331 | 331 | else { |
| 332 | element.textContent = innerHtml.trim() | |
| 332 | element.textContent = decodeHTMLEntities(innerHtml).trim() | |
| 333 | 333 | } |
| 334 | 334 | |
| 335 | 335 | lastIndex = closingIndex + closingMatch[0].length |
| @@ -397,7 +397,7 @@ function parseChildren(html: string, parent: ParsedElement): void { | ||
| 397 | 397 | parseChildren(innerHtml, element) |
| 398 | 398 | } |
| 399 | 399 | else { |
| 400 | element.textContent = innerHtml.trim() | |
| 400 | element.textContent = decodeHTMLEntities(innerHtml).trim() | |
| 401 | 401 | } |
| 402 | 402 | |
| 403 | 403 | lastIndex = tagRegex.lastIndex + closingMatch.index + closingMatch[0].length |
| @@ -420,7 +420,7 @@ function parseAttributes(attrString: string): Record<string, string> { | ||
| 420 | 420 | while (match !== null) { |
| 421 | 421 | const name = match[1] |
| 422 | 422 | const value = match[2] || match[3] || '' |
| 423 | attributes[name] = value | |
| 423 | attributes[name] = decodeHTMLEntities(value) | |
| 424 | 424 | match = attrRegex.exec(attrString) |
| 425 | 425 | } |
| 426 | 426 | |
| @@ -435,73 +435,117 @@ function querySelector(element: HTMLElement, selector: string): HTMLElement | nu | ||
| 435 | 435 | function querySelectorAll(element: HTMLElement, selector: string): HTMLElement[] { |
| 436 | 436 | const results: HTMLElement[] = [] |
| 437 | 437 | |
| 438 | // Simple selector parsing (supports tag, class, id, and attribute selectors) | |
| 439 | const parts = selector.match(/([.#[]?)([^\s.#[]+)/g) || [] | |
| 438 | for (const group of splitSelectorGroups(selector)) { | |
| 439 | const steps = parseSelectorSteps(group) | |
| 440 | let candidates = [element] | |
| 440 | 441 | |
| 441 | for (const part of parts) { | |
| 442 | if (part.startsWith('#')) { | |
| 443 | // ID selector | |
| 444 | const id = part.slice(1) | |
| 445 | const found = element.getElementById(id) | |
| 446 | if (found) | |
| 447 | results.push(found) | |
| 448 | } | |
| 449 | else if (part.startsWith('.')) { | |
| 450 | // Class selector | |
| 451 | const className = part.slice(1) | |
| 452 | results.push(...element.getElementsByClassName(className)) | |
| 453 | } | |
| 454 | else if (part.startsWith('[')) { | |
| 455 | // Attribute selector | |
| 456 | const attrMatch = part.match(/\[([\w:-]+)(?:=["']?([^"'\]]+)["']?)?\]/) | |
| 457 | if (attrMatch) { | |
| 458 | const [, attrName, attrValue] = attrMatch | |
| 459 | results.push(...filterByAttribute(element, attrName, attrValue)) | |
| 442 | for (const [index, step] of steps.entries()) { | |
| 443 | const next: HTMLElement[] = [] | |
| 444 | ||
| 445 | for (const candidate of candidates) { | |
| 446 | const pool = step.combinator === 'child' && index > 0 | |
| 447 | ? candidate.children | |
| 448 | : descendants(candidate) | |
| 449 | ||
| 450 | for (const item of pool) { | |
| 451 | if (matchesSimpleSelector(item, step.selector)) | |
| 452 | next.push(item) | |
| 453 | } | |
| 460 | 454 | } |
| 455 | ||
| 456 | candidates = uniqueElements(next) | |
| 457 | } | |
| 458 | ||
| 459 | results.push(...candidates) | |
| 460 | } | |
| 461 | ||
| 462 | return uniqueElements(results) | |
| 463 | } | |
| 464 | ||
| 465 | type SelectorCombinator = 'descendant' | 'child' | |
| 466 | ||
| 467 | interface SelectorStep { | |
| 468 | combinator: SelectorCombinator | |
| 469 | selector: string | |
| 470 | } | |
| 471 | ||
| 472 | function splitSelectorGroups(selector: string): string[] { | |
| 473 | const groups: string[] = [] | |
| 474 | let current = '' | |
| 475 | let bracketDepth = 0 | |
| 476 | let quote = '' | |
| 477 | ||
| 478 | for (const character of selector) { | |
| 479 | if ((character === '"' || character === '\'') && (!quote || quote === character)) | |
| 480 | quote = quote ? '' : character | |
| 481 | else if (!quote && character === '[') | |
| 482 | bracketDepth++ | |
| 483 | else if (!quote && character === ']') | |
| 484 | bracketDepth-- | |
| 485 | ||
| 486 | if (character === ',' && bracketDepth === 0 && !quote) { | |
| 487 | if (current.trim()) | |
| 488 | groups.push(current.trim()) | |
| 489 | current = '' | |
| 461 | 490 | } |
| 462 | 491 | else { |
| 463 | // Tag selector | |
| 464 | results.push(...element.getElementsByTagName(part)) | |
| 492 | current += character | |
| 465 | 493 | } |
| 466 | 494 | } |
| 467 | 495 | |
| 468 | // Handle compound selectors (e.g., "a.className" or "div#id") | |
| 469 | if (selector.includes('.') || selector.includes('#') || selector.includes('[')) { | |
| 470 | return results.filter((el) => { | |
| 471 | if (selector.includes('.')) { | |
| 472 | const classNames = selector.match(/\.([^\s.#[]+)/g)?.map(c => c.slice(1)) || [] | |
| 473 | const elementClasses = el.attributes.class?.split(/\s+/) || [] | |
| 474 | if (!classNames.every(cn => elementClasses.includes(cn))) | |
| 475 | return false | |
| 476 | } | |
| 496 | if (current.trim()) | |
| 497 | groups.push(current.trim()) | |
| 477 | 498 | |
| 478 | if (selector.includes('#')) { | |
| 479 | const idMatch = selector.match(/#([^\s.#[]+)/) | |
| 480 | if (idMatch && el.attributes.id !== idMatch[1]) | |
| 481 | return false | |
| 482 | } | |
| 499 | return groups | |
| 500 | } | |
| 483 | 501 | |
| 484 | const tagMatch = selector.match(/^([A-Za-z][\w:-]*)/) | |
| 485 | if (tagMatch && el.tagName.toLowerCase() !== tagMatch[1].toLowerCase()) | |
| 486 | return false | |
| 502 | function parseSelectorSteps(selector: string): SelectorStep[] { | |
| 503 | const steps: SelectorStep[] = [] | |
| 504 | let current = '' | |
| 505 | let bracketDepth = 0 | |
| 506 | let quote = '' | |
| 507 | let pendingCombinator: SelectorCombinator = 'descendant' | |
| 508 | ||
| 509 | const pushCurrent = (): void => { | |
| 510 | const value = current.trim() | |
| 511 | if (value) | |
| 512 | steps.push({ combinator: pendingCombinator, selector: value }) | |
| 513 | current = '' | |
| 514 | } | |
| 487 | 515 | |
| 488 | return true | |
| 489 | }) | |
| 516 | for (const character of selector) { | |
| 517 | if ((character === '"' || character === '\'') && (!quote || quote === character)) | |
| 518 | quote = quote ? '' : character | |
| 519 | else if (!quote && character === '[') | |
| 520 | bracketDepth++ | |
| 521 | else if (!quote && character === ']') | |
| 522 | bracketDepth-- | |
| 523 | ||
| 524 | if (!quote && bracketDepth === 0 && character === '>') { | |
| 525 | pushCurrent() | |
| 526 | pendingCombinator = 'child' | |
| 527 | } | |
| 528 | else if (!quote && bracketDepth === 0 && /\s/.test(character)) { | |
| 529 | if (current.trim()) { | |
| 530 | pushCurrent() | |
| 531 | pendingCombinator = 'descendant' | |
| 532 | } | |
| 533 | } | |
| 534 | else { | |
| 535 | current += character | |
| 536 | } | |
| 490 | 537 | } |
| 491 | 538 | |
| 492 | return results | |
| 539 | pushCurrent() | |
| 540 | return steps | |
| 493 | 541 | } |
| 494 | 542 | |
| 495 | function filterByAttribute(element: HTMLElement, attrName: string, attrValue?: string): HTMLElement[] { | |
| 543 | function descendants(element: HTMLElement): HTMLElement[] { | |
| 496 | 544 | const results: HTMLElement[] = [] |
| 497 | 545 | |
| 498 | 546 | function traverse(el: HTMLElement): void { |
| 499 | if (el.hasAttribute(attrName)) { | |
| 500 | if (!attrValue || el.getAttribute(attrName) === attrValue) { | |
| 501 | results.push(el) | |
| 502 | } | |
| 503 | } | |
| 504 | 547 | for (const child of el.children) { |
| 548 | results.push(child) | |
| 505 | 549 | traverse(child) |
| 506 | 550 | } |
| 507 | 551 | } |
| @@ -510,6 +554,72 @@ function filterByAttribute(element: HTMLElement, attrName: string, attrValue?: s | ||
| 510 | 554 | return results |
| 511 | 555 | } |
| 512 | 556 | |
| 557 | function matchesSimpleSelector(element: HTMLElement, selector: string): boolean { | |
| 558 | const tagMatch = selector.match(/^([A-Za-z][\w:-]*|\*)/) | |
| 559 | if (tagMatch && tagMatch[1] !== '*' && element.tagName.toLowerCase() !== tagMatch[1].toLowerCase()) | |
| 560 | return false | |
| 561 | ||
| 562 | const idMatches = [...selector.matchAll(/#([\w-]+)/g)] | |
| 563 | if (idMatches.some(match => element.getAttribute('id') !== match[1])) | |
| 564 | return false | |
| 565 | ||
| 566 | const elementClasses = element.getAttribute('class')?.split(/\s+/) || [] | |
| 567 | const classMatches = [...selector.matchAll(/\.([\w-]+)/g)] | |
| 568 | if (classMatches.some(match => !elementClasses.includes(match[1]))) | |
| 569 | return false | |
| 570 | ||
| 571 | const attributeMatches = [...selector.matchAll(/\[([\w:-]+)(?:\s*(\^=|\$=|\*=|~=|\|=|=)\s*["']?([^"'\]]*)["']?)?\]/g)] | |
| 572 | for (const match of attributeMatches) { | |
| 573 | const [, name, operator, expected] = match | |
| 574 | if (!element.hasAttribute(name)) | |
| 575 | return false | |
| 576 | if (!operator) | |
| 577 | continue | |
| 578 | ||
| 579 | const actual = element.getAttribute(name) || '' | |
| 580 | if (operator === '=' && actual !== expected) | |
| 581 | return false | |
| 582 | if (operator === '*=' && !actual.includes(expected)) | |
| 583 | return false | |
| 584 | if (operator === '^=' && !actual.startsWith(expected)) | |
| 585 | return false | |
| 586 | if (operator === '$=' && !actual.endsWith(expected)) | |
| 587 | return false | |
| 588 | if (operator === '~=' && !actual.split(/\s+/).includes(expected)) | |
| 589 | return false | |
| 590 | if (operator === '|=' && actual !== expected && !actual.startsWith(`${expected}-`)) | |
| 591 | return false | |
| 592 | } | |
| 593 | ||
| 594 | return Boolean(tagMatch || idMatches.length || classMatches.length || attributeMatches.length) | |
| 595 | } | |
| 596 | ||
| 597 | function uniqueElements(elements: HTMLElement[]): HTMLElement[] { | |
| 598 | return [...new Set(elements)] | |
| 599 | } | |
| 600 | ||
| 601 | function decodeHTMLEntities(value: string): string { | |
| 602 | const namedEntities: Record<string, string> = { | |
| 603 | amp: '&', | |
| 604 | apos: '\'', | |
| 605 | gt: '>', | |
| 606 | lt: '<', | |
| 607 | nbsp: '\u00A0', | |
| 608 | quot: '"', | |
| 609 | } | |
| 610 | ||
| 611 | return value.replace(/&(#x[\da-f]+|#\d+|[a-z]+);/gi, (entity, code: string) => { | |
| 612 | if (code[0] !== '#') | |
| 613 | return namedEntities[code.toLowerCase()] ?? entity | |
| 614 | ||
| 615 | const numeric = code[1].toLowerCase() === 'x' | |
| 616 | ? Number.parseInt(code.slice(2), 16) | |
| 617 | : Number.parseInt(code.slice(1), 10) | |
| 618 | ||
| 619 | return Number.isFinite(numeric) ? String.fromCodePoint(numeric) : entity | |
| 620 | }) | |
| 621 | } | |
| 622 | ||
| 513 | 623 | /** |
| 514 | 624 | * Helper to wait for a condition (useful for client-side rendered content) |
| 515 | 625 | * Note: This is a simple polling implementation since we can't execute JavaScript |
| @@ -117,6 +117,30 @@ describe('Web Scraper - Static HTML Parsing', () => { | ||
| 117 | 117 | expect(doc.querySelector('ytd-display-ad-renderer')?.textContent).toContain('Ad') |
| 118 | 118 | expect(doc.querySelector('[data-ad-slot]')?.tagName).toBe('ytd-display-ad-renderer') |
| 119 | 119 | }) |
| 120 | ||
| 121 | it('should follow descendant and child combinators for table rows', () => { | |
| 122 | const doc = parseHTML(` | |
| 123 | <table class="items"> | |
| 124 | <tbody> | |
| 125 | <tr class="odd"><td><a href="/club/startseite/verein/31">Club</a></td></tr> | |
| 126 | <tr class="even"><td><a href="/club/startseite/verein/42">Other</a></td></tr> | |
| 127 | </tbody> | |
| 128 | </table> | |
| 129 | `) | |
| 130 | ||
| 131 | expect(doc.querySelectorAll('table.items > tbody > tr')).toHaveLength(2) | |
| 132 | expect(doc.querySelectorAll('table.items a[href*="/verein/"]')).toHaveLength(2) | |
| 133 | expect(doc.querySelector('tr.odd > td > a')?.getAttribute('href')).toContain('/verein/31') | |
| 134 | }) | |
| 135 | ||
| 136 | it('should decode entities in text and attributes', () => { | |
| 137 | const doc = parseHTML('<a title="Brighton & Hove Albion">Brighton & Hove Albion €1.2bn</a>') | |
| 138 | const link = doc.querySelector('a') | |
| 139 | ||
| 140 | expect(link?.getAttribute('title')).toBe('Brighton & Hove Albion') | |
| 141 | expect(link?.textContent).toBe('Brighton & Hove\u00A0Albion €1.2bn') | |
| 142 | expect(extractText('<span>one</span><span>two</span>')).toBe('one two') | |
| 143 | }) | |
| 120 | 144 | }) |
| 121 | 145 | |
| 122 | 146 | describe('Client-Side Scraper - Detection', () => { |