| @@ -119,10 +119,10 @@ export async function fetchHTML(url: string, options: FetchHTMLOptions = {}): Pr |
| 119 | 119 | * Extract text content from HTML, stripping all tags |
| 120 | 120 | */ |
| 121 | 121 | export function extractText(html: string): string { |
| 122 | | - return html |
| 122 | + return decodeHTMLEntities(html |
| 123 | 123 | .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') |
| 124 | 124 | .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') |
| 125 | | - .replace(/<[^>]+>/g, '') |
| 125 | + .replace(/<[^>]+>/g, ' ')) |
| 126 | 126 | .replace(/\s+/g, ' ') |
| 127 | 127 | .trim() |
| 128 | 128 | } |
| @@ -329,7 +329,7 @@ function parseElement(html: string, startIndex: number, parent: ParsedElement | |
| 329 | 329 | parseChildren(innerHtml, element) |
| 330 | 330 | } |
| 331 | 331 | else { |
| 332 | | - element.textContent = innerHtml.trim() |
| 332 | + element.textContent = decodeHTMLEntities(innerHtml).trim() |
| 333 | 333 | } |
| 334 | 334 | |
| 335 | 335 | lastIndex = closingIndex + closingMatch[0].length |
| @@ -397,7 +397,7 @@ function parseChildren(html: string, parent: ParsedElement): void { |
| 397 | 397 | parseChildren(innerHtml, element) |
| 398 | 398 | } |
| 399 | 399 | else { |
| 400 | | - element.textContent = innerHtml.trim() |
| 400 | + element.textContent = decodeHTMLEntities(innerHtml).trim() |
| 401 | 401 | } |
| 402 | 402 | |
| 403 | 403 | lastIndex = tagRegex.lastIndex + closingMatch.index + closingMatch[0].length |
| @@ -420,7 +420,7 @@ function parseAttributes(attrString: string): Record<string, string> { |
| 420 | 420 | while (match !== null) { |
| 421 | 421 | const name = match[1] |
| 422 | 422 | const value = match[2] || match[3] || '' |
| 423 | | - attributes[name] = value |
| 423 | + attributes[name] = decodeHTMLEntities(value) |
| 424 | 424 | match = attrRegex.exec(attrString) |
| 425 | 425 | } |
| 426 | 426 | |
| @@ -435,73 +435,117 @@ function querySelector(element: HTMLElement, selector: string): HTMLElement | nu |
| 435 | 435 | function querySelectorAll(element: HTMLElement, selector: string): HTMLElement[] { |
| 436 | 436 | const results: HTMLElement[] = [] |
| 437 | 437 | |
| 438 | | - |
| 439 | | - const parts = selector.match(/([.#[]?)([^\s.#[]+)/g) || [] |
| 438 | + for (const group of splitSelectorGroups(selector)) { |
| 439 | + const steps = parseSelectorSteps(group) |
| 440 | + let candidates = [element] |
| 440 | 441 | |
| 441 | | - for (const part of parts) { |
| 442 | | - if (part.startsWith('#')) { |
| 443 | | - |
| 444 | | - const id = part.slice(1) |
| 445 | | - const found = element.getElementById(id) |
| 446 | | - if (found) |
| 447 | | - results.push(found) |
| 448 | | - } |
| 449 | | - else if (part.startsWith('.')) { |
| 450 | | - |
| 451 | | - const className = part.slice(1) |
| 452 | | - results.push(...element.getElementsByClassName(className)) |
| 453 | | - } |
| 454 | | - else if (part.startsWith('[')) { |
| 455 | | - |
| 456 | | - const attrMatch = part.match(/\[([\w:-]+)(?:=["']?([^"'\]]+)["']?)?\]/) |
| 457 | | - if (attrMatch) { |
| 458 | | - const [, attrName, attrValue] = attrMatch |
| 459 | | - results.push(...filterByAttribute(element, attrName, attrValue)) |
| 442 | + for (const [index, step] of steps.entries()) { |
| 443 | + const next: HTMLElement[] = [] |
| 444 | + |
| 445 | + for (const candidate of candidates) { |
| 446 | + const pool = step.combinator === 'child' && index > 0 |
| 447 | + ? candidate.children |
| 448 | + : descendants(candidate) |
| 449 | + |
| 450 | + for (const item of pool) { |
| 451 | + if (matchesSimpleSelector(item, step.selector)) |
| 452 | + next.push(item) |
| 453 | + } |
| 460 | 454 | } |
| 455 | + |
| 456 | + candidates = uniqueElements(next) |
| 457 | + } |
| 458 | + |
| 459 | + results.push(...candidates) |
| 460 | + } |
| 461 | + |
| 462 | + return uniqueElements(results) |
| 463 | +} |
| 464 | + |
| 465 | +type SelectorCombinator = 'descendant' | 'child' |
| 466 | + |
| 467 | +interface SelectorStep { |
| 468 | + combinator: SelectorCombinator |
| 469 | + selector: string |
| 470 | +} |
| 471 | + |
| 472 | +function splitSelectorGroups(selector: string): string[] { |
| 473 | + const groups: string[] = [] |
| 474 | + let current = '' |
| 475 | + let bracketDepth = 0 |
| 476 | + let quote = '' |
| 477 | + |
| 478 | + for (const character of selector) { |
| 479 | + if ((character === '"' || character === '\'') && (!quote || quote === character)) |
| 480 | + quote = quote ? '' : character |
| 481 | + else if (!quote && character === '[') |
| 482 | + bracketDepth++ |
| 483 | + else if (!quote && character === ']') |
| 484 | + bracketDepth-- |
| 485 | + |
| 486 | + if (character === ',' && bracketDepth === 0 && !quote) { |
| 487 | + if (current.trim()) |
| 488 | + groups.push(current.trim()) |
| 489 | + current = '' |
| 461 | 490 | } |
| 462 | 491 | else { |
| 463 | | - |
| 464 | | - results.push(...element.getElementsByTagName(part)) |
| 492 | + current += character |
| 465 | 493 | } |
| 466 | 494 | } |
| 467 | 495 | |
| 468 | | - |
| 469 | | - if (selector.includes('.') || selector.includes('#') || selector.includes('[')) { |
| 470 | | - return results.filter((el) => { |
| 471 | | - if (selector.includes('.')) { |
| 472 | | - const classNames = selector.match(/\.([^\s.#[]+)/g)?.map(c => c.slice(1)) || [] |
| 473 | | - const elementClasses = el.attributes.class?.split(/\s+/) || [] |
| 474 | | - if (!classNames.every(cn => elementClasses.includes(cn))) |
| 475 | | - return false |
| 476 | | - } |
| 496 | + if (current.trim()) |
| 497 | + groups.push(current.trim()) |
| 477 | 498 | |
| 478 | | - if (selector.includes('#')) { |
| 479 | | - const idMatch = selector.match(/#([^\s.#[]+)/) |
| 480 | | - if (idMatch && el.attributes.id !== idMatch[1]) |
| 481 | | - return false |
| 482 | | - } |
| 499 | + return groups |
| 500 | +} |
| 483 | 501 | |
| 484 | | - const tagMatch = selector.match(/^([A-Za-z][\w:-]*)/) |
| 485 | | - if (tagMatch && el.tagName.toLowerCase() !== tagMatch[1].toLowerCase()) |
| 486 | | - return false |
| 502 | +function parseSelectorSteps(selector: string): SelectorStep[] { |
| 503 | + const steps: SelectorStep[] = [] |
| 504 | + let current = '' |
| 505 | + let bracketDepth = 0 |
| 506 | + let quote = '' |
| 507 | + let pendingCombinator: SelectorCombinator = 'descendant' |
| 508 | + |
| 509 | + const pushCurrent = (): void => { |
| 510 | + const value = current.trim() |
| 511 | + if (value) |
| 512 | + steps.push({ combinator: pendingCombinator, selector: value }) |
| 513 | + current = '' |
| 514 | + } |
| 487 | 515 | |
| 488 | | - return true |
| 489 | | - }) |
| 516 | + for (const character of selector) { |
| 517 | + if ((character === '"' || character === '\'') && (!quote || quote === character)) |
| 518 | + quote = quote ? '' : character |
| 519 | + else if (!quote && character === '[') |
| 520 | + bracketDepth++ |
| 521 | + else if (!quote && character === ']') |
| 522 | + bracketDepth-- |
| 523 | + |
| 524 | + if (!quote && bracketDepth === 0 && character === '>') { |
| 525 | + pushCurrent() |
| 526 | + pendingCombinator = 'child' |
| 527 | + } |
| 528 | + else if (!quote && bracketDepth === 0 && /\s/.test(character)) { |
| 529 | + if (current.trim()) { |
| 530 | + pushCurrent() |
| 531 | + pendingCombinator = 'descendant' |
| 532 | + } |
| 533 | + } |
| 534 | + else { |
| 535 | + current += character |
| 536 | + } |
| 490 | 537 | } |
| 491 | 538 | |
| 492 | | - return results |
| 539 | + pushCurrent() |
| 540 | + return steps |
| 493 | 541 | } |
| 494 | 542 | |
| 495 | | -function filterByAttribute(element: HTMLElement, attrName: string, attrValue?: string): HTMLElement[] { |
| 543 | +function descendants(element: HTMLElement): HTMLElement[] { |
| 496 | 544 | const results: HTMLElement[] = [] |
| 497 | 545 | |
| 498 | 546 | function traverse(el: HTMLElement): void { |
| 499 | | - if (el.hasAttribute(attrName)) { |
| 500 | | - if (!attrValue || el.getAttribute(attrName) === attrValue) { |
| 501 | | - results.push(el) |
| 502 | | - } |
| 503 | | - } |
| 504 | 547 | for (const child of el.children) { |
| 548 | + results.push(child) |
| 505 | 549 | traverse(child) |
| 506 | 550 | } |
| 507 | 551 | } |
| @@ -510,6 +554,72 @@ function filterByAttribute(element: HTMLElement, attrName: string, attrValue?: s |
| 510 | 554 | return results |
| 511 | 555 | } |
| 512 | 556 | |
| 557 | +function matchesSimpleSelector(element: HTMLElement, selector: string): boolean { |
| 558 | + const tagMatch = selector.match(/^([A-Za-z][\w:-]*|\*)/) |
| 559 | + if (tagMatch && tagMatch[1] !== '*' && element.tagName.toLowerCase() !== tagMatch[1].toLowerCase()) |
| 560 | + return false |
| 561 | + |
| 562 | + const idMatches = [...selector.matchAll(/#([\w-]+)/g)] |
| 563 | + if (idMatches.some(match => element.getAttribute('id') !== match[1])) |
| 564 | + return false |
| 565 | + |
| 566 | + const elementClasses = element.getAttribute('class')?.split(/\s+/) || [] |
| 567 | + const classMatches = [...selector.matchAll(/\.([\w-]+)/g)] |
| 568 | + if (classMatches.some(match => !elementClasses.includes(match[1]))) |
| 569 | + return false |
| 570 | + |
| 571 | + const attributeMatches = [...selector.matchAll(/\[([\w:-]+)(?:\s*(\^=|\$=|\*=|~=|\|=|=)\s*["']?([^"'\]]*)["']?)?\]/g)] |
| 572 | + for (const match of attributeMatches) { |
| 573 | + const [, name, operator, expected] = match |
| 574 | + if (!element.hasAttribute(name)) |
| 575 | + return false |
| 576 | + if (!operator) |
| 577 | + continue |
| 578 | + |
| 579 | + const actual = element.getAttribute(name) || '' |
| 580 | + if (operator === '=' && actual !== expected) |
| 581 | + return false |
| 582 | + if (operator === '*=' && !actual.includes(expected)) |
| 583 | + return false |
| 584 | + if (operator === '^=' && !actual.startsWith(expected)) |
| 585 | + return false |
| 586 | + if (operator === '$=' && !actual.endsWith(expected)) |
| 587 | + return false |
| 588 | + if (operator === '~=' && !actual.split(/\s+/).includes(expected)) |
| 589 | + return false |
| 590 | + if (operator === '|=' && actual !== expected && !actual.startsWith(`${expected}-`)) |
| 591 | + return false |
| 592 | + } |
| 593 | + |
| 594 | + return Boolean(tagMatch || idMatches.length || classMatches.length || attributeMatches.length) |
| 595 | +} |
| 596 | + |
| 597 | +function uniqueElements(elements: HTMLElement[]): HTMLElement[] { |
| 598 | + return [...new Set(elements)] |
| 599 | +} |
| 600 | + |
| 601 | +function decodeHTMLEntities(value: string): string { |
| 602 | + const namedEntities: Record<string, string> = { |
| 603 | + amp: '&', |
| 604 | + apos: '\'', |
| 605 | + gt: '>', |
| 606 | + lt: '<', |
| 607 | + nbsp: '\u00A0', |
| 608 | + quot: '"', |
| 609 | + } |
| 610 | + |
| 611 | + return value.replace(/&(#x[\da-f]+|#\d+|[a-z]+);/gi, (entity, code: string) => { |
| 612 | + if (code[0] !== '#') |
| 613 | + return namedEntities[code.toLowerCase()] ?? entity |
| 614 | + |
| 615 | + const numeric = code[1].toLowerCase() === 'x' |
| 616 | + ? Number.parseInt(code.slice(2), 16) |
| 617 | + : Number.parseInt(code.slice(1), 10) |
| 618 | + |
| 619 | + return Number.isFinite(numeric) ? String.fromCodePoint(numeric) : entity |
| 620 | + }) |
| 621 | +} |
| 622 | + |
| 513 | 623 | |
| 514 | 624 | |
| 515 | 625 | |