also looking at this
fix: support structured DOM selectors
#847
6 files
+225
-57
Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.
| @@ -347,7 +347,16 @@ export class ExtractionPipeline<TIn = any, TOut = any> { | ||
| 347 | 347 | /** |
| 348 | 348 | * Built-in extractors for common patterns |
| 349 | 349 | */ |
| 350 | export const extractors = { | |
| 350 | export interface Extractors { | |
| 351 | text: (selector: string) => PipelineStep<Document, string[]> | |
| 352 | attr: (selector: string, attribute: string) => PipelineStep<Document, string[]> | |
| 353 | links: (selector?: string) => PipelineStep<Document, string[]> | |
| 354 | images: (selector?: string) => PipelineStep<Document, Array<{ src: string | null, alt: string | null, title: string | null }>> | |
| 355 | structured: (selector: string, schema: Record<string, string>) => PipelineStep<Document, Array<Record<string, any>>> | |
| 356 | jsonLd: () => PipelineStep<Document, any[]> | |
| 357 | } | |
| 358 | ||
| 359 | export const extractors: Extractors = { | |
| 351 | 360 | /** |
| 352 | 361 | * Extract text from elements |
| 353 | 362 | */ |
| @@ -362,7 +362,12 @@ Disallow: /narrower | ||
| 362 | 362 | ` |
| 363 | 363 | |
| 364 | 364 | function parserFor(userAgent: string) { |
| 365 | const parser = new RobotsParser({ userAgent, respectRobotsTxt: true }) | |
| 365 | const parser = new RobotsParser({ | |
| 366 | userAgent, | |
| 367 | respectRobotsTxt: true, | |
| 368 | cacheTime: 60_000, | |
| 369 | timeout: 5_000, | |
| 370 | }) | |
| 366 | 371 | return { parser, parsed: (parser as any).parse(fixture) } |
| 367 | 372 | } |
| 368 | 373 | |