fix: support structured DOM selectors

test/robots.test.tsmodified+6-1

Changes to test/robots.test.ts

		⋯361 linesShow the 361 lines above this hunk@@ -362,7 +362,12 @@ Disallow: /narrower
362	362	`
363	363
364	364	function parserFor(userAgent: string) {
365		- const parser = new RobotsParser({ userAgent, respectRobotsTxt: true })
	365	+ const parser = new RobotsParser({
	366	+ userAgent,
	367	+ respectRobotsTxt: true,
	368	+ cacheTime: 60_000,
	369	+ timeout: 5_000,
	370	+ })
366	371	return { parser, parsed: (parser as any).parse(fixture) }
367	372	}
368	373