fix(scanner): end a statement at the newline after a trailing line comment

packages/dtsx/src/extractor/scanner.tsmodified+92-13

Changes to packages/dtsx/src/extractor/scanner.ts

		⋯36 linesShow the 36 lines above this hunk@@ -37,6 +37,16 @@ const CH_AT = 64
37	37	const CH_DOLLAR = 36
38	38	const CH_UNDERSCORE = 95
39	39
	40	+// A declaration keyword and the name it declares, in the shape only a
	41	+// declaration has, on one line. See `startsDeclaration`. A name is never an
	42	+// operator word (`as // note` then `const satisfies T` continues), and a
	43	+// keyword never runs on into an escape (`export\u0056alue` is an identifier).
	44	+const DECL_NAME = String.raw`(?!(?:as|satisfies|in|of|instanceof|extends|is|keyof|typeof)(?![\p{L}\p{N}_$\\]))[\p{L}_$][\p{L}\p{N}_$]*`
	45	+const DECL_END = String.raw`(?![\p{L}\p{N}_$\\])`
	46	+const DECLARATION_START_RE = new RegExp(String.raw`^(?:(?:export|enum|var)${DECL_END}|import[ \t]+[\p{L}_$*{'"]|(?:const|let)[ \t]+${DECL_NAME}${DECL_END}|declare[ \t]+(?:const|let|var|function|class|enum|type|interface|namespace|module|global|abstract|async)${DECL_END}|abstract[ \t]+class${DECL_END}|interface[ \t]+${DECL_NAME}[ \t]*(?:[{<]|extends${DECL_END})|(?:namespace|module)[ \t]+${DECL_NAME}(?:\.${DECL_NAME})*[ \t]*\{|module[ \t]+['"]|type[ \t]+${DECL_NAME}[ \t]*=)`, 'u')
	47	+// A line break other than LF or CRLF. See `hasOtherLineBreak`.
	48	+const OTHER_LINE_BREAK_RE = /\r(?!\n)|[\u2028\u2029]/
	49	+
40	50	function isWhitespace(ch: number): boolean {
41	51	return ch === CH_SPACE || ch === CH_TAB || ch === CH_LF || ch === CH_CR
42	52	}
		⋯85 linesShow the 85 lines above this hunk@@ -128,6 +138,14 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
128	138	const nonExportedTypes = new Map<string, Declaration>()
129	139	const funcBodyIndices = new Set<number>()
130	140	let pos = 0
	141	+ // The last line comment `skipNonCode` stepped over: its `//` and the newline
	142	+ // that ends it. See `codeEndBefore`.
	143	+ let skippedCommentStart = -1
	144	+ let skippedCommentEnd = -1
	145	+ // Where the run of comments it belongs to starts, with only whitespace
	146	+ // between them, and where the last comment skipped ended. See `noteComment`.
	147	+ let commentRunStart = -1
	148	+ let lastCommentEnd = -1
131	149
132	150	// Skip BOM (byte order mark)
133	151	if (pos < len && source.charCodeAt(pos) === 0xFEFF)
		⋯206 linesShow the 206 lines above this hunk@@ -340,15 +358,23 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
340	358	if (ch === CH_SLASH && pos + 1 < len) {
341	359	const next = source.charCodeAt(pos + 1)
342	360	if (next === CH_SLASH) {
343		- // Line comment — use native indexOf for speed
	361	+ // Line comment – stop AT its newline, not past it. That newline can end
	362	+ // the statement the comment trails, and the ASI checks only run on a
	363	+ // newline: stepping over it read `export const a = 1 // note` on into
	364	+ // the next line, which dropped that declaration and typed `a` unknown.
344	365	const nl = source.indexOf('\n', pos + 2)
345		- pos = nl === -1 ? len : nl + 1
	366	+ const end = nl === -1 ? len : nl
	367	+ noteComment(end)
	368	+ skippedCommentStart = pos
	369	+ pos = skippedCommentEnd = end
346	370	return true
347	371	}
348	372	if (next === CH_STAR) {
349		- // Block comment — use native indexOf for speed
350		- const end = source.indexOf('*/', pos + 2)
351		- pos = end === -1 ? len : end + 2
	373	+ // Block comment – use native indexOf for speed
	374	+ const close = source.indexOf('*/', pos + 2)
	375	+ const end = close === -1 ? len : close + 2
	376	+ noteComment(end)
	377	+ pos = end
352	378	return true
353	379	}
354	380	// Check for regex literal: / not followed by / or *, preceded by non-expression char
		⋯54 linesShow the 54 lines above this hunk@@ -409,6 +435,47 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
409	435	}
410	436	}
411	437
	438	+ /**
	439	+ * Whether the source from the last skipped `//` up to `end` holds a lone CR,
	440	+ * U+2028 or U+2029. Comments are read to the next LF, so code after such a
	441	+ * break passes for comment text; that newline is then left as it was.
	442	+ */
	443	+ function hasOtherLineBreak(end: number): boolean {
	444	+ return OTHER_LINE_BREAK_RE.test(source.slice(skippedCommentStart, end))
	445	+ }
	446	+
	447	+ /** Record a comment from `pos` to `end`, extending the run before it if only whitespace separates them. */
	448	+ function noteComment(end: number): void {
	449	+ let gap = lastCommentEnd
	450	+ while (gap >= 0 && gap < pos && /\s/.test(source[gap])) gap++
	451	+ if (gap !== pos)
	452	+ commentRunStart = pos
	453	+ lastCommentEnd = end
	454	+ }
	455	+
	456	+ /**
	457	+ * `end`, moved back to the start of a line comment that runs up to it, so a
	458	+ * trailing `// note` stays out of the code read before it.
	459	+ */
	460	+ function codeEndBefore(end: number): number {
	461	+ // `end + 1` takes the LF in, so a CRLF's CR is not read as a lone one.
	462	+ return end === skippedCommentEnd && !hasOtherLineBreak(end + 1) ? skippedCommentStart : end
	463	+ }
	464	+
	465	+ /**
	466	+ * Whether a declaration starts at `pos`, the line after a skipped comment:
	467	+ * its keyword followed by the name it declares, which no expression or type
	468	+ * runs on into, unless the comment follows a `.` (`api. // note` then
	469	+ * `export()`). `function` and `class` are left out: either can be an operand
	470	+ * (`void // note` then `function f() {}`, a decorated class expression).
	471	+ */
	472	+ function startsDeclaration(): boolean {
	473	+ let before = commentRunStart - 1
	474	+ while (before >= 0 && /\s/.test(source[before])) before--
	475	+ const lineEnd = source.indexOf('\n', pos)
	476	+ return source.charCodeAt(before) !== CH_DOT && DECLARATION_START_RE.test(source.slice(pos, lineEnd === -1 ? len : lineEnd))
	477	+ }
	478	+
412	479	/** Index of the `//` that comments out `upTo` on this line, or -1. */
413	480	function lineCommentStart(lineStart: number, upTo: number): number {
414	481	let quote = 0
		⋯159 linesShow the 159 lines above this hunk@@ -574,7 +641,13 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
574	641	}
575	642	break
576	643	}
577		- const result = pos >= len || isTopLevelKeyword() || source.charCodeAt(pos) === CH_RBRACE
	644	+ let result = pos >= len || isTopLevelKeyword() || source.charCodeAt(pos) === CH_RBRACE
	645	+ // The newline of a trailing `// note` used to be stepped over, so none of
	646	+ // the ASI rules was written for it. It ends the statement only here, before
	647	+ // a `}`, the end of the file or a declaration; the other checks read on
	648	+ // past it, as they did.
	649	+ if (result && saved === skippedCommentEnd)
	650	+ result = !hasOtherLineBreak(pos) && (pos >= len || source.charCodeAt(pos) === CH_RBRACE || startsDeclaration())
578	651	pos = saved
579	652	return result
580	653	}
		⋯67 linesShow the 67 lines above this hunk@@ -648,6 +721,8 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
648	721	const ch = source.charCodeAt(pos)
649	722	if (ch !== CH_LF && ch !== CH_CR)
650	723	return false
	724	+ if (pos === skippedCommentEnd)
	725	+ return false // see checkASITopLevel
651	726	if (endsMidExpression(pos))
652	727	return false
653	728	const saved = pos
		⋯48 linesShow the 48 lines above this hunk@@ -702,6 +777,8 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
702	777	const ch = source.charCodeAt(pos)
703	778	if (ch !== CH_LF && ch !== CH_CR)
704	779	return false
	780	+ if (pos === skippedCommentEnd)
	781	+ return false // see checkASITopLevel
705	782	if (endsMidExpression(pos))
706	783	return false
707	784	const saved = pos
		⋯33 linesShow the 33 lines above this hunk@@ -741,6 +818,8 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
741	818	const ch = source.charCodeAt(pos)
742	819	if (ch !== CH_LF && ch !== CH_CR)
743	820	return false
	821	+ if (pos === skippedCommentEnd)
	822	+ return false // see checkASITopLevel
744	823	const saved = pos
745	824	pos++
746	825	if (ch === CH_CR && pos < len && source.charCodeAt(pos) === CH_LF)
		⋯1,542 linesShow the 1,542 lines above this hunk@@ -2289,7 +2368,7 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
2289	2368	break
2290	2369	pos++
2291	2370	}
2292		- typeAnnotation = sliceTrimmed(typeStart, pos)
	2371	+ typeAnnotation = sliceTrimmed(typeStart, codeEndBefore(pos))
2293	2372	}
2294	2373
2295	2374	// Initializer
		⋯68 linesShow the 68 lines above this hunk@@ -2364,7 +2443,7 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
2364	2443	}
2365	2444	break
2366	2445	}
2367		- initializerText = sliceTrimmed(initStart, pos)
	2446	+ initializerText = sliceTrimmed(initStart, codeEndBefore(pos))
2368	2447	if (initializerText.endsWith(' as const') || initializerText === 'const') {
2369	2448	isAsConst = true
2370	2449	if (!typeAnnotation) {
		⋯175 linesShow the 175 lines above this hunk@@ -2546,7 +2625,7 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
2546	2625	break
2547	2626	pos++
2548	2627	}
2549		- const typeBody = sliceTrimmed(typeStart, pos)
	2628	+ const typeBody = sliceTrimmed(typeStart, codeEndBefore(pos))
2550	2629	if (pos < len && source.charCodeAt(pos) === CH_SEMI)
2551	2630	pos++
2552	2631
		⋯768 linesShow the 768 lines above this hunk@@ -3321,7 +3400,7 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
3321	3400	break
3322	3401	pos++
3323	3402	}
3324		- vtype = sliceTrimmed(ts, pos)
	3403	+ vtype = sliceTrimmed(ts, codeEndBefore(pos))
3325	3404	}
3326	3405
3327	3406	let initText = ''
		⋯18 linesShow the 18 lines above this hunk@@ -3346,7 +3425,7 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
3346	3425	break
3347	3426	pos++
3348	3427	}
3349		- initText = sliceTrimmed(is2, pos)
	3428	+ initText = sliceTrimmed(is2, codeEndBefore(pos))
3350	3429	}
3351	3430	if (pos < len && source.charCodeAt(pos) === CH_SEMI)
3352	3431	pos++
		⋯78 linesShow the 78 lines above this hunk@@ -3431,7 +3510,7 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
3431	3510	break
3432	3511	pos++
3433	3512	}
3434		- const typeBody = sliceTrimmed(ts, pos)
	3513	+ const typeBody = sliceTrimmed(ts, codeEndBefore(pos))
3435	3514	if (pos < len && source.charCodeAt(pos) === CH_SEMI)
3436	3515	pos++
3437	3516	lines.push(`${indent}${prefix}type ${tname}${generics} = ${typeBody}`)
		⋯168 linesShow the 168 lines above this hunk@@ -3606,7 +3685,7 @@ function scanDeclarationsInternal(_source: string, _filename: string, _keepComme
3606	3685	/** Convert a default-export expression into an ambient binding plus export. */
3607	3686	function extractDefaultExpression(stmtStart: number, expressionStart: number): void {
3608	3687	skipToStatementEnd()
3609		- let expression = sliceTrimmed(expressionStart, pos)
	3688	+ let expression = sliceTrimmed(expressionStart, codeEndBefore(pos))
3610	3689	if (expression.charCodeAt(expression.length - 1) === CH_SEMI) expression = expression.slice(0, -1).trimEnd()
3611	3690	if (!expression) return
3612	3691