fix(lexer): leave interpolation mode while scanning string text

packages/lexer/tests/lexer_test.zigmodified+41-0

Changes to packages/lexer/tests/lexer_test.zig

		⋯305 linesShow the 305 lines above this hunk@@ -306,3 +306,44 @@ test "lexer: complex expression" {
306	306	try testing.expectEqual(TokenType.AmpersandAmpersand, tokens.items[9].type);
307	307	try testing.expectEqual(TokenType.Identifier, tokens.items[10].type);
308	308	}
	309	+
	310	+test "lexer: invalid escape after an interpolation ends the interpolation" {
	311	+ // `\ ` is not a valid escape. The lexer used to keep treating the rest of
	312	+ // the line as interpolation code, and the next `}` underflowed the brace
	313	+ // depth counter.
	314	+ var lexer = Lexer.init(testing.allocator, "\"${x}\\ }\"");
	315	+ var tokens = try lexer.tokenize();
	316	+ defer tokens.deinit(testing.allocator);
	317	+
	318	+ const expected = [_]TokenType{
	319	+ .StringInterpolationStart,
	320	+ .Identifier,
	321	+ .Invalid,
	322	+ .RightBrace,
	323	+ .UnterminatedString,
	324	+ .Eof,
	325	+ };
	326	+ try testing.expectEqual(expected.len, tokens.items.len);
	327	+ for (expected, tokens.items) |token_type, token| {
	328	+ try testing.expectEqual(token_type, token.type);
	329	+ }
	330	+}
	331	+
	332	+test "lexer: adjacent interpolations re-enter interpolation mode" {
	333	+ var lexer = Lexer.init(testing.allocator, "\"${a}${b}\"");
	334	+ var tokens = try lexer.tokenize();
	335	+ defer tokens.deinit(testing.allocator);
	336	+
	337	+ const expected = [_]TokenType{
	338	+ .StringInterpolationStart,
	339	+ .Identifier,
	340	+ .StringInterpolationMid,
	341	+ .Identifier,
	342	+ .StringInterpolationEnd,
	343	+ .Eof,
	344	+ };
	345	+ try testing.expectEqual(expected.len, tokens.items.len);
	346	+ for (expected, tokens.items) |token_type, token| {
	347	+ try testing.expectEqual(token_type, token.type);
	348	+ }
	349	+}