also looking at this
fix(lexer): leave interpolation mode while scanning string text
#928
2 files
+46
-1
| @@ -567,6 +567,10 @@ pub const Lexer = struct { | ||
| 567 | 567 | // The } that closes the interpolation expression was already consumed |
| 568 | 568 | // Update start to skip the } for the token lexeme |
| 569 | 569 | self.start = self.current; |
| 570 | // Scanning string text now, not interpolation code. Only a following | |
| 571 | // `${` re-enters interpolation, so an invalid escape or unterminated | |
| 572 | // string cannot leave the lexer counting braces of ordinary code. | |
| 573 | self.in_interpolation = false; | |
| 570 | 574 | |
| 571 | 575 | // Scan until next ${ or closing " |
| 572 | 576 | while (!self.isAtEnd()) { |
| @@ -649,12 +653,12 @@ pub const Lexer = struct { | ||
| 649 | 653 | if (self.peek() == '$' and self.peekNext() == '{') { |
| 650 | 654 | _ = self.advance(); // consume $ |
| 651 | 655 | _ = self.advance(); // consume { |
| 656 | self.in_interpolation = true; | |
| 652 | 657 | self.interpolation_brace_depth = 1; |
| 653 | 658 | return self.makeToken(.StringInterpolationMid); |
| 654 | 659 | } else { |
| 655 | 660 | // Closing " |
| 656 | 661 | _ = self.advance(); |
| 657 | self.in_interpolation = false; | |
| 658 | 662 | return self.makeToken(.StringInterpolationEnd); |
| 659 | 663 | } |
| 660 | 664 | } |
| @@ -306,3 +306,44 @@ test "lexer: complex expression" { | ||
| 306 | 306 | try testing.expectEqual(TokenType.AmpersandAmpersand, tokens.items[9].type); |
| 307 | 307 | try testing.expectEqual(TokenType.Identifier, tokens.items[10].type); |
| 308 | 308 | } |
| 309 | ||
| 310 | test "lexer: invalid escape after an interpolation ends the interpolation" { | |
| 311 | // `\ ` is not a valid escape. The lexer used to keep treating the rest of | |
| 312 | // the line as interpolation code, and the next `}` underflowed the brace | |
| 313 | // depth counter. | |
| 314 | var lexer = Lexer.init(testing.allocator, "\"${x}\\ }\""); | |
| 315 | var tokens = try lexer.tokenize(); | |
| 316 | defer tokens.deinit(testing.allocator); | |
| 317 | ||
| 318 | const expected = [_]TokenType{ | |
| 319 | .StringInterpolationStart, | |
| 320 | .Identifier, | |
| 321 | .Invalid, | |
| 322 | .RightBrace, | |
| 323 | .UnterminatedString, | |
| 324 | .Eof, | |
| 325 | }; | |
| 326 | try testing.expectEqual(expected.len, tokens.items.len); | |
| 327 | for (expected, tokens.items) |token_type, token| { | |
| 328 | try testing.expectEqual(token_type, token.type); | |
| 329 | } | |
| 330 | } | |
| 331 | ||
| 332 | test "lexer: adjacent interpolations re-enter interpolation mode" { | |
| 333 | var lexer = Lexer.init(testing.allocator, "\"${a}${b}\""); | |
| 334 | var tokens = try lexer.tokenize(); | |
| 335 | defer tokens.deinit(testing.allocator); | |
| 336 | ||
| 337 | const expected = [_]TokenType{ | |
| 338 | .StringInterpolationStart, | |
| 339 | .Identifier, | |
| 340 | .StringInterpolationMid, | |
| 341 | .Identifier, | |
| 342 | .StringInterpolationEnd, | |
| 343 | .Eof, | |
| 344 | }; | |
| 345 | try testing.expectEqual(expected.len, tokens.items.len); | |
| 346 | for (expected, tokens.items) |token_type, token| { | |
| 347 | try testing.expectEqual(token_type, token.type); | |
| 348 | } | |
| 349 | } | |