administrator/blaise
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

test(lexer): regression tests for UTF-8 bytes inside comments

Browse source 
Lock in the current behaviour: high UTF-8 bytes (2-, 3-, and 4-byte
sequences) inside brace, paren-star, and line comments are skipped
cleanly without disturbing the token stream that follows.
This commit is contained in:
Graeme Geldenhuys 2026-05-22 15:33:38 +01:00
parent aeb3e4ae8d
commit b7b3d9000f
1 changed files with 48 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
compiler/src/test/pascal/cp.test.lexer.pas
View file

@ -30,6 +30,10 @@ type
procedure TestLineComment_Skipped;
procedure TestBlockComment_Skipped;
procedure TestBlockComment_MultiLine_Skipped;
procedure TestBlockComment_UTF8Bytes_Skipped;
procedure TestLineComment_UTF8Bytes_Skipped;
procedure TestParenStarComment_UTF8Bytes_Skipped;
procedure TestBlockComment_UTF8_FollowedByCode;
{ Keywords }
procedure TestKeyword_Program;
@ -181,6 +185,50 @@ begin
AssertEquals('Kind after multiline {}', Ord(tkEnd), Ord(tok.Kind));
end;
procedure TLexerTests.TestBlockComment_UTF8Bytes_Skipped;
var
tok: TToken;
begin
(* U+00B1 PLUS-MINUS SIGN, U+2264 LESS-THAN-OR-EQUAL, U+1F600 GRINNING FACE.
High UTF-8 bytes inside a brace comment must not disturb the token stream. *)
SetLexer('{ tolerance: ' + #$C2#$B1 + ' x, ' + #$E2#$89#$A4 + ' y, ' +
#$F0#$9F#$98#$80 + ' z } begin');
tok := FLexer.Next;
AssertEquals('Kind after {} with UTF-8', Ord(tkBegin), Ord(tok.Kind));
end;
procedure TLexerTests.TestLineComment_UTF8Bytes_Skipped;
var
tok: TToken;
begin
SetLexer('// note: ' + #$E2#$86#$92 + ' arrow, ' + #$F0#$9F#$98#$80 + ' emoji' +
#10 + 'end');
tok := FLexer.Next;
AssertEquals('Kind after // with UTF-8', Ord(tkEnd), Ord(tok.Kind));
end;
procedure TLexerTests.TestParenStarComment_UTF8Bytes_Skipped;
var
tok: TToken;
begin
SetLexer('(* note ' + #$C2#$B1 + ' epsilon ' + #$E2#$89#$A4 + ' bound *) end');
tok := FLexer.Next;
AssertEquals('Kind after (* *) with UTF-8', Ord(tkEnd), Ord(tok.Kind));
end;
procedure TLexerTests.TestBlockComment_UTF8_FollowedByCode;
var
tok: TToken;
begin
{ Regression: high UTF-8 continuation bytes used to leak into the next
token boundary, causing spurious identifiers after the comment closed.
Verify the identifier following the comment lexes cleanly. }
SetLexer('{ ' + #$C2#$B1 + ' }' + #10 + 'foo');
tok := FLexer.Next;
AssertEquals('Kind after UTF-8 comment', Ord(tkIdent), Ord(tok.Kind));
AssertEquals('Value after UTF-8 comment', 'foo', tok.Value);
end;
{ Keywords }
procedure TLexerTests.TestKeyword_Program;