feat(lexer): #nnnn / #$hhhh Unicode-codepoint string literals

A `#`-prefixed numeric literal now denotes a Unicode codepoint and contributes
its UTF-8 encoding to the surrounding string constant — the coherent semantics
for Blaise's Char-less, UTF-8-native string type:

  #65      -> 'A'              (1 byte  $41)
  #$20AC   -> the euro sign    (3 bytes E2 82 AC)
  #$1F600  -> the grinning face(4 bytes F0 9F 98 80)

The literal is always a string, and adjacent `#` and '...' literals merge into
one compile-time constant (#72#73'!' -> 'HI!'), matching Pascal's literal-
merging tradition.  Decimal (#nnnn) and hexadecimal (#$hhhh) are both accepted;
the underlying tokeniser already captured #$-hex runs, so the change is confined
to UnescapeString + a new CodepointToUtf8.

Codepoints outside 0..U+10FFFF and the surrogate range U+D800..U+DFFF (not valid
Unicode scalar values) are a compile error, so emitted string constants are
always valid UTF-8.

Tests cover decimal ASCII, hex BMP (3-byte), astral (4-byte), and #+string
merging.  All four fixpoints green; full suite 3835.
This commit is contained in:
Graeme Geldenhuys 2026-06-26 10:40:40 +01:00
parent a6cdba7fbe
commit 09c2d1a0b9
2 changed files with 102 additions and 9 deletions

View file

@ -132,6 +132,7 @@ type
FFilename: string;
FDefines: TStringList; { conditional-compilation symbols, case-insensitive }
function MapKeyword(const AUpper: string): TTokenKind;
function CodepointToUtf8(ACodepoint: Integer): string;
function UnescapeString(const ARaw: string): string;
function ProcessTextBlock(const ARaw: string): string;
function DirectiveName(const AText: string): string;
@ -287,11 +288,41 @@ begin
Result := tkIdent; { keyword outside Phase 1 grammar treated as ident }
end;
function TLexer.CodepointToUtf8(ACodepoint: Integer): string;
{ Encode a Unicode scalar value as its UTF-8 byte sequence. A `#nnnn` / `#$hhhh`
literal denotes a codepoint (NOT a raw byte), so the string it contributes is
that codepoint's UTF-8 encoding: #65 -> 'A' (1 byte), #$20AC -> 3 bytes,
#$1F600 -> 4 bytes. Rejects values outside 0..U+10FFFF and the surrogate
range U+D800..U+DFFF, which are not valid scalar values. }
var
N: Integer;
begin
N := ACodepoint;
if (N < 0) or (N > $10FFFF) or ((N >= $D800) and (N <= $DFFF)) then
raise Exception.Create(Format(
'Invalid Unicode codepoint in character literal: %d (must be 0..$10FFFF, '
+ 'excluding surrogates $D800..$DFFF)', [N]));
if N <= $7F then
Result := Chr(N)
else if N <= $7FF then
Result := Chr($C0 or (N shr 6))
+ Chr($80 or (N and $3F))
else if N <= $FFFF then
Result := Chr($E0 or (N shr 12))
+ Chr($80 or ((N shr 6) and $3F))
+ Chr($80 or (N and $3F))
else
Result := Chr($F0 or (N shr 18))
+ Chr($80 or ((N shr 12) and $3F))
+ Chr($80 or ((N shr 6) and $3F))
+ Chr($80 or (N and $3F));
end;
function TLexer.UnescapeString(const ARaw: string): string;
{ ARaw is the full source span. Handles: 'text' with '' ' escaping,
#nn numeric char literals (decimal), and concatenated runs like
'abc'#13#10'def'. Uses OrdAt (0-based) so the body parses under both
FPC and the self-hosted Blaise compiler. }
#nnnn / #$hhhh Unicode-codepoint literals (decimal or hex, UTF-8 encoded),
and concatenated runs like 'abc'#13#10'def'. Uses OrdAt (0-based) so the body
parses under both FPC and the self-hosted Blaise compiler. }
var
I, Len, N, C: Integer;
begin
@ -327,18 +358,34 @@ begin
end;
end;
end
else if C = 35 then { '#' }
else if C = 35 then { '#' — a Unicode codepoint, decimal #nnnn or hex #$hhhh }
begin
I := I + 1;
N := 0;
while I < Len do
if (I < Len) and (OrdAt(ARaw, I) = 36) then { '$' -> hexadecimal }
begin
C := OrdAt(ARaw, I);
if (C < 48) or (C > 57) then Break;
N := N * 10 + (C - 48);
I := I + 1;
while I < Len do
begin
C := OrdAt(ARaw, I);
if (C >= 48) and (C <= 57) then N := N * 16 + (C - 48)
else if (C >= 65) and (C <= 70) then N := N * 16 + (C - 55) { A-F }
else if (C >= 97) and (C <= 102) then N := N * 16 + (C - 87) { a-f }
else Break;
I := I + 1;
end;
end
else
begin
while I < Len do
begin
C := OrdAt(ARaw, I);
if (C < 48) or (C > 57) then Break;
N := N * 10 + (C - 48);
I := I + 1;
end;
end;
Result := Result + Chr(N);
Result := Result + CodepointToUtf8(N);
end
else
I := I + 1;

View file

@ -82,6 +82,11 @@ type
procedure TestStringLit_Simple;
procedure TestStringLit_Empty;
procedure TestStringLit_EmbeddedQuote;
{ #nnnn / #$hhhh Unicode codepoint -> UTF-8 string literals }
procedure TestCodepoint_DecimalAscii;
procedure TestCodepoint_HexBmp_ThreeBytes;
procedure TestCodepoint_HexAstral_FourBytes;
procedure TestCodepoint_MergesWithStringAndEachOther;
{ Operators and punctuation }
procedure TestOp_Plus;
@ -449,6 +454,47 @@ begin
AssertEquals('Value', 'it''s', tok.Value);
end;
{ #nnnn / #$hhhh Unicode codepoint -> UTF-8 string literals }
procedure TLexerTests.TestCodepoint_DecimalAscii;
var tok: TToken;
begin
SetLexer('#65');
tok := FLexer.Next();
AssertEquals('Kind', Ord(tkStringLit), Ord(tok.Kind));
AssertEquals('Value', 'A', tok.Value); { codepoint 65 -> 1 byte $41 }
end;
procedure TLexerTests.TestCodepoint_HexBmp_ThreeBytes;
var tok: TToken;
begin
SetLexer('#$20AC'); { EURO SIGN -> UTF-8 E2 82 AC }
tok := FLexer.Next();
AssertEquals('Kind', Ord(tkStringLit), Ord(tok.Kind));
AssertEquals('len', 3, Length(tok.Value));
AssertEquals('Value', Chr($E2) + Chr($82) + Chr($AC), tok.Value);
end;
procedure TLexerTests.TestCodepoint_HexAstral_FourBytes;
var tok: TToken;
begin
SetLexer('#$1F600'); { GRINNING FACE -> UTF-8 F0 9F 98 80 }
tok := FLexer.Next();
AssertEquals('Kind', Ord(tkStringLit), Ord(tok.Kind));
AssertEquals('len', 4, Length(tok.Value));
AssertEquals('Value', Chr($F0) + Chr($9F) + Chr($98) + Chr($80), tok.Value);
end;
procedure TLexerTests.TestCodepoint_MergesWithStringAndEachOther;
var tok: TToken;
begin
{ Adjacent # and '...' literals merge into one compile-time string. }
SetLexer('#72#73''!'''); { 'H' 'I' '!' }
tok := FLexer.Next();
AssertEquals('Kind', Ord(tkStringLit), Ord(tok.Kind));
AssertEquals('Value', 'HI!', tok.Value);
end;
{ Operators and punctuation }
procedure TLexerTests.TestOp_Plus;