From 09c2d1a0b9a3dbcb82c1101162916b72f3b76b75 Mon Sep 17 00:00:00 2001 From: Graeme Geldenhuys Date: Fri, 26 Jun 2026 10:40:40 +0100 Subject: [PATCH] feat(lexer): #nnnn / #$hhhh Unicode-codepoint string literals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `#`-prefixed numeric literal now denotes a Unicode codepoint and contributes its UTF-8 encoding to the surrounding string constant — the coherent semantics for Blaise's Char-less, UTF-8-native string type: #65 -> 'A' (1 byte $41) #$20AC -> the euro sign (3 bytes E2 82 AC) #$1F600 -> the grinning face(4 bytes F0 9F 98 80) The literal is always a string, and adjacent `#` and '...' literals merge into one compile-time constant (#72#73'!' -> 'HI!'), matching Pascal's literal- merging tradition. Decimal (#nnnn) and hexadecimal (#$hhhh) are both accepted; the underlying tokeniser already captured #$-hex runs, so the change is confined to UnescapeString + a new CodepointToUtf8. Codepoints outside 0..U+10FFFF and the surrogate range U+D800..U+DFFF (not valid Unicode scalar values) are a compile error, so emitted string constants are always valid UTF-8. Tests cover decimal ASCII, hex BMP (3-byte), astral (4-byte), and #+string merging. All four fixpoints green; full suite 3835. --- compiler/src/main/pascal/uLexer.pas | 65 +++++++++++++++++++--- compiler/src/test/pascal/cp.test.lexer.pas | 46 +++++++++++++++ 2 files changed, 102 insertions(+), 9 deletions(-) diff --git a/compiler/src/main/pascal/uLexer.pas b/compiler/src/main/pascal/uLexer.pas index c82ee8c..7713dad 100644 --- a/compiler/src/main/pascal/uLexer.pas +++ b/compiler/src/main/pascal/uLexer.pas @@ -132,6 +132,7 @@ type FFilename: string; FDefines: TStringList; { conditional-compilation symbols, case-insensitive } function MapKeyword(const AUpper: string): TTokenKind; + function CodepointToUtf8(ACodepoint: Integer): string; function UnescapeString(const ARaw: string): string; function ProcessTextBlock(const ARaw: string): string; function DirectiveName(const AText: string): string; @@ -287,11 +288,41 @@ begin Result := tkIdent; { keyword outside Phase 1 grammar treated as ident } end; +function TLexer.CodepointToUtf8(ACodepoint: Integer): string; +{ Encode a Unicode scalar value as its UTF-8 byte sequence. A `#nnnn` / `#$hhhh` + literal denotes a codepoint (NOT a raw byte), so the string it contributes is + that codepoint's UTF-8 encoding: #65 -> 'A' (1 byte), #$20AC -> 3 bytes, + #$1F600 -> 4 bytes. Rejects values outside 0..U+10FFFF and the surrogate + range U+D800..U+DFFF, which are not valid scalar values. } +var + N: Integer; +begin + N := ACodepoint; + if (N < 0) or (N > $10FFFF) or ((N >= $D800) and (N <= $DFFF)) then + raise Exception.Create(Format( + 'Invalid Unicode codepoint in character literal: %d (must be 0..$10FFFF, ' + + 'excluding surrogates $D800..$DFFF)', [N])); + if N <= $7F then + Result := Chr(N) + else if N <= $7FF then + Result := Chr($C0 or (N shr 6)) + + Chr($80 or (N and $3F)) + else if N <= $FFFF then + Result := Chr($E0 or (N shr 12)) + + Chr($80 or ((N shr 6) and $3F)) + + Chr($80 or (N and $3F)) + else + Result := Chr($F0 or (N shr 18)) + + Chr($80 or ((N shr 12) and $3F)) + + Chr($80 or ((N shr 6) and $3F)) + + Chr($80 or (N and $3F)); +end; + function TLexer.UnescapeString(const ARaw: string): string; { ARaw is the full source span. Handles: 'text' with '' → ' escaping, - #nn numeric char literals (decimal), and concatenated runs like - 'abc'#13#10'def'. Uses OrdAt (0-based) so the body parses under both - FPC and the self-hosted Blaise compiler. } + #nnnn / #$hhhh Unicode-codepoint literals (decimal or hex, UTF-8 encoded), + and concatenated runs like 'abc'#13#10'def'. Uses OrdAt (0-based) so the body + parses under both FPC and the self-hosted Blaise compiler. } var I, Len, N, C: Integer; begin @@ -327,18 +358,34 @@ begin end; end; end - else if C = 35 then { '#' } + else if C = 35 then { '#' — a Unicode codepoint, decimal #nnnn or hex #$hhhh } begin I := I + 1; N := 0; - while I < Len do + if (I < Len) and (OrdAt(ARaw, I) = 36) then { '$' -> hexadecimal } begin - C := OrdAt(ARaw, I); - if (C < 48) or (C > 57) then Break; - N := N * 10 + (C - 48); I := I + 1; + while I < Len do + begin + C := OrdAt(ARaw, I); + if (C >= 48) and (C <= 57) then N := N * 16 + (C - 48) + else if (C >= 65) and (C <= 70) then N := N * 16 + (C - 55) { A-F } + else if (C >= 97) and (C <= 102) then N := N * 16 + (C - 87) { a-f } + else Break; + I := I + 1; + end; + end + else + begin + while I < Len do + begin + C := OrdAt(ARaw, I); + if (C < 48) or (C > 57) then Break; + N := N * 10 + (C - 48); + I := I + 1; + end; end; - Result := Result + Chr(N); + Result := Result + CodepointToUtf8(N); end else I := I + 1; diff --git a/compiler/src/test/pascal/cp.test.lexer.pas b/compiler/src/test/pascal/cp.test.lexer.pas index e95b633..c815183 100644 --- a/compiler/src/test/pascal/cp.test.lexer.pas +++ b/compiler/src/test/pascal/cp.test.lexer.pas @@ -82,6 +82,11 @@ type procedure TestStringLit_Simple; procedure TestStringLit_Empty; procedure TestStringLit_EmbeddedQuote; + { #nnnn / #$hhhh Unicode codepoint -> UTF-8 string literals } + procedure TestCodepoint_DecimalAscii; + procedure TestCodepoint_HexBmp_ThreeBytes; + procedure TestCodepoint_HexAstral_FourBytes; + procedure TestCodepoint_MergesWithStringAndEachOther; { Operators and punctuation } procedure TestOp_Plus; @@ -449,6 +454,47 @@ begin AssertEquals('Value', 'it''s', tok.Value); end; +{ #nnnn / #$hhhh Unicode codepoint -> UTF-8 string literals } + +procedure TLexerTests.TestCodepoint_DecimalAscii; +var tok: TToken; +begin + SetLexer('#65'); + tok := FLexer.Next(); + AssertEquals('Kind', Ord(tkStringLit), Ord(tok.Kind)); + AssertEquals('Value', 'A', tok.Value); { codepoint 65 -> 1 byte $41 } +end; + +procedure TLexerTests.TestCodepoint_HexBmp_ThreeBytes; +var tok: TToken; +begin + SetLexer('#$20AC'); { EURO SIGN -> UTF-8 E2 82 AC } + tok := FLexer.Next(); + AssertEquals('Kind', Ord(tkStringLit), Ord(tok.Kind)); + AssertEquals('len', 3, Length(tok.Value)); + AssertEquals('Value', Chr($E2) + Chr($82) + Chr($AC), tok.Value); +end; + +procedure TLexerTests.TestCodepoint_HexAstral_FourBytes; +var tok: TToken; +begin + SetLexer('#$1F600'); { GRINNING FACE -> UTF-8 F0 9F 98 80 } + tok := FLexer.Next(); + AssertEquals('Kind', Ord(tkStringLit), Ord(tok.Kind)); + AssertEquals('len', 4, Length(tok.Value)); + AssertEquals('Value', Chr($F0) + Chr($9F) + Chr($98) + Chr($80), tok.Value); +end; + +procedure TLexerTests.TestCodepoint_MergesWithStringAndEachOther; +var tok: TToken; +begin + { Adjacent # and '...' literals merge into one compile-time string. } + SetLexer('#72#73''!'''); { 'H' 'I' '!' } + tok := FLexer.Next(); + AssertEquals('Kind', Ord(tkStringLit), Ord(tok.Kind)); + AssertEquals('Value', 'HI!', tok.Value); +end; + { Operators and punctuation } procedure TLexerTests.TestOp_Plus;