rendered paste body// Tokenizer input file
setlength(TokenTypes, 3);
// White space
setlength(TokenTypes[0].Chars, 1);
TokenTypes[0].Name := 'whitespace';
TokenTypes[0].Chars[0].Charset := #9, #10, #13, #32;
TokenTypes[0].Chars[0].Repeat := True;
// Identifier
setlength(TokenTypes[1].Chars, 2);
TokenTypes[1].Name := 'identifier';
TokenTypes[1].Chars[0].Charset := 'a'..'z', '_';
TokenTypes[1].Chars[0].Repeat := False;
TokenTypes[1].Chars[1].Charset := 'a'..'z', '0'..'9', '_';
TokenTypes[1].Chars[1].Repeat := True;
// Plus
setlength(TokenTypes[2].Chars, 1);
TokenTypes[2].Name := 'plus';
TokenTypes[2].Chars[0].Charset := '+';
TokenTypes[2].Chars[0].Repeat := False;