nimlexer

nimony/src/nifler2/nimlexer.nim

• type TokKind = enum
    tkInvalid = (0, "tkInvalid")
    tkEof = (1, "\5BEOF\5D")
    tkSymbol = (2, "tkSymbol")
    tkAddr = (3, "addr")
    tkAnd = (4, "and")
    tkAs = (5, "as")
    tkAsm = (6, "asm")
    tkBind = (7, "bind")
    tkBlock = (8, "block")
    tkBreak = (9, "break")
    tkCase = (10, "case")
    tkCast = (11, "cast")
    tkConcept = (12, "concept")
    tkConst = (13, "const")
    tkContinue = (14, "continue")
    tkConverter = (15, "converter")
    tkDefer = (16, "defer")
    tkDiscard = (17, "discard")
    tkDistinct = (18, "distinct")
    tkDiv = (19, "div")
    tkDo = (20, "do")
    tkElif = (21, "elif")
    tkElse = (22, "else")
    tkEnd = (23, "end")
    tkEnum = (24, "enum")
    tkExcept = (25, "except")
    tkExport = (26, "export")
    tkFinally = (27, "finally")
    tkFor = (28, "for")
    tkFrom = (29, "from")
    tkFunc = (30, "func")
    tkIf = (31, "if")
    tkImport = (32, "import")
    tkIn = (33, "in")
    tkInclude = (34, "include")
    tkInterface = (35, "interface")
    tkIs = (36, "is")
    tkIsnot = (37, "isnot")
    tkIterator = (38, "iterator")
    tkLet = (39, "let")
    tkMacro = (40, "macro")
    tkMethod = (41, "method")
    tkMixin = (42, "mixin")
    tkMod = (43, "mod")
    tkNil = (44, "nil")
    tkNot = (45, "not")
    tkNotin = (46, "notin")
    tkObject = (47, "object")
    tkOf = (48, "of")
    tkOr = (49, "or")
    tkOut = (50, "out")
    tkProc = (51, "proc")
    tkPtr = (52, "ptr")
    tkRaise = (53, "raise")
    tkRef = (54, "ref")
    tkReturn = (55, "return")
    tkShl = (56, "shl")
    tkShr = (57, "shr")
    tkStatic = (58, "static")
    tkTemplate = (59, "template")
    tkTry = (60, "try")
    tkTuple = (61, "tuple")
    tkType = (62, "type")
    tkUsing = (63, "using")
    tkVar = (64, "var")
    tkWhen = (65, "when")
    tkWhile = (66, "while")
    tkXor = (67, "xor")
    tkYield = (68, "yield")
    tkIntLit = (69, "tkIntLit")
    tkInt8Lit = (70, "tkInt8Lit")
    tkInt16Lit = (71, "tkInt16Lit")
    tkInt32Lit = (72, "tkInt32Lit")
    tkInt64Lit = (73, "tkInt64Lit")
    tkUIntLit = (74, "tkUIntLit")
    tkUInt8Lit = (75, "tkUInt8Lit")
    tkUInt16Lit = (76, "tkUInt16Lit")
    tkUInt32Lit = (77, "tkUInt32Lit")
    tkUInt64Lit = (78, "tkUInt64Lit")
    tkFloatLit = (79, "tkFloatLit")
    tkFloat32Lit = (80, "tkFloat32Lit")
    tkFloat64Lit = (81, "tkFloat64Lit")
    tkFloat128Lit = (82, "tkFloat128Lit")
    tkStrLit = (83, "tkStrLit")
    tkRStrLit = (84, "tkRStrLit")
    tkTripleStrLit = (85, "tkTripleStrLit")
    tkGStrLit = (86, "tkGStrLit")
    tkGTripleStrLit = (87, "tkGTripleStrLit")
    tkCharLit = (88, "tkCharLit")
    tkCustomLit = (89, "tkCustomLit")
    tkParLe = (90, "\28")
    tkParRi = (91, "\29")
    tkBracketLe = (92, "\5B")
    tkBracketRi = (93, "\5D")
    tkCurlyLe = (94, "\7B")
    tkCurlyRi = (95, "\7D")
    tkBracketDotLe = (96, "\5B.")
    tkBracketDotRi = (97, ".\5D")
    tkCurlyDotLe = (98, "\7B.")
    tkCurlyDotRi = (99, ".\7D")
    tkParDotLe = (100, "\28.")
    tkParDotRi = (101, ".\29")
    tkComma = (102, ",")
    tkSemiColon = (103, ";")
    tkColon = (104, "\3A")
    tkColonColon = (105, "\3A\3A")
    tkEquals = (106, "=")
    tkDot = (107, ".")
    tkDotDot = (108, "..")
    tkBracketLeColon = (109, "\5B\3A")
    tkOpr = (110, "tkOpr")
    tkComment = (111, "tkComment")
    tkAccent = (112, "`")

• func dollar`.TokKind(e: TokKind): string

• type TokSpacing = enum
    tsLeading = (0, "tsLeading")
    tsTrailing = (1, "tsTrailing")
    tsEof = (2, "tsEof")

• func dollar`.TokSpacing(e: TokSpacing): string

• type Token = object
    kind: TokKind
    s: string
    indent: int32
    spacing: set[TokSpacing]
    line: int32
    col: int32
    base: int32
    suffixPos: int32
    iNumber: int64

• type Lexer = object
    buf: string
    pos: int64
    filename: string
    lineNumber: int32
    lineStart: int64
    currLineIndent: int32
    indentAhead: int32
    errors: seq

• const KeywordLow: TokKind
• const KeywordHigh: TokKind
• const OpChars: set[char]
• const SymChars: set[char]
• const SymStartChars: set[char]
• const UnicodeOperatorStartChars: set[char]

  ∙ ∘ × ★ ☆ ⊗ ⊘ ⊙ ⊛ ⊠ ⊡ ∩ ∧ ⊓ ⟑ ⟇ ⩓ ⩔ ■ □ ± ⊕ ⊖ ⊞ ⊟ ∪ ∨ ⊔ all start with one of these three bytes.

• proc ch(L: Lexer; i: int64): char
• proc nimIdentNormalize(s: string): string

  Nim's identifier equality made explicit: the first character counts as written, the rest is lowercased and underscores drop out. The lexer does not use it; the differential tools compare identifiers with it, because Nim's identifier cache cannot report the spelling it saw.

• func dollar`.UnicodeOprPred(e: UnicodeOprPred): string
• proc unicodeOprLen(buf: string; pos: int64): tuple[int64, UnicodeOprPred]

  Length and precedence class of the unicode operator at pos, or (0, uopNone). A byte in UnicodeOperatorStartChars that does not begin one of these is an ordinary identifier character, which is the whole reason this has to be consulted while scanning a symbol.

• proc getPrecedence(tok: Token): int64

  Nim computes an operator's precedence from its spelling, which is why the grammar's binary(...) takes this as a parameter rather than baking a precedence table into the generated parser.

• proc isRightAssoc(tok: Token): bool

  Only ^-like operators associate to the right, exactly as in parser.nim's isRightAssociative.

• proc isUnary(tok: Token): bool

  Space in front and none behind: -x is a prefix operator, a - x is not.

• proc isDotLike(tok: Token): bool
• proc isSigilLike(tok: Token): bool
• func dollar`.StringMode(e: StringMode): string
• proc openLexer(src: string; filename: string): Lexer
• proc next(L: var Lexer; tok: var Token)

  One token. Mirrors rawGetTok in compiler/lexer.nim branch for branch, because that is what the differential test compares against.

• iterator tokens(src: string; filename: string): Token

  Every token of src, tkEof last.