1 using System;
2 using System.Collections;
3
4
5
6
7
8 namespace System.Lex
9 {
10 public const int CONTINUE_TOKEN = -2;
11 public const int INVALID_TOKEN = -1;
12 public const int END_TOKEN = 0;
13 public class Token
14 {
15 public Token() :
16 id(INVALID_TOKEN), match(), line(1)
17 {
18 }
19 public Token(int id_) :
20 id(id_), match(), line(1)
21 {
22 }
23 public Token(int id_, const Lexeme& match_, int line_) :
24 id(id_), match(match_), line(line_)
25 {
26 }
27 public int id;
28 public Lexeme match;
29 public int line;
30 }
31 public bool NoWhiteSpaceBetweenTokens(const Token& first, const Token& second)
32 {
33 if (first.match.end == second.match.begin) return true;
34 return false;
35 }
36 public ustring GetEndTokenInfo()
37 {
38 return u"end of file";
39 }
40
41 public class TokenLine
42 {
43 public nothrow TokenLine() : tokens(), startState(0), endState(0)
44 {
45 }
46 public nothrow int TokenIndex(short columnNumber)
47 {
48 short col = 1;
49 int index = 0;
50 for (const Token& token : tokens)
51 {
52 short len = cast<short>(token.match.end - token.match.begin);
53 if (columnNumber >= col && columnNumber < col + len)
54 {
55 return index;
56 }
57 col = col + len;
58 ++index;
59 }
60 return -1;
61 }
62 public List<Token> tokens;
63 public int startState;
64 public int endState;
65 }
66
67 }