1
2
3
4
5
6 #include <sngcm/cmparser/LexerFileTokenParser.hpp>
7 #include <sngcm/cmlexer/LexerFileTokenLexer.hpp>
8 #include <sngcm/cmlexer/LexerFileTokenLexerTokens.hpp>
9 #include <soulng/util/TextUtils.hpp>
10 #include <soulng/util/Unicode.hpp>
11
12 namespace sngcm { namespace parser {
13
14 void ParseLexerFileLine(const std::u32string& lexerFileLine, sngcm::ast::SourceTokenFormatter* formatter, bool& inBlockComment)
15 {
16
17
18
19 std::u32string line(lexerFileLine);
20 line.append(U"\n");
21 LexerFileTokenLexer lexer(line, "", 0);
22 lexer.inBlockComment = inBlockComment;
23 lexer.SetSeparatorChar('\n');
24 ++lexer;
25 int i = 0;
26 formatter->BeginFormat();
27 while (*lexer != END)
28 {
29 switch (*lexer)
30 {
31 case WS: { soulng::lexer::Token token = lexer.GetToken(i); formatter->Spaces(token.match.ToString()); break; }
32 case LINECOMMENT: { soulng::lexer::Token token = lexer.GetToken(i); formatter->Comment(token.match.ToString()); return; }
33 case BLOCKCOMMENT:
34 {
35 soulng::lexer::Token token = lexer.GetToken(i);
36 formatter->Comment(token.match.ToString());
37 if (!soulng::util::EndsWith(ToUtf8(token.match.ToString()), "*/"))
38 {
39 inBlockComment = true;
40 return;
41 }
42 break;
43 }
44 case BLOCKCOMMENTLINE:
45 {
46 soulng::lexer::Token token = lexer.GetToken(i);
47 formatter->Comment(token.match.ToString());
48 if (token.match.ToString().find(U"*/") != std::u32string::npos)
49 {
50 inBlockComment = false;
51 }
52 return;
53 }
54 case KEYWORD: { soulng::lexer::Token token = lexer.GetToken(i); formatter->Keyword(token.match.ToString()); break; }
55 case ID: { soulng::lexer::Token token = lexer.GetToken(i); formatter->Identifier(token.match.ToString()); break; }
56 case CHARLIT: { soulng::lexer::Token token = lexer.GetToken(i); formatter->Char(token.match.ToString()); break; }
57 case STRINGLIT: { soulng::lexer::Token token = lexer.GetToken(i); formatter->String(token.match.ToString()); break; }
58 case NUMBER: { soulng::lexer::Token token = lexer.GetToken(i); formatter->Number(token.match.ToString()); break; }
59 case OTHER: { soulng::lexer::Token token = lexer.GetToken(i); formatter->Other(token.match.ToString()); break; }
60 }
61 ++lexer;
62 ++i;
63 }
64 formatter->EndFormat();
65 }
66
67 } }
68