1
2
3
4
5
6 #include <sngcpp/pp/PP.hpp>
7 #include <sngcpp/pp/PPLexer.hpp>
8 #include <sngcpp/pp/PPParser.hpp>
9 #include <sngcpp/pp/PPTokens.hpp>
10 #include <sngcpp/pp/TextTokens.hpp>
11 #include <sngcpp/pp/TextLexer.hpp>
12 #include <sngcpp/pp/Evaluator.hpp>
13 #include <sngcpp/lexer/CppLexer.hpp>
14 #include <sngcpp/lexer/CppKeywords.hpp>
15 #include <sngcpp/lexer/CppTokens.hpp>
16 #include <sngcpp/parser/Expression.hpp>
17 #include <soulng/util/MappedInputFile.hpp>
18 #include <soulng/util/Path.hpp>
19 #include <soulng/util/Unicode.hpp>
20 #include <iostream>
21
22 namespace sngcpp { namespace pp {
23
24 using namespace soulng::util;
25 using namespace soulng::unicode;
26
27 const char32_t* ndebugStr = U"NDEBUG";
28 const soulng::lexer::Lexeme ndebug(ndebugStr, ndebugStr + 6);
29
30 static int textTokenCppTokenMap[TextTokens::MAX];
31 static int ppTokenTextTokenMap[PPTokens::MAX];
32
33 void InitTextTokenCppTokenMap()
34 {
35 for (int i = 0; i < TextTokens::MAX; ++i)
36 {
37 textTokenCppTokenMap[i] = soulng::lexer::INVALID_TOKEN;
38 }
39 textTokenCppTokenMap[TextTokens::END] = CppTokens::END;
40 textTokenCppTokenMap[TextTokens::ID] = CppTokens::ID;
41 textTokenCppTokenMap[TextTokens::CHARLITERAL] = CppTokens::CHARLIT;
42 textTokenCppTokenMap[TextTokens::STRINGLITERAL] = CppTokens::STRINGLIT;
43 textTokenCppTokenMap[TextTokens::COLONCOLON] = CppTokens::COLONCOLON;
44 textTokenCppTokenMap[TextTokens::COMMA] = CppTokens::COMMA;
45 textTokenCppTokenMap[TextTokens::ASSIGN] = CppTokens::ASSIGN;
46 textTokenCppTokenMap[TextTokens::MULASSIGN] = CppTokens::MULASSIGN;
47 textTokenCppTokenMap[TextTokens::DIVASSIGN] = CppTokens::DIVASSIGN;
48 textTokenCppTokenMap[TextTokens::REMASSIGN] = CppTokens::REMASSIGN;
49 textTokenCppTokenMap[TextTokens::ADDASSIGN] = CppTokens::ADDASSIGN;
50 textTokenCppTokenMap[TextTokens::SUBASSIGN] = CppTokens::SUBASSIGN;
51 textTokenCppTokenMap[TextTokens::SHIFTRIGHTASSIGN] = CppTokens::SHIFTRIGHTASSIGN;
52 textTokenCppTokenMap[TextTokens::SHIFTLEFTASSIGN] = CppTokens::SHIFTLEFTASSIGN;
53 textTokenCppTokenMap[TextTokens::ANDASSIGN] = CppTokens::ANDASSIGN;
54 textTokenCppTokenMap[TextTokens::XORASSIGN] = CppTokens::XORASSIGN;
55 textTokenCppTokenMap[TextTokens::ORASSIGN] = CppTokens::ORASSIGN;
56 textTokenCppTokenMap[TextTokens::QUEST] = CppTokens::QUEST;
57 textTokenCppTokenMap[TextTokens::COLON] = CppTokens::COLON;
58 textTokenCppTokenMap[TextTokens::OROR] = CppTokens::OROR;
59 textTokenCppTokenMap[TextTokens::AMPAMP] = CppTokens::AMPAMP;
60 textTokenCppTokenMap[TextTokens::OR] = CppTokens::OR;
61 textTokenCppTokenMap[TextTokens::XOR] = CppTokens::XOR;
62 textTokenCppTokenMap[TextTokens::AMP] = CppTokens::AMP;
63 textTokenCppTokenMap[TextTokens::EQ] = CppTokens::EQ;
64 textTokenCppTokenMap[TextTokens::NEQ] = CppTokens::NEQ;
65 textTokenCppTokenMap[TextTokens::LEQ] = CppTokens::LEQ;
66 textTokenCppTokenMap[TextTokens::GEQ] = CppTokens::GEQ;
67 textTokenCppTokenMap[TextTokens::SPACESHIP] = CppTokens::SPACESHIP;
68 textTokenCppTokenMap[TextTokens::LANGLE] = CppTokens::LANGLE;
69 textTokenCppTokenMap[TextTokens::RANGLE] = CppTokens::RANGLE;
70 textTokenCppTokenMap[TextTokens::SHIFTLEFT] = CppTokens::SHIFTLEFT;
71 textTokenCppTokenMap[TextTokens::SHIFTRIGHT] = CppTokens::SHIFTRIGHT;
72 textTokenCppTokenMap[TextTokens::PLUS] = CppTokens::PLUS;
73 textTokenCppTokenMap[TextTokens::MINUS] = CppTokens::MINUS;
74 textTokenCppTokenMap[TextTokens::STAR] = CppTokens::STAR;
75 textTokenCppTokenMap[TextTokens::DIV] = CppTokens::DIV;
76 textTokenCppTokenMap[TextTokens::MOD] = CppTokens::MOD;
77 textTokenCppTokenMap[TextTokens::DOTSTAR] = CppTokens::DOTSTAR;
78 textTokenCppTokenMap[TextTokens::ARROWSTAR] = CppTokens::ARROWSTAR;
79 textTokenCppTokenMap[TextTokens::LPAREN] = CppTokens::LPAREN;
80 textTokenCppTokenMap[TextTokens::RPAREN] = CppTokens::RPAREN;
81 textTokenCppTokenMap[TextTokens::PLUSPLUS] = CppTokens::PLUSPLUS;
82 textTokenCppTokenMap[TextTokens::MINUSMINUS] = CppTokens::MINUSMINUS;
83 textTokenCppTokenMap[TextTokens::EXCLAMATION] = CppTokens::EXCLAMATION;
84 textTokenCppTokenMap[TextTokens::TILDE] = CppTokens::TILDE;
85 textTokenCppTokenMap[TextTokens::LBRACKET] = CppTokens::LBRACKET;
86 textTokenCppTokenMap[TextTokens::RBRACKET] = CppTokens::RBRACKET;
87 textTokenCppTokenMap[TextTokens::DOT] = CppTokens::DOT;
88 textTokenCppTokenMap[TextTokens::ARROW] = CppTokens::ARROW;
89 textTokenCppTokenMap[TextTokens::SEMICOLON] = CppTokens::SEMICOLON;
90 textTokenCppTokenMap[TextTokens::ELLIPSES] = CppTokens::ELLIPSES;
91 textTokenCppTokenMap[TextTokens::LBRACE] = CppTokens::LBRACE;
92 textTokenCppTokenMap[TextTokens::RBRACE] = CppTokens::RBRACE;
93 }
94
95 void InitPPTokenTextTokenMap()
96 {
97 for (int i = 0; i < PPTokens::MAX; ++i)
98 {
99 ppTokenTextTokenMap[i] = soulng::lexer::INVALID_TOKEN;
100 }
101 ppTokenTextTokenMap[PPTokens::END] = TextTokens::END;
102 ppTokenTextTokenMap[PPTokens::DEFINED] = TextTokens::DEFINED;
103 ppTokenTextTokenMap[PPTokens::ANGLEHEADERNAME] = TextTokens::ANGLEHEADERNAME;
104 ppTokenTextTokenMap[PPTokens::QUOTEHEADERNAME] = TextTokens::QUOTEHEADERNAME;
105 ppTokenTextTokenMap[PPTokens::PPNUMBER] = TextTokens::PPNUMBER;
106 ppTokenTextTokenMap[PPTokens::CHARLITERAL] = TextTokens::CHARLITERAL;
107 ppTokenTextTokenMap[PPTokens::STRINGLITERAL] = TextTokens::STRINGLITERAL;
108 ppTokenTextTokenMap[PPTokens::CHAR] = TextTokens::CHAR;
109 ppTokenTextTokenMap[PPTokens::ID] = TextTokens::ID;
110 ppTokenTextTokenMap[PPTokens::COLONCOLON] = TextTokens::COLONCOLON;
111 ppTokenTextTokenMap[PPTokens::COMMA] = TextTokens::COMMA;
112 ppTokenTextTokenMap[PPTokens::ASSIGN] = TextTokens::ASSIGN;
113 ppTokenTextTokenMap[PPTokens::MULASSIGN] = TextTokens::MULASSIGN;
114 ppTokenTextTokenMap[PPTokens::DIVASSIGN] = TextTokens::DIVASSIGN;
115 ppTokenTextTokenMap[PPTokens::REMASSIGN] = TextTokens::REMASSIGN;
116 ppTokenTextTokenMap[PPTokens::ADDASSIGN] = TextTokens::ADDASSIGN;
117 ppTokenTextTokenMap[PPTokens::SUBASSIGN] = TextTokens::SUBASSIGN;
118 ppTokenTextTokenMap[PPTokens::SHIFTRIGHTASSIGN] = TextTokens::SHIFTRIGHTASSIGN;
119 ppTokenTextTokenMap[PPTokens::SHIFTLEFTASSIGN] = TextTokens::SHIFTLEFTASSIGN;
120 ppTokenTextTokenMap[PPTokens::ANDASSIGN] = TextTokens::ANDASSIGN;
121 ppTokenTextTokenMap[PPTokens::XORASSIGN] = TextTokens::XORASSIGN;
122 ppTokenTextTokenMap[PPTokens::ORASSIGN] = TextTokens::ORASSIGN;
123 ppTokenTextTokenMap[PPTokens::QUEST] = TextTokens::QUEST;
124 ppTokenTextTokenMap[PPTokens::COLON] = TextTokens::COLON;
125 ppTokenTextTokenMap[PPTokens::OROR] = TextTokens::OROR;
126 ppTokenTextTokenMap[PPTokens::AMPAMP] = TextTokens::AMPAMP;
127 ppTokenTextTokenMap[PPTokens::OR] = TextTokens::OR;
128 ppTokenTextTokenMap[PPTokens::XOR] = TextTokens::XOR;
129 ppTokenTextTokenMap[PPTokens::AMP] = TextTokens::AMP;
130 ppTokenTextTokenMap[PPTokens::EQ] = TextTokens::EQ;
131 ppTokenTextTokenMap[PPTokens::NEQ] = TextTokens::NEQ;
132 ppTokenTextTokenMap[PPTokens::LEQ] = TextTokens::LEQ;
133 ppTokenTextTokenMap[PPTokens::GEQ] = TextTokens::GEQ;
134 ppTokenTextTokenMap[PPTokens::SPACESHIP] = TextTokens::SPACESHIP;
135 ppTokenTextTokenMap[PPTokens::LANGLE] = TextTokens::LANGLE;
136 ppTokenTextTokenMap[PPTokens::RANGLE] = TextTokens::RANGLE;
137 ppTokenTextTokenMap[PPTokens::SHIFTLEFT] = TextTokens::SHIFTLEFT;
138 ppTokenTextTokenMap[PPTokens::SHIFTRIGHT] = TextTokens::SHIFTRIGHT;
139 ppTokenTextTokenMap[PPTokens::PLUS] = TextTokens::PLUS;
140 ppTokenTextTokenMap[PPTokens::MINUS] = TextTokens::MINUS;
141 ppTokenTextTokenMap[PPTokens::STAR] = TextTokens::STAR;
142 ppTokenTextTokenMap[PPTokens::DIV] = TextTokens::DIV;
143 ppTokenTextTokenMap[PPTokens::MOD] = TextTokens::MOD;
144 ppTokenTextTokenMap[PPTokens::DOTSTAR] = TextTokens::DOTSTAR;
145 ppTokenTextTokenMap[PPTokens::ARROWSTAR] = TextTokens::ARROWSTAR;
146 ppTokenTextTokenMap[PPTokens::LPAREN] = TextTokens::LPAREN;
147 ppTokenTextTokenMap[PPTokens::RPAREN] = TextTokens::RPAREN;
148 ppTokenTextTokenMap[PPTokens::PLUSPLUS] = TextTokens::PLUSPLUS;
149 ppTokenTextTokenMap[PPTokens::MINUSMINUS] = TextTokens::MINUSMINUS;
150 ppTokenTextTokenMap[PPTokens::EXCLAMATION] = TextTokens::EXCLAMATION;
151 ppTokenTextTokenMap[PPTokens::TILDE] = TextTokens::TILDE;
152 ppTokenTextTokenMap[PPTokens::LBRACKET] = TextTokens::LBRACKET;
153 ppTokenTextTokenMap[PPTokens::RBRACKET] = TextTokens::RBRACKET;
154 ppTokenTextTokenMap[PPTokens::DOT] = TextTokens::DOT;
155 ppTokenTextTokenMap[PPTokens::ARROW] = TextTokens::ARROW;
156 ppTokenTextTokenMap[PPTokens::SEMICOLON] = TextTokens::SEMICOLON;
157 ppTokenTextTokenMap[PPTokens::ELLIPSES] = TextTokens::ELLIPSES;
158 ppTokenTextTokenMap[PPTokens::LBRACE] = TextTokens::LBRACE;
159 ppTokenTextTokenMap[PPTokens::RBRACE] = TextTokens::RBRACE;
160 }
161
162 std::std::vector<soulng::lexer::Token>MacroExpand(conststd::std::vector<soulng::lexer::Token>&tokens, constPP*pp)
163 {
164 bool prevWasMacro = false;
165 std::vector<soulng::lexer::Token> expandedTokens;
166 for (const soulng::lexer::Token& token : tokens)
167 {
168 if (token.id == TextTokens::ID)
169 {
170 auto it = pp->macroMap.find(token.match);
171 if (it != pp->macroMap.cend())
172 {
173 const std::std::vector<soulng::lexer::Token>&replacementTokens=it->second;
174 for (const soulng::lexer::Token& replacementToken : replacementTokens)
175 {
176 expandedTokens.push_back(replacementToken);
177 }
178 prevWasMacro = true;
179 }
180 else
181 {
182 expandedTokens.push_back(token);
183 }
184 }
185 else
186 {
187 if (prevWasMacro)
188 {
189 prevWasMacro = false;
190 if (token.id != TextTokens::WS)
191 {
192 expandedTokens.push_back(token);
193 }
194 }
195 else
196 {
197 expandedTokens.push_back(token);
198 }
199 }
200 }
201 return expandedTokens;
202 }
203
204 std::std::vector<soulng::lexer::Token>ConvertPPTokensToTextTokens(conststd::std::vector<soulng::lexer::Token>&ppTokens)
205 {
206 std::vector<soulng::lexer::Token> textTokens;
207 for (const soulng::lexer::Token& ppToken : ppTokens)
208 {
209 textTokens.push_back(std::move(soulng::lexer::Token(ppTokenTextTokenMap[ppToken.id], ppToken.match, ppToken.line)));
210 }
211 return textTokens;
212 }
213
214 const char32_t* zeroStr = U"0";
215 const soulng::lexer::Lexeme zeroLexeme(zeroStr, zeroStr + 1);
216 const soulng::lexer::Token zeroToken(CppTokens::INTLIT, zeroLexeme, 1);
217 const char32_t* oneStr = U"1";
218 const soulng::lexer::Lexeme oneLexeme(oneStr, oneStr + 1);
219 const soulng::lexer::Token oneToken(CppTokens::INTLIT, oneLexeme, 1);
220
221 std::std::vector<soulng::lexer::Token>ConvertTextTokensToCppTokens(conststd::std::vector<soulng::lexer::Token>&textTokens, constPP*pp)
222 {
223 std::vector<soulng::lexer::Token> cppTokens;
224 int state = 0;
225 for (const soulng::lexer::Token& textToken : textTokens)
226 {
227 switch (state)
228 {
229 case 0:
230 {
231 switch (textToken.id)
232 {
233 case TextTokens::KEYWORD:
234 case TextTokens::ID:
235 {
236 cppTokens.push_back(zeroToken);
237 break;
238 }
239 case TextTokens::PPNUMBER:
240 {
241 CppLexer lexer(textToken.match.begin, textToken.match.end, std::string(), 0);
242 ++lexer;
243 cppTokens.push_back(*lexer);
244 break;
245 }
246 case TextTokens::DEFINED:
247 {
248 state = 1;
249 break;
250 }
251 default:
252 {
253 cppTokens.push_back(std::move(soulng::lexer::Token(textTokenCppTokenMap[textToken.id], textToken.match, textToken.line)));
254 break;
255 }
256 }
257 break;
258 }
259 case 1:
260 {
261 switch (textToken.id)
262 {
263 case TextTokens::LPAREN:
264 {
265 state = 2;
266 break;
267 }
268 case TextTokens::ID:
269 {
270 if (pp->IsDefined(textToken.match))
271 {
272 cppTokens.push_back(oneToken);
273 }
274 else
275 {
276 cppTokens.push_back(zeroToken);
277 }
278 state = 0;
279 break;
280 }
281 default:
282 {
283 cppTokens.push_back(zeroToken);
284 state = 0;
285 break;
286 }
287 }
288 break;
289 }
290 case 2:
291 {
292 switch (textToken.id)
293 {
294 case TextTokens::ID:
295 {
296 if (pp->IsDefined(textToken.match))
297 {
298 cppTokens.push_back(oneToken);
299 }
300 else
301 {
302 cppTokens.push_back(zeroToken);
303 }
304 state = 3;
305 break;
306 }
307 default:
308 {
309 cppTokens.push_back(zeroToken);
310 state = 3;
311 }
312 }
313 break;
314 }
315 case 3:
316 {
317 if (textToken.id == TextTokens::RPAREN)
318 {
319 state = 0;
320 break;
321 }
322 break;
323 }
324 }
325 }
326 return cppTokens;
327 }
328
329 PP::PP(EvaluationContext& context_) :
330 fileIndex(0), save(true), process(true), processed(false), elseGroupProcessed(false), inIfGroup(false), verbose(false),
331 projectHeaderFileSet(nullptr), root(GetCurrentWorkingDirectory()), line(1), context(context_), rootMode(false), tokens(nullptr)
332 {
333 }
334
335 PP::~PP()
336 {
337 }
338
339 std::std::vector<soulng::lexer::Token>*PP::BeginDefine(constsoulng::lexer::Lexeme&id)
340 {
341 if (process)
342 {
343 return ¯oMap[id];
344 }
345 else
346 {
347 return nullptr;
348 }
349 }
350
351 void PP::EndDefine(std::std::vector<soulng::lexer::Token>&tokens)
352 {
353 if (process)
354 {
355 std::vector<soulng::lexer::Token> textTokens = ConvertPPTokensToTextTokens(tokens);
356 std::swap(tokens, textTokens);
357 }
358 }
359
360 void PP::Define(const soulng::lexer::Lexeme& lexeme)
361 {
362 std::std::vector<soulng::lexer::Token>*def=BeginDefine(lexeme);
363 EndDefine(*def);
364 }
365
366 bool PP::IsDefined(const soulng::lexer::Lexeme& lexeme) const
367 {
368 return macroMap.find(lexeme) != macroMap.cend();
369 }
370
371 bool PP::IsKeywordToken(const soulng::lexer::Token& token) const
372 {
373 soulng::lexer::KeywordMap* keywordMap = CppKeywords::GetKeywordMap();
374 return keywordMap->GetKeywordToken(token.match) != soulng::lexer::INVALID_TOKEN;
375 }
376
377 bool PP::IsProjectHeaderFile(const std::string& headerFilePath) const
378 {
379 if (projectHeaderFileSet)
380 {
381 return projectHeaderFileSet->IsProjectHeaderFile(headerFilePath);
382 }
383 return false;
384 }
385
386 void PP::Include(bool isAngleHeader, const std::string& headerName)
387 {
388 bool found = false;
389 std::string headerFilePath;
390 if (isAngleHeader)
391 {
392 for (const std::string& includeDir : includePath)
393 {
394 headerFilePath = GetFullPath(Path::Combine(Path::Combine(root, includeDir), headerName));
395 if (FileExists(headerFilePath))
396 {
397 found = true;
398 break;
399 }
400 }
401 }
402 else
403 {
404 headerFilePath = GetFullPath(Path::Combine(root, headerName));
405 if (FileExists(headerFilePath))
406 {
407 found = true;
408 }
409 }
410 if (found)
411 {
412 if (IsProjectHeaderFile(headerFilePath) || rootMode && Path::GetDirectoryName(headerFilePath) == root)
413 {
414 bool prevSave = save;
415 save = false;
416 ++fileIndex;
417 headerFilePaths.push_back(headerFilePath);
418 Preprocess(headerFilePath, this);
419 save = prevSave;
420 }
421 }
422 }
423
424 const char32_t* emptyStr = U"";
425
426 bool PP::Evaluate(const std::std::vector<soulng::lexer::Token>&exprPPTokens) const
427 {
428 std::vector<soulng::lexer::Token> exprTextTokens = ConvertPPTokensToTextTokens(exprPPTokens);
429 exprTextTokens = MacroExpand(exprTextTokens, this);
430 std::vector<soulng::lexer::Token> cppTokens = ConvertTextTokensToCppTokens(exprTextTokens, this);
431 CppLexer lexer(emptyStr, emptyStr, fileName, fileIndex);
432 lexer.SetLine(line);
433 lexer.SetTokens(cppTokens);
434 sngcpp::cppparser::ParsingContext ctx;
435 std::unique_ptr<sngcpp::ast::Node> expr = ExpressionParser::Parse(lexer, &ctx);
436 Evaluator evaluator(fileName, line, context);
437 expr->Accept(evaluator);
438 Value* value = evaluator.GetValue();
439 BoolValue* boolValue = value->ToBool(context);
440 return boolValue->GetValue();
441 }
442
443 void PP::If(const std::std::vector<soulng::lexer::Token>&exprPPTokens)
444 {
445 processStack.push(process);
446 bool processGroup = processStack.top() && Evaluate(exprPPTokens);
447 process = processGroup;
448 processedStack.push(processed);
449 processed = process;
450 inIfGroupStack.push(inIfGroup);
451 inIfGroup = true;
452 elseGroupProcessedStack.push(elseGroupProcessed);
453 elseGroupProcessed = false;
454 }
455
456 void PP::Ifdef(bool defined)
457 {
458 processStack.push(process);
459 bool processGroup = processStack.top() && defined;
460 process = processGroup;
461 processedStack.push(processed);
462 processed = process;
463 inIfGroupStack.push(inIfGroup);
464 inIfGroup = true;
465 elseGroupProcessedStack.push(elseGroupProcessed);
466 elseGroupProcessed = false;
467 }
468
469 void PP::Ifndef(bool defined)
470 {
471 processStack.push(process);
472 bool processGroup = processStack.top() && !defined;
473 process = processGroup;
474 processedStack.push(processed);
475 processed = process;
476 inIfGroupStack.push(inIfGroup);
477 inIfGroup = true;
478 elseGroupProcessedStack.push(elseGroupProcessed);
479 elseGroupProcessed = false;
480 }
481
482 void PP::Elif(const std::std::vector<soulng::lexer::Token>&exprPPTokens)
483 {
484 if (!inIfGroup)
485 {
486 throw std::runtime_error("error: " + fileName + ":" + std::to_string(line) + ": not in #if group");
487 }
488 if (processed)
489 {
490 process = false;
491 }
492 else
493 {
494 bool processGroup = Evaluate(exprPPTokens);
495 process = processGroup;
496 if (process)
497 {
498 processed = true;
499 }
500 }
501 }
502
503 void PP::Else()
504 {
505 if (!inIfGroup)
506 {
507 throw std::runtime_error("error: " + fileName + ":" + std::to_string(line) + ": not in #if group");
508 }
509 if (elseGroupProcessed)
510 {
511 throw std::runtime_error("error: " + fileName + ":" + std::to_string(line) + ": duplicate #else group");
512 }
513 if (processStack.top())
514 {
515 if (!processed)
516 {
517 process = true;
518 processed = true;
519 }
520 elseGroupProcessed = true;
521 }
522 else
523 {
524 process = false;
525 }
526 }
527
528 void PP::Endif()
529 {
530 if (!inIfGroup)
531 {
532 throw std::runtime_error("error: " + fileName + ":" + std::to_string(line) + ": not in #if group");
533 }
534 process = processStack.top();
535 processStack.pop();
536 processed = processedStack.top();
537 processedStack.pop();
538 elseGroupProcessed = elseGroupProcessedStack.top();
539 elseGroupProcessedStack.pop();
540 inIfGroup = inIfGroupStack.top();
541 inIfGroupStack.pop();
542 }
543
544 void PP::Emit(const char32_t* s)
545 {
546 if (save)
547 {
548 while (*s)
549 {
550 text.append(1, *s);
551 ++s;
552 }
553 }
554 }
555
556 void PP::Emit(const soulng::lexer::Lexeme& lexeme)
557 {
558 if (save)
559 {
560 const char32_t* p = lexeme.begin;
561 const char32_t* e = lexeme.end;
562 while (p != e)
563 {
564 text.append(1, *p);
565 ++p;
566 }
567 }
568 }
569
570 void PP::Emit(const char32_t* s, const soulng::lexer::Lexeme& lexeme, int tokenID)
571 {
572 Emit(s);
573 soulng::lexer::Lexeme match;
574 match.begin = lexeme.begin;
575 match.end = lexeme.end;
576 if (lexeme.end > lexeme.begin)
577 {
578 const char32_t* p = lexeme.end - 1;
579 while (p != lexeme.begin && (*p == '\r' || *p == '\n'))
580 {
581 --p;
582 }
583 ++p;
584 match.end = p;
585 }
586 tokens->push_back(soulng::lexer::Token(tokenID, match, line));
587 }
588
589 inline bool IsPPLine(const char32_t* p, const char32_t* e)
590 {
591 while (p != e&&( *p == ' ' || *p == '\t'))
592 {
593 ++p;
594 }
595 if (p != e && *p == '#') return true;
596 return false;
597 }
598
599 inline void SkipLineEnd(const char32_t*& p, const char32_t* e)
600 {
601 if (p != e && *p == '\r')
602 {
603 ++p;
604 }
605 if (p != e && *p == '\n')
606 {
607 ++p;
608 }
609 }
610
611 void GetLine(const char32_t*& p, const char32_t* e, const char32_t*& begin, const char32_t*& end, int& numNewLines)
612 {
613 numNewLines = 0;
614 begin = p;
615 end = e;
616 int state = 0;
617 while (p != e)
618 {
619 switch (state)
620 {
621 case 0:
622 {
623 switch (*p)
624 {
625 case '\r': case '\n':
626 {
627 SkipLineEnd(p, e);
628 end = p;
629 return;
630 }
631 case '/':
632 {
633 state = 1;
634 break;
635 }
636 case '\\':
637 {
638 state = 6;
639 break;
640 }
641 }
642 break;
643 }
644 case 1:
645 {
646 switch (*p)
647 {
648 case '\r': case '\n':
649 {
650 SkipLineEnd(p, e);
651 end = p;
652 return;
653 }
654 case '/':
655 {
656 state = 2;
657 break;
658 }
659 case '*':
660 {
661 state = 3;
662 break;
663 }
664 default:
665 {
666 state = 0;
667 break;
668 }
669 }
670 break;
671 }
672 case 2:
673 {
674 switch (*p)
675 {
676 case '\r': case '\n':
677 {
678 SkipLineEnd(p, e);
679 end = p;
680 return;
681 }
682 }
683 break;
684 }
685 case 3:
686 {
687 switch (*p)
688 {
689 case '*':
690 {
691 state = 4;
692 break;
693 }
694 case '\n':
695 {
696 ++numNewLines;
697 break;
698 }
699 }
700 break;
701 }
702 case 4:
703 {
704 switch (*p)
705 {
706 case '*':
707 {
708 break;
709 }
710 case '/':
711 {
712 ++p;
713 end = p;
714 return;
715 }
716 case '\n':
717 {
718 ++numNewLines;
719 state = 3;
720 break;
721 }
722 default:
723 {
724 state = 3;
725 break;
726 }
727 }
728 break;
729 }
730 case 6:
731 {
732 switch (*p)
733 {
734 case '\\':
735 {
736 break;
737 }
738 case '\r':
739 {
740 state = 7;
741 break;
742 }
743 case '\n':
744 {
745 ++numNewLines;
746 state = 0;
747 break;
748 }
749 default:
750 {
751 state = 0;
752 break;
753 }
754 }
755 break;
756 }
757 case 7:
758 {
759 switch (*p)
760 {
761 case '\r':
762 {
763 state = 7;
764 break;
765 }
766 case '\n':
767 {
768 ++numNewLines;
769 state = 0;
770 break;
771 }
772 case '\\':
773 {
774 state = 6;
775 break;
776 }
777 default:
778 {
779 state = 0;
780 break;
781 }
782 }
783 break;
784 }
785 }
786 ++p;
787 }
788 }
789
790 std::string ParseAngleHeaderName(const std::string& fileName, const soulng::lexer::Token& headerNameToken)
791 {
792 const char32_t* p = headerNameToken.match.begin;
793 const char32_t* e = headerNameToken.match.end;
794 const char32_t* begin = p;
795 const char32_t* end = e;
796 if (p != e && *p == '<')
797 {
798 ++p;
799 begin = p;
800 }
801 else
802 {
803 throw std::runtime_error("invalid header name in file '" + fileName + ":" + std::to_string(headerNameToken.line) + "': " + ToUtf8(std::u32string(headerNameToken.match.ToString())));
804 }
805 while (p != e && *p != '>')
806 {
807 ++p;
808 }
809 if (p != e && *p == '>')
810 {
811 end = p;
812 ++p;
813 }
814 else
815 {
816 throw std::runtime_error("invalid header name in file '" + fileName + ":" + std::to_string(headerNameToken.line) + "': " + ToUtf8(std::u32string(headerNameToken.match.ToString())));
817 }
818 if (p == e)
819 {
820 return ToUtf8(std::u32string(begin, end));
821 }
822 else
823 {
824 throw std::runtime_error("invalid header name in file '" + fileName + ":" + std::to_string(headerNameToken.line) + "': " + ToUtf8(std::u32string(headerNameToken.match.ToString())));
825 }
826 }
827
828 std::string ParseQuoteHeaderName(const std::string& fileName, const soulng::lexer::Token& headerNameToken)
829 {
830 const char32_t* p = headerNameToken.match.begin;
831 const char32_t* e = headerNameToken.match.end;
832 const char32_t* begin = p;
833 const char32_t* end = e;
834 if (p != e && *p == '"')
835 {
836 ++p;
837 begin = p;
838 }
839 else
840 {
841 throw std::runtime_error("invalid header name in file '" + fileName + ":" + std::to_string(headerNameToken.line) + "': " + ToUtf8(std::u32string(headerNameToken.match.ToString())));
842 }
843 while (p != e && *p != '"')
844 {
845 ++p;
846 }
847 if (p != e && *p == '"')
848 {
849 end = p;
850 ++p;
851 }
852 else
853 {
854 throw std::runtime_error("invalid header name in file '" + fileName + ":" + std::to_string(headerNameToken.line) + "': " + ToUtf8(std::u32string(headerNameToken.match.ToString())));
855 }
856 if (p == e)
857 {
858 return ToUtf8(std::u32string(begin, end));
859 }
860 else
861 {
862 throw std::runtime_error("invalid header name in file '" + fileName + ":" + std::to_string(headerNameToken.line) + "': " + ToUtf8(std::u32string(headerNameToken.match.ToString())));
863 }
864 }
865
866 void Preprocess(const std::string& fileName, PP* pp)
867 {
868 int prevLine = pp->line;
869 pp->line = 1;
870 std::string prevFileName = pp->fileName;
871 pp->fileName = fileName;
872 File* file = new File(std::move(ToUtf32(ReadFile(fileName))));
873 pp->files.push_back(std::unique_ptr<File>(file));
874 const char32_t* p = file->content.c_str();
875 const char32_t* e = file->content.c_str() + file->content.length();
876 while (p != e)
877 {
878 const char32_t* begin = p;
879 const char32_t* end = e;
880 int numNewLines = 0;
881 GetLine(p, e, begin, end, numNewLines);
882 if (IsPPLine(begin, end))
883 {
884 PPLexer lexer(begin, end, fileName, pp->fileIndex);
885 lexer.SetSeparatorChar('\n');
886 lexer.pp = pp;
887 lexer.SetLine(pp->line);
888 lexer.SetCountLines(false);
889 PPLineParser::Parse(lexer, pp);
890 if (pp->save)
891 {
892 for (const char32_t* p = begin; p != end; ++p)
893 {
894 if (*p != '\n' && *p != '\r')
895 {
896 pp->ctext.append(1, *p);
897 }
898 }
899 }
900 }
901 else if (pp->process)
902 {
903 TextLexer lexer(begin, end, fileName, pp->fileIndex);
904 lexer.SetSeparatorChar('\n');
905 lexer.pp = pp;
906 lexer.SetLine(pp->line);
907 lexer.SetCountLines(false);
908 std::vector<soulng::lexer::Token> tokens;
909 pp->tokens = &tokens;
910 ++lexer;
911 int i = 0;
912 while (*lexer != TextTokens::END)
913 {
914 tokens.push_back(lexer.GetToken(i++));
915 ++lexer;
916 }
917 tokens = MacroExpand(tokens, pp);
918 for (const soulng::lexer::Token& token : tokens)
919 {
920 if (token.id != TextTokens::BLOCKCOMMENT && token.id != TextTokens::LINECOMMENT)
921 {
922 pp->Emit(token.match);
923 }
924 if (pp->save)
925 {
926 pp->ctext.append(token.match.ToString());
927 }
928 }
929 }
930 for (int i = 0; i < numNewLines; ++i)
931 {
932 pp->Emit(U"\n");
933 ++pp->line;
934 }
935 pp->Emit(U"\n");
936 if (pp->save)
937 {
938 pp->ctext.append(U"\n");
939 }
940 ++pp->line;
941 }
942 pp->fileName = prevFileName;
943 pp->line = prevLine;
944 }
945
946 } }