top | up | prev | next

Lexer Class

Definition at line 40 of Lexer.hpp

Constructors

Lexer(Lexer&&) delete
Lexer(const Lexer&) delete
Lexer(const char32_t* start_, const char32_t* end_, const std::string& fileName_, int fileIndex_)
Lexer(const std::u32string& content_, const std::string& fileName_, int fileIndex_)

Member Functions

void AddError(const Span& span, const std::u32string& name)
const std::set<int>& BlockCommentStates() const
void ConvertExternal(Span& span)
const char32_t* End() const
std::u32string ErrorLines(const Span& span) const
std::u32string ErrorLines(const Token& token) const
std::vector<std::unique_ptr<std::exception>> Errors()
const std::string& FileName() const
LexerFlags Flags() const
void GetColumns(const Span& span, int32_t& startCol, int32_t& endCol) const
int GetCommentTokenId() const virtual
bool GetFlag(LexerFlags flag) const
KeywordMap* GetKeywordMap()
int GetKeywordToken(const Lexeme& lexeme) const
std::u32string GetMatch(const Span& span) const
int64_t GetPos() const
Span GetSpan() const
Token GetToken(int64_t pos) const
ParsingLog* Log() const
int NextState(int state, char32_t c) virtual
void NextToken()
const char32_t* Pos() const
void ResetFlag(LexerFlags flag)
std::u32string RestOfLine(int maxLineLength)
void Retract()
void SetBlockCommentStates(const std::set<int>& blockCommentStates_)
void SetCommentTokenId(int commentTokenId_)
void SetCountLines(bool countLines_)
void SetFlag(LexerFlags flag)
void SetKeywordMap(KeywordMap* keywordMap_)
void SetLine(int line_)
void SetLog(ParsingLog* log_)
void SetPos(int64_t pos)
void SetSeparatorChar(char32_t separatorChar_)
void SetSyncTokens(const std::vector<int>& syncTokens_)
void SetTokens(const std::vector<Token>& tokens_)
const char32_t* Start() const
bool Synchronize()
void ThrowExpectationFailure(const Span& span, const std::u32string& name)
TokenLine TokenizeLine(const std::u32string& line, int lineNumber, int startState)
int operator*() const
void operator++()
Lexer& operator=(Lexer&&) delete
Lexer& operator=(const Lexer&) delete

Member Variables

std::set<int> blockCommentStates
int commentTokenId
std::u32string content
bool countLines
std::vector<Token> current
const char32_t* end
std::vector<std::unique_ptr<std::exception>> errors
int fileIndex
std::string fileName
LexerFlags flags
KeywordMap* keywordMap
Lexeme lexeme
int32_t line
ParsingLog* log
const char32_t* pos
char32_t separatorChar
const char32_t* start
std::vector<int> syncTokens
Token token
std::vector<Token> tokens

Constructor Details

Lexer Constructor

soulng::lexer::Lexer::Lexer(Lexer &&) delete

Definition at line 46 of Lexer.hpp

Lexer Constructor

soulng::lexer::Lexer::Lexer(const Lexer &) delete

Definition at line 45 of Lexer.hpp

Lexer Constructor

soulng::lexer::Lexer::Lexer(const char32_t * start_, const char32_t * end_, const std::string& fileName_, int fileIndex_)

Definition at line 22 of Lexer.cpp :
 22 :
 23 content(), fileName(fileName_), fileIndex(fileIndex_), line(1), keywordMap(nullptr), start(start_), end(end_), pos(start), current(tokens.end()), 
 24 log(nullptr), countLines(true), separatorChar('\0'), flags(), commentTokenId(-1)
 25 {
 26 }

Declaration at line 44 of Lexer.hpp

Lexer Constructor

soulng::lexer::Lexer::Lexer(const std::u32string& content_, const std::string& fileName_, int fileIndex_)

Definition at line 16 of Lexer.cpp :
 16 :
 17 content(content_), fileName(fileName_), fileIndex(fileIndex_), line(1), keywordMap(nullptr), start(content.c_str()), end(content.c_str() + content.length()), pos(start), current(tokens.end()), 
 18 log(nullptr), countLines(true), separatorChar('\0'), flags(), commentTokenId(-1)
 19 {
 20 }

Declaration at line 43 of Lexer.hpp

Member Function Details

AddError Member Function

void soulng::lexer::Lexer::AddError(const Span & span, const std::u32string& name)

Definition at line 373 of Lexer.cpp
Declaration at line 72 of Lexer.hpp

Calls: soulng::lexer::Lexer::ErrorLines , soulng::lexer::Lexer::GetFlag , soulng::lexer::Lexer::GetToken , soulng::lexer::Lexer::SetFlag

BlockCommentStates Member Function

const std::set<int >& soulng::lexer::Lexer::BlockCommentStates() const

Definition at line 513 of Lexer.cpp :
514 {
515     return blockCommentStates;
516 }

Declaration at line 85 of Lexer.hpp

ConvertExternal Member Function

void soulng::lexer::Lexer::ConvertExternal(Span & span)

Definition at line 167 of Lexer.cpp :
168 {
169     Token startToken = GetToken(span.start);
170     span.start = static_cast<int>(startToken.match.begin - start);
171     Token endToken = GetToken(span.end);
172     span.end = static_cast<int>(endToken.match.end - start);
173 }

Declaration at line 61 of Lexer.hpp

Calls: soulng::lexer::Lexer::GetToken

End Member Function

const char32_t * soulng::lexer::Lexer::End() const

Definition at line 75 of Lexer.hpp :
 75 { return end; }

ErrorLines Member Function

std::u32string soulng::lexer::Lexer::ErrorLines(const Span & span) const

Definition at line 311 of Lexer.cpp
Declaration at line 69 of Lexer.hpp

Calls: soulng::lexer::Lexeme::ToString , soulng::lexer::Lexer::GetToken

Called by: soulng::lexer::Lexer::AddError , soulng::lexer::Lexer::ThrowExpectationFailure

ErrorLines Member Function

std::u32string soulng::lexer::Lexer::ErrorLines(const Token & token) const

Definition at line 295 of Lexer.cpp
Declaration at line 68 of Lexer.hpp

Calls: soulng::lexer::Lexeme::ToString

Errors Member Function

std::vector<std::unique_ptr<std::exception>> soulng::lexer::Lexer::Errors()

Definition at line 73 of Lexer.hpp :
 73 { return std::move(errors); }

FileName Member Function

const std::string& soulng::lexer::Lexer::FileName() const

Definition at line 59 of Lexer.hpp :
 59 { return fileName; }

Flags Member Function

LexerFlags soulng::lexer::Lexer::Flags() const

Definition at line 87 of Lexer.hpp :
 87 { return flags; }

GetColumns Member Function

void soulng::lexer::Lexer::GetColumns(const Span & span, int32_t& startCol, int32_t& endCol) const

Definition at line 342 of Lexer.cpp
Declaration at line 70 of Lexer.hpp

Calls: soulng::lexer::Lexer::GetToken

GetCommentTokenId Member Function

int soulng::lexer::Lexer::GetCommentTokenId() const virtual

Definition at line 94 of Lexer.hpp :
 94 { return -1; }

GetFlag Member Function

bool soulng::lexer::Lexer::GetFlag(LexerFlags flag) const

Definition at line 88 of Lexer.hpp :
 88 { return (flags & flag) != LexerFlags::none; }

Called by: soulng::lexer::Lexer::AddError , soulng::lexer::Lexer::Synchronize

GetKeywordMap Member Function

KeywordMap * soulng::lexer::Lexer::GetKeywordMap()

Definition at line 56 of Lexer.hpp :
 56 { return keywordMap; }

GetKeywordToken Member Function

int soulng::lexer::Lexer::GetKeywordToken(const Lexeme & lexeme) const

Definition at line 155 of Lexer.cpp :
156 {
157     if (keywordMap)
158     {
159         return keywordMap->GetKeywordToken(lexeme);
160     }
161     else
162     {
163         return INVALID_TOKEN;
164     }
165 }

Declaration at line 57 of Lexer.hpp

Calls: soulng::lexer::KeywordMap::GetKeywordToken

GetMatch Member Function

std::u32string soulng::lexer::Lexer::GetMatch(const Span & span) const

Definition at line 206 of Lexer.cpp
Declaration at line 67 of Lexer.hpp

Calls: soulng::lexer::Lexeme::ToString , soulng::lexer::Lexer::GetToken

GetPos Member Function

int64_t soulng::lexer::Lexer::GetPos() const

Definition at line 48 of Lexer.cpp :
 49 {
 50     int32_t p = static_cast<int32_t>(current - tokens.begin());
 51     return (static_cast<int64_t>(line) << 32) | static_cast<int64_t>(p);
 52 }

Declaration at line 52 of Lexer.hpp

Called by: soulng::lexer::Lexer::GetSpan , soulng::lexer::Lexer::NextToken

GetSpan Member Function

Span soulng::lexer::Lexer::GetSpan() const

Definition at line 60 of Lexer.hpp :
 60 { return Span(fileIndex, line, static_cast<int32_t>(GetPos())); }

Calls: soulng::lexer::Lexer::GetPos

GetToken Member Function

Token soulng::lexer::Lexer::GetToken(int64_t pos) const

Definition at line 175 of Lexer.cpp
Declaration at line 62 of Lexer.hpp

Called by: soulng::lexer::Lexer::AddError , soulng::lexer::Lexer::ConvertExternal , soulng::lexer::Lexer::ErrorLines , soulng::lexer::Lexer::GetColumns , soulng::lexer::Lexer::GetMatch , soulng::lexer::Lexer::ThrowExpectationFailure

Log Member Function

ParsingLog * soulng::lexer::Lexer::Log() const

Definition at line 78 of Lexer.hpp :
 78 { return log; }

NextState Member Function

int soulng::lexer::Lexer::NextState(int state, char32_t c) virtual

Definition at line 150 of Lexer.cpp :
151 {
152     return -1;
153 }

Declaration at line 54 of Lexer.hpp

Derived class overrides: TrivialLexer::NextState

Called by: soulng::lexer::Lexer::NextToken , soulng::lexer::Lexer::TokenizeLine

NextToken Member Function

void soulng::lexer::Lexer::NextToken()

Definition at line 60 of Lexer.cpp
Declaration at line 113 of Lexer.hpp

Calls: soulng::lexer::Lexer::GetPos , soulng::lexer::Lexer::NextState , soulng::lexer::Lexer::SetPos

Called by: soulng::lexer::Lexer::operator++

Pos Member Function

const char32_t * soulng::lexer::Lexer::Pos() const

Definition at line 76 of Lexer.hpp :
 76 { return pos; }

ResetFlag Member Function

void soulng::lexer::Lexer::ResetFlag(LexerFlags flag)

Definition at line 90 of Lexer.hpp :
 90 { flags = flags & ~flag; }

RestOfLine Member Function

std::u32string soulng::lexer::Lexer::RestOfLine(int maxLineLength)

Definition at line 387 of Lexer.cpp :
388 {
389     std::u32string restOfLine(current->match.ToString() + std::u32string(current->match.end, pos) + std::u32string(pos, LineEnd(end, pos)));
390     if (maxLineLength != 0)
391     {
392         restOfLine = restOfLine.substr(0, maxLineLength);
393     }
394     return restOfLine;
395 }

Declaration at line 79 of Lexer.hpp

Retract Member Function

void soulng::lexer::Lexer::Retract()

Definition at line 58 of Lexer.hpp :
 58 { token.match.end = pos; }

SetBlockCommentStates Member Function

void soulng::lexer::Lexer::SetBlockCommentStates(const std::set<int >& blockCommentStates_)

Definition at line 508 of Lexer.cpp :
509 {
510     blockCommentStates = blockCommentStates_;
511 }

Declaration at line 84 of Lexer.hpp

SetCommentTokenId Member Function

void soulng::lexer::Lexer::SetCommentTokenId(int commentTokenId_)

Definition at line 86 of Lexer.hpp :
 86 { commentTokenId = commentTokenId_; }

SetCountLines Member Function

void soulng::lexer::Lexer::SetCountLines(bool countLines_)

Definition at line 65 of Lexer.hpp :
 65 { countLines = countLines_; }

SetFlag Member Function

void soulng::lexer::Lexer::SetFlag(LexerFlags flag)

Definition at line 89 of Lexer.hpp :
 89 { flags = flags | flag; }

Called by: soulng::lexer::Lexer::AddError , soulng::lexer::Lexer::Synchronize

SetKeywordMap Member Function

void soulng::lexer::Lexer::SetKeywordMap(KeywordMap * keywordMap_)

Definition at line 55 of Lexer.hpp :
 55 { keywordMap = keywordMap_; }

SetLine Member Function

void soulng::lexer::Lexer::SetLine(int line_)

Definition at line 64 of Lexer.hpp :
 64 { line = line_; }

SetLog Member Function

void soulng::lexer::Lexer::SetLog(ParsingLog * log_)

Definition at line 77 of Lexer.hpp :
 77 { log = log_; }

SetPos Member Function

void soulng::lexer::Lexer::SetPos(int64_t pos)

Definition at line 54 of Lexer.cpp :
 55 {
 56     current = tokens.begin() + static_cast<int32_t>(pos);
 57     line = static_cast<int32_t>(pos >> 32);
 58 }

Declaration at line 53 of Lexer.hpp

Called by: soulng::lexer::Lexer::NextToken

SetSeparatorChar Member Function

void soulng::lexer::Lexer::SetSeparatorChar(char32_t separatorChar_)

Definition at line 80 of Lexer.hpp :
 80 { separatorChar = separatorChar_; }

SetSyncTokens Member Function

void soulng::lexer::Lexer::SetSyncTokens(const std::vector<int >& syncTokens_)

Definition at line 481 of Lexer.cpp :
482 {
483     syncTokens = syncTokens_;
484 }

Declaration at line 82 of Lexer.hpp

SetTokens Member Function

void soulng::lexer::Lexer::SetTokens(const std::vector<Token >& tokens_)

Definition at line 188 of Lexer.cpp
Declaration at line 63 of Lexer.hpp

Start Member Function

const char32_t * soulng::lexer::Lexer::Start() const

Definition at line 74 of Lexer.hpp :
 74 { return start; }

Synchronize Member Function

bool soulng::lexer::Lexer::Synchronize()

Definition at line 486 of Lexer.cpp
Declaration at line 83 of Lexer.hpp

Calls: soulng::lexer::Lexer::GetFlag , soulng::lexer::Lexer::SetFlag

ThrowExpectationFailure Member Function

void soulng::lexer::Lexer::ThrowExpectationFailure(const Span & span, const std::u32string& name)

Definition at line 367 of Lexer.cpp :
368 {
369     Token token = GetToken(span.start);
370     throw ParsingException("parsing error in '" + fileName + ":" + std::to_string(token.line) + "': " + ToUtf8(name) + " expected:\n" + ToUtf8(ErrorLines(span)), fileName, span);
371 }

Declaration at line 71 of Lexer.hpp

Calls: soulng::lexer::Lexer::ErrorLines , soulng::lexer::Lexer::GetToken

TokenizeLine Member Function

TokenLine soulng::lexer::Lexer::TokenizeLine(const std::u32string& line, int lineNumber, int startState)

Definition at line 397 of Lexer.cpp
Declaration at line 81 of Lexer.hpp

Calls: soulng::lexer::Lexer::NextState

operator* Member Function

int soulng::lexer::Lexer::operator*() const

Definition at line 50 of Lexer.hpp :
 50 { return current->id; }

operator++ Member Function

void soulng::lexer::Lexer::operator++()

Definition at line 32 of Lexer.cpp
Declaration at line 51 of Lexer.hpp

Calls: soulng::lexer::Lexer::NextToken

operator= Member Function

Lexer & soulng::lexer::Lexer::operator=(Lexer &&) delete

Definition at line 48 of Lexer.hpp

operator= Member Function

Lexer & soulng::lexer::Lexer::operator=(const Lexer &) delete

Definition at line 47 of Lexer.hpp

top | up | prev | next