Lexer Class

Definition

Line 15 of System.Lex/Lexer.cm

Inheritance

Constructors & Destructor

suppress Lexer(const Lexer&)
public Lexer(const String<uchar>& content_, const String<char>& fileName_, int fileIndex_)
public Lexer(const uchar* start_, const uchar* end_, const String<char>& fileName_, int fileIndex_)
public virtual ~Lexer()

Member Functions

public void AddError(const Span& span, const String<uchar>& name)
public nothrow const Set<int, Less<int>>& BlockCommentStates() const
public void ConvertExternal(Span& span)
public const uchar* End() const
public String<uchar> ErrorLines(const Span& span) const
public String<uchar> ErrorLines(const Token& token) const
public nothrow List<UniquePtr<Exception>> Errors()
public nothrow const List<int>& FarthestRuleContext() const
public const String<char>& FileName() const
public inline nothrow LexerFlags Flags() const
public char GetChar(long pos) const
public void GetColumns(const Span& span, int& startCol, int& endCol) const
protected virtual nothrow int GetCommentTokenId() const
public inline nothrow Span GetCurrentSpan() const
public double GetDouble(long pos) const
public String<char> GetFarthestError() const
public inline nothrow bool GetFlag(LexerFlags flag) const
public int GetInt(long pos) const
public KeywordMap* GetKeywordMap()
public int GetKeywordToken(const Lexeme& lexeme) const
public String<uchar> GetMatch(const Span& span) const
public nothrow String<char> GetParserStateStr() const
public long GetPos() const
public Span GetSpan() const
public Token GetToken(long pos) const
public uchar GetUChar(long pos) const
public wchar GetWChar(long pos) const
public ParsingLog* Log() const
public virtual int NextState(int state, uchar c)
public void NextToken()
public void PopRule()
public Span PopSpan()
public const uchar* Pos() const
public void PushRule(int ruleId)
public void PushSpan()
public inline nothrow void ResetFlag(LexerFlags flag)
public String<uchar> RestOfLine(int maxLineLength)
public void Retract()
public nothrow const List<int>& RuleContext() const
public void SetBlockCommentStates(const Set<int, Less<int>>& blockCommentStates_) const
public nothrow void SetCommentTokenId(int commentTokenId_)
public void SetCountLines(bool countLines_)
public inline nothrow void SetFlag(LexerFlags flag)
public void SetKeywordMap(KeywordMap* keywordMap_)
public void SetLine(int line_)
public void SetLog(ParsingLog* log_)
public void SetPos(long pos)
public nothrow void SetRuleNameListPtr(List<String<char>>* ruleNameListPtr_)
public void SetSeparatorChar(uchar separatorChar_)
public void SetSpan(long pos)
public nothrow void SetSyncTokens(const List<int>& syncTokens_)
public void SetTokens(const List<Token>& tokens_)
public const uchar* Start() const
public nothrow bool Synchronize()
public void ThrowExpectationFailure(const Span& span, const String<uchar>& name)
public void ThrowFarthestError()
public TokenLine TokenizeLine(const String<uchar>& line, int lineNumber, int startState)
public int operator*() const
public void operator++()
suppress void operator=(const Lexer&)

Member Variables

private Set<int, Less<int>> blockCommentStates
private int commentTokenId
private String<uchar> content
private bool countLines
private RandomAccessIter<Token, Token&, Token*> current
private Span currentSpan
private const uchar* end
private List<UniquePtr<Exception>> errors
private long farthestPos
private List<int> farthestRuleContext
private int fileIndex
private String<char> fileName
private LexerFlags flags
private KeywordMap* keywordMap
protected Lexeme lexeme
protected int line
private ParsingLog* log
private const uchar* pos
private List<int> ruleContext
private List<String<char>>* ruleNameListPtr
private uchar separatorChar
private Stack<Span> spanStack
private const uchar* start
private List<int> syncTokens
public Token token
private List<Token> tokens

Constructor& Destructor Details

Lexer Constructor

suppress Lexer(const Lexer&)

Definition
Line 27 of System.Lex/Lexer.cm


Lexer Constructor

public Lexer(const String<uchar>& content_, const String<char>& fileName_, int fileIndex_)

Definition
Line 17 of System.Lex/Lexer.cm


Lexer Constructor

public Lexer(const uchar* start_, const uchar* end_, const String<char>& fileName_, int fileIndex_)

Definition
Line 22 of System.Lex/Lexer.cm


~Lexer Destructor

public virtual ~Lexer()

Definition
Line 45 of System.Lex/Lexer.cm


Member Function Details

AddError Member Function

public void AddError(const Span& span, const String<uchar>& name)

Definition
Line 430 of System.Lex/Lexer.cm


BlockCommentStates Member Function

public nothrow const Set<int, Less<int>>& BlockCommentStates()

Definition
Line 33 of System.Lex/Lexer.cm


ConvertExternal Member Function

public void ConvertExternal(Span& span)

Definition
Line 266 of System.Lex/Lexer.cm


End Member Function

public const uchar* End()

Definition
Line 112 of System.Lex/Lexer.cm


ErrorLines Member Function

public String<uchar> ErrorLines(const Span& span)

Definition
Line 361 of System.Lex/Lexer.cm


ErrorLines Member Function

public String<uchar> ErrorLines(const Token& token)

Definition
Line 346 of System.Lex/Lexer.cm


Errors Member Function

public nothrow List<UniquePtr<Exception>> Errors()

Definition
Line 443 of System.Lex/Lexer.cm


FarthestRuleContext Member Function

public nothrow const List<int>& FarthestRuleContext()

Definition
Line 577 of System.Lex/Lexer.cm


FileName Member Function

public const String<char>& FileName()

Definition
Line 64 of System.Lex/Lexer.cm


Flags Member Function

public inline nothrow LexerFlags Flags()

Definition
Line 557 of System.Lex/Lexer.cm


GetChar Member Function

public char GetChar(long pos)

Definition
Line 289 of System.Lex/Lexer.cm


GetColumns Member Function

public void GetColumns(const Span& span, int& startCol, int& endCol)

Definition
Line 391 of System.Lex/Lexer.cm


GetCommentTokenId Member Function

protected virtual nothrow int GetCommentTokenId()

Definition
Line 41 of System.Lex/Lexer.cm


GetCurrentSpan Member Function

public inline nothrow Span GetCurrentSpan()

Definition
Line 95 of System.Lex/Lexer.cm


GetDouble Member Function

public double GetDouble(long pos)

Definition
Line 309 of System.Lex/Lexer.cm


GetFarthestError Member Function

public String<char> GetFarthestError()

Definition
Line 420 of System.Lex/Lexer.cm


GetFlag Member Function

public inline nothrow bool GetFlag(LexerFlags flag)

Definition
Line 561 of System.Lex/Lexer.cm


GetInt Member Function

public int GetInt(long pos)

Definition
Line 304 of System.Lex/Lexer.cm


GetKeywordMap Member Function

public KeywordMap* GetKeywordMap()

Definition
Line 56 of System.Lex/Lexer.cm


GetKeywordToken Member Function

public int GetKeywordToken(const Lexeme& lexeme)

Definition
Line 255 of System.Lex/Lexer.cm


GetMatch Member Function

public String<uchar> GetMatch(const Span& span)

Definition
Line 331 of System.Lex/Lexer.cm


GetParserStateStr Member Function

public nothrow String<char> GetParserStateStr()

Definition
Line 585 of System.Lex/Lexer.cm


GetPos Member Function

public long GetPos()

Definition
Line 156 of System.Lex/Lexer.cm


GetSpan Member Function

public Span GetSpan()

Definition
Line 68 of System.Lex/Lexer.cm


GetToken Member Function

public Token GetToken(long pos)

Definition
Line 277 of System.Lex/Lexer.cm


GetUChar Member Function

public uchar GetUChar(long pos)

Definition
Line 299 of System.Lex/Lexer.cm


GetWChar Member Function

public wchar GetWChar(long pos)

Definition
Line 294 of System.Lex/Lexer.cm


Log Member Function

public ParsingLog* Log()

Definition
Line 124 of System.Lex/Lexer.cm


NextState Member Function

public virtual int NextState(int state, uchar c)

Definition
Line 456 of System.Lex/Lexer.cm


NextToken Member Function

public void NextToken()

Definition
Line 166 of System.Lex/Lexer.cm


PopRule Member Function

public void PopRule()

Definition
Line 608 of System.Lex/Lexer.cm


PopSpan Member Function

public Span PopSpan()

Definition
Line 77 of System.Lex/Lexer.cm


Pos Member Function

public const uchar* Pos()

Definition
Line 116 of System.Lex/Lexer.cm


PushRule Member Function

public void PushRule(int ruleId)

Definition
Line 604 of System.Lex/Lexer.cm


PushSpan Member Function

public void PushSpan()

Definition
Line 72 of System.Lex/Lexer.cm


ResetFlag Member Function

public inline nothrow void ResetFlag(LexerFlags flag)

Definition
Line 569 of System.Lex/Lexer.cm


RestOfLine Member Function

public String<uchar> RestOfLine(int maxLineLength)

Definition
Line 447 of System.Lex/Lexer.cm


Retract Member Function

public void Retract()

Definition
Line 60 of System.Lex/Lexer.cm


RuleContext Member Function

public nothrow const List<int>& RuleContext()

Definition
Line 573 of System.Lex/Lexer.cm


SetBlockCommentStates Member Function

public void SetBlockCommentStates(const Set<int, Less<int>>& blockCommentStates_)

Definition
Line 29 of System.Lex/Lexer.cm


SetCommentTokenId Member Function

public nothrow void SetCommentTokenId(int commentTokenId_)

Definition
Line 37 of System.Lex/Lexer.cm


SetCountLines Member Function

public void SetCountLines(bool countLines_)

Definition
Line 103 of System.Lex/Lexer.cm


SetFlag Member Function

public inline nothrow void SetFlag(LexerFlags flag)

Definition
Line 565 of System.Lex/Lexer.cm


SetKeywordMap Member Function

public void SetKeywordMap(KeywordMap* keywordMap_)

Definition
Line 52 of System.Lex/Lexer.cm


SetLine Member Function

public void SetLine(int line_)

Definition
Line 99 of System.Lex/Lexer.cm


SetLog Member Function

public void SetLog(ParsingLog* log_)

Definition
Line 120 of System.Lex/Lexer.cm


SetPos Member Function

public void SetPos(long pos)

Definition
Line 161 of System.Lex/Lexer.cm


SetRuleNameListPtr Member Function

public nothrow void SetRuleNameListPtr(List<String<char>>* ruleNameListPtr_)

Definition
Line 581 of System.Lex/Lexer.cm


SetSeparatorChar Member Function

public void SetSeparatorChar(uchar separatorChar_)

Definition
Line 128 of System.Lex/Lexer.cm


SetSpan Member Function

public void SetSpan(long pos)

Definition
Line 83 of System.Lex/Lexer.cm


SetSyncTokens Member Function

public nothrow void SetSyncTokens(const List<int>& syncTokens_)

Definition
Line 531 of System.Lex/Lexer.cm


SetTokens Member Function

public void SetTokens(const List<Token>& tokens_)

Definition
Line 314 of System.Lex/Lexer.cm


Start Member Function

public const uchar* Start()

Definition
Line 108 of System.Lex/Lexer.cm


Synchronize Member Function

public nothrow bool Synchronize()

Definition
Line 535 of System.Lex/Lexer.cm


ThrowExpectationFailure Member Function

public void ThrowExpectationFailure(const Span& span, const String<uchar>& name)

Definition
Line 415 of System.Lex/Lexer.cm


ThrowFarthestError Member Function

public void ThrowFarthestError()

Definition
Line 426 of System.Lex/Lexer.cm


TokenizeLine Member Function

public TokenLine TokenizeLine(const String<uchar>& line, int lineNumber, int startState)

Definition
Line 460 of System.Lex/Lexer.cm


operator* Member Function

public int operator*()

Definition
Line 48 of System.Lex/Lexer.cm


operator++ Member Function

public void operator++()

Definition
Line 132 of System.Lex/Lexer.cm


operator= Member Function

suppress void operator=(const Lexer&)

Definition
Line 28 of System.Lex/Lexer.cm