Lexer Class

Definition

Line 15 of System.Lex/Lexer.cm

Inheritance

Constructors & Destructor

suppress Lexer(const Lexer&)
public Lexer(const String<uchar>& content_, const String<char>& fileName_, int fileIndex_)
public Lexer(const uchar* start_, const uchar* end_, const String<char>& fileName_, int fileIndex_)
public virtual ~Lexer()

Member Functions

public void AddError(const Span& span, const String<uchar>& name)
public nothrow const Set<int, Less<int>>& BlockCommentStates() const
public void ConvertExternal(Span& span)
public const uchar* End() const
public String<uchar> ErrorLines(const Span& span) const
public String<uchar> ErrorLines(const Token& token) const
public nothrow List<UniquePtr<Exception>> Errors()
public nothrow const List<int>& FarthestRuleContext() const
public const String<char>& FileName() const
public inline nothrow LexerFlags Flags() const
public char GetChar(long pos) const
public void GetColumns(const Span& span, int& startCol, int& endCol) const
protected virtual nothrow int GetCommentTokenId() const
public inline nothrow Span GetCurrentSpan() const
public double GetDouble(long pos) const
public String<char> GetFarthestError() const
public inline nothrow bool GetFlag(LexerFlags flag) const
public int GetInt(long pos) const
public KeywordMap* GetKeywordMap()
public int GetKeywordToken(const Lexeme& lexeme) const
public String<uchar> GetMatch(const Span& span) const
public nothrow String<char> GetParserStateStr() const
public long GetPos() const
public Span GetSpan() const
public Token GetToken(long pos) const
public uchar GetUChar(long pos) const
public wchar GetWChar(long pos) const
public ParsingLog* Log() const
public virtual int NextState(int state, uchar c)
public void NextToken()
public void PopRule()
public Span PopSpan()
public const uchar* Pos() const
public void PushRule(int ruleId)
public void PushSpan()
public inline nothrow void ResetFlag(LexerFlags flag)
public String<uchar> RestOfLine(int maxLineLength)
public void Retract()
public nothrow const List<int>& RuleContext() const
public void SetBlockCommentStates(const Set<int, Less<int>>& blockCommentStates_) const
public nothrow void SetCommentTokenId(int commentTokenId_)
public void SetCountLines(bool countLines_)
public inline nothrow void SetFlag(LexerFlags flag)
public void SetKeywordMap(KeywordMap* keywordMap_)
public void SetLine(int line_)
public void SetLog(ParsingLog* log_)
public void SetPos(long pos)
public nothrow void SetRuleNameListPtr(List<String<char>>* ruleNameListPtr_)
public void SetSeparatorChar(uchar separatorChar_)
public void SetSpan(long pos)
public nothrow void SetSyncTokens(const List<int>& syncTokens_)
public void SetTokens(const List<Token>& tokens_)
public const uchar* Start() const
public nothrow bool Synchronize()
public void ThrowExpectationFailure(const Span& span, const String<uchar>& name)
public void ThrowFarthestError()
public TokenLine TokenizeLine(const String<uchar>& line, int lineNumber, int startState)
public int operator*() const
public void operator++()
suppress void operator=(const Lexer&)

Member Variables

private Set<int, Less<int>> blockCommentStates
private int commentTokenId
private String<uchar> content
private bool countLines
private RandomAccessIter<Token, Token&, Token*> current
private Span currentSpan
private const uchar* end
private List<UniquePtr<Exception>> errors
private long farthestPos
private List<int> farthestRuleContext
private int fileIndex
private String<char> fileName
private LexerFlags flags
private KeywordMap* keywordMap
protected Lexeme lexeme
protected int line
private ParsingLog* log
private const uchar* pos
private List<int> ruleContext
private List<String<char>>* ruleNameListPtr
private uchar separatorChar
private Stack<Span> spanStack
private const uchar* start
private List<int> syncTokens
public Token token
private List<Token> tokens

Constructor& Destructor Details

Lexer Constructor

suppress Lexer(const Lexer&)

Definition
Line 28 of System.Lex/Lexer.cm


Lexer Constructor

public Lexer(const String<uchar>& content_, const String<char>& fileName_, int fileIndex_)

Definition
Line 17 of System.Lex/Lexer.cm


Lexer Constructor

public Lexer(const uchar* start_, const uchar* end_, const String<char>& fileName_, int fileIndex_)

Definition
Line 23 of System.Lex/Lexer.cm


~Lexer Destructor

public virtual ~Lexer()

Definition
Line 46 of System.Lex/Lexer.cm


Member Function Details

AddError Member Function

public void AddError(const Span& span, const String<uchar>& name)

Definition
Line 431 of System.Lex/Lexer.cm


BlockCommentStates Member Function

public nothrow const Set<int, Less<int>>& BlockCommentStates()

Definition
Line 34 of System.Lex/Lexer.cm


ConvertExternal Member Function

public void ConvertExternal(Span& span)

Definition
Line 267 of System.Lex/Lexer.cm


End Member Function

public const uchar* End()

Definition
Line 113 of System.Lex/Lexer.cm


ErrorLines Member Function

public String<uchar> ErrorLines(const Span& span)

Definition
Line 362 of System.Lex/Lexer.cm


ErrorLines Member Function

public String<uchar> ErrorLines(const Token& token)

Definition
Line 347 of System.Lex/Lexer.cm


Errors Member Function

public nothrow List<UniquePtr<Exception>> Errors()

Definition
Line 444 of System.Lex/Lexer.cm


FarthestRuleContext Member Function

public nothrow const List<int>& FarthestRuleContext()

Definition
Line 578 of System.Lex/Lexer.cm


FileName Member Function

public const String<char>& FileName()

Definition
Line 65 of System.Lex/Lexer.cm


Flags Member Function

public inline nothrow LexerFlags Flags()

Definition
Line 558 of System.Lex/Lexer.cm


GetChar Member Function

public char GetChar(long pos)

Definition
Line 290 of System.Lex/Lexer.cm


GetColumns Member Function

public void GetColumns(const Span& span, int& startCol, int& endCol)

Definition
Line 392 of System.Lex/Lexer.cm


GetCommentTokenId Member Function

protected virtual nothrow int GetCommentTokenId()

Definition
Line 42 of System.Lex/Lexer.cm


GetCurrentSpan Member Function

public inline nothrow Span GetCurrentSpan()

Definition
Line 96 of System.Lex/Lexer.cm


GetDouble Member Function

public double GetDouble(long pos)

Definition
Line 310 of System.Lex/Lexer.cm


GetFarthestError Member Function

public String<char> GetFarthestError()

Definition
Line 421 of System.Lex/Lexer.cm


GetFlag Member Function

public inline nothrow bool GetFlag(LexerFlags flag)

Definition
Line 562 of System.Lex/Lexer.cm


GetInt Member Function

public int GetInt(long pos)

Definition
Line 305 of System.Lex/Lexer.cm


GetKeywordMap Member Function

public KeywordMap* GetKeywordMap()

Definition
Line 57 of System.Lex/Lexer.cm


GetKeywordToken Member Function

public int GetKeywordToken(const Lexeme& lexeme)

Definition
Line 256 of System.Lex/Lexer.cm


GetMatch Member Function

public String<uchar> GetMatch(const Span& span)

Definition
Line 332 of System.Lex/Lexer.cm


GetParserStateStr Member Function

public nothrow String<char> GetParserStateStr()

Definition
Line 586 of System.Lex/Lexer.cm


GetPos Member Function

public long GetPos()

Definition
Line 157 of System.Lex/Lexer.cm


GetSpan Member Function

public Span GetSpan()

Definition
Line 69 of System.Lex/Lexer.cm


GetToken Member Function

public Token GetToken(long pos)

Definition
Line 278 of System.Lex/Lexer.cm


GetUChar Member Function

public uchar GetUChar(long pos)

Definition
Line 300 of System.Lex/Lexer.cm


GetWChar Member Function

public wchar GetWChar(long pos)

Definition
Line 295 of System.Lex/Lexer.cm


Log Member Function

public ParsingLog* Log()

Definition
Line 125 of System.Lex/Lexer.cm


NextState Member Function

public virtual int NextState(int state, uchar c)

Definition
Line 457 of System.Lex/Lexer.cm


NextToken Member Function

public void NextToken()

Definition
Line 167 of System.Lex/Lexer.cm


PopRule Member Function

public void PopRule()

Definition
Line 609 of System.Lex/Lexer.cm


PopSpan Member Function

public Span PopSpan()

Definition
Line 78 of System.Lex/Lexer.cm


Pos Member Function

public const uchar* Pos()

Definition
Line 117 of System.Lex/Lexer.cm


PushRule Member Function

public void PushRule(int ruleId)

Definition
Line 605 of System.Lex/Lexer.cm


PushSpan Member Function

public void PushSpan()

Definition
Line 73 of System.Lex/Lexer.cm


ResetFlag Member Function

public inline nothrow void ResetFlag(LexerFlags flag)

Definition
Line 570 of System.Lex/Lexer.cm


RestOfLine Member Function

public String<uchar> RestOfLine(int maxLineLength)

Definition
Line 448 of System.Lex/Lexer.cm


Retract Member Function

public void Retract()

Definition
Line 61 of System.Lex/Lexer.cm


RuleContext Member Function

public nothrow const List<int>& RuleContext()

Definition
Line 574 of System.Lex/Lexer.cm


SetBlockCommentStates Member Function

public void SetBlockCommentStates(const Set<int, Less<int>>& blockCommentStates_)

Definition
Line 30 of System.Lex/Lexer.cm


SetCommentTokenId Member Function

public nothrow void SetCommentTokenId(int commentTokenId_)

Definition
Line 38 of System.Lex/Lexer.cm


SetCountLines Member Function

public void SetCountLines(bool countLines_)

Definition
Line 104 of System.Lex/Lexer.cm


SetFlag Member Function

public inline nothrow void SetFlag(LexerFlags flag)

Definition
Line 566 of System.Lex/Lexer.cm


SetKeywordMap Member Function

public void SetKeywordMap(KeywordMap* keywordMap_)

Definition
Line 53 of System.Lex/Lexer.cm


SetLine Member Function

public void SetLine(int line_)

Definition
Line 100 of System.Lex/Lexer.cm


SetLog Member Function

public void SetLog(ParsingLog* log_)

Definition
Line 121 of System.Lex/Lexer.cm


SetPos Member Function

public void SetPos(long pos)

Definition
Line 162 of System.Lex/Lexer.cm


SetRuleNameListPtr Member Function

public nothrow void SetRuleNameListPtr(List<String<char>>* ruleNameListPtr_)

Definition
Line 582 of System.Lex/Lexer.cm


SetSeparatorChar Member Function

public void SetSeparatorChar(uchar separatorChar_)

Definition
Line 129 of System.Lex/Lexer.cm


SetSpan Member Function

public void SetSpan(long pos)

Definition
Line 84 of System.Lex/Lexer.cm


SetSyncTokens Member Function

public nothrow void SetSyncTokens(const List<int>& syncTokens_)

Definition
Line 532 of System.Lex/Lexer.cm


SetTokens Member Function

public void SetTokens(const List<Token>& tokens_)

Definition
Line 315 of System.Lex/Lexer.cm


Start Member Function

public const uchar* Start()

Definition
Line 109 of System.Lex/Lexer.cm


Synchronize Member Function

public nothrow bool Synchronize()

Definition
Line 536 of System.Lex/Lexer.cm


ThrowExpectationFailure Member Function

public void ThrowExpectationFailure(const Span& span, const String<uchar>& name)

Definition
Line 416 of System.Lex/Lexer.cm


ThrowFarthestError Member Function

public void ThrowFarthestError()

Definition
Line 427 of System.Lex/Lexer.cm


TokenizeLine Member Function

public TokenLine TokenizeLine(const String<uchar>& line, int lineNumber, int startState)

Definition
Line 461 of System.Lex/Lexer.cm


operator* Member Function

public int operator*()

Definition
Line 49 of System.Lex/Lexer.cm


operator++ Member Function

public void operator++()

Definition
Line 133 of System.Lex/Lexer.cm


operator= Member Function

suppress void operator=(const Lexer&)

Definition
Line 29 of System.Lex/Lexer.cm