1.2.5 Adding a Lexer

The lexer contains rules for matching input with regular expressions and returning corresponging tokens to the parser.

The lexer file syntax is described in this document.

To add a lexer to minilang:

• Right-click your project in the Solution Explorer and select 'Add | New Item...'.
• Select Visual C++ | Utility | Text File (.txt) item and name the file minilang_lexer.lexer .
• Add the following code to the lexer file:
  
  export module minilang.lexer;
  
  import minilang.token;
  import minilang.keyword;
  import minilang.expr;
  
  lexer MinilangLexer
  {
      rules
      {
          "{separators}" {}
          "{identifier}" { int64_t kw = lexer.GetKeywordToken(lexer.CurrentToken().match); if (kw == INVALID_TOKEN) return ID; else return kw; }
          "{integer}" { return INTEGER_LITERAL; }
          ";" { return SEMICOLON; }
          "\(" { return LPAREN; }
          "\)" { return RPAREN; }
          "\{" { return LBRACE; }
          "\}" { return RBRACE; }
          "\+" { return PLUS; }
          "-" { return MINUS; }
          "\*" { return MUL; }
          "/" { return DIV; }
          "%" { return MOD; }
          "!" { return NOT; }
          "==" { return EQ; }
          "!=" { return NEQ; }
          "<=" { return LEQ; }
          ">=" { return GEQ; }
          "<" { return LESS; }
          ">" { return GREATER; }
          "=" { return ASSIGN; }
          "," { return COMMA; }
      }
  }