Scanner Class Reference

Lexical analyzer for the GSC programming language. More...

#include <scanner.hpp>

Collaboration diagram for Scanner:

Public Member Functions
	Scanner (std::string_view program)
	Constructs a Scanner object.
void	scanTokens ()
	Scans the entire source code and generates a list of tokens.
std::vector< Token >	getTokens () const

Private Member Functions
void	addToken (TokenType type)
void	addToken (TokenType type, std::any literal)
void	scanToken ()
void	identifier ()
void	number ()
void	string ()
bool	match (const char &expected)
char	peek () const
char	peekNext () const
bool	isAtEnd () const
char	advance ()

Private Attributes
std::string_view	program
std::vector< Token >	tokens
int	start = 0
int	current = 0
int	line = 1

Detailed Description

Lexical analyzer for the GSC programming language.

The Scanner class is responsible for tokenizing the input source code. It reads the source code character by character and generates a list of tokens that represent the structure of the code.

Definition at line 23 of file scanner.hpp.

Constructor & Destructor Documentation

Scanner()

Scanner::Scanner

(

std::string_view

program

)

Constructs a Scanner object.

Parameters

program

The source code to be scanned.

Note

The program is stored as a string_view to avoid unnecessary copies.

Definition at line 167 of file scanner.cpp.

                                       : program{program} {
  tokens.reserve(256);
}

Member Function Documentation

addToken() [1/2]

void Scanner::addToken ( TokenType  type )

private

Definition at line 12 of file scanner.cpp.

{ addToken(type, nullptr); }

addToken() [2/2]

void Scanner::addToken ( TokenType  type, std::any  literal  )

private

Definition at line 14 of file scanner.cpp.

                                                     {
  std::string text{program.substr(start, current - start)};
  tokens.emplace_back(type, std::move(text), std::move(literal), line);
}

advance()

char Scanner::advance ( )

private

Definition at line 165 of file scanner.cpp.

{ return program[current++]; }

getTokens()

std::vector< Token > Scanner::getTokens

(

)

const

Definition at line 180 of file scanner.cpp.

{ return tokens; }

identifier()

void Scanner::identifier ( )

private

Definition at line 98 of file scanner.cpp.

                         {
  while (isAlphaNumeric(peek())) {
    advance();
  }
 
  std::string text = std::string{program.substr(start, current - start)};
 
  TokenType type = keywords.count(text) ? keywords.at(text) : IDENTIFIER;
  addToken(type);
}

isAtEnd()

bool Scanner::isAtEnd ( ) const

private

Definition at line 161 of file scanner.cpp.

                            {
  return current >= static_cast<int>(program.size());
}

match()

bool Scanner::match ( const char &  expected )

private

Definition at line 138 of file scanner.cpp.

                                        {
  if (isAtEnd() || program[current] != expected) {
    return false;
  }
 
  current++;
  return true;
}

number()

void Scanner::number ( )

private

Definition at line 109 of file scanner.cpp.

                     {
  while (isDigit(peek())) {
    advance();
  }
 
  int numberLiteral =
      std::stoi(std::string{program.substr(start, current - start)});
  addToken(NUMBER, numberLiteral);
}

peek()

char Scanner::peek ( ) const

private

Definition at line 147 of file scanner.cpp.

                         {
  if (isAtEnd()) {
    return '\0';
  }
  return program[current];
}

peekNext()

char Scanner::peekNext ( ) const

private

Definition at line 154 of file scanner.cpp.

                             {
  if (current + 1 >= static_cast<int>(program.size())) {
    return '\0';
  }
  return program[current + 1];
}

scanToken()

void Scanner::scanToken ( )

private

Definition at line 19 of file scanner.cpp.

                        {
  char c = advance();
  switch (c) {
  // One or two character tokens
  case '(':
    addToken(LEFT_PAREN);
    break;
  case ')':
    addToken(RIGHT_PAREN);
    break;
  case '{':
    addToken(LEFT_BRACE);
    break;
  case '}':
    addToken(RIGHT_BRACE);
    break;
  case '-':
    addToken(MINUS);
    break;
  case '+':
    addToken(PLUS);
    break;
  case ';':
    addToken(SEMICOLON);
    break;
  case '*':
    addToken(STAR);
    break;
  case '!':
    addToken(match('=') ? BANG_EQUAL : BANG);
    break;
  case '=':
    addToken(match('=') ? EQUAL_EQUAL : EQUAL);
    break;
  case '<':
    addToken(match('=') ? LESS_EQUAL : LESS);
    break;
  case '>':
    addToken(match('=') ? GREATER_EQUAL : GREATER);
    break;
 
  // Integer division vs. C-style comments
  case '/':
    if (match('/')) {
      while (peek() != '\n' && !isAtEnd())
        advance();
    } else {
      addToken(SLASH);
    }
    break;
 
  // Ignore white spaces
  case ' ':
  case '\r':
  case '\t':
    break;
 
  // New line
  case '\n':
    line++;
    break;
 
  // Identifiers
  case '"':
    string();
    break;
  default:
    if (isDigit(c)) {
      number();
    } else if (isAlpha(c)) {
      identifier();
    } else {
      // Report error but continue scanning
      error(line, "Unexpected character.");
    }
    break;
  };
}

scanTokens()

void Scanner::scanTokens

(

)

Scans the entire source code and generates a list of tokens.

Definition at line 171 of file scanner.cpp.

                         {
  while (!isAtEnd()) {
    start = current;
    scanToken();
  }
 
  tokens.emplace_back(END_OF_FILE, "", nullptr, line);
}

string()

void Scanner::string ( )

private

Definition at line 119 of file scanner.cpp.

                     {
  while (peek() != '"' && !isAtEnd()) {
    if (peek() == '\n')
      line++;
    advance();
  }
 
  if (isAtEnd()) {
    // Report error but finish scanning without crashing
    error(line, "Unterminated string.");
    return;
  }
 
  advance();
 
  std::string word{program.substr(start + 1, current - start - 2)};
  addToken(STRING, std::move(word));
}

Member Data Documentation

current

int Scanner::current = 0

private

Definition at line 28 of file scanner.hpp.

line

int Scanner::line = 1

private

Definition at line 29 of file scanner.hpp.

program

std::string_view Scanner::program

private

Definition at line 25 of file scanner.hpp.

start

int Scanner::start = 0

private

Definition at line 27 of file scanner.hpp.

tokens

std::vector<Token> Scanner::tokens

private

Definition at line 26 of file scanner.hpp.

---

The documentation for this class was generated from the following files:

• lib/gsc/scanner.hpp
• src/scanner.cpp