tokens.py

# An object that will hold the type of current input with its respective value
class Token:
    # type_ -> a type of token (from the list in constants)
    # value -> the actual value
    # pos_start -> Position of start of token (optional)
    # pos_end -> Position of end of token (optional)
    def __init__(self, type_, value=None, pos_start=None, pos_end=None):
        self.type = type_
        self.value = value
        if pos_start:
            self.pos_start = pos_start.copy()

            # Position end would be one ahead of start
            self.pos_end = pos_start.copy()
            self.pos_end.advance()
        if pos_end:
            self.pos_end = pos_end

    def matches(self, type_, value):
        return self.type == type_ and self.value == value

    def __repr__(self):
        if self.value:
            return f'{self.type}: {self.value}'
        return f'{self.type}'