#!/usr/bin/env python
# -*- coding: utf-8 -*-
#error, debug and warning
from functions import *
#The lexer/tokeniser object
class Lexer(object):
u"""
The lexer/tokeniser class. use the tokenise(string) method to return a list of tokens
Theoretically bug-free, it's hard to produce a hard to find bug in so few lines of code and it's been extensively tested
"""
def __init__(self):
u"""
Just sets up a couple of constants
"""
self.__forbidden_in_identifiers = ['"', "'", "[", "]", "(", ")", "{", "}"]
def tokenise(self, string):
u"""
Returns a list of all tokens in a string.
Token format is [a, b] where:
a is a character representing the token type
b (when present) is the token itself
"""
ret = []
tokens = []
ret = self.__next_token(string)
while ret != []:
tokens.append(ret[1:])
string = ret[0]
ret = self.__next_token(string)
return tokens
def __next_token(self, string):
u"""
Returns the next token in the string and
the string with the token trimmed off the front
"""
if len(string) == 0: return []
#Skip whitespace
while ord(string[0]) < 33:
string = string[1:]
ch = string[0]
#What is it?
if ch == "(":
return [string[1:], "("]
elif ch == ")":
return [string[1:], ")"]
elif ch == "[":
return [string[1:], "["]
elif ch == "]":
return [string[1:], "]"]
elif ch == "{":
return [string[1:], "{"]
elif ch == "}":
return [string[1:], "}"]
elif ch == '"':
[collectedstring, remnants] = self.__collect_string(string[1:], '"')
return [remnants, '"', collectedstring]
elif ch == "'":
[collectedstring, remnants] = self.__collect_string(string[1:], "'")
return [remnants, "'", collectedstring]
elif ch == ":":
return [string[1:], "'"]
else:
[collectedidentifier, remnants] = self.__collect_identifier(string)
return [remnants, "i", collectedidentifier]
def __collect_string(self, string, quote):
u"""
Returns the contents of the string we started collecting
Also returns the tail
"""
escape = False
chars = []
count = 1
for ch in string:
count += 1
if escape:
if ch == quote:
chars.append(quote)
else:
warning("Unrecognised string escape sequence: \\%s" % (ch))
escape = False
elif ch == quote:
return ["".join(chars), string[count:]]
elif ch == '\\':
escape = True
else:
chars.append(ch)
#Crap, we ran out of string
error("Ran out of string to collect, incomplete statement")
def __collect_identifier(self, string):
u"""
Returns the rest of an identifier name and the trailing string
Identifiers can be anything but:
-whitespace
-brackets (of any sort)
-quote symbols (all sorts)
"""
chars = []
count = 0
for ch in string:
count += 1
if self.__forbidden_in_identifiers.count(ch) > 0:
return ["".join(chars), string[count-1:]]
elif ord(ch) > 32:
chars.append(ch)
else:
return ["".join(chars), string[count:]]
return ["".join(chars), string[count:]]