timmeh

###################################################################	File: lexer.py# 	Author: Timothy Wakeham (timmeh)#	Date: 21/02/09##	Todo:#	-Create token descriptor class#	-Fix token descriptors#	-Token and token descriptors should subclass Collection#	-Do callback functions#	-Fix error handling#	-Error recovery##################################################################import reempty = ''digits = '0123456789'lchars = 'abcdefghijklmnopqrstuvwxyz'uchars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'chars = lchars + ucharsunderscore = '_'operators = '+-/*<>=!'class LexicalError(Exception):	def __init__(self, msg, lexer=None):		self.msg = msg		self.lexer = lexer	def __str__(self):		return self.msg		class Token:	def __init__(self, **kwargs):		for key, val in kwargs.iteritems():			setattr(self, key, val)		self.kwargs = kwargs.copy()	def __str__(self):		buffer = '-----------------------------\n'		for key, val in self.kwargs.iteritems():			buffer += '%s:%s\n' % (key, val)		#buffer += '-----------------------------'		return buffer[:-1]class Lexer:	def __init__(self, input_stream):		self.input_stream = input_stream		self.token_descriptors = {}			def register(self, type, initial, final=empty, callback=None):		self.token_descriptors[type] = [initial, final, callback]			def tokenise(self):		char_pos = 0		stream_len = len(self.input_stream)		token_list = []		token_pos = 1		while(char_pos <= stream_len):			lexical_error = 0			for token_type, token_descriptor in self.token_descriptors.iteritems():				lexical_error += 1				if char_pos >= stream_len: return token_list				if self.input_stream[char_pos] in token_descriptor[0]:					token_start = char_pos					while(self.input_stream[char_pos + token_pos] in token_descriptor[1]):						token_pos += 1						if char_pos + token_pos >= stream_len: break					token_end = char_pos + token_pos					token_value = self.input_stream[token_start:token_end]					token_list.append(Token(type=token_type, value=token_value, start=token_start, end=token_end, length=token_end-token_start))					char_pos += token_pos					token_pos = 1					break				print self.input_stream[char_pos]				#raise LexicalError('Unrecognised character - %s' % self.input_stream[char_pos])	lexer = Lexer('12+4-abc/80345')lexer.register('NUM', digits, digits)lexer.register('IDENT', chars, chars + digits + underscore)lexer.register('OPERATOR', operators)lexer.register('LBRACE', '(', empty)lexer.register('RBRACE', ')', empty)for token in lexer.tokenise():	print token