timmeh

import sre_parse, sre_compile, sysclass Scanner:    def __init__(self, lexicon, flags=0):        from sre_constants import BRANCH, SUBPATTERN        self.lexicon = lexicon        # combine phrases into a compound pattern        p = []        s = sre_parse.Pattern()        s.flags = flags        for phrase, action in lexicon:            p.append(sre_parse.SubPattern(s, [                (SUBPATTERN, (len(p)+1, sre_parse.parse(phrase, flags))),                ]))        p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])        s.groups = len(p)        self.scanner = sre_compile.compile(p)    def scan(self, string):        result = []        append = result.append        match = self.scanner.scanner(string).match        i = 0        while 1:            m = match()            if not m:                break            j = m.end()            if i == j:                break            action = self.lexicon[m.lastindex-1][1]            if callable(action):                self.match = m                action = action(self, m.group())            if action is not None:                append(action)            i = j        return result, string[i:]class token:	tokens = []	def __init__(self, scanner, value):		self.val = value		self.line = scanner.line		token.tokens.append(self)	def __str__(self):		return '<Token type:%s val:%s>' % (self.__class__, self.val)### identifier classclass t_ident(token):	def __init__(self, scanner, value):		token.__init__(self, scanner, value)### number classesclass t_num(token):	def __init__(self, scanner, value):		token.__init__(self, scanner, value)		class t_int(token):	def __init__(self, scanner, value):		token.__init__(self, scanner, value)		self.val = int(value)		class t_float(token):	def __init__(self, scanner, value):		token.__init__(self, scanner, value)		self.val = float(value)### operator classclass t_op(token):	def __init__(self, scanner, value):		token.__init__(self, scanner, value)### disposablesdef t_newline(scanner, value):	scanner.line += 1	return Nonedef t_whitespace(scanner, value):	return None	### other classesclass t_lbrac(token):	def __init__(self, scanner, value):		token.__init__(self, scanner, value)class t_rbrac(token):	def __init__(self, scanner, value):		token.__init__(self, scanner, value)ident = r'[a-zA-Z_][a-zA-Z0-9_]*'num = r'[0-9]+'integer_num = r'[\+-]?[0-9]+'float_num = r'[\+-]?[0-9]+\.[0-9]*'operator = '[\+\-*/\^<>=%]'newline = r'\n+'whitespace = r'[ \t]+'lbrac = r'\('rbrac = r'\)'patterns  = [	(ident, t_ident),	(float_num, t_float),	(integer_num, t_int),	(operator, t_op),	(newline, t_newline),	(whitespace, t_whitespace),	(lbrac, t_lbrac),	(rbrac, t_rbrac)	]class lexer:	def __init__(self, src, patterns):		self.scanner = Scanner(patterns)		self.scanner.line = 1		self.tokens = self.scanner.scan(src)[0]		self.src = src		self.lines = src.split('\n')		self.idx = 0		def next(self):		self.token = self.tokens[self.idx]		self.idx += 1		return self.token		test = lexer('print range(30)', patterns)print test.next()