All pastes #163568 Raw Edit

Mine

public python v1 · immutable
·published 2006-09-07 17:34 UTC
rendered paste body
#!/usr/bin/env python# -*- coding: utf-8 -*-reserved_words = ["p", "i", "?", "!", "#"]forbidden_in_identifiers = ['"', "'", "[", "]", "(", ")"]def error(string):	print string	raise SystemExitdef warning(string):	print stringdef debug(*args):	print "Debug message: %s" % ("".join(args))def tokenise(string):	tokens = []	ret = []	ret = next_token(string)	while ret != []:		tokens.append(ret[1:])		string = ret[0]		ret = next_token(string)	return tokensdef next_token(string):	if len(string) == 0: return []	#Skip whitespace	while ord(string[0]) < 33:		string = string[1:]	ch = string[0]	#What is it?	if ch == "(":		return [string[1:], "("]	elif ch == ")":		return [string[1:], ")"]	elif ch == "[":		return [string[1:], "["]	elif ch == "]":		return [string[1:], "]"]	elif ch == '"':		[collectedstring, remnants] = collect_string(string[1:])		return [remnants, '"', collectedstring]	elif ch == "'":		return [string[1:], "'"]	else:		[collectedidentifier, remnants] = collect_identifier(string) 		return [remnants, "i", collectedidentifier]		def collect_string(string):	escape = False	chars = []	count = 1	for ch in string:		count += 1		if escape:			if ch == '"':				chars.append('"')			else:				warning("Unrecognised string constant: \\%s" % (ch))			escape = False		elif ch == '"':			return ["".join(chars), string[count:]]		elif ch == '\\':			escape = True		else:			chars.append(ch)	#Crap, we ran out of string	error("Ran out of string to collect, incomplete statement")		def collect_identifier(string):	chars = []	count = 0	for ch in string:		count += 1		if forbidden_in_identifiers.count(ch) > 0:			return ["".join(chars), string[count-1:]]		elif ord(ch) > 32:			chars.append(ch)		else:			return ["".join(chars), string[count:]]	return ["".join(chars), string[count:]]if __name__ == "__main__":	print tokenise('(dfd "sdf" £566 s#C \'sdgfd [sf rey])')