rendered paste body#!/usr/bin/env python# -*- coding: utf-8 -*-reserved_words = ["p", "i", "?", "!", "#"]forbidden_in_identifiers = ['"', "'", "[", "]", "(", ")"]def error(string): print string raise SystemExitdef warning(string): print stringdef debug(*args): print "Debug message: %s" % ("".join(args))def tokenise(string): tokens = [] ret = [] ret = next_token(string) while ret != []: tokens.append(ret[1:]) string = ret[0] ret = next_token(string) return tokensdef next_token(string): if len(string) == 0: return [] #Skip whitespace while ord(string[0]) < 33: string = string[1:] ch = string[0] #What is it? if ch == "(": return [string[1:], "("] elif ch == ")": return [string[1:], ")"] elif ch == "[": return [string[1:], "["] elif ch == "]": return [string[1:], "]"] elif ch == '"': [collectedstring, remnants] = collect_string(string[1:]) return [remnants, '"', collectedstring] elif ch == "'": return [string[1:], "'"] else: [collectedidentifier, remnants] = collect_identifier(string) return [remnants, "i", collectedidentifier] def collect_string(string): escape = False chars = [] count = 1 for ch in string: count += 1 if escape: if ch == '"': chars.append('"') else: warning("Unrecognised string constant: \\%s" % (ch)) escape = False elif ch == '"': return ["".join(chars), string[count:]] elif ch == '\\': escape = True else: chars.append(ch) #Crap, we ran out of string error("Ran out of string to collect, incomplete statement") def collect_identifier(string): chars = [] count = 0 for ch in string: count += 1 if forbidden_in_identifiers.count(ch) > 0: return ["".join(chars), string[count-1:]] elif ord(ch) > 32: chars.append(ch) else: return ["".join(chars), string[count:]] return ["".join(chars), string[count:]]if __name__ == "__main__": print tokenise('(dfd "sdf" £566 s#C \'sdgfd [sf rey])')