rendered paste body#!/usr/bin/env python# encoding: utf-8"""SafeUnicode.pyCreated by Cody Brocious on 2006-10-16.Copyright (c) 2006 MP3tunes, LLC. All rights reserved."""import sysfrom decomp import decomp_chars, decomp_inverse, canon_charscodecs = ['UTF-8', 'EUC-JP', 'SJIS', 'Windows-1252', 'ISO-2022-JP', 'ISO-8859-2', 'ISO-8859-3', 'ISO-8859-4', 'ISO-8859-5', 'ISO-8859-6', 'ISO-8859-7', 'ISO-8859-8', 'ISO-8859-9', 'ISO-8859-10', 'ISO-8859-13', 'ISO-8859-14', 'ISO-8859-15', 'CP936', 'HZ', 'BIG-5', 'EUC-KR', 'UHC', 'ISO-2022-KR', 'Windows-1251', 'CP866', 'KOI8-R', 'ISO-8859-1', 'ASCII']def bruteForceDecode(input, errors): for codec in codecs: try: ret = input.decode(codec, **errors) return ret, codec except: pass raise Exceptiondef bruteForceEncode(input, errors): for codec in codecs: try: ret = input.encode(codec, **errors) return ret, codec except: pass raise Exceptionclass SafeUnicodeException(Exception): passclass SafeUnicode(object): def __init__(self, object=None, encoding=None, errors=None): if errors == None: self.__dict__['errors'] = {} else: self.__dict__['errors'] = {'errors' : errors} if object: self.__dict__['unicode'], self.__dict__['encoding'] = self.toUnicode(object, encoding) else: self.__dict__['unicode'] = u'' self.__dict__['encoding'] = encoding def toUnicode(self, input, encoding): errors = self.errors try: if input.__class__ == str: if encoding: try: return input.decode(encoding, **errors), encoding except: try: return unicode(input, encoding=encoding, **errors), encoding except: pass try: return bruteForceDecode(input, errors) except: # ENCODE, YOU FOOLS return unicode(input, **errors) elif input.__class__ == unicode: if encoding: try: input = input.encode(encoding) return input.decode(encoding, **errors), encoding except: pass elif encoding == 'force': try: temp, codec = bruteForceEncode(input, errors) return unicode(temp, codec, **errors), codec except: pass return input, None elif input.__class__ == SafeUnicode: if encoding and encoding != 'force': try: temp = input.__dict__['unicode'].encode(encoding) return unicode(temp, encoding, **errors), encoding except: pass elif encoding == 'force': try: temp, codec = bruteForceEncode(input, errors) return temp, codec except: pass return input.__dict__['unicode'], input.__dict__['encoding'] else: if encoding: return unicode(input, encoding=encoding, **errors), encoding else: return unicode(input, **errors), None except: try: return unicode(str(input), **errors), None except: raise SafeUnicodeException, 'Given string has no possible unicode encoding' def makeFilesystemSafe(self): if 'win32' == sys.platform: return SafeUnicode(self, 'iso-8859-1') else: return SafeUnicode(self, 'UTF-8') def __getattr__(self, name): unicode = self.__dict__.get('unicode') return getattr(unicode, name) def __setattr__(self, name, value): unicode = self.__dict__.get('unicode') setattr(unicode, name, value) def __delattr__(self, name): unicode = self.__dict__.get('unicode') delattr(unicode, name) def __getitem__(self, key): unicode = self.__dict__.get('unicode') return SafeUnicode(unicode[key], self.encoding) def __setitem__(self, key, value): unicode = self.__dict__.get('unicode') unicode[key] = value def __delitem__(self, key): unicode = self.__dict__.get('unicode') del unicode[key] def __repr__(self): unicode = self.__dict__.get('unicode') return `unicode` def __str__(self): unicode = self.__dict__.get('unicode') try: return unicode.encode(self.encoding, **self.errors) except: ret, codec = bruteForceEncode(unicode, self.errors) return ret def __int__(self): return int(self.__dict__.get('unicode')) def __float__(self): return float(self.__dict__.get('unicode')) def __unicode__(self): return self.__dict__.get('unicode') def __add__(self, right): if right.__class__ != SafeUnicode: right = SafeUnicode(right) return SafeUnicode(unicode(self) + unicode(right), encoding=self.encoding) def __eq__(self, right): if right.__class__ != SafeUnicode and right.__class__ != unicode: try: right = SafeUnicode(right) except: return False return unicode(self) == unicode(right) def __cmp__(self, right): if right.__class__ != SafeUnicode and right.__class__ != unicode: try: right = SafeUnicode(right) except: return -1 return unicode(self).__cmp__(unicode(right)) def __hash__(self): try: return unicode(self).__hash__() except: return 0 def __mod__(self, right): formats = [] if right.__class__ == tuple: for format in right: if format.__class__ == str: format = SafeUnicode(format) elif format.__class__ == unicode: format = SafeUnicode(format) formats.append(format) else: formats.append(right) return unicode(self) % tuple(formats) def __len__(self): return self.unicode.__len__() def __contains__(self, left): return unicode(SafeUnicode(left)) in unicode(self) def replace(self, find, replace): find = unicode(SafeUnicode(find)) replace = unicode(SafeUnicode(replace)) try: return SafeUnicode(unicode(self).replace(find, replace), encoding=self.encoding) except: return SafeUnicode(str(self).replace(find, replace), encoding=self.encoding) def getEncoding(self): return self.encoding def decompose(self): decomp_table = decomp_chars changed = False ret = u'' for i in range(len(self)): key = unicode(self[i]) if key in decomp_table: ret += decomp_table[key][1] changed = True else: ret += key if changed: return SafeUnicode(ret).decompose() return self def compose(self): comp_table = decomp_inverse ret = u'' slen = len(self) i = 0 while i < slen: found = None found_len = 0 found_char_len = 0 for char in comp_table: char_len = len(char) if self[i:i+char_len] == char: if found_len < char_len: found = comp_table[char][1] found_len = len(found) found_char_len = char_len if found == None: ret += unicode(self[i]) i += 1 else: ret += found i += found_char_len ret = SafeUnicode(ret) return ret def reorder(self): reorder = [char for char in unicode(self)] for i in range(len(reorder) - 1): first = reorder[i] second = reorder[i+1] if not first in canon_chars or \ not second in canon_chars: continue first_canon = canon_chars[first] second_canon = canon_chars[second] if second_canon == 0 or first_canon == 0: continue print `first`, `second`, canon_chars[first], canon_chars[second] if first_canon > second_canon: reorder[i+1] = first reorder[i] = second reorder = SafeUnicode(u''.join(reorder)) if reorder == self: return reorder return reorder.reorder() def normalize(self, type): if type == 'NFC': return self.normalizeNFC() elif type == 'NFD': return self.normalizeNFD() elif type == 'NFKC': return self.normalizeNFKC() elif type == 'NFKD': return self.normalizeNFKD() def normalizeNFC(self): return self.decompose().reorder().compose() def normalizeNFD(self): return self.decompose().reorder() def normalizeNFKC(self): return self def normalizeNFKD(self): return self