rendered paste body#!/usr/bin/env python# encoding: utf-8"""SafeUnicode.pyCreated by Cody Brocious on 2006-10-16.Copyright (c) 2006 MP3tunes, LLC. All rights reserved."""import sysfrom decomp import decomp_chars, decomp_inverse, canon_charscodecs = ['UTF-8', 'EUC-JP', 'SJIS', 'Windows-1252', 'ISO-2022-JP', 'ISO-8859-2', 'ISO-8859-3', 'ISO-8859-4', 'ISO-8859-5', 'ISO-8859-6', 'ISO-8859-7', 'ISO-8859-8', 'ISO-8859-9', 'ISO-8859-10', 'ISO-8859-13', 'ISO-8859-14', 'ISO-8859-15', 'CP936', 'HZ', 'BIG-5', 'EUC-KR', 'UHC', 'ISO-2022-KR', 'Windows-1251', 'CP866', 'KOI8-R', 'ISO-8859-1', 'ASCII']def bruteForceDecode(input, errors): for codec in codecs: try: ret = input.decode(codec, **errors) return ret, codec except: pass raise Exceptiondef bruteForceEncode(input, errors): for codec in codecs: try: ret = input.encode(codec, **errors) return ret, codec except: pass raise Exceptionclass SafeUnicodeException(Exception): passclass SafeUnicode(object): def __init__(self, object=None, encoding=None, errors=None): if errors == None: self.__dict__['errors'] = {} else: self.__dict__['errors'] = {'errors' : errors} if object: self.__dict__['unicode'], self.__dict__['encoding'] = self.toUnicode(object, encoding) else: self.__dict__['unicode'] = u'' self.__dict__['encoding'] = encoding def toUnicode(self, input, encoding): errors = self.errors try: if input.__class__ == str: if encoding: try: return input.decode(encoding, **errors), encoding except: try: return unicode(input, encoding=encoding, **errors), encoding except: pass try: return bruteForceDecode(input, errors) except: # ENCODE, YOU FOOLS return unicode(input, **errors) elif input.__class__ == unicode: if encoding: try: input = input.encode(encoding) return input.decode(encoding, **errors), encoding except: pass elif encoding == 'force': try: temp, codec = bruteForceEncode(input, errors) return unicode(temp, codec, **errors), codec except: pass return input, None elif input.__class__ == SafeUnicode: if encoding and encoding != 'force': try: temp = input.__dict__['unicode'].encode(encoding) return unicode(temp, encoding, **errors), encoding except: pass elif encoding == 'force': try: temp, codec = bruteForceEncode(input, errors) return temp, codec except: pass return input.__dict__['unicode'], input.__dict__['encoding'] else: if encoding: return unicode(input, encoding=encoding, **errors), encoding else: return unicode(input, **errors), None except: try: return unicode(str(input), **errors), None except: raise SafeUnicodeException, 'Given string has no possible unicode encoding' def makeFilesystemSafe(self): if 'win32' == sys.platform: return SafeUnicode(self, 'iso-8859-1') else: return SafeUnicode(self, 'UTF-8') def __getattr__(self, name): unicode = self.__dict__.get('unicode') return getattr(unicode, name) def __setattr__(self, name, value): unicode = self.__dict__.get('unicode') setattr(unicode, name, value) def __delattr__(self, name): unicode = self.__dict__.get('unicode') delattr(unicode, name) def __getitem__(self, key): unicode = self.__dict__.get('unicode') return SafeUnicode(unicode[key], self.encoding) def __setitem__(self, key, value): unicode = self.__dict__.get('unicode') unicode[key] = value def __delitem__(self, key): unicode = self.__dict__.get('unicode') del unicode[key] def __repr__(self): unicode = self.__dict__.get('unicode') return `unicode` def __str__(self): unicode = self.__dict__.get('unicode') try: return unicode.encode(self.encoding, **self.errors) except: ret, codec = bruteForceEncode(unicode, self.errors) return ret def __int__(self): return int(self.__dict__.get('unicode')) def __float__(self): return float(self.__dict__.get('unicode')) def __unicode__(self): return self.__dict__.get('unicode') def __add__(self, right): if right.__class__ != SafeUnicode: right = SafeUnicode(right) return SafeUnicode(unicode(self) + unicode(right), encoding=self.encoding) def __eq__(self, right): if right.__class__ != SafeUnicode and right.__class__ != unicode: try: right = SafeUnicode(right) except: return False return unicode(self) == unicode(right) def __cmp__(self, right): if right.__class__ != SafeUnicode and right.__class__ != unicode: try: right = SafeUnicode(right) except: return -1 return unicode(self).__cmp__(unicode(right)) def __hash__(self): try: return unicode(self).__hash__() except: return 0 def __mod__(self, right): formats = [] if right.__class__ == tuple: for format in right: if format.__class__ == str: format = SafeUnicode(format) elif format.__class__ == unicode: format = SafeUnicode(format) formats.append(format) else: formats.append(right) return unicode(self) % tuple(formats) def __len__(self): return self.unicode.__len__() def __contains__(self, left): return unicode(SafeUnicode(left)) in unicode(self) def replace(self, find, replace): find = unicode(SafeUnicode(find)) replace = unicode(SafeUnicode(replace)) try: return SafeUnicode(unicode(self).replace(find, replace), encoding=self.encoding) except: return SafeUnicode(str(self).replace(find, replace), encoding=self.encoding) def getEncoding(self): return self.encoding def decompose(self): decomp_table = decomp_chars changed = False ret = u'' for i in range(len(self)): key = unicode(self[i]) if key in decomp_table: ret += decomp_table[key][1] changed = True else: ret += key if changed: return SafeUnicode(ret).decompose() return self def compose(self): self_len = len(self) i = 0 out = u'' while i < self_len: starter = self[i] if starter in canon_chars and canon_chars[starter] != 0: i += 1 continue accents = [] i += 1 while i < self_len: accent = unicode(self[i]) if accent not in canon_chars or canon_chars[accent] == 0: break accents.append(accent) i += 1 if len(accents) == 0: out += starter continue out += self.composeCharacter(starter, accents) return SafeUnicode(out) def composeCharacter(self, starter, accents): accents_joined = u''.join(accents) combined = unicode(starter) + accents_joined char_len = len(combined) if combined in decomp_inverse: return decomp_inverse[combined][1] possibilities = [char for char in decomp_inverse if char[0] == starter and len(char) <= char_len] matches = [] for possible in possibilities: match = True for char in possible[1:]: if char not in accents: match = False if match: matches.append(possible) out = None if len(matches) == 0: return combined elif len(matches) == 1: out = matches[0] if out == None: low_comb = 0xFFFFFF for match in matches: if match[1] not in canon_chars: continue comb_class = canon_chars[match[1]] if comb_class < low_comb: out = match low_comb = comb_class if out == None: long_match = None long_match_len = 0 for match in matches: if len(match) >= long_match_len: long_match = match long_match_len = len(long_match) out = long_match remaining = [accent for accent in accents if accent not in out] out = decomp_inverse[out][1] out += u''.join(remaining) return out def reorder(self): reorder = [char for char in unicode(self)] changed = False for i in range(len(reorder) - 1): first = reorder[i] second = reorder[i+1] if not first in canon_chars or \ not second in canon_chars: continue first_canon = canon_chars[first] second_canon = canon_chars[second] if second_canon == 0 or first_canon == 0: continue if first_canon > second_canon: changed = True reorder[i+1] = first reorder[i] = second if not changed: return self reorder = SafeUnicode(u''.join(reorder)) return reorder.reorder() def normalize(self, type): if type == 'NFC': return self.normalizeNFC() elif type == 'NFD': return self.normalizeNFD() elif type == 'NFKC': return self.normalizeNFKC() elif type == 'NFKD': return self.normalizeNFKD() def normalizeNFC(self): return self.decompose().reorder().compose() def normalizeNFD(self): return self.decompose().reorder() def normalizeNFKC(self): return self def normalizeNFKD(self): return self