Miscellany

# TODO : need to removed javascript as wellimport re#p2 = re.compile('</?br>|</?p>' , re.I)p = re.compile('</?br>|</?p>' , re.I)def strip_html(text):    #text = text.decode('iso-8859-1')    # text = p2.sub('\n', text)    text = re.sub('[\n\r]+', ' ', text)    text = p.sub('\n', text)    def fixup(m):        text = m.group(0)        if text[:1] == "<":            mm = p.match(text)            if mm:                res = mm.group(1)                if res[:1] == "&":                    return handle_entity(res)                return ""            return "" # ignore tags        if text[:2] == "&#":            try:                if text[:3] == "&#x":                    return unichr(int(text[3:-1], 16))                else:                    return unichr(int(text[2:-1]))            except ValueError:                pass        elif text[:1] == "&":            return handle_entity(text)        return text # leave as is    text = re.sub("(?s)<[^>]*>|&#?\w+;", fixup, text)    text = re.sub('[ \t]+', ' ', text)    return text    import htmlentitydefsdef handle_entity(text):    entity = htmlentitydefs.entitydefs.get(text[1:-1])    if entity:        if entity[:2] == "&#":            try:                u = unichr(int(entity[2:-1]))                return u            except ValueError, e:                print e                pass        else:            return entity            #return unicode(entity, "iso-8859-1")