def strip_ml_tags(in_text): """Description: Removes all HTML/XML-like tags from the input text.""" # convert in_text to a mutable object (e.g. list) s_list = list(in_text) i,j = 0,0 while i < len(s_list): # iterate until a left-angle bracket is found if s_list[i] == '<': while s_list[i] != '>': # pop everything from the the left-angle bracket until the right-angle bracket s_list.pop(i) # pops the right-angle bracket, too s_list.pop(i) else: i=i+1 # convert the list back into text join_char=u'' return join_char.join(s_list)def addContent(country, topic, content): """ Get the data formated into a dict and insert it into SQL """ # Remove the html tags retext = strip_ml_tags(content) # Remove the &; entities retext = re.sub("&.*?;", "", retext) # Remove newlines and spaces retext = re.sub('((\n)+|(\t)+)', "", retext) # Get our records with a regular expression """More efficent REGEXP which needs converting into python AFAIK these are experimental PCRE lookaheads => '<?>' (Title:(?>(?P<title>.*?)URL:)(?>(?P<url>.*?)Description:)(?>(?P<description>.*?)(?=Title:|$)) """ fields = re.finditer(r"(Title:(?P<title>.*?)URL:(?P<url>.*?)Description:(?P<description>.*?)(?=Title:|$))", retext) # Helper variables for the loop failure = False count = 0 # Group each record into dicts and push to SQL for field in fields: # FIXME: if you know a better way to count the occurences of each # record in our fields iterator, please change this count += 1 # Each record into SQL data = field.groupdict() # Encode into UTF-8 for the utf table for string in data: string = string.encode('utf-8') try: cursor.execute("INSERT INTO python_links(country, topic, title, url, description) VALUES(%s, %s, %s, %s, %s);", (country, topic, data['title'].encode("utf-8"), data['url'], data['description'].encode("utf-8"))) #print query except: failure = True traceback.print_exc() if not failure: print "Done (%d records)" % (count