def strip_ml_tags(in_text):
"""Description: Removes all HTML/XML-like tags from the input text."""
# convert in_text to a mutable object (e.g. list)
s_list = list(in_text)
i,j = 0,0
while i < len(s_list):
# iterate until a left-angle bracket is found
if s_list[i] == '<':
while s_list[i] != '>':
# pop everything from the the left-angle bracket until the right-angle bracket
s_list.pop(i)
# pops the right-angle bracket, too
s_list.pop(i)
else:
i=i+1
# convert the list back into text
join_char=u''
return join_char.join(s_list)
def addContent(country, topic, content):
""" Get the data formated into a dict and insert it into SQL """
# Remove the html tags
retext = strip_ml_tags(content)
# Remove the &; entities
retext = re.sub("&.*?;", "", retext)
# Remove newlines and spaces
retext = re.sub('((\n)+|(\t)+)', "", retext)
# Get our records with a regular expression
"""More efficent REGEXP which needs converting into python
AFAIK these are experimental PCRE lookaheads => '<?>'
(Title:(?>(?P<title>.*?)URL:)(?>(?P<url>.*?)Description:)(?>(?P<description>.*?)(?=Title:|$)) """
fields = re.finditer(r"(Title:(?P<title>.*?)URL:(?P<url>.*?)Description:(?P<description>.*?)(?=Title:|$))", retext)
# Helper variables for the loop
failure = False
count = 0
# Group each record into dicts and push to SQL
for field in fields:
# FIXME: if you know a better way to count the occurences of each
# record in our fields iterator, please change this
count += 1
# Each record into SQL
data = field.groupdict()
# Encode into UTF-8 for the utf table
for string in data:
string = string.encode('utf-8')
try:
cursor.execute("INSERT INTO python_links(country, topic, title, url, description) VALUES(%s, %s, %s, %s, %s);", (country, topic, data['title'].encode("utf-8"), data['url'], data['description'].encode("utf-8")))
#print query
except:
failure = True
traceback.print_exc()
if not failure:
print "Done (%d records)" % (count