#!/usr/bin/env pythonfrom BeautifulSoup import BeautifulSoup class OrkutParser: def __init__(self, html): self.html = html self.soup = None self.parse_html() # list of ["profile_name", "profile_link", "profile_image", "scrap"] self.scraps = [] def get_account_name(self): # Get "Mayank" from the title - "orkut - Mayank's scrapbook" name = self.soup.html.title.string.strip() return name.split(" ")[2].split("'")[0] def parse_html(self, html=None): if html == None: html = self.html # 11th table holds the scraps start_outer = 11 table_obj = None tr_list = None scraps = [] # setup the parser self.soup = BeautifulSoup(html) # Starting parsing # step-1 get the table containing all the scraps - 11th table table_obj = self.soup("table")[start_outer] # step-2 get the table rows containing scrapper's profile link, image tr_list = self.parse_get_tr_list(table_obj) # step-3 get the relevant table data containing scrapper's name, scrap td_list = self.parse_get_td_list(tr_list) # step-4 get the relevant data from the table rows scraps = self.parse_get_scraps_data(td_list) # step-5 sanitize scraps list to be used directly self.scraps = self.sanitize_scraps(scraps) for scrap in self.scraps: print "Scrap from => " + scrap[0] print "Scrapper's link => " + scrap[1] print "Scrapper's image => " + scrap[2] print "Scrap => " + scrap[3] print "Total scraps = %d" % (len(self.scraps),) def parse_get_tr_list(self, table_obj): # scraps start from the 7th table row start_inner= 7 # number of scraps per page = 10 num_scraps = 10 tr_list = [] # each interesting table row is 4 rows apart for counter in range(start_inner, start_inner + 4*num_scraps, 4): tr_list.append(table_obj.contents[counter]) return tr_list def parse_get_td_list(self, tr_list): td_list = [] for tr in tr_list: if len(tr) >= 6: tmp_list = [] # 1st and 5th table coloumns are important for us # this will give us the link & picture of scrapper tmp_list.append(tr.contents[1]) # this will give us the name & scrap of scrapper tmp_list.append(tr.contents[5]) td_list.append(tmp_list) # return the list of lists return td_list def parse_get_scraps_data(self, td_list): # list of ["profile_name", "profile_link", "profile_image", "scrap"] scraps = [] profile_name = None profile_link = None image_link = None scrap = None for data in td_list: tmp_list = [] # get the profile link profile_link = data[0].contents[1]["href"] # get the profile image profile_image = data[0].contents[1].contents[0]["src"] # get the profile name profile_name = data[1].contents[1].contents[1].string # get the scrap scrap = "" # scraps will start from data[1].contents[1].contents[2] till the length of data[1].contents[1] # save type of data[1].contents[1].contents[1] so as to check for inline tags in scraps junk_obj = data[1].contents[1].contents[1] for i in range(2, len(data[1].contents[1])): if type(data[1].contents[1].contents[i]) == type(junk_obj): # if scrap contains an inline tag, get its string representation scrap += data[1].contents[1].contents[i].__repr__() else: # add as a textual scrap scrap += "%s" % (data[1].contents[1].contents[i].strip(),) # before adding to the scraps list, strip the newlines if any! tmp_list.append(profile_name.strip()) tmp_list.append(profile_link.strip()) tmp_list.append(profile_image.strip()) # remove the leading ":\n" from the scrap's text tmp_list.append(scrap[1:].strip()) scraps.append(tmp_list) return scraps def sanitize_scraps(self, scraps): # perforam any sanitization stuff here # for now, just returning a full profile address # and stripping scraps of ":" & stripping again sanitized_scraps = [] for scrap in scraps: tmp_list = [] tmp_list.append(scrap[0]) tmp_list.append("http://www.orkut.com" + scrap[1]) tmp_list.append(scrap[2]) tmp_list.append(scrap[3]) sanitized_scraps.append(tmp_list) return sanitized_scrapsif __name__ == "__main__": fr = open("Scrapbook.html", "r") html = fr.read() fr.close() parser = OrkutParser(html)