rendered paste body#!/usr/bin/env pythonimport feedparserimport tarfileimport urllibimport timeimport osimport sysimport termiosimport fcntlimport structMATCHLIMIT = 50CLEANDAYS = 21DB = os.environ['HOME'] + "/.extemp/"PAPERS = ('aljazeera', 'allafrica', 'bbc', 'bostonglobe', 'briefs', 'chicagotribune', 'cnn', 'csmonitor', 'dallasmorningnews', 'economist', 'financialtimes', 'foxnews', 'frenchpressagency', 'haaretz', 'houstonchronicle', 'iht', 'investorsbusiness', 'japantimes', 'jerusalempost', 'koreatimes', 'latimes', 'londontimes', 'middleeasttimes', 'moscowtimes', 'msnbc', 'newyorktimes', 'pravda', 'seattlepi', 'sfchronicle', 'spiegel', 'sydneymorningherald', 'taipeitimes', 'tehrantimes', 'washingtonpost', 'washingtontimes')class GetTtyAttribute(): def __init__(self): self.s = struct.pack("HHHH", 0, 0, 0, 0) self.fd_stdout = sys.stdout.fileno() self.x = fcntl.ioctl(self.fd_stdout, termios.TIOCGWINSZ, self.s) self.infotuple = struct.unpack("HHHH", self.x) self.height = self.infotuple[0] self.width = self.infotuple[1]class GetStatusInfo(): def __init__(self, paper, paperlength): self.paperdir = DB + "articles/" + paper + "/" self.filesize = 0 for root, dirs, files in os.walk(self.paperdir): self.filenum = len(files) self.spaces = paperlength - len(paper) + 1 self.output = paper + ":" + "%*s" % (self.spaces, "") + "%5s" % str(self.filenum) + " articles" class GetCleanInfo(): def __init__(self, paper, paperlength): self.paperdir = DB + "articles/" + paper + "/" self.currentdate = int(time.time()) self.cleandate = self.currentdate - (CLEANDAYS * 24 * 60 * 60) self.filesdeleted = 0 for root, dirs, files in os.walk(self.paperdir): for article in files: self.filedate = os.stat(self.paperdir + article).st_mtime if self.filedate < self.cleandate: os.remove(self.paperdir + article) self.filesdeleted += 1 self.spaces = paperlength - len(paper) + 1 self.output = paper + ":" + "%*s" % (self.spaces, "") + "%5s" % str(self.filesdeleted) + " deleted"class GetSearchInfo(): def __init__(self, paper): self.matches = [] firstpass = 0 for arg in sys.argv[3:]: searchterm = arg.lower() if firstpass == 0: arglist = "`" + arg + "'" for root, dirs, files in os.walk(paper): for article in files: temp = open(root + "/" + article, "r") temp2 = temp.read() temp3 = temp2.lower() if searchterm in temp3: self.matches.append(root + "/" + article) maxnumlen = len(str(len(self.matches))) firstpass = 1 else: arglist = arglist + " AND " + "`" + arg + "'" oldmatches = self.matches self.matches = [] for article in oldmatches: temp = open(article) temp2 = temp.read() temp3 = temp2.lower() if searchterm in temp3: self.matches.append(article) self.matchnum = len(self.matches) print("[Searching] " + "%*s" % (maxnumlen, len(self.matches)) + " matches for search terms: " + arglist)def downloadbar(completed, blocksize, totalsize): currenttime = float(time.time()) sizeinmb = float(totalsize) / float(1000000) totalblocks = float(totalsize) / float(blocksize) percdone = (float(completed) * float(100)) / float(totalblocks) sys.stdout.write("\r" + "[Downloading] " + "%.1f" % percdone + "% of " + "%.1f" % sizeinmb + " MB ") sys.stdout.flush() #####################################def main(): if os.getuid() == 0: sys.exit('You cannot run as root.') if not os.access(DB + "articles", os.F_OK): os.makedirs(DB + "articles") if len(sys.argv[1:]) == 0 or sys.argv[1] == 'help': help() elif sys.argv[1] == 'clean': clean() elif sys.argv[1] == 'install': install() elif sys.argv[1] == 'search': search() elif sys.argv[1] == 'status': status() elif sys.argv[1] == 'sync': sync() elif sys.argv[1] == 'update': update() elif sys.argv[1] == 'load': print("The option `load' is no longer used") else: sys.exit('Bad option `' + sys.argv[1] + '\', see extemp help')def help(): print('Usage: extemp [option] [paper] "[search phrase]"') print print('Options:') print print(' clean - delete old downloaded articles') print(' install - install latest software update') print(' search - search database for `search phrase\'') print(' status - print the status of the database') print(' sync - download the latest database snapshot') print(' update - download new articles into the database') print print('Papers:') print print('all - applies option to all of the papers below') print tty = GetTtyAttribute() longest = 0 for paper in PAPERS[0:]: if len(paper) > longest: longest = len(paper) wordsperline = tty.width / (longest + 1) if wordsperline < 1: sys.exit('The terminal is too narrow to display papers') wordnumber = 1 lastpaper = PAPERS[-1] for paper in PAPERS[0:]: spaces = longest - len(paper) + 1 if wordnumber == 1: linestring = paper + "%*s" % (spaces, "") wordnumber += 1 elif wordnumber < wordsperline: linestring = linestring + paper + "%*s" % (spaces, "") wordnumber += 1 elif wordnumber == wordsperline: linestring = linestring + paper + "%*s" % (spaces, "") print(linestring) wordnumber = 1 if paper == lastpaper and wordnumber != 1: print(linestring)def clean(): if len(sys.argv[2:]) == 0: sys.exit('No paper specified for `clean\'') elif sys.argv[2] in os.listdir(DB + "articles"): cleaninfo = GetCleanInfo(sys.argv[2], len(sys.argv[2])) print(cleaninfo.output) elif sys.argv[2] == "all": dbpapers = os.listdir(DB + "articles") dbpapers.sort() paperlength = 0 for paper in os.listdir(DB + "articles"): if len(paper) > paperlength: paperlength = len(paper) totaldelfiles = 0 for paper in dbpapers: cleaninfo = GetCleanInfo(paper, paperlength) print(cleaninfo.output) totaldelfiles += cleaninfo.filesdeleted print spaces = paperlength - len("TOTAL") + 1 print("TOTAL" + ":" + "%*s" % (spaces, "") + "%5s" % str(totaldelfiles) + " deleted") else: sys.exit('Paper `' + sys.argv[2] + "'is not a valid paper")def install(): print('This is the install function')def search(): if len(sys.argv[2:]) == 0: sys.exit('No paper specified for `search\'') elif len(sys.argv[3:]) == 0: sys.exit('No search terms were specified for `search\'') elif sys.argv[2] == "all": searchinfo = GetSearchInfo(DB + "articles/") elif sys.argv[2] in os.listdir(DB + "articles"): searchinfo = GetSearchInfo(DB + "articles/" + sys.argv[2]) else: sys.exit("Paper `" + sys.argv[2] + "' is not in the database") print("\n" + str(searchinfo.matchnum) + " matches, what do you want to do?\n") if searchinfo.matchnum > MATCHLIMIT: print("1. Open all " + str(searchinfo.matchnum) + " of them.") print("2. Open " + str(MATCHLIMIT) + " of them.") print("3. Open none of them.") print while True: answer = raw_input("Option Number: ") if answer == "1": action = "all" break elif answer == "2": action = "limit" break elif answer == "3": sys.exit() else: print("1. Open them.") print("2. Don't open them.") print while True: answer = raw_input("Option Number: ") if answer == "1": action = "all" break elif answer == "2": sys.exit() articlelist = "" if action == "all": for article in searchinfo.matches: articlelist = articlelist + " " + article elif action == "limit": jump = searchinfo.matchnum / MATCHLIMIT iter = 0 for num in range(0, searchinfo.matchnum, jump): articlelist = articlelist + " " + searchinfo.matches[num] iter += 1 if iter == MATCHLIMIT: break os.system("firefox " + articlelist + " &") def status(): if len(sys.argv[2:]) == 0: sys.exit('No paper specified for `status\'') elif sys.argv[2] in os.listdir(DB + "articles"): statusinfo = GetStatusInfo(sys.argv[2], len(sys.argv[2])) print(statusinfo.output) elif sys.argv[2] == "all": dbpapers = os.listdir(DB + "articles") dbpapers.sort() paperlength = 0 for paper in dbpapers: if len(paper) > paperlength: paperlength = len(paper) totalfiles = 0 for paper in dbpapers: statusinfo = GetStatusInfo(paper, paperlength) print(statusinfo.output) totalfiles += statusinfo.filenum print spaces = paperlength - len("TOTAL") + 1 print("TOTAL" + ":" + "%*s" % (spaces, "") + "%5s" % str(totalfiles) + " " + "articles") else: sys.exit('Paper `' + sys.argv[2] + "'is not a valid paper")def sync(): if len(sys.argv[2:]) == 0: sys.exit('No paper specified for `sync\'') elif sys.argv[2] == "all": try: print("[Searching] Looking for available snapshots ") ftppage = urllib.urlopen(FTPDIR + 'articles/').readlines() except: sys.exit("Failed to download list of available snapshots") for line in ftppage: splitline = line.split('"') if splitline[0] == '<li><a href=': possiblesnap = splitline[1] possiblesnapext = possiblesnap.split('.') if "tar" in possiblesnapext: snapshot = possiblesnap if os.access(DB + snapshot, os.F_OK): print("The most up to date snapshot is already installed") else: try: urllib.urlretrieve(FTPDIR + 'articles/' + snapshot, DB + snapshot, downloadbar) except: os.remove(DB + snapshot) sys.exit("\nDownloading the snapshot failed.") for file in os.listdir(DB): ext = file.split(".") if "tar" in ext and file != snapshot: sys.stdout.write("\n[Deleting] Removing old snapshot ") os.remove(DB + file) if os.access(DB + "articles", os.F_OK): print("\n[Deleting] Removing old database ") for root, dirs, files in os.walk(DB + "articles", topdown=False): for file in files: os.remove(root + "/" + file) for dir in dirs: os.removedirs(root + "/" + dir) print("[Extracting] Installing new database ") tarball = tarfile.open(DB + snapshot, 'r') tarball.extractall(DB) else: sys.exit("Bad paper `" + sys.argv[2] + "', only `all' can be synced")def update(): print("This is the update function")if __name__ == '__main__': main()