# -*- coding: utf-8 -*-"""This bot is used for summarize discussions spread over the whole wikiincluding all namespaces. It checks several users (at request), sequential (currently for the german wiki [de] only).!!! MAILER BOT HAS TO BE RUN DAILY TO WORK CORRECT AT CURRENT STATE !!!..."""# ====================================================================================================## ToDo-Liste (Bugs, Features, usw.):# http://de.wikipedia.org/wiki/Benutzer:DrTrigonBot/ToDo-Liste## READ THE *DOGMAS* FIRST!# # ====================================================================================================## (C) Dr. Trigon, 2008, 2009## Distributed under the terms of the MIT license.# (is part of DrTrigonBot-"framework")## keep the version of 'clean_user_sandbox.py', 'new_user.py', 'runbotrun.py', 'replace_tmpl.py', 'sum_disc-conf.py', ...# up to date with this:# Versioning scheme: A.B.CCCC# CCCC: beta, minor/dev relases, bugfixes, daily stuff, ...# B: bigger relases with tidy code and nice comments# A: really big release with multi lang. and toolserver support, ready# to use in pywikipedia framework, should also be contributed to it__version__='$Id: mailer.py 0.2.0000 2009-08-29 10:12:00Z drtrigon $'#import wikipedia, configimport dtbextimport re, sys, os, timeimport mathimport urllib, MySQLdbclass MailerRobot: ''' Robot which will check ... ''' #http://de.wikipedia.org/w/index.php?limit=50&title=Spezial:Beitrge&contribs=user&target=DrTrigon&namespace=3&year=&month=-1 #http://de.wikipedia.org/wiki/Spezial:Beitrge/DrTrigon _content_maxlen = 3000 def __init__(self): ''' constructor of SumDiscBot(), initialize needed vars ''' # modified due: http://de.wikipedia.org/wiki/Benutzer:DrTrigonBot/ToDo-Liste (id 28, 30, 17) self._userListPage = dtbext.wikipedia.Page(wikipedia.getSite(), u'Benutzer:DrTrigon/Entwurf/Vorlage:AutoMail') self._tmpl_regex = re.compile('\{\{Benutzer:DrTrigon/Entwurf/Vorlage:AutoMail(.*?)\}\}', re.S) self._today = int(time.time() / (60*60*24)) def run(self): ''' run SumDiscBot() ''' # modified due: http://de.wikipedia.org/wiki/Benutzer:DrTrigonBot/ToDo-Liste (id 24, 38, 17) ################################################## # move 'SendMail_TS', 'get_TS', 'ReferringPageGenerator_TS' and all '*_TS' into -> 'dtbext.toolserver' ('SendMail', 'get', ...) #print self.SendMail_TS('dr.trigon@surfeu.ch', 'subject', 'mail text') ## 'Benutzer:DrTrigon/Entwurf/Vorlage:AutoMail', 'Benutzer:DrTrigon', 'Hilfe:Single-User-Login', 'Hauptseite' #print self.get_TS(u'DrTrigon/Entwurf/Vorlage:AutoMail', db_direct=True) #print self.get_TS(u'DrTrigon', db_direct=True) #print self.get_TS(u'Single-User-Login', db_direct=True) #print self.get_TS(u'Hauptseite', db_direct=True) # #print self.get_TS(u'Benutzer:DrTrigon/Entwurf/Vorlage:AutoMail') #print self.get_TS(u'Benutzer:DrTrigon') #print self.get_TS(u'Hilfe:Single-User-Login') #print self.get_TS(u'Hauptseite') # very slow and probably not the correct result...?! #print self.ReferringPageGenerator_TS('abc') print self.UserInfo_TS('abc') print "Ende von 'mailer_ts.py'; toolserver wiki DB and mail access test." return ################################################## wikipedia.output(u'\03{lightgreen}* Processing Template Backlink List:\03{default}') for page in dtbext.pagegenerators.ReferringPageGenerator(self._userListPage): content = self._readPage(page) params = self._getMode(content) # get operating mode if not params: continue for item in params: days = self._today % int(item['Frequenz']) if not (days == 0): wikipedia.output(u'INFO: mail to %s will be sent in %i day(s)' % (item['Benutzer'], (int(item['Frequenz'])-days))) continue wikipedia.output(u'\03{lightblue}** Sending page %s as mail to user %s...\03{default}' % (page.aslink(), item['Benutzer'])) content = content.encode(config.textfile_encoding) #if not dtbext.wikipedia.SendMail(item['Benutzer'], page.title().encode(config.textfile_encoding), content.encode(config.textfile_encoding)): success = True j = 1 j_max = math.ceil(float(len(content)) / self._content_maxlen) for i in range(0, len(content), self._content_maxlen): text = content[i:(i+self._content_maxlen)] title = page.title() + (u' %i/%i' % (j, j_max)) success = success and dtbext.wikipedia.SendMail(item['Benutzer'], title.encode(config.textfile_encoding), text) j += 1 wikipedia.output(u"\03{lightblue}*** Mail '%s' sent. \03{default}" % title) if not success: wikipedia.output(u'!!! WARNING: mail could not be sent!') def _readPage(self, page, full=False): ''' read wiki page input: page returns: page content [string (unicode)] ''' # modified due: http://de.wikipedia.org/wiki/Benutzer:DrTrigonBot/ToDo-Liste (id 28) if full: (mode, info) = ('full', ' using "getFull()" mode') else: (mode, info) = ('default', '') #wikipedia.output(u'\03{lightblue}Reading Wiki at %s...\03{default}' % page.aslink()) wikipedia.output(u'\03{lightblue}Reading Wiki%s at %s...\03{default}' % (info, page.aslink())) try: #content = page.get() content = page.get(mode=mode) #if url in content: # durch history schon gegeben! (achtung dann bei multithreading... wobei ist ja thread per user...) # wikipedia.output(u'\03{lightaqua}** Dead link seems to have already been reported on %s\03{default}' % page.aslink()) # continue except (wikipedia.NoPage, wikipedia.IsRedirectPage): content = u'' return content def _getMode(self, content): ''' get operating mode from page with template by searching the template input: content [text] (page content) self-objects returns: params [dict] ''' # modified due: http://de.wikipedia.org/wiki/Benutzer:DrTrigonBot/ToDo-Liste (id 28, 17) tmpl_buf = self._tmpl_regex.search(content) params = [] if tmpl_buf: # enhanced: with template for item in tmpl_buf.groups(): tmpl_params = tmpl_buf.groups()[0] tmpl_params = re.sub('\n', '', tmpl_params) tmpl_params = re.sub('\|', "','", tmpl_params) tmpl_params = re.sub('=', "':'", tmpl_params) params.append( eval( "{" + tmpl_params[2:] + "'}" ) ) return params def SendMail_TS(self, user, subject, text, CCme = False): # user NICHT wiki username SONDERN email-adresse! # subject funktioniert (noch) nicht # CCme funktioniert (noch) nicht # anderes verzeichnis als lokal wre SEHR SINVOLL und GUT! filename = "mailbuf" try: f = open(filename, "w") f.write( text ) f.close() # http://www.elandsys.com/resources/sendmail/ os.system('/usr/sbin/sendmail %s < %s' % (user, filename)) os.remove(filename) return True except: return False def get_TS(self, title, force=False, get_redirect=False, throttle=True, sysop=False, change_edit_time=True, mode='default', plaintext=False, db_direct=False): # neu: # title Titel der Seite # db_direct True: direkt auf toolserver DB zugreifen / False: WikiProxy verwenden # - Wie kriege ich die bersetzung namespace <-> prfix (z.B. '2' <-> 'Benutzer:') ??? # - Wie greife ich auf externe DBs (z.B. 'cluster22') ??? # http://meta.wikimedia.org/wiki/User:Duesentrieb/Tools#Meta-Tools # >It will become obsolete when there is full text replication on the toolserver. if not db_direct: # http://meta.wikimedia.org/wiki/User:Duesentrieb/WikiProxy # http://toolserver.org/~daniel/WikiSense/WikiProxy.php #title = 'Benutzer:DrTrigon/Entwurf/Vorlage:AutoMail' req = 'http://toolserver.org/~daniel/WikiSense/WikiProxy.php?wiki=dewiki&title=%s' % urllib.quote(title) #req = 'http://toolserver.org/~daniel/WikiSense/WikiProxy.php' return wikipedia.getSite().getUrl(req, no_hostname = True) else: # - Wie kriege ich die bersetzung namespace <-> prfix (z.B. '2' <-> 'Benutzer:') ??? # - Wie greife ich auf externe DBs (z.B. 'cluster22') ??? # https://wiki.toolserver.org/view/Database_access#Python # http://www.wellho.net/resources/ex.php4?item=y115/sql1a.py # http://meta.wikimedia.org/wiki/Toolserver/Database # http://en.wikipedia.org/wiki/Wikipedia:Database_queries (http://en.wikipedia.org/wiki/index.php?curid=76871) # http://en.wikipedia.org/wiki/Wikipedia_talk:Database_queries # http://www.mediawiki.org/wiki/Manual:Database_layout # Establich a connection #db = MySQLdb.connect(db='enwiki_p', host="enwiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") db = MySQLdb.connect(db='dewiki_p', host="dewiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") # Run a MySQL query from Python and get the result set #db.query("""SELECT user_name, COUNT(*) FROM user, cur WHERE user_id=cur_user GROUP BY user_id ORDER BY user_id DESC LIMIT 20""") #db.query("""SELECT '%s' AS cur_title, cur_text FROM cur ORDER BY cur_title LIMIT 10""" % force) # 'Hauptseite', 'Benutzer:DrTrigon', 'Hilfe:Single-User-Login' # 'Hauptseite' (nicht alle Seiten vorhanden) db.query("""SELECT cur_title, cur_text, COUNT(*) FROM cur WHERE cur_title='%s' LIMIT 10""" % title) r = db.store_result() r = [row for row in r.fetch_row(10)] if (r[0][0] != None): db.close() return r # 'DrTrigon/Entwurf/Vorlage:AutoMail' (sollte alle Seiten enhalten) #db.query("""SELECT page_title, page_namespace, COUNT(*) FROM page WHERE page_title='DrTrigon/Entwurf/Vorlage:AutoMail' AND page_namespace=2 LIMIT 10""") #db.query("""SELECT page_title, page_namespace, COUNT(*) FROM page WHERE page_title='%s' LIMIT 10""" % title) #db.query("""SELECT page_title, page_namespace, COUNT(*) FROM page WHERE page_title='%s' AND page_namespace=%s LIMIT 10""" % (title, ns)) ##db.query("""SELECT page_title, page_namespace, COUNT(*) FROM page WHERE page_title='DrTrigon/Entwurf/Vorlage:AutoMail' AND page_namespace=2 LIMIT 10""") # namespaces? # >The text of the page itself is stored in the text table. To retrieve the text of an article, MediaWiki first searches for page_title in this # >table. Then, page_latest is used to search the revision table for rev_id, and rev_text_id is obtained in the process. The value obtained for # >rev_text_id is used to search for old_id in the text table to retrieve the text. #title = 'DrTrigon/Entwurf/Vorlage:AutoMail' #ns = 2 #db.query("""SELECT page_title, page_namespace, page_latest, COUNT(*) FROM page WHERE page_title='%s' AND page_namespace=%s LIMIT 10""" % (title, ns)) # wenn namespace eindeutig, d.h. Seite nur in einem ns vorhanden... db.query("""SELECT page_title, page_namespace, page_latest, COUNT(*) FROM page WHERE page_title='%s' LIMIT 10""" % title) r = db.store_result() r = [row for row in r.fetch_row(10)] page_latest = long(r[0][2]) db.query("""SELECT rev_id, rev_text_id, COUNT(*) FROM revision WHERE rev_id='%s' LIMIT 10""" % page_latest) r = db.store_result() r = [row for row in r.fetch_row(10)] rev_text_id = long(r[0][1]) db.query("""SELECT old_id, old_text, old_flags, COUNT(*) FROM text WHERE old_id='%s' LIMIT 10""" % rev_text_id) r = db.store_result() r = [row for row in r.fetch_row(10)] print r # DB://cluster22/4824034 #db = MySQLdb.connect(db='cluster22', host="dewiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") #db = MySQLdb.connect(db='sql-text22', host="dewiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") db = MySQLdb.connect(db='clematis', host="dewiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") db.query("""SELECT old_id, old_text, old_flags, COUNT(*) FROM text WHERE old_id='%s' LIMIT 10""" % "4824034") rr = db.store_result() rr = [row for row in rr.fetch_row(10)] print rr # _mysql_exceptions.OperationalError: (1044, "Access denied for user 'drtrigon'@'%.toolserver.org' to database 'cluster22'") db.close() #dbext.close() return r def ReferringPageGenerator_TS(self, a): # Establich a connection #db = MySQLdb.connect(db='enwiki_p', host="enwiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") db = MySQLdb.connect(db='dewiki_p', host="dewiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") # Run a MySQL query from Python and get the result set db.query("""SELECT cur_title, cur_text, COUNT(*) FROM cur WHERE cur_text LIKE '%{{Benutzer:DrTrigon/Entwurf/Vorlage:AutoMail%' ORDER BY cur_title LIMIT 10""") r = db.store_result() r = [row for row in r.fetch_row(10)] if (r[0][0] != None): db.close() return r return def UserInfo_TS(self, a): # Establich a connection #db = MySQLdb.connect(db='enwiki_p', host="enwiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") db = MySQLdb.connect(db='dewiki_p', host="dewiki-p.db.toolserver.org", read_default_file="/home/drtrigon/.my.cnf") # Run a MySQL query from Python and get the result set #db.query("""SELECT user_name, user_email, COUNT(*) FROM user WHERE user_name = 'DrTrigon' ORDER BY user_name LIMIT 10""") #db.query("""DESCRIBE user""") db.query("""DESCRIBE user_properties""") r = db.store_result() r = [row for row in r.fetch_row(10)] if (r[0][0] != None): db.close() return r returndef main(): bot = MailerRobot() # for several user's, but what about complete automation (continous running...) if len(wikipedia.handleArgs()) > 0: for arg in wikipedia.handleArgs(): if arg[:2] == "u'": arg = eval(arg) # for 'runbotrun.py' and unicode compatibility if (arg[:5] == "-auto") or (arg[:5] == "-cron"): bot.silent = True elif (arg == "-skip_clean_user_sandbox"): pass elif arg[:17] == "-compress_history": pass else: wikipedia.showHelp() return bot.run()if __name__ == "__main__": try: main() finally: wikipedia.stopme()