#!/usr/bin/python# Title:# test_train.py## CVS/RCS $Header:## Description: given a message pool full files, run through and score# or retrain based on the given scores## Copyright:# Eric S. Johansson (esj@harvee.org)# BSD License: see LICENSE or http://www.opensource.org/licenses/bsd-license.php## Requires:import sys# import reimport osimport os.pathimport pwdfrom stat import ST_SIZE,ST_MTIMEfrom email.Utils import parseaddrimport getoptfrom time import ctime, timeimport commandsimport cjsonimport crm114import syslogfrom optparse import OptionParser#from tpblue.log import logdelta_time = 5.0*24*60*60#sequence of events:# get mailbox path from configuration file# if who am I = None, admin:# iterate over all home directories for users UID >= 500# each user is checked for more than five days worth of message in a dumpster# all CRM114 .txt files will be truncateddef retrain (name, op, tr): """look in the message pool retrain everyone""" baseline_directory = os.path.join (name, "message_pool") cr_type = crm114.CRM114 if tr in 'bal': cr_type = crm114.CRM114_balanced if tr in 'var': cr_type = crm114.CRM114_var_limit analyzer = cr_type('/usr/local/crm/mailfilter.crm',os.path.join(name,'crm114/')) hist = {} working_directories = os.listdir(baseline_directory) for token in working_directories: # got the base name, check the date clot = os.path.join(baseline_directory,token) if os.path.isdir(clot): handle = open(os.path.join (clot,'message')) message = handle.read() handle = open(os.path.join (clot,'metadata')) metadata = handle.read() meta_dictionary = cjson.decode(metadata, all_unicode=True) #now train based on state if meta_dictionary['state'] == 'delivered': if "score" in op or "train" in op: score = analyzer.score_message (message) if "train" in op: analyzer.train_as_good(message, score) elif meta_dictionary['state'] == 'dumpster': if "score" in op or "train" in op: score = analyzer.score_message (message) if "train" in op: analyzer.train_as_bad(message, score) else: continue # print score, meta_dictionary['score'] , meta_dictionary['state'] # print score try: hist[int(score)] +=1 except: hist[int(score)] = 1 for i in hist.keys(): print "%s, %s"%(i, hist[i])def process_entry(user): """cleanup CRM 114 """ base = os.path.join(user,"crm114") # truncate perpetually increasing files file(os.path.join(base,"allmail.txt"), "w+").close() file(os.path.join(base,"rejected_by_css.txt"), "w+").close() file(os.path.join(base,"nonspamtext.txt"), "w+").close() file(os.path.join(base,"spamtext.txt"), "w+").close() spam = os.path.join(base,"spam.css") nonspam = os.path.join(base,"nonspam.css") os.system("rm %s "% spam) os.system("rm %s "% nonspam) os.system("cssutil -r %s >/tmp/cssbuild"% spam) os.system("cssutil -r %s >/tmp/cssbuild"% nonspam)def usage(): message = """sweepup [ -h ] | [ -p base_path ]-h: show this bit of text. Hopefully it will help p: path to twopenny-blue user configuration directory. can't count on configuration file-data because this may be running on copy of a user's files in a-nonstandard location."""# print messagedef main(): try: opts, args = getopt.getopt(sys.argv[1:], "hptsr:") except getopt.GetoptError: # print help information and exit: usage() sys.exit(2) user = None reset = False usage = "clean_configuratino.py [-p <path>] [-t train|score] [-r] [-m def|var|bal]" parser = OptionParser(usage) parser.add_option("-p", type="string", dest="path", default="", help="path to user CRM 114 data") parser.add_option("-t", type="string", dest="op", default="score", help="tell the system to train based on score. Leave off if just score") parser.add_option("-m", type="string", dest="tr", default="def", help="use default scoring (toe)") parser.add_option("-r", action="store_true", dest="reset", default=False, help="tell the system to reset training before scoring or training") (options, args) = parser.parse_args() if options.reset: process_entry( options.path) retrain(options.path, options.op, options.tr)if __name__ == "__main__": main()