rendered paste body#coding:utf-8
#
# (c) 2008 Jan-Simon Mller, dl9pf@gmx.de
# License: GPLv2
#
# Script for the opensuse weekly newsletter statistics
import re
import urllib
from BeautifulSoup import BeautifulSoup
from subprocess import Popen, PIPE
import sys
ISSUE = 60
##############################
# GET
# * lists.o.o subscriber count
#
##############################
# tnx to coolo ;)
print "contacting lists.opensuse.org ..."
cmd='wget -q -O - http://lists.opensuse.org/stats/general/general-mlmmj_subcount-day.png | pngtopnm | convert -crop 57x10+340+244 - - | convert -sample 115x20 - - | ocrad --filter numbers_only | cut -d " " -f1'
p = Popen(cmd, shell=True,stdin=PIPE, stdout=PIPE, close_fds=True)
(child_stdin, child_stdout) = (p.stdin, p.stdout)
listsubscriber_now = child_stdout.read().split("\n")[0]
print "...done"
##############################
# GET
# * USERS
# ** SIGNED/UNSIGNED
# * MEMBERS
# from users.opensuse.org
##############################
print "contacting users.opensuse.org ..."
#https://users.opensuse.org/
# fetch URL
MYURL = u"https://users.opensuse.org/"
x = urllib.urlopen(MYURL)
orig = unicode(x.read(), "utf-8")
#parse with BeautifulSoup
soup = BeautifulSoup(''.join(orig))
########################
# users, signed/unsigned
#grep for ths string
result = soup.find(text=re.compile('users support'))
#
prettify1 = result.split("users")[0]
prettify2 = prettify1.strip()
prettify3 = prettify2.split("of")
#
signed_now = prettify3[0].strip()
total_now = prettify3[1].strip()
#
#######################
# members
result = soup.find(text=re.compile('members'))
prettify1 = result.split("We have ")[1]
prettify2 = prettify1.split("openSUSE members.")[0]
prettify3 = prettify2.strip()
members_now = prettify3
#COUNTRIBUTORS="x (+x) of x (+x) registered contributors in the User Directory have signed the Guiding Principles. The board has acknowledged x [[Members|members]]. <br><br>"
print "...done"
##############################
# GET
# * forum-MEMBERS
# from forums.opensuse.org
##############################
print "contacting forums.opensuse.org..."
# fetch URL
MYURL = u"http://forums.opensuse.org/"
x = urllib.urlopen(MYURL)
orig = unicode(x.read(), "ISO-8859-1")
# parse with soup
soup = BeautifulSoup(''.join(orig))
# grep for string "Members:"
result = soup.find(text=re.compile('Members:'))
#prettify
forummembers_new = result.split("Members:")[1].strip().replace(",","")
print "...done"
##############################
# GET
# * global counters
# from https://build.opensuse.org/statistics
##############################
# fetch URL
buildprojects_new = "0"
buildpackages_new = "0"
buildrepositories_new = "0"
buildusers_new = "0"
if True:
print "contacting build.opensuse.org - this can take ages ..."
MYURL = u"https://build.opensuse.org/statistics"
x = urllib.urlopen(MYURL)
orig = unicode(x.read(), "utf-8")
# parse with soup
soup = BeautifulSoup(''.join(orig))
# grep for string "Members:"
myresult = soup.find("div", {"class" : "global_counters"})
if myresult:
buildprojects_new = str(myresult).split("projects")[0].split("<b>")[1].split("</b>")[0]
buildpackages_new = str(myresult).split("projects")[1].split("packages")[0].split("<b>")[1].split("</b>")[0]
buildrepositories_new = str(myresult).split("projects")[1].split("packages")[1].split("repositories")[0].split("<b>")[1].split("</b>")[0]
buildusers_new = str(myresult).split("projects")[1].split("packages")[1].split("repositories")[1].split("confirmed users")[0].split("<b>")[1].split("</b>")[0]
else:
print "login to build.opensuse.org with firefox"
sys.exit(1)
print "...done"
##############################
# GET
# * statistics from
# from bugzilla.novell.com
##############################
print "contacting bugzilla.novell.com ..."
MYURL = u"https://bugzilla.novell.com/report.cgi?bug_file_loc=&bug_file_loc_type=allwordssubstr&bug_id=&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=NEEDINFO&bug_status=REOPENED&bugidtype=include&chfieldfrom=&chfieldto=Now&chfieldvalue=&classification=openSUSE&email1=&email2=&emailassigned_to1=1&emailassigned_to2=1&emailcc2=1&emailqa_contact2=1&emailreporter2=1&emailtype1=substring&emailtype2=substring&field0-0-0=noop&keywords=&keywords_type=anywords&long_desc=&long_desc_type=fulltext&short_desc=&short_desc_type=allwordssubstr&status_whiteboard=&status_whiteboard_type=allwordssubstr&type0-0-0=noop&value0-0-0=&votes=&x_axis_field=bug_severity&y_axis_field=product&z_axis_field=&width=600&height=350&action=wrap&ctype=csv&format=table"
x = urllib.urlopen(MYURL)
orig_csv = unicode(x.read(), "utf-8")
# own dirty parser ...
rows = orig_csv.split("\n")
#remove rows[0]:
#"Product" / "Severity","Blocker","Critical","Major","Normal","Minor","Enhancement"
rows.remove(rows[0])
X = {}
Y = 0
for i in range(1,7):
X[i]=0
for j in rows:
X[i] = X[i]+int(str(j).split(",")[i])
Y = Y + int(str(j).split(",")[i])
bugzilla_blocker_new = X[1]
bugzilla_critical_new = X[2]
bugzilla_major_new = X[3]
bugzilla_normal_new = X[4]
bugzilla_minor_new = X[5]
bugzilla_enhancement_new = X[6]
bugzilla_total_new = Y
print "...done"
print listsubscriber_now+" subscribers to news.o.o"
print signed_now+" users signed the GP"
print total_now+" total users"
print members_now+" total members"
print forummembers_new+" forum members"
print buildprojects_new+" buildservice projects"
print buildpackages_new+" buildservice packages"
print buildrepositories_new+" buildservice repositories"
print buildusers_new+" buildservice users"
print str(bugzilla_blocker_new)+" bugzilla blocker"
print str(bugzilla_critical_new)+" bugzilla critical"
print str(bugzilla_major_new)+" bugzilla major"
print str(bugzilla_normal_new)+" bugzilla normal"
print str(bugzilla_minor_new)+" bugzilla minor"
print str(bugzilla_enhancement_new)+" bugzilla enhancement"
print str(bugzilla_total_new)+" bugzilla total"
#############################################################
# all new values are done - fetch old from last issue
#############################################################
OLDISSUE = int(ISSUE)-1
print "contacting en.opensuse.org for issue "+str(OLDISSUE)
MYURL = u"http://en.opensuse.org/index.php?title=OpenSUSE_Weekly_News/"+str(OLDISSUE)+u"&action=raw"
x = urllib.urlopen(MYURL)
orig = unicode(x.read(), "utf-8")
# parse with soup
soup = BeautifulSoup(''.join(orig))
# grep for string "listsubscriber_old"
result = soup.find(text=re.compile('non-unique subscribers'))
listsubscriber_old = result.split("non-unique")[0].split("has")[1].split("(")[0].strip()
# grep for string "signed_old","total_old", "members_old"
result = soup.find(text=re.compile('registered contributors in the'))
(x,y) = result.split("registered")[0].split("of")
signed_old = x.split("(")[0].strip()
total_old = y.split("(")[0].strip()
members_old = result.split("acknowledged")[1].split("[[members")[0].strip()
# grep for "forummembers_old"
result = soup.find(text=re.compile('registered users - most users'))
forummembers_old = result.split('have')[1].split("(")[0].strip()
# grep for build*_old
result = soup.find(text=re.compile('\) repositories by'))
buildprojects_old = result.split("projects")[0].split("hosts")[1].split("(")[0].strip()
buildpackages_old = result.split("projects")[1].split("packages")[0].split(",")[1].split("(")[0].strip()
buildrepositories_old = result.split("projects")[1].split("packages")[1].split("repositories")[0].split(",")[1].split("(")[0].strip()
buildusers_old = result.split("projects")[1].split("packages")[1].split("repositories")[1].split("by")[1].split("(")[0].strip()
# grep bugzilla stuff
result = soup.find(text=re.compile("The numbers for all openSUSE project products"))
bugzilla_total_old = result.split("All Open Reports: ")[1].split("(")[0].strip()
bugzilla_blocker_old = result.split("Blocker:")[1].split("(")[0].strip()
bugzilla_critical_old = result.split("Critical:")[1].split("(")[0].strip()
bugzilla_major_old = result.split("Major:")[1].split("(")[0].strip()
bugzilla_normal_old = result.split("Normal:")[1].split("(")[0].strip()
bugzilla_minor_old = result.split("Minor:")[1].split("(")[0].strip()
bugzilla_enhancement_old = result.split("Enhancements:")[1].split("(")[0].strip()
print "...done"
print "###########"
print listsubscriber_old+" old subscribers to news.o.o"
print signed_old+" old users signed the GP"
print total_old+" old total users"
print members_old+" old total members"
print forummembers_old+" old forum members"
print buildprojects_old+" old buildservice projects"
print buildpackages_old+" old buildservice packages"
print buildrepositories_old+" old buildservice repositories"
print buildusers_old+" old buildservice users"
print str(bugzilla_blocker_old)+" old bugzilla blocker"
print str(bugzilla_critical_old)+" old bugzilla critical"
print str(bugzilla_major_old)+" old bugzilla major"
print str(bugzilla_normal_old)+" old bugzilla normal"
print str(bugzilla_minor_old)+" old bugzilla minor"
print str(bugzilla_enhancement_old)+" old bugzilla enhancement"
print str(bugzilla_total_old)+" old bugzilla total"