#!/opt/local/bin/python##!/usr/bin/python# minidumper v. .01# Copyright, 1995-2005, Brian Mingus## This file is part of minidumper## This library is free software; you can redistribute it and/or# modify it under the terms of the GNU Lesser General Public# License as published by the Free Software Foundation; either# version 2.1 of the License, or (at your option) any later version.## This library is distributed in the hope that it will be useful,# but WITHOUT ANY WARRANTY; without even the implied warranty of# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU# Lesser General Public License for more details. # Takes a mediawiki xml dump, a list of the pages that you want from# the xml dump and extracts them to stdout# Example usage:# minidumper wikipedia-xml-dump.xml allowed_titles.list --tags=title,id > minidump.xmlfrom optparse import OptionParserimport reclass MiniDumper: def __init__(self): self.titleRE = re.compile(' <title>(.*)</title>\n') self.allowListRE = '' self.currentPage = '' self.xmlPOST = '</mediawiki>' self.firstTime = True # For determining when to pring xmlPRE self.counter = 0 # So we know when to quit (don't parse the whole dump unnecessarily) self.limit = 0 # Needs to be set to the length of the list of titles you gave me self.finished = False # We check this to see if counter == limit, and print the end of the dump if so self.xmlPRE = '\<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.3/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.3/ http://www.mediawiki.org/xml/export-0.3.xsd" version="0.3" xml:lang="en">\n\ <siteinfo>\n\ <sitename>Wikipedia</sitename>\n\ <base>http://en.wikipedia.org/wiki/Main_Page</base>\n\ <generator>MediaWiki 1.10alpha</generator>\n\ <case>first-letter</case>\n\ <namespaces>\n\ <namespace key="0" />\n\ </namespaces>\n\ </siteinfo>' def PageBuilder(self, thisLine): """ This takes lines of input and uses them to build pages. It should be fairly robust to interference, as the correct number of spaces are required on the <page> tags. When a page is completed (indicated by </page>) it is passed to AnalyzePage """ if thisLine == ' <page>\n': self.currentPage = thisLine elif thisLine == ' </page>\n': self.currentPage += thisLine self.AnalyzePage(self.currentPage) else: self.currentPage += thisLine def AllowList(self, listFileName): """ The allow list takes newline separated text file of page titles and returns a regular expression object that places boolean OR (|) before each title This was designed to work with a list of 150,000 titles, so pass it a big list if you want. """ f = open(listFileName,'r') titles = f.readlines() titles = [titles[i][:-1] for i in range(len(titles))] # removes trailing newlines titleList = [self.Escaper(title) for title in titles] self.limit = len(titleList) longLine = '' for title in range(len(titleList)): longLine += titleList[title] longLine = longLine[:-1] # remove trailing pipe print "Regex: " + longLine # TODO: why is this a list f.close() self.allowListRE = re.compile(longLine) def Escaper(self, string): """ Just puts a pipe inbetween each one and escapes it TODO: Remove trailing pipe! """ return str(re.escape(string) + '|') def AnalyzePage(self, page, allowedTags = []): """ AnalyzePage parses the page for its <title>tags</title> and compares it against the allow list You can optionally specify that you only want a subset of tags and the page will then be parsed by BeautifulSoup. It's a very cheap parse because we can use the SoupStrainer to only get the stuff we want. If the regular expression matches this pages title, send it to stdout """ # print "AnalyzePage: \n" + page titleMatch = self.titleRE.search(page) if titleMatch == None: return title = titleMatch.group(1) print "Title: " + title # Check title against supplied list if self.allowListRE.match(title) == None: return # It passed all of our checks - have it! print page self.counter += 1 if self.counter == self.limit: self.finished = Truedef main(): parser = OptionParser() parser.add_option("-t", "--tags", dest="tags", help="Comma-separated list of tags which are subordinate to the <page> tag to pay attention to. Current tags in the dump are page, title, id, revision, etc...") (options, args) = parser.parse_args() xmlFile = args[0] listFile = args[1] extractor = MiniDumper() extractor.AllowList(listFile) f = open(xmlFile) thisLine = None print extractor.xmlPRE while not thisLine == '': thisLine = f.readline() extractor.PageBuilder(thisLine) if extractor.finished == True: print extractor.xmlPOST returnmain()