Someone

 cat citeseerdownloaderv2.py #TODO: #       - Te puede aparecer un    <header status="deleted">  #       - CLASSPATH="/home/gus/workspace/CiteseerDownload/src/OAI-PMH.jar:/home/gus/xmlbeans-2.2.0/lib/xmlpublic.jar:/home/gus/xmlbeans-2.2.0/lib/xbean.jar:/home/gus/xmlbeans-2.2.0/lib/xbean_xpath.jar:/home/gus/xmlbeans-2.2.0/lib/jsr173_1.0_api.jar:/home/gus/workspace/CiteseerDownload/src/oai_dc.jar"#       - Capas que lo mejor seria generar un YAMLimport httplib, osimport org.openarchives.oai.x20 import java.io.Fileimport org.apache.xmlbeansdef get_listing(conn):        conn.request("GET", "/oai2?verb=ListIdentifiers&metadataPrefix=oai_dc")        response = conn.getresponse()        if (response.reason == "OK" and response.status == 200):                listidsdoc = org.openarchives.oai.x20.OAIPMHDocument.Factory.parse(response.read())                listids = listidsdoc.getOAIPMH().getListIdentifiers().getHeaderArray()                return listids        else:                print "Problema obtenindo la lista de archivos"def get_paper(id, i, out):        conn = httplib.HTTPConnection("export.arxiv.org")        url = "/oai2?verb=GetRecord&identifier=%s&metadataPrefix=oai_dc" % id        conn.request("GET", url)        response = conn.getresponse()        xmlopts = org.apache.xmlbeans.XmlOptions()        xmlopts.setSaveUseOpenFrag()         xmlopts.setUseDefaultNamespace()         rec = org.openarchives.oai.x20.OAIPMHDocument.Factory.parse(response.read(),xmlopts)        meta = rec.getOAIPMH().getGetRecord().getRecord().getMetadata()        dc = org.openarchives.oai.x20.oaiDc.DcDocument.Factory.parse(meta.toString(),xmlopts).getDc()        title = dc.getTitleArray(0).toString().split("<xml-fragment xmlns:oai_dc=\"http://www.openarchives.org/OAI/2.0/oai_dc/\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">")[1].split("</xml-fragment>")[0]        dsc = dc.getDescriptionArray(0).toString().split("<xml-fragment xmlns:oai_dc=\"http://www.openarchives.org/OAI/2.0/oai_dc/\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">")[1].split("</xml-fragment>")[0]        name = id.split(":")        name = name[2]        urlpdf = "http://arxiv.org/pdf/%s" % name        os.system("curl %s > /home/gus/papersV2/download/%s.pdf" % (urlpdf, name))        out.write("<File number=\"%d\" title=\"%s\" name=\"%s\" abstract=\"%s\" > </File> \n" % (i, title, name, dsc))        out.flush()        conn.close()def get_papers(out):        """ Main Loop """        out.write('<?xml version="1.0" encoding="ISO-8859-1"?> \n')            out.write('<root> \n')        conn = httplib.HTTPConnection("export.arxiv.org")        listids = get_listing(conn)        i=0        for id in listids[0:len(listids)/6]: #hay 10k en este repo vamo a encarar la tercera parte                i=i+1                print "%d/%d" % (i,len(listids)/6)                get_paper(id.getIdentifier(), i, out)        out.write('</root>')        out.close()out = open ('/home/gus/papersV2/download/summary.out', 'w') get_papers(out)