wlcode

#!/usr/bin/python

import lxml.html
import os
import sys
from pprint import pprint
import fnmatch

def locate(pattern, root=os.curdir, filesonly=True):
    for path, dirs, files in os.walk(os.path.abspath(root)):
        for filename in fnmatch.filter(files, pattern):
            yield os.path.join(path, filename)
        if not filesonly:
            for dd in dirs:
                yield os.path.join(path,dd)

#def main():
#  pprint(read_cable(sys.argv[1]))

def main():
   dir = sys.argv[1]
   i, limit = 0, 1000
   for f in locate("*.html", root=dir):
       pprint(readcable(f))
       i += 1
       if i > limit:
           break


def nom(lines, i):
  while lines[i] == '':
      i = i + 1
  return lines[i], i + 1

def readcable(f):
  # Read in HTML
  htm = open(f).read()
  idx = htm.find("Discussing cables")
  if idx < 0:
      raise "Invalid cable!"
  htm = htm[idx:]

  # Convert to TEXT
  t = lxml.html.fromstring(htm)

  # Do line-based processing from here.
  lines = t.text_content().encode('UTF8').split("\n")
  i = 5  # skip first 5 lines

  # Headers (check they're what we expected)
  junk, i = nom(lines, i); assert(junk == 'Reference ID');
  junk, i = nom(lines, i); assert(junk == 'Created');
  junk, i = nom(lines, i); assert(junk == 'Released');
  junk, i = nom(lines, i); assert(junk == 'Classification');
  junk, i = nom(lines, i); assert(junk == 'Origin');

  # Basic metadata just from line order
  cable = {}
  cable['refid'],          i = nom(lines, i);  
  cable['created'],        i = nom(lines, i); 
  cable['released'],       i = nom(lines, i); 
  cable['classification'], i = nom(lines, i); 
  cable['origin'],         i = nom(lines, i); 

  bodytext = lines[i:]


  # Rip TAGS: from body
  for line in bodytext:
    if line.startswith('TAGS:'):
      cable['tags'] = line.split(":", 1)[1].split()
      break

  # Rip SUBJECT: from body.  A little tricky; we're going
  # to look for a blank line to terminate.
  state = 0
  for line in bodytext:
    if state == 0:       # Look for "SUBJECT:"
      if line.startswith('SUBJECT:'): 
        cable['subject'] = line.split(":", 1)[1]
        state = 1
    elif state == 1:    # Collect until blank line
      if line.strip() == "":
        break            # Termination
      else:
        cable['subject'] += line

  cable['text'] = '\n'.join(bodytext)

  return cable 
  

if __name__ == '__main__':
  main()