zdic.net query draft code

NOTE
Python code is in section 1, javascript query from the search site is in section 2



SECTION 1
---------
# import re
import urllib, urllib2
import sys
import getopt
from lxml import etree, html

# from util import hook


# @hook.command('u')
# @hook.command
def zdic(inp):
    '''.zd/.zdic <Chinese word in characters> -- looks up <word> on zdic.net/cd/'''

    url = 'http://www.zdic.net/sousuo'
    # url = 'http://www.nciku.com/submitFeedback'

    params = {'lb_a': 'hp', 'lb_b': 'jq', 'lb_c': 'jq', 'tp': 'tp1', 'q': inp}
    # params = {'lb_a': 'hp', 'lb_b': 'jq', 'lb_c': 'jq', 'tp': 'tp1', 'q': inp}
    page = post_get(url, params)

#    page = http.get_html(url, term=inp)

#    words = page.xpath("//td[@class='word']")
#    defs = page.xpath("//div[@class='definition']")
    defs = page.xpath("//p[@class='zdct1']")
    pinyin = page.xpath("//span[@class='dicpy']")

    if not defs:
        return 'no definitions found'

#    out = defs[1].text_content().strip() + ': ' + ' '.join(
#            defs[0].text_content().split())

    out = [None]*2
    out[0] = defs[1].text_content().strip() + defs[2].text_content().strip()
    out[1] = defs[3].text_content().strip() + defs[4].text_content().strip()
    
    for c in out:
        if len(c) > 400:
            c = c[:c.rfind(' ', 0, 400)] + '...'

    return out


def post_get(url, params):
#    f = urllib.urlopen(url, urllib.urlencode(params))
#    
    user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0) Gecko/20100101 Firefox/8.0'
    headers = { 'User-Agent' : user_agent,
                'Host' : 'www.zdic.net',
                'Referer' : 'http://www.zdic.net/cd/' }

    data = urllib.urlencode(params)
    req = urllib2.Request(url, data, headers)
    import pdb; pdb.set_trace()
    f = urllib2.urlopen(req)
    text = f.read()
    outfile(text + '\n\n' + params['q'])
    return html.fromstring(text)





def main():
    # parse command line options
    try:
        opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
    except getopt.error, msg:
        print msg
        print "for help use --help"
        sys.exit(2)
    # process options
    for o, a in opts:
        if o in ("-h", "--help"):
            print __doc__
            sys.exit(0)
    # process arguments
    for arg in args:
        print zdic(arg) # process() is defined elsewhere


def outfile(text):
    of = open('zdret.txt', 'w')
    of.write(text)
    of.close()
    

if __name__ == "__main__":
    main()





SECTION 2
---------

RELEVANT CONTENT OF /CD/ (SEARCH PAGE)
<script language="javascript" src="/js/cd_sec.js" type="text/javascript"></script>

CONTENT OF /JS/CD_SEC.JS
<script language="javascript" src="/js/secpan.js" type="text/javascript"></script>


CONTENT OF /JS/SECPAN.JS
document.writeln("<div class=\"secpan\">");
document.writeln("<div class=\"sec_m\">");
document.writeln("<span id=\"tp1\">条目</span><div>|</div>");
document.writeln("<span id=\"tp2\">字典</span><div>|</div>");
document.writeln("<span id=\"tp3\">词典</span><div>|</div>");
document.writeln("<span id=\"tp4\">成语</span><div>|</div>");
document.writeln("<span id=\"tp5\">全站</span>");
document.writeln("</div>");
document.writeln("  <form method=\"post\" action=\"/sousuo/\" id=\"f1\" name=\"f1\">");
document.writeln("  <div class=\"tp_c\" id=\"tp_ts1\">请直接输入汉字或词语进行查询，支持拼音查询，例：“han”;“han4”;“han yu”;“han4 yu3”。</div>");
document.writeln("  <div class=\"tp_c\" id=\"tp_ts2\">");
document.writeln("    <label><input name=\"lb_a\" type=\"radio\" value=\"hp\" checked>汉字或拼音</label>");
document.writeln("    <label><input name=\"lb_a\" type=\"radio\" value=\"bis\">笔顺</label>");
document.writeln("    <label><input name=\"lb_a\" type=\"radio\" value=\"wb86\">五笔编码</label>");
document.writeln("    <label><input name=\"lb_a\" type=\"radio\" value=\"cj\">仓颉编码</label>");
document.writeln("    <label><input name=\"lb_a\" type=\"radio\" value=\"fc\">四角号码</label>");
document.writeln("    <label><input name=\"lb_a\" type=\"radio\" value=\"uno\">unicode</label>");
document.writeln("  </div>");
document.writeln("  <div class=\"tp_c\" id=\"tp_ts3\">");
document.writeln("    <label><input name=\"lb_b\" type=\"radio\" value=\"mh\" checked>模糊搜索</label>");
document.writeln("    <label><input name=\"lb_b\" type=\"radio\" value=\"jq\">精确搜索</label>");
document.writeln("  </div>");
document.writeln("  <div class=\"tp_c\" id=\"tp_ts4\">");
document.writeln("    <label><input name=\"lb_c\" type=\"radio\" value=\"mh\" checked>模糊搜索</label>");
document.writeln("    <label><input name=\"lb_c\" type=\"radio\" value=\"jq\">精确搜索</label>  ");
document.writeln("  </div>");
document.writeln("    <input name=\"tp\" id=\"tp\" type=\"hidden\" value=\"tp1\">");
document.writeln("    <div class=\"secpan_qb\">");
document.writeln("    <DIV class=\"query\">");
document.writeln("    <input class=\"q\" id=\"q\" name=\"q\" type=\"text\" value=\"\"  maxlength=\"30\">");
document.writeln("    </DIV>");
document.writeln("    <BUTTON class=\"btn\" type=\"submit\"></BUTTON>");
document.writeln("    </div>");
document.writeln("  </form>");
document.writeln("  <div class=\"tp_c\" id=\"tp_ts5\"></div>");
document.writeln("  <form action=\"http://www.google.com.hk/cse\" id=\"cse-search-box\" name=\"f2\" style=\"display:none\" target=\"_blank\">");
document.writeln("      <input type=\"hidden\" name=\"cx\" value=\"partner-pub-5789084355642416:bog9lf-6b3m\" />");
document.writeln("      <input type=\"hidden\" name=\"ie\" value=\"UTF-8\" />");
document.writeln("    <div class=\"secpan_qb\">");
document.writeln("    <DIV class=\"query\">      ");
document.writeln("      <input class=\"q\" type=\"text\" name=\"q\" value=\"\"/>");
document.writeln("    </div>");
document.writeln("    <BUTTON class=\"btn\" name=\"sa\" value=\"搜索\" type=\"submit\"></BUTTON>");
document.writeln("	</div>");
document.writeln("  </form>");
document.writeln("  <script type=\"text/javascript\" src=\"http://www.google.cn/cse/brand?form=cse-search-box&amp;lang=zh-Hans\"></script> ");
//document.writeln("<div class=\"tp_c\" id=\"tp_tx1\">模糊搜索请加问号：中？   |   <a href=\"/search/old.htm\">旧版搜索框</a></div>");
document.writeln("<div class=\"tp_c\" id=\"tp_tx1\"><a href=\"/zd/bs/\">【字典部首检字】</a>|<a href=\"/cd/bh/\">【词典部首检索】</a> | <a href=\"/search/old.htm\">旧版搜索框</a></div>");
document.writeln("    <div class=\"tp_c\" id=\"tp_tx2\"><a href=\"/zd/bs/\">【部首检字】</a>   <a href=\"/zd/py/\">【拼音索引】</a></div>");
document.writeln("    <div class=\"tp_c\" id=\"tp_tx3\"><a href=\"/cd/bh/\">【部首检索】</a></div>");
document.writeln("    <div class=\"tp_c\" id=\"tp_tx4\"><a href=\"/cy/bs/\">【部首检索】</a>   <a href=\"/cy/py/\">【拼音索引】</a></div>");
document.writeln("    <div class=\"tp_c\" id=\"tp_tx5\"></div>");
document.writeln("  <script type=\"text/javascript\" src=\"/js/cse.js\"></script>");
document.writeln("</div>    ");