a4_grading.py

import subprocess
import numpy as np

traintxt = open('data/ptb.char.train.txt').read()
traintxt = traintxt.replace('\n', '<eos>').split()
vocab_map = {}
vocab_ix = 0
for c in traintxt:
    if c not in vocab_map:
        vocab_map[c] = vocab_ix # note: zero-based vs lua: 1-based
        vocab_ix = vocab_ix + 1

#CHECK THAT THIS DICTIONARY IS THE SAME AS GENERATED BY LUA
#lua_map = dict(tup.split() for tup in subprocess.check_output(['luajit','a4_vocab.lua']).split("\n") if tup)
#for c,ix in vocab_map.iteritems():
    #assert ix == int(lua_map[c]) - 1

evaltxt = open('data/ptb.char.valid.txt').read()
evaltxt = evaltxt.replace('\n', '<eos>').split()
cmd = './run.sh'

proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout = subprocess.PIPE)

nll = 0
fails = 0

print "Flushing proc output until OK GO received"
while True:
    ret = proc.stdout.readline()
    print " -- ", ret.strip()
    if ret.strip() == "OK GO":
        break

print "Start test loop"

for i in range(len(evaltxt)-1):
    proc.stdin.write(evaltxt[i]+'\n')
    try:
        logprobs = map(float, proc.stdout.readline().split())
        assert len(logprobs) == len(vocab_map), "Unexpected response length %d"%len(logprobs)
        np.testing.assert_almost_equal(np.exp(logprobs).sum(), 1, decimal=4, err_msg="Nice try :)")
        nll -= logprobs[vocab_map[evaltxt[i+1]]]
    except Exception as e:
        fails += 1
        print "Fail #", fails, "errmsg:", str(e)
    if i%1000 == 0:
        avgnll  = nll / (i+1)
        perp = np.exp(5.6 * avgnll)
        print("%d\tPerplexity: %.6f\tavgnll: %.6f\tfails: %d" % (i, perp, avgnll,fails))