python 3 version

.gitignore

@@ -1,3 +1,4 @@
 data/
 models/
-*.pyc
+*.pyc
+.idea/

dan_sentiment.py

@@ -2,7 +2,8 @@
 from util.sentiment_util import *
 from util.math_util import *
 from util.adagrad import Adagrad
-import cPickle, time, argparse
+import time, argparse
+import _pickle as cPickle
 from collections import Counter
 
 # compute model accuracy on a given fold
@@ -38,7 +39,7 @@ def validate(data, fold, params, deep, f=relu):
 
         total += 1
 
-    print 'accuracy on ', fold, correct, total, str(correct / total), '\n'
+    print('accuracy on ',fold,int(correct),int(total), round(correct / total, 2), '\n')
     return correct / total
 
 # does both forward and backprop
@@ -180,13 +181,13 @@ def objective_and_grad(data, params, d, dh, len_voc, deep, labels, f=relu, df=dr
         for sent, label in split:
             c[label] += 1
             tot += 1
-        print c, tot
+        print(c, tot)
 
     if args['rand_We']:
-        print 'randomly initializing word embeddings...'
+        print('randomly initializing word embeddings...')
         orig_We = (random.rand(d, len_voc) * 2 - 1) * 0.08
     else:
-        print 'loading pretrained word embeddings...'
+        print('loading pretrained word embeddings...')
         orig_We = cPickle.load(open(args['We'], 'rb'))
 
     # output log and parameter file destinations
@@ -201,7 +202,7 @@ def objective_and_grad(data, params, d, dh, len_voc, deep, labels, f=relu, df=dr
     r = roll_params(params)
 
     dim = r.shape[0]
-    print 'parameter vector dimensionality:', dim
+    print('parameter vector dimensionality:', dim)
 
     log = open(log_file, 'w')
 
@@ -215,7 +216,7 @@ def objective_and_grad(data, params, d, dh, len_voc, deep, labels, f=relu, df=dr
 
         # create mini-batches
         random.shuffle(train)
-        batches = [train[x : x + args['batch_size']] for x in xrange(0, len(train), 
+        batches = [train[x : x + args['batch_size']] for x in range(0, len(train),
                    args['batch_size'])]
 
         epoch_error = 0.0
@@ -235,8 +236,8 @@ def objective_and_grad(data, params, d, dh, len_voc, deep, labels, f=relu, df=dr
             epoch_error += err
 
         # done with epoch
-        print time.time() - ep_t
-        print 'done with epoch ', epoch, ' epoch error = ', epoch_error, ' min error = ', min_error
+        print(time.time() - ep_t)
+        print('done with epoch ', epoch, ' epoch error = ', epoch_error, ' min error = ', min_error)
         lstring = 'done with epoch ' + str(epoch) + ' epoch error = ' + str(epoch_error) \
                  + ' min error = ' + str(min_error) + '\n'
         log.write(lstring)

preprocess/load_embeddings.py

@@ -1,14 +1,14 @@
 from numpy import *
-import cPickle, zipfile
+import _pickle as cPickle, zipfile
 
-vec_file = zipfile.ZipFile('../data/glove.840B.300d.zip', 'r').open('glove.840B.300d.txt', 'r')
-all_vocab = {}
-print 'loading vocab...'
+# vec_file = zipfile.ZipFile('../data/glove.840B.300d.zip', 'r').open('glove.840B.300d.txt', 'r')
+vec_file = open('../data/glove.840B.300d.txt', 'r')
+print('loading vocab...')
 wmap = cPickle.load(open('../data/sentiment/wordMapAll.bin', 'rb'))
 revMap = {}
 for word in wmap:
     revMap[wmap[word]] = word
-
+all_vocab = {}
 for line in vec_file:
     split = line.split()
     try:
@@ -18,25 +18,32 @@
     except:
         pass
 
-print len(wmap), len(all_vocab)
+print(len(wmap), len(all_vocab))
 d = len(all_vocab['the'])
 
-We = empty( (d, len(wmap)) )
+We = empty( (d, len(wmap)))
 
-print 'creating We for ', len(wmap), ' words'
+print('creating We for ', len(wmap), ' words')
 unknown = []
+wrong_shape = []
 
 for i in range(0, len(wmap)):
     word = revMap[i]
     try:
         We[:, i] = all_vocab[word]
     except KeyError:
         unknown.append(word)
-        print 'unknown: ', word
+        print('unknown: ', word)
         We[:, i] = all_vocab['unknown']
-
-print 'num unknowns: ', len(unknown)
-print We.shape
-
-print 'dumping...'
-cPickle.dump( We, open('../data/sentiment_We', 'wb'), protocol=cPickle.HIGHEST_PROTOCOL)
+    except ValueError:
+        print('value error', word, all_vocab[word][:3], all_vocab[word].shape)
+        wrong_shape.append(word)
+        start = len(all_vocab[word]) - d
+        We[:, i] = all_vocab[word][start:]
+
+print('num unknowns: ', len(unknown))
+print('num wrong shapes', len(wrong_shape))
+print(We.shape)
+
+print('dumping...')
+cPickle.dump( We, open('../data/sentiment_We', 'wb'))

preprocess/preprocess_imdb.py

@@ -1,5 +1,5 @@
 from glob import glob
-import cPickle
+import _pickle as cPickle
 import random
 
 def compute_vocab():
@@ -14,6 +14,7 @@ def compute_vocab():
     for fold in [trneg, trpos, tneg, tpos]:
         fold_docs = []
         for fname in fold:
+            print(fname)
             doc = []
             f = open(fname, 'r')
             for line in f:
@@ -52,17 +53,16 @@ def compute_vocab():
             elif i == 3:
                 test.append((doc, 1))
 
-    print len(train), len(test)
+    print(len(train), len(test))
 
     random.shuffle(train)
     random.shuffle(test)
 
     for x in range(3000, 3020):
-        print i, train[x][1], ' '.join(vocab[x] for x in train[x][0])
-        print '\n'
+        print(i, train[x][1], ' '.join(vocab[x] for x in train[x][0]))
+        print('\n')
 
-    cPickle.dump([train, test, vocab, vdict], open('../data/aclimdb/imdb_splits', 'wb'),\
-        protocol=cPickle.HIGHEST_PROTOCOL)
+    cPickle.dump([train, test, vocab, vdict], open('../data/aclimdb/imdb_splits', 'wb'))
 
 
 

preprocess/sentiment_trees.py

@@ -2,7 +2,7 @@
 # https://github.com/awni/semantic-rntn/blob/master/tree.py
 # credit to Awni Hannun
 
-import collections, cPickle
+import collections, _pickle as cPickle
 from nltk.corpus import stopwords
 from collections import Counter
 import string, sys
@@ -86,7 +86,7 @@ def mapWords(node,wordMap):
             node.word = wordMap[node.word]
 
 def loadWordMap():
-    import cPickle as pickle
+    import _pickle as pickle
     with open('wordMap.bin','r') as fid:
         return pickle.load(fid)
 
@@ -110,20 +110,20 @@ def buildWordMap():
     ddir = '../data/sentiment/'
 
     for file in [ddir + 'train.txt', ddir + 'dev.txt', ddir + 'test.txt']:
-        print "Reading trees.."
+        print("Reading trees..")
         with open(file,'r') as fid:
             trees = [Tree(l) for l in fid.readlines()]
 
-        print "Counting words.."
+        print("Counting words..")
         for tree in trees:
             leftTraverse(tree.root,nodeFn=countWords,args=words)
 
-        print len(words)
+        print(len(words))
 
-    wordMap = dict(zip(words.iterkeys(),xrange(len(words))))
+    wordMap = dict(zip(words.keys(), range(len(words))))
     wordMap[UNK] = len(words) # Add unknown as word
 
-    with open('../data/sentiment/wordMapAll.bin','w') as fid:
+    with open('../data/sentiment/wordMapAll.bin','wb') as fid:
         cPickle.dump(wordMap,fid)
 
     return wordMap
@@ -134,7 +134,7 @@ def loadTrees(dataSet='train', wmap=loadWordMap):
     """
     wordMap = wmap
     file = '../data/sentiment/%s.txt'%dataSet
-    print "Reading trees.."
+    print("Reading trees..")
     with open(file,'r') as fid:
         trees = [Tree(l) for l in fid.readlines()]
     for tree in trees:
@@ -152,8 +152,8 @@ def preprocess(sents, wmap, binary=False):
 def process_trees(wmap):
     for split in ['train', 'dev', 'test']:
         trees = loadTrees(dataSet=split, wmap=wmap)
-        print len(trees)
-        cPickle.dump(trees, open('../data/sentiment/' + split + '_alltrees', 'wb'), protocol=cPickle.HIGHEST_PROTOCOL)
+        print(len(trees))
+        cPickle.dump(trees, open('../data/sentiment/' + split + '_alltrees', 'wb'))
 
 
 def acquire_all_phrases(tree, phrases):
@@ -163,16 +163,16 @@ def acquire_all_phrases(tree, phrases):
 if __name__=='__main__':
 
     wmap = buildWordMap()
-    print 'num words: ', len(wmap)
+    print('num words: ', len(wmap))
     process_trees(wmap)
     train = cPickle.load(open('../data/sentiment/train_alltrees', 'rb'))
     dev = cPickle.load(open('../data/sentiment/dev_alltrees', 'rb'))
     test = cPickle.load(open('../data/sentiment/test_alltrees', 'rb'))
 
     revMap = {}
-    for k, v in wmap.iteritems():
+    for k, v in wmap.items():
         revMap[v] = k
-    print len(train), len(dev), len(test)
+    print(len(train), len(dev), len(test))
 
     # store train root labels
     t_sents = []
@@ -181,12 +181,12 @@ def acquire_all_phrases(tree, phrases):
         leftTraverse(tree.root,nodeFn=words_to_list,args=[sent,revMap])
         t_sents.append([sent, tree.root.label + 1])
 
-    print [revMap[x] for x in t_sents[0][0]]
-    print 'num train instances ', len(t_sents)
+    print([revMap[x] for x in t_sents[0][0]])
+    print('num train instances ', len(t_sents))
     c = Counter()
     for sent, label in t_sents:
         c[label] += 1
-    print c
+    print(c)
     cPickle.dump(t_sents, open('../data/sentiment/train-rootfine', 'wb'))
 
     # store both phrases and roots for dev / test
@@ -196,12 +196,12 @@ def acquire_all_phrases(tree, phrases):
         leftTraverse(tree.root,nodeFn=words_to_list,args=[sent,revMap])
         dev_sents.append([sent, tree.root.label + 1])
 
-    print [revMap[x] for x in dev_sents[0][0]]
-    print 'dev phrase length ', len(dev_sents)
+    print([revMap[x] for x in dev_sents[0][0]])
+    print('dev phrase length ', len(dev_sents))
     c = Counter()
     for sent, label in dev_sents:
         c[label] += 1
-    print c
+    print(c)
     cPickle.dump(dev_sents, open('../data/sentiment/dev-rootfine', 'wb'))
 
     test_sents = []
@@ -210,10 +210,10 @@ def acquire_all_phrases(tree, phrases):
         leftTraverse(tree.root,nodeFn=words_to_list,args=[sent,revMap])
         test_sents.append([sent, tree.root.label + 1])
 
-    print [revMap[x] for x in test_sents[0][0]]
-    print 'test phrase length ', len(test_sents)
+    print([revMap[x] for x in test_sents[0][0]])
+    print('test phrase length ', len(test_sents))
     c = Counter()
     for sent, label in test_sents:
         c[label] += 1
-    print c
+    print(c)
     cPickle.dump(test_sents, open('../data/sentiment/test-rootfine', 'wb'))

util/sentiment_util.py

@@ -1,5 +1,5 @@
 from numpy import *
-import cPickle
+import _pickle as cPickle
 
 def unroll_params(arr, d, dh, len_voc, deep=1, labels=2, wv=True):