text2index.py

#!/usr/bin/env python      
                                                                                                                                         
from scipy import *
import itertools
from optparse import OptionParser
import sys
import numpy as np
import pickle
                                                                                                            
parser = OptionParser()                                          
parser.add_option("-v", "--verbosity", action="store_true", dest="verbosity", help="Verbose?")


                                                                                                          
(options, args) = parser.parse_args()
if (len(args)<2):
        print "usage: -options input_filename vector_datafile output_filename"
        print options
	sys.exit(2)

inputFilename = args[0]
inputVocab = args[1]
outFilename = args[2]

print "input file", inputFilename
print "vocabulary file (for instance, generated by SRILM)", inputVocab
print "output file", outFilename


vocab_size = -1

print "reading vocabulary file"
inputVocablines = file(inputVocab).readlines()
print "Vocabulary size:",len(inputVocablines)
print "reading file data"
inputFile = file(inputFilename).readlines()
print "Corpus size:",len(inputFile)
outputFile = open(outFilename+'.dic', "wb" )

print "Building vocabulary dictionary"
i = 0
words_dict = {}

for line in inputVocablines:
  words_dict[line.split("\n")[0]] = i+1
  i+=1

words_dict['<unk>'] = 0
vocab_size=len(inputVocablines)
corpus_size=len(inputFile)
print "Building input data"
s_index=1
corpus = np.array([],dtype=int32)

for line in inputFile :
    s_line = line[:-1].split()
    sentence = np.empty([len(s_line)],dtype=int32)
    w_index = 0 #We begin at the start of line
    for w in s_line :
        if w in words_dict.keys() :
            index = words_dict[w]
        else : 
            index = words_dict['<unk>']
        sentence[w_index] = index
        w_index+=1
    
    corpus = np.append(corpus,sentence)
    
    s_index+=1
    if s_index % 1000 == 0 :
        print "Processed line", s_index,"out of", corpus_size


np.save(outFilename,corpus)
pickle.dump(words_dict, outputFile)
outputFile.close()