pushing scripts and data

Author: code-geek

gender_predictor.py

@@ -0,0 +1,94 @@
+from nltk import NaiveBayesClassifier, classify
+import name_loader
+import random
+
+
+class genderPredictor():
+
+    def getFeatures(self):
+        maleNames, femaleNames = self._loadNames()
+
+        featureset = list()
+
+        for nameTuple in maleNames:
+            features = self._nameFeatures(nameTuple[0])
+            male_prob, female_prob = self._getProbDistr(nameTuple)
+            features['male_prob'] = male_prob
+            features['female_prob'] = female_prob
+            featureset.append((features, 'M'))
+
+        for nameTuple in femaleNames:
+            features = self._nameFeatures(nameTuple[0])
+            male_prob, female_prob = self._getProbDistr(nameTuple)
+            features['male_prob'] = male_prob
+            features['female_prob'] = female_prob
+            featureset.append((features, 'F'))
+
+        return featureset
+
+    def trainAndTest(self, trainingPercent=0.80):
+        featureset = self.getFeatures()
+        random.shuffle(featureset)
+
+        name_count = len(featureset)
+
+        cut_point = int(name_count * trainingPercent)
+
+        train_set = featureset[:cut_point]
+        test_set = featureset[cut_point:]
+
+        self.train(train_set)
+
+        return self.test(test_set)
+
+    def classify(self, name):
+        feats = self._nameFeatures(name)
+        return self.classifier.classify(feats)
+
+    def train(self, train_set):
+        self.classifier = NaiveBayesClassifier.train(train_set)
+        return self.classifier
+
+    def test(self, test_set):
+        return classify.accuracy(self.classifier, test_set)
+
+    def _getProbDistr(self, nameTuple):
+        male_prob = (nameTuple[1] * 1.0) / (nameTuple[1] + nameTuple[2])
+        if male_prob == 1.0:
+            male_prob = 0.99
+        elif male_prob == 0.0:
+            male_prob = 0.01
+        else:
+            pass
+        female_prob = 1.0 - male_prob
+        return (male_prob, female_prob)
+
+    def getMostInformativeFeatures(self, n=5):
+        return self.classifier.most_informative_features(n)
+
+    def _loadNames(self):
+        return name_loader.getNameList()
+
+    def _nameFeatures(self, name):
+        name = name.upper()
+        return {
+            'last_letter': name[-1],
+            'last_two': name[-2:],
+            'last_three': name[-3:],
+            'last_is_vowel': (name[-1] in 'AEIOUY')
+        }
+
+
+if __name__ == "__main__":
+    gp = genderPredictor()
+    accuracy = gp.trainAndTest()
+    print ('Accuracy: %f' % accuracy)
+    print ('Most Informative Features')
+    feats = gp.getMostInformativeFeatures(10)
+    for feat in feats:
+        print ('\t%s = %s' % feat)
+    name = ''
+    while name != 'quit':
+        name = input('Enter name to classify: ')
+        name = name.strip()
+        print ('\n%s is classified as %s' % (name, gp.classify(name)))

name_loader.py

@@ -0,0 +1,66 @@
+import os
+import csv
+import pickle
+
+
+def getNameList():
+    if not os.path.exists('nepali_names.pickle'):
+        print('nepali_names.pickle does not exist, generating')
+
+        print('Extracting names from nepali_names.csv')
+        namesDict = extractNamesDict()
+
+        maleNames = list()
+        femaleNames = list()
+
+        print('Sorting Names')
+        for name in namesDict:
+            counts = namesDict[name]
+            tuple = (name, counts[0], counts[1])
+            if counts[0] > counts[1]:
+                maleNames.append(tuple)
+            elif counts[1] > counts[0]:
+                femaleNames.append(tuple)
+
+        names = (maleNames, femaleNames)
+
+        print('Saving nepali_names.pickle')
+        fw = open('nepali_names.pickle', 'wb')
+        pickle.dump(names, fw, -1)
+        fw.close()
+        print('Saved nepali_names.pickle')
+    else:
+        print('nepali_names.pickle exists, loading data')
+        f = open('nepali_names.pickle', 'rb')
+        names = pickle.load(f)
+        print('nepali_names.pickle loaded')
+
+    print('%d male names loaded, %d female names loaded' % (len(names[0]), len(names[1])))
+
+    return names
+
+
+def extractNamesDict():
+    names = dict()
+    genderMap = {'M': 0, 'F': 1}
+
+    file = open('nepali_names.csv', 'r')
+    rows = csv.reader(file, delimiter=',')
+
+    for row in rows:
+        name = row[0].upper()
+        gender = genderMap[row[1]]
+        count = int(row[2])
+
+        if name not in names:
+            names[name] = [0, 0]
+        names[name][gender] = names[name][gender] + count
+
+    file.close()
+    print('\tImported Nepali names file')
+
+    return names
+
+
+if __name__ == "__main__":
+    getNameList()

nepali_names.pickle

@@ -0,0 +1,42 @@
+gokaran,M,10
+gokaran prasad,M,1
+gokaran purus,M,1
+gokaran raj,M,2
+gokarana,M,3
+gokarda,M,1
+gokarn,M,40
+gokarn bahadur,M,8
+gokarn gairi,M,1
+gokarn jang,M,1
+gokarn khan,M,1
+gokarn lal,M,1
+gokarn maya,F,1
+gokarn prasad,M,7
+gokarn raj,M,1
+gokarna,F,4
+gokarna,M,783
+gokarna bahadhur,M,4
+gokarna bahadue,M,1
+gokarna bahadur,M,121
+gokarna bahadur aale,M,1
+gokarna bhadhur,M,3
+gokarna bhadur,M,1
+tejakumari,M,1
+tejal,F,2
+tejal maya,F,1
+tejal yata,F,1
+tejamai,F,1
+tejan,M,4
+tejanath,M,1
+tejanatha,M,1
+tejandar,M,1
+tejandra,M,2
+tejap,M,1
+tejar,M,1
+tejas,M,6
+tejasakar,M,1
+tejasbhi,F,1
+tejasbi,F,6
+tejasbi,M,1
+tejash,M,6
+tejashi,F,1