Done HW6 for group 4

classification/README

@@ -0,0 +1,31 @@
+������������� ��������� �� ������ SVM ��� ������ ���������� ������ ������������� VK
+
+- prediction_friends.py
+���� ����������� ������ ��������� �� ������ SVM
+��������� ��� �������: 
+
+python prediction_friends.py [ --learn_path <train_file_base> --test_path <predict_file_base> | --cv <file_base> ]
+
+��� 
+<train_file_base> - ���� ���������� �������� ���������� ������������� � ��������� �������:
+UID,Name,Surname,Sex,Age,Audios,Photos,InterestPages,Friends
+� �������� ������� ����� ������������ ���� vk_base.csv
+
+<predict_file_base> - ���� ���������� �������� ���������� ������������� � ��������� �������:
+UID,Name,Surname,Sex,Age,Audios,Photos,InterestPages,Friends, ������ � ���� Friends ����� ���� �������
+�������� 0 ��� None
+� �������� ������� ����� ������������ ���� vk_base.csv
+
+<file_base> - ���� ���������� �������� ���������� ������������� � ��������� �������:
+UID,Name,Surname,Sex,Age,Audios,Photos,InterestPages,Friends
+� �������� ������� ����� ������������ ���� vk_base.csv
+
+������� ��������:
+python prediction_friends.py - ������� ��������� ������������ �� ���������� �������� � ����� vk_base.csv
+
+python prediction_friends.py --cv <file_base> - ������� ��������� ������������ �� ���������� �������� � ����� file_base
+
+python prediction_friends.py --learn_path <train_file_base> --test_path <predict_file_base> - ������� ������������
+��� ����� predict_file_base
+
+

classification/api.py

@@ -1,23 +1,23 @@
-import requests
-
-__author__ = 'Nikolay Anokhin'
-
-
-class Api(object):
-
-    endpoint = ''
-
-    def __init__(self, access_token):
-        self.access_token = access_token
-
-    def call(self, method, **params):
-        request_params = params.copy()
-        request_params["access_token"] = self.access_token
-        try:
-            response = requests.get(self.endpoint.format(method=method), params=request_params)
-            if response.status_code != 200:
-                raise requests.exceptions.RequestException("Bad status code {}".format(response.status_code))
-            return response.json()
-        except requests.exceptions.RequestException as re:
-            print "A n API call failed with exception {}".format(re)
-            raise
+import requests
+
+__author__ = 'Nikolay Anokhin'
+
+
+class Api(object):
+    endpoint = ''
+
+    def __init__(self, access_token):
+        self.access_token = access_token
+
+    def call(self, method, **params):
+        request_params = params.copy()
+        if (len(self.access_token) > 0):
+            request_params["access_token"] = self.access_token
+        try:
+            response = requests.get(self.endpoint.format(method=method), params=request_params, timeout=3.0)
+            if response.status_code != 200:
+                raise requests.exceptions.RequestException("Bad status code {}".format(response.status_code))
+            return response.json()
+        except requests.exceptions.RequestException as re:
+            print "A n API call failed with exception {}".format(re)
+            return None

classification/prediction_friends.py

@@ -0,0 +1,152 @@
+import sklearn
+
+
+
+__author__ = 'Vladimir Kulpinov'
+
+import pickle
+import numpy
+import csv
+import user
+from sklearn import svm
+from sklearn import cross_validation
+import random
+import argparse
+
+
+class FriendsPredictor(object):
+    """
+    Model used to predict age of users in VK
+    """
+    # Initializer of training model
+    def __init__(self):
+        numpy.random.RandomState(1)
+        self.regressor = svm.SVR(C = 1000, kernel="rbf", max_iter=1000, gamma=0.6)
+
+    # Predict friends of list user
+    # return list of ages
+    def predict(self, x=[]):
+        return self.regressor.predict(x)
+
+    # Train of FriendPredictor with list of users
+    # and param_string
+    def fit(self, x=[], y=[]):
+        self.regressor.fit(x, y)
+        pass
+
+    # Save AgePredictor model in file with name filename
+    def save(self, filename=None):
+        if filename is None:
+            filename = "predict_model.dat"
+        with open(filename, 'wb') as f:
+            pickle.dump(self, f)
+
+    @staticmethod
+    def load(filename):
+        data = None
+        try:
+            with open(filename, 'rb') as f:
+                data = pickle.load(f)
+        except IOError:
+            print "File {} not found".format(filename)
+            exit(0)
+        return data
+
+def error_estimation(ages1, ages2):
+    """
+     Estimation of error of predict. Using MSE
+    """
+    if len(ages1) == 0 or len(ages2) != len(ages1):
+        return -1
+    else:
+        return (sum(map(lambda x, y: abs(x - y) ** 2, ages1, ages2)) / float(len(ages1))) ** 0.5
+
+def kfold_estimator(database_file = 'vk_base.csv'):
+    """
+    Function used to find best parameters for model
+    with CV 80%/20%
+    Input:
+        database_file : csv file with users information
+    """
+    users = []
+    with open(database_file, 'rb') as csvfile:
+        vk_base_reader = csv.reader(csvfile)
+        was_header = False
+        for row in vk_base_reader:
+            if (not was_header):
+                was_header = True
+            else:
+                u = user.User()
+                u.from_tsv(row)
+                users.append(u)
+                print u.to_tsv()
+
+
+    print "Database is read-out"
+
+    model = FriendsPredictor()
+
+    numpy.random.RandomState(1)
+    random.seed(1)
+    random.shuffle(users)
+    kf = cross_validation.KFold(len(users), n_folds=5, indices=True)
+    errors = []
+    for train, test in kf:
+        train_x = [users[i] for i in train]
+        predict_x = [users[i] for i in test]
+        labels = [int(users[i].count_friends) for i in test]
+        model.fit(train_x)
+        cur_err =  error_estimation(labels, model.predict(predict_x).tolist())
+        print cur_err
+        errors.append(cur_err)
+
+    estimation = sum(errors) / len(errors)
+    print "mean value: {}".format(estimation)
+    return estimation
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Linear regression method of gradient descent\nFinding number of friends')
+    parser.add_argument('learn_path', nargs=1)
+    parser.add_argument('test_path', nargs=1)
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+    users = []
+    with open(args.learn_path[0], 'rb') as csvfile:
+        vk_base_reader = csv.reader(csvfile)
+        was_header = False
+        for row in vk_base_reader:
+            if (not was_header):
+                was_header = True
+            else:
+                u = user.User()
+                u.from_tsv(row)
+                users.append(u)
+                #print u.to_tsv()
+
+    model = FriendsPredictor()
+    x = [u.get_predictable() for u in users]
+    y = [u.count_friends for u in users]
+    model.fit(x, y)
+    users = []
+
+    with open(args.test_path[0], 'rb') as csvfile:
+        vk_base_reader = csv.reader(csvfile)
+        was_header = False
+        for row in vk_base_reader:
+            if (not was_header):
+                was_header = True
+            else:
+                u = user.User()
+                u.from_tsv(row)
+                users.append(u)
+                #print u.to_tsv()
+
+    x = [u.get_predictable() for u in users]
+    answer = model.predict(x)
+    for i in answer:
+        print i
+
+if __name__ == "__main__":
+    main()

classification/user.py

@@ -1,24 +1,68 @@
-from datetime import date
-
-
-class User(object):
-
-    def __init__(self, uid, first_name, last_name):
-        self.uid = uid
-        self.first_name = first_name
-        self.last_name = last_name
-        self.sex = None
-        self.age = None
-
-    def set_age(self, birth_date):
-        if birth_date:
-            self.age = int((date.today() - birth_date).days / 365.2425)
-
-    def to_tsv(self):
-        return u'\t'.join([unicode(self.uid), unicode(self.first_name), unicode(self.last_name), unicode(self.sex), unicode(self.age)])
-
-    def __str__(self):
-        return unicode(self).encode('utf-8')
-
-    def __unicode__(self):
-        return u"User('{uid}', '{first}', '{last}')".format(uid=self.uid, first=self.first_name, last=self.last_name)
+from datetime import date
+
+
+class User(object):
+    def __init__(self, uid=None, first_name=None, last_name=None):
+        self.uid = uid
+        self.first_name = first_name
+        self.last_name = last_name
+        self.sex = 0
+        self.age = None
+        self.grad_university = None
+        self.relation = 0
+        self.is_exist_status = False
+        self.count_friends = 0
+        self.count_photos = 0
+        self.count_audios = 0
+        self.count_groups = 0
+        self.count_pages = 0
+
+    def set_age(self, birth_date):
+        if birth_date:
+            self.age = int((date.today() - birth_date).days / 365.2425)
+
+    def is_relevant(self):
+        return self.sex and self.sex > 0 and self.count_friends > 0 and self.count_friends < 500
+
+    def to_tsv(self, obfuscate=False):
+        if not obfuscate:
+            return '\t'.join([unicode(self.uid), unicode(self.first_name), unicode(self.last_name),
+                           unicode(self.sex), unicode(self.age), unicode(self.count_audios),
+                           unicode(self.count_photos), unicode(self.count_pages),
+                           unicode(self.count_friends)])
+        else:
+            return '\t'.join([unicode(1), unicode('Vasya'), unicode('Vasya'),
+                           unicode(self.sex), unicode(self.age), unicode(self.count_audios),
+                           unicode(self.count_photos), unicode(self.count_pages),
+                           unicode(self.count_friends)])
+
+
+    def from_tsv(self, row):
+        try:
+            self.uid = row[0]
+            self.first_name = row[1].decode('utf-8')
+            self.last_name = row[2].decode('utf-8')
+            self.sex = row[3]
+            self.age = row[4]
+            self.count_audios = row[5]
+            self.count_photos = row[6]
+            self.count_pages = row[7]
+            #self.count_groups = row[8]
+            self.count_friends = row[8]
+        except:
+            print "Error while reading user from row"
+            raise BaseException
+
+    def get_predictable(self):
+        result = []
+        result.append(int(self.sex))
+        result.append(int(self.count_photos))
+        result.append(int(self.count_pages))
+        result.append(int(self.count_audios))
+        return result
+
+    def __str__(self):
+        return unicode(self).encode('utf-8')
+
+    def __unicode__(self):
+        return u"User('{uid}', '{first}', '{last}')".format(uid=self.uid, first=self.first_name, last=self.last_name)