Integrate hab ml

.gitignore

@@ -1 +1,4 @@
-__pycache__/*
+__pycache__/*
+model/*
+DB/*
+.DS_Store

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "hab_ml"]
+	path = hab_ml
+	url = https://github.com/hab-spc/hab-ml.git

DB/labels.txt

@@ -0,0 +1,22 @@
+0	Acantharea
+1	Akashiwo
+2	Ceratium falcatiforme or fusus
+3	Ceratium furca
+4	Ceratium other
+5	Chaetoceros socialis
+6	Chattonella
+7	Ciliates
+8	Cochlodinium
+9	Dinophysis
+10	Eucampia
+11	Gyrodinium
+12	Lingulodinium polyedra
+13	Nauplii
+14	Polykrikos
+15	Prorocentrum micans
+16	Prorocentrum spp
+17	Pseudo-nitzschia chain
+18	Sand
+19	Straight diatom chains
+20	Thalassionema or Thalassiothrix chain
+21	detritus

DB/time_period.txt

@@ -0,0 +1 @@
+2019-05-30 09:58:00,2019-05-30 10:32:00,0.03,0.1,SPCP2

config/config.py

@@ -0,0 +1,111 @@
+""" configuration file to store constants
+"""
+from __future__ import absolute_import
+
+# Standard dist imports
+import os
+from pathlib import Path
+
+# Project level imports
+from constants.genericconstants import GenericConstants as CONST
+
+# Module level constants
+DEFAULT_ENV = CONST.DEV_ENV  # SVCL local environment
+# DEFAULT_ENV = CONST.PROD_ENV  # SPC Lab machine
+PROJECT_DIR = Path(__file__).resolve().parents[1]
+
+
+class Environment():
+    """Sets up Environment Variables, given the DEFAULT ENV
+
+    Set up all directory-related variables under here
+    """
+
+    def __init__(self, env_type=None):
+        """Initializes Environment()
+
+        Given the environment type (dev or prod), it sets up the model, data, and db direcotry related stuff. All variables need to be initialized with a string formatting so these are all RELATIVE PATHS
+        """
+        if env_type == CONST.DEV_ENV:
+            # Model and database items
+            root = '/data6/phytoplankton-db'
+            self.model_dir = '/data6/lekevin/hab-master/hab_rnd/hab-ml/experiments/hab_model_v1:20191023/'
+            self.data_dir = os.path.join(root, 'hab_in_situ/images', '{}')
+            self.meta_dir = os.path.join(root, 'csv')
+            self.hab_ml_main = os.path.join(PROJECT_DIR, 'hab_ml', '{}')
+            # SPC items
+            self.login_url = 'http://spc.ucsd.edu/data/admin/?next=/data/admin'
+
+        elif env_type == CONST.PROD_ENV:
+            self.model_dir = '/data6/plankton_test_db_new/model/20191023/00:51:01/'
+            self.data_dir = os.path.join(PROJECT_DIR, 'images', '{}')
+            self.hab_ml_main = os.path.join(PROJECT_DIR, 'hab_ml', '{}')
+
+
+class Config(Environment):
+    """Default Configs for training and inference
+    After initializing instance of Config, user can import configurations as a
+    state dictionary into other files. User can also add additional
+    configuration items by initializing them below.
+    Example for importing and using `opt`:
+    config.py
+        >> opt = Config()
+    main.py
+        >> from config import opt
+        >> lr = opt.lr
+
+    NOTE: all path related configurations should be set up in the Environment() class above to avoid issues with developing on a person vs production environment.
+
+    """
+    # SPC Submission dictionary
+    account_info = {'username': 'kevin',
+                    'password': 'ceratium'}
+    label_instance_name = 'hab_24'
+    tag = 'hab_24'
+    is_machine = True
+
+    summer2019_csv = 'DB/csv/hab_in_situ_summer2019.csv'
+
+    def __init__(self, env_type):
+        super().__init__(env_type)
+
+    def _parse(self, kwargs):
+        state_dict = self._state_dict()
+        for k, v in kwargs.items():
+            if k not in state_dict:
+                raise ValueError('UnKnown Option: "--%s"' % k)
+            setattr(self, k, v)
+
+        # print('======user config========')
+        # pprint(self._state_dict())
+        # print('==========end============')
+
+    def _state_dict(self):
+        """Return current configuration state
+        Allows user to view current state of the configurations
+        Example:
+        >>  from config import opt
+        >> print(opt._state_dict())
+        """
+        return {k: getattr(self, k) for k, _ in Config.__dict__.items() \
+                if not k.startswith('_')}
+
+
+def set_config(**kwargs):
+    """ Set configuration to train/test model
+    Able to set configurations dynamically without changing fixed value
+    within Config initialization. Keyword arguments in here will overwrite
+    preset configurations under `Config()`.
+    Example:
+    Below is an example for changing the print frequency of the loss and
+    accuracy logs.
+    >> opt = set_config(print_freq=50) # Default print_freq=10
+    >> ...
+    >> model, meter = train(trainer=music_trainer, data_loader=data_loader,
+                            print_freq=opt.print_freq) # PASSED HERE
+    """
+    opt._parse(kwargs)
+    return opt
+
+
+opt = Config(DEFAULT_ENV)

constants/genericconstants.py

@@ -0,0 +1,67 @@
+from collections import OrderedDict
+
+
+class GenericConstants:
+    RAW_DATA = 'raw'
+    PROCESSED_DATA = 'processed'
+    CURRENT_ENV = 'current_environment'
+    DEV_ENV = 'dev_mac'
+    PROD_ENV = 'prod_env'
+    LIVIS = 'livis'
+
+
+class SPCConstants:
+    LBL_SET_NAME = 'name'
+    TAG = 'tag'
+    IS_MCHN = 'is_machine'
+    MCHN_NAME = 'machine_name'
+
+
+class DBConstants:
+    date_table = 'date_sampled'
+
+    pre = 'image_'
+    # Image Info
+    IMG_FNAME = pre + 'filename'
+    IMG_ID = pre + 'id'
+    IMG_TSTAMP = pre + 'timestamp'
+    IMG_DATE = pre + 'date'
+    IMG_TIME = pre + 'time'
+    IMG_FSIZE = pre + 'file_size'
+    ECCENTRICITY = pre + 'eccentricity'
+    ORIENT = pre + 'orientation'
+    MJR_LEN = pre + 'major_axis_length'
+    MIN_LEN = pre + 'minor_axis_length'
+    HEIGHT = pre + 'height'
+    WIDTH = pre + 'width'
+    SOLIDITY = pre + 'solidity'
+    ASPT_RATIO = pre + 'aspect_ratio'
+    EST_VOL = pre + 'estimated_volume'
+    AREA = pre + 'area'
+
+    # Machine Learning Info
+    pre = 'ml_'
+    MODEL_NAME = pre + 'model_name'
+    USR_LBLS = pre + 'user_labels'
+    PRED = pre + 'prediction'
+    PROB = pre + 'probability'
+    PRED_TSTAMP = pre + 'prediction_timestamp'
+
+    # Annotation Info
+    pre = 'annot_'
+    IMG_STATUS = pre + 'image_status'
+    IMG_TAG = pre + 'image_tags'
+    ML_LBL = pre + 'machine_label'
+    HMN_LBL = pre + 'human_label'
+
+    def _state_dict(self, type='image'):
+        """Return current configuration state
+        Allows user to view current state of the configurations
+        Example:
+        >>  from genericconstants import DBConstants
+        >>  db = DBConstants()
+        >>  image_info_to_update = db._state_dict(type='image').values()
+        Out: odict_values(['image_filename', 'image_id', ..., 'image_area'])
+        """
+        return OrderedDict({k: getattr(self, k) for k, v in DBConstants().__dict__.items() \
+                            if not k.startswith('_') and v.startswith(type)})

helper.py

@@ -13,11 +13,17 @@
 class SPCDataTransformer():
     """Helper class"""
 
-    def __init__(self, data, logger=None):
-        assert isinstance(data, pd.DataFrame)
+    def __init__(self, data, csv_fname=None, classes=None, logger=None):
         # Initialize Logging
         self.logger = logger if logger else logging.getLogger('spcdata')
 
+        if csv_fname:
+            self.csv_fname = csv_fname
+            self.dataset = pd.read_csv(csv_fname)
+
+        if classes:
+            self.cls2idx, self.idx2cls = self.get_classes(classes)
+
         self.data = data
 
         # TODO accept this as parameter initialization. Need to keep constant
@@ -96,3 +102,50 @@ def _append_img_dir(self, data, image_dir, image_col='images'):
         df[image_col] = df['image_url'].apply(lambda x: os.path.join(
             image_dir, os.path.basename(x) + '.jpg'))
         return df
+
+    def get_predictions(self, pred_col='pred', verbose=False, decode=False, write=False):
+        """Get the prediction distribution"""
+        if self.idx2cls and decode:
+            self.dataset[pred_col] = self.dataset[pred_col].map(self.idx2cls)
+
+        if verbose:
+            print(self.dataset[pred_col].value_counts())
+        self.pred = self.dataset[pred_col].value_counts().to_dict()
+
+        if write:
+            # writes it in SPC formatting
+            with open('data/predictions.txt', 'w') as f:
+                for idx, row in self.dataset[['image_id', 'pred']].iterrows():
+                    f.write(row['image_id'] + ',' + str(row['pred']) + '\n')
+            f.close()
+            print('Finished writing')
+
+        return self.pred
+
+    def _export_labels(self):
+        pass
+
+    def get_classes(self, filename):
+        """Set class2idx, idx2class encoding/decoding dictionaries"""
+        class_list = SPCDataTransformer._parse_classes(filename)
+        cls2idx = {i: idx for idx, i in enumerate(sorted(class_list))}
+        idx2cls = {idx: i for idx, i in enumerate(sorted(class_list))}
+        return cls2idx, idx2cls
+
+    @staticmethod
+    def _parse_classes(filename):
+        """Parse MODE_data.info file"""
+        lbs_all_classes = []
+        with open(filename, 'r') as f:
+            label_counts = f.readlines()
+        label_counts = label_counts[:-1]
+        for i in label_counts:
+            class_counts = i.strip()
+            class_counts = class_counts.split()
+            class_name = ''
+            for j in class_counts:
+                if not j.isdigit():
+                    class_name += (' ' + j)
+            class_name = class_name.strip()
+            lbs_all_classes.append(class_name)
+        return lbs_all_classes