2.0.7

README.md

@@ -1,4 +1,4 @@
-# dkube-examples
+## dkube-examples
 
 This repository contains the DL examples ported to run on Dkube and showcase the features of Dkube platform.
 
@@ -35,3 +35,4 @@ Following examples are provided,
 ## Pytorch Examples
 
 **Will be added soon**
+

tensorflow/classification/mnist/digits/classifier/program/model.py

@@ -27,10 +27,8 @@
 
 FLAGS = None
 TF_TRAIN_STEPS = int(os.getenv('STEPS',1000))
-MODEL_DIR = os.getenv('DKUBE_JOB_OUTPUT_S3', None)
-DATA_DIR = os.getenv('DKUBE_INPUT_DATASETS', None)
-if DATA_DIR is not None:
-    DATA_DIR = DATA_DIR.split(",")[0]
+MODEL_DIR = "/opt/dkube/output"
+DATA_DIR = "/opt/dkube/input"
 BATCH_SIZE = int(os.getenv('BATCHSIZE', 10))
 EPOCHS = int(os.getenv('EPOCHS', 1))
 TF_MODEL_DIR = MODEL_DIR
@@ -40,13 +38,13 @@
 print ("TF_CONFIG: {}".format(os.getenv("TF_CONFIG", '{}')))
 
 def count_epochs(iterator):
-    cluster_spec = json.loads(os.getenv('TF_CONFIG',None))
-    role = cluster_spec['task']
-    host = cluster_spec['cluster'][role['type']][role['index']]
-    if len(cluster_spec['cluster'].keys()) > 1:
-     sess = tf.Session('grpc://'+ host)
-    else:
-     sess = tf.Session()
+    sess = tf.Session()
+    if os.getenv('TF_CONFIG') is not None:
+        cluster_spec = json.loads(os.getenv('TF_CONFIG',None))
+        role = cluster_spec['task']
+        host = cluster_spec['cluster'][role['type']][role['index']]
+        if len(cluster_spec['cluster'].keys()) > 1:
+            sess = tf.Session('grpc://'+ host)
     global steps_epoch
     if not steps_epoch:
         while True:
@@ -153,20 +151,21 @@ def model_fn(features, labels, mode, params):
     logging_hook = logger_hook({"loss": loss, "accuracy":accuracy[1] ,
         "step" : tf.train.get_or_create_global_step(), "steps_epoch": steps_epoch, "mode":"eval"}, every_n_iter=summary_interval)
     return tf.estimator.EstimatorSpec(
-        	mode=tf.estimator.ModeKeys.EVAL,
-        	loss=loss,
-        	eval_metric_ops={
-            	'accuracy':
-                	tf.metrics.accuracy(
-                    	labels=tf.argmax(labels, axis=1),
-                    	predictions=tf.argmax(logits, axis=1)),
-        	},
+                mode=tf.estimator.ModeKeys.EVAL,
+                loss=loss,
+                eval_metric_ops={
+                    'accuracy':
+                        tf.metrics.accuracy(
+                        labels=tf.argmax(labels, axis=1),
+                        predictions=tf.argmax(logits, axis=1)),
+                },
                 evaluation_hooks = [logging_hook])
 
 def main(unused_argv):
   try:
       fp = open(os.getenv('DKUBE_JOB_HP_TUNING_INFO_FILE', 'None'),'r')
       hyperparams = json.loads(fp.read())
+      hyperparams['num_epochs'] = EPOCHS
   except:
       hyperparams = { "learning_rate":1e-4, "batch_size":BATCH_SIZE, "num_epochs":EPOCHS }
       pass
@@ -233,7 +232,7 @@ def eval_input_fn():
                                       throttle_secs=1,
                                       start_delay_secs=1)
   tf.estimator.train_and_evaluate(mnist_classifier, train_spec, eval_spec)
-  if os.getenv('TF_CONFIG') != '':
+  if os.getenv('TF_CONFIG', '') != '':
         config = json.loads(os.getenv('TF_CONFIG'))
         if config['task']['type'] == 'master':
             mnist_classifier.export_savedmodel(MODEL_DIR, export_fn)
@@ -263,4 +262,6 @@ def run():
   tf.app.run(main=main)
 
 if __name__ == '__main__':
+    if os.getenv("STEPS") is None:
+        os.environ['STEPS'] = str(TF_TRAIN_STEPS)
     run()

tensorflow/classification/mnist/digits/hptuning/tuning.json

@@ -1,40 +1,31 @@
 {
-  "RequestCount": 1,
-  "OptimizationType": "maximize",
-  "OptimizationGoal": 0.99,
-  "ObjectiveValueName": "train_accuracy_1",
-  "MetricsNames": [
-   "train_accuracy_1"
-  ],
-  "ParameterConfigs": [
-   {
-    "name": "--learning_rate",
-    "parametertype": "double",
-    "feasible": {
-     "max": "0.05",
-     "min": "0.01"
+  "parallelTrialCount": 3,
+  "maxTrialCount": 6,
+  "maxFailedTrialCount": 3,
+  "objective": {
+    "type": "maximize",
+    "goal": 0.99,
+    "objectiveMetricName": "accuracy"
+  },
+  "algorithm": {
+    "algorithmName": "random"
+  },
+  "parameters": [
+    {
+      "name": "--learning_rate",
+      "parameterType": "double",
+      "feasibleSpace": {
+        "min": "0.01",
+        "max": "0.05"
+      }
+    },
+    {
+      "name": "--batch_size",
+      "parameterType": "int",
+      "feasibleSpace": {
+        "min": "100",
+        "max": "200"
+      }
     }
-   },
-   {
-    "name": "--batch_size",
-    "parametertype": "int",
-    "feasible": {
-     "max": "200",
-     "min": "100"
-    }
-   },
-   {
-    "name": "--num_epochs",
-    "parametertype": "int",
-    "feasible": {
-     "max": "10",
-     "min": "1"
-    }
-   }
-  ],
-  "SuggestionSpec": {
-   "requestNumber": 3,
-   "suggestionAlgorithm": "random"
-  }
+  ]
 }
-

tensorflow/classification/mnist/digits/hptuning/tuning.yaml

@@ -1,25 +1,21 @@
-optimizationtype: maximize
-objectivevaluename: train_accuracy_1
-optimizationgoal: 0.99
-requestcount: 1
-metricsnames:
- - train_accuracy_1
-parameterconfigs:
- - name: --learning_rate
-   parametertype: double
-   feasible:
-     min: "0.01"
-     max: "0.05"
- - name: --batch_size
-   parametertype: int
-   feasible:
-     min: "100"
-     max: "200"
- - name: --num_epochs
-   parametertype: int
-   feasible:
-     min: "1"
-     max: "10"
-suggestionSpec:
-    suggestionAlgorithm: "random"
-    requestNumber: 3
+parallelTrialCount: 3
+maxTrialCount: 6
+maxFailedTrialCount: 3
+objective:
+  type: maximize
+  goal: 0.99
+  objectiveMetricName: accuracy
+algorithm:
+  algorithmName: random
+parameters:
+  - name: --learning_rate
+    parameterType: double
+    feasibleSpace:
+      min: "0.01"
+      max: "0.05"
+  - name: --batch_size
+    parameterType: int
+    feasibleSpace:
+      min: "100"
+      max: "200"
+

tensorflow/classification/mnist/digits/pipeline/digits.ipynb

@@ -108,30 +108,42 @@
     "def d3pipeline(\n",
     "    #In notebook DKUBE_USER_ACCESS_TOKEN is automatically picked up from env variable\n",
     "    auth_token  = os.getenv(\"DKUBE_USER_ACCESS_TOKEN\"),\n",
-    "    #By default tf v1.12 image is used here, v1.10, v1.11 or v1.13 can be used. \n",
+    "    #By default tf v1.14 image is used here, v1.13 or v1.14 can be used. \n",
     "    #Or any other custom image name can be supplied.\n",
     "    #For custom private images, please input username/password\n",
-    "    training_container=json.dumps({'image':'docker.io/ocdr/dkube-datascience-tf-gpu:v1.12', 'username':'', 'password': ''}),\n",
+    "    training_container=json.dumps({'image':'docker.io/ocdr/d3-datascience-tf-cpu:v1.14', 'username':'', 'password': ''}),\n",
     "    #Name of the workspace in dkube. Update accordingly if different name is used while creating a workspace in dkube.\n",
     "    training_program=\"mnist\",\n",
     "    #Script to run inside the training container    \n",
     "    training_script=\"python model.py\",\n",
+    "    tuning = '{\"parallelTrialCount\":2,\"maxTrialCount\":4,\"maxFailedTrialCount\":2,\"objective\":{\"type\":\"maximize\",\"goal\":0.99,\"objectiveMetricName\":\"accuracy\"},\"algorithm\":{\"algorithmName\":\"random\"},\"parameters\":[{\"name\":\"--learning_rate\",\"parameterType\":\"double\",\"feasibleSpace\":{\"min\":\"0.01\",\"max\":\"0.05\"}},{\"name\":\"--batch_size\",\"parameterType\":\"int\",\"feasibleSpace\":{\"min\":\"100\",\"max\":\"200\"}}]}',\n",
     "    #Input datasets for training. Update accordingly if different name is used while creating dataset in dkube.    \n",
     "    training_datasets=json.dumps([\"mnist\"]),\n",
+    "    #Input dataset mount paths\n",
+    "    training_input_dataset_mounts=json.dumps([\"/opt/dkube/input\"]),\n",
+    "    #Output models for training.\n",
+    "    training_outputs=json.dumps([\"mnist\"]),\n",
+    "    #Output dataset mount paths\n",
+    "    training_output_mounts=json.dumps([\"/opt/dkube/output\"]),\n",
     "    #Request gpus as needed. Val 0 means no gpu, then training_container=docker.io/ocdr/dkube-datascience-tf-cpu:v1.12    \n",
-    "    training_gpus=1,\n",
+    "    training_gpus=0,\n",
     "    #Any envs to be passed to the training program    \n",
     "    training_envs=json.dumps([{\"steps\": 100}]),\n",
     "    #Device to be used for serving - dkube mnist example trained on gpu needs gpu for serving else set this param to 'cpu'\n",
-    "    serving_device='gpu'):\n",
+    "    serving_device='cpu',\n",
+    "    serving_container=json.dumps({'image':'docker.io/ocdr/mnist-example-preprocess:2.0.4', 'username':'', 'password': ''})\n",
+    "    ):\n",
     "\n",
     "    train       = dkube_training_op(auth_token, training_container,\n",
     "                                    program=training_program, run_script=training_script,\n",
-    "                                    datasets=training_datasets, ngpus=training_gpus,\n",
-    "                                    envs=training_envs)\n",
-    "    serving     = dkube_serving_op(auth_token, train.outputs['artifact'], device=serving_device).after(train)\n",
-    "    inference   = dkube_viewer_op(auth_token, serving.outputs['servingurl'],\n",
-    "                                  'digits', viewtype='inference').after(serving)"
+    "                                    datasets=training_datasets, outputs=training_outputs,\n",
+    "                                    input_dataset_mounts=training_input_dataset_mounts,\n",
+    "                                    output_mounts=training_output_mounts,\n",
+    "                                    ngpus=training_gpus,\n",
+    "                                    envs=training_envs, tuning=tuning)\n",
+    "    serving     = dkube_serving_op(auth_token, train.outputs['artifact'], device=serving_device, serving_container=serving_container).after(train)\n",
+    "    #inference   = dkube_viewer_op(auth_token, serving.outputs['servingurl'],\n",
+    "                                 #'digits', viewtype='inference').after(serving)"
    ]
   },
   {
@@ -173,6 +185,13 @@
    "source": [
     "run = client.run_pipeline(mnist_experiment.id, 'mnist_classifier_pipeline', 'dkube_mnist_pl.tar.gz', params={})"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -191,7 +210,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.3"
+   "version": "3.7.6"
   }
  },
  "nbformat": 4,

tensorflow/classification/resnet/catsdogs/classifier/program/model.py

@@ -16,11 +16,9 @@
     tf.logging.info('TF_CONFIG: {}'.format(os.environ["TF_CONFIG"]))
 
 FLAGS = None
-DATA_DIR = os.getenv('DKUBE_INPUT_DATASETS', None)
-if DATA_DIR is not None:
-    DATA_DIR = DATA_DIR.split(",")[0]
-MODEL_DIR = os.getenv('DKUBE_JOB_OUTPUT_S3', None)
-TFHUB_CACHE_DIR = os.getenv('TFHUB_CACHE_DIR',None)
+DATA_DIR = "/opt/dkube/input"
+MODEL_DIR = "/opt/dkube/output"
+TFHUB_CACHE_DIR = os.getenv('TFHUB_CACHE_DIR', "/opt/dkube/input")
 BATCH_SIZE = int(os.getenv('BATCHSIZE', 10))
 EPOCHS = int(os.getenv('EPOCHS', 1))
 TF_TRAIN_STEPS = int(os.getenv('STEPS',1000))
@@ -32,6 +30,8 @@
 #    os.makedirs(MODEL_DIR)
 
 def count_epochs(iterator):
+    if os.getenv('TF_CONFIG', None) == None:
+        return
     cluster_spec = json.loads(os.getenv('TF_CONFIG',None))
     role = cluster_spec['task']
     host = cluster_spec['cluster'][role['type']][role['index']]
@@ -127,6 +127,7 @@ def train(_):
     try:
       fp = open(os.getenv('DKUBE_JOB_HP_TUNING_INFO_FILE', 'None'),'r')
       hyperparams = json.loads(fp.read())
+      hyperparams['num_epochs'] = EPOCHS
     except:
       hyperparams = { "learning_rate":1e-3, "batch_size":BATCH_SIZE, "num_epochs":EPOCHS }
       pass
@@ -158,11 +159,13 @@ def train(_):
     }
     global TFHUB_CACHE_DIR
     if TFHUB_CACHE_DIR != None:
+        EXTRACT_PATH = "/tmp/tfhub-cache-dir"
         files = [os.path.join(TFHUB_CACHE_DIR, f) for f in tf.gfile.ListDirectory(TFHUB_CACHE_DIR) if f.endswith('tar.gz')]
         for fname in files:
             tar = tarfile.open(fname, "r:gz")
-            tar.extractall(TFHUB_CACHE_DIR)
+            tar.extractall(EXTRACT_PATH)
             tar.close()
+            TFHUB_CACHE_DIR = EXTRACT_PATH
     else:
         TFHUB_CACHE_DIR = params['module_spec']
 
@@ -195,7 +198,7 @@ def serving_input_receiver_fn():
         fn = lambda image: _img_string_to_tensor(image, input_img_size)
         features['inputs'] = tf.map_fn(fn, features['inputs'], dtype=tf.float32)
         return tf.estimator.export.ServingInputReceiver(features, received_tensors)
-    if os.getenv('TF_CONFIG') != '':
+    if os.getenv('TF_CONFIG', '') != '':
         config = json.loads(os.getenv('TF_CONFIG'))
         if config['task']['type'] == 'master':
             classifier.export_savedmodel(MODEL_DIR, serving_input_receiver_fn)
@@ -214,4 +217,6 @@ def run():
     tf.app.run(main=train)
 
 if __name__ == '__main__':
+    if os.getenv("STEPS") is None:
+        os.environ['STEPS'] = str(TF_TRAIN_STEPS)
     run()