Run livis

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
-images/*
+images/*
+LIVIS_automate/sampleImgs_static_html/*

LIVIS_automate/deploy.sh

@@ -1,52 +1,75 @@
 #!/usr/bin/env bash
-# This is the set of commands to execute on the local machine for:
+# This is the set of commands to execute on the local lab machine for:
+# 0. Run image sampling on odroid virtual machine
 # 1. Transfer the .tif images from the local machine to a folder in SVCL server
-# 2. Convert images using spcconvert and deploy model
-# 3. Upload predictions to static html
+# 2. Convert images using spcconvert
+# 3. Deploy model
+# 4. Upload predictions to static html
+# 5. Retrieve predicted images back to lab computer
 
+# NOTE: Steps 2-4 occurs on the SVCL server, hence the `deploy_remote.sh`
+# Step 0
 # ****************************************************************************
 
-# Step 1
-# We transfer the images on the local machine to the SVCL Server using scp command
-# Change the paths accordingly:
-
-#-- prompt for subdirectory name to save data to on local machine
-echo "Enter new data directory name to store on remote machine, followed by [ENTER]: "
+echo "Enter date and sample run for data storage location, followed by [ENTER]:"
 read date
 
-#--
-#open -a coRD
-#open 'smb://spcuser:odroid@192.168.1.110'
-#echo Select `data` directory when mounting and proceed to run camera system
-
-# include into this script ./run_camera.sh upon command
-# put watchdog on the data directory for the latest one after running camera
-# upload those images to the server
-# once imaging is done, run classifier upon yes command
+# Ssh into camera virtual machine (odroid) and runs the imaging
+read -p "Run LIVIS Imaging? [y/n]: " -n 1 -r
+echo    # (optional) move to a new line
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+    user=odroid
+    host=192.168.1.110
+    echo Logging into odroid remote server
+    source_dir="LIVIS/LIVIS_automate"
+    ssh $user@$host "cd $source_dir;./run_livis.sh $date"
+fi
+# Step 1
+# ****************************************************************************
+echo Uploading images to svcl server
 
-local_dir="/Users/ktl014/PycharmProjects/hab-master/LIVIS/images"
-source_path="$local_dir/$date/*"
+# Navigate to latest generated image dir
+samba_data_dir="/Volumes/data/*/"
+source_path="$(ls -td -- $samba_data_dir | head -1)"
+#source_path=/Volumes/data/1558641021/ #DEBUG purposes
 
-ssh_key="plankton@gpu6"
+# Prepare remote data storage location
+ssh_key="plankton@gpu2"
 svcl_dir="/data6/phytoplankton-db/hab_in_vitro"
-img_dir="$svcl_dir/images/$date/"
+img_dir="$svcl_dir/images/$date"
 dest_path="$ssh_key:$img_dir"
 
-# Upload images
+if [ -z "$(ls -A $source_path)" ]; then
+	echo "Images not found in $source_path. Check if images were taken"
+	exit 0
+fi	
+
+# Upload images to remote server
 ssh $ssh_key "mkdir -p $img_dir"
-scp -r $source_path $dest_path
+scp -r $source_path/* $dest_path
 
+# Step 2-4
 # ****************************************************************************
 
-# Step 2
 # Run `auto_script` on remote server to convert images and get predictions
-cd_dir="cd $svcl_dir/LIVIS_automate"
-activate_env="source activate hab_env"
-deploy="bash deploy_remote.sh $date"
-echo Deploying classification
-ssh plankton@gpu6 "$cd_dir;$activate_env;$deploy"
+# expects $date of format `20190523/001`. No 0000000s accompanied.
+
+read -p "Run LIVIS Deployment? [y/n]: " -n 1 -r
+echo    # (optional) move to a new line
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+    cd_dir="cd $svcl_dir/LIVIS_automate"
+    activate_env="source activate hab_env"
+    deploy="bash deploy_remote.sh $date"
+    echo Deploying classification
+    ssh plankton@gpu2 "$cd_dir;$activate_env;$deploy"
+fi
 
+# Step 5
 # ****************************************************************************
 
-# Step 3
-# Upload predictions to static html
+# Copy back all of the images to the local machine
+dest_path="/Volumes/LACIE\ SHARE/$date"
+mkdir -p dest_path
+scp -r "plankton@gpu2:/data6/phytoplankton-db/hab_in_vitro:images/$date" $dest_path

LIVIS_automate/deploy_remote.sh

@@ -2,28 +2,39 @@
 
 # This script lives in the server ar data6/phytoplankton-db/hab_invitro/LIVIS_automate
 
-# Step 2
-# Using SPC Convert
+# Step 1
+# Uncompress tar.gz. files
 root="/data6/phytoplankton-db/hab_in_vitro"
-cd "$root/LIVIS_automate/SPCConvert"
 
+# Receive data folder
 date=$1
 source activate hab_env
 img_dir="$root/images/$date" # test img_dir
-python spcconvert.py $img_dir
+
+# Uncompress all tar files and join into one folder
+echo Uncompressing tar files
+cd "$img_dir"
+for i in *.tar;do tar -xvf $i && rm $i; done
+mkdir -p "$img_dir/00000"
+for d in $img_dir/000000*; do cp -r $d/* "$img_dir/00000/"; rm -rf $d; done
+
+# Step 2
+# Convert images using SPCConvert
+cd "$root/LIVIS_automate/SPCConvert"
+python spcconvert.py "$img_dir/00000"
 
 # Step 3
 # Deploy classifier
 cd "$root/LIVIS_automate/hab-ml/"
-deploy_data="$root/images/$date"
-deploy_data="$img_dir" # test deploy data
+deploy_data="$root/images/$date/00000"
 model_dir='/data6/lekevin/hab-master/hab-spc/experiments/proro_run'
 python main.py --mode deploy --batch_size 16 --deploy_data $deploy_data --model_dir $model_dir --lab_config
 
 # Step 4
 cd "$root/LIVIS_automate/SPCConvert"
-html_dir="$root/images/${date}_static_html"
-url="$html_dir/spcdata.html"
+html_dir="$root/images/${date}/00000_static_html"
+url="$html_dir/templates/spcdata.html"
 pred="$html_dir/predictions.json"
-python addPredictions.py $url $pred
-echo Predictions updated on html!
+db="$html_dir/static/js/database.js"
+python addPredictions.py $url $pred $db
+echo Predictions added

LIVIS_automate/export_data.py

@@ -0,0 +1,41 @@
+""" Exports and updates meta.csv files
+
+- accepts text file of dates to upload
+-
+
+
+"""
+# Standard dist imports
+import argparse
+import glob
+import os
+import subprocess
+
+# Third party imports
+
+# Project level imports
+
+# Module level constants
+g
+
+# root_dir = '/Volumes/LACIE SHARE/'
+source_dir = '/data6/phytoplankton-db/hab_in_vitro/images'
+user_ip = 'plankton@gpu2:'
+dest_dir = '/data6/phytoplankton-db/hab_in_vitro/images'
+
+start_date = '20190530'
+end_date = '20190606'
+date_files = sorted(os.listdir(source_dir))
+if start_date != end_date:
+    desired_dates = date_files[
+                    date_files.index(start_date): date_files.index(end_date)+1] # include end date
+else:
+    desired_dates = list(start_date)
+
+# Append root directory to dates
+print('Exporting data for dates: {}'.format(desired_dates))
+for date in desired_dates:
+    rel_path = '001/00000_static_html'
+    source_json = os.path.join(source_dir, date, rel_path, 'pred.json')
+    dest_json = os.path.join(user_ip + dest_dir, date, rel_path, 'pred.json')
+    subprocess.call(['scp', source_json, dest_json])