From 7a57bcaa0ea0895f819420a1797ef7960f7dd376 Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Wed, 22 May 2019 16:40:29 -0700
Subject: [PATCH 01/13] initial upload

---
 LIVIS_automate/SPCConvert   |  2 +-
 LIVIS_automate/run_livis.sh | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100755 LIVIS_automate/run_livis.sh

diff --git a/LIVIS_automate/SPCConvert b/LIVIS_automate/SPCConvert
index 764df8f..c88bc7c 160000
--- a/LIVIS_automate/SPCConvert
+++ b/LIVIS_automate/SPCConvert
@@ -1 +1 @@
-Subproject commit 764df8f7c63ff158c25fd98344450545641457ec
+Subproject commit c88bc7c94141790a7dc25df5c7d296a1dcbdbdf3
diff --git a/LIVIS_automate/run_livis.sh b/LIVIS_automate/run_livis.sh
new file mode 100755
index 0000000..56b706a
--- /dev/null
+++ b/LIVIS_automate/run_livis.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+source_dir="/home/odroid"
+sh $source_dir/run_camera
+
+# Source directory
+source_path=$(ls -td -- $source_dir/data/*/ | head -1)
+
+echo "Enter date and sample run for data storage location, followed by [ENTER]:"
+read date
+
+ssh_key="plankton@gpu6"
+svcl_dir="/data6/phytoplankton-db/hab_in_vitro"
+img_dir="$svcl_dir/images/$date/"
+dest_path="$ssh_key:$img_dir"
+
+# Upload images
+ssh $ssh_key "mkdir -p $img_dir"
+scp -r $source_path $dest_path
\ No newline at end of file

From 6aaef7bb318ccf15cb90da7188badc6045a54aeb Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Wed, 22 May 2019 19:03:30 -0700
Subject: [PATCH 02/13] wip (testing deploy, livis, deploy_remote integration)

---
 LIVIS_automate/deploy.sh    | 35 ++++++++++++-----------------------
 LIVIS_automate/run_livis.sh | 21 ++++++++++-----------
 2 files changed, 22 insertions(+), 34 deletions(-)

diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index 0f2e003..a82f24a 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -6,47 +6,36 @@
 
 # ****************************************************************************
 
-# Step 1
-# We transfer the images on the local machine to the SVCL Server using scp command
-# Change the paths accordingly:
-
-#-- prompt for subdirectory name to save data to on local machine
-echo "Enter new data directory name to store on remote machine, followed by [ENTER]: "
+echo "Enter date and sample run for data storage location, followed by [ENTER]:"
 read date
 
-#--
-#open -a coRD
-#open 'smb://spcuser:odroid@192.168.1.110'
-#echo Select `data` directory when mounting and proceed to run camera system
+# Make data storage location
+#source_path="/Users/ktl014/PycharmProjects/hab-master/LIVIS/images"
+source_path="/Users/spcuser/Documents"
+source_path="$source_path/$date"
+mkdir -p $source_path
 
-# include into this script ./run_camera.sh upon command
-# put watchdog on the data directory for the latest one after running camera
-# upload those images to the server
-# once imaging is done, run classifier upon yes command
+# Run camera
+ssh odroid@192.168.1.110 "./run_livis.sh $date"
 
-local_dir="/Users/ktl014/PycharmProjects/hab-master/LIVIS/images"
-source_path="$local_dir/$date/*"
+# ****************************************************************************
 
+# Prerepare remote data storage location
 ssh_key="plankton@gpu6"
 svcl_dir="/data6/phytoplankton-db/hab_in_vitro"
 img_dir="$svcl_dir/images/$date/"
 dest_path="$ssh_key:$img_dir"
 
-# Upload images
+# Upload images to remote server
+source_path = "$source_path/0000000000"
 ssh $ssh_key "mkdir -p $img_dir"
 scp -r $source_path $dest_path
 
 # ****************************************************************************
 
-# Step 2
 # Run `auto_script` on remote server to convert images and get predictions
 cd_dir="cd $svcl_dir/LIVIS_automate"
 activate_env="source activate hab_env"
 deploy="bash deploy_remote.sh $date"
 echo Deploying classification
 ssh plankton@gpu6 "$cd_dir;$activate_env;$deploy"
-
-# ****************************************************************************
-
-# Step 3
-# Upload predictions to static html
\ No newline at end of file
diff --git a/LIVIS_automate/run_livis.sh b/LIVIS_automate/run_livis.sh
index 56b706a..67d566b 100755
--- a/LIVIS_automate/run_livis.sh
+++ b/LIVIS_automate/run_livis.sh
@@ -1,19 +1,18 @@
 #!/usr/bin/env bash
 
-source_dir="/home/odroid"
-sh $source_dir/run_camera
+date=$1
 
-# Source directory
-source_path=$(ls -td -- $source_dir/data/*/ | head -1)
+source_path="/home/odroid"
+cd source_dir
+sh run_camera
 
-echo "Enter date and sample run for data storage location, followed by [ENTER]:"
-read date
+# Source directory
+#source_path="$(ls -td -- $source_path/data/*/ | head -1)/0000000000"
+source_path="$source_path/data/1558380800/0000000000"
 
-ssh_key="plankton@gpu6"
-svcl_dir="/data6/phytoplankton-db/hab_in_vitro"
-img_dir="$svcl_dir/images/$date/"
-dest_path="$ssh_key:$img_dir"
+# Uncompress tar file
+tar -xvf "${source_path}.tar"
 
 # Upload images
-ssh $ssh_key "mkdir -p $img_dir"
+dest_path="/Users/spcuser/Documents/$date"
 scp -r $source_path $dest_path
\ No newline at end of file

From 85ad1a4ac4da040719deba2260cc1c00121a0b44 Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Wed, 22 May 2019 20:01:42 -0700
Subject: [PATCH 03/13] fix (run_livis): able to pass latest data directory
 back to pull

---
 LIVIS_automate/deploy.sh    | 15 +++++++++++++--
 LIVIS_automate/run_livis.sh | 10 ++--------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index a82f24a..2cbc6c1 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -16,7 +16,19 @@ source_path="$source_path/$date"
 mkdir -p $source_path
 
 # Run camera
-ssh odroid@192.168.1.110 "./run_livis.sh $date"
+user=odroid
+host=192.168.1.110
+echo Logging into odroid remote server
+source_dir="LIVIS/LIVIS_automate"
+data_dir=$(ssh $user@$host "cd $source_dir;./run_livis.sh $date")
+
+# Download to local directory
+ID=~/.ssh/id_rsa
+scp -ri $ID $user@$host:$data_dir $source_path
+
+# Uncompress tar file
+source_path="$source_path/0000000000"
+tar -xvf "${source_path}.tar"
 
 # ****************************************************************************
 
@@ -27,7 +39,6 @@ img_dir="$svcl_dir/images/$date/"
 dest_path="$ssh_key:$img_dir"
 
 # Upload images to remote server
-source_path = "$source_path/0000000000"
 ssh $ssh_key "mkdir -p $img_dir"
 scp -r $source_path $dest_path
 
diff --git a/LIVIS_automate/run_livis.sh b/LIVIS_automate/run_livis.sh
index 67d566b..ea7c48e 100755
--- a/LIVIS_automate/run_livis.sh
+++ b/LIVIS_automate/run_livis.sh
@@ -3,16 +3,10 @@
 date=$1
 
 source_path="/home/odroid"
-cd source_dir
+cd ~
 sh run_camera
 
 # Source directory
 #source_path="$(ls -td -- $source_path/data/*/ | head -1)/0000000000"
 source_path="$source_path/data/1558380800/0000000000"
-
-# Uncompress tar file
-tar -xvf "${source_path}.tar"
-
-# Upload images
-dest_path="/Users/spcuser/Documents/$date"
-scp -r $source_path $dest_path
\ No newline at end of file
+echo "$source_path"
\ No newline at end of file

From 364d901f2777c567a329290d3ef8f99a06380b87 Mon Sep 17 00:00:00 2001
From: "E. Orenstein" <e1orenst@ucsd.edu>
Date: Thu, 23 May 2019 12:09:09 -0700
Subject: [PATCH 04/13] fix (run_livis): pipeline completed

---
 LIVIS_automate/deploy.sh    | 21 ++++++++++++---------
 LIVIS_automate/run_livis.sh |  5 -----
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index 2cbc6c1..ad8b1a7 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -22,22 +22,25 @@ echo Logging into odroid remote server
 source_dir="LIVIS/LIVIS_automate"
 data_dir=$(ssh $user@$host "cd $source_dir;./run_livis.sh $date")
 
-# Download to local directory
-ID=~/.ssh/id_rsa
-scp -ri $ID $user@$host:$data_dir $source_path
-
-# Uncompress tar file
-source_path="$source_path/0000000000"
-tar -xvf "${source_path}.tar"
-
+# username: odroid, password: odroid
+#open 'smb://spcuser:odroid@192.168.1.110'
+source_path="$(ls -td -- /Volumes/data/*/ | head -1)0000000000"
+echo "$source_path"
+#source_path=/Volumes/data/1558380800/0000000000
 # ****************************************************************************
+echo Uploading images to svcl server
 
 # Prerepare remote data storage location
 ssh_key="plankton@gpu6"
 svcl_dir="/data6/phytoplankton-db/hab_in_vitro"
-img_dir="$svcl_dir/images/$date/"
+img_dir="$svcl_dir/images/$date"
 dest_path="$ssh_key:$img_dir"
 
+if [ -z "$(ls -A $source_path)" ]; then
+	echo "Images not found in $source_path. Check if images were taken"
+	exit 0
+fi	
+
 # Upload images to remote server
 ssh $ssh_key "mkdir -p $img_dir"
 scp -r $source_path $dest_path
diff --git a/LIVIS_automate/run_livis.sh b/LIVIS_automate/run_livis.sh
index ea7c48e..82ca6ce 100755
--- a/LIVIS_automate/run_livis.sh
+++ b/LIVIS_automate/run_livis.sh
@@ -5,8 +5,3 @@ date=$1
 source_path="/home/odroid"
 cd ~
 sh run_camera
-
-# Source directory
-#source_path="$(ls -td -- $source_path/data/*/ | head -1)/0000000000"
-source_path="$source_path/data/1558380800/0000000000"
-echo "$source_path"
\ No newline at end of file

From bb475c36d97f8aba40f5575a50fa15a55243d0a1 Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Thu, 23 May 2019 12:11:15 -0700
Subject: [PATCH 05/13] fix(run_livis): second upload of complete deployment

---
 LIVIS_automate/deploy_remote.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/LIVIS_automate/deploy_remote.sh b/LIVIS_automate/deploy_remote.sh
index 5bb45dd..2e370d8 100644
--- a/LIVIS_automate/deploy_remote.sh
+++ b/LIVIS_automate/deploy_remote.sh
@@ -15,15 +15,14 @@ python spcconvert.py $img_dir
 # Step 3
 # Deploy classifier
 cd "$root/LIVIS_automate/hab-ml/"
-deploy_data="$root/images/$date"
-deploy_data="$img_dir" # test deploy data
+deploy_data="$root/images/$date/0000000000"
 model_dir='/data6/lekevin/hab-master/hab-spc/experiments/proro_run'
 python main.py --mode deploy --batch_size 16 --deploy_data $deploy_data --model_dir $model_dir --lab_config
 
 # Step 4
 cd "$root/LIVIS_automate/SPCConvert"
-html_dir="$root/images/${date}_static_html"
+html_dir="$root/images/${date}/0000000000_static_html"
 url="$html_dir/spcdata.html"
 pred="$html_dir/predictions.json"
 python addPredictions.py $url $pred
-echo Predictions updated on html!
+

From ff883bf439d9638802c00e97c57efbe1934f32e1 Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Thu, 23 May 2019 19:14:11 -0700
Subject: [PATCH 06/13] fix(deploy_remote.sh): scp and uncompress image batches
 over 1000

---
 LIVIS_automate/deploy_remote.sh | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/LIVIS_automate/deploy_remote.sh b/LIVIS_automate/deploy_remote.sh
index 2e370d8..3ba8c0a 100644
--- a/LIVIS_automate/deploy_remote.sh
+++ b/LIVIS_automate/deploy_remote.sh
@@ -5,23 +5,31 @@
 # Step 2
 # Using SPC Convert
 root="/data6/phytoplankton-db/hab_in_vitro"
-cd "$root/LIVIS_automate/SPCConvert"
 
+# Receive data folder
 date=$1
 source activate hab_env
 img_dir="$root/images/$date" # test img_dir
-python spcconvert.py $img_dir
+
+# Uncompress all tar files and join into one folder
+cd "$img_dir"
+for i in *.tar;do tar -xvf $i && rm $i; done
+mkdir "$img_dir/00000"
+for d in $img_dir/000000*; do cp -r $d/* "$img_dir/00000/"; rm -rf $d; done
+
+cd "$root/LIVIS_automate/SPCConvert"
+python spcconvert.py "$img_dir/00000"
 
 # Step 3
 # Deploy classifier
 cd "$root/LIVIS_automate/hab-ml/"
-deploy_data="$root/images/$date/0000000000"
+deploy_data="$root/images/$date/00000"
 model_dir='/data6/lekevin/hab-master/hab-spc/experiments/proro_run'
 python main.py --mode deploy --batch_size 16 --deploy_data $deploy_data --model_dir $model_dir --lab_config
 
 # Step 4
 cd "$root/LIVIS_automate/SPCConvert"
-html_dir="$root/images/${date}/0000000000_static_html"
+html_dir="$root/images/${date}/00000_static_html"
 url="$html_dir/spcdata.html"
 pred="$html_dir/predictions.json"
 python addPredictions.py $url $pred

From a9baafa05c7f2b465b31f663f380556cdf744082 Mon Sep 17 00:00:00 2001
From: "E. Orenstein" <e1orenst@ucsd.edu>
Date: Thu, 23 May 2019 19:17:28 -0700
Subject: [PATCH 07/13] fix(deploy): scp correct dir

---
 LIVIS_automate/deploy.sh | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index ad8b1a7..67d5af7 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -20,17 +20,14 @@ user=odroid
 host=192.168.1.110
 echo Logging into odroid remote server
 source_dir="LIVIS/LIVIS_automate"
-data_dir=$(ssh $user@$host "cd $source_dir;./run_livis.sh $date")
+ssh $user@$host "cd $source_dir;./run_livis.sh $date"
 
-# username: odroid, password: odroid
-#open 'smb://spcuser:odroid@192.168.1.110'
-source_path="$(ls -td -- /Volumes/data/*/ | head -1)0000000000"
-echo "$source_path"
-#source_path=/Volumes/data/1558380800/0000000000
+source_path="$(ls -td -- /Volumes/data/*/ | head -1)"
+#source_path=/Volumes/data/1558641021/
 # ****************************************************************************
-echo Uploading images to svcl server
+echo Uploading images to svcl server. Uncompressing files first.
 
-# Prerepare remote data storage location
+# Prepare remote data storage location
 ssh_key="plankton@gpu6"
 svcl_dir="/data6/phytoplankton-db/hab_in_vitro"
 img_dir="$svcl_dir/images/$date"
@@ -43,11 +40,12 @@ fi
 
 # Upload images to remote server
 ssh $ssh_key "mkdir -p $img_dir"
-scp -r $source_path $dest_path
+scp -r $source_path/* $dest_path
 
 # ****************************************************************************
 
 # Run `auto_script` on remote server to convert images and get predictions
+# expects $date of format `20190523/001`. No 0000000s accompanied.
 cd_dir="cd $svcl_dir/LIVIS_automate"
 activate_env="source activate hab_env"
 deploy="bash deploy_remote.sh $date"

From 971223bca42430897758b54377b07b19c7bc2d47 Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Tue, 28 May 2019 10:34:27 -0700
Subject: [PATCH 08/13] copy back images to local machine for annotation step

---
 LIVIS_automate/deploy.sh        | 11 +++++++++--
 LIVIS_automate/deploy_remote.sh |  3 +--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index 67d5af7..c0b5f5a 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -28,7 +28,7 @@ source_path="$(ls -td -- /Volumes/data/*/ | head -1)"
 echo Uploading images to svcl server. Uncompressing files first.
 
 # Prepare remote data storage location
-ssh_key="plankton@gpu6"
+ssh_key="plankton@gpu2"
 svcl_dir="/data6/phytoplankton-db/hab_in_vitro"
 img_dir="$svcl_dir/images/$date"
 dest_path="$ssh_key:$img_dir"
@@ -50,4 +50,11 @@ cd_dir="cd $svcl_dir/LIVIS_automate"
 activate_env="source activate hab_env"
 deploy="bash deploy_remote.sh $date"
 echo Deploying classification
-ssh plankton@gpu6 "$cd_dir;$activate_env;$deploy"
+ssh plankton@gpu2 "$cd_dir;$activate_env;$deploy"
+
+# ****************************************************************************
+
+# Copy back all of the images to the local machine
+dest_path="/Users/spcuser/Documents/$date"
+mkdir -p dest_path
+scp -r "plankton@gpu2:/data6/phytoplankton-db/hab_in_vitro:images/$date" $dest_path
diff --git a/LIVIS_automate/deploy_remote.sh b/LIVIS_automate/deploy_remote.sh
index 3ba8c0a..418e30d 100644
--- a/LIVIS_automate/deploy_remote.sh
+++ b/LIVIS_automate/deploy_remote.sh
@@ -14,7 +14,7 @@ img_dir="$root/images/$date" # test img_dir
 # Uncompress all tar files and join into one folder
 cd "$img_dir"
 for i in *.tar;do tar -xvf $i && rm $i; done
-mkdir "$img_dir/00000"
+mkdir -p "$img_dir/00000"
 for d in $img_dir/000000*; do cp -r $d/* "$img_dir/00000/"; rm -rf $d; done
 
 cd "$root/LIVIS_automate/SPCConvert"
@@ -33,4 +33,3 @@ html_dir="$root/images/${date}/00000_static_html"
 url="$html_dir/spcdata.html"
 pred="$html_dir/predictions.json"
 python addPredictions.py $url $pred
-

From ba89864028c15c68a743b26edffbe6a5edfac085 Mon Sep 17 00:00:00 2001
From: "E. Orenstein" <e1orenst@ucsd.edu>
Date: Tue, 28 May 2019 12:22:15 -0700
Subject: [PATCH 09/13] update addpredictions cmd line

---
 LIVIS_automate/deploy.sh        | 2 +-
 LIVIS_automate/deploy_remote.sh | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index c0b5f5a..d7ebefc 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -55,6 +55,6 @@ ssh plankton@gpu2 "$cd_dir;$activate_env;$deploy"
 # ****************************************************************************
 
 # Copy back all of the images to the local machine
-dest_path="/Users/spcuser/Documents/$date"
+dest_path="/Volumes/LACIE\ SHARE/$date"
 mkdir -p dest_path
 scp -r "plankton@gpu2:/data6/phytoplankton-db/hab_in_vitro:images/$date" $dest_path
diff --git a/LIVIS_automate/deploy_remote.sh b/LIVIS_automate/deploy_remote.sh
index 418e30d..d99163e 100644
--- a/LIVIS_automate/deploy_remote.sh
+++ b/LIVIS_automate/deploy_remote.sh
@@ -32,4 +32,5 @@ cd "$root/LIVIS_automate/SPCConvert"
 html_dir="$root/images/${date}/00000_static_html"
 url="$html_dir/spcdata.html"
 pred="$html_dir/predictions.json"
-python addPredictions.py $url $pred
+db="$html_dir/js/database.js"
+python addPredictions.py $url $pred $db

From afe108227427aee63d56c606ca8276b415fc0ef2 Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Thu, 30 May 2019 14:50:22 -0700
Subject: [PATCH 10/13] give y/n options to run imaging or deployment or all

---
 LIVIS_automate/deploy.sh | 33 ++++++++++++++++++++++-----------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index d7ebefc..31146ae 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -16,11 +16,16 @@ source_path="$source_path/$date"
 mkdir -p $source_path
 
 # Run camera
-user=odroid
-host=192.168.1.110
-echo Logging into odroid remote server
-source_dir="LIVIS/LIVIS_automate"
-ssh $user@$host "cd $source_dir;./run_livis.sh $date"
+read -p "Run LIVIS Imaging? [y/n]: " -n 1 -r
+echo    # (optional) move to a new line
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+    user=odroid
+    host=192.168.1.110
+    echo Logging into odroid remote server
+    source_dir="LIVIS/LIVIS_automate"
+    ssh $user@$host "cd $source_dir;./run_livis.sh $date"
+fi
 
 source_path="$(ls -td -- /Volumes/data/*/ | head -1)"
 #source_path=/Volumes/data/1558641021/
@@ -46,15 +51,21 @@ scp -r $source_path/* $dest_path
 
 # Run `auto_script` on remote server to convert images and get predictions
 # expects $date of format `20190523/001`. No 0000000s accompanied.
-cd_dir="cd $svcl_dir/LIVIS_automate"
-activate_env="source activate hab_env"
-deploy="bash deploy_remote.sh $date"
-echo Deploying classification
-ssh plankton@gpu2 "$cd_dir;$activate_env;$deploy"
+
+read -p "Run LIVIS Deployment? [y/n]: " -n 1 -r
+echo    # (optional) move to a new line
+if [[ $REPLY =~ ^[Yy]$ ]]
+then
+    cd_dir="cd $svcl_dir/LIVIS_automate"
+    activate_env="source activate hab_env"
+    deploy="bash deploy_remote.sh $date"
+    echo Deploying classification
+    ssh plankton@gpu2 "$cd_dir;$activate_env;$deploy"
+fi
 
 # ****************************************************************************
 
 # Copy back all of the images to the local machine
-dest_path="/Volumes/LACIE\ SHARE/$date"
+dest_path="/Volumes/LACIE SHARE/$date"
 mkdir -p dest_path
 scp -r "plankton@gpu2:/data6/phytoplankton-db/hab_in_vitro:images/$date" $dest_path

From 28fc756ec25f40e7dcf031abf24787d3c8e108d2 Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Sun, 9 Jun 2019 16:33:49 -0700
Subject: [PATCH 11/13] implement changes from addPredictions in spcconvert

---
 .gitignore                      | 3 ++-
 LIVIS_automate/SPCConvert       | 2 +-
 LIVIS_automate/deploy.sh        | 4 ++--
 LIVIS_automate/deploy_remote.sh | 6 ++++--
 LIVIS_automate/hab-ml           | 2 +-
 LIVIS_automate/run_livis.sh     | 2 ++
 6 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/.gitignore b/.gitignore
index 85f61bb..b0a750c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 .DS_Store
-images/*
\ No newline at end of file
+images/*
+LIVIS_automate/sampleImgs_static_html/*
diff --git a/LIVIS_automate/SPCConvert b/LIVIS_automate/SPCConvert
index c88bc7c..d86a483 160000
--- a/LIVIS_automate/SPCConvert
+++ b/LIVIS_automate/SPCConvert
@@ -1 +1 @@
-Subproject commit c88bc7c94141790a7dc25df5c7d296a1dcbdbdf3
+Subproject commit d86a483a59a456b145f9ef90cd26cb0d41e978ec
diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index 31146ae..96d7899 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -30,7 +30,7 @@ fi
 source_path="$(ls -td -- /Volumes/data/*/ | head -1)"
 #source_path=/Volumes/data/1558641021/
 # ****************************************************************************
-echo Uploading images to svcl server. Uncompressing files first.
+echo Uploading images to svcl server
 
 # Prepare remote data storage location
 ssh_key="plankton@gpu2"
@@ -66,6 +66,6 @@ fi
 # ****************************************************************************
 
 # Copy back all of the images to the local machine
-dest_path="/Volumes/LACIE SHARE/$date"
+dest_path="/Volumes/LACIE\ SHARE/$date"
 mkdir -p dest_path
 scp -r "plankton@gpu2:/data6/phytoplankton-db/hab_in_vitro:images/$date" $dest_path
diff --git a/LIVIS_automate/deploy_remote.sh b/LIVIS_automate/deploy_remote.sh
index d99163e..04981f6 100644
--- a/LIVIS_automate/deploy_remote.sh
+++ b/LIVIS_automate/deploy_remote.sh
@@ -12,6 +12,7 @@ source activate hab_env
 img_dir="$root/images/$date" # test img_dir
 
 # Uncompress all tar files and join into one folder
+echo Uncompressing tar files
 cd "$img_dir"
 for i in *.tar;do tar -xvf $i && rm $i; done
 mkdir -p "$img_dir/00000"
@@ -30,7 +31,8 @@ python main.py --mode deploy --batch_size 16 --deploy_data $deploy_data --model_
 # Step 4
 cd "$root/LIVIS_automate/SPCConvert"
 html_dir="$root/images/${date}/00000_static_html"
-url="$html_dir/spcdata.html"
+url="$html_dir/templates/spcdata.html"
 pred="$html_dir/predictions.json"
-db="$html_dir/js/database.js"
+db="$html_dir/static/js/database.js"
 python addPredictions.py $url $pred $db
+echo Predictions added
diff --git a/LIVIS_automate/hab-ml b/LIVIS_automate/hab-ml
index 4919254..f75cc6a 160000
--- a/LIVIS_automate/hab-ml
+++ b/LIVIS_automate/hab-ml
@@ -1 +1 @@
-Subproject commit 491925447245473cadddebd44eb759ab598662fa
+Subproject commit f75cc6af92c66eeb2b39847b44f5473299b84a23
diff --git a/LIVIS_automate/run_livis.sh b/LIVIS_automate/run_livis.sh
index 82ca6ce..4db593a 100755
--- a/LIVIS_automate/run_livis.sh
+++ b/LIVIS_automate/run_livis.sh
@@ -5,3 +5,5 @@ date=$1
 source_path="/home/odroid"
 cd ~
 sh run_camera
+
+

From ace4f3a4b3af92986cc2d5deba35ec0e7396670e Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Sun, 9 Jun 2019 23:40:17 -0700
Subject: [PATCH 12/13] scp data to svcl server

---
 LIVIS_automate/export_data.py | 41 +++++++++++++++++++++++++++++++++++
 LIVIS_automate/run_livis.sh   |  9 --------
 2 files changed, 41 insertions(+), 9 deletions(-)
 create mode 100644 LIVIS_automate/export_data.py
 delete mode 100755 LIVIS_automate/run_livis.sh

diff --git a/LIVIS_automate/export_data.py b/LIVIS_automate/export_data.py
new file mode 100644
index 0000000..bb9e9c4
--- /dev/null
+++ b/LIVIS_automate/export_data.py
@@ -0,0 +1,41 @@
+""" Exports and updates meta.csv files
+
+- accepts text file of dates to upload
+-
+
+
+"""
+# Standard dist imports
+import argparse
+import glob
+import os
+import subprocess
+
+# Third party imports
+
+# Project level imports
+
+# Module level constants
+g
+
+# root_dir = '/Volumes/LACIE SHARE/'
+source_dir = '/data6/phytoplankton-db/hab_in_vitro/images'
+user_ip = 'plankton@gpu2:'
+dest_dir = '/data6/phytoplankton-db/hab_in_vitro/images'
+
+start_date = '20190530'
+end_date = '20190606'
+date_files = sorted(os.listdir(source_dir))
+if start_date != end_date:
+    desired_dates = date_files[
+                    date_files.index(start_date): date_files.index(end_date)+1] # include end date
+else:
+    desired_dates = list(start_date)
+
+# Append root directory to dates
+print('Exporting data for dates: {}'.format(desired_dates))
+for date in desired_dates:
+    rel_path = '001/00000_static_html'
+    source_json = os.path.join(source_dir, date, rel_path, 'pred.json')
+    dest_json = os.path.join(user_ip + dest_dir, date, rel_path, 'pred.json')
+    subprocess.call(['scp', source_json, dest_json])
diff --git a/LIVIS_automate/run_livis.sh b/LIVIS_automate/run_livis.sh
deleted file mode 100755
index 4db593a..0000000
--- a/LIVIS_automate/run_livis.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-date=$1
-
-source_path="/home/odroid"
-cd ~
-sh run_camera
-
-

From af6e82a2e43f8e8585a045158b32135973e7c6ec Mon Sep 17 00:00:00 2001
From: Kevin Le <ktl014@ucsd.edu>
Date: Tue, 16 Jul 2019 22:05:13 -0400
Subject: [PATCH 13/13] added more documentation for deploy scripts

---
 LIVIS_automate/deploy.sh        | 30 +++++++++++++++++-------------
 LIVIS_automate/deploy_remote.sh |  6 ++++--
 2 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh
index 96d7899..dfd2cd6 100755
--- a/LIVIS_automate/deploy.sh
+++ b/LIVIS_automate/deploy.sh
@@ -1,21 +1,20 @@
 #!/usr/bin/env bash
-# This is the set of commands to execute on the local machine for:
+# This is the set of commands to execute on the local lab machine for:
+# 0. Run image sampling on odroid virtual machine
 # 1. Transfer the .tif images from the local machine to a folder in SVCL server
-# 2. Convert images using spcconvert and deploy model
-# 3. Upload predictions to static html
+# 2. Convert images using spcconvert
+# 3. Deploy model
+# 4. Upload predictions to static html
+# 5. Retrieve predicted images back to lab computer
 
+# NOTE: Steps 2-4 occurs on the SVCL server, hence the `deploy_remote.sh`
+# Step 0
 # ****************************************************************************
 
 echo "Enter date and sample run for data storage location, followed by [ENTER]:"
 read date
 
-# Make data storage location
-#source_path="/Users/ktl014/PycharmProjects/hab-master/LIVIS/images"
-source_path="/Users/spcuser/Documents"
-source_path="$source_path/$date"
-mkdir -p $source_path
-
-# Run camera
+# Ssh into camera virtual machine (odroid) and runs the imaging
 read -p "Run LIVIS Imaging? [y/n]: " -n 1 -r
 echo    # (optional) move to a new line
 if [[ $REPLY =~ ^[Yy]$ ]]
@@ -26,12 +25,15 @@ then
     source_dir="LIVIS/LIVIS_automate"
     ssh $user@$host "cd $source_dir;./run_livis.sh $date"
 fi
-
-source_path="$(ls -td -- /Volumes/data/*/ | head -1)"
-#source_path=/Volumes/data/1558641021/
+# Step 1
 # ****************************************************************************
 echo Uploading images to svcl server
 
+# Navigate to latest generated image dir
+samba_data_dir="/Volumes/data/*/"
+source_path="$(ls -td -- $samba_data_dir | head -1)"
+#source_path=/Volumes/data/1558641021/ #DEBUG purposes
+
 # Prepare remote data storage location
 ssh_key="plankton@gpu2"
 svcl_dir="/data6/phytoplankton-db/hab_in_vitro"
@@ -47,6 +49,7 @@ fi
 ssh $ssh_key "mkdir -p $img_dir"
 scp -r $source_path/* $dest_path
 
+# Step 2-4
 # ****************************************************************************
 
 # Run `auto_script` on remote server to convert images and get predictions
@@ -63,6 +66,7 @@ then
     ssh plankton@gpu2 "$cd_dir;$activate_env;$deploy"
 fi
 
+# Step 5
 # ****************************************************************************
 
 # Copy back all of the images to the local machine
diff --git a/LIVIS_automate/deploy_remote.sh b/LIVIS_automate/deploy_remote.sh
index 04981f6..268b5de 100644
--- a/LIVIS_automate/deploy_remote.sh
+++ b/LIVIS_automate/deploy_remote.sh
@@ -2,8 +2,8 @@
 
 # This script lives in the server ar data6/phytoplankton-db/hab_invitro/LIVIS_automate
 
-# Step 2
-# Using SPC Convert
+# Step 1
+# Uncompress tar.gz. files
 root="/data6/phytoplankton-db/hab_in_vitro"
 
 # Receive data folder
@@ -18,6 +18,8 @@ for i in *.tar;do tar -xvf $i && rm $i; done
 mkdir -p "$img_dir/00000"
 for d in $img_dir/000000*; do cp -r $d/* "$img_dir/00000/"; rm -rf $d; done
 
+# Step 2
+# Convert images using SPCConvert
 cd "$root/LIVIS_automate/SPCConvert"
 python spcconvert.py "$img_dir/00000"