diff --git a/.gitignore b/.gitignore index 85f61bb..b0a750c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store -images/* \ No newline at end of file +images/* +LIVIS_automate/sampleImgs_static_html/* diff --git a/LIVIS_automate/SPCConvert b/LIVIS_automate/SPCConvert index 764df8f..d86a483 160000 --- a/LIVIS_automate/SPCConvert +++ b/LIVIS_automate/SPCConvert @@ -1 +1 @@ -Subproject commit 764df8f7c63ff158c25fd98344450545641457ec +Subproject commit d86a483a59a456b145f9ef90cd26cb0d41e978ec diff --git a/LIVIS_automate/deploy.sh b/LIVIS_automate/deploy.sh index 0f2e003..dfd2cd6 100755 --- a/LIVIS_automate/deploy.sh +++ b/LIVIS_automate/deploy.sh @@ -1,52 +1,75 @@ #!/usr/bin/env bash -# This is the set of commands to execute on the local machine for: +# This is the set of commands to execute on the local lab machine for: +# 0. Run image sampling on odroid virtual machine # 1. Transfer the .tif images from the local machine to a folder in SVCL server -# 2. Convert images using spcconvert and deploy model -# 3. Upload predictions to static html +# 2. Convert images using spcconvert +# 3. Deploy model +# 4. Upload predictions to static html +# 5. Retrieve predicted images back to lab computer +# NOTE: Steps 2-4 occurs on the SVCL server, hence the `deploy_remote.sh` +# Step 0 # **************************************************************************** -# Step 1 -# We transfer the images on the local machine to the SVCL Server using scp command -# Change the paths accordingly: - -#-- prompt for subdirectory name to save data to on local machine -echo "Enter new data directory name to store on remote machine, followed by [ENTER]: " +echo "Enter date and sample run for data storage location, followed by [ENTER]:" read date -#-- -#open -a coRD -#open 'smb://spcuser:odroid@192.168.1.110' -#echo Select `data` directory when mounting and proceed to run camera system - -# include into this script ./run_camera.sh upon command -# put watchdog on the data directory for the latest one after running camera -# upload those images to the server -# once imaging is done, run classifier upon yes command +# Ssh into camera virtual machine (odroid) and runs the imaging +read -p "Run LIVIS Imaging? [y/n]: " -n 1 -r +echo # (optional) move to a new line +if [[ $REPLY =~ ^[Yy]$ ]] +then + user=odroid + host=192.168.1.110 + echo Logging into odroid remote server + source_dir="LIVIS/LIVIS_automate" + ssh $user@$host "cd $source_dir;./run_livis.sh $date" +fi +# Step 1 +# **************************************************************************** +echo Uploading images to svcl server -local_dir="/Users/ktl014/PycharmProjects/hab-master/LIVIS/images" -source_path="$local_dir/$date/*" +# Navigate to latest generated image dir +samba_data_dir="/Volumes/data/*/" +source_path="$(ls -td -- $samba_data_dir | head -1)" +#source_path=/Volumes/data/1558641021/ #DEBUG purposes -ssh_key="plankton@gpu6" +# Prepare remote data storage location +ssh_key="plankton@gpu2" svcl_dir="/data6/phytoplankton-db/hab_in_vitro" -img_dir="$svcl_dir/images/$date/" +img_dir="$svcl_dir/images/$date" dest_path="$ssh_key:$img_dir" -# Upload images +if [ -z "$(ls -A $source_path)" ]; then + echo "Images not found in $source_path. Check if images were taken" + exit 0 +fi + +# Upload images to remote server ssh $ssh_key "mkdir -p $img_dir" -scp -r $source_path $dest_path +scp -r $source_path/* $dest_path +# Step 2-4 # **************************************************************************** -# Step 2 # Run `auto_script` on remote server to convert images and get predictions -cd_dir="cd $svcl_dir/LIVIS_automate" -activate_env="source activate hab_env" -deploy="bash deploy_remote.sh $date" -echo Deploying classification -ssh plankton@gpu6 "$cd_dir;$activate_env;$deploy" +# expects $date of format `20190523/001`. No 0000000s accompanied. + +read -p "Run LIVIS Deployment? [y/n]: " -n 1 -r +echo # (optional) move to a new line +if [[ $REPLY =~ ^[Yy]$ ]] +then + cd_dir="cd $svcl_dir/LIVIS_automate" + activate_env="source activate hab_env" + deploy="bash deploy_remote.sh $date" + echo Deploying classification + ssh plankton@gpu2 "$cd_dir;$activate_env;$deploy" +fi +# Step 5 # **************************************************************************** -# Step 3 -# Upload predictions to static html \ No newline at end of file +# Copy back all of the images to the local machine +dest_path="/Volumes/LACIE\ SHARE/$date" +mkdir -p dest_path +scp -r "plankton@gpu2:/data6/phytoplankton-db/hab_in_vitro:images/$date" $dest_path diff --git a/LIVIS_automate/deploy_remote.sh b/LIVIS_automate/deploy_remote.sh index 5bb45dd..268b5de 100644 --- a/LIVIS_automate/deploy_remote.sh +++ b/LIVIS_automate/deploy_remote.sh @@ -2,28 +2,39 @@ # This script lives in the server ar data6/phytoplankton-db/hab_invitro/LIVIS_automate -# Step 2 -# Using SPC Convert +# Step 1 +# Uncompress tar.gz. files root="/data6/phytoplankton-db/hab_in_vitro" -cd "$root/LIVIS_automate/SPCConvert" +# Receive data folder date=$1 source activate hab_env img_dir="$root/images/$date" # test img_dir -python spcconvert.py $img_dir + +# Uncompress all tar files and join into one folder +echo Uncompressing tar files +cd "$img_dir" +for i in *.tar;do tar -xvf $i && rm $i; done +mkdir -p "$img_dir/00000" +for d in $img_dir/000000*; do cp -r $d/* "$img_dir/00000/"; rm -rf $d; done + +# Step 2 +# Convert images using SPCConvert +cd "$root/LIVIS_automate/SPCConvert" +python spcconvert.py "$img_dir/00000" # Step 3 # Deploy classifier cd "$root/LIVIS_automate/hab-ml/" -deploy_data="$root/images/$date" -deploy_data="$img_dir" # test deploy data +deploy_data="$root/images/$date/00000" model_dir='/data6/lekevin/hab-master/hab-spc/experiments/proro_run' python main.py --mode deploy --batch_size 16 --deploy_data $deploy_data --model_dir $model_dir --lab_config # Step 4 cd "$root/LIVIS_automate/SPCConvert" -html_dir="$root/images/${date}_static_html" -url="$html_dir/spcdata.html" +html_dir="$root/images/${date}/00000_static_html" +url="$html_dir/templates/spcdata.html" pred="$html_dir/predictions.json" -python addPredictions.py $url $pred -echo Predictions updated on html! +db="$html_dir/static/js/database.js" +python addPredictions.py $url $pred $db +echo Predictions added diff --git a/LIVIS_automate/export_data.py b/LIVIS_automate/export_data.py new file mode 100644 index 0000000..bb9e9c4 --- /dev/null +++ b/LIVIS_automate/export_data.py @@ -0,0 +1,41 @@ +""" Exports and updates meta.csv files + +- accepts text file of dates to upload +- + + +""" +# Standard dist imports +import argparse +import glob +import os +import subprocess + +# Third party imports + +# Project level imports + +# Module level constants +g + +# root_dir = '/Volumes/LACIE SHARE/' +source_dir = '/data6/phytoplankton-db/hab_in_vitro/images' +user_ip = 'plankton@gpu2:' +dest_dir = '/data6/phytoplankton-db/hab_in_vitro/images' + +start_date = '20190530' +end_date = '20190606' +date_files = sorted(os.listdir(source_dir)) +if start_date != end_date: + desired_dates = date_files[ + date_files.index(start_date): date_files.index(end_date)+1] # include end date +else: + desired_dates = list(start_date) + +# Append root directory to dates +print('Exporting data for dates: {}'.format(desired_dates)) +for date in desired_dates: + rel_path = '001/00000_static_html' + source_json = os.path.join(source_dir, date, rel_path, 'pred.json') + dest_json = os.path.join(user_ip + dest_dir, date, rel_path, 'pred.json') + subprocess.call(['scp', source_json, dest_json]) diff --git a/LIVIS_automate/hab-ml b/LIVIS_automate/hab-ml index 4919254..f75cc6a 160000 --- a/LIVIS_automate/hab-ml +++ b/LIVIS_automate/hab-ml @@ -1 +1 @@ -Subproject commit 491925447245473cadddebd44eb759ab598662fa +Subproject commit f75cc6af92c66eeb2b39847b44f5473299b84a23