From d515c02c82c69e7ccf46dcf1584c8b685a12b5bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Fri, 14 Aug 2020 14:36:55 -0500 Subject: [PATCH 01/25] working on cache diagnostics --- diagnostics_dante/aggregate_csv.py | 12 +++++ diagnostics_dante/diagnostic.py | 69 ++++++++++++++++++++++++++ diagnostics_dante/generate_videos.py | 4 ++ diagnostics_dante/graph_diagnostics.py | 3 ++ diagnostics_dante/main.py | 26 ++++++++++ diagnostics_dante/test_videos.py | 39 +++++++++++++++ test_npz.py | 23 +++++++-- 7 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 diagnostics_dante/aggregate_csv.py create mode 100644 diagnostics_dante/diagnostic.py create mode 100644 diagnostics_dante/generate_videos.py create mode 100644 diagnostics_dante/graph_diagnostics.py create mode 100644 diagnostics_dante/main.py create mode 100644 diagnostics_dante/test_videos.py diff --git a/diagnostics_dante/aggregate_csv.py b/diagnostics_dante/aggregate_csv.py new file mode 100644 index 0000000..e3cb551 --- /dev/null +++ b/diagnostics_dante/aggregate_csv.py @@ -0,0 +1,12 @@ +import pandas as pd + +def aggregate_csv(path_to_csv): + df = pd.read_csv(path_to_csv) + df = df.groupby(['Shape','Size']).agg({'File Size': ['min', 'median', 'max'], 'Storage Time': ['min', 'median', 'max'], 'Retrieval Time': ['min', 'median', 'max']}) + + path_to_agg_csv = path_to_csv[:-4] + "_agg.csv" + df.to_csv(path_to_agg_csv) + return path_to_agg_csv + +# Test +#aggregate_csv("videos/diagnostics.csv") \ No newline at end of file diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py new file mode 100644 index 0000000..874a054 --- /dev/null +++ b/diagnostics_dante/diagnostic.py @@ -0,0 +1,69 @@ +import os +import sys +import inspect + +# Is there a better way? +currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) +parentdir = os.path.dirname(currentdir) +sys.path.insert(0,parentdir) + +from deeplens.full_manager.condition import Condition +from deeplens.full_manager.full_video_processing import CropSplitter +from deeplens.tracking.background import FixedCameraBGFGSegmenter +from deeplens.optimizer.deeplens import DeepLensOptimizer + +from deeplens.struct import * +from deeplens.utils import * +from deeplens.dataflow.map import * +from deeplens.full_manager.full_manager import * +from deeplens.utils.testing_utils import * +from deeplens.dataflow.agg import * +from deeplens.tracking.contour import * +from deeplens.tracking.event import * +from deeplens.core import * +from deeplens.simple_manager.manager import * + +from deeplens.utils.ui import play + +from deeplens.extern.cache import persist +from deeplens.struct import RawVideoStream + +sys.path.insert(0,currentdir) + +import time + +def diagnostic(video_path): + file_size = None + time_storage = None + time_retreive = None + + FILENAME = video_path #the video file that you want to load + LIMIT = 100 + + vstream = VideoStream(FILENAME, limit=LIMIT) #limit is the max number of frames + + t0 = time.time() + + cache = persist(vstream, 'cache.npz') #how big the size of the stored raw video is + + t1 = time.time() + + vstream = RawVideoStream('cache.npz', shape=(LIMIT,1080,1920,3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) + #do something + f = 0 + for v in vstream: + f += 1 + pass + + t2 = time.time() + + time_storage = t1 - t0 + time_retreive = t2 - t1 + + file_size = os.path.getsize('cache.npz') + + return (file_size, time_storage, time_retreive) + + +# Test +# print(diagnostic('../tcam.mp4')) \ No newline at end of file diff --git a/diagnostics_dante/generate_videos.py b/diagnostics_dante/generate_videos.py new file mode 100644 index 0000000..47162c2 --- /dev/null +++ b/diagnostics_dante/generate_videos.py @@ -0,0 +1,4 @@ +def generate_videos(video_path, shapes, max_pixels, min_pixels, reducing_factor): + # Video path should be w:h_sizeInPixels + path_to_videos = None + return path_to_videos \ No newline at end of file diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py new file mode 100644 index 0000000..316e5c4 --- /dev/null +++ b/diagnostics_dante/graph_diagnostics.py @@ -0,0 +1,3 @@ +def graph_diagnostics(path_to_agg_csv): + path_to_graphs = None # A folder of graphs + return path_to_graphs \ No newline at end of file diff --git a/diagnostics_dante/main.py b/diagnostics_dante/main.py new file mode 100644 index 0000000..1dd804a --- /dev/null +++ b/diagnostics_dante/main.py @@ -0,0 +1,26 @@ +from generate_videos import generate_videos +from test_videos import test_videos +from aggregate_csv import aggregate_csv +from graph_diagnostics import graph_diagnostics + +video_path = None +number_of_trials = 5 + +shapes = [(16,9), (4,3), (1,1), (20,1), (1,20)] # An array of aspect ratio tuples (width, height) + +# These may be replaced by default (8m to 30k reducing by a factor of two) +max_pixels = 8000000 # The max number of pixels per frame (approx) +min_pixels = 30000 # The min number of pixels per frame (approx) +reducing_factor = 2 # Number of pixels in frame reduced by this factor from max_pixels to min_pixels + +# Generate videos +path_to_videos = generate_videos(video_path, shapes, max_pixels, min_pixels, reducing_factor) + +# Test videos creating csv of results +path_to_csv = test_videos(path_to_videos, number_of_trials) + +# Aggregate csv by shape and size (min, med, max) +path_to_agg_csv = aggregate_csv(path_to_csv) + +# Graph aggregate csv and put results in new folder +path_to_graphs = graph_diagnostics(path_to_agg_csv) diff --git a/diagnostics_dante/test_videos.py b/diagnostics_dante/test_videos.py new file mode 100644 index 0000000..e3c9ac9 --- /dev/null +++ b/diagnostics_dante/test_videos.py @@ -0,0 +1,39 @@ +import os +from diagnostic import diagnostic + +def test_videos(path_to_videos, number_of_trials): + path_to_csv = path_to_videos + "diagnostics.csv" + + import csv + with open(path_to_csv, 'w', newline='') as file: + writer = csv.writer(file) + writer.writerow(["Shape", "Size", "Trial #", "File Size", "Storage Time", "Retrieval Time"]) + + directory = path_to_videos + + for video_path in os.listdir(directory): + filename = os.fsdecode(video_path) + if filename.endswith(".mp4"): + name = video_path[:-4] + shape = name.split('_')[0] + #size = name.split('_')[1] + + # For testing + size = 'ben' + + for trial_number in range(number_of_trials): + info = diagnostic(path_to_videos + video_path) + file_size = info[0] + time_storage = info[1] + time_retreive = info[2] + with open(path_to_csv, 'a', newline='') as file: + writer = csv.writer(file) + writer.writerow([shape, size, trial_number, file_size, time_storage, time_retreive]) + + continue + else: + continue + return path_to_csv + +# Test +#test_videos("videos/",5) \ No newline at end of file diff --git a/test_npz.py b/test_npz.py index 9ace197..71ae814 100644 --- a/test_npz.py +++ b/test_npz.py @@ -21,19 +21,36 @@ import cv2 import numpy as np +import time as t from deeplens.extern.cache import persist -FILENAME = '' #the video file that you want to load -vstream = VideoStream(FILENAME, limit=1000) #limit is the max number of frames +FILENAME = 'tcam.mp4' #the video file that you want to load +LIMIT = 100 +vstream = VideoStream(FILENAME, limit=LIMIT) #limit is the max number of frames + +t0 = t.time() size = persist(vstream, 'cache.npz') #how big the size of the stored raw video is +t1 = t.time() + from deeplens.struct import RawVideoStream -vstream = RawVideoStream('cache.npz', shape=(1000,1080,1920,3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) +vstream = RawVideoStream('cache.npz', shape=(LIMIT,1080,1920,3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) + #do something +f = 0 for v in vstream: + f += 1 + #print(f) pass +t2 = t.time() + +time_to_storage = t1 - t0 +time_to_retrieve = t2 - t1 +time_total = t2 - t0 + +print(f"Storage: {time_to_storage} + Retrieve: {time_to_retrieve} = Total: {time_total}") \ No newline at end of file From d655dcc273d1049902731113daaeb27056bf9798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Fri, 14 Aug 2020 19:15:46 -0500 Subject: [PATCH 02/25] almost done with graphing then size/shape --- diagnostics_dante/aggregate_csv.py | 7 ++-- diagnostics_dante/generate_videos.py | 2 ++ diagnostics_dante/graph_diagnostics.py | 45 +++++++++++++++++++++++++- diagnostics_dante/test_videos.py | 8 +++-- 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/diagnostics_dante/aggregate_csv.py b/diagnostics_dante/aggregate_csv.py index e3cb551..ead4bcd 100644 --- a/diagnostics_dante/aggregate_csv.py +++ b/diagnostics_dante/aggregate_csv.py @@ -3,10 +3,13 @@ def aggregate_csv(path_to_csv): df = pd.read_csv(path_to_csv) df = df.groupby(['Shape','Size']).agg({'File Size': ['min', 'median', 'max'], 'Storage Time': ['min', 'median', 'max'], 'Retrieval Time': ['min', 'median', 'max']}) + df.reset_index(inplace=True) + df.columns = df.columns.get_level_values(0) + df.columns = ['Shape', 'Size', 'File Size Min', 'File Size Median', 'File Size Max', 'Storage Time Min', 'Storage Time Median', 'Storage Time Max', 'Retrieval Time Min', 'Retrieval Time Median', 'Retrieval Time Max'] path_to_agg_csv = path_to_csv[:-4] + "_agg.csv" - df.to_csv(path_to_agg_csv) + df.to_csv(path_to_agg_csv, index=False) return path_to_agg_csv # Test -#aggregate_csv("videos/diagnostics.csv") \ No newline at end of file +aggregate_csv("results/diagnostics.csv") \ No newline at end of file diff --git a/diagnostics_dante/generate_videos.py b/diagnostics_dante/generate_videos.py index 47162c2..e96b502 100644 --- a/diagnostics_dante/generate_videos.py +++ b/diagnostics_dante/generate_videos.py @@ -1,3 +1,5 @@ +import os + def generate_videos(video_path, shapes, max_pixels, min_pixels, reducing_factor): # Video path should be w:h_sizeInPixels path_to_videos = None diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index 316e5c4..b253c4c 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -1,3 +1,46 @@ +import os +import pandas as pd +from matplotlib import pyplot as plt + def graph_diagnostics(path_to_agg_csv): + #os.mkdir('results/graphs') + + df = pd.read_csv(path_to_agg_csv) + + fig1 = plt.figure(figsize = (10, 5)) + plt.bar(df['Size'], height=df['File Size Max'], bottom=df['File Size Min']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Cache size (bytes)") + plt.title("(Shape) Storage Size") + plt.show() + + fig2 = plt.figure(figsize = (10, 5)) + + plt.bar(df['Size'], height=df['Storage Time Max'], bottom=df['Storage Time Min']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Storage Time (seconds)") + plt.title("(Shape) Storage Time") + plt.show() + + fig3 = plt.figure(figsize = (10, 5)) + + plt.bar(df['Size'], height=df['Retrieval Time Max'], bottom=df['Retrieval Time Min']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Retrieval Time (seconds)") + plt.title("(Shape) Retrieval Time") + plt.show() + + fig3 = plt.figure(figsize = (10, 5)) + + p1 = plt.bar(df['Size'], height=df['Storage Time Median']) + p2 = plt.bar(df['Size'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Total Time (seconds)") + plt.title("(Shape) Total Time") + plt.show() + path_to_graphs = None # A folder of graphs - return path_to_graphs \ No newline at end of file + return path_to_graphs + +# Test +graph_diagnostics("results/diagnostics_agg.csv") \ No newline at end of file diff --git a/diagnostics_dante/test_videos.py b/diagnostics_dante/test_videos.py index e3c9ac9..c382baa 100644 --- a/diagnostics_dante/test_videos.py +++ b/diagnostics_dante/test_videos.py @@ -2,7 +2,8 @@ from diagnostic import diagnostic def test_videos(path_to_videos, number_of_trials): - path_to_csv = path_to_videos + "diagnostics.csv" + os.mkdir('results') + path_to_csv = "results/diagnostics.csv" import csv with open(path_to_csv, 'w', newline='') as file: @@ -19,7 +20,7 @@ def test_videos(path_to_videos, number_of_trials): #size = name.split('_')[1] # For testing - size = 'ben' + size = 10 for trial_number in range(number_of_trials): info = diagnostic(path_to_videos + video_path) @@ -32,8 +33,9 @@ def test_videos(path_to_videos, number_of_trials): continue else: + print("There's a snake in my boot!") continue return path_to_csv # Test -#test_videos("videos/",5) \ No newline at end of file +test_videos("videos/",5) \ No newline at end of file From 6bfde40ca87859c92a481f85d198b071cb86a4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Mon, 17 Aug 2020 14:31:44 -0400 Subject: [PATCH 03/25] done with graphing but there is a big bug help :s --- diagnostics_dante/graph_diagnostics.py | 81 ++++++++++++++------------ 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index b253c4c..1ad170d 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -3,43 +3,50 @@ from matplotlib import pyplot as plt def graph_diagnostics(path_to_agg_csv): - #os.mkdir('results/graphs') - - df = pd.read_csv(path_to_agg_csv) - - fig1 = plt.figure(figsize = (10, 5)) - plt.bar(df['Size'], height=df['File Size Max'], bottom=df['File Size Min']) - plt.xlabel("Video Size (pixels)") - plt.ylabel("Cache size (bytes)") - plt.title("(Shape) Storage Size") - plt.show() - - fig2 = plt.figure(figsize = (10, 5)) - - plt.bar(df['Size'], height=df['Storage Time Max'], bottom=df['Storage Time Min']) - plt.xlabel("Video Size (pixels)") - plt.ylabel("Storage Time (seconds)") - plt.title("(Shape) Storage Time") - plt.show() - - fig3 = plt.figure(figsize = (10, 5)) - - plt.bar(df['Size'], height=df['Retrieval Time Max'], bottom=df['Retrieval Time Min']) - plt.xlabel("Video Size (pixels)") - plt.ylabel("Retrieval Time (seconds)") - plt.title("(Shape) Retrieval Time") - plt.show() - - fig3 = plt.figure(figsize = (10, 5)) - - p1 = plt.bar(df['Size'], height=df['Storage Time Median']) - p2 = plt.bar(df['Size'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) - plt.xlabel("Video Size (pixels)") - plt.ylabel("Total Time (seconds)") - plt.title("(Shape) Total Time") - plt.show() - - path_to_graphs = None # A folder of graphs + os.mkdir('results/graphs') + + dfo = pd.read_csv(path_to_agg_csv) + + for size, df in dfo.groupby('Size'): + fig0 = plt.figure(figsize = (10, 5)) + p1 = plt.bar(df['Shape'], height=df['Storage Time Median']) + p2 = plt.bar(df['Shape'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) + plt.xlabel("Shape") + plt.ylabel("Total Time (seconds)") + plt.title(f"{size} Total Time") + plt.savefig(f"results/graphs/{size}_totaltime.png") + + for shape, df in dfo.groupby('Shape'): + fig1 = plt.figure(figsize = (10, 5)) + plt.bar(df['Size'], height=df['File Size Max'], bottom=df['File Size Min']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Cache size (bytes)") + plt.title(f"{shape} File Size") + plt.savefig(f"results/graphs/{shape}_filesize.png") + + fig2 = plt.figure(figsize = (10, 5)) + plt.bar(df['Size'], height=df['Storage Time Max'], bottom=df['Storage Time Min']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Storage Time (seconds)") + plt.title(f"{shape} Storage Time") + plt.savefig(f"results/graphs/{shape}_storagetime.png") + + fig3 = plt.figure(figsize = (10, 5)) + plt.bar(df['Size'], height=df['Retrieval Time Max'], bottom=df['Retrieval Time Min']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Retrieval Time (seconds)") + plt.title(f"{shape} Retrieval Time") + plt.savefig(f"results/graphs/{shape}_retrievaltime.png") + + fig3 = plt.figure(figsize = (10, 5)) + p1 = plt.bar(df['Size'], height=df['Storage Time Median']) + p2 = plt.bar(df['Size'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Total Time (seconds)") + plt.title(f"{shape} Total Time") + plt.savefig(f"results/graphs/{shape}_totaltime.png") + + path_to_graphs = 'results/graphs' # A folder of graphs return path_to_graphs # Test From 13646bf424635536713532d4f90af5324d3b0861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Tue, 18 Aug 2020 00:46:04 -0400 Subject: [PATCH 04/25] testing done version 1 --- diagnostics_dante/diagnostic.py | 4 +++- diagnostics_dante/generate_videos.py | 27 ++++++++++++++++++++++---- diagnostics_dante/main.py | 8 +++++--- diagnostics_dante/test_videos.py | 29 ++++++---------------------- 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 874a054..562c674 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -32,7 +32,7 @@ import time -def diagnostic(video_path): +def diagnostic(video_path, size): file_size = None time_storage = None time_retreive = None @@ -42,6 +42,8 @@ def diagnostic(video_path): vstream = VideoStream(FILENAME, limit=LIMIT) #limit is the max number of frames + vstream[crop(0, 0, size[0], size[1])] + t0 = time.time() cache = persist(vstream, 'cache.npz') #how big the size of the stored raw video is diff --git a/diagnostics_dante/generate_videos.py b/diagnostics_dante/generate_videos.py index e96b502..ee21b3c 100644 --- a/diagnostics_dante/generate_videos.py +++ b/diagnostics_dante/generate_videos.py @@ -1,6 +1,25 @@ import os +import math -def generate_videos(video_path, shapes, max_pixels, min_pixels, reducing_factor): - # Video path should be w:h_sizeInPixels - path_to_videos = None - return path_to_videos \ No newline at end of file +def generate_videos(video_width, video_height, shapes, max_pixels, min_pixels, reducing_factor): + dictionary_of_video_sizes = {} + reducing_factor = 1/reducing_factor + for shape in shapes: + height_temp = video_height + width_temp = video_width + key = f"{shape[0]}:{shape[1]}" + dictionary_of_video_sizes[key] = [] + height_temp = math.ceil((shape[1] / shape[0]) * video_width) + if height_temp > video_height: + height_temp = video_height + width_temp = math.ceil((shape[0] / shape[1]) * video_height) + dictionary_of_video_sizes[key].append((width_temp, height_temp)) + while width_temp * height_temp > min_pixels: + height_temp = math.ceil(height_temp * math.sqrt(reducing_factor)) + width_temp = math.ceil(width_temp * math.sqrt(reducing_factor)) + dictionary_of_video_sizes[key].append((width_temp, height_temp)) + return dictionary_of_video_sizes + +# Test +#Dict = generate_videos(1920, 1080, [(16,9), (1,1), (1,20), (20,1), (2,3)], 1000000000000, 30000, 2) +#print(Dict) diff --git a/diagnostics_dante/main.py b/diagnostics_dante/main.py index 1dd804a..813c45c 100644 --- a/diagnostics_dante/main.py +++ b/diagnostics_dante/main.py @@ -3,7 +3,9 @@ from aggregate_csv import aggregate_csv from graph_diagnostics import graph_diagnostics -video_path = None +video_path = 'tcam.mp4' +video_width = 1920 +video_height = 1080 number_of_trials = 5 shapes = [(16,9), (4,3), (1,1), (20,1), (1,20)] # An array of aspect ratio tuples (width, height) @@ -14,10 +16,10 @@ reducing_factor = 2 # Number of pixels in frame reduced by this factor from max_pixels to min_pixels # Generate videos -path_to_videos = generate_videos(video_path, shapes, max_pixels, min_pixels, reducing_factor) +dictionary_of_video_sizes = generate_videos(video_width, video_height, shapes, max_pixels, min_pixels, reducing_factor) # Test videos creating csv of results -path_to_csv = test_videos(path_to_videos, number_of_trials) +path_to_csv = test_videos(video_path, dictionary_of_video_sizes, number_of_trials) # Aggregate csv by shape and size (min, med, max) path_to_agg_csv = aggregate_csv(path_to_csv) diff --git a/diagnostics_dante/test_videos.py b/diagnostics_dante/test_videos.py index c382baa..1244132 100644 --- a/diagnostics_dante/test_videos.py +++ b/diagnostics_dante/test_videos.py @@ -1,7 +1,7 @@ import os from diagnostic import diagnostic -def test_videos(path_to_videos, number_of_trials): +def test_videos(video_path, dictionary_of_video_sizes, number_of_trials): os.mkdir('results') path_to_csv = "results/diagnostics.csv" @@ -9,33 +9,16 @@ def test_videos(path_to_videos, number_of_trials): with open(path_to_csv, 'w', newline='') as file: writer = csv.writer(file) writer.writerow(["Shape", "Size", "Trial #", "File Size", "Storage Time", "Retrieval Time"]) - - directory = path_to_videos - for video_path in os.listdir(directory): - filename = os.fsdecode(video_path) - if filename.endswith(".mp4"): - name = video_path[:-4] - shape = name.split('_')[0] - #size = name.split('_')[1] - - # For testing - size = 10 - - for trial_number in range(number_of_trials): - info = diagnostic(path_to_videos + video_path) + for shape, sizes in dictionary_of_video_sizes: + for trial_number in range(number_of_trials): + for size in sizes: + info = diagnostic(video_path, size) file_size = info[0] time_storage = info[1] time_retreive = info[2] with open(path_to_csv, 'a', newline='') as file: writer = csv.writer(file) - writer.writerow([shape, size, trial_number, file_size, time_storage, time_retreive]) + writer.writerow([shape, size[0]*size[1], trial_number, file_size, time_storage, time_retreive]) - continue - else: - print("There's a snake in my boot!") - continue return path_to_csv - -# Test -test_videos("videos/",5) \ No newline at end of file From 637447f46b064128101f16b031bacb7c1d20a28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Tue, 18 Aug 2020 01:07:16 -0400 Subject: [PATCH 05/25] bug fixes --- diagnostics_dante/aggregate_csv.py | 2 +- diagnostics_dante/graph_diagnostics.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/diagnostics_dante/aggregate_csv.py b/diagnostics_dante/aggregate_csv.py index ead4bcd..fe44d0d 100644 --- a/diagnostics_dante/aggregate_csv.py +++ b/diagnostics_dante/aggregate_csv.py @@ -12,4 +12,4 @@ def aggregate_csv(path_to_csv): return path_to_agg_csv # Test -aggregate_csv("results/diagnostics.csv") \ No newline at end of file +#aggregate_csv("results/diagnostics.csv") \ No newline at end of file diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index 1ad170d..56682a3 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -50,4 +50,4 @@ def graph_diagnostics(path_to_agg_csv): return path_to_graphs # Test -graph_diagnostics("results/diagnostics_agg.csv") \ No newline at end of file +#graph_diagnostics("results/diagnostics_agg.csv") \ No newline at end of file From 7fa4f56268b386ea03734f6b3abc025b40750ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Tue, 18 Aug 2020 01:10:50 -0400 Subject: [PATCH 06/25] small fix can't find crop function tho --- diagnostics_dante/test_videos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/diagnostics_dante/test_videos.py b/diagnostics_dante/test_videos.py index 1244132..fcd9c82 100644 --- a/diagnostics_dante/test_videos.py +++ b/diagnostics_dante/test_videos.py @@ -10,7 +10,8 @@ def test_videos(video_path, dictionary_of_video_sizes, number_of_trials): writer = csv.writer(file) writer.writerow(["Shape", "Size", "Trial #", "File Size", "Storage Time", "Retrieval Time"]) - for shape, sizes in dictionary_of_video_sizes: + print(dictionary_of_video_sizes) + for shape, sizes in dictionary_of_video_sizes.items(): for trial_number in range(number_of_trials): for size in sizes: info = diagnostic(video_path, size) From ba5e49793c7db0ef6c2794cd26d91ba673c7bc4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Tue, 18 Aug 2020 11:19:43 -0400 Subject: [PATCH 07/25] works! except for the graphing bug --- diagnostics_dante/diagnostic.py | 2 +- diagnostics_dante/graph_diagnostics.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 562c674..fdf6082 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -42,7 +42,7 @@ def diagnostic(video_path, size): vstream = VideoStream(FILENAME, limit=LIMIT) #limit is the max number of frames - vstream[crop(0, 0, size[0], size[1])] + vstream[Crop(0, 0, size[0], size[1])] t0 = time.time() diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index 56682a3..4f49ce0 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -7,14 +7,14 @@ def graph_diagnostics(path_to_agg_csv): dfo = pd.read_csv(path_to_agg_csv) - for size, df in dfo.groupby('Size'): - fig0 = plt.figure(figsize = (10, 5)) - p1 = plt.bar(df['Shape'], height=df['Storage Time Median']) - p2 = plt.bar(df['Shape'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) - plt.xlabel("Shape") - plt.ylabel("Total Time (seconds)") - plt.title(f"{size} Total Time") - plt.savefig(f"results/graphs/{size}_totaltime.png") + # for size, df in dfo.groupby('Size'): + # fig0 = plt.figure(figsize = (10, 5)) + # p1 = plt.bar(df['Shape'], height=df['Storage Time Median']) + # p2 = plt.bar(df['Shape'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) + # plt.xlabel("Shape") + # plt.ylabel("Total Time (seconds)") + # plt.title(f"{size} Total Time") + # plt.savefig(f"results/graphs/{size}_totaltime.png") for shape, df in dfo.groupby('Shape'): fig1 = plt.figure(figsize = (10, 5)) From f9b8b892cf30e87d280a1c6ba4b52ab4ca8d40af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Tue, 18 Aug 2020 11:48:22 -0400 Subject: [PATCH 08/25] wow things were very buggy but it's fixed now --- diagnostics_dante/aggregate_csv.py | 10 ++++++++++ diagnostics_dante/diagnostic.py | 6 +++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/diagnostics_dante/aggregate_csv.py b/diagnostics_dante/aggregate_csv.py index fe44d0d..fb3fdef 100644 --- a/diagnostics_dante/aggregate_csv.py +++ b/diagnostics_dante/aggregate_csv.py @@ -7,9 +7,19 @@ def aggregate_csv(path_to_csv): df.columns = df.columns.get_level_values(0) df.columns = ['Shape', 'Size', 'File Size Min', 'File Size Median', 'File Size Max', 'Storage Time Min', 'Storage Time Median', 'Storage Time Max', 'Retrieval Time Min', 'Retrieval Time Median', 'Retrieval Time Max'] + df['Size'] = df['Size'].apply(lambda x: human_format(x)) + path_to_agg_csv = path_to_csv[:-4] + "_agg.csv" df.to_csv(path_to_agg_csv, index=False) return path_to_agg_csv +def human_format(num): + num = float('{:.3g}'.format(num)) + magnitude = 0 + while abs(num) >= 1000: + magnitude += 1 + num /= 1000.0 + return '{}{}'.format('{:f}'.format(num).rstrip('0').rstrip('.'), ['', 'K', 'M', 'B', 'T'][magnitude]) + # Test #aggregate_csv("results/diagnostics.csv") \ No newline at end of file diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index fdf6082..58376fc 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -42,7 +42,7 @@ def diagnostic(video_path, size): vstream = VideoStream(FILENAME, limit=LIMIT) #limit is the max number of frames - vstream[Crop(0, 0, size[0], size[1])] + vstream = vstream[Crop(0, 0, size[0], size[1])] t0 = time.time() @@ -50,7 +50,7 @@ def diagnostic(video_path, size): t1 = time.time() - vstream = RawVideoStream('cache.npz', shape=(LIMIT,1080,1920,3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) + vstream = RawVideoStream('cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) #do something f = 0 for v in vstream: @@ -68,4 +68,4 @@ def diagnostic(video_path, size): # Test -# print(diagnostic('../tcam.mp4')) \ No newline at end of file +print(diagnostic('../tcam.mp4', (1080, 1080))) \ No newline at end of file From 78741c482b9b9685415381c091d3fceb104732ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Wed, 26 Aug 2020 14:50:09 -0500 Subject: [PATCH 09/25] fixed bug adding boxplots --- diagnostics_dante/diagnostic.py | 2 +- diagnostics_dante/graph_diagnostics.py | 54 ++++++++++++++------------ diagnostics_dante/main.py | 2 +- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 58376fc..ab68da4 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -68,4 +68,4 @@ def diagnostic(video_path, size): # Test -print(diagnostic('../tcam.mp4', (1080, 1080))) \ No newline at end of file +#print(diagnostic('../tcam.mp4', (1080, 1080))) \ No newline at end of file diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index 4f49ce0..6f309b1 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -2,10 +2,12 @@ import pandas as pd from matplotlib import pyplot as plt -def graph_diagnostics(path_to_agg_csv): +def graph_diagnostics(path_to_csv, path_to_agg_csv): os.mkdir('results/graphs') + os.mkdir('results/graphs/spread') + os.mkdir('results/graphs/total_time') - dfo = pd.read_csv(path_to_agg_csv) + dfa = pd.read_csv(path_to_agg_csv) # for size, df in dfo.groupby('Size'): # fig0 = plt.figure(figsize = (10, 5)) @@ -16,27 +18,29 @@ def graph_diagnostics(path_to_agg_csv): # plt.title(f"{size} Total Time") # plt.savefig(f"results/graphs/{size}_totaltime.png") - for shape, df in dfo.groupby('Shape'): - fig1 = plt.figure(figsize = (10, 5)) - plt.bar(df['Size'], height=df['File Size Max'], bottom=df['File Size Min']) - plt.xlabel("Video Size (pixels)") - plt.ylabel("Cache size (bytes)") - plt.title(f"{shape} File Size") - plt.savefig(f"results/graphs/{shape}_filesize.png") - - fig2 = plt.figure(figsize = (10, 5)) - plt.bar(df['Size'], height=df['Storage Time Max'], bottom=df['Storage Time Min']) - plt.xlabel("Video Size (pixels)") - plt.ylabel("Storage Time (seconds)") - plt.title(f"{shape} Storage Time") - plt.savefig(f"results/graphs/{shape}_storagetime.png") - - fig3 = plt.figure(figsize = (10, 5)) - plt.bar(df['Size'], height=df['Retrieval Time Max'], bottom=df['Retrieval Time Min']) - plt.xlabel("Video Size (pixels)") - plt.ylabel("Retrieval Time (seconds)") - plt.title(f"{shape} Retrieval Time") - plt.savefig(f"results/graphs/{shape}_retrievaltime.png") + for shape, df in dfa.groupby('Shape'): + df = df.reset_index() + + # fig1 = plt.figure(figsize = (10, 5)) + # plt.bar(df['Size'], height=df['File Size Max'], bottom=df['File Size Min']) + # plt.xlabel("Video Size (pixels)") + # plt.ylabel("Cache size (bytes)") + # plt.title(f"{shape} File Size") + # plt.savefig(f"results/graphs/{shape}_filesize.png") + + # fig2 = plt.figure(figsize = (10, 5)) + # plt.bar(df['Size'], height=df['Storage Time Max'], bottom=df['Storage Time Min']) + # plt.xlabel("Video Size (pixels)") + # plt.ylabel("Storage Time (seconds)") + # plt.title(f"{shape} Storage Time") + # plt.savefig(f"results/graphs/{shape}_storagetime.png") + + # fig3 = plt.figure(figsize = (10, 5)) + # plt.bar(df['Size'], height=df['Retrieval Time Max'], bottom=df['Retrieval Time Min']) + # plt.xlabel("Video Size (pixels)") + # plt.ylabel("Retrieval Time (seconds)") + # plt.title(f"{shape} Retrieval Time") + # plt.savefig(f"results/graphs/{shape}_retrievaltime.png") fig3 = plt.figure(figsize = (10, 5)) p1 = plt.bar(df['Size'], height=df['Storage Time Median']) @@ -44,10 +48,10 @@ def graph_diagnostics(path_to_agg_csv): plt.xlabel("Video Size (pixels)") plt.ylabel("Total Time (seconds)") plt.title(f"{shape} Total Time") - plt.savefig(f"results/graphs/{shape}_totaltime.png") + plt.savefig(f"results/graphs/total_time/{shape}_totaltime.png") path_to_graphs = 'results/graphs' # A folder of graphs return path_to_graphs # Test -#graph_diagnostics("results/diagnostics_agg.csv") \ No newline at end of file +graph_diagnostics("diagnostics.csv", "results/diagnostics_agg.csv") \ No newline at end of file diff --git a/diagnostics_dante/main.py b/diagnostics_dante/main.py index 813c45c..452ea91 100644 --- a/diagnostics_dante/main.py +++ b/diagnostics_dante/main.py @@ -25,4 +25,4 @@ path_to_agg_csv = aggregate_csv(path_to_csv) # Graph aggregate csv and put results in new folder -path_to_graphs = graph_diagnostics(path_to_agg_csv) +path_to_graphs = graph_diagnostics(path_to_csv, path_to_agg_csv) From 0240f60b3899f64f60e1c6659d7df3a1e204f457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Wed, 26 Aug 2020 15:17:56 -0500 Subject: [PATCH 10/25] boxes done gotta fix bugs --- diagnostics_dante/graph_diagnostics.py | 73 +++++++++++++++----------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index 6f309b1..fefc463 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -5,53 +5,62 @@ def graph_diagnostics(path_to_csv, path_to_agg_csv): os.mkdir('results/graphs') os.mkdir('results/graphs/spread') + os.mkdir('results/graphs/file_size') os.mkdir('results/graphs/total_time') - dfa = pd.read_csv(path_to_agg_csv) + dfo = pd.read_csv(path_to_csv) + for shape, dfi in dfo.groupby('Shape'): + storage_time = [] + retrieval_time = [] - # for size, df in dfo.groupby('Size'): - # fig0 = plt.figure(figsize = (10, 5)) - # p1 = plt.bar(df['Shape'], height=df['Storage Time Median']) - # p2 = plt.bar(df['Shape'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) - # plt.xlabel("Shape") - # plt.ylabel("Total Time (seconds)") - # plt.title(f"{size} Total Time") - # plt.savefig(f"results/graphs/{size}_totaltime.png") + for size, df in dfi.groupby('Size'): + storage_time.append(df['Storage Time'].tolist()) + retrieval_time.append(df['Retrieval Time'].tolist()) + fig = plt.figure(figsize = (10, 5)) + ax2 = fig.add_axes([0, 0, 1, 1]) + bp2 = ax2.boxplot(storage_time) + plt.savefig(f"results/graphs/spread/{shape}_storagetime.png") + fig.clf() + + fig = plt.figure(figsize = (10, 5)) + ax3 = fig.add_axes([0, 0, 1, 1]) + bp3 = ax3.boxplot(retrieval_time) + plt.savefig(f"results/graphs/spread/{shape}_retrievaltime.png") + fig.clf() + + dfa = pd.read_csv(path_to_agg_csv) for shape, df in dfa.groupby('Shape'): df = df.reset_index() - # fig1 = plt.figure(figsize = (10, 5)) - # plt.bar(df['Size'], height=df['File Size Max'], bottom=df['File Size Min']) - # plt.xlabel("Video Size (pixels)") - # plt.ylabel("Cache size (bytes)") - # plt.title(f"{shape} File Size") - # plt.savefig(f"results/graphs/{shape}_filesize.png") - - # fig2 = plt.figure(figsize = (10, 5)) - # plt.bar(df['Size'], height=df['Storage Time Max'], bottom=df['Storage Time Min']) - # plt.xlabel("Video Size (pixels)") - # plt.ylabel("Storage Time (seconds)") - # plt.title(f"{shape} Storage Time") - # plt.savefig(f"results/graphs/{shape}_storagetime.png") - - # fig3 = plt.figure(figsize = (10, 5)) - # plt.bar(df['Size'], height=df['Retrieval Time Max'], bottom=df['Retrieval Time Min']) - # plt.xlabel("Video Size (pixels)") - # plt.ylabel("Retrieval Time (seconds)") - # plt.title(f"{shape} Retrieval Time") - # plt.savefig(f"results/graphs/{shape}_retrievaltime.png") - - fig3 = plt.figure(figsize = (10, 5)) + fig = plt.figure(figsize = (10, 5)) + plt.bar(df['Size'], height=df['File Size Median']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Cache size (bytes)") + plt.title(f"{shape} File Size") + plt.savefig(f"results/graphs/file_size/{shape}_filesize.png") + fig.clf() + + fig = plt.figure(figsize = (10, 5)) p1 = plt.bar(df['Size'], height=df['Storage Time Median']) p2 = plt.bar(df['Size'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) plt.xlabel("Video Size (pixels)") plt.ylabel("Total Time (seconds)") plt.title(f"{shape} Total Time") plt.savefig(f"results/graphs/total_time/{shape}_totaltime.png") + fig.clf() + + # for size, df in dfo.groupby('Size'): + # fig0 = plt.figure(figsize = (10, 5)) + # p1 = plt.bar(df['Shape'], height=df['Storage Time Median']) + # p2 = plt.bar(df['Shape'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) + # plt.xlabel("Shape") + # plt.ylabel("Total Time (seconds)") + # plt.title(f"{size} Total Time") + # plt.savefig(f"results/graphs/{size}_totaltime.png") path_to_graphs = 'results/graphs' # A folder of graphs return path_to_graphs # Test -graph_diagnostics("diagnostics.csv", "results/diagnostics_agg.csv") \ No newline at end of file +graph_diagnostics("results/diagnostics.csv", "results/diagnostics_agg.csv") \ No newline at end of file From 69ea8586bdda23345aa6317999fb815ca8d5bf80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Wed, 26 Aug 2020 15:34:02 -0500 Subject: [PATCH 11/25] bugs fixed ready to generate much data --- diagnostics_dante/graph_diagnostics.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index fefc463..84ff930 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -1,6 +1,7 @@ import os import pandas as pd from matplotlib import pyplot as plt +from aggregate_csv import human_format def graph_diagnostics(path_to_csv, path_to_agg_csv): os.mkdir('results/graphs') @@ -12,20 +13,26 @@ def graph_diagnostics(path_to_csv, path_to_agg_csv): for shape, dfi in dfo.groupby('Shape'): storage_time = [] retrieval_time = [] + sizes = [] for size, df in dfi.groupby('Size'): + sizes.append(human_format(size)) storage_time.append(df['Storage Time'].tolist()) retrieval_time.append(df['Retrieval Time'].tolist()) - fig = plt.figure(figsize = (10, 5)) - ax2 = fig.add_axes([0, 0, 1, 1]) - bp2 = ax2.boxplot(storage_time) + fig, ax = plt.subplots() + ax.set_title(f"{shape} Storage Time") + ax.boxplot(storage_time, labels=sizes) + plt.xlabel("Size (pixels)") + plt.ylabel("Time (seconds)") plt.savefig(f"results/graphs/spread/{shape}_storagetime.png") fig.clf() - fig = plt.figure(figsize = (10, 5)) - ax3 = fig.add_axes([0, 0, 1, 1]) - bp3 = ax3.boxplot(retrieval_time) + fig, ax = plt.subplots() + ax.set_title(f"{shape} Retrieval Time") + ax.boxplot(retrieval_time, labels=sizes) + plt.xlabel("Size (pixels)") + plt.ylabel("Time (seconds)") plt.savefig(f"results/graphs/spread/{shape}_retrievaltime.png") fig.clf() @@ -63,4 +70,4 @@ def graph_diagnostics(path_to_csv, path_to_agg_csv): return path_to_graphs # Test -graph_diagnostics("results/diagnostics.csv", "results/diagnostics_agg.csv") \ No newline at end of file +#graph_diagnostics("results/diagnostics.csv", "results/diagnostics_agg.csv") \ No newline at end of file From 595ae5884bf6137538e3893cfe9140718624728b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Thu, 27 Aug 2020 09:49:12 -0500 Subject: [PATCH 12/25] some enhacements --- diagnostics_dante/aggregate_csv.py | 1 + diagnostics_dante/graph_diagnostics.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/diagnostics_dante/aggregate_csv.py b/diagnostics_dante/aggregate_csv.py index fb3fdef..a44d8a1 100644 --- a/diagnostics_dante/aggregate_csv.py +++ b/diagnostics_dante/aggregate_csv.py @@ -7,6 +7,7 @@ def aggregate_csv(path_to_csv): df.columns = df.columns.get_level_values(0) df.columns = ['Shape', 'Size', 'File Size Min', 'File Size Median', 'File Size Max', 'Storage Time Min', 'Storage Time Median', 'Storage Time Max', 'Retrieval Time Min', 'Retrieval Time Median', 'Retrieval Time Max'] + df = df.sort_values(by='Size') df['Size'] = df['Size'].apply(lambda x: human_format(x)) path_to_agg_csv = path_to_csv[:-4] + "_agg.csv" diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index 84ff930..d3b81eb 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -36,6 +36,30 @@ def graph_diagnostics(path_to_csv, path_to_agg_csv): plt.savefig(f"results/graphs/spread/{shape}_retrievaltime.png") fig.clf() + dfa = pd.read_csv(path_to_agg_csv) + dfa['Shape'] = dfa['Shape'].apply(lambda x: sum(map(ord, x)) + ord(x[0])) + + fig = plt.figure(figsize = (10, 5)) + norm = plt.Normalize(dfa['Shape'].min(), dfa['Shape'].max()) + cmap = plt.get_cmap("magma") + plt.bar(dfa['Size'], height=dfa['File Size Median'], color=cmap(norm(dfa['Shape'].values))) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Cache size (bytes)") + plt.title(f"File Size") + plt.savefig(f"results/graphs/file_size/filesize.png") + fig.clf() + + fig = plt.figure(figsize = (10, 5)) + norm = plt.Normalize(dfa['Shape'].min(), dfa['Shape'].max()) + cmap = plt.get_cmap("magma") + p1 = plt.bar(dfa['Size'], height=dfa['Storage Time Median'], color=cmap(norm(dfa['Shape'].values))) + p2 = plt.bar(dfa['Size'], height=dfa['Retrieval Time Median'], bottom=dfa['Storage Time Median']) + plt.xlabel("Video Size (pixels)") + plt.ylabel("Total Time (seconds)") + plt.title(f"Total Time") + plt.savefig(f"results/graphs/total_time/totaltime.png") + fig.clf() + dfa = pd.read_csv(path_to_agg_csv) for shape, df in dfa.groupby('Shape'): df = df.reset_index() From a933eb426a494dd22b132d4a51bdda3d4ab797d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Mon, 31 Aug 2020 14:45:49 -0500 Subject: [PATCH 13/25] comaprison graphs done --- diagnostics_dante/compare_diagnostics.py | 54 ++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 diagnostics_dante/compare_diagnostics.py diff --git a/diagnostics_dante/compare_diagnostics.py b/diagnostics_dante/compare_diagnostics.py new file mode 100644 index 0000000..84bcade --- /dev/null +++ b/diagnostics_dante/compare_diagnostics.py @@ -0,0 +1,54 @@ +# Script that compares different results as line graph +# Take all folders in path folder +# Go in folder -> go in results -> take aggregate csv file +# Make a pandas df +# separate by shape +# add label that says what results they are from +# graph? +# add key and x, y, title + +import os +import pandas as pd +import matplotlib.pyplot as plt + +def graph_compare(path_to_many_results, shapes): + os.mkdir('comparison_results') + results = [] + for dirpath, dirnames, filenames in os.walk(path_to_many_results): + for fname in filenames: + if fname == "diagnostics_agg.csv": + agg_path = dirpath + '/' + fname + name = dirpath.replace(path_to_many_results + '/', '').replace('/results','') + results.append((name, agg_path)) + + dict_data_by_shapes = {} + for shape in shapes: + key = f"{shape[0]}:{shape[1]}" + dict_data_by_shapes[key] = [] + + for result in results: + name = result[0] + dfo = pd.read_csv(result[1]) + dfo['Total Time'] = dfo.apply(lambda x: x['Retrieval Time Median'] + x['Storage Time Median'], axis=1) + for shape, df in dfo.groupby('Shape'): + sizes = df['Size'].tolist() + data = df['Total Time'].tolist() + dict_data_by_shapes[shape].append((name, sizes, data)) + + for shape in dict_data_by_shapes: + fig = plt.figure(figsize = (10, 5)) + results = dict_data_by_shapes[shape] + for result in results: + plt.plot(result[1], result[2], label=result[0]) + plt.xlabel('Frame Size (pixels)') + plt.ylabel('Total Time (seconds)') + plt.title(f'Total Time of {shape} results') + plt.legend() + plt.savefig(f"comparison_results/{shape}_totaltime.png") + + + + + +# Test +#graph_compare("../../Results", [(16,9), (4,3), (1,1), (10,1), (1,10), (20,1), (1,20)]) \ No newline at end of file From 1faab4d9649759fd7cd2ffea29de3b3c1cf7047d Mon Sep 17 00:00:00 2001 From: "drgilmarin@uchicago.edu" Date: Mon, 14 Sep 2020 15:48:01 -0500 Subject: [PATCH 14/25] linux curr --- deeplens/extern/cache.py | 2 +- diagnostics_dante/diagnostic.py | 14 ++++++--- diagnostics_dante/main.py | 56 ++++++++++++++++----------------- requirements.txt | 2 +- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/deeplens/extern/cache.py b/deeplens/extern/cache.py index 7413e80..b0f1528 100644 --- a/deeplens/extern/cache.py +++ b/deeplens/extern/cache.py @@ -38,7 +38,7 @@ def delete_cache(file): def persist(vstream, file): shape = _array_shape(vstream) - fp = np.memmap(file, dtype='uint8', mode='w+', shape=shape,order='F') + fp = np.memmap(file, dtype='uint8', mode='w+', shape=shape,order='C') #only works for files, not derived stuff vstream.cap = None diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index ab68da4..7b93205 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -31,8 +31,11 @@ sys.path.insert(0,currentdir) import time +import os +import numpy as np def diagnostic(video_path, size): + os.system("sudo sync; echo 1 | sudo tee /proc/sys/vm/drop_caches >/dev/null") file_size = None time_storage = None time_retreive = None @@ -46,15 +49,16 @@ def diagnostic(video_path, size): t0 = time.time() - cache = persist(vstream, 'cache.npz') #how big the size of the stored raw video is + # /dev/shm/ + cache = persist(vstream, '/dev/shm/cache.npz') #how big the size of the stored raw video is t1 = time.time() - vstream = RawVideoStream('cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) + vstream = RawVideoStream('/dev/shm/cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) #do something - f = 0 + #f = 0 for v in vstream: - f += 1 + v['data'][0,0,0] pass t2 = time.time() @@ -62,7 +66,7 @@ def diagnostic(video_path, size): time_storage = t1 - t0 time_retreive = t2 - t1 - file_size = os.path.getsize('cache.npz') + file_size = os.path.getsize('/dev/shm/cache.npz') return (file_size, time_storage, time_retreive) diff --git a/diagnostics_dante/main.py b/diagnostics_dante/main.py index 452ea91..22d8c8f 100644 --- a/diagnostics_dante/main.py +++ b/diagnostics_dante/main.py @@ -1,28 +1,28 @@ -from generate_videos import generate_videos -from test_videos import test_videos -from aggregate_csv import aggregate_csv -from graph_diagnostics import graph_diagnostics - -video_path = 'tcam.mp4' -video_width = 1920 -video_height = 1080 -number_of_trials = 5 - -shapes = [(16,9), (4,3), (1,1), (20,1), (1,20)] # An array of aspect ratio tuples (width, height) - -# These may be replaced by default (8m to 30k reducing by a factor of two) -max_pixels = 8000000 # The max number of pixels per frame (approx) -min_pixels = 30000 # The min number of pixels per frame (approx) -reducing_factor = 2 # Number of pixels in frame reduced by this factor from max_pixels to min_pixels - -# Generate videos -dictionary_of_video_sizes = generate_videos(video_width, video_height, shapes, max_pixels, min_pixels, reducing_factor) - -# Test videos creating csv of results -path_to_csv = test_videos(video_path, dictionary_of_video_sizes, number_of_trials) - -# Aggregate csv by shape and size (min, med, max) -path_to_agg_csv = aggregate_csv(path_to_csv) - -# Graph aggregate csv and put results in new folder -path_to_graphs = graph_diagnostics(path_to_csv, path_to_agg_csv) +from generate_videos import generate_videos +from test_videos import test_videos +from aggregate_csv import aggregate_csv +from graph_diagnostics import graph_diagnostics + +video_path = '4KFootage.mkv' +video_width = 3840 +video_height = 2160 +number_of_trials = 50 + +shapes = [(16,9)] # An array of aspect ratio tuples (width, height) + +# These may be replaced by default (8m to 30k reducing by a factor of two) +max_pixels = 80000000 # The max number of pixels per frame (approx) +min_pixels = 30000 # The min number of pixels per frame (approx) +reducing_factor = 2 # Number of pixels in frame reduced by this factor from max_pixels to min_pixels + +# Generate videos +dictionary_of_video_sizes = generate_videos(video_width, video_height, shapes, max_pixels, min_pixels, reducing_factor) + +# Test videos creating csv of results +path_to_csv = test_videos(video_path, dictionary_of_video_sizes, number_of_trials) + +# Aggregate csv by shape and size (min, med, max) +#path_to_agg_csv = aggregate_csv(path_to_csv) + +# Graph aggregate csv and put results in new folder +#path_to_graphs = graph_diagnostics(path_to_csv, path_to_agg_csv) diff --git a/requirements.txt b/requirements.txt index 564dd8b..463fc62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -numpy==1.17.2 +numpy==1.19.1 opencv-python==4.1.1.26 absl-py==0.8.0 astor==0.8.0 From 4c1995103ceb5ccf75101178443a00f05c2859cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Wed, 16 Sep 2020 18:58:03 -0400 Subject: [PATCH 15/25] adding usage stats --- diagnostics_dante/diagnostic.py | 50 +++++++++++++++++++++++++++----- diagnostics_dante/test_videos.py | 8 +++-- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 7b93205..83d5ae1 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -33,12 +33,30 @@ import time import os import numpy as np +import psutil +from multiprocessing import Process, Value, Manager +from statistics import median + +def getResourceUsage(done, mlist): + cpu_usage = [] + mlist.append(cpu_usage) + ram_usage = [] + mlist.append(ram_usage) + while done.value: + cpu_usage.append(psutil.cpu_percent()) + ram_usage.append(psutil.virtual_memory().percent) + mlist[0] = cpu_usage + mlist[1] = ram_usage + time.sleep(1) + def diagnostic(video_path, size): os.system("sudo sync; echo 1 | sudo tee /proc/sys/vm/drop_caches >/dev/null") file_size = None time_storage = None + usage_storage = None time_retreive = None + usage_retrieve = None FILENAME = video_path #the video file that you want to load LIMIT = 100 @@ -47,29 +65,47 @@ def diagnostic(video_path, size): vstream = vstream[Crop(0, 0, size[0], size[1])] + manager = Manager() + mlist = manager.list() + done = Value('i', 1) + resource = Process(target=getResourceUsage, args=(done, mlist)) + resource.start() + t0 = time.time() # /dev/shm/ - cache = persist(vstream, '/dev/shm/cache.npz') #how big the size of the stored raw video is + cache = persist(vstream, 'cache.npz') #how big the size of the stored raw video is + + done.value -= 1 + resource.join() + usage_storage = mlist + + manager = Manager() + mlist = manager.list() + done = Value('i', 1) + resource = Process(target=getResourceUsage, args=(done, mlist)) + resource.start() t1 = time.time() - vstream = RawVideoStream('/dev/shm/cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) + vstream = RawVideoStream('cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) #do something - #f = 0 for v in vstream: - v['data'][0,0,0] + np.copy(v['data'], order='F') pass t2 = time.time() + done.value -= 1 + resource.join() + usage_retrieve = mlist + time_storage = t1 - t0 time_retreive = t2 - t1 - file_size = os.path.getsize('/dev/shm/cache.npz') - - return (file_size, time_storage, time_retreive) + file_size = os.path.getsize('cache.npz') + return (file_size, time_storage, time_retreive, median(usage_storage), max(usage_storage), median(usage_retrieve), max(usage_retrieve)) # Test #print(diagnostic('../tcam.mp4', (1080, 1080))) \ No newline at end of file diff --git a/diagnostics_dante/test_videos.py b/diagnostics_dante/test_videos.py index fcd9c82..fb90fab 100644 --- a/diagnostics_dante/test_videos.py +++ b/diagnostics_dante/test_videos.py @@ -8,7 +8,7 @@ def test_videos(video_path, dictionary_of_video_sizes, number_of_trials): import csv with open(path_to_csv, 'w', newline='') as file: writer = csv.writer(file) - writer.writerow(["Shape", "Size", "Trial #", "File Size", "Storage Time", "Retrieval Time"]) + writer.writerow(["Shape", "Size", "Trial #", "File Size", "Storage Time", "Retrieval Time", "Storage Usage Median", "Storage Usage Max", "Retrieval Usage Median", "Retrieval Usage Max"]) print(dictionary_of_video_sizes) for shape, sizes in dictionary_of_video_sizes.items(): @@ -18,8 +18,12 @@ def test_videos(video_path, dictionary_of_video_sizes, number_of_trials): file_size = info[0] time_storage = info[1] time_retreive = info[2] + usage_storage_median = info[3] + usage_storage_max = info[4] + usage_retrieve_median = info[5] + usage_retrieve_max = info[6] with open(path_to_csv, 'a', newline='') as file: writer = csv.writer(file) - writer.writerow([shape, size[0]*size[1], trial_number, file_size, time_storage, time_retreive]) + writer.writerow([shape, size[0]*size[1], trial_number, file_size, time_storage, time_retreive, usage_storage_median, usage_storage_max, usage_retrieve_median, usage_retrieve_max]) return path_to_csv From f6258f8998b237d5137c3f59a8c8039a70b83acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Wed, 16 Sep 2020 20:43:37 -0400 Subject: [PATCH 16/25] newest version --- diagnostics_dante/diagnostic.py | 70 ++++++++++++++++++++++++-------- diagnostics_dante/test_videos.py | 49 +++++++++++++++++++--- 2 files changed, 97 insertions(+), 22 deletions(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 83d5ae1..61af25f 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -39,24 +39,45 @@ def getResourceUsage(done, mlist): cpu_usage = [] - mlist.append(cpu_usage) ram_usage = [] - mlist.append(ram_usage) + read_count = [] + write_count = [] + read_bytes = [] + write_bytes = [] while done.value: cpu_usage.append(psutil.cpu_percent()) ram_usage.append(psutil.virtual_memory().percent) - mlist[0] = cpu_usage - mlist[1] = ram_usage + disk_tuple = psutil.disk_io_counters() + read_count.append(disk_tuple[0]) + write_count.append(disk_tuple[1]) + read_bytes.append(disk_tuple[2]) + write_bytes.append(disk_tuple[3]) time.sleep(1) + mlist.append(cpu_usage) + mlist.append(ram_usage) + mlist.append(read_count) + mlist.append(write_count) + mlist.append(read_bytes) + mlist.append(write_bytes) def diagnostic(video_path, size): os.system("sudo sync; echo 1 | sudo tee /proc/sys/vm/drop_caches >/dev/null") file_size = None time_storage = None - usage_storage = None - time_retreive = None - usage_retrieve = None + cpu_storage = None + ram_storage = None + read_count_storage = None + write_count_storage = None + read_bytes_storage = None + write_bytes_storage = None + time_retrieve = None + cpu_retrieve = None + ram_retrieve = None + read_count_retrieve = None + write_count_retrieve = None + read_bytes_retrieve = None + write_bytes_retrieve = None FILENAME = video_path #the video file that you want to load LIMIT = 100 @@ -74,11 +95,16 @@ def diagnostic(video_path, size): t0 = time.time() # /dev/shm/ - cache = persist(vstream, 'cache.npz') #how big the size of the stored raw video is + cache = persist(vstream, '/dev/shm/cache.npz') #how big the size of the stored raw video is done.value -= 1 resource.join() - usage_storage = mlist + cpu_storage = mlist[0] + ram_storage = mlist[1] + read_count_storage = mlist[2] + write_count_storage = mlist[3] + read_bytes_storage = mlist[4] + write_bytes_storage = mlist[5] manager = Manager() mlist = manager.list() @@ -88,7 +114,7 @@ def diagnostic(video_path, size): t1 = time.time() - vstream = RawVideoStream('cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) + vstream = RawVideoStream('/dev/shm/cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) #do something for v in vstream: np.copy(v['data'], order='F') @@ -98,14 +124,26 @@ def diagnostic(video_path, size): done.value -= 1 resource.join() - usage_retrieve = mlist + cpu_retrieve = mlist[0] + ram_retrieve = mlist[1] + read_count_retrieve = mlist[2] + write_count_retrieve = mlist[3] + read_bytes_retrieve = mlist[4] + write_bytes_retrieve = mlist[5] time_storage = t1 - t0 - time_retreive = t2 - t1 - - file_size = os.path.getsize('cache.npz') - - return (file_size, time_storage, time_retreive, median(usage_storage), max(usage_storage), median(usage_retrieve), max(usage_retrieve)) + time_retrieve = t2 - t1 + + file_size = os.path.getsize('/dev/shm/cache.npz') + + return (file_size, time_storage, time_retrieve, \ + median(cpu_storage), max(cpu_storage), median(ram_storage), max(ram_storage), \ + median(read_count_storage), max(read_count_storage), median(write_count_storage), max(write_count_storage), \ + median(read_bytes_storage), max(read_bytes_storage), median(write_bytes_storage), max(write_bytes_storage), \ + median(cpu_retrieve), max(cpu_retrieve), median(ram_retrieve), max(ram_retrieve), \ + median(read_count_retrieve), max(read_count_retrieve), median(write_count_retrieve), max(write_count_retrieve), \ + median(read_bytes_retrieve), max(read_bytes_retrieve), median(write_bytes_retrieve), max(write_bytes_retrieve), \ + ) # Test #print(diagnostic('../tcam.mp4', (1080, 1080))) \ No newline at end of file diff --git a/diagnostics_dante/test_videos.py b/diagnostics_dante/test_videos.py index fb90fab..4aeb580 100644 --- a/diagnostics_dante/test_videos.py +++ b/diagnostics_dante/test_videos.py @@ -8,7 +8,14 @@ def test_videos(video_path, dictionary_of_video_sizes, number_of_trials): import csv with open(path_to_csv, 'w', newline='') as file: writer = csv.writer(file) - writer.writerow(["Shape", "Size", "Trial #", "File Size", "Storage Time", "Retrieval Time", "Storage Usage Median", "Storage Usage Max", "Retrieval Usage Median", "Retrieval Usage Max"]) + writer.writerow(["Shape", "Size", "Trial #", "File Size", "Storage Time", "Retrieval Time", \ + "Storage CPU Median", "Storage CPU Max", "Storage RAM Median", "Storage RAM Max", \ + "Storage Read Count Median", "Storage Read Count Max", "Storage Write Count Median", "Storage Write Count Max", \ + "Storage Read Bytes Median", "Storage Read Bytes Max", "Storage Write Bytes Median", "Storage Write Bytes Max", \ + "Retrieval CPU Median", "Retrieval CPU Max", "Retrieval RAM Median", "Retrieval RAM Max", \ + "Retrieval Read Count Median", "Retrieval Read Count Max", "Retrieval Write Count Median", "Retrieval Write Count Max", \ + "Retrieval Read Bytes Median", "Retrieval Read Bytes Max", "Retrieval Write Bytes Median", "Retrieval Write Bytes Max", \ + ]) print(dictionary_of_video_sizes) for shape, sizes in dictionary_of_video_sizes.items(): @@ -18,12 +25,42 @@ def test_videos(video_path, dictionary_of_video_sizes, number_of_trials): file_size = info[0] time_storage = info[1] time_retreive = info[2] - usage_storage_median = info[3] - usage_storage_max = info[4] - usage_retrieve_median = info[5] - usage_retrieve_max = info[6] + + cpu_storage_median = info[3] + cpu_storage_max = info[4] + ram_storage_median = info[5] + ram_storage_max = info[6] + read_count_storage_median = info[7] + read_count_storage_max = info[8] + write_count_storage_median = info[9] + write_count_storage_max = info[10] + read_bytes_storage_median = info[11] + read_bytes_storage_max = info[12] + write_bytes_storage_median = info[13] + write_bytes_storage_max = info[14] + + cpu_retrieval_median = info[15] + cpu_retrieval_max = info[16] + ram_retrieval_median = info[17] + ram_retrieval_max = info[18] + read_count_retrieval_median = info[19] + read_count_retrieval_max = info[20] + write_count_retrieval_median = info[21] + write_count_retrieval_max = info[22] + read_bytes_retrieval_median = info[23] + read_bytes_retrieval_max = info[24] + write_bytes_retrieval_median = info[25] + write_bytes_retrieval_max = info[26] + with open(path_to_csv, 'a', newline='') as file: writer = csv.writer(file) - writer.writerow([shape, size[0]*size[1], trial_number, file_size, time_storage, time_retreive, usage_storage_median, usage_storage_max, usage_retrieve_median, usage_retrieve_max]) + writer.writerow([shape, size[0]*size[1], trial_number, file_size, time_storage, time_retreive, \ + cpu_storage_median, cpu_storage_max, ram_storage_median, ram_storage_max, \ + read_count_storage_median, read_count_storage_max, write_count_storage_median, write_count_storage_max, \ + read_bytes_storage_median, read_bytes_storage_max, write_bytes_storage_median, write_bytes_storage_max, \ + cpu_retrieval_median, cpu_retrieval_max, ram_retrieval_median, ram_retrieval_max, \ + read_count_retrieval_median, read_count_retrieval_max, write_count_retrieval_median, write_count_retrieval_max, \ + read_bytes_retrieval_median, read_bytes_retrieval_max, write_bytes_retrieval_median, write_bytes_retrieval_max, \ + ]) return path_to_csv From 8b4eed0cbf519a10e36a66382da590f78cf3744e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Wed, 16 Sep 2020 23:13:36 -0400 Subject: [PATCH 17/25] C --- diagnostics_dante/diagnostic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 61af25f..615b80a 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -117,7 +117,7 @@ def diagnostic(video_path, size): vstream = RawVideoStream('/dev/shm/cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) #do something for v in vstream: - np.copy(v['data'], order='F') + np.copy(v['data'], order='C') pass t2 = time.time() From db22b6a3c23168bab6262dd30def2dd0e4a4b90d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Sun, 27 Sep 2020 16:09:31 -0500 Subject: [PATCH 18/25] lables --- diagnostics_dante/aggregate_csv.py | 18 +++++++++++++++--- diagnostics_dante/compare_diagnostics.py | 14 +++++++------- diagnostics_dante/test_videos.py | 2 +- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/diagnostics_dante/aggregate_csv.py b/diagnostics_dante/aggregate_csv.py index a44d8a1..0006c30 100644 --- a/diagnostics_dante/aggregate_csv.py +++ b/diagnostics_dante/aggregate_csv.py @@ -2,10 +2,22 @@ def aggregate_csv(path_to_csv): df = pd.read_csv(path_to_csv) - df = df.groupby(['Shape','Size']).agg({'File Size': ['min', 'median', 'max'], 'Storage Time': ['min', 'median', 'max'], 'Retrieval Time': ['min', 'median', 'max']}) + df = df.groupby(['Shape','Size']).agg({'File Size': ['min', 'median', 'max'], 'Storage Time': ['min', 'median', 'max'], 'Retrieval Time': ['min', 'median', 'max'], \ + "Storage CPU Median": ['median'], "Storage CPU Max": ['max'], "Storage RAM Median": ['median'], "Storage RAM Max": ['max'], \ + "Storage Read Count Median": ['median'], "Storage Read Count Max": ['max'], "Storage Write Count Median": ['median'], "Storage Write Count Max": ['max'], \ + "Storage Read Bytes Median": ['median'], "Storage Read Bytes Max": ['max'], "Storage Write Bytes Median": ['median'], "Storage Write Bytes Max": ['max'], \ + "Retrieval CPU Median": ['median'], "Retrieval CPU Max": ['max'], "Retrieval RAM Median": ['median'], "Retrieval RAM Max": ['max'], \ + "Retrieval Read Count Median": ['median'], "Retrieval Read Count Max": ['max'], "Retrieval Write Count Median": ['median'], "Retrieval Write Count Max": ['max'], \ + "Retrieval Read Bytes Median": ['median'], "Retrieval Read Bytes Max": ['max'], "Retrieval Write Bytes Median": ['median'], "Retrieval Write Bytes Max": ['max']}) df.reset_index(inplace=True) df.columns = df.columns.get_level_values(0) - df.columns = ['Shape', 'Size', 'File Size Min', 'File Size Median', 'File Size Max', 'Storage Time Min', 'Storage Time Median', 'Storage Time Max', 'Retrieval Time Min', 'Retrieval Time Median', 'Retrieval Time Max'] + df.columns = ['Shape', 'Size', 'File Size Min', 'File Size Median', 'File Size Max', 'Storage Time Min', 'Storage Time Median', 'Storage Time Max', 'Retrieval Time Min', 'Retrieval Time Median', 'Retrieval Time Max', \ + "Storage CPU Median", "Storage CPU Max", "Storage RAM Median", "Storage RAM Max", \ + "Storage Read Count Median", "Storage Read Count Max", "Storage Write Count Median", "Storage Write Count Max", \ + "Storage Read Bytes Median", "Storage Read Bytes Max", "Storage Write Bytes Median", "Storage Write Bytes Max", \ + "Retrieval CPU Median", "Retrieval CPU Max", "Retrieval RAM Median", "Retrieval RAM Max", \ + "Retrieval Read Count Median", "Retrieval Read Count Max", "Retrieval Write Count Median", "Retrieval Write Count Max", \ + "Retrieval Read Bytes Median", "Retrieval Read Bytes Max", "Retrieval Write Bytes Median", "Retrieval Write Bytes Max"] df = df.sort_values(by='Size') df['Size'] = df['Size'].apply(lambda x: human_format(x)) @@ -23,4 +35,4 @@ def human_format(num): return '{}{}'.format('{:f}'.format(num).rstrip('0').rstrip('.'), ['', 'K', 'M', 'B', 'T'][magnitude]) # Test -#aggregate_csv("results/diagnostics.csv") \ No newline at end of file +aggregate_csv("results/diagnostics.csv") \ No newline at end of file diff --git a/diagnostics_dante/compare_diagnostics.py b/diagnostics_dante/compare_diagnostics.py index 84bcade..569cf14 100644 --- a/diagnostics_dante/compare_diagnostics.py +++ b/diagnostics_dante/compare_diagnostics.py @@ -12,7 +12,7 @@ import matplotlib.pyplot as plt def graph_compare(path_to_many_results, shapes): - os.mkdir('comparison_results') + #os.mkdir('comparison_results') results = [] for dirpath, dirnames, filenames in os.walk(path_to_many_results): for fname in filenames: @@ -32,23 +32,23 @@ def graph_compare(path_to_many_results, shapes): dfo['Total Time'] = dfo.apply(lambda x: x['Retrieval Time Median'] + x['Storage Time Median'], axis=1) for shape, df in dfo.groupby('Shape'): sizes = df['Size'].tolist() - data = df['Total Time'].tolist() + data = df["Retrieval Write Count Median"].tolist() dict_data_by_shapes[shape].append((name, sizes, data)) - + for shape in dict_data_by_shapes: fig = plt.figure(figsize = (10, 5)) results = dict_data_by_shapes[shape] for result in results: plt.plot(result[1], result[2], label=result[0]) plt.xlabel('Frame Size (pixels)') - plt.ylabel('Total Time (seconds)') - plt.title(f'Total Time of {shape} results') + #plt.ylabel("Storage Read Count Median") + #plt.title(f'"Storage Read Count Median" of {shape} results') plt.legend() - plt.savefig(f"comparison_results/{shape}_totaltime.png") + plt.savefig(f"comparison_results/{shape}_retrievalwritecount.png") # Test -#graph_compare("../../Results", [(16,9), (4,3), (1,1), (10,1), (1,10), (20,1), (1,20)]) \ No newline at end of file +graph_compare("../../Results Usage", [(16,9)]) \ No newline at end of file diff --git a/diagnostics_dante/test_videos.py b/diagnostics_dante/test_videos.py index 4aeb580..4f8382b 100644 --- a/diagnostics_dante/test_videos.py +++ b/diagnostics_dante/test_videos.py @@ -14,7 +14,7 @@ def test_videos(video_path, dictionary_of_video_sizes, number_of_trials): "Storage Read Bytes Median", "Storage Read Bytes Max", "Storage Write Bytes Median", "Storage Write Bytes Max", \ "Retrieval CPU Median", "Retrieval CPU Max", "Retrieval RAM Median", "Retrieval RAM Max", \ "Retrieval Read Count Median", "Retrieval Read Count Max", "Retrieval Write Count Median", "Retrieval Write Count Max", \ - "Retrieval Read Bytes Median", "Retrieval Read Bytes Max", "Retrieval Write Bytes Median", "Retrieval Write Bytes Max", \ + "Retrieval Read Bytes Median", "Retrieval Read Bytes Max", "Retrieval Write Bytes Median", "Retrieval Write Bytes Max" \ ]) print(dictionary_of_video_sizes) From b4a2e55cc38a3401681fd7861df1280bd1503a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Sun, 27 Sep 2020 16:15:03 -0500 Subject: [PATCH 19/25] fix --- diagnostics_dante/compare_diagnostics.py | 16 ++++------------ diagnostics_dante/graph_diagnostics.py | 9 --------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/diagnostics_dante/compare_diagnostics.py b/diagnostics_dante/compare_diagnostics.py index 569cf14..f9ffafd 100644 --- a/diagnostics_dante/compare_diagnostics.py +++ b/diagnostics_dante/compare_diagnostics.py @@ -1,18 +1,9 @@ -# Script that compares different results as line graph -# Take all folders in path folder -# Go in folder -> go in results -> take aggregate csv file -# Make a pandas df -# separate by shape -# add label that says what results they are from -# graph? -# add key and x, y, title - import os import pandas as pd import matplotlib.pyplot as plt def graph_compare(path_to_many_results, shapes): - #os.mkdir('comparison_results') + os.mkdir('comparison_results') results = [] for dirpath, dirnames, filenames in os.walk(path_to_many_results): for fname in filenames: @@ -32,6 +23,7 @@ def graph_compare(path_to_many_results, shapes): dfo['Total Time'] = dfo.apply(lambda x: x['Retrieval Time Median'] + x['Storage Time Median'], axis=1) for shape, df in dfo.groupby('Shape'): sizes = df['Size'].tolist() + # change to be for every col data = df["Retrieval Write Count Median"].tolist() dict_data_by_shapes[shape].append((name, sizes, data)) @@ -41,8 +33,8 @@ def graph_compare(path_to_many_results, shapes): for result in results: plt.plot(result[1], result[2], label=result[0]) plt.xlabel('Frame Size (pixels)') - #plt.ylabel("Storage Read Count Median") - #plt.title(f'"Storage Read Count Median" of {shape} results') + plt.ylabel("Storage Read Count Median") + plt.title(f'"Storage Read Count Median" of {shape} results') plt.legend() plt.savefig(f"comparison_results/{shape}_retrievalwritecount.png") diff --git a/diagnostics_dante/graph_diagnostics.py b/diagnostics_dante/graph_diagnostics.py index d3b81eb..28bb739 100644 --- a/diagnostics_dante/graph_diagnostics.py +++ b/diagnostics_dante/graph_diagnostics.py @@ -81,15 +81,6 @@ def graph_diagnostics(path_to_csv, path_to_agg_csv): plt.savefig(f"results/graphs/total_time/{shape}_totaltime.png") fig.clf() - # for size, df in dfo.groupby('Size'): - # fig0 = plt.figure(figsize = (10, 5)) - # p1 = plt.bar(df['Shape'], height=df['Storage Time Median']) - # p2 = plt.bar(df['Shape'], height=df['Retrieval Time Median'], bottom=df['Storage Time Median']) - # plt.xlabel("Shape") - # plt.ylabel("Total Time (seconds)") - # plt.title(f"{size} Total Time") - # plt.savefig(f"results/graphs/{size}_totaltime.png") - path_to_graphs = 'results/graphs' # A folder of graphs return path_to_graphs From 741f2c06ce98da6fd8d2be13c19ffc8ee5830448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Mon, 28 Sep 2020 23:04:33 -0500 Subject: [PATCH 20/25] wacky autoencoder --- diagnostics_dante/autoencoder.py | 73 ++++++++++++++++++++++++++++++++ diagnostics_dante/diagnostic.py | 8 +++- 2 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 diagnostics_dante/autoencoder.py diff --git a/diagnostics_dante/autoencoder.py b/diagnostics_dante/autoencoder.py new file mode 100644 index 0000000..518fbbe --- /dev/null +++ b/diagnostics_dante/autoencoder.py @@ -0,0 +1,73 @@ +import tensorflow.keras.layers +import tensorflow.keras.models +import tensorflow.keras.optimizers +import tensorflow.keras.datasets +import numpy +import matplotlib.pyplot + +# Encoder +x = tensorflow.keras.layers.Input(shape=(784), name="encoder_input") + +encoder_dense_layer1 = tensorflow.keras.layers.Dense(units=300, name="encoder_dense_1")(x) +encoder_activ_layer1 = tensorflow.keras.layers.LeakyReLU(name="encoder_leakyrelu_1")(encoder_dense_layer1) + +encoder_dense_layer2 = tensorflow.keras.layers.Dense(units=2, name="encoder_dense_2")(encoder_activ_layer1) +encoder_output = tensorflow.keras.layers.LeakyReLU(name="encoder_output")(encoder_dense_layer2) + +encoder = tensorflow.keras.models.Model(x, encoder_output, name="encoder_model") +encoder.summary() + +# Decoder +decoder_input = tensorflow.keras.layers.Input(shape=(2), name="decoder_input") + +decoder_dense_layer1 = tensorflow.keras.layers.Dense(units=300, name="decoder_dense_1")(decoder_input) +decoder_activ_layer1 = tensorflow.keras.layers.LeakyReLU(name="decoder_leakyrelu_1")(decoder_dense_layer1) + +decoder_dense_layer2 = tensorflow.keras.layers.Dense(units=784, name="decoder_dense_2")(decoder_activ_layer1) +decoder_output = tensorflow.keras.layers.LeakyReLU(name="decoder_output")(decoder_dense_layer2) + +decoder = tensorflow.keras.models.Model(decoder_input, decoder_output, name="decoder_model") +decoder.summary() + +# Autoencoder +ae_input = tensorflow.keras.layers.Input(shape=(784), name="AE_input") +ae_encoder_output = encoder(ae_input) +ae_decoder_output = decoder(ae_encoder_output) + +ae = tensorflow.keras.models.Model(ae_input, ae_decoder_output, name="AE") +ae.summary() + +# RMSE +def rmse(y_true, y_predict): + return tensorflow.keras.backend.mean(tensorflow.keras.backend.square(y_true-y_predict)) + +# AE Compilation +ae.compile(loss="mse", optimizer=tensorflow.keras.optimizers.Adam(lr=0.0005)) + +# Preparing MNIST Dataset +(x_train_orig, y_train), (x_test_orig, y_test) = tensorflow.keras.datasets.mnist.load_data() +x_train_orig = x_train_orig.astype("float32") / 255.0 +x_test_orig = x_test_orig.astype("float32") / 255.0 + +x_train = numpy.reshape(x_train_orig, newshape=(x_train_orig.shape[0], numpy.prod(x_train_orig.shape[1:]))) +x_test = numpy.reshape(x_test_orig, newshape=(x_test_orig.shape[0], numpy.prod(x_test_orig.shape[1:]))) + +# Training AE +ae.fit(x_train, x_train, epochs=20, batch_size=256, shuffle=True, validation_data=(x_test, x_test)) + +encoded_images = encoder.predict(x_train) +decoded_images = decoder.predict(encoded_images) +decoded_images_orig = numpy.reshape(decoded_images, newshape=(decoded_images.shape[0], 28, 28)) + +num_images_to_show = 5 +for im_ind in range(num_images_to_show): + plot_ind = im_ind*2 + 1 + rand_ind = numpy.random.randint(low=0, high=x_train.shape[0]) + matplotlib.pyplot.subplot(num_images_to_show, 2, plot_ind) + matplotlib.pyplot.imshow(x_train_orig[rand_ind, :, :], cmap="gray") + matplotlib.pyplot.subplot(num_images_to_show, 2, plot_ind+1) + matplotlib.pyplot.imshow(decoded_images_orig[rand_ind, :, :], cmap="gray") + +matplotlib.pyplot.figure() +matplotlib.pyplot.scatter(encoded_images[:, 0], encoded_images[:, 1], c=y_train) +matplotlib.pyplot.colorbar() \ No newline at end of file diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 615b80a..d7c3400 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -94,8 +94,9 @@ def diagnostic(video_path, size): t0 = time.time() - # /dev/shm/ - cache = persist(vstream, '/dev/shm/cache.npz') #how big the size of the stored raw video is + # compress vstream with autoencoder + + cache = persist(vstream, '/dev/shm/cache.npz') done.value -= 1 resource.join() @@ -115,6 +116,9 @@ def diagnostic(video_path, size): t1 = time.time() vstream = RawVideoStream('/dev/shm/cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) + + # decompress vstream + #do something for v in vstream: np.copy(v['data'], order='C') From 0cb783b062e22aaef4e909bbef6ef11501678d26 Mon Sep 17 00:00:00 2001 From: "drgilmarin@uchicago.edu" Date: Tue, 3 Nov 2020 13:21:45 -0600 Subject: [PATCH 21/25] testing compression --- diagnostics_dante/aggregate_csv.py | 2 +- diagnostics_dante/compare_diagnostics.py | 2 +- diagnostics_dante/diagnostic.py | 6 +- diagnostics_dante/generate_videos.py | 1 + diagnostics_dante/main.py | 2 +- diagnostics_dante/results/diagnostics.csv | 2 + diagnostics_dante/tfci.py | 297 ++++++++++++++++++++++ 7 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 diagnostics_dante/results/diagnostics.csv create mode 100644 diagnostics_dante/tfci.py diff --git a/diagnostics_dante/aggregate_csv.py b/diagnostics_dante/aggregate_csv.py index 0006c30..53a8783 100644 --- a/diagnostics_dante/aggregate_csv.py +++ b/diagnostics_dante/aggregate_csv.py @@ -35,4 +35,4 @@ def human_format(num): return '{}{}'.format('{:f}'.format(num).rstrip('0').rstrip('.'), ['', 'K', 'M', 'B', 'T'][magnitude]) # Test -aggregate_csv("results/diagnostics.csv") \ No newline at end of file +#aggregate_csv("results/diagnostics.csv") \ No newline at end of file diff --git a/diagnostics_dante/compare_diagnostics.py b/diagnostics_dante/compare_diagnostics.py index f9ffafd..31dda28 100644 --- a/diagnostics_dante/compare_diagnostics.py +++ b/diagnostics_dante/compare_diagnostics.py @@ -43,4 +43,4 @@ def graph_compare(path_to_many_results, shapes): # Test -graph_compare("../../Results Usage", [(16,9)]) \ No newline at end of file +#graph_compare("../../Results Usage", [(16,9)]) \ No newline at end of file diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index d7c3400..554c2de 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -1,6 +1,8 @@ import os import sys import inspect +import tfci +import tensorflow.compat.v1 as tf # Is there a better way? currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) @@ -94,7 +96,9 @@ def diagnostic(video_path, size): t0 = time.time() - # compress vstream with autoencoder + for v in vstream: + vi = v['data'] + tfci.compress('mbt2018-mean-msssim-8', vi, 't.tfci') cache = persist(vstream, '/dev/shm/cache.npz') diff --git a/diagnostics_dante/generate_videos.py b/diagnostics_dante/generate_videos.py index ee21b3c..57660f8 100644 --- a/diagnostics_dante/generate_videos.py +++ b/diagnostics_dante/generate_videos.py @@ -18,6 +18,7 @@ def generate_videos(video_width, video_height, shapes, max_pixels, min_pixels, r height_temp = math.ceil(height_temp * math.sqrt(reducing_factor)) width_temp = math.ceil(width_temp * math.sqrt(reducing_factor)) dictionary_of_video_sizes[key].append((width_temp, height_temp)) + dictionary_of_video_sizes['16:9'] = list(reversed(dictionary_of_video_sizes['16:9'])) return dictionary_of_video_sizes # Test diff --git a/diagnostics_dante/main.py b/diagnostics_dante/main.py index 22d8c8f..1af99ba 100644 --- a/diagnostics_dante/main.py +++ b/diagnostics_dante/main.py @@ -6,7 +6,7 @@ video_path = '4KFootage.mkv' video_width = 3840 video_height = 2160 -number_of_trials = 50 +number_of_trials = 1 shapes = [(16,9)] # An array of aspect ratio tuples (width, height) diff --git a/diagnostics_dante/results/diagnostics.csv b/diagnostics_dante/results/diagnostics.csv new file mode 100644 index 0000000..418005f --- /dev/null +++ b/diagnostics_dante/results/diagnostics.csv @@ -0,0 +1,2 @@ +Shape,Size,Trial #,File Size,Storage Time,Retrieval Time,Storage CPU Median,Storage CPU Max,Storage RAM Median,Storage RAM Max,Storage Read Count Median,Storage Read Count Max,Storage Write Count Median,Storage Write Count Max,Storage Read Bytes Median,Storage Read Bytes Max,Storage Write Bytes Median,Storage Write Bytes Max,Retrieval CPU Median,Retrieval CPU Max,Retrieval RAM Median,Retrieval RAM Max,Retrieval Read Count Median,Retrieval Read Count Max,Retrieval Write Count Median,Retrieval Write Count Max,Retrieval Read Bytes Median,Retrieval Read Bytes Max,Retrieval Write Bytes Median,Retrieval Write Bytes Max +16:9,16684,0,5005200,67.17157697677612,0.010992765426635742,18.7,48.0,43.1,44.4,352810,352850,75638,75685,9593881600,9599149056,4030365696,4030783488,20.5,20.5,42.9,42.9,352851,352851,75685,75685,9599165440,9599165440,4030783488,4030783488 diff --git a/diagnostics_dante/tfci.py b/diagnostics_dante/tfci.py new file mode 100644 index 0000000..5179cf6 --- /dev/null +++ b/diagnostics_dante/tfci.py @@ -0,0 +1,297 @@ +# Copyright 2019 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Converts an image between PNG and TFCI formats. + +Use this script to compress images with pre-trained models as published. See the +'models' subcommand for a list of available models. +""" + +import argparse +import os +import sys +import urllib + +from absl import app +from absl.flags import argparse_flags +import tensorflow.compat.v1 as tf + +import tensorflow_compression as tfc # pylint:disable=unused-import + +# Default URL to fetch metagraphs from. +URL_PREFIX = "https://storage.googleapis.com/tensorflow_compression/metagraphs" +# Default location to store cached metagraphs. +METAGRAPH_CACHE = "/tmp/tfc_metagraphs" + + +def read_png(image): + """Creates graph to load a PNG image file.""" + # string = tf.io.read_file(filename) + # image = tf.image.decode_image(string) + image = tf.expand_dims(image, 0) + return image + + +def write_png(filename, image): + """Creates graph to write a PNG image file.""" + image = tf.squeeze(image, 0) + if image.dtype.is_floating: + image = tf.round(image) + if image.dtype != tf.uint8: + image = tf.saturate_cast(image, tf.uint8) + string = tf.image.encode_png(image) + return tf.io.write_file(filename, string) + + +def load_cached(filename): + """Downloads and caches files from web storage.""" + pathname = os.path.join(METAGRAPH_CACHE, filename) + try: + with tf.io.gfile.GFile(pathname, "rb") as f: + string = f.read() + except tf.errors.NotFoundError: + url = URL_PREFIX + "/" + filename + try: + request = urllib.request.urlopen(url) + string = request.read() + finally: + request.close() + tf.io.gfile.makedirs(os.path.dirname(pathname)) + with tf.io.gfile.GFile(pathname, "wb") as f: + f.write(string) + return string + + +def import_metagraph(model): + """Imports a trained model metagraph into the current graph.""" + string = load_cached(model + ".metagraph") + metagraph = tf.MetaGraphDef() + metagraph.ParseFromString(string) + tf.train.import_meta_graph(metagraph) + return metagraph.signature_def + + +def instantiate_signature(signature_def): + """Fetches tensors defined in a signature from the graph.""" + graph = tf.get_default_graph() + inputs = { + k: graph.get_tensor_by_name(v.name) + for k, v in signature_def.inputs.items() + } + outputs = { + k: graph.get_tensor_by_name(v.name) + for k, v in signature_def.outputs.items() + } + return inputs, outputs + + +def compress_image(model, input_image): + """Compresses an image array into a bitstring.""" + with tf.Graph().as_default(): + # Load model metagraph. + signature_defs = import_metagraph(model) + inputs, outputs = instantiate_signature(signature_defs["sender"]) + + # Just one input tensor. + inputs = inputs["input_image"] + # Multiple output tensors, ordered alphabetically, without names. + outputs = [outputs[k] for k in sorted(outputs) if k.startswith("channel:")] + + # Run encoder. + with tf.Session() as sess: + arrays = sess.run(outputs, feed_dict={inputs: input_image}) + + # Pack data into bitstring. + # packed = tfc.PackedTensors() + # packed.model = model + # packed.pack(outputs, arrays) + return arrays + + +def compress(model, input_file, output_file, target_bpp=None, bpp_strict=False): + """Compresses a PNG file to a TFCI file.""" + if not output_file: + output_file = input_file + ".tfci" + + # Load image. + with tf.Graph().as_default(): + with tf.Session() as sess: + input_image = sess.run(read_png(input_file)) + num_pixels = input_image.shape[-2] * input_image.shape[-3] + + if not target_bpp: + # Just compress with a specific model. + bitstring = compress_image(model, input_image) + else: + # Get model list. + models = load_cached(model + ".models") + models = models.decode("ascii").split() + + # Do a binary search over all RD points. + lower = -1 + upper = len(models) + bpp = None + best_bitstring = None + best_bpp = None + while bpp != target_bpp and upper - lower > 1: + i = (upper + lower) // 2 + bitstring = compress_image(models[i], input_image) + bpp = 8 * len(bitstring) / num_pixels + is_admissible = bpp <= target_bpp or not bpp_strict + is_better = (best_bpp is None or + abs(bpp - target_bpp) < abs(best_bpp - target_bpp)) + if is_admissible and is_better: + best_bitstring = bitstring + best_bpp = bpp + if bpp < target_bpp: + lower = i + if bpp > target_bpp: + upper = i + if best_bpp is None: + assert bpp_strict + raise RuntimeError( + "Could not compress image to less than {} bpp.".format(target_bpp)) + bitstring = best_bitstring + + # Write bitstring to disk. + # with tf.io.gfile.GFile(output_file, "wb") as f: + # f.write(bitstring) + + +def decompress(input_file, output_file): + """Decompresses a TFCI file and writes a PNG file.""" + if not output_file: + output_file = input_file + ".png" + + with tf.Graph().as_default(): + # Unserialize packed data from disk. + with tf.io.gfile.GFile(input_file, "rb") as f: + packed = tfc.PackedTensors(f.read()) + + # Load model metagraph. + signature_defs = import_metagraph(packed.model) + inputs, outputs = instantiate_signature(signature_defs["receiver"]) + + # Multiple input tensors, ordered alphabetically, without names. + inputs = [inputs[k] for k in sorted(inputs) if k.startswith("channel:")] + # Just one output operation. + outputs = write_png(output_file, outputs["output_image"]) + + # Unpack data. + arrays = packed.unpack(inputs) + + # Run decoder. + with tf.Session() as sess: + sess.run(outputs, feed_dict=dict(zip(inputs, arrays))) + + +def list_models(): + url = URL_PREFIX + "/models.txt" + try: + request = urllib.request.urlopen(url) + print(request.read().decode("utf-8")) + finally: + request.close() + + +def parse_args(argv): + """Parses command line arguments.""" + parser = argparse_flags.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + # High-level options. + parser.add_argument( + "--url_prefix", + default=URL_PREFIX, + help="URL prefix for downloading model metagraphs.") + parser.add_argument( + "--metagraph_cache", + default=METAGRAPH_CACHE, + help="Directory where to cache model metagraphs.") + subparsers = parser.add_subparsers( + title="commands", dest="command", + help="Invoke ' -h' for more information.") + + # 'compress' subcommand. + compress_cmd = subparsers.add_parser( + "compress", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Reads a PNG file, compresses it using the given model, and " + "writes a TFCI file.") + compress_cmd.add_argument( + "model", + help="Unique model identifier. See 'models' command for options. If " + "'target_bpp' is provided, don't specify the index at the end of " + "the model identifier.") + compress_cmd.add_argument( + "--target_bpp", type=float, + help="Target bits per pixel. If provided, a binary search is used to try " + "to match the given bpp as close as possible. In this case, don't " + "specify the index at the end of the model identifier. It will be " + "automatically determined.") + compress_cmd.add_argument( + "--bpp_strict", action="store_true", + help="Try never to exceed 'target_bpp'. Ignored if 'target_bpp' is not " + "set.") + + # 'decompress' subcommand. + decompress_cmd = subparsers.add_parser( + "decompress", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Reads a TFCI file, reconstructs the image using the model " + "it was compressed with, and writes back a PNG file.") + + # Arguments for both 'compress' and 'decompress'. + for cmd, ext in ((compress_cmd, ".tfci"), (decompress_cmd, ".png")): + cmd.add_argument( + "input_file", + help="Input filename.") + cmd.add_argument( + "output_file", nargs="?", + help="Output filename (optional). If not provided, appends '{}' to " + "the input filename.".format(ext)) + + # 'models' subcommand. + subparsers.add_parser( + "models", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Lists available trained models. Requires an internet " + "connection.") + + # Parse arguments. + args = parser.parse_args(argv[1:]) + if args.command is None: + parser.print_usage() + sys.exit(2) + return args + + +def main(args): + # Command line can override these defaults. + global URL_PREFIX, METAGRAPH_CACHE + URL_PREFIX = args.url_prefix + METAGRAPH_CACHE = args.metagraph_cache + + # Invoke subcommand. + if args.command == "compress": + compress(args.model, args.input_file, args.output_file, + args.target_bpp, args.bpp_strict) + if args.command == "decompress": + decompress(args.input_file, args.output_file) + if args.command == "models": + list_models() + + +if __name__ == "__main__": + app.run(main, flags_parser=parse_args) From db90f2590913c31f1b92c673cd03f57485a3600e Mon Sep 17 00:00:00 2001 From: "drgilmarin@uchicago.edu" Date: Mon, 9 Nov 2020 21:13:47 -0600 Subject: [PATCH 22/25] first iteration of full pipeline --- diagnostics_dante/diagnostic.py | 16 +++- diagnostics_dante/tfci.py | 165 +++----------------------------- 2 files changed, 25 insertions(+), 156 deletions(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 554c2de..ccd99f1 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -96,9 +96,10 @@ def diagnostic(video_path, size): t0 = time.time() - for v in vstream: - vi = v['data'] - tfci.compress('mbt2018-mean-msssim-8', vi, 't.tfci') + os.mkdir('cache') + for i, v in enumerate(vstream): + tfci.compress('mbt2018-mean-msssim-8', v['data'], f"cache/{i}.tfci") + v['data'] = None cache = persist(vstream, '/dev/shm/cache.npz') @@ -121,7 +122,8 @@ def diagnostic(video_path, size): vstream = RawVideoStream('/dev/shm/cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) - # decompress vstream + for i, v in enumerate(vstream): + v['data'] = tfci.decompress(f"cache/{i}.tfci") #do something for v in vstream: @@ -142,7 +144,11 @@ def diagnostic(video_path, size): time_storage = t1 - t0 time_retrieve = t2 - t1 - file_size = os.path.getsize('/dev/shm/cache.npz') + total_size = 0 + for root, dirs, files in os.walk("cache"): + for f in files: + total_size += os.path.getsize(os.path.join(root, f)) + file_size = total_size return (file_size, time_storage, time_retrieve, \ median(cpu_storage), max(cpu_storage), median(ram_storage), max(ram_storage), \ diff --git a/diagnostics_dante/tfci.py b/diagnostics_dante/tfci.py index 5179cf6..db46a07 100644 --- a/diagnostics_dante/tfci.py +++ b/diagnostics_dante/tfci.py @@ -1,23 +1,3 @@ -# Copyright 2019 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Converts an image between PNG and TFCI formats. - -Use this script to compress images with pre-trained models as published. See the -'models' subcommand for a list of available models. -""" - import argparse import os import sys @@ -34,24 +14,15 @@ # Default location to store cached metagraphs. METAGRAPH_CACHE = "/tmp/tfc_metagraphs" - -def read_png(image): - """Creates graph to load a PNG image file.""" - # string = tf.io.read_file(filename) - # image = tf.image.decode_image(string) - image = tf.expand_dims(image, 0) - return image - - -def write_png(filename, image): +def write_array(image): """Creates graph to write a PNG image file.""" image = tf.squeeze(image, 0) if image.dtype.is_floating: image = tf.round(image) if image.dtype != tf.uint8: image = tf.saturate_cast(image, tf.uint8) - string = tf.image.encode_png(image) - return tf.io.write_file(filename, string) + # string = tf.image.encode_png(image) + return image def load_cached(filename): @@ -113,21 +84,17 @@ def compress_image(model, input_image): arrays = sess.run(outputs, feed_dict={inputs: input_image}) # Pack data into bitstring. - # packed = tfc.PackedTensors() - # packed.model = model - # packed.pack(outputs, arrays) - return arrays + packed = tfc.PackedTensors() + packed.model = model + packed.pack(outputs, arrays) + return packed.string -def compress(model, input_file, output_file, target_bpp=None, bpp_strict=False): - """Compresses a PNG file to a TFCI file.""" - if not output_file: - output_file = input_file + ".tfci" - +def compress(model, input_array, output_file, target_bpp=None, bpp_strict=False): # Load image. with tf.Graph().as_default(): with tf.Session() as sess: - input_image = sess.run(read_png(input_file)) + input_image = sess.run(tf.expand_dims(input_array, 0)) num_pixels = input_image.shape[-2] * input_image.shape[-3] if not target_bpp: @@ -165,15 +132,11 @@ def compress(model, input_file, output_file, target_bpp=None, bpp_strict=False): bitstring = best_bitstring # Write bitstring to disk. - # with tf.io.gfile.GFile(output_file, "wb") as f: - # f.write(bitstring) - + with tf.io.gfile.GFile(output_file, "wb") as f: + f.write(bitstring) -def decompress(input_file, output_file): - """Decompresses a TFCI file and writes a PNG file.""" - if not output_file: - output_file = input_file + ".png" +def decompress(input_file): with tf.Graph().as_default(): # Unserialize packed data from disk. with tf.io.gfile.GFile(input_file, "rb") as f: @@ -186,7 +149,7 @@ def decompress(input_file, output_file): # Multiple input tensors, ordered alphabetically, without names. inputs = [inputs[k] for k in sorted(inputs) if k.startswith("channel:")] # Just one output operation. - outputs = write_png(output_file, outputs["output_image"]) + outputs = write_array(outputs["output_image"]) # Unpack data. arrays = packed.unpack(inputs) @@ -194,104 +157,4 @@ def decompress(input_file, output_file): # Run decoder. with tf.Session() as sess: sess.run(outputs, feed_dict=dict(zip(inputs, arrays))) - - -def list_models(): - url = URL_PREFIX + "/models.txt" - try: - request = urllib.request.urlopen(url) - print(request.read().decode("utf-8")) - finally: - request.close() - - -def parse_args(argv): - """Parses command line arguments.""" - parser = argparse_flags.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - # High-level options. - parser.add_argument( - "--url_prefix", - default=URL_PREFIX, - help="URL prefix for downloading model metagraphs.") - parser.add_argument( - "--metagraph_cache", - default=METAGRAPH_CACHE, - help="Directory where to cache model metagraphs.") - subparsers = parser.add_subparsers( - title="commands", dest="command", - help="Invoke ' -h' for more information.") - - # 'compress' subcommand. - compress_cmd = subparsers.add_parser( - "compress", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description="Reads a PNG file, compresses it using the given model, and " - "writes a TFCI file.") - compress_cmd.add_argument( - "model", - help="Unique model identifier. See 'models' command for options. If " - "'target_bpp' is provided, don't specify the index at the end of " - "the model identifier.") - compress_cmd.add_argument( - "--target_bpp", type=float, - help="Target bits per pixel. If provided, a binary search is used to try " - "to match the given bpp as close as possible. In this case, don't " - "specify the index at the end of the model identifier. It will be " - "automatically determined.") - compress_cmd.add_argument( - "--bpp_strict", action="store_true", - help="Try never to exceed 'target_bpp'. Ignored if 'target_bpp' is not " - "set.") - - # 'decompress' subcommand. - decompress_cmd = subparsers.add_parser( - "decompress", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description="Reads a TFCI file, reconstructs the image using the model " - "it was compressed with, and writes back a PNG file.") - - # Arguments for both 'compress' and 'decompress'. - for cmd, ext in ((compress_cmd, ".tfci"), (decompress_cmd, ".png")): - cmd.add_argument( - "input_file", - help="Input filename.") - cmd.add_argument( - "output_file", nargs="?", - help="Output filename (optional). If not provided, appends '{}' to " - "the input filename.".format(ext)) - - # 'models' subcommand. - subparsers.add_parser( - "models", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description="Lists available trained models. Requires an internet " - "connection.") - - # Parse arguments. - args = parser.parse_args(argv[1:]) - if args.command is None: - parser.print_usage() - sys.exit(2) - return args - - -def main(args): - # Command line can override these defaults. - global URL_PREFIX, METAGRAPH_CACHE - URL_PREFIX = args.url_prefix - METAGRAPH_CACHE = args.metagraph_cache - - # Invoke subcommand. - if args.command == "compress": - compress(args.model, args.input_file, args.output_file, - args.target_bpp, args.bpp_strict) - if args.command == "decompress": - decompress(args.input_file, args.output_file) - if args.command == "models": - list_models() - - -if __name__ == "__main__": - app.run(main, flags_parser=parse_args) + return outputs \ No newline at end of file From 58a317cdde22eee6ae5279cdb6275cd35988222a Mon Sep 17 00:00:00 2001 From: "drgilmarin@uchicago.edu" Date: Mon, 9 Nov 2020 21:19:34 -0600 Subject: [PATCH 23/25] bugs --- diagnostics_dante/diagnostic.py | 3 +++ diagnostics_dante/main.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index ccd99f1..8126c5c 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -38,6 +38,7 @@ import psutil from multiprocessing import Process, Value, Manager from statistics import median +import shutil def getResourceUsage(done, mlist): cpu_usage = [] @@ -150,6 +151,8 @@ def diagnostic(video_path, size): total_size += os.path.getsize(os.path.join(root, f)) file_size = total_size + shutil.rmtree("cache") + return (file_size, time_storage, time_retrieve, \ median(cpu_storage), max(cpu_storage), median(ram_storage), max(ram_storage), \ median(read_count_storage), max(read_count_storage), median(write_count_storage), max(write_count_storage), \ diff --git a/diagnostics_dante/main.py b/diagnostics_dante/main.py index 1af99ba..672278e 100644 --- a/diagnostics_dante/main.py +++ b/diagnostics_dante/main.py @@ -22,7 +22,7 @@ path_to_csv = test_videos(video_path, dictionary_of_video_sizes, number_of_trials) # Aggregate csv by shape and size (min, med, max) -#path_to_agg_csv = aggregate_csv(path_to_csv) +path_to_agg_csv = aggregate_csv(path_to_csv) # Graph aggregate csv and put results in new folder -#path_to_graphs = graph_diagnostics(path_to_csv, path_to_agg_csv) +path_to_graphs = graph_diagnostics(path_to_csv, path_to_agg_csv) From 5b305c214198f609f9fd3f19c5edb41dcd23fa07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Tue, 17 Nov 2020 12:03:31 -0600 Subject: [PATCH 24/25] conventional cpmpression --- diagnostics_dante/diagnostic.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/diagnostics_dante/diagnostic.py b/diagnostics_dante/diagnostic.py index 8126c5c..40ae9d8 100644 --- a/diagnostics_dante/diagnostic.py +++ b/diagnostics_dante/diagnostic.py @@ -3,6 +3,7 @@ import inspect import tfci import tensorflow.compat.v1 as tf +import gzip # Is there a better way? currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) @@ -99,7 +100,10 @@ def diagnostic(video_path, size): os.mkdir('cache') for i, v in enumerate(vstream): - tfci.compress('mbt2018-mean-msssim-8', v['data'], f"cache/{i}.tfci") + f = gzip.GzipFile(f"cache/{i}.tfci", "w") + np.save(file=f, arr=v) + f.close() + #tfci.compress('mbt2018-mean-msssim-8', v['data'], f"cache/{i}.tfci") v['data'] = None cache = persist(vstream, '/dev/shm/cache.npz') @@ -124,6 +128,9 @@ def diagnostic(video_path, size): vstream = RawVideoStream('/dev/shm/cache.npz', shape=(LIMIT,size[1],size[0],3)) #retrieving the data (have to provide dimensions (num frames, w, h, channels) for i, v in enumerate(vstream): + f = gzip.GzipFile(f"cache/{i}.tfci", "w") + np.load(f) + f.close() v['data'] = tfci.decompress(f"cache/{i}.tfci") #do something From 36a7b0209a231f2ff6d41f8996bbd1ec59069d66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dante=20Ram=C3=B3n=20Gil-Mar=C3=ADn?= Date: Mon, 11 Jan 2021 11:49:51 -0400 Subject: [PATCH 25/25] revert later --- diagnostics_dante/results/diagnostics.csv | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 diagnostics_dante/results/diagnostics.csv diff --git a/diagnostics_dante/results/diagnostics.csv b/diagnostics_dante/results/diagnostics.csv deleted file mode 100644 index 418005f..0000000 --- a/diagnostics_dante/results/diagnostics.csv +++ /dev/null @@ -1,2 +0,0 @@ -Shape,Size,Trial #,File Size,Storage Time,Retrieval Time,Storage CPU Median,Storage CPU Max,Storage RAM Median,Storage RAM Max,Storage Read Count Median,Storage Read Count Max,Storage Write Count Median,Storage Write Count Max,Storage Read Bytes Median,Storage Read Bytes Max,Storage Write Bytes Median,Storage Write Bytes Max,Retrieval CPU Median,Retrieval CPU Max,Retrieval RAM Median,Retrieval RAM Max,Retrieval Read Count Median,Retrieval Read Count Max,Retrieval Write Count Median,Retrieval Write Count Max,Retrieval Read Bytes Median,Retrieval Read Bytes Max,Retrieval Write Bytes Median,Retrieval Write Bytes Max -16:9,16684,0,5005200,67.17157697677612,0.010992765426635742,18.7,48.0,43.1,44.4,352810,352850,75638,75685,9593881600,9599149056,4030365696,4030783488,20.5,20.5,42.9,42.9,352851,352851,75685,75685,9599165440,9599165440,4030783488,4030783488