From 11b18a8cca8732c307de4669f3d6c5aa3c415565 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Wed, 28 Oct 2020 23:48:15 -0400 Subject: [PATCH 01/11] bump pngquant to v2.13.0 --- src/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/install-dependencies.sh b/src/install-dependencies.sh index ba10eda..69abef8 100755 --- a/src/install-dependencies.sh +++ b/src/install-dependencies.sh @@ -15,7 +15,7 @@ PNGQUANT_EXE="$PNGQUANT_BUILD_DIR/pngquant" ZOPFLIPNG_BUILD_DIR="$HOME/zopfli" ZOPFLIPNG_EXE="$ZOPFLIPNG_BUILD_DIR/zopflipng" -PNGQUANT_VERSION_TAG="2.12.5" +PNGQUANT_VERSION_TAG="2.13.0" ZOPFLIPNG_VERSION_TAG="v2.2.0" LIBPNG_VERSION="1.6.37" LIBPNG_VERSION_FILE="v$LIBPNG_VERSION.tar.gz" From 3ebecfd7ea593d9d7ffc5af230577bcc93ea849e Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Wed, 28 Oct 2020 23:52:20 -0400 Subject: [PATCH 02/11] update zopflipng derivative to v2.3.0 includes patches to v1.0.3+, the last sequential patch applied is https://github.com/google/zopfli/commit/7113f4e96bd26df27c46d590df95e517b966f10d --- src/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/install-dependencies.sh b/src/install-dependencies.sh index 69abef8..c6f7fd0 100755 --- a/src/install-dependencies.sh +++ b/src/install-dependencies.sh @@ -16,7 +16,7 @@ ZOPFLIPNG_BUILD_DIR="$HOME/zopfli" ZOPFLIPNG_EXE="$ZOPFLIPNG_BUILD_DIR/zopflipng" PNGQUANT_VERSION_TAG="2.13.0" -ZOPFLIPNG_VERSION_TAG="v2.2.0" +ZOPFLIPNG_VERSION_TAG="v2.3.0" LIBPNG_VERSION="1.6.37" LIBPNG_VERSION_FILE="v$LIBPNG_VERSION.tar.gz" From 20c352a7d48f92ba09a77a34a2f159c556bce22c Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Thu, 29 Oct 2020 14:41:49 -0400 Subject: [PATCH 03/11] [crunch.py] add multiprocessing.Pool close and join methods --- src/crunch.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/crunch.py b/src/crunch.py index 6282dde..077ed99 100755 --- a/src/crunch.py +++ b/src/crunch.py @@ -148,11 +148,7 @@ def main(argv): # PNG validity test if not is_valid_png(png_path): sys.stderr.write( - ERROR_STRING - + " '" - + png_path - + "' is not a valid PNG file." - + os.linesep + ERROR_STRING + " '" + png_path + "' is not a valid PNG file." + os.linesep ) if is_gui(argv): log_error(png_path + " is not a valid PNG file.") @@ -180,9 +176,7 @@ def main(argv): + os.linesep ) if is_gui(argv): - log_error( - "pngquant was not found on the expected path " + PNGQUANT_EXE_PATH - ) + log_error("pngquant was not found on the expected path " + PNGQUANT_EXE_PATH) sys.exit(1) elif not os.path.exists(ZOPFLIPNG_EXE_PATH): sys.stderr.write( @@ -226,6 +220,8 @@ def main(argv): p = Pool(processes) try: p.map(optimize_png, png_path_list) + p.close() + p.join() except Exception as e: stdstream_lock.acquire() sys.stderr.write("-----" + os.linesep) @@ -272,9 +268,7 @@ def optimize_png(png_path): pngquant_options = ( " --quality=80-98 --skip-if-larger --force --strip --speed 1 --ext -crunch.png " ) - pngquant_command = ( - PNGQUANT_EXE_PATH + pngquant_options + shellquote(img.pre_filepath) - ) + pngquant_command = PNGQUANT_EXE_PATH + pngquant_options + shellquote(img.pre_filepath) try: subprocess.check_output(pngquant_command, stderr=subprocess.STDOUT, shell=True) except CalledProcessError as cpe: From db86e44c9c90ee27567afd112002e37f5ab752ef Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Thu, 29 Oct 2020 14:54:49 -0400 Subject: [PATCH 04/11] [crunch.py] convert to multiprocessing.Pool context manager confirms that Pool is terminated at the end of execution --- src/crunch.py | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/crunch.py b/src/crunch.py index 077ed99..823b0cd 100755 --- a/src/crunch.py +++ b/src/crunch.py @@ -217,24 +217,23 @@ def main(argv): + str(len(png_path_list)) + " image files..." ) - p = Pool(processes) - try: - p.map(optimize_png, png_path_list) - p.close() - p.join() - except Exception as e: - stdstream_lock.acquire() - sys.stderr.write("-----" + os.linesep) - sys.stderr.write( - ERROR_STRING - + " Error detected during execution of the request." - + os.linesep - ) - sys.stderr.write(str(e) + os.linesep) - stdstream_lock.release() - if is_gui(argv): - log_error(str(e)) - sys.exit(1) + with Pool(processes) as p: + try: + p.map(optimize_png, png_path_list) + p.join() + except Exception as e: + stdstream_lock.acquire() + sys.stderr.write("-----" + os.linesep) + sys.stderr.write( + ERROR_STRING + + " Error detected during execution of the request." + + os.linesep + ) + sys.stderr.write(str(e) + os.linesep) + stdstream_lock.release() + if is_gui(argv): + log_error(str(e)) + sys.exit(1) # end of successful processing, exit code 0 if is_gui(argv): From c1bba8761e4874dfadce52aa044c36f7b17c8a84 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Thu, 29 Oct 2020 15:04:00 -0400 Subject: [PATCH 05/11] [crunch.py] remove Pool.join() call --- src/crunch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/crunch.py b/src/crunch.py index 823b0cd..a729c42 100755 --- a/src/crunch.py +++ b/src/crunch.py @@ -220,7 +220,6 @@ def main(argv): with Pool(processes) as p: try: p.map(optimize_png, png_path_list) - p.join() except Exception as e: stdstream_lock.acquire() sys.stderr.write("-----" + os.linesep) From e49ea9e328624426e87d8c4ac1826c8b7d17c455 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Thu, 29 Oct 2020 15:13:28 -0400 Subject: [PATCH 06/11] [crunch.py] black source fmt --- src/crunch.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/crunch.py b/src/crunch.py index a729c42..66e859c 100755 --- a/src/crunch.py +++ b/src/crunch.py @@ -148,7 +148,11 @@ def main(argv): # PNG validity test if not is_valid_png(png_path): sys.stderr.write( - ERROR_STRING + " '" + png_path + "' is not a valid PNG file." + os.linesep + ERROR_STRING + + " '" + + png_path + + "' is not a valid PNG file." + + os.linesep ) if is_gui(argv): log_error(png_path + " is not a valid PNG file.") @@ -176,7 +180,9 @@ def main(argv): + os.linesep ) if is_gui(argv): - log_error("pngquant was not found on the expected path " + PNGQUANT_EXE_PATH) + log_error( + "pngquant was not found on the expected path " + PNGQUANT_EXE_PATH + ) sys.exit(1) elif not os.path.exists(ZOPFLIPNG_EXE_PATH): sys.stderr.write( @@ -266,7 +272,9 @@ def optimize_png(png_path): pngquant_options = ( " --quality=80-98 --skip-if-larger --force --strip --speed 1 --ext -crunch.png " ) - pngquant_command = PNGQUANT_EXE_PATH + pngquant_options + shellquote(img.pre_filepath) + pngquant_command = ( + PNGQUANT_EXE_PATH + pngquant_options + shellquote(img.pre_filepath) + ) try: subprocess.check_output(pngquant_command, stderr=subprocess.STDOUT, shell=True) except CalledProcessError as cpe: From 8eac43ff0c0076b99022bc2f12f427857ea62ee1 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Thu, 29 Oct 2020 15:15:34 -0400 Subject: [PATCH 07/11] add setup.cfg with flake8 line length config --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..43d7a3a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 90 From e457eebcadd5c0ef471b1c67cf25b5df0573b2a6 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Thu, 29 Oct 2020 15:17:16 -0400 Subject: [PATCH 08/11] [crunch.py] style: initial pass fmt with line length = 90 --- src/crunch.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/crunch.py b/src/crunch.py index 66e859c..a729c42 100755 --- a/src/crunch.py +++ b/src/crunch.py @@ -148,11 +148,7 @@ def main(argv): # PNG validity test if not is_valid_png(png_path): sys.stderr.write( - ERROR_STRING - + " '" - + png_path - + "' is not a valid PNG file." - + os.linesep + ERROR_STRING + " '" + png_path + "' is not a valid PNG file." + os.linesep ) if is_gui(argv): log_error(png_path + " is not a valid PNG file.") @@ -180,9 +176,7 @@ def main(argv): + os.linesep ) if is_gui(argv): - log_error( - "pngquant was not found on the expected path " + PNGQUANT_EXE_PATH - ) + log_error("pngquant was not found on the expected path " + PNGQUANT_EXE_PATH) sys.exit(1) elif not os.path.exists(ZOPFLIPNG_EXE_PATH): sys.stderr.write( @@ -272,9 +266,7 @@ def optimize_png(png_path): pngquant_options = ( " --quality=80-98 --skip-if-larger --force --strip --speed 1 --ext -crunch.png " ) - pngquant_command = ( - PNGQUANT_EXE_PATH + pngquant_options + shellquote(img.pre_filepath) - ) + pngquant_command = PNGQUANT_EXE_PATH + pngquant_options + shellquote(img.pre_filepath) try: subprocess.check_output(pngquant_command, stderr=subprocess.STDOUT, shell=True) except CalledProcessError as cpe: From 9bc1659a82a21cb34de00d7de801470f8e40ce88 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 30 Oct 2020 17:27:12 -0400 Subject: [PATCH 09/11] [crunch.py] revert context manager for Pool This is not available in Py2.7. Though Py2.7 is legacy code, we are going to continue to support here for now. --- src/crunch.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/crunch.py b/src/crunch.py index a729c42..1b482a4 100755 --- a/src/crunch.py +++ b/src/crunch.py @@ -217,22 +217,22 @@ def main(argv): + str(len(png_path_list)) + " image files..." ) - with Pool(processes) as p: - try: - p.map(optimize_png, png_path_list) - except Exception as e: - stdstream_lock.acquire() - sys.stderr.write("-----" + os.linesep) - sys.stderr.write( - ERROR_STRING - + " Error detected during execution of the request." - + os.linesep - ) - sys.stderr.write(str(e) + os.linesep) - stdstream_lock.release() - if is_gui(argv): - log_error(str(e)) - sys.exit(1) + p = Pool(processes) + try: + p.map(optimize_png, png_path_list) + except Exception as e: + stdstream_lock.acquire() + sys.stderr.write("-----" + os.linesep) + sys.stderr.write( + ERROR_STRING + + " Error detected during execution of the request." + + os.linesep + ) + sys.stderr.write(str(e) + os.linesep) + stdstream_lock.release() + if is_gui(argv): + log_error(str(e)) + sys.exit(1) # end of successful processing, exit code 0 if is_gui(argv): From 06f1b04345b2a7a90a6263be6198b8443d3ca800 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 30 Oct 2020 17:30:07 -0400 Subject: [PATCH 10/11] [bench.py] add new total files, total % reduction fields to report also adds better data descriptions --- benchmarks/img/bench.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/benchmarks/img/bench.py b/benchmarks/img/bench.py index 3df643c..f12fc15 100644 --- a/benchmarks/img/bench.py +++ b/benchmarks/img/bench.py @@ -48,11 +48,14 @@ def grouped(iterable, n): total_initial_size = sum(pre_size_list) total_final_size = sum(post_size_list) delta = total_initial_size - total_final_size +total_percent = (total_final_size / total_initial_size) * 100 +print(f"Total files: {len(percent_list)}") print(f"\nInitial:\t{total_initial_size:>8} B") print(f"Final: \t{total_final_size:>8} B") -print(f"Delta: -{delta} B") -print(f"Mean: {mean:.2f}%") +print(f"\nDelta: -{delta} B") +print(f"Total: {total_percent:.2f}% of initial size") +print(f"Mean image reduction: {mean:.2f}%") try: import numpy as np From e0b5a504f8c7bcc02c391d3078e86fa0d0acf5a8 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 30 Oct 2020 18:19:39 -0400 Subject: [PATCH 11/11] [crunch.py] add finally block with Pool.close and Pool.join --- src/crunch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/crunch.py b/src/crunch.py index 1b482a4..45c2700 100755 --- a/src/crunch.py +++ b/src/crunch.py @@ -233,6 +233,9 @@ def main(argv): if is_gui(argv): log_error(str(e)) sys.exit(1) + finally: + p.close() + p.join() # end of successful processing, exit code 0 if is_gui(argv):