From 45849c419cec67753ee556361ed05ff6e2b4c74e Mon Sep 17 00:00:00 2001 From: stktyagi Date: Sat, 7 Jun 2025 14:05:46 +0530 Subject: [PATCH 1/8] [feature] Feature implementation for collectstatic #246 Implementation of smart collectstatic logic to compare freeze hash before calling collectstatic, saving time and avoiding redundant calls. Fixes #246 --- images/common/collectstatic_control.py | 29 ++++++++++++++++++++++++++ images/common/init_command.sh | 2 +- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 images/common/collectstatic_control.py diff --git a/images/common/collectstatic_control.py b/images/common/collectstatic_control.py new file mode 100644 index 00000000..dc6c99ea --- /dev/null +++ b/images/common/collectstatic_control.py @@ -0,0 +1,29 @@ +import hashlib +import os +import subprocess + +import django +import redis +from django.conf import settings + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openwisp.settings") +django.setup() + + +def get_pip_freeze_hash(): + output = subprocess.check_output(["pip", "freeze"]).decode() + return hashlib.sha256(output.encode()).hexdigest() + + +def main(): + r = redis.Redis.from_url(settings.CACHES["default"]["LOCATION"]) + new_hash = get_pip_freeze_hash() + stored_hash = r.get("pip_freeze_hash") + if stored_hash is None or stored_hash.decode() != new_hash: + print("pip freeze hash changed or missing, running collectstatic...") + subprocess.run( + ["python", "manage.py", "collectstatic", "--noinput"], check=True + ) + r.set("pip_freeze_hash", new_hash) + else: + print("pip freeze hash unchanged, skipping collectstatic.") diff --git a/images/common/init_command.sh b/images/common/init_command.sh index 02e6bb9c..6c91da24 100644 --- a/images/common/init_command.sh +++ b/images/common/init_command.sh @@ -14,7 +14,7 @@ if [ "$MODULE_NAME" = 'dashboard' ]; then python manage.py migrate --noinput test -f "$SSH_PRIVATE_KEY_PATH" || ssh-keygen -t ed25519 -f "$SSH_PRIVATE_KEY_PATH" -N "" python load_init_data.py - python manage.py collectstatic --noinput + python /opt/openwisp/collectstatic_control.py start_uwsgi elif [ "$MODULE_NAME" = 'postfix' ]; then postfix_config From c3a02275184c7bdc60f7347ff079b40304a8a161 Mon Sep 17 00:00:00 2001 From: stktyagi Date: Sat, 7 Jun 2025 19:52:49 +0530 Subject: [PATCH 2/8] [fix] Add error handling and relative path #246 Added error handling in the collectstatic logic and usage of relative path instead of absolute in init_command.sh. Fixes #246 --- images/common/collectstatic_control.py | 29 +++++++++++++++++--------- images/common/init_command.sh | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/images/common/collectstatic_control.py b/images/common/collectstatic_control.py index dc6c99ea..95e7e815 100644 --- a/images/common/collectstatic_control.py +++ b/images/common/collectstatic_control.py @@ -1,6 +1,7 @@ import hashlib import os import subprocess +import sys import django import redis @@ -11,19 +12,27 @@ def get_pip_freeze_hash(): - output = subprocess.check_output(["pip", "freeze"]).decode() - return hashlib.sha256(output.encode()).hexdigest() + try: + pip_freeze_output = subprocess.check_output(["pip", "freeze"]).decode() + return hashlib.sha256(pip_freeze_output.encode()).hexdigest() + except subprocess.CalledProcessError as e: + print(f"Error running 'pip freeze': {e}", file=sys.stderr) + sys.exit(1) def main(): - r = redis.Redis.from_url(settings.CACHES["default"]["LOCATION"]) - new_hash = get_pip_freeze_hash() - stored_hash = r.get("pip_freeze_hash") - if stored_hash is None or stored_hash.decode() != new_hash: + redis_connection = redis.Redis.from_url(settings.CACHES["default"]["LOCATION"]) + current_pip_hash = get_pip_freeze_hash() + cached_pip_hash = redis_connection.get("pip_freeze_hash") + if cached_pip_hash is None or cached_pip_hash.decode() != current_pip_hash: print("pip freeze hash changed or missing, running collectstatic...") - subprocess.run( - ["python", "manage.py", "collectstatic", "--noinput"], check=True - ) - r.set("pip_freeze_hash", new_hash) + try: + subprocess.run( + ["python", "manage.py", "collectstatic", "--noinput"], check=True + ) + redis_connection.set("pip_freeze_hash", current_pip_hash) + except subprocess.CalledProcessError as e: + print(f"Error running 'collectstatic': {e}", file=sys.stderr) + sys.exit(1) else: print("pip freeze hash unchanged, skipping collectstatic.") diff --git a/images/common/init_command.sh b/images/common/init_command.sh index 6c91da24..fff68a51 100644 --- a/images/common/init_command.sh +++ b/images/common/init_command.sh @@ -14,7 +14,7 @@ if [ "$MODULE_NAME" = 'dashboard' ]; then python manage.py migrate --noinput test -f "$SSH_PRIVATE_KEY_PATH" || ssh-keygen -t ed25519 -f "$SSH_PRIVATE_KEY_PATH" -N "" python load_init_data.py - python /opt/openwisp/collectstatic_control.py + python collectstatic_control.py start_uwsgi elif [ "$MODULE_NAME" = 'postfix' ]; then postfix_config From ece64c4eb60a39982dbe49e5857179ef10ac1bb8 Mon Sep 17 00:00:00 2001 From: stktyagi Date: Sat, 14 Jun 2025 18:40:26 +0530 Subject: [PATCH 3/8] [chores] Simplify implementation with respect to env and update docs #246 Added COLLECTSTATIC_WHEN_DEPS_CHANGE env var to control collectstatic. Updated script and docs to support disabling it. Fixes #246 --- .env | 2 ++ docs/user/settings.rst | 8 ++++++ images/common/collectstatic.py | 50 ++++++++++++++++++++++++++++++++++ images/common/init_command.sh | 2 +- 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 images/common/collectstatic.py diff --git a/.env b/.env index 40d69959..573874aa 100644 --- a/.env +++ b/.env @@ -60,3 +60,5 @@ USE_OPENWISP_CELERY_FIRMWARE=True OPENWISP_CELERY_FIRMWARE_COMMAND_FLAGS=--concurrency=1 # Metric collection METRIC_COLLECTION=True +# collectstatic +COLLECTSTATIC_WHEN_DEPS_CHANGE=true diff --git a/docs/user/settings.rst b/docs/user/settings.rst index da816f0e..86fe6a59 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -1164,3 +1164,11 @@ NFS Server - **Valid Values:** STRING. - **Default:** ``10.0.0.0/8(rw,fsid=0,insecure,no_root_squash,no_subtree_check,sync)``. + +``COLLECTSTATIC_WHEN_DEPS_CHANGE`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- **Explanation:** Controls whether ``collectstatic`` is run automatically at container startup + based on a change in Python dependencies. +- **Valid Values:** ``true``, ``false`` +- **Default:** ``true`` diff --git a/images/common/collectstatic.py b/images/common/collectstatic.py new file mode 100644 index 00000000..da0d53ec --- /dev/null +++ b/images/common/collectstatic.py @@ -0,0 +1,50 @@ +"""" +Calls `collectstatic` only when dependencies have changed. +This helps speed up the startup sequence when using +public cloud providers. + +This behavior can be disabled by setting the env var +`COLLECTSTATIC_WHEN_DEPS_CHANGE` to `False`. +"""" + +import hashlib +import os +import subprocess +import sys + +import django +import redis +from django.conf import settings + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openwisp.settings") +django.setup() + + +def get_pip_freeze_hash(): + try: + pip_freeze_output = subprocess.check_output(["pip", "freeze"]).decode() + return hashlib.sha256(pip_freeze_output.encode()).hexdigest() + except subprocess.CalledProcessError as e: + print(f"Error running 'pip freeze': {e}", file=sys.stderr) + sys.exit(1) + + +def main(): + if os.environ.get("COLLECTSTATIC_WHEN_DEPS_CHANGE", "true").lower() == "false": + print("COLLECTSTATIC_WHEN_DEPS_CHANGE is false; skipping collectstatic.") + return + redis_connection = redis.Redis.from_url(settings.CACHES["default"]["LOCATION"]) + current_pip_hash = get_pip_freeze_hash() + cached_pip_hash = redis_connection.get("pip_freeze_hash") + if cached_pip_hash is None or cached_pip_hash.decode() != current_pip_hash: + print("pip freeze hash changed or missing, running collectstatic...") + try: + subprocess.run( + ["python", "manage.py", "collectstatic", "--noinput"], check=True + ) + redis_connection.set("pip_freeze_hash", current_pip_hash) + except subprocess.CalledProcessError as e: + print(f"Error running 'collectstatic': {e}", file=sys.stderr) + sys.exit(1) + else: + print("pip freeze hash unchanged, skipping collectstatic.") diff --git a/images/common/init_command.sh b/images/common/init_command.sh index fff68a51..b023589f 100644 --- a/images/common/init_command.sh +++ b/images/common/init_command.sh @@ -14,7 +14,7 @@ if [ "$MODULE_NAME" = 'dashboard' ]; then python manage.py migrate --noinput test -f "$SSH_PRIVATE_KEY_PATH" || ssh-keygen -t ed25519 -f "$SSH_PRIVATE_KEY_PATH" -N "" python load_init_data.py - python collectstatic_control.py + python collectstatic.py start_uwsgi elif [ "$MODULE_NAME" = 'postfix' ]; then postfix_config From e5e50e8a1ece25ea011f2de32cb6dddda7cde095 Mon Sep 17 00:00:00 2001 From: stktyagi Date: Sat, 14 Jun 2025 18:42:39 +0530 Subject: [PATCH 4/8] [chores] Simplify implementation with respect to env and update docs #246 Added COLLECTSTATIC_WHEN_DEPS_CHANGE env var to control collectstatic. Updated script and docs to support disabling it. Fixes #246 --- docs/user/settings.rst | 4 ++-- images/common/collectstatic.py | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/user/settings.rst b/docs/user/settings.rst index 86fe6a59..60e0a37b 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -1168,7 +1168,7 @@ NFS Server ``COLLECTSTATIC_WHEN_DEPS_CHANGE`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- **Explanation:** Controls whether ``collectstatic`` is run automatically at container startup - based on a change in Python dependencies. +- **Explanation:** Controls whether ``collectstatic`` is run automatically + at container startup based on a change in Python dependencies. - **Valid Values:** ``true``, ``false`` - **Default:** ``true`` diff --git a/images/common/collectstatic.py b/images/common/collectstatic.py index da0d53ec..96f58a06 100644 --- a/images/common/collectstatic.py +++ b/images/common/collectstatic.py @@ -1,11 +1,8 @@ -"""" -Calls `collectstatic` only when dependencies have changed. -This helps speed up the startup sequence when using -public cloud providers. +"""Calls `collectstatic` only when dependencies have changed. This helps speed up the startup sequence when using public cloud providers. This behavior can be disabled by setting the env var `COLLECTSTATIC_WHEN_DEPS_CHANGE` to `False`. -"""" +""" import hashlib import os From fd62513fcd8a8a1455cc5174fe2f106d9f88d5a5 Mon Sep 17 00:00:00 2001 From: stktyagi Date: Sat, 14 Jun 2025 18:47:14 +0530 Subject: [PATCH 5/8] [chores] Simplify implementation with respect to env and update docs #246 Added COLLECTSTATIC_WHEN_DEPS_CHANGE env var to control collectstatic. Updated script and docs to support disabling it. Fixes #246 --- images/common/collectstatic.py | 6 ++-- images/common/collectstatic_control.py | 38 -------------------------- 2 files changed, 3 insertions(+), 41 deletions(-) delete mode 100644 images/common/collectstatic_control.py diff --git a/images/common/collectstatic.py b/images/common/collectstatic.py index 96f58a06..9e461dc9 100644 --- a/images/common/collectstatic.py +++ b/images/common/collectstatic.py @@ -1,7 +1,7 @@ -"""Calls `collectstatic` only when dependencies have changed. This helps speed up the startup sequence when using public cloud providers. +"""Call `collectstatic` only when dependencies have changed. -This behavior can be disabled by setting the env var -`COLLECTSTATIC_WHEN_DEPS_CHANGE` to `False`. +This helps speed up startup on cloud platforms. To disable this behavior, +set the `COLLECTSTATIC_WHEN_DEPS_CHANGE` environment variable to `False`. """ import hashlib diff --git a/images/common/collectstatic_control.py b/images/common/collectstatic_control.py deleted file mode 100644 index 95e7e815..00000000 --- a/images/common/collectstatic_control.py +++ /dev/null @@ -1,38 +0,0 @@ -import hashlib -import os -import subprocess -import sys - -import django -import redis -from django.conf import settings - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "openwisp.settings") -django.setup() - - -def get_pip_freeze_hash(): - try: - pip_freeze_output = subprocess.check_output(["pip", "freeze"]).decode() - return hashlib.sha256(pip_freeze_output.encode()).hexdigest() - except subprocess.CalledProcessError as e: - print(f"Error running 'pip freeze': {e}", file=sys.stderr) - sys.exit(1) - - -def main(): - redis_connection = redis.Redis.from_url(settings.CACHES["default"]["LOCATION"]) - current_pip_hash = get_pip_freeze_hash() - cached_pip_hash = redis_connection.get("pip_freeze_hash") - if cached_pip_hash is None or cached_pip_hash.decode() != current_pip_hash: - print("pip freeze hash changed or missing, running collectstatic...") - try: - subprocess.run( - ["python", "manage.py", "collectstatic", "--noinput"], check=True - ) - redis_connection.set("pip_freeze_hash", current_pip_hash) - except subprocess.CalledProcessError as e: - print(f"Error running 'collectstatic': {e}", file=sys.stderr) - sys.exit(1) - else: - print("pip freeze hash unchanged, skipping collectstatic.") From 3c3dc32bbb7f00127fba268b2fc14c9e75b492d7 Mon Sep 17 00:00:00 2001 From: stktyagi Date: Wed, 18 Jun 2025 11:19:11 +0530 Subject: [PATCH 6/8] [Fix] Fix incorrect execution conditions for collectstatic #473 Post commit, if the env var is true we run collectstatic if pip freeze changes else we run collectstatic each time. Fix #473 --- images/common/collectstatic.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/images/common/collectstatic.py b/images/common/collectstatic.py index 9e461dc9..17ccbb0d 100644 --- a/images/common/collectstatic.py +++ b/images/common/collectstatic.py @@ -26,22 +26,28 @@ def get_pip_freeze_hash(): sys.exit(1) +def run_collectstatic(): + print("Running collectstatic...") + try: + subprocess.run( + ["python", "manage.py", "collectstatic", "--noinput"], check=True + ) + except subprocess.CalledProcessError as e: + print(f"Error running 'collectstatic': {e}", file=sys.stderr) + sys.exit(1) + + def main(): if os.environ.get("COLLECTSTATIC_WHEN_DEPS_CHANGE", "true").lower() == "false": - print("COLLECTSTATIC_WHEN_DEPS_CHANGE is false; skipping collectstatic.") + print("COLLECTSTATIC_WHEN_DEPS_CHANGE is false; running collectstatic.") + run_collectstatic() return redis_connection = redis.Redis.from_url(settings.CACHES["default"]["LOCATION"]) current_pip_hash = get_pip_freeze_hash() cached_pip_hash = redis_connection.get("pip_freeze_hash") if cached_pip_hash is None or cached_pip_hash.decode() != current_pip_hash: print("pip freeze hash changed or missing, running collectstatic...") - try: - subprocess.run( - ["python", "manage.py", "collectstatic", "--noinput"], check=True - ) - redis_connection.set("pip_freeze_hash", current_pip_hash) - except subprocess.CalledProcessError as e: - print(f"Error running 'collectstatic': {e}", file=sys.stderr) - sys.exit(1) + run_collectstatic() + redis_connection.set("pip_freeze_hash", current_pip_hash) else: print("pip freeze hash unchanged, skipping collectstatic.") From 82ac53a6af55e95485d70e0f40107a9c92e0a4d8 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 18 Jun 2025 11:45:04 -0300 Subject: [PATCH 7/8] [chores] Revised texts --- docs/user/settings.rst | 7 ++++--- images/common/collectstatic.py | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/user/settings.rst b/docs/user/settings.rst index 60e0a37b..13bdd7fb 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -1168,7 +1168,8 @@ NFS Server ``COLLECTSTATIC_WHEN_DEPS_CHANGE`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- **Explanation:** Controls whether ``collectstatic`` is run automatically - at container startup based on a change in Python dependencies. -- **Valid Values:** ``true``, ``false`` +- **Description:** Runs ``collectstatic`` at container startup only if + Python dependencies have changed. Set to ``false`` if you're using the + development version of the OpenWISP python modules. +- **Accepted Values:** ``true``, ``false`` - **Default:** ``true`` diff --git a/images/common/collectstatic.py b/images/common/collectstatic.py index 17ccbb0d..f699ef2f 100644 --- a/images/common/collectstatic.py +++ b/images/common/collectstatic.py @@ -1,7 +1,7 @@ -"""Call `collectstatic` only when dependencies have changed. +"""Run ``collectstatic`` only when dependencies have changed. -This helps speed up startup on cloud platforms. To disable this behavior, -set the `COLLECTSTATIC_WHEN_DEPS_CHANGE` environment variable to `False`. +Speeds up startup time on cloud platforms. To disable this behavior, set +the ``COLLECTSTATIC_WHEN_DEPS_CHANGE`` environment variable to ``False``. """ import hashlib @@ -46,8 +46,8 @@ def main(): current_pip_hash = get_pip_freeze_hash() cached_pip_hash = redis_connection.get("pip_freeze_hash") if cached_pip_hash is None or cached_pip_hash.decode() != current_pip_hash: - print("pip freeze hash changed or missing, running collectstatic...") + print("Changes in Python dependencies detected, running collectstatic...") run_collectstatic() redis_connection.set("pip_freeze_hash", current_pip_hash) else: - print("pip freeze hash unchanged, skipping collectstatic.") + print("No changes in Python dependencies, skipping collectstatic...") From 3de1bb366c8fd51ab887b680c2cd3d0fa760c3d4 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 18 Jun 2025 11:45:29 -0300 Subject: [PATCH 8/8] [chores] Removed unnecessary prints --- images/common/collectstatic.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/images/common/collectstatic.py b/images/common/collectstatic.py index f699ef2f..a0cbc036 100644 --- a/images/common/collectstatic.py +++ b/images/common/collectstatic.py @@ -27,7 +27,6 @@ def get_pip_freeze_hash(): def run_collectstatic(): - print("Running collectstatic...") try: subprocess.run( ["python", "manage.py", "collectstatic", "--noinput"], check=True @@ -39,7 +38,6 @@ def run_collectstatic(): def main(): if os.environ.get("COLLECTSTATIC_WHEN_DEPS_CHANGE", "true").lower() == "false": - print("COLLECTSTATIC_WHEN_DEPS_CHANGE is false; running collectstatic.") run_collectstatic() return redis_connection = redis.Redis.from_url(settings.CACHES["default"]["LOCATION"])