From 14ab61944e117c7333d1970ea508a96881095cc3 Mon Sep 17 00:00:00 2001 From: Mahathi-s154 <202117698+Mahathi-s154@users.noreply.github.com> Date: Wed, 18 Feb 2026 21:45:40 +0530 Subject: [PATCH 01/26] [tests:fix] Addressed Selenium unsaved changes test #902 Chromium v126 auto-accepts beforeunload dialogs in classic WebDriver sessions, causing Selenium tests waiting for alerts to fail. This change updates the test strategy to reliably detect unsaved changes without relying on deprecated beforeunload alert behavior. Fixes #902 Co-authored-by: Federico Capoano --- .../config/tests/test_selenium.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/openwisp_controller/config/tests/test_selenium.py b/openwisp_controller/config/tests/test_selenium.py index 5f94f2622..f718b3373 100644 --- a/openwisp_controller/config/tests/test_selenium.py +++ b/openwisp_controller/config/tests/test_selenium.py @@ -1,12 +1,15 @@ +import os import time from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.test import tag from django.urls.base import reverse +from selenium import webdriver from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys +from selenium.webdriver.common.utils import free_port from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import Select, WebDriverWait from swapper import load_model @@ -498,6 +501,37 @@ class TestDeviceAdminUnsavedChanges( ): browser = "chrome" + @classmethod + def get_chrome_webdriver(cls): + """ + Override the parent class method to enable BiDi mode and set + unhandledPromptBehavior to "ignore". This is required to test + beforeunload alerts, as Chromium v126+ auto-accepts them per + WebDriver standard. + + Ref: https://github.com/openwisp/openwisp-controller/issues/902 + """ + options = webdriver.ChromeOptions() + options.page_load_strategy = "eager" + if os.environ.get("SELENIUM_HEADLESS", False): + options.add_argument("--headless") + CHROME_BIN = os.environ.get("CHROME_BIN", None) + if CHROME_BIN: + options.binary_location = CHROME_BIN + options.add_argument("--window-size=1366,768") + options.add_argument("--ignore-certificate-errors") + options.add_argument("--no-sandbox") + options.add_argument("--disable-gpu") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-features=VizDisplayCompositor") + options.add_argument(f"--remote-debugging-port={free_port()}") + options.set_capability("goog:loggingPrefs", {"browser": "ALL"}) + # Enable BiDi mode and set unhandledPromptBehavior to "ignore" + # to allow testing beforeunload alerts (Chromium v126+). + options.enable_bidi = True + options.set_capability("unhandledPromptBehavior", "ignore") + return webdriver.Chrome(options=options) + def _is_unsaved_changes_alert_present(self): for entry in self.get_browser_logs(): if ( From 733f0c9cdde6fdfcfc0a78f5b3a64b43932e55ec Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 24 Feb 2026 13:12:52 -0300 Subject: [PATCH 02/26] [chores:tests] Fixed mocking in test_update_vpn_server_configuration The first call to vpn.save() was generting a real HTTP request, which failed in the CI of ansible-openwisp2. Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- openwisp_controller/config/tests/test_vpn.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/openwisp_controller/config/tests/test_vpn.py b/openwisp_controller/config/tests/test_vpn.py index 0703fba11..b47e9ffa5 100644 --- a/openwisp_controller/config/tests/test_vpn.py +++ b/openwisp_controller/config/tests/test_vpn.py @@ -827,33 +827,35 @@ def test_update_vpn_server_configuration(self): f"Cannot update configuration of {vpn.name} VPN server, " "webhook endpoint and authentication token are empty." ) - success_response = mock.Mock(spec=requests.Response) - success_response.status_code = 200 - success_response.raise_for_status = mock.Mock() with self.subTest("Webhook endpoint and authentication endpoint is present"): vpn.webhook_endpoint = "https://example.com" vpn.auth_token = "super-secret-token" - vpn.save() - vpn_client.refresh_from_db() + success_response = mock.Mock(spec=requests.Response) + success_response.status_code = 200 + success_response.raise_for_status = mock.Mock() with mock.patch( "openwisp_controller.config.tasks.logger.info" ) as mocked_logger, mock.patch( "requests.post", return_value=success_response ): + vpn.save() + vpn_client.refresh_from_db() post_save.send( instance=vpn_client, sender=vpn_client._meta.model, created=False ) - mocked_logger.assert_called_once_with( + expected_call = mock.call( f"Triggered update webhook of VPN Server UUID: {vpn.pk}" ) + mocked_logger.assert_has_calls([expected_call, expected_call]) + self.assertEqual(mocked_logger.call_count, 2) + fail_response = mock.Mock(spec=requests.Response) fail_response.status_code = 404 fail_response.raise_for_status.side_effect = requests.exceptions.HTTPError( "Not Found" ) - with mock.patch("logging.Logger.warning") as mocked_logger, mock.patch( "requests.post", return_value=fail_response ): From 9c1d46d2db9a5069c57b162f571a6e28df799d8f Mon Sep 17 00:00:00 2001 From: Atif <39148554+atif09@users.noreply.github.com> Date: Tue, 24 Feb 2026 21:59:25 +0530 Subject: [PATCH 03/26] [ci] Added automated backport workflow Co-authored-by: Federico Capoano --- .github/workflows/backport.yml | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/backport.yml diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000..19e4f83d6 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,43 @@ +name: Backport fixes to stable branch + +on: + push: + branches: + - master + - main + issue_comment: + types: [created] + +concurrency: + group: backport-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + backport-on-push: + if: github.event_name == 'push' + uses: openwisp/openwisp-utils/.github/workflows/reusable-backport.yml@master + with: + commit_sha: ${{ github.sha }} + secrets: + app_id: ${{ secrets.OPENWISP_BOT_APP_ID }} + private_key: ${{ secrets.OPENWISP_BOT_PRIVATE_KEY }} + + backport-on-comment: + if: > + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.issue.pull_request.merged_at != null && + github.event.issue.state == 'closed' && + contains(fromJSON('["MEMBER", "OWNER"]'), github.event.comment.author_association) && + startsWith(github.event.comment.body, '/backport') + uses: openwisp/openwisp-utils/.github/workflows/reusable-backport.yml@master + with: + pr_number: ${{ github.event.issue.number }} + comment_body: ${{ github.event.comment.body }} + secrets: + app_id: ${{ secrets.OPENWISP_BOT_APP_ID }} + private_key: ${{ secrets.OPENWISP_BOT_PRIVATE_KEY }} From fa2ce3d4dba356dd1e93b77e70fdaf98c8869d0e Mon Sep 17 00:00:00 2001 From: Gagan Deep Date: Tue, 24 Feb 2026 22:04:14 +0530 Subject: [PATCH 04/26] [fix] Use context variables in Vpn.auto_client for OpenVPN backend Bug: The `Vpn.auto_client` method was incorrectly using the Vpn.host field when generating the client configuration for the OpenVPN backend. This resulted in the `remote` directive being hardcoded in the `Template.config`, instead of being rendered from the provided context variables. Fix: The `Vpn.auto_client` method has been updated to use the correct context-based values. [backport 1.2] --- openwisp_controller/config/base/vpn.py | 2 +- openwisp_controller/config/tests/test_vpn.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/openwisp_controller/config/base/vpn.py b/openwisp_controller/config/base/vpn.py index 819cc1db7..5a1da28d5 100644 --- a/openwisp_controller/config/base/vpn.py +++ b/openwisp_controller/config/base/vpn.py @@ -648,7 +648,7 @@ def auto_client(self, auto_cert=True, template_backend_class=None): context_keys.pop("ip_address", None) context_keys.pop("vpn_subnet", None) auto = backend.auto_client( - host=self.host, + host=vpn_host, server=self.config[config_dict_key][0], **context_keys, ) diff --git a/openwisp_controller/config/tests/test_vpn.py b/openwisp_controller/config/tests/test_vpn.py index b47e9ffa5..211ad4a06 100644 --- a/openwisp_controller/config/tests/test_vpn.py +++ b/openwisp_controller/config/tests/test_vpn.py @@ -194,12 +194,13 @@ def test_auto_client(self): vpn = self._create_vpn() auto = vpn.auto_client() context_keys = vpn._get_auto_context_keys() - del context_keys["vpn_host"] del context_keys["vpn_port"] for key in context_keys.keys(): context_keys[key] = "{{%s}}" % context_keys[key] control = vpn.backend_class.auto_client( - host=vpn.host, server=self._vpn_config["openvpn"][0], **context_keys + host=context_keys.pop("vpn_host"), + server=self._vpn_config["openvpn"][0], + **context_keys, ) control["files"] = [ { @@ -224,14 +225,15 @@ def test_auto_client_auto_cert_False(self): vpn = self._create_vpn() auto = vpn.auto_client(auto_cert=False) context_keys = vpn._get_auto_context_keys() - del context_keys["vpn_host"] del context_keys["vpn_port"] for key in context_keys.keys(): context_keys[key] = "{{%s}}" % context_keys[key] for key in ["cert_path", "cert_contents", "key_path", "key_contents"]: del context_keys[key] control = vpn.backend_class.auto_client( - host=vpn.host, server=self._vpn_config["openvpn"][0], **context_keys + host=context_keys.pop("vpn_host"), + server=self._vpn_config["openvpn"][0], + **context_keys, ) control["files"] = [ { From 6fe7d90ac810f5bf7d58fea2516b2f263f794201 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Tue, 24 Feb 2026 13:51:22 -0300 Subject: [PATCH 05/26] [ci] Updated CI branches: added 1.2, removed gsoc25 branches [skip ci] --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ed6cc439..ea57cf6aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,7 @@ on: branches: - master - "1.1" - - gsoc25-map - - gsoc25-whois + - "1.2" jobs: build: From 7946420f4a05e9fd6d0a9f3d2fbe1a0937a0cac1 Mon Sep 17 00:00:00 2001 From: Abhigyan <170253496+UltraBot05@users.noreply.github.com> Date: Wed, 25 Feb 2026 01:10:02 +0530 Subject: [PATCH 06/26] [docs] Updated organization limits screenshot #1073 Closes #1073 Signed-off-by: UltraBot05 <170253496+UltraBot05@users.noreply.github.com> --- docs/user/organization-limits.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/organization-limits.rst b/docs/user/organization-limits.rst index 2b3a9d98b..8dbdfb132 100644 --- a/docs/user/organization-limits.rst +++ b/docs/user/organization-limits.rst @@ -8,7 +8,7 @@ To set these limits: 1. Navigate to **USERS & ORGANIZATIONS** on the left-hand navigation menu. 2. Go to **Organizations**. 3. Click on the specific organization you want to limit. -4. In the **CONTROLLER LIMIT** section, set the desired limit. +4. In the **CONTROLLER LIMITS** section, set the desired limit. Refer to the screenshot below for guidance: From 7da8e3f5eb059bad6a80b64983e1389e8a57b908 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 25 Feb 2026 17:22:38 -0300 Subject: [PATCH 07/26] [chores] CI: added "fail-on-error: false" to coveralls --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea57cf6aa..56c8a6a78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,3 +107,4 @@ jobs: uses: coverallsapp/github-action@v2 with: parallel-finished: true + fail-on-error: false From c747ecd02a7dfa8405e816cf21cbc5f783670684 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 25 Feb 2026 17:41:03 -0300 Subject: [PATCH 08/26] [chores] CI: added "fail-on-error: false" to coveralls (right place) --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56c8a6a78..594642dd8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,7 @@ jobs: format: cobertura flag-name: python-${{ matrix.env.env }} github-token: ${{ secrets.GITHUB_TOKEN }} + fail-on-error: false coveralls: needs: build From f12ae703150ef736ca768b8a2df9c9d4d3223a86 Mon Sep 17 00:00:00 2001 From: AlexV Date: Thu, 26 Feb 2026 21:05:18 +0100 Subject: [PATCH 09/26] [chores:fix] Add TestMigrationGraphIntegrity to render all project_states Co-authored-by: Alexandre Vincent --- .../connection/migrations/0001_initial.py | 5 ++- .../geo/migrations/0001_initial.py | 5 ++- .../tests/test_migrationgraph.py | 32 +++++++++++++++++++ runtests.py | 1 + 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 openwisp_controller/tests/test_migrationgraph.py diff --git a/openwisp_controller/connection/migrations/0001_initial.py b/openwisp_controller/connection/migrations/0001_initial.py index cd819c49a..24e902931 100644 --- a/openwisp_controller/connection/migrations/0001_initial.py +++ b/openwisp_controller/connection/migrations/0001_initial.py @@ -23,7 +23,10 @@ class Migration(migrations.Migration): swapper.dependency( *swapper.split(settings.AUTH_USER_MODEL), version="0004_default_groups" ), - swapper.dependency("config", "Device"), + swapper.dependency( + *swapper.split(settings.CONFIG_DEVICE_MODEL), + version="0004_add_device_model" + ), ] operations = [ diff --git a/openwisp_controller/geo/migrations/0001_initial.py b/openwisp_controller/geo/migrations/0001_initial.py index 10c5e7fc0..f1e2e2533 100644 --- a/openwisp_controller/geo/migrations/0001_initial.py +++ b/openwisp_controller/geo/migrations/0001_initial.py @@ -21,7 +21,10 @@ class Migration(migrations.Migration): swapper.dependency( *swapper.split(settings.AUTH_USER_MODEL), version="0004_default_groups" ), - swapper.dependency("config", "Device"), + swapper.dependency( + *swapper.split(settings.CONFIG_DEVICE_MODEL), + version="0004_add_device_model" + ), ] operations = [ diff --git a/openwisp_controller/tests/test_migrationgraph.py b/openwisp_controller/tests/test_migrationgraph.py new file mode 100644 index 000000000..852ed8325 --- /dev/null +++ b/openwisp_controller/tests/test_migrationgraph.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from django.db.migrations.loader import MigrationLoader +from django.test import SimpleTestCase, tag + + +@tag("slow") +class TestMigrationGraphIntegrity(SimpleTestCase): + def test_all_migration_states_render_apps(self) -> None: + """ + Fail fast if any migration target produces an invalid historical StateApps. + + This catches missing dependencies / broken swappable references that only + show up when Django renders historical models. + + Note : issues in third-party dependency migrations + will also be detected. + """ + loader = MigrationLoader(None, ignore_no_migrations=True) + + failures: list[str] = [] + for key in sorted(loader.graph.nodes.keys()): + try: + state = loader.project_state([key]) + _ = state.apps # triggers StateApps rendering + except (LookupError, ValueError, AttributeError, TypeError) as e: + failures.append(f"{key[0]}.{key[1]} -> {type(e).__name__}: {e}") + + if failures: + self.fail( + "Some migration states cannot be rendered:\n" + "\n".join(failures) + ) diff --git a/runtests.py b/runtests.py index 5c66e152e..84b6e448e 100755 --- a/runtests.py +++ b/runtests.py @@ -46,6 +46,7 @@ def run_tests(extra_args, settings_module, test_app): else: test_app = "openwisp2" app_dir = "tests/openwisp2/" + args += ["--exclude-tag", "slow"] # Run Django tests django_tests = run_tests(args, "openwisp2.settings", test_app) # Run pytest tests From 9b4223d550ecd11071b17b6ca0397c0abefce805 Mon Sep 17 00:00:00 2001 From: JgtAman <41839176+DragnEmperor@users.noreply.github.com> Date: Mon, 30 Jun 2025 23:56:49 +0530 Subject: [PATCH 10/26] [feature] Base logic for WHOIS lookup feature #1032 #1033 #1037 #1045 Closes #1032 Closes #1033 Closes #1037 Closes #1045 Signed-off-by: DragnEmperor Co-authored-by: Federico Capoano --- docs/developer/extending.rst | 1 + docs/index.rst | 1 + docs/user/settings.rst | 51 ++ docs/user/whois.rst | 80 +++ openwisp_controller/config/admin.py | 31 +- openwisp_controller/config/apps.py | 2 + openwisp_controller/config/base/device.py | 28 +- .../config/base/multitenancy.py | 20 + openwisp_controller/config/base/whois.py | 115 ++++ .../config/controller/views.py | 9 + .../config/migrations/0060_whoisinfo.py | 83 +++ openwisp_controller/config/models.py | 11 + openwisp_controller/config/settings.py | 10 + .../config/tests/test_device.py | 2 + openwisp_controller/config/whois/__init__.py | 0 openwisp_controller/config/whois/handlers.py | 29 + openwisp_controller/config/whois/service.py | 118 +++++ openwisp_controller/config/whois/tasks.py | 160 ++++++ .../config/whois/test_whois.py | 501 ++++++++++++++++++ openwisp_controller/config/whois/utils.py | 36 ++ requirements.txt | 1 + ...rganizationconfigsettings_whois_enabled.py | 79 +++ tests/openwisp2/sample_config/models.py | 10 + tests/openwisp2/settings.py | 9 + 24 files changed, 1376 insertions(+), 11 deletions(-) create mode 100644 docs/user/whois.rst create mode 100644 openwisp_controller/config/base/whois.py create mode 100644 openwisp_controller/config/migrations/0060_whoisinfo.py create mode 100644 openwisp_controller/config/whois/__init__.py create mode 100644 openwisp_controller/config/whois/handlers.py create mode 100644 openwisp_controller/config/whois/service.py create mode 100644 openwisp_controller/config/whois/tasks.py create mode 100644 openwisp_controller/config/whois/test_whois.py create mode 100644 openwisp_controller/config/whois/utils.py create mode 100644 tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py diff --git a/docs/developer/extending.rst b/docs/developer/extending.rst index eb29af94a..eda5c0b05 100644 --- a/docs/developer/extending.rst +++ b/docs/developer/extending.rst @@ -344,6 +344,7 @@ Once you have created the models, add the following to your CONFIG_VPNCLIENT_MODEL = "sample_config.VpnClient" CONFIG_ORGANIZATIONCONFIGSETTINGS_MODEL = "sample_config.OrganizationConfigSettings" CONFIG_ORGANIZATIONLIMITS_MODEL = "sample_config.OrganizationLimits" + CONFIG_WHOISINFO_MODEL = "sample_config.WHOISInfo" DJANGO_X509_CA_MODEL = "sample_pki.Ca" DJANGO_X509_CERT_MODEL = "sample_pki.Cert" GEO_LOCATION_MODEL = "sample_geo.Location" diff --git a/docs/index.rst b/docs/index.rst index 3372e6eae..6895e1c48 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,6 +48,7 @@ the OpenWISP architecture. user/zerotier.rst user/openvpn.rst user/subnet-division-rules.rst + user/whois.rst user/rest-api.rst user/settings.rst diff --git a/docs/user/settings.rst b/docs/user/settings.rst index ee32e4875..948b82ec8 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -761,3 +761,54 @@ recoverable failures, improving the reliability of the system. For more information on these settings, you can refer to the `the celery documentation regarding automatic retries for known errors. `_ + +.. _openwisp_controller_whois_enabled: + +``OPENWISP_CONTROLLER_WHOIS_ENABLED`` +------------------------------------- + +============ ========= +**type**: ``bool`` +**default**: ``False`` +============ ========= + +Allows enabling the optional :doc:`WHOIS Lookup feature `. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.2/whois-admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.2/whois-admin-setting.png + :alt: WHOIS admin setting + +After enabling this feature, you have to set +:ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT +` and +:ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY +`. + +.. warning:: + + If these three settings are not configured as expected, an + ``ImproperlyConfigured`` exception will be raised. + +.. _openwisp_controller_whois_geoip_account: + +``OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT`` +------------------------------------------- + +============ ======= +**type**: ``str`` +**default**: None +============ ======= + +Maxmind Account ID required for the :doc:`WHOIS Lookup feature `. + +.. _openwisp_controller_whois_geoip_key: + +``OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY`` +--------------------------------------- + +============ ======= +**type**: ``str`` +**default**: None +============ ======= + +Maxmind License Key required for the :doc:`WHOIS Lookup feature `. diff --git a/docs/user/whois.rst b/docs/user/whois.rst new file mode 100644 index 000000000..3039569a4 --- /dev/null +++ b/docs/user/whois.rst @@ -0,0 +1,80 @@ +WHOIS Lookup +============ + +.. important:: + + The **WHOIS Lookup** feature is **disabled by default**. + + To enable it, follow the `setup steps + `_ below. + +.. contents:: **Table of contents**: + :depth: 1 + :local: + +Overview +-------- + +The WHOIS Lookup feature displays information about the public IP address +used by devices to communicate with OpenWISP (via the ``last_ip`` field). +It helps identify the geographic location and ISP associated with the IP +address, which can be useful for troubleshooting network issues. + +The retrieved information pertains to the Autonomous System (ASN) +associated with the device's public IP address and includes: + +- ASN (Autonomous System Number) +- Organization name that owns the ASN +- CIDR block assigned to the ASN +- Physical address registered to the ASN +- Timezone of the ASN's registered location + +Trigger Conditions +------------------ + +A WHOIS lookup is triggered automatically when: + +- A new device is registered. +- A device fetches its checksum. + +However, the lookup will only run if **all** the following conditions are +met: + +- The device's last IP address is **public**. +- There is **no existing WHOIS record** for that IP. +- WHOIS lookup is **enabled** for the device's organization. + +Behavior with Shared IP Addresses +--------------------------------- + +If multiple devices share the same public IP address and one of them +switches to a different IP, the following occurs: + +- A lookup is triggered for the **new IP**. +- The WHOIS record for the **old IP** is deleted. +- The next time a device still using the old IP fetches its checksum, a + new lookup is triggered, ensuring up-to-date data. + +.. note:: + + When a device with an associated WHOIS record is deleted, its WHOIS + record is automatically removed. + +.. _controller_setup_whois_lookup: + +Setup Instructions +------------------ + +1. Create a MaxMind account: `Sign up here + `_. + + If you already have an account, just **Sign In**. + +2. Go to **Manage License Keys** in your MaxMind dashboard. +3. Generate a new license key and name it as you prefer. +4. Copy both the **Account ID** and **License Key**. +5. Set the following settings accordingly: + + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_ENABLED` to ``True``. + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT` to **Account ID**. + - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY` to **License Key**. diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index 9c758fd06..4ba8ea83b 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -1362,18 +1362,29 @@ def has_delete_permission(self, request, obj): limits_inline_position = 0 -if getattr(app_settings, "REGISTRATION_ENABLED", True): - class ConfigSettingsForm(AlwaysHasChangedMixin, forms.ModelForm): - class Meta: - widgets = {"context": FlatJsonWidget} - class ConfigSettingsInline(admin.StackedInline): - model = OrganizationConfigSettings - form = ConfigSettingsForm +class ConfigSettingsForm(AlwaysHasChangedMixin, forms.ModelForm): + class Meta: + widgets = {"context": FlatJsonWidget} + + +class ConfigSettingsInline(admin.StackedInline): + model = OrganizationConfigSettings + form = ConfigSettingsForm + + def get_fields(self, request, obj=None): + fields = [] + if app_settings.REGISTRATION_ENABLED: + fields += ["registration_enabled", "shared_secret"] + if app_settings.WHOIS_CONFIGURED: + fields += ["whois_enabled"] + fields += ["context"] + return fields + - OrganizationAdmin.save_on_top = True - OrganizationAdmin.inlines.insert(0, ConfigSettingsInline) - limits_inline_position = 1 +OrganizationAdmin.save_on_top = True +OrganizationAdmin.inlines.insert(0, ConfigSettingsInline) +limits_inline_position = 1 OrganizationAdmin.inlines.insert(limits_inline_position, OrganizationLimitsInline) diff --git a/openwisp_controller/config/apps.py b/openwisp_controller/config/apps.py index 973b46c63..4354fcd10 100644 --- a/openwisp_controller/config/apps.py +++ b/openwisp_controller/config/apps.py @@ -29,6 +29,7 @@ vpn_peers_changed, vpn_server_modified, ) +from .whois.handlers import connect_whois_handlers # ensure Device.hardware_id field is not flagged as unique # (because it's flagged as unique_together with organization) @@ -50,6 +51,7 @@ def ready(self, *args, **kwargs): self.register_dashboard_charts() self.register_menu_groups() self.notification_cache_update() + connect_whois_handlers() def __setmodels__(self): self.device_model = load_model("config", "Device") diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 763f25ecc..1921d5ad3 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -1,3 +1,4 @@ +from functools import cached_property from hashlib import md5 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied, ValidationError @@ -18,6 +19,7 @@ management_ip_changed, ) from ..validators import device_name_validator, mac_address_validator +from ..whois.service import WHOISService from .base import BaseModel @@ -118,6 +120,11 @@ class Meta: def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + # Initial value for last_ip is required in WHOIS + # to remove WHOIS info related to that ip address. + if app_settings.WHOIS_CONFIGURED: + self._changed_checked_fields.append("last_ip") + self._set_initial_values_for_changed_checked_fields() def _set_initial_values_for_changed_checked_fields(self): @@ -279,6 +286,8 @@ def save(self, *args, **kwargs): self.key = self.generate_key(shared_secret) state_adding = self._state.adding super().save(*args, **kwargs) + if app_settings.WHOIS_CONFIGURED: + self._check_last_ip() if state_adding and self.group and self.group.templates.exists(): self.create_default_config() # The value of "self._state.adding" will always be "False" @@ -299,7 +308,9 @@ def _check_changed_fields(self): self._get_initial_values_for_checked_fields() # Execute method for checked for each field in self._changed_checked_fields for field in self._changed_checked_fields: - getattr(self, f"_check_{field}_changed")() + method = getattr(self, f"_check_{field}_changed", None) + if callable(method): + method() def _is_deferred(self, field): """ @@ -509,3 +520,18 @@ def config_deactivated_clear_management_ip(cls, instance, *args, **kwargs): is changed to 'deactivated'. """ cls.objects.filter(pk=instance.device_id).update(management_ip="") + + @cached_property + def whois_service(self): + """ + Used as a shortcut to get WHOISService instance + for the device. + """ + return WHOISService(self) + + def _check_last_ip(self): + """Trigger WHOIS lookup if last_ip is not deferred.""" + if self._initial_last_ip == models.DEFERRED: + return + self.whois_service.trigger_whois_lookup() + self._initial_last_ip = self.last_ip diff --git a/openwisp_controller/config/base/multitenancy.py b/openwisp_controller/config/base/multitenancy.py index 9cdf15736..3095ab779 100644 --- a/openwisp_controller/config/base/multitenancy.py +++ b/openwisp_controller/config/base/multitenancy.py @@ -2,12 +2,15 @@ from copy import deepcopy import swapper +from django.core.exceptions import ValidationError from django.db import models from django.utils.translation import gettext_lazy as _ from jsonfield import JSONField from openwisp_utils.base import KeyField, UUIDModel +from openwisp_utils.fields import FallbackBooleanChoiceField +from .. import settings as app_settings from ..exceptions import OrganizationDeviceLimitExceeded from ..tasks import bulk_invalidate_config_get_cached_checksum @@ -31,6 +34,11 @@ class AbstractOrganizationConfigSettings(UUIDModel): verbose_name=_("shared secret"), help_text=_("used for automatic registration of devices"), ) + whois_enabled = FallbackBooleanChoiceField( + help_text=_("Whether the WHOIS lookup feature is enabled"), + fallback=app_settings.WHOIS_ENABLED, + verbose_name=_("WHOIS Enabled"), + ) context = JSONField( blank=True, default=dict, @@ -53,6 +61,18 @@ def __str__(self): def get_context(self): return deepcopy(self.context) + def clean(self): + if not app_settings.WHOIS_CONFIGURED and self.whois_enabled: + raise ValidationError( + { + "whois_enabled": _( + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "before enabling WHOIS feature." + ) + } + ) + return super().clean() + def save( self, force_insert=False, force_update=False, using=None, update_fields=None ): diff --git a/openwisp_controller/config/base/whois.py b/openwisp_controller/config/base/whois.py new file mode 100644 index 000000000..80af038f2 --- /dev/null +++ b/openwisp_controller/config/base/whois.py @@ -0,0 +1,115 @@ +from ipaddress import ip_address, ip_network + +from django.core.cache import cache +from django.core.exceptions import ValidationError +from django.db import models, transaction +from django.utils.translation import gettext_lazy as _ +from jsonfield import JSONField + +from openwisp_utils.base import TimeStampedEditableModel + +from ..whois.service import WHOISService +from ..whois.tasks import delete_whois_record + + +class AbstractWHOISInfo(TimeStampedEditableModel): + """ + Abstract model to store WHOIS information + for a device. + """ + + id = None + # Using ip_address as primary key to avoid redundant lookups + # and storage of duplicate WHOIS information per IP address. + # Whenever a device's last ip address changes, data related to + # previous IP address is deleted. If any device still has the + # previous IP address, they will trigger the lookup again + # ensuring latest WHOIS information is always available. + ip_address = models.GenericIPAddressField(db_index=True, primary_key=True) + isp = models.CharField( + max_length=100, + blank=True, + help_text=_("Organization for ASN"), + ) + asn = models.CharField( + max_length=6, + blank=True, + help_text=_("ASN"), + ) + timezone = models.CharField( + max_length=35, + blank=True, + help_text=_("Time zone"), + ) + address = JSONField( + default=dict, + help_text=_("Address"), + blank=True, + ) + cidr = models.CharField( + max_length=20, + blank=True, + help_text=_("CIDR"), + ) + + class Meta: + abstract = True + + def clean(self): + if ip_address(self.ip_address).is_private: + raise ValidationError( + { + "ip_address": _( + "WHOIS information cannot be created for private IP addresses." + ) + } + ) + if self.cidr: + try: + # strict is set to False to allow CIDR without a mask + # e.g. 192.168.1.12/24 with strict False normalizes to + # 192.168.1.0/24 else it would raise an error. + ip_network(self.cidr, strict=False) + except ValueError as e: + raise ValidationError( + {"cidr": _("Invalid CIDR format: %(error)s") % {"error": str(e)}} + ) + return super().clean() + + @staticmethod + def device_whois_info_delete_handler(instance, **kwargs): + """ + Delete WHOIS information for a device when the last IP address is removed or + when device is deleted. + """ + if instance._get_organization__config_settings().whois_enabled: + transaction.on_commit(lambda: delete_whois_record.delay(instance.last_ip)) + + # this method is kept here instead of in OrganizationConfigSettings because + # currently the caching is used only for WHOIS feature + @staticmethod + def invalidate_org_settings_cache(instance, **kwargs): + """ + Invalidate the cache for Organization settings on update/delete of + Organization settings instance. + """ + org_id = instance.organization_id + cache.delete(WHOISService.get_cache_key(org_id)) + + @property + def formatted_address(self): + """ + Used as default formatter for address field. + 'filter' is used to remove any None values + """ + return ", ".join( + filter( + None, + [ + self.address.get("city"), + self.address.get("country"), + self.address.get("continent"), + self.address.get("postal"), + ], + ) + ) diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py index 3dc714c0c..68023ea6d 100644 --- a/openwisp_controller/config/controller/views.py +++ b/openwisp_controller/config/controller/views.py @@ -153,6 +153,15 @@ def get(self, request, pk): # updates cache if ip addresses changed if updated: self.update_device_cache(device) + # When update fields are present then save() will run the WHOIS + # lookup. But if there are no update fields, we still want to + # trigger the WHOIS lookup if there is no record for the device's + # last_ip. + elif ( + app_settings.WHOIS_CONFIGURED + and not device.whois_service.get_device_whois_info() + ): + device.whois_service.trigger_whois_lookup() checksum_requested.send( sender=device.__class__, instance=device, request=request ) diff --git a/openwisp_controller/config/migrations/0060_whoisinfo.py b/openwisp_controller/config/migrations/0060_whoisinfo.py new file mode 100644 index 000000000..abacbdb8c --- /dev/null +++ b/openwisp_controller/config/migrations/0060_whoisinfo.py @@ -0,0 +1,83 @@ +# Generated by Django 5.2.1 on 2025-06-26 02:13 + +import django.utils.timezone +import jsonfield.fields +import model_utils.fields +from django.db import migrations, models + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("config", "0059_zerotier_templates_ow_zt_to_global"), + ] + + operations = [ + migrations.CreateModel( + name="WHOISInfo", + fields=[ + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="modified", + ), + ), + ( + "ip_address", + models.GenericIPAddressField( + db_index=True, primary_key=True, serialize=False + ), + ), + ( + "asn", + models.CharField(blank=True, help_text="ASN", max_length=6), + ), + ( + "timezone", + models.CharField(blank=True, help_text="Time zone", max_length=35), + ), + ( + "address", + jsonfield.fields.JSONField( + blank=True, default=dict, help_text="Address" + ), + ), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=20)), + ( + "isp", + models.CharField( + blank=True, + help_text="Organization for ASN", + max_length=100, + ), + ), + ], + options={ + "abstract": False, + "swappable": "CONFIG_WHOISINFO_MODEL", + }, + ), + migrations.AddField( + model_name="organizationconfigsettings", + name="whois_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the WHOIS lookup feature is enabled", + null=True, + verbose_name="WHOIS Enabled", + ), + ), + ] diff --git a/openwisp_controller/config/models.py b/openwisp_controller/config/models.py index 9795abba8..e36280319 100644 --- a/openwisp_controller/config/models.py +++ b/openwisp_controller/config/models.py @@ -10,6 +10,7 @@ from .base.tag import AbstractTaggedTemplate, AbstractTemplateTag from .base.template import AbstractTemplate from .base.vpn import AbstractVpn, AbstractVpnClient +from .base.whois import AbstractWHOISInfo class Device(AbstractDevice): @@ -111,3 +112,13 @@ class OrganizationLimits(AbstractOrganizationLimits): class Meta(AbstractOrganizationLimits.Meta): abstract = False swappable = swapper.swappable_setting("config", "OrganizationLimits") + + +class WHOISInfo(AbstractWHOISInfo): + """ + Stores WHOIS information for devices. + """ + + class Meta(AbstractWHOISInfo.Meta): + abstract = False + swappable = swapper.swappable_setting("config", "WHOISInfo") diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index 783bdcd49..a94617b34 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -1,6 +1,7 @@ import logging from django.conf import settings +from django.core.exceptions import ImproperlyConfigured from django.utils.translation import gettext_lazy as _ logger = logging.getLogger(__name__) @@ -67,3 +68,12 @@ def get_setting(option, default): "API_TASK_RETRY_OPTIONS", dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True), ) +WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None) +WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None) +WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False) +WHOIS_CONFIGURED = WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY +if WHOIS_ENABLED and not WHOIS_CONFIGURED: + raise ImproperlyConfigured( + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "when WHOIS_ENABLED is True." + ) diff --git a/openwisp_controller/config/tests/test_device.py b/openwisp_controller/config/tests/test_device.py index eb053f237..f40950440 100644 --- a/openwisp_controller/config/tests/test_device.py +++ b/openwisp_controller/config/tests/test_device.py @@ -524,6 +524,8 @@ def test_device_field_changed_checks(self): device.management_ip = "10.0.0.1" device.group_id = device_group.id device.organization_id = self._create_org().id + # assigning a random ip to last_ip + device.last_ip = "172.217.22.14" # Another query is generated due to "config.set_status_modified" # on name change with self.assertNumQueries(3): diff --git a/openwisp_controller/config/whois/__init__.py b/openwisp_controller/config/whois/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/whois/handlers.py b/openwisp_controller/config/whois/handlers.py new file mode 100644 index 000000000..6a785c4b6 --- /dev/null +++ b/openwisp_controller/config/whois/handlers.py @@ -0,0 +1,29 @@ +from django.db.models.signals import post_delete, post_save +from swapper import load_model + +from .. import settings as app_settings + + +def connect_whois_handlers(): + if not app_settings.WHOIS_CONFIGURED: + return + + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + post_delete.connect( + WHOISInfo.device_whois_info_delete_handler, + sender=Device, + dispatch_uid="device.delete_whois_info", + ) + post_save.connect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_save", + ) + post_delete.connect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_delete", + ) diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py new file mode 100644 index 000000000..81275a8e3 --- /dev/null +++ b/openwisp_controller/config/whois/service.py @@ -0,0 +1,118 @@ +from ipaddress import ip_address + +from django.core.cache import cache +from django.db import transaction +from swapper import load_model + +from openwisp_controller.config import settings as app_settings + +from .tasks import fetch_whois_details + + +class WHOISService: + """ + A handler class for managing the WHOIS functionality. + """ + + def __init__(self, device): + self.device = device + + @staticmethod + def get_cache_key(org_id): + """ + Used to get cache key for caching org settings of a device. + """ + return f"organization_config_{org_id}" + + @staticmethod + def is_valid_public_ip_address(ip): + """ + Check if given IP address is a valid public IP address. + """ + try: + return ip and ip_address(ip).is_global + except ValueError: + return False + + @staticmethod + def _get_whois_info_from_db(ip_address): + """ + For getting existing WHOISInfo for given IP from db if present. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + return WHOISInfo.objects.filter(ip_address=ip_address) + + @property + def is_whois_enabled(self): + """ + Check if the WHOIS lookup feature is enabled. + The OrganizationConfigSettings are cached as these settings + are not expected to change frequently. The timeout for the cache + is set to the same as the checksum cache timeout for consistency + with DeviceChecksumView. + """ + OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + Config = load_model("config", "Config") + + org_id = self.device.organization.pk + org_settings = cache.get(self.get_cache_key(org_id=org_id)) + if org_settings is None: + try: + org_settings = OrganizationConfigSettings.objects.get( + organization=org_id + ) + except OrganizationConfigSettings.DoesNotExist: + # If organization settings do not exist, fall back to global setting + return app_settings.WHOIS_ENABLED + cache.set( + self.get_cache_key(org_id=org_id), + org_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ) + return getattr(org_settings, "whois_enabled", app_settings.WHOIS_ENABLED) + + def _need_whois_lookup(self, new_ip): + """ + This is used to determine if the WHOIS lookup should be triggered + when the device is saved. + + The lookup is not triggered if: + - The new IP address is None or it is a private IP address. + - The WHOIS information of new ip is already present. + - WHOIS is disabled in the organization settings. (query from db) + """ + + # Check cheap conditions first before hitting the database + if not self.is_valid_public_ip_address(new_ip): + return False + + if self._get_whois_info_from_db(new_ip).exists(): + return False + + return self.is_whois_enabled + + def get_device_whois_info(self): + """ + If the WHOIS lookup feature is enabled and the device ``last_ip`` + is a public IP, it fetches WHOIS information for the network device. + """ + ip_address = self.device.last_ip + if not (self.is_valid_public_ip_address(ip_address) and self.is_whois_enabled): + return None + + return self._get_whois_info_from_db(ip_address=ip_address).first() + + def trigger_whois_lookup(self): + """ + Trigger WHOIS lookup based on the conditions of `_need_whois_lookup`. + Task is triggered on commit to ensure redundant data is not created. + """ + if self._need_whois_lookup(self.device.last_ip): + transaction.on_commit( + lambda: fetch_whois_details.delay( + device_pk=self.device.pk, + initial_ip_address=self.device._initial_last_ip, + new_ip_address=self.device.last_ip, + ) + ) diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py new file mode 100644 index 000000000..2ae434cd7 --- /dev/null +++ b/openwisp_controller/config/whois/tasks.py @@ -0,0 +1,160 @@ +import logging + +import requests +from celery import shared_task +from django.utils.translation import gettext as _ +from geoip2 import errors +from geoip2 import webservice as geoip2_webservice +from openwisp_notifications.signals import notify +from swapper import load_model + +from openwisp_utils.tasks import OpenwispCeleryTask + +from .. import settings as app_settings + +logger = logging.getLogger(__name__) + +EXCEPTION_MESSAGES = { + errors.AddressNotFoundError: _( + "No WHOIS information found for IP address {ip_address}" + ), + errors.AuthenticationError: _( + "Authentication failed for GeoIP2 service. " + "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." + ), + errors.OutOfQueriesError: _( + "Your account has run out of queries for the GeoIP2 service." + ), + errors.PermissionRequiredError: _( + "Your account does not have permission to access this service." + ), +} + + +class WHOISCeleryRetryTask(OpenwispCeleryTask): + """ + Base class for OpenWISP Celery tasks with retry support on failure. + """ + + # this is the exception related to networking errors + # that should trigger a retry of the task. + autoretry_for = (errors.HTTPError,) + + def on_failure(self, exc, task_id, args, kwargs, einfo): + """Notify the user about the failure of the WHOIS task.""" + Device = load_model("config", "Device") + + device_pk = kwargs.get("device_pk") + new_ip_address = kwargs.get("new_ip_address") + device = Device.objects.get(pk=device_pk) + + notify.send( + sender=device, + type="generic_message", + target=device, + action_object=device, + level="error", + message=_( + "Failed to fetch WHOIS details for device" + " [{notification.target}]({notification.target_link})" + ), + description=_( + f"WHOIS details could not be fetched for ip: {new_ip_address}." + ), + ) + logger.error(f"WHOIS lookup failed. Details: {exc}") + return super().on_failure(exc, task_id, args, kwargs, einfo) + + +# device_pk is used when task fails to report for which device failure occurred +@shared_task( + bind=True, + base=WHOISCeleryRetryTask, + **app_settings.API_TASK_RETRY_OPTIONS, +) +def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): + """ + Fetches the WHOIS details of the given IP address + and creates/updates the WHOIS record. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + # The task can be triggered for same ip address multiple times + # so we need to return early if WHOIS is already created. + if WHOISInfo.objects.filter(ip_address=new_ip_address).exists(): + return + + # Host is based on the db that is used to fetch the details. + # As we are using GeoLite2, 'geolite.info' host is used. + # Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example + ip_client = geoip2_webservice.Client( + account_id=app_settings.WHOIS_GEOIP_ACCOUNT, + license_key=app_settings.WHOIS_GEOIP_KEY, + host="geolite.info", + ) + + try: + data = ip_client.city(ip_address=new_ip_address) + + # Catching all possible exceptions raised by the geoip2 client + # and raising them with appropriate messages to be handled by the task + # retry mechanism. + except ( + errors.AddressNotFoundError, + errors.AuthenticationError, + errors.OutOfQueriesError, + errors.PermissionRequiredError, + ) as e: + exc_type = type(e) + message = EXCEPTION_MESSAGES.get(exc_type) + if exc_type is errors.AddressNotFoundError: + message = message.format(ip_address=new_ip_address) + raise exc_type(message) + except requests.RequestException as e: + raise e + + else: + # The attributes are always present in the response, + # but they can be None, so added fallbacks. + address = { + "city": data.city.name or "", + "country": data.country.name or "", + "continent": data.continent.name or "", + "postal": str(data.postal.code or ""), + } + + whois_obj = WHOISInfo( + isp=data.traits.autonomous_system_organization, + asn=data.traits.autonomous_system_number, + timezone=data.location.time_zone, + address=address, + cidr=data.traits.network, + ip_address=new_ip_address, + ) + whois_obj.full_clean() + whois_obj.save() + logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") + + # the following check ensures that for a case when device last_ip + # is not changed and there is no related WHOIS record, we do not + # delete the newly created record as both `initial_ip_address` and + # `new_ip_address` would be same for such case. + if initial_ip_address != new_ip_address: + # If any active devices are linked to the following record, + # then they will trigger this task and new record gets created + # with latest data. + delete_whois_record(ip_address=initial_ip_address) + + +@shared_task +def delete_whois_record(ip_address): + """ + Deletes the WHOIS record for the device's last IP address. + This is used when the device is deleted or its last IP address is changed. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + queryset = WHOISInfo.objects.filter(ip_address=ip_address) + if queryset.exists(): + queryset.delete() diff --git a/openwisp_controller/config/whois/test_whois.py b/openwisp_controller/config/whois/test_whois.py new file mode 100644 index 000000000..f5c93b6d3 --- /dev/null +++ b/openwisp_controller/config/whois/test_whois.py @@ -0,0 +1,501 @@ +import importlib +from unittest import mock + +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.db.models.signals import post_delete, post_save +from django.test import TestCase, TransactionTestCase, override_settings +from django.urls import reverse +from geoip2 import errors +from swapper import load_model + +from ...tests.utils import TestAdminMixin +from .. import settings as app_settings +from .handlers import connect_whois_handlers +from .utils import CreateWHOISMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") +Notification = load_model("openwisp_notifications", "Notification") + +notification_qs = Notification.objects.all() + + +class TestWHOIS(CreateWHOISMixin, TestAdminMixin, TestCase): + # Signals are connected when apps are loaded, + # and if WHOIS is Configured all related WHOIS + # handlers are also connected. Thus we need to + # disconnect them. + def _disconnect_signals(self): + post_delete.disconnect( + WHOISInfo.device_whois_info_delete_handler, + sender=Device, + dispatch_uid="device.delete_whois_info", + ) + post_save.disconnect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_save", + ) + post_delete.disconnect( + WHOISInfo.invalidate_org_settings_cache, + sender=OrganizationConfigSettings, + dispatch_uid="invalidate_org_config_cache_on_org_config_delete", + ) + + @override_settings( + OPENWISP_CONTROLLER_WHOIS_ENABLED=False, + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT=None, + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY=None, + ) + def test_whois_configuration_setting(self): + self._disconnect_signals() + org = self._get_org() + # reload app_settings to apply the overridden settings + importlib.reload(app_settings) + + with self.subTest("Test Signals not connected when WHOIS_CONFIGURED is False"): + # should not connect any handlers since WHOIS_CONFIGURED is False + connect_whois_handlers() + + assert not any( + "device.delete_whois_info" in str(r[0]) for r in post_delete.receivers + ) + assert not any( + "invalidate_org_config_cache_on_org_config_save" in str(r[0]) + for r in post_save.receivers + ) + assert not any( + "invalidate_org_config_cache_on_org_config_delete" in str(r[0]) + for r in post_delete.receivers + ) + + with self.subTest( + "Test WHOIS field hidden on admin when WHOIS_CONFIGURED is False" + ): + self._login() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertNotContains(response, 'name="config_settings-0-whois_enabled"') + + with self.subTest( + "Test ImproperlyConfigured raised when WHOIS_CONFIGURED is False" + ): + with override_settings(OPENWISP_CONTROLLER_WHOIS_ENABLED=True): + with self.assertRaises(ImproperlyConfigured): + # reload app_settings to apply the overridden settings + importlib.reload(app_settings) + + with override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT="test_account_id", + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY="test_license_key", + ): + importlib.reload(app_settings) + with self.subTest( + "Test WHOIS_CONFIGURED is True when both settings are set" + ): + self.assertTrue(app_settings.WHOIS_CONFIGURED) + + with self.subTest("Test Signals connected when WHOIS_CONFIGURED is True"): + connect_whois_handlers() + + assert any( + "device.delete_whois_info" in str(r[0]) + for r in post_delete.receivers + ) + assert any( + "invalidate_org_config_cache_on_org_config_save" in str(r[0]) + for r in post_save.receivers + ) + assert any( + "invalidate_org_config_cache_on_org_config_delete" in str(r[0]) + for r in post_delete.receivers + ) + + with self.subTest( + "Test WHOIS field visible on admin when WHOIS_CONFIGURED is True" + ): + self._login() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertContains(response, 'name="config_settings-0-whois_enabled"') + + def test_whois_enabled(self): + OrganizationConfigSettings.objects.all().delete() + device = self._create_device() + with self.subTest( + "Test WHOIS fallback when Organization settings do not exist" + ): + self.assertEqual( + device.whois_service.is_whois_enabled, app_settings.WHOIS_ENABLED + ) + + org_settings_obj = OrganizationConfigSettings( + organization=self._get_org(), whois_enabled=True + ) + + with self.subTest("Test WHOIS not configured does not allow enabling WHOIS"): + with mock.patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertRaises(ValidationError) as context_manager: + org_settings_obj.full_clean() + try: + self.assertEqual( + context_manager.exception.message_dict["whois_enabled"][0], + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "before enabling WHOIS feature.", + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + with mock.patch.object(app_settings, "WHOIS_CONFIGURED", True): + org_settings_obj.full_clean() + org_settings_obj.save() + + with self.subTest("Test setting WHOIS enabled to True"): + org_settings_obj.whois_enabled = True + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertEqual(getattr(org_settings_obj, "whois_enabled"), True) + + with self.subTest("Test setting WHOIS enabled to False"): + org_settings_obj.whois_enabled = False + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertEqual(getattr(org_settings_obj, "whois_enabled"), False) + + with self.subTest( + "Test setting WHOIS enabled to None fallbacks to global setting" + ): + # reload app_settings to ensure latest settings are applied + importlib.reload(app_settings) + org_settings_obj.whois_enabled = None + org_settings_obj.save(update_fields=["whois_enabled"]) + org_settings_obj.refresh_from_db(fields=["whois_enabled"]) + self.assertEqual( + getattr(org_settings_obj, "whois_enabled"), + app_settings.WHOIS_ENABLED, + ) + + +class TestWHOISInfoModel(CreateWHOISMixin, TestCase): + def test_whois_model_fields_validation(self): + """ + Test db_constraints and validators for WHOISInfo model fields. + """ + with self.assertRaises(ValidationError): + self._create_whois_info(isp="a" * 101) + + with self.assertRaises(ValidationError) as context_manager: + self._create_whois_info(ip_address="127.0.0.1") + try: + self.assertEqual( + context_manager.exception.message_dict["ip_address"][0], + "WHOIS information cannot be created for private IP addresses.", + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + with self.assertRaises(ValidationError): + self._create_whois_info(timezone="a" * 36) + + with self.assertRaises(ValidationError) as context_manager: + self._create_whois_info(cidr="InvalidCIDR") + try: + # Not using assertEqual here because we are adding error message raised by + # ipaddress module to the ValidationError message. + self.assertIn( + "Invalid CIDR format: 'InvalidCIDR'", + context_manager.exception.message_dict["cidr"][0], + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + with self.assertRaises(ValidationError): + self._create_whois_info(asn="InvalidASN") + + +class TestWHOISTransaction(CreateWHOISMixin, TransactionTestCase): + _WHOIS_GEOIP_CLIENT = ( + "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + ) + _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" + _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" + _WHOIS_TASKS_ERR_LOGGER = "openwisp_controller.config.whois.tasks.logger.error" + + def setUp(self): + super().setUp() + self.admin = self._get_admin() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") + def test_whois_task_called(self, mocked_task): + org = self._get_org() + connect_whois_handlers() + + with self.subTest("task called when last_ip is public"): + with mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_called() + mocked_set.assert_called_once() + mocked_task.reset_mock() + + with self.subTest("task called when last_ip is changed and is public"): + with mock.patch("django.core.cache.cache.get") as mocked_get: + device.last_ip = "172.217.22.10" + device.save() + mocked_task.assert_called() + mocked_get.assert_called_once() + mocked_task.reset_mock() + + with self.subTest("task not called when last_ip is private"): + device.last_ip = "10.0.0.1" + device.save() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("task not called when last_ip has related WHOISInfo"): + device.last_ip = "172.217.22.10" + self._create_whois_info(ip_address=device.last_ip) + device.save() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("task not called when WHOIS is disabled"): + Device.objects.all().delete() + org.config_settings.whois_enabled = False + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("task called via DeviceChecksumView when WHOIS is enabled"): + org.config_settings.whois_enabled = True + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + # config is required for checksum view to work + self._create_config(device=device) + # setting remote address field to a public IP to trigger WHOIS task + # since the view uses this header for tracking the device's IP + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR="172.217.22.10", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + + with self.subTest( + "task called via DeviceChecksumView when a device has no WHOIS record" + ): + WHOISInfo.objects.all().delete() + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") + def test_whois_multiple_orgs(self, mocked_task): + org2 = self._create_org(name="test org2", slug="test-org2") + OrganizationConfigSettings.objects.create( + organization=org2, whois_enabled=False + ) + + with self.subTest("Test task calls when device created with public last_ip"): + device1 = self._create_device( + last_ip="172.217.22.10", organization=self._get_org() + ) + mocked_task.assert_called() + mocked_task.reset_mock() + device2 = self._create_device(last_ip="172.217.22.11", organization=org2) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("Test task calls when last_ip is changed and is public"): + device1.last_ip = "172.217.22.12" + device1.save() + mocked_task.assert_called() + mocked_task.reset_mock() + device2.last_ip = "172.217.22.13" + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest("Test fetching WHOIS details"): + whois_obj1 = self._create_whois_info(ip_address=device1.last_ip) + self._create_whois_info(ip_address=device2.last_ip) + self.assertEqual(whois_obj1, device1.whois_service.get_device_whois_info()) + self.assertIsNone(device2.whois_service.get_device_whois_info()) + + with self.subTest("Test task calls in DeviceChecksumView when last_ip changes"): + # config is required for checksum view to work + self._create_config(device=device1) + self._create_config(device=device2) + response = self.client.get( + reverse("controller:device_checksum", args=[device1.pk]), + {"key": device1.key}, + REMOTE_ADDR="172.217.22.20", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device2.pk]), + {"key": device2.key}, + REMOTE_ADDR="172.217.22.30", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + "task called via DeviceChecksumView when a device has no WHOIS record" + ): + WHOISInfo.objects.all().delete() + response = self.client.get( + reverse("controller:device_checksum", args=[device1.pk]), + {"key": device1.key}, + REMOTE_ADDR=device1.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device2.pk]), + {"key": device2.key}, + REMOTE_ADDR=device2.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_INFO_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_whois_creation(self, mock_client, mock_info): + # helper function for asserting the model details with + # mocked api response + connect_whois_handlers() + + def _verify_whois_details(instance, ip_address): + self.assertEqual(instance.isp, "Google LLC") + self.assertEqual(instance.asn, "15169") + self.assertEqual(instance.timezone, "America/Los_Angeles") + self.assertEqual( + instance.address, + { + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + ) + self.assertEqual(instance.cidr, "172.217.22.0/24") + self.assertEqual(instance.ip_address, ip_address) + self.assertEqual( + instance.formatted_address, + "Mountain View, United States, North America, 94043", + ) + + # mocking the response from the geoip2 client + mock_response = mock.MagicMock() + mock_response.city.name = "Mountain View" + mock_response.country.name = "United States" + mock_response.continent.name = "North America" + mock_response.postal.code = "94043" + mock_response.traits.autonomous_system_organization = "Google LLC" + mock_response.traits.autonomous_system_number = 15169 + mock_response.traits.network = "172.217.22.0/24" + mock_response.location.time_zone = "America/Los_Angeles" + mock_client.return_value.city.return_value = mock_response + + with self.subTest("Test WHOIS create when device is created"): + device = self._create_device(last_ip="172.217.22.14") + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + + with self.subTest( + "Test WHOIS create & deletion of old record when last ip is updated" + ): + old_ip_address = device.last_ip + device.last_ip = "172.217.22.10" + device.save() + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + + # details related to old ip address should be deleted + self.assertEqual( + WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 0 + ) + + with self.subTest("Test WHOIS delete when device is deleted"): + ip_address = device.last_ip + device.delete(check_deactivated=False) + self.assertEqual(mock_info.call_count, 0) + mock_info.reset_mock() + + # WHOIS related to the device's last_ip should be deleted + self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) + + # we need to allow the task to propagate exceptions to ensure + # `on_failure` method is called and notifications are executed + @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_ERR_LOGGER) + @mock.patch(_WHOIS_TASKS_WARN_LOGGER) + @mock.patch(_WHOIS_TASKS_INFO_LOGGER) + def test_whois_task_failure_notification(self, mock_info, mock_warn, mock_error): + def assert_logging_on_exception( + exception, info_calls=0, warn_calls=0, error_calls=1 + ): + with self.subTest( + f"Test notifications and logging when {exception.__name__} is raised" + ), mock.patch(self._WHOIS_GEOIP_CLIENT, side_effect=exception("test")): + Device.objects.all().delete() # Clear existing devices + device = self._create_device(last_ip="172.217.22.14") + self.assertEqual(mock_info.call_count, info_calls) + self.assertEqual(mock_warn.call_count, warn_calls) + self.assertEqual(mock_error.call_count, error_calls) + self.assertEqual(notification_qs.count(), 1) + notification = notification_qs.first() + self.assertEqual(notification.actor, device) + self.assertEqual(notification.target, device) + self.assertEqual(notification.type, "generic_message") + self.assertIn( + "Failed to fetch WHOIS details for device", + notification.message, + ) + self.assertIn(device.last_ip, notification.description) + + mock_info.reset_mock() + mock_warn.reset_mock() + mock_error.reset_mock() + notification_qs.delete() + + # Test for all possible exceptions that can be raised by the geoip2 client + assert_logging_on_exception(errors.OutOfQueriesError) + assert_logging_on_exception(errors.AddressNotFoundError) + assert_logging_on_exception(errors.AuthenticationError) + assert_logging_on_exception(errors.PermissionRequiredError) diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py new file mode 100644 index 000000000..fd31ff6cf --- /dev/null +++ b/openwisp_controller/config/whois/utils.py @@ -0,0 +1,36 @@ +from swapper import load_model + +from ..tests.utils import CreateConfigMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class CreateWHOISMixin(CreateConfigMixin): + def _create_whois_info(self, **kwargs): + options = dict( + ip_address="172.217.22.14", + address={ + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + asn="15169", + isp="Google LLC", + timezone="America/Los_Angeles", + cidr="172.217.22.0/24", + ) + + options.update(kwargs) + w = WHOISInfo(**options) + w.full_clean() + w.save() + return w + + def setUp(self): + super().setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), whois_enabled=True + ) diff --git a/requirements.txt b/requirements.txt index 9775ceb98..5a8d14148 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,4 @@ shortuuid~=1.0.13 netaddr~=1.3.0 django-import-export~=4.3.14 jsonfield>=3.1.0,<4.0.0 +geoip2~=5.1.0 diff --git a/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py new file mode 100644 index 000000000..707a44bf5 --- /dev/null +++ b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py @@ -0,0 +1,79 @@ +# Generated by Django 5.2.1 on 2025-06-26 17:48 + +import django.utils.timezone +import jsonfield.fields +import model_utils.fields +from django.db import migrations, models + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("sample_config", "0007_alter_config_status"), + ] + + operations = [ + migrations.CreateModel( + name="WHOISInfo", + fields=[ + ( + "created", + model_utils.fields.AutoCreatedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="created", + ), + ), + ( + "modified", + model_utils.fields.AutoLastModifiedField( + default=django.utils.timezone.now, + editable=False, + verbose_name="modified", + ), + ), + ( + "ip_address", + models.GenericIPAddressField( + db_index=True, primary_key=True, serialize=False + ), + ), + ( + "isp", + models.CharField( + blank=True, help_text="Organization for ASN", max_length=100 + ), + ), + ("asn", models.CharField(blank=True, help_text="ASN", max_length=6)), + ( + "timezone", + models.CharField(blank=True, help_text="Time zone", max_length=35), + ), + ( + "address", + jsonfield.fields.JSONField( + blank=True, default=dict, help_text="Address" + ), + ), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=20)), + ("details", models.CharField(blank=True, max_length=64, null=True)), + ], + options={ + "abstract": False, + }, + ), + migrations.AddField( + model_name="organizationconfigsettings", + name="whois_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the WHOIS lookup feature is enabled", + null=True, + verbose_name="WHOIS Enabled", + ), + ), + ] diff --git a/tests/openwisp2/sample_config/models.py b/tests/openwisp2/sample_config/models.py index 8fec929db..e0e6448a5 100644 --- a/tests/openwisp2/sample_config/models.py +++ b/tests/openwisp2/sample_config/models.py @@ -13,6 +13,7 @@ ) from openwisp_controller.config.base.template import AbstractTemplate from openwisp_controller.config.base.vpn import AbstractVpn, AbstractVpnClient +from openwisp_controller.config.base.whois import AbstractWHOISInfo class DetailsModel(models.Model): @@ -111,3 +112,12 @@ class OrganizationLimits(DetailsModel, AbstractOrganizationLimits): class Meta(AbstractOrganizationLimits.Meta): abstract = False + + +class WHOISInfo(DetailsModel, AbstractWHOISInfo): + """ + Stores WHOIS information for devices. + """ + + class Meta(AbstractWHOISInfo.Meta): + abstract = False diff --git a/tests/openwisp2/settings.py b/tests/openwisp2/settings.py index c46e3e50a..ea1a79a56 100644 --- a/tests/openwisp2/settings.py +++ b/tests/openwisp2/settings.py @@ -208,6 +208,14 @@ OPENWISP_CONTROLLER_CONTEXT = {"vpnserver1": "vpn.testdomain.com"} OPENWISP_USERS_AUTH_API = True +# GEOIP Related Settings +OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT = os.getenv( + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT", "" +) +OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY = os.getenv( + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY", "" +) + TEST_RUNNER = "openwisp_utils.tests.TimeLoggingTestRunner" if os.environ.get("SAMPLE_APP", False): @@ -266,6 +274,7 @@ CONFIG_VPNCLIENT_MODEL = "sample_config.VpnClient" CONFIG_ORGANIZATIONCONFIGSETTINGS_MODEL = "sample_config.OrganizationConfigSettings" CONFIG_ORGANIZATIONLIMITS_MODEL = "sample_config.OrganizationLimits" + CONFIG_WHOISINFO_MODEL = "sample_config.WHOISInfo" DJANGO_X509_CA_MODEL = "sample_pki.Ca" DJANGO_X509_CERT_MODEL = "sample_pki.Cert" GEO_LOCATION_MODEL = "sample_geo.Location" From 2174fcb12f6d405d91b5fc883dd03958bd1bb29f Mon Sep 17 00:00:00 2001 From: JgtAman <41839176+DragnEmperor@users.noreply.github.com> Date: Tue, 22 Jul 2025 23:25:24 +0530 Subject: [PATCH 11/26] [feature] Added WHOIS details to the UI #1026 Closes #1026 Signed-off-by: DragnEmperor --- docs/user/rest-api.rst | 15 ++ docs/user/settings.rst | 4 +- docs/user/whois.rst | 17 ++ openwisp_controller/config/admin.py | 17 +- openwisp_controller/config/api/serializers.py | 5 +- .../{0060_whoisinfo.py => 0061_whoisinfo.py} | 2 +- .../config/static/whois/css/whois.css | 113 +++++++++++++ .../static/whois/images/whois_globe.svg | 3 + .../config/static/whois/js/whois.js | 56 +++++++ .../templates/admin/config/change_form.html | 3 + openwisp_controller/config/tests/test_api.py | 12 +- openwisp_controller/config/whois/mixins.py | 24 +++ .../config/whois/serializers.py | 28 ++++ .../config/whois/test_whois.py | 151 +++++++++++++++++- .../config/whois/tests_utils.py | 36 +++++ openwisp_controller/config/whois/utils.py | 49 +++--- 16 files changed, 491 insertions(+), 44 deletions(-) rename openwisp_controller/config/migrations/{0060_whoisinfo.py => 0061_whoisinfo.py} (97%) create mode 100644 openwisp_controller/config/static/whois/css/whois.css create mode 100644 openwisp_controller/config/static/whois/images/whois_globe.svg create mode 100644 openwisp_controller/config/static/whois/js/whois.js create mode 100644 openwisp_controller/config/whois/mixins.py create mode 100644 openwisp_controller/config/whois/serializers.py create mode 100644 openwisp_controller/config/whois/tests_utils.py diff --git a/docs/user/rest-api.rst b/docs/user/rest-api.rst index a4e15febe..c58d1ce93 100644 --- a/docs/user/rest-api.rst +++ b/docs/user/rest-api.rst @@ -68,6 +68,14 @@ List Devices GET /api/v1/controller/device/ +.. _device_list_whois: + +**WHOIS Details** + +If :doc:`WHOIS Lookup feature ` is enabled, each device in the list +response will also include a ``whois_info`` field with related brief WHOIS +information. + **Available filters** You can filter a list of devices based on their configuration status using @@ -145,6 +153,13 @@ Get Device Detail GET /api/v1/controller/device/{id}/ +.. _device_detail_whois: + +**WHOIS Details** + +If :doc:`WHOIS Lookup feature ` is enabled, the response will also +include a ``whois_info`` field with related detailed WHOIS information. + Download Device Configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/user/settings.rst b/docs/user/settings.rst index 948b82ec8..de49b9799 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -774,8 +774,8 @@ documentation regarding automatic retries for known errors. Allows enabling the optional :doc:`WHOIS Lookup feature `. -.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.2/whois-admin-setting.png - :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.2/whois-admin-setting.png +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-setting.png :alt: WHOIS admin setting After enabling this feature, you have to set diff --git a/docs/user/whois.rst b/docs/user/whois.rst index 3039569a4..8fc7d0e1c 100644 --- a/docs/user/whois.rst +++ b/docs/user/whois.rst @@ -78,3 +78,20 @@ Setup Instructions - Set :ref:`OPENWISP_CONTROLLER_WHOIS_ENABLED` to ``True``. - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT` to **Account ID**. - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY` to **License Key**. + +Viewing WHOIS Lookup Data +------------------------- + +Once the WHOIS Lookup feature is enabled and WHOIS data is available, the +retrieved details can be viewed in the following locations: + +- **Device Admin**: On the device's admin page, the WHOIS data is + displayed alongside the device's last IP address. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-details.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-details.png + :alt: WHOIS admin details + +- **Device REST API**: See WHOIS details in the :ref:`Device List + ` and :ref:`Device Detail ` + responses. diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index 4ba8ea83b..4ec225bb2 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -31,6 +31,7 @@ from swapper import load_model from openwisp_controller.config.views import get_default_values, get_relevant_templates +from openwisp_controller.config.whois.utils import get_whois_info from openwisp_users.admin import OrganizationAdmin from openwisp_users.multitenancy import MultitenantOrgFilter from openwisp_utils.admin import ( @@ -49,6 +50,7 @@ logger = logging.getLogger(__name__) prefix = "config/" +whois_prefix = "whois/" Config = load_model("config", "Config") Device = load_model("config", "Device") DeviceGroup = load_model("config", "DeviceGroup") @@ -569,12 +571,21 @@ class DeviceAdmin(MultitenantAdminMixin, BaseConfigAdmin, UUIDAdmin): fields.insert(0, "hardware_id") list_select_related = ("config", "organization") - class Media(BaseConfigAdmin.Media): + # overriding media property to allow testing in isolation + # as class Media is evaluated at import time making the + # settings not overridable in tests + @property + def media(self): js = BaseConfigAdmin.Media.js + [ f"{prefix}js/tabs.js", f"{prefix}js/management_ip.js", f"{prefix}js/relevant_templates.js", ] + css = BaseConfigAdmin.Media.css["all"] + if app_settings.WHOIS_CONFIGURED: + js.append(f"{whois_prefix}js/whois.js") + css += (f"{whois_prefix}css/whois.css",) + return super().media + forms.Media(js=js, css={"all": css}) def has_change_permission(self, request, obj=None): perm = super().has_change_permission(request) @@ -928,6 +939,10 @@ def get_extra_context(self, pk=None): ), } ) + # passing the whois details to the context to avoid + # the need to make an additional request in the js + if data := get_whois_info(pk): + ctx["device_whois_details"] = json.dumps(data) return ctx def add_view(self, request, form_url="", extra_context=None): diff --git a/openwisp_controller/config/api/serializers.py b/openwisp_controller/config/api/serializers.py index 7e67d8f5f..9402bbcbf 100644 --- a/openwisp_controller/config/api/serializers.py +++ b/openwisp_controller/config/api/serializers.py @@ -9,6 +9,7 @@ from ...serializers import BaseSerializer from .. import settings as app_settings +from ..whois.mixins import BriefWHOISMixin, WHOISMixin Template = load_model("config", "Template") Vpn = load_model("config", "Vpn") @@ -220,7 +221,7 @@ class DeviceListConfigSerializer(BaseConfigSerializer): templates = FilterTemplatesByOrganization(many=True, write_only=True) -class DeviceListSerializer(DeviceConfigSerializer): +class DeviceListSerializer(BriefWHOISMixin, DeviceConfigSerializer): config = DeviceListConfigSerializer(required=False) class Meta(BaseMeta): @@ -274,7 +275,7 @@ class DeviceDetailConfigSerializer(BaseConfigSerializer): templates = FilterTemplatesByOrganization(many=True) -class DeviceDetailSerializer(DeviceConfigSerializer): +class DeviceDetailSerializer(WHOISMixin, DeviceConfigSerializer): config = DeviceDetailConfigSerializer(allow_null=True) is_deactivated = serializers.BooleanField(read_only=True) diff --git a/openwisp_controller/config/migrations/0060_whoisinfo.py b/openwisp_controller/config/migrations/0061_whoisinfo.py similarity index 97% rename from openwisp_controller/config/migrations/0060_whoisinfo.py rename to openwisp_controller/config/migrations/0061_whoisinfo.py index abacbdb8c..d8148756b 100644 --- a/openwisp_controller/config/migrations/0060_whoisinfo.py +++ b/openwisp_controller/config/migrations/0061_whoisinfo.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ - ("config", "0059_zerotier_templates_ow_zt_to_global"), + ("config", "0060_cleanup_api_task_notification_types"), ] operations = [ diff --git a/openwisp_controller/config/static/whois/css/whois.css b/openwisp_controller/config/static/whois/css/whois.css new file mode 100644 index 000000000..44a7b5d1f --- /dev/null +++ b/openwisp_controller/config/static/whois/css/whois.css @@ -0,0 +1,113 @@ +:root { + --whois-bg-light: #f9fafb; + --whois-text-muted: rgba(0, 0, 0, 0.45); + --whois-border-radius: 5px; + --whois-padding: 15px; +} + +.whois-table, +.whois { + margin-top: 1.25em; + max-width: 500px; + width: 100%; + border-radius: var(--whois-border-radius); +} +.whois-table { + border-spacing: 0; + box-shadow: 0 0 0 1px #ccc; + border-style: none; +} +.whois-table th, +.whois-table td { + padding: var(--whois-padding) !important; + border: none; +} +.whois-table tr:first-child { + background-color: var(--whois-bg-light); +} +.whois-table tr:not(:first-child) { + background-color: white; +} +.whois-table th:first-of-type > div { + display: flex; + align-items: center; +} +/* For rounding the individual corners */ +.whois-table th:first-of-type { + border-top-left-radius: var(--whois-border-radius); + border-right: 1px solid var(--border-color); + border-bottom: 1px solid var(--border-color); +} +.whois-table th:last-of-type { + border-top-right-radius: var(--whois-border-radius); + border-bottom: 1px solid var(--border-color); +} +.whois-table tr:last-child td:first-child { + border-bottom-left-radius: var(--whois-border-radius); + border-right: 1px solid var(--border-color); +} +.whois-table tr:last-child td:last-child { + border-bottom-right-radius: var(--whois-border-radius); +} +.whois-table .ow-info-icon { + width: 1rem; + height: 1rem; + mask-repeat: no-repeat; + mask-position: center; + mask-size: contain; + display: inline-block; + flex-shrink: 0; + background: #777; + margin-left: 1em; +} +.whois-table .ow-info-icon:hover { + background: #007cba; +} + +.whois { + background: var(--whois-bg-light); +} +.whois summary { + background: white; + border-radius: var(--whois-border-radius); + list-style: none; + display: flex; + align-items: center; + color: var(--whois-text-muted) !important; + font-weight: 600; + cursor: pointer; +} +details.whois[open] summary { + padding-bottom: 0 !important; +} +.whois summary > div { + display: flex; + width: 100%; + justify-content: space-between; + margin-left: 1.4em; +} +.whois summary::-webkit-details-marker { + display: none; +} +.whois summary .mg-arrow { + transform: scale(1.75); + display: inline; +} +.whois > div { + padding: var(--whois-padding); + font-size: 1em; +} +.whois .additional-text { + display: block; +} +.whois .additional-text + .additional-text { + margin-top: 5px; +} +.whois-globe { + flex-shrink: 0; + mask: url(../images/whois_globe.svg) no-repeat center; + -webkit-mask: url(../images/whois_globe.svg) no-repeat center; + width: 1.1rem; + height: 1.1rem; + background-color: var(--whois-text-muted); +} diff --git a/openwisp_controller/config/static/whois/images/whois_globe.svg b/openwisp_controller/config/static/whois/images/whois_globe.svg new file mode 100644 index 000000000..74116f1a6 --- /dev/null +++ b/openwisp_controller/config/static/whois/images/whois_globe.svg @@ -0,0 +1,3 @@ + + + diff --git a/openwisp_controller/config/static/whois/js/whois.js b/openwisp_controller/config/static/whois/js/whois.js new file mode 100644 index 000000000..20f4339f5 --- /dev/null +++ b/openwisp_controller/config/static/whois/js/whois.js @@ -0,0 +1,56 @@ +"use strict"; + +if (typeof gettext === "undefined") { + var gettext = function (word) { + return word; + }; +} + +django.jQuery(function ($) { + const $addForm = $(".add-form"); + const $deviceForm = $("#device_form"); + + if ( + $addForm.length || + !$deviceForm.length || + typeof deviceWHOISDetails === "undefined" + ) { + return; + } + const $parentDiv = $("#overview-group .field-last_ip div:last"); + const tooltipText = gettext( + "This is the Organization associated with registered ASN", + ); + + $parentDiv.after( + ` + + + + + + + + +
+
+ ${gettext("ISP")} + +
+ 		${gettext("Country")}
${deviceWHOISDetails.isp}${deviceWHOISDetails.address.country}
+
+ + +
+ ${gettext("Additional Details")} +
+ +
+ ${gettext("ASN")}: ${deviceWHOISDetails.asn} + ${gettext("Timezone")}: ${deviceWHOISDetails.timezone} + ${gettext("Address")}: ${deviceWHOISDetails.formatted_address} + ${gettext("CIDR")}: ${deviceWHOISDetails.cidr} +
+
`, + ); +}); diff --git a/openwisp_controller/config/templates/admin/config/change_form.html b/openwisp_controller/config/templates/admin/config/change_form.html index 5e0c993a0..02536c00f 100644 --- a/openwisp_controller/config/templates/admin/config/change_form.html +++ b/openwisp_controller/config/templates/admin/config/change_form.html @@ -19,6 +19,9 @@ const owControllerApiHost = window.location; {% endif %} const owCommandApiEndpoint = '{{ commands_api_endpoint | safe }}'; + {% if device_whois_details %} + const deviceWHOISDetails = {{ device_whois_details | safe }}; + {% endif %} {{ block.super}} {% endblock extrahead %} diff --git a/openwisp_controller/config/tests/test_api.py b/openwisp_controller/config/tests/test_api.py index 068528610..fe12fe2cf 100644 --- a/openwisp_controller/config/tests/test_api.py +++ b/openwisp_controller/config/tests/test_api.py @@ -250,7 +250,9 @@ def test_device_create_with_devicegroup(self): def test_device_list_api(self): device = self._create_device() path = reverse("config_api:device_list") - with self.assertNumQueries(3): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(3): r = self.client.get(path) self.assertEqual(r.status_code, 200) with self.subTest("device list should show most recent first"): @@ -396,7 +398,9 @@ def test_device_filter_templates(self): def test_device_detail_api(self): d1 = self._create_device() path = reverse("config_api:device_detail", args=[d1.pk]) - with self.assertNumQueries(2): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(2): r = self.client.get(path) self.assertEqual(r.status_code, 200) self.assertEqual(r.data["config"], None) @@ -406,7 +410,9 @@ def test_device_detail_config_api(self): d1 = self._create_device() self._create_config(device=d1) path = reverse("config_api:device_detail", args=[d1.pk]) - with self.assertNumQueries(3): + with patch.object( + app_settings, "WHOIS_CONFIGURED", False + ), self.assertNumQueries(3): r = self.client.get(path) self.assertEqual(r.status_code, 200) self.assertNotEqual(r.data["config"], None) diff --git a/openwisp_controller/config/whois/mixins.py b/openwisp_controller/config/whois/mixins.py new file mode 100644 index 000000000..4e2ee2ea7 --- /dev/null +++ b/openwisp_controller/config/whois/mixins.py @@ -0,0 +1,24 @@ +from .. import settings as app_settings +from .serializers import BriefWHOISSerializer, WHOISSerializer + + +class WHOISMixin: + """Mixin to add WHOIS information to the device representation.""" + + serializer_class = WHOISSerializer + + def to_representation(self, obj): + data = super().to_representation(obj) + if app_settings.WHOIS_CONFIGURED and obj.whois_service.is_whois_enabled: + data["whois_info"] = self.get_whois_info(obj) + return data + + def get_whois_info(self, obj): + whois_obj = obj.whois_service.get_device_whois_info() + if not whois_obj: + return None + return self.serializer_class(whois_obj).data + + +class BriefWHOISMixin(WHOISMixin): + serializer_class = BriefWHOISSerializer diff --git a/openwisp_controller/config/whois/serializers.py b/openwisp_controller/config/whois/serializers.py new file mode 100644 index 000000000..fa859b8e6 --- /dev/null +++ b/openwisp_controller/config/whois/serializers.py @@ -0,0 +1,28 @@ +from rest_framework import serializers +from swapper import load_model + +WHOISInfo = load_model("config", "WHOISInfo") + + +class BriefWHOISSerializer(serializers.ModelSerializer): + """ + Serializer for brief representation of WHOIS model. + """ + + country = serializers.CharField(source="address.country", read_only=True) + + class Meta: + model = WHOISInfo + fields = ("isp", "country", "ip_address") + + +class WHOISSerializer(serializers.ModelSerializer): + """ + Serializer for detailed representation of WHOIS model. + """ + + address = serializers.JSONField() + + class Meta: + model = WHOISInfo + fields = "__all__" diff --git a/openwisp_controller/config/whois/test_whois.py b/openwisp_controller/config/whois/test_whois.py index f5c93b6d3..d98fb8336 100644 --- a/openwisp_controller/config/whois/test_whois.py +++ b/openwisp_controller/config/whois/test_whois.py @@ -1,17 +1,21 @@ import importlib from unittest import mock +from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.core.exceptions import ImproperlyConfigured, ValidationError from django.db.models.signals import post_delete, post_save -from django.test import TestCase, TransactionTestCase, override_settings +from django.test import TestCase, TransactionTestCase, override_settings, tag from django.urls import reverse from geoip2 import errors +from selenium.webdriver.common.by import By from swapper import load_model +from openwisp_utils.tests import SeleniumTestMixin + from ...tests.utils import TestAdminMixin from .. import settings as app_settings from .handlers import connect_whois_handlers -from .utils import CreateWHOISMixin +from .tests_utils import CreateWHOISMixin Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") @@ -183,6 +187,83 @@ def test_whois_enabled(self): app_settings.WHOIS_ENABLED, ) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_details_device_api(self): + """ + Test the WHOIS details API endpoint. + """ + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + self._login() + + with self.subTest( + "Device List API has whois_info when WHOIS_CONFIGURED is True" + ): + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data["results"][0]) + self.assertDictEqual( + response.data["results"][0]["whois_info"], + { + "isp": whois_obj.isp, + "country": whois_obj.address["country"], + "ip_address": whois_obj.ip_address, + }, + ) + + with self.subTest( + "Device Detail API has whois_info when WHOIS_CONFIGURED is True" + ): + + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data) + api_whois_info = response.data["whois_info"] + self.assertEqual(api_whois_info["isp"], whois_obj.isp) + self.assertEqual(api_whois_info["cidr"], whois_obj.cidr) + self.assertEqual(api_whois_info["asn"], whois_obj.asn) + self.assertEqual(api_whois_info["timezone"], whois_obj.timezone) + self.assertEqual(api_whois_info["address"], whois_obj.address) + + with self.subTest( + "Device List API has whois_info as None when no WHOIS Info exists" + ): + device.last_ip = "172.217.22.24" + device.save() + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data["results"][0]) + self.assertIsNone(response.data["results"][0]["whois_info"]) + + with self.subTest( + "Device Detail API has whois_info as None when no WHOIS Info exists" + ): + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertIn("whois_info", response.data) + self.assertIsNone(response.data["whois_info"]) + + with mock.patch.object(app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Device List API has no whois_info when WHOIS_CONFIGURED is False" + ): + response = self.client.get(reverse("config_api:device_list")) + self.assertEqual(response.status_code, 200) + self.assertNotIn("whois_info", response.data["results"][0]) + + with self.subTest( + "Device Detail API has no whois_info when WHOIS_CONFIGURED is False" + ): + response = self.client.get( + reverse("config_api:device_detail", args=[device.pk]) + ) + self.assertEqual(response.status_code, 200) + self.assertNotIn("whois_info", response.data) + class TestWHOISInfoModel(CreateWHOISMixin, TestCase): def test_whois_model_fields_validation(self): @@ -223,7 +304,7 @@ def test_whois_model_fields_validation(self): class TestWHOISTransaction(CreateWHOISMixin, TransactionTestCase): _WHOIS_GEOIP_CLIENT = ( - "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + "openwisp_controller.config.whois.tasks.geoip2_webservice.Client.city" ) _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" @@ -260,7 +341,7 @@ def test_whois_task_called(self, mocked_task): mocked_task.assert_not_called() mocked_task.reset_mock() - with self.subTest("task not called when last_ip has related WHOISInfo"): + with self.subTest("task not called when last_ip has related WHOIS Info"): device.last_ip = "172.217.22.10" self._create_whois_info(ip_address=device.last_ip) device.save() @@ -419,7 +500,7 @@ def _verify_whois_details(instance, ip_address): mock_response.traits.autonomous_system_number = 15169 mock_response.traits.network = "172.217.22.0/24" mock_response.location.time_zone = "America/Los_Angeles" - mock_client.return_value.city.return_value = mock_response + mock_client.return_value = mock_response with self.subTest("Test WHOIS create when device is created"): device = self._create_device(last_ip="172.217.22.14") @@ -499,3 +580,63 @@ def assert_logging_on_exception( assert_logging_on_exception(errors.AddressNotFoundError) assert_logging_on_exception(errors.AuthenticationError) assert_logging_on_exception(errors.PermissionRequiredError) + + +@tag("selenium_tests") +class TestWHOISSelenium(CreateWHOISMixin, SeleniumTestMixin, StaticLiveServerTestCase): + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_device_admin(self): + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + self.login() + + with self.subTest( + "WHOIS details visible in device admin when WHOIS_CONFIGURED is True" + ): + self.open(reverse("admin:config_device_change", args=[device.pk])) + table = self.find_element(By.CSS_SELECTOR, "table.whois-table") + rows = table.find_elements(By.TAG_NAME, "tr") + for row in rows: + if cells := row.find_elements(By.TAG_NAME, "td"): + self.assertEqual(cells[0].text, whois_obj.isp) + self.assertEqual(cells[1].text, whois_obj.address["country"]) + + details = self.find_element(By.CSS_SELECTOR, "details.whois") + self.web_driver.execute_script( + "arguments[0].setAttribute('open','')", details + ) + additional_text = details.find_elements(By.CSS_SELECTOR, ".additional-text") + self.assertIn(whois_obj.asn, additional_text[0].text) + self.assertIn(whois_obj.timezone, additional_text[1].text) + self.assertIn(whois_obj.formatted_address, additional_text[2].text) + self.assertIn(whois_obj.cidr, additional_text[3].text) + + with mock.patch.object(app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "WHOIS details not visible in device admin " + + "when WHOIS_CONFIGURED is False" + ): + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + + with self.subTest( + "WHOIS details not visible in device admin when WHOIS is disabled" + ): + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save(update_fields=["whois_enabled"]) + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + + with self.subTest( + "WHOIS details not visible in device admin when WHOIS Info does not exist" + ): + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save(update_fields=["whois_enabled"]) + WHOISInfo.objects.all().delete() + self.open(reverse("admin:config_device_change", args=[device.pk])) + self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") + self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") diff --git a/openwisp_controller/config/whois/tests_utils.py b/openwisp_controller/config/whois/tests_utils.py new file mode 100644 index 000000000..fd31ff6cf --- /dev/null +++ b/openwisp_controller/config/whois/tests_utils.py @@ -0,0 +1,36 @@ +from swapper import load_model + +from ..tests.utils import CreateConfigMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class CreateWHOISMixin(CreateConfigMixin): + def _create_whois_info(self, **kwargs): + options = dict( + ip_address="172.217.22.14", + address={ + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + asn="15169", + isp="Google LLC", + timezone="America/Los_Angeles", + cidr="172.217.22.0/24", + ) + + options.update(kwargs) + w = WHOISInfo(**options) + w.full_clean() + w.save() + return w + + def setUp(self): + super().setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), whois_enabled=True + ) diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py index fd31ff6cf..51919af08 100644 --- a/openwisp_controller/config/whois/utils.py +++ b/openwisp_controller/config/whois/utils.py @@ -1,36 +1,25 @@ from swapper import load_model -from ..tests.utils import CreateConfigMixin +from .. import settings as app_settings +from .serializers import WHOISSerializer +from .service import WHOISService Device = load_model("config", "Device") -WHOISInfo = load_model("config", "WHOISInfo") -OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") -class CreateWHOISMixin(CreateConfigMixin): - def _create_whois_info(self, **kwargs): - options = dict( - ip_address="172.217.22.14", - address={ - "city": "Mountain View", - "country": "United States", - "continent": "North America", - "postal": "94043", - }, - asn="15169", - isp="Google LLC", - timezone="America/Los_Angeles", - cidr="172.217.22.0/24", - ) - - options.update(kwargs) - w = WHOISInfo(**options) - w.full_clean() - w.save() - return w - - def setUp(self): - super().setUp() - OrganizationConfigSettings.objects.create( - organization=self._get_org(), whois_enabled=True - ) +def get_whois_info(pk): + if not app_settings.WHOIS_CONFIGURED or not pk: + return None + device = ( + Device.objects.select_related("organization__config_settings") + .filter(pk=pk) + .first() + ) + if not device or not device._get_organization__config_settings().whois_enabled: + return None + whois_obj = WHOISService(device).get_device_whois_info() + if not whois_obj: + return None + data = WHOISSerializer(whois_obj).data + data["formatted_address"] = getattr(whois_obj, "formatted_address", None) + return data From 66e14ae2dd331292a3f801dde39efa636770b6af Mon Sep 17 00:00:00 2001 From: JgtAman <41839176+DragnEmperor@users.noreply.github.com> Date: Sat, 20 Sep 2025 21:51:12 +0530 Subject: [PATCH 12/26] [feature] Allow estimating geographic location based on WHOIS data #1034 - Estimated geographic location logic, closes #1034. - Added notification and admin warning, closes #1035. - Set location estimated flag to ``False`` on manual update, closes #1036. - Added admin and API filters for estimated locations, closes #1028. --------- Signed-off-by: DragnEmperor --- docs/index.rst | 1 + docs/user/estimated-location.rst | 80 ++ docs/user/rest-api.rst | 43 + docs/user/settings.rst | 17 + docs/user/whois.rst | 39 +- openwisp_controller/config/admin.py | 2 +- openwisp_controller/config/base/device.py | 12 +- .../config/base/multitenancy.py | 14 + openwisp_controller/config/base/whois.py | 61 +- .../config/controller/views.py | 9 - .../config/management/__init__.py | 0 .../config/management/commands/__init__.py | 0 .../management/commands/clear_last_ip.py | 51 ++ ...s_approximate_location_enabled_and_more.py | 35 + openwisp_controller/config/settings.py | 5 + openwisp_controller/config/whois/service.py | 94 ++- openwisp_controller/config/whois/tasks.py | 48 +- .../config/whois/tests/__init__.py | 0 .../whois/{test_whois.py => tests/tests.py} | 215 +++-- .../config/whois/tests/utils.py | 152 ++++ .../config/whois/tests_utils.py | 36 - openwisp_controller/config/whois/utils.py | 59 +- openwisp_controller/geo/admin.py | 54 +- openwisp_controller/geo/api/filters.py | 15 + openwisp_controller/geo/api/serializers.py | 14 +- openwisp_controller/geo/api/views.py | 8 + openwisp_controller/geo/apps.py | 3 + openwisp_controller/geo/base/models.py | 73 ++ .../geo/estimated_location/__init__.py | 0 .../geo/estimated_location/handlers.py | 41 + .../geo/estimated_location/mixins.py | 30 + .../geo/estimated_location/tasks.py | 170 ++++ .../geo/estimated_location/tests/__init__.py | 0 .../geo/estimated_location/tests/tests.py | 761 ++++++++++++++++++ .../geo/estimated_location/tests/utils.py | 15 + .../geo/estimated_location/utils.py | 6 + .../migrations/0004_location_is_estimated.py | 21 + .../admin/geo/location/change_form.html | 11 + openwisp_controller/geo/tests/test_admin.py | 10 + ...s_approximate_location_enabled_and_more.py | 35 + .../migrations/0004_location_is_estimated.py | 21 + 41 files changed, 2064 insertions(+), 197 deletions(-) create mode 100644 docs/user/estimated-location.rst create mode 100644 openwisp_controller/config/management/__init__.py create mode 100644 openwisp_controller/config/management/commands/__init__.py create mode 100644 openwisp_controller/config/management/commands/clear_last_ip.py create mode 100644 openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py create mode 100644 openwisp_controller/config/whois/tests/__init__.py rename openwisp_controller/config/whois/{test_whois.py => tests/tests.py} (80%) create mode 100644 openwisp_controller/config/whois/tests/utils.py delete mode 100644 openwisp_controller/config/whois/tests_utils.py create mode 100644 openwisp_controller/geo/estimated_location/__init__.py create mode 100644 openwisp_controller/geo/estimated_location/handlers.py create mode 100644 openwisp_controller/geo/estimated_location/mixins.py create mode 100644 openwisp_controller/geo/estimated_location/tasks.py create mode 100644 openwisp_controller/geo/estimated_location/tests/__init__.py create mode 100644 openwisp_controller/geo/estimated_location/tests/tests.py create mode 100644 openwisp_controller/geo/estimated_location/tests/utils.py create mode 100644 openwisp_controller/geo/estimated_location/utils.py create mode 100644 openwisp_controller/geo/migrations/0004_location_is_estimated.py create mode 100644 openwisp_controller/geo/templates/admin/geo/location/change_form.html create mode 100644 tests/openwisp2/sample_config/migrations/0009_organizationconfigsettings_approximate_location_enabled_and_more.py create mode 100644 tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py diff --git a/docs/index.rst b/docs/index.rst index 6895e1c48..c6b268f55 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,6 +49,7 @@ the OpenWISP architecture. user/openvpn.rst user/subnet-division-rules.rst user/whois.rst + user/estimated-location.rst user/rest-api.rst user/settings.rst diff --git a/docs/user/estimated-location.rst b/docs/user/estimated-location.rst new file mode 100644 index 000000000..1f00a7440 --- /dev/null +++ b/docs/user/estimated-location.rst @@ -0,0 +1,80 @@ +Estimated Location +================== + +.. important:: + + The **Estimated Location** feature is **disabled by default**. + + Before enabling it, the :doc:`WHOIS Lookup feature ` must be + enabled. Then set + :ref:`OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED` to ``True`` + +.. contents:: **Table of contents**: + :depth: 1 + :local: + +Overview +-------- + +The Estimated Location feature automatically creates or updates a device’s +location based on latitude and longitude information retrieved from the +WHOIS Lookup feature. + +Trigger Conditions +------------------ + +Estimated Location is triggered when: + +- A **fresh WHOIS lookup** is performed for a device. +- Or when a WHOIS record already exists for the device’s IP **and**: + + - The device’s last IP address is **public**. + - WHOIS lookup and Estimated Location is **enabled** for the device’s + organization. + +Behavior +-------- + +The system will **attach the already existing matching location** of +another device with same ip to the current device if: + +- Only one device is found with that IP and it has a location. +- The current device **has no location** or that location is + **estimated**. + +If there are multiple devices with location for the same IP, the system +will **not attach any location** to the current device and a notification +will be sent suggesting the user to manually assign/create a location for +the device. + +If there is **no matching location**, a new estimated location is created +or the existing one is updated using coordinates from the WHOIS record, +but only if the existing location is estimated. + +If two devices share the same IP address and are assigned to the same +location, and the last IP of one of the devices is updated, the system +will create a new estimated location for that device. + +Visibility of Estimated Status +------------------------------ + +The estimated status of a location is visible on the location page if the +feature is enabled for the organization. The location admin page also +includes indicators for the estimated status. + +- The name of the location will have suffix **(Estimated Location : + )**. +- A warning on top of the page. +- **Is Estimated** field. + +Changes to the ``coordinates`` and ``geometry`` of the estimated location +will set the ``is_estimated`` field to ``False`` and remove the +"(Estimated Location)" suffix with IP from the location name. + +In REST API, the field will be visible in the :ref:`Device Location +`, :ref:`Location list +`, :ref:`Location Detail +` and :ref:`Location list (GeoJson) +` if the feature is **enabled**. The field can +also be used for filtering in the location list (including geojson) +endpoints and in the :ref:`Device List `. diff --git a/docs/user/rest-api.rst b/docs/user/rest-api.rst index c58d1ce93..fe2efba0e 100644 --- a/docs/user/rest-api.rst +++ b/docs/user/rest-api.rst @@ -76,6 +76,14 @@ If :doc:`WHOIS Lookup feature ` is enabled, each device in the list response will also include a ``whois_info`` field with related brief WHOIS information. +.. _device_list_estimated_filters: + +**Estimated Location Filters** + +if :doc:`Estimated Location feature ` is enabled, +devices can be filtered based on the estimated nature of their location +using the ``geo_is_estimated``. + **Available filters** You can filter a list of devices based on their configuration status using @@ -544,6 +552,13 @@ of certificate's organization as show in the example below: GET /api/v1/controller/cert/{common_name}/group/?org={org1_slug},{org2_slug} +.. |est_loc| replace:: Estimated Location feature + +.. _est_loc: estimated-location.html + +.. |estimated_details| replace:: If |est_loc|_ is enabled, the location + response will also include ``is_estimated`` status field. + Get Device Location ~~~~~~~~~~~~~~~~~~~ @@ -551,6 +566,12 @@ Get Device Location GET /api/v1/controller/device/{id}/location/ +.. _device_location_estimated: + +**Estimated Status** + +|estimated_details| + .. _create_device_location: Create Device Location @@ -787,6 +808,14 @@ List Locations GET /api/v1/controller/location/ +.. _location_list_estimated: + +**Estimated Status** + +|estimated_details| + +Locations can also be filtered using the ``is_estimated``. + **Available filters** You can filter using ``organization_id`` or ``organization_slug`` to get @@ -868,6 +897,12 @@ Get Location Details GET /api/v1/controller/location/{pk}/ +.. _location_detail_estimated: + +**Estimated Status** + +|estimated_details| + Change Location Details ~~~~~~~~~~~~~~~~~~~~~~~ @@ -910,6 +945,14 @@ List Locations with Devices Deployed (in GeoJSON Format) GET /api/v1/controller/location/geojson/ +.. _location_geojson_estimated: + +**Estimated Status** + +|estimated_details| + +Locations can also be filtered using the ``is_estimated``. + **Available filters** You can filter using ``organization_id`` or ``organization_slug`` to get diff --git a/docs/user/settings.rst b/docs/user/settings.rst index de49b9799..9a59bbf22 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -812,3 +812,20 @@ Maxmind Account ID required for the :doc:`WHOIS Lookup feature `. ============ ======= Maxmind License Key required for the :doc:`WHOIS Lookup feature `. + +.. _openwisp_controller_whois_estimated_location_enabled: + +``OPENWISP_CONTROLLER_WHOIS_ESTIMATED_LOCATION_ENABLED`` +-------------------------------------------------------- + +============ ========= +**type**: ``bool`` +**default**: ``False`` +============ ========= + +Allows enabling the optional :doc:`Estimated Location feature +`. + +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png + :alt: Estimated Location setting diff --git a/docs/user/whois.rst b/docs/user/whois.rst index 8fc7d0e1c..e9c63274c 100644 --- a/docs/user/whois.rst +++ b/docs/user/whois.rst @@ -28,6 +28,7 @@ associated with the device's public IP address and includes: - CIDR block assigned to the ASN - Physical address registered to the ASN - Timezone of the ASN's registered location +- Coordinates (Latitude and Longitude) Trigger Conditions ------------------ @@ -40,25 +41,23 @@ A WHOIS lookup is triggered automatically when: However, the lookup will only run if **all** the following conditions are met: +- The device is either **newly created** or has a **changed last IP**. - The device's last IP address is **public**. - There is **no existing WHOIS record** for that IP. - WHOIS lookup is **enabled** for the device's organization. -Behavior with Shared IP Addresses ---------------------------------- +Managing WHOIS Records +---------------------- -If multiple devices share the same public IP address and one of them -switches to a different IP, the following occurs: - -- A lookup is triggered for the **new IP**. -- The WHOIS record for the **old IP** is deleted. -- The next time a device still using the old IP fetches its checksum, a - new lookup is triggered, ensuring up-to-date data. +If a device updates its last IP address, lookup is triggered for the **new +IP** and the **WHOIS record for the old IP** is deleted if no active +devices are associated with that IP address. .. note:: When a device with an associated WHOIS record is deleted, its WHOIS - record is automatically removed. + record is automatically removed only if no active devices are + associated with that IP address. .. _controller_setup_whois_lookup: @@ -79,6 +78,26 @@ Setup Instructions - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT` to **Account ID**. - Set :ref:`OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY` to **License Key**. +6. Restart the application/containers if using ansible-openwisp2 or + docker. +7. Run the ``clear_last_ip`` management command to clear the last IP + address of **all active devices across organizations**. + + - If using ansible-openwisp2 (default directory is /opt/openwisp2, + unless changed in Ansible playbook configuration): + + .. code-block:: bash + + source /opt/openwisp2/env/bin/activate + python /opt/openwisp2/src/manage.py clear_last_ip + + - If using docker: + + .. code-block:: bash + + docker exec -it sh + python manage.py clear_last_ip + Viewing WHOIS Lookup Data ------------------------- diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index 4ec225bb2..af9e20320 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -1393,7 +1393,7 @@ def get_fields(self, request, obj=None): if app_settings.REGISTRATION_ENABLED: fields += ["registration_enabled", "shared_secret"] if app_settings.WHOIS_CONFIGURED: - fields += ["whois_enabled"] + fields += ["whois_enabled", "estimated_location_enabled"] fields += ["context"] return fields diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 1921d5ad3..525098737 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -287,7 +287,7 @@ def save(self, *args, **kwargs): state_adding = self._state.adding super().save(*args, **kwargs) if app_settings.WHOIS_CONFIGURED: - self._check_last_ip() + self._check_last_ip(creating=state_adding) if state_adding and self.group and self.group.templates.exists(): self.create_default_config() # The value of "self._state.adding" will always be "False" @@ -529,9 +529,13 @@ def whois_service(self): """ return WHOISService(self) - def _check_last_ip(self): - """Trigger WHOIS lookup if last_ip is not deferred.""" + def _check_last_ip(self, creating=False): + """ + Process details and location related to last_ip if last_ip has + changed or is being set for the first time. + """ if self._initial_last_ip == models.DEFERRED: return - self.whois_service.trigger_whois_lookup() + if creating or self.last_ip != self._initial_last_ip: + self.whois_service.process_ip_data_and_location() self._initial_last_ip = self.last_ip diff --git a/openwisp_controller/config/base/multitenancy.py b/openwisp_controller/config/base/multitenancy.py index 3095ab779..bceb5de42 100644 --- a/openwisp_controller/config/base/multitenancy.py +++ b/openwisp_controller/config/base/multitenancy.py @@ -39,6 +39,11 @@ class AbstractOrganizationConfigSettings(UUIDModel): fallback=app_settings.WHOIS_ENABLED, verbose_name=_("WHOIS Enabled"), ) + estimated_location_enabled = FallbackBooleanChoiceField( + help_text=_("Whether the estimated location feature is enabled"), + fallback=app_settings.ESTIMATED_LOCATION_ENABLED, + verbose_name=_("Estimated Location Enabled"), + ) context = JSONField( blank=True, default=dict, @@ -71,6 +76,15 @@ def clean(self): ) } ) + if not self.whois_enabled and self.estimated_location_enabled: + raise ValidationError( + { + "estimated_location_enabled": _( + "Estimated Location feature requires " + "WHOIS Lookup feature to be enabled." + ) + } + ) return super().clean() def save( diff --git a/openwisp_controller/config/base/whois.py b/openwisp_controller/config/base/whois.py index 80af038f2..4ae29aebb 100644 --- a/openwisp_controller/config/base/whois.py +++ b/openwisp_controller/config/base/whois.py @@ -1,10 +1,12 @@ from ipaddress import ip_address, ip_network +from django.contrib.gis.db.models import PointField from django.core.cache import cache from django.core.exceptions import ValidationError from django.db import models, transaction from django.utils.translation import gettext_lazy as _ from jsonfield import JSONField +from swapper import load_model from openwisp_utils.base import TimeStampedEditableModel @@ -51,6 +53,12 @@ class AbstractWHOISInfo(TimeStampedEditableModel): blank=True, help_text=_("CIDR"), ) + coordinates = PointField( + null=True, + blank=True, + help_text=_("Coordinates"), + srid=4326, + ) class Meta: abstract = True @@ -74,6 +82,20 @@ def clean(self): raise ValidationError( {"cidr": _("Invalid CIDR format: %(error)s") % {"error": str(e)}} ) + + if self.coordinates: + if not (-90 <= self.coordinates.y <= 90): + raise ValidationError( + {"coordinates": _("Latitude must be between -90 and 90 degrees.")} + ) + if not (-180 <= self.coordinates.x <= 180): + raise ValidationError( + { + "coordinates": _( + "Longitude must be between -180 and 180 degrees." + ) + } + ) return super().clean() @staticmethod @@ -82,8 +104,18 @@ def device_whois_info_delete_handler(instance, **kwargs): Delete WHOIS information for a device when the last IP address is removed or when device is deleted. """ - if instance._get_organization__config_settings().whois_enabled: - transaction.on_commit(lambda: delete_whois_record.delay(instance.last_ip)) + Device = load_model("config", "Device") + + last_ip = instance.last_ip + existing_devices = Device.objects.filter(_is_deactivated=False).filter( + last_ip=last_ip + ) + if ( + last_ip + and instance._get_organization__config_settings().whois_enabled + and not existing_devices.exists() + ): + transaction.on_commit(lambda: delete_whois_record.delay(last_ip)) # this method is kept here instead of in OrganizationConfigSettings because # currently the caching is used only for WHOIS feature @@ -113,3 +145,28 @@ def formatted_address(self): ], ) ) + + @property + def _location_name(self): + """ + Used to get location name based on the address and IP. + """ + address = self.formatted_address + if address: + parts = [part.strip() for part in address.split(",")[:2] if part.strip()] + location = ", ".join(parts) + return _(f"{location} (Estimated Location: {self.ip_address})") + return _(f"Estimated Location: {self.ip_address}") + + def _get_defaults_for_estimated_location(self): + """ + Used to get default values for creating or updating + an estimated location based on the WHOIS information. + """ + return { + "name": self._location_name, + "type": "outdoor", + "is_mobile": False, + "geometry": self.coordinates, + "address": self.formatted_address, + } diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py index 68023ea6d..3dc714c0c 100644 --- a/openwisp_controller/config/controller/views.py +++ b/openwisp_controller/config/controller/views.py @@ -153,15 +153,6 @@ def get(self, request, pk): # updates cache if ip addresses changed if updated: self.update_device_cache(device) - # When update fields are present then save() will run the WHOIS - # lookup. But if there are no update fields, we still want to - # trigger the WHOIS lookup if there is no record for the device's - # last_ip. - elif ( - app_settings.WHOIS_CONFIGURED - and not device.whois_service.get_device_whois_info() - ): - device.whois_service.trigger_whois_lookup() checksum_requested.send( sender=device.__class__, instance=device, request=request ) diff --git a/openwisp_controller/config/management/__init__.py b/openwisp_controller/config/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/management/commands/__init__.py b/openwisp_controller/config/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/management/commands/clear_last_ip.py b/openwisp_controller/config/management/commands/clear_last_ip.py new file mode 100644 index 000000000..4a5efd4c7 --- /dev/null +++ b/openwisp_controller/config/management/commands/clear_last_ip.py @@ -0,0 +1,51 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import OuterRef, Subquery +from swapper import load_model + + +class Command(BaseCommand): + help = "Clear the last IP address, if set, of active devices of all organizations." + + def add_arguments(self, parser): + parser.add_argument( + "--noinput", + "--no-input", + action="store_false", + dest="interactive", + help="Do NOT prompt the user for input of any kind.", + ) + return super().add_arguments(parser) + + def handle(self, *args, **options): + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + if options["interactive"]: + message = ["\n"] + message.append( + "This will clear last IP of all active devices across organizations!\n" + ) + message.append( + "Are you sure you want to do this?\n\n" + "Type 'yes' to continue, or 'no' to cancel: " + ) + if input("".join(message)) != "yes": + raise CommandError("Operation cancelled by user.") + + devices = Device.objects.filter(_is_deactivated=False).only("last_ip") + # Filter devices that have no WHOIS information for their last IP + devices = devices.exclude(last_ip=None).exclude( + last_ip__in=Subquery( + WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( + "ip_address" + ) + ), + ) + + updated_devices = devices.update(last_ip=None) + if updated_devices: + self.stdout.write( + f"Cleared last IP addresses for {updated_devices} active device(s)." + ) + else: + self.stdout.write("No active devices with last IP to clear.") diff --git a/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py b/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py new file mode 100644 index 000000000..8fe4d4c2c --- /dev/null +++ b/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 5.2.1 on 2025-07-10 18:09 + +import django.contrib.gis.db.models.fields +from django.db import migrations + +import openwisp_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("config", "0061_whoisinfo"), + ] + + operations = [ + migrations.AddField( + model_name="organizationconfigsettings", + name="estimated_location_enabled", + field=openwisp_utils.fields.FallbackBooleanChoiceField( + blank=True, + default=None, + fallback=False, + help_text="Whether the estimated location feature is enabled", + null=True, + verbose_name="Estimated Location Enabled", + ), + ), + migrations.AddField( + model_name="whoisinfo", + name="coordinates", + field=django.contrib.gis.db.models.fields.PointField( + blank=True, help_text="Coordinates", null=True, srid=4326 + ), + ), + ] diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index a94617b34..fa3efc750 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -77,3 +77,8 @@ def get_setting(option, default): "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + "when WHOIS_ENABLED is True." ) +ESTIMATED_LOCATION_ENABLED = get_setting("ESTIMATED_LOCATION_ENABLED", False) +if ESTIMATED_LOCATION_ENABLED and not WHOIS_ENABLED: + raise ImproperlyConfigured( + "WHOIS must be enabled before enabling ESTIMATED_LOCATION globally" + ) diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py index 81275a8e3..3e9dad6f1 100644 --- a/openwisp_controller/config/whois/service.py +++ b/openwisp_controller/config/whois/service.py @@ -1,4 +1,4 @@ -from ipaddress import ip_address +from ipaddress import ip_address as ip_addr from django.core.cache import cache from django.db import transaction @@ -6,7 +6,7 @@ from openwisp_controller.config import settings as app_settings -from .tasks import fetch_whois_details +from .tasks import fetch_whois_details, manage_estimated_locations class WHOISService: @@ -30,7 +30,7 @@ def is_valid_public_ip_address(ip): Check if given IP address is a valid public IP address. """ try: - return ip and ip_address(ip).is_global + return ip and ip_addr(ip).is_global except ValueError: return False @@ -43,20 +43,24 @@ def _get_whois_info_from_db(ip_address): return WHOISInfo.objects.filter(ip_address=ip_address) - @property - def is_whois_enabled(self): + @staticmethod + def get_org_config_settings(org_id): """ - Check if the WHOIS lookup feature is enabled. - The OrganizationConfigSettings are cached as these settings - are not expected to change frequently. The timeout for the cache - is set to the same as the checksum cache timeout for consistency - with DeviceChecksumView. + Retrieve and cache organization-specific configuration settings. + + Returns a "read-only" OrganizationConfigSettings instance for the + given organization. + If no settings exist for the organization, returns an empty instance to allow + fallback to global defaults. + + OrganizationConfigSettings are cached for performance, using the same timeout + as DeviceChecksumView for consistency. """ OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") Config = load_model("config", "Config") - org_id = self.device.organization.pk - org_settings = cache.get(self.get_cache_key(org_id=org_id)) + cache_key = WHOISService.get_cache_key(org_id=org_id) + org_settings = cache.get(cache_key) if org_settings is None: try: org_settings = OrganizationConfigSettings.objects.get( @@ -64,13 +68,38 @@ def is_whois_enabled(self): ) except OrganizationConfigSettings.DoesNotExist: # If organization settings do not exist, fall back to global setting - return app_settings.WHOIS_ENABLED + org_settings = OrganizationConfigSettings() cache.set( - self.get_cache_key(org_id=org_id), + cache_key, org_settings, timeout=Config._CHECKSUM_CACHE_TIMEOUT, ) - return getattr(org_settings, "whois_enabled", app_settings.WHOIS_ENABLED) + return org_settings + + @staticmethod + def check_estimate_location_configured(org_id): + if not org_id: + return False + if not app_settings.WHOIS_CONFIGURED: + return False + org_settings = WHOISService.get_org_config_settings(org_id=org_id) + return org_settings.estimated_location_enabled + + @property + def is_whois_enabled(self): + """ + Check if the WHOIS lookup feature is enabled. + """ + org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) + return org_settings.whois_enabled + + @property + def is_estimated_location_enabled(self): + """ + Check if the Estimated location feature is enabled. + """ + org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) + return org_settings.estimated_location_enabled def _need_whois_lookup(self, new_ip): """ @@ -92,6 +121,19 @@ def _need_whois_lookup(self, new_ip): return self.is_whois_enabled + def _need_estimated_location_management(self, new_ip): + """ + Used to determine if Estimated locations need to be created/updated + or not during WHOIS lookup. + """ + if not self.is_valid_public_ip_address(new_ip): + return False + + if not self.is_whois_enabled: + return False + + return self.is_estimated_location_enabled + def get_device_whois_info(self): """ If the WHOIS lookup feature is enabled and the device ``last_ip`` @@ -103,16 +145,28 @@ def get_device_whois_info(self): return self._get_whois_info_from_db(ip_address=ip_address).first() - def trigger_whois_lookup(self): + def process_ip_data_and_location(self): """ - Trigger WHOIS lookup based on the conditions of `_need_whois_lookup`. - Task is triggered on commit to ensure redundant data is not created. + Trigger WHOIS lookup based on the conditions of `_need_whois_lookup` + and also manage estimated locations based on the conditions of + `_need_estimated_location_management`. + Tasks are triggered on commit to ensure redundant data is not created. """ - if self._need_whois_lookup(self.device.last_ip): + new_ip = self.device.last_ip + if self._need_whois_lookup(new_ip): transaction.on_commit( lambda: fetch_whois_details.delay( device_pk=self.device.pk, initial_ip_address=self.device._initial_last_ip, - new_ip_address=self.device.last_ip, + new_ip_address=new_ip, + ) + ) + # To handle the case when WHOIS already exists as in that case + # WHOIS lookup is not triggered but we still need to + # manage estimated locations. + elif self._need_estimated_location_management(new_ip): + transaction.on_commit( + lambda: manage_estimated_locations.delay( + device_pk=self.device.pk, ip_address=new_ip ) ) diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py index 2ae434cd7..bafca7b78 100644 --- a/openwisp_controller/config/whois/tasks.py +++ b/openwisp_controller/config/whois/tasks.py @@ -2,15 +2,17 @@ import requests from celery import shared_task +from django.contrib.gis.geos import Point from django.utils.translation import gettext as _ from geoip2 import errors from geoip2 import webservice as geoip2_webservice -from openwisp_notifications.signals import notify from swapper import load_model +from openwisp_controller.geo.estimated_location.tasks import manage_estimated_locations from openwisp_utils.tasks import OpenwispCeleryTask from .. import settings as app_settings +from .utils import send_whois_task_notification logger = logging.getLogger(__name__) @@ -43,25 +45,9 @@ class WHOISCeleryRetryTask(OpenwispCeleryTask): def on_failure(self, exc, task_id, args, kwargs, einfo): """Notify the user about the failure of the WHOIS task.""" - Device = load_model("config", "Device") - device_pk = kwargs.get("device_pk") - new_ip_address = kwargs.get("new_ip_address") - device = Device.objects.get(pk=device_pk) - - notify.send( - sender=device, - type="generic_message", - target=device, - action_object=device, - level="error", - message=_( - "Failed to fetch WHOIS details for device" - " [{notification.target}]({notification.target_link})" - ), - description=_( - f"WHOIS details could not be fetched for ip: {new_ip_address}." - ), + send_whois_task_notification( + device_pk=device_pk, notify_type="whois_device_error" ) logger.error(f"WHOIS lookup failed. Details: {exc}") return super().on_failure(exc, task_id, args, kwargs, einfo) @@ -78,6 +64,7 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): Fetches the WHOIS details of the given IP address and creates/updates the WHOIS record. """ + Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") # The task can be triggered for same ip address multiple times @@ -85,6 +72,7 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): if WHOISInfo.objects.filter(ip_address=new_ip_address).exists(): return + device = Device.objects.get(pk=device_pk) # Host is based on the db that is used to fetch the details. # As we are using GeoLite2, 'geolite.info' host is used. # Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example @@ -123,7 +111,7 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): "continent": data.continent.name or "", "postal": str(data.postal.code or ""), } - + coordinates = Point(data.location.longitude, data.location.latitude, srid=4326) whois_obj = WHOISInfo( isp=data.traits.autonomous_system_organization, asn=data.traits.autonomous_system_number, @@ -131,19 +119,23 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): address=address, cidr=data.traits.network, ip_address=new_ip_address, + coordinates=coordinates, ) whois_obj.full_clean() whois_obj.save() logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") - # the following check ensures that for a case when device last_ip - # is not changed and there is no related WHOIS record, we do not - # delete the newly created record as both `initial_ip_address` and - # `new_ip_address` would be same for such case. - if initial_ip_address != new_ip_address: - # If any active devices are linked to the following record, - # then they will trigger this task and new record gets created - # with latest data. + if device._get_organization__config_settings().estimated_location_enabled: + manage_estimated_locations.delay( + device_pk=device_pk, ip_address=new_ip_address + ) + + # delete WHOIS record for initial IP if no devices are linked to it + if ( + not Device.objects.filter(_is_deactivated=False) + .filter(last_ip=initial_ip_address) + .exists() + ): delete_whois_record(ip_address=initial_ip_address) diff --git a/openwisp_controller/config/whois/tests/__init__.py b/openwisp_controller/config/whois/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/config/whois/test_whois.py b/openwisp_controller/config/whois/tests/tests.py similarity index 80% rename from openwisp_controller/config/whois/test_whois.py rename to openwisp_controller/config/whois/tests/tests.py index d98fb8336..82ad9e83b 100644 --- a/openwisp_controller/config/whois/test_whois.py +++ b/openwisp_controller/config/whois/tests/tests.py @@ -1,8 +1,11 @@ import importlib +from io import StringIO from unittest import mock +from django.contrib.gis.geos import Point from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.core.management import call_command from django.db.models.signals import post_delete, post_save from django.test import TestCase, TransactionTestCase, override_settings, tag from django.urls import reverse @@ -12,15 +15,15 @@ from openwisp_utils.tests import SeleniumTestMixin -from ...tests.utils import TestAdminMixin -from .. import settings as app_settings -from .handlers import connect_whois_handlers -from .tests_utils import CreateWHOISMixin +from ....tests.utils import TestAdminMixin +from ... import settings as app_settings +from ..handlers import connect_whois_handlers +from .utils import CreateWHOISMixin, WHOISTransactionMixin Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") -OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") Notification = load_model("openwisp_notifications", "Notification") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") notification_qs = Notification.objects.all() @@ -214,7 +217,6 @@ def test_whois_details_device_api(self): with self.subTest( "Device Detail API has whois_info when WHOIS_CONFIGURED is True" ): - response = self.client.get( reverse("config_api:device_detail", args=[device.pk]) ) @@ -264,6 +266,20 @@ def test_whois_details_device_api(self): self.assertEqual(response.status_code, 200) self.assertNotIn("whois_info", response.data) + def test_last_ip_management_command(self): + out = StringIO() + device = self._create_device(last_ip="172.217.22.11") + args = ["--noinput"] + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + self.assertIn( + "Cleared last IP addresses for 1 active device(s).", out.getvalue() + ) + device.refresh_from_db() + self.assertIsNone(device.last_ip) + + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + self.assertIn("No active devices with last IP to clear.", out.getvalue()) + class TestWHOISInfoModel(CreateWHOISMixin, TestCase): def test_whois_model_fields_validation(self): @@ -301,10 +317,31 @@ def test_whois_model_fields_validation(self): with self.assertRaises(ValidationError): self._create_whois_info(asn="InvalidASN") + # Common validation checks for longitude and latitude + coordinates_cases = [ + (150.0, 100.0, "Latitude must be between -90 and 90 degrees."), + (150.0, -100.0, "Latitude must be between -90 and 90 degrees."), + (200.0, 80.0, "Longitude must be between -180 and 180 degrees."), + (-200.0, -80.0, "Longitude must be between -180 and 180 degrees."), + ] + for longitude, latitude, expected_msg in coordinates_cases: + with self.assertRaises(ValidationError) as context_manager: + point = Point(longitude, latitude, srid=4326) + self._create_whois_info(coordinates=point) + try: + self.assertEqual( + context_manager.exception.message_dict["coordinates"][0], + expected_msg, + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + -class TestWHOISTransaction(CreateWHOISMixin, TransactionTestCase): +class TestWHOISTransaction( + CreateWHOISMixin, WHOISTransactionMixin, TransactionTestCase +): _WHOIS_GEOIP_CLIENT = ( - "openwisp_controller.config.whois.tasks.geoip2_webservice.Client.city" + "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" ) _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" @@ -316,76 +353,38 @@ def setUp(self): @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") - def test_whois_task_called(self, mocked_task): - org = self._get_org() + def test_whois_task_called(self, mocked_lookup_task): connect_whois_handlers() + self._task_called(mocked_lookup_task) - with self.subTest("task called when last_ip is public"): - with mock.patch("django.core.cache.cache.set") as mocked_set: - device = self._create_device(last_ip="172.217.22.14") - mocked_task.assert_called() - mocked_set.assert_called_once() - mocked_task.reset_mock() - - with self.subTest("task called when last_ip is changed and is public"): - with mock.patch("django.core.cache.cache.get") as mocked_get: - device.last_ip = "172.217.22.10" - device.save() - mocked_task.assert_called() - mocked_get.assert_called_once() - mocked_task.reset_mock() - - with self.subTest("task not called when last_ip is private"): - device.last_ip = "10.0.0.1" - device.save() - mocked_task.assert_not_called() - mocked_task.reset_mock() - - with self.subTest("task not called when last_ip has related WHOIS Info"): - device.last_ip = "172.217.22.10" + Device.objects.all().delete() # Clear existing devices + device = self._create_device() + with self.subTest( + "WHOIS lookup task not called when last_ip has related WhoIsInfo" + ): + device.organization.config_settings.whois_enabled = True + device.organization.config_settings.save() + device.last_ip = "172.217.22.14" self._create_whois_info(ip_address=device.last_ip) device.save() - mocked_task.assert_not_called() - mocked_task.reset_mock() - - with self.subTest("task not called when WHOIS is disabled"): - Device.objects.all().delete() - org.config_settings.whois_enabled = False - # Invalidates old org config settings cache - org.config_settings.save(update_fields=["whois_enabled"]) - device = self._create_device(last_ip="172.217.22.14") - mocked_task.assert_not_called() - mocked_task.reset_mock() - - with self.subTest("task called via DeviceChecksumView when WHOIS is enabled"): - org.config_settings.whois_enabled = True - # Invalidates old org config settings cache - org.config_settings.save(update_fields=["whois_enabled"]) - # config is required for checksum view to work - self._create_config(device=device) - # setting remote address field to a public IP to trigger WHOIS task - # since the view uses this header for tracking the device's IP - response = self.client.get( - reverse("controller:device_checksum", args=[device.pk]), - {"key": device.key}, - REMOTE_ADDR="172.217.22.10", - ) - self.assertEqual(response.status_code, 200) - mocked_task.assert_called() - mocked_task.reset_mock() + mocked_lookup_task.assert_not_called() + mocked_lookup_task.reset_mock() with self.subTest( - "task called via DeviceChecksumView when a device has no WHOIS record" + "WHOIS lookup task not called via DeviceChecksumView when " + "last_ip has related WhoIsInfo" ): WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) + self._create_config(device=device) response = self.client.get( reverse("controller:device_checksum", args=[device.pk]), {"key": device.key}, REMOTE_ADDR=device.last_ip, ) self.assertEqual(response.status_code, 200) - mocked_task.assert_called() - mocked_task.reset_mock() + mocked_lookup_task.assert_not_called() + mocked_lookup_task.reset_mock() @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") @@ -429,6 +428,7 @@ def test_whois_multiple_orgs(self, mocked_task): {"key": device1.key}, REMOTE_ADDR="172.217.22.20", ) + device1.refresh_from_db() self.assertEqual(response.status_code, 200) mocked_task.assert_called() mocked_task.reset_mock() @@ -437,12 +437,13 @@ def test_whois_multiple_orgs(self, mocked_task): {"key": device2.key}, REMOTE_ADDR="172.217.22.30", ) + device2.refresh_from_db() self.assertEqual(response.status_code, 200) mocked_task.assert_not_called() mocked_task.reset_mock() with self.subTest( - "task called via DeviceChecksumView when a device has no WHOIS record" + "Task not called via DeviceChecksumView when a device has no WHOIS record" ): WHOISInfo.objects.all().delete() response = self.client.get( @@ -451,7 +452,7 @@ def test_whois_multiple_orgs(self, mocked_task): REMOTE_ADDR=device1.last_ip, ) self.assertEqual(response.status_code, 200) - mocked_task.assert_called() + mocked_task.assert_not_called() mocked_task.reset_mock() response = self.client.get( reverse("controller:device_checksum", args=[device2.pk]), @@ -489,18 +490,11 @@ def _verify_whois_details(instance, ip_address): instance.formatted_address, "Mountain View, United States, North America, 94043", ) + self.assertEqual(instance.coordinates.x, 150.0) + self.assertEqual(instance.coordinates.y, 50.0) # mocking the response from the geoip2 client - mock_response = mock.MagicMock() - mock_response.city.name = "Mountain View" - mock_response.country.name = "United States" - mock_response.continent.name = "North America" - mock_response.postal.code = "94043" - mock_response.traits.autonomous_system_organization = "Google LLC" - mock_response.traits.autonomous_system_number = 15169 - mock_response.traits.network = "172.217.22.0/24" - mock_response.location.time_zone = "America/Los_Angeles" - mock_client.return_value = mock_response + mock_client.return_value.city.return_value = self._mocked_client_response() with self.subTest("Test WHOIS create when device is created"): device = self._create_device(last_ip="172.217.22.14") @@ -514,6 +508,7 @@ def _verify_whois_details(instance, ip_address): with self.subTest( "Test WHOIS create & deletion of old record when last ip is updated" + " when no other devices are linked to the old ip address" ): old_ip_address = device.last_ip device.last_ip = "172.217.22.10" @@ -531,12 +526,61 @@ def _verify_whois_details(instance, ip_address): WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 0 ) - with self.subTest("Test WHOIS delete when device is deleted"): + with self.subTest( + "Test WHOIS create & deletion of old record when last ip is updated" + " when other devices are linked to the old ip address" + ): + old_ip_address = device.last_ip + self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + device.last_ip = "172.217.22.11" + device.save() + self.assertEqual(mock_info.call_count, 1) + mock_info.reset_mock() + device.refresh_from_db() + + _verify_whois_details( + device.whois_service.get_device_whois_info(), device.last_ip + ) + + # details related to old ip address should be not be deleted + self.assertEqual( + WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 1 + ) + + with self.subTest( + "Test WHOIS not deleted when device is deleted and" + " other active devices are linked to the last_ip" + ): ip_address = device.last_ip device.delete(check_deactivated=False) self.assertEqual(mock_info.call_count, 0) mock_info.reset_mock() + # WHOIS related to the device's last_ip should be deleted + self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 1) + + Device.objects.all().delete() + with self.subTest( + "Test WHOIS deleted when device is deleted and" + " no other active devices are linked to the last_ip" + ): + device1 = self._create_device(last_ip="172.217.22.11") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + device2.deactivate() + mock_info.reset_mock() + ip_address = device1.last_ip + device1.delete(check_deactivated=False) + self.assertEqual(mock_info.call_count, 0) + mock_info.reset_mock() + # WHOIS related to the device's last_ip should be deleted self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) @@ -563,12 +607,13 @@ def assert_logging_on_exception( notification = notification_qs.first() self.assertEqual(notification.actor, device) self.assertEqual(notification.target, device) + self.assertEqual(notification.level, "error") self.assertEqual(notification.type, "generic_message") self.assertIn( "Failed to fetch WHOIS details for device", notification.message, ) - self.assertIn(device.last_ip, notification.description) + self.assertIn(device.last_ip, notification.rendered_description) mock_info.reset_mock() mock_warn.reset_mock() @@ -586,6 +631,16 @@ def assert_logging_on_exception( class TestWHOISSelenium(CreateWHOISMixin, SeleniumTestMixin, StaticLiveServerTestCase): @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) def test_whois_device_admin(self): + def _assert_no_js_errors(): + browser_logs = [] + for log in self.get_browser_logs(): + if self.browser == "chrome" and log["source"] != "console-api": + continue + elif log["message"] in ["wrong event specified: touchleave"]: + continue + browser_logs.append(log) + self.assertEqual(browser_logs, []) + whois_obj = self._create_whois_info() device = self._create_device(last_ip=whois_obj.ip_address) self.login() @@ -610,6 +665,7 @@ def test_whois_device_admin(self): self.assertIn(whois_obj.timezone, additional_text[1].text) self.assertIn(whois_obj.formatted_address, additional_text[2].text) self.assertIn(whois_obj.cidr, additional_text[3].text) + _assert_no_js_errors() with mock.patch.object(app_settings, "WHOIS_CONFIGURED", False): with self.subTest( @@ -619,6 +675,7 @@ def test_whois_device_admin(self): self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() with self.subTest( "WHOIS details not visible in device admin when WHOIS is disabled" @@ -629,6 +686,7 @@ def test_whois_device_admin(self): self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() with self.subTest( "WHOIS details not visible in device admin when WHOIS Info does not exist" @@ -640,3 +698,4 @@ def test_whois_device_admin(self): self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") + _assert_no_js_errors() diff --git a/openwisp_controller/config/whois/tests/utils.py b/openwisp_controller/config/whois/tests/utils.py new file mode 100644 index 000000000..7139ae883 --- /dev/null +++ b/openwisp_controller/config/whois/tests/utils.py @@ -0,0 +1,152 @@ +from unittest import mock + +from django.contrib.gis.geos import Point +from django.urls import reverse +from swapper import load_model + +from ...tests.utils import CreateConfigMixin + +Device = load_model("config", "Device") +WHOISInfo = load_model("config", "WHOISInfo") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class CreateWHOISMixin(CreateConfigMixin): + def _create_whois_info(self, **kwargs): + options = dict( + ip_address="172.217.22.14", + address={ + "city": "Mountain View", + "country": "United States", + "continent": "North America", + "postal": "94043", + }, + asn="15169", + isp="Google LLC", + timezone="America/Los_Angeles", + cidr="172.217.22.0/24", + coordinates=Point(150, 50, srid=4326), + ) + + options.update(kwargs) + w = WHOISInfo(**options) + w.full_clean() + w.save() + return w + + def setUp(self): + super().setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), whois_enabled=True + ) + + +class WHOISTransactionMixin: + @staticmethod + def _mocked_client_response(): + mock_response = mock.MagicMock() + mock_response.city.name = "Mountain View" + mock_response.country.name = "United States" + mock_response.continent.name = "North America" + mock_response.postal.code = "94043" + mock_response.traits.autonomous_system_organization = "Google LLC" + mock_response.traits.autonomous_system_number = 15169 + mock_response.traits.network = "172.217.22.0/24" + mock_response.location.time_zone = "America/Los_Angeles" + mock_response.location.latitude = 50 + mock_response.location.longitude = 150 + return mock_response + + def _task_called(self, mocked_task, task_name="WHOIS lookup"): + org = self._get_org() + + with self.subTest(f"{task_name} task called when last_ip is public"): + with mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_called() + mocked_set.assert_called_once() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task called when last_ip is changed and is public" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.last_ip = "172.217.22.10" + device.save() + mocked_task.assert_called() + mocked_set.assert_not_called() + mocked_get.assert_called_once() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip not updated"): + device.name = "default.test.Device2" + device.save() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip is private"): + device.last_ip = "10.0.0.1" + device.save() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when WHOIS is disabled"): + Device.objects.all().delete() + org.config_settings.whois_enabled = False + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + device = self._create_device(last_ip="172.217.22.14") + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task called via DeviceChecksumView when WHOIS is enabled" + ): + org.config_settings.whois_enabled = True + # Invalidates old org config settings cache + org.config_settings.save(update_fields=["whois_enabled"]) + # config is required for checksum view to work + device.refresh_from_db() + self._create_config(device=device) + # setting remote address field to a public IP to trigger WHOIS task + # since the view uses this header for tracking the device's IP + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR="172.217.22.10", + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task called via DeviceChecksumView for no WHOIS record" + ): + WHOISInfo.objects.all().delete() + device.refresh_from_db() + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task not called via DeviceChecksumView when WHOIS is disabled" + ): + WHOISInfo.objects.all().delete() + device.refresh_from_db() + org.config_settings.whois_enabled = False + org.config_settings.save(update_fields=["whois_enabled"]) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() diff --git a/openwisp_controller/config/whois/tests_utils.py b/openwisp_controller/config/whois/tests_utils.py deleted file mode 100644 index fd31ff6cf..000000000 --- a/openwisp_controller/config/whois/tests_utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from swapper import load_model - -from ..tests.utils import CreateConfigMixin - -Device = load_model("config", "Device") -WHOISInfo = load_model("config", "WHOISInfo") -OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") - - -class CreateWHOISMixin(CreateConfigMixin): - def _create_whois_info(self, **kwargs): - options = dict( - ip_address="172.217.22.14", - address={ - "city": "Mountain View", - "country": "United States", - "continent": "North America", - "postal": "94043", - }, - asn="15169", - isp="Google LLC", - timezone="America/Los_Angeles", - cidr="172.217.22.0/24", - ) - - options.update(kwargs) - w = WHOISInfo(**options) - w.full_clean() - w.save() - return w - - def setUp(self): - super().setUp() - OrganizationConfigSettings.objects.create( - organization=self._get_org(), whois_enabled=True - ) diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py index 51919af08..4a955810e 100644 --- a/openwisp_controller/config/whois/utils.py +++ b/openwisp_controller/config/whois/utils.py @@ -1,13 +1,66 @@ +from django.utils.translation import gettext_lazy as _ +from openwisp_notifications.signals import notify from swapper import load_model from .. import settings as app_settings -from .serializers import WHOISSerializer -from .service import WHOISService -Device = load_model("config", "Device") +MESSAGE_MAP = { + "whois_device_error": { + "type": "generic_message", + "level": "error", + "message": _( + "Failed to fetch WHOIS details for device" + " [{notification.target}]({notification.target_link})" + ), + "description": _("WHOIS details could not be fetched for ip: {ip_address}."), + }, + "estimated_location_error": { + "level": "error", + "type": "estimated_location_info", + "message": _( + "Unable to create estimated location for device " + "[{notification.target}]({notification.target_link}). " + "Please assign/create a location manually." + ), + "description": _("Multiple devices found for IP: {ip_address}"), + }, + "estimated_location_created": { + "type": "estimated_location_info", + "description": _("Estimated Location {notification.verb} for IP: {ip_address}"), + }, + "estimated_location_updated": { + "type": "estimated_location_info", + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " updated successfully." + ), + "description": _("Estimated Location updated for IP: {ip_address}"), + }, +} + + +def send_whois_task_notification(device_pk, notify_type, actor=None): + Device = load_model("config", "Device") + + device = Device.objects.get(pk=device_pk) + notify_details = MESSAGE_MAP[notify_type] + notify.send( + sender=actor or device, + target=device, + action_object=device, + ip_address=device.last_ip, + **notify_details, + ) def get_whois_info(pk): + from .serializers import WHOISSerializer + from .service import WHOISService + + Device = load_model("config", "Device") + if not app_settings.WHOIS_CONFIGURED or not pk: return None device = ( diff --git a/openwisp_controller/geo/admin.py b/openwisp_controller/geo/admin.py index 9064d657a..ffab0c99b 100644 --- a/openwisp_controller/geo/admin.py +++ b/openwisp_controller/geo/admin.py @@ -12,6 +12,8 @@ ) from swapper import load_model +from openwisp_controller.config import settings as config_app_settings +from openwisp_controller.config.whois.service import WHOISService from openwisp_users.multitenancy import MultitenantOrgFilter from ..admin import MultitenantAdminMixin @@ -98,6 +100,29 @@ class LocationAdmin(MultitenantAdminMixin, AbstractLocationAdmin): form = LocationForm inlines = [FloorPlanInline] list_select_related = ("organization",) + change_form_template = "admin/geo/location/change_form.html" + + def get_fields(self, request, obj=None): + fields = super().get_fields(request, obj) + org_id = obj.organization_id if obj else None + if not WHOISService.check_estimate_location_configured(org_id): + if "is_estimated" in fields: + fields.remove("is_estimated") + return fields + + def get_readonly_fields(self, request, obj=None): + fields = super().get_readonly_fields(request, obj) + org_id = obj.organization_id if obj else None + if obj and WHOISService.check_estimate_location_configured(org_id): + fields = fields + ("is_estimated",) + return fields + + def change_view(self, request, object_id, form_url="", extra_context=None): + obj = self.get_object(request, object_id) + org_id = obj.organization_id if obj else None + estimated_configured = WHOISService.check_estimate_location_configured(org_id) + extra_context = {"estimated_configured": estimated_configured} + return super().change_view(request, object_id, form_url, extra_context) LocationAdmin.list_display.insert(1, "organization") @@ -122,16 +147,39 @@ class DeviceLocationFilter(admin.SimpleListFilter): title = _("has geographic position set?") parameter_name = "with_geo" + def __init__(self, request, params, model, model_admin): + super().__init__(request, params, model, model_admin) + if config_app_settings.WHOIS_CONFIGURED: + self.title = _("geographic position") + def lookups(self, request, model_admin): + if config_app_settings.WHOIS_CONFIGURED: + return ( + ("outdoor", _("Outdoor")), + ("indoor", _("Indoor")), + ("estimated", _("Estimated")), + ("false", _("No Location")), + ) return ( ("true", _("Yes")), ("false", _("No")), ) def queryset(self, request, queryset): - if self.value(): - return queryset.filter(devicelocation__isnull=self.value() == "false") - return queryset + value = self.value() + if not value: + return queryset + if config_app_settings.WHOIS_CONFIGURED: + if value == "estimated": + return queryset.filter(devicelocation__location__is_estimated=True) + elif value in ("indoor", "outdoor"): + # estimated locations are outdoor by default + # so we need to exclude them from the result + return queryset.filter( + devicelocation__location__type=value, + devicelocation__location__is_estimated=False, + ) + return queryset.filter(devicelocation__isnull=self.value() == "false") # Prepend DeviceLocationInline to config.DeviceAdminExportable diff --git a/openwisp_controller/geo/api/filters.py b/openwisp_controller/geo/api/filters.py index 93a7c366e..2be821e58 100644 --- a/openwisp_controller/geo/api/filters.py +++ b/openwisp_controller/geo/api/filters.py @@ -1,6 +1,7 @@ from django.utils.translation import gettext_lazy as _ from django_filters import rest_framework as filters +from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.api.filters import ( DeviceListFilter as BaseDeviceListFilter, ) @@ -30,6 +31,20 @@ def filter_devicelocation(self, queryset, name, value): # Returns list of device that have devicelocation objects return queryset.exclude(devicelocation__isnull=value) + def filter_is_estimated(self, queryset, name, value): + return queryset.filter(devicelocation__location__is_estimated=value) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if config_app_settings.WHOIS_CONFIGURED: + self.filters["geo_is_estimated"] = filters.BooleanFilter( + field_name="devicelocation__location__is_estimated", + method=self.filter_is_estimated, + ) + self.filters["geo_is_estimated"].label = _( + "Is geographic location estimated?" + ) + class Meta: model = BaseDeviceListFilter.Meta.model geo_fields = [ diff --git a/openwisp_controller/geo/api/serializers.py b/openwisp_controller/geo/api/serializers.py index e4a800d90..0626efdab 100644 --- a/openwisp_controller/geo/api/serializers.py +++ b/openwisp_controller/geo/api/serializers.py @@ -11,6 +11,10 @@ from openwisp_utils.api.serializers import ValidatedModelSerializer from ...serializers import BaseSerializer +from ..estimated_location.mixins import ( + EstimatedLocationGeoJsonSerializer, + EstimatedLocationMixin, +) Device = load_model("config", "Device") Location = load_model("geo", "Location") @@ -31,7 +35,9 @@ class Meta: fields = "__all__" -class GeoJsonLocationSerializer(gis_serializers.GeoFeatureModelSerializer): +class GeoJsonLocationSerializer( + EstimatedLocationGeoJsonSerializer, gis_serializers.GeoFeatureModelSerializer +): device_count = IntegerField() class Meta: @@ -126,7 +132,7 @@ class Meta: read_only_fields = ("name",) -class LocationSerializer(BaseSerializer): +class LocationSerializer(EstimatedLocationMixin, BaseSerializer): floorplan = FloorPlanLocationSerializer(required=False, allow_null=True) class Meta: @@ -225,7 +231,9 @@ def update(self, instance, validated_data): return super().update(instance, validated_data) -class NestedtLocationSerializer(gis_serializers.GeoFeatureModelSerializer): +class NestedtLocationSerializer( + EstimatedLocationGeoJsonSerializer, gis_serializers.GeoFeatureModelSerializer +): class Meta: model = Location geo_field = "geometry" diff --git a/openwisp_controller/geo/api/views.py b/openwisp_controller/geo/api/views.py index b521c3d27..9b64ca084 100644 --- a/openwisp_controller/geo/api/views.py +++ b/openwisp_controller/geo/api/views.py @@ -11,6 +11,7 @@ from rest_framework_gis.pagination import GeoJsonPagination from swapper import load_model +from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.api.views import DeviceListCreateView from openwisp_users.api.filters import OrganizationManagedFilter from openwisp_users.api.mixins import FilterByOrganizationManaged, FilterByParentManaged @@ -51,6 +52,13 @@ class Meta(OrganizationManagedFilter.Meta): model = Location fields = OrganizationManagedFilter.Meta.fields + ["is_mobile", "type"] + def __init__(self, data=None, queryset=None, *, request=None, prefix=None): + super().__init__(data, queryset, request=request, prefix=prefix) + if config_app_settings.WHOIS_CONFIGURED: + self.filters["is_estimated"] = filters.BooleanFilter( + field_name="is_estimated" + ) + class FloorPlanOrganizationFilter(OrganizationManagedFilter): class Meta(OrganizationManagedFilter.Meta): diff --git a/openwisp_controller/geo/apps.py b/openwisp_controller/geo/apps.py index 90a626f8e..0e2cc6264 100644 --- a/openwisp_controller/geo/apps.py +++ b/openwisp_controller/geo/apps.py @@ -14,6 +14,8 @@ from openwisp_utils.admin_theme import register_dashboard_chart from openwisp_utils.admin_theme.menu import register_menu_group +from .estimated_location.handlers import register_estimated_location_notification_types + class GeoConfig(LociConfig): name = "openwisp_controller.geo" @@ -27,6 +29,7 @@ def ready(self): super().ready() self.register_dashboard_charts() self.register_menu_groups() + register_estimated_location_notification_types() if getattr(settings, "TESTING", False): self._add_params_to_test_config() diff --git a/openwisp_controller/geo/base/models.py b/openwisp_controller/geo/base/models.py index c7bc59798..c1225f838 100644 --- a/openwisp_controller/geo/base/models.py +++ b/openwisp_controller/geo/base/models.py @@ -1,4 +1,9 @@ +import re + from django.contrib.gis.db import models +from django.core.exceptions import ValidationError +from django.utils.translation import gettext +from django.utils.translation import gettext_lazy as _ from django_loci.base.models import ( AbstractFloorPlan, AbstractLocation, @@ -6,13 +11,81 @@ ) from swapper import get_model_name +from openwisp_controller.config.whois.service import WHOISService from openwisp_users.mixins import OrgMixin, ValidateOrgMixin class BaseLocation(OrgMixin, AbstractLocation): + _changed_checked_fields = ["is_estimated", "address", "geometry"] + + is_estimated = models.BooleanField( + default=False, + help_text=_("Whether the location's coordinates are estimated."), + ) + class Meta(AbstractLocation.Meta): abstract = True + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._set_initial_values_for_changed_checked_fields() + + def _set_initial_values_for_changed_checked_fields(self): + deferred_fields = self.get_deferred_fields() + for field in self._changed_checked_fields: + if field in deferred_fields: + setattr(self, f"_initial_{field}", models.DEFERRED) + else: + setattr(self, f"_initial_{field}", getattr(self, field)) + + def clean(self): + # Raise validation error if `is_estimated` is True but estimated feature is + # disabled. + if ( + (self._state.adding or self._initial_is_estimated != self.is_estimated) + and self.is_estimated + and not WHOISService.check_estimate_location_configured( + self.organization_id + ) + ): + raise ValidationError( + { + "is_estimated": _( + "Estimated Location feature required to be configured." + ) + } + ) + return super().clean() + + def save(self, *args, _set_estimated=False, **kwargs): + """ + Save the location object with special handling for estimated locations. + + Parameters: + _set_estimated: Boolean flag to indicate if this save is being performed + by the estimated location system. When False (default), + manual edits will clear the estimated status. + *args, **kwargs: Arguments passed to the parent save method. + + Returns: + The result of the parent save method. + """ + if WHOISService.check_estimate_location_configured(self.organization_id): + if not _set_estimated and ( + self._initial_address != self.address + or self._initial_geometry != self.geometry + ): + self.is_estimated = False + estimated_string = gettext("Estimated Location") + if self.name and estimated_string in self.name: + # remove string starting with "(Estimated Location" + self.name = re.sub( + rf"\s\({estimated_string}.*", "", self.name, flags=re.IGNORECASE + ) + else: + self.is_estimated = self._initial_is_estimated + return super().save(*args, **kwargs) + class BaseFloorPlan(OrgMixin, AbstractFloorPlan): location = models.ForeignKey(get_model_name("geo", "Location"), models.CASCADE) diff --git a/openwisp_controller/geo/estimated_location/__init__.py b/openwisp_controller/geo/estimated_location/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/geo/estimated_location/handlers.py b/openwisp_controller/geo/estimated_location/handlers.py new file mode 100644 index 000000000..60ea6f44a --- /dev/null +++ b/openwisp_controller/geo/estimated_location/handlers.py @@ -0,0 +1,41 @@ +from django.utils.translation import gettext_lazy as _ +from openwisp_notifications.types import register_notification_type +from swapper import load_model + +from openwisp_controller.config import settings as config_app_settings + + +def register_estimated_location_notification_types(): + """ + Register the notification types used by the Estimated Location module. + This is necessary to ensure that the notifications are properly configured. + """ + if not config_app_settings.WHOIS_CONFIGURED: + return + + Device = load_model("config", "Device") + Location = load_model("geo", "Location") + + register_notification_type( + "estimated_location_info", + { + "verbose_name": _("Estimated Location INFO"), + "verb": _("created"), + "level": "info", + "email_subject": _( + "Estimated Location: Created for device {notification.target}" + ), + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " {notification.verb} successfully." + ), + "target_link": ( + "openwisp_controller.geo.estimated_location.utils" + ".get_device_location_notification_target_url" + ), + "email_notification": False, + }, + models=[Device, Location], + ) diff --git a/openwisp_controller/geo/estimated_location/mixins.py b/openwisp_controller/geo/estimated_location/mixins.py new file mode 100644 index 000000000..f4d2fc93b --- /dev/null +++ b/openwisp_controller/geo/estimated_location/mixins.py @@ -0,0 +1,30 @@ +from openwisp_controller.config.whois.service import WHOISService + + +class EstimatedLocationMixin: + """ + Serializer mixin to add estimated location field to the serialized data + if the estimated location feature is configured and enabled for the organization. + """ + + def to_representation(self, obj): + data = super().to_representation(obj) + if WHOISService.check_estimate_location_configured(obj.organization_id): + data["is_estimated"] = obj.is_estimated + else: + data.pop("is_estimated", None) + return data + + +class EstimatedLocationGeoJsonSerializer(EstimatedLocationMixin): + """ + Extension of EstimatedLocationMixin for GeoJSON serialization. + """ + + def to_representation(self, obj): + data = super(EstimatedLocationMixin, self).to_representation(obj) + if WHOISService.check_estimate_location_configured(obj.organization_id): + data["properties"]["is_estimated"] = obj.is_estimated + else: + data["properties"].pop("is_estimated", None) + return data diff --git a/openwisp_controller/geo/estimated_location/tasks.py b/openwisp_controller/geo/estimated_location/tasks.py new file mode 100644 index 000000000..8ec080509 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tasks.py @@ -0,0 +1,170 @@ +import logging + +from celery import shared_task +from django.db import transaction +from swapper import load_model + +from openwisp_controller.config.whois.utils import send_whois_task_notification + +logger = logging.getLogger(__name__) + + +@shared_task +def manage_estimated_locations(device_pk, ip_address): + """ + Creates/updates estimated location for a device based on the latitude and + longitude or attaches an existing location. + Existing location here means a location of another device whose last_ip matches + the given ip_address. + Does not alters the existing location if it is not estimated. + + - If the current device has no location or location is estimate, either update + to an existing location; if it exists, else + + - A new location is created if current device has no location, or + if it does; it is updated using coords from WHOIS record if it is estimated. + + In case of multiple devices with same last_ip, the task will send a notification + to the user to resolve the conflict manually. + """ + Device = load_model("config", "Device") + Location = load_model("geo", "Location") + WHOISInfo = load_model("config", "WHOISInfo") + DeviceLocation = load_model("geo", "DeviceLocation") + + def _create_estimated_location(device_location, location_defaults): + with transaction.atomic(): + location = Location(**location_defaults, is_estimated=True) + location.full_clean() + location.save(_set_estimated=True) + device_location.location = location + device_location.full_clean() + device_location.save() + logger.info( + f"Estimated location saved successfully for {device_pk}" + f" for IP: {ip_address}" + ) + send_whois_task_notification( + device_pk=device_pk, + notify_type="estimated_location_created", + actor=location, + ) + + def _update_or_create_estimated_location( + device_location, whois_obj, attached_devices_exists=False + ): + # Used to update an existing location if it is estimated + # or create a new one if it doesn't exist + if whois_obj and whois_obj.coordinates: + location_defaults = { + **whois_obj._get_defaults_for_estimated_location(), + "organization_id": device.organization_id, + } + if current_location and current_location.is_estimated: + if attached_devices_exists: + # If there are other devices attached to the current location, + # we do not update it, but create a new one. + _create_estimated_location(device_location, location_defaults) + return + update_fields = [] + for attr, value in location_defaults.items(): + if getattr(current_location, attr) != value: + setattr(current_location, attr, value) + update_fields.append(attr) + if update_fields: + current_location.save( + update_fields=update_fields, _set_estimated=True + ) + logger.info( + f"Estimated location saved successfully for {device_pk}" + f" for IP: {ip_address}" + ) + send_whois_task_notification( + device_pk=device_pk, + notify_type="estimated_location_updated", + actor=current_location, + ) + elif not current_location: + # If there is no current location, we create a new one. + _create_estimated_location(device_location, location_defaults) + else: + logger.warning( + f"Coordinates not available for {device_pk} for IP: {ip_address}." + " Estimated location cannot be determined." + ) + return + + def _handle_attach_existing_location( + device, device_location, whois_obj, attached_devices_exists=False + ): + # For handling the case when WHOIS already exists for device's new last_ip + # then we attach the location of the device with same last_ip if it exists. + devices_with_location = ( + Device.objects.select_related("devicelocation") + .filter(organization_id=device.organization_id) + .filter(last_ip=ip_address, devicelocation__location__isnull=False) + .exclude(pk=device_pk) + ) + # If there are multiple devices with same last_ip then we need to inform + # the user to resolve the conflict manually. + if devices_with_location.count() > 1: + send_whois_task_notification( + device_pk=device_pk, notify_type="estimated_location_error" + ) + logger.error( + "Multiple devices with locations found with same " + f"last_ip {ip_address}. Please resolve the conflict manually." + ) + return + first_device = devices_with_location.first() + # If existing devices with same last_ip do not have any location + # then we create a new location based on WHOIS data. + if not first_device: + _update_or_create_estimated_location( + device_location, whois_obj, attached_devices_exists + ) + return + existing_location = first_device.devicelocation.location + # We need to remove any existing estimated location of the device + if current_location and not attached_devices_exists: + current_location.delete() + device_location.location = existing_location + device_location.full_clean() + device_location.save() + logger.info( + f"Estimated location saved successfully for {device_pk}" + f" for IP: {ip_address}" + ) + send_whois_task_notification( + device_pk=device_pk, + notify_type="estimated_location_updated", + actor=existing_location, + ) + + whois_obj = WHOISInfo.objects.filter(ip_address=ip_address).first() + device = ( + Device.objects.select_related("devicelocation__location", "organization") + .only("organization_id", "devicelocation") + .get(pk=device_pk) + ) + + if not (device_location := getattr(device, "devicelocation", None)): + device_location = DeviceLocation(content_object=device) + + attached_devices_exists = False + if current_location := device_location.location: + attached_devices_exists = ( + Device.objects.filter(devicelocation__location_id=current_location.pk) + .exclude(pk=device_pk) + .exists() + ) + + if not current_location or current_location.is_estimated: + _handle_attach_existing_location( + device, device_location, whois_obj, attached_devices_exists + ) + else: + logger.info( + f"Non Estimated location already set for {device_pk}. Update" + f" location manually as per IP: {ip_address}" + ) diff --git a/openwisp_controller/geo/estimated_location/tests/__init__.py b/openwisp_controller/geo/estimated_location/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwisp_controller/geo/estimated_location/tests/tests.py b/openwisp_controller/geo/estimated_location/tests/tests.py new file mode 100644 index 000000000..7b8eb1751 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tests/tests.py @@ -0,0 +1,761 @@ +import contextlib +import importlib +from unittest import mock + +from django.contrib.gis.geos import GEOSGeometry +from django.core.exceptions import ImproperlyConfigured, ValidationError +from django.test import TestCase, TransactionTestCase, override_settings +from django.urls import reverse +from openwisp_notifications.types import unregister_notification_type +from swapper import load_model + +from openwisp_controller.config import settings as config_app_settings +from openwisp_controller.config.whois.handlers import connect_whois_handlers +from openwisp_controller.config.whois.tests.utils import WHOISTransactionMixin + +from ....tests.utils import TestAdminMixin +from ...tests.utils import TestGeoMixin +from ..handlers import register_estimated_location_notification_types +from .utils import TestEstimatedLocationMixin + +Device = load_model("config", "Device") +Location = load_model("geo", "Location") +DeviceLocation = load_model("geo", "DeviceLocation") +WHOISInfo = load_model("config", "WHOISInfo") +Notification = load_model("openwisp_notifications", "Notification") +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + +notification_qs = Notification.objects.all() + + +class TestEstimatedLocation(TestAdminMixin, TestCase): + @override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT="test_account", + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY="test_key", + ) + def test_estimated_location_configuration_setting(self): + # reload app_settings to apply the overridden settings + importlib.reload(config_app_settings) + with self.subTest( + "ImproperlyConfigured raised when ESTIMATED_LOCATION_ENABLED is True " + "and WHOIS_ENABLED is False globally" + ): + with override_settings( + OPENWISP_CONTROLLER_WHOIS_ENABLED=False, + OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED=True, + ): + with self.assertRaises(ImproperlyConfigured): + # reload app_settings to apply the overridden settings + importlib.reload(config_app_settings) + + with self.subTest( + "Test WHOIS not enabled does not allow enabling Estimated Location" + ): + org_settings_obj = OrganizationConfigSettings(organization=self._get_org()) + with self.assertRaises(ValidationError) as context_manager: + org_settings_obj.whois_enabled = False + org_settings_obj.estimated_location_enabled = True + org_settings_obj.full_clean() + try: + self.assertEqual( + context_manager.exception.message_dict[ + "estimated_location_enabled" + ][0], + "Estimated Location feature requires " + + "WHOIS Lookup feature to be enabled.", + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + with self.subTest( + "Test Estimated Location field visible on admin when " + "WHOIS_CONFIGURED is True" + ): + self._login() + org = self._get_org() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertContains( + response, 'name="config_settings-0-estimated_location_enabled"' + ) + + with override_settings( + OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT=None, + OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY=None, + ): + importlib.reload(config_app_settings) + with self.subTest( + "Test Estimated Location field hidden on admin when " + "WHOIS_CONFIGURED is False" + ): + self._login() + org = self._get_org() + url = reverse( + "admin:openwisp_users_organization_change", + args=[org.pk], + ) + response = self.client.get(url) + self.assertNotContains( + response, 'name="config_settings-0-estimated_location_enabled"' + ) + + +class TestEstimatedLocationField(TestEstimatedLocationMixin, TestGeoMixin, TestCase): + location_model = Location + + def test_estimated_location_field(self): + org = self._get_org() + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + org.refresh_from_db() + with self.assertRaises(ValidationError) as context_manager: + self._create_location(organization=org, is_estimated=True) + try: + self.assertEqual( + context_manager.exception.message_dict["is_estimated"][0], + "Estimated Location feature required to be configured.", + ) + except AssertionError: + self.fail("ValidationError message not equal to expected message.") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_admin(self): + connect_whois_handlers() + admin = self._create_admin() + self.client.force_login(admin) + org = self._get_org() + location = self._create_location(organization=org, is_estimated=True) + path = reverse("admin:geo_location_change", args=[location.pk]) + response = self.client.get(path) + self.assertContains(response, "field-is_estimated") + self.assertContains( + response, "Whether the location's coordinates are estimated." + ) + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + response = self.client.get(path) + self.assertNotContains(response, "field-is_estimated") + self.assertNotContains( + response, "Whether the location's coordinates are estimated." + ) + + +class TestEstimatedLocationTransaction( + TestEstimatedLocationMixin, WHOISTransactionMixin, TransactionTestCase +): + _WHOIS_GEOIP_CLIENT = ( + "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + ) + _ESTIMATED_LOCATION_INFO_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.info" + ) + _ESTIMATED_LOCATION_ERROR_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.error" + ) + + def setUp(self): + super().setUp() + self.admin = self._get_admin() + # Unregister the notification type if it was previously registered + with contextlib.suppress(ImproperlyConfigured): + unregister_notification_type("estimated_location_info") + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True): + register_estimated_location_notification_types() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.manage_estimated_locations.delay" # noqa + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_task_called( + self, mocked_client, mocked_estimated_location_task + ): + connect_whois_handlers() + mocked_client.return_value.city.return_value = self._mocked_client_response() + + self._task_called( + mocked_estimated_location_task, task_name="Estimated location" + ) + + Device.objects.all().delete() + device = self._create_device() + with self.subTest( + "Estimated location task called when last_ip has related WhoIsInfo" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.organization.config_settings.whois_enabled = True + device.organization.config_settings.estimated_location_enabled = True + device.organization.config_settings.save() + device.last_ip = "172.217.22.14" + self._create_whois_info(ip_address=device.last_ip) + device.save() + mocked_set.assert_not_called() + # The cache `get` is called twice, once for `whois_enabled` and + # once for `estimated_location_enabled` + mocked_get.assert_called() + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimated location task not called via DeviceChecksumView when " + "last_ip has no related WhoIsInfo" + ): + WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) + self._create_config(device=device) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_creation_and_update(self, mock_client, mock_info): + connect_whois_handlers() + + def _verify_location_details(device, mocked_response): + location = device.devicelocation.location + mocked_location = mocked_response.location + address = ", ".join( + [ + mocked_response.city.name, + mocked_response.country.name, + mocked_response.continent.name, + mocked_response.postal.code, + ] + ) + ip_address = mocked_response.ip_address or device.last_ip + location_name = ( + ",".join(address.split(",")[:2]) + + f" (Estimated Location: {ip_address})" + ) + self.assertEqual(location.name, location_name) + self.assertEqual(location.address, address) + self.assertEqual( + location.geometry, + GEOSGeometry( + f"POINT({mocked_location.longitude} {mocked_location.latitude})", + srid=4326, + ), + ) + + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + + with self.subTest("Test Estimated location created when device is created"): + device = self._create_device(last_ip="172.217.22.14") + + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest("Test Estimated location updated when last ip is updated"): + device.last_ip = "172.217.22.10" + mocked_response.location.latitude = 50 + mocked_response.location.longitude = 150 + mocked_response.city.name = "New City" + mock_client.return_value.city.return_value = mocked_response + device.save() + device.refresh_from_db() + + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test Non Estimated Location not updated when last ip is updated" + ): + mocked_response.ip_address = device.last_ip + device.last_ip = "172.217.22.11" + device.devicelocation.location.is_estimated = False + mock_client.return_value.city.return_value = self._mocked_client_response() + device.devicelocation.location.save(_set_estimated=True) + device.save() + device.refresh_from_db() + + location = device.devicelocation.location + self.assertEqual(location.is_estimated, False) + self.assertEqual(location.is_mobile, False) + self.assertEqual(location.type, "outdoor") + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Non Estimated location already set for {device.pk}. Update" + f" location manually as per IP: {device.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test location shared for same IP when new device's location does not exist" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + mock_info.reset_mock() + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + mock_info.reset_mock() + + with self.subTest( + "Test location shared for same IP when new device's location is estimated" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + mock_info.reset_mock() + old_location = device2.devicelocation.location + device2.last_ip = "172.217.22.10" + device2.save() + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + device2.refresh_from_db() + + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.filter(pk=old_location.pk).count(), 0) + mock_info.reset_mock() + + with self.subTest( + "Test location not shared for same IP when new " + "device's location is not estimated" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.11", + ) + mock_info.reset_mock() + old_location = device2.devicelocation.location + old_location.is_estimated = False + old_location.save() + device2.last_ip = "172.217.22.10" + device2.save() + mock_info.assert_called_once_with( + f"Non Estimated location already set for {device2.pk}. Update" + f" location manually as per IP: {device2.last_ip}" + ) + device2.refresh_from_db() + + self.assertNotEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.filter(pk=old_location.pk).count(), 1) + mock_info.reset_mock() + + with self.subTest( + "Shared location not updated when either device's last_ip changes. " + "New location created for device with updated last_ip" + ): + Device.objects.all().delete() + device1 = self._create_device(last_ip="172.217.22.10") + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + mock_info.reset_mock() + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + device2.last_ip = "172.217.22.11" + device2.save() + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device2.pk}" + f" for IP: {device2.last_ip}" + ) + device2.refresh_from_db() + self.assertNotEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + mock_info.reset_mock() + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) + @mock.patch(_ESTIMATED_LOCATION_ERROR_LOGGER) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_notification(self, mock_client, mock_error, mock_info): + def _verify_notification(device, messages, notify_level="info"): + self.assertEqual(notification_qs.count(), 1) + notification = notification_qs.first() + device_location = getattr(device, "devicelocation", None) + actor = device + if device_location: + actor = device_location.location + self.assertEqual(notification.actor, actor) + self.assertEqual(notification.target, device) + self.assertEqual(notification.type, "estimated_location_info") + self.assertEqual(notification.level, notify_level) + for message in messages: + self.assertIn(message, notification.message) + self.assertIn(device.last_ip, notification.rendered_description) + self.assertIn("#devicelocation-group", notification.target_url) + + with self.subTest("Test Notification for location create"): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + device1 = self._create_device(last_ip="172.217.22.10") + messages = ["Estimated location", "created successfully"] + _verify_notification(device1, messages) + + with self.subTest("Test Notification for location update"): + notification_qs.delete() + # will have same location as first device + device2 = self._create_device( + name="11:22:33:44:55:66", + mac_address="11:22:33:44:55:66", + last_ip="172.217.22.10", + ) + messages = ["Estimated location", "updated successfully"] + _verify_notification(device2, messages) + + with self.subTest("Test Error Notification for conflicting locations"): + device2.last_ip = device1.last_ip + device2.save() + notification_qs.delete() + mock_info.reset_mock() + mock_error.reset_mock() + device3 = self._create_device( + name="11:22:33:44:55:77", + mac_address="11:22:33:44:55:77", + last_ip=device2.last_ip, + ) + mock_info.assert_not_called() + mock_error.assert_called_once_with( + f"Multiple devices with locations found with same " + f"last_ip {device3.last_ip}. Please resolve the conflict manually." + ) + messages = ["Unable to create estimated location for device"] + _verify_notification(device3, messages, "error") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimate_location_status_remove(self, mock_client): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip="172.217.22.10") + location = device.devicelocation.location + self.assertTrue(location.is_estimated) + org = self._get_org() + + with self.subTest( + "Test Estimated Status unchanged if Estimated feature is disabled" + ): + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + location.geometry = GEOSGeometry("POINT(12.512124 41.898903)", srid=4326) + location.save() + self.assertTrue(location.is_estimated) + self.assertIn(f"(Estimated Location: {device.last_ip})", location.name) + + with self.subTest( + "Test Estimated Status unchanged if Estimated feature is enabled" + " and desired fields not changed" + ): + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + location._set_initial_values_for_changed_checked_fields() + location.type = "outdoor" + location.is_mobile = True + location.save() + self.assertTrue(location.is_estimated) + + with self.subTest( + "Test Estimated Status changed if Estimated feature is enabled" + " and desired fields changed" + ): + location.geometry = GEOSGeometry("POINT(15.512124 45.898903)", srid=4326) + location.save() + self.assertFalse(location.is_estimated) + self.assertNotIn(f"(Estimated Location: {device.last_ip})", location.name) + + +class TestEstimatedLocationFieldFilters( + TestEstimatedLocationMixin, TestGeoMixin, TestCase +): + location_model = Location + object_location_model = DeviceLocation + + def setUp(self): + super().setUp() + admin = self._create_admin() + self.client.force_login(admin) + + def _create_device_location(self, **kwargs): + options = dict() + options.update(kwargs) + device_location = self.object_location_model(**options) + device_location.full_clean() + device_location.save() + return device_location + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_api_status_configured(self): + org1 = self._get_org() + org2 = self._create_org(name="org2") + OrganizationConfigSettings.objects.create( + organization=org2, + whois_enabled=False, + estimated_location_enabled=False, + ) + org1_location = self._create_location( + name="org1-location", organization=org1, is_estimated=True + ) + org2_location = self._create_location(name="org2-location", organization=org2) + org1_device = self._create_device(organization=org1) + org2_device = self._create_device(organization=org2) + self._create_device_location(content_object=org1_device, location=org1_location) + self._create_device_location(content_object=org2_device, location=org2_location) + + with self.subTest("Test Estimated Location in Locations List"): + path = reverse("geo_api:list_location") + with self.assertNumQueries(5): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + self.assertContains(response, org1_location.id) + self.assertContains(response, org2_location.id) + location1 = response.data["results"][1] + location2 = response.data["results"][0] + self.assertIn("is_estimated", location1) + self.assertNotIn("is_estimated", location2) + + with self.subTest("Test Estimated Location in Device Locations List"): + path = reverse("geo_api:device_location", args=[org1_device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertIn("is_estimated", response.data["location"]["properties"]) + + path = reverse("geo_api:device_location", args=[org2_device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertNotIn("is_estimated", response.data["location"]["properties"]) + + with self.subTest("Test Estimated Location in GeoJSON List"): + path = reverse("geo_api:location_geojson") + with self.assertNumQueries(3): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + for i in response.data["features"]: + if i["id"] == org1_location.id: + self.assertIn("is_estimated", i["properties"]) + self.assertTrue(i["properties"]["is_estimated"]) + elif i["id"] == org2_location.id: + self.assertNotIn("is_estimated", i["properties"]) + self.assertFalse(i["properties"]["is_estimated"]) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False) + def test_estimated_location_api_status_not_configured(self): + org = self._get_org() + location = self._create_location(name="org1-location", organization=org) + device = self._create_device(organization=org) + self._create_device_location(content_object=device, location=location) + + with self.subTest("Test Estimated status not in Locations List"): + path = reverse("geo_api:list_location") + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, location.id) + api_location = response.data["results"][0] + self.assertNotIn("is_estimated", api_location) + + with self.subTest("Test Estimated status not in Device Locations List"): + path = reverse("geo_api:device_location", args=[device.pk]) + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertNotIn("is_estimated", response.data["location"]["properties"]) + + with self.subTest("Test Estimated status not in GeoJSON Location List"): + path = reverse("geo_api:location_geojson") + with self.assertNumQueries(3): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + location_features = response.data["features"][0] + self.assertNotIn("is_estimated", location_features["properties"]) + self.assertFalse(location_features["properties"].get("is_estimated", False)) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_filter_list_api(self): + org = self._get_org() + location1 = self._create_location( + name="location1", is_estimated=True, organization=org + ) + location2 = self._create_location( + name="location2", is_estimated=False, organization=org + ) + device1 = self._create_device() + device2 = self._create_device( + name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" + ) + self._create_device_location(content_object=device1, location=location1) + self._create_device_location(content_object=device2, location=location2) + + path = reverse("geo_api:list_location") + + with self.subTest( + "Test Estimated Location filter available in location list " + "when WHOIS is configured" + ): + with self.assertNumQueries(4): + response = self.client.get(path, {"is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, location1.id) + self.assertNotContains(response, location2.id) + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location filter not available in location list " + "when WHOIS not configured" + ): + with self.assertNumQueries(5): + response = self.client.get(path, {"is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + self.assertContains(response, location1.id) + self.assertContains(response, location2.id) + + path = reverse("config_api:device_list") + + with self.subTest( + "Test Estimated Location filter available in device list " + "when WHOIS is configured" + ): + with self.assertNumQueries(3): + response = self.client.get(path, {"geo_is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 1) + self.assertContains(response, device1.id) + self.assertNotContains(response, device2.id) + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location filter not available in device list " + "when WHOIS not configured" + ): + with self.assertNumQueries(3): + response = self.client.get(path, {"geo_is_estimated": True}) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["count"], 2) + self.assertContains(response, device1.id) + self.assertContains(response, device2.id) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_estimated_location_filter_admin(self): + org = self._get_org() + estimated_location = self._create_location( + name="location1", is_estimated=True, organization=org + ) + outdoor_location = self._create_location(name="location2", organization=org) + indoor_location = self._create_location( + name="location3", organization=org, type="indoor" + ) + + estimated_device = self._create_device() + outdoor_device = self._create_device( + name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" + ) + indoor_device = self._create_device( + name="11:22:33:44:55:77", mac_address="11:22:33:44:55:77" + ) + self._create_device_location( + content_object=estimated_device, location=estimated_location + ) + self._create_device_location( + content_object=outdoor_device, location=outdoor_location + ) + self._create_device_location( + content_object=indoor_device, location=indoor_location + ) + + path = reverse("admin:config_device_changelist") + with self.subTest("Test All Locations Filter"): + response = self.client.get(path) + self.assertContains(response, estimated_device.id) + self.assertContains(response, outdoor_device.id) + self.assertContains(response, indoor_device.id) + + with self.subTest("Test Estimated Location Filter"): + response = self.client.get(path, {"with_geo": "estimated"}) + self.assertContains(response, estimated_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, indoor_device.id) + + with self.subTest("Test Outdoor Location Filter"): + response = self.client.get(path, {"with_geo": "outdoor"}) + self.assertContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + self.assertNotContains(response, indoor_device.id) + + with self.subTest("Test Indoor Location Filter"): + response = self.client.get(path, {"with_geo": "indoor"}) + self.assertContains(response, indoor_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + + with self.subTest("Test Indoor Location Filter"): + response = self.client.get(path, {"with_geo": "false"}) + self.assertNotContains(response, indoor_device.id) + self.assertNotContains(response, outdoor_device.id) + self.assertNotContains(response, estimated_device.id) + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + with self.subTest( + "Test Estimated Location Admin specific filters not available" + " when WHOIS not configured" + ): + for i in ["estimated", "outdoor", "indoor"]: + response = self.client.get(path, {"with_geo": i}) + self.assertContains(response, estimated_device.id) + self.assertContains(response, outdoor_device.id) + self.assertContains(response, indoor_device.id) diff --git a/openwisp_controller/geo/estimated_location/tests/utils.py b/openwisp_controller/geo/estimated_location/tests/utils.py new file mode 100644 index 000000000..77bb618ba --- /dev/null +++ b/openwisp_controller/geo/estimated_location/tests/utils.py @@ -0,0 +1,15 @@ +from swapper import load_model + +from openwisp_controller.config.whois.tests.utils import CreateWHOISMixin + +OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") + + +class TestEstimatedLocationMixin(CreateWHOISMixin): + def setUp(self): + super(CreateWHOISMixin, self).setUp() + OrganizationConfigSettings.objects.create( + organization=self._get_org(), + whois_enabled=True, + estimated_location_enabled=True, + ) diff --git a/openwisp_controller/geo/estimated_location/utils.py b/openwisp_controller/geo/estimated_location/utils.py new file mode 100644 index 000000000..ddd43e837 --- /dev/null +++ b/openwisp_controller/geo/estimated_location/utils.py @@ -0,0 +1,6 @@ +from openwisp_notifications.utils import _get_object_link + + +def get_device_location_notification_target_url(obj, field, absolute_url=True): + url = _get_object_link(obj._related_object(field), absolute_url) + return f"{url}#devicelocation-group" diff --git a/openwisp_controller/geo/migrations/0004_location_is_estimated.py b/openwisp_controller/geo/migrations/0004_location_is_estimated.py new file mode 100644 index 000000000..1fcdfca68 --- /dev/null +++ b/openwisp_controller/geo/migrations/0004_location_is_estimated.py @@ -0,0 +1,21 @@ +# Generated by Django 5.2.1 on 2025-06-25 19:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("geo", "0003_alter_devicelocation_floorplan_location"), + ] + + operations = [ + migrations.AddField( + model_name="location", + name="is_estimated", + field=models.BooleanField( + default=False, + help_text=("Whether the location's coordinates are estimated."), + ), + ), + ] diff --git a/openwisp_controller/geo/templates/admin/geo/location/change_form.html b/openwisp_controller/geo/templates/admin/geo/location/change_form.html new file mode 100644 index 000000000..96b4392e9 --- /dev/null +++ b/openwisp_controller/geo/templates/admin/geo/location/change_form.html @@ -0,0 +1,11 @@ +{% extends "admin/change_form.html" %} +{% load i18n admin_urls %} + +{% block messages %} + {{ block.super }} + {% if original and estimated_configured and original.is_estimated %} +
    +
  • {% trans "This location is estimated based on the device's last IP address. Please refine it for greater accuracy." %}
    • +
+ {% endif %} +{% endblock messages %} diff --git a/openwisp_controller/geo/tests/test_admin.py b/openwisp_controller/geo/tests/test_admin.py index 8d914dc8d..06a498cfe 100644 --- a/openwisp_controller/geo/tests/test_admin.py +++ b/openwisp_controller/geo/tests/test_admin.py @@ -137,6 +137,16 @@ def test_admin_menu_groups(self): html=True, ) + def test_location_readonly_fields(self): + location = self._create_location( + name="location1org", type="indoor", organization=self._get_org() + ) + self._create_admin() + self._login() + url = reverse(f"admin:{self.app_label}_location_change", args=[location.pk]) + response = self.client.get(url) + self.assertNotContains(response, ' Date: Wed, 10 Dec 2025 12:58:41 +0530 Subject: [PATCH 13/26] [feature] Updating stale WHOIS records #1058 Added required checks for ensuring WHOIS records older than a certain configurable threshold will be updated along with the coordinates. Closes #1058 Signed-off-by: DragnEmperor --- docs/user/estimated-location.rst | 15 +- docs/user/settings.rst | 13 ++ docs/user/whois.rst | 21 +- openwisp_controller/config/base/device.py | 2 + .../config/controller/views.py | 3 + .../management/commands/clear_last_ip.py | 9 +- .../migrations/0061_config_checksum_db.py | 3 +- .../{0061_whoisinfo.py => 0062_whoisinfo.py} | 2 +- ..._approximate_location_enabled_and_more.py} | 2 +- openwisp_controller/config/settings.py | 1 + openwisp_controller/config/whois/service.py | 197 +++++++++++++++- openwisp_controller/config/whois/tasks.py | 102 ++------- .../config/whois/tests/tests.py | 54 ++++- .../config/whois/tests/utils.py | 9 +- openwisp_controller/config/whois/utils.py | 6 +- .../geo/estimated_location/tasks.py | 212 ++++++++---------- .../geo/estimated_location/tests/tests.py | 107 ++++++++- .../admin/geo/location/change_form.html | 2 +- 18 files changed, 520 insertions(+), 240 deletions(-) rename openwisp_controller/config/migrations/{0061_whoisinfo.py => 0062_whoisinfo.py} (97%) rename openwisp_controller/config/migrations/{0062_organizationconfigsettings_approximate_location_enabled_and_more.py => 0063_organizationconfigsettings_approximate_location_enabled_and_more.py} (96%) diff --git a/docs/user/estimated-location.rst b/docs/user/estimated-location.rst index 1f00a7440..48b3c100f 100644 --- a/docs/user/estimated-location.rst +++ b/docs/user/estimated-location.rst @@ -16,14 +16,14 @@ Estimated Location Overview -------- -The Estimated Location feature automatically creates or updates a device’s -location based on latitude and longitude information retrieved from the -WHOIS Lookup feature. +This feature automatically creates or updates a device’s location based on +latitude and longitude information retrieved from the WHOIS Lookup +feature. Trigger Conditions ------------------ -Estimated Location is triggered when: +This feature is triggered when: - A **fresh WHOIS lookup** is performed for a device. - Or when a WHOIS record already exists for the device’s IP **and**: @@ -78,3 +78,10 @@ In REST API, the field will be visible in the :ref:`Device Location ` if the feature is **enabled**. The field can also be used for filtering in the location list (including geojson) endpoints and in the :ref:`Device List `. + +Managing Older Estimated Locations +---------------------------------- + +Whenever location related fields in WHOIS records are updated as per +:ref:`Managing WHOIS Older Records `; the location +will also be updated automatically. diff --git a/docs/user/settings.rst b/docs/user/settings.rst index 9a59bbf22..cf0d4a4de 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -829,3 +829,16 @@ Allows enabling the optional :doc:`Estimated Location feature .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png :alt: Estimated Location setting + +.. _openwisp_controller_whois_refresh_threshold_days: + +``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS`` +---------------------------------------------------- + +============ ======= +**type**: ``int`` +**default**: ``14`` +============ ======= + +Specifies the number of days after which the WHOIS information for a +device is considered stale and eligible for refresh. diff --git a/docs/user/whois.rst b/docs/user/whois.rst index e9c63274c..578bce597 100644 --- a/docs/user/whois.rst +++ b/docs/user/whois.rst @@ -15,10 +15,10 @@ WHOIS Lookup Overview -------- -The WHOIS Lookup feature displays information about the public IP address -used by devices to communicate with OpenWISP (via the ``last_ip`` field). -It helps identify the geographic location and ISP associated with the IP -address, which can be useful for troubleshooting network issues. +This feature displays information about the public IP address used by +devices to communicate with OpenWISP (via the ``last_ip`` field). It helps +identify the geographic location and ISP associated with the IP address, +which can be useful for troubleshooting network issues. The retrieved information pertains to the Autonomous System (ASN) associated with the device's public IP address and includes: @@ -36,6 +36,7 @@ Trigger Conditions A WHOIS lookup is triggered automatically when: - A new device is registered. +- An existing device's last IP address is changed. - A device fetches its checksum. However, the lookup will only run if **all** the following conditions are @@ -114,3 +115,15 @@ retrieved details can be viewed in the following locations: - **Device REST API**: See WHOIS details in the :ref:`Device List ` and :ref:`Device Detail ` responses. + +.. _whois_older_records: + +Managing Older WHOIS Records +---------------------------- + +If a record is older than :ref:`Threshold +`, it will be refreshed +automatically. + +The update mechanism will be triggered whenever a device is registered or +its last IP changes or fetches its checksum. diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 525098737..2a2d88fd0 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -538,4 +538,6 @@ def _check_last_ip(self, creating=False): return if creating or self.last_ip != self._initial_last_ip: self.whois_service.process_ip_data_and_location() + else: + self.whois_service.update_whois_info() self._initial_last_ip = self.last_ip diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py index 3dc714c0c..5e1c9fa10 100644 --- a/openwisp_controller/config/controller/views.py +++ b/openwisp_controller/config/controller/views.py @@ -153,6 +153,9 @@ def get(self, request, pk): # updates cache if ip addresses changed if updated: self.update_device_cache(device) + # check if WHOIS Info of device requires update + else: + device.whois_service.update_whois_info() checksum_requested.send( sender=device.__class__, instance=device, request=request ) diff --git a/openwisp_controller/config/management/commands/clear_last_ip.py b/openwisp_controller/config/management/commands/clear_last_ip.py index 4a5efd4c7..7c68a3d4b 100644 --- a/openwisp_controller/config/management/commands/clear_last_ip.py +++ b/openwisp_controller/config/management/commands/clear_last_ip.py @@ -4,7 +4,10 @@ class Command(BaseCommand): - help = "Clear the last IP address, if set, of active devices of all organizations." + help = ( + "Clears the last IP address (if set) for every active device" + " across all organizations." + ) def add_arguments(self, parser): parser.add_argument( @@ -29,11 +32,11 @@ def handle(self, *args, **options): "Are you sure you want to do this?\n\n" "Type 'yes' to continue, or 'no' to cancel: " ) - if input("".join(message)) != "yes": + if input("".join(message)).lower() != "yes": raise CommandError("Operation cancelled by user.") devices = Device.objects.filter(_is_deactivated=False).only("last_ip") - # Filter devices that have no WHOIS information for their last IP + # Filter out devices that have WHOIS information for their last IP devices = devices.exclude(last_ip=None).exclude( last_ip__in=Subquery( WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( diff --git a/openwisp_controller/config/migrations/0061_config_checksum_db.py b/openwisp_controller/config/migrations/0061_config_checksum_db.py index a572951cf..34319cae3 100644 --- a/openwisp_controller/config/migrations/0061_config_checksum_db.py +++ b/openwisp_controller/config/migrations/0061_config_checksum_db.py @@ -1,7 +1,6 @@ # Generated by Django for issue #1113 optimization from django.db import migrations, models -from swapper import load_model def populate_checksum_db(apps, schema_editor): @@ -13,7 +12,7 @@ def populate_checksum_db(apps, schema_editor): hence we use Config.objects.bulk_update() instead of Config.update_status_if_checksum_changed(). """ - Config = load_model("config", "Config") + Config = apps.get_model("config", "Config") chunk_size = 100 updated_configs = [] qs = ( diff --git a/openwisp_controller/config/migrations/0061_whoisinfo.py b/openwisp_controller/config/migrations/0062_whoisinfo.py similarity index 97% rename from openwisp_controller/config/migrations/0061_whoisinfo.py rename to openwisp_controller/config/migrations/0062_whoisinfo.py index d8148756b..4755b8485 100644 --- a/openwisp_controller/config/migrations/0061_whoisinfo.py +++ b/openwisp_controller/config/migrations/0062_whoisinfo.py @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ - ("config", "0060_cleanup_api_task_notification_types"), + ("config", "0061_config_checksum_db"), ] operations = [ diff --git a/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py b/openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py similarity index 96% rename from openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py rename to openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py index 8fe4d4c2c..bdb3dab64 100644 --- a/openwisp_controller/config/migrations/0062_organizationconfigsettings_approximate_location_enabled_and_more.py +++ b/openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py @@ -9,7 +9,7 @@ class Migration(migrations.Migration): dependencies = [ - ("config", "0061_whoisinfo"), + ("config", "0062_whoisinfo"), ] operations = [ diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index fa3efc750..8bcda862d 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -68,6 +68,7 @@ def get_setting(option, default): "API_TASK_RETRY_OPTIONS", dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True), ) +WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 14) WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None) WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None) WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False) diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py index 3e9dad6f1..71884d52d 100644 --- a/openwisp_controller/config/whois/service.py +++ b/openwisp_controller/config/whois/service.py @@ -1,12 +1,37 @@ +from datetime import timedelta from ipaddress import ip_address as ip_addr +import requests +from django.contrib.gis.geos import Point from django.core.cache import cache from django.db import transaction +from django.utils import timezone +from django.utils.translation import gettext as _ +from geoip2 import errors +from geoip2 import webservice as geoip2_webservice from swapper import load_model from openwisp_controller.config import settings as app_settings from .tasks import fetch_whois_details, manage_estimated_locations +from .utils import send_whois_task_notification + +EXCEPTION_MESSAGES = { + errors.AddressNotFoundError: _( + "No WHOIS information found for IP address {ip_address}" + ), + errors.AuthenticationError: _( + "Authentication failed for GeoIP2 service. " + "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." + ), + errors.OutOfQueriesError: _( + "Your account has run out of queries for the GeoIP2 service." + ), + errors.PermissionRequiredError: _( + "Your account does not have permission to access this service." + ), +} class WHOISService: @@ -17,6 +42,20 @@ class WHOISService: def __init__(self, device): self.device = device + @staticmethod + def get_geoip_client(): + """ + Used to get a GeoIP2 web service client instance. + Host is based on the db that is used to fetch the details. + As we are using GeoLite2, 'geolite.info' host is used. + Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example + """ + return geoip2_webservice.Client( + account_id=app_settings.WHOIS_GEOIP_ACCOUNT, + license_key=app_settings.WHOIS_GEOIP_KEY, + host="geolite.info", + ) + @staticmethod def get_cache_key(org_id): """ @@ -43,6 +82,15 @@ def _get_whois_info_from_db(ip_address): return WHOISInfo.objects.filter(ip_address=ip_address) + @staticmethod + def is_older(datetime): + """ + Check if given datetime is older than the refresh threshold. + """ + return (timezone.now() - datetime) >= timedelta( + days=app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + ) + @staticmethod def get_org_config_settings(org_id): """ @@ -90,6 +138,8 @@ def is_whois_enabled(self): """ Check if the WHOIS lookup feature is enabled. """ + if not app_settings.WHOIS_CONFIGURED: + return False org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) return org_settings.whois_enabled @@ -98,9 +148,57 @@ def is_estimated_location_enabled(self): """ Check if the Estimated location feature is enabled. """ + if not app_settings.WHOIS_CONFIGURED: + return False org_settings = self.get_org_config_settings(org_id=self.device.organization.pk) return org_settings.estimated_location_enabled + def process_whois_details(self, ip_address): + """ + Fetch WHOIS details for a given IP address and return only + the relevant information. + """ + ip_client = self.get_geoip_client() + try: + data = ip_client.city(ip_address=ip_address) + # Catching all possible exceptions raised by the geoip2 client + # and raising them with appropriate messages to be handled by the task + # retry mechanism. + except ( + errors.AddressNotFoundError, + errors.AuthenticationError, + errors.OutOfQueriesError, + errors.PermissionRequiredError, + ) as e: + exc_type = type(e) + message = EXCEPTION_MESSAGES.get(exc_type) + if exc_type is errors.AddressNotFoundError: + message = message.format(ip_address=ip_address) + raise exc_type(message) + except requests.RequestException as e: + raise e + else: + # The attributes are always present in the response, + # but they can be None, so added fallbacks. + address = { + "city": data.city.name or "", + "country": data.country.name or "", + "continent": data.continent.name or "", + "postal": str(data.postal.code or ""), + } + coordinates = Point( + data.location.longitude, data.location.latitude, srid=4326 + ) + return { + "isp": data.traits.autonomous_system_organization, + "asn": data.traits.autonomous_system_number, + "timezone": data.location.time_zone, + "address": address, + "coordinates": coordinates, + "cidr": data.traits.network, + "ip_address": ip_address, + } + def _need_whois_lookup(self, new_ip): """ This is used to determine if the WHOIS lookup should be triggered @@ -108,17 +206,17 @@ def _need_whois_lookup(self, new_ip): The lookup is not triggered if: - The new IP address is None or it is a private IP address. - - The WHOIS information of new ip is already present. + - The WHOIS information of new ip is present and is not older than + X days (defined by "WHOIS_REFRESH_THRESHOLD_DAYS"). - WHOIS is disabled in the organization settings. (query from db) """ # Check cheap conditions first before hitting the database if not self.is_valid_public_ip_address(new_ip): return False - - if self._get_whois_info_from_db(new_ip).exists(): + whois_obj = self._get_whois_info_from_db(ip_address=new_ip).first() + if whois_obj and not self.is_older(whois_obj.modified): return False - return self.is_whois_enabled def _need_estimated_location_management(self, new_ip): @@ -128,10 +226,8 @@ def _need_estimated_location_management(self, new_ip): """ if not self.is_valid_public_ip_address(new_ip): return False - if not self.is_whois_enabled: return False - return self.is_estimated_location_enabled def get_device_whois_info(self): @@ -145,7 +241,7 @@ def get_device_whois_info(self): return self._get_whois_info_from_db(ip_address=ip_address).first() - def process_ip_data_and_location(self): + def process_ip_data_and_location(self, force_lookup=False): """ Trigger WHOIS lookup based on the conditions of `_need_whois_lookup` and also manage estimated locations based on the conditions of @@ -153,12 +249,11 @@ def process_ip_data_and_location(self): Tasks are triggered on commit to ensure redundant data is not created. """ new_ip = self.device.last_ip - if self._need_whois_lookup(new_ip): + if force_lookup or self._need_whois_lookup(new_ip): transaction.on_commit( lambda: fetch_whois_details.delay( device_pk=self.device.pk, initial_ip_address=self.device._initial_last_ip, - new_ip_address=new_ip, ) ) # To handle the case when WHOIS already exists as in that case @@ -170,3 +265,87 @@ def process_ip_data_and_location(self): device_pk=self.device.pk, ip_address=new_ip ) ) + + def update_whois_info(self): + """ + Update the WHOIS information for the device. + """ + ip_address = self.device.last_ip + if not self.is_valid_public_ip_address(ip_address): + return + if not self.is_whois_enabled: + return + whois_obj = WHOISService._get_whois_info_from_db(ip_address=ip_address).first() + if whois_obj and self.is_older(whois_obj.modified): + fetch_whois_details.delay(device_pk=self.device.pk, initial_ip_address=None) + + def _create_or_update_whois(self, whois_details, whois_instance=None): + """ + Used to update an existing WHOIS instance; else, creates a new one. + Returns the updated or created WHOIS instance along with update fields. + """ + WHOISInfo = load_model("config", "WHOISInfo") + + update_fields = [] + if whois_instance: + for attr, value in whois_details.items(): + if getattr(whois_instance, attr) != value: + update_fields.append(attr) + setattr(whois_instance, attr, value) + if update_fields: + whois_instance.save(update_fields=update_fields) + else: + whois_instance = WHOISInfo(**whois_details) + whois_instance.full_clean() + whois_instance.save(force_insert=True) + return whois_instance, update_fields + + def _create_or_update_estimated_location( + self, location_defaults, attached_devices_exists + ): + """ + Create or update estimated location for the device based on the + given location defaults. + """ + Location = load_model("geo", "Location") + DeviceLocation = load_model("geo", "DeviceLocation") + + if not (device_location := getattr(self.device, "devicelocation", None)): + device_location = DeviceLocation(content_object=self.device) + + current_location = device_location.location + + if not current_location or ( + attached_devices_exists and current_location.is_estimated + ): + with transaction.atomic(): + current_location = Location(**location_defaults, is_estimated=True) + current_location.full_clean() + current_location.save(_set_estimated=True) + device_location.location = current_location + device_location.full_clean() + device_location.save() + send_whois_task_notification( + device=self.device, + notify_type="estimated_location_created", + actor=current_location, + ) + elif current_location.is_estimated: + update_fields = [] + for attr, value in location_defaults.items(): + if getattr(current_location, attr) != value: + setattr(current_location, attr, value) + update_fields.append(attr) + if update_fields: + with transaction.atomic(): + current_location.save( + update_fields=update_fields, _set_estimated=True + ) + + send_whois_task_notification( + device=self.device, + notify_type="estimated_location_updated", + actor=current_location, + ) + + return current_location diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py index bafca7b78..26b0a5037 100644 --- a/openwisp_controller/config/whois/tasks.py +++ b/openwisp_controller/config/whois/tasks.py @@ -1,11 +1,8 @@ import logging -import requests from celery import shared_task -from django.contrib.gis.geos import Point -from django.utils.translation import gettext as _ +from django.db import transaction from geoip2 import errors -from geoip2 import webservice as geoip2_webservice from swapper import load_model from openwisp_controller.geo.estimated_location.tasks import manage_estimated_locations @@ -16,23 +13,6 @@ logger = logging.getLogger(__name__) -EXCEPTION_MESSAGES = { - errors.AddressNotFoundError: _( - "No WHOIS information found for IP address {ip_address}" - ), - errors.AuthenticationError: _( - "Authentication failed for GeoIP2 service. " - "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " - "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." - ), - errors.OutOfQueriesError: _( - "Your account has run out of queries for the GeoIP2 service." - ), - errors.PermissionRequiredError: _( - "Your account does not have permission to access this service." - ), -} - class WHOISCeleryRetryTask(OpenwispCeleryTask): """ @@ -46,9 +26,7 @@ class WHOISCeleryRetryTask(OpenwispCeleryTask): def on_failure(self, exc, task_id, args, kwargs, einfo): """Notify the user about the failure of the WHOIS task.""" device_pk = kwargs.get("device_pk") - send_whois_task_notification( - device_pk=device_pk, notify_type="whois_device_error" - ) + send_whois_task_notification(device=device_pk, notify_type="whois_device_error") logger.error(f"WHOIS lookup failed. Details: {exc}") return super().on_failure(exc, task_id, args, kwargs, einfo) @@ -59,7 +37,7 @@ def on_failure(self, exc, task_id, args, kwargs, einfo): base=WHOISCeleryRetryTask, **app_settings.API_TASK_RETRY_OPTIONS, ) -def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): +def fetch_whois_details(self, device_pk, initial_ip_address): """ Fetches the WHOIS details of the given IP address and creates/updates the WHOIS record. @@ -67,65 +45,29 @@ def fetch_whois_details(self, device_pk, initial_ip_address, new_ip_address): Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") - # The task can be triggered for same ip address multiple times - # so we need to return early if WHOIS is already created. - if WHOISInfo.objects.filter(ip_address=new_ip_address).exists(): - return - - device = Device.objects.get(pk=device_pk) - # Host is based on the db that is used to fetch the details. - # As we are using GeoLite2, 'geolite.info' host is used. - # Refer: https://geoip2.readthedocs.io/en/latest/#sync-web-service-example - ip_client = geoip2_webservice.Client( - account_id=app_settings.WHOIS_GEOIP_ACCOUNT, - license_key=app_settings.WHOIS_GEOIP_KEY, - host="geolite.info", - ) - - try: - data = ip_client.city(ip_address=new_ip_address) - - # Catching all possible exceptions raised by the geoip2 client - # and raising them with appropriate messages to be handled by the task - # retry mechanism. - except ( - errors.AddressNotFoundError, - errors.AuthenticationError, - errors.OutOfQueriesError, - errors.PermissionRequiredError, - ) as e: - exc_type = type(e) - message = EXCEPTION_MESSAGES.get(exc_type) - if exc_type is errors.AddressNotFoundError: - message = message.format(ip_address=new_ip_address) - raise exc_type(message) - except requests.RequestException as e: - raise e - - else: - # The attributes are always present in the response, - # but they can be None, so added fallbacks. - address = { - "city": data.city.name or "", - "country": data.country.name or "", - "continent": data.continent.name or "", - "postal": str(data.postal.code or ""), - } - coordinates = Point(data.location.longitude, data.location.latitude, srid=4326) - whois_obj = WHOISInfo( - isp=data.traits.autonomous_system_organization, - asn=data.traits.autonomous_system_number, - timezone=data.location.time_zone, - address=address, - cidr=data.traits.network, - ip_address=new_ip_address, - coordinates=coordinates, + with transaction.atomic(): + device = Device.objects.get(pk=device_pk) + new_ip_address = device.last_ip + WHOISService = device.whois_service + + # If there is existing WHOIS older record then it needs to be updated + whois_obj = WHOISInfo.objects.filter(ip_address=new_ip_address).first() + if whois_obj and not WHOISService.is_older(whois_obj.modified): + return + + fetched_details = WHOISService.process_whois_details(new_ip_address) + whois_obj, update_fields = WHOISService._create_or_update_whois( + fetched_details, whois_obj ) - whois_obj.full_clean() - whois_obj.save() logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") if device._get_organization__config_settings().estimated_location_enabled: + # the estimated location task should not run if old record is updated + # and location related fields are not updated + if update_fields and not any( + i in update_fields for i in ["address", "coordinates"] + ): + return manage_estimated_locations.delay( device_pk=device_pk, ip_address=new_ip_address ) diff --git a/openwisp_controller/config/whois/tests/tests.py b/openwisp_controller/config/whois/tests/tests.py index 82ad9e83b..3accc02c6 100644 --- a/openwisp_controller/config/whois/tests/tests.py +++ b/openwisp_controller/config/whois/tests/tests.py @@ -1,4 +1,5 @@ import importlib +from datetime import timedelta from io import StringIO from unittest import mock @@ -9,6 +10,7 @@ from django.db.models.signals import post_delete, post_save from django.test import TestCase, TransactionTestCase, override_settings, tag from django.urls import reverse +from django.utils import timezone from geoip2 import errors from selenium.webdriver.common.by import By from swapper import load_model @@ -341,7 +343,7 @@ class TestWHOISTransaction( CreateWHOISMixin, WHOISTransactionMixin, TransactionTestCase ): _WHOIS_GEOIP_CLIENT = ( - "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + "openwisp_controller.config.whois.service.geoip2_webservice.Client" ) _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" @@ -584,6 +586,56 @@ def _verify_whois_details(instance, ip_address): # WHOIS related to the device's last_ip should be deleted self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_whois_update(self, mock_client): + connect_whois_handlers() + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + threshold = app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + + whois_obj = self._create_whois_info() + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + + with self.subTest("Test WHOIS update when older than X days for new device"): + mocked_response.traits.autonomous_system_number = 11111 + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip=whois_obj.ip_address) + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(11111)) + + with self.subTest( + "Test WHOIS update when older than X days for existing device" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + device = self._create_device(last_ip="172.217.22.11") + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(11111)) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + mocked_response.traits.autonomous_system_number = 22222 + mock_client.return_value.city.return_value = mocked_response + device.save() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(22222)) + + with self.subTest( + "Test WHOIS update when older than X days for existing device " + "from DeviceChecksum View" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + device = self._create_device(last_ip="172.217.22.11") + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(22222)) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + mocked_response.traits.autonomous_system_number = 33333 + mock_client.return_value.city.return_value = mocked_response + device.save() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj.asn, str(33333)) + # we need to allow the task to propagate exceptions to ensure # `on_failure` method is called and notifications are executed @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) diff --git a/openwisp_controller/config/whois/tests/utils.py b/openwisp_controller/config/whois/tests/utils.py index 7139ae883..03822270a 100644 --- a/openwisp_controller/config/whois/tests/utils.py +++ b/openwisp_controller/config/whois/tests/utils.py @@ -61,10 +61,13 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): org = self._get_org() with self.subTest(f"{task_name} task called when last_ip is public"): - with mock.patch("django.core.cache.cache.set") as mocked_set: + with mock.patch( + "django.core.cache.cache.get", side_effect=[None, org.config_settings] + ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: device = self._create_device(last_ip="172.217.22.14") mocked_task.assert_called() mocked_set.assert_called_once() + mocked_get.assert_called() mocked_task.reset_mock() with self.subTest( @@ -77,7 +80,7 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): device.save() mocked_task.assert_called() mocked_set.assert_not_called() - mocked_get.assert_called_once() + mocked_get.assert_called() mocked_task.reset_mock() with self.subTest(f"{task_name} task not called when last_ip not updated"): @@ -122,7 +125,7 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): mocked_task.reset_mock() with self.subTest( - f"{task_name} task called via DeviceChecksumView for no WHOIS record" + f"{task_name} task not called via DeviceChecksumView for no WHOIS record" ): WHOISInfo.objects.all().delete() device.refresh_from_db() diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py index 4a955810e..c1b0430e9 100644 --- a/openwisp_controller/config/whois/utils.py +++ b/openwisp_controller/config/whois/utils.py @@ -41,10 +41,10 @@ } -def send_whois_task_notification(device_pk, notify_type, actor=None): +def send_whois_task_notification(device, notify_type, actor=None): Device = load_model("config", "Device") - - device = Device.objects.get(pk=device_pk) + if not isinstance(device, Device): + device = Device.objects.get(pk=device) notify_details = MESSAGE_MAP[notify_type] notify.send( sender=actor or device, diff --git a/openwisp_controller/geo/estimated_location/tasks.py b/openwisp_controller/geo/estimated_location/tasks.py index 8ec080509..c900af12c 100644 --- a/openwisp_controller/geo/estimated_location/tasks.py +++ b/openwisp_controller/geo/estimated_location/tasks.py @@ -1,7 +1,6 @@ import logging from celery import shared_task -from django.db import transaction from swapper import load_model from openwisp_controller.config.whois.utils import send_whois_task_notification @@ -9,6 +8,71 @@ logger = logging.getLogger(__name__) +def _handle_attach_existing_location( + device, device_location, ip_address, existing_device_location +): + """ + Helper function to: + 1. Attach existing device's location (same last_ip) to current device, else + 2. Update current location of device using WHOIS data; if it exists, else + 3. Create a new estimated location for the device using WHOIS data. + """ + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + current_location = device_location.location + attached_devices_exists = None + if current_location is not None: + attached_devices_exists = ( + Device.objects.filter(devicelocation__location_id=current_location.pk) + .exclude(pk=device.pk) + .exists() + ) + if existing_device_location: + existing_location = existing_device_location.location + device_location.location = existing_location + device_location.full_clean() + device_location.save() + logger.info( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {ip_address}" + ) + # We need to remove existing estimated location of the device + # if it is not shared + if attached_devices_exists is False: + current_location.delete() + send_whois_task_notification( + device=device, + notify_type="estimated_location_updated", + actor=existing_location, + ) + return + # If existing devices with same last_ip do not have any location + # then we create a new location based on WHOIS data. + whois_obj = WHOISInfo.objects.filter(ip_address=ip_address).first() + if not whois_obj or not whois_obj.coordinates: + logger.warning( + f"Coordinates not available for {device.pk} for IP: {ip_address}." + " Estimated location cannot be determined." + ) + return + + location_defaults = { + **whois_obj._get_defaults_for_estimated_location(), + "organization_id": device.organization_id, + } + # create new location if no location exists for device or the estimated location + # of device is shared. + whois_service = device.whois_service + whois_service._create_or_update_estimated_location( + location_defaults, attached_devices_exists + ) + logger.info( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {ip_address}" + ) + + @shared_task def manage_estimated_locations(device_pk, ip_address): """ @@ -28,140 +92,40 @@ def manage_estimated_locations(device_pk, ip_address): to the user to resolve the conflict manually. """ Device = load_model("config", "Device") - Location = load_model("geo", "Location") - WHOISInfo = load_model("config", "WHOISInfo") DeviceLocation = load_model("geo", "DeviceLocation") - def _create_estimated_location(device_location, location_defaults): - with transaction.atomic(): - location = Location(**location_defaults, is_estimated=True) - location.full_clean() - location.save(_set_estimated=True) - device_location.location = location - device_location.full_clean() - device_location.save() - logger.info( - f"Estimated location saved successfully for {device_pk}" - f" for IP: {ip_address}" - ) - send_whois_task_notification( - device_pk=device_pk, - notify_type="estimated_location_created", - actor=location, - ) - - def _update_or_create_estimated_location( - device_location, whois_obj, attached_devices_exists=False - ): - # Used to update an existing location if it is estimated - # or create a new one if it doesn't exist - if whois_obj and whois_obj.coordinates: - location_defaults = { - **whois_obj._get_defaults_for_estimated_location(), - "organization_id": device.organization_id, - } - if current_location and current_location.is_estimated: - if attached_devices_exists: - # If there are other devices attached to the current location, - # we do not update it, but create a new one. - _create_estimated_location(device_location, location_defaults) - return - update_fields = [] - for attr, value in location_defaults.items(): - if getattr(current_location, attr) != value: - setattr(current_location, attr, value) - update_fields.append(attr) - if update_fields: - current_location.save( - update_fields=update_fields, _set_estimated=True - ) - logger.info( - f"Estimated location saved successfully for {device_pk}" - f" for IP: {ip_address}" - ) - send_whois_task_notification( - device_pk=device_pk, - notify_type="estimated_location_updated", - actor=current_location, - ) - elif not current_location: - # If there is no current location, we create a new one. - _create_estimated_location(device_location, location_defaults) - else: - logger.warning( - f"Coordinates not available for {device_pk} for IP: {ip_address}." - " Estimated location cannot be determined." - ) - return - - def _handle_attach_existing_location( - device, device_location, whois_obj, attached_devices_exists=False - ): - # For handling the case when WHOIS already exists for device's new last_ip - # then we attach the location of the device with same last_ip if it exists. - devices_with_location = ( - Device.objects.select_related("devicelocation") - .filter(organization_id=device.organization_id) - .filter(last_ip=ip_address, devicelocation__location__isnull=False) - .exclude(pk=device_pk) - ) - # If there are multiple devices with same last_ip then we need to inform - # the user to resolve the conflict manually. - if devices_with_location.count() > 1: - send_whois_task_notification( - device_pk=device_pk, notify_type="estimated_location_error" - ) - logger.error( - "Multiple devices with locations found with same " - f"last_ip {ip_address}. Please resolve the conflict manually." - ) - return - first_device = devices_with_location.first() - # If existing devices with same last_ip do not have any location - # then we create a new location based on WHOIS data. - if not first_device: - _update_or_create_estimated_location( - device_location, whois_obj, attached_devices_exists - ) - return - existing_location = first_device.devicelocation.location - # We need to remove any existing estimated location of the device - if current_location and not attached_devices_exists: - current_location.delete() - device_location.location = existing_location - device_location.full_clean() - device_location.save() - logger.info( - f"Estimated location saved successfully for {device_pk}" - f" for IP: {ip_address}" - ) - send_whois_task_notification( - device_pk=device_pk, - notify_type="estimated_location_updated", - actor=existing_location, - ) + device = Device.objects.select_related("devicelocation__location").get(pk=device_pk) - whois_obj = WHOISInfo.objects.filter(ip_address=ip_address).first() - device = ( - Device.objects.select_related("devicelocation__location", "organization") - .only("organization_id", "devicelocation") - .get(pk=device_pk) + devices_with_location = ( + Device.objects.only("devicelocation") + .select_related("devicelocation__location") + .filter(organization_id=device.organization_id) + .filter(last_ip=ip_address, devicelocation__location__isnull=False) + .exclude(pk=device.pk) ) + # multiple devices can have same last_ip in cases like usage of proxy + if devices_with_location.count() > 1: + send_whois_task_notification( + device=device, notify_type="estimated_location_error" + ) + logger.error( + "Multiple devices with locations found with same " + f"last_ip {ip_address}. Please resolve the conflict manually." + ) + return + if not (device_location := getattr(device, "devicelocation", None)): device_location = DeviceLocation(content_object=device) - attached_devices_exists = False - if current_location := device_location.location: - attached_devices_exists = ( - Device.objects.filter(devicelocation__location_id=current_location.pk) - .exclude(pk=device_pk) - .exists() - ) + current_location = device_location.location if not current_location or current_location.is_estimated: + existing_device_location = getattr( + devices_with_location.first(), "devicelocation", None + ) _handle_attach_existing_location( - device, device_location, whois_obj, attached_devices_exists + device, device_location, ip_address, existing_device_location ) else: logger.info( diff --git a/openwisp_controller/geo/estimated_location/tests/tests.py b/openwisp_controller/geo/estimated_location/tests/tests.py index 7b8eb1751..8760a7796 100644 --- a/openwisp_controller/geo/estimated_location/tests/tests.py +++ b/openwisp_controller/geo/estimated_location/tests/tests.py @@ -1,11 +1,13 @@ import contextlib import importlib +from datetime import timedelta from unittest import mock from django.contrib.gis.geos import GEOSGeometry from django.core.exceptions import ImproperlyConfigured, ValidationError from django.test import TestCase, TransactionTestCase, override_settings from django.urls import reverse +from django.utils import timezone from openwisp_notifications.types import unregister_notification_type from swapper import load_model @@ -16,6 +18,7 @@ from ....tests.utils import TestAdminMixin from ...tests.utils import TestGeoMixin from ..handlers import register_estimated_location_notification_types +from ..tasks import manage_estimated_locations from .utils import TestEstimatedLocationMixin Device = load_model("config", "Device") @@ -147,7 +150,7 @@ class TestEstimatedLocationTransaction( TestEstimatedLocationMixin, WHOISTransactionMixin, TransactionTestCase ): _WHOIS_GEOIP_CLIENT = ( - "openwisp_controller.config.whois.tasks.geoip2_webservice.Client" + "openwisp_controller.config.whois.service.geoip2_webservice.Client" ) _ESTIMATED_LOCATION_INFO_LOGGER = ( "openwisp_controller.geo.estimated_location.tasks.logger.info" @@ -174,7 +177,9 @@ def test_estimated_location_task_called( self, mocked_client, mocked_estimated_location_task ): connect_whois_handlers() - mocked_client.return_value.city.return_value = self._mocked_client_response() + mocked_response = self._mocked_client_response() + mocked_client.return_value.city.return_value = mocked_response + threshold = config_app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 self._task_called( mocked_estimated_location_task, task_name="Estimated location" @@ -182,6 +187,8 @@ def test_estimated_location_task_called( Device.objects.all().delete() device = self._create_device() + self._create_config(device=device) + with self.subTest( "Estimated location task called when last_ip has related WhoIsInfo" ): @@ -207,7 +214,6 @@ def test_estimated_location_task_called( ): WHOISInfo.objects.all().delete() self._create_whois_info(ip_address=device.last_ip) - self._create_config(device=device) response = self.client.get( reverse("controller:device_checksum", args=[device.pk]), {"key": device.key}, @@ -217,10 +223,80 @@ def test_estimated_location_task_called( mocked_estimated_location_task.assert_not_called() mocked_estimated_location_task.reset_mock() + with self.subTest( + "Estimate location task not called when address/coordinates not updated" + ): + WHOISInfo.objects.all().delete() + whois_obj = self._create_whois_info(ip_address=device.last_ip) + WHOISInfo.objects.filter(pk=whois_obj.pk).update( + modified=timezone.now() - timedelta(days=threshold) + ) + device.save() + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_not_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimate location task called when address/coordinates updated" + ): + WHOISInfo.objects.all().delete() + whois_obj = self._create_whois_info(ip_address=device.last_ip) + WHOISInfo.objects.filter(pk=whois_obj.pk).update( + modified=timezone.now() - timedelta(days=threshold) + ) + mocked_response.city.name = "New city" + mocked_client.return_value.city.return_value = mocked_response + device.save() + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + mocked_response.city.name = "New city 2" + mocked_client.return_value.city.return_value = mocked_response + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_called() + + mocked_response.location.latitude = 60 + mocked_client.return_value.city.return_value = mocked_response + device.save() + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + mocked_response.location.longitude = 160 + mocked_client.return_value.city.return_value = mocked_response + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_estimated_location_task.assert_called() + mocked_estimated_location_task.reset_mock() + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.send_whois_task_notification" # noqa + ) + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" # noqa + ) + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.manage_estimated_locations.delay" # noqa + ) @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) @mock.patch(_WHOIS_GEOIP_CLIENT) - def test_estimated_location_creation_and_update(self, mock_client, mock_info): + def test_estimated_location_creation_and_update( + self, mock_client, mock_info, _mocked_task, _mocked_notify, _mocked_notify2 + ): connect_whois_handlers() def _verify_location_details(device, mocked_response): @@ -254,6 +330,8 @@ def _verify_location_details(device, mocked_response): with self.subTest("Test Estimated location created when device is created"): device = self._create_device(last_ip="172.217.22.14") + with self.assertNumQueries(14): + manage_estimated_locations(device.pk, device.last_ip) location = device.devicelocation.location mocked_response.ip_address = device.last_ip @@ -274,6 +352,8 @@ def _verify_location_details(device, mocked_response): mocked_response.city.name = "New City" mock_client.return_value.city.return_value = mocked_response device.save() + with self.assertNumQueries(8): + manage_estimated_locations(device.pk, device.last_ip) device.refresh_from_db() location = device.devicelocation.location @@ -297,6 +377,8 @@ def _verify_location_details(device, mocked_response): mock_client.return_value.city.return_value = self._mocked_client_response() device.devicelocation.location.save(_set_estimated=True) device.save() + with self.assertNumQueries(2): + manage_estimated_locations(device.pk, device.last_ip) device.refresh_from_db() location = device.devicelocation.location @@ -315,12 +397,15 @@ def _verify_location_details(device, mocked_response): ): Device.objects.all().delete() device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) mock_info.reset_mock() device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66", last_ip="172.217.22.10", ) + with self.assertNumQueries(8): + manage_estimated_locations(device2.pk, device2.last_ip) self.assertEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk @@ -336,15 +421,20 @@ def _verify_location_details(device, mocked_response): ): Device.objects.all().delete() device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66", last_ip="172.217.22.11", ) + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.reset_mock() old_location = device2.devicelocation.location device2.last_ip = "172.217.22.10" device2.save() + # 3 queries related to notifications cleanup + with self.assertNumQueries(16): + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Estimated location saved successfully for {device2.pk}" f" for IP: {device2.last_ip}" @@ -363,17 +453,21 @@ def _verify_location_details(device, mocked_response): ): Device.objects.all().delete() device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66", last_ip="172.217.22.11", ) + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.reset_mock() old_location = device2.devicelocation.location old_location.is_estimated = False old_location.save() device2.last_ip = "172.217.22.10" device2.save() + with self.assertNumQueries(2): + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Non Estimated location already set for {device2.pk}. Update" f" location manually as per IP: {device2.last_ip}" @@ -392,17 +486,21 @@ def _verify_location_details(device, mocked_response): ): Device.objects.all().delete() device1 = self._create_device(last_ip="172.217.22.10") + manage_estimated_locations(device1.pk, device1.last_ip) device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66", last_ip="172.217.22.10", ) + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.reset_mock() self.assertEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk ) device2.last_ip = "172.217.22.11" device2.save() + with self.assertNumQueries(14): + manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Estimated location saved successfully for {device2.pk}" f" for IP: {device2.last_ip}" @@ -480,6 +578,7 @@ def test_estimate_location_status_remove(self, mock_client): location = device.devicelocation.location self.assertTrue(location.is_estimated) org = self._get_org() + connect_whois_handlers() with self.subTest( "Test Estimated Status unchanged if Estimated feature is disabled" diff --git a/openwisp_controller/geo/templates/admin/geo/location/change_form.html b/openwisp_controller/geo/templates/admin/geo/location/change_form.html index 96b4392e9..af2fc9ceb 100644 --- a/openwisp_controller/geo/templates/admin/geo/location/change_form.html +++ b/openwisp_controller/geo/templates/admin/geo/location/change_form.html @@ -1,4 +1,4 @@ -{% extends "admin/change_form.html" %} +{% extends "admin/django_loci/location_change_form.html" %} {% load i18n admin_urls %} {% block messages %} From 84cd30597327f3fe7d05b19f333400fd78d96b71 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Thu, 26 Feb 2026 16:07:50 -0300 Subject: [PATCH 14/26] [chores] Improved WHOIS & Estimated Location features Fixed multiple issues discovered during testing and review across WHOIS, estimated location, and device IP handling, and consolidated QA and maintenance changes. Key changes: - Ensure clear_last_ip command calls model save() to trigger signals and cache invalidation - Fix WHOIS and estimated location bugs, including shared location recreation and modified timestamp updates - Improve handling of unchanged IPs in DeviceChecksumView - Rename and adjust migrations for estimated location settings - Optimize queries and improve error handling (global HttpError handling) - Improve UI (CSS/JS/templates) and documentation - Improved efficiency of `manage_estimated_locations` - Resolved N+1 query issue in `LocationListCreateView` - Fixed HTTP request inside `transaction.atomic()` - Shifted to Celery `send_task` to retain dependency flow - Removed whois info from device list API to avoid N+1 - Simplified auto-naming of estimated locations - Updated location update description messages - Increased WHOIS refresh threshold to 90 days (~92% reduction in API calls) - WHOIS data refresh now executes on transaction commit - Skips `delete_whois_record` if `initial_ip_address` is None - Fixed late-binding closure issue with `_initial_last_ip` - Error notifications now only sent if `device_pk` is not None - CIDR field length increased to accommodate IPv6 - Added `is_estimated` filter and column to admin - Added `is_estimated` to `LocationSerializer` and API filter - Improved test coverage - Fixed parallel test 401 errors - Several other improvements and fixes for problems found during testing Co-authored-by: DragnEmperor Co-authored-by: Gagan Deep --- docs/index.rst | 2 +- docs/user/estimated-location.rst | 142 ++-- docs/user/intro.rst | 5 + docs/user/rest-api.rst | 8 +- docs/user/settings.rst | 54 +- docs/user/whois.rst | 92 +-- openwisp_controller/config/admin.py | 2 +- openwisp_controller/config/api/serializers.py | 4 +- openwisp_controller/config/base/device.py | 12 +- openwisp_controller/config/base/whois.py | 31 +- .../config/controller/views.py | 3 - .../management/commands/clear_last_ip.py | 44 +- .../config/migrations/0062_whoisinfo.py | 9 +- ...gs_estimated_location_enabled_and_more.py} | 0 openwisp_controller/config/settings.py | 22 +- .../config/static/whois/css/whois.css | 3 - .../config/static/whois/js/whois.js | 24 +- .../templates/admin/config/change_form.html | 7 +- .../config/tests/test_admin.py | 3 + .../config/tests/test_device.py | 10 + openwisp_controller/config/whois/commands.py | 66 ++ openwisp_controller/config/whois/mixins.py | 16 +- .../config/whois/serializers.py | 12 - openwisp_controller/config/whois/service.py | 118 ++-- openwisp_controller/config/whois/tasks.py | 106 ++- .../config/whois/tests/tests.py | 642 ++++++++++++++---- .../config/whois/tests/utils.py | 57 +- openwisp_controller/config/whois/utils.py | 46 +- openwisp_controller/geo/admin.py | 37 +- openwisp_controller/geo/api/serializers.py | 9 +- openwisp_controller/geo/api/views.py | 25 +- openwisp_controller/geo/base/models.py | 56 +- .../geo/estimated_location/handlers.py | 14 +- .../geo/estimated_location/mixins.py | 13 +- .../geo/estimated_location/tasks.py | 70 +- .../geo/estimated_location/tests/tests.py | 522 ++++++++++---- .../geo/estimated_location/tests/utils.py | 19 + .../geo/estimated_location/utils.py | 45 +- .../migrations/0004_location_is_estimated.py | 1 + .../admin/geo/location/change_form.html | 4 +- openwisp_controller/geo/tests/test_api.py | 7 +- requirements.txt | 2 +- ...rganizationconfigsettings_whois_enabled.py | 10 +- .../migrations/0004_location_is_estimated.py | 1 + 44 files changed, 1656 insertions(+), 719 deletions(-) rename openwisp_controller/config/migrations/{0063_organizationconfigsettings_approximate_location_enabled_and_more.py => 0063_organizationconfigsettings_estimated_location_enabled_and_more.py} (100%) create mode 100644 openwisp_controller/config/whois/commands.py diff --git a/docs/index.rst b/docs/index.rst index c6b268f55..8bdb49d88 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,9 +47,9 @@ the OpenWISP architecture. user/vxlan-wireguard.rst user/zerotier.rst user/openvpn.rst - user/subnet-division-rules.rst user/whois.rst user/estimated-location.rst + user/subnet-division-rules.rst user/rest-api.rst user/settings.rst diff --git a/docs/user/estimated-location.rst b/docs/user/estimated-location.rst index 48b3c100f..55c9188da 100644 --- a/docs/user/estimated-location.rst +++ b/docs/user/estimated-location.rst @@ -1,13 +1,19 @@ Estimated Location ================== +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :alt: Estimated location flag + .. important:: The **Estimated Location** feature is **disabled by default**. - Before enabling it, the :doc:`WHOIS Lookup feature ` must be - enabled. Then set - :ref:`OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED` to ``True`` + Before enabling it, the :ref:`WHOIS Lookup feature + ` must be enabled. + + Then set :ref:`OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED` to + ``True``. .. contents:: **Table of contents**: :depth: 1 @@ -16,72 +22,102 @@ Estimated Location Overview -------- -This feature automatically creates or updates a device’s location based on -latitude and longitude information retrieved from the WHOIS Lookup +This feature automatically creates or updates a device's location based on +latitude and longitude information retrieved from the :doc:`whois` feature. -Trigger Conditions ------------------- +It is very useful for those users who have devices scattered across +different geographic regions and would like some help to place the devices +on the map, while being gently reminded to improve the precision of the +location with a direct link for doing so. -This feature is triggered when: +It also significantly reduces the effort required to assign a geographic +location manually when many devices are deployed in large buildings like +schools, offices, hospitals, libraries, etc. Improve the precision of the +estimated location just once and all the other devices sharing the same +public IP will automatically inherit the same location. -- A **fresh WHOIS lookup** is performed for a device. -- Or when a WHOIS record already exists for the device’s IP **and**: +The feature is not useful in the following scenarios: - - The device’s last IP address is **public**. - - WHOIS lookup and Estimated Location is **enabled** for the device’s - organization. +- Most devices are deployed in one single location. +- Most devices are mobile (e.g. moving vehicles). -Behavior --------- +Visibility of Estimated Status +------------------------------ -The system will **attach the already existing matching location** of -another device with same ip to the current device if: +The estimated status of a location is visible in the admin interface in +several ways: -- Only one device is found with that IP and it has a location. -- The current device **has no location** or that location is - **estimated**. +- The location name will mention the *IP address* from which it was + estimated. +- A *warning message* appears at the top of the location list page as in + the image below. -If there are multiple devices with location for the same IP, the system -will **not attach any location** to the current device and a notification -will be sent suggesting the user to manually assign/create a location for -the device. + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/estimated-warning.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/estimated-warning.png + :alt: Estimated location warning -If there is **no matching location**, a new estimated location is created -or the existing one is updated using coordinates from the WHOIS record, -but only if the existing location is estimated. +- The *Is Estimated?* flag is displayed both in the location list page and + in the location detail page, as in the images below. -If two devices share the same IP address and are assigned to the same -location, and the last IP of one of the devices is updated, the system -will create a new estimated location for that device. + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-list.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-list.png + :alt: Estimated location admin list -Visibility of Estimated Status ------------------------------- + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/is-estimated-flag.png + :alt: Estimated location flag + +- The device list page also allows filtering devices which are associated + with estimated locations as shown in the image below. -The estimated status of a location is visible on the location page if the -feature is enabled for the organization. The location admin page also -includes indicators for the estimated status. + .. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/filter-devices-by-estimated-location.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/filter-devices-by-estimated-location.png + :alt: Filter devices associated to estimated locations -- The name of the location will have suffix **(Estimated Location : - )**. -- A warning on top of the page. -- **Is Estimated** field. +Any change to the geographic coordinates of an estimated location will set +the ``is_estimated`` field to ``False``. -Changes to the ``coordinates`` and ``geometry`` of the estimated location -will set the ``is_estimated`` field to ``False`` and remove the -"(Estimated Location)" suffix with IP from the location name. +When manually increasing the precision of estimated locations, it is +highly recommended to also change the auto-generated location name. -In REST API, the field will be visible in the :ref:`Device Location -`, :ref:`Location list +In the REST API, the ``is_estimated`` field is visible in the :ref:`Device +Location `, :ref:`Location list `, :ref:`Location Detail -` and :ref:`Location list (GeoJson) -` if the feature is **enabled**. The field can -also be used for filtering in the location list (including geojson) -endpoints and in the :ref:`Device List `. +` and :ref:`Location list (GeoJSON) +` endpoints if the feature is enabled. The +field can also be used for filtering in the location list endpoints, +including the GeoJSON endpoint, and in the :ref:`Device List +`. + +Triggers and Record Management +------------------------------ + +The feature is triggered automatically when all the following conditions +are met: + +- A WHOIS lookup is performed. +- The last IP is a public IP address. +- Both WHOIS lookup and Estimated Location features are enabled for the + device's organization. + +If no matching location exists, a new estimated location is created using +coordinates from the WHOIS record. If an estimated location already +exists, it will be updated with the new coordinates. + +If another device with the same IP already has a location, the system will +assign the same location for any device having the same IP and not being +assigned to any other location. + +If two devices share the same IP and are assigned to the same location, +and one of them updates its last IP, the system will create a new +estimated location for that device. -Managing Older Estimated Locations ----------------------------------- +When multiple devices with the same IP already have a location assigned +but the locations differ, the system will send a notification to network +administrators asking to manually resolve the ambiguity. -Whenever location related fields in WHOIS records are updated as per -:ref:`Managing WHOIS Older Records `; the location -will also be updated automatically. +When WHOIS records are updated as described in :ref:`the WHOIS Lookup +section `, any related estimated +location will also be updated, if needed and only if the estimated +location has not been manually modified to increase precision. diff --git a/docs/user/intro.rst b/docs/user/intro.rst index 98c4b0abe..1c4b123ae 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -35,6 +35,8 @@ following features: - **VPN management**: automatically provision VPN tunnel configurations, including cryptographic keys and IP addresses, e.g.: :doc:`OpenVPN `, :doc:`WireGuard ` +- :doc:`whois`: display information about the public IP address used by + devices to communicate with OpenWISP - :doc:`import-export` It exposes various :doc:`REST API endpoints `. @@ -96,6 +98,9 @@ The geographic app is based on `django-loci geographic coordinates of the devices, as well as their indoor coordinates on floor plan images. +It also provides an :doc:`estimated-location` feature which automatically +creates or updates device locations based on WHOIS data. + It exposes various :doc:`REST API endpoints `. Subnet Division App diff --git a/docs/user/rest-api.rst b/docs/user/rest-api.rst index fe2efba0e..2ec0b48ed 100644 --- a/docs/user/rest-api.rst +++ b/docs/user/rest-api.rst @@ -80,9 +80,9 @@ information. **Estimated Location Filters** -if :doc:`Estimated Location feature ` is enabled, +If :doc:`Estimated Location feature ` is enabled, devices can be filtered based on the estimated nature of their location -using the ``geo_is_estimated``. +using the ``geo_is_estimated`` filter. **Available filters** @@ -814,7 +814,7 @@ List Locations |estimated_details| -Locations can also be filtered using the ``is_estimated``. +Locations can also be filtered using the ``is_estimated`` filter. **Available filters** @@ -951,7 +951,7 @@ List Locations with Devices Deployed (in GeoJSON Format) |estimated_details| -Locations can also be filtered using the ``is_estimated``. +Locations can also be filtered using the ``is_estimated`` filter. **Available filters** diff --git a/docs/user/settings.rst b/docs/user/settings.rst index cf0d4a4de..aa59425ef 100644 --- a/docs/user/settings.rst +++ b/docs/user/settings.rst @@ -774,8 +774,8 @@ documentation regarding automatic retries for known errors. Allows enabling the optional :doc:`WHOIS Lookup feature `. -.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-setting.png - :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-setting.png +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-setting.png :alt: WHOIS admin setting After enabling this feature, you have to set @@ -796,10 +796,10 @@ After enabling this feature, you have to set ============ ======= **type**: ``str`` -**default**: None +**default**: ``""`` ============ ======= -Maxmind Account ID required for the :doc:`WHOIS Lookup feature `. +MaxMind Account ID required for the :doc:`WHOIS Lookup feature `. .. _openwisp_controller_whois_geoip_key: @@ -808,15 +808,28 @@ Maxmind Account ID required for the :doc:`WHOIS Lookup feature `. ============ ======= **type**: ``str`` -**default**: None +**default**: ``""`` ============ ======= -Maxmind License Key required for the :doc:`WHOIS Lookup feature `. +MaxMind License Key required for the :doc:`WHOIS Lookup feature `. -.. _openwisp_controller_whois_estimated_location_enabled: +.. _openwisp_controller_whois_refresh_threshold_days: + +``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS`` +---------------------------------------------------- -``OPENWISP_CONTROLLER_WHOIS_ESTIMATED_LOCATION_ENABLED`` --------------------------------------------------------- +============ ======= +**type**: ``int`` +**default**: ``90`` +============ ======= + +Specifies the number of days after which the WHOIS information for a +device is considered stale and eligible for refresh. + +.. _openwisp_controller_estimated_location_enabled: + +``OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED`` +-------------------------------------------------- ============ ========= **type**: ``bool`` @@ -826,19 +839,14 @@ Maxmind License Key required for the :doc:`WHOIS Lookup feature `. Allows enabling the optional :doc:`Estimated Location feature `. -.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png - :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-location-setting.png - :alt: Estimated Location setting +.. warning:: -.. _openwisp_controller_whois_refresh_threshold_days: + :ref:`OPENWISP_CONTROLLER_WHOIS_ENABLED + ` must be set to ``True`` before + enabling this feature. Enabling estimated locations while + ``WHOIS_ENABLED=False`` will raise ``ImproperlyConfigured`` at startup + time. -``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS`` ----------------------------------------------------- - -============ ======= -**type**: ``int`` -**default**: ``14`` -============ ======= - -Specifies the number of days after which the WHOIS information for a -device is considered stale and eligible for refresh. +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-setting.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/estimated-locations/admin-setting.png + :alt: Estimated Location setting diff --git a/docs/user/whois.rst b/docs/user/whois.rst index 578bce597..264b2b163 100644 --- a/docs/user/whois.rst +++ b/docs/user/whois.rst @@ -1,12 +1,15 @@ WHOIS Lookup ============ +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :alt: WHOIS admin details + .. important:: The **WHOIS Lookup** feature is **disabled by default**. - To enable it, follow the `setup steps - `_ below. + To enable it, follow the :ref:`controller_setup_whois_lookup` below. .. contents:: **Table of contents**: :depth: 1 @@ -20,6 +23,13 @@ devices to communicate with OpenWISP (via the ``last_ip`` field). It helps identify the geographic location and ISP associated with the IP address, which can be useful for troubleshooting network issues. +.. note:: + + Once WHOIS Lookups are enabled, no manual intervention is needed: + everything is handled automatically, including the refresh of old + data. See :ref:`controller_whois_auto_management` for more + information. + The retrieved information pertains to the Autonomous System (ASN) associated with the device's public IP address and includes: @@ -30,35 +40,10 @@ associated with the device's public IP address and includes: - Timezone of the ASN's registered location - Coordinates (Latitude and Longitude) -Trigger Conditions ------------------- - -A WHOIS lookup is triggered automatically when: - -- A new device is registered. -- An existing device's last IP address is changed. -- A device fetches its checksum. - -However, the lookup will only run if **all** the following conditions are -met: - -- The device is either **newly created** or has a **changed last IP**. -- The device's last IP address is **public**. -- There is **no existing WHOIS record** for that IP. -- WHOIS lookup is **enabled** for the device's organization. - -Managing WHOIS Records ----------------------- - -If a device updates its last IP address, lookup is triggered for the **new -IP** and the **WHOIS record for the old IP** is deleted if no active -devices are associated with that IP address. - .. note:: - When a device with an associated WHOIS record is deleted, its WHOIS - record is automatically removed only if no active devices are - associated with that IP address. + This data also serves as a base for the :doc:`Estimated Location + feature <./estimated-location>`. .. _controller_setup_whois_lookup: @@ -82,7 +67,9 @@ Setup Instructions 6. Restart the application/containers if using ansible-openwisp2 or docker. 7. Run the ``clear_last_ip`` management command to clear the last IP - address of **all active devices across organizations**. + address of any active device which doesn't have WHOIS info yet across + all organizations (which will trigger the WHOIS lookup at the next + config checksum check). - If using ansible-openwisp2 (default directory is /opt/openwisp2, unless changed in Ansible playbook configuration): @@ -90,7 +77,7 @@ Setup Instructions .. code-block:: bash source /opt/openwisp2/env/bin/activate - python /opt/openwisp2/src/manage.py clear_last_ip + python /opt/openwisp2/manage.py clear_last_ip - If using docker: @@ -103,27 +90,40 @@ Viewing WHOIS Lookup Data ------------------------- Once the WHOIS Lookup feature is enabled and WHOIS data is available, the -retrieved details can be viewed in the following locations: +retrieved details can be viewed as follows: -- **Device Admin**: On the device's admin page, the WHOIS data is +- **Device Admin**: on the device's admin page, the WHOIS data is displayed alongside the device's last IP address. -.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-details.png - :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois-admin-details.png +.. image:: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png + :target: https://raw.githubusercontent.com/openwisp/openwisp-controller/docs/docs/1.3/whois/admin-details.png :alt: WHOIS admin details -- **Device REST API**: See WHOIS details in the :ref:`Device List - ` and :ref:`Device Detail ` - responses. +- **Device REST API**: Refer to :ref:`Device List ` and + :ref:`Device Detail `. + +.. _controller_whois_auto_management: -.. _whois_older_records: +Triggers and Record Management +------------------------------ -Managing Older WHOIS Records ----------------------------- +A WHOIS lookup is triggered automatically when: -If a record is older than :ref:`Threshold -`, it will be refreshed -automatically. +- A new device registers and the last IP is a public IP address. +- A device's last IP address changes and is a public IP address. +- A device fetches its checksum **and** either no WHOIS record exists for + the IP or the existing record is older than the :ref:`configured + threshold `. -The update mechanism will be triggered whenever a device is registered or -its last IP changes or fetches its checksum. +The lookup will only run if the device's last IP address is **public** and +WHOIS lookup is **enabled** for the device's organization. + +When a device updates its last IP address, a WHOIS lookup is triggered for +the **new IP** and the **WHOIS record for the old IP** is deleted, unless +any active devices are associated with that IP address. + +.. note:: + + When a device with an associated WHOIS record is deleted, its WHOIS + record is automatically removed, but only if no other active devices + are associated with the same IP address. diff --git a/openwisp_controller/config/admin.py b/openwisp_controller/config/admin.py index af9e20320..7c328a353 100644 --- a/openwisp_controller/config/admin.py +++ b/openwisp_controller/config/admin.py @@ -942,7 +942,7 @@ def get_extra_context(self, pk=None): # passing the whois details to the context to avoid # the need to make an additional request in the js if data := get_whois_info(pk): - ctx["device_whois_details"] = json.dumps(data) + ctx["device_whois_details"] = data return ctx def add_view(self, request, form_url="", extra_context=None): diff --git a/openwisp_controller/config/api/serializers.py b/openwisp_controller/config/api/serializers.py index 9402bbcbf..f07627e60 100644 --- a/openwisp_controller/config/api/serializers.py +++ b/openwisp_controller/config/api/serializers.py @@ -9,7 +9,7 @@ from ...serializers import BaseSerializer from .. import settings as app_settings -from ..whois.mixins import BriefWHOISMixin, WHOISMixin +from ..whois.mixins import WHOISMixin Template = load_model("config", "Template") Vpn = load_model("config", "Vpn") @@ -221,7 +221,7 @@ class DeviceListConfigSerializer(BaseConfigSerializer): templates = FilterTemplatesByOrganization(many=True, write_only=True) -class DeviceListSerializer(BriefWHOISMixin, DeviceConfigSerializer): +class DeviceListSerializer(DeviceConfigSerializer): config = DeviceListConfigSerializer(required=False) class Meta(BaseMeta): diff --git a/openwisp_controller/config/base/device.py b/openwisp_controller/config/base/device.py index 2a2d88fd0..d985ccaab 100644 --- a/openwisp_controller/config/base/device.py +++ b/openwisp_controller/config/base/device.py @@ -119,12 +119,14 @@ class Meta: verbose_name_plural = app_settings.DEVICE_VERBOSE_NAME[1] def __init__(self, *args, **kwargs): + if ( + app_settings.WHOIS_CONFIGURED + and "last_ip" not in self._changed_checked_fields + ): + # Initial value for last_ip is required in WHOIS + # to remove WHOIS info related to that ip address. + self._changed_checked_fields = [*self._changed_checked_fields, "last_ip"] super().__init__(*args, **kwargs) - # Initial value for last_ip is required in WHOIS - # to remove WHOIS info related to that ip address. - if app_settings.WHOIS_CONFIGURED: - self._changed_checked_fields.append("last_ip") - self._set_initial_values_for_changed_checked_fields() def _set_initial_values_for_changed_checked_fields(self): diff --git a/openwisp_controller/config/base/whois.py b/openwisp_controller/config/base/whois.py index 4ae29aebb..6a2abac38 100644 --- a/openwisp_controller/config/base/whois.py +++ b/openwisp_controller/config/base/whois.py @@ -5,8 +5,6 @@ from django.core.exceptions import ValidationError from django.db import models, transaction from django.utils.translation import gettext_lazy as _ -from jsonfield import JSONField -from swapper import load_model from openwisp_utils.base import TimeStampedEditableModel @@ -34,7 +32,7 @@ class AbstractWHOISInfo(TimeStampedEditableModel): help_text=_("Organization for ASN"), ) asn = models.CharField( - max_length=6, + max_length=10, blank=True, help_text=_("ASN"), ) @@ -43,13 +41,13 @@ class AbstractWHOISInfo(TimeStampedEditableModel): blank=True, help_text=_("Time zone"), ) - address = JSONField( + address = models.JSONField( default=dict, help_text=_("Address"), blank=True, ) cidr = models.CharField( - max_length=20, + max_length=49, blank=True, help_text=_("CIDR"), ) @@ -81,8 +79,7 @@ def clean(self): except ValueError as e: raise ValidationError( {"cidr": _("Invalid CIDR format: %(error)s") % {"error": str(e)}} - ) - + ) from e if self.coordinates: if not (-90 <= self.coordinates.y <= 90): raise ValidationError( @@ -104,17 +101,8 @@ def device_whois_info_delete_handler(instance, **kwargs): Delete WHOIS information for a device when the last IP address is removed or when device is deleted. """ - Device = load_model("config", "Device") - last_ip = instance.last_ip - existing_devices = Device.objects.filter(_is_deactivated=False).filter( - last_ip=last_ip - ) - if ( - last_ip - and instance._get_organization__config_settings().whois_enabled - and not existing_devices.exists() - ): + if last_ip and instance._get_organization__config_settings().whois_enabled: transaction.on_commit(lambda: delete_whois_record.delay(last_ip)) # this method is kept here instead of in OrganizationConfigSettings because @@ -155,8 +143,13 @@ def _location_name(self): if address: parts = [part.strip() for part in address.split(",")[:2] if part.strip()] location = ", ".join(parts) - return _(f"{location} (Estimated Location: {self.ip_address})") - return _(f"Estimated Location: {self.ip_address}") + # Use named placeholders so translators receive the template + return _("%(location)s: %(ip)s") % { + "location": location, + "ip": self.ip_address, + } + # Use named placeholder for consistency + return _("Estimated Location: %(ip)s") % {"ip": self.ip_address} def _get_defaults_for_estimated_location(self): """ diff --git a/openwisp_controller/config/controller/views.py b/openwisp_controller/config/controller/views.py index 5e1c9fa10..3dc714c0c 100644 --- a/openwisp_controller/config/controller/views.py +++ b/openwisp_controller/config/controller/views.py @@ -153,9 +153,6 @@ def get(self, request, pk): # updates cache if ip addresses changed if updated: self.update_device_cache(device) - # check if WHOIS Info of device requires update - else: - device.whois_service.update_whois_info() checksum_requested.send( sender=device.__class__, instance=device, request=request ) diff --git a/openwisp_controller/config/management/commands/clear_last_ip.py b/openwisp_controller/config/management/commands/clear_last_ip.py index 7c68a3d4b..55ae9ba20 100644 --- a/openwisp_controller/config/management/commands/clear_last_ip.py +++ b/openwisp_controller/config/management/commands/clear_last_ip.py @@ -1,11 +1,11 @@ -from django.core.management.base import BaseCommand, CommandError -from django.db.models import OuterRef, Subquery -from swapper import load_model +from django.core.management.base import BaseCommand + +from openwisp_controller.config.whois.commands import clear_last_ip_command class Command(BaseCommand): help = ( - "Clears the last IP address (if set) for every active device" + "Clears the last IP address (if set) for active devices without WHOIS records" " across all organizations." ) @@ -17,38 +17,10 @@ def add_arguments(self, parser): dest="interactive", help="Do NOT prompt the user for input of any kind.", ) - return super().add_arguments(parser) def handle(self, *args, **options): - Device = load_model("config", "Device") - WHOISInfo = load_model("config", "WHOISInfo") - - if options["interactive"]: - message = ["\n"] - message.append( - "This will clear last IP of all active devices across organizations!\n" - ) - message.append( - "Are you sure you want to do this?\n\n" - "Type 'yes' to continue, or 'no' to cancel: " - ) - if input("".join(message)).lower() != "yes": - raise CommandError("Operation cancelled by user.") - - devices = Device.objects.filter(_is_deactivated=False).only("last_ip") - # Filter out devices that have WHOIS information for their last IP - devices = devices.exclude(last_ip=None).exclude( - last_ip__in=Subquery( - WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( - "ip_address" - ) - ), + clear_last_ip_command( + stdout=self.stdout, + stderr=self.stderr, + interactive=options["interactive"], ) - - updated_devices = devices.update(last_ip=None) - if updated_devices: - self.stdout.write( - f"Cleared last IP addresses for {updated_devices} active device(s)." - ) - else: - self.stdout.write("No active devices with last IP to clear.") diff --git a/openwisp_controller/config/migrations/0062_whoisinfo.py b/openwisp_controller/config/migrations/0062_whoisinfo.py index 4755b8485..8899f49c3 100644 --- a/openwisp_controller/config/migrations/0062_whoisinfo.py +++ b/openwisp_controller/config/migrations/0062_whoisinfo.py @@ -1,7 +1,6 @@ # Generated by Django 5.2.1 on 2025-06-26 02:13 import django.utils.timezone -import jsonfield.fields import model_utils.fields from django.db import migrations, models @@ -41,7 +40,7 @@ class Migration(migrations.Migration): ), ( "asn", - models.CharField(blank=True, help_text="ASN", max_length=6), + models.CharField(blank=True, help_text="ASN", max_length=10), ), ( "timezone", @@ -49,11 +48,9 @@ class Migration(migrations.Migration): ), ( "address", - jsonfield.fields.JSONField( - blank=True, default=dict, help_text="Address" - ), + models.JSONField(blank=True, default=dict, help_text="Address"), ), - ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=20)), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=49)), ( "isp", models.CharField( diff --git a/openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py b/openwisp_controller/config/migrations/0063_organizationconfigsettings_estimated_location_enabled_and_more.py similarity index 100% rename from openwisp_controller/config/migrations/0063_organizationconfigsettings_approximate_location_enabled_and_more.py rename to openwisp_controller/config/migrations/0063_organizationconfigsettings_estimated_location_enabled_and_more.py diff --git a/openwisp_controller/config/settings.py b/openwisp_controller/config/settings.py index 8bcda862d..6c591c489 100644 --- a/openwisp_controller/config/settings.py +++ b/openwisp_controller/config/settings.py @@ -68,18 +68,26 @@ def get_setting(option, default): "API_TASK_RETRY_OPTIONS", dict(max_retries=5, retry_backoff=True, retry_backoff_max=600, retry_jitter=True), ) -WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 14) -WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", None) -WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", None) +WHOIS_GEOIP_ACCOUNT = get_setting("WHOIS_GEOIP_ACCOUNT", "") +WHOIS_GEOIP_KEY = get_setting("WHOIS_GEOIP_KEY", "") WHOIS_ENABLED = get_setting("WHOIS_ENABLED", False) -WHOIS_CONFIGURED = WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY +WHOIS_CONFIGURED = bool(WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY) if WHOIS_ENABLED and not WHOIS_CONFIGURED: raise ImproperlyConfigured( - "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " - + "when WHOIS_ENABLED is True." + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY must be set " + + "when OPENWISP_CONTROLLER_WHOIS_ENABLED is True." + ) +WHOIS_REFRESH_THRESHOLD_DAYS = get_setting("WHOIS_REFRESH_THRESHOLD_DAYS", 90) +if not ( + isinstance(WHOIS_REFRESH_THRESHOLD_DAYS, int) and WHOIS_REFRESH_THRESHOLD_DAYS > 0 +): + raise ImproperlyConfigured( + "OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS must be a positive integer" ) ESTIMATED_LOCATION_ENABLED = get_setting("ESTIMATED_LOCATION_ENABLED", False) if ESTIMATED_LOCATION_ENABLED and not WHOIS_ENABLED: raise ImproperlyConfigured( - "WHOIS must be enabled before enabling ESTIMATED_LOCATION globally" + "OPENWISP_CONTROLLER_WHOIS_ENABLED must be set to True before " + "setting OPENWISP_CONTROLLER_ESTIMATED_LOCATION_ENABLED to True." ) diff --git a/openwisp_controller/config/static/whois/css/whois.css b/openwisp_controller/config/static/whois/css/whois.css index 44a7b5d1f..2530cf296 100644 --- a/openwisp_controller/config/static/whois/css/whois.css +++ b/openwisp_controller/config/static/whois/css/whois.css @@ -77,9 +77,6 @@ font-weight: 600; cursor: pointer; } -details.whois[open] summary { - padding-bottom: 0 !important; -} .whois summary > div { display: flex; width: 100%; diff --git a/openwisp_controller/config/static/whois/js/whois.js b/openwisp_controller/config/static/whois/js/whois.js index 20f4339f5..7b8c20be7 100644 --- a/openwisp_controller/config/static/whois/js/whois.js +++ b/openwisp_controller/config/static/whois/js/whois.js @@ -1,11 +1,17 @@ "use strict"; -if (typeof gettext === "undefined") { - var gettext = function (word) { +var gettext = + window.gettext || + function (word) { return word; }; +// For XSS prevention +function escapeHtml(text) { + if (text == null) return ""; + const div = document.createElement("div"); + div.textContent = text; + return div.innerHTML; } - django.jQuery(function ($) { const $addForm = $(".add-form"); const $deviceForm = $("#device_form"); @@ -34,8 +40,8 @@ django.jQuery(function ($) { ${gettext("Country")} - ${deviceWHOISDetails.isp} - ${deviceWHOISDetails.address.country} + ${escapeHtml(deviceWHOISDetails.isp)} + ${escapeHtml(deviceWHOISDetails.address.country)}
@@ -46,10 +52,10 @@ django.jQuery(function ($) {
- ${gettext("ASN")}: ${deviceWHOISDetails.asn} - ${gettext("Timezone")}: ${deviceWHOISDetails.timezone} - ${gettext("Address")}: ${deviceWHOISDetails.formatted_address} - ${gettext("CIDR")}: ${deviceWHOISDetails.cidr} + ${gettext("ASN")}: ${escapeHtml(deviceWHOISDetails.asn)} + ${gettext("Timezone")}: ${escapeHtml(deviceWHOISDetails.timezone)} + ${gettext("Address")}: ${escapeHtml(deviceWHOISDetails.formatted_address)} + ${gettext("CIDR")}: ${escapeHtml(deviceWHOISDetails.cidr)}
`, ); diff --git a/openwisp_controller/config/templates/admin/config/change_form.html b/openwisp_controller/config/templates/admin/config/change_form.html index 02536c00f..c4099580e 100644 --- a/openwisp_controller/config/templates/admin/config/change_form.html +++ b/openwisp_controller/config/templates/admin/config/change_form.html @@ -12,6 +12,9 @@ {% endblock %} {% block extrahead %} + {% if device_whois_details %} + {{ device_whois_details|json_script:"device-whois-details" }} + {% endif %} {{ block.super}} diff --git a/openwisp_controller/config/tests/test_admin.py b/openwisp_controller/config/tests/test_admin.py index 3be6f448a..aa9e0ebca 100644 --- a/openwisp_controller/config/tests/test_admin.py +++ b/openwisp_controller/config/tests/test_admin.py @@ -2267,14 +2267,17 @@ def _verify_template_queries(self, config, count): self.assertEqual(response.status_code, 200) # ensuring queries are consistent for different number of templates + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_1(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 1) + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_5(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 5) + @patch.object(app_settings, "WHOIS_CONFIGURED", False) def test_templates_fetch_queries_10(self): config = self._create_config(organization=self._get_org()) self._verify_template_queries(config, 10) diff --git a/openwisp_controller/config/tests/test_device.py b/openwisp_controller/config/tests/test_device.py index f40950440..9603606d3 100644 --- a/openwisp_controller/config/tests/test_device.py +++ b/openwisp_controller/config/tests/test_device.py @@ -531,6 +531,16 @@ def test_device_field_changed_checks(self): with self.assertNumQueries(3): device._check_changed_fields() + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_changed_checked_fields_no_duplicates(self): + """Ensure `_changed_checked_fields` contains `last_ip` only once. + This prevents duplicates if __init__ is invoked multiple times. + """ + device = Device() + # Simulate __init__ being called again on the same instance + device.__init__() + self.assertEqual(device._changed_checked_fields.count("last_ip"), 1) + def test_exceed_organization_device_limit(self): org = self._get_org() org.config_limits.device_limit = 1 diff --git a/openwisp_controller/config/whois/commands.py b/openwisp_controller/config/whois/commands.py new file mode 100644 index 000000000..dafbe92e8 --- /dev/null +++ b/openwisp_controller/config/whois/commands.py @@ -0,0 +1,66 @@ +from django.core.management.base import CommandError +from django.db.models import OuterRef, Subquery +from swapper import load_model + +from openwisp_controller.config import settings as app_settings + + +def clear_last_ip_command(stdout, stderr, interactive=True): + """ + Clears the last IP address (if set) for active devices without WHOIS records + across all organizations. + """ + if not app_settings.WHOIS_CONFIGURED: + raise CommandError( + "WHOIS lookup is not configured. Set " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT " + "and OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY to enable this command." + ) + + Device = load_model("config", "Device") + WHOISInfo = load_model("config", "WHOISInfo") + + if interactive: # pragma: no cover + message = ["\n"] + message.append( + "This will clear the last IP of any active device which doesn't " + "have WHOIS info yet!\n" + ) + message.append( + "Are you sure you want to do this?\n\n" + "Type 'yes' to continue, or 'no' to cancel: " + ) + if input("".join(message)).lower() != "yes": + raise CommandError("Operation cancelled by user.") + + devices = ( + Device.objects.filter(_is_deactivated=False).select_related( + "organization__config_settings" + ) + # include the FK field 'organization' in .only() so the related + # `organization__config_settings` traversal is not deferred + .only("last_ip", "organization", "key") + ) + # Filter out devices that have WHOIS information for their last IP + devices = devices.exclude(last_ip=None).exclude( + last_ip__in=Subquery( + WHOISInfo.objects.filter(ip_address=OuterRef("last_ip")).values( + "ip_address" + ) + ), + ) + # We cannot use a queryset-level update here because it bypasses model save() + # and signals, which are required to properly invalidate related caches + # (e.g. DeviceChecksumView.get_device). To ensure correct behavior and + # future compatibility, each device is saved individually. + updated_devices = 0 + for device in devices.iterator(): + device.last_ip = None + device.save(update_fields=["last_ip"]) + updated_devices += 1 + if updated_devices: + stdout.write( + f"Cleared the last IP addresses for {updated_devices} active device(s)." + ) + else: + stdout.write("No active devices with last IP to clear.") diff --git a/openwisp_controller/config/whois/mixins.py b/openwisp_controller/config/whois/mixins.py index 4e2ee2ea7..3cd70949b 100644 --- a/openwisp_controller/config/whois/mixins.py +++ b/openwisp_controller/config/whois/mixins.py @@ -1,5 +1,5 @@ from .. import settings as app_settings -from .serializers import BriefWHOISSerializer, WHOISSerializer +from .serializers import WHOISSerializer class WHOISMixin: @@ -7,18 +7,14 @@ class WHOISMixin: serializer_class = WHOISSerializer - def to_representation(self, obj): - data = super().to_representation(obj) - if app_settings.WHOIS_CONFIGURED and obj.whois_service.is_whois_enabled: - data["whois_info"] = self.get_whois_info(obj) - return data - def get_whois_info(self, obj): whois_obj = obj.whois_service.get_device_whois_info() if not whois_obj: return None return self.serializer_class(whois_obj).data - -class BriefWHOISMixin(WHOISMixin): - serializer_class = BriefWHOISSerializer + def to_representation(self, obj): + data = super().to_representation(obj) + if app_settings.WHOIS_CONFIGURED and obj.whois_service.is_whois_enabled: + data["whois_info"] = self.get_whois_info(obj) + return data diff --git a/openwisp_controller/config/whois/serializers.py b/openwisp_controller/config/whois/serializers.py index fa859b8e6..028fc9426 100644 --- a/openwisp_controller/config/whois/serializers.py +++ b/openwisp_controller/config/whois/serializers.py @@ -4,18 +4,6 @@ WHOISInfo = load_model("config", "WHOISInfo") -class BriefWHOISSerializer(serializers.ModelSerializer): - """ - Serializer for brief representation of WHOIS model. - """ - - country = serializers.CharField(source="address.country", read_only=True) - - class Meta: - model = WHOISInfo - fields = ("isp", "country", "ip_address") - - class WHOISSerializer(serializers.ModelSerializer): """ Serializer for detailed representation of WHOIS model. diff --git a/openwisp_controller/config/whois/service.py b/openwisp_controller/config/whois/service.py index 71884d52d..d6f72d016 100644 --- a/openwisp_controller/config/whois/service.py +++ b/openwisp_controller/config/whois/service.py @@ -2,36 +2,19 @@ from ipaddress import ip_address as ip_addr import requests +from celery import current_app from django.contrib.gis.geos import Point from django.core.cache import cache from django.db import transaction from django.utils import timezone -from django.utils.translation import gettext as _ from geoip2 import errors from geoip2 import webservice as geoip2_webservice from swapper import load_model from openwisp_controller.config import settings as app_settings -from .tasks import fetch_whois_details, manage_estimated_locations -from .utils import send_whois_task_notification - -EXCEPTION_MESSAGES = { - errors.AddressNotFoundError: _( - "No WHOIS information found for IP address {ip_address}" - ), - errors.AuthenticationError: _( - "Authentication failed for GeoIP2 service. " - "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " - "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." - ), - errors.OutOfQueriesError: _( - "Your account has run out of queries for the GeoIP2 service." - ), - errors.PermissionRequiredError: _( - "Your account does not have permission to access this service." - ), -} +from .tasks import fetch_whois_details +from .utils import EXCEPTION_MESSAGES, send_whois_task_notification class WHOISService: @@ -71,6 +54,7 @@ def is_valid_public_ip_address(ip): try: return ip and ip_addr(ip).is_global except ValueError: + # ip_address() from the stdlib raises ValueError for malformed strings return False @staticmethod @@ -83,11 +67,14 @@ def _get_whois_info_from_db(ip_address): return WHOISInfo.objects.filter(ip_address=ip_address) @staticmethod - def is_older(datetime): + def is_older(dt): """ Check if given datetime is older than the refresh threshold. + Raises TypeError if datetime is naive (not timezone-aware). """ - return (timezone.now() - datetime) >= timedelta( + if not timezone.is_aware(dt): + raise TypeError("datetime must be timezone-aware") + return (timezone.now() - dt) >= timedelta( days=app_settings.WHOIS_REFRESH_THRESHOLD_DAYS ) @@ -125,7 +112,7 @@ def get_org_config_settings(org_id): return org_settings @staticmethod - def check_estimate_location_configured(org_id): + def check_estimated_location_enabled(org_id): if not org_id: return False if not app_settings.WHOIS_CONFIGURED: @@ -174,9 +161,9 @@ def process_whois_details(self, ip_address): message = EXCEPTION_MESSAGES.get(exc_type) if exc_type is errors.AddressNotFoundError: message = message.format(ip_address=ip_address) - raise exc_type(message) - except requests.RequestException as e: - raise e + raise exc_type(message) from e + except requests.RequestException: + raise else: # The attributes are always present in the response, # but they can be None, so added fallbacks. @@ -186,16 +173,23 @@ def process_whois_details(self, ip_address): "continent": data.continent.name or "", "postal": str(data.postal.code or ""), } - coordinates = Point( - data.location.longitude, data.location.latitude, srid=4326 - ) + # Coordinates may be None in WHOIS response + # WHOISInfo.timezone is a non-nullable CharField, so store empty + # string when missing to avoid IntegrityError on save. + time_zone, coordinates = "", None + if location := data.location: + if location.latitude is not None and location.longitude is not None: + coordinates = Point( + location.longitude, location.latitude, srid=4326 + ) + time_zone = location.time_zone or "" return { - "isp": data.traits.autonomous_system_organization, - "asn": data.traits.autonomous_system_number, - "timezone": data.location.time_zone, + "isp": str(data.traits.autonomous_system_organization or ""), + "asn": str(data.traits.autonomous_system_number or ""), + "timezone": time_zone, "address": address, "coordinates": coordinates, - "cidr": data.traits.network, + "cidr": str(data.traits.network or ""), "ip_address": ip_address, } @@ -210,25 +204,35 @@ def _need_whois_lookup(self, new_ip): X days (defined by "WHOIS_REFRESH_THRESHOLD_DAYS"). - WHOIS is disabled in the organization settings. (query from db) """ - # Check cheap conditions first before hitting the database + if not self.is_whois_enabled: + return False if not self.is_valid_public_ip_address(new_ip): return False whois_obj = self._get_whois_info_from_db(ip_address=new_ip).first() if whois_obj and not self.is_older(whois_obj.modified): return False - return self.is_whois_enabled + return True def _need_estimated_location_management(self, new_ip): """ Used to determine if Estimated locations need to be created/updated or not during WHOIS lookup. """ + if not self.is_whois_enabled: + return False if not self.is_valid_public_ip_address(new_ip): return False - if not self.is_whois_enabled: + if not self.is_estimated_location_enabled: return False - return self.is_estimated_location_enabled + return True + + def trigger_estimated_location_task(self, ip_address): + """Helper method to trigger the estimated location task.""" + current_app.send_task( + "whois_estimated_location_task", + kwargs={"device_pk": self.device.pk, "ip_address": ip_address}, + ) def get_device_whois_info(self): """ @@ -238,7 +242,6 @@ def get_device_whois_info(self): ip_address = self.device.last_ip if not (self.is_valid_public_ip_address(ip_address) and self.is_whois_enabled): return None - return self._get_whois_info_from_db(ip_address=ip_address).first() def process_ip_data_and_location(self, force_lookup=False): @@ -249,11 +252,12 @@ def process_ip_data_and_location(self, force_lookup=False): Tasks are triggered on commit to ensure redundant data is not created. """ new_ip = self.device.last_ip + initial_ip = self.device._initial_last_ip if force_lookup or self._need_whois_lookup(new_ip): transaction.on_commit( lambda: fetch_whois_details.delay( device_pk=self.device.pk, - initial_ip_address=self.device._initial_last_ip, + initial_ip_address=initial_ip, ) ) # To handle the case when WHOIS already exists as in that case @@ -261,14 +265,16 @@ def process_ip_data_and_location(self, force_lookup=False): # manage estimated locations. elif self._need_estimated_location_management(new_ip): transaction.on_commit( - lambda: manage_estimated_locations.delay( - device_pk=self.device.pk, ip_address=new_ip + lambda: self.trigger_estimated_location_task( + ip_address=new_ip, ) ) def update_whois_info(self): """ - Update the WHOIS information for the device. + Update existing WHOIS data for the device + when the data is older than + ``OPENWISP_CONTROLLER_WHOIS_REFRESH_THRESHOLD_DAYS``. """ ip_address = self.device.last_ip if not self.is_valid_public_ip_address(ip_address): @@ -277,7 +283,12 @@ def update_whois_info(self): return whois_obj = WHOISService._get_whois_info_from_db(ip_address=ip_address).first() if whois_obj and self.is_older(whois_obj.modified): - fetch_whois_details.delay(device_pk=self.device.pk, initial_ip_address=None) + transaction.on_commit( + lambda: fetch_whois_details.delay( + device_pk=self.device.pk, + initial_ip_address=None, + ) + ) def _create_or_update_whois(self, whois_details, whois_instance=None): """ @@ -285,15 +296,18 @@ def _create_or_update_whois(self, whois_details, whois_instance=None): Returns the updated or created WHOIS instance along with update fields. """ WHOISInfo = load_model("config", "WHOISInfo") - update_fields = [] if whois_instance: for attr, value in whois_details.items(): + # whois_details already coerce to string most values if getattr(whois_instance, attr) != value: update_fields.append(attr) setattr(whois_instance, attr, value) - if update_fields: - whois_instance.save(update_fields=update_fields) + # bump modified time so staleness check + # doesn't re-trigger the lookup + update_fields.append("modified") + whois_instance.modified = timezone.now() + whois_instance.save(update_fields=update_fields) else: whois_instance = WHOISInfo(**whois_details) whois_instance.full_clean() @@ -325,11 +339,11 @@ def _create_or_update_estimated_location( device_location.location = current_location device_location.full_clean() device_location.save() - send_whois_task_notification( - device=self.device, - notify_type="estimated_location_created", - actor=current_location, - ) + send_whois_task_notification( + device=self.device, + notify_type="estimated_location_created", + actor=current_location, + ) elif current_location.is_estimated: update_fields = [] for attr, value in location_defaults.items(): @@ -341,11 +355,9 @@ def _create_or_update_estimated_location( current_location.save( update_fields=update_fields, _set_estimated=True ) - send_whois_task_notification( device=self.device, notify_type="estimated_location_updated", actor=current_location, ) - return current_location diff --git a/openwisp_controller/config/whois/tasks.py b/openwisp_controller/config/whois/tasks.py index 26b0a5037..78955600e 100644 --- a/openwisp_controller/config/whois/tasks.py +++ b/openwisp_controller/config/whois/tasks.py @@ -1,11 +1,11 @@ import logging from celery import shared_task +from django.core.cache import cache from django.db import transaction from geoip2 import errors from swapper import load_model -from openwisp_controller.geo.estimated_location.tasks import manage_estimated_locations from openwisp_utils.tasks import OpenwispCeleryTask from .. import settings as app_settings @@ -23,11 +23,32 @@ class WHOISCeleryRetryTask(OpenwispCeleryTask): # that should trigger a retry of the task. autoretry_for = (errors.HTTPError,) + def on_success(self, retval, task_id, args, kwargs): + """Mark the task as successfully completed.""" + task_key = f"{self.name}_last_operation" + cache.set(task_key, "success", None) + return super().on_success(retval, task_id, args, kwargs) + def on_failure(self, exc, task_id, args, kwargs, einfo): - """Notify the user about the failure of the WHOIS task.""" - device_pk = kwargs.get("device_pk") - send_whois_task_notification(device=device_pk, notify_type="whois_device_error") - logger.error(f"WHOIS lookup failed. Details: {exc}") + """ + Notify the user about the failure of the WHOIS task. + + Notifications are sent only once when task fails for the first time. + Subsequent failures do not trigger notifications until a successful run occurs. + """ + device_pk = kwargs.get("device_pk") or (args[0] if args else None) + if device_pk is not None: + # All exceptions are treated globally to prevent notification spam. + # The cache key is global (not per-device) to avoid spamming admins + # with multiple notifications for the same recurring issue. + task_key = f"{self.name}_last_operation" + last_operation = cache.get(task_key) + if last_operation != "errored": + cache.set(task_key, "errored", None) + send_whois_task_notification( + device=device_pk, notify_type="whois_device_error" + ) + logger.error(f"WHOIS lookup failed. Details: {exc}") return super().on_failure(exc, task_id, args, kwargs, einfo) @@ -45,50 +66,65 @@ def fetch_whois_details(self, device_pk, initial_ip_address): Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") - with transaction.atomic(): - device = Device.objects.get(pk=device_pk) - new_ip_address = device.last_ip - WHOISService = device.whois_service + try: + device = Device.objects.select_related("devicelocation").get(pk=device_pk) + except Device.DoesNotExist: + logger.warning(f"Device {device_pk} not found, skipping WHOIS lookup") + return + new_ip_address = device.last_ip + whois_service = device.whois_service + # If there is existing WHOIS older record then it needs to be updated + whois_obj = WHOISInfo.objects.filter(ip_address=new_ip_address).first() + if whois_obj and not whois_service.is_older(whois_obj.modified): + return + # WARNING: execute HTTP requests before transaction lock is acquired + fetched_details = whois_service.process_whois_details(new_ip_address) - # If there is existing WHOIS older record then it needs to be updated - whois_obj = WHOISInfo.objects.filter(ip_address=new_ip_address).first() - if whois_obj and not WHOISService.is_older(whois_obj.modified): - return - - fetched_details = WHOISService.process_whois_details(new_ip_address) - whois_obj, update_fields = WHOISService._create_or_update_whois( + with transaction.atomic(): + whois_obj, update_fields = whois_service._create_or_update_whois( fetched_details, whois_obj ) logger.info(f"Successfully fetched WHOIS details for {new_ip_address}.") - - if device._get_organization__config_settings().estimated_location_enabled: - # the estimated location task should not run if old record is updated - # and location related fields are not updated - if update_fields and not any( - i in update_fields for i in ["address", "coordinates"] - ): - return - manage_estimated_locations.delay( - device_pk=device_pk, ip_address=new_ip_address + if initial_ip_address: + transaction.on_commit( + # execute synchronously as we're already in a background task + lambda: delete_whois_record(ip_address=initial_ip_address) ) - - # delete WHOIS record for initial IP if no devices are linked to it + if not device._get_organization__config_settings().estimated_location_enabled: + return + # the estimated location task should not run if old record is updated + # and location related fields are not updated + device_location = getattr(device, "devicelocation", None) if ( - not Device.objects.filter(_is_deactivated=False) - .filter(last_ip=initial_ip_address) - .exists() + device_location + and device_location.location + and update_fields + and not any(i in update_fields for i in ["address", "coordinates"]) ): - delete_whois_record(ip_address=initial_ip_address) + return + transaction.on_commit( + lambda: whois_service.trigger_estimated_location_task( + ip_address=new_ip_address, + ) + ) @shared_task -def delete_whois_record(ip_address): +def delete_whois_record(ip_address, force=False): """ Deletes the WHOIS record for the device's last IP address. This is used when the device is deleted or its last IP address is changed. + 'force' parameter is used to delete the record without checking for linked devices. """ + Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") - queryset = WHOISInfo.objects.filter(ip_address=ip_address) - if queryset.exists(): + if force: queryset.delete() + else: + if ( + not Device.objects.filter(_is_deactivated=False) + .filter(last_ip=ip_address) + .exists() + ): + queryset.delete() diff --git a/openwisp_controller/config/whois/tests/tests.py b/openwisp_controller/config/whois/tests/tests.py index 3accc02c6..c3622ec96 100644 --- a/openwisp_controller/config/whois/tests/tests.py +++ b/openwisp_controller/config/whois/tests/tests.py @@ -1,17 +1,23 @@ +import copy import importlib from datetime import timedelta from io import StringIO from unittest import mock +from uuid import uuid4 +from django.conf import settings from django.contrib.gis.geos import Point from django.contrib.staticfiles.testing import StaticLiveServerTestCase +from django.core.cache import cache from django.core.exceptions import ImproperlyConfigured, ValidationError -from django.core.management import call_command +from django.core.management import CommandError, call_command from django.db.models.signals import post_delete, post_save from django.test import TestCase, TransactionTestCase, override_settings, tag from django.urls import reverse from django.utils import timezone from geoip2 import errors +from requests.exceptions import RequestException +from selenium.common.exceptions import UnexpectedAlertPresentException from selenium.webdriver.common.by import By from swapper import load_model @@ -20,16 +26,28 @@ from ....tests.utils import TestAdminMixin from ... import settings as app_settings from ..handlers import connect_whois_handlers +from ..service import WHOISService +from ..tasks import delete_whois_record, fetch_whois_details +from ..utils import get_whois_info, send_whois_task_notification from .utils import CreateWHOISMixin, WHOISTransactionMixin +Config = load_model("config", "Config") Device = load_model("config", "Device") WHOISInfo = load_model("config", "WHOISInfo") Notification = load_model("openwisp_notifications", "Notification") OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") -notification_qs = Notification.objects.all() +MODIFIED_CACHE = copy.deepcopy(settings.CACHES) +# add key_prefix to avoid conflicts in parallel tests +MODIFIED_CACHE["default"]["KEY_PREFIX"] = "whois_failure" +def _notification_qs(): + return Notification.objects.all() + + +# SESSION_ENGINE set to DB to avoid conflicts in parallel tests +@override_settings(SESSION_ENGINE="django.contrib.sessions.backends.db") class TestWHOIS(CreateWHOISMixin, TestAdminMixin, TestCase): # Signals are connected when apps are loaded, # and if WHOIS is Configured all related WHOIS @@ -59,6 +77,8 @@ def _disconnect_signals(self): ) def test_whois_configuration_setting(self): self._disconnect_signals() + self.addCleanup(importlib.reload, app_settings) + # ensure organization exists for admin page checks org = self._get_org() # reload app_settings to apply the overridden settings importlib.reload(app_settings) @@ -66,7 +86,6 @@ def test_whois_configuration_setting(self): with self.subTest("Test Signals not connected when WHOIS_CONFIGURED is False"): # should not connect any handlers since WHOIS_CONFIGURED is False connect_whois_handlers() - assert not any( "device.delete_whois_info" in str(r[0]) for r in post_delete.receivers ) @@ -135,16 +154,23 @@ def test_whois_configuration_setting(self): response = self.client.get(url) self.assertContains(response, 'name="config_settings-0-whois_enabled"') + def test_is_older_requires_timezone_aware(self): + """Verify is_older raises TypeError for naive datetimes.""" + naive_dt = timezone.now().replace(tzinfo=None) + with self.assertRaises(TypeError): + WHOISService.is_older(naive_dt) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) def test_whois_enabled(self): OrganizationConfigSettings.objects.all().delete() device = self._create_device() + with self.subTest( "Test WHOIS fallback when Organization settings do not exist" ): self.assertEqual( device.whois_service.is_whois_enabled, app_settings.WHOIS_ENABLED ) - org_settings_obj = OrganizationConfigSettings( organization=self._get_org(), whois_enabled=True ) @@ -154,30 +180,25 @@ def test_whois_enabled(self): app_settings, "WHOIS_CONFIGURED", False ), self.assertRaises(ValidationError) as context_manager: org_settings_obj.full_clean() - try: - self.assertEqual( - context_manager.exception.message_dict["whois_enabled"][0], - "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " - + "before enabling WHOIS feature.", - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") - - with mock.patch.object(app_settings, "WHOIS_CONFIGURED", True): - org_settings_obj.full_clean() - org_settings_obj.save() + self.assertEqual( + context_manager.exception.message_dict["whois_enabled"][0], + "WHOIS_GEOIP_ACCOUNT and WHOIS_GEOIP_KEY must be set " + + "before enabling WHOIS feature.", + ) with self.subTest("Test setting WHOIS enabled to True"): org_settings_obj.whois_enabled = True + org_settings_obj.full_clean() org_settings_obj.save(update_fields=["whois_enabled"]) org_settings_obj.refresh_from_db(fields=["whois_enabled"]) - self.assertEqual(getattr(org_settings_obj, "whois_enabled"), True) + self.assertTrue(org_settings_obj.whois_enabled) with self.subTest("Test setting WHOIS enabled to False"): org_settings_obj.whois_enabled = False + org_settings_obj.full_clean() org_settings_obj.save(update_fields=["whois_enabled"]) org_settings_obj.refresh_from_db(fields=["whois_enabled"]) - self.assertEqual(getattr(org_settings_obj, "whois_enabled"), False) + self.assertFalse(org_settings_obj.whois_enabled) with self.subTest( "Test setting WHOIS enabled to None fallbacks to global setting" @@ -185,10 +206,11 @@ def test_whois_enabled(self): # reload app_settings to ensure latest settings are applied importlib.reload(app_settings) org_settings_obj.whois_enabled = None + org_settings_obj.full_clean() org_settings_obj.save(update_fields=["whois_enabled"]) org_settings_obj.refresh_from_db(fields=["whois_enabled"]) self.assertEqual( - getattr(org_settings_obj, "whois_enabled"), + org_settings_obj.whois_enabled, app_settings.WHOIS_ENABLED, ) @@ -202,19 +224,11 @@ def test_whois_details_device_api(self): self._login() with self.subTest( - "Device List API has whois_info when WHOIS_CONFIGURED is True" + "Device List API does not have whois_info (removed to avoid N+1)" ): response = self.client.get(reverse("config_api:device_list")) self.assertEqual(response.status_code, 200) - self.assertIn("whois_info", response.data["results"][0]) - self.assertDictEqual( - response.data["results"][0]["whois_info"], - { - "isp": whois_obj.isp, - "country": whois_obj.address["country"], - "ip_address": whois_obj.ip_address, - }, - ) + self.assertNotIn("whois_info", response.data["results"][0]) with self.subTest( "Device Detail API has whois_info when WHOIS_CONFIGURED is True" @@ -232,14 +246,14 @@ def test_whois_details_device_api(self): self.assertEqual(api_whois_info["address"], whois_obj.address) with self.subTest( - "Device List API has whois_info as None when no WHOIS Info exists" + "Device List API does not have whois_info when no WHOIS Info exists" ): device.last_ip = "172.217.22.24" device.save() + device.refresh_from_db() response = self.client.get(reverse("config_api:device_list")) self.assertEqual(response.status_code, 200) - self.assertIn("whois_info", response.data["results"][0]) - self.assertIsNone(response.data["results"][0]["whois_info"]) + self.assertNotIn("whois_info", response.data["results"][0]) with self.subTest( "Device Detail API has whois_info as None when no WHOIS Info exists" @@ -258,7 +272,6 @@ def test_whois_details_device_api(self): response = self.client.get(reverse("config_api:device_list")) self.assertEqual(response.status_code, 200) self.assertNotIn("whois_info", response.data["results"][0]) - with self.subTest( "Device Detail API has no whois_info when WHOIS_CONFIGURED is False" ): @@ -268,20 +281,102 @@ def test_whois_details_device_api(self): self.assertEqual(response.status_code, 200) self.assertNotIn("whois_info", response.data) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_device_list_api_whois_no_nplus1(self): + """ + Test that WHOIS info doesn't cause N+1 queries in device list API. + Should use constant queries regardless of device count. + """ + self._login() + path = reverse("config_api:device_list") + # Create 3 devices with WHOIS info + for i in range(3): + device = self._create_device( + name=f"device{i}", + mac_address=f"00:11:22:33:44:{i:02x}", + last_ip=f"172.217.22.{i + 1}", + ) + WHOISInfo.objects.create( + ip_address=device.last_ip, + isp="Test ISP", + asn="12345", + address={"city": "Test City", "country": "Test Country"}, + cidr=f"172.217.22.{i + 1}/32", + ) + with self.subTest(f"Device List API with WHOIS info: {i}"): + with self.assertNumQueries(4): + response = self.client.get(path) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(response.data["results"]), i + 1) + for result in response.data["results"]: + self.assertNotIn("whois_info", result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) def test_last_ip_management_command(self): out = StringIO() device = self._create_device(last_ip="172.217.22.11") args = ["--noinput"] call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) self.assertIn( - "Cleared last IP addresses for 1 active device(s).", out.getvalue() + "Cleared the last IP addresses for 1 active device(s).", out.getvalue() ) device.refresh_from_db() self.assertIsNone(device.last_ip) - + out.seek(0) + out.truncate(0) call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) self.assertIn("No active devices with last IP to clear.", out.getvalue()) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_last_ip_management_command_queries(self): + out = StringIO() + self._create_device(last_ip="172.217.22.11") + self._create_device( + name="default.test.device2", + last_ip="172.217.22.12", + mac_address="11:22:33:44:55:66", + ) + self._create_device( + name="default.test.device3", + last_ip="172.217.22.13", + mac_address="22:33:44:55:66:77", + ) + self._create_device( + name="default.test.device4", mac_address="66:33:44:55:66:77" + ) + args = ["--noinput"] + # 4 queries (3 for each device's last_ip update) and 1 for fetching devices + with self.assertNumQueries(4): + call_command("clear_last_ip", *args, stdout=out, stderr=StringIO()) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_last_ip_management_command_invalidates_cache(self): + device = self._create_device(last_ip="172.217.22.11") + self._create_config(device=device) + call_command("clear_last_ip", "--noinput", stdout=StringIO()) + device.refresh_from_db() + self.assertEqual(device.last_ip, None) + # We will use the DeviceChecksumView to set the last_ip again to + # the same value to verify that the command invalidates the cache + # and the view is able to send the same IP again. + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR="172.217.22.11", + ) + self.assertEqual(response.status_code, 200) + device.refresh_from_db() + self.assertEqual(device.last_ip, "172.217.22.11") + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", False) + def test_clear_last_ip_command_not_enabled(self): + """Test that clear_last_ip command raises error when WHOIS is not configured.""" + out = StringIO() + err = StringIO() + with self.assertRaises(CommandError) as context: + call_command("clear_last_ip", "--noinput", stdout=out, stderr=err) + self.assertIn("WHOIS lookup is not configured", str(context.exception)) + class TestWHOISInfoModel(CreateWHOISMixin, TestCase): def test_whois_model_fields_validation(self): @@ -290,35 +385,24 @@ def test_whois_model_fields_validation(self): """ with self.assertRaises(ValidationError): self._create_whois_info(isp="a" * 101) - with self.assertRaises(ValidationError) as context_manager: self._create_whois_info(ip_address="127.0.0.1") - try: - self.assertEqual( - context_manager.exception.message_dict["ip_address"][0], - "WHOIS information cannot be created for private IP addresses.", - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") - + self.assertEqual( + context_manager.exception.message_dict["ip_address"][0], + "WHOIS information cannot be created for private IP addresses.", + ) with self.assertRaises(ValidationError): self._create_whois_info(timezone="a" * 36) - with self.assertRaises(ValidationError) as context_manager: self._create_whois_info(cidr="InvalidCIDR") - try: - # Not using assertEqual here because we are adding error message raised by - # ipaddress module to the ValidationError message. - self.assertIn( - "Invalid CIDR format: 'InvalidCIDR'", - context_manager.exception.message_dict["cidr"][0], - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") - + # Not using assertEqual here because we are adding error message raised by + # ipaddress module to the ValidationError message. + self.assertIn( + "Invalid CIDR format: 'InvalidCIDR'", + context_manager.exception.message_dict["cidr"][0], + ) with self.assertRaises(ValidationError): - self._create_whois_info(asn="InvalidASN") - + self._create_whois_info(asn="InvalidASNNumber") # Common validation checks for longitude and latitude coordinates_cases = [ (150.0, 100.0, "Latitude must be between -90 and 90 degrees."), @@ -330,15 +414,14 @@ def test_whois_model_fields_validation(self): with self.assertRaises(ValidationError) as context_manager: point = Point(longitude, latitude, srid=4326) self._create_whois_info(coordinates=point) - try: - self.assertEqual( - context_manager.exception.message_dict["coordinates"][0], - expected_msg, - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") + self.assertEqual( + context_manager.exception.message_dict["coordinates"][0], + expected_msg, + ) +# SESSION_ENGINE set to DB to avoid conflicts in parallel tests +@override_settings(SESSION_ENGINE="django.contrib.sessions.backends.db") class TestWHOISTransaction( CreateWHOISMixin, WHOISTransactionMixin, TransactionTestCase ): @@ -348,27 +431,72 @@ class TestWHOISTransaction( _WHOIS_TASKS_INFO_LOGGER = "openwisp_controller.config.whois.tasks.logger.info" _WHOIS_TASKS_WARN_LOGGER = "openwisp_controller.config.whois.tasks.logger.warning" _WHOIS_TASKS_ERR_LOGGER = "openwisp_controller.config.whois.tasks.logger.error" + _WHOIS_TASK_NAME = "openwisp_controller.config.whois.tasks.fetch_whois_details" def setUp(self): super().setUp() self.admin = self._get_admin() + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_process_whois_details_handles_missing_coordinates(self, mock_client): + """Ensure WHOIS processing tolerates missing coordinates in response.""" + connect_whois_handlers() + mocked_response = self._mocked_client_response() + # simulate missing coordinates + mocked_response.location = None + mock_client.return_value.city.return_value = mocked_response + device = self._create_device(last_ip="172.217.22.14") + whois_details = device.whois_service.process_whois_details(device.last_ip) + self.assertIsNone(whois_details.get("coordinates")) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.delay") def test_whois_task_called(self, mocked_lookup_task): connect_whois_handlers() self._task_called(mocked_lookup_task) + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() + + with self.subTest("WHOIS lookup task called when last_ip is public"): + with mock.patch( + "django.core.cache.cache.get", return_value=None + ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_lookup_task.assert_called() + mocked_set.assert_called_once_with( + f"organization_config_{org.pk}", + org.config_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ) + mocked_get.assert_called() + mocked_lookup_task.reset_mock() + + with self.subTest( + "WHOIS lookup task called when last_ip is changed and is public" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.last_ip = "172.217.22.10" + device.save() + device.refresh_from_db() + mocked_lookup_task.assert_called() + mocked_set.assert_not_called() + mocked_get.assert_called() + mocked_lookup_task.reset_mock() - Device.objects.all().delete() # Clear existing devices - device = self._create_device() with self.subTest( "WHOIS lookup task not called when last_ip has related WhoIsInfo" ): - device.organization.config_settings.whois_enabled = True - device.organization.config_settings.save() device.last_ip = "172.217.22.14" self._create_whois_info(ip_address=device.last_ip) device.save() + device.refresh_from_db() + device.organization.config_settings.refresh_from_db() mocked_lookup_task.assert_not_called() mocked_lookup_task.reset_mock() @@ -409,9 +537,12 @@ def test_whois_multiple_orgs(self, mocked_task): with self.subTest("Test task calls when last_ip is changed and is public"): device1.last_ip = "172.217.22.12" device1.save() + device1.refresh_from_db() mocked_task.assert_called() mocked_task.reset_mock() device2.last_ip = "172.217.22.13" + device2.save() + device2.refresh_from_db() mocked_task.assert_not_called() mocked_task.reset_mock() @@ -474,26 +605,27 @@ def test_whois_creation(self, mock_client, mock_info): connect_whois_handlers() def _verify_whois_details(instance, ip_address): - self.assertEqual(instance.isp, "Google LLC") - self.assertEqual(instance.asn, "15169") - self.assertEqual(instance.timezone, "America/Los_Angeles") + mock_response = self._mocked_client_response() self.assertEqual( - instance.address, - { - "city": "Mountain View", - "country": "United States", - "continent": "North America", - "postal": "94043", - }, + instance.isp, mock_response.traits.autonomous_system_organization ) - self.assertEqual(instance.cidr, "172.217.22.0/24") - self.assertEqual(instance.ip_address, ip_address) self.assertEqual( - instance.formatted_address, - "Mountain View, United States, North America, 94043", + instance.asn, str(mock_response.traits.autonomous_system_number) ) - self.assertEqual(instance.coordinates.x, 150.0) - self.assertEqual(instance.coordinates.y, 50.0) + self.assertEqual(instance.timezone, mock_response.location.time_zone) + mock_address = { + "city": mock_response.city.name, + "country": mock_response.country.name, + "continent": mock_response.continent.name, + "postal": mock_response.postal.code, + } + self.assertEqual(instance.address, mock_address) + self.assertEqual(instance.cidr, "172.217.22.0/24") + self.assertEqual(instance.ip_address, ip_address) + formatted_address = WHOISInfo(address=mock_address).formatted_address + self.assertEqual(instance.formatted_address, formatted_address) + self.assertEqual(instance.coordinates.x, mock_response.location.longitude) + self.assertEqual(instance.coordinates.y, mock_response.location.latitude) # mocking the response from the geoip2 client mock_client.return_value.city.return_value = self._mocked_client_response() @@ -515,6 +647,7 @@ def _verify_whois_details(instance, ip_address): old_ip_address = device.last_ip device.last_ip = "172.217.22.10" device.save() + device.refresh_from_db() self.assertEqual(mock_info.call_count, 1) mock_info.reset_mock() device.refresh_from_db() @@ -522,7 +655,6 @@ def _verify_whois_details(instance, ip_address): _verify_whois_details( device.whois_service.get_device_whois_info(), device.last_ip ) - # details related to old ip address should be deleted self.assertEqual( WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 0 @@ -540,14 +672,13 @@ def _verify_whois_details(instance, ip_address): ) device.last_ip = "172.217.22.11" device.save() + device.refresh_from_db() self.assertEqual(mock_info.call_count, 1) mock_info.reset_mock() device.refresh_from_db() - _verify_whois_details( device.whois_service.get_device_whois_info(), device.last_ip ) - # details related to old ip address should be not be deleted self.assertEqual( WHOISInfo.objects.filter(ip_address=old_ip_address).count(), 1 @@ -561,7 +692,6 @@ def _verify_whois_details(instance, ip_address): device.delete(check_deactivated=False) self.assertEqual(mock_info.call_count, 0) mock_info.reset_mock() - # WHOIS related to the device's last_ip should be deleted self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 1) @@ -582,7 +712,6 @@ def _verify_whois_details(instance, ip_address): device1.delete(check_deactivated=False) self.assertEqual(mock_info.call_count, 0) mock_info.reset_mock() - # WHOIS related to the device's last_ip should be deleted self.assertEqual(WHOISInfo.objects.filter(ip_address=ip_address).count(), 0) @@ -594,7 +723,6 @@ def test_whois_update(self, mock_client): mock_client.return_value.city.return_value = mocked_response threshold = app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 new_time = timezone.now() - timedelta(days=threshold) - whois_obj = self._create_whois_info() WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) @@ -606,7 +734,28 @@ def test_whois_update(self, mock_client): self.assertEqual(whois_obj.asn, str(11111)) with self.subTest( - "Test WHOIS update when older than X days for existing device" + "Test WHOIS update not running for invalid ip address " + "even if older than X days" + ): + Device.objects.all().delete() + WHOISInfo.objects.all().delete() + # to check ip address fallbacks are working if device created by + # bypassing validations + device = Device.objects.create( + name="default.test.device", + organization=self._get_org(), + mac_address=self.TEST_MAC_ADDRESS, + last_ip="InvalidIP", + ) + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj, None) + device.save() + device.refresh_from_db() + whois_obj = device.whois_service.get_device_whois_info() + self.assertEqual(whois_obj, None) + + with self.subTest( + "Test WHOIS update not running if whois disabled even if older than X days" ): Device.objects.all().delete() WHOISInfo.objects.all().delete() @@ -616,16 +765,22 @@ def test_whois_update(self, mock_client): WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) mocked_response.traits.autonomous_system_number = 22222 mock_client.return_value.city.return_value = mocked_response + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save() device.save() + device.refresh_from_db() whois_obj = device.whois_service.get_device_whois_info() - self.assertEqual(whois_obj.asn, str(22222)) + self.assertEqual(whois_obj, None) with self.subTest( - "Test WHOIS update when older than X days for existing device " - "from DeviceChecksum View" + "Test WHOIS update when older than X days for existing device" ): Device.objects.all().delete() WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() device = self._create_device(last_ip="172.217.22.11") whois_obj = device.whois_service.get_device_whois_info() self.assertEqual(whois_obj.asn, str(22222)) @@ -633,9 +788,57 @@ def test_whois_update(self, mock_client): mocked_response.traits.autonomous_system_number = 33333 mock_client.return_value.city.return_value = mocked_response device.save() + device.refresh_from_db() whois_obj = device.whois_service.get_device_whois_info() self.assertEqual(whois_obj.asn, str(33333)) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_device_last_ip_deferred_checks(self): + whois_obj = self._create_whois_info() + self._create_device(last_ip=whois_obj.ip_address) + # Deferred fields remained deferred + device = Device.objects.only("id", "created").first() + with self.subTest("Test deferred fields remained deferred after last_ip check"): + device._check_last_ip() + self.assertTrue(device._is_deferred("last_ip")) + + with self.subTest( + "Test update_whois does not run if last_ip is deferred" + ), mock.patch( + "openwisp_controller.config.whois.service.WHOISService.update_whois_info" + ) as mock_update_whois: + threshold = app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + device._check_last_ip() + mock_update_whois.assert_not_called() + mock_update_whois.reset_mock() + + def test_create_or_update_whois_updates_modified_unchanged_details(self): + """ + Test that _create_or_update_whois updates the modified field + even when the WHOIS details are unchanged. + """ + whois_obj = self._create_whois_info(ip_address="172.217.22.50") + device = self._create_device(last_ip=whois_obj.ip_address) + old_modified = whois_obj.modified + whois_details = { + "isp": whois_obj.isp, + "asn": whois_obj.asn, + "timezone": whois_obj.timezone, + "address": whois_obj.address, + "cidr": whois_obj.cidr, + "coordinates": whois_obj.coordinates, + } + updated_obj, _ = device.whois_service._create_or_update_whois( + whois_details, whois_obj + ) + updated_obj.refresh_from_db() + self.assertGreater( + updated_obj.modified, + old_modified, + ) + # we need to allow the task to propagate exceptions to ensure # `on_failure` method is called and notifications are executed @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) @@ -645,38 +848,213 @@ def test_whois_update(self, mock_client): @mock.patch(_WHOIS_TASKS_INFO_LOGGER) def test_whois_task_failure_notification(self, mock_info, mock_warn, mock_error): def assert_logging_on_exception( - exception, info_calls=0, warn_calls=0, error_calls=1 + exception, info_calls=0, warn_calls=0, error_calls=1, notification_count=1 ): with self.subTest( - f"Test notifications and logging when {exception.__name__} is raised" - ), mock.patch(self._WHOIS_GEOIP_CLIENT, side_effect=exception("test")): + f"Test notification and logging when {exception.__name__} is raised" + ), mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exception("test") Device.objects.all().delete() # Clear existing devices device = self._create_device(last_ip="172.217.22.14") self.assertEqual(mock_info.call_count, info_calls) self.assertEqual(mock_warn.call_count, warn_calls) self.assertEqual(mock_error.call_count, error_calls) - self.assertEqual(notification_qs.count(), 1) - notification = notification_qs.first() - self.assertEqual(notification.actor, device) - self.assertEqual(notification.target, device) - self.assertEqual(notification.level, "error") - self.assertEqual(notification.type, "generic_message") - self.assertIn( - "Failed to fetch WHOIS details for device", - notification.message, - ) - self.assertIn(device.last_ip, notification.rendered_description) - + if notification_count > 0: + notification_qs = _notification_qs() + self.assertEqual(notification_qs.count(), notification_count) + notification = notification_qs.first() + self.assertEqual(notification.actor, device) + self.assertEqual(notification.target, device) + self.assertEqual(notification.level, "error") + self.assertEqual(notification.type, "generic_message") + self.assertIn( + "Failed to fetch WHOIS details for device", + notification.message, + ) + self.assertIn(device.last_ip, notification.rendered_description) mock_info.reset_mock() mock_warn.reset_mock() mock_error.reset_mock() - notification_qs.delete() + _notification_qs().delete() # Test for all possible exceptions that can be raised by the geoip2 client + # Notification are sent only one time when any of the following exceptions + # are raised first time. assert_logging_on_exception(errors.OutOfQueriesError) - assert_logging_on_exception(errors.AddressNotFoundError) - assert_logging_on_exception(errors.AuthenticationError) - assert_logging_on_exception(errors.PermissionRequiredError) + assert_logging_on_exception(errors.AddressNotFoundError, notification_count=0) + assert_logging_on_exception(errors.AuthenticationError, notification_count=0) + assert_logging_on_exception( + errors.PermissionRequiredError, notification_count=0 + ) + assert_logging_on_exception(RequestException, notification_count=0) + cache.clear() + + @override_settings(CACHES=MODIFIED_CACHE) + @override_settings(CELERY_TASK_EAGER_PROPAGATES=False) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_whois_task_failure_cache(self): + permanent_errors = [ + errors.AddressNotFoundError, + errors.OutOfQueriesError, + errors.AuthenticationError, + errors.PermissionRequiredError, + ] + + def trigger_error_and_assert_cached(exc, notification_count=0): + with mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exc("test") + Device.objects.all().delete() + device = self._create_device(last_ip="172.217.22.14") + cache_key = f"{self._WHOIS_TASK_NAME}_last_operation" + self.assertEqual(cache.get(cache_key), "errored") + self.assertEqual(_notification_qs().count(), notification_count) + _notification_qs().delete() + return device + + # simulate that no matter which permanent error is raised first, + # the rest of the permanent errors should use the cache + for first_error in permanent_errors: + with self.subTest(f"Cache populated by {first_error.__name__}"): + cache.clear() + trigger_error_and_assert_cached(first_error, 1) + for subsequent_error in permanent_errors: + if subsequent_error is first_error: + continue + with self.subTest( + f"Cache reused when {subsequent_error.__name__} occurs " + f"after {first_error.__name__}" + ): + trigger_error_and_assert_cached(subsequent_error, 0) + + with self.subTest("Test cache updated on success"), mock.patch( + self._WHOIS_GEOIP_CLIENT + ) as mock_client: + Device.objects.all().delete() + mocked_response = self._mocked_client_response() + mocked_response.location = None + mock_client.return_value.city.return_value = mocked_response + self._create_device(last_ip="172.217.22.14") + cache_key = f"{self._WHOIS_TASK_NAME}_last_operation" + self.assertEqual(cache.get(cache_key), "success") + self.assertEqual(_notification_qs().count(), 0) + cache.clear() + + @override_settings(CELERY_TASK_EAGER_PROPAGATES=True) + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.tasks.fetch_whois_details.retry") + def test_whois_task_retry_mechanism(self, mock_retry): + def assert_retry_on_exception(exception, should_retry): + with self.subTest( + f"Test retry mechanism when {exception.__name__} is raised" + ), mock.patch(self._WHOIS_GEOIP_CLIENT) as mock_client: + mock_client.return_value.city.side_effect = exception("test") + Device.objects.all().delete() + mock_retry.reset_mock() + mock_retry.side_effect = exception("test") + with self.assertRaises(exception): + self._create_device(last_ip="172.217.22.14") + if should_retry: + self.assertEqual(mock_retry.call_count, 1) + assert isinstance(mock_retry.call_args.kwargs["exc"], exception) + else: + self.assertEqual(mock_retry.call_count, 0) + + assert_retry_on_exception(errors.HTTPError, should_retry=True) + assert_retry_on_exception(errors.OutOfQueriesError, should_retry=False) + assert_retry_on_exception(errors.AddressNotFoundError, should_retry=False) + assert_retry_on_exception(errors.AuthenticationError, should_retry=False) + assert_retry_on_exception(errors.PermissionRequiredError, should_retry=False) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_TASKS_WARN_LOGGER) + def test_fetch_whois_details_device_not_found(self, mock_warn): + invalid_pk = uuid4() + fetch_whois_details(device_pk=invalid_pk, initial_ip_address="10.0.0.1") + mock_warn.assert_called_once_with( + f"Device {invalid_pk} not found, skipping WHOIS lookup" + ) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_fetch_whois_details_record_already_exists(self, mock_client): + whois_obj = self._create_whois_info() + device = self._create_device(last_ip=whois_obj.ip_address) + mock_client.return_value.city.return_value = self._mocked_client_response() + fetch_whois_details(device_pk=device.pk, initial_ip_address="10.0.0.1") + mock_client.assert_not_called() + + def test_send_whois_task_notification_with_invalid_device_pk(self): + invalid_pk = uuid4() + result = send_whois_task_notification( + device=invalid_pk, notify_type="whois_device_error" + ) + self.assertIsNone(result) + + def test_delete_whois_record_force(self): + whois_obj = self._create_whois_info() + ip_address = whois_obj.ip_address + delete_whois_record(ip_address=ip_address, force=True) + self.assertFalse(WHOISInfo.objects.filter(ip_address=ip_address).exists()) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_device_whois_disabled(self): + org = self._get_org() + org.config_settings.whois_enabled = False + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.14") + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_with_none_pk(self): + result = get_whois_info(pk=None) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + def test_get_whois_info_device_not_found(self): + result = get_whois_info(pk=uuid4()) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_get_whois_info_not_found(self, mock_client): + mock_client.return_value.city.return_value = self._mocked_client_response() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.all().delete() + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_get_whois_info_returns_data_with_formatted_address(self, mock_client): + mock_client.return_value.city.return_value = self._mocked_client_response() + org = self._get_org() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.filter(ip_address=device.last_ip).delete() + org.config_settings.whois_enabled = True + org.config_settings.save() + WHOISInfo.objects.create( + ip_address=device.last_ip, + address={ + "city": "Mountain View", + "country": "United States", + "postal": "94043", + }, + ) + result = get_whois_info(pk=device.pk) + self.assertIsNotNone(result) + self.assertEqual( + result["formatted_address"], "Mountain View, United States, 94043" + ) + + @mock.patch.object(app_settings, "WHOIS_CONFIGURED", False) + def test_get_whois_info_when_not_configured(self): + device = self._create_device(last_ip="172.217.22.14") + result = get_whois_info(pk=device.pk) + self.assertIsNone(result) @tag("selenium_tests") @@ -707,7 +1085,6 @@ def _assert_no_js_errors(): if cells := row.find_elements(By.TAG_NAME, "td"): self.assertEqual(cells[0].text, whois_obj.isp) self.assertEqual(cells[1].text, whois_obj.address["country"]) - details = self.find_element(By.CSS_SELECTOR, "details.whois") self.web_driver.execute_script( "arguments[0].setAttribute('open','')", details @@ -735,6 +1112,7 @@ def _assert_no_js_errors(): org = self._get_org() org.config_settings.whois_enabled = False org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") @@ -746,8 +1124,44 @@ def _assert_no_js_errors(): org = self._get_org() org.config_settings.whois_enabled = True org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) WHOISInfo.objects.all().delete() self.open(reverse("admin:config_device_change", args=[device.pk])) self.wait_for_invisibility(By.CSS_SELECTOR, "table.whois-table") self.wait_for_invisibility(By.CSS_SELECTOR, "details.whois") _assert_no_js_errors() + + with self.subTest("Check XSS protection in WHOIS details admin view"): + whois_data = { + "ip_address": device.last_ip, + "isp": "", + "timezone": "", + "address": { + "city": "Mountain View", + "country": "", + "continent": "North America", + "postal": "94043", + }, + } + WHOISInfo.objects.all().delete() + self._create_whois_info(**whois_data) + try: + self.open(reverse("admin:config_device_change", args=[device.pk])) + table = self.find_element(By.CSS_SELECTOR, "table.whois-table") + rows = table.find_elements(By.TAG_NAME, "tr") + for row in rows: + if cells := row.find_elements(By.TAG_NAME, "td"): + self.assertIn("onerror", cells[0].text) + self.assertIn("script", cells[1].text) + details = self.find_element(By.CSS_SELECTOR, "details.whois") + self.web_driver.execute_script( + "arguments[0].setAttribute('open','')", details + ) + additional_text = details.find_elements( + By.CSS_SELECTOR, ".additional-text" + ) + self.assertIn("script", additional_text[1].text) + self.assertIn("script", additional_text[2].text) + _assert_no_js_errors() + except UnexpectedAlertPresentException: + self.fail("XSS vulnerability detected in WHOIS details admin view.") diff --git a/openwisp_controller/config/whois/tests/utils.py b/openwisp_controller/config/whois/tests/utils.py index 03822270a..ce36e5a4d 100644 --- a/openwisp_controller/config/whois/tests/utils.py +++ b/openwisp_controller/config/whois/tests/utils.py @@ -27,7 +27,6 @@ def _create_whois_info(self, **kwargs): cidr="172.217.22.0/24", coordinates=Point(150, 50, srid=4326), ) - options.update(kwargs) w = WHOISInfo(**options) w.full_clean() @@ -59,39 +58,27 @@ def _mocked_client_response(): def _task_called(self, mocked_task, task_name="WHOIS lookup"): org = self._get_org() - - with self.subTest(f"{task_name} task called when last_ip is public"): - with mock.patch( - "django.core.cache.cache.get", side_effect=[None, org.config_settings] - ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: - device = self._create_device(last_ip="172.217.22.14") - mocked_task.assert_called() - mocked_set.assert_called_once() - mocked_get.assert_called() - mocked_task.reset_mock() - - with self.subTest( - f"{task_name} task called when last_ip is changed and is public" - ): - with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( - "django.core.cache.cache.set" - ) as mocked_set: - device.last_ip = "172.217.22.10" - device.save() - mocked_task.assert_called() - mocked_set.assert_not_called() - mocked_get.assert_called() + device = self._create_device(last_ip="172.217.22.14") mocked_task.reset_mock() with self.subTest(f"{task_name} task not called when last_ip not updated"): device.name = "default.test.Device2" device.save() + device.refresh_from_db() mocked_task.assert_not_called() mocked_task.reset_mock() with self.subTest(f"{task_name} task not called when last_ip is private"): device.last_ip = "10.0.0.1" device.save() + device.refresh_from_db() + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest(f"{task_name} task not called when last_ip is invalid"): + device.last_ip = "invalid_ip" + device.save() + device.refresh_from_db() mocked_task.assert_not_called() mocked_task.reset_mock() @@ -100,6 +87,7 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): org.config_settings.whois_enabled = False # Invalidates old org config settings cache org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) device = self._create_device(last_ip="172.217.22.14") mocked_task.assert_not_called() mocked_task.reset_mock() @@ -110,6 +98,7 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): org.config_settings.whois_enabled = True # Invalidates old org config settings cache org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) # config is required for checksum view to work device.refresh_from_db() self._create_config(device=device) @@ -125,7 +114,8 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): mocked_task.reset_mock() with self.subTest( - f"{task_name} task not called via DeviceChecksumView for no WHOIS record" + f"{task_name} task not called via DeviceChecksumView " + "if no WHOIS record and IP unchanged" ): WHOISInfo.objects.all().delete() device.refresh_from_db() @@ -145,6 +135,25 @@ def _task_called(self, mocked_task, task_name="WHOIS lookup"): device.refresh_from_db() org.config_settings.whois_enabled = False org.config_settings.save(update_fields=["whois_enabled"]) + org.config_settings.refresh_from_db(fields=["whois_enabled"]) + response = self.client.get( + reverse("controller:device_checksum", args=[device.pk]), + {"key": device.key}, + REMOTE_ADDR=device.last_ip, + ) + self.assertEqual(response.status_code, 200) + mocked_task.assert_not_called() + mocked_task.reset_mock() + + with self.subTest( + f"{task_name} task not called explicitly via DeviceChecksumView for " + "stale records" + ), mock.patch( + "openwisp_controller.config.whois.service.WHOISService.is_older", + return_value=True, + ): + WHOISInfo.objects.all().delete() + self._create_whois_info(ip_address=device.last_ip) response = self.client.get( reverse("controller:device_checksum", args=[device.pk]), {"key": device.key}, diff --git a/openwisp_controller/config/whois/utils.py b/openwisp_controller/config/whois/utils.py index c1b0430e9..1a51a6d44 100644 --- a/openwisp_controller/config/whois/utils.py +++ b/openwisp_controller/config/whois/utils.py @@ -1,4 +1,5 @@ from django.utils.translation import gettext_lazy as _ +from geoip2 import errors from openwisp_notifications.signals import notify from swapper import load_model @@ -14,37 +15,32 @@ ), "description": _("WHOIS details could not be fetched for ip: {ip_address}."), }, - "estimated_location_error": { - "level": "error", - "type": "estimated_location_info", - "message": _( - "Unable to create estimated location for device " - "[{notification.target}]({notification.target_link}). " - "Please assign/create a location manually." - ), - "description": _("Multiple devices found for IP: {ip_address}"), - }, - "estimated_location_created": { - "type": "estimated_location_info", - "description": _("Estimated Location {notification.verb} for IP: {ip_address}"), - }, - "estimated_location_updated": { - "type": "estimated_location_info", - "message": _( - "Estimated location [{notification.actor}]({notification.actor_link})" - " for device" - " [{notification.target}]({notification.target_link})" - " updated successfully." - ), - "description": _("Estimated Location updated for IP: {ip_address}"), - }, +} + +EXCEPTION_MESSAGES = { + errors.AddressNotFoundError: _( + "No WHOIS information found for IP address {ip_address}" + ), + errors.AuthenticationError: _( + "Authentication failed for GeoIP2 service. " + "Check your OPENWISP_CONTROLLER_WHOIS_GEOIP_ACCOUNT and " + "OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY settings." + ), + errors.OutOfQueriesError: _( + "Your account has run out of queries for the GeoIP2 service." + ), + errors.PermissionRequiredError: _( + "Your account does not have permission to access this service." + ), } def send_whois_task_notification(device, notify_type, actor=None): Device = load_model("config", "Device") if not isinstance(device, Device): - device = Device.objects.get(pk=device) + device = Device.objects.filter(pk=device).first() + if not device: + return notify_details = MESSAGE_MAP[notify_type] notify.send( sender=actor or device, diff --git a/openwisp_controller/geo/admin.py b/openwisp_controller/geo/admin.py index ffab0c99b..f08960b8c 100644 --- a/openwisp_controller/geo/admin.py +++ b/openwisp_controller/geo/admin.py @@ -102,26 +102,43 @@ class LocationAdmin(MultitenantAdminMixin, AbstractLocationAdmin): list_select_related = ("organization",) change_form_template = "admin/geo/location/change_form.html" + # Adding is_estimated field via 'get_' methods to allow testing in + # isolation as class level insertions are evaluated at import time + # making it unsuitable as per current testing setup. + def get_list_display(self, request): + list_display = list(super().get_list_display(request)) # makes a copy + if config_app_settings.WHOIS_CONFIGURED: + list_display.insert(list_display.index("created"), "is_estimated") + return list_display + + def get_list_filter(self, request): + list_filter = list(super().get_list_filter(request)) # makes a copy + if config_app_settings.WHOIS_CONFIGURED: + list_filter.append("is_estimated") + return list_filter + def get_fields(self, request, obj=None): - fields = super().get_fields(request, obj) - org_id = obj.organization_id if obj else None - if not WHOISService.check_estimate_location_configured(org_id): - if "is_estimated" in fields: - fields.remove("is_estimated") + fields = list(super().get_fields(request, obj)) # makes a copy + # do not show the is_estimated field on add pages + # or if the estimated location feature is not enabled for the organization + if not obj or not WHOISService.check_estimated_location_enabled( + obj.organization_id + ): + fields.remove("is_estimated") return fields def get_readonly_fields(self, request, obj=None): fields = super().get_readonly_fields(request, obj) - org_id = obj.organization_id if obj else None - if obj and WHOISService.check_estimate_location_configured(org_id): - fields = fields + ("is_estimated",) + if obj and WHOISService.check_estimated_location_enabled(obj.organization_id): + fields = ("is_estimated",) + fields # creates a new tuple return fields def change_view(self, request, object_id, form_url="", extra_context=None): obj = self.get_object(request, object_id) org_id = obj.organization_id if obj else None - estimated_configured = WHOISService.check_estimate_location_configured(org_id) - extra_context = {"estimated_configured": estimated_configured} + estimated_enabled = WHOISService.check_estimated_location_enabled(org_id) + extra_context = extra_context or {} + extra_context["estimated_enabled"] = estimated_enabled return super().change_view(request, object_id, form_url, extra_context) diff --git a/openwisp_controller/geo/api/serializers.py b/openwisp_controller/geo/api/serializers.py index 0626efdab..3d7cc61e3 100644 --- a/openwisp_controller/geo/api/serializers.py +++ b/openwisp_controller/geo/api/serializers.py @@ -12,7 +12,7 @@ from ...serializers import BaseSerializer from ..estimated_location.mixins import ( - EstimatedLocationGeoJsonSerializer, + EstimatedLocationGeoJsonMixin, EstimatedLocationMixin, ) @@ -36,7 +36,7 @@ class Meta: class GeoJsonLocationSerializer( - EstimatedLocationGeoJsonSerializer, gis_serializers.GeoFeatureModelSerializer + EstimatedLocationGeoJsonMixin, gis_serializers.GeoFeatureModelSerializer ): device_count = IntegerField() @@ -166,7 +166,8 @@ def validate(self, data): def to_representation(self, instance): request = self.context["request"] data = super().to_representation(instance) - floorplans = instance.floorplan_set.all().order_by("-modified") + # floorplan_set is already prefetched and ordered in the view + floorplans = instance.floorplan_set.all() floorplan_list = [] for floorplan in floorplans: dict_ = { @@ -232,7 +233,7 @@ def update(self, instance, validated_data): class NestedtLocationSerializer( - EstimatedLocationGeoJsonSerializer, gis_serializers.GeoFeatureModelSerializer + EstimatedLocationGeoJsonMixin, gis_serializers.GeoFeatureModelSerializer ): class Meta: model = Location diff --git a/openwisp_controller/geo/api/views.py b/openwisp_controller/geo/api/views.py index 9b64ca084..ae87c34df 100644 --- a/openwisp_controller/geo/api/views.py +++ b/openwisp_controller/geo/api/views.py @@ -1,5 +1,5 @@ from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.db.models import Count +from django.db.models import Count, Prefetch from django.http import Http404 from django.utils.translation import gettext_lazy as _ from django_filters import rest_framework as filters @@ -52,11 +52,15 @@ class Meta(OrganizationManagedFilter.Meta): model = Location fields = OrganizationManagedFilter.Meta.fields + ["is_mobile", "type"] - def __init__(self, data=None, queryset=None, *, request=None, prefix=None): - super().__init__(data, queryset, request=request, prefix=prefix) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # This is evaluated at runtime, which makes it suited + # For the automated testing strategy we are using. + # Defining this at class definition does not allow flexible testing. if config_app_settings.WHOIS_CONFIGURED: self.filters["is_estimated"] = filters.BooleanFilter( - field_name="is_estimated" + field_name="is_estimated", + label=_("Is geographic location estimated?"), ) @@ -223,8 +227,10 @@ class GeoJsonLocationList( Shows only locations which are assigned to devices. """ - queryset = Location.objects.filter(devicelocation__isnull=False).annotate( - device_count=Count("devicelocation") + queryset = ( + Location.objects.filter(devicelocation__isnull=False) + .annotate(device_count=Count("devicelocation")) + .order_by("-created") ) serializer_class = GeoJsonLocationSerializer pagination_class = GeoJsonLocationListPagination @@ -319,7 +325,12 @@ class FloorPlanDetailView( class LocationListCreateView(ProtectedAPIMixin, generics.ListCreateAPIView): serializer_class = LocationSerializer - queryset = Location.objects.order_by("-created") + queryset = Location.objects.prefetch_related( + Prefetch( + "floorplan_set", + queryset=FloorPlan.objects.order_by("-created"), + ) + ).order_by("-created") pagination_class = ListViewPagination filter_backends = [filters.DjangoFilterBackend] filterset_class = LocationOrganizationFilter diff --git a/openwisp_controller/geo/base/models.py b/openwisp_controller/geo/base/models.py index c1225f838..c26aca6cb 100644 --- a/openwisp_controller/geo/base/models.py +++ b/openwisp_controller/geo/base/models.py @@ -1,8 +1,7 @@ -import re +from typing import ClassVar from django.contrib.gis.db import models from django.core.exceptions import ValidationError -from django.utils.translation import gettext from django.utils.translation import gettext_lazy as _ from django_loci.base.models import ( AbstractFloorPlan, @@ -16,9 +15,14 @@ class BaseLocation(OrgMixin, AbstractLocation): - _changed_checked_fields = ["is_estimated", "address", "geometry"] + _changed_checked_fields: ClassVar[list[str]] = [ + "is_estimated", + "address", + "geometry", + ] is_estimated = models.BooleanField( + _("Is Estimated?"), default=False, help_text=_("Whether the location's coordinates are estimated."), ) @@ -41,12 +45,14 @@ def _set_initial_values_for_changed_checked_fields(self): def clean(self): # Raise validation error if `is_estimated` is True but estimated feature is # disabled. + estimated_status_changed = ( + self._initial_is_estimated is not models.DEFERRED + and self._initial_is_estimated != self.is_estimated + ) if ( - (self._state.adding or self._initial_is_estimated != self.is_estimated) + (self._state.adding or estimated_status_changed) and self.is_estimated - and not WHOISService.check_estimate_location_configured( - self.organization_id - ) + and not WHOISService.check_estimated_location_enabled(self.organization_id) ): raise ValidationError( { @@ -64,27 +70,35 @@ def save(self, *args, _set_estimated=False, **kwargs): Parameters: _set_estimated: Boolean flag to indicate if this save is being performed by the estimated location system. When False (default), - manual edits will clear the estimated status. + manual edits will clear the estimated status (only if estimated location + feature is enabled). *args, **kwargs: Arguments passed to the parent save method. Returns: The result of the parent save method. """ - if WHOISService.check_estimate_location_configured(self.organization_id): - if not _set_estimated and ( - self._initial_address != self.address - or self._initial_geometry != self.geometry - ): + changed_fields = set() + if WHOISService.check_estimated_location_enabled(self.organization_id): + address_changed = ( + self._initial_address is not models.DEFERRED + and self._initial_address != self.address + ) + geometry_changed = ( + self._initial_geometry is not models.DEFERRED + and self._initial_geometry != self.geometry + ) + if not _set_estimated and (address_changed or geometry_changed): self.is_estimated = False - estimated_string = gettext("Estimated Location") - if self.name and estimated_string in self.name: - # remove string starting with "(Estimated Location" - self.name = re.sub( - rf"\s\({estimated_string}.*", "", self.name, flags=re.IGNORECASE - ) - else: + changed_fields = {"is_estimated"} + # Manual changes to is_estimated discarded if feature not enabled + elif self._initial_is_estimated is not models.DEFERRED: self.is_estimated = self._initial_is_estimated - return super().save(*args, **kwargs) + changed_fields = {"is_estimated"} + if update_fields := kwargs.get("update_fields"): + kwargs["update_fields"] = set(update_fields) | changed_fields + result = super().save(*args, **kwargs) + self._set_initial_values_for_changed_checked_fields() + return result class BaseFloorPlan(OrgMixin, AbstractFloorPlan): diff --git a/openwisp_controller/geo/estimated_location/handlers.py b/openwisp_controller/geo/estimated_location/handlers.py index 60ea6f44a..2b66a6d94 100644 --- a/openwisp_controller/geo/estimated_location/handlers.py +++ b/openwisp_controller/geo/estimated_location/handlers.py @@ -4,6 +4,8 @@ from openwisp_controller.config import settings as config_app_settings +from .utils import MESSAGE_MAP + def register_estimated_location_notification_types(): """ @@ -19,18 +21,10 @@ def register_estimated_location_notification_types(): register_notification_type( "estimated_location_info", { + **MESSAGE_MAP["estimated_location_created"], "verbose_name": _("Estimated Location INFO"), "verb": _("created"), - "level": "info", - "email_subject": _( - "Estimated Location: Created for device {notification.target}" - ), - "message": _( - "Estimated location [{notification.actor}]({notification.actor_link})" - " for device" - " [{notification.target}]({notification.target_link})" - " {notification.verb} successfully." - ), + "email_subject": _("Estimated location created for {notification.target}"), "target_link": ( "openwisp_controller.geo.estimated_location.utils" ".get_device_location_notification_target_url" diff --git a/openwisp_controller/geo/estimated_location/mixins.py b/openwisp_controller/geo/estimated_location/mixins.py index f4d2fc93b..222eaa8b7 100644 --- a/openwisp_controller/geo/estimated_location/mixins.py +++ b/openwisp_controller/geo/estimated_location/mixins.py @@ -1,6 +1,8 @@ from openwisp_controller.config.whois.service import WHOISService +# These mixins are required as estimated location is an organization level feature. +# Also, adding it this way makes it read-only as well. class EstimatedLocationMixin: """ Serializer mixin to add estimated location field to the serialized data @@ -9,21 +11,22 @@ class EstimatedLocationMixin: def to_representation(self, obj): data = super().to_representation(obj) - if WHOISService.check_estimate_location_configured(obj.organization_id): + if WHOISService.check_estimated_location_enabled(obj.organization_id): data["is_estimated"] = obj.is_estimated else: data.pop("is_estimated", None) return data -class EstimatedLocationGeoJsonSerializer(EstimatedLocationMixin): +class EstimatedLocationGeoJsonMixin: """ - Extension of EstimatedLocationMixin for GeoJSON serialization. + Serializer mixin to add estimated location field to the serialized GeoJSON data + if the estimated location feature is configured and enabled for the organization. """ def to_representation(self, obj): - data = super(EstimatedLocationMixin, self).to_representation(obj) - if WHOISService.check_estimate_location_configured(obj.organization_id): + data = super().to_representation(obj) + if WHOISService.check_estimated_location_enabled(obj.organization_id): data["properties"]["is_estimated"] = obj.is_estimated else: data["properties"].pop("is_estimated", None) diff --git a/openwisp_controller/geo/estimated_location/tasks.py b/openwisp_controller/geo/estimated_location/tasks.py index c900af12c..58eb863d1 100644 --- a/openwisp_controller/geo/estimated_location/tasks.py +++ b/openwisp_controller/geo/estimated_location/tasks.py @@ -28,7 +28,10 @@ def _handle_attach_existing_location( .exclude(pk=device.pk) .exists() ) - if existing_device_location: + if ( + existing_device_location + and existing_device_location.location != device_location.location + ): existing_location = existing_device_location.location device_location.location = existing_location device_location.full_clean() @@ -61,6 +64,18 @@ def _handle_attach_existing_location( **whois_obj._get_defaults_for_estimated_location(), "organization_id": device.organization_id, } + # Create new location only if location is changed. + if ( + attached_devices_exists + and current_location + and current_location.geometry == location_defaults.get("geometry") + and current_location.name == location_defaults.get("name") + ): + logger.debug( + f"Estimated location unchanged for {device.pk}" + f" for IP: {ip_address}, keeping existing location" + ) + return # create new location if no location exists for device or the estimated location # of device is shared. whois_service = device.whois_service @@ -73,7 +88,7 @@ def _handle_attach_existing_location( ) -@shared_task +@shared_task(name="whois_estimated_location_task") def manage_estimated_locations(device_pk, ip_address): """ Creates/updates estimated location for a device based on the latitude and @@ -82,7 +97,7 @@ def manage_estimated_locations(device_pk, ip_address): the given ip_address. Does not alters the existing location if it is not estimated. - - If the current device has no location or location is estimate, either update + - If the current device has no location or location is estimated, either update to an existing location; if it exists, else - A new location is created if current device has no location, or @@ -94,18 +109,34 @@ def manage_estimated_locations(device_pk, ip_address): Device = load_model("config", "Device") DeviceLocation = load_model("geo", "DeviceLocation") - device = Device.objects.select_related("devicelocation__location").get(pk=device_pk) - - devices_with_location = ( - Device.objects.only("devicelocation") + try: + device = Device.objects.select_related("devicelocation__location").get( + pk=device_pk + ) + except Device.DoesNotExist: + logger.warning( + f"Device {device_pk} not found, skipping manage_estimated_locations" + ) + return + devices_with_location = list( + # "devicelocation" and "devicelocation__location" must be in only() to + # prevent Django from deferring them, which would conflict with + # select_related(). Django raises FieldError if a relation field is + # both deferred and traversed via select_related. + Device.objects.only( + "id", "name", "last_ip", "devicelocation", "devicelocation__location" + ) .select_related("devicelocation__location") - .filter(organization_id=device.organization_id) - .filter(last_ip=ip_address, devicelocation__location__isnull=False) - .exclude(pk=device.pk) + .filter( + organization_id=device.organization_id, + last_ip=ip_address, + devicelocation__location__isnull=False, + ) + # evaluated to LIMIT query, we need to know if there's more than 1 result + .exclude(pk=device.pk)[:2] ) - # multiple devices can have same last_ip in cases like usage of proxy - if devices_with_location.count() > 1: + if len(devices_with_location) > 1: send_whois_task_notification( device=device, notify_type="estimated_location_error" ) @@ -114,16 +145,19 @@ def manage_estimated_locations(device_pk, ip_address): f"last_ip {ip_address}. Please resolve the conflict manually." ) return - + # if device doesn't have a location yet, initialize a draft if not (device_location := getattr(device, "devicelocation", None)): device_location = DeviceLocation(content_object=device) - current_location = device_location.location - if not current_location or current_location.is_estimated: - existing_device_location = getattr( - devices_with_location.first(), "devicelocation", None - ) + # existing device location + try: + existing_device_location = getattr( + devices_with_location[0], "devicelocation", None + ) + # no existing device location + except IndexError: + existing_device_location = None _handle_attach_existing_location( device, device_location, ip_address, existing_device_location ) diff --git a/openwisp_controller/geo/estimated_location/tests/tests.py b/openwisp_controller/geo/estimated_location/tests/tests.py index 8760a7796..70794f44c 100644 --- a/openwisp_controller/geo/estimated_location/tests/tests.py +++ b/openwisp_controller/geo/estimated_location/tests/tests.py @@ -2,6 +2,7 @@ import importlib from datetime import timedelta from unittest import mock +from uuid import uuid4 from django.contrib.gis.geos import GEOSGeometry from django.core.exceptions import ImproperlyConfigured, ValidationError @@ -14,6 +15,7 @@ from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.whois.handlers import connect_whois_handlers from openwisp_controller.config.whois.tests.utils import WHOISTransactionMixin +from openwisp_controller.geo import estimated_location from ....tests.utils import TestAdminMixin from ...tests.utils import TestGeoMixin @@ -21,6 +23,7 @@ from ..tasks import manage_estimated_locations from .utils import TestEstimatedLocationMixin +Config = load_model("config", "Config") Device = load_model("config", "Device") Location = load_model("geo", "Location") DeviceLocation = load_model("geo", "DeviceLocation") @@ -28,7 +31,9 @@ Notification = load_model("openwisp_notifications", "Notification") OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") -notification_qs = Notification.objects.all() + +def _notification_qs(): + return Notification.objects.all() class TestEstimatedLocation(TestAdminMixin, TestCase): @@ -37,7 +42,8 @@ class TestEstimatedLocation(TestAdminMixin, TestCase): OPENWISP_CONTROLLER_WHOIS_GEOIP_KEY="test_key", ) def test_estimated_location_configuration_setting(self): - # reload app_settings to apply the overridden settings + # reload app_settings to apply the overridden + self.addCleanup(importlib.reload, config_app_settings) importlib.reload(config_app_settings) with self.subTest( "ImproperlyConfigured raised when ESTIMATED_LOCATION_ENABLED is True " @@ -59,16 +65,11 @@ def test_estimated_location_configuration_setting(self): org_settings_obj.whois_enabled = False org_settings_obj.estimated_location_enabled = True org_settings_obj.full_clean() - try: - self.assertEqual( - context_manager.exception.message_dict[ - "estimated_location_enabled" - ][0], - "Estimated Location feature requires " - + "WHOIS Lookup feature to be enabled.", - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") + self.assertEqual( + context_manager.exception.message_dict["estimated_location_enabled"][0], + "Estimated Location feature requires " + + "WHOIS Lookup feature to be enabled.", + ) with self.subTest( "Test Estimated Location field visible on admin when " @@ -116,13 +117,10 @@ def test_estimated_location_field(self): org.refresh_from_db() with self.assertRaises(ValidationError) as context_manager: self._create_location(organization=org, is_estimated=True) - try: - self.assertEqual( - context_manager.exception.message_dict["is_estimated"][0], - "Estimated Location feature required to be configured.", - ) - except AssertionError: - self.fail("ValidationError message not equal to expected message.") + self.assertEqual( + context_manager.exception.message_dict["is_estimated"][0], + "Estimated Location feature required to be configured.", + ) @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) def test_estimated_location_admin(self): @@ -131,33 +129,70 @@ def test_estimated_location_admin(self): self.client.force_login(admin) org = self._get_org() location = self._create_location(organization=org, is_estimated=True) - path = reverse("admin:geo_location_change", args=[location.pk]) - response = self.client.get(path) - self.assertContains(response, "field-is_estimated") - self.assertContains( - response, "Whether the location's coordinates are estimated." - ) + change_path = reverse("admin:geo_location_change", args=[location.pk]) + add_path = reverse("admin:geo_location_add") + + with self.subTest( + "is-estimated field visible when estimated location is enabled" + ): + response = self.client.get(change_path) + self.assertContains(response, "field-is_estimated") + self.assertContains( + response, "Whether the location's coordinates are estimated." + ) + with self.subTest( + "is-estimated field not visible in add pages because auto-managed" + ): + response = self.client.get(add_path) + self.assertNotContains(response, "field-is_estimated") + org.config_settings.estimated_location_enabled = False org.config_settings.save() + org.config_settings.refresh_from_db() + + with self.subTest( + "is-estimated field hidden when estimated location is disabled" + ): + response = self.client.get(change_path) + self.assertNotContains(response, "field-is_estimated") + self.assertNotContains( + response, "Whether the location's coordinates are estimated." + ) + with self.subTest( + "double-check is-estimated field is not " + "leaking if estimated location is disabled" + ): + response = self.client.get(add_path) + self.assertNotContains(response, "field-is_estimated") + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False) + def test_estimated_location_admin_add_whois_disabled(self): + admin = self._create_admin() + self.client.force_login(admin) + path = reverse("admin:geo_location_add") response = self.client.get(path) self.assertNotContains(response, "field-is_estimated") - self.assertNotContains( - response, "Whether the location's coordinates are estimated." - ) class TestEstimatedLocationTransaction( - TestEstimatedLocationMixin, WHOISTransactionMixin, TransactionTestCase + TestEstimatedLocationMixin, WHOISTransactionMixin, TestGeoMixin, TransactionTestCase ): + location_model = Location + object_location_model = DeviceLocation + _WHOIS_GEOIP_CLIENT = ( "openwisp_controller.config.whois.service.geoip2_webservice.Client" ) _ESTIMATED_LOCATION_INFO_LOGGER = ( "openwisp_controller.geo.estimated_location.tasks.logger.info" ) + _ESTIMATED_LOCATION_WARNING_LOGGER = ( + "openwisp_controller.geo.estimated_location.tasks.logger.warning" + ) _ESTIMATED_LOCATION_ERROR_LOGGER = ( "openwisp_controller.geo.estimated_location.tasks.logger.error" ) + _WHOIS_TASK_NAME = "openwisp_controller.config.whois.tasks.fetch_whois_details" def setUp(self): super().setUp() @@ -170,7 +205,7 @@ def setUp(self): @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) @mock.patch( - "openwisp_controller.geo.estimated_location.tasks.manage_estimated_locations.delay" # noqa + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 ) @mock.patch(_WHOIS_GEOIP_CLIENT) def test_estimated_location_task_called( @@ -186,8 +221,50 @@ def test_estimated_location_task_called( ) Device.objects.all().delete() - device = self._create_device() - self._create_config(device=device) + WHOISInfo.objects.all().delete() + org = self._get_org() + org.config_settings.whois_enabled = True + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + + with self.subTest("Estimated location task called when last_ip is public"): + with mock.patch( + "django.core.cache.cache.get", return_value=None + ) as mocked_get, mock.patch("django.core.cache.cache.set") as mocked_set: + device = self._create_device(last_ip="172.217.22.14") + mocked_estimated_location_task.assert_called() + expected_cache_set_calls = [ + mock.call( + f"organization_config_{org.pk}", + org.config_settings, + timeout=Config._CHECKSUM_CACHE_TIMEOUT, + ), + mock.call( + f"{self._WHOIS_TASK_NAME}_last_operation", "success", None + ), + ] + mocked_set.assert_has_calls(expected_cache_set_calls) + mocked_get.assert_called() + mocked_estimated_location_task.reset_mock() + + with self.subTest( + "Estimated location task called when last_ip is changed and is public" + ): + with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( + "django.core.cache.cache.set" + ) as mocked_set: + device.last_ip = "172.217.22.10" + device.save() + device.refresh_from_db() + mocked_estimated_location_task.assert_called() + expected_cache_set_calls = [ + mock.call( + f"{self._WHOIS_TASK_NAME}_last_operation", "success", None + ), + ] + mocked_set.assert_has_calls(expected_cache_set_calls) + mocked_get.assert_called() + mocked_estimated_location_task.reset_mock() with self.subTest( "Estimated location task called when last_ip has related WhoIsInfo" @@ -195,12 +272,12 @@ def test_estimated_location_task_called( with mock.patch("django.core.cache.cache.get") as mocked_get, mock.patch( "django.core.cache.cache.set" ) as mocked_set: - device.organization.config_settings.whois_enabled = True - device.organization.config_settings.estimated_location_enabled = True - device.organization.config_settings.save() + self._create_config(device=device) device.last_ip = "172.217.22.14" self._create_whois_info(ip_address=device.last_ip) device.save() + device.refresh_from_db() + device.organization.config_settings.refresh_from_db() mocked_set.assert_not_called() # The cache `get` is called twice, once for `whois_enabled` and # once for `estimated_location_enabled` @@ -210,7 +287,7 @@ def test_estimated_location_task_called( with self.subTest( "Estimated location task not called via DeviceChecksumView when " - "last_ip has no related WhoIsInfo" + "last_ip already has related WhoIsInfo" ): WHOISInfo.objects.all().delete() self._create_whois_info(ip_address=device.last_ip) @@ -231,7 +308,9 @@ def test_estimated_location_task_called( WHOISInfo.objects.filter(pk=whois_obj.pk).update( modified=timezone.now() - timedelta(days=threshold) ) + self._create_object_location(content_object=device) device.save() + device.refresh_from_db() mocked_estimated_location_task.assert_not_called() mocked_estimated_location_task.reset_mock() response = self.client.get( @@ -254,43 +333,17 @@ def test_estimated_location_task_called( mocked_response.city.name = "New city" mocked_client.return_value.city.return_value = mocked_response device.save() - mocked_estimated_location_task.assert_called() - mocked_estimated_location_task.reset_mock() - mocked_response.city.name = "New city 2" - mocked_client.return_value.city.return_value = mocked_response - response = self.client.get( - reverse("controller:device_checksum", args=[device.pk]), - {"key": device.key}, - REMOTE_ADDR=device.last_ip, - ) - self.assertEqual(response.status_code, 200) - mocked_estimated_location_task.assert_called() - - mocked_response.location.latitude = 60 - mocked_client.return_value.city.return_value = mocked_response - device.save() - mocked_estimated_location_task.assert_called() - mocked_estimated_location_task.reset_mock() - mocked_response.location.longitude = 160 - mocked_client.return_value.city.return_value = mocked_response - response = self.client.get( - reverse("controller:device_checksum", args=[device.pk]), - {"key": device.key}, - REMOTE_ADDR=device.last_ip, - ) - self.assertEqual(response.status_code, 200) + device.refresh_from_db() mocked_estimated_location_task.assert_called() mocked_estimated_location_task.reset_mock() @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.service.send_whois_task_notification") @mock.patch( - "openwisp_controller.config.whois.service.send_whois_task_notification" # noqa - ) - @mock.patch( - "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" # noqa + "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" ) @mock.patch( - "openwisp_controller.geo.estimated_location.tasks.manage_estimated_locations.delay" # noqa + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 ) @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) @mock.patch(_WHOIS_GEOIP_CLIENT) @@ -302,19 +355,17 @@ def test_estimated_location_creation_and_update( def _verify_location_details(device, mocked_response): location = device.devicelocation.location mocked_location = mocked_response.location - address = ", ".join( - [ - mocked_response.city.name, - mocked_response.country.name, - mocked_response.continent.name, - mocked_response.postal.code, - ] - ) + mock_address = { + "city": mocked_response.city.name, + "country": mocked_response.country.name, + "continent": mocked_response.continent.name, + "postal": mocked_response.postal.code, + } + address = WHOISInfo(address=mock_address).formatted_address ip_address = mocked_response.ip_address or device.last_ip - location_name = ( - ",".join(address.split(",")[:2]) - + f" (Estimated Location: {ip_address})" - ) + location_name = WHOISInfo( + address=mock_address, ip_address=ip_address + )._location_name self.assertEqual(location.name, location_name) self.assertEqual(location.address, address) self.assertEqual( @@ -330,9 +381,8 @@ def _verify_location_details(device, mocked_response): with self.subTest("Test Estimated location created when device is created"): device = self._create_device(last_ip="172.217.22.14") - with self.assertNumQueries(14): + with self.assertNumQueries(13): manage_estimated_locations(device.pk, device.last_ip) - location = device.devicelocation.location mocked_response.ip_address = device.last_ip self.assertEqual(location.is_estimated, True) @@ -352,9 +402,9 @@ def _verify_location_details(device, mocked_response): mocked_response.city.name = "New City" mock_client.return_value.city.return_value = mocked_response device.save() - with self.assertNumQueries(8): - manage_estimated_locations(device.pk, device.last_ip) device.refresh_from_db() + with self.assertNumQueries(7): + manage_estimated_locations(device.pk, device.last_ip) location = device.devicelocation.location mocked_response.ip_address = device.last_ip @@ -368,18 +418,41 @@ def _verify_location_details(device, mocked_response): ) mock_info.reset_mock() + with self.subTest("Test Estimated location Name when address not available"): + device.last_ip = "172.217.22.11" + mocked_response.city.name = "" + mocked_response.country.name = "" + mocked_response.continent.name = "" + mocked_response.postal.code = "" + mock_client.return_value.city.return_value = mocked_response + device.save() + device.refresh_from_db() + with self.assertNumQueries(7): + manage_estimated_locations(device.pk, device.last_ip) + + location = device.devicelocation.location + mocked_response.ip_address = device.last_ip + self.assertEqual(location.is_estimated, True) + self.assertEqual(location.is_mobile, False) + _verify_location_details(device, mocked_response) + mock_info.assert_called_once_with( + f"Estimated location saved successfully for {device.pk}" + f" for IP: {device.last_ip}" + ) + mock_info.reset_mock() + with self.subTest( "Test Non Estimated Location not updated when last ip is updated" ): mocked_response.ip_address = device.last_ip - device.last_ip = "172.217.22.11" + device.last_ip = "172.217.22.12" device.devicelocation.location.is_estimated = False mock_client.return_value.city.return_value = self._mocked_client_response() device.devicelocation.location.save(_set_estimated=True) device.save() + device.refresh_from_db() with self.assertNumQueries(2): manage_estimated_locations(device.pk, device.last_ip) - device.refresh_from_db() location = device.devicelocation.location self.assertEqual(location.is_estimated, False) @@ -404,7 +477,7 @@ def _verify_location_details(device, mocked_response): mac_address="11:22:33:44:55:66", last_ip="172.217.22.10", ) - with self.assertNumQueries(8): + with self.assertNumQueries(7): manage_estimated_locations(device2.pk, device2.last_ip) self.assertEqual( @@ -433,13 +506,13 @@ def _verify_location_details(device, mocked_response): device2.last_ip = "172.217.22.10" device2.save() # 3 queries related to notifications cleanup - with self.assertNumQueries(16): + device2.refresh_from_db() + with self.assertNumQueries(15): manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Estimated location saved successfully for {device2.pk}" f" for IP: {device2.last_ip}" ) - device2.refresh_from_db() self.assertEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk @@ -466,13 +539,13 @@ def _verify_location_details(device, mocked_response): old_location.save() device2.last_ip = "172.217.22.10" device2.save() + device2.refresh_from_db() with self.assertNumQueries(2): manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Non Estimated location already set for {device2.pk}. Update" f" location manually as per IP: {device2.last_ip}" ) - device2.refresh_from_db() self.assertNotEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk @@ -499,24 +572,118 @@ def _verify_location_details(device, mocked_response): ) device2.last_ip = "172.217.22.11" device2.save() - with self.assertNumQueries(14): + device2.refresh_from_db() + with self.assertNumQueries(13): manage_estimated_locations(device2.pk, device2.last_ip) mock_info.assert_called_once_with( f"Estimated location saved successfully for {device2.pk}" f" for IP: {device2.last_ip}" ) - device2.refresh_from_db() self.assertNotEqual( device1.devicelocation.location.pk, device2.devicelocation.location.pk ) mock_info.reset_mock() @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_ESTIMATED_LOCATION_WARNING_LOGGER) + def test_manage_estimated_location_device_not_found(self, mock_warn): + invalid_pk = uuid4() + manage_estimated_locations(device_pk=invalid_pk, ip_address="10.0.0.1") + mock_warn.assert_called_once_with( + f"Device {invalid_pk} not found, skipping manage_estimated_locations" + ) + + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_estimated_location_handling_on_whois_update( + self, mock_client, mock_send_task + ): + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + threshold = config_app_settings.WHOIS_REFRESH_THRESHOLD_DAYS + 1 + new_time = timezone.now() - timedelta(days=threshold) + org = self._get_org() + org.config_settings.estimated_location_enabled = False + org.config_settings.save() + device = self._create_device(last_ip="172.217.22.10") + with self.assertRaises(Device.devicelocation.RelatedObjectDoesNotExist): + # Accessing devicelocation to verify it doesn't exist (raises if not) + device.devicelocation + org.config_settings.estimated_location_enabled = True + org.config_settings.save() + whois_obj = device.whois_service.get_device_whois_info() + WHOISInfo.objects.filter(pk=whois_obj.pk).update(modified=new_time) + device.name = "test.new.name" + device.save() + device.refresh_from_db() + # location created so can safely access devicelocation + # Accessing devicelocation to verify it exists (raises if not) + device.devicelocation + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_unchanged_whois_data_no_location_recreation(self, mock_client, _): + """Ensure identical WHOIS results do not recreate a shared Location when + devices reuse the same IP.""" + connect_whois_handlers() + mocked_response = self._mocked_client_response() + mock_client.return_value.city.return_value = mocked_response + shared_ip = "20.49.19.19" + device1 = self._create_device( + name="device-a", + mac_address="00:11:22:33:44:55", + last_ip=shared_ip, + ) + device2 = self._create_device( + name="device-b", + mac_address="00:11:22:33:44:66", + last_ip=shared_ip, + ) + original_location = device1.devicelocation.location + self.assertEqual(original_location.pk, device2.devicelocation.location.pk) + location_count = Location.objects.count() + notification_count = _notification_qs().count() + # Clear the last ip for both devices, so setting them again + # will trigger the WHOIS lookup flow. + for device in (device1, device2): + device.last_ip = "" + device.save(update_fields=["last_ip"]) + device.refresh_from_db() + # We set the same shared IP again. This simulates device fetching checksum. + for device in (device1, device2): + device.last_ip = shared_ip + device.save(update_fields=["last_ip"]) + device.refresh_from_db() + # The location object should remain unchanged since the WHOIS data is the same. + self.assertEqual(original_location.pk, device1.devicelocation.location.pk) + self.assertEqual( + device1.devicelocation.location.pk, device2.devicelocation.location.pk + ) + self.assertEqual(Location.objects.count(), location_count) + self.assertTrue(Location.objects.filter(pk=original_location.pk).exists()) + self.assertEqual(_notification_qs().count(), notification_count) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) @mock.patch(_ESTIMATED_LOCATION_INFO_LOGGER) @mock.patch(_ESTIMATED_LOCATION_ERROR_LOGGER) @mock.patch(_WHOIS_GEOIP_CLIENT) - def test_estimated_location_notification(self, mock_client, mock_error, mock_info): + def test_estimated_location_notification( + self, mock_client, mock_error, mock_info, _ + ): def _verify_notification(device, messages, notify_level="info"): + notification_qs = _notification_qs() self.assertEqual(notification_qs.count(), 1) notification = notification_qs.first() device_location = getattr(device, "devicelocation", None) @@ -540,7 +707,7 @@ def _verify_notification(device, messages, notify_level="info"): _verify_notification(device1, messages) with self.subTest("Test Notification for location update"): - notification_qs.delete() + _notification_qs().delete() # will have same location as first device device2 = self._create_device( name="11:22:33:44:55:66", @@ -551,9 +718,7 @@ def _verify_notification(device, messages, notify_level="info"): _verify_notification(device2, messages) with self.subTest("Test Error Notification for conflicting locations"): - device2.last_ip = device1.last_ip - device2.save() - notification_qs.delete() + _notification_qs().delete() mock_info.reset_mock() mock_error.reset_mock() device3 = self._create_device( @@ -566,12 +731,50 @@ def _verify_notification(device, messages, notify_level="info"): f"Multiple devices with locations found with same " f"last_ip {device3.last_ip}. Please resolve the conflict manually." ) - messages = ["Unable to create estimated location for device"] + messages = [ + "Unable to create estimated location for device", + "Please assign/create a location manually.", + ] _verify_notification(device3, messages, "error") @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch("openwisp_controller.config.whois.service.send_whois_task_notification") + @mock.patch( + "openwisp_controller.geo.estimated_location.tasks.send_whois_task_notification" + ) + @mock.patch( + "openwisp_controller.config.whois.service.WHOISService.trigger_estimated_location_task" # noqa: E501 + ) + @mock.patch(_WHOIS_GEOIP_CLIENT) + def test_manage_estimated_locations_no_coordinates_warning( + self, mock_client, _mocked_task, _mocked_notify, _mocked_notify2 + ): + with mock.patch.object(estimated_location.tasks.logger, "warning") as mock_warn: + connect_whois_handlers() + mock_client.return_value.city.return_value = self._mocked_client_response() + device = self._create_device(last_ip="172.217.22.14") + WHOISInfo.objects.filter(ip_address=device.last_ip).delete() + WHOISInfo.objects.create( + ip_address=device.last_ip, + address={ + "city": "Mountain View", + "country": "United States", + "postal": "94043", + }, + ) + manage_estimated_locations(device.pk, device.last_ip) + mock_warn.assert_called_once_with( + f"Coordinates not available for {device.pk} for IP: {device.last_ip}. " + "Estimated location cannot be determined." + ) + + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + @mock.patch( + "openwisp_controller.config.whois.service.current_app.send_task", + side_effect=TestEstimatedLocationMixin.run_task, + ) @mock.patch(_WHOIS_GEOIP_CLIENT) - def test_estimate_location_status_remove(self, mock_client): + def test_estimate_location_status_remove(self, mock_client, _): mocked_response = self._mocked_client_response() mock_client.return_value.city.return_value = mocked_response device = self._create_device(last_ip="172.217.22.10") @@ -585,10 +788,12 @@ def test_estimate_location_status_remove(self, mock_client): ): org.config_settings.estimated_location_enabled = False org.config_settings.save() + org.config_settings.refresh_from_db() location.geometry = GEOSGeometry("POINT(12.512124 41.898903)", srid=4326) location.save() + location.refresh_from_db() self.assertTrue(location.is_estimated) - self.assertIn(f"(Estimated Location: {device.last_ip})", location.name) + self.assertIn(f": {device.last_ip}", location.name) with self.subTest( "Test Estimated Status unchanged if Estimated feature is enabled" @@ -596,10 +801,12 @@ def test_estimate_location_status_remove(self, mock_client): ): org.config_settings.estimated_location_enabled = True org.config_settings.save() + org.config_settings.refresh_from_db() location._set_initial_values_for_changed_checked_fields() location.type = "outdoor" location.is_mobile = True location.save() + location.refresh_from_db() self.assertTrue(location.is_estimated) with self.subTest( @@ -607,9 +814,12 @@ def test_estimate_location_status_remove(self, mock_client): " and desired fields changed" ): location.geometry = GEOSGeometry("POINT(15.512124 45.898903)", srid=4326) - location.save() + location.save(update_fields=["geometry"]) + location.refresh_from_db() self.assertFalse(location.is_estimated) - self.assertNotIn(f"(Estimated Location: {device.last_ip})", location.name) + # Note: Name is no longer automatically cleaned up when + # is_estimated becomes False. Users must update the name manually + # if desired class TestEstimatedLocationFieldFilters( @@ -623,14 +833,6 @@ def setUp(self): admin = self._create_admin() self.client.force_login(admin) - def _create_device_location(self, **kwargs): - options = dict() - options.update(kwargs) - device_location = self.object_location_model(**options) - device_location.full_clean() - device_location.save() - return device_location - @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) def test_estimated_location_api_status_configured(self): org1 = self._get_org() @@ -646,21 +848,20 @@ def test_estimated_location_api_status_configured(self): org2_location = self._create_location(name="org2-location", organization=org2) org1_device = self._create_device(organization=org1) org2_device = self._create_device(organization=org2) - self._create_device_location(content_object=org1_device, location=org1_location) - self._create_device_location(content_object=org2_device, location=org2_location) + self._create_object_location(content_object=org1_device, location=org1_location) + self._create_object_location(content_object=org2_device, location=org2_location) with self.subTest("Test Estimated Location in Locations List"): path = reverse("geo_api:list_location") - with self.assertNumQueries(5): + with self.assertNumQueries(4): response = self.client.get(path) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["count"], 2) self.assertContains(response, org1_location.id) self.assertContains(response, org2_location.id) - location1 = response.data["results"][1] - location2 = response.data["results"][0] - self.assertIn("is_estimated", location1) - self.assertNotIn("is_estimated", location2) + results_by_id = {item["id"]: item for item in response.data["results"]} + self.assertIn("is_estimated", results_by_id[str(org1_location.id)]) + self.assertNotIn("is_estimated", results_by_id[str(org2_location.id)]) with self.subTest("Test Estimated Location in Device Locations List"): path = reverse("geo_api:device_location", args=[org1_device.pk]) @@ -668,7 +869,6 @@ def test_estimated_location_api_status_configured(self): response = self.client.get(path) self.assertEqual(response.status_code, 200) self.assertIn("is_estimated", response.data["location"]["properties"]) - path = reverse("geo_api:device_location", args=[org2_device.pk]) with self.assertNumQueries(4): response = self.client.get(path) @@ -681,20 +881,19 @@ def test_estimated_location_api_status_configured(self): response = self.client.get(path) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["count"], 2) - for i in response.data["features"]: - if i["id"] == org1_location.id: - self.assertIn("is_estimated", i["properties"]) - self.assertTrue(i["properties"]["is_estimated"]) - elif i["id"] == org2_location.id: - self.assertNotIn("is_estimated", i["properties"]) - self.assertFalse(i["properties"]["is_estimated"]) + results_by_id = {item["id"]: item for item in response.data["features"]} + location1_result = results_by_id[str(org1_location.id)] + location2_result = results_by_id[str(org2_location.id)] + self.assertIn("is_estimated", location1_result["properties"]) + self.assertTrue(location1_result["properties"]["is_estimated"]) + self.assertNotIn("is_estimated", location2_result["properties"]) @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False) def test_estimated_location_api_status_not_configured(self): org = self._get_org() location = self._create_location(name="org1-location", organization=org) device = self._create_device(organization=org) - self._create_device_location(content_object=device, location=location) + self._create_object_location(content_object=device, location=location) with self.subTest("Test Estimated status not in Locations List"): path = reverse("geo_api:list_location") @@ -721,7 +920,6 @@ def test_estimated_location_api_status_not_configured(self): self.assertEqual(response.data["count"], 1) location_features = response.data["features"][0] self.assertNotIn("is_estimated", location_features["properties"]) - self.assertFalse(location_features["properties"].get("is_estimated", False)) @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) def test_estimated_location_filter_list_api(self): @@ -732,13 +930,15 @@ def test_estimated_location_filter_list_api(self): location2 = self._create_location( name="location2", is_estimated=False, organization=org ) + location3 = self._create_location( + name="location3", is_estimated=False, organization=org + ) device1 = self._create_device() device2 = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" ) - self._create_device_location(content_object=device1, location=location1) - self._create_device_location(content_object=device2, location=location2) - + self._create_object_location(content_object=device1, location=location1) + self._create_object_location(content_object=device2, location=location2) path = reverse("geo_api:list_location") with self.subTest( @@ -751,16 +951,17 @@ def test_estimated_location_filter_list_api(self): self.assertEqual(response.data["count"], 1) self.assertContains(response, location1.id) self.assertNotContains(response, location2.id) + self.assertNotContains(response, location3.id) with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): with self.subTest( "Test Estimated Location filter not available in location list " "when WHOIS not configured" ): - with self.assertNumQueries(5): + with self.assertNumQueries(4): response = self.client.get(path, {"is_estimated": True}) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data["count"], 2) + self.assertEqual(response.data["count"], 3) self.assertContains(response, location1.id) self.assertContains(response, location2.id) @@ -789,6 +990,39 @@ def test_estimated_location_filter_list_api(self): self.assertContains(response, device1.id) self.assertContains(response, device2.id) + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) + def test_location_admin_estimated_field(self): + org = self._get_org() + estimated_location = self._create_location( + name="location1", is_estimated=True, organization=org + ) + estimated_device = self._create_device() + self._create_object_location( + content_object=estimated_device, location=estimated_location + ) + path = reverse("admin:geo_location_changelist") + response = self.client.get(path) + + with self.subTest("Test location Admin estimated field displayed"): + self.assertContains(response, "column-is_estimated") + + with self.subTest("Test location admin estimated field sorting enabled"): + self.assertContains(response, "sortable column-is_estimated") + + with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): + response = self.client.get(path) + with self.subTest( + "Test location admin estimated field not displayed " + "when WHOIS not configured" + ): + self.assertNotContains(response, "column-is_estimated") + + with self.subTest( + "Test location admin estimated field sorting not enabled " + "when WHOIS not configured" + ): + self.assertNotContains(response, "sortable column-is_estimated") + @mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", True) def test_estimated_location_filter_admin(self): org = self._get_org() @@ -799,7 +1033,6 @@ def test_estimated_location_filter_admin(self): indoor_location = self._create_location( name="location3", organization=org, type="indoor" ) - estimated_device = self._create_device() outdoor_device = self._create_device( name="11:22:33:44:55:66", mac_address="11:22:33:44:55:66" @@ -807,54 +1040,61 @@ def test_estimated_location_filter_admin(self): indoor_device = self._create_device( name="11:22:33:44:55:77", mac_address="11:22:33:44:55:77" ) - self._create_device_location( + self._create_object_location( content_object=estimated_device, location=estimated_location ) - self._create_device_location( + self._create_object_location( content_object=outdoor_device, location=outdoor_location ) - self._create_device_location( + self._create_object_location( content_object=indoor_device, location=indoor_location ) - path = reverse("admin:config_device_changelist") + with self.subTest("Test All Locations Filter"): response = self.client.get(path) self.assertContains(response, estimated_device.id) self.assertContains(response, outdoor_device.id) self.assertContains(response, indoor_device.id) + self.assertContains(response, "3 Devices") with self.subTest("Test Estimated Location Filter"): response = self.client.get(path, {"with_geo": "estimated"}) self.assertContains(response, estimated_device.id) self.assertNotContains(response, outdoor_device.id) self.assertNotContains(response, indoor_device.id) + self.assertContains(response, "1 Device") with self.subTest("Test Outdoor Location Filter"): response = self.client.get(path, {"with_geo": "outdoor"}) self.assertContains(response, outdoor_device.id) self.assertNotContains(response, estimated_device.id) self.assertNotContains(response, indoor_device.id) + self.assertContains(response, "1 Device") with self.subTest("Test Indoor Location Filter"): response = self.client.get(path, {"with_geo": "indoor"}) self.assertContains(response, indoor_device.id) self.assertNotContains(response, outdoor_device.id) self.assertNotContains(response, estimated_device.id) + self.assertContains(response, "1 Device") - with self.subTest("Test Indoor Location Filter"): + with self.subTest("Test No Location Filter"): response = self.client.get(path, {"with_geo": "false"}) self.assertNotContains(response, indoor_device.id) self.assertNotContains(response, outdoor_device.id) self.assertNotContains(response, estimated_device.id) + self.assertContains(response, "0 Devices") with mock.patch.object(config_app_settings, "WHOIS_CONFIGURED", False): with self.subTest( "Test Estimated Location Admin specific filters not available" " when WHOIS not configured" ): - for i in ["estimated", "outdoor", "indoor"]: - response = self.client.get(path, {"with_geo": i}) - self.assertContains(response, estimated_device.id) - self.assertContains(response, outdoor_device.id) - self.assertContains(response, indoor_device.id) + for filter_value in ["estimated", "outdoor", "indoor"]: + with self.subTest(filter_value=filter_value): + response = self.client.get(path, {"with_geo": filter_value}) + self.assertContains(response, estimated_device.id) + self.assertContains(response, outdoor_device.id) + self.assertContains(response, indoor_device.id) + self.assertContains(response, "3 Devices") diff --git a/openwisp_controller/geo/estimated_location/tests/utils.py b/openwisp_controller/geo/estimated_location/tests/utils.py index 77bb618ba..0ca26f335 100644 --- a/openwisp_controller/geo/estimated_location/tests/utils.py +++ b/openwisp_controller/geo/estimated_location/tests/utils.py @@ -1,15 +1,34 @@ from swapper import load_model +from openwisp_controller.config.whois.tasks import fetch_whois_details from openwisp_controller.config.whois.tests.utils import CreateWHOISMixin +from ..tasks import manage_estimated_locations + OrganizationConfigSettings = load_model("config", "OrganizationConfigSettings") class TestEstimatedLocationMixin(CreateWHOISMixin): def setUp(self): + # skip the org config settings creation from the parent mixin super(CreateWHOISMixin, self).setUp() OrganizationConfigSettings.objects.create( organization=self._get_org(), whois_enabled=True, estimated_location_enabled=True, ) + + # helper to mock send_task to directly call the task function synchronously + @staticmethod + def run_task(name, args=None, kwargs=None, **_): + # This mock intercepts all Celery send_task calls and executes them + # synchronously for testing purposes. We need to handle both task names + # because when a device is created/saved, the WHOIS lookup task may be + # triggered first, which then triggers the estimated location task. + # Without handling both, the assertion would fail and break the test flow. + if name == "whois_estimated_location_task": + return manage_estimated_locations(*args or [], **kwargs or {}) + elif name == "openwisp_controller.config.whois.tasks.fetch_whois_details": + return fetch_whois_details(*args or [], **kwargs or {}) + # Let other tasks pass through (they may be mocked elsewhere) + return None diff --git a/openwisp_controller/geo/estimated_location/utils.py b/openwisp_controller/geo/estimated_location/utils.py index ddd43e837..f7516afdc 100644 --- a/openwisp_controller/geo/estimated_location/utils.py +++ b/openwisp_controller/geo/estimated_location/utils.py @@ -1,6 +1,49 @@ -from openwisp_notifications.utils import _get_object_link +from django.utils.translation import gettext_lazy as _ + +from openwisp_controller.config.whois.utils import MESSAGE_MAP + +# Mutating the existing MESSAGE_MAP to include estimated location messages +MESSAGE_MAP.update( + { + "estimated_location_error": { + "level": "error", + "type": "estimated_location_info", + "message": _( + "Unable to create estimated location for device " + "[{notification.target}]({notification.target_link}). " + "Please assign/create a location manually." + ), + "description": _("Multiple devices found for IP: {ip_address}"), + }, + "estimated_location_created": { + "type": "estimated_location_info", + "level": "info", + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " {notification.verb} successfully." + ), + "description": _("Geographic coordinates inferred from IP: {ip_address}"), + }, + "estimated_location_updated": { + "type": "estimated_location_info", + "level": "info", + "message": _( + "Estimated location [{notification.actor}]({notification.actor_link})" + " for device" + " [{notification.target}]({notification.target_link})" + " updated successfully." + ), + "description": _("Geographic coordinates updated for IP: {ip_address}"), + }, + } +) def get_device_location_notification_target_url(obj, field, absolute_url=True): + # importing here to avoid "AppRegistryNotReady" + from openwisp_notifications.utils import _get_object_link + url = _get_object_link(obj._related_object(field), absolute_url) return f"{url}#devicelocation-group" diff --git a/openwisp_controller/geo/migrations/0004_location_is_estimated.py b/openwisp_controller/geo/migrations/0004_location_is_estimated.py index 1fcdfca68..45ac400da 100644 --- a/openwisp_controller/geo/migrations/0004_location_is_estimated.py +++ b/openwisp_controller/geo/migrations/0004_location_is_estimated.py @@ -16,6 +16,7 @@ class Migration(migrations.Migration): field=models.BooleanField( default=False, help_text=("Whether the location's coordinates are estimated."), + verbose_name="Is Estimated?", ), ), ] diff --git a/openwisp_controller/geo/templates/admin/geo/location/change_form.html b/openwisp_controller/geo/templates/admin/geo/location/change_form.html index af2fc9ceb..5c1bd85a4 100644 --- a/openwisp_controller/geo/templates/admin/geo/location/change_form.html +++ b/openwisp_controller/geo/templates/admin/geo/location/change_form.html @@ -1,9 +1,9 @@ {% extends "admin/django_loci/location_change_form.html" %} -{% load i18n admin_urls %} +{% load i18n %} {% block messages %} {{ block.super }} - {% if original and estimated_configured and original.is_estimated %} + {% if original and estimated_enabled and original.is_estimated %}
  • {% trans "This location is estimated based on the device's last IP address. Please refine it for greater accuracy." %}
diff --git a/openwisp_controller/geo/tests/test_api.py b/openwisp_controller/geo/tests/test_api.py index 4ebff1c9b..d8743fa8c 100644 --- a/openwisp_controller/geo/tests/test_api.py +++ b/openwisp_controller/geo/tests/test_api.py @@ -1,6 +1,7 @@ import json import tempfile import uuid +from unittest.mock import patch from django.contrib.auth import get_user_model from django.contrib.gis.geos import Point @@ -12,6 +13,7 @@ from rest_framework.authtoken.models import Token from swapper import load_model +from openwisp_controller.config import settings as config_app_settings from openwisp_controller.config.tests.utils import ( CreateConfigTemplateMixin, CreateDeviceMixin, @@ -533,7 +535,7 @@ def test_filter_location_list(self): path = reverse("geo_api:list_location") with self.subTest("Test without organization filtering"): - with self.assertNumQueries(5): + with self.assertNumQueries(4): response = self.client.get(path) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["count"], 2) @@ -583,6 +585,7 @@ def test_filter_location_list(self): self.assertContains(response, org1_location.id) self.assertNotContains(response, org2_location.id) + @patch.object(config_app_settings, "WHOIS_CONFIGURED", False) def test_post_location_list(self): path = reverse("geo_api:list_location") coords = json.loads(Point(2, 23).geojson) @@ -614,6 +617,7 @@ def test_get_location_detail(self): else: self.fail("NoReverseMatch not raised as expected") + @patch.object(config_app_settings, "WHOIS_CONFIGURED", False) def test_put_location_detail(self): l1 = self._create_location() path = reverse("geo_api:detail_location", args=[l1.pk]) @@ -691,6 +695,7 @@ def test_delete_location_detail(self): response = self.client.delete(path) self.assertEqual(response.status_code, 204) + @patch.object(config_app_settings, "WHOIS_CONFIGURED", False) def test_create_location_with_floorplan(self): path = reverse("geo_api:list_location") fl_image = self._get_simpleuploadedfile() diff --git a/requirements.txt b/requirements.txt index 5a8d14148..386f4b9a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,4 @@ shortuuid~=1.0.13 netaddr~=1.3.0 django-import-export~=4.3.14 jsonfield>=3.1.0,<4.0.0 -geoip2~=5.1.0 +geoip2>=5.1.0,<6.0.0 diff --git a/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py index 707a44bf5..d9125877c 100644 --- a/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py +++ b/tests/openwisp2/sample_config/migrations/0008_whoisinfo_organizationconfigsettings_whois_enabled.py @@ -1,7 +1,6 @@ # Generated by Django 5.2.1 on 2025-06-26 17:48 import django.utils.timezone -import jsonfield.fields import model_utils.fields from django.db import migrations, models @@ -46,22 +45,21 @@ class Migration(migrations.Migration): blank=True, help_text="Organization for ASN", max_length=100 ), ), - ("asn", models.CharField(blank=True, help_text="ASN", max_length=6)), + ("asn", models.CharField(blank=True, help_text="ASN", max_length=10)), ( "timezone", models.CharField(blank=True, help_text="Time zone", max_length=35), ), ( "address", - jsonfield.fields.JSONField( - blank=True, default=dict, help_text="Address" - ), + models.JSONField(blank=True, default=dict, help_text="Address"), ), - ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=20)), + ("cidr", models.CharField(blank=True, help_text="CIDR", max_length=49)), ("details", models.CharField(blank=True, max_length=64, null=True)), ], options={ "abstract": False, + "swappable": "CONFIG_WHOISINFO_MODEL", }, ), migrations.AddField( diff --git a/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py b/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py index e3ad65809..1c2e65e19 100644 --- a/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py +++ b/tests/openwisp2/sample_geo/migrations/0004_location_is_estimated.py @@ -16,6 +16,7 @@ class Migration(migrations.Migration): field=models.BooleanField( default=False, help_text="Whether the location's coordinates are estimated.", + verbose_name="Is Estimated?", ), ), ] From 6ab2ca2f821fbc8b6539c6fd0bb36c9fc226c033 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Wed, 4 Mar 2026 17:55:08 -0300 Subject: [PATCH 15/26] [docs] Port changelog for 1.2.1 --- CHANGES.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 4c2b804ce..2f0de6b15 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,18 @@ Version 1.3.0 [unreleased] Work in progress. +Version 1.2.1 [2026-03-04] +-------------------------- + +Bugfixes +~~~~~~~~ + +- Use context variables in Vpn.auto_client for OpenVPN backend +- Fixed 500 FieldError in DeviceLocationView `#1110 + `_ +- Fixed MultiValueDictKeyError on empty device form submission `#1057 + `_ + Version 1.2.0 [2025-10-24] -------------------------- From 6623550723d1b97c6553f921d9f39edf56c3750f Mon Sep 17 00:00:00 2001 From: Sarthak Tyagi <142912014+stktyagi@users.noreply.github.com> Date: Fri, 6 Mar 2026 19:05:38 +0530 Subject: [PATCH 16/26] [ci] Enabled CI failure bot --- .github/workflows/bot-ci-failure.yml | 72 ++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/bot-ci-failure.yml diff --git a/.github/workflows/bot-ci-failure.yml b/.github/workflows/bot-ci-failure.yml new file mode 100644 index 000000000..c99d899cc --- /dev/null +++ b/.github/workflows/bot-ci-failure.yml @@ -0,0 +1,72 @@ +name: CI Failure Bot + +on: + workflow_run: + workflows: ["OpenWISP Controller CI Build"] + types: + - completed + +permissions: + pull-requests: write + actions: read + contents: read + +concurrency: + group: ci-failure-${{ github.repository }}-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.head_branch }} + cancel-in-progress: true + +jobs: + find-pr: + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'failure' }} + outputs: + pr_number: ${{ steps.pr.outputs.number }} + steps: + - name: Find PR Number + id: pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + PR_NUMBER_PAYLOAD: ${{ github.event.workflow_run.pull_requests[0].number }} + EVENT_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + PR_NUMBER="$PR_NUMBER_PAYLOAD" + if [ -n "$PR_NUMBER" ]; then + echo "Found PR #$PR_NUMBER from workflow payload." + echo "number=$PR_NUMBER" >> $GITHUB_OUTPUT + exit 0 + fi + HEAD_SHA="$EVENT_HEAD_SHA" + echo "Payload empty. Searching for PR via Commits API..." + PR_NUMBER=$(gh api repos/$REPO/commits/$HEAD_SHA/pulls -q '.[0].number' 2>/dev/null || true) + if [ -n "$PR_NUMBER" ] && [ "$PR_NUMBER" != "null" ]; then + echo "Found PR #$PR_NUMBER using Commits API." + echo "number=$PR_NUMBER" >> $GITHUB_OUTPUT + exit 0 + fi + echo "API lookup failed/empty. Scanning open PRs for matching head SHA..." + PR_NUMBER=$(gh pr list --repo "$REPO" --state open --limit 100 --json number,headRefOid --jq ".[] | select(.headRefOid == \"$HEAD_SHA\") | .number" | head -n 1) + if [ -n "$PR_NUMBER" ]; then + echo "Found PR #$PR_NUMBER by scanning open PRs." + echo "number=$PR_NUMBER" >> $GITHUB_OUTPUT + exit 0 + fi + echo "::warning::No open PR found. This workflow run might not be attached to an open PR." + exit 0 + + call-ci-failure-bot: + needs: find-pr + if: ${{ needs.find-pr.outputs.pr_number != '' }} + uses: openwisp/openwisp-utils/.github/workflows/reusable-bot-ci-failure.yml@master + with: + pr_number: ${{ needs.find-pr.outputs.pr_number }} + head_sha: ${{ github.event.workflow_run.head_sha }} + head_repo: ${{ github.event.workflow_run.head_repository.full_name }} + base_repo: ${{ github.repository }} + run_id: ${{ github.event.workflow_run.id }} + pr_author: ${{ github.event.workflow_run.actor.login }} + actor: ${{ github.actor }} + secrets: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + APP_ID: ${{ secrets.OPENWISP_BOT_APP_ID }} + PRIVATE_KEY: ${{ secrets.OPENWISP_BOT_PRIVATE_KEY }} From 9eff7573d5a7613538126d50b90ef3463b2afca6 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 6 Mar 2026 15:15:20 -0300 Subject: [PATCH 17/26] [change:ux] Improved help text of configuration variable fields The previous help texts were confusing and too technical. [backport 1.2] --- openwisp_controller/config/base/config.py | 8 ++--- .../config/base/device_group.py | 8 ++--- .../config/base/multitenancy.py | 3 +- openwisp_controller/config/base/template.py | 7 ++--- .../config/migrations/0023_update_context.py | 7 +++-- .../0028_template_default_values.py | 6 ++-- .../config/migrations/0036_device_group.py | 8 +++-- .../migrations/0049_devicegroup_context.py | 4 +-- ...0051_organizationconfigsettings_context.py | 4 +-- .../sample_config/migrations/0001_initial.py | 30 +++++++++++-------- 10 files changed, 46 insertions(+), 39 deletions(-) diff --git a/openwisp_controller/config/base/config.py b/openwisp_controller/config/base/config.py index b34ff9d1d..efb42ff65 100644 --- a/openwisp_controller/config/base/config.py +++ b/openwisp_controller/config/base/config.py @@ -85,10 +85,10 @@ class AbstractConfig(ChecksumCacheMixin, BaseConfig): blank=True, default=dict, help_text=_( - "Additional " - '' - "context (configuration variables) in JSON format" + "allows overriding " + '' + "configuration variables" ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, dump_kwargs={"indent": 4}, diff --git a/openwisp_controller/config/base/device_group.py b/openwisp_controller/config/base/device_group.py index 64fc77e5d..366d61667 100644 --- a/openwisp_controller/config/base/device_group.py +++ b/openwisp_controller/config/base/device_group.py @@ -42,8 +42,9 @@ class AbstractDeviceGroup(OrgMixin, TimeStampedEditableModel): load_kwargs={"object_pairs_hook": collections.OrderedDict}, dump_kwargs={"indent": 4}, help_text=_( - "Group meta data, use this field to store data which is related" - " to this group and can be retrieved via the REST API." + "Store custom metadata related to this group. This field is intended " + "for arbitrary data that does not affect device configuration and can " + "be retrieved via the REST API for integrations or external tools." ), verbose_name=_("Metadata"), ) @@ -53,8 +54,7 @@ class AbstractDeviceGroup(OrgMixin, TimeStampedEditableModel): load_kwargs={"object_pairs_hook": collections.OrderedDict}, dump_kwargs={"indent": 4}, help_text=_( - "This field can be used to add meta data for the group" - ' or to add "Configuration Variables" to the devices.' + "Define configuration variables available to all devices in this group" ), verbose_name=_("Configuration Variables"), ) diff --git a/openwisp_controller/config/base/multitenancy.py b/openwisp_controller/config/base/multitenancy.py index bceb5de42..c2434e413 100644 --- a/openwisp_controller/config/base/multitenancy.py +++ b/openwisp_controller/config/base/multitenancy.py @@ -50,7 +50,8 @@ class AbstractOrganizationConfigSettings(UUIDModel): load_kwargs={"object_pairs_hook": collections.OrderedDict}, dump_kwargs={"indent": 4}, help_text=_( - 'This field can be used to add "Configuration Variables"' " to the devices." + "Define reusable configuration variables available " + "to all devices in this organization" ), verbose_name=_("Configuration Variables"), ) diff --git a/openwisp_controller/config/base/template.py b/openwisp_controller/config/base/template.py index d7b0a7dc9..25e89eee3 100644 --- a/openwisp_controller/config/base/template.py +++ b/openwisp_controller/config/base/template.py @@ -97,10 +97,9 @@ class AbstractTemplate(ShareableOrgMixinUniqueName, BaseConfig): default=dict, blank=True, help_text=_( - "A dictionary containing the default " - "values for the variables used by this " - "template; these default variables will " - "be used during schema validation." + "Define default values for the variables used in this template. " + "These values are used during validation and when a variable is " + "not provided by the device, group, or organization." ), load_kwargs={"object_pairs_hook": OrderedDict}, dump_kwargs={"indent": 4}, diff --git a/openwisp_controller/config/migrations/0023_update_context.py b/openwisp_controller/config/migrations/0023_update_context.py index 278ee8036..68edd30da 100644 --- a/openwisp_controller/config/migrations/0023_update_context.py +++ b/openwisp_controller/config/migrations/0023_update_context.py @@ -18,9 +18,10 @@ class Migration(migrations.Migration): default=dict, dump_kwargs={"ensure_ascii": False, "indent": 4}, help_text=( - 'Additional ' - "context (configuration variables) in JSON format" + "allows overriding " + '' + "configuration variables" ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, ), diff --git a/openwisp_controller/config/migrations/0028_template_default_values.py b/openwisp_controller/config/migrations/0028_template_default_values.py index 47a0909b6..13651f4dd 100644 --- a/openwisp_controller/config/migrations/0028_template_default_values.py +++ b/openwisp_controller/config/migrations/0028_template_default_values.py @@ -18,9 +18,9 @@ class Migration(migrations.Migration): default=dict, dump_kwargs={"ensure_ascii": False, "indent": 4}, help_text=( - "A dictionary containing the default values for " - "the variables used by this template; these default " - "variables will be used during schema validation." + "Define default values for the variables used in this template. " + "These values are used during validation and when a variable is " + "not provided by the device, group, or organization." ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, verbose_name="Default Values", diff --git a/openwisp_controller/config/migrations/0036_device_group.py b/openwisp_controller/config/migrations/0036_device_group.py index 6197bbb51..f3ca39ced 100644 --- a/openwisp_controller/config/migrations/0036_device_group.py +++ b/openwisp_controller/config/migrations/0036_device_group.py @@ -62,9 +62,11 @@ class Migration(migrations.Migration): dump_kwargs={"ensure_ascii": False, "indent": 4}, load_kwargs={"object_pairs_hook": collections.OrderedDict}, help_text=( - "Group meta data, use this field to store data which is" - " related to this group and can be retrieved via the" - " REST API." + "Store custom metadata related to this group. " + "This field is intended for arbitrary data that " + "does not affect device configuration and can " + "be retrieved via the REST API for integrations " + "or external tools." ), verbose_name="Metadata", ), diff --git a/openwisp_controller/config/migrations/0049_devicegroup_context.py b/openwisp_controller/config/migrations/0049_devicegroup_context.py index adb108d75..34733f4db 100644 --- a/openwisp_controller/config/migrations/0049_devicegroup_context.py +++ b/openwisp_controller/config/migrations/0049_devicegroup_context.py @@ -20,8 +20,8 @@ class Migration(migrations.Migration): default=dict, dump_kwargs={"ensure_ascii": False, "indent": 4}, help_text=( - "This field can be used to add meta data for the group" - ' or to add "Configuration Variables" to the devices.' + "Define configuration variables available " + "to all devices in this group" ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, verbose_name="Configuration Variables", diff --git a/openwisp_controller/config/migrations/0051_organizationconfigsettings_context.py b/openwisp_controller/config/migrations/0051_organizationconfigsettings_context.py index a72a6f173..b643e2e12 100644 --- a/openwisp_controller/config/migrations/0051_organizationconfigsettings_context.py +++ b/openwisp_controller/config/migrations/0051_organizationconfigsettings_context.py @@ -20,8 +20,8 @@ class Migration(migrations.Migration): default=dict, dump_kwargs={"indent": 4}, help_text=( - 'This field can be used to add "Configuration Variables"' - " to the devices." + "Define reusable configuration variables available " + "to all devices in this organization" ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, verbose_name="Configuration Variables", diff --git a/tests/openwisp2/sample_config/migrations/0001_initial.py b/tests/openwisp2/sample_config/migrations/0001_initial.py index d41f4fd89..c6bb22c89 100644 --- a/tests/openwisp2/sample_config/migrations/0001_initial.py +++ b/tests/openwisp2/sample_config/migrations/0001_initial.py @@ -121,9 +121,10 @@ class Migration(migrations.Migration): default=dict, dump_kwargs={"ensure_ascii": False, "indent": 4}, help_text=( - 'Additional ' - "context (configuration variables) in JSON format" + "allows overriding " + '' + "configuration variables" ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, ), @@ -579,9 +580,10 @@ class Migration(migrations.Migration): default=dict, dump_kwargs={"ensure_ascii": False, "indent": 4}, help_text=( - "A dictionary containing the default values for the " - "variables used by this template; these default variables " - "will be used during schema validation." + "Define default values for the variables used " + "in this template. These values are used during " + "validation and when a variable is not provided " + "by the device, group, or organization." ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, verbose_name="Default Values", @@ -709,8 +711,8 @@ class Migration(migrations.Migration): default=dict, dump_kwargs={"indent": 4}, help_text=( - 'This field can be used to add "Configuration Variables"' - " to the devices." + "Define reusable configuration variables " + "available to all devices in this organization" ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, verbose_name="Configuration Variables", @@ -765,9 +767,11 @@ class Migration(migrations.Migration): dump_kwargs={"ensure_ascii": False, "indent": 4}, load_kwargs={"object_pairs_hook": collections.OrderedDict}, help_text=( - "Group meta data, use this field to store data which is" - " related to this group and can be retrieved via the" - " REST API." + "Store custom metadata related to this group. " + "This field is intended for arbitrary data that " + "does not affect device configuration and can " + "be retrieved via the REST API for integrations " + "or external tools." ), verbose_name="Metadata", ), @@ -779,8 +783,8 @@ class Migration(migrations.Migration): default=dict, dump_kwargs={"ensure_ascii": False, "indent": 4}, help_text=( - "This field can be used to add meta data for the group" - ' or to add "Configuration Variables" to the devices.' + "Define configuration variables available " + "to all devices in this group" ), load_kwargs={"object_pairs_hook": collections.OrderedDict}, verbose_name="Configuration Variables", From 1ebf7b6e092ffc1ddd3f918efcfb752890ade572 Mon Sep 17 00:00:00 2001 From: Federico Capoano Date: Fri, 6 Mar 2026 17:12:52 -0300 Subject: [PATCH 18/26] [docs] Port changelog for 1.2.2 --- CHANGES.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 2f0de6b15..8ba6d354e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -6,6 +6,18 @@ Version 1.3.0 [unreleased] Work in progress. +Version 1.2.2 [2026-03-06] +-------------------------- + +Changes +~~~~~~~ + +Other changes ++++++++++++++ + +- Improved help text of configuration variable fields +- Minor fixes in the test suite + Version 1.2.1 [2026-03-04] -------------------------- From 9e98d3e1c4180ec64ad88075bb8afda4afbe4943 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Mar 2026 11:02:26 +0000 Subject: [PATCH 19/26] [deps] Update responses requirement from ~=0.25.8 to ~=0.26.0 Updates the requirements on [responses](https://github.com/getsentry/responses) to permit the latest version. - [Release notes](https://github.com/getsentry/responses/releases) - [Changelog](https://github.com/getsentry/responses/blob/master/CHANGES) - [Commits](https://github.com/getsentry/responses/compare/0.25.8...0.26.0) --- updated-dependencies: - dependency-name: responses dependency-version: 0.26.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 0929f923f..c358df8a7 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,4 +3,4 @@ pytest-timeout~=2.3.0 openwisp-utils[qa,selenium,channels-test] @ https://github.com/openwisp/openwisp-utils/archive/refs/heads/1.3.tar.gz django_redis~=6.0.0 mock-ssh-server~=0.9.1 -responses~=0.25.8 +responses~=0.26.0 From 1da4d8ec258635a79502124849e8f3e7ba333a13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Mar 2026 11:02:32 +0000 Subject: [PATCH 20/26] [deps] Update pytest-timeout requirement from ~=2.3.0 to ~=2.4.0 Updates the requirements on [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) to permit the latest version. - [Commits](https://github.com/pytest-dev/pytest-timeout/compare/2.3.0...2.4.0) --- updated-dependencies: - dependency-name: pytest-timeout dependency-version: 2.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index c358df8a7..8744aa9d9 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ pytest-cov~=7.0.0 -pytest-timeout~=2.3.0 +pytest-timeout~=2.4.0 openwisp-utils[qa,selenium,channels-test] @ https://github.com/openwisp/openwisp-utils/archive/refs/heads/1.3.tar.gz django_redis~=6.0.0 mock-ssh-server~=0.9.1 From 0e2d8f73c428a8fb223f1de7a0b029df3e2ed84f Mon Sep 17 00:00:00 2001 From: Sarthak Tyagi <142912014+stktyagi@users.noreply.github.com> Date: Mon, 9 Mar 2026 03:13:11 +0530 Subject: [PATCH 21/26] [fix] Fixed mentioning by CI faillure bot --- .github/workflows/bot-ci-failure.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bot-ci-failure.yml b/.github/workflows/bot-ci-failure.yml index c99d899cc..17f32eb97 100644 --- a/.github/workflows/bot-ci-failure.yml +++ b/.github/workflows/bot-ci-failure.yml @@ -21,6 +21,7 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'failure' }} outputs: pr_number: ${{ steps.pr.outputs.number }} + pr_author: ${{ steps.pr.outputs.author }} steps: - name: Find PR Number id: pr @@ -30,10 +31,21 @@ jobs: PR_NUMBER_PAYLOAD: ${{ github.event.workflow_run.pull_requests[0].number }} EVENT_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} run: | + emit_pr() { + local pr_number="$1" + local pr_author + pr_author=$(gh pr view "$pr_number" --repo "$REPO" --json author --jq '.author.login // empty' 2>/dev/null || echo "") + if [ -z "$pr_author" ]; then + pr_author="${{ github.event.workflow_run.actor.login }}" + echo "::warning::Could not fetch PR author for PR #$pr_number; falling back to @$pr_author" + fi + echo "number=$pr_number" >> "$GITHUB_OUTPUT" + echo "author=$pr_author" >> "$GITHUB_OUTPUT" + } PR_NUMBER="$PR_NUMBER_PAYLOAD" if [ -n "$PR_NUMBER" ]; then echo "Found PR #$PR_NUMBER from workflow payload." - echo "number=$PR_NUMBER" >> $GITHUB_OUTPUT + emit_pr "$PR_NUMBER" exit 0 fi HEAD_SHA="$EVENT_HEAD_SHA" @@ -41,14 +53,14 @@ jobs: PR_NUMBER=$(gh api repos/$REPO/commits/$HEAD_SHA/pulls -q '.[0].number' 2>/dev/null || true) if [ -n "$PR_NUMBER" ] && [ "$PR_NUMBER" != "null" ]; then echo "Found PR #$PR_NUMBER using Commits API." - echo "number=$PR_NUMBER" >> $GITHUB_OUTPUT + emit_pr "$PR_NUMBER" exit 0 fi echo "API lookup failed/empty. Scanning open PRs for matching head SHA..." PR_NUMBER=$(gh pr list --repo "$REPO" --state open --limit 100 --json number,headRefOid --jq ".[] | select(.headRefOid == \"$HEAD_SHA\") | .number" | head -n 1) if [ -n "$PR_NUMBER" ]; then echo "Found PR #$PR_NUMBER by scanning open PRs." - echo "number=$PR_NUMBER" >> $GITHUB_OUTPUT + emit_pr "$PR_NUMBER" exit 0 fi echo "::warning::No open PR found. This workflow run might not be attached to an open PR." @@ -64,8 +76,8 @@ jobs: head_repo: ${{ github.event.workflow_run.head_repository.full_name }} base_repo: ${{ github.repository }} run_id: ${{ github.event.workflow_run.id }} - pr_author: ${{ github.event.workflow_run.actor.login }} - actor: ${{ github.actor }} + pr_author: ${{ needs.find-pr.outputs.pr_author }} + actor: ${{ github.event.workflow_run.actor.login }} secrets: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} APP_ID: ${{ secrets.OPENWISP_BOT_APP_ID }} From 0b897dd150e60aedf251c1156ec3bc1f65a6acb9 Mon Sep 17 00:00:00 2001 From: Pushpit Kamboj Date: Fri, 13 Mar 2026 22:39:59 +0530 Subject: [PATCH 22/26] [CI] Add changelog bot workflow --- .github/workflows/bot-changelog.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/bot-changelog.yml diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml new file mode 100644 index 000000000..6558c0a27 --- /dev/null +++ b/.github/workflows/bot-changelog.yml @@ -0,0 +1,15 @@ +name: Changelog Bot + +on: + # Trigger when a PR review is submitted with approval + pull_request_review: + types: [submitted] + +jobs: + changelog: + # Call the reusable workflow from openwisp-utils + uses: openwisp/openwisp-utils/.github/workflows/reusable-bot-changelog.yml@master + secrets: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + OPENWISP_BOT_APP_ID: ${{ secrets.OPENWISP_BOT_APP_ID }} + OPENWISP_BOT_PRIVATE_KEY: ${{ secrets.OPENWISP_BOT_PRIVATE_KEY }} From 11679f567d839c79ce65c73a9c5055f40e54e017 Mon Sep 17 00:00:00 2001 From: Pushpit Kamboj Date: Sat, 14 Mar 2026 11:12:54 +0530 Subject: [PATCH 23/26] [change] Add PR approved checks --- .github/workflows/bot-changelog.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml index 6558c0a27..a09d16793 100644 --- a/.github/workflows/bot-changelog.yml +++ b/.github/workflows/bot-changelog.yml @@ -7,7 +7,11 @@ on: jobs: changelog: - # Call the reusable workflow from openwisp-utils + if: | + github.event.review.state == 'approved' && + (github.event.review.author_association == 'OWNER' || + github.event.review.author_association == 'MEMBER' || + github.event.review.author_association == 'COLLABORATOR') uses: openwisp/openwisp-utils/.github/workflows/reusable-bot-changelog.yml@master secrets: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} From 2b416b59152f3fb3fa4111ac07cecfe79cea7fc8 Mon Sep 17 00:00:00 2001 From: Pushpit Kamboj Date: Fri, 27 Mar 2026 11:58:59 +0530 Subject: [PATCH 24/26] [ci] Remove old workflow with new trigger and runner yml files --- .github/workflows/bot-changelog-runner.yml | 54 +++++++++++++++++++++ .github/workflows/bot-changelog-trigger.yml | 39 +++++++++++++++ .github/workflows/bot-changelog.yml | 19 -------- 3 files changed, 93 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/bot-changelog-runner.yml create mode 100644 .github/workflows/bot-changelog-trigger.yml delete mode 100644 .github/workflows/bot-changelog.yml diff --git a/.github/workflows/bot-changelog-runner.yml b/.github/workflows/bot-changelog-runner.yml new file mode 100644 index 000000000..73a32d636 --- /dev/null +++ b/.github/workflows/bot-changelog-runner.yml @@ -0,0 +1,54 @@ +name: Changelog Bot Runner + +on: + workflow_run: + workflows: ["Changelog Bot Trigger"] + types: + - completed + +permissions: + actions: read + contents: read + pull-requests: write + issues: write + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + fetch-metadata: + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion == 'success' + outputs: + pr_number: ${{ steps.metadata.outputs.pr_number }} + steps: + - name: Download PR metadata + id: download + uses: actions/download-artifact@v4 + with: + name: changelog-metadata + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + continue-on-error: true + + - name: Read PR metadata + if: steps.download.outcome == 'success' + id: metadata + run: | + PR_NUMBER=$(cat pr_number) + if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then + echo "::error::Invalid PR number: $PR_NUMBER" + exit 1 + fi + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + + changelog: + needs: fetch-metadata + if: needs.fetch-metadata.outputs.pr_number != '' + uses: openwisp/openwisp-utils/.github/workflows/reusable-bot-changelog.yml@master + with: + pr_number: ${{ needs.fetch-metadata.outputs.pr_number }} + secrets: + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + OPENWISP_BOT_APP_ID: ${{ secrets.OPENWISP_BOT_APP_ID }} + OPENWISP_BOT_PRIVATE_KEY: ${{ secrets.OPENWISP_BOT_PRIVATE_KEY }} \ No newline at end of file diff --git a/.github/workflows/bot-changelog-trigger.yml b/.github/workflows/bot-changelog-trigger.yml new file mode 100644 index 000000000..b98a6a4be --- /dev/null +++ b/.github/workflows/bot-changelog-trigger.yml @@ -0,0 +1,39 @@ +name: Changelog Bot Trigger + +on: + pull_request_review: + types: [submitted] + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + check: + if: | + github.event.review.state == 'approved' && + (github.event.review.author_association == 'OWNER' || + github.event.review.author_association == 'MEMBER' || + github.event.review.author_association == 'COLLABORATOR') + runs-on: ubuntu-latest + steps: + - name: Check for noteworthy PR + id: check + env: + PR_TITLE: ${{ github.event.pull_request.title }} + run: | + if echo "$PR_TITLE" | grep -qiE '^\[(feature|fix|change)\]'; then + echo "has_noteworthy=true" >> $GITHUB_OUTPUT + fi + + - name: Save PR metadata + if: steps.check.outputs.has_noteworthy == 'true' + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: echo "$PR_NUMBER" > pr_number + + - name: Upload PR metadata + if: steps.check.outputs.has_noteworthy == 'true' + uses: actions/upload-artifact@v4 + with: + name: changelog-metadata + path: pr_number \ No newline at end of file diff --git a/.github/workflows/bot-changelog.yml b/.github/workflows/bot-changelog.yml deleted file mode 100644 index a09d16793..000000000 --- a/.github/workflows/bot-changelog.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Changelog Bot - -on: - # Trigger when a PR review is submitted with approval - pull_request_review: - types: [submitted] - -jobs: - changelog: - if: | - github.event.review.state == 'approved' && - (github.event.review.author_association == 'OWNER' || - github.event.review.author_association == 'MEMBER' || - github.event.review.author_association == 'COLLABORATOR') - uses: openwisp/openwisp-utils/.github/workflows/reusable-bot-changelog.yml@master - secrets: - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - OPENWISP_BOT_APP_ID: ${{ secrets.OPENWISP_BOT_APP_ID }} - OPENWISP_BOT_PRIVATE_KEY: ${{ secrets.OPENWISP_BOT_PRIVATE_KEY }} From e33a1e07daa09d64dc976d727ac7c5a619332fbd Mon Sep 17 00:00:00 2001 From: Pushpit Kamboj Date: Fri, 27 Mar 2026 13:02:06 +0530 Subject: [PATCH 25/26] [fix] QA checks --- .github/workflows/bot-changelog-runner.yml | 2 +- .github/workflows/bot-changelog-trigger.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bot-changelog-runner.yml b/.github/workflows/bot-changelog-runner.yml index 73a32d636..f54737eed 100644 --- a/.github/workflows/bot-changelog-runner.yml +++ b/.github/workflows/bot-changelog-runner.yml @@ -51,4 +51,4 @@ jobs: secrets: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} OPENWISP_BOT_APP_ID: ${{ secrets.OPENWISP_BOT_APP_ID }} - OPENWISP_BOT_PRIVATE_KEY: ${{ secrets.OPENWISP_BOT_PRIVATE_KEY }} \ No newline at end of file + OPENWISP_BOT_PRIVATE_KEY: ${{ secrets.OPENWISP_BOT_PRIVATE_KEY }} diff --git a/.github/workflows/bot-changelog-trigger.yml b/.github/workflows/bot-changelog-trigger.yml index b98a6a4be..a41854196 100644 --- a/.github/workflows/bot-changelog-trigger.yml +++ b/.github/workflows/bot-changelog-trigger.yml @@ -6,7 +6,7 @@ on: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - + jobs: check: if: | @@ -36,4 +36,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: changelog-metadata - path: pr_number \ No newline at end of file + path: pr_number From 18e754d1c60c0a73c9d28b42a711e33f7b13114b Mon Sep 17 00:00:00 2001 From: Pushpit Kamboj Date: Thu, 23 Apr 2026 15:10:40 +0530 Subject: [PATCH 26/26] [fix] Remove js env, changed permissions for changelog-bot --- .github/workflows/bot-changelog-runner.yml | 14 +++++++------- .github/workflows/bot-changelog-trigger.yml | 5 ++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bot-changelog-runner.yml b/.github/workflows/bot-changelog-runner.yml index f54737eed..218a7cce1 100644 --- a/.github/workflows/bot-changelog-runner.yml +++ b/.github/workflows/bot-changelog-runner.yml @@ -8,23 +8,19 @@ on: permissions: actions: read - contents: read - pull-requests: write - issues: write - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: fetch-metadata: runs-on: ubuntu-latest if: github.event.workflow_run.conclusion == 'success' + permissions: + actions: read outputs: pr_number: ${{ steps.metadata.outputs.pr_number }} steps: - name: Download PR metadata id: download - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: changelog-metadata github-token: ${{ secrets.GITHUB_TOKEN }} @@ -45,6 +41,10 @@ jobs: changelog: needs: fetch-metadata if: needs.fetch-metadata.outputs.pr_number != '' + permissions: + contents: read + pull-requests: write + issues: write uses: openwisp/openwisp-utils/.github/workflows/reusable-bot-changelog.yml@master with: pr_number: ${{ needs.fetch-metadata.outputs.pr_number }} diff --git a/.github/workflows/bot-changelog-trigger.yml b/.github/workflows/bot-changelog-trigger.yml index a41854196..0472f2209 100644 --- a/.github/workflows/bot-changelog-trigger.yml +++ b/.github/workflows/bot-changelog-trigger.yml @@ -4,8 +4,7 @@ on: pull_request_review: types: [submitted] -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true +permissions: {} jobs: check: @@ -33,7 +32,7 @@ jobs: - name: Upload PR metadata if: steps.check.outputs.has_noteworthy == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: changelog-metadata path: pr_number