From dd238b67999c1b82886b5486335efb3baec18894 Mon Sep 17 00:00:00 2001 From: Ian Hoffman Date: Mon, 23 Aug 2021 23:12:53 -0700 Subject: [PATCH 1/4] Support Whatsapp (WIP) --- app/airq/controllers/api.py | 15 ++++- app/airq/lib/twilio.py | 19 ++++-- app/airq/models/clients.py | 32 +++++++-- ...e054d3a_support_whatsapp_as_enum_member.py | 67 +++++++++++++++++++ app/tests/test_clients.py | 4 +- app/tests/test_sms.py | 6 +- 6 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 app/migrations/versions/a3c90e054d3a_support_whatsapp_as_enum_member.py diff --git a/app/airq/controllers/api.py b/app/airq/controllers/api.py index a9c02de..20fc925 100644 --- a/app/airq/controllers/api.py +++ b/app/airq/controllers/api.py @@ -23,23 +23,36 @@ def _get_supported_locale(locale: str) -> str: def sms_reply(locale: str) -> str: supported_locale = _get_supported_locale(locale) g.locale = supported_locale + zipcode = request.values.get("Body", "").strip() + phone_number = request.values.get("From", "").strip() + if phone_number.startswith("whatsapp:"): + phone_number = phone_number.lstrip("whatsapp:") + identifier_type = ClientIdentifierType.WHATSAPP + else: + identifier_type = ClientIdentifierType.PHONE_NUMBER + response = commands.handle_command( - zipcode, phone_number, ClientIdentifierType.PHONE_NUMBER, supported_locale + zipcode, phone_number, identifier_type, supported_locale ) + return response.serialize() def test_command(locale: str) -> str: supported_locale = _get_supported_locale(locale) g.locale = supported_locale + command = request.args.get("command", "").strip() + if request.headers.getlist("X-Forwarded-For"): ip = request.headers.getlist("X-Forwarded-For")[0] else: ip = request.remote_addr + response = commands.handle_command( command, ip, ClientIdentifierType.IP, supported_locale ) + return response.as_html() diff --git a/app/airq/lib/twilio.py b/app/airq/lib/twilio.py index cd2077b..40366ae 100644 --- a/app/airq/lib/twilio.py +++ b/app/airq/lib/twilio.py @@ -2,11 +2,12 @@ import logging import typing -from airq import config - from twilio.base.exceptions import TwilioRestException from twilio.rest import Client +from airq import config +from airq.models.clients import ClientIdentifierType + logger = logging.getLogger(__name__) @@ -15,6 +16,9 @@ class TwilioErrorCode(enum.IntEnum): OUT_OF_REGION = 21408 UNSUBSCRIBED = 21610 + # See https://www.twilio.com/docs/whatsapp/tutorial/send-whatsapp-notification-messages-templates#encountering-error-code-63016 + NO_CONVERSATION = 63016 + @classmethod def from_exc(cls, exc: TwilioRestException) -> typing.Optional["TwilioErrorCode"]: for m in cls: @@ -23,14 +27,21 @@ def from_exc(cls, exc: TwilioRestException) -> typing.Optional["TwilioErrorCode" return None -def send_sms( - body: str, to_number: str, locale: str, media: typing.Optional[str] = None +def send_message( + body: str, + to_number: str, + type_code: ClientIdentifierType, + locale: str, + media: typing.Optional[str] = None, ): from_number = config.TWILIO_NUMBERS.get(locale) if not from_number: logger.exception("Couldn't find a Twilio number for %s", locale) return + if type_code == ClientIdentifierType.WHATSAPP: + from_number = "whatsapp:" + from_number + kwargs = dict(body=body, to=to_number, from_=from_number) if media: kwargs["media_url"] = media diff --git a/app/airq/models/clients.py b/app/airq/models/clients.py index 8966068..76f8987 100644 --- a/app/airq/models/clients.py +++ b/app/airq/models/clients.py @@ -21,8 +21,6 @@ from airq.lib.readings import Pm25 from airq.lib.readings import Readings from airq.lib.sms import coerce_phone_number -from airq.lib.twilio import send_sms -from airq.lib.twilio import TwilioErrorCode from airq.models.events import Event from airq.models.events import EventType from airq.models.zipcodes import Zipcode @@ -34,6 +32,11 @@ class ClientIdentifierType(enum.Enum): PHONE_NUMBER = 1 IP = 2 + WHATSAPP = 3 + + @property + def can_receive_messages(self) -> bool: + return self != self.IP class ClientQuery(BaseQuery): @@ -69,7 +72,7 @@ def get_by_phone_number(self, phone_number: str) -> typing.Optional["Client"]: # def filter_phones(self) -> "ClientQuery": - return self.filter(Client.type_code == ClientIdentifierType.PHONE_NUMBER) + return self.filter(Client.type_code != ClientIdentifierType.IP) def filter_inactive_since(self, timestamp: float) -> "ClientQuery": return self.filter(Client.last_activity_at < timestamp).filter( @@ -202,6 +205,10 @@ def __repr__(self) -> str: # and therefore shouldn't include it in its state EVENT_RESPONSE_TIME = datetime.timedelta(hours=1) + @property + def identifier_type(self) -> ClientIdentifierType: + return ClientIdentifierType(self.type_code) + @classmethod def get_share_window(self) -> typing.Tuple[int, int]: ts = timestamp() @@ -346,12 +353,24 @@ def is_in_send_window(self) -> bool: return send_start <= dt.hour < send_end def send_message(self, message: str, media: typing.Optional[str] = None) -> bool: - if self.type_code == ClientIdentifierType.PHONE_NUMBER: + from airq.lib.twilio import send_message + from airq.lib.twilio import TwilioErrorCode + + if self.identifier_type.can_receive_messages: try: - send_sms(message, self.identifier, self.locale, media=media) + send_message( + message, self.identifier, self.identifier_type, self.locale, media=media + ) except TwilioRestException as e: code = TwilioErrorCode.from_exc(e) - if code: + if code == TwilioErrorCode.NO_CONVERSATION: + logger.exception( + 'Sent non-template message "%s" to %s outside of a conversation', + message, + self, + ) + return False + elif code: logger.warning( "Disabling alerts for recipient %s: %s", self, @@ -362,7 +381,6 @@ def send_message(self, message: str, media: typing.Optional[str] = None) -> bool else: raise else: - # Other clients types don't yet support message sending. logger.info("Not messaging client %s: %s", self.id, message) return True diff --git a/app/migrations/versions/a3c90e054d3a_support_whatsapp_as_enum_member.py b/app/migrations/versions/a3c90e054d3a_support_whatsapp_as_enum_member.py new file mode 100644 index 0000000..4c0e34e --- /dev/null +++ b/app/migrations/versions/a3c90e054d3a_support_whatsapp_as_enum_member.py @@ -0,0 +1,67 @@ +"""support whatsapp as enum member + +Revision ID: a3c90e054d3a +Revises: 3dad84c99218 +Create Date: 2021-08-24 05:28:52.434323 + +""" +from alembic import op +import sqlalchemy as sa + +# Cribbed from https://markrailton.com/blog/creating-migrations-when-changing-an-enum-in-python-using-sql-alchemy + +# revision identifiers, used by Alembic. +revision = "a3c90e054d3a" +down_revision = "3dad84c99218" +branch_labels = None +depends_on = None + +# Enum 'type' for PostgreSQL +enum_name = "clientidentifiertype" +# Set temporary enum 'type' for PostgreSQL +tmp_enum_name = "tmp_" + enum_name + +# Options for Enum +old_options = ("PHONE_NUMBER", "IP") +new_options = sorted(old_options + ("WHATSAPP",)) + +# Create enum fields +old_type = sa.Enum(*old_options, name=enum_name) +new_type = sa.Enum(*new_options, name=enum_name) + + +def upgrade(): + # Rename current enum type to tmp_ + op.execute("ALTER TYPE " + enum_name + " RENAME TO " + tmp_enum_name) + # Create new enum type in db + new_type.create(op.get_bind()) + # Update column to use new enum type + op.execute( + "ALTER TABLE clients ALTER COLUMN type_code TYPE " + + enum_name + + " USING type_code::text::" + + enum_name + ) + # Drop old enum type + op.execute("DROP TYPE " + tmp_enum_name) + + +def downgrade(): + # Instantiate db query + op.execute( + "DELETE FROM events WHERE client_id IN (SELECT id FROM clients WHERE type_code = 'WHATSAPP')" + ) + op.execute("DELETE FROM clients WHERE type_code = 'WHATSAPP'") + # Rename enum type to tmp_ + op.execute("ALTER TYPE " + enum_name + " RENAME TO " + tmp_enum_name) + # Create enum type using old values + old_type.create(op.get_bind()) + # Set enum type as type for event_type column + op.execute( + "ALTER TABLE clients ALTER COLUMN type_code TYPE " + + enum_name + + " USING type_code::text::" + + enum_name + ) + # Drop temp enum type + op.execute("DROP TYPE " + tmp_enum_name) diff --git a/app/tests/test_clients.py b/app/tests/test_clients.py index 2b16b71..3380c42 100644 --- a/app/tests/test_clients.py +++ b/app/tests/test_clients.py @@ -222,7 +222,7 @@ def test_send_message_raises_known_error_code(self): self.assertTrue(client.is_enabled_for_alerts) self.assertEqual(0, Event.query.count()) with mock.patch( - "airq.models.clients.send_sms", + "airq.models.clients.send_message", side_effect=TwilioRestException( "", "", code=TwilioErrorCode.OUT_OF_REGION.value ), @@ -239,7 +239,7 @@ def test_send_message_raises_unknown_error_code(self): self.assertTrue(client.is_enabled_for_alerts) self.assertEqual(0, Event.query.count()) with mock.patch( - "airq.models.clients.send_sms", + "airq.models.clients.send_message", side_effect=TwilioRestException("", "", code=77), ): with self.assertRaises(Exception): diff --git a/app/tests/test_sms.py b/app/tests/test_sms.py index 17fa8ce..9a5c9c1 100644 --- a/app/tests/test_sms.py +++ b/app/tests/test_sms.py @@ -739,12 +739,10 @@ def test_solicit_feedback(self): self.clock.advance() response = self.client.post( - "/sms/en", data={"Body": "This is some feedback", "From": "+13333333333"} + "/sms/en", data={"Body": "E", "From": "+13333333333"} ) self.assertEqual(200, response.status_code) - self.assert_event( - client_id, EventType.FEEDBACK_RECEIVED, feedback="This is some feedback" - ) + self.assert_event(client_id, EventType.FEEDBACK_RECEIVED, feedback="E") self.assertEqual(3, Event.query.count()) self.clock.advance() From e68c8bf8f0d705a14904e381a06b6cf5cf24c7b1 Mon Sep 17 00:00:00 2001 From: Ian Hoffman Date: Mon, 23 Aug 2021 23:14:53 -0700 Subject: [PATCH 2/4] test fixes --- app/tests/test_clients.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/test_clients.py b/app/tests/test_clients.py index 3380c42..4a05fe6 100644 --- a/app/tests/test_clients.py +++ b/app/tests/test_clients.py @@ -222,7 +222,7 @@ def test_send_message_raises_known_error_code(self): self.assertTrue(client.is_enabled_for_alerts) self.assertEqual(0, Event.query.count()) with mock.patch( - "airq.models.clients.send_message", + "airq.lib.twilio.send_message", side_effect=TwilioRestException( "", "", code=TwilioErrorCode.OUT_OF_REGION.value ), @@ -239,7 +239,7 @@ def test_send_message_raises_unknown_error_code(self): self.assertTrue(client.is_enabled_for_alerts) self.assertEqual(0, Event.query.count()) with mock.patch( - "airq.models.clients.send_message", + "airq.lib.twilio.send_message", side_effect=TwilioRestException("", "", code=77), ): with self.assertRaises(Exception): From 37a7497bb1ca7ef779fcfc0f95849efe74ab0821 Mon Sep 17 00:00:00 2001 From: Ian Hoffman Date: Mon, 23 Aug 2021 23:16:19 -0700 Subject: [PATCH 3/4] wip --- app/airq/models/clients.py | 6 +++++- app/tests/test_sms.py | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/airq/models/clients.py b/app/airq/models/clients.py index 76f8987..da962c2 100644 --- a/app/airq/models/clients.py +++ b/app/airq/models/clients.py @@ -359,7 +359,11 @@ def send_message(self, message: str, media: typing.Optional[str] = None) -> bool if self.identifier_type.can_receive_messages: try: send_message( - message, self.identifier, self.identifier_type, self.locale, media=media + message, + self.identifier, + self.identifier_type, + self.locale, + media=media, ) except TwilioRestException as e: code = TwilioErrorCode.from_exc(e) diff --git a/app/tests/test_sms.py b/app/tests/test_sms.py index 9a5c9c1..17fa8ce 100644 --- a/app/tests/test_sms.py +++ b/app/tests/test_sms.py @@ -739,10 +739,12 @@ def test_solicit_feedback(self): self.clock.advance() response = self.client.post( - "/sms/en", data={"Body": "E", "From": "+13333333333"} + "/sms/en", data={"Body": "This is some feedback", "From": "+13333333333"} ) self.assertEqual(200, response.status_code) - self.assert_event(client_id, EventType.FEEDBACK_RECEIVED, feedback="E") + self.assert_event( + client_id, EventType.FEEDBACK_RECEIVED, feedback="This is some feedback" + ) self.assertEqual(3, Event.query.count()) self.clock.advance() From dcb2113171f58c174616a3f031f554595c12ffb1 Mon Sep 17 00:00:00 2001 From: Ian Hoffman Date: Wed, 25 Aug 2021 22:13:58 -0700 Subject: [PATCH 4/4] wip --- app/airq/models/clients.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/airq/models/clients.py b/app/airq/models/clients.py index da962c2..5292c2e 100644 --- a/app/airq/models/clients.py +++ b/app/airq/models/clients.py @@ -428,6 +428,8 @@ def maybe_notify(self) -> bool: if was_alerted_recently and abs(curr_aqi - last_aqi) < 50: return False + # Warning! This is a template message used by Whatsapp. If you change it, make sure to add the new copy + # in the Twilio console or we won't be able to send it to Whatsapp users. message = gettext( 'Air quality in %(city)s %(zipcode)s has changed to %(curr_aqi_level)s (AQI %(curr_aqi)s).\n\nReply "M" for Menu or "E" to end alerts.', city=self.zipcode.city.name,