From d9dac219637fa3fd546c9fedbdf7c8fbf456d9a9 Mon Sep 17 00:00:00 2001 From: Ojasvi Harsola Date: Thu, 19 Jan 2023 19:09:16 +0530 Subject: [PATCH 1/3] Xola Connector Enhancement 1. Convert the Purchase Orders time to Pacific Time 2. Add Payment Method in the Orders stream --- .../sample_files/configured_catalog.json | 13 +++++++++++++ .../source-xola/source_xola/schemas/orders.json | 3 +++ .../connectors/source-xola/source_xola/source.py | 12 ++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-xola/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-xola/sample_files/configured_catalog.json index 48e1f6ba8c0f..b5e4a732cf8b 100644 --- a/airbyte-integrations/connectors/source-xola/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-xola/sample_files/configured_catalog.json @@ -22,6 +22,19 @@ } } }, + "items":{ + "type":["null","array"], + "default":[], + "items":{ + "type":["null","object"], + "properties":{ + "id":{ + "title":"paymentMethod", + "type":["null","string"] + } + } + } + }, "order_id":{ "type":["null","string"] }, diff --git a/airbyte-integrations/connectors/source-xola/source_xola/schemas/orders.json b/airbyte-integrations/connectors/source-xola/source_xola/schemas/orders.json index 52550f53ba6c..b0354413d8ca 100644 --- a/airbyte-integrations/connectors/source-xola/source_xola/schemas/orders.json +++ b/airbyte-integrations/connectors/source-xola/source_xola/schemas/orders.json @@ -25,6 +25,9 @@ "travelers":{ "type":["null","string"] }, + "paymentMethod":{ + "type":["null","string"] + }, "source":{ "type":["null","string"] }, diff --git a/airbyte-integrations/connectors/source-xola/source_xola/source.py b/airbyte-integrations/connectors/source-xola/source_xola/source.py index 75a260f1bbf0..72c443c8aa3c 100644 --- a/airbyte-integrations/connectors/source-xola/source_xola/source.py +++ b/airbyte-integrations/connectors/source-xola/source_xola/source.py @@ -14,6 +14,7 @@ import logging import traceback import re +import pytz LOGGER = logging.getLogger() @@ -174,6 +175,10 @@ def request_params( params['seller'] = self.seller_id return params + + def parse_datetime_to_pst(self, datetime_str): + datetime_object = datetime.fromisoformat(datetime_str) + return datetime_object.astimezone(pytz.timezone('US/Pacific')).strftime("%Y-%m-%dT%H:%M:%SZ") def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ @@ -191,9 +196,12 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp resp["tags"] = ",".join([tag['id'] for tag in data["tags"]]) else: resp["tags"] = "" + + if "items" in data.keys(): + resp["paymentMethod"] = ",".join([item['paymentMethod'] for item in data["items"]]) resp["order_id"] = data["id"] - if "createdAt" in data.keys(): resp["createdAt"] = data["createdAt"] + if "createdAt" in data.keys(): resp["createdAt"] = self.parse_datetime_to_pst(data["createdAt"]) if "customerName" in data.keys(): resp["customerName"] = data["customerName"] if "customerEmail" in data.keys(): resp["customerEmail"] = data["customerEmail"] @@ -215,7 +223,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp #if "quantity" in data.keys(): resp["quantity"] = data["quantity"] if "event" in data.keys(): resp["event"] = data["event"] if "amount" in data.keys(): resp["amount"] = data["amount"] - if "updatedAt" in data.keys(): resp["updatedAt"] = data["updatedAt"] + if "updatedAt" in data.keys(): resp["updatedAt"] = self.parse_datetime_to_pst(data["updatedAt"]) if "type" in data.keys(): resp["type"] = data["type"] modified_response.append(resp) From 19b06000b63c366485c1f1920ed26d36ab23eb91 Mon Sep 17 00:00:00 2001 From: Ojasvi Harsola Date: Fri, 20 Jan 2023 12:34:58 +0530 Subject: [PATCH 2/3] update xola connector version --- .../2cfcd96e-72ab-11ec-90d6-0242ac120003.json | 2 +- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-xola/Dockerfile | 2 +- .../connectors/source-xola/source_xola/source.py | 13 +++---------- 5 files changed, 7 insertions(+), 14 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2cfcd96e-72ab-11ec-90d6-0242ac120003.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2cfcd96e-72ab-11ec-90d6-0242ac120003.json index 241b3cc4ca8b..268f9590b6f7 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2cfcd96e-72ab-11ec-90d6-0242ac120003.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2cfcd96e-72ab-11ec-90d6-0242ac120003.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "2cfcd96e-72ab-11ec-90d6-0242ac120003", "name": "Xola", "dockerRepository": "blotout/source-xola", - "dockerImageTag": "0.1.23", + "dockerImageTag": "0.1.25", "documentationUrl": "https://github.com/blotoutio/source-xola/README.md" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d7726f055bd9..2c160f1b42c5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -669,7 +669,7 @@ - sourceDefinitionId: 2cfcd96e-72ab-11ec-90d6-0242ac120003 name: Xola dockerRepository: blotout/source-xola - dockerImageTag: 0.1.23 + dockerImageTag: 0.1.25 documentationUrl: https://github.com/blotoutio/source-xola/README.md - sourceDefinitionId: c7264528-791f-11ec-90d6-0242ac120003 name: Twitter Ads diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index af3ad02ed311..4483961a6a5f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6353,7 +6353,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [ ] -- dockerImage: "blotout/source-xola:0.1.23" +- dockerImage: "blotout/source-xola:0.1.25" spec: "documentationUrl": "https://github.com/blotoutio/source-xola/README.md" "connectionSpecification": diff --git a/airbyte-integrations/connectors/source-xola/Dockerfile b/airbyte-integrations/connectors/source-xola/Dockerfile index fb2b619d5eef..d715b75607c3 100644 --- a/airbyte-integrations/connectors/source-xola/Dockerfile +++ b/airbyte-integrations/connectors/source-xola/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.23 +LABEL io.airbyte.version=0.1.25 LABEL io.airbyte.name=blotout/source-xola diff --git a/airbyte-integrations/connectors/source-xola/source_xola/source.py b/airbyte-integrations/connectors/source-xola/source_xola/source.py index 72c443c8aa3c..1bacd401253b 100644 --- a/airbyte-integrations/connectors/source-xola/source_xola/source.py +++ b/airbyte-integrations/connectors/source-xola/source_xola/source.py @@ -1,4 +1,3 @@ -import datetime from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Dict @@ -14,7 +13,6 @@ import logging import traceback import re -import pytz LOGGER = logging.getLogger() @@ -176,10 +174,6 @@ def request_params( params['seller'] = self.seller_id return params - def parse_datetime_to_pst(self, datetime_str): - datetime_object = datetime.fromisoformat(datetime_str) - return datetime_object.astimezone(pytz.timezone('US/Pacific')).strftime("%Y-%m-%dT%H:%M:%SZ") - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ TODO: Override this method to define how a response is parsed. @@ -201,7 +195,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp resp["paymentMethod"] = ",".join([item['paymentMethod'] for item in data["items"]]) resp["order_id"] = data["id"] - if "createdAt" in data.keys(): resp["createdAt"] = self.parse_datetime_to_pst(data["createdAt"]) + if "createdAt" in data.keys(): resp["createdAt"] = data["createdAt"] if "customerName" in data.keys(): resp["customerName"] = data["customerName"] if "customerEmail" in data.keys(): resp["customerEmail"] = data["customerEmail"] @@ -219,11 +213,10 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp resp["createdBy"] = data["createdBy"] else: resp["createdBy"] = "" - - #if "quantity" in data.keys(): resp["quantity"] = data["quantity"] + if "event" in data.keys(): resp["event"] = data["event"] if "amount" in data.keys(): resp["amount"] = data["amount"] - if "updatedAt" in data.keys(): resp["updatedAt"] = self.parse_datetime_to_pst(data["updatedAt"]) + if "updatedAt" in data.keys(): resp["updatedAt"] = data["updatedAt"] if "type" in data.keys(): resp["type"] = data["type"] modified_response.append(resp) From ad9d4b318634077ffa19aae3b489d3bc92f9d136 Mon Sep 17 00:00:00 2001 From: Dharmendra Date: Fri, 17 Mar 2023 11:53:54 +0530 Subject: [PATCH 3/3] xola cxhanges --- .../source-xola/source_xola/schemas/transactions.json | 3 --- .../connectors/source-xola/source_xola/source.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-xola/source_xola/schemas/transactions.json b/airbyte-integrations/connectors/source-xola/source_xola/schemas/transactions.json index f67b7661d829..0582bc551952 100644 --- a/airbyte-integrations/connectors/source-xola/source_xola/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-xola/source_xola/schemas/transactions.json @@ -16,9 +16,6 @@ "createdAt": { "type":["null","string"] }, - "updatedAt": { - "type": ["null", "string"] - }, "createdBy": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-xola/source_xola/source.py b/airbyte-integrations/connectors/source-xola/source_xola/source.py index 1bacd401253b..a7e59c336cd5 100644 --- a/airbyte-integrations/connectors/source-xola/source_xola/source.py +++ b/airbyte-integrations/connectors/source-xola/source_xola/source.py @@ -454,7 +454,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class Transactions(IncrementalXolaStream): primary_key = "id" seller_id = None - cursor_field = "updatedAt" + cursor_field = "createdAt" def __init__(self, seller_id: str, x_api_key: str, **kwargs): super().__init__(x_api_key, **kwargs)