From 49ef17cb5ebe73ba938b73c0342fc0eeb5d856ec Mon Sep 17 00:00:00 2001 From: Jakob van Santen Date: Wed, 16 Aug 2023 17:13:53 +0200 Subject: [PATCH 1/2] Use hash of body as datapoint id --- ampel/lsst/ingest/LSSTDataPointShaper.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ampel/lsst/ingest/LSSTDataPointShaper.py b/ampel/lsst/ingest/LSSTDataPointShaper.py index 74563b0..bb94b5b 100755 --- a/ampel/lsst/ingest/LSSTDataPointShaper.py +++ b/ampel/lsst/ingest/LSSTDataPointShaper.py @@ -49,10 +49,13 @@ def process( # type: ignore[override] Non detection limit don't have an identifier. """ + id = hash_payload( + encode(dict(sorted(photo_dict.items()))), + size=-self.digest_size * 8, + ) if "diaSourceId" in photo_dict: - id = photo_dict["diaSourceId"] tags.append("LSST_DP") - sourceid_list.add(id) + sourceid_list.add(photo_dict["diaSourceId"]) elif "diaForcedSourceId" in photo_dict: id = photo_dict["diaForcedSourceId"] tags.append("LSST_FP") @@ -63,10 +66,6 @@ def process( # type: ignore[override] tags.append("LSST_OBJ") else: # Nondetection Limit - id = hash_payload( - encode(dict(sorted(photo_dict.items()))), - size=-self.digest_size * 8, - ) tags.append("LSST_ND") print("OOOOOO do we have upper limits?") ret_list.append( @@ -79,5 +78,5 @@ def process( # type: ignore[override] return [ dp for dp in ret_list - if not (dp["id"] in sourceid_list and "LSST_FP" in dp["tag"]) + if not (dp["body"].get("diaSourceId") in sourceid_list and "LSST_FP" in dp["tag"]) ] From 073330c62b6596fb6265f8b64fa33280a8a1504c Mon Sep 17 00:00:00 2001 From: wombaugh Date: Tue, 22 Aug 2023 10:20:38 +0200 Subject: [PATCH 2/2] Consistently use hashed IDs + consistently reject duplicated forced dps. --- ampel/lsst/ingest/LSSTDataPointShaper.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ampel/lsst/ingest/LSSTDataPointShaper.py b/ampel/lsst/ingest/LSSTDataPointShaper.py index bb94b5b..3aa366e 100755 --- a/ampel/lsst/ingest/LSSTDataPointShaper.py +++ b/ampel/lsst/ingest/LSSTDataPointShaper.py @@ -57,12 +57,10 @@ def process( # type: ignore[override] tags.append("LSST_DP") sourceid_list.add(photo_dict["diaSourceId"]) elif "diaForcedSourceId" in photo_dict: - id = photo_dict["diaForcedSourceId"] tags.append("LSST_FP") elif "diaObjectId" in photo_dict: # DiaObject # diaObjectId is also used in (prv)diaSource and diaForcedPhotometry # if other fields are added, check if they contain diaObjectId - id = photo_dict["diaObjectId"] tags.append("LSST_OBJ") else: # Nondetection Limit @@ -78,5 +76,5 @@ def process( # type: ignore[override] return [ dp for dp in ret_list - if not (dp["body"].get("diaSourceId") in sourceid_list and "LSST_FP" in dp["tag"]) + if not (dp["body"].get("diaForcedSourceId") in sourceid_list and "LSST_FP" in dp["tag"]) ]