From d1076ad5275dac21049464dfe2ad05fddd9531aa Mon Sep 17 00:00:00 2001 From: Daniel Palafox Date: Mon, 13 Jul 2026 10:06:07 -0400 Subject: [PATCH] fix: update delete test data script and add instructions --- .../scripts/mongo-scripts-guide.md | 30 ++ assertion-service-2/scripts/setup-scripts.md | 15 - .../scripts/test-data/delete_documents.py | 162 ++++++++--- .../scripts/test-data/insert_documents.py | 272 ------------------ .../scripts/test-data/json/delete.json | 4 - assertion-service-2/scripts/utils/config.py | 3 +- 6 files changed, 148 insertions(+), 338 deletions(-) create mode 100644 assertion-service-2/scripts/mongo-scripts-guide.md delete mode 100644 assertion-service-2/scripts/setup-scripts.md delete mode 100644 assertion-service-2/scripts/test-data/insert_documents.py delete mode 100644 assertion-service-2/scripts/test-data/json/delete.json diff --git a/assertion-service-2/scripts/mongo-scripts-guide.md b/assertion-service-2/scripts/mongo-scripts-guide.md new file mode 100644 index 000000000..1b55e3d16 --- /dev/null +++ b/assertion-service-2/scripts/mongo-scripts-guide.md @@ -0,0 +1,30 @@ +# MONGO DB Scripts Setup Guide + +## Run scripts via helper + +Use `run-script.sh` from your local machine — no manual SSH or docker exec needed. + +```bash +# Interactive shell inside the container +./run-script.sh --username --server --assertion-docker + +# Run a script +./run-script.sh --username --server --assertion-docker --script test_connection.py +./run-script.sh --username --server --assertion-docker --script query-fixes/backfill_orcid_record.py +``` + +## Clean test data + +Use one of the options below to remove all affiliations shown in Affiliation Manager for a single member. + +```bash +# Option 1: use Salesforce member ID +./run-script.sh --username --server --assertion-docker --script "test-data/delete_documents.py --database assertionservice --collections assertion --member-salesforce-id " + +# Option 2: use internal member ID (memberservice.member._id) +./run-script.sh --username --server --assertion-docker --script "test-data/delete_documents.py --database assertionservice --collections assertion --member-id " +``` + +Notes: +- Use exactly one selector: `--member-salesforce-id` or `--member-id`. +- The script will prompt for confirmation before deleting data. diff --git a/assertion-service-2/scripts/setup-scripts.md b/assertion-service-2/scripts/setup-scripts.md deleted file mode 100644 index 73687cee9..000000000 --- a/assertion-service-2/scripts/setup-scripts.md +++ /dev/null @@ -1,15 +0,0 @@ -# MONGO DB Scripts Setup Guide - -## Run scripts via helper - -Use `run-script.sh` from your local machine — no manual SSH or docker exec needed. - -```bash -# Interactive shell inside the container -./run-script.sh --username --server --assertion-docker - -# Run a script -./run-script.sh --username --server --assertion-docker --script test_connection.py -./run-script.sh --username --server --assertion-docker --script query-fixes/backfill_orcid_record.py -./run-script.sh --username --server --assertion-docker --script test-data/delete_documents.py -``` diff --git a/assertion-service-2/scripts/test-data/delete_documents.py b/assertion-service-2/scripts/test-data/delete_documents.py index 893fb6a75..cade6de33 100644 --- a/assertion-service-2/scripts/test-data/delete_documents.py +++ b/assertion-service-2/scripts/test-data/delete_documents.py @@ -2,9 +2,7 @@ """ MongoDB Data Cleanup Script -This script removes documents from specified MongoDB collections using -filters defined in a JSON file. It is intended for targeted documents of -invalid, obsolete, or incorrect data. +This script removes affiliations for a target member from MongoDB. Related ticket: https://app.clickup.com/t/9014437828/PD-4941 @@ -17,34 +15,35 @@ --collections One or more collection names to clean up. ---file_name - Path to a JSON file containing the document filters to delete. +--member-id + Internal member ID. When provided, the script deletes all affiliations + listed for that member in Affiliation Manager (assertion.member_id). + +--member-salesforce-id + Salesforce member ID. The script resolves it via memberservice.member, + then deletes all affiliations listed in Affiliation Manager for the + resolved member ID. Behavior -------- -For each specified collection, the script reads the filters from the -provided JSON file and deletes documents that match those filters. - -Example JSON format: -[ - { "_id": "603f675aec57dc0008ae0952" }, - { "email": "test@orcid.org" } -] +The script builds the delete query from member scope and removes all matching +documents in the target collection. Usage ----- -python delete_documents.py --database --collections [collection2 ...] --file_name +python delete_documents.py --database assertionservice --collections assertion --member-id +python delete_documents.py --database assertionservice --collections assertion --member-salesforce-id Examples -------- -python delete_documents.py --database assertionservice --collections assertions orcid_record --file_name json/delete.json +python delete_documents.py --database assertionservice --collections assertion --member-id 689f6f2abc1234def56789ab """ import argparse -import json +import re import sys from pathlib import Path -from typing import List, Dict, Any +from typing import List, Dict, Any, Optional from pymongo.errors import OperationFailure from bson import ObjectId @@ -61,13 +60,27 @@ logger = setup_logger(__name__, log_file='delete_documents.log') +SALESFORCE_ID_PATTERN = re.compile(r"^[a-zA-Z0-9]{18}$") + class DeleteDocuments: - def __init__(self, connection: MongoDBConnection, collection_name: str, file_name: str): + def __init__( + self, + connection: MongoDBConnection, + collection_name: str, + inline_filters: Optional[List[Dict[str, Any]]] = None + ): self.connection = connection self.collection = connection.get_collection(collection_name) - self.file_name = file_name + self.inline_filters = inline_filters or [] + + def _load_items(self) -> Optional[List[Dict[str, Any]]]: + if self.inline_filters: + return self.inline_filters + + logger.error("No inline filters were provided") + return None def find_problematic_collections(self) -> List[Dict[str, Any]]: """ @@ -76,15 +89,9 @@ def find_problematic_collections(self) -> List[Dict[str, Any]]: Returns: List of problematic documents """ - try: - with open(self.file_name, 'r') as f: - items = json.load(f) - except FileNotFoundError: - print(f"Error: {self.file_name} not found.") - return - except json.JSONDecodeError: - print(f"Error: {self.file_name} contains invalid JSON.") - return + items = self._load_items() + if not items: + return [] cleaned_items = [self.prepare_item(item) for item in items] @@ -94,8 +101,8 @@ def find_problematic_collections(self) -> List[Dict[str, Any]]: logger.info(f"Searching for documents in collection '{self.collection.name}' where '{query}'") documents = list(self.collection.find(query)) for doc in documents: - logger.info(f"Document with _id '{doc['_id']}' is problematic") - logger.info(f"Found {len(documents)} documents to fix") + logger.info(f"Document with _id '{doc['_id']}' found") + logger.info(f"Found {len(documents)} documents to delete") return documents except OperationFailure as e: logger.error(f"Failed to query documents: {e}") @@ -106,11 +113,11 @@ def find_problematic_collections(self) -> List[Dict[str, Any]]: def print_report(self, documents: List[Dict[str, Any]]): if not documents: - logger.info("No problematic documents found") + logger.info("No assertions found") return logger.info("\n" + "="*80) - logger.info("PROBLEMATIC DOCUMENTS REPORT") + logger.info("ASSERTIONS DOCUMENTS REPORT") logger.info("="*80) status_counts = {} @@ -118,22 +125,22 @@ def print_report(self, documents: List[Dict[str, Any]]): status = doc.get('_id', 'Unknown') status_counts[status] = status_counts.get(status, 0) + 1 - logger.info(f"\nTotal documents to fix: {len(documents)}") + logger.info(f"\nTotal documents to delete: {len(documents)}") logger.info("\n" + "="*80) - def fix_documents(self, documents: List[Dict[str, Any]]) -> int: + def delete_documents(self, documents: List[Dict[str, Any]]) -> int: """ - Fix the status of problematic documents. + Delete the specified documents. Returns: - Number of documents successfully updated + Number of documents successfully deleted """ if not documents: - logger.info("No documents to fix") + logger.info("No documents to delete") return 0 - logger.info(f"\n Applying fixes to {len(documents)} documents...") + logger.info(f"\n Deleting {len(documents)} documents...") document_ids = [doc['_id'] for doc in documents if '_id' in doc and doc['_id'] is not None] @@ -192,18 +199,56 @@ def parse_arguments(): MONGO_URI or MONGO_DB - MongoDB connection string MONGO_DATABASE or DATABASE - Database name (default: assertionservice) MONGO_COLLECTION or COLLECTION - Collection name (default: assertion) - FILE_NAME - Collection name (default: json/delete.json) """ ) parser.add_argument('--mongo-uri', help='MongoDB URI (overrides env)') parser.add_argument('--database', help='MongoDB database name (overrides env)') parser.add_argument('--collections', help='MongoDB collection(s) name') - parser.add_argument('--file_name', help='File that contains the documents to delete') + parser.add_argument('--member-id', dest='member_id', help='Internal member ID to clean all affiliations for') + parser.add_argument( + '--member-salesforce-id', + dest='member_salesforce_id', + help='Salesforce ID to resolve member and clean all affiliations for that member' + ) return parser.parse_args() +def resolve_member_id_from_salesforce_id(mongo_uri: str, salesforce_id: str) -> Optional[str]: + if not SALESFORCE_ID_PATTERN.match(salesforce_id): + logger.error( + "Invalid Salesforce ID: %r. Expected exactly 18 alphanumeric characters.", + salesforce_id + ) + return None + + members_connection = MongoDBConnection(mongo_uri, 'memberservice') + try: + if not members_connection.connect(): + logger.error("Failed to connect to memberservice database to resolve Salesforce ID") + return None + + member_collection = members_connection.get_collection('member') + if member_collection is None: + logger.error("Could not access memberservice.member collection") + return None + + member = member_collection.find_one({'salesforce_id': salesforce_id}, {'_id': 1, 'salesforce_id': 1}) + if not member: + logger.error("No member found for salesforce_id=%s", salesforce_id) + return None + + member_id = str(member.get('_id')) + logger.info("Resolved member salesforce_id=%s to member_id=%s", salesforce_id, member_id) + return member_id + except Exception as e: + logger.error("Unexpected error resolving member by salesforce id: %s", e) + return None + finally: + members_connection.disconnect() + + def main(): args = parse_arguments() @@ -212,7 +257,33 @@ def main(): mongo_uri = args.mongo_uri or config.mongo_uri database = args.database or config.mongo_database collections = args.collections or config.mongo_collection - file_name = args.file_name or config.file_name + member_id = args.member_id + member_salesforce_id = args.member_salesforce_id + + if member_id and member_salesforce_id: + logger.error("Use either --member-id or --member-salesforce-id, not both") + return 1 + + if not member_id and not member_salesforce_id: + logger.error("A member selector is required. Use --member-id or --member-salesforce-id") + return 1 + + if member_salesforce_id: + resolved_member_id = resolve_member_id_from_salesforce_id(mongo_uri, member_salesforce_id) + if not resolved_member_id: + return 1 + member_id = resolved_member_id + + if member_id and collections != 'assertion': + logger.warning( + "Member cleanup is intended for Affiliation Manager data in 'assertion'. " + "Current collection is '%s'.", + collections, + ) + + inline_filters: List[Dict[str, Any]] = [] + if member_id: + inline_filters = [{'member_id': member_id}] logger.info("="*80) logger.info("MongoDB Delete Documents Script") @@ -220,7 +291,8 @@ def main(): logger.info(f"MongoDB URI: {mongo_uri[:20]}..." if len(mongo_uri) > 20 else f"MongoDB URI: {mongo_uri}") logger.info(f"Database: {database}") logger.info(f"Collection: {collections}") - logger.info(f"Filename: {file_name}") + if member_id: + logger.info(f"Member ID scope: {member_id}") logger.info("="*80 + "\n") connection = MongoDBConnection(mongo_uri, database) @@ -230,14 +302,14 @@ def main(): logger.error("Failed to connect to MongoDB. Exiting.") return 1 - fixer = DeleteDocuments(connection, collections, file_name) + fixer = DeleteDocuments(connection, collections, inline_filters) documents = fixer.find_problematic_collections() fixer.print_report(documents) if not documents: - logger.info("\n No fixes needed. All documents are correct.") + logger.info("\n No deletions needed.") return 0 logger.info("\n" + "="*80) @@ -254,7 +326,7 @@ def main(): logger.info("\n\n Operation cancelled by user") return 1 - deleted_count = fixer.fix_documents(documents) + deleted_count = fixer.delete_documents(documents) if deleted_count > 0: if not fixer.verify_fixes(): diff --git a/assertion-service-2/scripts/test-data/insert_documents.py b/assertion-service-2/scripts/test-data/insert_documents.py deleted file mode 100644 index 61da58fb9..000000000 --- a/assertion-service-2/scripts/test-data/insert_documents.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -""" -MongoDB Insert Documents Script - -This script inserts documents into a specified MongoDB collection using -data defined in a JSON file. It is intended for persisting test data -for validation and troubleshooting workflows. - -Related ticket: -https://app.clickup.com/t/9014437828/PD-5160 - -Parameters ----------- ---database - Name of the MongoDB database containing the collection. - ---collection - Collection name where the data will be inserted. - ---file_name - Path to a JSON file containing one document or a list of documents to insert. - -Behavior --------- -The script reads document data from the provided JSON file and inserts -it into the specified collection. - -Example JSON format: -[ - { - "email": "sideshow.bob.orcid@mailinator.com", - "affiliation_section": "SERVICE" - } -] - -Usage ------ -python insert_documents.py --database --collection --file_name - -Examples --------- -python3 insert_documents.py --file_name json/assertions.json -python3 insert_documents.py --collection orcid_record --file_name json/orcid_records.json -python3 insert_documents.py --database memberservice --collection member --file_name json/members.json -python3 insert_documents.py --database userservice --collection jhi_user --file_name json/users.json -""" - -import argparse -import sys -from pathlib import Path -from typing import List, Dict, Any, Optional - -from bson import json_util -from pymongo.errors import OperationFailure - -CURRENT_DIR = Path(__file__).resolve().parent -UTILS_DIR = CURRENT_DIR.parent / "utils" - -if str(UTILS_DIR) not in sys.path: - sys.path.insert(0, str(UTILS_DIR)) - -from logger_config import setup_logger -from db_connection import MongoDBConnection -from config import Config - - -logger = setup_logger(__name__, log_file='insert_documents.log') - - -def remove_mongo_oid(document: Dict[str, Any]) -> Dict[str, Any]: - cleaned_document = document.copy() - - if ( - "_id" in cleaned_document - and isinstance(cleaned_document["_id"], dict) - and "$oid" in cleaned_document["_id"] - ): - cleaned_document.pop("_id") - - return cleaned_document - - -class InsertDocuments: - - def __init__(self, connection: MongoDBConnection, collection_name: str, file_name: str): - self.connection = connection - self.collection = connection.get_collection(collection_name) - self.file_name = file_name - - def analyze_documents(self) -> Optional[List[Dict[str, Any]]]: - """ - Read document data from JSON file. - - Returns: - A list of documents, or None if the file is invalid. - """ - try: - with open(self.file_name, 'r', encoding='utf-8') as f: - documents = json_util.loads(f.read()) - - if isinstance(documents, dict): - cleaned_documents = [remove_mongo_oid(documents)] - logger.info("Loaded 1 document from JSON file") - return cleaned_documents - - if isinstance(documents, list): - if not all(isinstance(document, dict) for document in documents): - logger.error( - f"Error: {self.file_name} must contain only JSON objects." - ) - return None - - cleaned_documents = [ - remove_mongo_oid(document) for document in documents - ] - logger.info(f"Loaded {len(cleaned_documents)} documents from JSON file") - return cleaned_documents - - logger.error( - f"Error: {self.file_name} must contain either a JSON object " - f"or a JSON array of objects." - ) - return None - - except FileNotFoundError: - logger.error(f"Error: {self.file_name} not found.") - return None - except ValueError as e: - logger.error(f"Error: {self.file_name} contains invalid JSON. {e}") - return None - - def print_report(self, documents: List[Dict[str, Any]]): - if not documents: - logger.info("No documents found") - return - - logger.info("\n" + "=" * 80) - logger.info("DOCUMENT INSERT REPORT") - logger.info("=" * 80) - logger.info(f"Documents loaded: {len(documents)}") - logger.info(f"Target collection: {self.collection.name}") - logger.info("=" * 80) - - def insert_documents(self, documents: List[Dict[str, Any]]) -> int: - """ - Insert documents into the collection. - - Returns: - Number of documents successfully inserted - """ - if not documents: - logger.info("No documents to insert") - return 0 - - try: - logger.info(f"\nInserting {len(documents)} document(s)...") - - if len(documents) == 1: - result = self.collection.insert_one(documents[0]) - logger.info(f"Successfully inserted 1 document. (ID: {result.inserted_id})") - return 1 - - result = self.collection.insert_many(documents) - logger.info(f"Successfully inserted {len(result.inserted_ids)} documents") - return len(result.inserted_ids) - - except OperationFailure as e: - logger.error(f"Failed to insert documents: {e}") - return 0 - except Exception as e: - logger.error(f"Unexpected error during insertion: {e}") - return 0 - - -def parse_arguments(): - parser = argparse.ArgumentParser( - description='Insert documents into a MongoDB collection', - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - python insert_documents.py - python insert_documents.py --database assertionservice --collection assertion --file_name delete.json - -Environment Variables: - MONGO_URI or MONGO_DB - MongoDB connection string - MONGO_DATABASE or DATABASE - Database name - MONGO_COLLECTION or COLLECTION - Collection name - FILE_NAME - JSON file name - """ - ) - - parser.add_argument('--mongo-uri', help='MongoDB URI (overrides env)') - parser.add_argument('--database', help='MongoDB database name (overrides env)') - parser.add_argument('--collection', help='MongoDB collection name') - parser.add_argument('--file_name', help='File containing document(s) to insert') - - return parser.parse_args() - - -def main(): - args = parse_arguments() - - config = Config() - - mongo_uri = args.mongo_uri or config.mongo_uri - database = args.database or config.mongo_database - collection = args.collection or config.mongo_collection - file_name = args.file_name or config.file_name - - logger.info("=" * 80) - logger.info("MongoDB Insert Documents Script") - logger.info("=" * 80) - logger.info(f"MongoDB URI: {mongo_uri[:20]}..." if len(mongo_uri) > 20 else f"MongoDB URI: {mongo_uri}") - logger.info(f"Database: {database}") - logger.info(f"Collection: {collection}") - logger.info(f"Filename: {file_name}") - logger.info("=" * 80 + "\n") - - connection = MongoDBConnection(mongo_uri, database) - - try: - if not connection.connect(): - logger.error("Failed to connect to MongoDB. Exiting.") - return 1 - - inserter = InsertDocuments(connection, collection, file_name) - - documents = inserter.analyze_documents() - - if not documents: - logger.info("\nNo documents to insert.") - return 0 - - inserter.print_report(documents) - - logger.info("\n" + "=" * 80) - logger.info("WARNING: This will modify the database!") - logger.info(f"{len(documents)} document(s) will be inserted into collection '{collection}'") - logger.info("=" * 80) - - try: - response = input("\nDo you want to proceed? (yes/no): ").strip().lower() - if response not in ['yes', 'y']: - logger.info("\nOperation cancelled by user") - return 0 - except (KeyboardInterrupt, EOFError): - logger.info("\n\nOperation cancelled by user") - return 1 - - inserted_count = inserter.insert_documents(documents) - - if inserted_count == 0: - logger.error("No documents were inserted") - return 1 - - logger.info("\n" + "=" * 80) - logger.info("Script completed successfully") - logger.info("=" * 80) - return 0 - - except KeyboardInterrupt: - logger.info("\n\nOperation cancelled by user (Ctrl+C)") - return 1 - except Exception as e: - logger.error(f"\nUnexpected error: {e}", exc_info=True) - return 1 - finally: - connection.disconnect() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/assertion-service-2/scripts/test-data/json/delete.json b/assertion-service-2/scripts/test-data/json/delete.json deleted file mode 100644 index c83f60375..000000000 --- a/assertion-service-2/scripts/test-data/json/delete.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - { "email": "orcid-record-1@testingmissingorcidids.com" }, - { "email": "qa+mp0307220827@orcid.org" } -] diff --git a/assertion-service-2/scripts/utils/config.py b/assertion-service-2/scripts/utils/config.py index 9ca5c54c6..1b394d80d 100644 --- a/assertion-service-2/scripts/utils/config.py +++ b/assertion-service-2/scripts/utils/config.py @@ -4,7 +4,7 @@ """ import os -from typing import Dict, Optional +from typing import Dict class Config: @@ -20,7 +20,6 @@ def __init__(self): self.mongo_uri = self._get_mongo_uri() self.mongo_database = self._get_env('MONGO_DATABASE', 'assertionservice') self.mongo_collection = self._get_env('MONGO_COLLECTION', 'assertion') - self.file_name = self._get_env('FILE_NAME', 'json/delete.json') def _get_mongo_uri(self) -> str: return (