diff --git a/ReBACH_Logs_Summary_Description.md b/ReBACH_Logs_Summary_Description.md index d67e615..1a6330c 100644 --- a/ReBACH_Logs_Summary_Description.md +++ b/ReBACH_Logs_Summary_Description.md @@ -4,17 +4,17 @@ | Log Message | Description | |-------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Total matched unique articles | Total number of articles whose article_ids are found both in ReDATA and curation storage. This excludes already preserved articles.
**Note**: A preserved article can be in either of the archival storages. | +| Total matched unique articles | Total number of articles whose article_ids are found both in ReDATA and curation storage during preprocessing. This excludes already preserved articles.
**Note**: A preserved article can be in either of the archival storages. | | Total unmatched unique articles | Total number of articles whose article_ids are found in ReDATA but **not** in curation storage. This excludes already preserved articles. | | Total matched article versions | Total number of article versions whose article_ids are found in ReDATA and curation storage contains a folder of the version. This excludes already preserved versions. | | Total unmatched article versions | Total number of article versions whose article_ids are found in ReDATA but curation storage **does not** contain a folder of the version. This excludes already preserved versions. This number should always be zero. If non-zero, check the logs. | -| Total skipped unique articles | Total number of articles that are already preserved and are not processed. These articles have at least one version in archival staging storage or archival storage . | -| Total skipped article versions | Total number of article versions that are already preserved and are not processed. | +| Total skipped unique articles | Total number of articles that are found already preserved during preprocessing and are not processed. These articles have at least one version in archival staging storage or archival storage . | +| Total skipped article versions | Total number of article versions that are found already preserved during preprocessing and are not processed. | | Total articles | Total number of articles in ReDATA, both published and unpublished. | | Total published articles/article versions | Total number of published articles against total number of published versions in ReDATA. | | Total count of already preserved (skipped) articles / article versions | Total count of already preserved articles against total count of already preserved article versions.
**Note:** These figures include all articles and article versions already preserved in either of the remote archival storages and are skipped during processing. | -| Total count of articles with fetch error / articles | Total number of articles with error while fetching items either from ReDATA or curation storage against total number of published articles.
**Note**: Articles with fetch error are skipped during processing. | -| Total count of article versions with fetch error / article versions | Total number of article versions with error while fetching items either from ReDATA or curation storage against total number of published article versions.
**Note**: Article versions with fetch error are skipped during processing. | +| Total count of articles with fetch error / articles | Total number of articles with error while fetching items either from ReDATA (either API errors or file download errors) or curation storage against total number of published articles.
**Note**: Articles with fetch error are skipped during processing. | +| Total count of article versions with fetch error / article versions | Total number of article versions with error while fetching items either from ReDATA (either API errors or file download errors) or curation storage against total number of published article versions.
**Note**: Article versions with fetch error are skipped during processing. | | Total articles versions matched/published (unskipped) | Total number of article versions with folders in curation storage against total number of published article versions in ReDATA.
**Note**: The two numbers should be equal if nothing went wrong. If the first number is less than the second, it means there was an issue matching at least one of the published article versions that are published in ReDATA. The logs should be examined | | Total articles versions processed/matched | Total number of article versions successfully processed against total number of article versions in ReDATA with a folder in curation storage.
**Note**: The first number will always be less than or equal to the second. If the numbers are not equal, the log should be checked. | | Total count of already preserved article versions in archival storage | Total number of already preserved article versions in archival storage.
**Note:** These article versions are skipped during processing. | diff --git a/app.py b/app.py index d9aca10..9202787 100644 --- a/app.py +++ b/app.py @@ -154,14 +154,14 @@ def main(): log.write_log_in_file('info', " ", True) log.write_log_in_file('info', "------- Fetching articles -------", True) article_obj = Article(config, log, args.ids) - article_data, already_preserved_counts_dict = article_obj.get_articles() + article_data, skipped_items_counts_dict = article_obj.get_articles() - already_preserved_articles_count = len(already_preserved_counts_dict['already_preserved_article_ids']) - already_preserved_versions_count = already_preserved_counts_dict['already_preserved_versions'] - locally_preserved_article_version_count = already_preserved_counts_dict['articles_locally_preserved'] + already_preserved_articles_count = len(skipped_items_counts_dict['already_preserved_article_ids']) + already_preserved_versions_count = skipped_items_counts_dict['already_preserved_versions'] + locally_preserved_article_version_count = skipped_items_counts_dict['articles_locally_preserved'] - articles_with_error_count = len(already_preserved_counts_dict['articles_with_error']) - article_versions_with_error_count = already_preserved_counts_dict['article_versions_with_error'] + articles_with_fetch_error_count = len(skipped_items_counts_dict['articles_with_fetch_error']) + article_versions_with_fetch_error_count = skipped_items_counts_dict['article_versions_with_fetch_error'] published_articles_count = 0 published_articles_versions_count = 0 published_unpublished_count = 0 @@ -174,9 +174,9 @@ def main(): log.write_log_in_file('info', "Fetched: " + f"Total articles: {published_unpublished_count}, " - + f"Published articles: {published_articles_count + already_preserved_articles_count + articles_with_error_count}, " + + f"Published articles: {published_articles_count + already_preserved_articles_count + articles_with_fetch_error_count}, " + "Published article versions: " - + f"{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}", + + f"{published_articles_versions_count + already_preserved_versions_count + article_versions_with_fetch_error_count}", True) print(" ") @@ -195,8 +195,8 @@ def main(): print(" ") # Start articles processing after completing fetching data from API - processed_articles_versions_count, ap_trust_preserved_article_version_count, wasabi_preserved_versions \ - = article_obj.process_articles(article_data) + processed_articles_versions_count, ap_trust_preserved_article_version_count, wasabi_preserved_versions, articles_with_processing_error, \ + articles_versions_with_processing_error = article_obj.process_articles(article_data) # Start collections processing after completing fetching data from API and articles processing. processed_collections_versions_count, already_preserved_collections_counts = collection_obj.process_collections(collection_data) @@ -214,25 +214,26 @@ def main(): log.write_log_in_file('info', "Total published articles/article versions: \t\t\t\t\t" - + f'{published_articles_count + already_preserved_articles_count + articles_with_error_count} / ' - + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}', + + f'{published_articles_count + already_preserved_articles_count + articles_with_fetch_error_count} / ' + + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_fetch_error_count}', True) log.write_log_in_file('info', "Total count of already preserved (skipped) articles / article versions: \t\t" - + f'{already_preserved_articles_count} / {already_preserved_versions_count}', + + f'{already_preserved_articles_count} / ' + + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_fetch_error_count}', True) log.write_log_in_file('info', "Total count of articles with fetch error / articles: \t\t\t\t" - + f'{articles_with_error_count} / ' - + f'{published_articles_count + already_preserved_articles_count + articles_with_error_count}', + + f'{articles_with_fetch_error_count + articles_with_processing_error} / ' + + f'{published_articles_count + already_preserved_articles_count + articles_with_fetch_error_count}', True) log.write_log_in_file('info', "Total count of article versions with fetch error / article versions: \t\t" - + f'{article_versions_with_error_count} / ' - + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_error_count}', + + f'{article_versions_with_fetch_error_count + articles_versions_with_processing_error} / ' + + f'{published_articles_versions_count + already_preserved_versions_count + article_versions_with_fetch_error_count}', True) if article_obj.processor.duplicate_bag_in_preservation_storage_count > 0: diff --git a/figshare/Article.py b/figshare/Article.py index 162e258..378e1a7 100644 --- a/figshare/Article.py +++ b/figshare/Article.py @@ -29,7 +29,6 @@ class Article: def __init__(self, config, log, ids): self.config_obj = config figshare_config = self.config_obj.figshare_config() - # self.aptrust_config = self.config_obj.aptrust_config() self.system_config = self.config_obj.system_config() self.api_endpoint = figshare_config["url"] self.api_token = figshare_config["token"] @@ -54,11 +53,16 @@ def __init__(self, config, log, ids): self.matched_curation_folder_list = [] self.no_matched = 0 self.no_unmatched = 0 - self.already_preserved_counts_dict = {'already_preserved_article_ids': set(), 'already_preserved_versions': 0, - 'wasabi_preserved_versions': 0, 'ap_trust_preserved_versions': 0, - 'articles_with_error': set(), 'article_versions_with_error': 0, 'articles_staged': 0, - 'articles_locally_preserved': 0 - } + + # This dict is used to store counts of skipped items. Set articles_with_fetch_error keeps article_ids that failed during preprocessing. + # while set articles_with_processing_error keeps articled_ids that failed during processing and their intersection is o to avoid + # double counting. Items in articles_(version)_with_processing_error are included in published_articles_(version) count. + self.skipped_items_counts_dict = {'already_preserved_article_ids': set(), 'already_preserved_versions': 0, + 'wasabi_preserved_versions': 0, 'ap_trust_preserved_versions': 0, + 'articles_with_fetch_error': set(), 'article_versions_with_fetch_error': 0, 'articles_staged': 0, + 'articles_with_processing_error': set(), + 'articles_locally_preserved': 0, 'articles_versions_with_processing_error': 0 + } self.skipped_article_versions = {} self.processor = Integration(self.config_obj, self.logs) @@ -123,7 +127,7 @@ def get_articles(self): if (retries > self.retries): break - return article_data, self.already_preserved_counts_dict + return article_data, self.skipped_items_counts_dict def article_loop(self, articles, page_size, page, article_data): no_of_article = 0 @@ -167,8 +171,9 @@ def __get_article_versions(self, article): article_version = 'v' + str(version['version']).zfill(2) if version['version'] <= 9 \ else 'v' + str(version['version']) article_id = str(article['id']) - self.skipped_article_versions[article_id] = [] - self.skipped_article_versions[article_id].append(article_version) + if article_id not in self.skipped_article_versions.keys(): + self.skipped_article_versions[article_id] = set() + self.skipped_article_versions[article_id].add(article_version) continue metadata.append(version_data) else: @@ -277,8 +282,8 @@ def __get_article_metadata_by_version(self, version, article_id): f"Curation folder for article {article_id} version {version['version']} not found.", True) self.logs.write_log_in_file("info", "Aborting execution.", True, True) - self.already_preserved_counts_dict['articles_with_error'].add(article_id) - self.already_preserved_counts_dict['article_versions_with_error'] += 1 + self.skipped_items_counts_dict['articles_with_fetch_error'].add(article_id) + self.skipped_items_counts_dict['article_versions_with_fetch_error'] += 1 self.logs.write_log_in_file("error", f"Curation folder for article {article_id} version {version['version']} not found." + " Article version will be skipped.", @@ -325,7 +330,7 @@ def __get_article_metadata_by_version(self, version, article_id): # Comparison in archival staging storage (final local) if compare_hash(version_md5, version_local_final_preserved_list): already_preserved = True - self.already_preserved_counts_dict['articles_locally_preserved'] += 1 + self.skipped_items_counts_dict['articles_locally_preserved'] += 1 self.logs.write_log_in_file("info", f"Article {article_id} version {version['version']} " + "already preserved in archival staging storage", @@ -334,7 +339,7 @@ def __get_article_metadata_by_version(self, version, article_id): # Comparison in archival storage (final remote) if compare_hash(version_md5, version_final_storage_preserved_list): already_preserved = in_ap_trust = True - self.already_preserved_counts_dict['ap_trust_preserved_versions'] += 1 + self.skipped_items_counts_dict['ap_trust_preserved_versions'] += 1 self.logs.write_log_in_file("info", f"Article {article_id} version {version['version']} " + "already preserved in archival storage.", @@ -351,7 +356,7 @@ def __get_article_metadata_by_version(self, version, article_id): # Comparison in alternative archival staging storage (remote) if compare_hash(version_md5, version_staging_storage_preserved_list): - self.already_preserved_counts_dict['wasabi_preserved_versions'] += 1 + self.skipped_items_counts_dict['wasabi_preserved_versions'] += 1 self.logs.write_log_in_file("info", f"Article {article_id} version {version['version']} " + "already preserved in alternative archival staging storage.", True) @@ -359,8 +364,8 @@ def __get_article_metadata_by_version(self, version, article_id): in_alternative_remote_storage = True if already_preserved and upload_item and in_alternative_remote_storage: - self.already_preserved_counts_dict['already_preserved_article_ids'].add(article_id) - self.already_preserved_counts_dict['already_preserved_versions'] += 1 + self.skipped_items_counts_dict['already_preserved_article_ids'].add(article_id) + self.skipped_items_counts_dict['already_preserved_versions'] += 1 if in_ap_trust: for version_hash in version_final_storage_preserved_list: if version_hash[0] == version_md5 and version_hash[1] != payload_size: @@ -370,12 +375,12 @@ def __get_article_metadata_by_version(self, version, article_id): True) return None elif not already_preserved and upload_item and in_alternative_remote_storage: - self.already_preserved_counts_dict['already_preserved_article_ids'].add(article_id) - self.already_preserved_counts_dict['already_preserved_versions'] += 1 + self.skipped_items_counts_dict['already_preserved_article_ids'].add(article_id) + self.skipped_items_counts_dict['already_preserved_versions'] += 1 return None elif already_preserved and not upload_item and in_alternative_remote_storage: - self.already_preserved_counts_dict['already_preserved_article_ids'].add(article_id) - self.already_preserved_counts_dict['already_preserved_versions'] += 1 + self.skipped_items_counts_dict['already_preserved_article_ids'].add(article_id) + self.skipped_items_counts_dict['already_preserved_versions'] += 1 if in_ap_trust: for version_hash in version_final_storage_preserved_list: if version_hash[0] == version_md5 and version_hash[1] != payload_size: @@ -385,8 +390,8 @@ def __get_article_metadata_by_version(self, version, article_id): True) return None elif already_preserved and not upload_item and not in_alternative_remote_storage: - self.already_preserved_counts_dict['already_preserved_article_ids'].add(article_id) - self.already_preserved_counts_dict['already_preserved_versions'] += 1 + self.skipped_items_counts_dict['already_preserved_article_ids'].add(article_id) + self.skipped_items_counts_dict['already_preserved_versions'] += 1 if in_ap_trust: for version_hash in version_final_storage_preserved_list: if version_hash[0] == version_md5 and version_hash[1] != payload_size: @@ -439,6 +444,45 @@ def set_version_metadata(self, version_data, files, private_version_no, version_ } } + """ + Fetch article version files list, even if version files are private. + Could return None if request fails, otherwise returns a list + :param article_id int value. + :param version int value. + """ + def get_article_version_files(self, article_id: int, version: int): + success = False + retries = 1 + page_size = 100 + page = 1 + while not success and retries <= self.retries: + try: + page_empty = False + header = {'Authorization': 'token ' + self.api_token} + article_version_files_api = self.api_endpoint + "/articles/{0}/versions/{1}/files".format(article_id, version) + article_version_files = [] + while not page_empty: + params = {'page': page, 'page_size': page_size} + get_response = requests.get(article_version_files_api, headers=header, params=params, timeout=self.retry_wait) + if get_response.status_code == 200: + if len(get_response.json()) > 0: + article_version_files += get_response.json() + page += 1 + else: + page_empty = True + success = True + else: + retries = self.retries_if_error(f"{article_id} Public API not reachable. Retry {retries}", + get_response.status_code, retries) + if retries > self.retries: + return None + return article_version_files + except requests.exceptions.RequestException as e: + error_code = get_response.status_code if 'get_response' in locals() else 500 + retries = self.retries_if_error(f"{e}. Retry {retries}", error_code, retries) + if retries > self.retries: + return None + def private_article_for_files(self, version_data): get_response = requests.get(version_data['url_private_api'], headers={'Authorization': 'token ' + self.api_token}, @@ -922,9 +966,9 @@ def find_matched_articles(self, articles): self.logs.write_log_in_file("info", f"Total matched article versions: {self.no_matched}.", True) self.logs.write_log_in_file("info", f"Total unmatched article versions: {self.no_unmatched}.", True) self.logs.write_log_in_file("info", "Total skipped unique articles: " - + f"{len(self.already_preserved_counts_dict['already_preserved_article_ids'])}.", True) + + f"{len(self.skipped_items_counts_dict['already_preserved_article_ids'])}.", True) self.logs.write_log_in_file("info", "Total skipped article versions: " - + f"{self.already_preserved_counts_dict['already_preserved_versions']}.", True) + + f"{self.skipped_items_counts_dict['already_preserved_versions']}.", True) if len(set(unmatched_articles)) > 0 or len(self.article_non_match_info) > 0: self.logs.write_log_in_file("warning", "There were unmatched articles or article versions." @@ -965,14 +1009,19 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold success = success & self.__save_json_in_metadata(version_data, folder_name) if check_files and copy_files: - try: - # download all files and verify hash with downloaded file. - delete_now = self.__download_files(version_data['files'], version_data, folder_name) - except Exception as e: - self.logs.write_log_in_file("error", f"{str(e)} for {'_'.join(os.path.basename(folder_name).split('_')[0:-1])}" , True) - if self.system_config['continue-on-error'] == "False": - self.logs.write_log_in_file("info", "Aborting execution.", True, True) + article_version_files = self.get_article_version_files(int(version_data['id']), int(version_data['version'])) + if article_version_files is None and len(version_data['files']) > 1: + self.logs.write_log_in_file("error", f"Error retrieving file list for article {version_data['id']} {version_no}.", True) delete_now = True + else: + try: + # download all files and verify hash with downloaded file. + delete_now = self.__download_files(article_version_files, version_data, folder_name) + except Exception as e: + self.logs.write_log_in_file("error", f"{str(e)} for {'_'.join(os.path.basename(folder_name).split('_')[0:-1])}." , True) + if self.system_config['continue-on-error'] == "False": + self.logs.write_log_in_file("info", "Aborting execution.", True, True) + delete_now = True # check if download process has error or not. if (delete_now is False): @@ -1005,6 +1054,12 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold # if download process has any errors then delete complete folder self.logs.write_log_in_file("info", "Download process had an error so complete folder is being deleted.", True) self.delete_folder(check_dir) + if version_data['id'] not in self.skipped_items_counts_dict['articles_with_fetch_error']: + self.skipped_items_counts_dict['articles_with_processing_error'].add(version_data['id']) + self.skipped_items_counts_dict['articles_versions_with_processing_error'] += 1 + if str(version_data['id']) not in self.skipped_article_versions.keys(): + self.skipped_article_versions[str(version_data['id'])] = set() + self.skipped_article_versions[str(version_data['id'])].add(format_version(version_data['version'])) success = False else: if check_files or copy_files: @@ -1173,8 +1228,9 @@ def process_articles(self, articles): if (value_post_process != 0): self.logs.write_log_in_file("error", f"{version_data['id']} version {version_data['version']} - " + "Post-processing script failed.", True) - return processed_count, self.already_preserved_counts_dict['ap_trust_preserved_versions'], \ - self.already_preserved_counts_dict['wasabi_preserved_versions'] + return processed_count, self.skipped_items_counts_dict['ap_trust_preserved_versions'], \ + self.skipped_items_counts_dict['wasabi_preserved_versions'], len(self.skipped_items_counts_dict['articles_with_processing_error']), \ + self.skipped_items_counts_dict['articles_versions_with_processing_error'] """ Preservation and Curation directory access check while processing.