diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 04175b61e1..cbf37868a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -61,15 +61,15 @@ jobs: run: | poetry sync - - name: Run Server-side linting with pytest - # Only run on new files for now-- for all changes, filter is ACMRTUXB - # Check manage.py to prevent a crash if no files are selected. + - name: Run Server-side linting with Ruff run: | - poetry run pylint $(git diff --name-only --diff-filter=A origin/${{github.base_ref}} | grep -E "(.py$)") manage.py + poetry run ruff check designsafe/apps - - name: Run Server-side formatting with black + - name: Run Server-side formatting with Ruff + # Only run on new files for now-- for all changes, filter is ACMRTUXB + # Check manage.py to prevent a crash if no files are selected. run: | - poetry run black $(git diff --name-only --diff-filter=A origin/${{github.base_ref}} | grep -E "(.py$)") manage.py --check + poetry run ruff format $(git diff --name-only --diff-filter=A origin/${{github.base_ref}} | grep -E "(.py$)") manage.py --check Client_Side_Unit_Tests: runs-on: ubuntu-22.04 diff --git a/designsafe/apps/accounts/admin.py b/designsafe/apps/accounts/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/accounts/admin.py +++ b/designsafe/apps/accounts/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/accounts/apps.py b/designsafe/apps/accounts/apps.py index 3c809d22b6..3222e5d9f8 100644 --- a/designsafe/apps/accounts/apps.py +++ b/designsafe/apps/accounts/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig + class DesignSafeAccountsConfig(AppConfig): name = 'designsafe.apps.accounts' label = 'designsafe_accounts' diff --git a/designsafe/apps/accounts/forms.py b/designsafe/apps/accounts/forms.py index 72903f0022..4b366797dd 100644 --- a/designsafe/apps/accounts/forms.py +++ b/designsafe/apps/accounts/forms.py @@ -1,19 +1,23 @@ -import re import logging +import re + from django import forms from django.contrib.auth import get_user_model from django.urls import reverse -from django.utils.translation import gettext as _ from django.utils.safestring import mark_safe -from django.utils.html import escape +from django.utils.translation import gettext as _ +from pytas.http import TASClient from snowpenguin.django.recaptcha2.fields import ReCaptchaField from snowpenguin.django.recaptcha2.widgets import ReCaptchaWidget - -from .models import (DesignSafeProfile, NotificationPreferences, - DesignSafeProfileNHInterests, DesignSafeProfileResearchActivities, - DesignSafeProfileNHTechnicalDomains) from termsandconditions.models import TermsAndConditions, UserTermsAndConditions -from pytas.http import TASClient + +from .models import ( + DesignSafeProfile, + DesignSafeProfileNHInterests, + DesignSafeProfileNHTechnicalDomains, + DesignSafeProfileResearchActivities, + NotificationPreferences, +) LOGGER = logging.getLogger(__name__) @@ -161,7 +165,7 @@ class ChangePasswordForm(forms.Form): def __init__(self, *args, **kwargs): self._username = kwargs.pop('username') - super(ChangePasswordForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def clean(self): cleaned_data = self.cleaned_data @@ -182,8 +186,8 @@ def clean(self): err_msg = mark_safe( 'The current password you provided is incorrect. Please try again. ' 'If you do not remember your current password you can ' - 'reset your password with an email ' - 'confirmation.' % reset_link) + f'reset your password with an email ' + 'confirmation.') self.add_error('current_password', err_msg) def save(self): @@ -218,7 +222,8 @@ def clean(self): try: tas = TASClient() user = tas.get_user(username=username) - except: + except Exception: + LOGGER.exception("") msg = 'The username provided does not match an existing user.' self.add_error('username', msg) raise forms.ValidationError(msg) @@ -244,7 +249,7 @@ class UserProfileForm(forms.Form): gender = forms.ChoiceField(label='Gender', choices=GENDER_OPTIONS) def __init__(self, *args, **kwargs): - super(UserProfileForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class TasUserProfileAdminForm(forms.Form): @@ -380,7 +385,7 @@ class UserRegistrationForm(UserProfileForm, ProfessionalProfileForm): 'bio', 'website', 'orcid_id', 'professional_level', 'research_activities'] def __init__(self, *args, **kwargs): - super(UserRegistrationForm, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.fields['institutionId'].choices = get_institution_choices() self.fields['institutionId'].choices += (('-1', 'My Institution is not listed'),) self.fields['countryId'].choices = get_country_choices() @@ -438,7 +443,7 @@ def save(self, source='DesignSafe', pi_eligibility=INELIGIBLE): safe_data = tas_data.copy() safe_data['password'] = safe_data['confirmPassword'] = '********' - LOGGER.info('Attempting new user registration: %s' % safe_data) + LOGGER.info('Attempting new user registration: %s', safe_data) tas_user = TASClient().save_user(None, tas_data) # create local user @@ -447,7 +452,7 @@ def save(self, source='DesignSafe', pi_eligibility=INELIGIBLE): # the user should not exist user = UserModel.objects.get(username=data['username']) LOGGER.warning('On TAS registration, local user already existed? ' - 'user=%s' % user) + 'user=%s', user) except UserModel.DoesNotExist: user = UserModel.objects.create_user( username=data['username'], @@ -486,13 +491,13 @@ def save(self, source='DesignSafe', pi_eligibility=INELIGIBLE): pro_profile.save() # terms of use - LOGGER.info('Prior to Registration, %s %s <%s> agreed to Terms of Use' % ( - data['firstName'], data['lastName'], data['email'])) + LOGGER.info('Prior to Registration, %s %s <%s> agreed to Terms of Use', + data['firstName'], data['lastName'], data['email']) try: terms = TermsAndConditions.get_active() user_terms = UserTermsAndConditions(user=user, terms=terms) user_terms.save() - except: + except Exception: LOGGER.exception('Error saving UserTermsAndConditions for user=%s', user) return tas_user diff --git a/designsafe/apps/accounts/integrations.py b/designsafe/apps/accounts/integrations.py index af8d4780f2..f610bd46a3 100644 --- a/designsafe/apps/accounts/integrations.py +++ b/designsafe/apps/accounts/integrations.py @@ -1,7 +1,7 @@ -from django.conf import settings -from importlib import import_module import logging +from importlib import import_module +from django.conf import settings logger = logging.getLogger(__name__) @@ -11,15 +11,19 @@ def get_integrations(): for app in settings.INSTALLED_APPS: try: - mod = import_module('%s.integrations' % app) + mod = import_module(f"{app}.integrations") try: app_integrations += mod.provide_integrations() except AttributeError: continue - except: - logger.warning('Call to module.provide_integrations fail for module: %s' % mod.__name__) - except: + except Exception: + logger.exception("") + logger.warning( + "Call to module.provide_integrations fail for module: %s", + mod.__name__, + ) + except Exception: + logger.exception("") continue return app_integrations - diff --git a/designsafe/apps/accounts/models.py b/designsafe/apps/accounts/models.py index ddd0ce232a..414e2607e2 100644 --- a/designsafe/apps/accounts/models.py +++ b/designsafe/apps/accounts/models.py @@ -1,16 +1,16 @@ +import logging + +import six from django.conf import settings -from django.db import models -from django.db import connections, DatabaseError -from django.utils.translation import gettext_lazy as _ from django.core.mail import send_mail +from django.db import DatabaseError, connections, models +from django.utils.translation import gettext_lazy as _ from pytas.http import TASClient -import logging -import six logger = logging.getLogger(__name__) -class NEESUser(object): +class NEESUser: def __init__(self, **kwargs): for k, v in six.iteritems(kwargs): @@ -123,6 +123,6 @@ class NotificationPreferences(models.Model): class Meta: permissions = ( - ('view_notification_subscribers', 'Can view list of users subscribed to a ' - 'notification type'), + ('view_notification_subscribers', ('Can view list of users subscribed to a ' + 'notification type')), ) diff --git a/designsafe/apps/accounts/tasks.py b/designsafe/apps/accounts/tasks.py index 1ed33b62f5..8e22022bb7 100644 --- a/designsafe/apps/accounts/tasks.py +++ b/designsafe/apps/accounts/tasks.py @@ -1,16 +1,13 @@ import csv import io import logging -from django.conf import settings + # from agavepy.agave import Agave, AgaveException from celery import shared_task -from requests import HTTPError +from django.conf import settings from django.contrib.auth import get_user_model from pytas.models import User as TASUser -from django.db.models import Q -from designsafe.apps.accounts.models import (DesignSafeProfile, - NotificationPreferences) - +from requests import HTTPError logger = logging.getLogger(__name__) @@ -59,14 +56,14 @@ def create_report(username, list_name): research_activities = [activity['description'] for activity in activities] # order of items as required by user - writer.writerow([user_profile.lastName if user_profile.lastName else user_profile.lastName, - user_profile.firstName if user_profile.firstName else user_profile.firstName, + writer.writerow([user_profile.lastName, + user_profile.firstName, user_profile.email, user_profile.phone, user_profile.institution, user_profile.title, designsafe_user.profile.professional_level, - designsafe_user.profile.bio if designsafe_user.profile.bio else designsafe_user.profile.bio, + designsafe_user.profile.bio, nh_interests if nh_interests else None, research_activities if research_activities else None, user_profile, @@ -79,13 +76,14 @@ def create_report(username, list_name): else: writer.writerow(['Unable to find user data for username "' + user.username + '"', ]) - except: + except Exception: + logger.exception("") continue User = get_user_model().objects.get(username=username) client = User.agave_oauth.client - setattr(csv_file, 'name', 'user_report.csv') + csv_file.name = 'user_report.csv' client.files.importData( filePath=username, fileName='user_report.csv', @@ -95,7 +93,7 @@ def create_report(username, list_name): csv_file.close() - except (HTTPError, AgaveException): + except (HTTPError, AgaveException): # noqa logger.exception('Failed to create user report.', extra={'user': username, 'systemId': settings.AGAVE_STORAGE_SYSTEM}) diff --git a/designsafe/apps/accounts/tests.py b/designsafe/apps/accounts/tests.py index 08f775d209..034d9abdf1 100644 --- a/designsafe/apps/accounts/tests.py +++ b/designsafe/apps/accounts/tests.py @@ -1,10 +1,11 @@ -from django.test import TestCase -from django.contrib.auth import get_user_model, signals -from django.contrib.auth.models import Permission -from django.urls import reverse from unittest import skip -from mock import patch +from unittest.mock import patch + import pytest +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Permission +from django.test import TestCase +from django.urls import reverse pytestmark = pytest.mark.django_db @@ -64,7 +65,7 @@ def test_mailing_list_subscribe(self): resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertContains( - resp, '"{0}","{1}"'.format(ds_user.get_full_name(), ds_user.email)) + resp, f'"{ds_user.get_full_name()}","{ds_user.email}"') def test_mailing_list_unsubscribe(self): """ @@ -83,7 +84,7 @@ def test_mailing_list_unsubscribe(self): resp = self.client.get(url) self.assertEqual(resp.status_code, 200) self.assertNotContains( - resp, '"{0}","{1}"'.format(ds_user.get_full_name(), ds_user.email)) + resp, f'"{ds_user.get_full_name()}","{ds_user.email}"') @skip("Need to mock celery call") def test_user_report_access(self): diff --git a/designsafe/apps/accounts/urls.py b/designsafe/apps/accounts/urls.py index 7d54f11d58..13fc05db97 100644 --- a/designsafe/apps/accounts/urls.py +++ b/designsafe/apps/accounts/urls.py @@ -1,6 +1,7 @@ -from django.urls import include, re_path as url +from django.urls import re_path as url from django.urls import reverse from django.utils.translation import gettext_lazy as _ + from designsafe.apps.accounts import views urlpatterns = [ diff --git a/designsafe/apps/accounts/views.py b/designsafe/apps/accounts/views.py index 44e7f2988e..84f6d4d6d1 100644 --- a/designsafe/apps/accounts/views.py +++ b/designsafe/apps/accounts/views.py @@ -1,24 +1,29 @@ -from django.shortcuts import render +import logging +import re + +import requests +from django.conf import settings from django.contrib import messages from django.contrib.auth import get_user_model, logout from django.contrib.auth.decorators import login_required, permission_required from django.core.exceptions import ObjectDoesNotExist -from django.urls import reverse -from django.conf import settings from django.db.models import Q from django.http import HttpResponse, HttpResponseRedirect +from django.shortcuts import render +from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from designsafe.apps.accounts import forms, integrations -from designsafe.apps.accounts.models import (NEESUser, DesignSafeProfile, - NotificationPreferences) -from designsafe.apps.accounts.tasks import create_report from pytas.http import TASClient from pytas.models import User as TASUser -import logging -import requests -import re from termsandconditions.models import TermsAndConditions +from designsafe.apps.accounts import forms, integrations +from designsafe.apps.accounts.models import ( + DesignSafeProfile, + NEESUser, + NotificationPreferences, +) +from designsafe.apps.accounts.tasks import create_report + logger = logging.getLogger(__name__) @@ -42,9 +47,9 @@ def manage_profile(request): try: demographics = django_user.profile - except ObjectDoesNotExist as e: + except ObjectDoesNotExist: demographics = {} - logger.info('exception e:{} {}'.format(type(e), e)) + logger.exception("") context = { 'title': 'Manage Account', @@ -144,10 +149,9 @@ def nees_migration(request, step=None): if len(nees_user_match) == 0: messages.error(request, 'We were unable to locate a NEEShub account for the email ' - 'address %s. Please confirm that you entered the ' + f'address {email_address}. Please confirm that you entered the ' 'email address correctly and try again. If you feel this ' - 'is in error, please submit a support ticket.' % - email_address) + 'is in error, please submit a support ticket.') return HttpResponseRedirect(reverse('designsafe_accounts:nees_migration')) else: tas_by_username = None @@ -156,11 +160,11 @@ def nees_migration(request, step=None): tas_api = TASClient() try: tas_by_username = tas_api.get_user(username=nees_user_match[0].username) - except: + except Exception: logger.exception('Error checking for existing TAS users') try: tas_by_email = tas_api.get_user(email=nees_user_match[0].email) - except: + except Exception: logger.exception('Error checking for existing TAS users') context = { @@ -217,7 +221,7 @@ def nees_migration(request, step=None): return HttpResponseRedirect('/') except Exception as e: logger.exception('Error saving user!') - logger.info('error: {}'.format(e)) + logger.info('error: %s', e) error_type = e.args[1] if len(e.args) > 1 else '' @@ -232,7 +236,7 @@ def nees_migration(request, step=None): 'This email is already registered. If you already have an ' 'account with TACC, please log in using those credentials.') form._errors.setdefault('email', [err_msg]) - err_msg = '%s Did you forget your password?' % ( + err_msg = '{} Did you forget your password?'.format( err_msg, reverse('designsafe_accounts:password_reset')) elif 'PasswordInvalidException' in error_type: @@ -326,8 +330,8 @@ def profile_edit(request): ds_profile.professional_level = pro_data['professional_level'] ds_profile.nh_interests_primary = pro_data['nh_interests_primary'] - except ObjectDoesNotExist as e: - logger.info('exception e: {} {}'.format(type(e), e )) + except ObjectDoesNotExist: + logger.exception("") ds_profile = DesignSafeProfile( user=user, bio=pro_data['bio'], @@ -415,7 +419,7 @@ def _process_password_reset_request(request, form): logger.info('Processing password reset request for username: "%s"', username) resp = tas.request_password_reset(user['username'], source='DesignSafe') logger.debug(resp) - except Exception as e: + except Exception: logger.exception('Failed password reset request') return True @@ -496,12 +500,11 @@ def mailing_list_subscription(request, list_name): try: su = get_user_model().objects.filter( Q(notification_preferences__isnull=True) | - Q(**{"notification_preferences__{}".format(list_name): True})) - subscribers += list('"{0}","{1}"'.format(u.get_full_name(), - u.email) for u in su) + Q(**{f"notification_preferences__{list_name}": True})) + subscribers += [f'"{u.get_full_name()}","{u.email}"' for u in su] - except TypeError as e: - logger.warning('Invalid list name: {}'.format(list_name)) + except TypeError: + logger.warning('Invalid list name: %s', list_name) return HttpResponse('\n'.join(subscribers), content_type='text/csv') diff --git a/designsafe/apps/api/admin.py b/designsafe/apps/api/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/api/admin.py +++ b/designsafe/apps/api/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/api/agave/__init__.py b/designsafe/apps/api/agave/__init__.py index 3a66b4ece9..8f96e72c60 100644 --- a/designsafe/apps/api/agave/__init__.py +++ b/designsafe/apps/api/agave/__init__.py @@ -4,16 +4,17 @@ we should create an agave.utils or agave.libs module for these to live in. """ import logging +from urllib.parse import urlparse + import requests from agavepy.agave import Agave, load_resource -from tapipy.tapis import Tapis from django.conf import settings -from urllib.parse import urlparse +from tapipy.tapis import Tapis logger = logging.getLogger(__name__) -AGAVE_RESOURCES = load_resource(getattr(settings, 'AGAVE_TENANT_BASEURL')) -tenant_id = urlparse(getattr(settings, "TAPIS_TENANT_BASEURL")).hostname.split( +AGAVE_RESOURCES = load_resource(settings.AGAVE_TENANT_BASEURL) +tenant_id = urlparse(settings.TAPIS_TENANT_BASEURL).hostname.split( "." )[0] @@ -71,7 +72,7 @@ def impersonate_service_account(username): :param str username: Username to impersonate. """ - url = '/'.join([settings.AGAVE_TENANT_BASEURL, 'token']) + url = f"{settings.AGAVE_TENANT_BASEURL}/token" cred = (settings.AGAVE_CLIENT_KEY, settings.AGAVE_CLIENT_SECRET) if getattr(settings, 'AGAVE_USE_SANDBOX', False): @@ -81,7 +82,7 @@ def impersonate_service_account(username): 'grant_type': 'admin_password', 'username': settings.DS_ADMIN_USERNAME, 'password': settings.DS_ADMIN_PASSWORD, - 'token_username': '/'.join([settings.AGAVE_USER_STORE_ID, username]), + 'token_username': f"{settings.AGAVE_USER_STORE_ID}/{username}", 'scope': 'PRODUCTION', } response = requests.post(url, data=body, auth=cred) diff --git a/designsafe/apps/api/ai_keywords/urls.py b/designsafe/apps/api/ai_keywords/urls.py index 00a430b52e..4f2a98afe2 100644 --- a/designsafe/apps/api/ai_keywords/urls.py +++ b/designsafe/apps/api/ai_keywords/urls.py @@ -1,6 +1,7 @@ """URLs for RAG-based keyword suggestions.""" from django.urls import path + from .views import KeywordsView urlpatterns = [ diff --git a/designsafe/apps/api/ai_keywords/utils.py b/designsafe/apps/api/ai_keywords/utils.py index e5505d72db..28c9ab5479 100644 --- a/designsafe/apps/api/ai_keywords/utils.py +++ b/designsafe/apps/api/ai_keywords/utils.py @@ -1,9 +1,11 @@ """Utility functions for AI keyword extraction and ChromaDB ingest.""" import json + from django.conf import settings -from designsafe.apps.api.publications_v2.models import Publication + from designsafe.apps.api.ai_keywords.views import RAG +from designsafe.apps.api.publications_v2.models import Publication def _retrieve_keywords(pub: Publication) -> list[str]: diff --git a/designsafe/apps/api/ai_keywords/views.py b/designsafe/apps/api/ai_keywords/views.py index ece6c5d333..a8452b1535 100644 --- a/designsafe/apps/api/ai_keywords/views.py +++ b/designsafe/apps/api/ai_keywords/views.py @@ -2,16 +2,18 @@ import json import logging + from chromadb import HttpClient, Settings -from typing_extensions import TypedDict -from langchain_core.documents import Document -from langgraph.graph import START, StateGraph -from langchain_chroma import Chroma +from django.conf import settings +from django.http import HttpRequest, JsonResponse from langchain.prompts import PromptTemplate +from langchain_chroma import Chroma +from langchain_core.documents import Document from langchain_openai import ChatOpenAI, OpenAIEmbeddings -from django.http import HttpRequest, JsonResponse -from django.conf import settings -from designsafe.apps.api.views import BaseApiView, ApiException +from langgraph.graph import START, StateGraph +from typing_extensions import TypedDict + +from designsafe.apps.api.views import ApiException, BaseApiView logger = logging.getLogger(__name__) @@ -49,7 +51,7 @@ def get(self, request: HttpRequest): resp_list = _parse_keywords_response(resp["answer"]) except (AttributeError, TypeError, ValueError): logger.exception("Error decoding answer") - logger.debug(f"Raw answer: {resp['answer']}") + logger.debug("Raw answer: %s", resp['answer']) resp_list = [] return JsonResponse({"response": resp_list}) @@ -124,9 +126,9 @@ def __init__(self): top_p=0.1, ) - except Exception as e: + except Exception: logger.exception("Error initializing RAG") - raise e + raise def retrieve(self, state: State): """Retrieve relevant documents from vector store based on the question.""" diff --git a/designsafe/apps/api/datafiles/handlers.py b/designsafe/apps/api/datafiles/handlers.py index c80958da91..146c5a097b 100644 --- a/designsafe/apps/api/datafiles/handlers.py +++ b/designsafe/apps/api/datafiles/handlers.py @@ -1,16 +1,20 @@ import logging + from django.core.exceptions import PermissionDenied from django.urls import reverse from tapipy.errors import BaseTapyException, InternalServerError, UnauthorizedError + from designsafe.apps.api.datafiles.notifications import notify -from designsafe.apps.api.datafiles.operations import tapis_operations -from designsafe.apps.api.datafiles.operations import googledrive_operations -from designsafe.apps.api.datafiles.operations import dropbox_operations -from designsafe.apps.api.datafiles.operations import box_operations -from designsafe.apps.api.datafiles.operations import shared_operations +from designsafe.apps.api.datafiles.operations import ( + box_operations, + dropbox_operations, + googledrive_operations, + shared_operations, + tapis_operations, +) from designsafe.apps.api.exceptions import ApiException -from designsafe.libs.common.decorators import retry from designsafe.apps.workspace.api.views import test_system_needs_keys +from designsafe.libs.common.decorators import retry logger = logging.getLogger(__name__) @@ -44,7 +48,9 @@ def datafiles_get_handler(api, client, scheme, system, path, operation, username system_needs_keys = test_system_needs_keys(client, username, system, path) if system_needs_keys: logger.error( - f"Keys for user {username} must be manually pushed to system: {system_needs_keys.id}" + "Keys for user %s must be manually pushed to system: %s", + username, + system_needs_keys.id, ) raise except BaseTapyException as exc: @@ -58,15 +64,15 @@ def datafiles_post_handler(api, username, client, scheme, system, raise PermissionDenied op = getattr(operations_mapping[api], operation) - operation in notify_actions and notify(username, operation, '{} operation has started.'.format(operation.capitalize()), 'INFO', {}) + operation in notify_actions and notify(username, operation, f'{operation.capitalize()} operation has started.', 'INFO', {}) try: result = op(client, system, path, tapis_tracking_id=tapis_tracking_id, **body) - operation in notify_actions and notify(username, operation, '{} operation was successful.'.format(operation.capitalize()), 'SUCCESS', result) + operation in notify_actions and notify(username, operation, f'{operation.capitalize()} operation was successful.', 'SUCCESS', result) return result - except Exception as exc: - operation in notify_actions and notify(username, operation, 'File operation {} could not be completed.'.format(operation.capitalize()), 'ERROR', {}) - raise exc + except Exception: + operation in notify_actions and notify(username, operation, f'File operation {operation.capitalize()} could not be completed.', 'ERROR', {}) + raise def datafiles_put_handler(api, username, client, scheme, system, @@ -75,18 +81,18 @@ def datafiles_put_handler(api, username, client, scheme, system, raise PermissionDenied op = getattr(operations_mapping[api], operation) - operation in notify_actions and notify(username, operation, '{} operation has started.'.format(operation.capitalize()), 'INFO', {}) + operation in notify_actions and notify(username, operation, f'{operation.capitalize()} operation has started.', 'INFO', {}) try: result = op(client, system, path, username=username, tapis_tracking_id=tapis_tracking_id, **body) if operation == 'copy' and system != body.get('dest_system', None): - notify(username, operation, 'Your file transfer request has been received and will be processed shortly.'.format(operation.capitalize()), 'SUCCESS', result) + notify(username, operation, 'Your file transfer request has been received and will be processed shortly.', 'SUCCESS', result) else: - operation in notify_actions and notify(username, operation, '{} operation was successful.'.format(operation.capitalize()), 'SUCCESS', result) + operation in notify_actions and notify(username, operation, f'{operation.capitalize()} operation was successful.', 'SUCCESS', result) return result - except Exception as exc: - operation in notify_actions and notify(username, operation, 'File operation {} could not be completed.'.format(operation.capitalize()), 'ERROR', {}) - raise exc + except Exception: + operation in notify_actions and notify(username, operation, f'File operation {operation.capitalize()} could not be completed.', 'ERROR', {}) + raise def resource_unconnected_handler(api): @@ -117,21 +123,21 @@ def resource_expired_handler(api): if api == 'googledrive': message = 'While you previously granted this application access to Google Drive, ' \ 'that grant appears to be no longer valid. Please ' \ - 'disconnect and reconnect your Google Drive account ' \ - 'to continue using Google Drive data.' % reverse('googledrive_integration:index') + 'disconnect and reconnect your Google Drive account ' \ + 'to continue using Google Drive data.'.format(reverse('googledrive_integration:index')) return ApiException(status=403, message=message) if api == 'dropbox': message = 'While you previously granted this application access to Dropbox, ' \ 'that grant appears to be no longer valid. Please ' \ - 'disconnect and reconnect your Dropbox.com account ' \ - 'to continue using Dropbox data.' % reverse('dropbox_integration:index') + 'disconnect and reconnect your Dropbox.com account ' \ + 'to continue using Dropbox data.'.format(reverse('dropbox_integration:index')) raise ApiException(status=403, message=message) if api == 'box': # user needs to reconnect with Box message = 'While you previously granted this application access to Box, ' \ 'that grant appears to be no longer valid. Please ' \ - 'disconnect and reconnect your Box.com account ' \ - 'to continue using Box data.' % reverse('box_integration:index') + 'disconnect and reconnect your Box.com account ' \ + 'to continue using Box data.'.format(reverse('box_integration:index')) raise ApiException(status=403, message=message) else: message = 'There was an error accessing this storage system.' diff --git a/designsafe/apps/api/datafiles/models.py b/designsafe/apps/api/datafiles/models.py index b4ee5868dc..c5f480da15 100644 --- a/designsafe/apps/api/datafiles/models.py +++ b/designsafe/apps/api/datafiles/models.py @@ -1,6 +1,6 @@ -from django.conf import settings from django.db import models + class DataFilesSurveyResult(models.Model): project_id = models.CharField(max_length=255) comments = models.CharField(max_length=4096, blank=True, default='') diff --git a/designsafe/apps/api/datafiles/operations/box_operations.py b/designsafe/apps/api/datafiles/operations/box_operations.py index 3cda0bf0bb..98f6fc16de 100644 --- a/designsafe/apps/api/datafiles/operations/box_operations.py +++ b/designsafe/apps/api/datafiles/operations/box_operations.py @@ -1,8 +1,6 @@ -from boxsdk.object.folder import Folder -from boxsdk.object.file import File -from boxsdk.exception import BoxAPIException import io import logging + logger = logging.getLogger(__name__) def listing(client, system, path, offset=0, limit=100, *args, **kwargs): @@ -16,7 +14,7 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): items = folder.get_items(limit=limit, offset=offset, fields=['name', 'size', 'type', 'modified_at', 'id']) - mapped_items = map(lambda f: { + mapped_items = ({ 'system': None, 'type': 'dir' if f.type == 'folder' else 'file', 'format': 'folder' if f.type == 'folder' else 'raw', @@ -26,7 +24,7 @@ def listing(client, system, path, offset=0, limit=100, *args, **kwargs): 'length': f.size if f.type != 'web_link' else '', 'lastModified': f.modified_at, - }, items) + } for f in items) return {'listing': list(mapped_items)} @@ -56,7 +54,7 @@ def copy(client, src_system, src_path, dest_system, dest_path, filetype='file', elif filetype == 'dir': file_to_copy = client.folder(src_path) - file_copy = file_to_copy.copy(destination_folder) + file_to_copy.copy(destination_folder) return {} diff --git a/designsafe/apps/api/datafiles/operations/dropbox_operations.py b/designsafe/apps/api/datafiles/operations/dropbox_operations.py index c38e445c3f..6796c3d9d1 100644 --- a/designsafe/apps/api/datafiles/operations/dropbox_operations.py +++ b/designsafe/apps/api/datafiles/operations/dropbox_operations.py @@ -1,11 +1,12 @@ -import os import io import logging -from dropbox import Dropbox -from dropbox.files import FolderMetadata +import os + from django.conf import settings from django.contrib.auth import get_user_model from django.core.files.uploadedfile import TemporaryUploadedFile +from dropbox import Dropbox +from dropbox.files import FolderMetadata logger = logging.getLogger(__name__) @@ -22,9 +23,7 @@ def listing( else: files = client.files_list_folder_continue(nextPageToken) - listing = list( - map( - lambda f: { + listing = [{ "system": "dropbox", "type": ("dir" if isinstance(f, FolderMetadata) else "file"), "format": ("folder" if isinstance(f, FolderMetadata) else "raw"), @@ -35,10 +34,7 @@ def listing( "lastModified": ( None if isinstance(f, FolderMetadata) else f.server_modified ), - }, - files.entries, - ) - ) + } for f in files.entries] nextPageToken = files.cursor if files.has_more else None @@ -83,14 +79,14 @@ def preview(client: Dropbox, system, path, *args, **kwargs): file_type = "object" elif file_ext in settings.SUPPORTED_MS_OFFICE: file_type = "ms-office" - url = "https://view.officeapps.live.com/op/view.aspx?src={}".format(url) + url = f"https://view.officeapps.live.com/op/view.aspx?src={url}" elif file_ext in settings.SUPPORTED_VIDEO_EXTS: file_type = "video" # url = '/api/datafiles/media/agave/private/{}/{}'.format(system, path) elif file_ext in settings.SUPPORTED_IPYNB_PREVIEW_EXTS: file_type = "ipynb" tmp = url.replace("https://", "") - url = "https://nbviewer.jupyter.org/urls/{tmp}".format(tmp=tmp) + url = f"https://nbviewer.jupyter.org/urls/{tmp}" else: file_type = "other" return {"href": url, "fileType": file_type} @@ -110,10 +106,10 @@ def copy( src_file_name = os.path.basename(src_path) if src_system != dest_system: + from designsafe.apps.api.datafiles.handlers import resource_unconnected_handler from designsafe.apps.api.datafiles.operations.transfer_operations import ( transfer, ) - from designsafe.apps.api.datafiles.handlers import resource_unconnected_handler from designsafe.apps.api.datafiles.views import get_client user = get_user_model().objects.get(username=username) @@ -141,7 +137,7 @@ def copy( src_path = "/" + src_path if dest_path and not dest_path.startswith("/"): dest_path = "/" + dest_path - dest_path = "{}/{}".format(dest_path, src_file_name) + dest_path = f"{dest_path}/{src_file_name}" src_client.files_copy(src_path, dest_path, autorename=True) @@ -149,7 +145,7 @@ def copy( def download(client: Dropbox, system, path, *args, **kwargs): - logger.debug("Downloading file from Dropbox: {}".format(path)) + logger.debug("Downloading file from Dropbox: %s", path) if not path.startswith("/"): path = "/" + path (meta, response) = client.files_download(path) @@ -169,7 +165,7 @@ def upload( ): if path and not path.startswith("/"): path = "/" + path - upload_path = "{}/{}".format(path, uploaded_file.name).replace("//", "/") + upload_path = f"{path}/{uploaded_file.name}".replace("//", "/") client.files_upload(uploaded_file.read(), upload_path, autorename=True) return {"result": "OK"} @@ -177,7 +173,7 @@ def upload( def mkdir(client, system, path, dir_name, *args, **kwargs): if path and not path.startswith("/"): path = "/" + path - folder_path = "{}/{}".format(path, dir_name).replace("//", "/") + folder_path = f"{path}/{dir_name}".replace("//", "/") client.files_create_folder(folder_path) return {"name": dir_name, "path": folder_path} diff --git a/designsafe/apps/api/datafiles/operations/googledrive_operations.py b/designsafe/apps/api/datafiles/operations/googledrive_operations.py index 3cea5f478d..f62c742fbf 100644 --- a/designsafe/apps/api/datafiles/operations/googledrive_operations.py +++ b/designsafe/apps/api/datafiles/operations/googledrive_operations.py @@ -1,13 +1,11 @@ -import urllib -import os import io +import logging +import os + import magic -import datetime from django.conf import settings -from requests.exceptions import HTTPError -import logging -from elasticsearch_dsl import Q -from googleapiclient.http import MediaIoBaseUpload, MediaIoBaseDownload +from googleapiclient.http import MediaIoBaseDownload, MediaIoBaseUpload + # from portal.libs.elasticsearch.indexes import IndexedFile # from portal.apps.search.tasks import agave_indexer, agave_listing_indexer @@ -19,25 +17,25 @@ def listing(client, system, path, offset=None, limit=100, nextPageToken=None, *a path = 'root' fields = "mimeType, name, id, modifiedTime, fileExtension, size, parents, webViewLink" - listing_call = client.files().list(q="'{}' in parents and trashed=False".format(path), - fields="files({}), nextPageToken".format(fields), pageSize=limit, pageToken=nextPageToken)\ + listing_call = client.files().list(q=f"'{path}' in parents and trashed=False", + fields=f"files({fields}), nextPageToken", pageSize=limit, pageToken=nextPageToken)\ .execute() listing = listing_call.get('files') scroll_token = listing_call.get('nextPageToken') reached_end = not bool(scroll_token) - listing = list(map(lambda f: { + listing = [{ 'system': None, 'type': 'dir' if f['mimeType'] == 'application/vnd.google-apps.folder' else 'file', 'format': 'folder' if f['mimeType'] == 'application/vnd.google-apps.folder' else 'raw', 'mimeType': f['mimeType'], 'path': f['id'], 'name': f['name'], - 'length': int(f['size']) if 'size' in f.keys() else 0, + 'length': int(f['size']) if 'size' in f else 0, 'lastModified': f['modifiedTime'], '_links': { 'self': {'href': f['webViewLink']} } - }, listing)) + } for f in listing] return {'listing': listing, 'nextPageToken': scroll_token, 'reachedEnd': reached_end} @@ -68,7 +66,7 @@ def walk_all(client, system, path, limit=100): def upload(client, system, path, uploaded_file, *args, **kwargs): if not path: path = 'root' - filename = os.path.basename(uploaded_file.name) + os.path.basename(uploaded_file.name) mimetype = magic.from_buffer(uploaded_file.getvalue(), mime=True) media = MediaIoBaseUpload(uploaded_file, mimetype=mimetype) file_meta = { @@ -113,7 +111,7 @@ def download(client, system, path, *args, **kwargs): downloader = MediaIoBaseDownload(fh, request) done = False while done is False: - status, done = downloader.next_chunk() + _status, done = downloader.next_chunk() fh.name = file_name return fh @@ -125,7 +123,7 @@ def preview(client, system, path, *args, **kwargs): file_name = path.strip('/').split('/')[-1] file_ext = os.path.splitext(file_name)[1].lower() - url = '/api/datafiles/media/googledrive/private/googledrive/{}'.format(path) + url = f'/api/datafiles/media/googledrive/private/googledrive/{path}' if file_ext in settings.SUPPORTED_TEXT_PREVIEW_EXTS: file_type = 'text' @@ -133,25 +131,27 @@ def preview(client, system, path, *args, **kwargs): file_type = 'image' elif file_ext in settings.SUPPORTED_OBJECT_PREVIEW_EXTS: file_type = 'object' - url = 'https://docs.google.com/gview?url={}&embedded=true'.format(url) + url = f'https://docs.google.com/gview?url={url}&embedded=true' elif file_ext in settings.SUPPORTED_MS_OFFICE: file_type = 'ms-office' - url = 'https://view.officeapps.live.com/op/view.aspx?src={}'.\ - format(url) + url = f'https://view.officeapps.live.com/op/view.aspx?src={url}' elif file_ext in settings.SUPPORTED_VIDEO_EXTS: file_type = 'video' # url = '/api/datafiles/media/agave/private/{}/{}'.format(system, path) elif file_ext in settings.SUPPORTED_IPYNB_PREVIEW_EXTS: file_type = 'ipynb' tmp = url.replace('https://', '') - url = 'https://nbviewer.jupyter.org/urls/{tmp}'.format(tmp=tmp) + url = f'https://nbviewer.jupyter.org/urls/{tmp}' else: file_type = 'other' return {'href': url, 'fileType': file_type} def copy(client, src_system, src_path, dest_system, dest_path, filename, filetype='file', *args, **kwargs): - from designsafe.apps.api.datafiles.operations.transfer_operations import transfer, transfer_folder + from designsafe.apps.api.datafiles.operations.transfer_operations import ( + transfer, + transfer_folder, + ) if not src_path: src_path = 'root' # Google drive doesn't have a robust copy API, so this endpoint uses generic transfer methods. diff --git a/designsafe/apps/api/datafiles/operations/shared_operations.py b/designsafe/apps/api/datafiles/operations/shared_operations.py index 17a18f664c..cfb06b6c22 100644 --- a/designsafe/apps/api/datafiles/operations/shared_operations.py +++ b/designsafe/apps/api/datafiles/operations/shared_operations.py @@ -1,15 +1,12 @@ -import urllib -import os -import datetime -import io -from django.conf import settings -from requests.exceptions import HTTPError import logging -import json + from elasticsearch_dsl import Q -import magic + +from designsafe.apps.api.datafiles.operations.tapis_operations import ( + listing as agave_listing, +) from designsafe.apps.data.models.elasticsearch import IndexedFile -from designsafe.apps.api.datafiles.operations.tapis_operations import preview, copy, download, download_bytes, listing as agave_listing + # from portal.libs.elasticsearch.indexes import IndexedFile # from portal.apps.search.tasks import agave_indexer, agave_listing_indexer @@ -50,7 +47,6 @@ def listing(client, system, path, username, offset=0, limit=100, *args, **kwargs nested_filter.path = 'legacyPermissions' nested_filter.query = pems_filter - file_path = '/' home_filter = Q('prefix', **{'path._exact': '/' + username}) system_filter = Q('term', **{'system._exact': 'designsafe.storage.default'}) query = Q('bool', must_not=home_filter, filter=[nested_filter, system_filter]) diff --git a/designsafe/apps/api/datafiles/operations/tapis_operations.py b/designsafe/apps/api/datafiles/operations/tapis_operations.py index 1d50debd42..7481fae8fc 100644 --- a/designsafe/apps/api/datafiles/operations/tapis_operations.py +++ b/designsafe/apps/api/datafiles/operations/tapis_operations.py @@ -4,18 +4,22 @@ import os import urllib from pathlib import Path + +import httpx +import requests import tapipy +from django.conf import settings +from elasticsearch_dsl import Q + +from designsafe.apps.api.datafiles.models import PublicationSymlink from designsafe.apps.api.datafiles.utils import * +from designsafe.apps.api.filemeta.models import FileMetaModel +from designsafe.apps.api.filemeta.tasks import ( + copy_file_meta_async, + move_file_meta_async, +) from designsafe.apps.data.models.elasticsearch import IndexedFile from designsafe.apps.data.tasks import agave_indexer, agave_listing_indexer -from designsafe.apps.api.filemeta.models import FileMetaModel -from designsafe.apps.api.filemeta.tasks import move_file_meta_async, copy_file_meta_async -from designsafe.apps.api.datafiles.models import PublicationSymlink -from django.conf import settings -from elasticsearch_dsl import Q -import requests -import httpx -from requests.exceptions import HTTPError logger = logging.getLogger(__name__) @@ -53,7 +57,7 @@ def listing(client, system, path, offset=0, limit=100, q=None, *args, **kwargs): return search(client, system, path, offset=0, limit=100, query_string=q, **kwargs) raw_listing = client.files.listFiles( systemId=system, - path=urllib.parse.quote((path or '/')), + path=urllib.parse.quote(path or '/'), offset=int(offset), limit=int(limit), headers={"X-Tapis-Tracking-ID": kwargs.get("tapis_tracking_id", "")} @@ -189,7 +193,7 @@ def search(client, system, path, offset=0, limit=100, query_string='', **kwargs) # check old publication files # stopgap until all /published-data paths are indexed if system == "designsafe.storage.published" and path.startswith("/published-data"): - legacy_pub_path = path.lstrip("/published-data") + legacy_pub_path = path.removeprefix("/published-data") legacy_path_filter = Q('term', **{'path._comps': legacy_pub_path}) path_comp_filter = path_comp_filter | legacy_path_filter @@ -350,7 +354,8 @@ def copy(client, src_system, src_path, dest_system, dest_path, *args, **kwargs): client.files.listFiles(systemId=dest_system, path=os.path.join(dest_path, src_file_name)) dst_file_name = rename_duplicate_path(src_file_name) full_dest_path = os.path.join(dest_path.strip('/'), dst_file_name) - except: + except Exception: + logger.exception("") dst_file_name = src_file_name full_dest_path = os.path.join(dest_path.strip('/'), src_file_name) @@ -555,7 +560,8 @@ def upload(client, system, path, uploaded_file, webkit_relative_path=None, *args meta=metadata ) return dict(resp) - except: + except Exception: + logger.exception("") return dict(resp) @@ -611,15 +617,14 @@ def preview(client, system, path, href="", max_uses=3, lifetime=600, *args, **kw file_type = 'object' elif file_ext in settings.SUPPORTED_MS_OFFICE: file_type = 'ms-office' - url = 'https://view.officeapps.live.com/op/view.aspx?src={}'.\ - format(url) + url = f'https://view.officeapps.live.com/op/view.aspx?src={url}' elif file_ext in settings.SUPPORTED_VIDEO_EXTS: file_type = 'video' # url = '/api/datafiles/media/agave/private/{}/{}'.format(system, path) elif file_ext in settings.SUPPORTED_IPYNB_PREVIEW_EXTS: file_type = 'ipynb' tmp = url.replace('https://', '') - url = 'https://nbviewer.jupyter.org/urls/{tmp}'.format(tmp=tmp) + url = f'https://nbviewer.jupyter.org/urls/{tmp}' else: file_type = 'other' diff --git a/designsafe/apps/api/datafiles/operations/transfer_operations.py b/designsafe/apps/api/datafiles/operations/transfer_operations.py index f40d2126e2..aafa050e32 100644 --- a/designsafe/apps/api/datafiles/operations/transfer_operations.py +++ b/designsafe/apps/api/datafiles/operations/transfer_operations.py @@ -1,7 +1,9 @@ -from designsafe.apps.api.datafiles.operations import tapis_operations -from designsafe.apps.api.datafiles.operations import googledrive_operations -from designsafe.apps.api.datafiles.operations import dropbox_operations -from designsafe.apps.api.datafiles.operations import box_operations +from designsafe.apps.api.datafiles.operations import ( + box_operations, + dropbox_operations, + googledrive_operations, + tapis_operations, +) api_mapping = { 'googledrive': { @@ -39,7 +41,6 @@ def transfer(src_client, dest_client, src_api, dest_api, src_system, dest_system file_bytes = _download(src_client, src_system, src_path) _upload(dest_client, dest_system, dest_path, file_bytes) - return def transfer_folder(src_client, dest_client, src_api, dest_api, src_system, dest_system, src_path, dest_path, dirname, *args, **kwargs): diff --git a/designsafe/apps/api/datafiles/tests.py b/designsafe/apps/api/datafiles/tests.py index 6eafecefb3..22e27a5951 100644 --- a/designsafe/apps/api/datafiles/tests.py +++ b/designsafe/apps/api/datafiles/tests.py @@ -1,6 +1,12 @@ -import pytest import json -from designsafe.apps.api.datafiles.models import DataFilesSurveyResult, DataFilesSurveyCounter + +import pytest + +from designsafe.apps.api.datafiles.models import ( + DataFilesSurveyCounter, + DataFilesSurveyResult, +) + @pytest.mark.django_db def test_survey_post(client, regular_user): diff --git a/designsafe/apps/api/datafiles/urls.py b/designsafe/apps/api/datafiles/urls.py index 68d85d93c5..92b0da642a 100644 --- a/designsafe/apps/api/datafiles/urls.py +++ b/designsafe/apps/api/datafiles/urls.py @@ -1,5 +1,10 @@ from django.urls import re_path as url -from designsafe.apps.api.datafiles.views import DataFilesView, TransferFilesView, MicrosurveyView + +from designsafe.apps.api.datafiles.views import ( + DataFilesView, + MicrosurveyView, + TransferFilesView, +) urlpatterns = [ url(r'^transfer/(?P[\w.-]+)/$', TransferFilesView.as_view(), name='file_transfer'), diff --git a/designsafe/apps/api/datafiles/utils.py b/designsafe/apps/api/datafiles/utils.py index d19a6ce1bb..3fc30cca2a 100644 --- a/designsafe/apps/api/datafiles/utils.py +++ b/designsafe/apps/api/datafiles/utils.py @@ -2,13 +2,14 @@ import json import logging import os -import re from copy import deepcopy + import exifread from celery import shared_task -from designsafe.apps.api.agave import service_account from PIL import Image +from designsafe.apps.api.agave import service_account + logger = logging.getLogger(__name__) ALLOWED_IMAGE_TYPES = ['png', 'gif', 'jpg', 'jpeg', 'webp'] @@ -41,8 +42,8 @@ def rename_duplicate_path(file_name): """ _ext = os.path.splitext(file_name)[1].lower() _name = os.path.splitext(file_name)[0] - now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H-%M-%S') - return '{}_{}{}'.format(_name, now, _ext) + now = datetime.datetime.utc().strftime('%Y-%m-%d %H-%M-%S') + return f'{_name}_{now}{_ext}' def query_file_meta(system, path): diff --git a/designsafe/apps/api/datafiles/views.py b/designsafe/apps/api/datafiles/views.py index 90b90e8914..d50c90a920 100644 --- a/designsafe/apps/api/datafiles/views.py +++ b/designsafe/apps/api/datafiles/views.py @@ -1,20 +1,34 @@ import json import logging from hashlib import sha256 + from boxsdk.exception import BoxOAuthException -from django.http import JsonResponse from django.conf import settings +from django.http import JsonResponse from dropbox.exceptions import AuthError as DropboxAuthError from google.auth.exceptions import GoogleAuthError from requests.exceptions import HTTPError -from designsafe.libs.common.utils import check_group_membership -from designsafe.apps.api.datafiles.handlers import datafiles_get_handler, datafiles_post_handler, datafiles_put_handler, resource_unconnected_handler, resource_expired_handler -from designsafe.apps.api.datafiles.operations.transfer_operations import transfer, transfer_folder -from designsafe.apps.api.datafiles.notifications import notify -from designsafe.apps.api.datafiles.models import DataFilesSurveyResult, DataFilesSurveyCounter -from designsafe.apps.api.views import BaseApiView + from designsafe.apps.api.agave import service_account +from designsafe.apps.api.datafiles.handlers import ( + datafiles_get_handler, + datafiles_post_handler, + datafiles_put_handler, + resource_expired_handler, + resource_unconnected_handler, +) +from designsafe.apps.api.datafiles.models import ( + DataFilesSurveyCounter, + DataFilesSurveyResult, +) +from designsafe.apps.api.datafiles.notifications import notify +from designsafe.apps.api.datafiles.operations.transfer_operations import ( + transfer, + transfer_folder, +) from designsafe.apps.api.utils import get_client_ip +from designsafe.apps.api.views import BaseApiView +from designsafe.libs.common.utils import check_group_membership logger = logging.getLogger(__name__) metrics = logging.getLogger('metrics') @@ -172,17 +186,17 @@ def put(self, request, format): notify(request.user.username, 'transfer', 'Copy operation has started.', 'INFO', {}) try: if format == 'folder': - resp = transfer_folder(src_client, dest_client, **body) + transfer_folder(src_client, dest_client, **body) notify(request.user.username, 'transfer', 'Files were successfully copied.', 'SUCCESS', {}) return JsonResponse({'success': True}) else: - resp = transfer(src_client, dest_client, **body) + transfer(src_client, dest_client, **body) notify(request.user.username, 'transfer', 'Files were successfully copied.', 'SUCCESS', {}) return JsonResponse({'success': True}) except Exception as exc: notify(request.user.username, 'transfer', 'Copy operation has failed.', 'ERROR', {}) logger.info(exc) - raise exc + raise class MicrosurveyView(BaseApiView): diff --git a/designsafe/apps/api/decorators.py b/designsafe/apps/api/decorators.py index 5d16b818b7..c0a6b46d49 100644 --- a/designsafe/apps/api/decorators.py +++ b/designsafe/apps/api/decorators.py @@ -2,21 +2,20 @@ django specific utils or try and make it as general as possible. """ import logging -from functools import wraps from base64 import b64decode +from functools import wraps from urllib.parse import urlparse -from django.conf import settings -from django.http import HttpRequest -from django.contrib.auth import get_user_model -from django.contrib.auth import login -from django.core.exceptions import ObjectDoesNotExist + import jwt as pyjwt +from cryptography.exceptions import UnsupportedAlgorithm from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.serialization import load_der_public_key -from cryptography.exceptions import UnsupportedAlgorithm -from tapipy.tapis import Tapis +from django.conf import settings +from django.contrib.auth import get_user_model, login +from django.core.exceptions import ObjectDoesNotExist +from django.http import HttpRequest from tapipy.errors import BaseTapyException +from tapipy.tapis import Tapis #pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -41,8 +40,8 @@ def _decode_jwt(jwt): try: decoded = pyjwt.decode(jwt, key, issuer=settings.AGAVE_JWT_ISSUER) - except pyjwt.exceptions.DecodeError as exc: - logger.exception('Could not decode JWT. %s', exc) + except pyjwt.exceptions.DecodeError: + logger.exception('Could not decode JWT.') return {} return decoded @@ -74,7 +73,7 @@ def tapis_jwt_login(func): #pylint: disable=missing-docstring @wraps(func) def decorated_function(request: HttpRequest, *args, **kwargs): - tenant_id = urlparse(getattr(settings, "TAPIS_TENANT_BASEURL")).hostname.split( + tenant_id = urlparse(settings.TAPIS_TENANT_BASEURL).hostname.split( "." )[0] if request.user.is_authenticated: diff --git a/designsafe/apps/api/exceptions.py b/designsafe/apps/api/exceptions.py index 91c9809f46..190db82504 100644 --- a/designsafe/apps/api/exceptions.py +++ b/designsafe/apps/api/exceptions.py @@ -22,8 +22,10 @@ class ApiException(RequestException): extra: Extra information of exception. """ - def __init__(self, message = None, status = None, extra = {}, *args, **kwargs): - super(ApiException, self).__init__(*args, **kwargs) + def __init__(self, message = None, status = None, extra = None, *args, **kwargs): + if extra is None: + extra = {} + super().__init__(*args, **kwargs) response = self.response or Response() response.status_code = status or response.status_code response.reason = message or response.reason diff --git a/designsafe/apps/api/filemeta/management/commands/populate_filemeta_table_from_tavpisv2_metadata.py b/designsafe/apps/api/filemeta/management/commands/populate_filemeta_table_from_tavpisv2_metadata.py index c1b23a9dff..ca1987069e 100644 --- a/designsafe/apps/api/filemeta/management/commands/populate_filemeta_table_from_tavpisv2_metadata.py +++ b/designsafe/apps/api/filemeta/management/commands/populate_filemeta_table_from_tavpisv2_metadata.py @@ -5,9 +5,9 @@ # pylint: disable=logging-fstring-interpolation # pylint: disable=no-member -import os -import logging import json +import logging +import os from django.core.management.base import BaseCommand @@ -60,12 +60,14 @@ def populate_filemeta_table(dry_run, do_not_update_existing): Update the filemeta table from Tapisv2-based metadata. """ logger.info( - f"Updating filemeta table from tapisv2-based metadata." - f" dry_run={dry_run} do_not_update_existing={do_not_update_existing}" + "Updating filemeta table from tapisv2-based metadata." + " dry_run=%s do_not_update_existing=%s", + dry_run, + do_not_update_existing, ) v2_file_meta_data = get_all_v2_file_meta() - logger.info(f"Processing {len(v2_file_meta_data)} tapisv2-based metadata entries") + logger.info("Processing %s tapisv2-based metadata entries", len(v2_file_meta_data)) updated = 0 already_exists = 0 @@ -92,9 +94,11 @@ def populate_filemeta_table(dry_run, do_not_update_existing): logger.info( "Successfully updated filemeta table from tapisv2-based metadata" - f"\n {len(v2_file_meta_data)} tapisv2-based metadata entries." - f"\n {already_exists} entries already existed in filemeta table." - f"\n {updated} entries were updated/created in filemeta table" + "\n {len(v2_file_meta_data)} tapisv2-based metadata entries." + "\n %s entries already existed in filemeta table." + "\n %s entries were updated/created in filemeta table", + already_exists, + updated, ) diff --git a/designsafe/apps/api/filemeta/tasks.py b/designsafe/apps/api/filemeta/tasks.py index 209a4cd6be..00d05df95c 100644 --- a/designsafe/apps/api/filemeta/tasks.py +++ b/designsafe/apps/api/filemeta/tasks.py @@ -1,7 +1,9 @@ """Utils for bulk move/copy of file metadata objects.""" import os + from celery import shared_task + from designsafe.apps.api.filemeta.models import FileMetaModel diff --git a/designsafe/apps/api/filemeta/tests.py b/designsafe/apps/api/filemeta/tests.py index cba2d9d0e5..39190b790a 100644 --- a/designsafe/apps/api/filemeta/tests.py +++ b/designsafe/apps/api/filemeta/tests.py @@ -1,7 +1,8 @@ -import pytest import json + +import pytest from django.db import IntegrityError -from django.test import Client + from designsafe.apps.api.filemeta.models import FileMetaModel diff --git a/designsafe/apps/api/filemeta/urls.py b/designsafe/apps/api/filemeta/urls.py index a666913589..c7467ef9fb 100644 --- a/designsafe/apps/api/filemeta/urls.py +++ b/designsafe/apps/api/filemeta/urls.py @@ -1,7 +1,8 @@ """File Meta API routes""" from django.urls import path -from .views import FileMetaView, CreateFileMetaView + +from .views import CreateFileMetaView, FileMetaView urlpatterns = [ path("/", FileMetaView.as_view(), name="filemeta-get"), diff --git a/designsafe/apps/api/filemeta/views.py b/designsafe/apps/api/filemeta/views.py index 014cccabc3..f1f28046a4 100644 --- a/designsafe/apps/api/filemeta/views.py +++ b/designsafe/apps/api/filemeta/views.py @@ -1,14 +1,15 @@ """File Meta view""" -import logging import json -from django.http import JsonResponse, HttpRequest +import logging + +from django.http import HttpRequest, JsonResponse + from designsafe.apps.api.datafiles.operations.tapis_operations import listing from designsafe.apps.api.exceptions import ApiException from designsafe.apps.api.filemeta.models import FileMetaModel from designsafe.apps.api.views import AuthenticatedAllowJwtApiView - logger = logging.getLogger(__name__) @@ -31,8 +32,10 @@ def check_access(request, system_id: str, path: str, check_for_writable_access=F try: listing(request.user.tapis_oauth.client, system_id, path) except Exception as exc: # pylint:disable=broad-exception-caught - logger.error( - f"user cannot access any related metadata as listing failed for {system_id}/{path} with error {str(exc)}." + logger.exception( + "user cannot access any related metadata as listing failed for %s/%s", + system_id, + path, ) raise ApiException("User forbidden to access metadata", status=403) from exc @@ -49,7 +52,7 @@ def get(self, request: HttpRequest, system_id: str, path: str): result = {} try: - logger.debug(f"Get file metadata. system:{system_id} path:{path}") + logger.debug("Get file metadata. system:%s path:%s", system_id, path) file_meta = FileMetaModel.get_by_path_and_system( system=system_id, path=path ) @@ -73,7 +76,8 @@ def post(self, request: HttpRequest): value = json.loads(request.body) if "system" not in value or "path" not in value: logger.error( - f"Unable to create or update file metadata as system and path not in payload: {value}" + "Unable to create or update file metadata as system and path not in payload: %s", + value, ) raise ApiException("System and path are required in payload", status=400) @@ -86,14 +90,14 @@ def post(self, request: HttpRequest): try: logger.info( - f"Creating or updating file metadata. system:{system_id} path:{path}" + "Creating or updating file metadata. system:%s path:%s", system_id, path ) FileMetaModel.create_or_update_file_meta(value) return JsonResponse({"result": "OK"}) except Exception as exc: logger.exception( - f"Unable to create or update file metadata: {system_id}/{path}" + "Unable to create or update file metadata: %s/%s", system_id, path ) raise ApiException( "Unable to create or update file metadata", status=500 diff --git a/designsafe/apps/api/licenses/urls.py b/designsafe/apps/api/licenses/urls.py index 83be3bd978..424f19e2c3 100644 --- a/designsafe/apps/api/licenses/urls.py +++ b/designsafe/apps/api/licenses/urls.py @@ -1,4 +1,5 @@ from django.urls import re_path as url + from designsafe.apps.api.licenses.views import LicenseView urlpatterns = [ diff --git a/designsafe/apps/api/licenses/views.py b/designsafe/apps/api/licenses/views.py index 8904b201f1..3dea478803 100644 --- a/designsafe/apps/api/licenses/views.py +++ b/designsafe/apps/api/licenses/views.py @@ -1,11 +1,13 @@ """Views for the licenses API.""" +import logging + +from django.apps import apps from django.contrib.auth import get_user_model -from django.http.response import HttpResponseForbidden, HttpResponseNotFound from django.http import JsonResponse -from django.apps import apps +from django.http.response import HttpResponseForbidden, HttpResponseNotFound + from designsafe.apps.api.views import AuthenticatedAllowJwtApiView -import logging logger = logging.getLogger(__name__) diff --git a/designsafe/apps/api/mixins.py b/designsafe/apps/api/mixins.py index 69ce3fff15..1a28f18c92 100644 --- a/designsafe/apps/api/mixins.py +++ b/designsafe/apps/api/mixins.py @@ -1,15 +1,17 @@ -from django.utils.decorators import method_decorator +import json +import logging + from django.contrib.auth.decorators import login_required -from designsafe.apps.api.decorators import agave_jwt_login -from django.http import HttpResponse from django.core.serializers.json import DjangoJSONEncoder -import logging -import json +from django.http import HttpResponse +from django.utils.decorators import method_decorator + +from designsafe.apps.api.decorators import agave_jwt_login logger = logging.getLevelName(__name__) -class JSONResponseMixin(object): +class JSONResponseMixin: """ View mixin to return a JSON response. We're building one so we can put any extra code in here. @@ -20,7 +22,7 @@ def render_to_json_response(self, context, **response_kwargs): return HttpResponse(json.dumps(context, cls=DjangoJSONEncoder), **response_kwargs) -class SecureMixin(object): +class SecureMixin: """View mixin to ensure the user has access to a secured view This Mixin first checks if the request is done using a JWT. If this is not the case @@ -31,4 +33,4 @@ class SecureMixin(object): @method_decorator(agave_jwt_login) @method_decorator(login_required, name='dispatch') def dispatch(self, request, *args, **kwargs): - return super(SecureMixin, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) diff --git a/designsafe/apps/api/models.py b/designsafe/apps/api/models.py index 330effd5f0..f3fa2afa37 100644 --- a/designsafe/apps/api/models.py +++ b/designsafe/apps/api/models.py @@ -1,4 +1,3 @@ -from django.db import models # Create your models here. #from designsafe.apps.api.agave.models.fields import * diff --git a/designsafe/apps/api/notifications/apps.py b/designsafe/apps/api/notifications/apps.py index 00d517e1bb..68c106a2f4 100644 --- a/designsafe/apps/api/notifications/apps.py +++ b/designsafe/apps/api/notifications/apps.py @@ -7,5 +7,4 @@ class NotificationsConfig(AppConfig): verbose_name = 'Designsafe Notifications' def ready(self): - from designsafe.apps.api.notifications.receivers import (send_notification_ws, - send_broadcast_ws) + pass diff --git a/designsafe/apps/api/notifications/models.py b/designsafe/apps/api/notifications/models.py index 70176c09ac..ff2e65bdd8 100644 --- a/designsafe/apps/api/notifications/models.py +++ b/designsafe/apps/api/notifications/models.py @@ -1,9 +1,10 @@ -from django.db import models -from django.core.serializers.json import DjangoJSONEncoder import datetime -import logging import json +import logging + import six +from django.core.serializers.json import DjangoJSONEncoder +from django.db import models logger = logging.getLogger(__name__) @@ -65,7 +66,7 @@ def save(self, *args, **kwargs): logger.debug('Keys with error: %s . Value: %s', key, self.extra[key]) raise - super(BaseNotify, self).save(*args, **kwargs) + super().save(*args, **kwargs) @property def extra_content(self): @@ -89,7 +90,7 @@ def mark_deleted(self): self.save() def to_dict(self): - event_data = super(Notification, self).to_dict() + event_data = super().to_dict() event_data.update({ 'user': self.user, 'read': self.read, @@ -101,7 +102,7 @@ class Broadcast(BaseNotify): group = models.CharField(max_length=20) def to_dict(self): - event_data = super(Broadcast, self).to_dict() + event_data = super().to_dict() event_data.update({ 'group': self.group }) diff --git a/designsafe/apps/api/notifications/receivers.py b/designsafe/apps/api/notifications/receivers.py index 2b7c564157..65c593a022 100644 --- a/designsafe/apps/api/notifications/receivers.py +++ b/designsafe/apps/api/notifications/receivers.py @@ -1,15 +1,16 @@ """Signal receivers for notifications""" -import logging import json -from django.db.models.signals import post_save -from django.dispatch import receiver -from django.contrib.auth import get_user_model +import logging + from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from designsafe.apps.api.notifications.models import Notification, Broadcast -from designsafe.apps.onboarding.models import SetupEvent +from django.contrib.auth import get_user_model +from django.db.models.signals import post_save +from django.dispatch import receiver +from designsafe.apps.api.notifications.models import Broadcast, Notification +from designsafe.apps.onboarding.models import SetupEvent logger = logging.getLogger(__name__) diff --git a/designsafe/apps/api/notifications/tests.py b/designsafe/apps/api/notifications/tests.py index b0faf72ef8..6c4d1af3c8 100644 --- a/designsafe/apps/api/notifications/tests.py +++ b/designsafe/apps/api/notifications/tests.py @@ -1,18 +1,18 @@ import json +import logging import os -from django.test import TestCase -from django.test import Client +from unittest import skip +from unittest.mock import patch +from urllib.parse import urlencode + from django.contrib.auth import get_user_model from django.db.models.signals import post_save -from mock import patch -from urllib.parse import urlencode -from unittest import skip +from django.test import Client, TestCase from django.urls import reverse -from designsafe.apps.api.notifications.models import Notification -from .receivers import send_notification_ws -import logging +from designsafe.apps.api.notifications.models import Notification +from .receivers import send_notification_ws logger = logging.getLogger(__name__) diff --git a/designsafe/apps/api/notifications/urls.py b/designsafe/apps/api/notifications/urls.py index 6faab4b567..42380be6d9 100644 --- a/designsafe/apps/api/notifications/urls.py +++ b/designsafe/apps/api/notifications/urls.py @@ -1,6 +1,9 @@ from django.urls import re_path as url -from designsafe.apps.api.notifications.views.api import ManageNotificationsView, NotificationsBadgeView +from designsafe.apps.api.notifications.views.api import ( + ManageNotificationsView, + NotificationsBadgeView, +) urlpatterns = [ url(r'^$', ManageNotificationsView.as_view(), name='index'), diff --git a/designsafe/apps/api/notifications/views/api.py b/designsafe/apps/api/notifications/views/api.py index 7e569938ab..7e549936ae 100644 --- a/designsafe/apps/api/notifications/views/api.py +++ b/designsafe/apps/api/notifications/views/api.py @@ -1,10 +1,11 @@ -import logging import json +import logging + from django.http import HttpResponse, JsonResponse + +from designsafe.apps.api.mixins import JSONResponseMixin, SecureMixin from designsafe.apps.api.notifications.models import Notification from designsafe.apps.api.views import BaseApiView -from designsafe.apps.api.mixins import JSONResponseMixin, SecureMixin - logger = logging.getLogger(__name__) diff --git a/designsafe/apps/api/projects/fixtures.py b/designsafe/apps/api/projects/fixtures.py index 7a88609b1c..a90d755371 100644 --- a/designsafe/apps/api/projects/fixtures.py +++ b/designsafe/apps/api/projects/fixtures.py @@ -1,4 +1,5 @@ import datetime + from dateutil.tz import tzoffset exp_instance_meta =\ diff --git a/designsafe/apps/api/projects/models.py b/designsafe/apps/api/projects/models.py index 630ecbb72c..a844a022fe 100644 --- a/designsafe/apps/api/projects/models.py +++ b/designsafe/apps/api/projects/models.py @@ -1,17 +1,16 @@ -import six import json import logging -import xml.etree.ElementTree as ET -from designsafe.apps.data.models.agave.metadata import (BaseMetadataResource, - BaseMetadataPermissionResource) +import six + +from designsafe.apps.api.agave import to_camel_case from designsafe.apps.data.models.agave.files import BaseFileResource +from designsafe.apps.data.models.agave.metadata import ( + BaseMetadataPermissionResource, + BaseMetadataResource, +) from designsafe.apps.data.models.agave.systems import BaseSystemResource from designsafe.apps.data.models.agave.systems import roles as system_roles -from designsafe.apps.api.agave import to_camel_case -from designsafe.apps.data.models.agave.base import Model as MetadataModel -from designsafe.apps.data.models.agave import fields -from django.contrib.auth import get_user_model logger = logging.getLogger(__name__) @@ -33,7 +32,7 @@ def __init__(self, agave_client, **kwargs): 'name': Project.NAME } defaults.update(kwargs) - super(Project, self).__init__(agave_client, **defaults) + super().__init__(agave_client, **defaults) # initialize properties cache attributes self._project_directory = None @@ -105,7 +104,7 @@ def collaborators(self): return [pem.username for pem in permissions] def add_collaborator(self, username): - logger.info('Adding collaborator "{}" to project "{}"'.format(username, self.uuid)) + logger.info(f'Adding collaborator "{username}" to project "{self.uuid}"') # Set permissions on the metadata record pem = BaseMetadataPermissionResource(self.uuid, self._agave) @@ -118,7 +117,7 @@ def add_collaborator(self, username): self.project_system.add_role(username, system_roles.USER) def remove_collaborator(self, username): - logger.info('Removing collaborator "{}" from project "{}"'.format(username, self.uuid)) + logger.info(f'Removing collaborator "{username}" from project "{self.uuid}"') team_members = self.value.get('teamMembers', []) # logger.info(coPis) @@ -176,7 +175,7 @@ def co_pis(self): return self.value.get('coPis', []) def add_co_pi(self, username): - logger.info('Adding Co PI "{}" to project "{}"'.format(username, self.uuid)) + logger.info(f'Adding Co PI "{username}" to project "{self.uuid}"') coPis = self.value.get('coPis', []) @@ -185,7 +184,7 @@ def add_co_pi(self, username): self.add_collaborator(username) def remove_co_pi(self, username): - logger.info('Removing Co PI "{}" from project "{}"'.format(username, self.uuid)) + logger.info(f'Removing Co PI "{username}" from project "{self.uuid}"') coPis = self.value.get('coPis', []) # logger.info(coPis) @@ -233,7 +232,7 @@ def project_system(self): @property def project_system_id(self): - return 'project-{}'.format(self.uuid) + return f'project-{self.uuid}' @property def project_data_listing(self, path='/'): diff --git a/designsafe/apps/api/projects/tests.py b/designsafe/apps/api/projects/tests.py index 0b844416a2..59dad9f88c 100644 --- a/designsafe/apps/api/projects/tests.py +++ b/designsafe/apps/api/projects/tests.py @@ -1,6 +1,13 @@ -from designsafe.apps.api.projects.fixtures import exp_instance_meta, exp_instance_resp, exp_entity_meta, exp_entity_json import pytest +from designsafe.apps.api.projects.fixtures import ( + exp_entity_json, + exp_entity_meta, + exp_instance_meta, + exp_instance_resp, +) + + @pytest.mark.skip(reason="TODOv3: Update projects with Tapisv3") @pytest.mark.django_db def test_project_instance_get(client, mock_tapis_client, authenticated_user): diff --git a/designsafe/apps/api/projects/urls.py b/designsafe/apps/api/projects/urls.py index 7c3de86b5a..6a0361070a 100644 --- a/designsafe/apps/api/projects/urls.py +++ b/designsafe/apps/api/projects/urls.py @@ -11,15 +11,18 @@ """ from django.urls import re_path as url -from designsafe.apps.api.projects.views import (ProjectListingView, + +from designsafe.apps.api.projects.views import ( + AmendPublicationView, + NeesPublicationView, ProjectCollectionView, ProjectDataView, ProjectInstanceView, + ProjectListingView, ProjectMetaView, ProjectNotificationView, PublicationView, - AmendPublicationView, - NeesPublicationView) +) urlpatterns = [ url(r'^publication/(?P[A-Z\-]+-[0-9]+)?(v(?P[0-9]+))?/?', PublicationView.as_view(), name='publication'), diff --git a/designsafe/apps/api/projects/views.py b/designsafe/apps/api/projects/views.py index 305a0ef1ad..921f6ecde3 100644 --- a/designsafe/apps/api/projects/views.py +++ b/designsafe/apps/api/projects/views.py @@ -1,40 +1,44 @@ """Views""" import copy -import logging import json -from celery import group, chain -from django.urls import reverse +import logging + +from celery import chain, group from django.conf import settings -from django.http.response import HttpResponseForbidden -from django.http import JsonResponse, HttpResponseBadRequest from django.contrib.auth import get_user_model -from django.core.mail import send_mail -from django.utils.decorators import method_decorator from django.contrib.auth.decorators import login_required -from designsafe.apps.api.decorators import agave_jwt_login +from django.http import HttpResponseBadRequest, JsonResponse +from django.http.response import HttpResponseForbidden +from django.urls import reverse +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt + from designsafe.apps.api import tasks -from designsafe.apps.api.views import BaseApiView +from designsafe.apps.api.agave import ( + get_service_account_client, + to_camel_case, +) +from designsafe.apps.api.decorators import agave_jwt_login from designsafe.apps.api.mixins import SecureMixin from designsafe.apps.api.projects.models import Project -from designsafe.apps.projects.models.agave.base import Project as BaseProject -from designsafe.apps.projects.models.categories import Category -from designsafe.apps.api.agave import get_service_account_client, service_account, to_camel_case -from designsafe.apps.data.models.agave.metadata import BaseMetadataPermissionResource +from designsafe.apps.api.publications.operations import initilize_publication +from designsafe.apps.api.utils import get_client_ip +from designsafe.apps.api.views import BaseApiView from designsafe.apps.data.models.agave.files import BaseFileResource +from designsafe.apps.data.models.agave.metadata import BaseMetadataPermissionResource from designsafe.apps.data.models.agave.util import AgaveJSONEncoder -from designsafe.apps.accounts.models import DesignSafeProfile +from designsafe.apps.data.models.elasticsearch import IndexedPublication +from designsafe.apps.projects.models.agave.base import Project as BaseProject from designsafe.apps.projects.models.utils import lookup_model as project_lookup_model from designsafe.libs.common.decorators import profile as profile_fn -from designsafe.apps.api.publications.operations import initilize_publication +from designsafe.libs.elasticsearch.docs.publication_legacy import ( + BaseESPublicationLegacy, +) from designsafe.libs.elasticsearch.docs.publications import BaseESPublication -from designsafe.libs.elasticsearch.docs.publication_legacy import BaseESPublicationLegacy -from designsafe.apps.data.models.elasticsearch import IndexedPublication from designsafe.libs.elasticsearch.utils import new_es_client -from django.views.decorators.csrf import csrf_exempt -from elasticsearch_dsl import Q -from designsafe.apps.api.utils import get_client_ip + logger = logging.getLogger(__name__) -metrics = logging.getLogger('metrics.{name}'.format(name=__name__)) +metrics = logging.getLogger(f'metrics.{__name__}') def template_project_storage_system(project): @@ -251,22 +255,22 @@ def get(self, request, file_mgr_name=None, system_id=None, offset=0, limit=100): offset = request.GET.get('offset', 0) limit = request.GET.get('limit', 100) if query_string is not None: - projects = Project.ES_search(agave_client=client, query_string=query_string, **{'offset': offset, 'limit': limit}) + projects = Project.ES_search(agave_client=client, query_string=query_string, offset=offset, limit=limit) data = {'projects': projects} return JsonResponse(data, encoder=AgaveJSONEncoder) # Add metadata fields to project listings for workspace browser if system_id: - projects = Project.list_projects(agave_client=client, **{'path': '', 'type': 'dir', 'system': system_id}) + projects = Project.list_projects(agave_client=client, path='', type='dir', system=system_id) for p in projects: p.path = '' p.name = p.value['title'] - p.system = 'project-{}'.format(p.uuid) + p.system = f'project-{p.uuid}' data = { 'children': projects, 'path': 'Projects', } else: - projects = Project.list_projects(agave_client=client, **{'offset': offset, 'limit': limit}) + projects = Project.list_projects(agave_client=client, offset=offset, limit=limit) data = {'projects': projects} return JsonResponse(data, encoder=AgaveJSONEncoder) diff --git a/designsafe/apps/api/projects_v2/conftest.py b/designsafe/apps/api/projects_v2/conftest.py index 65d4d4a4ef..9358b6410d 100644 --- a/designsafe/apps/api/projects_v2/conftest.py +++ b/designsafe/apps/api/projects_v2/conftest.py @@ -1,16 +1,17 @@ """Fixtures related to project metadata.""" import pytest + from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.projects_v2.operations.graph_operations import ( + add_node_to_project, + initialize_project_graph, +) from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( - create_project_metdata, - create_entity_metadata, + FileObj, add_file_associations, + create_entity_metadata, + create_project_metdata, set_file_tags, - FileObj, -) -from designsafe.apps.api.projects_v2.operations.graph_operations import ( - initialize_project_graph, - add_node_to_project, ) diff --git a/designsafe/apps/api/projects_v2/constants.py b/designsafe/apps/api/projects_v2/constants.py index 8cd2abc2df..e99a3a923d 100644 --- a/designsafe/apps/api/projects_v2/constants.py +++ b/designsafe/apps/api/projects_v2/constants.py @@ -97,10 +97,10 @@ { "id": "niche", "name": ( - ( + "National Full-Scale Testing Infrastructure for Community Hardening " "in Extreme Wind, Surge, and Wave Events (NICHE)" - ) + ), }, { diff --git a/designsafe/apps/api/projects_v2/elasticsearch.py b/designsafe/apps/api/projects_v2/elasticsearch.py index 24d2260754..19f687c0e7 100644 --- a/designsafe/apps/api/projects_v2/elasticsearch.py +++ b/designsafe/apps/api/projects_v2/elasticsearch.py @@ -1,9 +1,10 @@ """Elasticsearch model for published works""" -from elasticsearch_dsl import Document -from elasticsearch.exceptions import NotFoundError import networkx as nx from django.conf import settings +from elasticsearch.exceptions import NotFoundError +from elasticsearch_dsl import Document + from designsafe.apps.api.projects_v2.models import ProjectMetadata from designsafe.apps.api.projects_v2.operations.project_publish_operations import ( add_values_to_tree, diff --git a/designsafe/apps/api/projects_v2/management/commands/migrate_projects_tapis_systems_from_v2_to_v3.py b/designsafe/apps/api/projects_v2/management/commands/migrate_projects_tapis_systems_from_v2_to_v3.py index 4326fbe788..ea2f13a449 100644 --- a/designsafe/apps/api/projects_v2/management/commands/migrate_projects_tapis_systems_from_v2_to_v3.py +++ b/designsafe/apps/api/projects_v2/management/commands/migrate_projects_tapis_systems_from_v2_to_v3.py @@ -7,13 +7,13 @@ # pylint: disable=logging-fstring-interpolation # pylint: disable=no-member - import logging import os -from tapipy.tapis import Tapis from django.conf import settings from django.core.management.base import BaseCommand +from tapipy.tapis import Tapis + from designsafe.apps.api.projects_v2.tests.schema_integration import iterate_entities try: @@ -49,7 +49,7 @@ def set_workspace_permissions(client: Tapis, username: str, system_id: str, role files_pems = {"reader": "READ", "writer": "MODIFY"} - logger.info(f"Adding {username} permissions to Tapis system {system_id}") + logger.info("Adding %s, permissions to Tapis system %s", username, system_id) client.systems.grantUserPerms( systemId=system_id, userName=username, permissions=system_pems[role] ) @@ -140,9 +140,7 @@ def add_arguments(self, parser): "ensuring they are synchronized with their V2 counterparts.", ) - def handle( - self, *args, **options - ): # pylint: disable=too-many-statements disable=too-many-locals + def handle(self, *args, **options): # pylint: disable=too-many-statements disable=too-many-locals dry_run = options["dry_run"] update_existing = options["update_existing"] @@ -166,23 +164,11 @@ def handle( else "/corral-repl/projects/NHERI/community" # community data system has as a special path ) title = project["value"]["title"] - description = ( - project["value"]["description"] - if "description" in project["value"] - else "" - ) + description = project["value"].get("description", "") pi = project["value"]["pi"] # pylint: disable=invalid-name - co_pis = project["value"]["coPis"] if "coPis" in project["value"] else [] - team_members = ( - project["value"]["teamMembers"] - if "teamMembers" in project["value"] - else [] - ) - guest_members = ( - project["value"]["guestMembers"] - if ("guestMembers" in project["value"]) - else [] - ) + co_pis = project["value"].get("coPis", []) + team_members = project["value"].get("teamMembers", []) + guest_members = project["value"].get("guestMembers", []) guest_members = [u["user"] for u in guest_members if u is not None] all_writers = ( @@ -195,7 +181,7 @@ def handle( ) except Exception: # pylint: disable=broad-exception-caught: tapis_v2_system_roles = [] - logger.error(f"Unable to get roles on uuid:{uuid}") + logger.exception("Unable to get roles on uuid: %s", uuid) users_from_roles = [u["username"] for u in tapis_v2_system_roles] users_from_roles_not_listed_elsewhere = [ u for u in users_from_roles if u not in all_writers @@ -219,6 +205,7 @@ def handle( try: client.systems.getSystem(systemId=system) except Exception: # pylint: disable=broad-exception-caught + logger.exception("") system_exists = False if not dry_run: @@ -233,7 +220,7 @@ def handle( all_users ) for user in user_to_remove: - logger.info(f"removing user: {user}") + logger.info("removing user: %s", user) remove_user(client, system, user) create = not system_exists @@ -257,7 +244,7 @@ def handle( for user in all_readers: set_workspace_permissions(client, user, system, "reader") except Exception: # pylint: disable=broad-exception-caught - logger.exception(f"Error for system:{system}") + logger.exception("Error for system: %s", system) else: system_exists_text = ( "System already exists" @@ -265,8 +252,7 @@ def handle( else "System does not exist" ) logger.info( - f"Running in dry-run mode. No changes will be made. " - f"" - f"Note: {system_exists_text}" + "Running in dry-run mode. No changes will be made. Note: %s", + system_exists_text, ) logger.info("Successfully migrated systems") diff --git a/designsafe/apps/api/projects_v2/migration_utils/_tests/pub_symlink_integration.py b/designsafe/apps/api/projects_v2/migration_utils/_tests/pub_symlink_integration.py index d473a193a5..4bb3244d15 100644 --- a/designsafe/apps/api/projects_v2/migration_utils/_tests/pub_symlink_integration.py +++ b/designsafe/apps/api/projects_v2/migration_utils/_tests/pub_symlink_integration.py @@ -14,7 +14,6 @@ def symlink_mapping_integration_test(): mapping = construct_symlink_mapping(pub.project_id) for value in mapping.values(): for path in value: - if len(path) > max_len: - max_len = len(path) + max_len = max(max_len, len(path)) print(max_len) diff --git a/designsafe/apps/api/projects_v2/migration_utils/file_obj_ingest.py b/designsafe/apps/api/projects_v2/migration_utils/file_obj_ingest.py index 152f95432b..5132c0bba6 100644 --- a/designsafe/apps/api/projects_v2/migration_utils/file_obj_ingest.py +++ b/designsafe/apps/api/projects_v2/migration_utils/file_obj_ingest.py @@ -2,19 +2,21 @@ import json import os -from typing import Iterator +from collections.abc import Iterator + import requests -from urllib3.util import Retry +from django.conf import settings from requests import Session -from requests.exceptions import RetryError from requests.adapters import HTTPAdapter -from django.conf import settings +from requests.exceptions import RetryError +from urllib3.util import Retry + from designsafe.apps.api.agave import get_service_account_client_v2 as service_account -from designsafe.apps.api.projects_v2.schema_models._field_models import FileObj from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( add_file_associations, ) +from designsafe.apps.api.projects_v2.schema_models._field_models import FileObj def iterate_entities(name="designsafe.project") -> Iterator[dict]: diff --git a/designsafe/apps/api/projects_v2/migration_utils/graph_constructor.py b/designsafe/apps/api/projects_v2/migration_utils/graph_constructor.py index 0c0d9c2d80..99b0c6e896 100644 --- a/designsafe/apps/api/projects_v2/migration_utils/graph_constructor.py +++ b/designsafe/apps/api/projects_v2/migration_utils/graph_constructor.py @@ -1,23 +1,25 @@ """Utils for constructing project trees from legacy association IDs.""" -import json -from typing import TypedDict, Optional -from uuid import uuid4 import copy +import json from pathlib import Path +from typing import TypedDict +from uuid import uuid4 + import networkx as nx from django.utils.text import slugify + from designsafe.apps.api.agave import get_service_account_client_v2 as service_account -from designsafe.apps.data.models.elasticsearch import IndexedPublication -from designsafe.apps.projects.managers.publication import FIELD_MAP -from designsafe.apps.api.projects_v2.schema_models import PATH_SLUGS -from designsafe.apps.projects.models.categories import Category from designsafe.apps.api.projects_v2 import constants as names -from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata from designsafe.apps.api.projects_v2.migration_utils.publication_transforms import ( - transform_entity, construct_users, + transform_entity, ) +from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata +from designsafe.apps.api.projects_v2.schema_models import PATH_SLUGS +from designsafe.apps.data.models.elasticsearch import IndexedPublication +from designsafe.apps.projects.managers.publication import FIELD_MAP +from designsafe.apps.projects.models.categories import Category # map metadata 'name' field to allowed (direct) children ALLOWED_RELATIONS = { @@ -86,7 +88,7 @@ def get_entities_by_project_id(project_id: str) -> list[dict]: def get_path(graph: nx.DiGraph, node_id: str): """Iterate through all predecessors in the graph (inclusive)""" shortest_path = nx.shortest_path(graph, "NODE_ROOT", node_id) - path_uuids = map(lambda node: graph.nodes[node]["uuid"], shortest_path) + path_uuids = (graph.nodes[node]["uuid"] for node in shortest_path) return list(path_uuids) @@ -134,7 +136,7 @@ def construct_graph_from_db(project_id) -> nx.DiGraph: # support multiple versions. project_graph.add_node( root_node_id, - **{"uuid": None, "name": None, "projectType": "other", "order": 0}, + uuid=None, name=None, projectType="other", order=0, ) base_node_id = f"NODE_project_{uuid4()}" project_graph.add_node(base_node_id, **base_node_data) @@ -277,7 +279,7 @@ def construct_publication_graph( def construct_entity_filepaths( - entity_listing: list[dict], pub_graph: nx.DiGraph, version: Optional[int] = None + entity_listing: list[dict], pub_graph: nx.DiGraph, version: int | None = None ): """ Walk the publication graph and construct base file paths for each node. @@ -309,7 +311,7 @@ def construct_entity_filepaths( def construct_entity_filepaths_legacy( - pub_graph: nx.DiGraph, version: Optional[int] = None + pub_graph: nx.DiGraph, version: int | None = None ): """ Walk the publication graph and construct base file paths for each node. @@ -345,7 +347,7 @@ def get_entity_orders( return prj_orders.to_dict().get("orders", []) -def transform_pub_entities(project_id: str, version: Optional[int] = None): +def transform_pub_entities(project_id: str, version: int | None = None): """Validate publication entities against their corresponding model.""" entity_listing = get_entities_from_publication(project_id, version=version) base_pub_meta = IndexedPublication.from_id(project_id, revision=version).to_dict() @@ -356,7 +358,7 @@ def transform_pub_entities(project_id: str, version: Optional[int] = None): ) path_mappings = [] - for _, node_data in pub_graph.nodes.items(): + for node_data in pub_graph.nodes.values(): node_entity = next( (e for e in entity_listing if e["uuid"] == node_data["uuid"]), None ) diff --git a/designsafe/apps/api/projects_v2/migration_utils/project_db_ingest.py b/designsafe/apps/api/projects_v2/migration_utils/project_db_ingest.py index 165f9a63eb..79eb5493f6 100644 --- a/designsafe/apps/api/projects_v2/migration_utils/project_db_ingest.py +++ b/designsafe/apps/api/projects_v2/migration_utils/project_db_ingest.py @@ -1,23 +1,25 @@ """Utilities for ingesting projects using the ProjectMetadata db model.""" -from datetime import datetime, timezone -from pydantic import ValidationError -from elasticsearch_dsl import Q +from datetime import UTC, datetime + import networkx as nx -from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata +from elasticsearch_dsl import Q +from pydantic import ValidationError + from designsafe.apps.api.projects_v2.migration_utils.graph_constructor import ( - get_entities_by_project_id, - construct_graph_from_db, combine_pub_versions, + construct_graph_from_db, + get_entities_by_project_id, +) +from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata +from designsafe.apps.api.projects_v2.operations.graph_operations import ( + _renormalize_ordering, ) from designsafe.apps.api.projects_v2.schema_models import SCHEMA_MAPPING from designsafe.apps.api.projects_v2.tests.schema_integration import ( iterate_entities, iterate_pubs, ) -from designsafe.apps.api.projects_v2.operations.graph_operations import ( - _renormalize_ordering, -) from designsafe.apps.api.publications_v2.models import Publication from designsafe.apps.data.models.elasticsearch import IndexedPublication @@ -78,9 +80,9 @@ def ingest_entities_by_name(name): ] try: value_model = schema_model.model_validate(entity["value"]) - except ValidationError as err: + except ValidationError: print(entity) - raise err + raise try: prj = ProjectMetadata.objects.get( name="designsafe.project", uuid__in=entity["associationIds"] @@ -177,7 +179,7 @@ def fix_guest_members(): print(prj.project_id) for guest in legacy_guests: already_added = any( - ( + u for u in existing_users if ( @@ -186,7 +188,7 @@ def fix_guest_members(): and u["lname"] == guest["lname"] and u.get("email", "") == guest.get("email", "") ) - ) + ) if not already_added: prj.value["users"].append( @@ -250,7 +252,7 @@ def ingest_publications(): project_id=pub["projectId"], defaults={ "created": datetime.fromisoformat(pub["created"]).replace( - tzinfo=timezone.utc + tzinfo=UTC ), "tree": pub_graph_json, "value": pub_base, @@ -312,7 +314,7 @@ def ingest_tombstones(): defaults={ "is_published": False, "created": datetime.fromisoformat(pub["created"]).replace( - tzinfo=timezone.utc + tzinfo=UTC ), "tree": pub_graph_json, "value": pub_base, diff --git a/designsafe/apps/api/projects_v2/migration_utils/pub_symlink_operations.py b/designsafe/apps/api/projects_v2/migration_utils/pub_symlink_operations.py index 9652c56180..6c6e507c11 100644 --- a/designsafe/apps/api/projects_v2/migration_utils/pub_symlink_operations.py +++ b/designsafe/apps/api/projects_v2/migration_utils/pub_symlink_operations.py @@ -3,17 +3,19 @@ import os import subprocess from pathlib import Path + import networkx as nx from django.conf import settings -from designsafe.libs.common.context_managers import Workdir -from designsafe.apps.api.publications_v2.models import Publication + +from designsafe.apps.api.datafiles.models import PublicationSymlink from designsafe.apps.api.projects_v2 import constants from designsafe.apps.api.projects_v2.operations.path_operations import ( construct_entity_filepaths, construct_published_path_mappings, update_path_mappings, ) -from designsafe.apps.api.datafiles.models import PublicationSymlink +from designsafe.apps.api.publications_v2.models import Publication +from designsafe.libs.common.context_managers import Workdir published_path = settings.DESIGNSAFE_PUBLISHED_PATH dataset_path = settings.PUBLISHED_DATASET_PATH @@ -111,12 +113,12 @@ def migrate_publication_metadata(project_id): latest_version = 1 base_meta_node = next( - ( + node for node in pub_tree.nodes if pub_tree.nodes[node]["name"] == constants.PROJECT and pub_tree.nodes[node].get("version", latest_version) == latest_version - ) + ) base_meta_value = pub_tree.nodes[base_meta_node]["value"] pub.value = base_meta_value diff --git a/designsafe/apps/api/projects_v2/migration_utils/publication_file_operations.py b/designsafe/apps/api/projects_v2/migration_utils/publication_file_operations.py index 1ca93227e4..f6cc2f5760 100644 --- a/designsafe/apps/api/projects_v2/migration_utils/publication_file_operations.py +++ b/designsafe/apps/api/projects_v2/migration_utils/publication_file_operations.py @@ -1,8 +1,9 @@ """Operations to format published data in accordance with the project structure""" -import shutil import os +import shutil from pathlib import Path + from designsafe.apps.api.projects_v2.migration_utils.graph_constructor import ( transform_pub_entities, ) @@ -21,11 +22,11 @@ def format_publication_data( pub_graph, path_mappings = transform_pub_entities(project_id, version) base_project = next( - ( + node for node in pub_graph if pub_graph.nodes[node]["name"] == "designsafe.project" - ) + ) prj_value = pub_graph.nodes[base_project] diff --git a/designsafe/apps/api/projects_v2/migration_utils/publication_transforms.py b/designsafe/apps/api/projects_v2/migration_utils/publication_transforms.py index 6cbba3be24..41549582a6 100644 --- a/designsafe/apps/api/projects_v2/migration_utils/publication_transforms.py +++ b/designsafe/apps/api/projects_v2/migration_utils/publication_transforms.py @@ -2,8 +2,10 @@ from pathlib import Path from typing import TypedDict + from django.conf import settings from django.contrib.auth import get_user_model + from designsafe.apps.api.projects_v2.schema_models import SCHEMA_MAPPING user_model = get_user_model() @@ -27,7 +29,7 @@ def convert_v1_user(user): } -def get_user_info(username: str, role: str = None) -> dict: +def get_user_info(username: str, role: str | None = None) -> dict: """Construct a user object from info in the db.""" try: user_obj = user_model.objects.get(username=username) @@ -141,7 +143,7 @@ def get_file_obj_by_uuid(uuid): return file_obj for inner_dict in tags.values(): - for key in inner_dict.keys(): + for key in inner_dict: tag_name = next( (tag["label"] for tag in tag_options if tag["name"] == key), None ) diff --git a/designsafe/apps/api/projects_v2/models/__init__.py b/designsafe/apps/api/projects_v2/models/__init__.py index 58b001e946..8621221c12 100644 --- a/designsafe/apps/api/projects_v2/models/__init__.py +++ b/designsafe/apps/api/projects_v2/models/__init__.py @@ -1,2 +1,5 @@ """Projects V2 API models.""" -from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata + +from designsafe.apps.api.projects_v2.models.project_metadata import ( + ProjectMetadata as ProjectMetadata, +) diff --git a/designsafe/apps/api/projects_v2/models/project_metadata.py b/designsafe/apps/api/projects_v2/models/project_metadata.py index b8d5bb3289..4e5873c57f 100644 --- a/designsafe/apps/api/projects_v2/models/project_metadata.py +++ b/designsafe/apps/api/projects_v2/models/project_metadata.py @@ -1,12 +1,14 @@ """Models for representing project metadata""" import uuid -from django.utils import timezone -from django.db import models -from django.core.validators import MinLengthValidator -from django.core.serializers.json import DjangoJSONEncoder -from django.contrib.auth import get_user_model + from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.serializers.json import DjangoJSONEncoder +from django.core.validators import MinLengthValidator +from django.db import models from django.dispatch import receiver +from django.utils import timezone + from designsafe.apps.api.projects_v2 import constants user_model = get_user_model() diff --git a/designsafe/apps/api/projects_v2/operations/_tests/path_operations_unit_test.py b/designsafe/apps/api/projects_v2/operations/_tests/path_operations_unit_test.py index 0b7e68a681..ad6eb3f419 100644 --- a/designsafe/apps/api/projects_v2/operations/_tests/path_operations_unit_test.py +++ b/designsafe/apps/api/projects_v2/operations/_tests/path_operations_unit_test.py @@ -1,11 +1,12 @@ """Unit tests for the path_operations module""" -import pytest -import networkx as nx from pathlib import Path + +import networkx as nx + from designsafe.apps.api.projects_v2.operations._tests.path_operations_fixtures import ( - PUBLISHED_SIM_FIXTURE, PUBLISHED_OTHER_FIXTURE, + PUBLISHED_SIM_FIXTURE, ) from designsafe.apps.api.projects_v2.operations.path_operations import ( construct_entity_filepaths, @@ -58,29 +59,29 @@ def test_path_mappings_added_to_graph_sim(): # Assert that path mappings are applied to node metadata assert any( - ( + fo for fo in updated_graph.nodes["NODE_401565e2-2271-4b2c-b822-f89935a5a15f"][ "value" ]["fileObjs"] if fo["path"] == "/test/PRJ-3462/Simulation--parameter-influence-on-tropical-cyclone-rainfall/data/Model--tcr-model/Input--simulation-input/Output--simulation-of-parameter-influence-on-tropical-cyclone-rainfall/data/spatial_important_update.mat" - ) + ) assert any( - ( + fo for fo in updated_graph.nodes["NODE_401565e2-2271-4b2c-b822-f89935a5a15f"][ "value" ]["fileObjs"] if fo["path"] == "/test/PRJ-3462/Simulation--parameter-influence-on-tropical-cyclone-rainfall/data/Model--tcr-model/Input--simulation-input/Output--simulation-of-parameter-influence-on-tropical-cyclone-rainfall/data/spatial_important_update(1).mat" - ) + ) # Assert that file path mappings are reflected in file tags assert any( - ( + tag for tag in updated_graph.nodes["NODE_401565e2-2271-4b2c-b822-f89935a5a15f"][ "value" @@ -88,10 +89,10 @@ def test_path_mappings_added_to_graph_sim(): if tag["tagName"] == "tropical cyclone" and tag["path"] == "/test/PRJ-3462/Simulation--parameter-influence-on-tropical-cyclone-rainfall/data/Model--tcr-model/Input--simulation-input/Output--simulation-of-parameter-influence-on-tropical-cyclone-rainfall/data/spatial_important_update.mat" - ) + ) assert any( - ( + tag for tag in updated_graph.nodes["NODE_401565e2-2271-4b2c-b822-f89935a5a15f"][ "value" @@ -99,7 +100,7 @@ def test_path_mappings_added_to_graph_sim(): if tag["tagName"] == "tropical cyclone (duplicate filename)" and tag["path"] == "/test/PRJ-3462/Simulation--parameter-influence-on-tropical-cyclone-rainfall/data/Model--tcr-model/Input--simulation-input/Output--simulation-of-parameter-influence-on-tropical-cyclone-rainfall/data/spatial_important_update(1).mat" - ) + ) diff --git a/designsafe/apps/api/projects_v2/operations/_tests/project_meta_unit_test.py b/designsafe/apps/api/projects_v2/operations/_tests/project_meta_unit_test.py index a3e155b615..fe273f1b65 100644 --- a/designsafe/apps/api/projects_v2/operations/_tests/project_meta_unit_test.py +++ b/designsafe/apps/api/projects_v2/operations/_tests/project_meta_unit_test.py @@ -1,26 +1,27 @@ """Tests for operations that create/update project metadata.""" -import pytest import networkx as nx +import pytest from django import db + from designsafe.apps.api.projects_v2 import constants -from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( - create_project_metdata, - create_entity_metadata, - change_project_type, - add_file_associations, - remove_file_associations, - add_file_tags, - remove_file_tags, - ProjectMetadata, - FileObj, - FileTag, -) from designsafe.apps.api.projects_v2.operations.graph_operations import ( - initialize_project_graph, add_node_to_project, + initialize_project_graph, remove_nodes_from_project, reorder_project_nodes, ) +from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( + FileObj, + FileTag, + ProjectMetadata, + add_file_associations, + add_file_tags, + change_project_type, + create_entity_metadata, + create_project_metdata, + remove_file_associations, + remove_file_tags, +) @pytest.mark.django_db diff --git a/designsafe/apps/api/projects_v2/operations/_tests/publish_unit_test.py b/designsafe/apps/api/projects_v2/operations/_tests/publish_unit_test.py index 9224691bc3..63efaaabad 100644 --- a/designsafe/apps/api/projects_v2/operations/_tests/publish_unit_test.py +++ b/designsafe/apps/api/projects_v2/operations/_tests/publish_unit_test.py @@ -1,24 +1,19 @@ """Tests for operations to convert metadata for publication""" import pytest -import networkx as nx -from django import db + from designsafe.apps.api.projects_v2 import constants -from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( - create_project_metdata, - create_entity_metadata, - add_file_associations, - set_file_tags, - FileObj, -) from designsafe.apps.api.projects_v2.operations.graph_operations import ( - initialize_project_graph, add_node_to_project, ) - +from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( + FileObj, + add_file_associations, + create_entity_metadata, +) from designsafe.apps.api.projects_v2.operations.project_publish_operations import ( - get_publication_subtree, get_publication_full_tree, + get_publication_subtree, ) @@ -51,9 +46,9 @@ def test_publication_subtree(project_with_associations): assert expected_tag_path_1 in entity_tag_paths assert expected_tag_path_2 in entity_tag_paths assert path_mapping[mc_node] == { - f"/path/to/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/file1", - f"/path/to/other/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/file1(1)", - f"/path/to/dir1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/dir1", + "/path/to/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/file1", + "/path/to/other/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/file1(1)", + "/path/to/dir1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/dir1", } @@ -87,9 +82,9 @@ def test_publication_subtree_with_version(project_with_associations): assert expected_tag_path_2 in entity_tag_paths assert path_mapping[mc_node] == { - f"/path/to/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment--V2/data/Model-config--test-entity/data/file1", - f"/path/to/other/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment--V2/data/Model-config--test-entity/data/file1(1)", - f"/path/to/dir1": "/published-data-test/PRJ-1234/Experiment--test-experiment--V2/data/Model-config--test-entity/data/dir1", + "/path/to/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment--V2/data/Model-config--test-entity/data/file1", + "/path/to/other/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment--V2/data/Model-config--test-entity/data/file1(1)", + "/path/to/dir1": "/published-data-test/PRJ-1234/Experiment--test-experiment--V2/data/Model-config--test-entity/data/dir1", } @@ -155,13 +150,13 @@ def test_tree_multiple_experiments(project_with_associations): ) assert full_path_mapping[mc_node_1] == { - f"/path/to/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/file1", - f"/path/to/other/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/file1(1)", - f"/path/to/dir1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/dir1", + "/path/to/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/file1", + "/path/to/other/file1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/file1(1)", + "/path/to/dir1": "/published-data-test/PRJ-1234/Experiment--test-experiment/data/Model-config--test-entity/data/dir1", } assert full_path_mapping[mc_node_2] == { - f"/path/to/file3": "/published-data-test/PRJ-1234/Experiment--test-experiment-2/data/Model-config--test-entity-2/data/file3", - f"/path/to/other/file3": "/published-data-test/PRJ-1234/Experiment--test-experiment-2/data/Model-config--test-entity-2/data/file3(1)", - f"/path/to/dir2": "/published-data-test/PRJ-1234/Experiment--test-experiment-2/data/Model-config--test-entity-2/data/dir2", + "/path/to/file3": "/published-data-test/PRJ-1234/Experiment--test-experiment-2/data/Model-config--test-entity-2/data/file3", + "/path/to/other/file3": "/published-data-test/PRJ-1234/Experiment--test-experiment-2/data/Model-config--test-entity-2/data/file3(1)", + "/path/to/dir2": "/published-data-test/PRJ-1234/Experiment--test-experiment-2/data/Model-config--test-entity-2/data/dir2", } diff --git a/designsafe/apps/api/projects_v2/operations/datacite_operations.py b/designsafe/apps/api/projects_v2/operations/datacite_operations.py index 6d36d4cb16..72c453c54b 100644 --- a/designsafe/apps/api/projects_v2/operations/datacite_operations.py +++ b/designsafe/apps/api/projects_v2/operations/datacite_operations.py @@ -1,16 +1,17 @@ """Operations to format and manage Datacite DOIs""" import datetime -from typing import Optional import json -import requests + import networkx as nx +import requests from django.conf import settings + from designsafe.apps.api.projects_v2 import constants from designsafe.apps.api.projects_v2.schema_models import PATH_SLUGS -def format_datacite_date_range(date_start: str, date_end: Optional[str]): +def format_datacite_date_range(date_start: str, date_end: str | None): """Convert a date range to datacite JSON""" date_string = date_start if date_end: @@ -33,7 +34,7 @@ def dedup_collectors(collectors: list[dict]) -> list[dict]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements def get_datacite_json( - pub_graph: nx.DiGraph, entity_uuid: str, version: Optional[int] = 1 + pub_graph: nx.DiGraph, entity_uuid: str, version: int | None = 1 ): """ Generate datacite payload for a publishable entity. `pub_graph` is the output of @@ -149,11 +150,11 @@ def get_datacite_json( datacite_json["titles"] = [ {"title": f"{title}, in {base_meta['title']}"} - for title in set([entity_meta["title"]]) + for title in {entity_meta["title"]} ] else: datacite_json["titles"] = [ - {"title": title} for title in set([entity_meta["title"]]) + {"title": title} for title in {entity_meta["title"]} ] datacite_json["publisher"] = "Designsafe-CI" @@ -192,7 +193,7 @@ def get_datacite_json( "description": desc, "lang": "en-Us", } - for desc in set([base_meta["description"], entity_meta["description"]]) + for desc in {base_meta["description"], entity_meta["description"]} ] if not is_other: @@ -293,7 +294,7 @@ def get_datacite_json( return datacite_json -def upsert_datacite_json(datacite_json: dict, doi: Optional[str] = None): +def upsert_datacite_json(datacite_json: dict, doi: str | None = None): """ Create a draft DOI in datacite with the specified metadata. If a DOI is specified, the metadata for that DOI is updated instead. diff --git a/designsafe/apps/api/projects_v2/operations/graph_operations.py b/designsafe/apps/api/projects_v2/operations/graph_operations.py index 6399d35d9f..762c2cbee9 100644 --- a/designsafe/apps/api/projects_v2/operations/graph_operations.py +++ b/designsafe/apps/api/projects_v2/operations/graph_operations.py @@ -1,9 +1,11 @@ """Utilities for adding, removing, and reordering nodes in the project graph.""" -import uuid import copy +import uuid + import networkx as nx from django.db import transaction + from designsafe.apps.api.projects_v2 import constants from designsafe.apps.api.projects_v2.models import ProjectMetadata @@ -76,7 +78,7 @@ def _reorder_nodes(graph: nx.DiGraph, node_id: str, new_index: int): old_index = _graph.nodes[node_id]["order"] sorted_siblings: list[str] = sorted( - list(_graph.successors(parent)), key=lambda n: _graph.nodes[n]["order"] + _graph.successors(parent), key=lambda n: _graph.nodes[n]["order"] ) if new_index > old_index: @@ -116,7 +118,7 @@ def initialize_project_graph(project_id: str): # type Other projects have a "null" parent node above the project root, to # support multiple versions. project_graph.add_node( - root_node_id, **{"uuid": None, "name": None, "projectType": project_type} + root_node_id, uuid=None, name=None, projectType=project_type ) base_node_id = f"NODE_project_{uuid.uuid4()}" project_graph.add_node(base_node_id, **base_node_data) diff --git a/designsafe/apps/api/projects_v2/operations/path_operations.py b/designsafe/apps/api/projects_v2/operations/path_operations.py index b61c56b9cd..29e78a8857 100644 --- a/designsafe/apps/api/projects_v2/operations/path_operations.py +++ b/designsafe/apps/api/projects_v2/operations/path_operations.py @@ -2,30 +2,31 @@ in the directory structure. UNUSED for now, pending stakeholder approval. """ -from typing import Optional import copy -import tempfile -import subprocess +import logging import shutil +import subprocess +import tempfile from pathlib import Path -import logging + import networkx as nx from celery import shared_task -from django.utils.text import slugify from django.conf import settings -from designsafe.libs.common.context_managers import Workdir, AsyncTaskContext -from designsafe.apps.api.publications_v2.models import Publication -from designsafe.apps.api.projects_v2.schema_models import PATH_SLUGS +from django.utils.text import slugify + from designsafe.apps.api.projects_v2.operations.project_archive_operations import ( ranch_archive_webhook, ) +from designsafe.apps.api.projects_v2.schema_models import PATH_SLUGS +from designsafe.apps.api.publications_v2.models import Publication +from designsafe.libs.common.context_managers import AsyncTaskContext, Workdir logger = logging.getLogger(__name__) def construct_entity_filepaths( pub_graph: nx.DiGraph, - version: Optional[int] = None, + version: int | None = None, dataset_root=None, strip_legacy_versions=False, ): @@ -247,9 +248,9 @@ def generate_manifests_for_project(project_id: str): subprocess.CalledProcessError, subprocess.SubprocessError, FileNotFoundError, - ) as exc: + ): logger.debug("Failed to generate manifest at %s", path) - raise exc + raise logger.debug("Finished generating manifests for publication %s", project_id) diff --git a/designsafe/apps/api/projects_v2/operations/project_archive_operations.py b/designsafe/apps/api/projects_v2/operations/project_archive_operations.py index 89d1546bf1..26e5c80f6d 100644 --- a/designsafe/apps/api/projects_v2/operations/project_archive_operations.py +++ b/designsafe/apps/api/projects_v2/operations/project_archive_operations.py @@ -2,16 +2,17 @@ Utility to create a ZIP archive of a publication and its metadata. """ -import os import json import logging -from typing import Optional +import os + import requests -from requests.auth import HTTPBasicAuth -from django.conf import settings from celery import shared_task -from designsafe.libs.common.context_managers import AsyncTaskContext +from django.conf import settings +from requests.auth import HTTPBasicAuth + from designsafe.apps.api.publications_v2.models import Publication +from designsafe.libs.common.context_managers import AsyncTaskContext logger = logging.getLogger(__name__) @@ -54,7 +55,7 @@ def ranch_archive_webhook(project_id: str): @shared_task -def archive_publication_async(project_id: str, version: Optional[str] = 1): +def archive_publication_async(project_id: str, version: str | None = 1): """async wrapper around archive""" with AsyncTaskContext(): create_metadata_file(project_id, version) diff --git a/designsafe/apps/api/projects_v2/operations/project_email_operations.py b/designsafe/apps/api/projects_v2/operations/project_email_operations.py index 7420122707..2aeb590494 100644 --- a/designsafe/apps/api/projects_v2/operations/project_email_operations.py +++ b/designsafe/apps/api/projects_v2/operations/project_email_operations.py @@ -1,11 +1,11 @@ """Utilities to email users/staff in response to project lifecycle changes.""" -from typing import Optional + from django.conf import settings from django.core.mail import send_mail -def send_project_permissions_alert(project_id: str, version: Optional[int], error: str): +def send_project_permissions_alert(project_id: str, version: int | None, error: str): """ Alert dev team when a project has encountered a permission error during publication. """ diff --git a/designsafe/apps/api/projects_v2/operations/project_meta_operations.py b/designsafe/apps/api/projects_v2/operations/project_meta_operations.py index 23b2ac84ae..082f9e6a1f 100644 --- a/designsafe/apps/api/projects_v2/operations/project_meta_operations.py +++ b/designsafe/apps/api/projects_v2/operations/project_meta_operations.py @@ -2,20 +2,21 @@ import operator import re -from typing import Optional + import requests from django.db import models, transaction from pydantic import BaseModel + +from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.projects_v2.models import ProjectMetadata +from designsafe.apps.api.projects_v2.operations import graph_operations from designsafe.apps.api.projects_v2.schema_models import SCHEMA_MAPPING from designsafe.apps.api.projects_v2.schema_models.base import ( + BaseProject, FileObj, FileTag, PartialEntityWithFiles, - BaseProject, ) -from designsafe.apps.api.projects_v2 import constants -from designsafe.apps.api.projects_v2.models import ProjectMetadata -from designsafe.apps.api.projects_v2.operations import graph_operations def create_project_metdata(value): @@ -83,12 +84,12 @@ def get_changed_users(old_value: BaseProject, new_value: BaseProject): Diff users between incoming and existing project metadata to determine which users need permissions to be added/removed via Tapis. """ - old_users = set( - (u.username for u in old_value.users if u.username and u.role != "guest") - ) - new_users = set( - (u.username for u in new_value.users if u.username and u.role != "guest") - ) + old_users = { + u.username for u in old_value.users if u.username and u.role != "guest" + } + new_users = { + u.username for u in new_value.users if u.username and u.role != "guest" + } users_to_add = list(new_users - old_users) users_to_remove = list(old_users - new_users) @@ -276,8 +277,8 @@ class GithubReleaseParams(BaseModel): org: str repo: str tag: str - description: Optional[str] - title: Optional[str] + description: str | None + title: str | None def validate_github_release(github_url: str) -> GithubReleaseParams: diff --git a/designsafe/apps/api/projects_v2/operations/project_publish_operations.py b/designsafe/apps/api/projects_v2/operations/project_publish_operations.py index 042fead2ee..2d8465ac59 100644 --- a/designsafe/apps/api/projects_v2/operations/project_publish_operations.py +++ b/designsafe/apps/api/projects_v2/operations/project_publish_operations.py @@ -1,26 +1,33 @@ """Utils for generating published metadata""" -from typing import Optional, Literal -import subprocess -import os -import shutil import copy import datetime -from pathlib import Path import logging +import os +import shutil +import subprocess +from pathlib import Path +from typing import Literal + +import networkx as nx import requests +from celery import shared_task from django.conf import settings from django.db import close_old_connections -import networkx as nx -from celery import shared_task -from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.ai_keywords.utils import add_publications_to_chroma +from designsafe.apps.api.projects_v2 import constants from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata from designsafe.apps.api.projects_v2.operations.datacite_operations import ( get_datacite_json, + get_doi_publication_date, publish_datacite_doi, upsert_datacite_json, - get_doi_publication_date, +) +from designsafe.apps.api.projects_v2.operations.path_operations import ( + construct_entity_filepaths, + generate_sha512_manifest, + update_path_mappings, ) from designsafe.apps.api.projects_v2.operations.project_archive_operations import ( create_metadata_file, @@ -29,23 +36,17 @@ from designsafe.apps.api.projects_v2.operations.project_email_operations import ( send_project_permissions_alert, ) -from designsafe.apps.api.projects_v2.operations.path_operations import ( - construct_entity_filepaths, - update_path_mappings, - generate_sha512_manifest, -) from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( validate_github_release, ) -from designsafe.apps.api.publications_v2.models import Publication -from designsafe.apps.api.publications_v2.elasticsearch import index_publication -from designsafe.apps.data.tasks import agave_indexer -from designsafe.apps.api.publications_v2.tasks import ingest_pub_fedora_async -from designsafe.libs.common.context_managers import AsyncTaskContext -from designsafe.apps.api.ai_keywords.utils import add_publications_to_chroma from designsafe.apps.api.publications_v2.agents.neo4j_publication_ingest import ( neo4j_ingest_publication_async, ) +from designsafe.apps.api.publications_v2.elasticsearch import index_publication +from designsafe.apps.api.publications_v2.models import Publication +from designsafe.apps.api.publications_v2.tasks import ingest_pub_fedora_async +from designsafe.apps.data.tasks import agave_indexer +from designsafe.libs.common.context_managers import AsyncTaskContext logger = logging.getLogger(__name__) @@ -286,15 +287,14 @@ def _update_path_mappings(pub_graph: nx.DiGraph, project_uuid: str): def get_publication_subtree( project_id: str, entity_uuid: str, - version: Optional[int] = None, - version_info: Optional[int] = None, + version: int | None = None, + version_info: int | None = None, ) -> tuple[str, nx.DiGraph]: """ Obtain the subtree for a single publishable entity (experiment/simulation/etc) and add version information if relevant. The subtree includes the root node and any nodes associated with the UUID. """ - project_uuid = ProjectMetadata.get_project_by_id(project_id).uuid tree_with_values = add_values_to_tree(project_id) pub_root = next( ( @@ -366,8 +366,8 @@ def fix_publication_dates(incoming_tree: nx.DiGraph): def get_publication_full_tree( project_id: str, entity_uuids: list[str], - version: Optional[int] = None, - version_info: Optional[str] = None, + version: int | None = None, + version_info: str | None = None, ): """Combine subtrees to create the full publishable metadata object.""" full_path_mapping = {} @@ -412,11 +412,11 @@ class PipelineAmendFailure(Exception): def copy_publication_files( - path_mapping: dict, + path_mapping: dict[str, dict[str, str]], project_id: str, project_uuid: str, - dirs_to_lock: Optional[list[str]] = None, - version: Optional[int] = None, + dirs_to_lock: list[str] | None = None, + version: int | None = None, ): """ Copy files from My Projects to the published area on Corral. @@ -426,12 +426,10 @@ def copy_publication_files( logger.debug("Copying publication files for %s", project_id) os.chmod("/corral-repl/tacc/NHERI/published", 0o755) try: - # pub_root_dir = str(Path(f"{settings.DESIGNSAFE_PUBLISHED_PATH}") / pub_dirname) - for node_id in path_mapping: - for src_path in path_mapping[node_id]: - + for src_to_dest_map in path_mapping.values(): + for src_path in src_to_dest_map: # Convert root path from Tapis system root to Corral mount root src_path_obj = ( Path(settings.DESIGNSAFE_PROJECTS_PATH) @@ -442,7 +440,7 @@ def copy_publication_files( raise ProjectFileNotFound(f"File not found: {src_path}") dest_path_obj = Path(settings.DESIGNSAFE_PUBLISHED_PATH) / Path( - path_mapping[node_id][src_path].lstrip("/") + src_to_dest_map[src_path].lstrip("/") ) os.makedirs(dest_path_obj.parent, exist_ok=True) @@ -475,7 +473,7 @@ def copy_publication_files( logger.debug("Alerting due to data transfer failure for %s", project_id) logger.error(exc) send_project_permissions_alert(project_id, version, str(exc)) - raise exc + raise finally: os.chmod("/corral-repl/tacc/NHERI/published", 0o555) @@ -485,12 +483,10 @@ def copy_github_release(tree: nx.DiGraph, version: int = 1): """Download GitHub release archive to Corral.""" gh_node = next( - ( - node - for node in tree.nodes - if tree.nodes[node]["name"] == "designsafe.project" - and tree.nodes[node]["version"] == version - ) + node + for node in tree.nodes + if tree.nodes[node]["name"] == "designsafe.project" + and tree.nodes[node]["version"] == version ) base_path = tree.nodes[gh_node]["basePath"] corral_path = f"/corral-repl/tacc/NHERI/published{base_path}/data" @@ -503,15 +499,13 @@ def copy_github_release(tree: nx.DiGraph, version: int = 1): os.chmod("/corral-repl/tacc/NHERI/published", 0o755) try: - os.makedirs(corral_path, exist_ok=True) repo_zip_url = ( requests.get(github_release_url, timeout=10).json().get("zipball_url") ) repo_zip_contents = requests.get(repo_zip_url, stream=True, timeout=600) with open(full_corral_path, "wb") as file: - for chunk in repo_zip_contents.iter_content(chunk_size=10 * 1024): - file.write(chunk) + file.writelines(repo_zip_contents.iter_content(chunk_size=10 * 1024)) finally: os.chmod("/corral-repl/tacc/NHERI/published", 0o555) @@ -545,12 +539,11 @@ def create_publication_manifests( generate_sha512_manifest(path) -# pylint: disable=too-many-locals, too-many-branches, too-many-statements def publish_project( project_id: str, entity_uuids: list[str], - version: Optional[int] = 1, - version_info: Optional[str] = None, + version: int | None = 1, + version_info: str | None = None, dry_run: bool = False, ): """ @@ -578,12 +571,10 @@ def publish_project( project_uuid = ProjectMetadata.get_project_by_id(project_id).uuid base_meta_node = next( - ( - node - for node in pub_tree.nodes - if pub_tree.nodes[node]["name"] == constants.PROJECT - and pub_tree.nodes[node].get("version", version) == version - ) + node + for node in pub_tree.nodes + if pub_tree.nodes[node]["name"] == constants.PROJECT + and pub_tree.nodes[node].get("version", version) == version ) project_type = pub_tree.nodes[base_meta_node]["value"]["projectType"] @@ -658,8 +649,8 @@ def publish_project( def publish_project_async( project_id: str, entity_uuids: list[str], - version: Optional[int] = 1, - version_info: Optional[str] = None, + version: int | None = 1, + version_info: str | None = None, dry_run: bool = False, ): """Async wrapper around publication""" @@ -674,8 +665,10 @@ def publish_project_async( ) try: add_publications_to_chroma(publications=[meta]) - except Exception as e: # pylint: disable=broad-except - logger.error("Error adding publication to Chroma vector store: %s", e) + except Exception: + logger.exception( + "Error adding publication %s to Chroma vector store", project_id + ) except Exception as exc: raise PipelinePublishFailure( f"""Publication of {project_id} failed with {type(exc).__name__}: {exc} diff --git a/designsafe/apps/api/projects_v2/operations/project_system_operations.py b/designsafe/apps/api/projects_v2/operations/project_system_operations.py index 53cba5e965..d85d519c0f 100644 --- a/designsafe/apps/api/projects_v2/operations/project_system_operations.py +++ b/designsafe/apps/api/projects_v2/operations/project_system_operations.py @@ -3,13 +3,14 @@ # from portal.utils.encryption import createKeyPair import logging from typing import Literal + from celery import shared_task -from tapipy.tapis import Tapis from django.conf import settings -from designsafe.apps.api.agave import service_account -from designsafe.libs.common.context_managers import AsyncTaskContext from tapipy.errors import BaseTapyException +from tapipy.tapis import Tapis +from designsafe.apps.api.agave import service_account +from designsafe.libs.common.context_managers import AsyncTaskContext logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ def set_workspace_permissions( files_pems = {"reader": "READ", "writer": "MODIFY"} - logger.info(f"Adding {username} permissions to Tapis system {system_id}") + logger.info("Adding %s permissions to Tapis system %s", username, system_id) client.systems.grantUserPerms( systemId=system_id, userName=username, permissions=system_pems[role] ) @@ -100,7 +101,9 @@ def create_workspace_dir(project_uuid: str) -> str: return path -def create_workspace_system(client, project_uuid: str, project_id: str = None) -> str: +def create_workspace_system( + client, project_uuid: str, project_id: str | None = None +) -> str: system_id = f"project-{project_uuid}" system_args = { "id": system_id, @@ -145,7 +148,7 @@ def increment_workspace_count(force=None) -> int: def setup_project_file_system( - project_uuid: str, users: list[str], project_id: str = None + project_uuid: str, users: list[str], project_id: str | None = None ): """ Create a workspace system owned by user whose client is passed. diff --git a/designsafe/apps/api/projects_v2/schema_models/__init__.py b/designsafe/apps/api/projects_v2/schema_models/__init__.py index 556f7a4512..da9b8133b6 100644 --- a/designsafe/apps/api/projects_v2/schema_models/__init__.py +++ b/designsafe/apps/api/projects_v2/schema_models/__init__.py @@ -3,13 +3,12 @@ from designsafe.apps.api.projects_v2.schema_models import ( base, experimental, + field_recon, hybrid_sim, simulation, - field_recon, ) from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel - SCHEMA_MAPPING: dict[str, MetadataModel] = { constants.PROJECT: base.BaseProject, # Experimental diff --git a/designsafe/apps/api/projects_v2/schema_models/_field_models.py b/designsafe/apps/api/projects_v2/schema_models/_field_models.py index 22e2909545..8abb17db43 100644 --- a/designsafe/apps/api/projects_v2/schema_models/_field_models.py +++ b/designsafe/apps/api/projects_v2/schema_models/_field_models.py @@ -1,13 +1,17 @@ """Utiity models used in multiple field types""" +import logging from datetime import datetime from functools import partial -from typing import Annotated, Literal, Optional +from typing import Annotated, Literal + +from django.contrib.auth import get_user_model from pydantic import AliasChoices, BaseModel, BeforeValidator, ConfigDict, Field from pydantic.alias_generators import to_camel -from django.contrib.auth import get_user_model from pytas.http import TASClient +logger = logging.getLogger(__name__) + class MetadataModel(BaseModel): """Subclass BaseModel with custom config.""" @@ -34,19 +38,19 @@ def to_fedora_json(self) -> dict: class ProjectUser(MetadataModel): """Model for project users.""" - order: Optional[int] = None - guest: Optional[bool] = None - fname: Optional[str] = None - lname: Optional[str] = None - name: Optional[str] = None - email: Optional[str] = None - inst: Optional[str] = None - user: Optional[str] = None - username: Optional[str] = None - orcid_id: Optional[str] = None - role: Optional[Literal["pi", "co_pi", "team_member", "guest"]] = None + order: int | None = None + guest: bool | None = None + fname: str | None = None + lname: str | None = None + name: str | None = None + email: str | None = None + inst: str | None = None + user: str | None = None + username: str | None = None + orcid_id: str | None = None + role: Literal["pi", "co_pi", "team_member", "guest"] | None = None - authorship: Optional[bool] = None + authorship: bool | None = None @classmethod def from_username(cls, username: str, role: str = "team_member", **kwargs): @@ -66,7 +70,7 @@ def from_username(cls, username: str, role: str = "team_member", **kwargs): except user_model.DoesNotExist: try: tas_client = TASClient() - tas_user: Optional[dict] = tas_client.get_user(username=username) + tas_user: dict | None = tas_client.get_user(username=username) if not tas_user: return cls(username=username, role=role, guest=False) return cls( @@ -79,21 +83,21 @@ def from_username(cls, username: str, role: str = "team_member", **kwargs): **kwargs, ) # pylint:disable=broad-exception-caught - except Exception as _: - print(username) - print("unrecoverable username") + except Exception: + logger.exception("Unrecoverable username: %s", username) + return cls(username=username, role=role, guest=False) class GuestMember(MetadataModel): """Model for guest members.""" - order: Optional[int] = None + order: int | None = None guest: bool = True fname: str lname: str - inst: Optional[str] = None - email: Optional[str] = None + inst: str | None = None + email: str | None = None user: str @@ -105,7 +109,7 @@ class ProjectAward(MetadataModel): "" ) number: str = "" - funding_source: Optional[str] = None + funding_source: str | None = None class AssociatedProject(MetadataModel): @@ -114,11 +118,11 @@ class AssociatedProject(MetadataModel): # only title guaranteed type: str = "Linked Dataset" title: str - href: Optional[str] = "" + href: str | None = "" href_type: str = "URL" - order: Optional[int] = None + order: int | None = None # Some test projects have this weird attribute. - delete: Optional[bool] = None + delete: bool | None = None # Some legacy projects have a doi attribute. doi: str = "" @@ -146,9 +150,9 @@ def to_fedora_json(self): class FileTag(MetadataModel): """Model for file tags.""" - file_uuid: Optional[str] = Field(default=None, exclude=True) + file_uuid: str | None = Field(default=None, exclude=True) tag_name: str - path: Optional[str] = None + path: str | None = None class FileObj(MetadataModel): @@ -157,11 +161,11 @@ class FileObj(MetadataModel): system: str name: str path: str - legacy_path: Optional[str] = None + legacy_path: str | None = None type: Literal["file", "dir"] - length: Optional[int] = None - last_modified: Optional[str] = None - uuid: Optional[str] = None + length: int | None = None + last_modified: str | None = None + uuid: str | None = None class HazmapperMap(MetadataModel): @@ -171,7 +175,7 @@ class HazmapperMap(MetadataModel): uuid: str path: str deployment: str - href: Optional[str] = None + href: str | None = None class Ref(MetadataModel): @@ -193,7 +197,7 @@ class NaturalHazardEvent(MetadataModel): event_name: str event_start: datetime - event_end: Optional[datetime] = None + event_end: datetime | None = None location: str latitude: str longitude: str diff --git a/designsafe/apps/api/projects_v2/schema_models/_field_transforms.py b/designsafe/apps/api/projects_v2/schema_models/_field_transforms.py index ac234b4366..92583135a4 100644 --- a/designsafe/apps/api/projects_v2/schema_models/_field_transforms.py +++ b/designsafe/apps/api/projects_v2/schema_models/_field_transforms.py @@ -1,5 +1,5 @@ """Transforms for converting legacy metadata to fit our Pydantic schemas.""" -from typing import Optional, TypedDict +from typing import TypedDict def handle_award_number(award: list[dict] | list[str] | str) -> list[dict]: @@ -25,10 +25,7 @@ def handle_legacy_authors(author_list: list): if not bool(author_list): return [] if isinstance(author_list[0], str): - author_map = map( - lambda author: {"name": author, "guest": False, "authorship": True}, - author_list, - ) + author_map = ({"name": author, "guest": False, "authorship": True} for author in author_list) return list(author_map) return author_list @@ -52,10 +49,10 @@ class DropdownValueDict(TypedDict): def handle_dropdown_value( - dropdown_value: Optional[str | DropdownValueDict], + dropdown_value: str | DropdownValueDict | None, options: list[DropdownValueDict], - fallback: Optional[DropdownValueDict] = None, -) -> Optional[DropdownValueDict]: + fallback: DropdownValueDict | None = None, +) -> DropdownValueDict | None: """Look up value if a string id/value is passed.""" if not dropdown_value: return None diff --git a/designsafe/apps/api/projects_v2/schema_models/base.py b/designsafe/apps/api/projects_v2/schema_models/base.py index 3c25b28519..5b4ad93754 100644 --- a/designsafe/apps/api/projects_v2/schema_models/base.py +++ b/designsafe/apps/api/projects_v2/schema_models/base.py @@ -1,27 +1,29 @@ """Pydantic schema models for base-level project entities.""" from datetime import datetime -from typing import Literal, Optional, Annotated +from typing import Annotated, Literal + from pydantic import ( - BeforeValidator, AliasChoices, + BeforeValidator, ConfigDict, - model_validator, Field, + model_validator, ) + from designsafe.apps.api.projects_v2.constants import ( - NATURAL_HAZARD_TYPES, FIELD_RESEARCH_TYPES, + NATURAL_HAZARD_TYPES, OTHER_DATA_TYPES, ) -from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel from designsafe.apps.api.projects_v2.schema_models._field_models import ( AssociatedProject, DropdownValue, - FileTag, FileObj, + FileTag, GuestMember, HazmapperMap, + MetadataModel, NaturalHazardEvent, ProjectAward, ProjectUser, @@ -69,10 +71,10 @@ class BaseProject(MetadataModel): guest_members: Annotated[ list[GuestMember], BeforeValidator(handle_array_of_none) ] = [] - pi: Optional[str] = None + pi: str | None = None co_pis: list[str] = [] data_type: Annotated[ - Optional[DropdownValue], + DropdownValue | None, BeforeValidator( lambda v: handle_dropdown_value( v, OTHER_DATA_TYPES, fallback={"id": "other", "name": v} @@ -94,15 +96,15 @@ class BaseProject(MetadataModel): ] = [] associated_projects: list[AssociatedProject] = [] referenced_data: list[ReferencedWork] = [] - ef: Optional[str] = None + ef: str | None = None keywords: Annotated[list[str], BeforeValidator(handle_keywords)] = [] nh_event: str = "" nh_event_start: Annotated[ - Optional[datetime], BeforeValidator(lambda v: v or None) + datetime | None, BeforeValidator(lambda v: v or None) ] = None nh_event_end: Annotated[ - Optional[datetime], BeforeValidator(lambda v: v or None) + datetime | None, BeforeValidator(lambda v: v or None) ] = None nh_location: str = "" nh_latitude: str = "" @@ -119,7 +121,7 @@ class BaseProject(MetadataModel): BeforeValidator(lambda v: handle_dropdown_values(v, FIELD_RESEARCH_TYPES)), ] = [] dois: list[str] = [] - license: Optional[str] = None + license: str | None = None file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] @@ -128,15 +130,15 @@ class BaseProject(MetadataModel): facilities: list[DropdownValue] = [] # These fields are ONLY present on publication PRJ-1665 - natural_hazard_type: Optional[str] = None - natural_hazard_event: Optional[str] = None - coverage_temporal: Optional[str] = None - lat_long_name: Optional[str] = None + natural_hazard_type: str | None = None + natural_hazard_event: str | None = None + coverage_temporal: str | None = None + lat_long_name: str | None = None tombstone: bool = False - tombstone_message: Optional[str] = None + tombstone_message: str | None = None - github_url: Optional[str] = None + github_url: str | None = None def construct_users(self) -> list[ProjectUser]: """Fill in missing user information from the database.""" diff --git a/designsafe/apps/api/projects_v2/schema_models/experimental.py b/designsafe/apps/api/projects_v2/schema_models/experimental.py index 4e868053fb..2c295e4779 100644 --- a/designsafe/apps/api/projects_v2/schema_models/experimental.py +++ b/designsafe/apps/api/projects_v2/schema_models/experimental.py @@ -1,28 +1,30 @@ """Pydantic models for Experimental entities""" import itertools -from typing import Optional, Annotated -from pydantic import BeforeValidator, Field, ConfigDict, model_validator, AliasChoices -from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel +from typing import Annotated + +from pydantic import AliasChoices, BeforeValidator, ConfigDict, Field, model_validator + +from designsafe.apps.api.projects_v2.constants import ( + EQUIPMENT_TYPES, + EXPERIMENT_TYPES, + FACILITY_OPTIONS, +) from designsafe.apps.api.projects_v2.schema_models._field_models import ( AssociatedProject, DropdownValue, FileObj, FileTag, + MetadataModel, ProjectUser, Ref, ReferencedWork, ) from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( handle_array_of_none, - handle_legacy_authors, handle_dropdown_value, handle_keywords, -) -from designsafe.apps.api.projects_v2.constants import ( - FACILITY_OPTIONS, - EQUIPMENT_TYPES, - EXPERIMENT_TYPES, + handle_legacy_authors, ) equipment_type_options = list(itertools.chain(*EQUIPMENT_TYPES.values())) @@ -41,7 +43,7 @@ class Experiment(MetadataModel): keywords: Annotated[list[str], BeforeValidator(handle_keywords)] = [] facility: Annotated[ - Optional[DropdownValue], + DropdownValue | None, BeforeValidator(lambda v: handle_dropdown_value(v, FACILITY_OPTIONS)), Field( validation_alias=AliasChoices("facility", "experimentalFacility"), @@ -50,19 +52,19 @@ class Experiment(MetadataModel): experimental_facility_other: str = Field(default="", exclude=True) experiment_type: Annotated[ - Optional[DropdownValue], + DropdownValue | None, BeforeValidator(lambda v: handle_dropdown_value(v, experiment_type_options)), ] = None experiment_type_other: str = Field(default="", exclude=True) equipment_type: Annotated[ - Optional[DropdownValue], + DropdownValue | None, BeforeValidator(lambda v: handle_dropdown_value(v, equipment_type_options)), ] = None equipment_type_other: str = Field(default="", exclude=True) - procedure_start: Optional[str] = None - procedure_end: Optional[str] = None + procedure_start: str | None = None + procedure_end: str | None = None authors: Annotated[list[ProjectUser], BeforeValidator(handle_legacy_authors)] = [] project: list[str] = [] @@ -140,16 +142,16 @@ class ExperimentModelConfig(MetadataModel): file_objs: list[FileObj] = [] # Deprecated/legacy fields - drawing: Optional[list[str]] = Field( + drawing: list[str] | None = Field( default=None, exclude=True, alias="modelDrawing" ) - tags: Optional[dict] = Field(default=None, exclude=True) - image: Optional[dict] = Field(default=None, exclude=True) - lat: Optional[str] = Field(default=None, exclude=True) - lon: Optional[str] = Field(default=None, exclude=True) - video: Optional[dict] = Field(default=None, exclude=True) - spatial: Optional[str] = Field(default=None, exclude=True) - coverage: Optional[str] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) + image: dict | None = Field(default=None, exclude=True) + lat: str | None = Field(default=None, exclude=True) + lon: str | None = Field(default=None, exclude=True) + video: dict | None = Field(default=None, exclude=True) + spatial: str | None = Field(default=None, exclude=True) + coverage: str | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -175,18 +177,18 @@ class ExperimentSensor(MetadataModel): file_objs: list[FileObj] = [] # Deprecated legacy fields - sensor_list_type: Optional[str] = None - sensor_drawing: Optional[list[str]] = None - tags: Optional[dict] = Field(default=None, exclude=True) + sensor_list_type: str | None = None + sensor_drawing: list[str] | None = None + tags: dict | None = Field(default=None, exclude=True) - sensor_lists: Optional[list[str]] = Field(default=None, exclude=True) - event_type: Optional[str] = None + sensor_lists: list[str] | None = Field(default=None, exclude=True) + event_type: str | None = None # This field is ONLY present on pub PRJ-1649 - analysis: Optional[list[str]] = None + analysis: list[str] | None = None # This field ONLY Present on sensor 8078182091498319385-242ac11c-0001-012 - load: Optional[list[str]] = Field(default=None, exclude=True) + load: list[str] | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -215,8 +217,8 @@ class ExperimentEvent(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) - load: Optional[list[str]] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) + load: list[str] | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -242,9 +244,9 @@ class ExperimentAnalysis(MetadataModel): dois: list[str] = [] - tags: Optional[dict] = Field(default=None, exclude=True) - reference: Optional[str] = Field(default=None, exclude=True) - referencedoi: Optional[str] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) + reference: str | None = Field(default=None, exclude=True) + referencedoi: str | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" diff --git a/designsafe/apps/api/projects_v2/schema_models/field_recon.py b/designsafe/apps/api/projects_v2/schema_models/field_recon.py index ee541dc57d..ea799c361c 100644 --- a/designsafe/apps/api/projects_v2/schema_models/field_recon.py +++ b/designsafe/apps/api/projects_v2/schema_models/field_recon.py @@ -1,26 +1,28 @@ """Pydantic schema models for Field Recon entities""" -from typing import Annotated, Optional import itertools -from pydantic import BeforeValidator, Field, AliasChoices -from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel +from typing import Annotated + +from pydantic import AliasChoices, BeforeValidator, Field + +from designsafe.apps.api.projects_v2.constants import ( + FR_EQUIPMENT_TYPES, + FR_OBSERVATION_TYPES, +) from designsafe.apps.api.projects_v2.schema_models._field_models import ( AssociatedProject, DropdownValue, FileObj, FileTag, + MetadataModel, ProjectUser, ReferencedWork, ) from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( handle_array_of_none, - handle_legacy_authors, handle_dropdown_values, handle_keywords, -) -from designsafe.apps.api.projects_v2.constants import ( - FR_EQUIPMENT_TYPES, - FR_OBSERVATION_TYPES, + handle_legacy_authors, ) equipment_type_options = list(itertools.chain(*FR_EQUIPMENT_TYPES.values())) @@ -38,8 +40,8 @@ class Mission(MetadataModel): keywords: Annotated[list[str], BeforeValidator(handle_keywords)] = [] event: str = "" - date_start: Optional[str] = None - date_end: Optional[str] = None + date_start: str | None = None + date_end: str | None = None location: str = "" latitude: str = "" longitude: str = "" @@ -51,7 +53,7 @@ class Mission(MetadataModel): tombstone: bool = False # Deprecate these later - facility: Optional[DropdownValue] = None + facility: DropdownValue | None = None def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -114,7 +116,7 @@ class FieldReconReport(MetadataModel): dois: list[str] = [] # deprecated, only appears in test projects - facility: Optional[DropdownValue] = None + facility: DropdownValue | None = None missions: list[str] = Field(default=[], exclude=True) referenced_datas: list[ReferencedWork] = Field(default=[], exclude=True) @@ -166,8 +168,8 @@ class FieldReconCollection(MetadataModel): list[DropdownValue], BeforeValidator(lambda v: handle_dropdown_values(v, FR_OBSERVATION_TYPES)), ] = [] - date_start: Optional[str] = None - date_end: Optional[str] = None + date_start: str | None = None + date_end: str | None = None data_collectors: list[ProjectUser] = [] guest_data_collectors: list[str] = [] @@ -198,7 +200,7 @@ class SocialScienceCollection(MetadataModel): sample_approach: Annotated[list[str], BeforeValidator(handle_array_of_none)] = [] sample_size: str = "" date_start: str - date_end: Optional[str] = None + date_end: str | None = None data_collectors: list[ProjectUser] = [] location: str = "" latitude: str = "" @@ -300,7 +302,7 @@ class GeoscienceCollection(MetadataModel): BeforeValidator(lambda v: handle_dropdown_values(v, FR_OBSERVATION_TYPES)), ] = [] date_start: str - date_end: Optional[str] = None + date_end: str | None = None location: str = "" latitude: str = "" longitude: str = "" diff --git a/designsafe/apps/api/projects_v2/schema_models/hybrid_sim.py b/designsafe/apps/api/projects_v2/schema_models/hybrid_sim.py index 4372e75a9c..349a5d490e 100644 --- a/designsafe/apps/api/projects_v2/schema_models/hybrid_sim.py +++ b/designsafe/apps/api/projects_v2/schema_models/hybrid_sim.py @@ -1,24 +1,26 @@ """Pydantic schema models for Hybrid Simulation entities""" -from typing import Annotated, Optional +from typing import Annotated + from pydantic import BeforeValidator, Field, model_validator -from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel + +from designsafe.apps.api.projects_v2.constants import HYBRID_SIM_TYPES from designsafe.apps.api.projects_v2.schema_models._field_models import ( AssociatedProject, DropdownValue, FileObj, FileTag, + MetadataModel, ProjectUser, Ref, ReferencedWork, ) from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( handle_array_of_none, - handle_legacy_authors, handle_dropdown_value, handle_keywords, + handle_legacy_authors, ) -from designsafe.apps.api.projects_v2.constants import HYBRID_SIM_TYPES class HybridSimulation(MetadataModel): @@ -30,9 +32,9 @@ class HybridSimulation(MetadataModel): DropdownValue, BeforeValidator(lambda v: handle_dropdown_value(v, HYBRID_SIM_TYPES)), ] - simulation_type_other: Optional[str] = Field(exclude=True, default=None) - procedure_start: Optional[str] = None - procedure_end: Optional[str] = None + simulation_type_other: str | None = Field(exclude=True, default=None) + procedure_start: str | None = None + procedure_end: str | None = None referenced_data: list[ReferencedWork] = [] related_work: list[AssociatedProject] = [] keywords: Annotated[list[str], BeforeValidator(handle_keywords)] = [] @@ -40,7 +42,7 @@ class HybridSimulation(MetadataModel): project: list[str] = [] dois: list[str] = [] - facility: Optional[DropdownValue] = None + facility: DropdownValue | None = None tombstone: bool = False @@ -96,7 +98,7 @@ class HybridSimGlobalModel(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -121,7 +123,7 @@ class HybridSimCoordinator(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -147,7 +149,7 @@ class HybridSimSimSubstructure(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -172,7 +174,7 @@ class HybridSimExpSubstructure(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -197,7 +199,7 @@ class HybridSimCoordinatorOutput(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -223,7 +225,7 @@ class HybridSimSimOutput(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -249,7 +251,7 @@ class HybridSimExpOutput(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -273,9 +275,9 @@ class HybridSimAnalysis(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) - reference: Optional[str] = None - referencedoi: Optional[str] = None + tags: dict | None = Field(default=None, exclude=True) + reference: str | None = None + referencedoi: str | None = None def to_fedora_json(self): """Metadata representation for the Fedora repository""" @@ -298,7 +300,7 @@ class HybridSimReport(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - tags: Optional[dict] = Field(default=None, exclude=True) + tags: dict | None = Field(default=None, exclude=True) def to_fedora_json(self): """Metadata representation for the Fedora repository""" diff --git a/designsafe/apps/api/projects_v2/schema_models/simulation.py b/designsafe/apps/api/projects_v2/schema_models/simulation.py index 7f27de2747..2e61101c02 100644 --- a/designsafe/apps/api/projects_v2/schema_models/simulation.py +++ b/designsafe/apps/api/projects_v2/schema_models/simulation.py @@ -1,24 +1,26 @@ """Pydantic models for Simulation entities.""" -from typing import Optional, Annotated +from typing import Annotated + from pydantic import BeforeValidator, ConfigDict, Field, model_validator -from designsafe.apps.api.projects_v2.schema_models._field_models import MetadataModel + +from designsafe.apps.api.projects_v2.constants import SIMULATION_TYPES from designsafe.apps.api.projects_v2.schema_models._field_models import ( AssociatedProject, DropdownValue, FileObj, FileTag, + MetadataModel, ProjectUser, Ref, ReferencedWork, ) from designsafe.apps.api.projects_v2.schema_models._field_transforms import ( handle_array_of_none, - handle_legacy_authors, handle_dropdown_value, handle_keywords, + handle_legacy_authors, ) -from designsafe.apps.api.projects_v2.constants import SIMULATION_TYPES class Simulation(MetadataModel): @@ -30,7 +32,7 @@ class Simulation(MetadataModel): DropdownValue, BeforeValidator(lambda v: handle_dropdown_value(v, SIMULATION_TYPES)), ] - simulation_type_other: Optional[str] = Field(exclude=True, default=None) + simulation_type_other: str | None = Field(exclude=True, default=None) referenced_data: list[ReferencedWork] = [] related_work: list[AssociatedProject] = [] keywords: Annotated[list[str], BeforeValidator(handle_keywords)] = [] @@ -38,7 +40,7 @@ class Simulation(MetadataModel): project: list[str] = [] dois: list[str] = [] - facility: Optional[DropdownValue] = None + facility: DropdownValue | None = None tombstone: bool = False @model_validator(mode="after") @@ -172,8 +174,8 @@ class SimulationAnalysis(MetadataModel): file_tags: list[FileTag] = [] file_objs: list[FileObj] = [] - reference: Optional[str] = None - referencedoi: Optional[str] = None + reference: str | None = None + referencedoi: str | None = None def to_fedora_json(self): """Metadata representation for the Fedora repository""" diff --git a/designsafe/apps/api/projects_v2/tasks.py b/designsafe/apps/api/projects_v2/tasks.py index dcdebaf4e4..3041934255 100644 --- a/designsafe/apps/api/projects_v2/tasks.py +++ b/designsafe/apps/api/projects_v2/tasks.py @@ -5,20 +5,10 @@ from django.contrib.auth import get_user_model from django.core.mail import send_mail -# pylint: disable=unused-import -from designsafe.apps.api.projects_v2.operations.project_system_operations import ( - add_users_to_project_async, - remove_users_from_project_async, -) -from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata -from designsafe.apps.api.projects_v2.operations.project_publish_operations import ( - publish_project_async, - amend_publication_async, -) -from designsafe.apps.api.projects_v2.operations.project_archive_operations import ( - archive_publication_async, -) from designsafe.apps.api.projects_v2.elasticsearch import reindex_projects +from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata + +# pylint: disable=unused-import from designsafe.libs.common.context_managers import AsyncTaskContext diff --git a/designsafe/apps/api/projects_v2/tests/schema_integration.py b/designsafe/apps/api/projects_v2/tests/schema_integration.py index 5f65fc7828..b0cf601b1a 100644 --- a/designsafe/apps/api/projects_v2/tests/schema_integration.py +++ b/designsafe/apps/api/projects_v2/tests/schema_integration.py @@ -1,22 +1,24 @@ """Integration-type tests to confirm that Pydantic schemas are exhaustive.""" import json -from typing import Iterator +from collections.abc import Iterator + import networkx as nx from pydantic import BaseModel, ValidationError -from designsafe.apps.api.projects_v2.operations.datacite_operations import ( - get_datacite_json, -) -from designsafe.apps.api.projects_v2.schema_models.base import BaseProject + +from designsafe.apps.api.agave import get_service_account_client_v2 as service_account from designsafe.apps.api.projects_v2.migration_utils.graph_constructor import ( transform_pub_entities, ) -from designsafe.apps.api.agave import get_service_account_client_v2 as service_account -from designsafe.apps.api.publications.operations import listing as list_pubs from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata +from designsafe.apps.api.projects_v2.operations.datacite_operations import ( + get_datacite_json, +) from designsafe.apps.api.projects_v2.operations.project_publish_operations import ( get_publication_full_tree, ) +from designsafe.apps.api.projects_v2.schema_models.base import BaseProject +from designsafe.apps.api.publications.operations import listing as list_pubs def update_project(uuid, new_value): diff --git a/designsafe/apps/api/projects_v2/urls.py b/designsafe/apps/api/projects_v2/urls.py index 9a62d775a0..c79a4b8841 100644 --- a/designsafe/apps/api/projects_v2/urls.py +++ b/designsafe/apps/api/projects_v2/urls.py @@ -1,16 +1,17 @@ """Placeholder API routes""" from django.urls import path + from designsafe.apps.api.projects_v2.views import ( - ProjectsView, - ProjectInstanceView, - ProjectEntityView, + ProjectEntityAssociationsView, ProjectEntityOrderView, ProjectEntityValidateView, - ProjectEntityAssociationsView, + ProjectEntityView, ProjectFileAssociationsView, ProjectFileTagsView, + ProjectInstanceView, ProjectPreviewView, + ProjectsView, ) urlpatterns = [ diff --git a/designsafe/apps/api/projects_v2/views.py b/designsafe/apps/api/projects_v2/views.py index 26ea738637..3c3c91f4cb 100644 --- a/designsafe/apps/api/projects_v2/views.py +++ b/designsafe/apps/api/projects_v2/views.py @@ -1,57 +1,57 @@ """Placeholder views module""" -import logging import json +import logging + import networkx as nx -from django.http import HttpRequest, JsonResponse from django.conf import settings +from django.db import models +from django.http import HttpRequest, JsonResponse from django.utils.decorators import method_decorator from django.views.decorators.csrf import csrf_exempt -from django.db import models -from designsafe.libs.common.utils import check_group_membership -from designsafe.apps.api.views import BaseApiView, ApiException -from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata -from designsafe.apps.api.projects_v2.schema_models.base import BaseProject -from designsafe.apps.api.projects_v2.tasks import alert_sensitive_data + +from designsafe.apps.api.decorators import tapis_jwt_login +from designsafe.apps.api.projects_v2 import constants from designsafe.apps.api.projects_v2.migration_utils.graph_constructor import ( ALLOWED_RELATIONS, ) -from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.projects_v2.models.project_metadata import ProjectMetadata from designsafe.apps.api.projects_v2.operations.graph_operations import ( - reorder_project_nodes, add_node_to_project, - remove_nodes_from_project, initialize_project_graph, remove_nodes_for_entity, + remove_nodes_from_project, + reorder_project_nodes, ) from designsafe.apps.api.projects_v2.operations.project_meta_operations import ( - patch_metadata, + MissingGithubFile, add_file_associations, - set_file_associations, + change_project_type, create_entity_metadata, + create_project_metdata, delete_entity, + get_changed_users, + patch_metadata, remove_file_associations, + set_file_associations, set_file_tags, - change_project_type, - create_project_metdata, - get_changed_users, validate_github_release, - MissingGithubFile, ) from designsafe.apps.api.projects_v2.operations.project_publish_operations import ( add_values_to_tree, validate_entity_selection, ) from designsafe.apps.api.projects_v2.operations.project_system_operations import ( - increment_workspace_count, - setup_project_file_system, add_users_to_project_async, + increment_workspace_count, remove_users_from_project_async, + setup_project_file_system, ) -from designsafe.apps.api.projects_v2.schema_models.base import FileObj -from designsafe.apps.api.decorators import tapis_jwt_login +from designsafe.apps.api.projects_v2.schema_models.base import BaseProject, FileObj +from designsafe.apps.api.projects_v2.tasks import alert_sensitive_data from designsafe.apps.api.utils import get_client_ip - +from designsafe.apps.api.views import ApiException, BaseApiView +from designsafe.libs.common.utils import check_group_membership logger = logging.getLogger(__name__) metrics = logging.getLogger("metrics") @@ -142,7 +142,6 @@ def get(self, request: HttpRequest): def post(self, request: HttpRequest): """Create a new project.""" - user = request.user if not request.user.is_authenticated: raise ApiException("Unauthenticated user", status=401) req_body = json.loads(request.body) @@ -284,7 +283,7 @@ def patch(self, request: HttpRequest, project_id: str): }, ) - if "githubUrl" in request_body.keys(): + if "githubUrl" in request_body: try: request_body["githubUrl"] = request_body["githubUrl"].strip() github_params = validate_github_release(request_body["githubUrl"]) diff --git a/designsafe/apps/api/projects_v2/views_unit_test.py b/designsafe/apps/api/projects_v2/views_unit_test.py index a6c5723458..074762d204 100644 --- a/designsafe/apps/api/projects_v2/views_unit_test.py +++ b/designsafe/apps/api/projects_v2/views_unit_test.py @@ -1,6 +1,7 @@ -import pytest import json +import pytest + @pytest.mark.django_db def test_get_project_instance_unauthed(client, project_with_associations): diff --git a/designsafe/apps/api/publications/operations.py b/designsafe/apps/api/publications/operations.py index 85a75f9707..1786d006fc 100644 --- a/designsafe/apps/api/publications/operations.py +++ b/designsafe/apps/api/publications/operations.py @@ -1,16 +1,21 @@ -from designsafe.apps.data.models.elasticsearch import IndexedPublication, IndexedPublicationLegacy -from designsafe.apps.api.publications import search_utils -from designsafe.apps.api.agave import get_service_account_client -from designsafe.libs.elasticsearch.exceptions import DocumentNotFound -from designsafe.libs.elasticsearch.utils import new_es_client -from django.contrib.auth import get_user_model -from elasticsearch_dsl import Q -import os -import requests import datetime import json -import urllib import logging +import os +import urllib + +import requests +from django.contrib.auth import get_user_model +from elasticsearch_dsl import Q + +from designsafe.apps.api.agave import get_service_account_client +from designsafe.apps.api.publications import search_utils +from designsafe.apps.data.models.elasticsearch import ( + IndexedPublication, + IndexedPublicationLegacy, +) +from designsafe.libs.elasticsearch.exceptions import DocumentNotFound +from designsafe.libs.elasticsearch.utils import new_es_client logger = logging.getLogger(__name__) @@ -20,7 +25,7 @@ def _get_user_by_username(hit, username): if not users: try: user_obj = get_user_model().objects.get(username=username) - return "{}, {}".format(user_obj.last_name, user_obj.first_name) + return f"{user_obj.last_name}, {user_obj.first_name}" except: return username user = next(_user for _user in users if _user['username'] == username) @@ -164,7 +169,7 @@ def metrics(project_id, *args, **kwargs): """retrieve metrics for a given project ID""" client = get_service_account_client() - query = {'name': 'designsafe.metrics.{}'.format(project_id)} + query = {'name': f'designsafe.metrics.{project_id}'} metrics_meta = client.meta.listMetadata(q=json.dumps(query))[0] return metrics_meta diff --git a/designsafe/apps/api/publications/search_utils.py b/designsafe/apps/api/publications/search_utils.py index 89b14f3942..f1ad2b7d24 100644 --- a/designsafe/apps/api/publications/search_utils.py +++ b/designsafe/apps/api/publications/search_utils.py @@ -1,5 +1,7 @@ -from elasticsearch_dsl import Q import logging + +from elasticsearch_dsl import Q + logger = logging.getLogger(__name__) def experimental_facility_query(facility): diff --git a/designsafe/apps/api/publications/urls.py b/designsafe/apps/api/publications/urls.py index c7addaa968..b7c29fd3f0 100644 --- a/designsafe/apps/api/publications/urls.py +++ b/designsafe/apps/api/publications/urls.py @@ -1,8 +1,14 @@ from django.urls import re_path as url -from designsafe.apps.api.publications.views import PublicationListingView, PublicationDetailView, PublicationDataCiteView, PublicationDataCiteEventsView, PublicationClarivateView -from django.http import JsonResponse from django.views.decorators.cache import cache_page +from designsafe.apps.api.publications.views import ( + PublicationClarivateView, + PublicationDataCiteEventsView, + PublicationDataCiteView, + PublicationDetailView, + PublicationListingView, +) + urlpatterns = [ diff --git a/designsafe/apps/api/publications/views.py b/designsafe/apps/api/publications/views.py index 6f9d925cf8..953b7370d4 100644 --- a/designsafe/apps/api/publications/views.py +++ b/designsafe/apps/api/publications/views.py @@ -1,20 +1,17 @@ -from designsafe.apps.api.views import BaseApiView -from django.http import JsonResponse, HttpResponseForbidden -from designsafe.apps.api.agave import get_service_account_client -from django.conf import settings +import logging + +import requests +from django.http import JsonResponse from requests.exceptions import HTTPError + +from designsafe.apps.api.agave import get_service_account_client from designsafe.apps.api.publications import operations from designsafe.apps.api.publications.operations import ( + build_citation_json, clarivate_single_api, clarivate_wos_exp_single_api_basic, - build_citation_json, ) -from designsafe.apps.projects.managers import datacite as DataciteManager -from django.utils.decorators import method_decorator -import json -import logging -import requests - +from designsafe.apps.api.views import BaseApiView logger = logging.getLogger(__name__) diff --git a/designsafe/apps/api/publications_v2/agents/docs_rag_agent.py b/designsafe/apps/api/publications_v2/agents/docs_rag_agent.py index 3c636df56b..3c0c615138 100644 --- a/designsafe/apps/api/publications_v2/agents/docs_rag_agent.py +++ b/designsafe/apps/api/publications_v2/agents/docs_rag_agent.py @@ -4,9 +4,10 @@ import os from collections.abc import Callable + import chromadb -from chromadb.config import Settings as ChromaSettings import pydantic +from chromadb.config import Settings as ChromaSettings from pydantic_ai import Agent from pydantic_ai.models.openai import OpenAIResponsesModel from pydantic_ai.providers.openai import OpenAIProvider diff --git a/designsafe/apps/api/publications_v2/agents/elasticsearch_agent.py b/designsafe/apps/api/publications_v2/agents/elasticsearch_agent.py index 329eadec2b..d8d09bc5ac 100644 --- a/designsafe/apps/api/publications_v2/agents/elasticsearch_agent.py +++ b/designsafe/apps/api/publications_v2/agents/elasticsearch_agent.py @@ -1,7 +1,7 @@ """Agentic tools for performing Elasticsearch queries/aggregations.""" -from pydantic import BaseModel from elasticsearch_dsl import Q +from pydantic import BaseModel from designsafe.apps.api.publications_v2.elasticsearch import IndexedPublication diff --git a/designsafe/apps/api/publications_v2/agents/intent_agent.py b/designsafe/apps/api/publications_v2/agents/intent_agent.py index e337742b7d..92260035bd 100644 --- a/designsafe/apps/api/publications_v2/agents/intent_agent.py +++ b/designsafe/apps/api/publications_v2/agents/intent_agent.py @@ -1,17 +1,14 @@ """RAG agent for retrieving publications from the OpenAI document store.""" from typing import Literal -import openai +import openai +from django.conf import settings from pydantic import BaseModel from pydantic_ai import Agent from pydantic_ai.models.openai import OpenAIModel from pydantic_ai.providers.openai import OpenAIProvider - -from django.conf import settings - - oai_rag_client = openai.AsyncOpenAI( base_url=settings.OPENAI_API_URL, api_key=settings.OPENAI_API_KEY ) diff --git a/designsafe/apps/api/publications_v2/agents/neo4j_publication_ingest.py b/designsafe/apps/api/publications_v2/agents/neo4j_publication_ingest.py index 85748cd87c..6bda36f3ad 100644 --- a/designsafe/apps/api/publications_v2/agents/neo4j_publication_ingest.py +++ b/designsafe/apps/api/publications_v2/agents/neo4j_publication_ingest.py @@ -2,13 +2,14 @@ import os from urllib import parse + +import httpx import neo4j -from neo4j import Transaction -from django.conf import settings import networkx as nx -import httpx import openai from celery import shared_task +from django.conf import settings +from neo4j import Transaction from pydantic_ai.format_prompt import format_as_xml SUMMARIZE_PROMPT = """ @@ -143,7 +144,7 @@ def merge_files( project_id: str, collection_id: str, file_objs: list[dict], - file_tags: list[dict] = None, + file_tags: list[dict] | None = None, ): """Merge files and tags into the graph db.""" if not file_tags: diff --git a/designsafe/apps/api/publications_v2/agents/neo4j_rag_agent.py b/designsafe/apps/api/publications_v2/agents/neo4j_rag_agent.py index e974a0ca32..58bfe912dd 100644 --- a/designsafe/apps/api/publications_v2/agents/neo4j_rag_agent.py +++ b/designsafe/apps/api/publications_v2/agents/neo4j_rag_agent.py @@ -1,6 +1,7 @@ """RAG agent using Neo4J to construct queries.""" from pydantic_ai import Agent + from designsafe.apps.api.publications_v2.agents.docs_rag_agent import ( documentation_lookup as documentation_search, ) @@ -11,7 +12,6 @@ publication_graph_search, ) - AGENT_INSTRUCTIONS = """ You are an agent that routes user requests to exactly one first tool. Choose the single best tool using the priority order below, then answer using tool output. diff --git a/designsafe/apps/api/publications_v2/agents/openai_rag_agent.py b/designsafe/apps/api/publications_v2/agents/openai_rag_agent.py index 4d4ff72509..cacd3a187e 100644 --- a/designsafe/apps/api/publications_v2/agents/openai_rag_agent.py +++ b/designsafe/apps/api/publications_v2/agents/openai_rag_agent.py @@ -1,22 +1,20 @@ """RAG agent for retrieving publications from the OpenAI document store.""" -from typing import Literal from collections.abc import Callable -from asgiref.sync import sync_to_async +from typing import Literal + import networkx as nx import openai - +from asgiref.sync import sync_to_async +from django.conf import settings +from elasticsearch_dsl import Q from pydantic import BaseModel from pydantic_ai import Agent, RunContext from pydantic_ai.models.openai import OpenAIModel from pydantic_ai.providers.openai import OpenAIProvider -from elasticsearch_dsl import Q - -from django.conf import settings from designsafe.apps.api.publications_v2.elasticsearch import IndexedPublication - OAI_VECTOR_STORE_ID = "vs_68891f2f54948191bd1053afdf542987" oai_rag_client = openai.AsyncOpenAI( @@ -61,13 +59,13 @@ class AgentDeps(BaseModel): "ALWAYS include links verbatim. Do not truncate them or alter their formatting. ", # "ALWAYS explain in detail why each referenced publication was chosen. ", "DO NOT use the legacyPath attribute on files to construct URLs. ", - "If the question relates to attributes of a publication, DO NOT discuss publications that aren't relevant to those attributes." + ("If the question relates to attributes of a publication, DO NOT discuss publications that aren't relevant to those attributes." """Construct keywords according to the following guidelines: 1. Carefully read the entire query to understand its core topics and main ideas. 2. Extract up to 6 relevant keywords or key phrases that accurately reflect the key concepts discussed in the query. ONLY use words or phrases directly mentioned in the query. 3. If any keywords of key phrases are plural, add the singular form. If the keyword contains a number, include a version where the number is written out. For example if the keyword contains the character '1', write it out as 'one' and add it to the list. 4. Key phrases can contain a maximum of 2 words separated by spaces. - 5. Avoid common stop words like "the", "and", or "with". Also, avoid overly generic terms unless they are critical to the topic.""", + 5. Avoid common stop words like "the", "and", or "with". Also, avoid overly generic terms unless they are critical to the topic."""), "Provide the full result link at the end so that users can access it to explore the full set of DesignSafe publications.", ), ) @@ -150,7 +148,7 @@ async def vector_lookup( """Look up publications in the OpenAI vector store.""" await ctx.deps.response_callback( - f"Performing lookup with the following parameters: {str(search_params)}" + f"Performing lookup with the following parameters: {search_params!s}" ) vector_results: list[PublicationRagResult] = [] @@ -174,13 +172,13 @@ async def vector_lookup( "Publication lookup complete. Constructing your response..." ) - recovered_project_ids = set((res.project_id for res in vector_results)) + recovered_project_ids = {res.project_id for res in vector_results} _, es_search_results = await sync_to_async(get_es_results)( search_params, max_num_results=ctx.deps.result_size ) - es_project_ids = set((hit.meta.id for hit in es_search_results.hits)) + es_project_ids = {hit.meta.id for hit in es_search_results.hits} hits_in_both = recovered_project_ids.intersection(es_project_ids) es_only = es_project_ids - recovered_project_ids @@ -198,7 +196,7 @@ async def vector_lookup( n for n in project_dict["nodes"] if n["name"] == "designsafe.project" ) - null_keys = [k for k in base_meta["value"].keys() if not base_meta["value"][k]] + null_keys = [k for k in base_meta["value"] if not base_meta["value"][k]] for null_key in null_keys: del base_meta["value"][null_key] @@ -208,7 +206,7 @@ async def vector_lookup( for node in project_tree.successors("NODE_ROOT"): doi_entity = project_tree.nodes[node] null_keys = [ - k for k in doi_entity["value"].keys() if not doi_entity["value"][k] + k for k in doi_entity["value"] if not doi_entity["value"][k] ] for null_key in null_keys: del doi_entity["value"][null_key] diff --git a/designsafe/apps/api/publications_v2/agents/process_mkdocs.py b/designsafe/apps/api/publications_v2/agents/process_mkdocs.py index b10f064d6f..d65225262b 100644 --- a/designsafe/apps/api/publications_v2/agents/process_mkdocs.py +++ b/designsafe/apps/api/publications_v2/agents/process_mkdocs.py @@ -2,14 +2,14 @@ Utils for parsing mkdocs documentation for ingestion into a vector DB. """ -import re import logging import os +import re from pathlib import Path -from typing import List, Dict from urllib.parse import urljoin -from pydantic import BaseModel + import yaml +from pydantic import BaseModel logger = logging.getLogger(__name__) @@ -23,14 +23,14 @@ class Document(BaseModel): metadata: dict -def parse_mkdocs_config(config_path: str) -> Dict: +def parse_mkdocs_config(config_path: str) -> dict: """Parse MkDocs configuration to understand the site structure.""" with open(config_path, "r", encoding="utf-8") as f: config = yaml.safe_load(f) return config -def extract_nav_structure(nav: List, parent_path: str = "") -> List[Dict]: +def extract_nav_structure(nav: list, parent_path: str = "") -> list[dict]: """Extract navigation structure from MkDocs nav configuration.""" pages = [] @@ -81,7 +81,7 @@ def build_url(md_file: str) -> str: # pylint: disable=too-many-locals -def extract_sections_from_markdown(content: str, file_info: Dict) -> List[Document]: +def extract_sections_from_markdown(content: str, file_info: dict) -> list[Document]: """Extract sections from markdown content with proper metadata.""" documents = [] @@ -216,7 +216,7 @@ def process_markdown_files(docs_dir: str = "/docs"): file_path = docs_path / page_info["file"] if file_path.exists(): - logger.info(f"Processing: {page_info['file']}") + logger.info("Processing: %s", page_info["file"]) with open(file_path, "r", encoding="utf-8") as f: content = f.read() @@ -226,10 +226,12 @@ def process_markdown_files(docs_dir: str = "/docs"): all_documents.extend(documents) logger.info( - f" Extracted {len(documents)} sections from {page_info['file']}" + " Extracted %s sections from %s", + len(documents), + page_info["file"], ) else: - logger.warning(f"File not found: {file_path}") + logger.warning("File not found: %s", file_path) - logger.info(f"Total documents extracted: {len(all_documents)}") + logger.info("Total documents extracted: %s", len(all_documents)) return all_documents diff --git a/designsafe/apps/api/publications_v2/agents/process_url.py b/designsafe/apps/api/publications_v2/agents/process_url.py index 5c9c53fab7..10d02ea276 100644 --- a/designsafe/apps/api/publications_v2/agents/process_url.py +++ b/designsafe/apps/api/publications_v2/agents/process_url.py @@ -7,19 +7,15 @@ - SimCenter Tools (quoFEM, EE-UQ, Hydro, WE-UQ, PBE, R2D) """ -import time import logging -from typing import List, Set, Optional -from urllib.parse import urljoin, urlparse - +import time from collections import deque - - -from pydantic import BaseModel +from urllib.parse import urljoin, urlparse # Web scraping imports import requests from bs4 import BeautifulSoup +from pydantic import BaseModel # Configure logging logging.basicConfig( @@ -162,7 +158,7 @@ def __init__(self, delay: float = 0.3, max_pages: int = 0): {"User-Agent": "DesignSafe-RAG-Bot/1.0 (Educational/Research Purpose)"} ) - def fetch_page(self, url: str) -> Optional[BeautifulSoup]: + def fetch_page(self, url: str) -> BeautifulSoup | None: """Fetch and parse a single page, following redirects.""" try: response = self.session.get(url, timeout=30, allow_redirects=True) @@ -177,17 +173,17 @@ def fetch_page(self, url: str) -> Optional[BeautifulSoup]: if "url=" in content.lower(): redirect_url = content.split("url=")[-1].strip() full_redirect = urljoin(url, redirect_url) - logger.info(f" Following redirect to {full_redirect}") + logger.info(" Following redirect to %s", full_redirect) return self.fetch_page(full_redirect) return soup - except Exception as e: - logger.error(f"Error fetching {url}: {e}") + except Exception: + logger.exception("Error fetching %s", url) return None def get_links( self, soup: BeautifulSoup, base_url: str, current_url: str - ) -> Set[str]: + ) -> set[str]: """Extract all internal documentation links from a page.""" links = set() parsed_base = urlparse(base_url) @@ -196,11 +192,7 @@ def get_links( href = a["href"] # Skip anchors, external links, and special links - if ( - href.startswith("#") - or href.startswith("mailto:") - or href.startswith("javascript:") - ): + if href.startswith(("#", "mailto:", "javascript:")): continue # Resolve relative URLs @@ -248,7 +240,7 @@ def get_links( def extract_mkdocs_content( self, soup: BeautifulSoup, url: str, source_name: str - ) -> List[Document]: + ) -> list[Document]: """Extract content from MkDocs-style pages (DesignSafe uses ReadTheDocs theme).""" documents = [] @@ -351,7 +343,7 @@ def extract_mkdocs_content( def extract_sphinx_content( self, soup: BeautifulSoup, url: str, source_name: str - ) -> List[Document]: + ) -> list[Document]: """Extract content from Sphinx-style pages (SimCenter).""" documents = [] @@ -394,7 +386,7 @@ def extract_sphinx_content( if sections: for section in sections: - section_id = section.get("id", "") + section.get("id", "") header = section.find(["h1", "h2", "h3", "h4"]) section_title = header.get_text(strip=True) if header else page_title @@ -443,7 +435,7 @@ def extract_sphinx_content( def extract_jupyter_book_content( self, soup: BeautifulSoup, url: str, source_name: str - ) -> List[Document]: + ) -> list[Document]: """Extract content from Jupyter Book / Sphinx Book Theme pages.""" documents = [] @@ -481,7 +473,7 @@ def extract_jupyter_book_content( if sections: for section in sections: - section_id = section.get("id", "") + section.get("id", "") header = section.find(["h1", "h2", "h3", "h4"]) section_title = header.get_text(strip=True) if header else page_title @@ -575,14 +567,14 @@ def normalize_url(self, url: str) -> str: url = url[:-10] return url - def scrape_site(self, source_key: str) -> List[Document]: + def scrape_site(self, source_key: str) -> list[Document]: """Scrape an entire documentation site.""" source = DOC_SOURCES[source_key] base_url = source["base_url"] source_name = source["name"] doc_type = source["type"] - logger.info(f"Scraping {source_name} from {base_url}") + logger.info("Scraping %s from %s", source_name, base_url) all_documents = [] visited = set() @@ -599,7 +591,7 @@ def scrape_site(self, source_key: str) -> List[Document]: visited.add(normalized_url) limit_str = str(self.max_pages) if self.max_pages > 0 else "unlimited" - logger.info(f" [{len(visited)}/{limit_str}] {url}") + logger.info(" [%s/%s] %s", len(visited), limit_str, url) soup = self.fetch_page(url) if not soup: @@ -626,6 +618,8 @@ def scrape_site(self, source_key: str) -> List[Document]: time.sleep(self.delay) logger.info( - f" Scraped {len(visited)} pages, extracted {len(all_documents)} document chunks" + "Scraped %s pages, extracted %s document chunks", + len(visited), + len(all_documents), ) return all_documents diff --git a/designsafe/apps/api/publications_v2/agents/setup_chromadb.py b/designsafe/apps/api/publications_v2/agents/setup_chromadb.py index 94ecf26bda..e87431d119 100644 --- a/designsafe/apps/api/publications_v2/agents/setup_chromadb.py +++ b/designsafe/apps/api/publications_v2/agents/setup_chromadb.py @@ -2,21 +2,26 @@ Util for populating the ChromaDB database. """ +import logging import os import uuid + import chromadb -from chromadb.config import Settings as ChromaSettings from chromadb.api import ClientAPI +from chromadb.config import Settings as ChromaSettings from chromadb.utils import embedding_functions + from designsafe.apps.api.publications_v2.agents.process_mkdocs import ( - process_markdown_files, Document, + process_markdown_files, ) from designsafe.apps.api.publications_v2.agents.process_url import ( - WebScraper, DOC_SOURCES, + WebScraper, ) +logger = logging.getLogger(__name__) + OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") OPENAI_EMBEDDING_MODEL = os.environ.get( "OPENAI_EMBEDDING_MODEL", "text-embedding-ada-002" @@ -53,6 +58,7 @@ def ingest_documents(chroma_client: ClientAPI, docs: list[Document]): collection.add(documents=documents, metadatas=metadata, ids=ids) # pylint:disable=broad-exception-caught except Exception: + logger.exception("Error ingesting documents") print(f"EXCEPTION: {documents}") print(f"indexed embeddings for slice {docs_slice.start}-{docs_slice.stop}") diff --git a/designsafe/apps/api/publications_v2/agents/tools/schema_utils.py b/designsafe/apps/api/publications_v2/agents/tools/schema_utils.py index bcbfd81a35..fd8381eff0 100644 --- a/designsafe/apps/api/publications_v2/agents/tools/schema_utils.py +++ b/designsafe/apps/api/publications_v2/agents/tools/schema_utils.py @@ -3,6 +3,7 @@ """ from typing import Any + import neo4j NODE_PROPERTIES_QUERY = """ diff --git a/designsafe/apps/api/publications_v2/agents/tools/text2cypher_tool.py b/designsafe/apps/api/publications_v2/agents/tools/text2cypher_tool.py index 51a052714f..1694d86268 100644 --- a/designsafe/apps/api/publications_v2/agents/tools/text2cypher_tool.py +++ b/designsafe/apps/api/publications_v2/agents/tools/text2cypher_tool.py @@ -3,11 +3,12 @@ """ import logging -from neo4j import GraphDatabase, AsyncGraphDatabase, Query, AsyncDriver + +from django.conf import settings +from neo4j import AsyncDriver, AsyncGraphDatabase, GraphDatabase, Query from neo4j.exceptions import Neo4jError from openai import AsyncOpenAI from pydantic_ai import Agent, RunContext -from django.conf import settings from designsafe.apps.api.publications_v2.agents.tools.schema_utils import get_schema @@ -39,6 +40,7 @@ async def raise_if_write_attempted(client: AsyncDriver, query: str) -> None: :param query: Query string to sanitize :type query: str """ + # ruff: disable[SIM117] async with client.session() as sess: async with await sess.begin_transaction(timeout=30) as tx: result = await tx.run(query) diff --git a/designsafe/apps/api/publications_v2/elasticsearch.py b/designsafe/apps/api/publications_v2/elasticsearch.py index fd18e2f3d2..cdb9c65c02 100644 --- a/designsafe/apps/api/publications_v2/elasticsearch.py +++ b/designsafe/apps/api/publications_v2/elasticsearch.py @@ -1,8 +1,9 @@ """Elasticsearch model for published works""" -from elasticsearch_dsl import Document -from elasticsearch.exceptions import NotFoundError from django.conf import settings +from elasticsearch.exceptions import NotFoundError +from elasticsearch_dsl import Document + from designsafe.apps.api.publications_v2.models import Publication diff --git a/designsafe/apps/api/publications_v2/models.py b/designsafe/apps/api/publications_v2/models.py index 8c42413e8f..cb54cb7a94 100644 --- a/designsafe/apps/api/publications_v2/models.py +++ b/designsafe/apps/api/publications_v2/models.py @@ -1,8 +1,8 @@ """Model for published works.""" -from django.utils import timezone -from django.db import models from django.core.serializers.json import DjangoJSONEncoder +from django.db import models +from django.utils import timezone class Publication(models.Model): diff --git a/designsafe/apps/api/publications_v2/operations/fedora_graph_operations.py b/designsafe/apps/api/publications_v2/operations/fedora_graph_operations.py index ea83058f75..c6706e9845 100644 --- a/designsafe/apps/api/publications_v2/operations/fedora_graph_operations.py +++ b/designsafe/apps/api/publications_v2/operations/fedora_graph_operations.py @@ -2,23 +2,25 @@ import os import urllib + import networkx as nx from fido.fido import Fido -from designsafe.libs.common.context_managers import AsyncTaskContext -from designsafe.apps.api.publications_v2.models import Publication -from designsafe.apps.api.projects_v2.schema_models import SCHEMA_MAPPING, PATH_SLUGS -from designsafe.apps.api.projects_v2.schema_models.base import BaseProject + from designsafe.apps.api.projects_v2 import constants +from designsafe.apps.api.projects_v2.schema_models import PATH_SLUGS, SCHEMA_MAPPING +from designsafe.apps.api.projects_v2.schema_models.base import BaseProject +from designsafe.apps.api.publications_v2.models import Publication +from designsafe.libs.common.context_managers import AsyncTaskContext from designsafe.libs.fedora.fedora_operations import ( + PUBLICATIONS_MOUNT_ROOT, create_fc_version, - upload_manifest, fedora_post, fedora_update, - get_sha1_hash, - get_fido_output, - get_child_paths, generate_manifest, - PUBLICATIONS_MOUNT_ROOT, + get_child_paths, + get_fido_output, + get_sha1_hash, + upload_manifest, ) prov_predecessor_mapping = { @@ -321,11 +323,11 @@ def generate_manifest_other(project_id, version=1): pub_tree = nx.node_link_graph(pub.tree) version_node = next( - ( + node for node in pub_tree.successors("NODE_ROOT") if pub_tree.nodes[node]["version"] == version - ) + ) version_meta = pub_tree.nodes[version_node] @@ -334,7 +336,7 @@ def generate_manifest_other(project_id, version=1): file_tags = version_meta["value"].get("fileTags", []) if version and version > 1: - project_id = f"{project_id}v{str(version)}" + project_id = f"{project_id}v{version!s}" manifest = [] archive_path = os.path.join(PUBLICATIONS_MOUNT_ROOT, base_path.lstrip("/")) for path in get_child_paths(archive_path): diff --git a/designsafe/apps/api/publications_v2/tasks.py b/designsafe/apps/api/publications_v2/tasks.py index a24fbff187..37f26c416a 100644 --- a/designsafe/apps/api/publications_v2/tasks.py +++ b/designsafe/apps/api/publications_v2/tasks.py @@ -1,9 +1,10 @@ """Async tasks related to published work""" from celery import shared_task -from designsafe.libs.common.context_managers import AsyncTaskContext + from designsafe.apps.api.publications_v2.operations.fedora_graph_operations import ( ingest_pub_fedora, ) +from designsafe.libs.common.context_managers import AsyncTaskContext @shared_task() diff --git a/designsafe/apps/api/publications_v2/urls.py b/designsafe/apps/api/publications_v2/urls.py index c13b361e6e..70c64da491 100644 --- a/designsafe/apps/api/publications_v2/urls.py +++ b/designsafe/apps/api/publications_v2/urls.py @@ -1,11 +1,12 @@ """Publication API routes""" from django.urls import path, re_path + from designsafe.apps.api.publications_v2.views import ( - PublicationListingView, + PublicationAmendView, PublicationDetailView, + PublicationListingView, PublicationPublishView, - PublicationAmendView, PublicationVersionView, ) diff --git a/designsafe/apps/api/publications_v2/views.py b/designsafe/apps/api/publications_v2/views.py index a0c8f573ca..7f5921ea07 100644 --- a/designsafe/apps/api/publications_v2/views.py +++ b/designsafe/apps/api/publications_v2/views.py @@ -1,18 +1,20 @@ """Views for published data""" -import logging import json +import logging + import networkx as nx from django.http import HttpRequest, JsonResponse -from designsafe.apps.api.views import BaseApiView, ApiException -from designsafe.apps.api.publications_v2.models import Publication -from designsafe.apps.api.publications_v2.elasticsearch import IndexedPublication + from designsafe.apps.api.projects_v2.operations.project_publish_operations import ( - publish_project_async, amend_publication_async, + publish_project_async, ) from designsafe.apps.api.projects_v2.views import get_project_for_user +from designsafe.apps.api.publications_v2.elasticsearch import IndexedPublication +from designsafe.apps.api.publications_v2.models import Publication from designsafe.apps.api.utils import get_client_ip +from designsafe.apps.api.views import ApiException, BaseApiView logger = logging.getLogger(__name__) metrics = logging.getLogger("metrics") @@ -251,6 +253,7 @@ def get(self, request: HttpRequest, project_id, version=None): for (_, node) in pub_tree.nodes.data() ]: for tag in file_tag_arr: + # ruff: disable[PERF402] file_tags.append(tag) tree_json = nx.tree_data(pub_tree, "NODE_ROOT") diff --git a/designsafe/apps/api/publications_v2/websocket_consumers.py b/designsafe/apps/api/publications_v2/websocket_consumers.py index 32fb5630b3..380d733a33 100644 --- a/designsafe/apps/api/publications_v2/websocket_consumers.py +++ b/designsafe/apps/api/publications_v2/websocket_consumers.py @@ -2,7 +2,9 @@ import json import logging + from channels.generic.websocket import AsyncWebsocketConsumer + from designsafe.apps.api.publications_v2.agents.neo4j_rag_agent import ( combined_agent as HybridSearchAgent, ) @@ -43,7 +45,7 @@ async def receive(self, text_data=None, bytes_data=None): "ip": ip_addr, "operation": "chat.feedback", "sessionId": getattr(session, "session_key", ""), - "user": getattr(user, "username"), + "user": user.username, "info": {"value": payload, "history": json_data.get("history")}, }, ) @@ -77,13 +79,13 @@ async def chat_message(self, event): "ip": ip_addr, "operation": "chat.response", "sessionId": getattr(session, "session_key", ""), - "user": getattr(user, "username"), + "user": user.username, "info": {"query": event["payload"], "response": res}, }, ) # pylint:disable=broad-exception-caught - except Exception as exc: - logger.debug(exc) + except Exception: + logger.exception("") await self.send( json.dumps( { diff --git a/designsafe/apps/api/search/searchmanager/base.py b/designsafe/apps/api/search/searchmanager/base.py index 3a07b13bbe..881fb81373 100644 --- a/designsafe/apps/api/search/searchmanager/base.py +++ b/designsafe/apps/api/search/searchmanager/base.py @@ -1,6 +1,7 @@ -from elasticsearch import TransportError, ConnectionTimeout +from elasticsearch import ConnectionTimeout, TransportError -class BaseSearchManager(object): + +class BaseSearchManager: """ Wraps elastic search result object This class wraps the elasticsearch result object @@ -72,8 +73,7 @@ def all(self): def results(self, offset): res = self._search.execute() limit = offset + self._page_size - if res.hits.total.value < limit: - limit = res.hits.total.value + limit = min(limit, res.hits.total.value) if offset > limit: offset = 0 @@ -101,4 +101,4 @@ def __getattr__(self, name): return val else: raise AttributeError( - 'Search class has no attribute \'{}\''.format(name)) + f'Search class has no attribute \'{name}\'') diff --git a/designsafe/apps/api/search/searchmanager/cms.py b/designsafe/apps/api/search/searchmanager/cms.py index 410c85745d..9ce9bc6399 100644 --- a/designsafe/apps/api/search/searchmanager/cms.py +++ b/designsafe/apps/api/search/searchmanager/cms.py @@ -4,9 +4,9 @@ """ -import logging -from elasticsearch_dsl import Q, Index from django.conf import settings +from elasticsearch_dsl import Index, Q + from designsafe.apps.api.search.searchmanager.base import BaseSearchManager @@ -27,10 +27,10 @@ def __init__(self, request=None, **kwargs): 'date': 'date' } - super(CMSSearchManager, self).__init__(cms_index, cms_index.search()) + super().__init__(cms_index, cms_index.search()) def construct_query(self, system=None, file_path=None): - cms_index_name = list(Index(settings.ES_INDEX_PREFIX.format('cms')).get_alias().keys())[0] + cms_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('cms')).get_alias().keys())) cms_query = Q( 'bool', must=[ diff --git a/designsafe/apps/api/search/searchmanager/community.py b/designsafe/apps/api/search/searchmanager/community.py index 02cce3d9aa..400607b657 100644 --- a/designsafe/apps/api/search/searchmanager/community.py +++ b/designsafe/apps/api/search/searchmanager/community.py @@ -5,10 +5,12 @@ import logging + +from django.conf import settings +from elasticsearch_dsl import Index, Q, Search + from designsafe.apps.api.search.searchmanager.base import BaseSearchManager from designsafe.apps.data.models.elasticsearch import IndexedFile -from elasticsearch_dsl import Q, Search, Index -from django.conf import settings logger = logging.getLogger(__name__) @@ -29,12 +31,12 @@ def __init__(self, request=None, **kwargs): split_query[i] = "*" + c + "*" self.query_string = " ".join(split_query) - super(CommunityDataSearchManager, self).__init__( + super().__init__( IndexedFile, IndexedFile.search()) def construct_query(self, system=None, file_path=None): - files_index_name = list(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())[0] + files_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())) ngram_query = Q("query_string", query=self.query_string, fields=["name"], diff --git a/designsafe/apps/api/search/searchmanager/lookups.py b/designsafe/apps/api/search/searchmanager/lookups.py index 7ff2a6953e..eaf41bc451 100644 --- a/designsafe/apps/api/search/searchmanager/lookups.py +++ b/designsafe/apps/api/search/searchmanager/lookups.py @@ -2,9 +2,11 @@ :synopsis: Function to lookup manager classes """ -from importlib import import_module import logging +from importlib import import_module + from django.conf import settings + from designsafe.apps.api.exceptions import ApiException #pylint: disable=invalid-name diff --git a/designsafe/apps/api/search/searchmanager/private_data.py b/designsafe/apps/api/search/searchmanager/private_data.py index 9adafc5ea3..4a796578e2 100644 --- a/designsafe/apps/api/search/searchmanager/private_data.py +++ b/designsafe/apps/api/search/searchmanager/private_data.py @@ -4,10 +4,12 @@ """ import logging + +from django.conf import settings +from elasticsearch_dsl import Index, Q, Search + from designsafe.apps.api.search.searchmanager.base import BaseSearchManager from designsafe.apps.data.models.elasticsearch import IndexedFile -from elasticsearch_dsl import Q, Search, Index -from django.conf import settings logger = logging.getLogger(__name__) @@ -24,12 +26,12 @@ def __init__(self, request=None, **kwargs): self.query_string = kwargs.get('query_string').replace("/", "\\/") self.username = kwargs.get('username') - super(PrivateDataSearchManager, self).__init__( + super().__init__( IndexedFile, IndexedFile.search()) def construct_query(self, system, file_path=None): - files_index_name = list(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())[0] + files_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())) if system == settings.AGAVE_STORAGE_SYSTEM: storage_prefix_query = Q({'prefix': {'path._exact': '/' + self.username}}) diff --git a/designsafe/apps/api/search/searchmanager/publications.py b/designsafe/apps/api/search/searchmanager/publications.py index b46751d4f6..4c9483eda4 100644 --- a/designsafe/apps/api/search/searchmanager/publications.py +++ b/designsafe/apps/api/search/searchmanager/publications.py @@ -4,16 +4,17 @@ """ +import json import logging -from designsafe.apps.api.search.searchmanager.base import BaseSearchManager -from designsafe.apps.data.models.elasticsearch import IndexedPublication -from elasticsearch_dsl import Q, Search, Index -from django.conf import settings import urllib -import json from functools import reduce + +from django.conf import settings +from elasticsearch_dsl import Index, Q, Search + +from designsafe.apps.api.search.searchmanager.base import BaseSearchManager +from designsafe.apps.data.models.elasticsearch import IndexedPublication from designsafe.libs.elasticsearch.docs.publications import BaseESPublication -from designsafe.libs.elasticsearch.docs.publication_legacy import BaseESPublicationLegacy logger = logging.getLogger(__name__) @@ -34,7 +35,7 @@ def __init__(self, request=None, **kwargs): else: self.query_string = kwargs.get('query_string').replace("/", "\\/") - super(PublicationsSearchManager, self).__init__( + super().__init__( IndexedPublication, Search()) def experimental_facility_query(self, facility_name): @@ -137,7 +138,7 @@ def simulation_query(self): simulation_type = self.query_dict['advancedFilters']['simulation']['simulationType'] facility_name = self.query_dict['advancedFilters']['simulation']['facility'] if facility_name: - expt_query = expt_query & self.facility_query(facility_name) + expt_query & self.facility_query(facility_name) # noqa if not self.query_dict['typeFilters']['simulation'] and not simulation_type: return None sim_query = Q('term', **{'project.value.projectType._exact': 'simulation'}) @@ -151,7 +152,7 @@ def field_recon_query(self): nh_event = self.query_dict['advancedFilters']['field_recon']['naturalHazardEvent'] facility_name = self.query_dict['advancedFilters']['field_recon']['facility'] if facility_name: - expt_query = expt_query & self.facility_query(facility_name) + expt_query & self.facility_query(facility_name) # noqa if not self.query_dict['typeFilters']['field_recon'] and not (nh_type or nh_event): return None fr_query = Q('term', **{'project.value.projectType._exact': 'field_recon'}) @@ -175,10 +176,10 @@ def other_query(self): return q def hybrid_sim_query(self): - sim_type = data_type = self.query_dict['advancedFilters']['hybrid_simulation']['hybridSimulationType'] + sim_type = self.query_dict['advancedFilters']['hybrid_simulation']['hybridSimulationType'] facility_name = self.query_dict['advancedFilters']['hybrid_simulation']['facility'] if facility_name: - expt_query = expt_query & self.facility_query(facility_name) + expt_query & self.facility_query(facility_name) # noqa if not self.query_dict['typeFilters']['hybrid_simulation'] and not sim_type: return None q = Q('term', **{'project.value.projectType._exact': 'hybrid_simulation'}) @@ -270,11 +271,11 @@ def construct_query(self, system=None, file_path=None, **kwargs): "authors.lname", "name" ] - published_index_name = list(Index(settings.ES_INDEX_PREFIX.format('publications')).get_alias().keys())[0] - legacy_index_name = list(Index(settings.ES_INDEX_PREFIX.format('publications-legacy')).get_alias().keys())[0] + published_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('publications')).get_alias().keys())) + next(iter(Index(settings.ES_INDEX_PREFIX.format('publications-legacy')).get_alias().keys())) - ds_user_query = Q({"nested": + Q({"nested": {"path": "users", "ignore_unmapped": True, "query": {"query_string": @@ -284,7 +285,7 @@ def construct_query(self, system=None, file_path=None, **kwargs): "user.username"], "lenient": True}}} }) - nees_pi_query = Q({"nested": + Q({"nested": {"path": "pis", "ignore_unmapped": True, "query": {"query_string": @@ -293,7 +294,7 @@ def construct_query(self, system=None, file_path=None, **kwargs): "pis.lastName"], "lenient": True}}} }) - pub_query = Q('query_string', query=self.query_string, default_operator='and', fields=project_query_fields) + Q('query_string', query=self.query_string, default_operator='and', fields=project_query_fields) published_query = Q( 'bool', must=[ diff --git a/designsafe/apps/api/search/searchmanager/publications_legacy.py b/designsafe/apps/api/search/searchmanager/publications_legacy.py index 61ca9a1b1d..0972410ff5 100644 --- a/designsafe/apps/api/search/searchmanager/publications_legacy.py +++ b/designsafe/apps/api/search/searchmanager/publications_legacy.py @@ -5,12 +5,16 @@ import logging + +from django.conf import settings +from elasticsearch_dsl import Index, Q, Search + from designsafe.apps.api.search.searchmanager.base import BaseSearchManager from designsafe.apps.data.models.elasticsearch import IndexedPublication -from elasticsearch_dsl import Q, Search, Index -from django.conf import settings +from designsafe.libs.elasticsearch.docs.publication_legacy import ( + BaseESPublicationLegacy, +) from designsafe.libs.elasticsearch.docs.publications import BaseESPublication -from designsafe.libs.elasticsearch.docs.publication_legacy import BaseESPublicationLegacy logger = logging.getLogger(__name__) @@ -25,7 +29,7 @@ def __init__(self, request=None, **kwargs): else: self.query_string = kwargs.get('query_string').replace("/", "\\/") - super(LegacyPublicationsSearchManager, self).__init__( + super().__init__( IndexedPublication, Search()) def construct_query(self, system=None, file_path=None, **kwargs): @@ -38,8 +42,8 @@ def construct_query(self, system=None, file_path=None, **kwargs): "pis", "name" ] - published_index_name = list(Index(settings.ES_INDEX_PREFIX.format('publications')).get_alias().keys())[0] - legacy_index_name = list(Index(settings.ES_INDEX_PREFIX.format('publications-legacy')).get_alias().keys())[0] + next(iter(Index(settings.ES_INDEX_PREFIX.format('publications')).get_alias().keys())) + legacy_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('publications-legacy')).get_alias().keys())) filter_queries = [] if kwargs.get('type_filters'): for type_filter in kwargs['type_filters']: @@ -95,7 +99,7 @@ def listing(self, system=None, file_path=None, offset=0, limit=100, **kwargs): children = [] for hit in res: try: - getattr(hit, 'projectId') + hit.projectId # noqa hit_to_file = BaseESPublication.hit_to_file(hit) children.append(hit_to_file) except AttributeError: diff --git a/designsafe/apps/api/search/searchmanager/publications_site_search.py b/designsafe/apps/api/search/searchmanager/publications_site_search.py index 5364633ef3..5b3bd1cf6c 100644 --- a/designsafe/apps/api/search/searchmanager/publications_site_search.py +++ b/designsafe/apps/api/search/searchmanager/publications_site_search.py @@ -5,12 +5,16 @@ import logging + +from django.conf import settings +from elasticsearch_dsl import Index, Q, Search + from designsafe.apps.api.search.searchmanager.base import BaseSearchManager from designsafe.apps.data.models.elasticsearch import IndexedPublication -from elasticsearch_dsl import Q, Search, Index -from django.conf import settings +from designsafe.libs.elasticsearch.docs.publication_legacy import ( + BaseESPublicationLegacy, +) from designsafe.libs.elasticsearch.docs.publications import BaseESPublication -from designsafe.libs.elasticsearch.docs.publication_legacy import BaseESPublicationLegacy logger = logging.getLogger(__name__) @@ -25,7 +29,7 @@ def __init__(self, request=None, **kwargs): else: self.query_string = kwargs.get('query_string').replace("/", "\\/") - super(PublicationsSiteSearchManager, self).__init__( + super().__init__( IndexedPublication, Search()) def construct_query(self, system=None, file_path=None, **kwargs): @@ -49,8 +53,8 @@ def construct_query(self, system=None, file_path=None, **kwargs): "authors.lname", "name" ] - published_index_name = list(Index(settings.ES_INDEX_PREFIX.format('publications')).get_alias().keys())[0] - legacy_index_name = list(Index(settings.ES_INDEX_PREFIX.format('publications-legacy')).get_alias().keys())[0] + published_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('publications')).get_alias().keys())) + legacy_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('publications-legacy')).get_alias().keys())) filter_queries = [] if kwargs.get('type_filters'): for type_filter in kwargs['type_filters']: @@ -102,7 +106,7 @@ def listing(self, system=None, file_path=None, offset=0, limit=100, **kwargs): children = [] for hit in res: try: - getattr(hit, 'projectId') + hit.projectId # noqa hit_to_file = BaseESPublication.hit_to_file(hit) children.append(hit_to_file) except AttributeError: diff --git a/designsafe/apps/api/search/searchmanager/published_files.py b/designsafe/apps/api/search/searchmanager/published_files.py index 8cf98b81d3..241274401b 100644 --- a/designsafe/apps/api/search/searchmanager/published_files.py +++ b/designsafe/apps/api/search/searchmanager/published_files.py @@ -5,10 +5,12 @@ import logging + +from django.conf import settings +from elasticsearch_dsl import Index, Q, Search + from designsafe.apps.api.search.searchmanager.base import BaseSearchManager from designsafe.apps.data.models.elasticsearch import IndexedFile -from elasticsearch_dsl import Q, Search, Index -from django.conf import settings logger = logging.getLogger(__name__) @@ -23,12 +25,12 @@ def __init__(self, request=None, **kwargs): else: self.query_string = kwargs.get('query_string').replace("/", "\\/") - super(PublishedDataSearchManager, self).__init__( + super().__init__( IndexedFile, IndexedFile.search()) def construct_query(self, system=None, file_path=None): - files_index_name = list(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())[0] + files_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())) ngram_query = Q("query_string", query=self.query_string, fields=["name"], minimum_should_match='80%', diff --git a/designsafe/apps/api/search/searchmanager/shared_data.py b/designsafe/apps/api/search/searchmanager/shared_data.py index fcb3bd5cb0..712b1e3b10 100644 --- a/designsafe/apps/api/search/searchmanager/shared_data.py +++ b/designsafe/apps/api/search/searchmanager/shared_data.py @@ -5,10 +5,12 @@ import logging + +from django.conf import settings +from elasticsearch_dsl import Index, Q + from designsafe.apps.api.search.searchmanager.base import BaseSearchManager from designsafe.apps.data.models.elasticsearch import IndexedFile -from elasticsearch_dsl import Q, search, Index -from django.conf import settings logger = logging.getLogger(__name__) @@ -25,12 +27,12 @@ def __init__(self, request=None, **kwargs): self.query_string = kwargs.get('query_string').replace("/", "\\/") self.username = kwargs.get('username') - super(SharedDataSearchManager, self).__init__( + super().__init__( IndexedFile, IndexedFile.search()) def construct_query(self, system, file_path=None): - files_index_name = list(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())[0] + files_index_name = next(iter(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())) if system == settings.AGAVE_STORAGE_SYSTEM: storage_prefix_query = Q({'prefix': {'path._exact': '/' + self.username}}) @@ -71,7 +73,7 @@ def listing(self, system, file_path, user_context=None, offset=None, limit=None) search = search.query(Q('bool', must_not=[Q({'prefix': {'path._exact': '/'+user_context}})])) search = search.filter("term", system=system) - search = search.query(Q('bool', must_not=[Q({'prefix': {'path._exact': '{}/.Trash'.format(user_context)}})])) + search = search.query(Q('bool', must_not=[Q({'prefix': {'path._exact': f'{user_context}/.Trash'}})])) res = search.execute() children = [] diff --git a/designsafe/apps/api/search/searchmanager/tests.py b/designsafe/apps/api/search/searchmanager/tests.py index 81970034bc..01b1f3fb15 100644 --- a/designsafe/apps/api/search/searchmanager/tests.py +++ b/designsafe/apps/api/search/searchmanager/tests.py @@ -1,26 +1,29 @@ -import os -import json -import datetime from unittest import skip -from mock import patch, MagicMock +from unittest.mock import MagicMock, patch + from django.test import TestCase -from django.contrib.auth import get_user_model -from django.conf import settings -from datetime import timedelta -from elasticsearch_dsl import Search +from elasticsearch_dsl import Search +from designsafe.apps.api.search.searchmanager.community import ( + CommunityDataSearchManager, +) from designsafe.apps.api.search.searchmanager.lookups import SearchLookupManager - -from designsafe.apps.api.search.searchmanager.private_data import PrivateDataSearchManager -from designsafe.apps.api.search.searchmanager.community import CommunityDataSearchManager -from designsafe.apps.api.search.searchmanager.published_files import PublishedDataSearchManager +from designsafe.apps.api.search.searchmanager.private_data import ( + PrivateDataSearchManager, +) +from designsafe.apps.api.search.searchmanager.publications import ( + PublicationsSearchManager, +) +from designsafe.apps.api.search.searchmanager.published_files import ( + PublishedDataSearchManager, +) from designsafe.apps.api.search.searchmanager.shared_data import SharedDataSearchManager -from designsafe.apps.api.search.searchmanager.publications import PublicationsSearchManager - -from designsafe.apps.data.models.elasticsearch import IndexedPublication, IndexedPublicationLegacy -from designsafe.apps.data.models.elasticsearch import IndexedFile +from designsafe.apps.data.models.elasticsearch import ( + IndexedFile, + IndexedPublication, + IndexedPublicationLegacy, +) -from designsafe.apps.api.exceptions import ApiException @skip("Need to mock Elasticsearch calls") class TestLookupManager(TestCase): @@ -176,7 +179,7 @@ def test_listing(self, mock_search, mock_pub, mock_leg_pub): mock_pub().to_file.return_value = {'type': 'pub'} mock_leg_pub().to_file.return_value = {'type': 'leg_pub'} - res = fm.listing(**{'type_filters': []}) + res = fm.listing(type_filters=[]) expected_result = { 'trail': [{'name': '$SEARCH', 'path': '/$SEARCH'}], 'name': '$SEARCH', diff --git a/designsafe/apps/api/search/tests.py b/designsafe/apps/api/search/tests.py index 44062c47a3..74cb82d439 100644 --- a/designsafe/apps/api/search/tests.py +++ b/designsafe/apps/api/search/tests.py @@ -1,10 +1,8 @@ -from mock import Mock, patch, MagicMock, PropertyMock +from unittest.mock import patch + from django.test import TestCase -from django.contrib.auth import get_user_model -from django.conf import settings from django.urls import reverse -from designsafe.apps.api.search.views import SearchView -import json + class SearchViewTests(TestCase): diff --git a/designsafe/apps/api/search/urls.py b/designsafe/apps/api/search/urls.py index 3b646d53a2..f794d42e90 100644 --- a/designsafe/apps/api/search/urls.py +++ b/designsafe/apps/api/search/urls.py @@ -1,4 +1,5 @@ from django.urls import re_path as url + from designsafe.apps.api.search.views import SearchView """ diff --git a/designsafe/apps/api/search/views.py b/designsafe/apps/api/search/views.py index 9e8909e948..00afaf879a 100644 --- a/designsafe/apps/api/search/views.py +++ b/designsafe/apps/api/search/views.py @@ -5,18 +5,23 @@ access. """ import logging -import operator -from elasticsearch_dsl import Q, Search, Index -from elasticsearch import TransportError, ConnectionTimeout -from django.http import (HttpResponseBadRequest, - JsonResponse) + from django.conf import settings -from designsafe.apps.api.views import BaseApiView +from django.http import HttpResponseBadRequest, JsonResponse +from elasticsearch import ConnectionTimeout, TransportError +from elasticsearch_dsl import Index, Search -from designsafe.apps.api.search.searchmanager.community import CommunityDataSearchManager -from designsafe.apps.api.search.searchmanager.published_files import PublishedDataSearchManager from designsafe.apps.api.search.searchmanager.cms import CMSSearchManager -from designsafe.apps.api.search.searchmanager.publications_site_search import PublicationsSiteSearchManager +from designsafe.apps.api.search.searchmanager.community import ( + CommunityDataSearchManager, +) +from designsafe.apps.api.search.searchmanager.publications_site_search import ( + PublicationsSiteSearchManager, +) +from designsafe.apps.api.search.searchmanager.published_files import ( + PublishedDataSearchManager, +) +from designsafe.apps.api.views import BaseApiView logger = logging.getLogger(__name__) @@ -25,17 +30,17 @@ class SearchView(BaseApiView): """Main view to handle sitewise search requests""" def get(self, request): """GET handler.""" - q = request.GET.get('query_string') + request.GET.get('query_string') offset = int(request.GET.get('offset', 0)) limit = int(request.GET.get('limit', 10)) if limit > 500: return HttpResponseBadRequest("limit must not exceed 500") type_filter = request.GET.get('type_filter', 'all') doc_type_map = { - list(Index(settings.ES_INDEX_PREFIX.format('publications')).get_alias().keys())[0]: 'publication', - list(Index(settings.ES_INDEX_PREFIX.format('publications-legacy')).get_alias().keys())[0]: 'publication', - list(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())[0]: 'file', - list(Index(settings.ES_INDEX_PREFIX.format('cms')).get_alias().keys())[0]: 'modelresult' + next(iter(Index(settings.ES_INDEX_PREFIX.format('publications')).get_alias().keys())): 'publication', + next(iter(Index(settings.ES_INDEX_PREFIX.format('publications-legacy')).get_alias().keys())): 'publication', + next(iter(Index(settings.ES_INDEX_PREFIX.format('files')).get_alias().keys())): 'file', + next(iter(Index(settings.ES_INDEX_PREFIX.format('cms')).get_alias().keys())): 'modelresult' } public_files_query = CommunityDataSearchManager(request).construct_query() | PublishedDataSearchManager(request).construct_query() @@ -83,8 +88,8 @@ def get(self, request): if r.meta.doc_type == 'publication' and hasattr(r, 'users'): users = r.users pi = r.project.value.pi - pi_user = [x for x in users if x.username==pi][0] - d["piLabel"] = "{}, {}".format(pi_user.last_name, pi_user.first_name) + pi_user = next(x for x in users if x.username==pi) + d["piLabel"] = f"{pi_user.last_name}, {pi_user.first_name}" hits.append(d) out['hits'] = hits diff --git a/designsafe/apps/api/systems/ssh_keys_manager.py b/designsafe/apps/api/systems/ssh_keys_manager.py index a47483eeb3..08d672ff0f 100644 --- a/designsafe/apps/api/systems/ssh_keys_manager.py +++ b/designsafe/apps/api/systems/ssh_keys_manager.py @@ -4,8 +4,8 @@ """ import logging -import paramiko +import paramiko logger = logging.getLogger(__name__) @@ -93,14 +93,14 @@ def _get_add_pub_key_command(self, system_id, public_key): :return str: command """ comment = self._get_pub_key_comment(system_id) - string = " ".join([public_key, comment]) + string = f"{public_key} {comment}" command = ( 'if [ ! -f "~/.ssh/authorized_keys" ]; then ' "mkdir -p ~/.ssh/ && touch ~/.ssh/authorized_keys " "&& chmod 0600 ~/.ssh/authorized_keys; fi && " - 'grep -q -F "{string}" ~/.ssh/authorized_keys || ' - 'echo "{string}" >> ~/.ssh/authorized_keys' - ).format(string=string) + f'grep -q -F "{string}" ~/.ssh/authorized_keys || ' + f'echo "{string}" >> ~/.ssh/authorized_keys' + ) return command def add_public_key( @@ -135,7 +135,7 @@ def add_public_key( if status == -1: logger.info("No response from the server") elif status == 0: - logger.info(f"Public key added successfully to {hostname}") + logger.info("Public key added successfully to %s", hostname) elif status > 0: error_lines = "" for line in stderr.readlines(): diff --git a/designsafe/apps/api/systems/urls.py b/designsafe/apps/api/systems/urls.py index cb1a20ab17..db8f2aaa2b 100644 --- a/designsafe/apps/api/systems/urls.py +++ b/designsafe/apps/api/systems/urls.py @@ -1,6 +1,7 @@ """Publication API routes""" from django.urls import path + from .views import SystemKeysView urlpatterns = [ diff --git a/designsafe/apps/api/systems/utils.py b/designsafe/apps/api/systems/utils.py index e6f1fea32d..d001ec9587 100644 --- a/designsafe/apps/api/systems/utils.py +++ b/designsafe/apps/api/systems/utils.py @@ -4,13 +4,14 @@ """ import logging + from paramiko.ssh_exception import ( AuthenticationException, ChannelException, SSHException, ) -from .ssh_keys_manager import KeysManager, KeyCannotBeAdded +from .ssh_keys_manager import KeyCannotBeAdded, KeysManager logger = logging.getLogger(__name__) @@ -41,7 +42,7 @@ def add_pub_key_to_resource( mgr = KeysManager(user, password, token) message = "add_pub_key_to_resource" - logger.info(f"Adding public key for user {user.username} on system {system_id}") + logger.info("Adding public key for user %s on system %s", user.username, system_id) try: if hostname is None: sys = user.tapis_oauth.client.systems.getSystem(systemId=system_id) diff --git a/designsafe/apps/api/systems/views.py b/designsafe/apps/api/systems/views.py index 3c2f027d3a..c9642faa48 100644 --- a/designsafe/apps/api/systems/views.py +++ b/designsafe/apps/api/systems/views.py @@ -3,12 +3,17 @@ :synopsis: Systems views """ -import logging import json +import logging + from django.http import JsonResponse + from designsafe.apps.api.views import AuthenticatedApiView -from designsafe.apps.onboarding.steps.system_access_v3 import create_system_credentials_with_keys +from designsafe.apps.onboarding.steps.system_access_v3 import ( + create_system_credentials_with_keys, +) from designsafe.utils.encryption import createKeyPair + from .utils import add_pub_key_to_resource logger = logging.getLogger(__name__) @@ -30,7 +35,7 @@ def post(self, request): system_id = body["systemId"] logger.info( - f"Resetting credentials for user {request.user.username} on system {system_id}" + "Resetting credentials for user %s on system %s", request.user.username, system_id ) (priv_key_str, publ_key_str) = createKeyPair() diff --git a/designsafe/apps/api/tasks.py b/designsafe/apps/api/tasks.py index 0727ceeee0..aac948b770 100644 --- a/designsafe/apps/api/tasks.py +++ b/designsafe/apps/api/tasks.py @@ -1,28 +1,22 @@ -import shutil +import json import logging -import re import os -import sys -import json -import urllib.request, urllib.parse, urllib.error -from datetime import datetime +import shutil + from celery import shared_task -from django.urls import reverse -from django.contrib.auth import get_user_model -from pytas.models import User as TASUser from django.conf import settings -from requests.exceptions import HTTPError +from django.contrib.auth import get_user_model +from django.core.mail import send_mail +from django.urls import reverse +from elasticsearch.helpers import bulk -from designsafe.apps.api.notifications.models import Notification, Broadcast +from designsafe.apps.accounts.models import DesignSafeProfile from designsafe.apps.api.agave import get_service_account_client +from designsafe.apps.api.notifications.models import Notification +from designsafe.apps.data.tasks import agave_indexer from designsafe.apps.projects.models.elasticsearch import IndexedProject -from designsafe.apps.data.models.elasticsearch import IndexedPublication from designsafe.libs.elasticsearch.docs.publications import BaseESPublication from designsafe.libs.elasticsearch.utils import new_es_client -from designsafe.apps.data.tasks import agave_indexer -from elasticsearch_dsl.query import Q -from elasticsearch.helpers import bulk -from django.core.mail import send_mail logger = logging.getLogger(__name__) @@ -44,12 +38,12 @@ def box_download(self, username, src_resource, src_file_id, dest_resource, dest_ src_resource, src_file_id, username, dest_resource, dest_file_id) try: - target_path = reverse('designsafe_data:data_depot', + reverse('designsafe_data:data_depot', args=[src_resource, src_file_id]) n = Notification(event_type='data', status=Notification.INFO, operation='box_download_start', - message='Starting download of file %s from box.' % (src_file_id,), + message=f'Starting download of file {src_file_id} from box.', user=username, # extra={'target_path': target_path}) extra={'id': dest_file_id} @@ -58,7 +52,7 @@ def box_download(self, username, src_resource, src_file_id, dest_resource, dest_ user = get_user_model().objects.get(username=username) - from designsafe.apps.api.data import BoxFileManager, AgaveFileManager + from designsafe.apps.api.data import AgaveFileManager, BoxFileManager agave_fm = AgaveFileManager(user) dest_real_path = agave_fm.get_file_real_path(dest_file_id) @@ -80,24 +74,24 @@ def box_download(self, username, src_resource, src_file_id, dest_resource, dest_ pems_indexing=True, index_full_path=True, levels=levels) - target_path = reverse('designsafe_data:data_depot', + reverse('designsafe_data:data_depot', args=[dest_resource, dest_file_id]) n = Notification(event_type='data', status=Notification.SUCCESS, operation='box_download_end', - message='File %s was copied from box successfully!' % (src_file_id, ), + message=f'File {src_file_id} was copied from box successfully!', user=username, extra={'id': dest_file_id} ) n.save() - except: + except Exception: logger.exception('Unexpected task failure: box_download', extra={ 'username': username, 'box_file_id': src_file_id, 'to_resource': dest_resource, 'dest_file_id': dest_file_id }) - target_path = reverse('designsafe_data:data_depot', + reverse('designsafe_data:data_depot', args=[src_resource, src_file_id]) n = Notification(event_type='data', status=Notification.ERROR, @@ -184,14 +178,13 @@ def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_fi :param dest_file_id: the box id of the destination folder :return: """ - logger.debug('Importing file %s://%s for user %s to %s://%s' % ( - src_resource, src_file_id, username, dest_resource, dest_file_id)) + logger.debug('Importing file %s://%s for user %s to %s://%s', src_resource, src_file_id, username, dest_resource, dest_file_id) try: n = Notification(event_type = 'data', status = Notification.INFO, operation = 'box_upload_start', - message = 'Starting import of file %s into box.' % src_file_id, + message = f'Starting import of file {src_file_id} into box.', user = username, # extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)}) extra={'id': src_file_id}) @@ -203,7 +196,8 @@ def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_fi file_type, file_id = box_fm.parse_file_id(dest_file_id) if file_type != 'folder': - logger.warn('Cannot import to a file destination!') + logger.warning('Cannot import to a file destination!') + # ruff: disable [TRY002] raise Exception('You can only import files to a folder!', status=400, extra={ 'src_resource': src_resource, @@ -225,7 +219,7 @@ def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_fi else: logger.error('Unable to upload %s: file does not exist!', file_real_path) - except: + except Exception: logger.exception('Upload to Box failed!') logger.debug('Box upload task complete.') @@ -233,12 +227,12 @@ def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_fi n = Notification(event_type = 'data', status = Notification.SUCCESS, operation = 'box_upload_end', - message = 'File(s) %s succesfully uploaded into box!' % src_file_id, + message = f'File(s) {src_file_id} succesfully uploaded into box!', user = username, extra={'id': src_file_id}) # extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), dest_resource, dest_file_id)}) n.save() - except: + except Exception: logger.exception('Unexpected task failure: box_upload', extra={ 'username': username, 'src_resource': src_resource, @@ -262,7 +256,7 @@ def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_fi def box_upload_file(box_file_manager, box_folder_id, file_real_path): - file_path, file_name = os.path.split(file_real_path) + _file_path, file_name = os.path.split(file_real_path) with open(file_real_path, 'rb') as file_handle: box_folder = box_file_manager.box_api.folder(box_folder_id) uploaded_file = box_folder.upload_stream(file_handle, file_name) @@ -281,7 +275,7 @@ def box_upload_directory(box_file_manager, box_parent_folder_id, dir_real_path): :return: The new box folder. """ - dirparentpath, dirname = os.path.split(dir_real_path) + _dirparentpath, dirname = os.path.split(dir_real_path) box_parent_folder = box_file_manager.box_api.folder(box_parent_folder_id) logger.info('Create directory %s in box folder/%s', dirname, box_parent_folder_id) box_folder = box_parent_folder.create_subfolder(dirname) @@ -313,7 +307,7 @@ def copy_public_to_mydata(self, username, src_resource, src_file_id, dest_resour n = Notification(event_type = 'data', status = 'INFO', operation = 'copy_public_to_mydata_start', - message = 'Copying folder/files %s from public data to your private data. Please wait...' % (src_file_id, ), + message = f'Copying folder/files {src_file_id} from public data to your private data. Please wait...', user = username, extra={ 'system': dest_resource, @@ -372,7 +366,7 @@ def copy_public_to_mydata(self, username, src_resource, src_file_id, dest_resour }) # extra = {}) n.save() - except: + except Exception: logger.exception('Unexpected task failure') n = Notification(event_type = 'data', @@ -415,21 +409,21 @@ def set_project_id(self, project_uuid): id_metas = service.meta.listMetadata(q='{"name": "designsafe.project.id"}') logger.debug(json.dumps(id_metas, indent=4)) if not len(id_metas): - raise Exception('No project Id found') + raise Exception('No project Id found') # noqa id_meta = id_metas[0] project_id = int(id_meta['value']['id']) project_id = project_id + 1 for i in range(10): - _projs = service.meta.listMetadata(q='{{"name": "designsafe.project", "value.projectId": {} }}'.format(project_id)) + _projs = service.meta.listMetadata(q=f'{{"name": "designsafe.project", "value.projectId": {project_id} }}') if len(_projs): project_id = project_id + 1 - project.project_id = 'PRJ-{}'.format(str(project_id)) + project.project_id = f'PRJ-{project_id!s}' project.save(service) logger.debug('updated project id=%s', project.uuid) id_meta['value']['id'] = project_id - new_metadata = service.meta.updateMetadata(body=id_meta, uuid=id_meta['uuid']) + service.meta.updateMetadata(body=id_meta, uuid=id_meta['uuid']) logger.debug('updated id record=%s', id_meta['uuid']) index_or_update_project.apply_async(args=[project.uuid], queue='api') @@ -462,7 +456,6 @@ def index_projects_listing(projects): ------- Void """ - from designsafe.apps.projects.models.elasticsearch import IndexedProject idx = IndexedProject.Index.name client = IndexedProject._get_connection() ops = [] @@ -555,8 +548,8 @@ def reindex_projects(self): for listing in list_all_projects(): try: index_projects_listing(listing) - except Exception as e: - logger.exception(e) + except Exception: + logger.exception("") @shared_task(bind=True, max_retries=5) @@ -583,18 +576,18 @@ def copy_publication_files_to_corral(self, project_id, revision=None, selected_f filepaths = list(set(filepaths)) filepaths = sorted(filepaths) - base_path = ''.join(['/', publication.projectId]) + f'/{publication.projectId}' os.chmod('/corral-repl/tacc/NHERI/published', 0o755) - prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id) + prefix_dest = f'/corral-repl/tacc/NHERI/published/{project_id}' if revision: - prefix_dest += 'v{}'.format(revision) + prefix_dest += f'v{revision}' if not os.path.isdir(prefix_dest): os.mkdir(prefix_dest) prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format(publication.project['uuid']) for filepath in filepaths: - local_src_path = '{}/{}'.format(prefix_src, filepath) - local_dst_path = '{}/{}'.format(prefix_dest, filepath) + local_src_path = f'{prefix_src}/{filepath}' + local_dst_path = f'{prefix_dest}/{filepath}' logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path) if os.path.isdir(local_src_path): try: @@ -610,8 +603,7 @@ def copy_publication_files_to_corral(self, project_id, revision=None, selected_f os.chmod(local_dst_path, 0o555) except OSError as exc: logger.info(exc) - except IOError as exc: - logger.info(exc) + else: try: if not os.path.isdir(os.path.dirname(local_dst_path)): @@ -626,8 +618,6 @@ def copy_publication_files_to_corral(self, project_id, revision=None, selected_f os.chmod(local_dst_path, 0o444) except OSError as exc: logger.info(exc) - except IOError as exc: - logger.info(exc) os.chmod(prefix_dest, 0o555) os.chmod('/corral-repl/tacc/NHERI/published', 0o555) @@ -636,7 +626,7 @@ def copy_publication_files_to_corral(self, project_id, revision=None, selected_f index_path = '/' + project_id if revision: - index_path += 'v{}'.format(revision) + index_path += f'v{revision}' agave_indexer.apply_async(kwargs={'systemId': 'designsafe.storage.published', 'filePath': index_path, 'recurse':True}, queue='indexing') @@ -656,7 +646,7 @@ def freeze_publication_meta(self, project_id, entity_uuids=None, revision=None, revised_authors ) except Exception as exc: - logger.error('Proj Id: %s. %s', project_id, exc, exc_info=True) + logger.exception('Proj Id: %s.', project_id) raise self.retry(exc=exc) @@ -670,10 +660,13 @@ def amend_publication_data(self, project_id, amendments=None, authors=None, revi :param list of entity_uuid strings: Main entity uuid. """ from designsafe.apps.projects.managers import publication as PublicationManager - from designsafe.libs.fedora.fedora_operations import amend_project_fedora, ingest_project_experimental - from designsafe.libs.fedora.sim_operations import ingest_project_sim + from designsafe.libs.fedora.fedora_operations import ( + amend_project_fedora, + ingest_project_experimental, + ) from designsafe.libs.fedora.fr_operations import ingest_project_fr from designsafe.libs.fedora.hyb_sim_operations import ingest_project_hyb_sim + from designsafe.libs.fedora.sim_operations import ingest_project_sim try: amended_pub = PublicationManager.amend_publication(project_id, amendments, authors, revision) PublicationManager.amend_datacite_doi(amended_pub) @@ -689,7 +682,7 @@ def amend_publication_data(self, project_id, amendments=None, authors=None, revi if amended_pub.project.value.projectType == 'field_recon': ingest_project_fr(project_id, version=revision, amend=True) except Exception as exc: - logger.error('Proj Id: %s. %s', project_id, exc, exc_info=True) + logger.exception('Proj Id: %s.', project_id) raise self.retry(exc=exc) @@ -719,7 +712,7 @@ def save_publication(self, project_id, entity_uuids=None, revision=None, revised revised_authors=revised_authors ) except Exception as exc: - logger.error('Proj Id: %s. %s', project_id, exc, exc_info=True) + logger.exception('Proj Id: %s.', project_id) raise self.retry(exc=exc) @shared_task(bind=True) @@ -728,7 +721,7 @@ def zip_publication_files(self, project_id, revision=None): try: PublicationManager.archive(project_id=project_id, revision=revision) except Exception as exc: - logger.error('Zip Proj Id: %s. %s', project_id, exc, exc_info=True) + logger.exception('Zip Proj Id: %s.', project_id) raise self.retry(exc=exc) @shared_task(bind=True) @@ -744,7 +737,7 @@ def swap_file_tag_uuids(self, project_id, revision=None): try: PublicationManager.fix_file_tags(project_id, revision=revision) except Exception as exc: - logger.error('File Tag Correction Error: %s. %s', project_id, exc, exc_info=True) + logger.exception('File Tag Correction Error: %s', project_id) raise self.retry(exc=exc) @shared_task(bind=True) @@ -768,9 +761,7 @@ def fedora_ingest_other(self, project_id): @shared_task(bind=True, max_retries=5, default_retry_delay=60) def save_to_fedora(self, project_id, revision=None): - import requests - import magic - from designsafe.libs.elasticsearch.docs.publications import BaseESPublication + from designsafe.libs.elasticsearch.docs.publications import BaseESPublication try: es_client = new_es_client() pub = BaseESPublication(project_id=project_id, revision=revision, using=es_client) @@ -781,7 +772,9 @@ def save_to_fedora(self, project_id, revision=None): ingest_project(project_id, version=revision) return if pub.project.value.projectType == 'experimental': - from designsafe.libs.fedora.fedora_operations import ingest_project_experimental + from designsafe.libs.fedora.fedora_operations import ( + ingest_project_experimental, + ) ingest_project_experimental(project_id, version=revision) return if pub.project.value.projectType == 'simulation': @@ -798,6 +791,7 @@ def save_to_fedora(self, project_id, revision=None): return except Exception as exc: + logger.exception("") logger.error('Proj Id: %s. %s', project_id, exc) raise self.retry(exc=exc) @@ -808,41 +802,41 @@ def set_facl_project(self, project_uuid, usernames): job_body = { 'parameters': { 'username': username, - 'directory': 'projects/{}'.format(project_uuid) + 'directory': f'projects/{project_uuid}' }, 'name': 'setfacl', 'appId': 'setfacl_corral3-0.1' } res = client.jobs.submit(body=job_body) - logger.debug('set facl project: {}'.format(res)) + logger.debug('set facl project: %s', res) @shared_task(bind=True, max_retries=3, default_retry_delay=60) def email_project_admins(self, project_id, project_uuid, project_title, project_url, username): #contact project admins regarding publication of sensitive information - service = get_service_account_client() + get_service_account_client() admins = settings.PROJECT_ADMINS_EMAIL user = get_user_model().objects.get(username=username) for admin in admins: - email_body = """ + email_body = f"""

Hello,

The following Field Research project has been created with the intent of publishing sensitive information:
- {prjID} - {title} + {project_id} - {project_title}

Contact PI:
- {name} - {email} + {user.get_full_name()} - {user.email}

Link to Project:
- {url}. + {project_url}.

This is a programmatically generated message. Do NOT reply to this message. - """.format(name=user.get_full_name(), email=user.email, title=project_title, prjID=project_id, url=project_url) + """ send_mail( "DesignSafe PII Alert", @@ -853,31 +847,31 @@ def email_project_admins(self, project_id, project_uuid, project_title, project_ @shared_task(bind=True, max_retries=3, default_retry_delay=60) def email_collaborator_added_to_project(self, project_id, project_uuid, project_title, project_url, team_members_to_add, co_pis_to_add): - service = get_service_account_client() + get_service_account_client() for username in team_members_to_add + co_pis_to_add: collab_users = get_user_model().objects.filter(username=username) if collab_users: for collab_user in collab_users: - email_body = """ -

Hi {name},


-

You have been added to the project {title} (ID: {prjID}).


-

You can visit the project using the url {url}.

+ email_body = f""" +

Hi {collab_user.get_full_name()},


+

You have been added to the project {project_title} (ID: {project_id}).


+

You can visit the project using the url {project_url}.

You must log in to view this project.

You can now start working on the project. Please use your TACC account to access the DesignSafe-CI website or to ask for help.

Thanks,
The DesignSafe-CI Team.

This is a programmatically generated message. Do NOT reply to this message. - """.format(name=collab_user.get_full_name(), title=project_title, prjID=project_id, url=project_url) + """ try: collab_user.profile.send_mail("You have been added to a DesignSafe project!", email_body) - except DesignSafeProfile.DoesNotExist as err: - logger.info("Could not send email to user {}".format(collab_user)) + except DesignSafeProfile.DoesNotExist: + logger.info("Could not send email to user %s", collab_user) send_mail( "You have been added to a project!", email_body, settings.DEFAULT_FROM_EMAIL, [collab_user.email], - html_message=body) + html_message=body) # noqa @shared_task(bind=True, max_retries=3, default_retry_delay=60) def email_user_publication_request_confirmation(self, username): @@ -894,8 +888,8 @@ def email_user_publication_request_confirmation(self, username): """.format(pub_url="https://www.designsafe-ci.org/data/browser/public/", ticket_url="https://www.designsafe-ci.org/help/new-ticket/") try: user.profile.send_mail(email_subject, email_body) - except Exception as e: - logger.info("Could not send email to user {}".format(user)) + except Exception: # noqa + logger.info(f"Could not send email to user {user}") # noqa send_mail( email_subject, email_body, @@ -930,9 +924,9 @@ def check_published_files(self, project_id, revision=None, selected_files=None): file_path.strip('/') for file_path in filepaths if (file_path != '.Trash') ] - pub_directory = '/corral-repl/tacc/NHERI/published/{}'.format(project_id) + pub_directory = f'/corral-repl/tacc/NHERI/published/{project_id}' if revision: - pub_directory += 'v{}'.format(revision) + pub_directory += f'v{revision}' #navigate through publication files paths and #compare to the previous list of files @@ -956,34 +950,34 @@ def check_published_files(self, project_id, revision=None, selected_files=None): #send email if there are files/folders missing/empty if(missing_files or empty_folders): #log for potential later queries - logger.info("check_published_files missing files: " + project_id + " " + str(missing_files)) - logger.info("check_published_files empty folders: " + project_id + " " + str(empty_folders)) + logger.info("check_published_files missing files: %s %s", project_id, str(missing_files)) + logger.info("check_published_files empty folders: %s %s", project_id, str(empty_folders)) #send email to dev admins - service = get_service_account_client() + get_service_account_client() prj_admins = settings.DEV_PROJECT_ADMINS_EMAIL for admin in prj_admins: - email_body = """ + email_body = f"""

Hello,

The following project has been published with either missing files/folders or empty folders:
- {prjID} - revision {revision} + {project_id} - revision {revision}
- Path to publication files: {pubFiles} + Path to publication files: {pub_directory}

These are the missing files/folders for this publication:
- {missingFiles} + {missing_files}

These are the empty folders for this publication:
- {emptyFolders} + {empty_folders}

This is a programmatically generated message. Do NOT reply to this message. - """.format(pubFiles=pub_directory, prjID=project_id, missingFiles=missing_files, emptyFolders = empty_folders,revision=revision) + """ send_mail( "DesignSafe Alert: Published Project has missing files/folders", diff --git a/designsafe/apps/api/tasks_unit_test.py b/designsafe/apps/api/tasks_unit_test.py index e9021f5332..c4bc114644 100644 --- a/designsafe/apps/api/tasks_unit_test.py +++ b/designsafe/apps/api/tasks_unit_test.py @@ -1,7 +1,6 @@ -from django.test import TestCase, RequestFactory -from django.conf import settings -from django.contrib.auth import get_user_model -from mock import patch, MagicMock +from unittest.mock import MagicMock, patch + +from django.test import TestCase TEST_PROJECT = {'uuid': '6262924605326814745-242ac11c-0001-012', 'schemaId': None, @@ -37,12 +36,12 @@ def test_project_listing(self, mock_client): self.assertEqual(prj_result, [TEST_PROJECT]) @patch('designsafe.apps.api.tasks.bulk') - @patch('designsafe.apps.projects.models.elasticsearch.IndexedProject') + @patch('designsafe.apps.api.tasks.IndexedProject') def test_project_indexer(self, mock_index, mock_bulk): mock_connection = MagicMock() mock_index.Index.name = 'designsafe-test-projects' mock_index._get_connection.return_value = mock_connection - from designsafe.apps.api.tasks import index_projects_listing + from designsafe.apps.api.tasks import index_projects_listing index_projects_listing([TEST_PROJECT]) expected_doc = {**TEST_PROJECT} diff --git a/designsafe/apps/api/tests.py b/designsafe/apps/api/tests.py index 1ad34c673e..581cbbf38d 100644 --- a/designsafe/apps/api/tests.py +++ b/designsafe/apps/api/tests.py @@ -1,15 +1,16 @@ -from django.test import TestCase, RequestFactory -from django.conf import settings -from django.contrib.auth import get_user_model - -from designsafe.apps.projects.models.agave.experimental import ExperimentalProject, ModelConfig, FileModel - -# from agavepy.agave import Agave -import mock import json - import logging + +# from agavepy.agave import Agave import pytest +from django.contrib.auth import get_user_model +from django.test import TestCase + +from designsafe.apps.projects.models.agave.experimental import ( + ExperimentalProject, + FileModel, + ModelConfig, +) pytestmark = pytest.mark.django_db diff --git a/designsafe/apps/api/urls.py b/designsafe/apps/api/urls.py index 4fad4e8bda..3713b3416a 100644 --- a/designsafe/apps/api/urls.py +++ b/designsafe/apps/api/urls.py @@ -1,9 +1,10 @@ # pylint: disable=missing-docstring -from django.urls import re_path as url, include, path +from django.urls import include, path +from django.urls import re_path as url + from designsafe.apps.api.views import LoggerApi -from django.http import JsonResponse -from .views import SystemQueueProxyApi -from .views import SystemOverviewProxyApi + +from .views import SystemOverviewProxyApi, SystemQueueProxyApi urlpatterns = [ path("projects/v2", include('designsafe.apps.api.projects_v2.urls')), diff --git a/designsafe/apps/api/users/tasks.py b/designsafe/apps/api/users/tasks.py index 415c8bb61e..8f822b32cd 100644 --- a/designsafe/apps/api/users/tasks.py +++ b/designsafe/apps/api/users/tasks.py @@ -1,6 +1,7 @@ """ Celery tasks for users api """ from celery import shared_task + from .utils import _get_latest_allocations diff --git a/designsafe/apps/api/users/urls.py b/designsafe/apps/api/users/urls.py index 6cd6bccaae..9ba82d43cb 100644 --- a/designsafe/apps/api/users/urls.py +++ b/designsafe/apps/api/users/urls.py @@ -1,5 +1,13 @@ -from django.urls import path, re_path as url -from designsafe.apps.api.users.views import SearchView, AuthenticatedView, UsageView, ProjectUserView, PublicView +from django.urls import path +from django.urls import re_path as url + +from designsafe.apps.api.users.views import ( + AuthenticatedView, + ProjectUserView, + PublicView, + SearchView, + UsageView, +) urlpatterns = [ path("project-lookup/", ProjectUserView.as_view()), diff --git a/designsafe/apps/api/users/utils.py b/designsafe/apps/api/users/utils.py index 3d93baa099..09a42b6fe4 100644 --- a/designsafe/apps/api/users/utils.py +++ b/designsafe/apps/api/users/utils.py @@ -1,14 +1,15 @@ """Utility functions for user API""" -import logging import json -from pytas.http import TASClient +import logging + from django.conf import settings -from django.db.models import Q -from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth import get_user_model -from designsafe.apps.workspace.models.allocations import UserAllocations +from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Q +from pytas.http import TASClient +from designsafe.apps.workspace.models.allocations import UserAllocations logger = logging.getLogger(__name__) @@ -125,7 +126,7 @@ def get_allocations(user, force=False): username = user.username try: if force: - logger.info(f"Forcing TAS allocation retrieval for user:{username}") + logger.info("Forcing TAS allocation retrieval for user:%s", username) raise ObjectDoesNotExist result = {"hosts": {}} result.update(UserAllocations.objects.get(user=user).value) diff --git a/designsafe/apps/api/users/views.py b/designsafe/apps/api/users/views.py index 44349228e0..f3cd5b8d19 100644 --- a/designsafe/apps/api/users/views.py +++ b/designsafe/apps/api/users/views.py @@ -1,17 +1,19 @@ -import logging import json -from designsafe.apps.api.mixins import SecureMixin -from designsafe.apps.api.users import utils as users_utils +import logging + from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist from django.forms.models import model_to_dict -from django.http import HttpResponseNotFound, JsonResponse, HttpRequest +from django.http import HttpRequest, HttpResponseNotFound, JsonResponse from django.views.generic.base import View -from django.core.exceptions import ObjectDoesNotExist +from elasticsearch_dsl import Q from pytas.http import TASClient -from designsafe.apps.api.views import BaseApiView, ApiException + +from designsafe.apps.api.mixins import SecureMixin +from designsafe.apps.api.users import utils as users_utils +from designsafe.apps.api.views import ApiException, BaseApiView from designsafe.apps.data.models.elasticsearch import IndexedFile, IndexedPublication from designsafe.libs.elasticsearch.utils import new_es_client -from elasticsearch_dsl import Q logger = logging.getLogger(__name__) @@ -102,6 +104,7 @@ def get(self, request): user_tas = TASClient().get_user(username=q) res_dict["profile"] = {"institution": user_tas["institution"]} except Exception: + logger.exception("") logger.info("No Profile.") return JsonResponse(res_dict) diff --git a/designsafe/apps/api/utils.py b/designsafe/apps/api/utils.py index 0d9c2d1be2..1ae82be883 100644 --- a/designsafe/apps/api/utils.py +++ b/designsafe/apps/api/utils.py @@ -1,6 +1,7 @@ """Utilities for APIs """ import logging + from django.conf import settings #pylint: disable=invalid-name @@ -18,10 +19,7 @@ def is_jwt(request): decorate protected views. """ val = request.META.get(settings.AGAVE_JWT_HEADER) - if val: - return True - - return False + return bool(val) def get_client_ip(request): diff --git a/designsafe/apps/api/views.py b/designsafe/apps/api/views.py index 94e81ce6e0..c95ef7bc77 100644 --- a/designsafe/apps/api/views.py +++ b/designsafe/apps/api/views.py @@ -1,17 +1,20 @@ -from django.views.decorators.csrf import csrf_exempt -from django.views.generic import View -from django.http import JsonResponse, HttpResponse, Http404 +import json +import logging +from logging import getLevelName + +import requests +from django.conf import settings from django.core.exceptions import PermissionDenied +from django.http import Http404, HttpResponse, JsonResponse from django.utils.decorators import method_decorator -from django.conf import settings +from django.views.decorators.csrf import csrf_exempt +from django.views.generic import View from requests.exceptions import HTTPError -from .exceptions import ApiException -import logging -from logging import getLevelName -import json -from designsafe.apps.api.decorators import tapis_jwt_login from tapipy.errors import BaseTapyException -import requests + +from designsafe.apps.api.decorators import tapis_jwt_login + +from .exceptions import ApiException logger = logging.getLogger(__name__) @@ -29,17 +32,17 @@ def dispatch(self, request, *args, **kwargs): """ try: return super().dispatch(request, *args, **kwargs) - except (PermissionDenied, Http404) as e: + except (PermissionDenied, Http404): # log information but re-raise exception to let django handle response - logger.error(e, exc_info=True) - raise e + logger.exception("") + raise except ApiException as e: status = e.response.status_code or 400 message = e.response.reason extra = e.extra if status != 404: - logger.error( - "%s: %s", message, e.response.text, exc_info=True, extra=extra + logger.exception( + "%s: %s", message, e.response.text, extra=extra ) else: logger.info("Error %s", message, exc_info=True, extra=extra) @@ -69,20 +72,17 @@ def dispatch(self, request, *args, **kwargs): }, ) else: - logger.error( + logger.exception( "%s: %s", message, e.response.text, - exc_info=True, extra={ "username": request.user.username, "session_key": request.session.session_key, }, ) else: - logger.error( - e, - exc_info=True, + logger.exception( extra={ "username": request.user.username, "session_key": request.session.session_key, @@ -90,8 +90,8 @@ def dispatch(self, request, *args, **kwargs): ) message = str(e) return JsonResponse({"message": message}, status=status) - except Exception as e: # pylint: disable=broad-except - logger.error(e, exc_info=True) + except Exception: # pylint: disable=broad-except + logger.exception("") return JsonResponse({"message": "Something went wrong here..."}, status=500) @@ -105,7 +105,7 @@ def dispatch(self, request, *args, **kwargs): if not request.user.is_authenticated: return JsonResponse({"message": "Unauthenticated user"}, status=401) - return super(AuthenticatedApiView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) class AuthenticatedAllowJwtApiView(AuthenticatedApiView): @@ -117,7 +117,7 @@ class AuthenticatedAllowJwtApiView(AuthenticatedApiView): @method_decorator(tapis_jwt_login) def dispatch(self, request, *args, **kwargs): """Returns 401 if user is not authenticated like AuthenticatedApiView but allows JWT access.""" - return super(AuthenticatedAllowJwtApiView, self).dispatch( + return super().dispatch( request, *args, **kwargs ) @@ -176,7 +176,7 @@ def get(self, request, hostname): "status": "success" }) except requests.exceptions.RequestException as e: - logger.exception("Proxy API Error:", str(e)) + logger.exception("Proxy API Error") return JsonResponse({ "response": None, "status": "error", @@ -201,7 +201,7 @@ def get(self, request): }, status=200) except requests.exceptions.RequestException as e: - logger.exception("Proxy API Error:", str(e)) + logger.exception("Proxy API Error") return JsonResponse({ "response": None, "status": "error", diff --git a/designsafe/apps/applications/admin.py b/designsafe/apps/applications/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/applications/admin.py +++ b/designsafe/apps/applications/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/applications/models.py b/designsafe/apps/applications/models.py index 137941ffae..e69de29bb2 100644 --- a/designsafe/apps/applications/models.py +++ b/designsafe/apps/applications/models.py @@ -1 +0,0 @@ -from django.db import models diff --git a/designsafe/apps/applications/tests.py b/designsafe/apps/applications/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/applications/tests.py +++ b/designsafe/apps/applications/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/applications/urls.py b/designsafe/apps/applications/urls.py index 9591c6849d..465c649c2b 100644 --- a/designsafe/apps/applications/urls.py +++ b/designsafe/apps/applications/urls.py @@ -1,8 +1,6 @@ -from django.urls import include, re_path as url -from django.urls import reverse -from django.utils.translation import gettext_lazy as _ -from designsafe.apps.applications import views +from django.urls import re_path as url +from designsafe.apps.applications import views urlpatterns = [ url(r'^$', views.index, name='index'), diff --git a/designsafe/apps/applications/views.py b/designsafe/apps/applications/views.py index 455e30d07d..3211de2c31 100644 --- a/designsafe/apps/applications/views.py +++ b/designsafe/apps/applications/views.py @@ -1,18 +1,20 @@ +# ruff: noqa # from agavepy.agave import Agave, AgaveException, load_resource -from designsafe.apps.licenses.models import LICENSE_TYPES, get_license_info -from designsafe.apps.notifications.views import get_number_unread_notifications -from designsafe.libs.common.decorators import profile as profile_fn -from designsafe.apps.api.tasks import index_or_update_project +import json +import logging + from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.serializers.json import DjangoJSONEncoder from django.http import HttpResponse -from django.shortcuts import render, redirect +from django.shortcuts import render from requests import HTTPError -from designsafe.apps.api.utils import get_client_ip -import json -import logging +from designsafe.apps.api.tasks import index_or_update_project +from designsafe.apps.api.utils import get_client_ip +from designsafe.apps.licenses.models import LICENSE_TYPES, get_license_info +from designsafe.apps.notifications.views import get_number_unread_notifications +from designsafe.libs.common.decorators import profile as profile_fn logger = logging.getLogger(__name__) metrics = logging.getLogger('metrics') @@ -23,7 +25,7 @@ @login_required def index(request): context = {} - token_key = getattr(settings, 'AGAVE_TOKEN_SESSION_ID') + token_key = settings.AGAVE_TOKEN_SESSION_ID if token_key in request.session: context['session'] = { 'agave': json.dumps(request.session[token_key]) @@ -89,7 +91,7 @@ def call_api(request, service): } if lic_type is not None: _, license_models = get_license_info() - license_model = [x for x in license_models if x.license_type == lic_type][0] + license_model = next(x for x in license_models if x.license_type == lic_type) lic = license_model.objects.filter(user=request.user).first() data['license']['enabled'] = lic is not None else: @@ -439,7 +441,7 @@ def call_api(request, service): appId = request.GET.get('appId') query = request.GET.get('q') - ds_admin_client = Agave(api_server=getattr(settings, 'AGAVE_TENANT_BASEURL'), token=getattr(settings, 'AGAVE_SUPER_TOKEN')) + ds_admin_client = Agave(api_server=settings.AGAVE_TENANT_BASEURL, token=settings.AGAVE_SUPER_TOKEN) if request.method == 'GET': if appId and pems: @@ -486,16 +488,15 @@ def call_api(request, service): data = ds_admin_client.meta.updateMetadataPermissionsForUser(body=body, uuid=uuid, username=username) else: - return HttpResponse('Unexpected service: %s' % service, status=400) + return HttpResponse(f'Unexpected service: {service}', status=400) except AgaveException as e: - logger.error('Failed to execute {0} API call due to AgaveException={1}'.format( - service, e)) + logger.error(f'Failed to execute {service} API call due to AgaveException={e}') return HttpResponse(json.dumps(e), content_type='application/json', status=400) except HTTPError as e: try: json_response = e.response.json() - metrics.info('Failed to execute {0} API call due to HTTPError={1}'.format( + metrics.info('Failed to execute {} API call due to HTTPError={}'.format( service, json_response.get('message')), extra={ 'agent': request.META.get('HTTP_USER_AGENT'), 'ip': get_client_ip(request), @@ -504,19 +505,20 @@ def call_api(request, service): 'user': request.user.username, 'info': {} }) - logger.error('Failed to execute {0} API call due to HTTPError={1}'.format( - service, json_response.get('message'))) + logger.error('Failed to execute %s API call due to HTTPError=%s', + service, json_response.get('message')) return HttpResponse(json.dumps(json_response.get('message')), content_type='application/json', status=400) except Exception as e: + logger.exception("") return HttpResponse(json.dumps(e), content_type='application/json', status=400) except Exception as e: - metrics.info('Failed to execute {0} API call due to Exception={1}'.format( - service, e), extra={ + logger.exception("") + metrics.info(f'Failed to execute {service} API call due to Exception={e}', extra={ 'agent': request.META.get('HTTP_USER_AGENT'), 'ip': get_client_ip(request), 'operation': 'agave.meta.listMetadata', @@ -524,14 +526,13 @@ def call_api(request, service): 'user': request.user.username, 'info': {} }) - logger.error('Failed to execute {0} API call due to Exception={1}'.format( - service, e), extra={ + logger.error('Failed to execute %s API call due to Exception=%s', service, e, extra={ 'agent': request.META.get('HTTP_USER_AGENT'), 'ip': get_client_ip(request), 'user': request.user.username }) return HttpResponse( - json.dumps({'status': 'error', 'message': '{}'.format(e)}), + json.dumps({'status': 'error', 'message': f'{e}'}), content_type='application/json', status=400) return HttpResponse(json.dumps(data, cls=DjangoJSONEncoder), diff --git a/designsafe/apps/auth/apps.py b/designsafe/apps/auth/apps.py index aeb437e17a..d005a97d4a 100644 --- a/designsafe/apps/auth/apps.py +++ b/designsafe/apps/auth/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig + class DesignSafeAuthConfig(AppConfig): name = 'designsafe.apps.auth' label = 'designsafe_auth' diff --git a/designsafe/apps/auth/backends.py b/designsafe/apps/auth/backends.py index 8cd5122f57..c4c1a51be8 100644 --- a/designsafe/apps/auth/backends.py +++ b/designsafe/apps/auth/backends.py @@ -2,17 +2,19 @@ import logging import re + from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.backends import ModelBackend -from tapipy.tapis import Tapis +from django.core.exceptions import ValidationError +from pytas.http import TASClient from tapipy.errors import BaseTapyException +from tapipy.tapis import Tapis + from designsafe.apps.accounts.models import DesignSafeProfile, NotificationPreferences from designsafe.apps.api.users.utils import get_user_data from designsafe.apps.auth.models import TapisOAuthToken -from django.core.exceptions import ValidationError from designsafe.apps.auth.tasks import update_institution_from_tas -from pytas.http import TASClient logger = logging.getLogger(__name__) @@ -30,25 +32,26 @@ def authenticate(self, request, username=None, password=None, **kwargs): tas_user = None if request is not None: self.logger.info( - 'Attempting login via TAS for user "%s" from IP "%s"' - % (username, request.META.get("REMOTE_ADDR")) - ) + 'Attempting login via TAS for user "%s" from IP "%s"', username, request.META.get("REMOTE_ADDR") + ) + else: self.logger.info( - 'Attempting login via TAS for user "%s" from IP "%s"' - % (username, "unknown") - ) + 'Attempting login via TAS for user "%s" from IP "%s"', username, "unknown" + ) + try: # Check if this user is valid on the mail server if self.tas.authenticate(username, password): tas_user = self.tas.get_user(username=username) - self.logger.info('Login successful for user "%s"' % username) + self.logger.info('Login successful for user "%s"', username) else: raise ValidationError( "Authentication Error", "Your username or password is incorrect.", ) except Exception as e: + self.logger.exception("") self.logger.warning(e.args) if re.search(r"PendingEmailConfirmation", e.args[1]): raise ValidationError( @@ -70,8 +73,8 @@ def authenticate(self, request, username=None, password=None, **kwargs): except UserModel.DoesNotExist: # Create a user in Django's local database self.logger.info( - 'Creating local user record for "%s" from TAS Profile' - % username + 'Creating local user record for "%s" from TAS Profile', + username ) user = UserModel.objects.create_user( username=username, @@ -106,8 +109,8 @@ def authenticate(self, *args, **kwargs): token = kwargs["token"] logger.info( - 'Attempting login via Tapis with token "%s"' - % TapisOAuthToken().get_masked_token(token) + 'Attempting login via Tapis with token "%s"', + TapisOAuthToken().get_masked_token(token), ) client = Tapis(base_url=settings.TAPIS_TENANT_BASEURL, access_token=token) diff --git a/designsafe/apps/auth/backends_unit_test.py b/designsafe/apps/auth/backends_unit_test.py index fc8d8bf007..1d3daae720 100644 --- a/designsafe/apps/auth/backends_unit_test.py +++ b/designsafe/apps/auth/backends_unit_test.py @@ -1,8 +1,10 @@ +from unittest.mock import Mock + import pytest from django.contrib.auth import get_user_model -from mock import Mock -from tapipy.tapis import TapisResult from tapipy.errors import BaseTapyException +from tapipy.tapis import TapisResult + from designsafe.apps.auth.backends import TapisOAuthBackend from designsafe.apps.auth.views import launch_setup_checks diff --git a/designsafe/apps/auth/middleware.py b/designsafe/apps/auth/middleware.py index 10a9e81336..dcbb1b2472 100644 --- a/designsafe/apps/auth/middleware.py +++ b/designsafe/apps/auth/middleware.py @@ -3,12 +3,14 @@ """ import logging + from django.contrib.auth import logout from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from django.http import HttpResponseRedirect from django.urls import reverse from tapipy.errors import BaseTapyException + from designsafe.apps.auth.models import TapisOAuthToken logger = logging.getLogger(__name__) @@ -48,7 +50,8 @@ def process_request(self, request): return logger.info( - f"Tapis OAuth token expired for user {request.user.username}. Refreshing token" + "Tapis OAuth token expired for user %s. Refreshing token", + request.user.username ) with transaction.atomic(): # Get a lock on this user's token row in db. diff --git a/designsafe/apps/auth/models.py b/designsafe/apps/auth/models.py index 73e61918d9..263b2f4e3b 100644 --- a/designsafe/apps/auth/models.py +++ b/designsafe/apps/auth/models.py @@ -4,8 +4,9 @@ import logging import time from urllib.parse import urlparse -from django.db import models + from django.conf import settings +from django.db import models from tapipy.tapis import Tapis logger = logging.getLogger(__name__) @@ -13,7 +14,7 @@ TOKEN_EXPIRY_THRESHOLD = 600 -tenant_id = urlparse(getattr(settings, "TAPIS_TENANT_BASEURL")).hostname.split( +tenant_id = urlparse(settings.TAPIS_TENANT_BASEURL).hostname.split( "." )[0] @@ -80,10 +81,10 @@ def client(self): :rtype: :class:Tapis """ return Tapis( - base_url=getattr(settings, "TAPIS_TENANT_BASEURL"), + base_url=settings.TAPIS_TENANT_BASEURL, tenant_id=tenant_id, - client_id=getattr(settings, "TAPIS_CLIENT_ID"), - client_key=getattr(settings, "TAPIS_CLIENT_KEY"), + client_id=settings.TAPIS_CLIENT_ID, + client_key=settings.TAPIS_CLIENT_KEY, access_token=self.access_token, refresh_token=self.refresh_token, ) diff --git a/designsafe/apps/auth/models_unit_test.py b/designsafe/apps/auth/models_unit_test.py index cbd9d159c2..57c0ae255b 100644 --- a/designsafe/apps/auth/models_unit_test.py +++ b/designsafe/apps/auth/models_unit_test.py @@ -1,6 +1,8 @@ -import pytest import time from datetime import timedelta + +import pytest + from designsafe.apps.auth.models import TapisOAuthToken pytestmark = pytest.mark.django_db diff --git a/designsafe/apps/auth/tasks.py b/designsafe/apps/auth/tasks.py index 8252f0417a..efb2cdc441 100644 --- a/designsafe/apps/auth/tasks.py +++ b/designsafe/apps/auth/tasks.py @@ -1,13 +1,16 @@ """ Celery tasks for user onboarding and other user-related tasks. """ import logging -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta + +from celery import shared_task from django.conf import settings from django.contrib.auth import get_user_model from django.core.mail import send_mail -from celery import shared_task from pytas.http import TASClient + from designsafe.apps.api.notifications.models import Notification + logger = logging.getLogger(__name__) @@ -48,7 +51,7 @@ def new_user_alert(username): @shared_task() def clear_old_notifications(): """Delete notifications older than 30 days to prevent them cluttering the db.""" - time_cutoff = datetime.now() - timedelta(days=30) + time_cutoff = datetime.now(tz=UTC) - timedelta(days=30) Notification.objects.filter(datetime__lte=time_cutoff).delete() @@ -58,6 +61,7 @@ def update_institution_from_tas(self, username): try: tas_model = TASClient().get_user(username=username) except Exception as exc: + logger.exception("") raise self.retry(exc=exc) user_model.profile.institution = tas_model.get("institution", None) user_model.profile.orcid_id = tas_model.get("orcidId", None) diff --git a/designsafe/apps/auth/templatetags/auth_extras.py b/designsafe/apps/auth/templatetags/auth_extras.py index dffd4566f8..fc43191d2e 100644 --- a/designsafe/apps/auth/templatetags/auth_extras.py +++ b/designsafe/apps/auth/templatetags/auth_extras.py @@ -1,5 +1,4 @@ from django import template -from django.contrib.auth.models import Group register = template.Library() diff --git a/designsafe/apps/auth/test_tasks.py b/designsafe/apps/auth/test_tasks.py index 1eb3fb8e9d..4809847d84 100644 --- a/designsafe/apps/auth/test_tasks.py +++ b/designsafe/apps/auth/test_tasks.py @@ -1,6 +1,7 @@ -import pytest from unittest import mock +import pytest + @pytest.fixture def mock_get_user_model(authenticated_user): diff --git a/designsafe/apps/auth/urls.py b/designsafe/apps/auth/urls.py index fc956002c4..53aa4ee613 100644 --- a/designsafe/apps/auth/urls.py +++ b/designsafe/apps/auth/urls.py @@ -4,6 +4,7 @@ """ from django.urls import path + from designsafe.apps.auth import views app_name = "designsafe_auth" diff --git a/designsafe/apps/auth/views.py b/designsafe/apps/auth/views.py index 1b47e1dc11..d5914cb1f8 100644 --- a/designsafe/apps/auth/views.py +++ b/designsafe/apps/auth/views.py @@ -3,19 +3,22 @@ """ import logging -import time import secrets +import time + import requests from django.conf import settings from django.contrib import messages from django.contrib.auth import authenticate, login, logout from django.contrib.auth.views import LogoutView as DjangoLogoutView -from django.urls import reverse -from django.http import HttpResponseRedirect, HttpResponseBadRequest +from django.http import HttpResponseBadRequest, HttpResponseRedirect from django.shortcuts import render +from django.urls import reverse + from designsafe.apps.api.users.tasks import cache_allocations from designsafe.apps.api.utils import get_client_ip from designsafe.apps.onboarding.execute import execute_setup_steps, new_user_setup_check + from .models import TapisOAuthToken logger = logging.getLogger(__name__) @@ -45,8 +48,8 @@ def tapis_oauth(request): "designsafe-ci.org", "www.designsafe-ci.org" ) - tenant_base_url = getattr(settings, "TAPIS_TENANT_BASEURL") - client_id = getattr(settings, "TAPIS_CLIENT_ID") + tenant_base_url = settings.TAPIS_TENANT_BASEURL + client_id = settings.TAPIS_CLIENT_ID METRICS.debug(f"user:{request.user.username} starting oauth redirect login") @@ -171,7 +174,11 @@ def dispatch(self, request): logout(request) - logger.info('The user %s is attempting to logout via Tapis with token "%s"' % (username, token[:8].ljust(len(token), '-'))) + logger.info( + 'The user %s is attempting to logout via Tapis with token "%s"', + username, + token[:8].ljust(len(token), "-"), + ) logout_endpoint = f"{settings.TAPIS_TENANT_BASEURL}/v3/oauth2/logout?redirect_url=https://{request.get_host()}{settings.LOGOUT_REDIRECT_URL}" response = HttpResponseRedirect(logout_endpoint) diff --git a/designsafe/apps/auth/views_unit_test.py b/designsafe/apps/auth/views_unit_test.py index 1fdb0d67ea..20c2272fe9 100644 --- a/designsafe/apps/auth/views_unit_test.py +++ b/designsafe/apps/auth/views_unit_test.py @@ -1,12 +1,12 @@ """DesignSafe Auth Tapis OAuth flow view tests""" import pytest -from django.contrib.auth import get_user from django.conf import settings -from django.urls import reverse +from django.contrib.auth import get_user from django.http import HttpResponseRedirect -from designsafe.apps.auth.views import launch_setup_checks +from django.urls import reverse +from designsafe.apps.auth.views import launch_setup_checks TEST_STATE = "ABCDEFG123456" diff --git a/designsafe/apps/box_integration/admin.py b/designsafe/apps/box_integration/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/box_integration/admin.py +++ b/designsafe/apps/box_integration/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/box_integration/integrations.py b/designsafe/apps/box_integration/integrations.py index 74eb6a88a1..cfc11467e1 100644 --- a/designsafe/apps/box_integration/integrations.py +++ b/designsafe/apps/box_integration/integrations.py @@ -1,4 +1,3 @@ -from django.urls import reverse def provide_integrations(): diff --git a/designsafe/apps/box_integration/models.py b/designsafe/apps/box_integration/models.py index 4b4d2360b8..59cf6b89a0 100644 --- a/designsafe/apps/box_integration/models.py +++ b/designsafe/apps/box_integration/models.py @@ -1,7 +1,7 @@ + from boxsdk import Client, OAuth2 -from django.db import models from django.conf import settings -import json +from django.db import models class BoxUserToken(models.Model): diff --git a/designsafe/apps/box_integration/tasks.py b/designsafe/apps/box_integration/tasks.py index f6f01b88f8..edd3860fff 100644 --- a/designsafe/apps/box_integration/tasks.py +++ b/designsafe/apps/box_integration/tasks.py @@ -1,6 +1,7 @@ -from django.contrib.auth import get_user_model import logging +from django.contrib.auth import get_user_model + logger = logging.getLogger(__name__) diff --git a/designsafe/apps/box_integration/tests/test_views.py b/designsafe/apps/box_integration/tests/test_views.py index eb9396659c..6df1dda31c 100644 --- a/designsafe/apps/box_integration/tests/test_views.py +++ b/designsafe/apps/box_integration/tests/test_views.py @@ -1,10 +1,11 @@ -from django.test import TestCase +from unittest import mock, skip + +from boxsdk.object.user import User from django.contrib.auth import get_user_model +from django.test import TestCase from django.urls import reverse -from unittest import skip -from boxsdk.object.user import User + from designsafe.apps.box_integration.models import BoxUserToken -import mock @skip('This test is not working') diff --git a/designsafe/apps/box_integration/urls.py b/designsafe/apps/box_integration/urls.py index 2146c4d76e..f8157ada03 100644 --- a/designsafe/apps/box_integration/urls.py +++ b/designsafe/apps/box_integration/urls.py @@ -1,4 +1,5 @@ from django.urls import re_path as url + from designsafe.apps.box_integration import views urlpatterns = [ diff --git a/designsafe/apps/box_integration/views.py b/designsafe/apps/box_integration/views.py index 259257bda5..d480ec41bd 100644 --- a/designsafe/apps/box_integration/views.py +++ b/designsafe/apps/box_integration/views.py @@ -1,15 +1,16 @@ -from boxsdk import OAuth2, Client -from boxsdk.exception import BoxOAuthException, BoxException +import logging + +from boxsdk import Client, OAuth2 +from boxsdk.exception import BoxException, BoxOAuthException from django.conf import settings -from django.contrib.auth.decorators import login_required from django.contrib import messages -from django.urls import reverse -from django.http import (HttpResponseRedirect, HttpResponseBadRequest) +from django.contrib.auth.decorators import login_required +from django.http import HttpResponseBadRequest, HttpResponseRedirect from django.shortcuts import render +from django.urls import reverse + from designsafe.apps.box_integration.models import BoxUserToken from designsafe.apps.box_integration.tasks import check_connection -import logging - logger = logging.getLogger(__name__) @@ -24,17 +25,15 @@ def index(request): context['box_connection'] = box_user except BoxOAuthException: # authentication failed - logger.warning('Box oauth token for user=%s failed to authenticate' % - request.user.username) + logger.warning('Box oauth token for user=%sfailed to authenticate', request.user.username) context['box_connection'] = False except BoxException: # session layer exception - logger.warning('Box API error when testing oauth token for user=%s' % - request.user.username) + logger.warning('Box API error when testing oauth token for user=%s', request.user.username) context['box_connection'] = False except BoxUserToken.DoesNotExist: - logger.debug('BoxUserToken does not exist for user=%s' % request.user.username) + logger.debug('BoxUserToken does not exist for user=%s', request.user.username) return render(request, 'designsafe/apps/box_integration/index.html', context) @@ -63,7 +62,7 @@ def oauth2_callback(request): else: return HttpResponseBadRequest('Unexpected request') - if not (state == box['state']): + if state != box['state']: return HttpResponseBadRequest('Request expired') try: @@ -83,8 +82,8 @@ def oauth2_callback(request): box_user_id=box_user.id, ) token.save() - except BoxException as e: - logger.exception('Unable to complete Box integration setup: %s' % e) + except BoxException: + logger.exception('Unable to complete Box integration setup') messages.error(request, 'Oh no! An unexpected error occurred while trying to set ' 'up the Box.com application. Please try again.') @@ -99,9 +98,10 @@ def disconnect(request): box_user_token = request.user.box_user_token box_user_token.delete() except BoxUserToken.DoesNotExist: - logger.warn('Disconnect Box; BoxUserToken does not exist.', + logger.warning('Disconnect Box; BoxUserToken does not exist.', extra={'user': request.user}) - except: + except Exception: + logger.exception("") logger.error('Disconnect Box; BoxUserToken delete error.', extra={'user': request.user}) messages.success( diff --git a/designsafe/apps/box_integration/webhooks.py b/designsafe/apps/box_integration/webhooks.py index 97e37a361f..92f0445256 100644 --- a/designsafe/apps/box_integration/webhooks.py +++ b/designsafe/apps/box_integration/webhooks.py @@ -1,7 +1,8 @@ +import logging + from django.http.response import HttpResponse from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_http_methods -import logging logger = logging.getLogger(__name__) diff --git a/designsafe/apps/cms_plugins/admin.py b/designsafe/apps/cms_plugins/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/cms_plugins/admin.py +++ b/designsafe/apps/cms_plugins/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/cms_plugins/cms_plugins.py b/designsafe/apps/cms_plugins/cms_plugins.py index 05d1b32f2a..572e78a238 100644 --- a/designsafe/apps/cms_plugins/cms_plugins.py +++ b/designsafe/apps/cms_plugins/cms_plugins.py @@ -1,14 +1,15 @@ try: import urllib.parse except ImportError: - from urllib import parse as urlparse + pass from cms.plugin_base import CMSPluginBase from cms.plugin_pool import plugin_pool -from designsafe.apps.cms_plugins.models import ResponsiveEmbedPlugin from django.conf import settings from django.utils.translation import gettext as _ +from designsafe.apps.cms_plugins.models import ResponsiveEmbedPlugin + class CMSResponsiveEmbedPlugin(CMSPluginBase): model = ResponsiveEmbedPlugin # model where plugin data are saved diff --git a/designsafe/apps/cms_plugins/context_processors.py b/designsafe/apps/cms_plugins/context_processors.py index 04431bcd72..b2d9e8a631 100644 --- a/designsafe/apps/cms_plugins/context_processors.py +++ b/designsafe/apps/cms_plugins/context_processors.py @@ -1,6 +1,7 @@ -from menus.menu_pool import MenuPool import logging +from menus.menu_pool import MenuPool + logger = logging.getLogger(__name__) diff --git a/designsafe/apps/cms_plugins/menu.py b/designsafe/apps/cms_plugins/menu.py index 7ad05206c4..6e6019578c 100644 --- a/designsafe/apps/cms_plugins/menu.py +++ b/designsafe/apps/cms_plugins/menu.py @@ -1,10 +1,10 @@ -from menus.base import NavigationNode +import logging +from importlib import import_module + from cms.menu_bases import CMSAttachMenu -from menus.menu_pool import menu_pool from django.conf import settings -from importlib import import_module from django.utils.translation import gettext_lazy as _ -import logging +from menus.base import NavigationNode logger = logging.getLogger(__name__) @@ -14,7 +14,7 @@ def _menu_nodes_for_apps(menu_type): node_id = 1 for app in settings.INSTALLED_APPS: try: - mod = import_module('%s.urls' % app) + mod = import_module(f'{app}.urls') try: items = mod.menu_items(type=menu_type) if items is not None: @@ -34,10 +34,12 @@ def _menu_nodes_for_apps(menu_type): except AttributeError: continue except TypeError: - logger.warning('Call to %s.cms_menu_nodes should return an iterable!') - except: - logger.warning('Call to %s.cms_menu_nodes failed!' % mod.__name__) - except: + logger.warning('Call to %s.cms_menu_nodes should return an iterable!', mod.__name__) + except Exception: + logger.exception("") + logger.warning('Call to %s.cms_menu_nodes failed!', mod.__name__) + except Exception: + logger.exception("") continue return nodes diff --git a/designsafe/apps/cms_plugins/models.py b/designsafe/apps/cms_plugins/models.py index e1b4c15aa4..9f1982e18e 100644 --- a/designsafe/apps/cms_plugins/models.py +++ b/designsafe/apps/cms_plugins/models.py @@ -1,5 +1,6 @@ -from django.db import models from cms.models import CMSPlugin +from django.db import models + class ResponsiveEmbedPlugin(CMSPlugin): url = models.CharField(max_length=255) diff --git a/designsafe/apps/cms_plugins/tests.py b/designsafe/apps/cms_plugins/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/cms_plugins/tests.py +++ b/designsafe/apps/cms_plugins/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/cms_plugins/views.py b/designsafe/apps/cms_plugins/views.py index 91ea44a218..b8e4ee02a7 100644 --- a/designsafe/apps/cms_plugins/views.py +++ b/designsafe/apps/cms_plugins/views.py @@ -1,3 +1,2 @@ -from django.shortcuts import render # Create your views here. diff --git a/designsafe/apps/dashboard/tests.py b/designsafe/apps/dashboard/tests.py index 8bfab4afd1..82a4d2f711 100644 --- a/designsafe/apps/dashboard/tests.py +++ b/designsafe/apps/dashboard/tests.py @@ -1,11 +1,12 @@ import json import logging from unittest import skip -from elasticsearch_dsl import Q + +from django.contrib.auth import get_user_model from django.test import TestCase -from django.contrib.auth import get_user_model, signals -from django.contrib.auth.models import Permission from django.urls import reverse +from elasticsearch_dsl import Q + from designsafe.apps.data.models.elasticsearch import IndexedFile logger = logging.getLogger(__name__) diff --git a/designsafe/apps/dashboard/urls.py b/designsafe/apps/dashboard/urls.py index f68ad7ad8b..906686722b 100644 --- a/designsafe/apps/dashboard/urls.py +++ b/designsafe/apps/dashboard/urls.py @@ -1,6 +1,5 @@ -from django.urls import include, re_path as url -from django.urls import reverse -from django.utils.translation import gettext_lazy as _ +from django.urls import re_path as url + from designsafe.apps.dashboard import views urlpatterns = [ diff --git a/designsafe/apps/dashboard/views.py b/designsafe/apps/dashboard/views.py index e91f7fc421..68e7d248b7 100644 --- a/designsafe/apps/dashboard/views.py +++ b/designsafe/apps/dashboard/views.py @@ -1,8 +1,8 @@ -from django.shortcuts import render -from django.contrib.auth.decorators import login_required, permission_required -from django.http import JsonResponse import logging +from django.contrib.auth.decorators import login_required +from django.shortcuts import render + logger = logging.getLogger(__name__) @login_required diff --git a/designsafe/apps/data/admin.py b/designsafe/apps/data/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/data/admin.py +++ b/designsafe/apps/data/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/data/apps.py b/designsafe/apps/data/apps.py index 8ad2fdb9b5..846068859f 100644 --- a/designsafe/apps/data/apps.py +++ b/designsafe/apps/data/apps.py @@ -1,12 +1,12 @@ """Data app config.""" import logging import socket -from urllib3.connection import HTTPConnection + from django.apps import AppConfig from django.conf import settings -from elasticsearch_dsl.connections import connections from elasticsearch import Urllib3HttpConnection - +from elasticsearch_dsl.connections import connections +from urllib3.connection import HTTPConnection # pylint: disable=invalid-name logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ class KeepAliveConnection(Urllib3HttpConnection): Solution from Github issue: https://github.com/elastic/elasticsearch-py/issues/966#issuecomment-819823632 """ def __init__(self, *args, **kwargs): - super(KeepAliveConnection, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if not (settings.DEBUG or getattr(settings, "TEST", False)): self.pool.conn_kw['socket_options'] = HTTPConnection.default_socket_options + [ (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), @@ -37,7 +37,7 @@ class DataConfig(AppConfig): def ready(self): # pylint:disable=too-many-locals """Run stuff when app is ready.""" - enable_sniffing = (not settings.DESIGNSAFE_ENVIRONMENT == 'dev') + enable_sniffing = (settings.DESIGNSAFE_ENVIRONMENT != 'dev') try: connections.create_connection('default', hosts=settings.ES_CONNECTIONS[settings.DESIGNSAFE_ENVIRONMENT]['hosts'], @@ -58,24 +58,39 @@ def ready(self): # pylint:disable=too-many-locals # We need to import every class so we can later setup the reverse # reverse relations. # pylint: disable=unused-import + from designsafe.apps.data.models.agave.base import set_lazy_rels from designsafe.apps.projects.models.agave.experimental import ( # noqa: F401 - ExperimentalProject, FileModel as ExpFileModel, DataTag as ExpDataTag, - Experiment, Analysis, ModelConfig, SensorList, Event, Report as ExpReport - ) - from designsafe.apps.projects.models.agave.simulation import ( # noqa: F401 - SimulationProject, FileModel as SimFileModel, DataTag as SimDataTag, - Simulation, Model, Input, Output, Analysis as SimAnalysis, Report as SimReport + Analysis, + Event, + Experiment, + ExperimentalProject, + ModelConfig, + SensorList, ) from designsafe.apps.projects.models.agave.hybrid_simulation import ( # noqa: F401 - HybridSimulationProject, FileModel as HybFileModel, DataTag as HybDataTag, - HybridSimulation, GlobalModel, Coordinator, SimSubstructure, ExpSubstructure, - CoordinatorOutput, SimOutput, ExpOutput, Analysis as HybAnalysis, - Report as HybReport + Coordinator, + CoordinatorOutput, + ExpOutput, + ExpSubstructure, + GlobalModel, + HybridSimulation, + HybridSimulationProject, + SimOutput, + SimSubstructure, ) from designsafe.apps.projects.models.agave.rapid import ( # noqa: F401 - FieldReconProject, FileModel as FRFileModel, DataTag as FRDataTag, - Mission, Instrument, ReferencedData, Collection, Report as FRReport + Collection, + FieldReconProject, + Instrument, + Mission, + ReferencedData, + ) + from designsafe.apps.projects.models.agave.simulation import ( # noqa: F401 + Input, + Model, + Output, + Simulation, + SimulationProject, ) - from designsafe.apps.data.models.agave.base import set_lazy_rels set_lazy_rels() - super(DataConfig, self).ready() + super().ready() diff --git a/designsafe/apps/data/indexer.py b/designsafe/apps/data/indexer.py index bf706b9038..fda0ae9953 100644 --- a/designsafe/apps/data/indexer.py +++ b/designsafe/apps/data/indexer.py @@ -1,4 +1,3 @@ -import os import logging logger = logging.getLogger(__name__) diff --git a/designsafe/apps/data/management/commands/es_setup.py b/designsafe/apps/data/management/commands/es_setup.py index ba0b227602..dc0724d7dc 100644 --- a/designsafe/apps/data/management/commands/es_setup.py +++ b/designsafe/apps/data/management/commands/es_setup.py @@ -1,9 +1,10 @@ +import getpass import logging -from django.core.management import BaseCommand, CommandError -from six.moves import input -from django.conf import settings + import elasticsearch -import getpass +from django.conf import settings +from django.core.management import BaseCommand + logger = logging.getLogger(__name__) from designsafe.libs.elasticsearch.indices import init @@ -39,7 +40,7 @@ def handle(self, *args, **options): local_es_client = elasticsearch.Elasticsearch(settings.ES_CONNECTIONS[local]['hosts'], timeout=300) remote_es_client = elasticsearch.Elasticsearch(settings.ES_CONNECTIONS[remote]['hosts'], - **{'http_auth': "designsafe_{}:{}".format(remote, password)}, + http_auth=f"designsafe_{remote}:{password}", timeout=300) def reindex(source_index, target_index, query=None): try: @@ -50,10 +51,10 @@ def reindex(source_index, target_index, query=None): target_index=target_index, query=query, ) - logger.info(f"Reindexed {response} documents from {source_index} to {target_index}") + logger.info("Reindexed %s documents from %s to %s", response, source_index, target_index) except elasticsearch.helpers.BulkIndexError as e: - logger.error(f"BulkIndexError: {e.errors}") - logger.error(f"Failed to reindex documents from {source_index} to {target_index}") + logger.error("BulkIndexError: %s", e.errors) + logger.error("Failed to reindex documents from %s to %s", source_index, target_index) return False return response @@ -67,7 +68,7 @@ def reindex(source_index, target_index, query=None): failed_indexes = [] for index in indexes: - query = {"query": {"prefix": {"path._exact": "/{}".format(username)}}} if "files" in index else None + query = {"query": {"prefix": {"path._exact": f"/{username}"}}} if "files" in index else None result = reindex( source_index=index.format(remote), target_index=index.format(local), @@ -77,7 +78,12 @@ def reindex(source_index, target_index, query=None): failed_indexes.append(index.format(local)) logger.info("Finished indexing.") if failed_indexes: - logger.error(f"Successfully reindex {len(indexes) - len(failed_indexes)}" - f" of {len(indexes)} indexes\n" - f"failed indexes are: {', '.join(failed_indexes)}\n" - f"(check logs above for errors related to failures)") + logger.error( + "Successfully reindex %s" + " of %s indexes\n" + "failed indexes are: %s\n" + "(check logs above for errors related to failures)", + len(indexes) - len(failed_indexes), + len(indexes), + ", ".join(failed_indexes), + ) diff --git a/designsafe/apps/data/management/commands/reindex.py b/designsafe/apps/data/management/commands/reindex.py index fa984e590a..7ca7e1dc2a 100644 --- a/designsafe/apps/data/management/commands/reindex.py +++ b/designsafe/apps/data/management/commands/reindex.py @@ -1,13 +1,13 @@ """Reindex command""" import json import logging + +import elasticsearch import six -from django.core.management.base import BaseCommand, CommandError from django.conf import settings -import elasticsearch +from django.core.management.base import BaseCommand, CommandError from elasticsearch import TransportError - logger = logging.getLogger(__name__) class Command(BaseCommand): @@ -49,7 +49,7 @@ def handle(self, *args, **options): from_index = options.get('from_index') to_index = options.get('to_index') doc_type = options.get('doc_type') - all_docs = options.get('all_docs') + options.get('all_docs') remote_host = options.get('remote_host') sample = options.get('sample') size = options.get('size') @@ -80,45 +80,43 @@ def handle(self, *args, **options): if remote_host: - if remote_host.startswith('http://') \ - or remote_host.startswith('https://'): + if remote_host.startswith(('http://', 'https://')): hosts = [remote_host] - body['source']['remote'] = {'host': '{}:9200'.format(hosts[0])} + body['source']['remote'] = {'host': f'{hosts[0]}:9200'} else: hosts = settings.ES_CONNECTIONS.get(remote_host, {}).get('hosts', []) - body['source']['remote'] = {'host': 'http://{}:9200'.format(hosts[0])} + body['source']['remote'] = {'host': f'http://{hosts[0]}:9200'} else: hosts = settings.ES_CONNECTIONS[settings.DESIGNSAFE_ENVIRONMENT]['hosts'] if not hosts: - raise CommandError('No valid remote hosts given. Remote host value given: {}'.\ - format(remote_host)) + raise CommandError(f'No valid remote hosts given. Remote host value given: {remote_host}') if doc_type: body['source']['type'] = doc_type - self.stdout.write('doc_type: %s' % doc_type) + self.stdout.write(f'doc_type: {doc_type}') es_local = elasticsearch.Elasticsearch( settings.ES_CONNECTIONS[settings.DESIGNSAFE_ENVIRONMENT]['hosts'], request_timeout=120) es_remote = elasticsearch.Elasticsearch(hosts, request_timeout=120) - self.stdout.write('local conn: %s' % settings.ES_CONNECTIONS[settings.DESIGNSAFE_ENVIRONMENT]['hosts']) - self.stdout.write('remote conn: %s' % hosts) + self.stdout.write('local conn: {}'.format(settings.ES_CONNECTIONS[settings.DESIGNSAFE_ENVIRONMENT]['hosts'])) + self.stdout.write(f'remote conn: {hosts}') remote_index = es_remote.indices.get(from_index) try: - local_index = es_local.indices.get(to_index) + es_local.indices.get(to_index) except TransportError as err: if err.status_code == 404: - self.stdout.write('Creating index: %s' % to_index) + self.stdout.write(f'Creating index: {to_index}') index_mapping = remote_index[from_index]['mappings'] index_settings = remote_index[from_index]['settings'] - self.stdout.write('Using Settings: %s' % json.dumps(index_settings)) + self.stdout.write(f'Using Settings: {json.dumps(index_settings)}') index_settings['index'].pop('uuid', None) index_settings['index'].pop('creation_date') index_settings['index'].pop('version') index_settings['index'].pop('provided_name', None) self.remove_fielddata(index_mapping) - self.stdout.write('Using mapping: %s' % json.dumps(index_mapping)) - local_index = es_local.indices.create( + self.stdout.write(f'Using mapping: {json.dumps(index_mapping)}') + es_local.indices.create( to_index, body={ "settings": index_settings, @@ -126,6 +124,6 @@ def handle(self, *args, **options): }) else: raise - self.stdout.write('body to use: %s' % body) + self.stdout.write(f'body to use: {body}') resp = es_local.reindex(body=body, request_timeout=options.get('timeout', 120)) self.stdout.write(json.dumps(resp)) diff --git a/designsafe/apps/data/management/commands/swap_reindex.py b/designsafe/apps/data/management/commands/swap_reindex.py index 438ea6740f..ebc289b65d 100644 --- a/designsafe/apps/data/management/commands/swap_reindex.py +++ b/designsafe/apps/data/management/commands/swap_reindex.py @@ -1,12 +1,11 @@ -import logging -from django.core.management import BaseCommand, CommandError -from django.conf import settings import elasticsearch -from elasticsearch import TransportError +from django.conf import settings +from django.core.management import BaseCommand from elasticsearch_dsl import Index -from elasticsearch_dsl.connections import connections + from designsafe.libs.elasticsearch.indices import setup_index + class Command(BaseCommand): """ This command reindexes all documents in the default files index in order to @@ -41,18 +40,18 @@ def handle(self, *args, **options): reindex_index_alias = default_index_alias + '-reindex' if not swap_only: - confirm = input('This will delete any documents in the index "{}" and recreate the index. Continue? (Y/n) '.format(reindex_index_alias)) + confirm = input(f'This will delete any documents in the index "{reindex_index_alias}" and recreate the index. Continue? (Y/n) ') if confirm != 'Y': raise SystemExit # Set up a fresh reindexing alias. setup_index(index_config, force=True, reindex=True) try: - default_index_name = list(Index(default_index_alias, using=es_client).get_alias().keys())[0] - reindex_index_name = list(Index(reindex_index_alias, using=es_client).get_alias().keys())[0] - except Exception as e: + default_index_name = next(iter(Index(default_index_alias, using=es_client).get_alias().keys())) + reindex_index_name = next(iter(Index(reindex_index_alias, using=es_client).get_alias().keys())) + except Exception as exc: self.stdout.write('Unable to lookup required indices by alias. Have you set up both a default and a reindexing index?') - raise SystemExit + raise SystemExit from exc if not swap_only: # Reindex file metadata from the default index to the reindexing index @@ -71,5 +70,5 @@ def handle(self, *args, **options): # Re-initialize the new reindexing index to save space. if cleanup: - reindex_index_name = list(Index(reindex_index_alias, using=es_client).get_alias().keys())[0] + reindex_index_name = next(iter(Index(reindex_index_alias, using=es_client).get_alias().keys())) Index(reindex_index_name, using=es_client).delete(ignore=404) diff --git a/designsafe/apps/data/management/commands/tests.py b/designsafe/apps/data/management/commands/tests.py index 69e37b5a05..7228d8e517 100644 --- a/designsafe/apps/data/management/commands/tests.py +++ b/designsafe/apps/data/management/commands/tests.py @@ -1,24 +1,22 @@ -from mock import Mock, patch, MagicMock, PropertyMock, call -from django.test import TestCase -from django.contrib.auth import get_user_model -from django.conf import settings +from unittest.mock import MagicMock, patch + from django.core.management import call_command -import datetime +from django.test import TestCase class TestSwapReindex(TestCase): def setUp(self): self.patch_setup = patch('designsafe.apps.data.management.commands.swap_reindex.setup_index') - self.patch_connections = patch('designsafe.apps.data.management.commands.swap_reindex.connections') + #self.patch_connections = patch('designsafe.apps.data.management.commands.swap_reindex.connections') self.patch_elasticsearch = patch('designsafe.apps.data.management.commands.swap_reindex.elasticsearch') self.mock_setup = self.patch_setup.start() - self.mock_connections = self.patch_connections.start() + #self.mock_connections = self.patch_connections.start() self.mock_elasticsearch = self.patch_elasticsearch.start() self.addCleanup(self.patch_setup.stop) - self.addCleanup(self.patch_connections.stop) + # self.addCleanup(self.patch_connections.stop) self.addCleanup(self.patch_elasticsearch.stop) @patch('designsafe.apps.data.management.commands.swap_reindex.Command.handle') diff --git a/designsafe/apps/data/models/agave/base.py b/designsafe/apps/data/models/agave/base.py index 0cbea5246e..4f4282c025 100644 --- a/designsafe/apps/data/models/agave/base.py +++ b/designsafe/apps/data/models/agave/base.py @@ -1,10 +1,12 @@ """ Base classes to handle agave metadata objects """ +import datetime import inspect -import six import json -import re import logging -import datetime +import re + +import six + from designsafe.apps.api import tasks from designsafe.apps.projects.models import Category @@ -13,7 +15,7 @@ REGISTRY = {} LAZY_OPS = [] -class RelatedQuery(object): +class RelatedQuery: def __init__(self, uuid=None, uuids=None, related_obj_name=None, rel_cls=None): self.uuid = uuid self.uuids = uuids or [] @@ -71,7 +73,7 @@ def serialize(self, value): return self.to_python(value) def register_lazy_rel(cls, field_name, related_obj_name, multiple, rel_cls): - reg_key = '{}.{}'.format(cls.model_name, cls.__name__) + reg_key = f'{cls.model_name}.{cls.__name__}' LAZY_OPS.append((reg_key, field_name, RelatedQuery(related_obj_name=related_obj_name, rel_cls=rel_cls)) @@ -86,7 +88,7 @@ def set_lazy_rels(): del LAZY_OPS[:] def register_class(cls, name, model_name): - registry_key = '{}.{}'.format(model_name, name) + registry_key = f'{model_name}.{name}' if REGISTRY.get(registry_key) is None: REGISTRY[registry_key] = cls @@ -102,17 +104,17 @@ def spinal_to_camelcase(string): else: first = comps[0] camel = ''.join(x.capitalize() or '_' for x in comps[1:]) - camel = ''.join([first, camel]) + camel = f'{first}{camel}' return camel -class Manager(object): +class Manager: def __init__(self, model_cls): self.model_cls = model_cls self.agave_client = None def set_client(self, agave_client): self.agave_client = agave_client - setattr(self.model_cls._meta, 'agave_client', agave_client) + self.model_cls._meta.agave_client = agave_client return self def get(self, agave_client, uuid=None, project_id=None): @@ -121,7 +123,7 @@ def get(self, agave_client, uuid=None, project_id=None): elif project_id is not None: metas = agave_client.meta.listMetadata( privileged=False, - q='{{"value.projectId": "{project_id}"}}'.format(project_id=project_id)) + q=f'{{"value.projectId": "{project_id}"}}') if len(metas): meta = metas[0] else: @@ -143,12 +145,12 @@ def list(self, agave_client, association_id=None): for meta in metas: yield self.model_cls(**meta) -class Links(object): +class Links: def __init__(self, values): for attrname, val in six.iteritems(values): setattr(self, attrname, val) -class Options(object): +class Options: """Options class to store model's _meta data """ _model = None @@ -184,7 +186,7 @@ class BaseModel(type): Metaclass for metadata models """ def __new__(cls, name, bases, attrs): - super_new = super(BaseModel, cls).__new__ + super_new = super().__new__ # Also ensure initialization is only performed for subclasses of Model # (excluding Model class itself). @@ -202,11 +204,11 @@ def __new__(cls, name, bases, attrs): model_name = attrs.get('model_name') new_class.add_to_class('_meta', Options(model_name)) if attrs.get('_is_nested', False): - setattr(new_class, 'model_name', None) + new_class.model_name = None else: - setattr(new_class, 'model_name', model_name) + new_class.model_name = model_name - setattr(new_class, '_is_nested', attrs.pop('_is_nested', False)) + new_class._is_nested = attrs.pop('_is_nested', False) for obj_name, obj in list(attrs.items()): new_class.add_to_class(obj_name, obj) @@ -223,7 +225,7 @@ def add_to_class(cls, name, value): def _prepare(cls): opts = cls._meta if cls.__doc__ is None: - cls.__doc__ = "%s(%s)" % (cls.__name__, ", ".join(f.name for f in opts._fields)) + cls.__doc__ = "{}({})".format(cls.__name__, ", ".join(f.name for f in opts._fields)) #if not opts.manager: # if any(f.name == 'objects' for f in opts.fields): @@ -235,13 +237,13 @@ def _prepare(cls): # manager.auto_created = True # cls.add_to_class('objects', manager) if not opts.model_manager: - setattr(cls._meta, 'model_manager', Manager(cls)) + cls._meta.model_manager = Manager(cls) if not isinstance(opts.model_manager, Manager): - raise ValueError("Model Manager must be a Manager class.") + raise ValueError("Model Manager must be a Manager class.") # noqa -class Model(object, metaclass=BaseModel): +class Model(metaclass=BaseModel): """Metadata model""" def __init__(self, **kwargs): @@ -259,13 +261,10 @@ def __init__(self, **kwargs): self.name = None self.parent = None #logger.debug('kwargs: %s', json.dumps(kwargs, indent=4)) - cls = self.__class__ - opts = self._meta #logger.debug('_is_nested: %s', self._is_nested) self.update(**kwargs) def update(self, **kwargs): - cls = self.__class__ opts = self._meta _setattr = setattr if self._is_nested: @@ -303,7 +302,7 @@ def update(self, **kwargs): if self.name is None: self.name = self._meta.model_name - super(Model, self).__init__() + super().__init__() def __getattribute__(self, name): opts = object.__getattribute__(self, '_meta') @@ -403,7 +402,6 @@ def to_dict(self): return dict_obj def to_body_dict(self): - from designsafe.apps.data.models.agave.fields import ListField dict_obj = {} if not self._is_nested: @@ -485,7 +483,7 @@ def associate(self, value): def manager(cls): return cls._meta.model_manager -class BaseAgaveResource(object): +class BaseAgaveResource: """ Base Class that all Agave API Resource objects inherit from. """ @@ -509,7 +507,7 @@ def __getattr__(self, name): if camel_name in self._wrapped: return self._wrapped.get(camel_name) - raise AttributeError('\'{0}\' has no attribute \'{1}\''.format(self.__class__.__name__, name)) + raise AttributeError(f'\'{self.__class__.__name__}\' has no attribute \'{name}\'') def __setattr__(self, name, value): if name != '_wrapped' and name != '_agave': @@ -518,4 +516,4 @@ def __setattr__(self, name, value): self._wrapped[camel_name] = value return - super(BaseAgaveResource, self).__setattr__(name, value) \ No newline at end of file + super().__setattr__(name, value) \ No newline at end of file diff --git a/designsafe/apps/data/models/agave/fields.py b/designsafe/apps/data/models/agave/fields.py index dbebf54547..dc668d1d75 100644 --- a/designsafe/apps/data/models/agave/fields.py +++ b/designsafe/apps/data/models/agave/fields.py @@ -1,19 +1,19 @@ """ Base field classes """ -import six -import datetime -import dateutil.parser import collections +import datetime +import logging from decimal import Decimal -from designsafe.apps.data.models.agave.base import register_lazy_rel -import logging +import dateutil.parser + +from designsafe.apps.data.models.agave.base import register_lazy_rel logger = logging.getLogger(__name__) -class NOT_PROVIDED(object): +class NOT_PROVIDED: pass -class BaseField(object): +class BaseField: """ Base field class """ related_model = None @@ -64,7 +64,7 @@ def get_default(self): if not isinstance(self.default, NOT_PROVIDED): return self.default else: - raise ValueError('No default set') + raise ValueError('No default set') # noqa def to_python(self, value): return value @@ -78,7 +78,7 @@ def clean(self, value): class CharField(BaseField): """ Char Field """ def __init__(self, *args, **kwargs): - super(CharField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def to_python(self, value): return str(value) @@ -90,12 +90,12 @@ class UuidField(CharField): """ Uuid Field """ def __init__(self, *args, **kwargs): kwargs['schema_field'] = True - super(UuidField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class DateTimeField(BaseField): """ Date Time Field """ def __init__(self, *args, **kwargs): - super(DateTimeField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def to_python(self, value): if not isinstance(value, datetime): @@ -110,7 +110,7 @@ def serialize(self, value): class IntField(BaseField): """ Int Field """ def __init__(self, *args, **kwargs): - super(IntField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def to_python(self, value): return int(value) @@ -121,7 +121,7 @@ def serialize(self, value): class DecimalField(BaseField): """ Decimal Field """ def __init__(self, *args, **kwargs): - super(DecimalField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def to_python(self, value): return Decimal(value) @@ -136,7 +136,7 @@ def __init__(self, verbose_name, list_cls=None, *args, **kwargs): kwargs['default'] = kwargs.get('default', []) kwargs['list_cls'] = list_cls kwargs['verbose_name'] = verbose_name - super(ListField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def to_python(self, value): try: @@ -183,7 +183,7 @@ class NestedObjectField(BaseField): def __init__(self, nested_cls, *args, **kwargs): kwargs['nested_cls'] = nested_cls kwargs['default'] = kwargs.get('default', {}) - super(NestedObjectField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def serialize(self, value): return value.to_body_dict() @@ -193,11 +193,11 @@ class RelatedObjectField(BaseField): def __init__(self, related, multiple=False, related_name=None, *args, **kwargs): kwargs['related_name'] = related_name kwargs['related'] = related - super(RelatedObjectField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.multiple = multiple def contribute_to_class(self, cls, name): """ Register the field with the model class it belongs to. """ - super(RelatedObjectField, self).contribute_to_class(cls, name) - related_name = self.related_name or '%s_set' % cls.__name__.lower() + super().contribute_to_class(cls, name) + related_name = self.related_name or f'{cls.__name__.lower()}_set' register_lazy_rel(self.related, related_name, cls.model_name, self.multiple, cls) diff --git a/designsafe/apps/data/models/agave/files.py b/designsafe/apps/data/models/agave/files.py index d6c6abde9a..b08f36cc0e 100644 --- a/designsafe/apps/data/models/agave/files.py +++ b/designsafe/apps/data/models/agave/files.py @@ -1,17 +1,23 @@ import json import logging -import re import os -import six -import urllib.request, urllib.parse, urllib.error +import re +import urllib.error import urllib.parse +import urllib.request + +import six from requests.exceptions import HTTPError -from designsafe.apps.data.models.agave.base import BaseAgaveResource -from designsafe.apps.data.models.agave.metadata import BaseMetadataResource, BaseMetadataPermissionResource -from designsafe.apps.data.models.agave.systems import roles as system_roles_list + # from agavepy.agave import AgaveException # from agavepy.async import AgaveAsyncResponse, TimeoutError, Error from designsafe.apps.api import tasks +from designsafe.apps.data.models.agave.base import BaseAgaveResource +from designsafe.apps.data.models.agave.metadata import ( + BaseMetadataPermissionResource, + BaseMetadataResource, +) +from designsafe.apps.data.models.agave.systems import roles as system_roles_list logger = logging.getLogger(__name__) @@ -24,7 +30,7 @@ class BaseFileMetadata(BaseMetadataResource): def __init__(self, agave_client, file_obj=None, **kwargs): meta_objs = [] if file_obj: - query = '{{"associationIds": "{}", "name": "designsafe.file"}}'.format(file_obj.uuid) + query = f'{{"associationIds": "{file_obj.uuid}", "name": "designsafe.file"}}' meta_objs = agave_client.meta.listMetadata(q=query) logger.info(meta_objs) if meta_objs: @@ -43,7 +49,7 @@ def __init__(self, agave_client, file_obj=None, **kwargs): defaults['associationIds'].append(project_uuid) else: defaults = kwargs - super(BaseFileMetadata, self).__init__(agave_client, **defaults) + super().__init__(agave_client, **defaults) @classmethod def search(cls, agave_client, q): @@ -104,7 +110,7 @@ def match_pems_to_project(self, project_uuid = None): if not project_uuid: return self - project_roles = self._agave.systems.listRoles(systemId='project-{}'.format(project_uuid)) + project_roles = self._agave.systems.listRoles(systemId=f'project-{project_uuid}') project_roles = [x for x in project_roles if x['username'] != 'ds_admin'] meta_pems = BaseMetadataPermissionResource.list_permissions(self.uuid, self._agave) meta_pems_users = self._update_pems_with_system_roles(project_roles, meta_pems) @@ -115,12 +121,12 @@ def match_pems_to_project(self, project_uuid = None): def save(self): if self.uuid is None: - super(BaseFileMetadata, self).save() + super().save() #self.match_pems_to_project() if self.value.get('projectUUID'): tasks.check_project_meta_pems.apply_async(args=[self.uuid], queue='api') else: - super(BaseFileMetadata, self).save() + super().save() class BaseFileResource(BaseAgaveResource): """Represents an Agave Files API Resource""" @@ -170,7 +176,7 @@ class BaseFileResource(BaseAgaveResource): SUPPORTED_MS_OFFICE) def __init__(self, agave_client, system, path, **kwargs): - super(BaseFileResource, self).__init__(agave_client, system=system, path=path, + super().__init__(agave_client, system=system, path=path, **kwargs) self._children = None self._metadata = None @@ -184,11 +190,11 @@ def __repr__(self): @property def agave_uri(self): - return 'agave://{}/{}'.format(self.system, self.path) + return f'agave://{self.system}/{self.path}' @property def id(self): - return '/'.join([self.system, self.path]) + return f'{self.system}/{self.path}' @property def children(self): @@ -239,7 +245,7 @@ def trail(self): trail_comps = [{'name': path_comps[i] or '/', 'system': self.system, 'path': '/'.join(path_comps[0:i+1]) or '/', - } for i in range(0, len(path_comps))] + } for i in range(len(path_comps))] return trail_comps @trail.setter @@ -254,6 +260,7 @@ def metadata(self): return self._metadata except Exception as exc: + logger.exception("") logger.debug('Couldn\'t get metadata %s', exc) @property @@ -277,10 +284,11 @@ def uuid(self): return meta_q.get('associationIds') return None except Exception as exc: + logger.exception("") logger.debug('Couldn\'t get uuid %s', exc) def to_dict(self): - ser = super(BaseFileResource, self).to_dict() + ser = super().to_dict() if self._children is not None: ser['children'] = [c.to_dict() for c in self._children] @@ -319,7 +327,7 @@ def copy(self, dest_path, file_name=None): if file_name is None: file_name = self.name - body = {'action': 'copy', 'path': '/'.join([dest_path, file_name])} + body = {'action': 'copy', 'path': f'{dest_path}/{file_name}'} copy_result = self._agave.files.manage(systemId=self.system, filePath=urllib.parse.quote(self.path), body=body) @@ -475,7 +483,7 @@ def move(self, dest_path, file_name=None): """ if file_name is None: file_name = self.name - body = {'action': 'move', 'path': '/'.join([dest_path, file_name])} + body = {'action': 'move', 'path': f'{dest_path}/{file_name}'} move_result = self._agave.files.manage(systemId=self.system, filePath=urllib.parse.quote(self.path), body=body) @@ -506,8 +514,7 @@ def share(self, username, permission, recursive=False): permission_body = {'username': username, 'permission': permission, 'recursive': recursive} - logger.info('Updating file permissions on {}: {}'.format(self.agave_uri, - permission_body)) + logger.info('Updating file permissions on %s: %s', self.agave_uri, permission_body) self._agave.files.updatePermissions( systemId=self.system, filePath=urllib.parse.quote(self.path), @@ -555,7 +562,7 @@ def upload(self, upload_file): class BaseAgaveFileHistoryRecord(BaseAgaveResource): def __init__(self, agave_client, **kwargs): - super(BaseAgaveFileHistoryRecord, self).__init__(agave_client) + super().__init__(agave_client) self.status = None self.description = None self.createdBy = None @@ -575,7 +582,7 @@ def __str__(self): self.description) def __repr__(self): - return ''.format(str(self)) + return f'' class BaseFilePermissionResource(BaseAgaveResource): @@ -601,7 +608,7 @@ def __init__(self, agave_client, agave_file, **kwargs): 'username': None } defaults.update(**kwargs) - super(BaseFilePermissionResource, self).__init__(agave_client, **defaults) + super().__init__(agave_client, **defaults) self.agave_file = agave_file diff --git a/designsafe/apps/data/models/agave/metadata.py b/designsafe/apps/data/models/agave/metadata.py index f3451531d1..75e4adccdb 100644 --- a/designsafe/apps/data/models/agave/metadata.py +++ b/designsafe/apps/data/models/agave/metadata.py @@ -1,5 +1,6 @@ import json import logging + from designsafe.apps.data.models.agave.base import BaseAgaveResource logger = logging.getLogger(__name__) @@ -20,7 +21,7 @@ def __init__(self, agave_client, **kwargs): 'value': {} } defaults.update(kwargs) - super(BaseMetadataResource, self).__init__(agave_client, **defaults) + super().__init__(agave_client, **defaults) @property def request_body(self): @@ -51,18 +52,17 @@ def save(self): :rtype: :class:`BaseMetadataResource` """ if self.uuid is None: - logger.info('Saving "{}" metadata: {}'.format(self.name, self.request_body)) + logger.info('Saving "%s" metadata: %s', self.name, self.request_body) result = self._agave.meta.addMetadata(body=self.request_body) else: - logger.info('Updating "{}" metadata {}: {}'.format(self.name, self.uuid, - self.request_body)) + logger.info('Updating "%s" metadata %s: %s', self.name, self.uuid, self.request_body) result = self._agave.meta.updateMetadata(uuid=self.uuid, body=self.request_body) self._wrapped.update(**result) return self def delete(self): - logger.info('Deleting "{}" metadata {}'.format(self.name, self.uuid)) + logger.info('Deleting "%s" metadata %s', self.name, self.uuid) self._agave.meta.deleteMetadata(uuid=self.uuid) return self @@ -84,7 +84,7 @@ def __init__(self, metadata_uuid, agave_client, **kwargs): 'username': None } defaults.update(**kwargs) - super(BaseMetadataPermissionResource, self).__init__(agave_client, **defaults) + super().__init__(agave_client, **defaults) self.metadata_uuid = metadata_uuid @property @@ -123,10 +123,7 @@ def permission_bit(self, value): elif value == 'WRITE': self.read = False self.write = True - elif value == 'READ_WRITE': - self.read = True - self.write = True - elif value == 'ALL': + elif value == 'READ_WRITE' or value == 'ALL': self.read = True self.write = True elif value == 'NONE': @@ -147,8 +144,7 @@ def save(self): :return: self :rtype: :class:`BaseMetadataPermissionResource` """ - logger.info('Updating metadata permissions: {} {}'.format(self.metadata_uuid, - self.request_body)) + logger.info('Updating metadata permissions: %s %s', self.metadata_uuid, self.request_body) result = self._agave.meta.updateMetadataPermissions(uuid=self.metadata_uuid, body=self.request_body) self._wrapped.update(**result) diff --git a/designsafe/apps/data/models/agave/systems/__init__.py b/designsafe/apps/data/models/agave/systems/__init__.py index d4f4ac936c..119b778ea3 100644 --- a/designsafe/apps/data/models/agave/systems/__init__.py +++ b/designsafe/apps/data/models/agave/systems/__init__.py @@ -1 +1 @@ -from . systems import BaseSystemResource +from .systems import BaseSystemResource as BaseSystemResource diff --git a/designsafe/apps/data/models/agave/systems/systems.py b/designsafe/apps/data/models/agave/systems/systems.py index b7fd2f207c..6c187bc452 100644 --- a/designsafe/apps/data/models/agave/systems/systems.py +++ b/designsafe/apps/data/models/agave/systems/systems.py @@ -1,6 +1,6 @@ import logging + from designsafe.apps.data.models.agave.base import BaseAgaveResource -from . import roles logger = logging.getLogger(__name__) @@ -8,7 +8,7 @@ class BaseSystemResource(BaseAgaveResource): def __init__(self, agave_client, **kwargs): - super(BaseSystemResource, self).__init__(agave_client, **kwargs) + super().__init__(agave_client, **kwargs) @property def request_body(self): @@ -23,12 +23,12 @@ def add_role(self, username, role): 'username': username, 'role': role } - logger.info('Granting system role on {}: {}'.format(self.id, role_body)) + logger.info('Granting system role on %s: %s', self.id, role_body) self._agave.systems.updateRole(systemId=self.id, body=role_body) return self def remove_role(self, username): - logger.info('Revoking system roles on {} for user {}'.format(self.id, username)) + logger.info('Revoking system roles on %s for user %s', self.id, username) self._agave.systems.deleteRoleForUser(systemId=self.id, username=username) return self diff --git a/designsafe/apps/data/models/agave/util.py b/designsafe/apps/data/models/agave/util.py index 629c9bd373..8b2b8f905a 100644 --- a/designsafe/apps/data/models/agave/util.py +++ b/designsafe/apps/data/models/agave/util.py @@ -1,8 +1,9 @@ """ Agave models util module.""" import logging + from django.core.serializers.json import DjangoJSONEncoder -from designsafe.apps.data.models.agave.base import BaseAgaveResource -from designsafe.apps.data.models.agave.base import BaseModel + +from designsafe.apps.data.models.agave.base import BaseAgaveResource, BaseModel logger = logging.getLogger(__name__) @@ -24,7 +25,7 @@ class AgaveJSONEncoder(DjangoJSONEncoder): def default(self, o): """Serializer function""" - if isinstance(o, BaseAgaveResource) or isinstance(o, BaseModel): + if isinstance(o, (BaseAgaveResource, BaseModel)): return o.to_dict() - return super(AgaveJSONEncoder, self).default(o) + return super().default(o) diff --git a/designsafe/apps/data/models/elasticsearch.py b/designsafe/apps/data/models/elasticsearch.py index c9eec0af5b..eca5067de2 100644 --- a/designsafe/apps/data/models/elasticsearch.py +++ b/designsafe/apps/data/models/elasticsearch.py @@ -1,17 +1,27 @@ -import logging -import json import datetime +import logging + from django.conf import settings -from django.db import models -from elasticsearch_dsl.connections import connections -from elasticsearch_dsl import (Search, Document, Date, Nested, - analyzer, Object, Text, Long, - Boolean, Keyword, - GeoPoint, MetaField, Index) +from elasticsearch_dsl import ( + Boolean, + Date, + Document, + Keyword, + Long, + MetaField, + Nested, + Object, + Text, +) from elasticsearch_dsl.query import Q -from elasticsearch import TransportError, ConnectionTimeout -from designsafe.libs.elasticsearch.analyzers import path_analyzer, file_analyzer, file_pattern_analyzer, reverse_file_analyzer + +from designsafe.libs.elasticsearch.analyzers import ( + file_analyzer, + file_pattern_analyzer, + path_analyzer, + reverse_file_analyzer, +) from designsafe.libs.elasticsearch.exceptions import DocumentNotFound from designsafe.libs.elasticsearch.utils import file_uuid_sha256 @@ -66,15 +76,15 @@ def save(self, *args, **kwargs): """ Sets `lastUpdated` attribute on save. Otherwise see elasticsearch_dsl.Document.save() """ - self.lastUpdated = datetime.datetime.now() - return super(IndexedFile, self).save(*args, **kwargs) + self.lastUpdated = datetime.datetime.now(tz=datetime.UTC) + return super().save(*args, **kwargs) def update(self, *args, **kwargs): """ Sets `lastUpdated` attribute on save. Otherwise see elasticsearch_dsl.Document.update() """ - lastUpdated = datetime.datetime.now() - return super(IndexedFile, self).update(lastUpdated=lastUpdated, *args, **kwargs) + lastUpdated = datetime.datetime.now(tz=datetime.UTC) + return super().update(*args, **kwargs, lastUpdated=lastUpdated) @classmethod def from_path(cls, system, path): @@ -361,19 +371,20 @@ def from_id(cls, project_id, revision=None, using='default'): raise DocumentNotFound() if revision: - revision_filter = Q('match', **{'revision': revision}) + revision_filter = Q('match', revision=revision) else: # Search for documents where revision is not specified. - revision_filter = ~Q('exists', **{'field': 'revision'}) + revision_filter = ~Q('exists', field='revision') id_filter = Q('term', **{'projectId._exact': project_id}) search = cls.search(using=using).filter(id_filter & revision_filter) try: res = search.execute() - except Exception as e: - raise e + except Exception: + logger.exception("") + raise if res.hits.total.value > 1: - id_filter = Q('term', **{'_id': res[0].meta.id}) + id_filter = Q('term', _id=res[0].meta.id) # Delete all files indexed with the same system/path, except the first result delete_query = id_filter & ~id_filter cls.search(using=using).filter(delete_query).delete() @@ -382,7 +393,7 @@ def from_id(cls, project_id, revision=None, using='default'): return cls.get(res[0].meta.id, using=using) else: raise DocumentNotFound("No document found for " - "{}".format(project_id)) + f"{project_id}") @classmethod def max_revision(cls, project_id, using='default'): @@ -509,10 +520,11 @@ def from_id(cls, project_id): search = cls.search().filter(id_filter) try: res = search.execute() - except Exception as e: - raise e + except Exception: + logger.exception("") + raise if res.hits.total.value > 1: - id_filter = Q('term', **{'_id': res[0].meta.id}) + id_filter = Q('term', _id=res[0].meta.id) # Delete all files indexed with the same system/path, except the first result delete_query = id_filter & ~id_filter cls.search().filter(delete_query).delete() @@ -521,7 +533,7 @@ def from_id(cls, project_id): return cls.get(res[0].meta.id) else: raise DocumentNotFound("No document found for " - "{}".format(project_id)) + f"{project_id}") class Index: name = settings.ES_INDICES['publications_legacy']['alias'] diff --git a/designsafe/apps/data/tasks.py b/designsafe/apps/data/tasks.py index cbeeb66bc9..bbff69a2d9 100644 --- a/designsafe/apps/data/tasks.py +++ b/designsafe/apps/data/tasks.py @@ -1,8 +1,9 @@ import logging + from celery import shared_task -from django.conf import settings -from designsafe.apps.api.agave import get_tg458981_client, get_service_account_client -from designsafe.libs.elasticsearch.utils import index_level, walk_levels, index_listing + +from designsafe.apps.api.agave import get_service_account_client, get_tg458981_client +from designsafe.libs.elasticsearch.utils import index_level, index_listing, walk_levels logger = logging.getLogger(__name__) @@ -23,7 +24,7 @@ def agave_indexer(self, systemId, filePath='/', recurse=True, update_pems=False, filePath, folders, files = walk_levels(client, systemId, filePath, ignore_hidden=ignore_hidden).__next__() index_level(filePath, folders, files, systemId, reindex=reindex) except Exception as exc: - logger.debug(exc) + logger.exception("") raise self.retry(exc=exc) if recurse: diff --git a/designsafe/apps/data/urls.py b/designsafe/apps/data/urls.py index aee6640a82..0df0ad5ba5 100644 --- a/designsafe/apps/data/urls.py +++ b/designsafe/apps/data/urls.py @@ -1,12 +1,13 @@ from django.urls import re_path as url +from django.urls import reverse +from django.utils.translation import gettext_lazy as _ + from designsafe.apps.data.views.base import ( + DataDepotLegacyPublishedView, + DataDepotPublishedView, DataDepotView, FileMediaView, - DataDepotPublishedView, - DataDepotLegacyPublishedView ) -from django.urls import reverse -from django.utils.translation import gettext_lazy as _ urlpatterns = [ url( diff --git a/designsafe/apps/data/views/base.py b/designsafe/apps/data/views/base.py index eebc357810..2c8c487f01 100644 --- a/designsafe/apps/data/views/base.py +++ b/designsafe/apps/data/views/base.py @@ -1,26 +1,23 @@ import json import logging + import networkx as nx -from designsafe.apps.api.exceptions import ApiException -from designsafe.apps.notifications.views import get_number_unread_notifications from django.conf import settings from django.core.exceptions import PermissionDenied -from django.urls import reverse from django.http import Http404, HttpResponse from django.shortcuts import resolve_url +from django.urls import reverse from django.utils.decorators import method_decorator -from django.views.generic.base import TemplateView, View from django.views.decorators.csrf import ensure_csrf_cookie -from designsafe.libs.common.decorators import profile - -from designsafe.libs.elasticsearch.docs.publications import BaseESPublication -from designsafe.libs.elasticsearch.docs.publication_legacy import BaseESPublicationLegacy +from django.views.generic.base import TemplateView, View -from designsafe.apps.projects.managers.base import ProjectsManager -from designsafe.apps.api.agave import service_account +from designsafe.apps.api.exceptions import ApiException from designsafe.apps.api.publications_v2.models import Publication -from designsafe.apps.api.projects_v2.operations.datacite_operations import get_datacite_json -import json +from designsafe.apps.notifications.views import get_number_unread_notifications +from designsafe.libs.common.decorators import profile +from designsafe.libs.elasticsearch.docs.publication_legacy import ( + BaseESPublicationLegacy, +) logger = logging.getLogger(__name__) @@ -131,7 +128,7 @@ def get_google_scholar_context(project_id): class BasePublicTemplate(TemplateView): def get_context_data(self, **kwargs): - context = super(BasePublicTemplate, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) context['unreadNotifications'] = 0 return context @@ -153,12 +150,12 @@ def login_redirect(request): @method_decorator(ensure_csrf_cookie) def dispatch(self, request, *args, **kwargs): try: - return super(DataDepotView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) except PermissionDenied: return DataDepotView.login_redirect(request) def get_context_data(self, **kwargs): - context = super(DataDepotView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) logger.info('Get context Data') if self.request.user.is_authenticated: @@ -189,7 +186,7 @@ def dispatch(self, request, *args, **kwargs): return self.login_rediect(request) def get_context_data(self, **kwargs): - context = super(DataBrowserTestView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) logger.info('Get context Data') context['unreadNotifications'] = get_number_unread_notifications(self.request) @@ -201,7 +198,7 @@ def get_context_data(self, **kwargs): else: resource = 'public' - fm_cls = lookup_file_manager(resource) + fm_cls = lookup_file_manager(resource) # noqa if fm_cls is None: raise Http404('Unknown resource') @@ -234,7 +231,7 @@ def get_context_data(self, **kwargs): }, } - sources_api = SourcesApi() + sources_api = SourcesApi() # noqa source_id = resource if source_id == 'agave': if fm is not None and fm.is_shared(file_path): @@ -267,7 +264,7 @@ def get_system_dirname(self, system_id): dirname = self.systems_mappings.get(system_id) if dirname is None and system_id.startswith('project-'): prjuuid = system_id.replace('project-', '') - dirname = 'projects/{prjuuid}'.format(prjuuid=prjuuid) + dirname = f'projects/{prjuuid}' return dirname @@ -277,12 +274,10 @@ def get(self, request, file_mgr_name, system_id, file_path): raise Http404('Resource not Found') filename = file_path.rsplit('/', 1)[1] - filepath = '{corral}/{sys_dirname}/{file_path}'.format( - corral=self.corral, sys_dirname=self.get_system_dirname(system_id), - file_path=file_path) + filepath = f'{self.corral}/{self.get_system_dirname(system_id)}/{file_path}' response = HttpResponse() - response['Content-Disposition'] = 'attachment; filename={filename}'.format(filename=filename) - response['X-Accel-Redirect'] = '/internal-resource/{filepath}'.format(filepath=filepath) + response['Content-Disposition'] = f'attachment; filename={filename}' + response['X-Accel-Redirect'] = f'/internal-resource/{filepath}' return response @@ -295,13 +290,13 @@ class DataDepotPublishedView(TemplateView): @method_decorator(ensure_csrf_cookie) def dispatch(self, request, *args, **kwargs): - return super(DataDepotPublishedView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) def get_context_data(self, **kwargs): """ Update context data to add publication. """ - context = super(DataDepotPublishedView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) try: scholar_context, datacite_context, title = get_google_scholar_context(kwargs['project_id']) context['dc_context'] = [json.dumps(ctx) for ctx in datacite_context] @@ -309,7 +304,7 @@ def get_context_data(self, **kwargs): context['citation_title'] = f"{kwargs['project_id']} | {title}" except Exception: # If we can't generate DataCite JSON, render the page without meta tags. - pass + logger.exception("") if self.request.user.is_authenticated: context['angular_init'] = json.dumps({ @@ -331,11 +326,11 @@ class DataDepotLegacyPublishedView(TemplateView): @method_decorator(ensure_csrf_cookie) def dispatch(self, request, *args, **kwargs): - return super(DataDepotLegacyPublishedView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) def get_context_data(self, **kwargs): """Update context data to add publication.""" - context = super(DataDepotLegacyPublishedView, self).get_context_data(**kwargs) + context = super().get_context_data(**kwargs) logger.info('Get context Data') nees_id = kwargs['project_id'].strip('.groups').strip('/') logger.debug('nees_id: %s', nees_id) @@ -344,7 +339,7 @@ def get_context_data(self, **kwargs): context['neesId'] = nees_id.split('/')[0] context['citation_title'] = pub.title context['citation_date'] = getattr(pub, 'startDate', '') - experiments = getattr(pub, 'experiments') + experiments = pub.experiments if experiments and len(experiments): context['doi'] = getattr(pub.experiments[0], 'doi', '') exp_users = [getattr(exp, 'creators', []) for exp in experiments] diff --git a/designsafe/apps/data/views/mixins.py b/designsafe/apps/data/views/mixins.py index 6cd5322c0d..c438b40fcd 100644 --- a/designsafe/apps/data/views/mixins.py +++ b/designsafe/apps/data/views/mixins.py @@ -1,21 +1,15 @@ -from django.views.generic.base import View as BaseView -from django.utils.decorators import method_decorator -from django.contrib.auth.decorators import login_required -from django.core.serializers.json import DjangoJSONEncoder -from django.http import HttpResponse, StreamingHttpResponse -# from agavepy.agave import Agave, AgaveException -from django.contrib.auth import get_user_model -from django.conf import settings -import datetime -import copy import json import logging -import requests -from requests.exceptions import ConnectionError, HTTPError + +# from agavepy.agave import Agave, AgaveException +from django.contrib.auth.decorators import login_required +from django.core.serializers.json import DjangoJSONEncoder +from django.http import HttpResponse +from django.utils.decorators import method_decorator logger = logging.getLogger(__name__) -class JSONResponseMixin(object): +class JSONResponseMixin: """ View mixin to return a JSON response. We're building one so we can put any extra code in here. @@ -26,10 +20,10 @@ def render_to_json_response(self, context, **response_kwargs): response_kwargs['content_type'] = 'application/json' return HttpResponse(json.dumps(context, cls=DjangoJSONEncoder), **response_kwargs) -class SecureMixin(object): +class SecureMixin: """ View mixin to use login_required """ @method_decorator(login_required) def dispatch(self, request, *args, **kwargs): - return super(SecureMixin, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) diff --git a/designsafe/apps/djangoRT/__init__.py b/designsafe/apps/djangoRT/__init__.py old mode 100755 new mode 100644 diff --git a/designsafe/apps/djangoRT/admin.py b/designsafe/apps/djangoRT/admin.py old mode 100755 new mode 100644 index 8c38f3f3da..b97a94f645 --- a/designsafe/apps/djangoRT/admin.py +++ b/designsafe/apps/djangoRT/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/djangoRT/forms.py b/designsafe/apps/djangoRT/forms.py old mode 100755 new mode 100644 index 51b5e0aee3..d48dec176b --- a/designsafe/apps/djangoRT/forms.py +++ b/designsafe/apps/djangoRT/forms.py @@ -22,7 +22,7 @@ def validate(self, value): """ Check if value consists only of valid emails. """ # Use the parent's handling of required fields, etc. - super(MultiEmailField, self).validate(value) + super().validate(value) for email in value: validate_email(email.strip()) @@ -60,7 +60,6 @@ class TicketGuestForm(BaseTicketForm): Anonymous users ticket form. Adds a CAPTCHA to reduce spam submissions. """ captcha = ReCaptchaField(widget=ReCaptchaWidget) - pass class ReplyForm(forms.Form): """ diff --git a/designsafe/apps/djangoRT/models.py b/designsafe/apps/djangoRT/models.py old mode 100755 new mode 100644 index 137941ffae..e69de29bb2 --- a/designsafe/apps/djangoRT/models.py +++ b/designsafe/apps/djangoRT/models.py @@ -1 +0,0 @@ -from django.db import models diff --git a/designsafe/apps/djangoRT/rtModels.py b/designsafe/apps/djangoRT/rtModels.py old mode 100755 new mode 100644 index 23ea853078..fc4ae20b0e --- a/designsafe/apps/djangoRT/rtModels.py +++ b/designsafe/apps/djangoRT/rtModels.py @@ -1,4 +1,4 @@ - +# ruff:disable[N999] class Ticket: diff --git a/designsafe/apps/djangoRT/rtUtil.py b/designsafe/apps/djangoRT/rtUtil.py old mode 100755 new mode 100644 index bb9a7e284c..4540441e12 --- a/designsafe/apps/djangoRT/rtUtil.py +++ b/designsafe/apps/djangoRT/rtUtil.py @@ -1,8 +1,10 @@ -import rt +# ruff: disable [N999] +import logging +from datetime import datetime + import pytz +import rt from django.conf import settings -from datetime import datetime -import logging logger = logging.getLogger(__name__) @@ -26,9 +28,9 @@ def __init__(self, queue=settings.DJANGO_RT['RT_QUEUE']): def getUserTickets(self, userEmail, show_resolved=False): if show_resolved: - query = 'Requestor="%s"' % userEmail + query = f'Requestor="{userEmail}"' else: - query = 'Requestor="%s" AND Status!="resolved" AND Status!="closed"' % userEmail + query = f'Requestor="{userEmail}" AND Status!="resolved" AND Status!="closed"' ticket_list = self.tracker.search( Queue=rt.ALL_QUEUES, @@ -39,7 +41,7 @@ def getUserTickets(self, userEmail, show_resolved=False): for ticket in ticket_list: ticket['id'] = ticket['id'].replace('ticket/', '') ticket['LastUpdated'] = datetime.strptime(ticket['LastUpdated'], - '%a %b %d %X %Y',) + '%a %b %d %X %Y',).astimezone() return ticket_list @@ -48,7 +50,7 @@ def getTicket(self, ticket_id): ticket['id'] = ticket['id'].replace('ticket/', '') ticket['LastUpdated'] = datetime.strptime(ticket['LastUpdated'], - '%a %b %d %X %Y',) + '%a %b %d %X %Y',).astimezone() return ticket @@ -57,7 +59,7 @@ def getTicketHistory(self, ticket_id): local_tz = pytz.timezone('US/Central') for ticket in ticketHistory: - ticket['Created'] = datetime.strptime(ticket['Created'], '%Y-%m-%d %X') + ticket['Created'] = datetime.strptime(ticket['Created'], '%Y-%m-%d %X').astimezone() ticket['Created'] = ticket['Created'].replace(tzinfo=pytz.UTC) ticket['Created'] = ticket['Created'].astimezone(local_tz) @@ -73,7 +75,9 @@ def createTicket(self, ticket): Subject=ticket.subject, Requestor=ticket.requestor, Cc=",".join(ticket.cc), Text=ticket.problem_description.replace('\n', '\n ')) - def replyToTicket(self, ticket_id, text='', files=[]): + def replyToTicket(self, ticket_id, text='', files=None): + if files is None: + files = [] return self.tracker.reply(ticket_id, text=text, files=files) def commentOnTicket(self, ticket_id, text=''): diff --git a/designsafe/apps/djangoRT/urls.py b/designsafe/apps/djangoRT/urls.py old mode 100755 new mode 100644 diff --git a/designsafe/apps/djangoRT/views.py b/designsafe/apps/djangoRT/views.py old mode 100755 new mode 100644 index afc0064bae..7642324ab8 --- a/designsafe/apps/djangoRT/views.py +++ b/designsafe/apps/djangoRT/views.py @@ -1,18 +1,24 @@ -from django.shortcuts import render -from django.http import HttpResponse, HttpResponseRedirect, JsonResponse, HttpResponseBadRequest -from django.core.exceptions import PermissionDenied -from django.urls import reverse -from designsafe.apps.djangoRT import rtUtil, forms, rtModels -from django.contrib.auth.decorators import login_required +import json +import logging + +import requests +from django.conf import settings from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.core.exceptions import PermissionDenied from django.core.files.base import ContentFile -from django.conf import settings +from django.http import ( + HttpResponse, + HttpResponseBadRequest, + HttpResponseRedirect, + JsonResponse, +) +from django.shortcuts import render +from django.urls import reverse + from designsafe.apps.api.exceptions import ApiException from designsafe.apps.api.views import BaseApiView -import logging -import mimetypes -import json -import requests +from designsafe.apps.djangoRT import forms, rtModels, rtUtil logger = logging.getLogger(__name__) @@ -72,7 +78,7 @@ def ticketcreate(request): form = form_cls(request.POST, request.FILES) if form.is_valid(): - requestor_meta = '%s %s <%s>' % ( + requestor_meta = '{} {} <{}>'.format( form.cleaned_data['first_name'], form.cleaned_data['last_name'], request.user.email @@ -89,8 +95,8 @@ def ticketcreate(request): ('HTTP Referer', form.cleaned_data['http_referer']), ) - header = '\n'.join('[%s] %s' % m for m in meta) - ticket_body = '%s\n\n%s\n\n---\n%s' % ( + header = '\n'.join('[{}] {}'.format(*m) for m in meta) + ticket_body = '{}\n\n{}\n\n---\n{}'.format( header, form.cleaned_data['problem_description'], requestor_meta @@ -101,7 +107,7 @@ def ticketcreate(request): requestor=request.user.email, cc=form.cleaned_data.get('cc', '')) - logger.debug('Creating ticket for user: %s' % form.cleaned_data) + logger.debug('Creating ticket for user: %s', form.cleaned_data) rt = rtUtil.DjangoRt() ticket_id = rt.createTicket(ticket) @@ -192,11 +198,14 @@ def ticketclose(request, ticketId): if request.method == 'POST': form = forms.CloseForm(request.POST) - if form.is_valid(): - if (rt.commentOnTicket(ticketId, text=form.cleaned_data['reply']) and - rt.closeTicket(ticketId)): - return HttpResponseRedirect(reverse('djangoRT:ticketdetail', - args=[ticketId])) + if ( + form.is_valid() + and rt.commentOnTicket(ticketId, text=form.cleaned_data["reply"]) + and rt.closeTicket(ticketId) + ): + return HttpResponseRedirect( + reverse("djangoRT:ticketdetail", args=[ticketId]) + ) else: form = forms.CloseForm(initial=data) return render(request, 'djangoRT/ticketClose.html', { @@ -224,7 +233,7 @@ def post(self, request): """ data = json.loads(request.body) email = request.user.email if request.user.is_authenticated else data['email'] - name = "{} {}".format(request.user.first_name, request.user.last_name) if request.user.is_authenticated else data['name'] + name = f"{request.user.first_name} {request.user.last_name}" if request.user.is_authenticated else data['name'] subject = data['subject'] body = data['body'] project_id = data['projectId'] @@ -278,10 +287,7 @@ def post(self, request): if subject is None or email is None or body is None: return HttpResponseBadRequest() - requestor_meta = '%s <%s>' % ( - name, - email - ) + requestor_meta = f'{name} <{email}>' meta = ( ('Opened by', request.user.username), @@ -291,20 +297,16 @@ def post(self, request): ('Project Title', project_title), ) - header = '\n'.join('[%s] %s' % m for m in meta) + header = '\n'.join('[{}] {}'.format(*m) for m in meta) - ticket_body = '%s\n\n%s\n\n---\n%s' % ( - header, - body, - requestor_meta - ) + ticket_body = f'{header}\n\n{body}\n\n---\n{requestor_meta}' ticket = rtModels.Ticket(subject=subject, problem_description="\n ".join(ticket_body.splitlines()), requestor=email, cc='') - logger.debug(f'Creating ticket for user: {name} email: {email}') + logger.debug('Creating ticket for user: %s email: %s', name, email) ticket_id = rt.createTicket(ticket) diff --git a/designsafe/apps/dropbox_integration/admin.py b/designsafe/apps/dropbox_integration/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/dropbox_integration/admin.py +++ b/designsafe/apps/dropbox_integration/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/dropbox_integration/models.py b/designsafe/apps/dropbox_integration/models.py index d6238d8077..4e122a76a1 100644 --- a/designsafe/apps/dropbox_integration/models.py +++ b/designsafe/apps/dropbox_integration/models.py @@ -1,6 +1,6 @@ -from dropbox import Dropbox from django.conf import settings from django.db import models +from dropbox import Dropbox class DropboxUserToken(models.Model): diff --git a/designsafe/apps/dropbox_integration/tests.py b/designsafe/apps/dropbox_integration/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/dropbox_integration/tests.py +++ b/designsafe/apps/dropbox_integration/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/dropbox_integration/urls.py b/designsafe/apps/dropbox_integration/urls.py index 4dcbd6a6cd..a529bdb03f 100644 --- a/designsafe/apps/dropbox_integration/urls.py +++ b/designsafe/apps/dropbox_integration/urls.py @@ -1,4 +1,5 @@ from django.urls import re_path as url + from designsafe.apps.dropbox_integration import views urlpatterns = [ diff --git a/designsafe/apps/dropbox_integration/views.py b/designsafe/apps/dropbox_integration/views.py index 0a30142905..da64d2690c 100644 --- a/designsafe/apps/dropbox_integration/views.py +++ b/designsafe/apps/dropbox_integration/views.py @@ -1,18 +1,18 @@ """Views for Dropbox integration in DesignSafe.""" import logging -from dropbox.oauth import DropboxOAuth2Flow, BadRequestException, BadStateException -from dropbox.exceptions import AuthError -from dropbox import Dropbox + from django.conf import settings -from django.contrib.auth.decorators import login_required from django.contrib import messages -from django.urls import reverse +from django.contrib.auth.decorators import login_required from django.http import HttpResponseRedirect -from django.views.decorators.csrf import csrf_exempt from django.shortcuts import render -from designsafe.apps.dropbox_integration.models import DropboxUserToken +from django.urls import reverse +from django.views.decorators.csrf import csrf_exempt +from dropbox.exceptions import AuthError +from dropbox.oauth import BadRequestException, BadStateException, DropboxOAuth2Flow +from designsafe.apps.dropbox_integration.models import DropboxUserToken logger = logging.getLogger(__name__) diff --git a/designsafe/apps/geo/apps.py b/designsafe/apps/geo/apps.py index 5238dad187..2a1af07ec0 100644 --- a/designsafe/apps/geo/apps.py +++ b/designsafe/apps/geo/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig + class DesignGeoConfig(AppConfig): name = 'designsafe.apps.geo' label = 'designsafe_geo' diff --git a/designsafe/apps/geo/tests.py b/designsafe/apps/geo/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/geo/tests.py +++ b/designsafe/apps/geo/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/geo/urls.py b/designsafe/apps/geo/urls.py index de7ce860e4..9e98587368 100644 --- a/designsafe/apps/geo/urls.py +++ b/designsafe/apps/geo/urls.py @@ -1,5 +1,5 @@ -from django.urls import include, re_path as url -from django.urls import reverse +from django.urls import re_path as url + from designsafe.apps.geo import views urlpatterns = [ diff --git a/designsafe/apps/geo/views.py b/designsafe/apps/geo/views.py index 36af58d20b..80b2bc9ec8 100644 --- a/designsafe/apps/geo/views.py +++ b/designsafe/apps/geo/views.py @@ -1,7 +1,7 @@ -from django.shortcuts import render -from django.contrib.auth.decorators import login_required import logging +from django.contrib.auth.decorators import login_required +from django.shortcuts import render logger = logging.getLogger(__name__) metrics_logger = logging.getLogger('metrics') diff --git a/designsafe/apps/googledrive_integration/admin.py b/designsafe/apps/googledrive_integration/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/googledrive_integration/admin.py +++ b/designsafe/apps/googledrive_integration/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/googledrive_integration/integrations.py b/designsafe/apps/googledrive_integration/integrations.py index 130cc35932..de5556221d 100644 --- a/designsafe/apps/googledrive_integration/integrations.py +++ b/designsafe/apps/googledrive_integration/integrations.py @@ -1,4 +1,3 @@ -from django.urls import reverse def provide_integrations(): diff --git a/designsafe/apps/googledrive_integration/models.py b/designsafe/apps/googledrive_integration/models.py index f4bb8bf248..ee2d563040 100644 --- a/designsafe/apps/googledrive_integration/models.py +++ b/designsafe/apps/googledrive_integration/models.py @@ -1,16 +1,15 @@ -from google.auth.transport.requests import Request -from googleapiclient import discovery -import google.oauth2.credentials -from django.conf import settings -from django.db import models +import base64 import logging import os - -import base64 import pickle -from django.utils import encoding +import google.oauth2.credentials import jsonpickle +from django.conf import settings +from django.db import models +from django.utils import encoding +from google.auth.transport.requests import Request +from googleapiclient import discovery logger = logging.getLogger(__name__) @@ -22,7 +21,7 @@ class CredentialsField(models.Field): def __init__(self, *args, **kwargs): if 'null' not in kwargs: kwargs['null'] = True - super(CredentialsField, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def get_internal_type(self): return 'BinaryField' diff --git a/designsafe/apps/googledrive_integration/tasks.py b/designsafe/apps/googledrive_integration/tasks.py index 63bc6bbeb5..a7dcd804d4 100644 --- a/designsafe/apps/googledrive_integration/tasks.py +++ b/designsafe/apps/googledrive_integration/tasks.py @@ -1,7 +1,6 @@ -from django.conf import settings -from django.contrib.auth import get_user_model import logging -import inspect + +from django.contrib.auth import get_user_model logger = logging.getLogger(__name__) diff --git a/designsafe/apps/googledrive_integration/tests.py b/designsafe/apps/googledrive_integration/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/googledrive_integration/tests.py +++ b/designsafe/apps/googledrive_integration/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/googledrive_integration/urls.py b/designsafe/apps/googledrive_integration/urls.py index abd4ca59e4..77bc2ab815 100644 --- a/designsafe/apps/googledrive_integration/urls.py +++ b/designsafe/apps/googledrive_integration/urls.py @@ -1,4 +1,5 @@ from django.urls import re_path as url + from designsafe.apps.googledrive_integration import views urlpatterns = [ diff --git a/designsafe/apps/googledrive_integration/views.py b/designsafe/apps/googledrive_integration/views.py index 9b81c7ece5..308d0f938a 100644 --- a/designsafe/apps/googledrive_integration/views.py +++ b/designsafe/apps/googledrive_integration/views.py @@ -1,21 +1,22 @@ +import logging + import google_auth_oauthlib.flow import requests -import os -from django.db import IntegrityError from django.conf import settings -from django.contrib.auth.decorators import login_required -from django.utils.decorators import method_decorator from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.db import IntegrityError +from django.http import ( + HttpResponseBadRequest, + HttpResponseRedirect, +) +from django.shortcuts import render from django.urls import reverse -from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseBadRequest, - Http404) from django.views.decorators.csrf import csrf_exempt -from django.views.generic import View -from django.shortcuts import render + from designsafe.apps.googledrive_integration.models import GoogleDriveUserToken from designsafe.apps.googledrive_integration.tasks import check_connection -import logging logger = logging.getLogger(__name__) CLIENT_CONFIG = {'web': { @@ -34,15 +35,14 @@ def index(request): try: googledrive_user = check_connection(request.user.username) context['googledrive_connection'] = googledrive_user - except Exception as e: + except Exception: # authentication failed - logger.exception('google drive failed to authenticate: {}'.format(e)) - logger.warning('Google Drive oauth token for user=%s failed to authenticate' % - request.user.username) + logger.exception('google drive failed to authenticate') + logger.warning('Google Drive oauth token for user=%s failed to authenticate', request.user.username) context['googledrive_connection'] = False - except: - logger.debug('GoogleDriveUserToken does not exist for user=%s' % request.user.username) + except Exception: + logger.exception('GoogleDriveUserToken does not exist for user=%s', request.user.username) return render(request, 'designsafe/apps/googledrive_integration/index.html', context) @@ -78,7 +78,7 @@ def oauth2_callback(request): else: return HttpResponseBadRequest('Unexpected request') - if not (state == googledrive['state']): + if state != googledrive['state']: return HttpResponseBadRequest('Request expired') try: @@ -104,19 +104,19 @@ def oauth2_callback(request): token.save() - except IntegrityError as e: + except IntegrityError: # Auth flow completed previously, and no refresh_token granted. Need to disconnect to get # another refresh_token. logger.debug('GoogleDriveUserToken refresh_token cannot be null, revoking previous access and restart flow.') - revoke = requests.post('https://accounts.google.com/o/oauth2/revoke', + requests.post('https://accounts.google.com/o/oauth2/revoke', params={'token': credentials.token}, headers = {'content-type': 'application/x-www-form-urlencoded'}) HttpResponseRedirect(reverse('googledrive_integration:initialize_token')) - except Exception as e: - logger.exception('Unable to complete Google Drive integration setup: %s' % e) + except Exception: + logger.exception('Unable to complete Google Drive integration setup') messages.error(request, 'Oh no! An unexpected error occurred while trying to set ' 'up the Google Drive application. Please try again.') @@ -134,7 +134,7 @@ def disconnect(request): params={'token': googledrive_user_token.credential.token}, headers = {'content-type': 'application/x-www-form-urlencoded'}) - status_code = getattr(revoke, 'status_code') + status_code = revoke.status_code googledrive_user_token.delete() @@ -145,18 +145,18 @@ def disconnect(request): else: logger.error('Disconnect Google Drive; google drive account revoke error.', extra={'user': request.user}) - logger.debug('status code:{}'.format(status_code)) + logger.debug('status code: %s', status_code) return HttpResponseRedirect(reverse('googledrive_integration:index')) except GoogleDriveUserToken.DoesNotExist: - logger.warn('Disconnect Google Drive; GoogleDriveUserToken does not exist.', + logger.warning('Disconnect Google Drive; GoogleDriveUserToken does not exist.', extra={'user': request.user}) - except Exception as e: + except Exception: logger.error('Disconnect Google Drive; GoogleDriveUserToken delete error.', extra={'user': request.user}) - logger.exception('google drive delete error: {}'.format(e)) + logger.exception('google drive delete error:') messages.success(request, 'Your Google Drive account has been disconnected from DesignSafe.') diff --git a/designsafe/apps/licenses/admin.py b/designsafe/apps/licenses/admin.py index f92194cd1f..4af2d080ad 100644 --- a/designsafe/apps/licenses/admin.py +++ b/designsafe/apps/licenses/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from designsafe.apps.licenses import models diff --git a/designsafe/apps/licenses/models.py b/designsafe/apps/licenses/models.py index a9eb23099c..00abe2b336 100644 --- a/designsafe/apps/licenses/models.py +++ b/designsafe/apps/licenses/models.py @@ -1,6 +1,7 @@ +import logging + from django.conf import settings from django.db import models -import logging logger = logging.getLogger(__name__) @@ -32,7 +33,7 @@ class Meta: abstract = True def __str__(self): - return "%s: %s" % (self.license_type, self.user.username) + return f"{self.license_type}: {self.user.username}" def license_as_str(self): self.license_file_content = self.license_file_content.replace('\r\n', '\n') diff --git a/designsafe/apps/licenses/tests.py b/designsafe/apps/licenses/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/licenses/tests.py +++ b/designsafe/apps/licenses/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/licenses/views.py b/designsafe/apps/licenses/views.py index 91ea44a218..b8e4ee02a7 100644 --- a/designsafe/apps/licenses/views.py +++ b/designsafe/apps/licenses/views.py @@ -1,3 +1,2 @@ -from django.shortcuts import render # Create your views here. diff --git a/designsafe/apps/nco/api_urls.py b/designsafe/apps/nco/api_urls.py index 58fcda6983..b2c2dfcdd1 100644 --- a/designsafe/apps/nco/api_urls.py +++ b/designsafe/apps/nco/api_urls.py @@ -1,9 +1,10 @@ from django.urls import re_path as url + from designsafe.apps.nco.views import ( - ProjectsListView, FiltersListView, - TtcGrantsView, + ProjectsListView, TtcFacilitiesView, + TtcGrantsView, TtcGrantTypesView, TtcHazardTypesView, ) diff --git a/designsafe/apps/nco/apps.py b/designsafe/apps/nco/apps.py index 8946a0ecd1..f1641707cf 100644 --- a/designsafe/apps/nco/apps.py +++ b/designsafe/apps/nco/apps.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals from django.apps import AppConfig diff --git a/designsafe/apps/nco/managers/projects.py b/designsafe/apps/nco/managers/projects.py index 842c79dbfd..5cfd899920 100644 --- a/designsafe/apps/nco/managers/projects.py +++ b/designsafe/apps/nco/managers/projects.py @@ -4,21 +4,21 @@ :synopsis: Nco projects manager. Should only do read actions. Unless it is to update the mongo database. """ -from __future__ import unicode_literals, absolute_import -import six -import logging -from datetime import datetime, timedelta import calendar +import logging +from datetime import UTC, datetime, timedelta + +import six from dateutil.parser import parse as datetime_parse -from designsafe.libs.mongo.load_projects import MongoProjectsHelper -from designsafe.apps.projects.managers.base import ProjectsManager -from designsafe.apps.api.agave import service_account +from designsafe.apps.api.agave import service_account +from designsafe.apps.projects.managers.base import ProjectsManager +from designsafe.libs.mongo.load_projects import MongoProjectsHelper logger = logging.getLogger(__name__) -class NcoProjectsManager(object): +class NcoProjectsManager: """Nco Projects Manager.""" def __init__(self, user): @@ -34,7 +34,7 @@ def __init__(self, user): def _process_time_filter(self, value): """Convert from time filter value to date range.""" - date = datetime.utcnow() + date = datetime.now(tz=UTC) def this_week(): """Get this week range.""" diff --git a/designsafe/apps/nco/managers/ttc_grants.py b/designsafe/apps/nco/managers/ttc_grants.py index 2f0e30e004..6090100267 100644 --- a/designsafe/apps/nco/managers/ttc_grants.py +++ b/designsafe/apps/nco/managers/ttc_grants.py @@ -3,20 +3,15 @@ .. module:: designsafe.apps.nco.managers.ttc_grants :synopsis: Nco ttc_grants manager. Should only do read actions. """ -from __future__ import unicode_literals, absolute_import -import six import logging -from datetime import datetime, timedelta -import calendar -from dateutil.parser import parse as datetime_parse -from designsafe.libs.mongo.load_ttc_grants import MongoTTCHelper -from designsafe.apps.api.agave import service_account +from designsafe.apps.api.agave import service_account +from designsafe.libs.mongo.load_ttc_grants import MongoTTCHelper logger = logging.getLogger(__name__) -class NcoTtcGrantsManager(object): +class NcoTtcGrantsManager: """Nco TTC Grants Manager.""" def __init__(self, user): diff --git a/designsafe/apps/nco/urls.py b/designsafe/apps/nco/urls.py index cb56baf148..1da6903418 100644 --- a/designsafe/apps/nco/urls.py +++ b/designsafe/apps/nco/urls.py @@ -1,4 +1,5 @@ from django.urls import re_path as url + from designsafe.apps.nco.views import NcoIndexView, NcoTtcGrantsView urlpatterns = [ diff --git a/designsafe/apps/nco/views/__init__.py b/designsafe/apps/nco/views/__init__.py index 104cfe8a09..afba9ffaff 100644 --- a/designsafe/apps/nco/views/__init__.py +++ b/designsafe/apps/nco/views/__init__.py @@ -1,20 +1,20 @@ -from designsafe.apps.nco.views.base import NcoIndexView, NcoTtcGrantsView from designsafe.apps.nco.views.api import ( - ProjectsListView, FiltersListView, - TtcGrantsView, + ProjectsListView, TtcFacilitiesView, + TtcGrantsView, TtcGrantTypesView, TtcHazardTypesView, ) +from designsafe.apps.nco.views.base import NcoIndexView, NcoTtcGrantsView __all__ = [ - "NcoIndexView", - "ProjectsListView", "FiltersListView", + "NcoIndexView", "NcoTtcGrantsView", - "TtcGrantsView", + "ProjectsListView", "TtcFacilitiesView", "TtcGrantTypesView", + "TtcGrantsView", "TtcHazardTypesView", ] diff --git a/designsafe/apps/nco/views/api.py b/designsafe/apps/nco/views/api.py index f09efa558c..a4eecc32ba 100644 --- a/designsafe/apps/nco/views/api.py +++ b/designsafe/apps/nco/views/api.py @@ -5,11 +5,11 @@ """ import json import logging + from designsafe.apps.api.views import BaseApiView from designsafe.apps.nco.managers import NcoProjectsManager, NcoTtcGrantsManager from designsafe.libs.mongo.response import MongoJsonResponse - logger = logging.getLogger(__name__) diff --git a/designsafe/apps/nco/views/base.py b/designsafe/apps/nco/views/base.py index 422e6a1a94..d84d6ea16c 100644 --- a/designsafe/apps/nco/views/base.py +++ b/designsafe/apps/nco/views/base.py @@ -4,7 +4,9 @@ :synopsis: Views for NCO app. """ import logging + from django.views.generic.base import TemplateView + # from designsafe.libs.mongo.load_projects import MongoProjectsHelper # from designsafe.apps.api.agave import service_account @@ -15,8 +17,6 @@ class NcoIndexView(TemplateView): """Nco Index view.""" - pass class NcoTtcGrantsView(TemplateView): """NCO TTC Grants view.""" - pass \ No newline at end of file diff --git a/designsafe/apps/notifications/admin.py b/designsafe/apps/notifications/admin.py index 76a75e4060..985a7dbbaa 100644 --- a/designsafe/apps/notifications/admin.py +++ b/designsafe/apps/notifications/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from designsafe.apps.notifications.models import SiteMessage # Register your models here. diff --git a/designsafe/apps/notifications/apps.py b/designsafe/apps/notifications/apps.py index 6b35c61bfe..d7b8273c0b 100644 --- a/designsafe/apps/notifications/apps.py +++ b/designsafe/apps/notifications/apps.py @@ -1,7 +1,7 @@ from designsafe.apps.signals.signals import ds_event -class Event(object): +class Event: @classmethod def send_event(self, event_type, users, event_data): ds_event.send(sender=self.__class__, event_type = event_type, users = users, **event_data); diff --git a/designsafe/apps/notifications/models.py b/designsafe/apps/notifications/models.py index 48e558494c..e2ab485735 100644 --- a/designsafe/apps/notifications/models.py +++ b/designsafe/apps/notifications/models.py @@ -1,10 +1,13 @@ +import cgi +import datetime +import json + from django.db import models from django.dispatch import receiver + from designsafe.apps.signals.signals import generic_event + from .apps import Event -import datetime -import json -import cgi class Notification(models.Model): diff --git a/designsafe/apps/notifications/tests.py b/designsafe/apps/notifications/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/notifications/tests.py +++ b/designsafe/apps/notifications/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/notifications/urls.py b/designsafe/apps/notifications/urls.py index f8db7f6556..46b1ce4738 100644 --- a/designsafe/apps/notifications/urls.py +++ b/designsafe/apps/notifications/urls.py @@ -1,8 +1,8 @@ -from django.urls import include, re_path as url +from django.urls import re_path as url from django.urls import reverse from django.utils.translation import gettext_lazy as _ -from designsafe.apps.notifications import views +from designsafe.apps.notifications import views urlpatterns = [ url(r'^$', views.index, name='index'), diff --git a/designsafe/apps/notifications/views.py b/designsafe/apps/notifications/views.py index a625343b9b..26407263f1 100644 --- a/designsafe/apps/notifications/views.py +++ b/designsafe/apps/notifications/views.py @@ -1,13 +1,14 @@ import json import logging from itertools import chain + from django.core import serializers from django.http import HttpResponse from django.shortcuts import render + from designsafe.apps.api.notifications.models import Notification from designsafe.apps.notifications.models import Notification as LegacyNotification - LOGGER = logging.getLogger(__name__) @@ -19,7 +20,7 @@ def get_number_unread_notifications(request): # nondeleted = JobNotification.objects.filter(deleted=False, user=str(request.user)).count() unread = Notification.objects.filter(deleted=False, read=False, user=str(request.user)).count() # LOGGER.info('nondeleted: {}'.format(nondeleted)) - LOGGER.info('unread: {}'.format(unread)) + LOGGER.info('unread: %s', unread) return unread @@ -46,7 +47,7 @@ def notifications(request): def delete_notification(request): body = json.loads(request.body) pk = body['pk'] - LOGGER.info('pk: {}'.format(pk)) + LOGGER.info('pk: %s', pk) if pk == 'all': items = Notification.objects.filter(deleted=False, user=str(request.user)) for i in items: diff --git a/designsafe/apps/onboarding/api/urls.py b/designsafe/apps/onboarding/api/urls.py index ecda14042d..1fc4e21914 100644 --- a/designsafe/apps/onboarding/api/urls.py +++ b/designsafe/apps/onboarding/api/urls.py @@ -1,8 +1,8 @@ """ URL routing for the onboarding API. """ from django.urls import path -from designsafe.apps.onboarding.api import views +from designsafe.apps.onboarding.api import views app_name = "onboarding_api" urlpatterns = [ diff --git a/designsafe/apps/onboarding/api/views.py b/designsafe/apps/onboarding/api/views.py index 7172ec18dd..c4f61a6588 100644 --- a/designsafe/apps/onboarding/api/views.py +++ b/designsafe/apps/onboarding/api/views.py @@ -1,29 +1,31 @@ """"API views for the Onboarding app.""" -import logging import json +import logging import math + from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import user_passes_test +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.http import ( Http404, - JsonResponse, HttpResponseBadRequest, + JsonResponse, ) -from django.contrib.auth.decorators import user_passes_test -from django.contrib.auth import get_user_model -from django.core.exceptions import PermissionDenied, ObjectDoesNotExist from django.utils.decorators import method_decorator -from designsafe.libs.common.utils import check_onboarding_admin, check_group_membership -from designsafe.apps.api.views import AuthenticatedApiView, ApiException + from designsafe.apps.api.users.utils import q_to_model_queries -from designsafe.apps.onboarding.models import SetupEvent, SetupEventEncoder +from designsafe.apps.api.views import ApiException, AuthenticatedApiView from designsafe.apps.onboarding.execute import ( - log_setup_state, - load_setup_step, - execute_single_step, execute_setup_steps, + execute_single_step, + load_setup_step, + log_setup_state, ) +from designsafe.apps.onboarding.models import SetupEvent, SetupEventEncoder from designsafe.apps.onboarding.state import SetupState +from designsafe.libs.common.utils import check_group_membership, check_onboarding_admin logger = logging.getLogger(__name__) @@ -307,7 +309,7 @@ def get(self, request): total = math.ceil(len(results) / limit) offset = (page - 1) * limit - users_in_page = results[offset : limit * page] # noqa: E203 + users_in_page = results[offset : limit * page] account_setup_steps = getattr(settings, "PORTAL_USER_ACCOUNT_SETUP_STEPS", []) # Assemble an array with the User data we care about diff --git a/designsafe/apps/onboarding/api/views_unit_test.py b/designsafe/apps/onboarding/api/views_unit_test.py index 910aa31ad2..e2878c9ed1 100644 --- a/designsafe/apps/onboarding/api/views_unit_test.py +++ b/designsafe/apps/onboarding/api/views_unit_test.py @@ -1,12 +1,14 @@ -import pytest -import logging import json -from mock import MagicMock -from django.http import JsonResponse +import logging +from unittest.mock import MagicMock + +import pytest from django.db.models import signals +from django.http import JsonResponse + +from designsafe.apps.onboarding.api.views import SetupStepView, get_user_onboarding from designsafe.apps.onboarding.models import SetupEvent from designsafe.apps.onboarding.state import SetupState -from designsafe.apps.onboarding.api.views import SetupStepView, get_user_onboarding logger = logging.getLogger(__name__) @@ -41,17 +43,17 @@ def mocked_get_tas_client(mocker): def test_get_user_unauthenticated_forbidden(client, regular_user): - response = client.get("/api/onboarding/user/{}/".format(regular_user.username)) + response = client.get(f"/api/onboarding/user/{regular_user.username}/") assert response.status_code == 401 def test_get_other_user_forbidden(client, authenticated_user, onboarding_admin_user): - response = client.get("/api/onboarding/user/{}/".format(onboarding_admin_user.username)) + response = client.get(f"/api/onboarding/user/{onboarding_admin_user.username}/") assert response.status_code == 403 def test_get_user_as_staff(client, authenticated_onboarding_admin, regular_user): - response = client.get("/api/onboarding/user/{}/".format(regular_user.username)) + response = client.get(f"/api/onboarding/user/{regular_user.username}/") assert response.status_code == 200 result = json.loads(response.content)["response"] assert result["username"] == regular_user.username @@ -77,7 +79,7 @@ def test_get_non_existent_user_as_staff(client, authenticated_onboarding_admin): def test_get_user_as_user(client, settings, authenticated_user, mock_steps): # A user should be able to retrieve their own setup event info response = client.get( - "/api/onboarding/user/{}".format(authenticated_user.username), follow=True + f"/api/onboarding/user/{authenticated_user.username}", follow=True ) result = response.json()["response"] @@ -98,7 +100,7 @@ def test_retry_step(client, settings, authenticated_user, mock_retry_step, mocke "designsafe.apps.onboarding.api.views.execute_single_step" ) response = client.get( - "/api/onboarding/user/{}".format(authenticated_user.username), follow=True + f"/api/onboarding/user/{authenticated_user.username}", follow=True ) mock_execute_single_step.apply_async.assert_called_with( args=[ @@ -119,14 +121,14 @@ def test_retry_step(client, settings, authenticated_user, mock_retry_step, mocke def test_incomplete_post(client, authenticated_user): # post should return HttpResponseBadRequest (400) if fields are missing response = client.post( - "/api/onboarding/user/{}/".format(authenticated_user), + f"/api/onboarding/user/{authenticated_user}/", content_type="application/json", data=json.dumps({"action": "user_confirm"}), ) assert response.status_code == 400 response = client.post( - "/api/onboarding/user/{}/".format(authenticated_user), + f"/api/onboarding/user/{authenticated_user}/", content_type="application/json", data=json.dumps({"step": "setupstep"}), ) @@ -146,7 +148,7 @@ def test_client_action(regular_user, rf): def test_reset_not_staff(client, authenticated_user): response = client.post( - "/api/onboarding/user/{}/".format(authenticated_user.username), + f"/api/onboarding/user/{authenticated_user.username}/", content_type="application/json", data=json.dumps( { @@ -178,7 +180,7 @@ def test_reset(rf, onboarding_admin_user, regular_user, mocked_log_setup_state): def test_complete_not_staff(client, authenticated_user): response = client.post( - "/api/onboarding/user/{}/".format(authenticated_user.username), + f"/api/onboarding/user/{authenticated_user.username}/", content_type="application/json", data=json.dumps( { @@ -194,7 +196,7 @@ def test_complete( client, authenticated_onboarding_admin, regular_user, mock_steps, mocked_executor ): response = client.post( - "/api/onboarding/user/{}/".format(regular_user.username), + f"/api/onboarding/user/{regular_user.username}/", content_type="application/json", data=json.dumps( { diff --git a/designsafe/apps/onboarding/conftest.py b/designsafe/apps/onboarding/conftest.py index a258003ffd..e6ef13b907 100644 --- a/designsafe/apps/onboarding/conftest.py +++ b/designsafe/apps/onboarding/conftest.py @@ -1,6 +1,7 @@ """ Pytest fixtures for the onboarding app. """ import pytest + from designsafe.apps.onboarding.models import SetupEvent from designsafe.apps.onboarding.state import SetupState diff --git a/designsafe/apps/onboarding/execute.py b/designsafe/apps/onboarding/execute.py index 34892985c4..3732feb7ce 100644 --- a/designsafe/apps/onboarding/execute.py +++ b/designsafe/apps/onboarding/execute.py @@ -1,15 +1,17 @@ """Methods for executing setup steps for a user.""" import logging -from inspect import isclass from importlib import import_module +from inspect import isclass + from celery import shared_task from django.conf import settings from django.contrib.auth import get_user_model -from designsafe.apps.onboarding.state import SetupState + +from designsafe.apps.accounts.models import DesignSafeProfile from designsafe.apps.onboarding.models import SetupEvent +from designsafe.apps.onboarding.state import SetupState from designsafe.apps.onboarding.steps.abstract import AbstractStep -from designsafe.apps.accounts.models import DesignSafeProfile logger = logging.getLogger(__name__) @@ -61,6 +63,7 @@ def load_setup_step(user, step): raise ValueError(f"Setup step {step} is not a class") setup_step = call(user) if not isinstance(setup_step, AbstractStep): + # ruff: disable[TRY004] raise ValueError(f"Setup step {step} is not a subclass of AbstractStep") return setup_step @@ -85,7 +88,7 @@ def process_setup_step(setup_step): except Exception as err: # pylint: disable=broad-except logger.exception("Problem processing setup step") setup_step.state = SetupState.ERROR - setup_step.log(f"Exception: {str(err)}") + setup_step.log(f"Exception: {err!s}") @shared_task() diff --git a/designsafe/apps/onboarding/execute_unit_test.py b/designsafe/apps/onboarding/execute_unit_test.py index 8bb771981d..5de4a4d6eb 100644 --- a/designsafe/apps/onboarding/execute_unit_test.py +++ b/designsafe/apps/onboarding/execute_unit_test.py @@ -1,20 +1,21 @@ +from unittest.mock import MagicMock + import pytest -from mock import MagicMock from django.db.models import signals -from designsafe.apps.onboarding.steps.test_steps import MockProcessingCompleteStep + from designsafe.apps.accounts.models import DesignSafeProfile -from designsafe.apps.onboarding.models import SetupEvent -from designsafe.apps.onboarding.state import SetupState from designsafe.apps.onboarding.execute import ( + StepExecuteException, execute_setup_steps, execute_single_step, - prepare_setup_steps, load_setup_step, log_setup_state, new_user_setup_check, - StepExecuteException, + prepare_setup_steps, ) - +from designsafe.apps.onboarding.models import SetupEvent +from designsafe.apps.onboarding.state import SetupState +from designsafe.apps.onboarding.steps.test_steps import MockProcessingCompleteStep pytestmark = pytest.mark.django_db diff --git a/designsafe/apps/onboarding/models.py b/designsafe/apps/onboarding/models.py index 28ea73f814..0e1764e62a 100644 --- a/designsafe/apps/onboarding/models.py +++ b/designsafe/apps/onboarding/models.py @@ -1,8 +1,8 @@ """Onboarding models.""" -from django.db import models from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder +from django.db import models class SetupEvent(models.Model): diff --git a/designsafe/apps/onboarding/models_unit_test.py b/designsafe/apps/onboarding/models_unit_test.py index 0c2d3d74d3..bc05285e7d 100644 --- a/designsafe/apps/onboarding/models_unit_test.py +++ b/designsafe/apps/onboarding/models_unit_test.py @@ -1,8 +1,8 @@ -from designsafe.apps.onboarding.state import SetupState -from designsafe.apps.onboarding.models import SetupEvent -from django.db.models import signals import pytest +from django.db.models import signals +from designsafe.apps.onboarding.models import SetupEvent +from designsafe.apps.onboarding.state import SetupState pytestmark = pytest.mark.django_db diff --git a/designsafe/apps/onboarding/steps/abstract.py b/designsafe/apps/onboarding/steps/abstract.py index c4946ee07e..37e46c19b2 100644 --- a/designsafe/apps/onboarding/steps/abstract.py +++ b/designsafe/apps/onboarding/steps/abstract.py @@ -1,8 +1,10 @@ """Abstract class for user setup steps.""" from abc import ABCMeta, abstractmethod -from six import add_metaclass + from django.conf import settings +from six import add_metaclass + from designsafe.apps.onboarding.models import SetupEvent from designsafe.apps.onboarding.state import SetupState @@ -102,7 +104,7 @@ def custom_status(self): Called when displaying this step in the client. Should return a string that displays a custom status """ - return None + return @abstractmethod def prepare(self): diff --git a/designsafe/apps/onboarding/steps/abstract_unit_test.py b/designsafe/apps/onboarding/steps/abstract_unit_test.py index 93ff1425a0..e345bc3d72 100644 --- a/designsafe/apps/onboarding/steps/abstract_unit_test.py +++ b/designsafe/apps/onboarding/steps/abstract_unit_test.py @@ -1,8 +1,9 @@ +import pytest +from django.db.models import signals + from designsafe.apps.onboarding.models import SetupEvent from designsafe.apps.onboarding.state import SetupState -from django.db.models import signals from designsafe.apps.onboarding.steps.test_steps import MockStep -import pytest @pytest.fixture(autouse=True) diff --git a/designsafe/apps/onboarding/steps/access_unit_test.py b/designsafe/apps/onboarding/steps/access_unit_test.py index db21f95c34..319a18686f 100644 --- a/designsafe/apps/onboarding/steps/access_unit_test.py +++ b/designsafe/apps/onboarding/steps/access_unit_test.py @@ -1,15 +1,17 @@ -from django.test import TestCase, RequestFactory +from unittest.mock import ANY, patch + +import pytest from django.contrib.auth import get_user_model -from mock import patch, ANY +from django.test import RequestFactory, TestCase + from designsafe.apps.onboarding.state import SetupState from designsafe.apps.onboarding.steps.access import RequestAccessStep -import pytest @pytest.mark.django_db(transaction=True) class TestRequestAccessStep(TestCase): def setUp(self): - super(TestRequestAccessStep, self).setUp() + super().setUp() # Create a test user User = get_user_model() @@ -19,7 +21,7 @@ def setUp(self): self.staff.is_staff = True def tearDown(self): - super(TestRequestAccessStep, self).tearDown() + super().tearDown() @patch("designsafe.apps.onboarding.steps.access.RequestAccessStep.log") def test_prepare(self, mock_log): diff --git a/designsafe/apps/onboarding/steps/allocation.py b/designsafe/apps/onboarding/steps/allocation.py index 3d3dff3877..15336711e1 100644 --- a/designsafe/apps/onboarding/steps/allocation.py +++ b/designsafe/apps/onboarding/steps/allocation.py @@ -1,8 +1,8 @@ """Allocation step for Onboarding.""" -from designsafe.apps.onboarding.steps.abstract import AbstractStep -from designsafe.apps.onboarding.state import SetupState from designsafe.apps.api.users.utils import get_allocations +from designsafe.apps.onboarding.state import SetupState +from designsafe.apps.onboarding.steps.abstract import AbstractStep class AllocationStep(AbstractStep): diff --git a/designsafe/apps/onboarding/steps/allocation_unit_test.py b/designsafe/apps/onboarding/steps/allocation_unit_test.py index ed03eddd18..d1b6d94495 100644 --- a/designsafe/apps/onboarding/steps/allocation_unit_test.py +++ b/designsafe/apps/onboarding/steps/allocation_unit_test.py @@ -1,7 +1,9 @@ -from designsafe.apps.onboarding.steps.allocation import AllocationStep -from mock import ANY +from unittest.mock import ANY + import pytest +from designsafe.apps.onboarding.steps.allocation import AllocationStep + @pytest.fixture def get_allocations_mock(mocker): diff --git a/designsafe/apps/onboarding/steps/project_membership.py b/designsafe/apps/onboarding/steps/project_membership.py index e5961049a1..92a2319140 100644 --- a/designsafe/apps/onboarding/steps/project_membership.py +++ b/designsafe/apps/onboarding/steps/project_membership.py @@ -1,13 +1,14 @@ """Project Membership Step for Onboarding.""" import logging -from requests.auth import HTTPBasicAuth + from django.conf import settings from pytas.http import TASClient +from requests.auth import HTTPBasicAuth from rt import Rt -from designsafe.apps.onboarding.steps.abstract import AbstractStep -from designsafe.apps.onboarding.state import SetupState +from designsafe.apps.onboarding.state import SetupState +from designsafe.apps.onboarding.steps.abstract import AbstractStep logger = logging.getLogger(__name__) @@ -76,7 +77,7 @@ def send_project_request(self, request): tracker = self.get_tracker() ticket_text = f""" User {self.user.username} is requesting access to DesignSafe Computation and Data Resources. - System administrator please visit {request.build_absolute_uri(f'/onboarding/setup/{self.user.username}')} + System administrator please visit {request.build_absolute_uri(f"/onboarding/setup/{self.user.username}")} to complete this request. """ @@ -92,9 +93,8 @@ def send_project_request(self, request): tracker.logout() if not result: - raise Exception( # pylint: disable=broad-exception-raised - "Could not create ticket" - ) + # ruff: disable[TRY002] + raise Exception("Could not create ticket") self.state = SetupState.STAFFWAIT self.log( @@ -102,9 +102,8 @@ def send_project_request(self, request): data={"ticket": result}, ) else: - raise Exception( # pylint: disable=broad-exception-raised - "Could not log in to RT" - ) + # ruff: disable[TRY002] + raise Exception("Could not log in to RT") except Exception as err: # pylint: disable=broad-except logger.exception( msg="Could not create ticket on behalf of user during ProjectMembershipStep" @@ -135,7 +134,7 @@ def add_to_project(self): self.fail( f"{self.user.username} could not be added to {self.project['title']} due to error {reason}" ) - raise exc + raise def deny_project_request(self): """Deny a project request and close the ticket""" @@ -144,7 +143,7 @@ def deny_project_request(self): if event.data and "ticket" in event.data: ticket_id = event.data["ticket"] tracker = self.get_tracker() - request_text = f"""Your request for access to DesignSafe Computation and Data Resources has been + request_text = """Your request for access to DesignSafe Computation and Data Resources has been denied. You can still access the publicly available web pages and data. If you believe this is an error, please submit a help ticket. """ if tracker.login(): @@ -202,9 +201,7 @@ def client_action(self, action, data, request): f"Portal access request approved by {request.user.username}" ) except Exception as err: # pylint: disable=broad-except - logger.exception( - msg=f"Error during staff_approve on {self.step_name()}" - ) + logger.exception("Error during staff_approve on %s", self.step_name()) logger.error(err.args) self.fail( "An error occurred while trying to add this user to the project" diff --git a/designsafe/apps/onboarding/steps/project_membership_unit_test.py b/designsafe/apps/onboarding/steps/project_membership_unit_test.py index d51cb8474d..605625f22f 100644 --- a/designsafe/apps/onboarding/steps/project_membership_unit_test.py +++ b/designsafe/apps/onboarding/steps/project_membership_unit_test.py @@ -1,10 +1,12 @@ -from django.conf import settings -from designsafe.apps.onboarding.steps.project_membership import ProjectMembershipStep -from designsafe.apps.onboarding.models import SetupEvent -from mock import MagicMock, ANY -import pytest import json import os +from unittest.mock import ANY, MagicMock + +import pytest +from django.conf import settings + +from designsafe.apps.onboarding.models import SetupEvent +from designsafe.apps.onboarding.steps.project_membership import ProjectMembershipStep @pytest.fixture diff --git a/designsafe/apps/onboarding/steps/system_access.py b/designsafe/apps/onboarding/steps/system_access.py index 24e6c6b184..e6acd5c64f 100644 --- a/designsafe/apps/onboarding/steps/system_access.py +++ b/designsafe/apps/onboarding/steps/system_access.py @@ -1,8 +1,10 @@ """System Allocation Access Step for Onboarding.""" import logging -from designsafe.apps.onboarding.state import SetupState + from designsafe.apps.api.users.utils import get_allocations +from designsafe.apps.onboarding.state import SetupState + from .project_membership import ProjectMembershipStep logger = logging.getLogger(__name__) @@ -44,8 +46,8 @@ def has_required_systems(self): # If the intersection of the set of systems and resources has # items, the user has the necessary allocation return len(set(systems).intersection(resources)) > 0 - except Exception as exc: # pylint: disable=broad-except - logger.error(exc) + except Exception: + logger.exception("") self.fail("We were unable to retrieve your allocations.") return False diff --git a/designsafe/apps/onboarding/steps/system_access_unit_test.py b/designsafe/apps/onboarding/steps/system_access_unit_test.py index 6cad3c5ea2..79eb032e73 100644 --- a/designsafe/apps/onboarding/steps/system_access_unit_test.py +++ b/designsafe/apps/onboarding/steps/system_access_unit_test.py @@ -1,9 +1,11 @@ -from django.conf import settings -from designsafe.apps.onboarding.steps.system_access import SystemAccessStep -import pytest import json import os +import pytest +from django.conf import settings + +from designsafe.apps.onboarding.steps.system_access import SystemAccessStep + @pytest.fixture def tas_client(mocker): diff --git a/designsafe/apps/onboarding/steps/system_access_v3.py b/designsafe/apps/onboarding/steps/system_access_v3.py index dee4c82668..13af9f3ec9 100644 --- a/designsafe/apps/onboarding/steps/system_access_v3.py +++ b/designsafe/apps/onboarding/steps/system_access_v3.py @@ -1,19 +1,20 @@ """System Access Step for Onboarding.""" import logging + from tapipy.errors import ( - NotFoundError, BaseTapyException, ForbiddenError, + NotFoundError, UnauthorizedError, ) -from designsafe.apps.onboarding.steps.abstract import AbstractStep -from designsafe.apps.onboarding.state import SetupState + from designsafe.apps.api.agave import get_service_account_client, get_tg458981_client from designsafe.apps.api.tasks import agave_indexer +from designsafe.apps.onboarding.state import SetupState +from designsafe.apps.onboarding.steps.abstract import AbstractStep from designsafe.libs.common.decorators import retry - logger = logging.getLogger(__name__) @@ -31,7 +32,9 @@ def create_system_credentials_with_keys( # pylint: disable=too-many-arguments """ Set an RSA key pair as the user's auth credential on a Tapis system. """ - logger.info(f"Creating user credential for {username} on Tapis system {system_id}") + logger.info( + "Creating user credential for %s on Tapis system %s", username, system_id + ) data = {"privateKey": private_key, "publicKey": public_key} client.systems.createUserCredential( systemId=system_id, @@ -54,7 +57,11 @@ def create_system_credentials( # pylint: disable=too-many-arguments """ Setup user's auth credential on a Tapis system using TMS. """ - logger.info(f"Creating user credential for {username} on Tapis system {system_id} using TMS") + logger.info( + "Creating user credential for %s on Tapis system %s using TMS", + username, + system_id, + ) client.systems.createUserCredential( systemId=system_id, userName=username, @@ -65,7 +72,7 @@ def create_system_credentials( # pylint: disable=too-many-arguments def set_user_permissions(user, system_id): """Apply read/write/execute permissions to files and read permissions on the system.""" - logger.info(f"Adding {user.username} permissions to Tapis system {system_id}") + logger.info("Adding %s permissions to Tapis system %s", user.username, system_id) client = get_service_account_client() client.systems.grantUserPerms( systemId=system_id, userName=user.username, permissions=["READ"] @@ -138,7 +145,10 @@ def process(self): try: create_system_credentials( - self.user.tapis_oauth.client, self.user.username, system, createTmsKeys=True + self.user.tapis_oauth.client, + self.user.username, + system, + createTmsKeys=True, ) self.log(f"Successfully created credentials for system: {system}") except BaseTapyException as exc: diff --git a/designsafe/apps/onboarding/steps/test_steps.py b/designsafe/apps/onboarding/steps/test_steps.py index 7447c0cc98..b7abf7b78a 100644 --- a/designsafe/apps/onboarding/steps/test_steps.py +++ b/designsafe/apps/onboarding/steps/test_steps.py @@ -1,4 +1,5 @@ -from mock import MagicMock +from unittest.mock import MagicMock + from designsafe.apps.onboarding.state import SetupState from designsafe.apps.onboarding.steps.abstract import AbstractStep @@ -122,10 +123,10 @@ def client_action(self, action, data, request): return if action == "staff_approve": - self.complete("Approved by {user}".format(user=request.user.username)) + self.complete(f"Approved by {request.user.username}") self.staff_approve_spy(action, data, request) elif action == "staff_deny": - self.fail("Denied by {user}".format(user=request.user.username)) + self.fail(f"Denied by {request.user.username}") self.staff_deny_spy(action, data, request) @@ -148,7 +149,7 @@ def description(self): return "Long description of a mock step that results in error upon processing" def process(self): - raise Exception("MockErrorStep") + raise Exception("MockErrorStep") # noqa class MockInvalidStepClass: diff --git a/designsafe/apps/onboarding/urls.py b/designsafe/apps/onboarding/urls.py index a5554205b9..b641f7b3d2 100644 --- a/designsafe/apps/onboarding/urls.py +++ b/designsafe/apps/onboarding/urls.py @@ -1,6 +1,7 @@ """URLs for the Onboarding app.""" -from django.urls import re_path, path +from django.urls import path, re_path + from designsafe.apps.onboarding.views import OnboardingView app_name = "workbench" diff --git a/designsafe/apps/onboarding/views.py b/designsafe/apps/onboarding/views.py index e86da1e4df..801363696d 100644 --- a/designsafe/apps/onboarding/views.py +++ b/designsafe/apps/onboarding/views.py @@ -1,9 +1,9 @@ """Views for Onboarding""" -from django.views.generic.base import TemplateView +from django.contrib.auth.decorators import login_required from django.utils.decorators import method_decorator from django.views.decorators.csrf import ensure_csrf_cookie -from django.contrib.auth.decorators import login_required +from django.views.generic.base import TemplateView @method_decorator(login_required, name="dispatch") diff --git a/designsafe/apps/projects/admin.py b/designsafe/apps/projects/admin.py index c26e686c78..0aec2ad17c 100644 --- a/designsafe/apps/projects/admin.py +++ b/designsafe/apps/projects/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from designsafe.apps.projects.models.categories import Category, CategoryOrder diff --git a/designsafe/apps/projects/apps.py b/designsafe/apps/projects/apps.py index 9f39670609..6ae72d670d 100644 --- a/designsafe/apps/projects/apps.py +++ b/designsafe/apps/projects/apps.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals from django.apps import AppConfig diff --git a/designsafe/apps/projects/management/commands/check_entities_relations.py b/designsafe/apps/projects/management/commands/check_entities_relations.py index 93d29831da..16088b5342 100644 --- a/designsafe/apps/projects/management/commands/check_entities_relations.py +++ b/designsafe/apps/projects/management/commands/check_entities_relations.py @@ -1,13 +1,12 @@ """Management command to delete null entity relations""" -import six import json import logging -from django.core.management.base import BaseCommand, CommandError -from django.conf import settings + +import six +from django.core.management.base import BaseCommand from requests.exceptions import HTTPError -from designsafe.apps.projects.models.agave.base import Project + from designsafe.apps.api.agave import get_service_account_client -from designsafe.apps.data.models.agave.base import Model as MetadataModel from designsafe.apps.projects.models.utils import lookup_model logger = logging.getLogger(__name__) @@ -28,7 +27,7 @@ class Command(BaseCommand): error is raised. """ def __init__(self, *args, **kwargs): - super(Command, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.prj = None self.client = None diff --git a/designsafe/apps/projects/management/commands/projects_report.py b/designsafe/apps/projects/management/commands/projects_report.py index a45df8e113..daa9bcc8a6 100644 --- a/designsafe/apps/projects/management/commands/projects_report.py +++ b/designsafe/apps/projects/management/commands/projects_report.py @@ -1,13 +1,12 @@ """Command to print NSF report""" -import six -import json import csv +import json import logging -from django.core.management.base import BaseCommand, CommandError -from django.conf import settings + from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist -from requests.exceptions import HTTPError +from django.core.management.base import BaseCommand + from designsafe.apps.api.agave import get_service_account_client logger = logging.getLogger(__name__) @@ -16,7 +15,7 @@ class Command(BaseCommand): """Returns a csv file with necessary details for report""" def __init__(self, *args, **kwargs): - super(Command, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.client = None def add_arguments(self, parser): diff --git a/designsafe/apps/projects/managers/base.py b/designsafe/apps/projects/managers/base.py index e5074a7ad1..9829989fc1 100644 --- a/designsafe/apps/projects/managers/base.py +++ b/designsafe/apps/projects/managers/base.py @@ -3,17 +3,16 @@ .. :module: designsafe.apps.projects.managers.base :synopsis: Base manager for projects. """ -from __future__ import unicode_literals, absolute_import -import logging import json -from designsafe.apps.projects.models.utils import lookup_model as project_lookup_model +import logging +from designsafe.apps.projects.models.utils import lookup_model as project_lookup_model LOG = logging.getLogger(__name__) -class ProjectsManager(object): +class ProjectsManager: """Base projects manager.""" def __init__(self, agave_client=None, user=None): @@ -44,17 +43,15 @@ def get_project_by_id(self, project_id): metas = self._ac.meta.listMetadata( q=json.dumps({"value.projectId": project_id}) ) - assert metas, "No project with id: {project_id} found.".format( - project_id=project_id - ) + assert metas, f"No project with id: {project_id} found." if len(metas) > 1: affected_uuids = [] for meta in metas: affected_uuids.append(meta['uuid']) LOG.info( - "More than one record with project id: {prj_id} found. " - "Affected project UUIDs: {uuids}" - .format(prj_id=project_id, uuids=affected_uuids) + f"More than one record with project id: {project_id} found. " + f"Affected project UUIDs: {affected_uuids}" + ) prj_json = metas[0] diff --git a/designsafe/apps/projects/managers/datacite.py b/designsafe/apps/projects/managers/datacite.py index 611bd18268..45c7fc251d 100644 --- a/designsafe/apps/projects/managers/datacite.py +++ b/designsafe/apps/projects/managers/datacite.py @@ -4,11 +4,11 @@ :synopsis: Python interface to Datacite's rest API. Visit: https://support.datacite.org/docs/api for more info. """ -from __future__ import unicode_literals, absolute_import import json import logging -from django.conf import settings + import requests +from django.conf import settings from requests import HTTPError LOGGER = logging.getLogger(__name__) @@ -29,7 +29,7 @@ def get_doi(doi): :rtype: dict """ res = requests.get( - '{api_url}/{doi}'.format(api_url=DOIS_URL, doi=doi), + f'{DOIS_URL}/{doi}', auth=(DATACITE_USER, DATACITE_PASS), ) res.raise_for_status() @@ -93,7 +93,7 @@ def create_or_update_doi(attributes=None, doi=None): datacite_attrs.update(attributes) attributes = datacite_attrs http_verb = requests.put - url = "{base_url}/{doi}".format(base_url=DOIS_URL, doi=doi) + url = f"{DOIS_URL}/{doi}" payload['data']['id'] = doi attributes.pop('prefix', None) else: @@ -175,7 +175,7 @@ def delete_doi(doi): :param str doi: DOI to delete. """ res = requests.delete( - "{base_url}/{doi}".format(base_url=DOIS_URL, doi=doi), + f"{DOIS_URL}/{doi}", auth=(DATACITE_USER, DATACITE_PASS) ) res.raise_for_status() diff --git a/designsafe/apps/projects/managers/publication.py b/designsafe/apps/projects/managers/publication.py index f3c2309906..f5b1084038 100644 --- a/designsafe/apps/projects/managers/publication.py +++ b/designsafe/apps/projects/managers/publication.py @@ -5,14 +5,15 @@ Visit: https://support.datacite.org/docs/api for more info. """ -from html.entities import entitydefs -import os import json -import zipfile import logging +import os +import zipfile + from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from pytas.http import TASClient + from designsafe import settings from designsafe.apps.api.agave import service_account from designsafe.apps.data.models.agave.files import BaseFileResource @@ -23,7 +24,6 @@ from designsafe.libs.elasticsearch.docs.publications import BaseESPublication from designsafe.libs.elasticsearch.utils import new_es_client - logger = logging.getLogger(__name__) TARGET_BASE = 'https://www.designsafe-ci.org/data/browser/public/designsafe.storage.published/{project_id}' ENTITY_TARGET_BASE = 'https://www.designsafe-ci.org/data/browser/public/designsafe.storage.published/{project_id}/#details-{entity_uuid}' @@ -146,7 +146,7 @@ def draft_publication( if entity: if revision: entity_url = ENTITY_TARGET_BASE.format( - project_id='{}v{}'.format(project_id, revision), + project_id=f'{project_id}v{revision}', entity_uuid=ent_uuid ) original_entities = getattr(pub, FIELD_MAP[entity.name]) @@ -192,7 +192,7 @@ def draft_publication( if revision: # Versions should not update certain fields # Add version number to DataCite info - prj_url = TARGET_BASE.format(project_id='{}v{}'.format(project_id, revision)) + prj_url = TARGET_BASE.format(project_id=f'{project_id}v{revision}') pub = BaseESPublication(project_id=project_id, revision=revision) prj.title = pub.project.value.title prj.team_order = pub.project.value.teamOrder @@ -204,7 +204,7 @@ def draft_publication( # append links to previous versions in DOI... relatedIdentifiers = [] for ver in range(1, revision): - id = '{}v{}'.format(project_id, ver) if ver!=1 else project_id + id = f'{project_id}v{ver}' if ver!=1 else project_id relatedIdentifiers.append( { 'relatedIdentifierType': 'URL', @@ -535,7 +535,7 @@ def check_complete_tags(tags): def fix_tags_path(entity): for tag in entity['value']['fileTags']: try: - pub_base = "{}v{}".format(project_id, revision) if revision else project_id + pub_base = f"{project_id}v{revision}" if revision else project_id pub_file = BaseFileResource.listing( service_account(), system="designsafe.storage.published", @@ -543,7 +543,7 @@ def fix_tags_path(entity): ) tag['fileUuid'] = pub_file.uuid except Exception as err: - logger.info('error: {}'.format(err)) + logger.info(f'error: {err}') continue def fix_tags_no_path(entity): @@ -551,20 +551,20 @@ def fix_tags_no_path(entity): proj_other = BaseFileResource.listing(service_account(), system="project-{}".format(entity['uuid']), path="") for child in proj_other.children: try: - pub_base = "{}v{}".format(project_id, revision) if revision else project_id - pub_file = BaseFileResource.listing(service_account(), system="designsafe.storage.published", path="{}{}".format(pub_base, child.path)) + pub_base = f"{project_id}v{revision}" if revision else project_id + pub_file = BaseFileResource.listing(service_account(), system="designsafe.storage.published", path=f"{pub_base}{child.path}") proj_file = BaseFileResource.listing(service_account(), system="project-{}".format(entity['uuid']), path=child.path) for tag in entity['value']['fileTags']: if tag['fileUuid'] == proj_file.uuid: tag['fileUuid'] = pub_file.uuid except Exception as err: - logger.info('error: {}'.format(err)) + logger.info(f'error: {err}') continue else: for fobj in entity['fileObjs']: try: - pub_base = "{}v{}".format(project_id, revision) if revision else project_id + pub_base = f"{project_id}v{revision}" if revision else project_id pub_file = BaseFileResource.listing(service_account(), system="designsafe.storage.published", path="{}{}".format(pub_base, fobj['path'])) proj_file = BaseFileResource.listing(service_account(), system="project-{}".format(pub_dict['project']['uuid']), path=fobj['path']) for tag in entity['value']['fileTags']: @@ -572,7 +572,7 @@ def fix_tags_no_path(entity): tag['fileUuid'] = pub_file.uuid except Exception as err: - logger.info('error: {}'.format(err)) + logger.info(f'error: {err}') continue for entname in entities_to_check: @@ -603,11 +603,11 @@ def archive(project_id, revision=None): es_client = new_es_client() pub = BaseESPublication(project_id=project_id, revision=revision, using=es_client) if revision: - archive_prefix = '{}v{}'.format(pub.projectId, revision) + archive_prefix = f'{pub.projectId}v{revision}' else: archive_prefix = pub.projectId - archive_name = '{}_archive.zip'.format(archive_prefix) - metadata_name = '{}_metadata.json'.format(archive_prefix) + archive_name = f'{archive_prefix}_archive.zip' + metadata_name = f'{archive_prefix}_metadata.json' pub_dir = settings.DESIGNSAFE_PUBLISHED_PATH arc_dir = os.path.join(pub_dir, 'archives/') archive_path = os.path.join(arc_dir, archive_name) @@ -625,8 +625,8 @@ def set_perms(dir, octal, subdir=None): os.chmod(os.path.join(root, d), octal) for f in files: os.chmod(os.path.join(root, f), octal) - except Exception as e: - logger.exception("Failed to set permissions for {}".format(dir)) + except Exception: + logger.exception(f"Failed to set permissions for {dir}") os.chmod(dir, 0o555) # compress published files into a zip archive @@ -634,7 +634,7 @@ def create_archive(): arc_source = os.path.join(pub_dir, archive_prefix) try: - logger.debug("Creating archive for {}".format(archive_prefix)) + logger.debug(f"Creating archive for {archive_prefix}") zf = zipfile.ZipFile(archive_path, mode='w', allowZip64=True) for dirs, _, files in os.walk(arc_source): for f in files: @@ -643,8 +643,8 @@ def create_archive(): zf.write(os.path.join(dirs, f), os.path.join(dirs.replace(pub_dir, ''), f)) zf.write(metadata_path, metadata_name) zf.close() - except Exception as e: - logger.exception("Archive creation failed for {}".format(arc_source)) + except Exception: + logger.exception(f"Archive creation failed for {arc_source}") finally: set_perms(pub_dir, 0o555, arc_source) set_perms(arc_dir, 0o555) @@ -665,7 +665,7 @@ def create_metadata(): project_uuid = pub_dict['project']['uuid'] try: - logger.debug("Creating metadata for {}".format(archive_prefix)) + logger.debug(f"Creating metadata for {archive_prefix}") if pub_dict['project']['value']['projectType'] in entity_type_map: ent_type = entity_type_map[pub_dict['project']['value']['projectType']] entity_uuids = [] @@ -699,7 +699,7 @@ def create_metadata(): set_perms(arc_dir, 0o755) create_metadata() create_archive() - except Exception as e: + except Exception: logger.exception('Failed to archive publication!') diff --git a/designsafe/apps/projects/models/agave/base.json b/designsafe/apps/projects/models/agave/base.json new file mode 100644 index 0000000000..650672be08 --- /dev/null +++ b/designsafe/apps/projects/models/agave/base.json @@ -0,0 +1,81 @@ +{ + "contributors": [ + { + "contributorType": "HostingInstitution", + "nameType": "Organizational", + "name": "University of Central Florida (ucf.edu)" + } + ], + "creators": [ + { + "name": "Rafi, Md Nafees Fuad", + "givenName": "Md Nafees Fuad", + "familyName": "Rafi", + "affiliation": [ + { + "name": "University of Central Florida (ucf.edu)", + "schemeUri": null, + "affiliationIdentifier": null, + "affiliationIdentifierScheme": null + } + ] + }, + { + "name": "Hasan, Samiul", + "givenName": "Samiul", + "familyName": "Hasan", + "affiliation": [ + { + "name": "University of Central Florida (ucf.edu)", + "schemeUri": null, + "affiliationIdentifier": null, + "affiliationIdentifierScheme": null + } + ] + } + ], + "titles": [ + { + "title": "Evacuation Traffic Prediction for Dynamic Transportation Networks over Multiple Hurricanes: A Generalized Graph Learning Approach" + } + ], + "publisher": "Designsafe-CI", + "publicationYear": 2026, + "types": { "resourceType": "Project", "resourceTypeGeneral": "Dataset" }, + "version": 1, + "descriptions": [ + { + "descriptionType": "Abstract", + "description": "This project contains the dataset used for the study titled \u201cEvacuation Traffic Prediction for Dynamic Transportation Networks over Multiple Hurricanes: A Generalized Graph Learning Approach.\u201d The study develops a Dynamic Graph Learning (DGL) framework to predict traffic conditions during unseen hurricane evacuation periods. The project focuses on evacuation traffic flow prediction using detector-based traffic data, dynamic transportation network representation, evacuation order information, and spatial attributes related to hurricane events. It also introduces a modular training approach to show the incremental learning of the DGL framework from multiple hurricanes.\nThe dataset includes processed traffic detector (MVDS) observations from multiple historical hurricanes in Florida, along with multiple evacuation-related features used for model training and evaluation. The dynamic graph structure represents changes in the active transportation sensor network over time, allowing the model to account for real-time detector availability and evolving roadway conditions during evacuation events.\nThis project is unique because it supports network-wide generalized evacuation traffic prediction across multiple hurricanes rather than training and evaluating a model on a single event. The data can be reused by researchers, transportation agencies, and emergency management practitioners to develop, compare, or validate traffic prediction models for hurricane evacuation planning and real-time operational decision support. The project may also support future studies on dynamic graph learning, traffic forecasting, transportation resilience, and disaster response modeling.\nThe intended audience includes researchers in transportation engineering, artificial intelligence, disaster resilience, emergency management, and infrastructure systems, as well as practitioners interested in evacuation traffic forecasting and network-wide traffic monitoring during hurricane events.", + "lang": "en-Us" + } + ], + "subjects": [ + { "subject": "evacuation traffic prediction" }, + { "subject": "transportation networks" }, + { "subject": "emergency management" }, + { "subject": "dynamic graph learning" } + ], + "language": "English", + "identifiers": [{ "identifierType": "Project ID", "identifier": "PRJ-6362" }], + "fundingReferences": [ + { + "awardTitle": "EAGER-SAI: Exploring Pathways of Adaptive Infrastructure Management with Rapidly Intensifying Hurricanes", + "awardNumber": "2122135", + "funderName": "NSF" + } + ], + "relatedIdentifiers": [], + "rightsList": [ + { + "lang": "en", + "rights": "Open Data Commons Attribution", + "rightsUri": null, + "schemeUri": null, + "rightsIdentifier": null, + "rightsIdentifierScheme": null + } + ], + "url": "https://www.designsafe-ci.org/data/browser/public/designsafe.storage.published/PRJ-6362", + "prefix": "doi:10.17603/DS2" +} diff --git a/designsafe/apps/projects/models/agave/base.py b/designsafe/apps/projects/models/agave/base.py index 029a50c2b2..55db0921e6 100644 --- a/designsafe/apps/projects/models/agave/base.py +++ b/designsafe/apps/projects/models/agave/base.py @@ -1,17 +1,21 @@ """Base""" import datetime +import json import logging + import six -import json -import os from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from pytas.http import TASClient -from designsafe.apps.data.models.agave.base import Model as MetadataModel + from designsafe.apps.data.models.agave import fields -from designsafe.apps.data.models.elasticsearch import IndexedPublication, IndexedPublicationLegacy +from designsafe.apps.data.models.agave.base import Model as MetadataModel +from designsafe.apps.data.models.elasticsearch import ( + IndexedPublication, +) from designsafe.libs.elasticsearch.exceptions import DocumentNotFound + logger = logging.getLogger(__name__) class FileObjModel(MetadataModel): @@ -30,7 +34,7 @@ class RelatedEntity(MetadataModel): def to_body_dict(self): """Serialize to Agave's REST API payload JSON.""" - body_dict = super(RelatedEntity, self).to_body_dict() + body_dict = super().to_body_dict() body_dict['_relatedFields'] = [] for attrname, field in six.iteritems(self._meta._related_fields): body_dict['_relatedFields'].append(attrname) @@ -127,7 +131,7 @@ class Project(MetadataModel): @property def system(self): - return 'project-{uuid}'.format(uuid=self.uuid) + return f'project-{self.uuid}' @property def collaborators(self): @@ -138,7 +142,7 @@ def collaborators(self): } def to_body_dict(self): - body_dict = super(Project, self).to_body_dict() + body_dict = super().to_body_dict() body_dict['_related'] = {} for attrname, field in six.iteritems(self._meta._related_fields): body_dict['_related'][attrname] = field.rel_cls.model_name @@ -227,7 +231,7 @@ def save(self, ag): prj = self.manager().get(ag, self.uuid) if prj.project_id and prj.project_id != 'None': self.project_id = prj.project_id - super(Project, self).save(ag) + super().save(ag) def related_entities(self, offset=0, limit=300): from designsafe.apps.projects.models.utils import lookup_model @@ -418,7 +422,7 @@ def to_datacite_json(self,project=None): ) if getattr(self, 'data_type', False): - attributes['types']['resourceType'] += '/{}'.format(self.data_type) + attributes['types']['resourceType'] += f'/{self.data_type}' attributes['types']['resourceTypeGeneral'] = 'Dataset' attributes['descriptions'] = [ @@ -534,7 +538,7 @@ def generate_creators(authors): details = {} if user_obj and user_tas: - author_name = "{} {}".format(user_obj.first_name, user_obj.last_name) + author_name = f"{user_obj.first_name} {user_obj.last_name}" details.update({ "@type": "Person", 'name': author_name, diff --git a/designsafe/apps/projects/models/agave/experimental.py b/designsafe/apps/projects/models/agave/experimental.py index e44d807326..9025cff4af 100644 --- a/designsafe/apps/projects/models/agave/experimental.py +++ b/designsafe/apps/projects/models/agave/experimental.py @@ -1,9 +1,13 @@ """Experimental project models""" import logging -import six -from designsafe.apps.data.models.agave.base import Model as MetadataModel + from designsafe.apps.data.models.agave import fields -from designsafe.apps.projects.models.agave.base import RelatedEntity, Project, FileObjModel +from designsafe.apps.data.models.agave.base import Model as MetadataModel +from designsafe.apps.projects.models.agave.base import ( + FileObjModel, + Project, + RelatedEntity, +) logger = logging.getLogger(__name__) @@ -65,7 +69,7 @@ class Experiment(RelatedEntity): def to_datacite_json(self, project=None): """Serialize object to datacite JSON.""" - attributes = super(Experiment, self).to_datacite_json() + attributes = super().to_datacite_json() if hasattr(self, 'experimental_facility') and len(self.experimental_facility) and ('None' not in self.experimental_facility): attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.experimental_facility.title(), } @@ -103,9 +107,7 @@ def to_datacite_json(self, project=None): 'awardNumber': award['number'], "funderName": award.get("fundingSource", "N/A"), }) - attributes['types']['resourceType'] = "Experiment/{experiment_type}".format( - experiment_type=self.experiment_type.title() - ) + attributes['types']['resourceType'] = f"Experiment/{self.experiment_type.title()}" # related works are not required, so they can be missing... attributes['relatedIdentifiers'] = [] for r_work in self.related_work: @@ -131,7 +133,7 @@ def to_datacite_json(self, project=None): def to_dataset_json(self): """Serialize object to dataset JSON.""" - attributes = super(Experiment, self).to_dataset_json() + attributes = super().to_dataset_json() attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.experimental_facility.title(), } ] @@ -142,9 +144,7 @@ def to_dataset_json(self): "name": self.experimental_facility, } ] - attributes['types']['resourceType'] = "Experiment/{experiment_type}".format( - experiment_type=self.experiment_type.title() - ) + attributes['types']['resourceType'] = f"Experiment/{self.experiment_type.title()}" return attributes class Analysis(RelatedEntity): diff --git a/designsafe/apps/projects/models/agave/hybrid_simulation.py b/designsafe/apps/projects/models/agave/hybrid_simulation.py index a2d6ec3c86..7081f17005 100644 --- a/designsafe/apps/projects/models/agave/hybrid_simulation.py +++ b/designsafe/apps/projects/models/agave/hybrid_simulation.py @@ -1,9 +1,13 @@ """Hybrid Simulation project models""" import logging -import six -from designsafe.apps.data.models.agave.base import Model as MetadataModel + from designsafe.apps.data.models.agave import fields -from designsafe.apps.projects.models.agave.base import RelatedEntity, Project, FileObjModel +from designsafe.apps.data.models.agave.base import Model as MetadataModel +from designsafe.apps.projects.models.agave.base import ( + FileObjModel, + Project, + RelatedEntity, +) logger = logging.getLogger(__name__) @@ -76,15 +80,11 @@ class HybridSimulation(RelatedEntity): def to_datacite_json(self, project=None): """Serialize object to datacite JSON.""" - attributes = super(HybridSimulation, self).to_datacite_json() + attributes = super().to_datacite_json() if self.simulation_type_other: - attributes['types']['resourceType'] = "Simulation/{simulation_type}".format( - simulation_type=self.simulation_type_other.title() - ) + attributes['types']['resourceType'] = f"Simulation/{self.simulation_type_other.title()}" else: - attributes['types']['resourceType'] = "Simulation/{simulation_type}".format( - simulation_type=self.simulation_type.title() - ) + attributes['types']['resourceType'] = f"Simulation/{self.simulation_type.title()}" if self.facility: attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.facility["name"], } @@ -147,7 +147,7 @@ def to_datacite_json(self, project=None): def to_dataset_json(self): """Serialize object to dataset JSON.""" - attributes = super(HybridSimulation, self).to_dataset_json() + attributes = super().to_dataset_json() attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.facility.title(), } ] @@ -159,13 +159,9 @@ def to_dataset_json(self): } ] if self.simulation_type_other: - attributes['types']['resourceType'] = "Simulation/{simulation_type}".format( - simulation_type=self.simulation_type_other.title() - ) + attributes['types']['resourceType'] = f"Simulation/{self.simulation_type_other.title()}" else: - attributes['types']['resourceType'] = "Simulation/{simulation_type}".format( - simulation_type=self.simulation_type.title() - ) + attributes['types']['resourceType'] = f"Simulation/{self.simulation_type.title()}" return attributes diff --git a/designsafe/apps/projects/models/agave/rapid.py b/designsafe/apps/projects/models/agave/rapid.py index a97d1479ce..e61b8c1acf 100644 --- a/designsafe/apps/projects/models/agave/rapid.py +++ b/designsafe/apps/projects/models/agave/rapid.py @@ -1,9 +1,13 @@ """Rapid project models""" import logging -import json -from designsafe.apps.data.models.agave.base import Model as MetadataModel + from designsafe.apps.data.models.agave import fields -from designsafe.apps.projects.models.agave.base import RelatedEntity, Project, FileObjModel +from designsafe.apps.data.models.agave.base import Model as MetadataModel +from designsafe.apps.projects.models.agave.base import ( + FileObjModel, + Project, + RelatedEntity, +) logger = logging.getLogger(__name__) @@ -19,7 +23,7 @@ def __init__(self, *args, **kwargs): kwargs['value']['nhTypes'] = [nh_type] if nh_type_other: kwargs['value']['nhTypes'] = [nh_type_other] - super(FieldReconProject, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) model_name = 'designsafe.project' team_members = fields.ListField('Team Members') @@ -79,10 +83,8 @@ class Mission(RelatedEntity): def to_datacite_json(self, project=None): """Serialize object to datacite JSON.""" - attributes = super(Mission, self).to_datacite_json() - attributes['types']['resourceType'] = "Mission/{location}".format( - location=self.location.title() - ) + attributes = super().to_datacite_json() + attributes['types']['resourceType'] = f"Mission/{self.location.title()}" if self.facility: attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.facility["name"], } @@ -145,7 +147,7 @@ def to_datacite_json(self, project=None): def to_dataset_json(self): """Serialize object to dataset JSON.""" - attributes = super(Mission, self).to_dataset_json() + attributes = super().to_dataset_json() attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.facility.title(), } ] @@ -156,9 +158,7 @@ def to_dataset_json(self): "name": self.facility, } ] - attributes['types']['resourceType'] = "Mission/{location}".format( - location=self.location.title() - ) + attributes['types']['resourceType'] = f"Mission/{self.location.title()}" return attributes @@ -270,7 +270,7 @@ class Report(RelatedEntity): def to_datacite_json(self, project=None): """Serialize object to datacite JSON.""" - attributes = super(Report, self).to_datacite_json() + attributes = super().to_datacite_json() attributes['types']['resourceType'] = "Project/Report" if self.facility: attributes["subjects"] = attributes.get("subjects", []) + [ @@ -334,7 +334,7 @@ def to_datacite_json(self, project=None): def to_dataset_json(self): """Serialize object to dataset JSON.""" - attributes = super(Report, self).to_dataset_json() + attributes = super().to_dataset_json() attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.facility.title(), } ] diff --git a/designsafe/apps/projects/models/agave/simulation.py b/designsafe/apps/projects/models/agave/simulation.py index de91a6bd11..daf9c5d3b6 100644 --- a/designsafe/apps/projects/models/agave/simulation.py +++ b/designsafe/apps/projects/models/agave/simulation.py @@ -1,9 +1,13 @@ """Simulation project models""" import logging -import six -from designsafe.apps.data.models.agave.base import Model as MetadataModel + from designsafe.apps.data.models.agave import fields -from designsafe.apps.projects.models.agave.base import RelatedEntity, Project, FileObjModel +from designsafe.apps.data.models.agave.base import Model as MetadataModel +from designsafe.apps.projects.models.agave.base import ( + FileObjModel, + Project, + RelatedEntity, +) logger = logging.getLogger(__name__) @@ -60,15 +64,11 @@ class Simulation(RelatedEntity): def to_datacite_json(self, project=None): """Serialize object to datacite JSON.""" - attributes = super(Simulation, self).to_datacite_json() + attributes = super().to_datacite_json() if self.simulation_type_other: - attributes['types']['resourceType'] = "Simulation/{simulation_type}".format( - simulation_type=self.simulation_type_other.title() - ) + attributes['types']['resourceType'] = f"Simulation/{self.simulation_type_other.title()}" else: - attributes['types']['resourceType'] = "Simulation/{simulation_type}".format( - simulation_type=self.simulation_type.title() - ) + attributes['types']['resourceType'] = f"Simulation/{self.simulation_type.title()}" if self.facility: attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.facility["name"], } @@ -131,7 +131,7 @@ def to_datacite_json(self, project=None): def to_dataset_json(self): """Serialize object to dataset JSON.""" - attributes = super(Simulation, self).to_dataset_json() + attributes = super().to_dataset_json() attributes["subjects"] = attributes.get("subjects", []) + [ {"subject": self.facility.title(), } ] @@ -143,13 +143,9 @@ def to_dataset_json(self): } ] if self.simulation_type_other: - attributes['types']['resourceType'] = "Simulation/{simulation_type}".format( - simulation_type=self.simulation_type_other.title() - ) + attributes['types']['resourceType'] = f"Simulation/{self.simulation_type_other.title()}" else: - attributes['types']['resourceType'] = "Simulation/{simulation_type}".format( - simulation_type=self.simulation_type.title() - ) + attributes['types']['resourceType'] = f"Simulation/{self.simulation_type.title()}" return attributes diff --git a/designsafe/apps/projects/models/elasticsearch.py b/designsafe/apps/projects/models/elasticsearch.py index 1c77416e4c..da07b967e4 100644 --- a/designsafe/apps/projects/models/elasticsearch.py +++ b/designsafe/apps/projects/models/elasticsearch.py @@ -1,16 +1,17 @@ -from __future__ import unicode_literals, absolute_import import logging -import json + from django.conf import settings -from django.db import models -from elasticsearch_dsl.connections import connections -from elasticsearch_dsl import (Search, Document, Date, Nested, - analyzer, Object, Text, Long, - Boolean, Keyword, - GeoPoint, MetaField) -from elasticsearch_dsl.query import Q -from elasticsearch import TransportError, ConnectionTimeout -from designsafe.libs.elasticsearch.analyzers import path_analyzer +from elasticsearch_dsl import ( + Boolean, + Date, + Document, + Keyword, + Long, + MetaField, + Nested, + Object, + Text, +) #pylint: disable=invalid-name logger = logging.getLogger(__name__) diff --git a/designsafe/apps/projects/models/utils.py b/designsafe/apps/projects/models/utils.py index 2073843397..41e21e6ebc 100644 --- a/designsafe/apps/projects/models/utils.py +++ b/designsafe/apps/projects/models/utils.py @@ -1,10 +1,13 @@ """Utilities for projects models.""" import logging -from designsafe.apps.projects.models.agave import base -from designsafe.apps.projects.models.agave import experimental -from designsafe.apps.projects.models.agave import simulation -from designsafe.apps.projects.models.agave import hybrid_simulation -from designsafe.apps.projects.models.agave import rapid + +from designsafe.apps.projects.models.agave import ( + base, + experimental, + hybrid_simulation, + rapid, + simulation, +) from designsafe.apps.projects.models.agave.base import Project logger = logging.getLogger(__name__) @@ -67,7 +70,7 @@ def lookup_model(entity=None, name=None): if isinstance(entity, dict): ename = '{ptype}_project'.format(ptype=entity['value'].get('projectType', '')) else: - ename = '{ptype}_project'.format(ptype=entity.value.project_type) + ename = f'{entity.value.project_type}_project' modules = [base, experimental, simulation, hybrid_simulation, rapid] else: @@ -91,6 +94,6 @@ def lookup_model(entity=None, name=None): if cls is None and ename.endswith('_project'): return Project elif cls is None: - raise AttributeError("Model '{model}' needed for '{name}' does not exist".format(model=name, name=ename)) + raise AttributeError(f"Model '{name}' needed for '{ename}' does not exist") return cls diff --git a/designsafe/apps/projects/tests.py b/designsafe/apps/projects/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/projects/tests.py +++ b/designsafe/apps/projects/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/projects/views.py b/designsafe/apps/projects/views.py index 91ea44a218..b8e4ee02a7 100644 --- a/designsafe/apps/projects/views.py +++ b/designsafe/apps/projects/views.py @@ -1,3 +1,2 @@ -from django.shortcuts import render # Create your views here. diff --git a/designsafe/apps/rapid/apps.py b/designsafe/apps/rapid/apps.py index 4b58cf6380..fd0063020b 100644 --- a/designsafe/apps/rapid/apps.py +++ b/designsafe/apps/rapid/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig + class DesignRapidConfig(AppConfig): name = 'designsafe.apps.rapid' label = 'designsafe_rapid' diff --git a/designsafe/apps/rapid/forms.py b/designsafe/apps/rapid/forms.py index b1bc5c174b..e3182d0d73 100644 --- a/designsafe/apps/rapid/forms.py +++ b/designsafe/apps/rapid/forms.py @@ -1,8 +1,7 @@ -from django import forms -from django.core.validators import MinValueValidator, MaxValueValidator -from designsafe.apps.rapid.models import RapidNHEvent, RapidNHEventType import logging +from django import forms +from django.core.validators import MaxValueValidator, MinValueValidator logger = logging.getLogger(__name__) diff --git a/designsafe/apps/rapid/models.py b/designsafe/apps/rapid/models.py index e61e8b0ff2..732f0400d6 100644 --- a/designsafe/apps/rapid/models.py +++ b/designsafe/apps/rapid/models.py @@ -1,9 +1,16 @@ -from django.conf import settings -from datetime import datetime import logging -from elasticsearch_dsl import (Document, Text, Date, Nested, - Boolean, GeoPoint, MetaField, Text, - Keyword) + +from django.conf import settings +from elasticsearch_dsl import ( + Date, + Document, + GeoPoint, + Keyword, + MetaField, + Nested, + Text, +) + #from designsafe.connections import connections logger = logging.getLogger(__name__) @@ -65,4 +72,4 @@ class Meta: def save(self, **kwargs): # self.created_date = datetime.utcnow() - return super(RapidNHEvent, self).save(**kwargs) + return super().save(**kwargs) diff --git a/designsafe/apps/rapid/tests.py b/designsafe/apps/rapid/tests.py index 4e4cc3d6ca..57536a63c6 100644 --- a/designsafe/apps/rapid/tests.py +++ b/designsafe/apps/rapid/tests.py @@ -1,12 +1,10 @@ -from django.test import TestCase -from django.urls import reverse -from django.contrib.auth import get_user_model, models, signals -from designsafe.apps.rapid.models import RapidNHEvent from unittest import skip -import mock -import requests_mock + import pytest -import os, settings +from django.contrib.auth import get_user_model, models +from django.test import TestCase +from django.urls import reverse + class RapidTests(TestCase): """ diff --git a/designsafe/apps/rapid/urls.py b/designsafe/apps/rapid/urls.py index a4028b9a0c..0f507fbd8d 100644 --- a/designsafe/apps/rapid/urls.py +++ b/designsafe/apps/rapid/urls.py @@ -1,7 +1,6 @@ -from django.urls import include, re_path as url, path -from django.urls import reverse -from designsafe.apps.rapid import views +from django.urls import re_path as url +from designsafe.apps.rapid import views urlpatterns = [ url(r'^event-types/?$', views.get_event_types, name="get_event_types"), diff --git a/designsafe/apps/rapid/views.py b/designsafe/apps/rapid/views.py index 801a356403..b63e5fcefc 100644 --- a/designsafe/apps/rapid/views.py +++ b/designsafe/apps/rapid/views.py @@ -1,31 +1,31 @@ # pylint: disable=invalid-name -import uuid -import os -import json import io -from datetime import datetime -from PIL import Image -from elasticsearch import TransportError, ConnectionTimeout -from django.urls import reverse -from django.http import JsonResponse, HttpResponseRedirect, HttpResponseNotFound, HttpResponseBadRequest -from django.shortcuts import render -from django.contrib.auth.decorators import login_required, user_passes_test -from django.contrib.auth import get_user_model, models -from django.db.models import Q -from tapipy.tapis import Tapis -from django.views.decorators.cache import cache_page - - +import json import logging -from designsafe.apps.rapid.models import RapidNHEventType, RapidNHEvent -from designsafe.apps.rapid import forms as rapid_forms +import os +import uuid +from datetime import UTC, datetime + import requests -from django.http import JsonResponse +from django.contrib.auth import get_user_model, models +from django.contrib.auth.decorators import login_required, user_passes_test +from django.http import ( + HttpResponseBadRequest, + HttpResponseNotFound, + HttpResponseRedirect, + JsonResponse, +) +from django.shortcuts import render +from django.urls import reverse +from django.views.decorators.cache import cache_page from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET -from django.core.cache import cache -import threading +from elasticsearch import ConnectionTimeout, TransportError +from PIL import Image + +from designsafe.apps.rapid import forms as rapid_forms +from designsafe.apps.rapid.models import RapidNHEvent, RapidNHEventType logger = logging.getLogger(__name__) metrics_logger = logging.getLogger('metrics') @@ -98,7 +98,7 @@ def opentopo_data(request): response_data = response.json() return JsonResponse(response_data, safe=False, status=response.status_code) except requests.RequestException as e: - logger.error(f"Error fetching URL {OPEN_TOPO_REQUEST_FOR_WHOLE_WORLD}: {str(e)}") + logger.error("Error fetching URL %s: %s,", OPEN_TOPO_REQUEST_FOR_WHOLE_WORLD, e) return JsonResponse({'error': str(e)}, status=500) @@ -155,7 +155,7 @@ def admin_create_event(request): ev.event_date = form.cleaned_data["event_date"] ev.event_type = form.cleaned_data["event_type"] ev.title = form.cleaned_data["title"] - ev.created_date = datetime.utcnow() + ev.created_date = datetime.now(tz=UTC) if request.FILES: try: image_uuid = str(uuid.uuid1()) @@ -163,7 +163,8 @@ def admin_create_event(request): thumb = thumbnail_image(f) handle_uploaded_image(thumb, image_uuid) ev.main_image_uuid = image_uuid - except: + except Exception: + logger.exception("") return HttpResponseBadRequest("Hmm, a bad file perhaps?") try: ev.save(refresh=True) @@ -188,7 +189,8 @@ def admin_create_event(request): def admin_edit_event(request, event_id): try: event = RapidNHEvent.get(event_id) - except: + except Exception: + logger.exception("") return HttpResponseNotFound() metrics_logger.info('Rapid Admin edit event', @@ -232,7 +234,8 @@ def admin_edit_event(request, event_id): thumb = thumbnail_image(f) handle_uploaded_image(thumb, image_uuid) event.main_image_uuid = image_uuid - except: + except Exception: + logger.exception("") return HttpResponseBadRequest("Hmm, a bad file perhaps?") if old_image_uuid: os.remove(os.path.join(settings.DESIGNSAFE_UPLOAD_PATH, 'RAPID', 'images', old_image_uuid)) @@ -268,14 +271,15 @@ def admin_delete_event(request, event_id): try: event = RapidNHEvent.get(event_id) - except: + except Exception: + logger.exception("") return HttpResponseNotFound() if request.method == 'POST': image_uuid = event.main_image_uuid try: os.remove(os.path.join(settings.DESIGNSAFE_UPLOAD_PATH, 'RAPID', 'images', image_uuid)) - except: - pass + except Exception: + logger.exception("") event.delete(refresh=True) return HttpResponseRedirect(reverse('designsafe_rapid:admin')) @@ -285,7 +289,8 @@ def admin_delete_event(request, event_id): def admin_event_datasets(request, event_id): try: event = RapidNHEvent.get(event_id) - except: + except Exception: + logger.exception("") return HttpResponseNotFound() context = { @@ -299,7 +304,8 @@ def admin_event_datasets(request, event_id): def admin_event_add_dataset(request, event_id): try: event = RapidNHEvent.get(event_id) - except: + except Exception: + logger.exception("") return HttpResponseNotFound() form = rapid_forms.RapidNHEventDatasetForm(request.POST or None) @@ -333,7 +339,8 @@ def admin_event_add_dataset(request, event_id): def admin_event_edit_dataset(request, event_id, dataset_id): try: event = RapidNHEvent.get(event_id) - except: + except Exception: + logger.exception("") return HttpResponseNotFound() dataset = next((d for d in event.datasets if d.id == dataset_id), None) @@ -375,7 +382,8 @@ def admin_event_edit_dataset(request, event_id, dataset_id): def admin_event_delete_dataset(request, event_id, dataset_id): try: event = RapidNHEvent.get(event_id) - except: + except Exception: + logger.exception("") return HttpResponseNotFound() if request.method == 'POST': event.datasets = [ds for ds in event.datasets if ds.id != dataset_id] diff --git a/designsafe/apps/search/apps.py b/designsafe/apps/search/apps.py index ee0663af76..1544a86f3d 100644 --- a/designsafe/apps/search/apps.py +++ b/designsafe/apps/search/apps.py @@ -1,5 +1,6 @@ from django.apps import AppConfig + class DesignSafeSearchConfig(AppConfig): name = 'designsafe.apps.search' label = 'designsafe_search' diff --git a/designsafe/apps/search/search_indexes.py b/designsafe/apps/search/search_indexes.py index 110a00c6cd..3cfe96cb66 100644 --- a/designsafe/apps/search/search_indexes.py +++ b/designsafe/apps/search/search_indexes.py @@ -1,19 +1,20 @@ import logging import re -from haystack import indexes + +from cms.models import CMSPlugin, Title + +# from cms.toolbar.toolbar import CMSToolbar +from django.conf import settings from django.db.models import Q from django.template import RequestContext -from django.utils import timezone + #rom djangocms_text_ckeditor.models import Text #from django.utils.text import smart_split from django.test import RequestFactory - -from django.utils.html import strip_tags +from django.utils import timezone from django.utils.encoding import force_str - -from cms.models import Title, CMSPlugin, Page -# from cms.toolbar.toolbar import CMSToolbar -from django.conf import settings +from django.utils.html import strip_tags +from haystack import indexes logger = logging.getLogger(__name__) @@ -57,14 +58,15 @@ def prepare(self, obj): request = rf.get("/") request.session = {} request.LANGUAGE_CODE = settings.LANGUAGE_CODE - self.prepared_data = super(TextPluginIndex, self).prepare(page) + self.prepared_data = super().prepare(page) plugins = CMSPlugin.objects.filter(placeholder__in=obj.page.placeholders.all()) text = '' for base_plugin in plugins: try: instance, plugin_type = base_plugin.get_plugin_instance() except Exception as e: - logger.debug(f"{type(e)}: {e}") + logger.exception("") + logger.debug("%s: %s", type(e), {e}) continue if instance is None: # this is an empty plugin diff --git a/designsafe/apps/search/tasks.py b/designsafe/apps/search/tasks.py index 99fc7e2a08..9228c69b4e 100644 --- a/designsafe/apps/search/tasks.py +++ b/designsafe/apps/search/tasks.py @@ -1,8 +1,9 @@ -from celery import shared_task -import datetime import logging -from django.core.management import call_command + +from celery import shared_task from django.conf import settings +from django.core.management import call_command + from designsafe.apps.data.tasks import agave_indexer logger = logging.getLogger(__name__) diff --git a/designsafe/apps/search/tests.py b/designsafe/apps/search/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/search/tests.py +++ b/designsafe/apps/search/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/search/urls.py b/designsafe/apps/search/urls.py index ba5be2f08e..61372e376d 100644 --- a/designsafe/apps/search/urls.py +++ b/designsafe/apps/search/urls.py @@ -1,6 +1,5 @@ -from django.urls import include, re_path as url -from django.urls import reverse -from django.utils.translation import gettext_lazy as _ +from django.urls import re_path as url + from designsafe.apps.search import views urlpatterns = [ diff --git a/designsafe/apps/search/views.py b/designsafe/apps/search/views.py index e020284299..38c5e6675c 100644 --- a/designsafe/apps/search/views.py +++ b/designsafe/apps/search/views.py @@ -1,17 +1,7 @@ -from django.core import serializers -from django.http import HttpResponse -from django.shortcuts import render -from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_POST # from agavepy.agave import AgaveException -from requests import HTTPError -from django.contrib.auth import get_user_model - -from itertools import chain - -import json import logging +from django.shortcuts import render logger = logging.getLogger(__name__) diff --git a/designsafe/apps/signals/admin.py b/designsafe/apps/signals/admin.py index 8c38f3f3da..b97a94f645 100644 --- a/designsafe/apps/signals/admin.py +++ b/designsafe/apps/signals/admin.py @@ -1,3 +1,2 @@ -from django.contrib import admin # Register your models here. diff --git a/designsafe/apps/signals/tests.py b/designsafe/apps/signals/tests.py index 7ce503c2dd..4929020489 100644 --- a/designsafe/apps/signals/tests.py +++ b/designsafe/apps/signals/tests.py @@ -1,3 +1,2 @@ -from django.test import TestCase # Create your tests here. diff --git a/designsafe/apps/signals/websocket_consumers.py b/designsafe/apps/signals/websocket_consumers.py index d762a13aff..c70b4c792e 100644 --- a/designsafe/apps/signals/websocket_consumers.py +++ b/designsafe/apps/signals/websocket_consumers.py @@ -1,6 +1,7 @@ """Websocket consumers""" import json + from channels.generic.websocket import AsyncWebsocketConsumer from channels.layers import get_channel_layer diff --git a/designsafe/apps/token_access/admin.py b/designsafe/apps/token_access/admin.py index 65d66c7848..8a677eb119 100644 --- a/designsafe/apps/token_access/admin.py +++ b/designsafe/apps/token_access/admin.py @@ -1,4 +1,5 @@ from django.contrib import admin + from .models import Token diff --git a/designsafe/apps/token_access/middleware.py b/designsafe/apps/token_access/middleware.py index afb738c68f..d6978b9d62 100644 --- a/designsafe/apps/token_access/middleware.py +++ b/designsafe/apps/token_access/middleware.py @@ -1,7 +1,8 @@ from django.http import HttpResponseForbidden -from .models import Token from django.utils.deprecation import MiddlewareMixin +from .models import Token + class TokenAuthenticationMiddleware(MiddlewareMixin): """ diff --git a/designsafe/apps/token_access/models.py b/designsafe/apps/token_access/models.py index 68534af297..8a2afb580b 100644 --- a/designsafe/apps/token_access/models.py +++ b/designsafe/apps/token_access/models.py @@ -1,8 +1,9 @@ -from django.db import models -from django.conf import settings import binascii import os +from django.conf import settings +from django.db import models + class Token(models.Model): """ @@ -17,7 +18,7 @@ class Token(models.Model): def save(self, *args, **kwargs): if not self.token: self.token = Token.generate_token() - return super(Token, self).save(*args, **kwargs) + return super().save(*args, **kwargs) @staticmethod def generate_token(): @@ -25,7 +26,7 @@ def generate_token(): @property def header(self): - return 'Token {0}'.format(self.token) + return f'Token {self.token}' def __str__(self): return self.token diff --git a/designsafe/apps/token_access/tests.py b/designsafe/apps/token_access/tests.py index bfcc2ca96c..50b0e85451 100644 --- a/designsafe/apps/token_access/tests.py +++ b/designsafe/apps/token_access/tests.py @@ -1,7 +1,8 @@ +import requests_mock from django.contrib.auth import get_user_model from django.test import TestCase + from .models import Token -import requests_mock class TokenAccessTest(TestCase): @@ -17,7 +18,7 @@ def test_model_save(self): def test_model_header(self): token = Token.objects.get(user__username='ds_user') - expected_header = 'Token {0}'.format(token.token) + expected_header = f'Token {token.token}' self.assertEqual(expected_header, token.header) def test_token_access_middleware(self): diff --git a/designsafe/apps/webhooks/unit_test.py b/designsafe/apps/webhooks/unit_test.py index 0720b44510..b1c51c4fdf 100644 --- a/designsafe/apps/webhooks/unit_test.py +++ b/designsafe/apps/webhooks/unit_test.py @@ -1,15 +1,17 @@ import json import os +from unittest.mock import MagicMock, patch from urllib.parse import urlencode -from mock import patch, MagicMock -from django.test import TestCase, TransactionTestCase, override_settings + from django.contrib.auth import get_user_model from django.db.models import signals +from django.test import TestCase, TransactionTestCase, override_settings from django.urls import reverse from tapipy.tapis import TapisResult + +from designsafe.apps.api.exceptions import ApiException from designsafe.apps.api.notifications.models import Notification from designsafe.apps.api.notifications.receivers import send_notification_ws -from designsafe.apps.api.exceptions import ApiException from designsafe.apps.webhooks.views import validate_tapis_job @@ -33,7 +35,6 @@ def setUp(self): def tearDown(self): self.user_model_patcher.stop() - pass def test_valid_job(self): job = validate_tapis_job("id", "ds_user") diff --git a/designsafe/apps/webhooks/urls.py b/designsafe/apps/webhooks/urls.py index 35338f7161..9d5cc134dd 100644 --- a/designsafe/apps/webhooks/urls.py +++ b/designsafe/apps/webhooks/urls.py @@ -2,8 +2,8 @@ """ from django.urls import path -from designsafe.apps.webhooks import views +from designsafe.apps.webhooks import views app_name = "webhooks" urlpatterns = [ diff --git a/designsafe/apps/webhooks/views.py b/designsafe/apps/webhooks/views.py index 47b6bfd694..c46bd62bc7 100644 --- a/designsafe/apps/webhooks/views.py +++ b/designsafe/apps/webhooks/views.py @@ -6,24 +6,22 @@ import json import logging +from django.conf import settings from django.contrib.auth import get_user_model -from django.views.decorators.csrf import csrf_exempt -from django.utils.decorators import method_decorator +from django.core.exceptions import ObjectDoesNotExist from django.db import transaction from django.http import HttpResponse, HttpResponseBadRequest -from django.core.exceptions import ObjectDoesNotExist -from django.conf import settings - +from django.utils.decorators import method_decorator +from django.views.decorators.csrf import csrf_exempt from requests import HTTPError from tapipy.errors import BaseTapyException +from designsafe.apps.api.exceptions import ApiException from designsafe.apps.api.notifications.models import Notification from designsafe.apps.api.tasks import agave_indexer from designsafe.apps.api.views import BaseApiView -from designsafe.apps.api.exceptions import ApiException from designsafe.apps.workspace.api.utils import check_job_for_timeout - logger = logging.getLogger(__name__) TERMINAL_JOB_STATES = ["FINISHED", "CANCELLED", "FAILED"] @@ -46,7 +44,10 @@ def validate_tapis_job(job_uuid, job_owner, disallowed_states=None): # Validate the job UUID against the owner if job_data.owner != job_owner: logger.error( - f"Tapis job (owner='{job_data.owner}', status='{job_data.status}) for this event (owner='{job_owner}') is not valid" + "Tapis job (owner='%s', status='%s) for this event (owner='%s') is not valid", + job_data.owner, + job_data.status, + job_owner, ) raise ApiException("Unable to find a related valid job for this notification.") @@ -92,7 +93,10 @@ def post(self, request, *args, **kwargs): # Do nothing on job status not in portal notification states if job_status not in settings.PORTAL_JOB_NOTIFICATION_STATES: logger.info( - f"Job UUID {job_uuid} for owner {username} entered {job_status} state (no notification sent)" + "Job UUID %s for owner %s entered %s state (no notification sent)", + job_uuid, + username, + job_status, ) return HttpResponse("OK") @@ -100,7 +104,7 @@ def post(self, request, *args, **kwargs): if job_status == job_old_status: return HttpResponse("OK") - logger.info(f"JOB STATUS CHANGE: UUID={job_uuid} status={job_status}") + logger.info("JOB STATUS CHANGE: UUID=%s status=%s", job_uuid, job_status) event_data = { Notification.EVENT_TYPE: "job", @@ -143,7 +147,7 @@ def post(self, request, *args, **kwargs): return HttpResponse("OK") except (ObjectDoesNotExist, BaseTapyException, ApiException) as exc: - logger.exception(exc) + logger.exception("") return HttpResponseBadRequest(f"ERROR: {exc}") @@ -193,7 +197,7 @@ def post(self, request, *args, **kwargs): } except (HTTPError, BaseTapyException, ApiException) as exc: - logger.exception(exc) + logger.exception("") return HttpResponseBadRequest(f"ERROR: {exc}") Notification.objects.create(**event_data) diff --git a/designsafe/apps/workspace/admin.py b/designsafe/apps/workspace/admin.py index 5a215622c4..a21a27aad4 100644 --- a/designsafe/apps/workspace/admin.py +++ b/designsafe/apps/workspace/admin.py @@ -9,9 +9,9 @@ from designsafe.apps.workspace.models.app_descriptions import AppDescription from designsafe.apps.workspace.models.app_entries import ( AppListingEntry, - AppVariant, - AppTrayCategory, AppTag, + AppTrayCategory, + AppVariant, ) admin.site.register(AppDescription) diff --git a/designsafe/apps/workspace/api/urls.py b/designsafe/apps/workspace/api/urls.py index 1b239767a8..d30f5e8f60 100644 --- a/designsafe/apps/workspace/api/urls.py +++ b/designsafe/apps/workspace/api/urls.py @@ -2,8 +2,8 @@ """ from django.urls import path, re_path -from designsafe.apps.workspace.api import views +from designsafe.apps.workspace.api import views app_name = "workspace_api" urlpatterns = [ diff --git a/designsafe/apps/workspace/api/utils.py b/designsafe/apps/workspace/api/utils.py index 35d2b9f704..5bbdf7441e 100644 --- a/designsafe/apps/workspace/api/utils.py +++ b/designsafe/apps/workspace/api/utils.py @@ -1,7 +1,7 @@ """Workspace API Utils""" import json -from typing import Union + from tapipy.tapis import TapisResult @@ -13,7 +13,7 @@ def get_tapis_timeout_error_messages(job_id): ] -def _get_job_notes(job_notes: Union[str, TapisResult]): +def _get_job_notes(job_notes: str | TapisResult): """ Normalize `job.notes` as in older version of Tapis `notes` is a JSON-formatted string but this is being changed to a TapisResult object. Once all tenants are migrated to diff --git a/designsafe/apps/workspace/api/views.py b/designsafe/apps/workspace/api/views.py index 8f83fdc0c7..052b409d1a 100644 --- a/designsafe/apps/workspace/api/views.py +++ b/designsafe/apps/workspace/api/views.py @@ -3,32 +3,32 @@ :synopsys: Views to handle Workspace API """ -import logging import json +import logging from urllib.parse import urlparse -from django.http import JsonResponse + from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.db.models import F, Count +from django.db.models import Count, F from django.db.models.lookups import GreaterThan +from django.http import JsonResponse from django.urls import reverse +from tapipy.errors import BaseTapyException, InternalServerError, UnauthorizedError from tapipy.tapis import TapisResult -from tapipy.errors import InternalServerError, UnauthorizedError, BaseTapyException + from designsafe.apps.api.exceptions import ApiException -from designsafe.apps.api.users.utils import get_user_data -from designsafe.apps.api.views import AuthenticatedApiView +from designsafe.apps.api.users.utils import get_allocations, get_user_data from designsafe.apps.api.utils import get_client_ip +from designsafe.apps.api.views import AuthenticatedApiView from designsafe.apps.licenses.models import LICENSE_TYPES, get_license_info -from designsafe.libs.tapis.serializers import BaseTapisResultSerializer +from designsafe.apps.onboarding.steps.system_access_v3 import create_system_credentials +from designsafe.apps.workspace.api.utils import check_job_for_timeout from designsafe.apps.workspace.models.app_descriptions import AppDescription from designsafe.apps.workspace.models.app_entries import ( AppTrayCategory, AppVariant, ) -from designsafe.apps.api.users.utils import get_allocations -from designsafe.apps.workspace.api.utils import check_job_for_timeout -from designsafe.apps.onboarding.steps.system_access_v3 import create_system_credentials - +from designsafe.libs.tapis.serializers import BaseTapisResultSerializer logger = logging.getLogger(__name__) METRICS = logging.getLogger(f"metrics.{__name__}") @@ -79,7 +79,7 @@ def _get_user_app_license(license_type, user): def _get_systems( - user: object, can_exec: bool, systems: list = None, list_type: str = "ALL" + user: object, can_exec: bool, systems: list | None = None, list_type: str = "ALL" ) -> list: """List of all enabled systems of the specified can_exec type available for the user.""" tapis = user.tapis_oauth.client @@ -163,7 +163,9 @@ def test_system_needs_keys( return False except (InternalServerError, UnauthorizedError): logger.exception( - f"TMS_KEYS credential generation failed for system: {system_id}, user: {username}" + "TMS_KEYS credential generation failed for system: %s, user: %s", + system_id, + username, ) raise @@ -309,9 +311,7 @@ def _get_private_apps(self, user): listType="MINE", limit=-1, ) - my_apps = list( - map( - lambda app: { + my_apps = [{ "app_id": app.id, "app_type": "tapis", "bundle_id": None, @@ -322,10 +322,7 @@ def _get_private_apps(self, user): "label": getattr(app.notes, "label", app.id), "shortLabel": getattr(app.notes, "shortLabel", None), "version": app.version, - }, - apps_listing, - ) - ) + } for app in apps_listing] return my_apps @@ -739,13 +736,13 @@ def _submit_job(self, request, body, tapis, username): # job_post['parameterSet']['envVariables'] = job_post['parameterSet'].get('envVariables', []) + [license_var] # Test file listing on relevant systems to determine whether keys need to be pushed manually - for system_id in list( - set([job_post["archiveSystemId"], job_post["execSystemId"]]) - ): + for system_id in {job_post["archiveSystemId"], job_post["execSystemId"]}: system_needs_keys = test_system_needs_keys(tapis, username, system_id) if system_needs_keys: logger.info( - f"Keys for user {username} must be manually pushed to system: {system_needs_keys.id}" + "Keys for user %s must be manually pushed to system: %s", + username, + system_needs_keys.id, ) return {"execSys": system_needs_keys} @@ -826,7 +823,7 @@ def _submit_job(self, request, body, tapis, username): } ] - logger.info(f"user: {username} is submitting job: {job_post}") + logger.info("user: %s is submitting job: %s", username, job_post) response = tapis.jobs.submitJob( **job_post, headers={"X-Tapis-Tracking-ID": f"portals.{request.session.session_key}"}, diff --git a/designsafe/apps/workspace/cms_plugins.py b/designsafe/apps/workspace/cms_plugins.py index d5570627e6..00d445c496 100644 --- a/designsafe/apps/workspace/cms_plugins.py +++ b/designsafe/apps/workspace/cms_plugins.py @@ -1,15 +1,17 @@ """CMS plugins for Tools & Applications pages.""" import logging + from cms.plugin_base import CMSPluginBase from cms.plugin_pool import plugin_pool -from designsafe.apps.workspace.models.app_entries import ( - AppListingEntry, -) + from designsafe.apps.workspace.models.app_cms_plugins import ( AppCategoryListingPlugin, - RelatedAppsPlugin, AppVariantsPlugin, + RelatedAppsPlugin, +) +from designsafe.apps.workspace.models.app_entries import ( + AppListingEntry, ) logger = logging.getLogger(__name__) diff --git a/designsafe/apps/workspace/models/allocations.py b/designsafe/apps/workspace/models/allocations.py index b85200f8d0..5619eb0fa4 100644 --- a/designsafe/apps/workspace/models/allocations.py +++ b/designsafe/apps/workspace/models/allocations.py @@ -3,8 +3,8 @@ """ -from django.db import models from django.conf import settings +from django.db import models class UserAllocations(models.Model): diff --git a/designsafe/apps/workspace/models/app_cms_plugins.py b/designsafe/apps/workspace/models/app_cms_plugins.py index 2ba760254a..bfa405cd56 100644 --- a/designsafe/apps/workspace/models/app_cms_plugins.py +++ b/designsafe/apps/workspace/models/app_cms_plugins.py @@ -2,9 +2,10 @@ from cms.models.pluginmodel import CMSPlugin from django.db import models + from designsafe.apps.workspace.models.app_entries import ( - AppTrayCategory, AppListingEntry, + AppTrayCategory, ) diff --git a/designsafe/apps/workspace/models/app_descriptions.py b/designsafe/apps/workspace/models/app_descriptions.py index c07c7347b5..ed79b17f10 100644 --- a/designsafe/apps/workspace/models/app_descriptions.py +++ b/designsafe/apps/workspace/models/app_descriptions.py @@ -1,4 +1,5 @@ import logging + from django.db import models #pylint: disable=invalid-name @@ -10,7 +11,7 @@ class AppDescription(models.Model): appdescription = models.TextField(help_text='App dropdown description text for apps that have a dropdown.') def __str__(self): - return "%s" % (self.appid) + return f"{self.appid}" def desc_to_dict(self): return {'appId': self.appid, 'appDescription': self.appdescription} diff --git a/designsafe/apps/workspace/models/elasticsearch.py b/designsafe/apps/workspace/models/elasticsearch.py index 797d5e5fe5..95d1e7ba8f 100644 --- a/designsafe/apps/workspace/models/elasticsearch.py +++ b/designsafe/apps/workspace/models/elasticsearch.py @@ -1,18 +1,16 @@ import logging -import json + from django.conf import settings -from django.db import models -from elasticsearch_dsl.connections import connections -from elasticsearch_dsl import (Search, DocType, Date, Nested, - analyzer, Object, Text, Long, - Boolean, Keyword, - GeoPoint, MetaField) -from elasticsearch_dsl.query import Q -from elasticsearch import TransportError -from designsafe.libs.elasticsearch.analyzers import path_analyzer -from designsafe.libs.elasticsearch.utils import get_sha256_hash, new_es_client +from elasticsearch_dsl import ( + Date, + Document, + Keyword, + MetaField, + Nested, + Text, +) #pylint: disable=invalid-name logger = logging.getLogger(__name__) diff --git a/designsafe/apps/workspace/tests.py b/designsafe/apps/workspace/tests.py index 0e09e88518..5a724eec6b 100644 --- a/designsafe/apps/workspace/tests.py +++ b/designsafe/apps/workspace/tests.py @@ -1,10 +1,12 @@ import json -from mock import patch -from django.test import TestCase -from .models.app_descriptions import AppDescription from unittest import skip -from django.urls import reverse +from unittest.mock import patch + from django.contrib.auth import get_user_model +from django.test import TestCase +from django.urls import reverse + +from .models.app_descriptions import AppDescription @skip("TODOv3: Update apps api with Tapisv3") @@ -39,7 +41,7 @@ class TestAppsApiViews(TestCase): @classmethod def setUpClass(cls): - super(TestAppsApiViews, cls).setUpClass() + super().setUpClass() cls.mock_client_patcher = patch( "designsafe.apps.auth.models.TapisOAuthToken.client" ) @@ -48,7 +50,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.mock_client_patcher.stop() - super(TestAppsApiViews, cls).tearDownClass() + super().tearDownClass() def setUp(self): user = get_user_model().objects.get(pk=2) diff --git a/designsafe/apps/workspace/urls.py b/designsafe/apps/workspace/urls.py index 00cc09812a..fcad844b30 100644 --- a/designsafe/apps/workspace/urls.py +++ b/designsafe/apps/workspace/urls.py @@ -1,6 +1,7 @@ """Workspace URLs """ from django.urls import re_path + from designsafe.apps.workspace import views urlpatterns = [ diff --git a/designsafe/apps/workspace/utils.py b/designsafe/apps/workspace/utils.py index a84eeb9a96..670e1097cd 100644 --- a/designsafe/apps/workspace/utils.py +++ b/designsafe/apps/workspace/utils.py @@ -23,13 +23,10 @@ def setup_identity_file(username, agave, system, dir_path): # Create .Identity file with user's username identity_file = io.StringIO(username) - setattr(identity_file, 'name', '.Identity') + identity_file.name = '.Identity' return agave.files.importData( systemId=system, - filePath='/{username}/{dir_path}'.format( - username=username, - dir_path=dir_path - ), + filePath=f'/{username}/{dir_path}', fileToUpload=identity_file, fileName='.Identity' ) diff --git a/designsafe/apps/workspace/views.py b/designsafe/apps/workspace/views.py index 6c2ab98af8..715261bded 100644 --- a/designsafe/apps/workspace/views.py +++ b/designsafe/apps/workspace/views.py @@ -2,10 +2,10 @@ .. :module: apps.workspace.views :synopsis: Views to handle Workspace """ -from django.views.generic.base import TemplateView +from django.contrib.auth.decorators import login_required from django.utils.decorators import method_decorator from django.views.decorators.csrf import ensure_csrf_cookie -from django.contrib.auth.decorators import login_required +from django.views.generic.base import TemplateView @method_decorator(login_required, name="dispatch") @@ -16,4 +16,4 @@ class WorkspaceView(TemplateView): @method_decorator(ensure_csrf_cookie) def dispatch(self, request, *args, **kwargs): """Overwrite dispatch to ensure csrf cookie""" - return super(WorkspaceView, self).dispatch(request, *args, **kwargs) + return super().dispatch(request, *args, **kwargs) diff --git a/poetry.lock b/poetry.lock index 14724b9dcd..dc724091f7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -8408,6 +8408,34 @@ files = [ requests = "*" six = "*" +[[package]] +name = "ruff" +version = "0.16.7" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "ruff-0.16.7-py3-none-linux_armv6l.whl", hash = "sha256:727307773e7c7f9181d3ed3a2484186e56c1fa1874255911c74585eb2c7c19f9"}, + {file = "ruff-0.16.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9d61c258deabf58f34c67bd4bb4d939c7f2e6b5f0e59c1cdd1cf771b11cde929"}, + {file = "ruff-0.16.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ab81118df8945e0193d0240712aa4496573595b75185c3636ed825592a0f728"}, + {file = "ruff-0.16.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c196c968874fc8019da8e7163de7a1a370f111e2309b4b7dfea0fce950198d0"}, + {file = "ruff-0.16.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac8c3bd0a7e10ad31e6ce51e7a99f3cb772e69aecdd6b9ea7e99b362f62a62c0"}, + {file = "ruff-0.16.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398d3988edde000b5c75dc1b3f584708da9bc990de069c18909142580fec1af9"}, + {file = "ruff-0.16.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce05b62b770a8217c4646a9c4139fca00efe8fe5d71f87df2b243ff20d4584d1"}, + {file = "ruff-0.16.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af1b576fddb9d9ef2ececfb5fadcd6a624b25070ed85e3cfcfe449fc3ff6a7b9"}, + {file = "ruff-0.16.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ce7f8f22df67c93ed96c717f9128eadb797144ac2bad475cf536f31d6100c55"}, + {file = "ruff-0.16.7-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:06d0e93d04f392996435ebd600c153f65b47d73fbec2415aa99c5ee5756b3a5f"}, + {file = "ruff-0.16.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:142151a5e7b93c1b11111337142f89dd2fbfee92161225c99a97222f22e32656"}, + {file = "ruff-0.16.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e6651f97a342d8b35d54d8991544ca22169b86dc54111cb604666940c431b750"}, + {file = "ruff-0.16.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ef140c6eb935fa9a84c9c607dfb2cb1b85843c192e79265b0c54f35f557ea8e5"}, + {file = "ruff-0.16.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:53e39506a730fadeee0d998ed5946f30671f0db240c6c7c73bdabbe33604bb6f"}, + {file = "ruff-0.16.7-py3-none-win32.whl", hash = "sha256:2ea3470fcebcbc5df2fb0c6f3b90333fa9084c534e0111c038fa4a6ab9f1c4b7"}, + {file = "ruff-0.16.7-py3-none-win_amd64.whl", hash = "sha256:7ac26aca826e9e21d0f1cb25b54ac660760a9fdd094d3e4df9848232be98cfc6"}, + {file = "ruff-0.16.7-py3-none-win_arm64.whl", hash = "sha256:aab7f39e2c9df6c596216070f98eef1207b94f8516cca20c808826974971855b"}, + {file = "ruff-0.16.7.tar.gz", hash = "sha256:5f71d004ac1263b22fa39462ac5ae618a4b77d58981af2cc79bf79a29c12b1a6"}, +] + [[package]] name = "s3transfer" version = "0.16.0" @@ -10342,5 +10370,5 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" -python-versions = "^3.12" -content-hash = "3dbff3a6aa0c95fca8dff317946ca5bcc3893b03d3a5dc73cd80d4cabdd80dfe" +python-versions = ">=3.12,<4.0" +content-hash = "29953776df72907eb241bac6a1b9f3ca1701e20f11350ce9310daeeed1d338d5" diff --git a/pyproject.toml b/pyproject.toml index b2743d0c7e..d89975aad4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "designsafe" version = "6.0" description = "" -requires-python = "^3.12" +requires-python = ">=3.12,<4.0" readme = "README.md" dependencies = [ @@ -103,3 +103,18 @@ pytest-django = "^4.5.2" pytest-cov = "^2.10.0" pytest-mock = "^3.6.1" requests-mock = "^1.12.1" +ruff = "^0.16.7" + + +[tool.ruff.lint] +extend-select=["G"] +ignore = ["E501", "RUF012"] + +[tool.ruff] +extend-exclude = ["**/migrations/", + "*test.py", + "*tests.py", + "designsafe/apps/projects", + "designsafe/apps/api/projects", + "designsafe/apps/api/publications" + ] \ No newline at end of file