diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 0000000..54dda93 --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,20 @@ +name: Ruff +on: + push: + branches: [master] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/ruff-action@v3 + - run: ruff format --check + - run: ruff check . diff --git a/.project b/.project deleted file mode 100644 index d7973bc..0000000 --- a/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - explgbk - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/.pydevproject b/.pydevproject deleted file mode 100644 index 40e9f40..0000000 --- a/.pydevproject +++ /dev/null @@ -1,5 +0,0 @@ - - -Default -python 2.7 - diff --git a/dal/__init__.py b/dal/__init__.py deleted file mode 100755 index d2d6b66..0000000 --- a/dal/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'mshankar@slac.stanford.edu' diff --git a/dal/exp_cache.py b/dal/exp_cache.py index ffae464..ea6e972 100644 --- a/dal/exp_cache.py +++ b/dal/exp_cache.py @@ -1,61 +1,98 @@ import os import json -import time import datetime import pytz import dateutil.relativedelta import logging import re -import requests import threading import sched -from pymongo import ASCENDING, DESCENDING, ReadPreference -from bson import ObjectId +from pymongo import DESCENDING from kafka import KafkaConsumer -from kafka.errors import KafkaError -from context import logbookclient, instrument_scientists_run_table_defintions, usergroups, kafka_producer, local_kafka_events, reload_named_caches -from dal.explgbk import get_experiments_for_instrument, get_poc_feedback_changes, get_poc_feedback_document +from context import ( + logbookclient, + usergroups, + kafka_producer, + local_kafka_events, + reload_named_caches, +) +from dal.explgbk import ( + get_experiments_for_instrument, + get_poc_feedback_changes, + get_poc_feedback_document, +) -__author__ = 'mshankar@slac.stanford.edu' +__author__ = "mshankar@slac.stanford.edu" logger = logging.getLogger(__name__) all_experiment_names = set() roles_with_post_privileges = [] -class PeriodicUpdates(): + +class PeriodicUpdates: """ Gather experiment names to be updated periodically in the future. """ + def __init__(self): self.lock = threading.Lock() self.experiment_names = set() + def add(self, experiment_name): with self.lock: try: self.experiment_names.add(experiment_name) - except Exception as e: - logger.exception("Exception adding %s to periodic updater", experiment_name) + except Exception: + logger.exception( + "Exception adding %s to periodic updater", experiment_name + ) + def getAndReset(self): with self.lock: ret = self.experiment_names self.experiment_names = set() return ret + periodic_updates = PeriodicUpdates() + def init_app(app): - if 'experiments' not in list(logbookclient['explgbk_cache'].list_collection_names()): - logbookclient['explgbk_cache']['experiments'].create_index( [("name", "text" ), ("description", "text" ), ("instrument", "text" ), ("contact_info", "text" ), ("params.PNR", "text" )] ) - if 'operations' not in list(logbookclient['explgbk_cache'].list_collection_names()): - logbookclient['explgbk_cache']['operations'].create_index( [("name", DESCENDING)], unique=True) - logbookclient['explgbk_cache']['operations'].insert_one({"name": "explgbk_cache_rebuild", "initiated": datetime.datetime.utcfromtimestamp(0.0), "completed": datetime.datetime.utcfromtimestamp(0.0)}) + if "experiments" not in list( + logbookclient["explgbk_cache"].list_collection_names() + ): + logbookclient["explgbk_cache"]["experiments"].create_index( + [ + ("name", "text"), + ("description", "text"), + ("instrument", "text"), + ("contact_info", "text"), + ("params.PNR", "text"), + ] + ) + if "operations" not in list(logbookclient["explgbk_cache"].list_collection_names()): + logbookclient["explgbk_cache"]["operations"].create_index( + [("name", DESCENDING)], unique=True + ) + logbookclient["explgbk_cache"]["operations"].insert_one( + { + "name": "explgbk_cache_rebuild", + "initiated": datetime.datetime.utcfromtimestamp(0.0), + "completed": datetime.datetime.utcfromtimestamp(0.0), + } + ) global roles_with_post_privileges - roles_with_post_privileges = [x["name"] for x in logbookclient["site"]["roles"].find({"app": "LogBook", "privileges": { "$in": ["post"] }}, {"name": 1, "_id": 0})] + roles_with_post_privileges = [ + x["name"] + for x in logbookclient["site"]["roles"].find( + {"app": "LogBook", "privileges": {"$in": ["post"]}}, {"name": 1, "_id": 0} + ) + ] __load_experiment_names() scheduler = sched.scheduler() @@ -64,51 +101,66 @@ def init_app(app): def __periodic(scheduler, interval, action, actionargs=()): # This is the function that runs periodically - scheduler.enter(interval, 1, __periodic, (scheduler, interval, action, actionargs)) - last_rebuild = logbookclient['explgbk_cache']['operations'].find_one({"name": "explgbk_cache_rebuild"}) + scheduler.enter( + interval, 1, __periodic, (scheduler, interval, action, actionargs) + ) + last_rebuild = logbookclient["explgbk_cache"]["operations"].find_one( + {"name": "explgbk_cache_rebuild"} + ) db_time_utc = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) if (db_time_utc - last_rebuild["initiated"]).total_seconds() > interval: action(*actionargs) else: - logger.info("Skipping the periodic cache rebuild as we rebuilt the cache at %s within the specified interval %s", last_rebuild["initiated"], interval) + logger.info( + "Skipping the periodic cache rebuild as we rebuilt the cache at %s within the specified interval %s", + last_rebuild["initiated"], + interval, + ) def __kickoff_cache_update_thread(): # This runs in a background thread; the scheduler.run is blocking and will block forever. - __periodic(scheduler, 24*60*60, __update_experiments_info) + __periodic(scheduler, 24 * 60 * 60, __update_experiments_info) scheduler.run() __cache_update_thread = threading.Thread(target=__kickoff_cache_update_thread) __cache_update_thread.start() - def _non_immediate_update(): try: global periodic_updates exps = periodic_updates.getAndReset() - logger.info("Processing the non immediate cache for %s experiments", len(exps)) + logger.info( + "Processing the non immediate cache for %s experiments", len(exps) + ) for exp in exps: try: update_single_experiment_info(exp) - except: + except Exception: logger.exception("Exception in periodic updater updating %s", exp) - except: + except Exception: logger.exception("Exception in periodic updater") + def __non_immediate_periodic(scheduler, interval, action, actionargs=()): - scheduler.enter(interval, 1, __non_immediate_periodic, (scheduler, interval, action, actionargs)) + scheduler.enter( + interval, + 1, + __non_immediate_periodic, + (scheduler, interval, action, actionargs), + ) action(*actionargs) nonimmediatesched = sched.scheduler() def __kickoff_non_immediate_thread(): - __non_immediate_periodic(nonimmediatesched, 5*60, _non_immediate_update) + __non_immediate_periodic(nonimmediatesched, 5 * 60, _non_immediate_update) nonimmediatesched.run() - __non_immediate_updater_thread = threading.Thread(target=__kickoff_non_immediate_thread) + __non_immediate_updater_thread = threading.Thread( + target=__kickoff_non_immediate_thread + ) __non_immediate_updater_thread.start() - - def reload_cache(experiment_name=None): """ Reload the experiment cache from the database. @@ -124,68 +176,120 @@ def reload_cache(experiment_name=None): __update_experiments_info() + def get_experiments(): """ Get a list of experiments from the database. Returns basic information and also some info on the first and last runs. """ - return list(logbookclient['explgbk_cache']['experiments'].find({})) + return list(logbookclient["explgbk_cache"]["experiments"].find({})) + def get_cached_experiment_info(experiment_id): """ Returns basic information and also some info on the first and last runs. """ - return logbookclient['explgbk_cache']['experiments'].find_one({"_id": experiment_id}) + return logbookclient["explgbk_cache"]["experiments"].find_one( + {"_id": experiment_id} + ) + def get_experiments_starting_in_time_frame(start_time, end_time): """ Get a list of experiments whose start_time is in the given time range. """ - return list(logbookclient['explgbk_cache']['experiments'].find({"$and": [ - {"start_time": {"$gte": start_time}}, - {"start_time": {"$lte": end_time}} - ]}, {"name": 1, "start_time": 1})) + return list( + logbookclient["explgbk_cache"]["experiments"].find( + { + "$and": [ + {"start_time": {"$gte": start_time}}, + {"start_time": {"$lte": end_time}}, + ] + }, + {"name": 1, "start_time": 1}, + ) + ) + def get_sorted_experiments_ids(sort_criteria): """ Get only the experiment ids sorted according to the sort criteria. Sort criteria is a JSON array of [[sort_attr, sort_direction]] """ - return [ x["_id"] for x in logbookclient['explgbk_cache']['experiments'].find({}, { "_id": 1, "name": 1 }).sort(sort_criteria)] + return [ + x["_id"] + for x in logbookclient["explgbk_cache"]["experiments"] + .find({}, {"_id": 1, "name": 1}) + .sort(sort_criteria) + ] + def get_experiments_for_user(uid): """ Get a list of experiments for which the user has read access. """ sitedb = logbookclient["site"] - cachedb = logbookclient['explgbk_cache'] groups = usergroups.get_user_posix_groups(uid) groups.append("uid:" + uid) # See if the user has any global read privileges - global_read_roles_for_user = [ x for x in sitedb["roles"].find({"players": {"$in": groups}, "privileges": "read"}) ] + global_read_roles_for_user = [ + x + for x in sitedb["roles"].find( + {"players": {"$in": groups}, "privileges": "read"} + ) + ] if global_read_roles_for_user: - logger.debug("User %s has read privileges for all experiments from the site database", uid) + logger.debug( + "User %s has read privileges for all experiments from the site database", + uid, + ) return get_experiments() - global_read_roles = set([(x["app"], x["name"]) for x in sitedb["roles"].find({"privileges": "read"}, {"_id": 0, "app": 1, "name": 1})]) + global_read_roles = set( + [ + (x["app"], x["name"]) + for x in sitedb["roles"].find( + {"privileges": "read"}, {"_id": 0, "app": 1, "name": 1} + ) + ] + ) # Check for instrument level privileges - instrument_roles_for_user = [x for x in sitedb["instruments"].aggregate([{"$match": {"roles.players": {"$in": groups}}}, {"$unwind": "$roles"}])] + instrument_roles_for_user = [ + x + for x in sitedb["instruments"].aggregate( + [{"$match": {"roles.players": {"$in": groups}}}, {"$unwind": "$roles"}] + ) + ] exp_for_uid = {} for irole in instrument_roles_for_user: if (irole["roles"]["app"], irole["roles"]["name"]) in global_read_roles: - logger.debug("User %s has read permission for instrument %s because of role %s/%s", uid, irole["_id"], irole["roles"]["app"], irole["roles"]["name"] ) + logger.debug( + "User %s has read permission for instrument %s because of role %s/%s", + uid, + irole["_id"], + irole["roles"]["app"], + irole["roles"]["name"], + ) for exp in get_experiments_for_instrument(irole["_id"]): exp_for_uid[exp["_id"]] = exp # Now for experiments for which the user is directly a collaborator - for exp in list(logbookclient['explgbk_cache']["experiments"].find({"players": {"$in": groups}})): + for exp in list( + logbookclient["explgbk_cache"]["experiments"].find({"players": {"$in": groups}}) + ): exp_for_uid[exp["_id"]] = exp return list(exp_for_uid.values()) + def get_direct_experiments_for_user(uid): """ Get a list of experiments for which the user is a direct collaborator. This information typically comes in from the URAWI BTR. """ - return list(logbookclient['explgbk_cache']["experiments"].find({"players": {"$in": ["uid:" + uid]}}, {"name": 1, "instrument": 1})) + return list( + logbookclient["explgbk_cache"]["experiments"].find( + {"players": {"$in": ["uid:" + uid]}}, {"name": 1, "instrument": 1} + ) + ) + def get_cached_experiment_names(): """ @@ -194,6 +298,7 @@ def get_cached_experiment_names(): global all_experiment_names return list(all_experiment_names) + def get_experiments_with_post_privileges(userid, active_exps): """ Get the list of experiments that the logged in user has post privileges for. @@ -203,25 +308,55 @@ def get_experiments_with_post_privileges(userid, active_exps): groups = usergroups.get_user_posix_groups(userid) u_a_g = ["uid:" + userid] + groups logger.debug("Looking for experiments with post privileges for %s", u_a_g) - site_roles = [ x for x in logbookclient["site"]["roles"].find({"app": "LogBook", "privileges": { "$in": ["post"]}, "players": { "$in": u_a_g }})] + site_roles = [ + x + for x in logbookclient["site"]["roles"].find( + { + "app": "LogBook", + "privileges": {"$in": ["post"]}, + "players": {"$in": u_a_g}, + } + ) + ] if site_roles: logger.debug("User %s has post privileges for all experiments") postable_exps = get_experiments() else: - postable_exps = list(logbookclient['explgbk_cache']["experiments"].find({"post_players": {"$in": u_a_g}})) + postable_exps = list( + logbookclient["explgbk_cache"]["experiments"].find( + {"post_players": {"$in": u_a_g}} + ) + ) # Sort - ret_exps = [ { attr : x.get(attr, None) for attr in ["_id", "name", "instrument", "description", "start_time", "end_time", "posix_group", "params"] } for x in postable_exps ] - active_exp_names = [ x["name"] for x in active_exps if "name" in x ] + ret_exps = [ + { + attr: x.get(attr, None) + for attr in [ + "_id", + "name", + "instrument", + "description", + "start_time", + "end_time", + "posix_group", + "params", + ] + } + for x in postable_exps + ] + active_exp_names = [x["name"] for x in active_exps if "name" in x] for exp in ret_exps: if exp["name"] in active_exp_names: exp["is_active"] = True return ret_exps + def get_experiment_stats(): """ Get various computed/cached stats for the experiments """ - return list(logbookclient['explgbk_cache']['experiment_stats'].find({})) + return list(logbookclient["explgbk_cache"]["experiment_stats"].find({})) + def get_experiment_daily_data_breakdown(report_type, instrument): """ @@ -230,40 +365,95 @@ def get_experiment_daily_data_breakdown(report_type, instrument): """ if report_type == "file_sizes": if not instrument or instrument == "ALL": - return list(logbookclient['explgbk_cache']['experiment_stats'].aggregate([ - { "$unwind": "$dataDailyBreakdown" }, - { "$replaceRoot": {"newRoot": "$dataDailyBreakdown" }}, - { "$group": { "_id": "$_id", "total_size": {"$sum": {"$divide": ["$total_size", 1024]}}}}, - { "$sort": { "_id": -1 }} - ])) + return list( + logbookclient["explgbk_cache"]["experiment_stats"].aggregate( + [ + {"$unwind": "$dataDailyBreakdown"}, + {"$replaceRoot": {"newRoot": "$dataDailyBreakdown"}}, + { + "$group": { + "_id": "$_id", + "total_size": { + "$sum": {"$divide": ["$total_size", 1024]} + }, + } + }, + {"$sort": {"_id": -1}}, + ] + ) + ) else: - return list(logbookclient['explgbk_cache']['experiment_stats'].aggregate([ - { "$lookup": { "from": "experiments", "localField": "_id", "foreignField": "_id", "as": "exp"}}, - { "$unwind": "$exp" }, - { "$match": { "exp.instrument": instrument }}, - { "$unwind": "$dataDailyBreakdown" }, - { "$replaceRoot": {"newRoot": "$dataDailyBreakdown" }}, - { "$group": { "_id": "$_id", "total_size": {"$sum": {"$divide": ["$total_size", 1024]}}}}, - { "$sort": { "_id": -1 }} - ])) + return list( + logbookclient["explgbk_cache"]["experiment_stats"].aggregate( + [ + { + "$lookup": { + "from": "experiments", + "localField": "_id", + "foreignField": "_id", + "as": "exp", + } + }, + {"$unwind": "$exp"}, + {"$match": {"exp.instrument": instrument}}, + {"$unwind": "$dataDailyBreakdown"}, + {"$replaceRoot": {"newRoot": "$dataDailyBreakdown"}}, + { + "$group": { + "_id": "$_id", + "total_size": { + "$sum": {"$divide": ["$total_size", 1024]} + }, + } + }, + {"$sort": {"_id": -1}}, + ] + ) + ) elif report_type == "run_counts": if not instrument or instrument == "ALL": - return list(logbookclient['explgbk_cache']['experiment_stats'].aggregate([ - { "$unwind": "$runDailyBreakdown" }, - { "$replaceRoot": {"newRoot": "$runDailyBreakdown" }}, - { "$group": { "_id": "$_id", "total_runs": {"$sum": "$run_count"}}}, - { "$sort": { "_id": -1 }} - ])) + return list( + logbookclient["explgbk_cache"]["experiment_stats"].aggregate( + [ + {"$unwind": "$runDailyBreakdown"}, + {"$replaceRoot": {"newRoot": "$runDailyBreakdown"}}, + { + "$group": { + "_id": "$_id", + "total_runs": {"$sum": "$run_count"}, + } + }, + {"$sort": {"_id": -1}}, + ] + ) + ) else: - return list(logbookclient['explgbk_cache']['experiment_stats'].aggregate([ - { "$lookup": { "from": "experiments", "localField": "_id", "foreignField": "_id", "as": "exp"}}, - { "$unwind": "$exp" }, - { "$match": { "exp.instrument": instrument }}, - { "$unwind": "$runDailyBreakdown" }, - { "$replaceRoot": {"newRoot": "$runDailyBreakdown" }}, - { "$group": { "_id": "$_id", "total_runs": {"$sum": "$run_count"}}}, - { "$sort": { "_id": -1 }} - ])) + return list( + logbookclient["explgbk_cache"]["experiment_stats"].aggregate( + [ + { + "$lookup": { + "from": "experiments", + "localField": "_id", + "foreignField": "_id", + "as": "exp", + } + }, + {"$unwind": "$exp"}, + {"$match": {"exp.instrument": instrument}}, + {"$unwind": "$runDailyBreakdown"}, + {"$replaceRoot": {"newRoot": "$runDailyBreakdown"}}, + { + "$group": { + "_id": "$_id", + "total_runs": {"$sum": "$run_count"}, + } + }, + {"$sort": {"_id": -1}}, + ] + ) + ) + def does_experiment_exist(experiment_name): """ @@ -272,23 +462,29 @@ def does_experiment_exist(experiment_name): So, we are avoiding a hit to the database by caching just the names themselves in memory. """ global all_experiment_names - if experiment_name in all_experiment_names: # Check the cache first. + if experiment_name in all_experiment_names: # Check the cache first. return True expdb = logbookclient[experiment_name] collnames = list(expdb.list_collection_names()) - if 'info' in collnames: + if "info" in collnames: return True return False + def text_search_for_experiments(search_terms): """ Search the experiment cache for experiments matching the search terms. Use search terms separated by spaces. The backslash escapes the space for literal searches. Use the minus character to suppress a word. """ - matching_entries = list(logbookclient['explgbk_cache']['experiments'].find({ "$text": { "$search": search_terms }})) - return sorted(matching_entries, key=lambda x : x["name"]) + matching_entries = list( + logbookclient["explgbk_cache"]["experiments"].find( + {"$text": {"$search": search_terms}} + ) + ) + return sorted(matching_entries, key=lambda x: x["name"]) + def search_experiments_for_common_fields(search_term, sort_criteria): """ @@ -296,62 +492,117 @@ def search_experiments_for_common_fields(search_term, sort_criteria): The fields searched are _id, name, contact_info, description """ patt = re.compile(search_term) - matching_entries = list(logbookclient['explgbk_cache']['experiments'].find( - { "$or": [ - { "_id": { "$regex": patt } }, - { "name": { "$regex": patt } }, - { "contact_info": { "$regex": patt } }, - { "description": { "$regex": patt } } - ]}, {"_id": 1, "name": 1}).sort(sort_criteria)) + matching_entries = list( + logbookclient["explgbk_cache"]["experiments"] + .find( + { + "$or": [ + {"_id": {"$regex": patt}}, + {"name": {"$regex": patt}}, + {"contact_info": {"$regex": patt}}, + {"description": {"$regex": patt}}, + ] + }, + {"_id": 1, "name": 1}, + ) + .sort(sort_criteria) + ) return matching_entries + def get_recently_updated_experiments(offset_secs): """ Return a list of experiment names which have run.last_run.begin_time gte specified offset """ offset_time = datetime.datetime.now() - datetime.timedelta(seconds=offset_secs) - return [ x["name"] for x in list(logbookclient['explgbk_cache']['experiments'].find({"last_run.begin_time": { "$gte": offset_time }}, {"name": 1})) ] + return [ + x["name"] + for x in list( + logbookclient["explgbk_cache"]["experiments"].find( + {"last_run.begin_time": {"$gte": offset_time}}, {"name": 1} + ) + ) + ] + def get_all_param_names_matching_regex(rgx): """ Get all param names in all experiments matching the incoming regex. """ patt = re.compile(rgx) - apns = logbookclient['explgbk_cache']["experiments"].distinct("all_param_names") - return [ x for x in apns if patt.match(x) ] + apns = logbookclient["explgbk_cache"]["experiments"].distinct("all_param_names") + return [x for x in apns if patt.match(x)] + def get_experiments_proposal_mappings(): """ Get all the experiments with their PNR's if present. """ - return list(logbookclient['explgbk_cache']["experiments"].find({}, {"name": 1, "params.PNR": 1, "instrument": 1})) + return list( + logbookclient["explgbk_cache"]["experiments"].find( + {}, {"name": 1, "params.PNR": 1, "instrument": 1} + ) + ) def get_potentially_active_users(cutoff_date): """ Returns the set of users in experiment whose end date is after the specified date. """ - ret = list(logbookclient['explgbk_cache']["experiments"].aggregate([ - { "$match": { "end_time": {"$gte": cutoff_date } } }, - { "$group": { "_id": 0, "players": { "$push": "$players" }}}, - { "$project": { "_id": 0, "players": 1 } }, - { "$project": {"allusers": { "$reduce": { "input": "$players", "initialValue": [], "in": { "$concatArrays" : ["$$value", "$$this"] } }}}}, - { "$unwind": "$allusers"}, - { "$project": { "userid": "$allusers" }}, - { "$match": { "userid": {"$regex": "^uid:.*"}}}, - { "$group": {"_id": "$userid"}}, - { "$project": {"_id": 0, "userid": { "$replaceOne": { "input": "$_id", "find": "uid:", "replacement": "" } } }}, - { "$sort": { "userid": 1}} - ])) - return [ x["userid"] for x in ret] + ret = list( + logbookclient["explgbk_cache"]["experiments"].aggregate( + [ + {"$match": {"end_time": {"$gte": cutoff_date}}}, + {"$group": {"_id": 0, "players": {"$push": "$players"}}}, + {"$project": {"_id": 0, "players": 1}}, + { + "$project": { + "allusers": { + "$reduce": { + "input": "$players", + "initialValue": [], + "in": {"$concatArrays": ["$$value", "$$this"]}, + } + } + } + }, + {"$unwind": "$allusers"}, + {"$project": {"userid": "$allusers"}}, + {"$match": {"userid": {"$regex": "^uid:.*"}}}, + {"$group": {"_id": "$userid"}}, + { + "$project": { + "_id": 0, + "userid": { + "$replaceOne": { + "input": "$_id", + "find": "uid:", + "replacement": "", + } + }, + } + }, + {"$sort": {"userid": 1}}, + ] + ) + ) + return [x["userid"] for x in ret] def __load_experiment_names(): - """ We cache the list of experimemt names to speedup authz/other operations. + """We cache the list of experimemt names to speedup authz/other operations. This reloads the cached list of experiment names from the explgbk_cache """ global all_experiment_names - all_experiment_names = set([x["name"] for x in logbookclient['explgbk_cache']['experiments'].find({}, {"name": 1, "_id":0})]) + all_experiment_names = set( + [ + x["name"] + for x in logbookclient["explgbk_cache"]["experiments"].find( + {}, {"name": 1, "_id": 0} + ) + ] + ) + def __update_experiments_info(): """ @@ -362,14 +613,18 @@ def __update_experiments_info(): logger.info("Updating the experiment info cached in 'explgbk_cache'.") database_names = sorted(list(logbookclient.list_database_names()), reverse=True) db_time_utc = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - logbookclient['explgbk_cache']['operations'].update_one({"name": "explgbk_cache_rebuild"}, {"$set": {"initiated": db_time_utc}}) + logbookclient["explgbk_cache"]["operations"].update_one( + {"name": "explgbk_cache_rebuild"}, {"$set": {"initiated": db_time_utc}} + ) for experiment_name in database_names: if experiment_name in ["admin", "config", "local", "site"]: continue update_single_experiment_info(experiment_name) db_time_utc = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - logbookclient['explgbk_cache']['operations'].update_one({"name": "explgbk_cache_rebuild"}, {"$set": {"completed": db_time_utc}}) - kafka_producer.send("explgbk_cache", { "cache_rebuild": True } ) + logbookclient["explgbk_cache"]["operations"].update_one( + {"name": "explgbk_cache_rebuild"}, {"$set": {"completed": db_time_utc}} + ) + kafka_producer.send("explgbk_cache", {"cache_rebuild": True}) def update_single_experiment_info(experiment_name, crud="Update"): @@ -379,97 +634,267 @@ def update_single_experiment_info(experiment_name, crud="Update"): global all_experiment_names if crud == "Delete": all_experiment_names.remove(experiment_name) - logbookclient['explgbk_cache']['experiments'].delete_one({"_id": experiment_name}) - logbookclient['explgbk_cache']['experiment_stats'].delete_one({"_id": experiment_name}) + logbookclient["explgbk_cache"]["experiments"].delete_one( + {"_id": experiment_name} + ) + logbookclient["explgbk_cache"]["experiment_stats"].delete_one( + {"_id": experiment_name} + ) return - logger.debug("Gathering the experiment info cached in 'explgbk_cache' for experiment %s", experiment_name) + logger.debug( + "Gathering the experiment info cached in 'explgbk_cache' for experiment %s", + experiment_name, + ) expdb = logbookclient[experiment_name] collnames = list(expdb.list_collection_names()) - if 'info' in collnames: + if "info" in collnames: info = expdb["info"].find_one({}, {"latest_setup": 0}) - if 'name' not in info or 'instrument' not in info: - logger.error("Database %s has a info collection but the info object does not have an instrument or name. Note this could also be a timing issue if you are using secondaries", experiment_name) + if "name" not in info or "instrument" not in info: + logger.error( + "Database %s has a info collection but the info object does not have an instrument or name. Note this could also be a timing issue if you are using secondaries", + experiment_name, + ) return all_experiment_names.add(experiment_name) - expinfo = { "_id": experiment_name } + expinfo = {"_id": experiment_name} roles = [x for x in expdb["roles"].find()] all_players = set() - list(map(lambda x : all_players.update(x.get('players', [])), roles)) + list(map(lambda x: all_players.update(x.get("players", [])), roles)) expinfo["players"] = list(all_players) post_players = set() - list(map(lambda x : post_players.update(x.get('players', [])), expdb["roles"].find({"name": {"$in": roles_with_post_privileges}}, {"_id": 0, "players": 1}))) + list( + map( + lambda x: post_players.update(x.get("players", [])), + expdb["roles"].find( + {"name": {"$in": roles_with_post_privileges}}, + {"_id": 0, "players": 1}, + ), + ) + ) expinfo["post_players"] = list(post_players) - if 'runs' in collnames: + if "runs" in collnames: run_count = expdb["runs"].count_documents({}) - expinfo['run_count'] = run_count + expinfo["run_count"] = run_count if run_count: - last_run = expdb["runs"].find({}, { "num": 1, "begin_time": 1, "end_time": 1 } ).sort([("begin_time", -1)]).limit(1)[0] - first_run = expdb["runs"].find({}, { "num": 1, "begin_time": 1, "end_time": 1 } ).sort([("begin_time", 1)]).limit(1)[0] - expinfo["first_run"] = { "num": first_run["num"], - "begin_time": first_run["begin_time"], - "end_time": first_run["end_time"] - } - expinfo["last_run"] = { "num": last_run["num"], - "begin_time": last_run["begin_time"], - "end_time": last_run["end_time"] - } - runDailyBreakdown = list(expdb["runs"].aggregate([ - {"$group": { "_id": {"$dateToParts": { "date": {"$convert": { "input": "$begin_time", "to": "date" }}}}, "run_count": {"$sum": 1}}}, - {"$project": { "_id.year": 1, "_id.month": 1, "_id.day": 1, "run_count": 1 }}, - {"$group": { "_id": {"$dateFromParts": { "year": "$_id.year", "month": "$_id.month", "day": "$_id.day" } }, "run_count": {"$sum": "$run_count"}}}, - {"$sort": {"_id": 1}} - ])) + last_run = ( + expdb["runs"] + .find({}, {"num": 1, "begin_time": 1, "end_time": 1}) + .sort([("begin_time", -1)]) + .limit(1)[0] + ) + first_run = ( + expdb["runs"] + .find({}, {"num": 1, "begin_time": 1, "end_time": 1}) + .sort([("begin_time", 1)]) + .limit(1)[0] + ) + expinfo["first_run"] = { + "num": first_run["num"], + "begin_time": first_run["begin_time"], + "end_time": first_run["end_time"], + } + expinfo["last_run"] = { + "num": last_run["num"], + "begin_time": last_run["begin_time"], + "end_time": last_run["end_time"], + } + runDailyBreakdown = list( + expdb["runs"].aggregate( + [ + { + "$group": { + "_id": { + "$dateToParts": { + "date": { + "$convert": { + "input": "$begin_time", + "to": "date", + } + } + } + }, + "run_count": {"$sum": 1}, + } + }, + { + "$project": { + "_id.year": 1, + "_id.month": 1, + "_id.day": 1, + "run_count": 1, + } + }, + { + "$group": { + "_id": { + "$dateFromParts": { + "year": "$_id.year", + "month": "$_id.month", + "day": "$_id.day", + } + }, + "run_count": {"$sum": "$run_count"}, + } + }, + {"$sort": {"_id": 1}}, + ] + ) + ) if runDailyBreakdown: - logbookclient['explgbk_cache']['experiment_stats'].update_one({"_id": experiment_name}, { "$set": { "runDailyBreakdown": runDailyBreakdown } }, upsert=True) - - expinfo["all_param_names"] = [x["_id"] for x in expdb["runs"].aggregate([ - { "$project": { "params": { "$objectToArray": "$params" }}}, - { "$unwind": "$params" }, - { "$group": { "_id": "$params.k", "total": { "$sum": 1 }}}, - { "$sort": { "_id": 1 }} - ])] + logbookclient["explgbk_cache"]["experiment_stats"].update_one( + {"_id": experiment_name}, + {"$set": {"runDailyBreakdown": runDailyBreakdown}}, + upsert=True, + ) + + expinfo["all_param_names"] = [ + x["_id"] + for x in expdb["runs"].aggregate( + [ + {"$project": {"params": {"$objectToArray": "$params"}}}, + {"$unwind": "$params"}, + {"$group": {"_id": "$params.k", "total": {"$sum": 1}}}, + {"$sort": {"_id": 1}}, + ] + ) + ] else: logger.debug("No runs in experiment " + experiment_name) try: - if 'file_catalog' in collnames: + if "file_catalog" in collnames: + def asTime(val): if not val: return None elif isinstance(val[0]["create_timestamp"], datetime.datetime): return val[0]["create_timestamp"] elif isinstance(val[0]["create_timestamp"], str): - return datetime.datetime.strptime(val[0]["create_timestamp"], '%Y-%m-%dT%H:%M:%SZ') + return datetime.datetime.strptime( + val[0]["create_timestamp"], "%Y-%m-%dT%H:%M:%SZ" + ) else: return None - f_file = asTime(list(expdb['file_catalog'].find({}).sort([("create_timestamp", -1)]).limit(1))) - l_file = asTime(list(expdb['file_catalog'].find({}).sort([("create_timestamp", 1)]).limit(1))) + + f_file = asTime( + list( + expdb["file_catalog"] + .find({}) + .sort([("create_timestamp", -1)]) + .limit(1) + ) + ) + l_file = asTime( + list( + expdb["file_catalog"] + .find({}) + .sort([("create_timestamp", 1)]) + .limit(1) + ) + ) if f_file and l_file: - attrs = ['years', 'months', 'days', 'hours', 'minutes'] - human_readable_diff = lambda delta: ['%d %s' % (getattr(delta, attr), getattr(delta, attr) > 1 and attr or attr[:-1]) for attr in attrs if getattr(delta, attr)] - expinfo['file_timestamps'] = { + attrs = ["years", "months", "days", "hours", "minutes"] + + def human_readable_diff(delta): + return [ + "%d %s" + % ( + getattr(delta, attr), + getattr(delta, attr) > 1 and attr or attr[:-1], + ) + for attr in attrs + if getattr(delta, attr) + ] + + expinfo["file_timestamps"] = { "first_file_ts": f_file, "last_file_ts": l_file, "duration": (l_file - f_file).total_seconds(), - "hr_duration": human_readable_diff(dateutil.relativedelta.relativedelta (f_file, l_file)) + "hr_duration": human_readable_diff( + dateutil.relativedelta.relativedelta(f_file, l_file) + ), } - dataSummary = [x for x in expdb['file_catalog'].aggregate([ { "$group" : - { "_id" : None, - "totalDataSize": { "$sum": { "$divide": ["$size", 1024*1024*1024*1.0 ] } }, - "totalFiles": { "$sum": 1 } - } - } ])] + dataSummary = [ + x + for x in expdb["file_catalog"].aggregate( + [ + { + "$group": { + "_id": None, + "totalDataSize": { + "$sum": { + "$divide": [ + "$size", + 1024 * 1024 * 1024 * 1.0, + ] + } + }, + "totalFiles": {"$sum": 1}, + } + } + ] + ) + ] if dataSummary: - expinfo['totalDataSize'] = dataSummary[0]['totalDataSize'] - expinfo['totalFiles'] = dataSummary[0]['totalFiles'] - dataDailyBreakdown = [x for x in expdb['file_catalog'].aggregate([ - {"$group": { "_id": {"$dateToParts": { "date": {"$convert": { "input": "$create_timestamp", "to": "date" }}}}, "total_size": {"$sum": "$size"}}}, - {"$project": { "_id.year": 1, "_id.month": 1, "_id.day": 1, "total_size": 1 }}, - {"$group": { "_id": {"$dateFromParts": { "year": "$_id.year", "month": "$_id.month", "day": "$_id.day" } }, "total_size": {"$sum": {"$divide": [ "$total_size", 1024*1024*1024]}}}} - ], allowDiskUse=True)] + expinfo["totalDataSize"] = dataSummary[0]["totalDataSize"] + expinfo["totalFiles"] = dataSummary[0]["totalFiles"] + dataDailyBreakdown = [ + x + for x in expdb["file_catalog"].aggregate( + [ + { + "$group": { + "_id": { + "$dateToParts": { + "date": { + "$convert": { + "input": "$create_timestamp", + "to": "date", + } + } + } + }, + "total_size": {"$sum": "$size"}, + } + }, + { + "$project": { + "_id.year": 1, + "_id.month": 1, + "_id.day": 1, + "total_size": 1, + } + }, + { + "$group": { + "_id": { + "$dateFromParts": { + "year": "$_id.year", + "month": "$_id.month", + "day": "$_id.day", + } + }, + "total_size": { + "$sum": { + "$divide": [ + "$total_size", + 1024 * 1024 * 1024, + ] + } + }, + } + }, + ], + allowDiskUse=True, + ) + ] if dataDailyBreakdown: - logbookclient['explgbk_cache']['experiment_stats'].update_one({"_id": experiment_name}, { "$set": { "dataDailyBreakdown": dataDailyBreakdown } }, upsert=True) - except Exception as e: + logbookclient["explgbk_cache"]["experiment_stats"].update_one( + {"_id": experiment_name}, + {"$set": {"dataDailyBreakdown": dataDailyBreakdown}}, + upsert=True, + ) + except Exception: logger.exception("Exception computing the file parameters") poc_feedback_changes = get_poc_feedback_changes(experiment_name) @@ -477,24 +902,43 @@ def asTime(val): poc_feedback_doc = get_poc_feedback_document(experiment_name) expinfo["poc_feedback"] = { "num_items": len(poc_feedback_changes), - "num_items_4_5": len(list(filter(lambda kv: kv[0] not in ["basic-scheduled", "basic-actual"] and (kv[1] in [ "4", "5" ]), poc_feedback_doc.items()))), + "num_items_4_5": len( + list( + filter( + lambda kv: ( + kv[0] not in ["basic-scheduled", "basic-actual"] + and (kv[1] in ["4", "5"]) + ), + poc_feedback_doc.items(), + ) + ) + ), "last_modified_by": poc_feedback_changes[-1]["modified_by"], - "last_modified_at": poc_feedback_changes[-1]["modified_at"] - } + "last_modified_at": poc_feedback_changes[-1]["modified_at"], + } expinfo.update(info) expinfo["_id"] = experiment_name - logbookclient['explgbk_cache']['experiments'].replace_one({"_id": experiment_name}, expinfo, upsert=True) + logbookclient["explgbk_cache"]["experiments"].replace_one( + {"_id": experiment_name}, expinfo, upsert=True + ) - logger.info("Updated the experiment info cached in 'explgbk_cache' for experiment %s", experiment_name) + logger.info( + "Updated the experiment info cached in 'explgbk_cache' for experiment %s", + experiment_name, + ) else: - logger.error("Database %s does not have a info collection. Note this could also be a timing issue if you are using secondaries", experiment_name) + logger.error( + "Database %s does not have a info collection. Note this could also be a timing issue if you are using secondaries", + experiment_name, + ) def __establish_local_kafka_consumers__(): """ This processes from the local queue """ + def processMessage(msg): try: logger.debug("Kafka/local Message %s", msg) @@ -507,37 +951,62 @@ def processMessage(msg): reload_named_caches(info["named_cache"]) __load_experiment_names() elif message_type in ["experiments", "roles", "samples"]: - if 'experiment_name' in info: - experiment_name = info['experiment_name'] - logger.info("Got a Kafka/local message %s for experiment %s - building the cache entry", message_type, experiment_name) - crud = info.get("CRUD", "Update") if message_type == "experiments" else "Update" + if "experiment_name" in info: + experiment_name = info["experiment_name"] + logger.info( + "Got a Kafka/local message %s for experiment %s - building the cache entry", + message_type, + experiment_name, + ) + crud = ( + info.get("CRUD", "Update") + if message_type == "experiments" + else "Update" + ) update_single_experiment_info(experiment_name, crud=crud) else: - logger.error("Kafka/local message in immediate topics without an experiment name %s", message_type) + logger.error( + "Kafka/local message in immediate topics without an experiment name %s", + message_type, + ) else: - logger.debug("Not re-building immediately for a non immediate topic %s", message_type) + logger.debug( + "Not re-building immediately for a non immediate topic %s", + message_type, + ) global periodic_updates - if 'experiment_name' in info: - experiment_name = info['experiment_name'] + if "experiment_name" in info: + experiment_name = info["experiment_name"] periodic_updates.add(experiment_name) else: - logger.error("Kafka/local message in non-immediate topics without an experiment name %s", message_type) - except Exception as e: + logger.error( + "Kafka/local message in non-immediate topics without an experiment name %s", + message_type, + ) + except Exception: logger.exception("Exception processing Kafka/local message.") + def worker(): while True: msg = local_kafka_events.get() processMessage(msg) local_kafka_events.task_done() + local_msg_thread = threading.Thread(target=worker) local_msg_thread.start() + def __establish_kafka_consumers(): """ Establish Kafka consumers that listen to new experiments and runs and updates the cache. """ + def subscribe_kafka(): - consumer = KafkaConsumer(bootstrap_servers=os.environ.get("KAFKA_BOOTSTRAP_SERVER", "localhost:9092").split(",")) + consumer = KafkaConsumer( + bootstrap_servers=os.environ.get( + "KAFKA_BOOTSTRAP_SERVER", "localhost:9092" + ).split(",") + ) consumer.subscribe(["experiments", "explgbk_cache"]) for msg in consumer: @@ -552,7 +1021,7 @@ def subscribe_kafka(): reload_named_caches(info["named_cache"]) __load_experiment_names() - except Exception as e: + except Exception: logger.exception("Exception processing Kafka message.") # Create thread for kafka consumer diff --git a/dal/imagestores/gridfs.py b/dal/imagestores/gridfs.py index 1e55976..bbb6f4a 100644 --- a/dal/imagestores/gridfs.py +++ b/dal/imagestores/gridfs.py @@ -9,12 +9,15 @@ logger = logging.getLogger(__name__) + class GridFSIS(ImageStore): - def store_file_and_return_url(self, experiment_name, filename, mimetype, filecontents): + def store_file_and_return_url( + self, experiment_name, filename, mimetype, filecontents + ): expdb = logbookclient[experiment_name] fs = GridFS(expdb) fid = fs.put(filecontents) - return "mongo://"+str(fid) + return "mongo://" + str(fid) def return_url_contents(self, experiment_name, remote_url): mtch = re.match("mongo://([\w]*)/(.*)", remote_url) diff --git a/dal/imagestores/tar.py b/dal/imagestores/tar.py deleted file mode 100644 index f248893..0000000 --- a/dal/imagestores/tar.py +++ /dev/null @@ -1,45 +0,0 @@ -from dal.imagestores.imagestore import ImageStore -import os -import logging -import requests -import tarfile -import io - -from bson import ObjectId - -from context import logbookclient, LOGBOOK_SITE - -logger = logging.getLogger(__name__) - -class TarIS(ImageStore): - def __get_experiment_results_folder__(self, experiment_name): - if LOGBOOK_SITE == "LCLS": - expdb = logbookclient[experiment_name] - instrument = expdb['info'].find_one()["instrument"].lower() - results = os.path.join("/reg/d/psdm/", instrument, experiment_name, "results") - return results - return None - - def store_file_and_return_url(self, experiment_name, filename, mimetype, filecontents): - results_folder = self.__get_experiment_results_folder__(experiment_name) - if not ( results_folder and os.path.exists(results_folder) and os.path.isdir(results_folder) ): - raise Exception("Missing results folder for experiment %s %s" % (experiment_name, results_folder)) - archive_folder = os.path.join(results_folder, "archive") - if not os.path.exists(archive_folder): - os.mkdir(archive_folder) - with tarfile.open(os.path.join(archive_folder, "attachments.tar"), "a") as t: - tinfo = tarfile.TarInfo(str(ObjectId())) - filecontents.seek(0, 2) - tinfo.size = filecontents.tell() - filecontents.seek(0, 0) - t.addfile(tinfo, filecontents) - return "tar://"+tinfo.name - - def return_url_contents(self, experiment_name, remote_url): - attachments_file = os.path.join(self.__get_experiment_results_folder__(experiment_name), "archive", "attachments.tar") - if not ( attachments_file and os.path.exists(attachments_file)): - raise Exception("Missing attachments tar file for experiment %s %s" % (experiment_name, attachments_file)) - - fid = remote_url.replace("tar://", "") - with tarfile.open(attachments_file, "r") as t: - return io.BytesIO(t.extractfile(fid).read()) diff --git a/dal/run_control.py b/dal/run_control.py index f064ea5..7661a3d 100644 --- a/dal/run_control.py +++ b/dal/run_control.py @@ -1,56 +1,81 @@ -''' +""" Run control business logic. -''' +""" -import json import datetime import logging -import re -import requests -from pymongo import ASCENDING, DESCENDING, ReturnDocument, ReadPreference +from pymongo import DESCENDING, ReturnDocument, ReadPreference from bson import ObjectId from context import logbookclient from dal.utils import escape_chars_for_mongo -__author__ = 'mshankar@slac.stanford.edu' +__author__ = "mshankar@slac.stanford.edu" logger = logging.getLogger(__name__) -def start_run(experiment_name, run_type, user_specified_run_number=None, user_specified_start_time=None, user_specified_sample=None, params=None): - ''' +def start_run( + experiment_name, + run_type, + user_specified_run_number=None, + user_specified_start_time=None, + user_specified_sample=None, + params=None, +): + """ Start a new run for the specified experiment If the user_specified_run_number is not specified; we use the next_runnum autoincrement counter. - ''' - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.PRIMARY) + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) if not user_specified_run_number: - next_run_num_doc = expdb['counters'].find_one_and_update({ "_id" : "next_runnum"}, {'$inc': {'seq': 1}}, return_document=ReturnDocument.AFTER) + next_run_num_doc = expdb["counters"].find_one_and_update( + {"_id": "next_runnum"}, + {"$inc": {"seq": 1}}, + return_document=ReturnDocument.AFTER, + ) if not next_run_num_doc: - raise Exception("Could not update run number counter for experiment %s" % experiment_name) + raise Exception( + "Could not update run number counter for experiment %s" + % experiment_name + ) next_run_num = next_run_num_doc["seq"] logger.info("Next run for experiment %s is %s", experiment_name, next_run_num) else: next_run_num = user_specified_run_number - logger.info("Next run for experiment %s is from the user %s", experiment_name, next_run_num) - - begin_time = user_specified_start_time if user_specified_start_time else datetime.datetime.utcnow() + logger.info( + "Next run for experiment %s is from the user %s", + experiment_name, + next_run_num, + ) + + begin_time = ( + user_specified_start_time + if user_specified_start_time + else datetime.datetime.utcnow() + ) run_doc = { - "num" : next_run_num, - "type" : run_type, - "begin_time" : begin_time, - "end_time" : None, - "params" : {}, - "editable_params" : {}} + "num": next_run_num, + "type": run_type, + "begin_time": begin_time, + "end_time": None, + "params": {}, + "editable_params": {}, + } if user_specified_sample: user_sample = expdb.samples.find_one({"name": user_specified_sample}) if user_sample: run_doc["sample"] = user_sample["_id"] else: - raise Exception("Could not find sample %s for experiment %s" % (user_specified_sample, experiment_name)) + raise Exception( + "Could not find sample %s for experiment %s" + % (user_specified_sample, experiment_name) + ) else: current_sample = expdb.current.find_one({"_id": "sample"}) if current_sample: @@ -58,27 +83,34 @@ def start_run(experiment_name, run_type, user_specified_run_number=None, user_sp if params: run_doc["params"] = params - result = expdb['runs'].insert_one(run_doc) - return expdb['runs'].find_one({"num": next_run_num}) + expdb["runs"].insert_one(run_doc) + return expdb["runs"].find_one({"num": next_run_num}) + def get_current_run(experiment_name): - ''' + """ Get the run document for the run with the maximum run number. - ''' - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.PRIMARY) + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) current_run_doc = list(expdb.runs.find().sort([("num", DESCENDING)]).limit(1)) return current_run_doc[0] if current_run_doc else None + def get_run_doc_for_run_num(experiment_name, run_num): """ Get the run document for the specified run number """ - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.PRIMARY) + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) run_doc = expdb.runs.find_one({"num": run_num}) if run_doc: return run_doc return None + def get_specified_run_params_for_all_runs(experiment_name, run_params): """ Get the specified run parameters for all runs in the experiment. @@ -90,6 +122,7 @@ def get_specified_run_params_for_all_runs(experiment_name, run_params): projection_op["params." + escape_chars_for_mongo(run_param)] = 1 return [x for x in expdb.runs.find({}, projection_op)] + def map_param_editable_to_run_nums(experiment_name, param_editable): """ Pass in a run param name or an editable name. @@ -99,7 +132,10 @@ def map_param_editable_to_run_nums(experiment_name, param_editable): That is, if there is an editable param with the same name as a run param, we use the editable param as the source of the pivot. """ expdb = logbookclient[experiment_name] - def __getval__(rn, parts): # Should get you editable_params.TAG.value from { "num" : 23, "editable_params" : { "TAG" : { "value" : "Ravenclaw" } } } + + def __getval__( + rn, parts + ): # Should get you editable_params.TAG.value from { "num" : 23, "editable_params" : { "TAG" : { "value" : "Ravenclaw" } } } ret = rn for part in parts: ret = ret[part] @@ -116,7 +152,9 @@ def __pivot__(rns, fqn): return ret fqn = "editable_params." + escape_chars_for_mongo(param_editable) + ".value" - editables = list(expdb.runs.find({fqn: {"$exists": 1}}, {"_id": 0, "num": 1, fqn: 1})) + editables = list( + expdb.runs.find({fqn: {"$exists": 1}}, {"_id": 0, "num": 1, fqn: 1}) + ) if editables and len(editables) > 0: logger.debug("Found an editable param with name %s", fqn) return __pivot__(editables, fqn) @@ -129,6 +167,7 @@ def __pivot__(rns, fqn): return {} + def get_run_nums_matching_params(experiment_name, query_document): """ Get an array of run numbers for all runs that have the specified value for the specified parameter. @@ -136,8 +175,11 @@ def get_run_nums_matching_params(experiment_name, query_document): """ expdb = logbookclient[experiment_name] projection_op = {"num": 1} - query = { "params." + escape_chars_for_mongo(k) : v for k,v in query_document.items() } - return [ x["num"] for x in expdb.runs.find(query, projection_op) ] + query = { + "params." + escape_chars_for_mongo(k): v for k, v in query_document.items() + } + return [x["num"] for x in expdb.runs.find(query, projection_op)] + def get_run_nums_matching_editable_regex(experiment_name, param_name, incoming_regex): """ @@ -146,8 +188,14 @@ def get_run_nums_matching_editable_regex(experiment_name, param_name, incoming_r """ expdb = logbookclient[experiment_name] projection_op = {"num": 1} - query = { "editable_params." + escape_chars_for_mongo(param_name) + ".value": { "$regex": incoming_regex, "$options": "i" }} - return [ x["num"] for x in expdb.runs.find(query, projection_op) ] + query = { + "editable_params." + escape_chars_for_mongo(param_name) + ".value": { + "$regex": incoming_regex, + "$options": "i", + } + } + return [x["num"] for x in expdb.runs.find(query, projection_op)] + def get_sample_for_run(experiment_name, run_num): """ @@ -157,33 +205,51 @@ def get_sample_for_run(experiment_name, run_num): run_doc = expdb.runs.find_one({"num": run_num}) if not run_doc: return None - if 'sample' not in run_doc: + if "sample" not in run_doc: return None return expdb.samples.find_one({"_id": run_doc["sample"]}) def end_run(experiment_name, user_specified_end_time=None): - ''' + """ End the current run; this is mostly a matter of filling in the end time - ''' - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.PRIMARY) + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) current_run_doc = get_current_run(experiment_name) - end_time = user_specified_end_time if user_specified_end_time else datetime.datetime.utcnow() - return expdb.runs.find_one_and_update({"num": current_run_doc["num"]}, {'$set': {'end_time': end_time}}, return_document=ReturnDocument.AFTER) + end_time = ( + user_specified_end_time + if user_specified_end_time + else datetime.datetime.utcnow() + ) + return expdb.runs.find_one_and_update( + {"num": current_run_doc["num"]}, + {"$set": {"end_time": end_time}}, + return_document=ReturnDocument.AFTER, + ) + def is_run_closed(experiment_name, run_num): - ''' + """ Check if the specified run is closed - ''' - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.PRIMARY) + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) run_doc = expdb.runs.find_one({"num": run_num}) - if run_doc and run_doc.get('end_time', None): + if run_doc and run_doc.get("end_time", None): return True return False + def add_run_params(experiment_name, run_doc, run_params): - ''' + """ Add run parameters to the specified run. - ''' + """ expdb = logbookclient[experiment_name] - return expdb.runs.find_one_and_update({"num": run_doc["num"]}, {'$set': run_params }, return_document=ReturnDocument.AFTER) + return expdb.runs.find_one_and_update( + {"num": run_doc["num"]}, + {"$set": run_params}, + return_document=ReturnDocument.AFTER, + ) diff --git a/explgbk/__init__.py b/explgbk/__init__.py new file mode 100644 index 0000000..1d6e087 --- /dev/null +++ b/explgbk/__init__.py @@ -0,0 +1,7 @@ +""" +EXPLGBK - Experiment Logbook +LCLS, Cryo and UED data management logbook +""" + +__version__ = "0.1.0" +__author__ = "mshankar@slac.stanford.edu" diff --git a/explgbk/app.py b/explgbk/app.py new file mode 100755 index 0000000..073d704 --- /dev/null +++ b/explgbk/app.py @@ -0,0 +1,67 @@ +import json +import logging +import os +import sys + +from explgbk.dal import exp_cache +from explgbk.context import security +from flask import Flask +from flask_socket_util import socket_service +from explgbk.blueprints.pages import pages_blueprint +from explgbk.blueprints.api import explgbk_blueprint + + +root = logging.getLogger() +root.setLevel(logging.getLevelName(os.environ.get("LOG_LEVEL", "INFO"))) +logging.getLogger("kafka").setLevel(logging.INFO) +logging.getLogger("engineio").setLevel(logging.WARN) +logging.getLogger("flask_authnz").setLevel(logging.WARN) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +ch.setFormatter(formatter) +root.addHandler(ch) + +logger = logging.getLogger(__name__) + +__author__ = "mshankar@slac.stanford.edu" + + +# Initialize application. +app = Flask("explgbk") +# Set the expiration for static files +app.config["SEND_FILE_MAX_AGE_DEFAULT"] = 60 * 60 +app.secret_key = "This is a secret key that is somewhat temporary." +app.debug = False + + +@app.template_filter("json") +def jinga2_jsonfilter(value): + return json.dumps(value) + + +# Register routes. +app.register_blueprint(pages_blueprint) +app.register_blueprint(explgbk_blueprint) + +socket_service.init_app( + app, + security, + kafkatopics=[ + "experiments", + "elog", + "runs", + "shifts", + "samples", + "file_catalog", + "workflow_jobs", + ], +) + +exp_cache.init_app(app) + +logger.info("Server initialization complete") + +if __name__ == "__main__": + print("Please use gunicorn for development as well.") + sys.exit(-1) diff --git a/explgbk/blueprints/__init__.py b/explgbk/blueprints/__init__.py new file mode 100644 index 0000000..be44f69 --- /dev/null +++ b/explgbk/blueprints/__init__.py @@ -0,0 +1,3 @@ +""" +Flask blueprints for EXPLGBK application +""" diff --git a/services/explgbk.py b/explgbk/blueprints/api.py similarity index 56% rename from services/explgbk.py rename to explgbk/blueprints/api.py index eeeb6c6..484d376 100755 --- a/services/explgbk.py +++ b/explgbk/blueprints/api.py @@ -1,4 +1,4 @@ -''' +""" Code for the business logic. Here's where you do the actual business logic using functions from the dal's business object. The public methods here are expected to be Flask blueprint endpoints. @@ -6,7 +6,7 @@ Events are published into Kafka and the Websocket layer here. Use security's authentication_required and authorization_required decorators to enforce authz/authn. -''' +""" import os import json @@ -17,91 +17,232 @@ import abc import requests -import context +from explgbk import context from functools import wraps -from collections import OrderedDict from datetime import datetime, timedelta import types import hashlib import urllib import base64 import pytz -import jwt import smtplib from email.message import EmailMessage -from flask import Blueprint, jsonify, request, url_for, Response, stream_with_context, send_file, \ - abort, redirect, make_response, g, current_app - -from dal.explgbk import LgbkException, get_experiment_info, save_new_experiment_setup, register_new_experiment, \ - get_instruments, get_currently_active_experiments, switch_experiment, get_elog_entries, post_new_log_entry, get_specific_elog_entry, \ - get_specific_shift, get_experiment_files, get_experiment_runs, get_all_run_tables, get_runtable_data, get_runtable_sources, \ - create_update_user_run_table_def, update_editable_param_for_run, get_instrument_station_list, update_existing_experiment, \ - create_update_instrument, get_experiment_shifts, get_shift_for_experiment_by_name, close_shift_for_experiment, \ - create_update_shift, get_latest_shift, get_samples, create_sample, update_sample, get_sample_for_experiment_by_name, \ - make_sample_current, register_file_for_experiment, search_elog_for_text, delete_run_table, get_current_sample_name, \ - get_elogs_for_run_num, get_elogs_for_run_num_range, get_elogs_for_specified_id, get_collaborators, get_role_object, \ - add_collaborator_to_role, remove_collaborator_from_role, delete_elog_entry, modify_elog_entry, clone_experiment, rename_experiment, \ - instrument_standby, get_experiment_files_for_run, get_elog_authors, get_elog_entries_by_author, get_elog_tags, get_elog_entries_by_tag, \ - get_elogs_for_date_range, clone_sample, get_modal_param_definitions, lock_unlock_experiment, get_elog_emails, \ - get_elog_email_subscriptions, elog_email_subscribe, elog_email_unsubscribe, get_elog_email_subscriptions_emails, \ - get_poc_feedback_changes, add_poc_feedback_item, clone_run_table_definition, replace_system_run_table_definition, \ - delete_system_run_table, get_instrument_elogs, post_related_elog_entry, get_related_instrument_elog_entries, \ - get_elog_tree_for_specified_id, get_workflow_definitions, get_dm_locations, get_workflow_triggers, \ - create_update_wf_definition, get_workflow_jobs, get_workflow_job_doc, create_wf_job, delete_wf_job, update_wf_job, \ - file_available_at_location, get_collaborators_list_for_experiment, get_site_naming_conventions, delete_sample_for_experiment, \ - get_global_roles, add_player_to_global_role, remove_player_from_global_role, get_site_config, file_not_available_at_location, \ - get_experiment_run_document, get_experiment_files_for_run_for_live_mode, get_switch_history, delete_experiment, migrate_attachments_to_local_store, \ - get_complete_elog_tree_for_specified_id, get_site_file_types, add_player_to_instrument_role, remove_player_from_instrument_role, \ - delete_wf_definition, get_elog_entries_by_regex, get_run_param_descriptions, add_update_run_param_descriptions, change_sample_for_run, \ - add_update_experiment_params, get_ques_proposal_details, import_users_from_URAWI, get_poc_feedback_document, get_poc_feedback_experiments, \ - get_experiment_files_for_run_for_live_mode_at_location, get_active_experiment_name_for_instrument_station, \ - get_experiment_files_for_live_mode_at_location, get_run_numbers_with_tag, stop_current_sample, get_tag_to_run_numbers, \ - get_tags_for_runs, clone_system_template_run_tables_into_experiment, get_projects, get_project_info, create_project, update_project, \ - get_project_grids, get_project_grid, add_grid_to_project, update_project_grid, link_grid_to_experiment, get_exp_file_counts_by_extension, \ - questionnaire_cache_refresh - - -from dal.run_control import start_run, get_current_run, end_run, add_run_params, get_run_doc_for_run_num, get_sample_for_run, \ - get_specified_run_params_for_all_runs, is_run_closed, get_run_nums_matching_params, get_run_nums_matching_editable_regex, \ - map_param_editable_to_run_nums - -from dal.utils import JSONEncoder, escape_chars_for_mongo, replaceInfNan - -from dal.exp_cache import get_experiments, get_experiments_for_user, does_experiment_exist, reload_cache as reload_experiment_cache, \ - text_search_for_experiments, get_experiment_stats, get_experiment_daily_data_breakdown, \ - get_experiments_with_post_privileges, get_cached_experiment_names, get_all_param_names_matching_regex, get_experiments_proposal_mappings, \ - update_single_experiment_info, get_experiments_starting_in_time_frame, get_sorted_experiments_ids, get_cached_experiment_info, \ - search_experiments_for_common_fields, get_direct_experiments_for_user, get_potentially_active_users, get_recently_updated_experiments - -from dal.imagestores import parseImageStoreURL - -__author__ = 'mshankar@slac.stanford.edu' - -explgbk_blueprint = Blueprint('experiment_logbook_api', __name__) +from flask import ( + Blueprint, + jsonify, + request, + Response, + stream_with_context, + send_file, + abort, + redirect, + make_response, + g, + current_app, +) + +from explgbk.dal.explgbk import ( + LgbkException, + get_experiment_info, + save_new_experiment_setup, + register_new_experiment, + get_instruments, + get_currently_active_experiments, + switch_experiment, + get_elog_entries, + post_new_log_entry, + get_specific_elog_entry, + get_specific_shift, + get_experiment_files, + get_experiment_runs, + get_all_run_tables, + get_runtable_data, + get_runtable_sources, + create_update_user_run_table_def, + update_editable_param_for_run, + get_instrument_station_list, + update_existing_experiment, + create_update_instrument, + get_experiment_shifts, + get_shift_for_experiment_by_name, + close_shift_for_experiment, + create_update_shift, + get_latest_shift, + get_samples, + create_sample, + update_sample, + get_sample_for_experiment_by_name, + make_sample_current, + register_file_for_experiment, + search_elog_for_text, + delete_run_table, + get_current_sample_name, + get_elogs_for_run_num, + get_elogs_for_run_num_range, + get_elogs_for_specified_id, + get_collaborators, + get_role_object, + add_collaborator_to_role, + remove_collaborator_from_role, + delete_elog_entry, + modify_elog_entry, + clone_experiment, + rename_experiment, + instrument_standby, + get_experiment_files_for_run, + get_elog_authors, + get_elog_entries_by_author, + get_elog_tags, + get_elog_entries_by_tag, + get_elogs_for_date_range, + clone_sample, + get_modal_param_definitions, + lock_unlock_experiment, + get_elog_emails, + get_elog_email_subscriptions, + elog_email_subscribe, + elog_email_unsubscribe, + get_elog_email_subscriptions_emails, + add_poc_feedback_item, + clone_run_table_definition, + replace_system_run_table_definition, + delete_system_run_table, + get_instrument_elogs, + post_related_elog_entry, + get_related_instrument_elog_entries, + get_elog_tree_for_specified_id, + get_workflow_definitions, + get_dm_locations, + get_workflow_triggers, + create_update_wf_definition, + get_workflow_jobs, + get_workflow_job_doc, + create_wf_job, + delete_wf_job, + update_wf_job, + file_available_at_location, + get_collaborators_list_for_experiment, + get_site_naming_conventions, + delete_sample_for_experiment, + get_global_roles, + add_player_to_global_role, + remove_player_from_global_role, + get_site_config, + file_not_available_at_location, + get_experiment_run_document, + get_experiment_files_for_run_for_live_mode, + get_switch_history, + delete_experiment, + migrate_attachments_to_local_store, + get_complete_elog_tree_for_specified_id, + get_site_file_types, + add_player_to_instrument_role, + remove_player_from_instrument_role, + delete_wf_definition, + get_elog_entries_by_regex, + get_run_param_descriptions, + add_update_run_param_descriptions, + change_sample_for_run, + add_update_experiment_params, + get_ques_proposal_details, + import_users_from_URAWI, + get_poc_feedback_document, + get_poc_feedback_experiments, + get_experiment_files_for_run_for_live_mode_at_location, + get_active_experiment_name_for_instrument_station, + get_experiment_files_for_live_mode_at_location, + get_run_numbers_with_tag, + stop_current_sample, + get_tag_to_run_numbers, + get_tags_for_runs, + clone_system_template_run_tables_into_experiment, + get_projects, + get_project_info, + create_project, + update_project, + get_project_grids, + get_project_grid, + add_grid_to_project, + update_project_grid, + link_grid_to_experiment, + get_exp_file_counts_by_extension, + questionnaire_cache_refresh, +) + + +from explgbk.dal.run_control import ( + start_run, + get_current_run, + end_run, + add_run_params, + get_run_doc_for_run_num, + get_sample_for_run, + get_specified_run_params_for_all_runs, + is_run_closed, + get_run_nums_matching_params, + get_run_nums_matching_editable_regex, + map_param_editable_to_run_nums, +) + +from explgbk.dal.utils import JSONEncoder, escape_chars_for_mongo, replaceInfNan + +from explgbk.dal.exp_cache import ( + get_experiments_for_user, + does_experiment_exist, + reload_cache as reload_experiment_cache, + text_search_for_experiments, + get_experiment_stats, + get_experiment_daily_data_breakdown, + get_experiments_with_post_privileges, + get_cached_experiment_names, + get_all_param_names_matching_regex, + get_experiments_proposal_mappings, + update_single_experiment_info, + get_experiments_starting_in_time_frame, + get_sorted_experiments_ids, + get_cached_experiment_info, + search_experiments_for_common_fields, + get_direct_experiments_for_user, + get_potentially_active_users, + get_recently_updated_experiments, +) + +from explgbk.dal.imagestores import parseImageStoreURL + +__author__ = "mshankar@slac.stanford.edu" + +explgbk_blueprint = Blueprint("experiment_logbook_api", __name__) + def addHeaders(resp): # We don't send html with this blueprint; so we use that as a default. - if 'Content-Type' not in resp.headers or resp.headers['Content-Type'].startswith('text/html'): - resp.headers['Content-Type'] = 'application/json; charset=utf-8' + if "Content-Type" not in resp.headers or resp.headers["Content-Type"].startswith( + "text/html" + ): + resp.headers["Content-Type"] = "application/json; charset=utf-8" return resp + explgbk_blueprint.after_request(addHeaders) logger = logging.getLogger(__name__) + def logAndAbort(error_msg, ret_status=500): logger.error(error_msg) return Response(error_msg, status=ret_status) + def experiment_exists(wrapped_function): """ Decorator to make sure experiment_name in the argument to the ws call exists. """ + @wraps(wrapped_function) def function_interceptor(*args, **kwargs): - experiment_name = kwargs.get('experiment_name', None) + experiment_name = kwargs.get("experiment_name", None) if experiment_name and does_experiment_exist(experiment_name): exp_info = get_experiment_info(experiment_name) g.experiment_name = experiment_name @@ -109,41 +250,54 @@ def function_interceptor(*args, **kwargs): g.exp_info = exp_info return wrapped_function(*args, **kwargs) else: - logger.error("Experiment %s does not exist in the experiment cache", experiment_name) + logger.error( + "Experiment %s does not exist in the experiment cache", experiment_name + ) abort(404) return None return function_interceptor + def experiment_exists_and_unlocked(wrapped_function): """ Decorator to make sure experiment_name in the argument to the ws call exists. """ + @wraps(wrapped_function) def function_interceptor(*args, **kwargs): - experiment_name = kwargs.get('experiment_name', None) + experiment_name = kwargs.get("experiment_name", None) if experiment_name and does_experiment_exist(experiment_name): exp_info = get_experiment_info(experiment_name) - if exp_info.get("is_locked", False) and set(["POST", "PUT", "DELETE"]) & set(request.url_rule.methods): - logger.error("Experiment %s is locked; methods that modify data are not allowed. To change data, please unlock the experiment.", experiment_name) - abort(423) # Webdav locked. + if exp_info.get("is_locked", False) and set( + ["POST", "PUT", "DELETE"] + ) & set(request.url_rule.methods): + logger.error( + "Experiment %s is locked; methods that modify data are not allowed. To change data, please unlock the experiment.", + experiment_name, + ) + abort(423) # Webdav locked. return None g.experiment_name = experiment_name g.instrument = exp_info["instrument"] g.exp_info = exp_info return wrapped_function(*args, **kwargs) else: - logger.error("Experiment %s does not exist in the experiment cache", experiment_name) + logger.error( + "Experiment %s does not exist in the experiment cache", experiment_name + ) abort(404) return None return function_interceptor + def instrument_exists(wrapped_function): """ Decorator to pull the instrument name from the request in the absence of an experiment. For example, when switching an experiment etc. """ + @wraps(wrapped_function) def function_interceptor(*args, **kwargs): info = request.json @@ -156,6 +310,7 @@ def function_interceptor(*args, **kwargs): logger.error("No instrument specified in call") abort(404) return None + return function_interceptor @@ -165,30 +320,31 @@ def function_interceptor(*args, **kwargs): @context.security.authorization_required("read") def svc_getexpinfo(experiment_name): """ -
Gets the basic info for the specified experiment - 
{
-  "success": true,
-  "value": {
-    "_id": "diadaq13",
-    "name": "diadaq13",
-    "description": "Testing the DAQ system software of the instrument",
-    "instrument": "DIA",
-    "registration_time": "2013-05-24T03:52:25+00:00",
-    "start_time": "2013-05-24T03:50:49+00:00",
-    "end_time": "2013-05-24T03:50:50+00:00",
-    "leader_account": "gapon",
-    "contact_info": "Igor Gaponenko (gapon@slac.stanford.edu)",
-    "posix_group": "diadaq13",
-    "params": {
-      "DATA_PATH": "/reg/data/ana01/",
-      "dm_locations": "NERSC"
-    }
-  }
-}
-
+
Gets the basic info for the specified experiment + 
{
+      "success": true,
+      "value": {
+        "_id": "diadaq13",
+        "name": "diadaq13",
+        "description": "Testing the DAQ system software of the instrument",
+        "instrument": "DIA",
+        "registration_time": "2013-05-24T03:52:25+00:00",
+        "start_time": "2013-05-24T03:50:49+00:00",
+        "end_time": "2013-05-24T03:50:50+00:00",
+        "leader_account": "gapon",
+        "contact_info": "Igor Gaponenko (gapon@slac.stanford.edu)",
+        "posix_group": "diadaq13",
+        "params": {
+          "DATA_PATH": "/reg/data/ana01/",
+          "dm_locations": "NERSC"
+        }
+      }
+    }
+
""" info = get_experiment_info(experiment_name) - return JSONEncoder().encode({'success': True, 'value': info}) + return JSONEncoder().encode({"success": True, "value": info}) + @explgbk_blueprint.route("/lgbk//ws/internalinfo", methods=["GET"]) @experiment_exists @@ -204,13 +360,15 @@ def svc_get_internal_info(experiment_name): info = get_experiment_info(experiment_name) ret = { "instrument": info["instrument"], - "proposal_id": __map_experiment_to_URAWI_proposal__(experiment_name, info)["proposal_id"], + "proposal_id": __map_experiment_to_URAWI_proposal__(experiment_name, info)[ + "proposal_id" + ], "start_time": info.get("start_time"), "end_time": info.get("end_time"), "posix_group": info.get("posix_group"), "params": info.get("params", {}), } - return JSONEncoder().encode({'success': True, 'value': ret}) + return JSONEncoder().encode({"success": True, "value": ret}) @explgbk_blueprint.route("/lgbk//ws/info/setup", methods=["POST"]) @@ -224,19 +382,26 @@ def svc_saveexpinfosetup(experiment_name): """ setup_details = json.loads(request.data.decode("utf-8")) logger.info("Saving setup %s", setup_details) - save_new_experiment_setup(experiment_name, setup_details, context.security.get_current_user_id()) + save_new_experiment_setup( + experiment_name, setup_details, context.security.get_current_user_id() + ) return jsonify({"success": True}) + class LegacyCatSort(metaclass=abc.ABCMeta): def __init__(self): - self.legacy_run_period = -1 # Run periods older than this run are categorized into a legacy bucket + self.legacy_run_period = ( + -1 + ) # Run periods older than this run are categorized into a legacy bucket + @abc.abstractmethod def set_legacy_cutoff(self, legacy_run_period): - """ Set the run period before which all experiment are bucketed into a legacy bucket""" + """Set the run period before which all experiment are bucketed into a legacy bucket""" raise NotImplementedError + def __estimate_run_period__(self, exp): - """ Estimate the LCLS run period """ + """Estimate the LCLS run period""" erp = 0 override = exp.get("params", {}).get("run_period", None) if override: @@ -246,42 +411,79 @@ def __estimate_run_period__(self, exp): if erp and erp <= self.legacy_run_period: return 1 ins = exp["instrument"] - ctx_lg_cutoff = context.instrument_definitions.get(ins, {}).get("params", {}).get("legacy_cutoff", None) + ctx_lg_cutoff = ( + context.instrument_definitions.get(ins, {}) + .get("params", {}) + .get("legacy_cutoff", None) + ) if ctx_lg_cutoff and erp <= int(ctx_lg_cutoff): return 1 return erp + class CategorizerWithLegacy(LegacyCatSort): def __bucket_run_period__(self, exp): rp = self.__estimate_run_period__(exp) if rp == 1: return "Previous" return "Run " + str(rp) if rp else "null" + def __call__(self, exp): return self.__bucket_run_period__(exp) + def set_legacy_cutoff(self, legacy_run_period): self.legacy_run_period = legacy_run_period + class SorterWithLegacy(LegacyCatSort): def __init__(self): - self.legacy_run_period = -1 # Run periods older than this run are categorized into a legacy bucket + self.legacy_run_period = ( + -1 + ) # Run periods older than this run are categorized into a legacy bucket + def __call__(self, exp): return self.__estimate_run_period__(exp) + def set_legacy_cutoff(self, legacy_run_period): self.legacy_run_period = legacy_run_period categorizers = { - "instrument": [(lambda exp : exp.get("instrument", None))], - "instrument_lastrunyear": [(lambda exp : exp.get("instrument", None)), (lambda exp : exp["last_run"]["begin_time"].year if "last_run" in exp else None)], - "instrument_runperiod": [(lambda exp : exp.get("instrument", None)), CategorizerWithLegacy() ], - } + "instrument": [(lambda exp: exp.get("instrument", None))], + "instrument_lastrunyear": [ + (lambda exp: exp.get("instrument", None)), + (lambda exp: exp["last_run"]["begin_time"].year if "last_run" in exp else None), + ], + "instrument_runperiod": [ + (lambda exp: exp.get("instrument", None)), + CategorizerWithLegacy(), + ], +} sorters = { "name": ((lambda exp: exp["name"]), False), - "lastrunyear": ((lambda exp: exp["last_run"]["begin_time"] if "last_run" in exp else exp["start_time"]), True), - "runperiod": ((lambda exp: exp["last_run"]["begin_time"] if "last_run" in exp else exp["start_time"]), True), - } + "lastrunyear": ( + ( + lambda exp: ( + exp["last_run"]["begin_time"] + if "last_run" in exp + else exp["start_time"] + ) + ), + True, + ), + "runperiod": ( + ( + lambda exp: ( + exp["last_run"]["begin_time"] + if "last_run" in exp + else exp["start_time"] + ) + ), + True, + ), +} + def categorize(explist, categorizers, sorter): ret = {} @@ -291,13 +493,13 @@ def categorize(explist, categorizers, sorter): cur_dict = ret for n, categorizer in enumerate(categorizers): key = categorizer(exp) - if (n+1) == len(categorizers): + if (n + 1) == len(categorizers): if key not in cur_dict: - cur_dict[key] = [] + cur_dict[key] = [] cur_dict[key].append(exp) else: if key not in cur_dict: - cur_dict[key] = {} + cur_dict[key] = {} cur_dict = cur_dict[key] return ret @@ -339,15 +541,22 @@ def svc_get_experiments(): srt.set_legacy_cutoff(legacy_run_period) if categorizer and sortby: - return JSONEncoder().encode({"success": True, "value": categorize(experiments, categorizer, sortby)}) + return JSONEncoder().encode( + {"success": True, "value": categorize(experiments, categorizer, sortby)} + ) if sortby: - return JSONEncoder().encode({"success": True, "value": sorted(experiments, key=sortby[0], reverse=sortby[1])}) + return JSONEncoder().encode( + { + "success": True, + "value": sorted(experiments, key=sortby[0], reverse=sortby[1]), + } + ) return JSONEncoder().encode({"success": True, "value": experiments}) + @explgbk_blueprint.route("/lgbk/ws/sorted_experiment_ids", methods=["GET"]) @context.security.authentication_required -@context.security.authorization_required("experiment_create") def svc_get_sorted_experiments_ids(): """ Get just the experiment ids ( normalized names ) sorted by the specified criteria. @@ -359,7 +568,10 @@ def svc_get_sorted_experiments_ids(): """ sort_criteria = json.loads(request.args.get("sort", '[["start_time", -1]]')) logger.info("Sorting by " + json.dumps(sort_criteria)) - return JSONEncoder().encode({"success": True, "value": get_sorted_experiments_ids(sort_criteria)}) + return JSONEncoder().encode( + {"success": True, "value": get_sorted_experiments_ids(sort_criteria)} + ) + @explgbk_blueprint.route("/lgbk/ws/get_experiment_infos", methods=["POST"]) @context.security.authentication_required @@ -369,13 +581,14 @@ def svc_get_experiment_infos(): Pass in a JSON array of experiment ids to get the experiment infos for them as an array. """ exp_ids = request.json - ret = [ get_cached_experiment_info(exp_id) for exp_id in exp_ids ] - ret = [ x for x in ret if x ] + ret = [get_cached_experiment_info(exp_id) for exp_id in exp_ids] + ret = [x for x in ret if x] for exp in ret: if "all_param_names" in exp: del exp["all_param_names"] return JSONEncoder().encode({"success": True, "value": ret}) + @explgbk_blueprint.route("/lgbk/ws/ops_search_exp_infos", methods=["GET"]) @context.security.authentication_required @context.security.authorization_required("experiment_create") @@ -387,9 +600,13 @@ def svc_ops_search_experiment_infos(): """ search_term = request.args["search_text"] sort_criteria = json.loads(request.args.get("sort", '[["start_time", -1]]')) - matching_experiment_ids = [x["_id"] for x in search_experiments_for_common_fields(search_term, sort_criteria)] + matching_experiment_ids = [ + x["_id"] + for x in search_experiments_for_common_fields(search_term, sort_criteria) + ] return JSONEncoder().encode({"success": True, "value": matching_experiment_ids}) + def __map_experiment_to_URAWI_proposal__(expname, ep): einfo = {"name": expname, "instrument": ep.get("instrument", "N/A")} if ep.get("params", {}).get("PNR", None): @@ -399,7 +616,7 @@ def __map_experiment_to_URAWI_proposal__(expname, ep): einfo["proposal_id"] = expname[3:7].upper() elif len(expname) == 8: # Older experiments where we used to drop the L - einfo["proposal_id"] = 'L' + expname[3:6].upper() + einfo["proposal_id"] = "L" + expname[3:6].upper() else: # Possibly internal commissioning experiments which do not have a proposal id einfo["proposal_id"] = expname @@ -426,7 +643,10 @@ def svc_get_experiments_to_proposal(): return JSONEncoder().encode({"success": True, "value": ret}) -@explgbk_blueprint.route("/lgbk/ws/experiments_with_user_as_collaborator", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk/ws/experiments_with_user_as_collaborator", methods=["GET"] +) def svc_get_experiments_with_user_as_collaborator(): """ Returns a list of experiment for which the user is listed as a collaborator in the roles @@ -438,8 +658,11 @@ def svc_get_experiments_with_user_as_collaborator(): uid = request.args.get("uid", None) if not uid: return logAndAbort("Please specify a uid") - - return JSONEncoder().encode({"success": True, "value": get_direct_experiments_for_user(uid)}) + + return JSONEncoder().encode( + {"success": True, "value": get_direct_experiments_for_user(uid)} + ) + @explgbk_blueprint.route("/lgbk/ws/search_experiment_info", methods=["GET"]) @context.security.authentication_required @@ -450,9 +673,11 @@ def svc_search_experiment_info(): """ search_terms = request.args.get("search_text", "") experiments = get_experiments_for_user(context.security.get_current_user_id()) - matching_experiment_names = [x["name"] for x in text_search_for_experiments(search_terms)] + matching_experiment_names = [ + x["name"] for x in text_search_for_experiments(search_terms) + ] user_matches = [x for x in experiments if x["name"] in matching_experiment_names] - return jsonify({'success': True, 'value': user_matches}) + return jsonify({"success": True, "value": user_matches}) @explgbk_blueprint.route("/lgbk/ws/postable_experiments", methods=["GET"]) @@ -465,7 +690,15 @@ def svc_get_experiments_with_post_privileges(): Else we query the experiment cache and return those. """ userid = context.security.get_current_user_id() - return jsonify({'success': True, 'value': get_experiments_with_post_privileges(userid, get_currently_active_experiments())}) + return jsonify( + { + "success": True, + "value": get_experiments_with_post_privileges( + userid, get_currently_active_experiments() + ), + } + ) + @explgbk_blueprint.route("/lgbk/ws/get_cached_experiment_names", methods=["GET"]) def svc_get_cached_experiment_names(): @@ -473,7 +706,7 @@ def svc_get_cached_experiment_names(): Get a list of the cached experiment names. Mainly meant for debugging. """ - return jsonify({'success': True, 'value': get_cached_experiment_names()}) + return jsonify({"success": True, "value": get_cached_experiment_names()}) @explgbk_blueprint.route("/lgbk/ws/experiment_names_updated_within", methods=["GET"]) @@ -484,8 +717,13 @@ def svc_get_recently_updated_experiments(): This may be enhanced in the future to include elog/workflow_jobs etc. :param: Specify the time offset as offset_seconds """ - offset_secs = int(request.args.get("offset_secs", str(8*60*60))) # Defaults to a shift - return jsonify({'success': True, 'value': get_recently_updated_experiments(offset_secs)}) + offset_secs = int( + request.args.get("offset_secs", str(8 * 60 * 60)) + ) # Defaults to a shift + return jsonify( + {"success": True, "value": get_recently_updated_experiments(offset_secs)} + ) + @explgbk_blueprint.route("/lgbk/ws/instruments", methods=["GET"]) @context.security.authentication_required @@ -493,7 +731,8 @@ def svc_get_instruments(): """ Get the list of instruments """ - return jsonify({'success': True, 'value': get_instruments()}) + return jsonify({"success": True, "value": get_instruments()}) + @explgbk_blueprint.route("/lgbk/ws/experiment_stats", methods=["GET"]) @context.security.authentication_required @@ -501,7 +740,8 @@ def svc_get_experiment_stats(): """ Get various experiment stats """ - return jsonify({'success': True, 'value': get_experiment_stats()}) + return jsonify({"success": True, "value": get_experiment_stats()}) + @explgbk_blueprint.route("/lgbk/ws/experiment_daily_data_breakdown", methods=["GET"]) @context.security.authentication_required @@ -511,7 +751,13 @@ def svc_get_experiment_daily_data_breakdown(): """ instrument = request.args.get("instrument", "ALL") report_type = request.args.get("report_type", "file_sizes") - return JSONEncoder().encode({'success': True, 'value': get_experiment_daily_data_breakdown(report_type, instrument)}) + return JSONEncoder().encode( + { + "success": True, + "value": get_experiment_daily_data_breakdown(report_type, instrument), + } + ) + @explgbk_blueprint.route("/lgbk/ws/instrument_station_list", methods=["GET"]) @context.security.authentication_required @@ -519,7 +765,10 @@ def svc_instrument_station_list(): """ Get the list of possible instrument/station pairs as a list. """ - return JSONEncoder().encode({'success': True, 'value': get_instrument_station_list()}) + return JSONEncoder().encode( + {"success": True, "value": get_instrument_station_list()} + ) + @explgbk_blueprint.route("/lgbk/ws/activeexperiments", methods=["GET"]) @context.security.authentication_required @@ -527,7 +776,9 @@ def svc_get_active_experiments(): """ Get the list of currently active experiments at each instrument/station. """ - return JSONEncoder().encode({'success': True, 'value': get_currently_active_experiments()}) + return JSONEncoder().encode( + {"success": True, "value": get_currently_active_experiments()} + ) @explgbk_blueprint.route("/lgbk/ws/potentiallyactiveusers", methods=["GET"]) @@ -542,14 +793,19 @@ def svc_get_potentially_active_users(): """ cutoff_days_str = request.args.get("cutoff_days", None) if not cutoff_days_str: - cutoff_date = datetime.utcnow() - timedelta(days=365*3) + cutoff_date = datetime.utcnow() - timedelta(days=365 * 3) else: cutoff_date = datetime.utcnow() - timedelta(days=int(cutoff_days_str)) logger.info("Getting users from experiments after %s", cutoff_date) - return JSONEncoder().encode({'success': True, 'value': get_potentially_active_users(cutoff_date)}) + return JSONEncoder().encode( + {"success": True, "value": get_potentially_active_users(cutoff_date)} + ) + -@explgbk_blueprint.route("/lgbk/ws/activeexperiment_for_instrument_station", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk/ws/activeexperiment_for_instrument_station", methods=["GET"] +) def svc_get_active_experiment_for_instrument_station(): """ Get the currently active experiment for a particular instrument/station. @@ -560,11 +816,16 @@ def svc_get_active_experiment_for_instrument_station(): if not instrument_name: return logAndAbort("Please pass in the instrument name, for example, XPP") station_num = int(request.args.get("station", "0")) - active_experiment = get_active_experiment_name_for_instrument_station(instrument_name, station_num) + active_experiment = get_active_experiment_name_for_instrument_station( + instrument_name, station_num + ) if active_experiment: - return JSONEncoder().encode({'success': True, 'value': active_experiment}) + return JSONEncoder().encode({"success": True, "value": active_experiment}) else: - return logAndAbort("Cannot find a valid active experiment for %s/%s, found %s" % (instrument_name, station_num, active_experiment)) + return logAndAbort( + "Cannot find a valid active experiment for %s/%s, found %s" + % (instrument_name, station_num, active_experiment) + ) @explgbk_blueprint.route("/lgbk/ws/usergroups", methods=["GET"]) @@ -575,7 +836,7 @@ def svc_getUserGroupsForAuthenticatedUser(): """ userid = context.security.get_current_user_id() groups = context.usergroups.get_user_posix_groups(userid) - return jsonify({'success': True, 'value': { "userid": userid, "groups": groups }}) + return jsonify({"success": True, "value": {"userid": userid, "groups": groups}}) @explgbk_blueprint.route("/lgbk/ws/instruments/", methods=["POST"]) @@ -589,7 +850,7 @@ def svc_create_instrument(): if not info: return logAndAbort("Creating instrument missing info document") - necessary_keys = set(['_id', 'description']) + necessary_keys = set(["_id", "description"]) missing_keys = necessary_keys - info.keys() if missing_keys: return logAndAbort("Creating instrument missing keys %s" % missing_keys) @@ -599,10 +860,13 @@ def svc_create_instrument(): (status, errormsg) = create_update_instrument(info["_id"], True, info) if status: - context.kafka_producer.send("instruments", {"instrument_name" : info["_id"], "CRUD": "Create", "value": info }) - return jsonify({'success': True}) + context.kafka_producer.send( + "instruments", + {"instrument_name": info["_id"], "CRUD": "Create", "value": info}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) @explgbk_blueprint.route("/lgbk/ws/instruments/", methods=["PUT"]) @@ -616,11 +880,11 @@ def svc_update_instrument(insid): if not info: return logAndAbort("Creating instrument missing info document") - necessary_keys = set(['_id', 'description']) + necessary_keys = set(["_id", "description"]) missing_keys = necessary_keys - info.keys() if missing_keys: return logAndAbort("Creating instrument missing keys %s" % missing_keys) - + if info["_id"] != insid: return logAndAbort("Instrument names do not match") @@ -629,10 +893,13 @@ def svc_update_instrument(insid): (status, errormsg) = create_update_instrument(info["_id"], False, info) if status: - context.kafka_producer.send("instruments", {"instrument_name" : info["_id"], "CRUD": "Update", "value": info }) - return jsonify({'success': True}) + context.kafka_producer.send( + "instruments", + {"instrument_name": info["_id"], "CRUD": "Update", "value": info}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) @explgbk_blueprint.route("/lgbk/ws/global_roles", methods=["GET"]) @@ -641,6 +908,7 @@ def svc_update_instrument(insid): def svc_get_global_roles(): return JSONEncoder().encode({"success": True, "value": get_global_roles()}) + @explgbk_blueprint.route("/lgbk/ws/add_player_to_global_role", methods=["POST"]) @context.security.authentication_required @context.security.authorization_required("manage_groups") @@ -656,7 +924,10 @@ def svc_add_player_to_global_role(): if not player: return logAndAbort("Please specify a player") - return JSONEncoder().encode({"success": True, "value": add_player_to_global_role(player, role)}) + return JSONEncoder().encode( + {"success": True, "value": add_player_to_global_role(player, role)} + ) + @explgbk_blueprint.route("/lgbk/ws/remove_player_from_global_role", methods=["POST"]) @context.security.authentication_required @@ -673,7 +944,10 @@ def svc_remove_player_from_global_role(): if not player: return logAndAbort("Please specify a player") - return JSONEncoder().encode({"success": True, "value": remove_player_from_global_role(player, role)}) + return JSONEncoder().encode( + {"success": True, "value": remove_player_from_global_role(player, role)} + ) + @explgbk_blueprint.route("/lgbk/ws/add_player_to_instrument_role", methods=["POST"]) @context.security.authentication_required @@ -693,9 +967,17 @@ def svc_add_player_to_instrument_role(): if not player: return logAndAbort("Please specify a player") - return JSONEncoder().encode({"success": True, "value": add_player_to_instrument_role(instrument, player, role)}) + return JSONEncoder().encode( + { + "success": True, + "value": add_player_to_instrument_role(instrument, player, role), + } + ) + -@explgbk_blueprint.route("/lgbk/ws/remove_player_from_instrument_role", methods=["POST"]) +@explgbk_blueprint.route( + "/lgbk/ws/remove_player_from_instrument_role", methods=["POST"] +) @context.security.authentication_required @context.security.authorization_required("manage_groups") def svc_remove_player_from_instrument_role(): @@ -713,7 +995,13 @@ def svc_remove_player_from_instrument_role(): if not player: return logAndAbort("Please specify a player") - return JSONEncoder().encode({"success": True, "value": remove_player_from_instrument_role(instrument, player, role)}) + return JSONEncoder().encode( + { + "success": True, + "value": remove_player_from_instrument_role(instrument, player, role), + } + ) + @explgbk_blueprint.route("/lgbk/ws/lookup_experiment_in_urawi", methods=["GET"]) @context.security.authentication_required @@ -725,9 +1013,14 @@ def svc_lookup_experiment_in_URAWI(): proposal_id = request.args.get("PNR", None) run_period = request.args.get("run_period", None) urawi_doc = get_ques_proposal_details(experiment_name, run_period, proposal_id) - if urawi_doc and urawi_doc.get("URAWI", {}) and urawi_doc["URAWI"].get("proposalNo", None): - return jsonify({'success': True, "value": urawi_doc}) - return jsonify({'success': False}) + if ( + urawi_doc + and urawi_doc.get("URAWI", {}) + and urawi_doc["URAWI"].get("proposalNo", None) + ): + return jsonify({"success": True, "value": urawi_doc}) + return jsonify({"success": False}) + @explgbk_blueprint.route("/lgbk/ws/register_new_experiment", methods=["POST"]) @context.security.authentication_required @@ -739,7 +1032,9 @@ def svc_register_new_experiment(): """ experiment_name = request.args.get("experiment_name", None) if not experiment_name: - return logAndAbort("Experiment registration missing experiment_name in query parameters") + return logAndAbort( + "Experiment registration missing experiment_name in query parameters" + ) if does_experiment_exist(experiment_name): return logAndAbort("Experiment %s already exists" % experiment_name) @@ -747,31 +1042,63 @@ def svc_register_new_experiment(): if not info: return logAndAbort("Experiment registration missing info document") - necessary_keys = set(['instrument', 'start_time', 'end_time', 'leader_account', 'contact_info']) + necessary_keys = set( + ["instrument", "start_time", "end_time", "leader_account", "contact_info"] + ) missing_keys = necessary_keys - info.keys() if missing_keys: return logAndAbort("Experiment registration missing keys %s" % missing_keys) if info["instrument"] not in set([x["_id"] for x in get_instruments()]): - return logAndAbort("The instrument specified %s is not a valid instrument" % info["instrument"]) - if 'posix_group' in info and len(info["posix_group"].strip()) < 1: - del info['posix_group'] + return logAndAbort( + "The instrument specified %s is not a valid instrument" + % info["instrument"] + ) + if "posix_group" in info and len(info["posix_group"].strip()) < 1: + del info["posix_group"] (status, errormsg) = register_new_experiment(experiment_name, info) try: import_users_from_URAWI(experiment_name) - except: + except Exception: logger.exception("Exception importing users from URAWI") if status: - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Create", "value": info }) - context.kafka_producer.send("shifts", {"experiment_name" : experiment_name, "CRUD": "Create", "value": get_latest_shift(experiment_name) }) + context.kafka_producer.send( + "experiments", + {"experiment_name": experiment_name, "CRUD": "Create", "value": info}, + ) + context.kafka_producer.send( + "shifts", + { + "experiment_name": experiment_name, + "CRUD": "Create", + "value": get_latest_shift(experiment_name), + }, + ) role_obj = get_role_object(experiment_name, "LogBook/Editor") - role_obj.update({'collaborators_added': [x for x in get_collaborators_list_for_experiment(experiment_name)], 'collaborators_removed': [], 'requestor': context.security.get_current_user_id()}) - context.kafka_producer.send("roles", {"experiment_name" : experiment_name, "instrument": get_experiment_info(experiment_name)["instrument"], "CRUD": "Update", "value": role_obj }) - return jsonify({'success': True}) + role_obj.update( + { + "collaborators_added": [ + x for x in get_collaborators_list_for_experiment(experiment_name) + ], + "collaborators_removed": [], + "requestor": context.security.get_current_user_id(), + } + ) + context.kafka_producer.send( + "roles", + { + "experiment_name": experiment_name, + "instrument": get_experiment_info(experiment_name)["instrument"], + "CRUD": "Update", + "value": role_obj, + }, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + @explgbk_blueprint.route("/lgbk/ws/experiment_edit_info", methods=["POST"]) @context.security.authentication_required @@ -783,23 +1110,31 @@ def svc_experiment_edit_info(): """ experiment_name = request.args.get("experiment_name", None) if not experiment_name: - return logAndAbort("Experiment registration missing experiment_name in query parameters") + return logAndAbort( + "Experiment registration missing experiment_name in query parameters" + ) info = request.json if not info: return logAndAbort("Experiment registration missing info document") - necessary_keys = set(['instrument', 'start_time', 'end_time', 'leader_account', 'contact_info']) + necessary_keys = set( + ["instrument", "start_time", "end_time", "leader_account", "contact_info"] + ) missing_keys = necessary_keys - info.keys() if missing_keys: return logAndAbort("Experiment registration missing keys %s" % missing_keys) (status, errormsg) = update_existing_experiment(experiment_name, info) if status: - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Update", "value": info}) - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + {"experiment_name": experiment_name, "CRUD": "Update", "value": info}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + @explgbk_blueprint.route("/lgbk/ws/clone_experiment", methods=["POST"]) @context.security.authentication_required @@ -810,30 +1145,49 @@ def svc_clone_experiment(): """ experiment_name = request.args.get("experiment_name", None) if not experiment_name: - return logAndAbort("Experiment clone missing experiment_name in query parameters") + return logAndAbort( + "Experiment clone missing experiment_name in query parameters" + ) src_experiment_name = request.args.get("src_experiment_name", None) if not src_experiment_name: - return logAndAbort("Experiment clone missing src_experiment_name in query parameters") + return logAndAbort( + "Experiment clone missing src_experiment_name in query parameters" + ) info = request.json if not info: return logAndAbort("Experiment clone missing info document") - copy_specs = {k.replace("copy_", "") : v for k,v in info.items() if k.startswith("copy_")} - info = {k : v for k,v in info.items() if not k.startswith("copy_")} + copy_specs = { + k.replace("copy_", ""): v for k, v in info.items() if k.startswith("copy_") + } + info = {k: v for k, v in info.items() if not k.startswith("copy_")} - necessary_keys = set(['start_time', 'end_time']) + necessary_keys = set(["start_time", "end_time"]) missing_keys = necessary_keys - info.keys() if missing_keys: return logAndAbort("Experiment clone missing keys %s" % missing_keys) - (status, errormsg) = clone_experiment(experiment_name, src_experiment_name, info, copy_specs) + (status, errormsg) = clone_experiment( + experiment_name, src_experiment_name, info, copy_specs + ) if status: - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Create", "value": info }) - context.kafka_producer.send("shifts", {"experiment_name" : experiment_name, "CRUD": "Create", "value": get_latest_shift(experiment_name) }) - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + {"experiment_name": experiment_name, "CRUD": "Create", "value": info}, + ) + context.kafka_producer.send( + "shifts", + { + "experiment_name": experiment_name, + "CRUD": "Create", + "value": get_latest_shift(experiment_name), + }, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + @explgbk_blueprint.route("/lgbk/ws/rename_experiment", methods=["POST"]) @context.security.authentication_required @@ -844,22 +1198,39 @@ def svc_rename_experiment(): """ experiment_name = request.args.get("experiment_name", None) if not experiment_name: - return logAndAbort("Experiment rename missing experiment_name in query parameters") + return logAndAbort( + "Experiment rename missing experiment_name in query parameters" + ) new_experiment_name = request.args.get("new_experiment_name", None) if not new_experiment_name: - return logAndAbort("Experiment clone missing src_experiment_name in query parameters") + return logAndAbort( + "Experiment clone missing src_experiment_name in query parameters" + ) old_info = copy.copy(get_experiment_info(experiment_name)) (status, errormsg) = rename_experiment(experiment_name, new_experiment_name) if status: - context.kafka_producer.send("experiments", {"experiment_name" : new_experiment_name, "CRUD": "Create", "value": get_experiment_info(new_experiment_name) }) - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Delete", "value": old_info }) - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + { + "experiment_name": new_experiment_name, + "CRUD": "Create", + "value": get_experiment_info(new_experiment_name), + }, + ) + context.kafka_producer.send( + "experiments", + {"experiment_name": experiment_name, "CRUD": "Delete", "value": old_info}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + -@explgbk_blueprint.route("/lgbk//ws/add_update_experiment_params", methods=["POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/add_update_experiment_params", methods=["POST"] +) @context.security.authentication_required @experiment_exists_and_unlocked @context.security.authorization_required("manage_groups") @@ -870,12 +1241,18 @@ def svc_add_update_experiment_params(experiment_name): """ params = request.json if not params: - return logAndAbort("Please send the experiment parameters as a simple name value pair JSON dictionary") + return logAndAbort( + "Please send the experiment parameters as a simple name value pair JSON dictionary" + ) status, errormsg = add_update_experiment_params(experiment_name, params) if status: info = get_experiment_info(experiment_name) - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Update", "value": info }) - return jsonify({'success': status, 'errormsg': errormsg}) + context.kafka_producer.send( + "experiments", + {"experiment_name": experiment_name, "CRUD": "Update", "value": info}, + ) + return jsonify({"success": status, "errormsg": errormsg}) + @explgbk_blueprint.route("/lgbk//", methods=["DELETE"]) @context.security.authentication_required @@ -888,12 +1265,18 @@ def svc_delete_experiment(experiment_name): old_info = copy.copy(get_experiment_info(experiment_name)) (status, errormsg) = delete_experiment(experiment_name) if status: - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Delete", "value": old_info }) - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + {"experiment_name": experiment_name, "CRUD": "Delete", "value": old_info}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + -@explgbk_blueprint.route("/lgbk//migrate_attachments", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/lgbk//migrate_attachments", methods=["GET", "POST"] +) @context.security.authentication_required @context.security.authorization_required("experiment_delete") def svc_migrate_attachments(experiment_name): @@ -902,7 +1285,8 @@ def svc_migrate_attachments(experiment_name): The sysadmin is then expected to use mongodump --archive to make a backup of the experiment which should then include the attachments as well. """ (status, errormsg) = migrate_attachments_to_local_store(experiment_name) - return jsonify({'success': status, 'errormsg': errormsg}) + return jsonify({"success": status, "errormsg": errormsg}) + @explgbk_blueprint.route("/lgbk/ws/lock_unlock_experiment", methods=["POST"]) @context.security.authentication_required @@ -913,14 +1297,23 @@ def svc_lock_unlock_experiment(): """ experiment_name = request.args.get("experiment_name", None) if not experiment_name: - return logAndAbort("Experiment lock/unlock missing experiment_name in query parameters") + return logAndAbort( + "Experiment lock/unlock missing experiment_name in query parameters" + ) (status, errormsg) = lock_unlock_experiment(experiment_name) if status: - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Update", "value": get_experiment_info(experiment_name)}) - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + { + "experiment_name": experiment_name, + "CRUD": "Update", + "value": get_experiment_info(experiment_name), + }, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) @explgbk_blueprint.route("/lgbk//ws/update_schedule", methods=["POST"]) @@ -933,38 +1326,52 @@ def svc_update_experiment_schedule(experiment_name): """ new_start_time = request.args.get("start_time", None) if not new_start_time: - return logAndAbort("Please specify the new start time as a start_time query parameter") + return logAndAbort( + "Please specify the new start time as a start_time query parameter" + ) new_end_time = request.args.get("end_time", None) if not new_end_time: - return logAndAbort("Please specify the new start time as a end_time query parameter") - start_time = datetime.strptime(new_start_time, '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=pytz.UTC) - end_time = datetime.strptime(new_end_time, '%Y-%m-%dT%H:%M:%S.%fZ').replace(tzinfo=pytz.UTC) + return logAndAbort( + "Please specify the new start time as a end_time query parameter" + ) + start_time = datetime.strptime(new_start_time, "%Y-%m-%dT%H:%M:%S.%fZ").replace( + tzinfo=pytz.UTC + ) + end_time = datetime.strptime(new_end_time, "%Y-%m-%dT%H:%M:%S.%fZ").replace( + tzinfo=pytz.UTC + ) if start_time >= end_time: return logAndAbort("Please specify a end time that is after the start time") if (end_time - start_time).total_seconds() < 3600: - return logAndAbort("We do not allow experiments with durations less than an hour") + return logAndAbort( + "We do not allow experiments with durations less than an hour" + ) info = get_experiment_info(experiment_name) - - tz = pytz.timezone('America/Los_Angeles') - logger.info("About to change the schedule for %s from %s to %s currently this is %s to %s", + + tz = pytz.timezone("America/Los_Angeles") + logger.info( + "About to change the schedule for %s from %s to %s currently this is %s to %s", experiment_name, - start_time.astimezone(tz).strftime('%b/%d/%Y %H:%M:%S'), - end_time.astimezone(tz).strftime('%b/%d/%Y %H:%M:%S'), - info["start_time"].strftime('%b/%d/%Y %H:%M:%S'), - info["end_time"].strftime('%b/%d/%Y %H:%M:%S') + start_time.astimezone(tz).strftime("%b/%d/%Y %H:%M:%S"), + end_time.astimezone(tz).strftime("%b/%d/%Y %H:%M:%S"), + info["start_time"].strftime("%b/%d/%Y %H:%M:%S"), + info["end_time"].strftime("%b/%d/%Y %H:%M:%S"), ) - info["start_time"] = start_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ') - info["end_time"] = end_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + info["start_time"] = start_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + info["end_time"] = end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") (status, errormsg) = update_existing_experiment(experiment_name, info) if status: - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Update", "value": info}) - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + {"experiment_name": experiment_name, "CRUD": "Update", "value": info}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) @explgbk_blueprint.route("/lgbk/ws/reload_experiment_cache", methods=["GET"]) @@ -977,11 +1384,11 @@ def svc_reload_experiment_cache(): experiment_name = request.args.get("experiment_name", None) if experiment_name: reload_experiment_cache(experiment_name=experiment_name) - return jsonify({'success': True}) - + return jsonify({"success": True}) reload_experiment_cache() - return jsonify({'success': True}) + return jsonify({"success": True}) + @explgbk_blueprint.route("/lgbk/ws/reload_named_cache", methods=["GET"]) @context.security.authentication_required @@ -992,11 +1399,14 @@ def svc_reload_named_cache(): """ cache_name = request.args.get("cache_name", None) if cache_name: - context.kafka_producer.send("explgbk_cache", { "named_cache": cache_name } ) - return jsonify({'success': True}) - return jsonify({'success': False}) + context.kafka_producer.send("explgbk_cache", {"named_cache": cache_name}) + return jsonify({"success": True}) + return jsonify({"success": False}) + -@explgbk_blueprint.route("/lgbk/ws/rebuild_experiment_cache_for_experiment", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk/ws/rebuild_experiment_cache_for_experiment", methods=["GET"] +) @context.security.authentication_required @context.security.authorization_required("edit") def svc_rebuild_experiment_cache(): @@ -1006,8 +1416,15 @@ def svc_rebuild_experiment_cache(): experiment_name = request.args.get("experiment_name", None) if experiment_name: update_single_experiment_info(experiment_name, "Update") - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Update", "value": get_experiment_info(experiment_name)}) - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + { + "experiment_name": experiment_name, + "CRUD": "Update", + "value": get_experiment_info(experiment_name), + }, + ) + return jsonify({"success": True}) @explgbk_blueprint.route("/lgbk/ws/switch_experiment", methods=["POST"]) @@ -1020,41 +1437,69 @@ def svc_switch_experiment(): """ info = request.json if not info: - return jsonify({'success': False, 'errormsg': "No data supplied.."}) + return jsonify({"success": False, "errormsg": "No data supplied.."}) experiment_name = info.get("experiment_name", None) if not experiment_name: - return jsonify({'success': False, 'errormsg': "No experiment name"}) + return jsonify({"success": False, "errormsg": "No experiment name"}) instrument = info.get("instrument", None) if not instrument: - return jsonify({'success': False, 'errormsg': "No instrument given"}) + return jsonify({"success": False, "errormsg": "No instrument given"}) - if 'station' not in info: - return jsonify({'success': False, 'errormsg': "No station given."}) + if "station" not in info: + return jsonify({"success": False, "errormsg": "No station given."}) station = int(info.get("station")) info_from_database = get_experiment_info(experiment_name) if not info_from_database: - return jsonify({'success': False, 'errormsg': "Experiment does not exist in the database"}) + return jsonify( + {"success": False, "errormsg": "Experiment does not exist in the database"} + ) if info_from_database["instrument"] != instrument: - return jsonify({'success': False, 'errormsg': "Trying to switch experiment on instrument %s for experiment on %s" % (instrument, info_from_database["instrument"])}) - - if experiment_name in [ x.get('name', '') for x in get_currently_active_experiments() ]: - return jsonify({'success': False, 'errormsg': "Trying to switch experiment %s onto instrument %s but it is already currently active" % (experiment_name, instrument)}) - - if context.LOGBOOK_SITE in ["LCLS"] and os.path.exists("/reg/g/psdm/data/ExpNameDb/experiment-db.dat"): - logger.info("Checking to see if the experiment exists in the old style file naming list of experiments - /reg/g/psdm/data/ExpNameDb/experiment-db.dat") - with open("/reg/g/psdm/data/ExpNameDb/experiment-db.dat", 'r') as f: + return jsonify( + { + "success": False, + "errormsg": "Trying to switch experiment on instrument %s for experiment on %s" + % (instrument, info_from_database["instrument"]), + } + ) + + if experiment_name in [ + x.get("name", "") for x in get_currently_active_experiments() + ]: + return jsonify( + { + "success": False, + "errormsg": "Trying to switch experiment %s onto instrument %s but it is already currently active" + % (experiment_name, instrument), + } + ) + + if context.LOGBOOK_SITE in ["LCLS"] and os.path.exists( + "/reg/g/psdm/data/ExpNameDb/experiment-db.dat" + ): + logger.info( + "Checking to see if the experiment exists in the old style file naming list of experiments - /reg/g/psdm/data/ExpNameDb/experiment-db.dat" + ) + with open("/reg/g/psdm/data/ExpNameDb/experiment-db.dat", "r") as f: lines = f.readlines() - if experiment_name in [ x.split()[2] for x in lines ]: - return jsonify({'success': False, 'errormsg': "The experiment %s is using old style file names as it exists in /reg/g/psdm/data/ExpNameDb/experiment-db.dat. Mixing and matching file name conventions is not supported." % (experiment_name)}) + if experiment_name in [x.split()[2] for x in lines]: + return jsonify( + { + "success": False, + "errormsg": "The experiment %s is using old style file names as it exists in /reg/g/psdm/data/ExpNameDb/experiment-db.dat. Mixing and matching file name conventions is not supported." + % (experiment_name), + } + ) userid = context.security.get_current_user_id() - previously_active_experiment = get_active_experiment_name_for_instrument_station(instrument, station) + previously_active_experiment = get_active_experiment_name_for_instrument_station( + instrument, station + ) (status, errormsg) = switch_experiment(instrument, station, experiment_name, userid) if status: @@ -1066,16 +1511,32 @@ def svc_switch_experiment(): } if previously_active_experiment: expswdoc["previous_experiment_name"] = previously_active_experiment["name"] - context.kafka_producer.send("experiment_switch", {"experiment_name" : experiment_name, "value": expswdoc}) + context.kafka_producer.send( + "experiment_switch", {"experiment_name": experiment_name, "value": expswdoc} + ) # We may add/remove operator_uid's etc Rebuild the caches for affected instruments. - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Update", "value": get_experiment_info(experiment_name) }) + context.kafka_producer.send( + "experiments", + { + "experiment_name": experiment_name, + "CRUD": "Update", + "value": get_experiment_info(experiment_name), + }, + ) if previously_active_experiment: - context.kafka_producer.send("experiments", {"experiment_name" : previously_active_experiment["name"], "CRUD": "Update", "value": get_experiment_info(previously_active_experiment["name"]) }) - - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + { + "experiment_name": previously_active_experiment["name"], + "CRUD": "Update", + "value": get_experiment_info(previously_active_experiment["name"]), + }, + ) + + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) @explgbk_blueprint.route("/lgbk/ws/instrument_standby", methods=["POST"]) @@ -1088,33 +1549,42 @@ def svc_instrument_standby(): """ info = request.json if not info: - return jsonify({'success': False, 'errormsg': "No data supplied.."}) + return jsonify({"success": False, "errormsg": "No data supplied.."}) instrument = info.get("instrument", None) if not instrument: - return jsonify({'success': False, 'errormsg': "No instrument given"}) + return jsonify({"success": False, "errormsg": "No instrument given"}) - if not "station" in info: - return jsonify({'success': False, 'errormsg': "No station given."}) + if "station" not in info: + return jsonify({"success": False, "errormsg": "No station given."}) station = info.get("station", None) userid = context.security.get_current_user_id() - previously_active_experiment = get_active_experiment_name_for_instrument_station(instrument, station) + previously_active_experiment = get_active_experiment_name_for_instrument_station( + instrument, station + ) (status, errormsg) = instrument_standby(instrument, station, userid) if status: - context.kafka_producer.send("instrument_standby", {"value": { - "instrument": instrument, - "station": station, - "userid": userid - }}) + context.kafka_producer.send( + "instrument_standby", + {"value": {"instrument": instrument, "station": station, "userid": userid}}, + ) if previously_active_experiment: - context.kafka_producer.send("experiments", {"experiment_name" : previously_active_experiment["name"], "CRUD": "Update", "value": get_experiment_info(previously_active_experiment["name"]) }) - return jsonify({'success': True}) + context.kafka_producer.send( + "experiments", + { + "experiment_name": previously_active_experiment["name"], + "CRUD": "Update", + "value": get_experiment_info(previously_active_experiment["name"]), + }, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + @explgbk_blueprint.route("/lgbk/ws/instrument_switch_history", methods=["GET"]) @context.security.authentication_required @@ -1123,7 +1593,14 @@ def svc_instrument_switch_history(): """ Get the history of experiment switches for an instrument/station. """ - return JSONEncoder().encode({"success": True, "value":get_switch_history(request.args["instrument"], int(request.args["station"]))}) + return JSONEncoder().encode( + { + "success": True, + "value": get_switch_history( + request.args["instrument"], int(request.args["station"]) + ), + } + ) @explgbk_blueprint.route("/lgbk//ws/has_role", methods=["GET"]) @@ -1136,20 +1613,43 @@ def svc_has_role(experiment_name): # Should this check for a privilege? For now, we stop at roles. role_fq_name = request.args.get("role_fq_name", None) if not role_fq_name: - return logAndAbort("Please pass in a fully qualified role name like LogBook/Editor") + return logAndAbort( + "Please pass in a fully qualified role name like LogBook/Editor" + ) application_name, role_name = role_fq_name.split("/") - return JSONEncoder().encode({"success": True, - "value": { - "role_fq_name": role_fq_name, "application_name": application_name, "role_name": role_name, - "hasRole": context.roleslookup.has_slac_user_role(context.security.get_current_user_id(), application_name, role_name, experiment_name, instrument=g.instrument) - }}) + return JSONEncoder().encode( + { + "success": True, + "value": { + "role_fq_name": role_fq_name, + "application_name": application_name, + "role_name": role_name, + "hasRole": context.roleslookup.has_slac_user_role( + context.security.get_current_user_id(), + application_name, + role_name, + experiment_name, + instrument=g.instrument, + ), + }, + } + ) + @explgbk_blueprint.route("/lgbk//ws/elog", methods=["GET"]) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_elog_entries(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_elog_entries(experiment_name, sample_name=request.args.get("sampleName", None))}) + return JSONEncoder().encode( + { + "success": True, + "value": get_elog_entries( + experiment_name, sample_name=request.args.get("sampleName", None) + ), + } + ) + @explgbk_blueprint.route("/lgbk//ws/attachment", methods=["GET"]) @context.security.authentication_required @@ -1159,7 +1659,12 @@ def svc_get_elog_attachment(experiment_name): entry_id = request.args.get("entry_id", None) attachment_id = request.args.get("attachment_id", None) prefer_preview = request.args.get("prefer_preview", "False").lower() == "true" - logger.info("Fetching attachment %s for entry %s prefer_preview is %s", attachment_id, entry_id, prefer_preview) + logger.info( + "Fetching attachment %s for entry %s prefer_preview is %s", + attachment_id, + entry_id, + prefer_preview, + ) entry = get_specific_elog_entry(experiment_name, entry_id) for attachment in entry.get("attachments", None): if str(attachment.get("_id", None)) == attachment_id: @@ -1169,21 +1674,31 @@ def svc_get_elog_attachment(experiment_name): logger.debug("Returning preview") remote_url = attachment.get("preview_url", None) else: - return send_file('static/attachment.png') + return send_file("static/attachment.png") else: logger.debug("Returning main document") remote_url = attachment.get("url", None) - file_type = attachment['type'] + file_type = attachment["type"] if remote_url: - urlcontents = parseImageStoreURL(remote_url).return_url_contents(experiment_name, remote_url) + urlcontents = parseImageStoreURL(remote_url).return_url_contents( + experiment_name, remote_url + ) resp = make_response(send_file(urlcontents, mimetype=file_type)) - if not (attachment["type"].startswith("image") or "preview_url" in attachment): - resp.headers["Content-Disposition"] = 'attachment; filename="' + attachment["name"] + '"' + if not ( + attachment["type"].startswith("image") + or "preview_url" in attachment + ): + resp.headers["Content-Disposition"] = ( + 'attachment; filename="' + attachment["name"] + '"' + ) return resp - return Response("Cannot find attachment " + attachment_id , status=404) + return Response("Cannot find attachment " + attachment_id, status=404) + -@explgbk_blueprint.route("/lgbk//ws/ext_preview/", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/ext_preview/", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -1196,11 +1711,14 @@ def svc_get_ext_preview(experiment_name, path): """ path = path.replace("", experiment_name) m = hashlib.md5() - m.update(experiment_name.encode('utf-8')) - m.update(context.PREVIEW_PREFIX_SHARED_SECRET.encode('utf-8')) + m.update(experiment_name.encode("utf-8")) + m.update(context.PREVIEW_PREFIX_SHARED_SECRET.encode("utf-8")) # path = path.replace("", urllib.parse.quote(base64.standard_b64encode(m.hexdigest().encode()))) response = make_response(redirect(context.PREVIEW_PREFIX + "/" + path)) - response.set_cookie("LGBK_EXT_PREVIEW", urllib.parse.quote(base64.standard_b64encode(m.hexdigest().encode()))) + response.set_cookie( + "LGBK_EXT_PREVIEW", + urllib.parse.quote(base64.standard_b64encode(m.hexdigest().encode())), + ) return response @@ -1209,39 +1727,83 @@ def send_elog_as_email(experiment_name, elog_doc, email_to): Send the elog document as an emails to the specified list. """ try: - full_email_addresses = [ x + "@slac.stanford.edu" if '@' not in x else x for x in email_to] - logger.info("Sending elog " + elog_doc["content"] + " for experiment " + experiment_name + " as an email to " + ",".join(full_email_addresses)); - if(len(list(filter(lambda x: "@" in x , full_email_addresses)))) != len(full_email_addresses): - logger.error("Not all addresss in the email To list have a @ character. Not sending mail %s", full_email_addresses) + full_email_addresses = [ + x + "@slac.stanford.edu" if "@" not in x else x for x in email_to + ] + logger.info( + "Sending elog " + + elog_doc["content"] + + " for experiment " + + experiment_name + + " as an email to " + + ",".join(full_email_addresses) + ) + if (len(list(filter(lambda x: "@" in x, full_email_addresses)))) != len( + full_email_addresses + ): + logger.error( + "Not all addresss in the email To list have a @ character. Not sending mail %s", + full_email_addresses, + ) return False + def generateEMailMsgFromELogDoc(elog_doc): msg = EmailMessage() - tz = pytz.timezone('America/Los_Angeles') - msg_by_at = (elog_doc.get("author", ""), elog_doc.get("relevance_time", datetime.utcnow()).astimezone(tz).strftime('%b/%d/%Y %H:%M:%S')) - if 'title' in elog_doc: + tz = pytz.timezone("America/Los_Angeles") + msg_by_at = ( + elog_doc.get("author", ""), + elog_doc.get("relevance_time", datetime.utcnow()) + .astimezone(tz) + .strftime("%b/%d/%Y %H:%M:%S"), + ) + if "title" in elog_doc: msg.make_mixed() htmlmsg = EmailMessage() htmlmsg.make_alternative() htmlmsg.add_alternative( - "

By:{0} at:{1}

\n".format(*msg_by_at) - + elog_doc["content"], subtype='html') + "

By:{0} at:{1}

\n".format( + *msg_by_at + ) + + elog_doc["content"], + subtype="html", + ) msg.attach(htmlmsg) else: - msg.set_content("By: {0} at: {1}\n\n".format(*msg_by_at) + elog_doc["content"]) + msg.set_content( + "By: {0} at: {1}\n\n".format(*msg_by_at) + elog_doc["content"] + ) msg.make_mixed() for attachment in elog_doc.get("attachments", []): - if 'type' in attachment and '/' in attachment['type']: - maintype, subtype = attachment['type'].split('/', 1) + if "type" in attachment and "/" in attachment["type"]: + maintype, subtype = attachment["type"].split("/", 1) else: maintype, subtype = "application", "data" - with parseImageStoreURL(attachment["url"]).return_url_contents(experiment_name, attachment["url"]) as imgget: - msg.add_attachment(imgget.read(), maintype=maintype, subtype=subtype, filename=attachment['name']) + with parseImageStoreURL(attachment["url"]).return_url_contents( + experiment_name, attachment["url"] + ) as imgget: + msg.add_attachment( + imgget.read(), + maintype=maintype, + subtype=subtype, + filename=attachment["name"], + ) return msg msg = generateEMailMsgFromELogDoc(elog_doc) - msg['Subject'] = '' + "Elog message for " + experiment_name + " " + ("run {0} ".format(elog_doc["run_num"]) if elog_doc.get("run_num", None) else "") + elog_doc.get("title", "") - msg['From'] = 'exp_logbook_robot@slac.stanford.edu' - msg['To'] = ", ".join(full_email_addresses) + msg["Subject"] = ( + "" + + "Elog message for " + + experiment_name + + " " + + ( + "run {0} ".format(elog_doc["run_num"]) + if elog_doc.get("run_num", None) + else "" + ) + + elog_doc.get("title", "") + ) + msg["From"] = "exp_logbook_robot@slac.stanford.edu" + msg["To"] = ", ".join(full_email_addresses) parent_msg = msg while elog_doc.get("parent", None): elog_doc = get_specific_elog_entry(experiment_name, elog_doc["parent"]) @@ -1249,23 +1811,31 @@ def generateEMailMsgFromELogDoc(elog_doc): parent_msg.attach(child_message) parent_msg = child_message - with smtplib.SMTP(os.environ.get("EMAIL_SERVER_HOST", "smtp.slac.stanford.edu"), int(os.environ.get("EMAIL_SERVER_PORT", "25"))) as s: - mailstatus = s.sendmail(msg['From'], full_email_addresses, msg.as_string()) + with smtplib.SMTP( + os.environ.get("EMAIL_SERVER_HOST", "smtp.slac.stanford.edu"), + int(os.environ.get("EMAIL_SERVER_PORT", "25")), + ) as s: + mailstatus = s.sendmail(msg["From"], full_email_addresses, msg.as_string()) if mailstatus: logger.warn(mailstatus) s.quit() except Exception: - logger.exception("Exception sending elog emails for experiment " + experiment_name) + logger.exception( + "Exception sending elog emails for experiment " + experiment_name + ) return True -@explgbk_blueprint.route("/lgbk//ws/elog//complete_elog_tree", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws/elog//complete_elog_tree", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_complete_elog_tree_for_specified_id(experiment_name, entry_id): complete_tree = get_complete_elog_tree_for_specified_id(experiment_name, entry_id) - return JSONEncoder().encode({'success': True, 'value': complete_tree}) + return JSONEncoder().encode({"success": True, "value": complete_tree}) @explgbk_blueprint.route("/lgbk//ws/new_elog_entry", methods=["POST"]) @@ -1287,22 +1857,40 @@ def svc_post_new_elog_entry(experiment_name): if request.form.get("author", None): author = request.form["author"] logger.debug("Overriding the author with %s", author) - if not author.endswith("opr") and (author == "" or not context.usergroups.get_userids_matching_pattern(author)): + if not author.endswith("opr") and ( + author == "" or not context.usergroups.get_userids_matching_pattern(author) + ): raise Exception(f"Cannot override author with non-existent user {author}") optional_args = {} parent = request.form.get("parent", None) if parent: - logger.debug("We are creating a followup entry for " + parent + " for experiment " + experiment_name) + logger.debug( + "We are creating a followup entry for " + + parent + + " for experiment " + + experiment_name + ) parent_entry = get_specific_elog_entry(experiment_name, parent) if parent_entry: - optional_args["parent"] = parent_entry["_id"] # This should give back the oid - parent_root = parent_entry.get("root", None) # This should give back the oid - optional_args["root"] = parent_root if parent_root else parent_entry["_id"] # both should be oids + optional_args["parent"] = parent_entry[ + "_id" + ] # This should give back the oid + parent_root = parent_entry.get( + "root", None + ) # This should give back the oid + optional_args["root"] = ( + parent_root if parent_root else parent_entry["_id"] + ) # both should be oids else: - return logAndAbort("Cannot find parent entry for followup log message for experiment " + experiment_name + " for parent oid " + parent) + return logAndAbort( + "Cannot find parent entry for followup log message for experiment " + + experiment_name + + " for parent oid " + + parent + ) - run_num_str = request.form.get("run_num", None); + run_num_str = request.form.get("run_num", None) if run_num_str: if run_num_str == "current": current_run_doc = get_current_run(experiment_name) @@ -1312,21 +1900,29 @@ def svc_post_new_elog_entry(experiment_name): try: run_num = int(run_num_str) except ValueError: - run_num = run_num_str # Cryo uses strings for run numbers. + run_num = run_num_str # Cryo uses strings for run numbers. run_doc = get_run_doc_for_run_num(experiment_name, run_num) if not run_doc: - return JSONEncoder().encode({'success': False, 'errormsg': "Cannot find run with specified run number - " + str(run_num) + " for experiment " + experiment_name}) + return JSONEncoder().encode( + { + "success": False, + "errormsg": "Cannot find run with specified run number - " + + str(run_num) + + " for experiment " + + experiment_name, + } + ) optional_args["run_num"] = run_num - log_title = request.form.get("log_title", None); + log_title = request.form.get("log_title", None) if log_title: optional_args["title"] = log_title - shift = request.form.get("shift", None); + shift = request.form.get("shift", None) if shift: shift_obj = get_specific_shift(experiment_name, shift) if shift_obj: - optional_args["shift"] = shift_obj["_id"] # We should get a oid here + optional_args["shift"] = shift_obj["_id"] # We should get a oid here log_emails = request.form.get("log_emails", None) if log_emails: @@ -1348,7 +1944,9 @@ def svc_post_new_elog_entry(experiment_name): is_issue = request.form.get("elog_support_issue", None) if is_issue: logger.info("Issue detected in experiment %s", experiment_name) - optional_args["tags"] = optional_args["tags"] + ["ISSUE"] if "tags" in optional_args else ["ISSUE"] + optional_args["tags"] = ( + optional_args["tags"] + ["ISSUE"] if "tags" in optional_args else ["ISSUE"] + ) jira_ticket = request.form.get("jira_ticket", None) if jira_ticket: logger.info("Associated with an existing JIRA ticket %s", jira_ticket) @@ -1356,14 +1954,26 @@ def svc_post_new_elog_entry(experiment_name): is_roadblock = request.form.get("elog_support_roadblock", None) if is_roadblock: logger.info("Roadblock detected in experiment %s", experiment_name) - optional_args["tags"] = optional_args["tags"] + ["ROADBLOCK"] if "tags" in optional_args else ["ROADBLOCK"] - - post_to_elogs = [ k.replace('post_to_elog_', '') for k, v in request.form.items() if k.startswith('post_to_elog_') and v.lower() == "on" ] + optional_args["tags"] = ( + optional_args["tags"] + ["ROADBLOCK"] + if "tags" in optional_args + else ["ROADBLOCK"] + ) + + post_to_elogs = [ + k.replace("post_to_elog_", "") + for k, v in request.form.items() + if k.startswith("post_to_elog_") and v.lower() == "on" + ] # Alternate knob for cross posting to the intrument elog (if it exists). - xpost_instrument_elog = json.loads(request.form.get("xpost_instrument_elog", "false").lower()) + xpost_instrument_elog = json.loads( + request.form.get("xpost_instrument_elog", "false").lower() + ) if xpost_instrument_elog: - instrument_elogs = get_instrument_elogs(experiment_name, include_site_spanning_elogs=False) + instrument_elogs = get_instrument_elogs( + experiment_name, include_site_spanning_elogs=False + ) if instrument_elogs: post_to_elogs.extend(instrument_elogs) post_to_elogs = list(set(post_to_elogs)) @@ -1379,48 +1989,82 @@ def svc_post_new_elog_entry(experiment_name): logger.info(filename) files.append((filename, upload)) try: - inserted_doc = post_new_log_entry(experiment_name, author, log_content, files, **optional_args) + inserted_doc = post_new_log_entry( + experiment_name, author, log_content, files, **optional_args + ) except LgbkException as e: - return JSONEncoder().encode({'success': False, 'errormsg': str(e), 'value': None}) - if 'run_num' in inserted_doc: - sample_obj = get_sample_for_run(experiment_name, inserted_doc['run_num']) + return JSONEncoder().encode( + {"success": False, "errormsg": str(e), "value": None} + ) + if "run_num" in inserted_doc: + sample_obj = get_sample_for_run(experiment_name, inserted_doc["run_num"]) if sample_obj: - inserted_doc['sample'] = sample_obj['name'] - context.kafka_producer.send("elog", {"experiment_name" : experiment_name, "CRUD": "Create", "value": inserted_doc}) + inserted_doc["sample"] = sample_obj["name"] + context.kafka_producer.send( + "elog", + {"experiment_name": experiment_name, "CRUD": "Create", "value": inserted_doc}, + ) logger.debug("Published the new elog entry for %s", experiment_name) # Send an email out if a list of emails was specified. email_to = inserted_doc.get("email_to", []) if not email_to and "root" in inserted_doc: - email_to = get_specific_elog_entry(experiment_name, inserted_doc["root"]).get("email_to", []) + email_to = get_specific_elog_entry(experiment_name, inserted_doc["root"]).get( + "email_to", [] + ) email_to.extend(get_elog_email_subscriptions_emails(experiment_name)) if email_to and not skip_email: - logger.debug("Sending emails for new elog entry in experiment %s to %s", experiment_name, ",".join(email_to)) + logger.debug( + "Sending emails for new elog entry in experiment %s to %s", + experiment_name, + ",".join(email_to), + ) send_elog_as_email(experiment_name, inserted_doc, email_to) if "root" in inserted_doc: - root_posted = get_specific_elog_entry(experiment_name, inserted_doc["root"]).get("post_to_elogs", []) + root_posted = get_specific_elog_entry( + experiment_name, inserted_doc["root"] + ).get("post_to_elogs", []) if root_posted: post_to_elogs.extend(root_posted) - for post_to_elog in post_to_elogs: + for post_to_elog in post_to_elogs: post_to_elog = post_to_elog.replace(" ", "_") logger.debug("Cross posting entry to %s", post_to_elog) - rel_ins_doc = post_related_elog_entry(post_to_elog, experiment_name, inserted_doc["_id"]) + rel_ins_doc = post_related_elog_entry( + post_to_elog, experiment_name, inserted_doc["_id"] + ) if rel_ins_doc: logger.debug("Publishing cross post entry to %s", post_to_elog) - context.kafka_producer.send("elog", {"experiment_name" : post_to_elog, "CRUD": "Create", "value": rel_ins_doc}) + context.kafka_producer.send( + "elog", + { + "experiment_name": post_to_elog, + "CRUD": "Create", + "value": rel_ins_doc, + }, + ) email_to = get_elog_email_subscriptions_emails(post_to_elog) if not author.endswith("opr") and author not in email_to: - logger.info("Adding the author %s as an email recipient of xpost elog entries", author) + logger.info( + "Adding the author %s as an email recipient of xpost elog entries", + author, + ) email_to.append(author) if email_to and not skip_email: - logger.debug("Sending emails for cross posted elog entry in experiment %s to %s", post_to_elog, ",".join(email_to)) + logger.debug( + "Sending emails for cross posted elog entry in experiment %s to %s", + post_to_elog, + ",".join(email_to), + ) send_elog_as_email(post_to_elog, rel_ins_doc, email_to) - return JSONEncoder().encode({'success': True, 'value': inserted_doc}) + return JSONEncoder().encode({"success": True, "value": inserted_doc}) -@explgbk_blueprint.route("/lgbk//ws/modify_elog_entry", methods=["POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/modify_elog_entry", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("edit") @@ -1444,7 +2088,11 @@ def svc_modify_elog_entry(experiment_name): run_num = None if not entry_id or not log_content: - return logAndAbort("Please pass in the _id of the elog entry for " + experiment_name + " and the new content") + return logAndAbort( + "Please pass in the _id of the elog entry for " + + experiment_name + + " and the new content" + ) files = [] for upload in request.files.getlist("files"): @@ -1453,18 +2101,55 @@ def svc_modify_elog_entry(experiment_name): logger.info(filename) files.append((filename, upload)) - status = modify_elog_entry(experiment_name, entry_id, context.security.get_current_user_id(), log_content, email_to, tags, files, title, run_num=run_num) + status = modify_elog_entry( + experiment_name, + entry_id, + context.security.get_current_user_id(), + log_content, + email_to, + tags, + files, + title, + run_num=run_num, + ) if status: modified_entry = get_specific_elog_entry(experiment_name, entry_id) - context.kafka_producer.send("elog", {"experiment_name" : experiment_name, "CRUD": "Update", "value": modified_entry}) - previous_version = get_specific_elog_entry(experiment_name, modified_entry["previous_version"]) - context.kafka_producer.send("elog", {"experiment_name" : experiment_name, "CRUD": "Create", "value": previous_version}) - for instr_elog_name, instr_elog_entry in get_related_instrument_elog_entries(experiment_name, entry_id).items(): - context.kafka_producer.send("elog", {"experiment_name" : instr_elog_name, "CRUD": "Update", "value": instr_elog_entry}) + context.kafka_producer.send( + "elog", + { + "experiment_name": experiment_name, + "CRUD": "Update", + "value": modified_entry, + }, + ) + previous_version = get_specific_elog_entry( + experiment_name, modified_entry["previous_version"] + ) + context.kafka_producer.send( + "elog", + { + "experiment_name": experiment_name, + "CRUD": "Create", + "value": previous_version, + }, + ) + for instr_elog_name, instr_elog_entry in get_related_instrument_elog_entries( + experiment_name, entry_id + ).items(): + context.kafka_producer.send( + "elog", + { + "experiment_name": instr_elog_name, + "CRUD": "Update", + "value": instr_elog_entry, + }, + ) email_to = modified_entry.get("email_to", None) if not email_to and "root" in modified_entry: - email_to = get_specific_elog_entry(experiment_name, modified_entry["root"]).get("email_to", None) + email_to = get_specific_elog_entry( + experiment_name, modified_entry["root"] + ).get("email_to", None) if email_to: send_elog_as_email(experiment_name, modified_entry, email_to) @@ -1476,57 +2161,139 @@ def svc_modify_elog_entry(experiment_name): @experiment_exists @context.security.authorization_required("read") def svc_search_elog(experiment_name): - search_text = request.args.get("search_text", "") - run_num_str = request.args.get("run_num", None) + search_text = request.args.get("search_text", "") + run_num_str = request.args.get("run_num", None) start_run_num_str = request.args.get("start_run_num", None) - end_run_num_str = request.args.get("end_run_num", None) + end_run_num_str = request.args.get("end_run_num", None) start_date_str = request.args.get("start_date", None) - end_date_str = request.args.get("end_date", None) + end_date_str = request.args.get("end_date", None) tag_filter = request.args.get("tag", None) id_str = request.args.get("_id", None) if run_num_str: - return JSONEncoder().encode({"success": True, "value": get_elogs_for_run_num(experiment_name, int(run_num_str))}) + return JSONEncoder().encode( + { + "success": True, + "value": get_elogs_for_run_num(experiment_name, int(run_num_str)), + } + ) elif start_run_num_str and end_run_num_str: - return JSONEncoder().encode({"success": True, "value": get_elogs_for_run_num_range(experiment_name, int(start_run_num_str), int(end_run_num_str))}) + return JSONEncoder().encode( + { + "success": True, + "value": get_elogs_for_run_num_range( + experiment_name, int(start_run_num_str), int(end_run_num_str) + ), + } + ) elif id_str: - return JSONEncoder().encode({"success": True, "value": get_elogs_for_specified_id(experiment_name, id_str)}) + return JSONEncoder().encode( + { + "success": True, + "value": get_elogs_for_specified_id(experiment_name, id_str), + } + ) elif start_date_str and end_date_str: - return JSONEncoder().encode({"success": True, "value": get_elogs_for_date_range(experiment_name, datetime.strptime(start_date_str, '%Y-%m-%dT%H:%M:%S.%fZ'), datetime.strptime(end_date_str, '%Y-%m-%dT%H:%M:%S.%fZ'))}) + return JSONEncoder().encode( + { + "success": True, + "value": get_elogs_for_date_range( + experiment_name, + datetime.strptime(start_date_str, "%Y-%m-%dT%H:%M:%S.%fZ"), + datetime.strptime(end_date_str, "%Y-%m-%dT%H:%M:%S.%fZ"), + ), + } + ) elif search_text.startswith("t:"): - return JSONEncoder().encode({"success": True, "value": get_elog_entries_by_tag(experiment_name, search_text[2:])}) + return JSONEncoder().encode( + { + "success": True, + "value": get_elog_entries_by_tag(experiment_name, search_text[2:]), + } + ) elif search_text.startswith("x:"): - return JSONEncoder().encode({"success": True, "value": get_elog_entries_by_regex(experiment_name, search_text[2:])}) + return JSONEncoder().encode( + { + "success": True, + "value": get_elog_entries_by_regex(experiment_name, search_text[2:]), + } + ) elif len(search_text) < 1 and tag_filter: - return JSONEncoder().encode({"success": True, "value": get_elog_entries_by_tag(experiment_name, tag_filter)}) + return JSONEncoder().encode( + { + "success": True, + "value": get_elog_entries_by_tag(experiment_name, tag_filter), + } + ) else: combined_results = {} if search_text in get_elog_authors(experiment_name): - combined_results.update({ x["_id"] : x for x in get_elog_entries_by_author(experiment_name, search_text) }) + combined_results.update( + { + x["_id"]: x + for x in get_elog_entries_by_author(experiment_name, search_text) + } + ) if search_text in get_elog_tags(experiment_name): - combined_results.update({ x["_id"] : x for x in get_elog_entries_by_tag(experiment_name, search_text) }) + combined_results.update( + { + x["_id"]: x + for x in get_elog_entries_by_tag(experiment_name, search_text) + } + ) - combined_results.update({ x["_id"] : x for x in search_elog_for_text(experiment_name, search_text) }) + combined_results.update( + {x["_id"]: x for x in search_elog_for_text(experiment_name, search_text)} + ) if tag_filter: tag_entries = get_elog_entries_by_tag(experiment_name, tag_filter) - combined_results = { x : combined_results[x] for x in set([ y["_id"] for y in tag_entries ]) & combined_results.keys() } + combined_results = { + x: combined_results[x] + for x in set([y["_id"] for y in tag_entries]) & combined_results.keys() + } + + return JSONEncoder().encode( + { + "success": True, + "value": list( + sorted(combined_results.values(), key=lambda x: x["insert_time"]) + ), + } + ) - return JSONEncoder().encode({"success": True, "value": list(sorted(combined_results.values(), key=lambda x : x["insert_time"]))}) -@explgbk_blueprint.route("/lgbk//ws/delete_elog_entry", methods=["DELETE"]) +@explgbk_blueprint.route( + "/lgbk//ws/delete_elog_entry", methods=["DELETE"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("delete") def svc_delete_elog_entry(experiment_name): - entry_id = request.args.get("_id", None) + entry_id = request.args.get("_id", None) if not entry_id: - return logAndAbort("Please pass in the _id of the elog entry for " + experiment_name) - status = delete_elog_entry(experiment_name, entry_id, context.security.get_current_user_id()) + return logAndAbort( + "Please pass in the _id of the elog entry for " + experiment_name + ) + status = delete_elog_entry( + experiment_name, entry_id, context.security.get_current_user_id() + ) if status: entry = get_specific_elog_entry(experiment_name, entry_id) - context.kafka_producer.send("elog", {"experiment_name" : experiment_name, "CRUD": "Update", "value": entry}) - for instr_elog_name, instr_elog_entry in get_related_instrument_elog_entries(experiment_name, entry_id).items(): - context.kafka_producer.send("elog", {"experiment_name" : instr_elog_name, "CRUD": "Update", "value": instr_elog_entry}) + context.kafka_producer.send( + "elog", + {"experiment_name": experiment_name, "CRUD": "Update", "value": entry}, + ) + for instr_elog_name, instr_elog_entry in get_related_instrument_elog_entries( + experiment_name, entry_id + ).items(): + context.kafka_producer.send( + "elog", + { + "experiment_name": instr_elog_name, + "CRUD": "Update", + "value": instr_elog_entry, + }, + ) return JSONEncoder().encode({"success": status}) @@ -1540,80 +2307,158 @@ def svc_cross_post_elog_entry(experiment_name): """ entry_id = request.args.get("_id", None) if not entry_id: - return logAndAbort("Please pass in the _id of the elog entry for " + experiment_name) + return logAndAbort( + "Please pass in the _id of the elog entry for " + experiment_name + ) existing_entry = get_specific_elog_entry(experiment_name, entry_id) if not existing_entry: return logAndAbort("Cannot find log entry in " + experiment_name) post_to_elogs_str = request.args.get("post_to_elogs", None) if not post_to_elogs_str: - return logAndAbort("Please pass in the names of the instrument elogs as post_to_elogs") + return logAndAbort( + "Please pass in the names of the instrument elogs as post_to_elogs" + ) post_to_elogs = post_to_elogs_str.split(",") - all_elogs_for_id = get_elog_tree_for_specified_id(experiment_name, existing_entry["_id"]) + all_elogs_for_id = get_elog_tree_for_specified_id( + experiment_name, existing_entry["_id"] + ) for post_to_elog in post_to_elogs: post_to_elog = post_to_elog.replace(" ", "_") logger.debug("Cross posting entry to %s", post_to_elog) for elog_for_id in all_elogs_for_id: - logger.debug("Cross posting entry to %s %s", post_to_elog, elog_for_id["_id"]) - rel_ins_doc = post_related_elog_entry(post_to_elog, experiment_name, elog_for_id["_id"]) + logger.debug( + "Cross posting entry to %s %s", post_to_elog, elog_for_id["_id"] + ) + rel_ins_doc = post_related_elog_entry( + post_to_elog, experiment_name, elog_for_id["_id"] + ) if rel_ins_doc: - context.kafka_producer.send("elog", {"experiment_name" : post_to_elog, "CRUD": "Create", "value": rel_ins_doc}) + context.kafka_producer.send( + "elog", + { + "experiment_name": post_to_elog, + "CRUD": "Create", + "value": rel_ins_doc, + }, + ) return JSONEncoder().encode({"success": True}) + @explgbk_blueprint.route("/lgbk//ws/elog_emails", methods=["GET"]) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_elog_emails(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_elog_emails(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_elog_emails(experiment_name)} + ) + -@explgbk_blueprint.route("/lgbk//ws/elog_email_subscriptions", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/elog_email_subscriptions", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_elog_email_subscriptions(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_elog_email_subscriptions(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_elog_email_subscriptions(experiment_name)} + ) + -@explgbk_blueprint.route("/lgbk//ws/elog_email_subscribe", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/elog_email_subscribe", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_elog_email_subscribe(experiment_name): - return JSONEncoder().encode({"success": True, "value": elog_email_subscribe(experiment_name, context.security.get_current_user_id())}) + return JSONEncoder().encode( + { + "success": True, + "value": elog_email_subscribe( + experiment_name, context.security.get_current_user_id() + ), + } + ) + -@explgbk_blueprint.route("/lgbk//ws/elog_email_unsubscribe", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/elog_email_unsubscribe", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_elog_email_unsubscribe(experiment_name): - return JSONEncoder().encode({"success": True, "value": elog_email_unsubscribe(experiment_name, context.security.get_current_user_id())}) + return JSONEncoder().encode( + { + "success": True, + "value": elog_email_unsubscribe( + experiment_name, context.security.get_current_user_id() + ), + } + ) + @explgbk_blueprint.route("/lgbk//ws/get_elog_tags", methods=["GET"]) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_elog_tags(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_elog_tags(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_elog_tags(experiment_name)} + ) + -@explgbk_blueprint.route("/lgbk//ws/get_instrument_elogs", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/get_instrument_elogs", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_instrument_elogs(experiment_name): - include_site_spanning_elogs = json.loads(request.args.get("include_site_spanning_elogs", "true")) - include_instrument_elogs = json.loads(request.args.get("include_instrument_elogs", "true")) - return JSONEncoder().encode({"success": True, "value": get_instrument_elogs(experiment_name, include_instrument_elogs=include_instrument_elogs, include_site_spanning_elogs=include_site_spanning_elogs)}) + include_site_spanning_elogs = json.loads( + request.args.get("include_site_spanning_elogs", "true") + ) + include_instrument_elogs = json.loads( + request.args.get("include_instrument_elogs", "true") + ) + return JSONEncoder().encode( + { + "success": True, + "value": get_instrument_elogs( + experiment_name, + include_instrument_elogs=include_instrument_elogs, + include_site_spanning_elogs=include_site_spanning_elogs, + ), + } + ) + @explgbk_blueprint.route("/lgbk//ws/files", methods=["GET"]) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_files(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_experiment_files(experiment_name, sample_name=request.args.get("sampleName", None))}) + return JSONEncoder().encode( + { + "success": True, + "value": get_experiment_files( + experiment_name, sample_name=request.args.get("sampleName", None) + ), + } + ) + -@explgbk_blueprint.route("/lgbk//ws/file_counts_by_extension", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/file_counts_by_extension", methods=["GET"] +) @experiment_exists def svc_exp_file_counts_by_extension(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_exp_file_counts_by_extension(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_exp_file_counts_by_extension(experiment_name)} + ) + @explgbk_blueprint.route("/lgbk//ws//files", methods=["GET"]) @context.security.authentication_required @@ -1623,10 +2468,15 @@ def svc_get_files_for_run(experiment_name, run_num): try: rnum = int(run_num) except ValueError: - rnum = run_num_str # Cryo uses strings for run numbers. - return JSONEncoder().encode({"success": True, "value": get_experiment_files_for_run(experiment_name, rnum)}) + rnum = run_num # Cryo uses strings for run numbers. + return JSONEncoder().encode( + {"success": True, "value": get_experiment_files_for_run(experiment_name, rnum)} + ) -@explgbk_blueprint.route("/lgbk//ws//files_for_live_mode", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws//files_for_live_mode", methods=["GET"] +) def svc_get_files_for_run_for_live_mode(experiment_name, run_num): """ Get a minimal set of information for psana live mode. @@ -1635,10 +2485,19 @@ def svc_get_files_for_run_for_live_mode(experiment_name, run_num): try: rnum = int(run_num) except ValueError: - rnum = run_num_str # Cryo uses strings for run numbers. - return JSONEncoder().encode({"success": True, "value": get_experiment_files_for_run_for_live_mode(experiment_name, rnum)}) + rnum = run_num # Cryo uses strings for run numbers. + return JSONEncoder().encode( + { + "success": True, + "value": get_experiment_files_for_run_for_live_mode(experiment_name, rnum), + } + ) + -@explgbk_blueprint.route("/lgbk//ws//files_for_live_mode_at_location", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws//files_for_live_mode_at_location", + methods=["GET"], +) def svc_get_files_for_run_for_live_mode_at_location(experiment_name, run_num): """ Similar to files_for_live_mode. @@ -1649,18 +2508,32 @@ def svc_get_files_for_run_for_live_mode_at_location(experiment_name, run_num): try: rnum = int(run_num) except ValueError: - rnum = run_num_str # Cryo uses strings for run numbers. + rnum = run_num # Cryo uses strings for run numbers. location = request.args.get("location", None) if not location: - return logAndAbort("Please pass in a valid data management location name using the location parameter") + return logAndAbort( + "Please pass in a valid data management location name using the location parameter" + ) run_doc = get_run_doc_for_run_num(experiment_name, rnum) if not run_doc: - return logAndAbort("Cannot find run number %s for experiment %s" % (rnum, experiment_name)) + return logAndAbort( + "Cannot find run number %s for experiment %s" % (rnum, experiment_name) + ) + + return JSONEncoder().encode( + { + "success": True, + "value": get_experiment_files_for_run_for_live_mode_at_location( + experiment_name, rnum, location + ), + } + ) - return JSONEncoder().encode({"success": True, "value": get_experiment_files_for_run_for_live_mode_at_location(experiment_name, rnum, location)}) -@explgbk_blueprint.route("/lgbk//ws/files_for_live_mode_at_location", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/files_for_live_mode_at_location", methods=["GET"] +) def svc_get_files_for_live_mode_at_location(experiment_name): """ Similar to files_for_live_mode_for_run except for all runs. @@ -1670,7 +2543,9 @@ def svc_get_files_for_live_mode_at_location(experiment_name): """ location = request.args.get("location", None) if not location: - return logAndAbort("Please pass in a valid data management location name using the location parameter") + return logAndAbort( + "Please pass in a valid data management location name using the location parameter" + ) ret = get_experiment_files_for_live_mode_at_location(experiment_name, location) return JSONEncoder().encode({"success": True, "value": ret}) @@ -1682,7 +2557,17 @@ def svc_get_files_for_live_mode_at_location(experiment_name): @context.security.authorization_required("read") def svc_get_runs(experiment_name): include_run_params = json.loads(request.args.get("includeParams", "true")) - return JSONEncoder().encode({"success": True, "value": get_experiment_runs(experiment_name, include_run_params, sample_name=request.args.get("sampleName", None))}) + return JSONEncoder().encode( + { + "success": True, + "value": get_experiment_runs( + experiment_name, + include_run_params, + sample_name=request.args.get("sampleName", None), + ), + } + ) + @explgbk_blueprint.route("/lgbk//ws/runs/", methods=["GET"]) @context.security.authentication_required @@ -1692,7 +2577,7 @@ def svc_get_run_document(experiment_name, run_num): try: rnum = int(run_num) except ValueError: - rnum = run_num # Cryo uses strings for run numbers. + rnum = run_num # Cryo uses strings for run numbers. run_doc = get_experiment_run_document(experiment_name, rnum) if not run_doc: return logAndAbort("Cannot find run document for " + rnum) @@ -1714,25 +2599,29 @@ def svc_get_legacy_runs(experiment_name): The times are im PDT. """ run_infos_src = get_experiment_runs(experiment_name, False, sample_name=None) - tz = pytz.timezone('America/Los_Angeles') + tz = pytz.timezone("America/Los_Angeles") run_infos = [] for ri in run_infos_src: - rinf = { "begin_time": int(ri["begin_time"].astimezone(tz).timestamp()), + rinf = { + "begin_time": int(ri["begin_time"].astimezone(tz).timestamp()), "run_num": ri["num"], - "run_type": ri.get("type", "DATA") - } + "run_type": ri.get("type", "DATA"), + } if "end_time" in ri and ri["end_time"]: rinf["end_time"] = int(ri["end_time"].astimezone(tz).timestamp()) run_infos.append(rinf) return JSONEncoder().encode({"success": True, "value": run_infos}) + @explgbk_blueprint.route("/lgbk//ws/shifts", methods=["GET"]) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_shifts(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_experiment_shifts(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_experiment_shifts(experiment_name)} + ) @explgbk_blueprint.route("/lgbk//ws/run_tables", methods=["GET"]) @@ -1740,7 +2629,9 @@ def svc_get_shifts(experiment_name): @experiment_exists @context.security.authorization_required("read") def svc_get_runtables(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_all_run_tables(experiment_name, g.instrument)}) + return JSONEncoder().encode( + {"success": True, "value": get_all_run_tables(experiment_name, g.instrument)} + ) @explgbk_blueprint.route("/lgbk//ws/run_table_data", methods=["GET"]) @@ -1750,16 +2641,36 @@ def svc_get_runtables(experiment_name): def svc_get_runtable_data(experiment_name): tableName = request.args.get("tableName") sampleName = request.args.get("sampleName", None) - return JSONEncoder().encode({"success": True, "value": list(map(replaceInfNan, get_runtable_data(experiment_name, g.instrument, tableName, sampleName=sampleName)))}) + return JSONEncoder().encode( + { + "success": True, + "value": list( + map( + replaceInfNan, + get_runtable_data( + experiment_name, g.instrument, tableName, sampleName=sampleName + ), + ) + ), + } + ) + -@explgbk_blueprint.route("/lgbk//ws/run_table_sources", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/run_table_sources", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_runtable_sources(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_runtable_sources(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_runtable_sources(experiment_name)} + ) -@explgbk_blueprint.route("/lgbk//ws/create_update_user_run_table_def", methods=["POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/create_update_user_run_table_def", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -1768,14 +2679,18 @@ def svc_create_update_user_run_table_def(experiment_name): Create or update an existing user definition table. """ logger.error(json.dumps(request.json, indent=2)) - (status, errormsg, rtbl_obj) = create_update_user_run_table_def(experiment_name, g.instrument, request.json) + (status, errormsg, rtbl_obj) = create_update_user_run_table_def( + experiment_name, g.instrument, request.json + ) if status: return JSONEncoder().encode({"success": True, "value": rtbl_obj}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) -@explgbk_blueprint.route("/lgbk//ws/run_table_editable_update", methods=["POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/run_table_editable_update", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -1792,14 +2707,25 @@ def svc_run_table_editable_update(experiment_name): userid = context.security.get_current_user_id() # params.Calibrations is a legacy of old LCLS1 experiments. - if not source.startswith('editable_params.') and not source.startswith('params.Calibrations/'): + if not source.startswith("editable_params.") and not source.startswith( + "params.Calibrations/" + ): return logAndAbort("We can only change editable parameters.") - if source.endswith('.value'): + if source.endswith(".value"): source = source.replace(".value", "") - return JSONEncoder().encode({"success": True, "result": update_editable_param_for_run(experiment_name, runnum, source, value, userid)}) + return JSONEncoder().encode( + { + "success": True, + "result": update_editable_param_for_run( + experiment_name, runnum, source, value, userid + ), + } + ) -@explgbk_blueprint.route("/lgbk//ws/clone_run_table_def", methods=["POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/clone_run_table_def", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -1807,11 +2733,18 @@ def svc_clone_run_table_def(experiment_name): existing_run_table_name = request.args.get("existing_run_table_name", None) new_run_table_name = request.args.get("new_run_table_name", None) if not existing_run_table_name or not new_run_table_name: - return logAndAbort("Please specify the a table to clone along with the new name") - (status, errormsg, val) = clone_run_table_definition(experiment_name, existing_run_table_name, new_run_table_name) + return logAndAbort( + "Please specify the a table to clone along with the new name" + ) + (status, errormsg, val) = clone_run_table_definition( + experiment_name, existing_run_table_name, new_run_table_name + ) return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": val}) -@explgbk_blueprint.route("/lgbk//ws/replace_system_run_table_def", methods=["POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/replace_system_run_table_def", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -1819,16 +2752,32 @@ def svc_replace_system_run_table_def(experiment_name): existing_run_table_name = request.form.get("existing_run_table_name", None) system_run_table_name = request.form.get("system_run_table_name", None) if not existing_run_table_name or not system_run_table_name: - return logAndAbort("Please specify the a table to use as a replacement along with the system run table name") + return logAndAbort( + "Please specify the a table to use as a replacement along with the system run table name" + ) is_instrument = json.loads(request.form.get("is_instrument", "false").lower()) - logger.debug("Making %s a system run table called %s. Is instrument %s", existing_run_table_name, system_run_table_name, is_instrument) + logger.debug( + "Making %s a system run table called %s. Is instrument %s", + existing_run_table_name, + system_run_table_name, + is_instrument, + ) is_template = json.loads(request.form.get("is_template", "false").lower()) - (status, errormsg, val) = replace_system_run_table_definition(experiment_name, existing_run_table_name, system_run_table_name, instrument=g.instrument if is_instrument else None, is_template=is_template ) + (status, errormsg, val) = replace_system_run_table_definition( + experiment_name, + existing_run_table_name, + system_run_table_name, + instrument=g.instrument if is_instrument else None, + is_template=is_template, + ) return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": val}) -@explgbk_blueprint.route("/lgbk//ws/clone_system_template_run_tables", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws/clone_system_template_run_tables", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -1841,7 +2790,10 @@ def svc_clone_system_template_run_tables(experiment_name): clone_system_template_run_tables_into_experiment(experiment_name, g.instrument) return JSONEncoder().encode({"success": True}) -@explgbk_blueprint.route("/lgbk//ws/delete_run_table", methods=["DELETE"]) + +@explgbk_blueprint.route( + "/lgbk//ws/delete_run_table", methods=["DELETE"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -1853,15 +2805,28 @@ def svc_delete_run_table(experiment_name): if is_system_run_table: logger.debug("Deleting system run table") if context.security.check_privilege_for_experiment("ops_page", None, None): - status, errormsg = delete_system_run_table(experiment_name, g.instrument, table_name) - return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": None}) + status, errormsg = delete_system_run_table( + experiment_name, g.instrument, table_name + ) + return JSONEncoder().encode( + {"success": status, "errormsg": errormsg, "value": None} + ) else: - return {"success": False, "errormsg": "Not enough permissions to perform this operation", "value": None} + return { + "success": False, + "errormsg": "Not enough permissions to perform this operation", + "value": None, + } else: status, errormsg = delete_run_table(experiment_name, table_name) - return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": None}) + return JSONEncoder().encode( + {"success": status, "errormsg": errormsg, "value": None} + ) + -@explgbk_blueprint.route("/lgbk//ws/runtables/export_as_csv", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/runtables/export_as_csv", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -1875,20 +2840,28 @@ def svc_runtable_export_as_csv(experiment_name): if not tableName: return logAndAbort("Please specify the table name to export.") sampleName = request.args.get("sampleName", None) - tbldefs = [x for x in filter(lambda x : x["name"] == tableName and not x.get("is_template", False), get_all_run_tables(experiment_name, g.instrument))] + tbldefs = [ + x + for x in filter( + lambda x: x["name"] == tableName and not x.get("is_template", False), + get_all_run_tables(experiment_name, g.instrument), + ) + ] if len(tbldefs) != 1: return logAndAbort("Cannot find the definition for table " + tableName) - coltups = [ ("Run Number", "num") ] # List of tuples of label and attr name + coltups = [("Run Number", "num")] # List of tuples of label and attr name for cdef in tbldefs[0].get("coldefs", []): coltups.append((cdef["label"], cdef["source"])) - tz = pytz.timezone('America/Los_Angeles') + tz = pytz.timezone("America/Los_Angeles") si = io.StringIO() si.write(",".join([x[0] for x in coltups]) + "\n") + def __tocsv__(obj): if isinstance(obj, datetime): - return obj.astimezone(tz).strftime('%Y-%m-%d %H:%M:%S') + return obj.astimezone(tz).strftime("%Y-%m-%d %H:%M:%S") else: return str(obj) + def __ldget__(obj, attrpath, deflt): prts = attrpath.split(".") for prt in prts: @@ -1896,14 +2869,25 @@ def __ldget__(obj, attrpath, deflt): if not obj: return deflt return __tocsv__(obj) - for dt in list(map(replaceInfNan, get_runtable_data(experiment_name, g.instrument, tableName, sampleName=sampleName))): - si.write(",".join([ __ldget__(dt, ct[1], "") for ct in coltups]) + "\n") + + for dt in list( + map( + replaceInfNan, + get_runtable_data( + experiment_name, g.instrument, tableName, sampleName=sampleName + ), + ) + ): + si.write(",".join([__ldget__(dt, ct[1], "") for ct in coltups]) + "\n") resp = make_response(si.getvalue()) - resp.headers["Content-Disposition"] = "attachment; filename="+tableName+".csv" + resp.headers["Content-Disposition"] = "attachment; filename=" + tableName + ".csv" resp.headers["Content-type"] = "text/csv" return resp -@explgbk_blueprint.route("/run_control//ws/start_run", methods=["GET", "POST"]) + +@explgbk_blueprint.route( + "/run_control//ws/start_run", methods=["GET", "POST"] +) @context.security.authentication_required @experiment_exists_and_unlocked @context.security.authorization_required("post") @@ -1923,12 +2907,19 @@ def svc_start_run(experiment_name): # LCLS uses int run numbers; Cryo users strings. rn = int(user_specified_run_number) user_specified_run_number = rn - logger.debug("Converting user specified run number as int %s", user_specified_run_number) + logger.debug( + "Converting user specified run number as int %s", + user_specified_run_number, + ) except ValueError: logger.debug("Using run number as is %s", user_specified_run_number) pass user_specified_start_time_str = request.args.get("start_time", None) - user_specified_start_time = datetime.strptime(user_specified_start_time_str, '%Y-%m-%dT%H:%M:%S.%fZ') if user_specified_start_time_str else None + user_specified_start_time = ( + datetime.strptime(user_specified_start_time_str, "%Y-%m-%dT%H:%M:%S.%fZ") + if user_specified_start_time_str + else None + ) user_specified_sample = request.args.get("sample", None) # Here's where we can put validations on starting a new run. @@ -1937,41 +2928,72 @@ def svc_start_run(experiment_name): run_doc = get_current_run(experiment_name) if run_doc and not run_doc.get("end_time", None): - logger.warn("Previous run for experiment %s was not closed; closing it", experiment_name) + logger.warn( + "Previous run for experiment %s was not closed; closing it", experiment_name + ) try: run_doc = __end_run_and_publish_message__(experiment_name) - except: - logger.exception("Exception closing previous unclosed run for experiment %s", experiment_name) - - params = { escape_chars_for_mongo(k) : v for k, v in request.get_json().items() } if request.is_json else None + except Exception: + logger.exception( + "Exception closing previous unclosed run for experiment %s", + experiment_name, + ) + + params = ( + {escape_chars_for_mongo(k): v for k, v in request.get_json().items()} + if request.is_json + else None + ) - run_doc = start_run(experiment_name, run_type, user_specified_run_number, user_specified_start_time, user_specified_sample, params=params) + run_doc = start_run( + experiment_name, + run_type, + user_specified_run_number, + user_specified_start_time, + user_specified_sample, + params=params, + ) - sample_obj = get_sample_for_run(experiment_name, run_doc['num']) + sample_obj = get_sample_for_run(experiment_name, run_doc["num"]) if sample_obj: - run_doc['sample'] = sample_obj['name'] + run_doc["sample"] = sample_obj["name"] - context.kafka_producer.send("runs", {"experiment_name" : experiment_name, "CRUD": "Create", "value": run_doc}) + context.kafka_producer.send( + "runs", {"experiment_name": experiment_name, "CRUD": "Create", "value": run_doc} + ) logger.debug("Published the new run for %s", experiment_name) return JSONEncoder().encode({"success": True, "value": run_doc}) + def __end_run_and_publish_message__(experiment_name, user_specified_end_time=None): run_doc = end_run(experiment_name, user_specified_end_time) - sample_obj = get_sample_for_run(experiment_name, run_doc['num']) + sample_obj = get_sample_for_run(experiment_name, run_doc["num"]) if sample_obj: - run_doc['sample'] = sample_obj['name'] + run_doc["sample"] = sample_obj["name"] try: - run_doc["duration"] = (run_doc["end_time"] - run_doc["begin_time"]).total_seconds() - except: - logger.exception("Exception computing duration for run %s for experiment %s", run_doc["num"], experiment_name) - run_doc["file_catalog"] = get_experiment_files_for_run(experiment_name, run_doc['num']) - context.kafka_producer.send("runs", {"experiment_name" : experiment_name, "CRUD": "Update", "value": run_doc}) + run_doc["duration"] = ( + run_doc["end_time"] - run_doc["begin_time"] + ).total_seconds() + except Exception: + logger.exception( + "Exception computing duration for run %s for experiment %s", + run_doc["num"], + experiment_name, + ) + run_doc["file_catalog"] = get_experiment_files_for_run( + experiment_name, run_doc["num"] + ) + context.kafka_producer.send( + "runs", {"experiment_name": experiment_name, "CRUD": "Update", "value": run_doc} + ) return run_doc -@explgbk_blueprint.route("/run_control//ws/end_run", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/run_control//ws/end_run", methods=["GET", "POST"] +) @context.security.authentication_required @experiment_exists_and_unlocked @context.security.authorization_required("post") @@ -1980,13 +3002,19 @@ def svc_end_run(experiment_name): End the current run; ending the current run is mostly setting the end time. """ user_specified_end_time_str = request.args.get("end_time", None) - user_specified_end_time = datetime.strptime(user_specified_end_time_str, '%Y-%m-%dT%H:%M:%S.%fZ') if user_specified_end_time_str else None + user_specified_end_time = ( + datetime.strptime(user_specified_end_time_str, "%Y-%m-%dT%H:%M:%S.%fZ") + if user_specified_end_time_str + else None + ) run_doc = __end_run_and_publish_message__(experiment_name, user_specified_end_time) return JSONEncoder().encode({"success": True, "value": run_doc}) -@explgbk_blueprint.route("/run_control//ws/current_run", methods=["GET"]) +@explgbk_blueprint.route( + "/run_control//ws/current_run", methods=["GET"] +) @explgbk_blueprint.route("/lgbk//ws/current_run", methods=["GET"]) @experiment_exists def svc_current_run(experiment_name): @@ -1999,7 +3027,11 @@ def svc_current_run(experiment_name): logger.error("Current run for experiment %s does not exist", experiment_name) return JSONEncoder().encode({"success": False, "value": None}) if skipClosedRuns and run_doc.get("end_time", None): - logger.error("Current run %s for experiment %s is already closed", run_doc.get("num", ""), experiment_name) + logger.error( + "Current run %s for experiment %s is already closed", + run_doc.get("num", ""), + experiment_name, + ) return JSONEncoder().encode({"success": False, "value": None}) try: @@ -2011,7 +3043,10 @@ def svc_current_run(experiment_name): return JSONEncoder().encode({"success": True, "value": run_doc}) -@explgbk_blueprint.route("/run_control//ws/add_run_params", methods=["POST"]) + +@explgbk_blueprint.route( + "/run_control//ws/add_run_params", methods=["POST"] +) @explgbk_blueprint.route("/lgbk//ws/add_run_params", methods=["POST"]) @context.security.authentication_required @experiment_exists_and_unlocked @@ -2027,7 +3062,7 @@ def svc_add_run_params(experiment_name): try: rnum = int(user_specified_run_number) except ValueError: - rnum = user_specified_run_number # Cryo uses strings for run numbers. + rnum = user_specified_run_number # Cryo uses strings for run numbers. current_run_doc = get_run_doc_for_run_num(experiment_name, rnum) else: current_run_doc = get_current_run(experiment_name) @@ -2037,18 +3072,26 @@ def svc_add_run_params(experiment_name): params = request.json if len(params) <= 0: - logger.warn("No run parameters were specified in add_run_params.") # This is not an error; merely something to help with debugging. + logger.warn( + "No run parameters were specified in add_run_params." + ) # This is not an error; merely something to help with debugging. return JSONEncoder().encode({"success": True}) - run_params = {"params." + escape_chars_for_mongo(k) : v for k, v in params.items() } + run_params = {"params." + escape_chars_for_mongo(k): v for k, v in params.items()} run_doc_after = add_run_params(experiment_name, current_run_doc, run_params) - sample_obj = get_sample_for_run(experiment_name, run_doc_after['num']) + sample_obj = get_sample_for_run(experiment_name, run_doc_after["num"]) if sample_obj: - run_doc_after['sample'] = sample_obj['name'] + run_doc_after["sample"] = sample_obj["name"] - context.kafka_producer.send("runs", {"experiment_name" : experiment_name, "CRUD": "Update", "value": run_doc_after}) + context.kafka_producer.send( + "runs", + {"experiment_name": experiment_name, "CRUD": "Update", "value": run_doc_after}, + ) return JSONEncoder().encode({"success": True, "value": run_doc_after}) -@explgbk_blueprint.route("/lgbk//ws/close_shift", methods=["GET", "POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/close_shift", methods=["GET", "POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -2063,12 +3106,18 @@ def svc_close_shift(experiment_name): (status, errormsg) = close_shift_for_experiment(experiment_name, shift_name) if status: shift_doc = get_shift_for_experiment_by_name(experiment_name, shift_name) - context.kafka_producer.send("shifts", {"experiment_name" : experiment_name, "CRUD": "Update", "value": shift_doc}) - return jsonify({'success': True}) + context.kafka_producer.send( + "shifts", + {"experiment_name": experiment_name, "CRUD": "Update", "value": shift_doc}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) -@explgbk_blueprint.route("/lgbk//ws/create_update_shift", methods=["POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/create_update_shift", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -2083,7 +3132,9 @@ def svc_create_update_shift(experiment_name): create_str = request.args.get("create", None) if not create_str: - return logAndAbort("Creating shift must have a boolean create parameter indicating if the shift is created or updated.") + return logAndAbort( + "Creating shift must have a boolean create parameter indicating if the shift is created or updated." + ) createp = create_str.lower() in set(["yes", "true", "t", "1"]) logger.debug("Create update shift is %s for %s", createp, create_str) @@ -2091,7 +3142,7 @@ def svc_create_update_shift(experiment_name): if not info: return logAndAbort("Creating shift missing info document") - necessary_keys = set(['name', 'leader', 'begin_time']) + necessary_keys = set(["name", "leader", "begin_time"]) missing_keys = necessary_keys - info.keys() if missing_keys: return logAndAbort("Creating shift missing keys %s" % missing_keys) @@ -2099,10 +3150,17 @@ def svc_create_update_shift(experiment_name): (status, errormsg) = create_update_shift(experiment_name, shift_name, createp, info) if status: shift_doc = get_shift_for_experiment_by_name(experiment_name, shift_name) - context.kafka_producer.send("shifts", {"experiment_name" : experiment_name, "CRUD": "Create" if createp else "Update", "value": shift_doc }) - return jsonify({'success': True}) + context.kafka_producer.send( + "shifts", + { + "experiment_name": experiment_name, + "CRUD": "Create" if createp else "Update", + "value": shift_doc, + }, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) @explgbk_blueprint.route("/lgbk//ws/get_latest_shift", methods=["GET"]) @@ -2110,7 +3168,9 @@ def svc_create_update_shift(experiment_name): @experiment_exists @context.security.authorization_required("read") def svc_get_latest_shift(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_latest_shift(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_latest_shift(experiment_name)} + ) @explgbk_blueprint.route("/lgbk//ws/samples", methods=["GET"]) @@ -2119,7 +3179,9 @@ def svc_get_latest_shift(experiment_name): @experiment_exists @context.security.authorization_required("read") def svc_get_samples(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_samples(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_samples(experiment_name)} + ) @explgbk_blueprint.route("/lgbk//ws/samples/", methods=["POST"]) @@ -2136,24 +3198,45 @@ def svc_create_sample(experiment_name): if "_id" in sampledetails: return logAndAbort("Cannot specify an id when creating a sample") - if 'create_associated_run' in sampledetails and sampledetails['create_associated_run']: + if ( + "create_associated_run" in sampledetails + and sampledetails["create_associated_run"] + ): current_run = get_current_run(experiment_name) if current_run and not is_run_closed(experiment_name, current_run["num"]): - return jsonify({'success': False, 'errormsg': ("Cannot switch to and create a run if the current run %s is still open %s" % (current_run["num"], experiment_name))}) - del sampledetails['create_associated_run'] + return jsonify( + { + "success": False, + "errormsg": ( + "Cannot switch to and create a run if the current run %s is still open %s" + % (current_run["num"], experiment_name) + ), + } + ) + del sampledetails["create_associated_run"] automatically_create_associated_run = True else: automatically_create_associated_run = False - (status, errormsg) = create_sample(experiment_name, sampledetails, automatically_create_associated_run) + (status, errormsg) = create_sample( + experiment_name, sampledetails, automatically_create_associated_run + ) if status: - sample_doc = get_sample_for_experiment_by_name(experiment_name, sampledetails["name"]) - context.kafka_producer.send("samples", {"experiment_name" : experiment_name, "CRUD": "Create", "value": sample_doc }) - return jsonify({'success': True}) + sample_doc = get_sample_for_experiment_by_name( + experiment_name, sampledetails["name"] + ) + context.kafka_producer.send( + "samples", + {"experiment_name": experiment_name, "CRUD": "Create", "value": sample_doc}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + -@explgbk_blueprint.route("/lgbk//ws/samples/", methods=["PUT"]) +@explgbk_blueprint.route( + "/lgbk//ws/samples/", methods=["PUT"] +) @context.security.authentication_required @experiment_exists_and_unlocked @context.security.authorization_required("post") @@ -2167,20 +3250,36 @@ def svc_update_sample(experiment_name, sampleid): (status, errormsg) = update_sample(experiment_name, sampleid, sampledetails) if status: - sample_doc = get_sample_for_experiment_by_name(experiment_name, sampledetails["name"]) - context.kafka_producer.send("samples", {"experiment_name" : experiment_name, "CRUD": "Update", "value": sample_doc }) - return jsonify({'success': True}) + sample_doc = get_sample_for_experiment_by_name( + experiment_name, sampledetails["name"] + ) + context.kafka_producer.send( + "samples", + {"experiment_name": experiment_name, "CRUD": "Update", "value": sample_doc}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) -@explgbk_blueprint.route("/lgbk//ws/samples/", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws/samples/", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_sample_by_name(experiment_name, sample_name): - return JSONEncoder().encode({"success": True, "value": get_sample_for_experiment_by_name(experiment_name, sample_name)}) + return JSONEncoder().encode( + { + "success": True, + "value": get_sample_for_experiment_by_name(experiment_name, sample_name), + } + ) -@explgbk_blueprint.route("/lgbk//ws/samples/", methods=["DELETE"]) + +@explgbk_blueprint.route( + "/lgbk//ws/samples/", methods=["DELETE"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("edit") @@ -2188,20 +3287,32 @@ def svc_delete_sample(experiment_name, sample_name): status, errormsg, obj = delete_sample_for_experiment(experiment_name, sample_name) if status: sample_doc = get_sample_for_experiment_by_name(experiment_name, sample_name) - context.kafka_producer.send("samples", {"experiment_name" : experiment_name, "CRUD": "Delete", "value": sample_doc }) - return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": obj}) + context.kafka_producer.send( + "samples", + {"experiment_name": experiment_name, "CRUD": "Delete", "value": sample_doc}, + ) + return JSONEncoder().encode( + {"success": status, "errormsg": errormsg, "value": obj} + ) else: - return jsonify({'success': status, 'errormsg': errormsg}) + return jsonify({"success": status, "errormsg": errormsg}) -@explgbk_blueprint.route("/lgbk//ws/current_sample_name", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/current_sample_name", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_current_sample(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_current_sample_name(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_current_sample_name(experiment_name)} + ) + -@explgbk_blueprint.route("/lgbk//ws/clone_sample", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/clone_sample", methods=["GET", "POST"] +) @context.security.authentication_required @experiment_exists_and_unlocked @context.security.authorization_required("post") @@ -2217,15 +3328,23 @@ def svc_clone_sample(experiment_name): if not new_sample_name: return logAndAbort("Please pass in the new sample name as a parameter") - (status, errormsg) = clone_sample(experiment_name, existing_sample_name, new_sample_name) + (status, errormsg) = clone_sample( + experiment_name, existing_sample_name, new_sample_name + ) if status: sample_doc = get_sample_for_experiment_by_name(experiment_name, new_sample_name) - context.kafka_producer.send("samples", {"experiment_name" : experiment_name, "CRUD": "Create", "value": sample_doc }) - return jsonify({'success': True}) + context.kafka_producer.send( + "samples", + {"experiment_name": experiment_name, "CRUD": "Create", "value": sample_doc}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + -@explgbk_blueprint.route("/lgbk//ws/make_sample_current", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/make_sample_current", methods=["GET", "POST"] +) @context.security.authentication_required @experiment_exists_and_unlocked @context.security.authorization_required("post") @@ -2237,14 +3356,29 @@ def svc_make_sample_current(experiment_name): (status, errormsg) = make_sample_current(experiment_name, sample_name) if status: sample_doc = get_sample_for_experiment_by_name(experiment_name, sample_name) - context.kafka_producer.send("samples", {"experiment_name" : experiment_name, "CRUD": "Make_Current", "value": sample_doc }) + context.kafka_producer.send( + "samples", + { + "experiment_name": experiment_name, + "CRUD": "Make_Current", + "value": sample_doc, + }, + ) userid = context.security.get_current_user_id() - post_new_log_entry(experiment_name, userid, "Sample {} was activated by {}".format(sample_name, userid), []) - return jsonify({'success': True}) + post_new_log_entry( + experiment_name, + userid, + "Sample {} was activated by {}".format(sample_name, userid), + [], + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + -@explgbk_blueprint.route("/lgbk//ws/stop_current_sample", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/stop_current_sample", methods=["GET", "POST"] +) @context.security.authentication_required @experiment_exists_and_unlocked @context.security.authorization_required("post") @@ -2258,13 +3392,20 @@ def svc_stop_current_sample(experiment_name): # sample_doc = get_sample_for_experiment_by_name(experiment_name, sample_name) # context.kafka_producer.send("samples", {"experiment_name" : experiment_name, "CRUD": "Stop", "value": sample_doc }) userid = context.security.get_current_user_id() - post_new_log_entry(experiment_name, userid, "Sample {} was stopped by {}".format(sample_name, userid), []) - return jsonify({'success': True}) + post_new_log_entry( + experiment_name, + userid, + "Sample {} was stopped by {}".format(sample_name, userid), + [], + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) -@explgbk_blueprint.route("/lgbk//ws/change_sample_for_run", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/change_sample_for_run", methods=["GET"] +) @context.security.authentication_required @experiment_exists_and_unlocked @context.security.authorization_required("edit") @@ -2273,18 +3414,20 @@ def svc_change_sample_for_run(experiment_name): if not sample_name: return logAndAbort("We need a sample_name as a parameter") - run_num_str = request.args.get("run_num", None); + run_num_str = request.args.get("run_num", None) if not run_num_str: return logAndAbort("We need a run_num as a parameter") try: run_num = int(run_num_str) except ValueError: - run_num = run_num_str # Cryo uses strings for run numbers. + run_num = run_num_str # Cryo uses strings for run numbers. status, errormsg = change_sample_for_run(experiment_name, run_num, sample_name) run_doc = get_run_doc_for_run_num(experiment_name, run_num) - context.kafka_producer.send("runs", {"experiment_name" : experiment_name, "CRUD": "Update", "value": run_doc}) - return jsonify({'success': status, 'errormsg': errormsg}) + context.kafka_producer.send( + "runs", {"experiment_name": experiment_name, "CRUD": "Update", "value": run_doc} + ) + return jsonify({"success": status, "errormsg": errormsg}) @explgbk_blueprint.route("/lgbk//ws/register_file", methods=["POST"]) @@ -2303,30 +3446,39 @@ def svc_register_file(experiment_name): if not info: return logAndAbort("Please pass in the file json to register a new file.") - necessary_keys = set(['path']) + necessary_keys = set(["path"]) def attach_current_run(flinfo): - if 'run_num' not in flinfo.keys(): - current_run_num = get_current_run(experiment_name)['num'] - logger.info("Associating file %s with current run %s", flinfo['path'], current_run_num) - flinfo['run_num'] = current_run_num + if "run_num" not in flinfo.keys(): + current_run_num = get_current_run(experiment_name)["num"] + logger.info( + "Associating file %s with current run %s", + flinfo["path"], + current_run_num, + ) + flinfo["run_num"] = current_run_num def convert_timestamps(flinfo): for k in flinfo.keys(): - if k.endswith('_timestamp'): - flinfo[k] = datetime.strptime(flinfo[k], '%Y-%m-%dT%H:%M:%SZ') + if k.endswith("_timestamp"): + flinfo[k] = datetime.strptime(flinfo[k], "%Y-%m-%dT%H:%M:%SZ") flinfo[k] = flinfo[k] + timedelta(microseconds=1) - if 'create_timestamp' not in flinfo: - flinfo['create_timestamp'] = datetime.utcnow() - if 'modify_timestamp' not in flinfo: - flinfo['modify_timestamp'] = datetime.utcnow() + if "create_timestamp" not in flinfo: + flinfo["create_timestamp"] = datetime.utcnow() + if "modify_timestamp" not in flinfo: + flinfo["modify_timestamp"] = datetime.utcnow() if isinstance(info, list): ret_status = [] for finfo in info: missing_keys = necessary_keys - finfo.keys() if missing_keys: - ret_status.append({'success': False, 'errormsg': "File registration missing keys %s" % missing_keys}) + ret_status.append( + { + "success": False, + "errormsg": "File registration missing keys %s" % missing_keys, + } + ) continue attach_current_run(finfo) @@ -2334,10 +3486,17 @@ def convert_timestamps(flinfo): (status, errormsg) = register_file_for_experiment(experiment_name, finfo) if status: - context.kafka_producer.send("file_catalog", {"experiment_name" : experiment_name, "CRUD": "Create", "value": finfo }) - ret_status.append({'success': True}) + context.kafka_producer.send( + "file_catalog", + { + "experiment_name": experiment_name, + "CRUD": "Create", + "value": finfo, + }, + ) + ret_status.append({"success": True}) else: - ret_status.append({'success': False, 'errormsg': errormsg}) + ret_status.append({"success": False, "errormsg": errormsg}) return jsonify(ret_status) else: missing_keys = necessary_keys - info.keys() @@ -2349,19 +3508,25 @@ def convert_timestamps(flinfo): (status, errormsg) = register_file_for_experiment(experiment_name, info) if status: - context.kafka_producer.send("file_catalog", {"experiment_name" : experiment_name, "CRUD": "Create", "value": info }) - return jsonify({'success': True}) + context.kafka_producer.send( + "file_catalog", + {"experiment_name": experiment_name, "CRUD": "Create", "value": info}, + ) + return jsonify({"success": True}) else: - return jsonify({'success': False, 'errormsg': errormsg}) + return jsonify({"success": False, "errormsg": errormsg}) + -@explgbk_blueprint.route("/lgbk//ws/file_available_at_location", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/file_available_at_location", methods=["GET", "POST"] +) @context.security.authentication_required @experiment_exists def svc_file_available_at_location(experiment_name): location = request.args.get("location", None) if not location: return logAndAbort("Please specify the location.") - if not location in [x["name"] for x in get_dm_locations(experiment_name)]: + if location not in [x["name"] for x in get_dm_locations(experiment_name)]: return logAndAbort("Please specify a valid location") file_path = request.args.get("file_path", None) if not file_path: @@ -2373,11 +3538,20 @@ def svc_file_available_at_location(experiment_name): run_num = int(run_num) except ValueError: pass - file_info = file_available_at_location(experiment_name, run_num, file_path, location) - context.kafka_producer.send("file_catalog", {"experiment_name" : experiment_name, "CRUD": "Update", "value": file_info }) - return jsonify({'success': True}) + file_info = file_available_at_location( + experiment_name, run_num, file_path, location + ) + context.kafka_producer.send( + "file_catalog", + {"experiment_name": experiment_name, "CRUD": "Update", "value": file_info}, + ) + return jsonify({"success": True}) -@explgbk_blueprint.route("/lgbk//ws/check_and_move_run_files_to_location", methods=["GET", "POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/check_and_move_run_files_to_location", + methods=["GET", "POST"], +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -2385,7 +3559,7 @@ def svc_check_and_move_run_files_to_location(experiment_name): location = request.args.get("location", None) if not location: return logAndAbort("Please specify the location.") - if not location in [x["name"] for x in get_dm_locations(experiment_name)]: + if location not in [x["name"] for x in get_dm_locations(experiment_name)]: return logAndAbort("Please specify a valid location") run_num = request.args.get("run_num", None) if not run_num: @@ -2394,7 +3568,9 @@ def svc_check_and_move_run_files_to_location(experiment_name): run_num = int(run_num) except ValueError: pass - restore_missing_files = json.loads(request.args.get("restore_missing_files", "false")) + restore_missing_files = json.loads( + request.args.get("restore_missing_files", "false") + ) file_types_to_restore = request.args.get("file_types_to_restore", "").split(",") file_patterns = [] if not file_types_to_restore: @@ -2407,39 +3583,82 @@ def svc_check_and_move_run_files_to_location(experiment_name): site_config = get_site_config() if site_config.get("dm_mover_prefix", None): - files_for_run = {x["path"] : x for x in get_experiment_files_for_run(experiment_name, run_num)} + files_for_run = { + x["path"]: x for x in get_experiment_files_for_run(experiment_name, run_num) + } + def __any_pattern__(f): - return any(map(lambda x : re.match(x, f["path"]), file_patterns)) - files_for_run = {x["path"] : x for x in filter(lambda f: __any_pattern__(f), files_for_run.values())} - resp = requests.post(site_config["dm_mover_prefix"] + "ws/" + experiment_name + "/check_files_for_run", json={ - "location": location, - "run_num": run_num, - "experiment_name": experiment_name, - "instrument": get_experiment_info(experiment_name)["instrument"], - "restore_missing_files": restore_missing_files, - "files": [x["path"] for x in files_for_run.values()] - }).json() + return any(map(lambda x: re.match(x, f["path"]), file_patterns)) + + files_for_run = { + x["path"]: x + for x in filter(lambda f: __any_pattern__(f), files_for_run.values()) + } + resp = requests.post( + site_config["dm_mover_prefix"] + + "ws/" + + experiment_name + + "/check_files_for_run", + json={ + "location": location, + "run_num": run_num, + "experiment_name": experiment_name, + "instrument": get_experiment_info(experiment_name)["instrument"], + "restore_missing_files": restore_missing_files, + "files": [x["path"] for x in files_for_run.values()], + }, + ).json() for mfile, status in resp.get("files", {}).items(): file_info = files_for_run.get(mfile, None) - if status == "present" and file_info and not location in file_info.get("locations", {}).keys(): + if ( + status == "present" + and file_info + and location not in file_info.get("locations", {}).keys() + ): logger.debug("I think %s is not there but it is already there", mfile) file_available_at_location(experiment_name, run_num, mfile, location) - elif status != "present" and file_info and location in file_info.get("locations", {}).keys(): + elif ( + status != "present" + and file_info + and location in file_info.get("locations", {}).keys() + ): logger.debug("I think %s is there but it has been removed", mfile) - file_not_available_at_location(experiment_name, run_num, mfile, location) - - return JSONEncoder().encode({'success': True, "value": {"run_files": get_experiment_files_for_run(experiment_name, run_num), "matching_files": [x["path"] for x in files_for_run.values()], "dmstatus": resp.get("files", {})} }) + file_not_available_at_location( + experiment_name, run_num, mfile, location + ) + + return JSONEncoder().encode( + { + "success": True, + "value": { + "run_files": get_experiment_files_for_run(experiment_name, run_num), + "matching_files": [x["path"] for x in files_for_run.values()], + "dmstatus": resp.get("files", {}), + }, + } + ) else: - return JSONEncoder().encode({'success': False, "errormsg": "This site has not been configured with a mover endpoint."}) + return JSONEncoder().encode( + { + "success": False, + "errormsg": "This site has not been configured with a mover endpoint.", + } + ) + @explgbk_blueprint.route("/lgbk//ws/collaborators", methods=["GET"]) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_collaborators(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_collaborators(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_collaborators(experiment_name)} + ) -@explgbk_blueprint.route("/lgbk//ws/exp_posix_group_members", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws/exp_posix_group_members", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -2453,12 +3672,20 @@ def svc_get_posix_group_members(experiment_name): if exp_info.get("posix_group", None) != experiment_name: return JSONEncoder().encode({"success": True, "value": []}) try: - return JSONEncoder().encode({"success": True, "value": context.usergroups.get_group_members(experiment_name)}) - except: + return JSONEncoder().encode( + { + "success": True, + "value": context.usergroups.get_group_members(experiment_name), + } + ) + except Exception: logger.exception("Exception getting posix members for %s", experiment_name) return JSONEncoder().encode({"success": True, "value": []}) -@explgbk_blueprint.route("/lgbk//ws/toggle_role", methods=["GET", "POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/toggle_role", methods=["GET", "POST"] +) @experiment_exists @context.security.authentication_required @context.security.authorization_required("manage_groups") @@ -2470,7 +3697,7 @@ def svc_toggle_role(experiment_name): role_obj = get_role_object(experiment_name, role_fq_name) collaborators_before = get_collaborators_list_for_experiment(experiment_name) - if role_obj and 'players' in role_obj and uid in role_obj['players']: + if role_obj and "players" in role_obj and uid in role_obj["players"]: status = remove_collaborator_from_role(experiment_name, uid, role_fq_name) collaborators_after = get_collaborators_list_for_experiment(experiment_name) collaborators_removed = collaborators_before - collaborators_after @@ -2483,12 +3710,34 @@ def svc_toggle_role(experiment_name): if status: role_obj = get_role_object(experiment_name, role_fq_name) - role_obj.update({'collaborators_added': [x for x in collaborators_added], 'collaborators_removed': [x for x in collaborators_removed], 'requestor': context.security.get_current_user_id() }) - context.kafka_producer.send("roles", {"experiment_name" : experiment_name, "instrument": get_experiment_info(experiment_name)["instrument"], "CRUD": "Update", "value": role_obj }) + role_obj.update( + { + "collaborators_added": [x for x in collaborators_added], + "collaborators_removed": [x for x in collaborators_removed], + "requestor": context.security.get_current_user_id(), + } + ) + context.kafka_producer.send( + "roles", + { + "experiment_name": experiment_name, + "instrument": get_experiment_info(experiment_name)["instrument"], + "CRUD": "Update", + "value": role_obj, + }, + ) + + return JSONEncoder().encode( + { + "success": status, + "message": "Did not match any entries" if not status else "", + } + ) - return JSONEncoder().encode({"success": status, "message": "Did not match any entries" if not status else ""}) -@explgbk_blueprint.route("/lgbk//ws/add_collaborator", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/add_collaborator", methods=["GET", "POST"] +) @experiment_exists @context.security.authentication_required @context.security.authorization_required("manage_groups") @@ -2506,12 +3755,33 @@ def svc_add_collaborator(experiment_name): if status: role_obj = get_role_object(experiment_name, role_fq_name) - role_obj.update({'collaborators_added': [x for x in collaborators_added], 'collaborators_removed': [x for x in collaborators_removed], 'requestor': context.security.get_current_user_id() }) - context.kafka_producer.send("roles", {"experiment_name" : experiment_name, "instrument": get_experiment_info(experiment_name)["instrument"], "CRUD": "Update", "value": role_obj }) - return JSONEncoder().encode({"success": status, "message": "Did not match any entries" if not status else ""}) + role_obj.update( + { + "collaborators_added": [x for x in collaborators_added], + "collaborators_removed": [x for x in collaborators_removed], + "requestor": context.security.get_current_user_id(), + } + ) + context.kafka_producer.send( + "roles", + { + "experiment_name": experiment_name, + "instrument": get_experiment_info(experiment_name)["instrument"], + "CRUD": "Update", + "value": role_obj, + }, + ) + return JSONEncoder().encode( + { + "success": status, + "message": "Did not match any entries" if not status else "", + } + ) -@explgbk_blueprint.route("/lgbk//ws/remove_collaborator", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/remove_collaborator", methods=["GET", "POST"] +) @experiment_exists @context.security.authentication_required @context.security.authorization_required("manage_groups") @@ -2520,19 +3790,36 @@ def svc_remove_collaborator(experiment_name): if not uid: return logAndAbort("Please specify a uid") - collaborator_roles = next(filter(lambda x : x["uid"] == uid, get_collaborators(experiment_name))) - for role in collaborator_roles['roles']: + collaborator_roles = next( + filter(lambda x: x["uid"] == uid, get_collaborators(experiment_name)) + ) + for role in collaborator_roles["roles"]: collaborators_before = get_collaborators_list_for_experiment(experiment_name) remove_collaborator_from_role(experiment_name, uid, role) collaborators_after = get_collaborators_list_for_experiment(experiment_name) collaborators_added = [] collaborators_removed = collaborators_before - collaborators_after role_obj = get_role_object(experiment_name, role) - role_obj.update({'collaborators_added': [x for x in collaborators_added], 'collaborators_removed': [x for x in collaborators_removed], 'requestor': context.security.get_current_user_id() }) - context.kafka_producer.send("roles", {"experiment_name" : experiment_name, "instrument": get_experiment_info(experiment_name)["instrument"], "CRUD": "Update", "value": role_obj }) + role_obj.update( + { + "collaborators_added": [x for x in collaborators_added], + "collaborators_removed": [x for x in collaborators_removed], + "requestor": context.security.get_current_user_id(), + } + ) + context.kafka_producer.send( + "roles", + { + "experiment_name": experiment_name, + "instrument": get_experiment_info(experiment_name)["instrument"], + "CRUD": "Update", + "value": role_obj, + }, + ) return JSONEncoder().encode({"success": True, "message": "Removed collaborator"}) + @explgbk_blueprint.route("/lgbk//ws/sync_posix_group", methods=["GET"]) @experiment_exists @context.security.authentication_required @@ -2559,11 +3846,26 @@ def svc_sync_posix_group(experiment_name): collabs_to_add_to_posix_group = exp_collabs - posix_group_members collabs_to_remove_from_posix_group = posix_group_members - exp_collabs role_obj = {} - role_obj.update({'collaborators_added': [x for x in collabs_to_add_to_posix_group], 'collaborators_removed': [x for x in collabs_to_remove_from_posix_group], 'requestor': context.security.get_current_user_id() }) + role_obj.update( + { + "collaborators_added": [x for x in collabs_to_add_to_posix_group], + "collaborators_removed": [x for x in collabs_to_remove_from_posix_group], + "requestor": context.security.get_current_user_id(), + } + ) logger.info(role_obj) - context.kafka_producer.send("roles", {"experiment_name" : experiment_name, "instrument": get_experiment_info(experiment_name)["instrument"], "CRUD": "Update", "value": role_obj }) + context.kafka_producer.send( + "roles", + { + "experiment_name": experiment_name, + "instrument": get_experiment_info(experiment_name)["instrument"], + "CRUD": "Update", + "value": role_obj, + }, + ) return JSONEncoder().encode({"success": True, "value": role_obj}) + @explgbk_blueprint.route("/lgbk/ws/get_matching_uids", methods=["GET"]) @context.security.authentication_required def get_matching_uids(): @@ -2578,13 +3880,20 @@ def get_matching_uids(): ret = ret + context.usergroups.get_userids_matching_pattern(uid) return JSONEncoder().encode({"success": True, "value": ret}) + @explgbk_blueprint.route("/lgbk/ws/get_matching_groups", methods=["GET"]) @context.security.authentication_required def get_matching_groups(): group_name = request.args.get("group_name", None) if not group_name: return logAndAbort("Please specify a group_name") - return JSONEncoder().encode({"success": True, "value": context.usergroups.get_groups_matching_pattern(group_name)}) + return JSONEncoder().encode( + { + "success": True, + "value": context.usergroups.get_groups_matching_pattern(group_name), + } + ) + def sync_collaborators_with_user_portal(experiment_name): logger.info("Importing collaborators from URAWI for %s", experiment_name) @@ -2596,10 +3905,27 @@ def sync_collaborators_with_user_portal(experiment_name): if collaborators_added: role_obj = get_role_object(experiment_name, "LogBook/Writer") - role_obj.update({'collaborators_added': [x for x in collaborators_added], 'collaborators_removed': [x for x in collaborators_removed], 'requestor': context.security.get_current_user_id() }) - context.kafka_producer.send("roles", {"experiment_name" : experiment_name, "instrument": get_experiment_info(experiment_name)["instrument"], "CRUD": "Update", "value": role_obj }) - -@explgbk_blueprint.route("/lgbk//ws/sync_collaborators_with_user_portal", methods=["GET"]) + role_obj.update( + { + "collaborators_added": [x for x in collaborators_added], + "collaborators_removed": [x for x in collaborators_removed], + "requestor": context.security.get_current_user_id(), + } + ) + context.kafka_producer.send( + "roles", + { + "experiment_name": experiment_name, + "instrument": get_experiment_info(experiment_name)["instrument"], + "CRUD": "Update", + "value": role_obj, + }, + ) + + +@explgbk_blueprint.route( + "/lgbk//ws/sync_collaborators_with_user_portal", methods=["GET"] +) @experiment_exists @context.security.authentication_required @context.security.authorization_required("edit") @@ -2609,12 +3935,17 @@ def svc_sync_collaborators_with_user_portal(experiment_name): return JSONEncoder().encode({"success": True}) -@explgbk_blueprint.route("/lgbk/ws/sync_collaborators_with_user_portal_for_upcoming_experiments", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk/ws/sync_collaborators_with_user_portal_for_upcoming_experiments", + methods=["GET"], +) def svc_sync_collaborators_with_user_portal_for_upcoming_experiments(): """ This is used primarily from a cron job to sync with URAWI and get the latest set of collaborators. """ - upcoming_experiments = get_experiments_starting_in_time_frame(datetime.utcnow() - timedelta(days=2), datetime.utcnow() + timedelta(days=10)) + upcoming_experiments = get_experiments_starting_in_time_frame( + datetime.utcnow() - timedelta(days=2), datetime.utcnow() + timedelta(days=10) + ) for upcoming_experiment in upcoming_experiments: if "name" in upcoming_experiment: sync_collaborators_with_user_portal(upcoming_experiment["name"]) @@ -2628,9 +3959,17 @@ def svc_get_modal_param_definitions(): if not modal_type: return logAndAbort("Please specify a modal_type") param_defs = get_modal_param_definitions(modal_type) - return JSONEncoder().encode({"success": True, "value": param_defs if param_defs else { "_id": modal_type, "params": [] }}) + return JSONEncoder().encode( + { + "success": True, + "value": param_defs if param_defs else {"_id": modal_type, "params": []}, + } + ) + -@explgbk_blueprint.route("/lgbk//ws/get_modal_param_definitions", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/get_modal_param_definitions", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -2639,9 +3978,14 @@ def svc_get_modal_param_definitions_for_experiment(experiment_name): if not modal_type: return logAndAbort("Please specify a modal_type") param_defs = get_modal_param_definitions(modal_type) - return JSONEncoder().encode({"success": True, "value": param_defs if param_defs else {}}) + return JSONEncoder().encode( + {"success": True, "value": param_defs if param_defs else {}} + ) + -@explgbk_blueprint.route("/lgbk//ws/get_feedback_document", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/get_feedback_document", methods=["GET"] +) @experiment_exists def get_feedback_document(experiment_name): """ @@ -2651,36 +3995,52 @@ def get_feedback_document(experiment_name): return JSONEncoder().encode({"success": True, "value": poc_feedback_doc}) -@explgbk_blueprint.route("/lgbk//ws/add_feedback_item", methods=["POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/add_feedback_item", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("feedback_write") def add_feedback_item(experiment_name): - item_name = request.form.get("item_name", None) + item_name = request.form.get("item_name", None) item_value = request.form.get("item_value", "") if not item_name: return logAndAbort("Please specify the item name (item_name)") - add_poc_feedback_item(experiment_name, item_name, item_value, context.security.get_current_user_id()) + add_poc_feedback_item( + experiment_name, item_name, item_value, context.security.get_current_user_id() + ) # We treat the poc_feedback similar to experiment params and send a Kafka message. Mostly this is to rebuild the cache. info = get_experiment_info(experiment_name) - context.kafka_producer.send("experiments", {"experiment_name" : experiment_name, "CRUD": "Update", "value": info }) + context.kafka_producer.send( + "experiments", + {"experiment_name": experiment_name, "CRUD": "Update", "value": info}, + ) return JSONEncoder().encode({"success": True}) + # Tableau integration items - we follow https://jira.slac.stanford.edu/browse/PSWA-61 as much as possible. @explgbk_blueprint.route("/lgbk/ws/poc_feedback/schema", methods=["GET"]) def get_poc_feedback_schema(): feedback_defs_file = "static/json/{}/feedback.json".format(context.LOGBOOK_SITE) if not os.path.exists(feedback_defs_file): - return JSONEncoder().encode({"status": "error", "message": "No schema definition found for this site {}".format(context.LOGBOOK_SITE)}) + return JSONEncoder().encode( + { + "status": "error", + "message": "No schema definition found for this site {}".format( + context.LOGBOOK_SITE + ), + } + ) with open(feedback_defs_file, "r") as f: defs = json.load(f) + # Process the definitions and change some attribute names/values. Use hint for title, value for default_value, data_type for type def __map_schema_attrs__(o): if isinstance(o, list): - return [ __map_schema_attrs__(x) for x in o ] + return [__map_schema_attrs__(x) for x in o] elif isinstance(o, dict): - ret = { "title": o["hint"] } + ret = {"title": o["hint"]} if "groups" in o and "toggler" not in o: ret["groups"] = __map_schema_attrs__(o["groups"]) else: @@ -2690,15 +4050,26 @@ def __map_schema_attrs__(o): if "datatype" in o: ret["type"] = o["datatype"] else: - ret["type"] = {"readonly_text": "Number", "input": "String", "checkbox": "Boolean", "selector": "Number"}.get(o["type"], "String") - if o["type"] == "selector" or o["type"] == "checkbox" and o.get("notes", False): + ret["type"] = { + "readonly_text": "Number", + "input": "String", + "checkbox": "Boolean", + "selector": "Number", + }.get(o["type"], "String") + if ( + o["type"] == "selector" + or o["type"] == "checkbox" + and o.get("notes", False) + ): ret["id_notes"] = o["id"] + "-notes" if "groups" in o: ret["groups"] = __map_schema_attrs__(o["groups"]) return ret + defs = __map_schema_attrs__(defs) - return JSONEncoder().encode({"status": "success", "defs": defs }) + return JSONEncoder().encode({"status": "success", "defs": defs}) + @explgbk_blueprint.route("/lgbk/ws/poc_feedback/experiments", methods=["GET"]) def svc_get_poc_feedback_experiments(): @@ -2706,11 +4077,14 @@ def svc_get_poc_feedback_experiments(): Return all the experiments that have non-trivial poc feedback. """ exps = get_poc_feedback_experiments() - tz = pytz.timezone('America/Los_Angeles') + tz = pytz.timezone("America/Los_Angeles") ret = [] for exp in exps: - x = { "exper_name": exp["name"], "instr_name": exp["instrument"] } - if exp.get("params", {}).get("PNR", None) and exp.get("params", {}).get("PNR") == "N/A": + x = {"exper_name": exp["name"], "instr_name": exp["instrument"]} + if ( + exp.get("params", {}).get("PNR", None) + and exp.get("params", {}).get("PNR") == "N/A" + ): logger.debug("Skipping internal experiment %s", exp["name"]) continue if exp.get("params", {}).get("PNR", None): @@ -2721,25 +4095,44 @@ def svc_get_poc_feedback_experiments(): x["proposalNo"] = exp["name"][3:7].upper() elif len(exp["name"]) == 8: # Older experiments where we used to drop the L - x["proposalNo"] = 'L' + exp["name"][3:6].upper() + x["proposalNo"] = "L" + exp["name"][3:6].upper() else: pass if "proposalNo" not in x: - logger.debug("No point sending POC feedback if we cannot map to a URAWI proposal for %s", exp["name"]) + logger.debug( + "No point sending POC feedback if we cannot map to a URAWI proposal for %s", + exp["name"], + ) continue - if exp.get("poc_feedback", {}).get("last_modified_by", None): x["last_report_uid"] = exp["poc_feedback"]["last_modified_by"] - if exp.get("poc_feedback", {}).get("last_modified_at", None): x["last_report_time"] = exp["poc_feedback"]["last_modified_at"].astimezone(tz).strftime('%Y-%m-%d %H:%M:%S') - if exp.get("poc_feedback", {}).get("last_modified_at", None): x["last_modified_at_utc"] = exp["poc_feedback"]["last_modified_at"] - if exp.get("poc_feedback", {}).get("num_items", None): x["num_items"] = exp["poc_feedback"]["num_items"] - if exp.get("poc_feedback", {}).get("num_items_4_5", None): x["num_items_4_5"] = exp["poc_feedback"]["num_items_4_5"] - if exp.get("last_run", {}).get("begin_time", None): x["last_run_begin"] = exp["last_run"]["begin_time"].astimezone(tz).strftime('%Y-%m-%d %H:%M:%S') + if exp.get("poc_feedback", {}).get("last_modified_by", None): + x["last_report_uid"] = exp["poc_feedback"]["last_modified_by"] + if exp.get("poc_feedback", {}).get("last_modified_at", None): + x["last_report_time"] = ( + exp["poc_feedback"]["last_modified_at"] + .astimezone(tz) + .strftime("%Y-%m-%d %H:%M:%S") + ) + if exp.get("poc_feedback", {}).get("last_modified_at", None): + x["last_modified_at_utc"] = exp["poc_feedback"]["last_modified_at"] + if exp.get("poc_feedback", {}).get("num_items", None): + x["num_items"] = exp["poc_feedback"]["num_items"] + if exp.get("poc_feedback", {}).get("num_items_4_5", None): + x["num_items_4_5"] = exp["poc_feedback"]["num_items_4_5"] + if exp.get("last_run", {}).get("begin_time", None): + x["last_run_begin"] = ( + exp["last_run"]["begin_time"] + .astimezone(tz) + .strftime("%Y-%m-%d %H:%M:%S") + ) ret.append(x) return JSONEncoder().encode({"status": "success", "experiments": ret}) -@explgbk_blueprint.route("/lgbk//ws/get_run_params_for_all_runs", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/get_run_params_for_all_runs", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -2752,13 +4145,17 @@ def svc_get_specified_run_params_for_all_runs(experiment_name): """ param_names_str = request.args.get("param_names", None) if not param_names_str: - return logAndAbort("Please specify the run parameters as a comma separated list in the parameter param_names)") + return logAndAbort( + "Please specify the run parameters as a comma separated list in the parameter param_names)" + ) param_names = param_names_str.split(",") param_values = get_specified_run_params_for_all_runs(experiment_name, param_names) return JSONEncoder().encode({"success": True, "value": param_values}) -@explgbk_blueprint.route("/lgbk//ws/get_runs_matching_params", methods=["POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/get_runs_matching_params", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -2775,7 +4172,10 @@ def svc_get_runs_matching_param_values(experiment_name): run_numbers = get_run_nums_matching_params(experiment_name, query_doc) return JSONEncoder().encode({"success": True, "value": run_numbers}) -@explgbk_blueprint.route("/lgbk//ws/get_runs_matching_editable", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws/get_runs_matching_editable", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -2794,13 +4194,18 @@ def svc_get_run_nums_matching_editable_regex(experiment_name): return logAndAbort("Please specify the regex to match the value against") try: - run_numbers = get_run_nums_matching_editable_regex(experiment_name, param_name, incoming_regex) + run_numbers = get_run_nums_matching_editable_regex( + experiment_name, param_name, incoming_regex + ) return JSONEncoder().encode({"success": True, "value": run_numbers}) except Exception as e: logger.exception(e) return logAndAbort("Exception fetching run numbers " + str(e)) -@explgbk_blueprint.route("/lgbk//ws/get_runs_with_tag", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws/get_runs_with_tag", methods=["GET"] +) @experiment_exists def svc_get_run_nums_matching_tag(experiment_name): """ @@ -2813,6 +4218,7 @@ def svc_get_run_nums_matching_tag(experiment_name): run_numbers = get_run_numbers_with_tag(experiment_name, tag) return JSONEncoder().encode({"success": True, "value": run_numbers}) + @explgbk_blueprint.route("/lgbk//ws/get_tags_to_runs", methods=["GET"]) @context.security.authentication_required @experiment_exists @@ -2821,7 +4227,10 @@ def svc_get_tag_to_run_numbers(experiment_name): """ Get a dict of tag to the run number that have elog statements containing the tag """ - return JSONEncoder().encode({"success": True, "value": get_tag_to_run_numbers(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_tag_to_run_numbers(experiment_name)} + ) + @explgbk_blueprint.route("/lgbk//ws/get_runs_to_tags", methods=["GET"]) @context.security.authentication_required @@ -2831,9 +4240,14 @@ def svc_get_tags_for_runs(experiment_name): """ Get a dict of run number to the union of tags for all elog statments associated with that run """ - return JSONEncoder().encode({"success": True, "value": get_tags_for_runs(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_tags_for_runs(experiment_name)} + ) + -@explgbk_blueprint.route("/lgbk//ws//get_tags_for_run", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws//get_tags_for_run", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -2845,12 +4259,17 @@ def svc_get_tags_for_run(experiment_name, run_num): try: run_num = int(run_num_str) except ValueError: - run_num = run_num_str # Cryo uses strings for run numbers. + run_num = run_num_str # Cryo uses strings for run numbers. tags_for_runs = get_tags_for_runs(experiment_name) - return JSONEncoder().encode({"success": True, "value": tags_for_runs.get(run_num, [])}) + return JSONEncoder().encode( + {"success": True, "value": tags_for_runs.get(run_num, [])} + ) + -@explgbk_blueprint.route("/lgbk//ws/map_param_editable_to_run_nums", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/map_param_editable_to_run_nums", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -2864,30 +4283,51 @@ def svc_map_param_editable_to_run_nums(experiment_name): param_name = request.args.get("param_name", None) if not param_name: return logAndAbort("Please specify the parameter name") - return JSONEncoder().encode({"success": True, "value": map_param_editable_to_run_nums(experiment_name, param_name)}) + return JSONEncoder().encode( + { + "success": True, + "value": map_param_editable_to_run_nums(experiment_name, param_name), + } + ) + @explgbk_blueprint.route("/lgbk//ws/dm_locations", methods=["GET"]) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_dm_locations(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_dm_locations(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_dm_locations(experiment_name)} + ) + -@explgbk_blueprint.route("/lgbk//ws/workflow_definitions", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/workflow_definitions", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_wf_definitions(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_workflow_definitions(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_workflow_definitions(experiment_name)} + ) + -@explgbk_blueprint.route("/lgbk//ws/workflow_triggers", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/workflow_triggers", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_wf_triggers(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_workflow_triggers(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_workflow_triggers(experiment_name)} + ) -@explgbk_blueprint.route("/lgbk//ws/create_update_workflow_def", methods=["POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/create_update_workflow_def", methods=["POST"] +) @context.security.authentication_required @context.security.authorization_required("post") def svc_create_update_wf_definition(experiment_name): @@ -2898,26 +4338,70 @@ def svc_create_update_wf_definition(experiment_name): if not info: return logAndAbort("Please pass in the workflow definition as a JSON document.") - necessary_keys = set(['name', 'executable', 'trigger', 'location', 'parameters']) + necessary_keys = set(["name", "executable", "trigger", "location", "parameters"]) missing_keys = necessary_keys - info.keys() if missing_keys: - return JSONEncoder().encode({"success": False, "errormsg": "Create/update workflow missing keys %s" % missing_keys, "value": None}) - if info['trigger'] not in [x["value"] for x in get_workflow_triggers(experiment_name)]: - return JSONEncoder().encode({"success": False, "errormsg": "Invalid trigger %s in create/update workflow" % info['trigger'], "value": None}) - if info['location'] not in [ x["name"] for x in get_dm_locations(experiment_name) if "jid_prefix" in x and x["jid_prefix"] ]: - return JSONEncoder().encode({"success": False, "errormsg": "Invalid location %s in create/update workflow" % info['location'], "value": None}) + return JSONEncoder().encode( + { + "success": False, + "errormsg": "Create/update workflow missing keys %s" % missing_keys, + "value": None, + } + ) + if info["trigger"] not in [ + x["value"] for x in get_workflow_triggers(experiment_name) + ]: + return JSONEncoder().encode( + { + "success": False, + "errormsg": "Invalid trigger %s in create/update workflow" + % info["trigger"], + "value": None, + } + ) + if info["location"] not in [ + x["name"] + for x in get_dm_locations(experiment_name) + if "jid_prefix" in x and x["jid_prefix"] + ]: + return JSONEncoder().encode( + { + "success": False, + "errormsg": "Invalid location %s in create/update workflow" + % info["location"], + "value": None, + } + ) info["run_as_user"] = context.security.get_current_user_id() - if info["run_as_user"] == "root" or (len(info["run_as_user"]) == 6 and info["run_as_user"].endswith("opr")): - return JSONEncoder().encode({"success": False, "errormsg": "Cannot create a workflow definition for the user %s for security reasons" % info["run_as_user"], "value": None}) - current_wfdefs = { x["name"] : x for x in get_workflow_definitions(experiment_name) } + if info["run_as_user"] == "root" or ( + len(info["run_as_user"]) == 6 and info["run_as_user"].endswith("opr") + ): + return JSONEncoder().encode( + { + "success": False, + "errormsg": "Cannot create a workflow definition for the user %s for security reasons" + % info["run_as_user"], + "value": None, + } + ) + current_wfdefs = {x["name"]: x for x in get_workflow_definitions(experiment_name)} if info["name"] in current_wfdefs.keys() and "_id" not in info: - return JSONEncoder().encode({"success": False, "errormsg": "There already exists a workflow definition for %s" % info["name"], "value": None}) - + return JSONEncoder().encode( + { + "success": False, + "errormsg": "There already exists a workflow definition for %s" + % info["name"], + "value": None, + } + ) (status, errormsg, val) = create_update_wf_definition(experiment_name, info) return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": val}) -@explgbk_blueprint.route("/lgbk//ws/workflow_definitions/", methods=["DELETE"]) + +@explgbk_blueprint.route( + "/lgbk//ws/workflow_definitions/", methods=["DELETE"] +) @context.security.authentication_required @context.security.authorization_required("post") def svc_create_delete_wf_definition(experiment_name, defid): @@ -2927,14 +4411,20 @@ def svc_create_delete_wf_definition(experiment_name, defid): (status, errormsg, val) = delete_wf_definition(experiment_name, defid) return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": val}) + @explgbk_blueprint.route("/lgbk//ws/workflow_jobs", methods=["GET"]) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_wf_jobs(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_workflow_jobs(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_workflow_jobs(experiment_name)} + ) + -@explgbk_blueprint.route("/lgbk//ws/workflow//", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/workflow//", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") @@ -2944,30 +4434,64 @@ def svc_get_wf_job_action(experiment_name, job_id, action): """ wf_job = get_workflow_job_doc(experiment_name, job_id) if not wf_job: - return logAndAbort("Cannot find workflow in experiment %s for id %s" % (experiment_name, job_id), 404) + return logAndAbort( + "Cannot find workflow in experiment %s for id %s" + % (experiment_name, job_id), + 404, + ) if action not in ["job_statuses", "job_details", "job_log_file"]: - return logAndAbort("For security reasons, action %s is not proxied thru the logbook" % (action), 405) + return logAndAbort( + "For security reasons, action %s is not proxied thru the logbook" + % (action), + 405, + ) + def __proxy_JID__(location): - logger.debug("Calling the JID at %s", (location["jid_prefix"]+"jid/ws/"+action)) + logger.debug( + "Calling the JID at %s", (location["jid_prefix"] + "jid/ws/" + action) + ) client_cert_params = {} if "jid_client_key" in location and "jid_client_cert" in location: - client_cert_params["cert"] = (location["jid_client_cert"], location["jid_client_key"]) + client_cert_params["cert"] = ( + location["jid_client_cert"], + location["jid_client_key"], + ) if "jid_ca_cert" in location: client_cert_params["verify"] = location["jid_ca_cert"] - arp_token = context.generateArpToken(context.security.get_current_user_id(), experiment_name) + arp_token = context.generateArpToken( + context.security.get_current_user_id(), experiment_name + ) # The ARP does an exact match on userid in wf_job; so for these mostly innocent calls, we change the useird on the job doc wf_job["user"] = context.security.get_current_user_id() - req = requests.post(location["jid_prefix"]+"jid/ws/"+experiment_name+"/"+action, data=JSONEncoder().encode(wf_job), stream=True, headers={"Content-Type": "application/json", "Authorization": "Bearer " + arp_token}, **client_cert_params) + req = requests.post( + location["jid_prefix"] + "jid/ws/" + experiment_name + "/" + action, + data=JSONEncoder().encode(wf_job), + stream=True, + headers={ + "Content-Type": "application/json", + "Authorization": "Bearer " + arp_token, + }, + **client_cert_params, + ) resp = Response(stream_with_context(req.iter_content(chunk_size=1024))) return resp - location = { x["name"] : x for x in get_dm_locations(experiment_name) }.get(wf_job['def']['location'], None) + location = {x["name"]: x for x in get_dm_locations(experiment_name)}.get( + wf_job["def"]["location"], None + ) if not location: - return logAndAbort("Cannot determine workflow location in experiment %s for id %s %s" % (experiment_name, job_id, wf_job), 500) + return logAndAbort( + "Cannot determine workflow location in experiment %s for id %s %s" + % (experiment_name, job_id, wf_job), + 500, + ) return __proxy_JID__(location) -@explgbk_blueprint.route("/lgbk//ws/create_workflow_job", methods=["POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/create_workflow_job", methods=["POST"] +) @context.security.authentication_required @context.security.authorization_required("post") def svc_create_workflow_job(experiment_name): @@ -2978,23 +4502,48 @@ def svc_create_workflow_job(experiment_name): if not info: return logAndAbort("Please pass in the workflow job as a JSON document.") - necessary_keys = set(['job_name', 'run_num']) + necessary_keys = set(["job_name", "run_num"]) missing_keys = necessary_keys - info.keys() if missing_keys: - return JSONEncoder().encode({"success": False, "errormsg": "Create/update workflow job missing keys %s" % missing_keys, "value": None}) + return JSONEncoder().encode( + { + "success": False, + "errormsg": "Create/update workflow job missing keys %s" % missing_keys, + "value": None, + } + ) - def_id = { x["name"]: x for x in get_workflow_definitions(experiment_name) }.get(info["job_name"], {}).get("_id", None) + def_id = {x["name"]: x for x in get_workflow_definitions(experiment_name)}.get( + info["job_name"], {} + ).get("_id", None) if not def_id: - return JSONEncoder().encode({"success": False, "errormsg": "Cannot find job definition for %s " % info["job_name"], "value": None}) - wf_job_doc = { "run_num": int(info["run_num"]), "def_id": def_id, "user": context.security.get_current_user_id(), "status": "START" } + return JSONEncoder().encode( + { + "success": False, + "errormsg": "Cannot find job definition for %s " % info["job_name"], + "value": None, + } + ) + wf_job_doc = { + "run_num": int(info["run_num"]), + "def_id": def_id, + "user": context.security.get_current_user_id(), + "status": "START", + } (status, errormsg, val) = create_wf_job(experiment_name, wf_job_doc) if status: - context.kafka_producer.send("workflow_jobs", {"experiment_name" : experiment_name, "CRUD": "Create", "value": val }) + context.kafka_producer.send( + "workflow_jobs", + {"experiment_name": experiment_name, "CRUD": "Create", "value": val}, + ) return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": val}) -@explgbk_blueprint.route("/lgbk//ws/delete_workflow_job", methods=["GET", "POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/delete_workflow_job", methods=["GET", "POST"] +) @context.security.authentication_required @context.security.authorization_required("post") def svc_delete_workflow_job(experiment_name): @@ -3007,11 +4556,17 @@ def svc_delete_workflow_job(experiment_name): (status, errormsg, val) = delete_wf_job(experiment_name, job_id) if status: - context.kafka_producer.send("workflow_jobs", {"experiment_name" : experiment_name, "CRUD": "Delete", "value": val }) + context.kafka_producer.send( + "workflow_jobs", + {"experiment_name": experiment_name, "CRUD": "Delete", "value": val}, + ) return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": val}) -@explgbk_blueprint.route("/lgbk//ws/update_workflow_job", methods=["POST"]) + +@explgbk_blueprint.route( + "/lgbk//ws/update_workflow_job", methods=["POST"] +) @context.security.authentication_required @context.security.authorization_required("post") def svc_update_workflow_job(experiment_name): @@ -3030,17 +4585,22 @@ def svc_update_workflow_job(experiment_name): if not wf_id: return logAndAbort("Please pass in the job _id in a JSON document.") - allowed_keys = ['status', 'counters', 'tool_id', 'log_file_path'] - wf_updates = { k: info[k] for k in allowed_keys if k in info and info[k] } + allowed_keys = ["status", "counters", "tool_id", "log_file_path"] + wf_updates = {k: info[k] for k in allowed_keys if k in info and info[k]} (status, errormsg, val) = update_wf_job(experiment_name, wf_id, wf_updates) if status: - context.kafka_producer.send("workflow_jobs", {"experiment_name" : experiment_name, "CRUD": "Update", "value": val }) + context.kafka_producer.send( + "workflow_jobs", + {"experiment_name": experiment_name, "CRUD": "Update", "value": val}, + ) return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": val}) -@explgbk_blueprint.route("/lgbk//ws/kill_workflow_job", methods=["GET", "POST"]) +@explgbk_blueprint.route( + "/lgbk//ws/kill_workflow_job", methods=["GET", "POST"] +) @context.security.authentication_required @context.security.authorization_required("post") def svc_kill_workflow_job(experiment_name): @@ -3051,28 +4611,51 @@ def svc_kill_workflow_job(experiment_name): if not job_id: return logAndAbort("Please pass in the workflow job id.") wf_job = get_workflow_job_doc(experiment_name, job_id) - location_config = { x["name"] : x for x in get_dm_locations(experiment_name)} + location_config = {x["name"]: x for x in get_dm_locations(experiment_name)} location = location_config[wf_job["def"]["location"]] client_cert_params = {} if "jid_client_key" in location and "jid_client_cert" in location: - client_cert_params["cert"] = (location["jid_client_cert"], location["jid_client_key"]) + client_cert_params["cert"] = ( + location["jid_client_cert"], + location["jid_client_key"], + ) if "jid_ca_cert" in location: client_cert_params["verify"] = location["jid_ca_cert"] user_for_token = context.security.get_current_user_id() if user_for_token != wf_job["user"]: user_for_token = wf_job["user"] - logger.warning("Only for the kill job, we generate the token as the user %s running the job %s instead of the logged in user %s", user_for_token, wf_job["user"], context.security.get_current_user_id()) + logger.warning( + "Only for the kill job, we generate the token as the user %s running the job %s instead of the logged in user %s", + user_for_token, + wf_job["user"], + context.security.get_current_user_id(), + ) arp_token = context.generateArpToken(user_for_token, experiment_name) - resp = requests.post(location["jid_prefix"] + "jid/ws/"+experiment_name+"/"+"kill_job", data=JSONEncoder().encode(wf_job), headers={"Content-Type": "application/json", "Authorization": "Bearer " + arp_token}, **client_cert_params) + resp = requests.post( + location["jid_prefix"] + "jid/ws/" + experiment_name + "/" + "kill_job", + data=JSONEncoder().encode(wf_job), + headers={ + "Content-Type": "application/json", + "Authorization": "Bearer " + arp_token, + }, + **client_cert_params, + ) respdoc = resp.json()["value"] - (status, errormsg, val) = update_wf_job(experiment_name, job_id, {"status": respdoc.get("status", wf_job["status"])}) + (status, errormsg, val) = update_wf_job( + experiment_name, job_id, {"status": respdoc.get("status", wf_job["status"])} + ) if status: - context.kafka_producer.send("workflow_jobs", {"experiment_name" : experiment_name, "CRUD": "Update", "value": val }) + context.kafka_producer.send( + "workflow_jobs", + {"experiment_name": experiment_name, "CRUD": "Update", "value": val}, + ) return JSONEncoder().encode({"success": status, "errormsg": errormsg, "value": val}) -@explgbk_blueprint.route("/lgbk//ws/generate_arp_token", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws/generate_arp_token", methods=["GET"] +) @context.security.authentication_required @context.security.authorization_required("post") def svc_workflow_generate_arp_token(experiment_name): @@ -3086,19 +4669,27 @@ def svc_workflow_generate_arp_token(experiment_name): """ token_lifetime = int(request.args.get("token_lifetime", "1")) if token_lifetime > 480: - return logAndAbort("Cannot generate tokens with lifetimes larger than 480 minutes") + return logAndAbort( + "Cannot generate tokens with lifetimes larger than 480 minutes" + ) if request.headers.get("X-Forwarded-Auth-Type", None) != "Kerberos": return logAndAbort("Tokens are support only with the ws-kerb endpoint") - arp_token = context.generateArpToken(context.security.get_current_user_id(), experiment_name, token_lifetime) + arp_token = context.generateArpToken( + context.security.get_current_user_id(), experiment_name, token_lifetime + ) return JSONEncoder().encode({"success": True, "value": arp_token}) + @explgbk_blueprint.route("/lgbk/naming_conventions", methods=["GET"]) @context.security.authentication_required def svc_get_site_naming_conventions(): """ Get the site config """ - return JSONEncoder().encode({"success": True, "value": get_site_naming_conventions()}) + return JSONEncoder().encode( + {"success": True, "value": get_site_naming_conventions()} + ) + @explgbk_blueprint.route("/lgbk/filemanager_file_types", methods=["GET"]) @context.security.authentication_required @@ -3108,16 +4699,26 @@ def svc_get_site_filemanager_file_types(): """ return JSONEncoder().encode({"success": True, "value": get_site_file_types()}) -@explgbk_blueprint.route("/lgbk//ws/run_param_descriptions", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws/run_param_descriptions", methods=["GET"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("read") def svc_get_run_param_descriptions(experiment_name): - return JSONEncoder().encode({"success": True, "value": get_run_param_descriptions(experiment_name)}) + return JSONEncoder().encode( + {"success": True, "value": get_run_param_descriptions(experiment_name)} + ) -@explgbk_blueprint.route("/run_control//ws/add_update_run_param_descriptions", methods=["POST"]) -@explgbk_blueprint.route("/lgbk//ws/add_update_run_param_descriptions", methods=["POST"]) +@explgbk_blueprint.route( + "/run_control//ws/add_update_run_param_descriptions", + methods=["POST"], +) +@explgbk_blueprint.route( + "/lgbk//ws/add_update_run_param_descriptions", methods=["POST"] +) @context.security.authentication_required @experiment_exists @context.security.authorization_required("post") @@ -3145,7 +4746,10 @@ def svc_get_params_matching_prefix(): matches = get_all_param_names_matching_regex(pattern) return JSONEncoder().encode({"success": True, "value": matches}) -@explgbk_blueprint.route("/lgbk//ws//get_params_matching_prefix", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk//ws//get_params_matching_prefix", methods=["GET"] +) def svc_get_params_matching_prefix_for_run(experiment_name, run_num): """ Return parameter names that match a prefix for an experiment/run @@ -3159,16 +4763,18 @@ def svc_get_params_matching_prefix_for_run(experiment_name, run_num): try: run_num = int(run_num_str) except ValueError: - run_num = run_num_str # Cryo uses strings for run numbers. + run_num = run_num_str # Cryo uses strings for run numbers. run_doc = get_run_doc_for_run_num(experiment_name, run_num) if not run_doc: return JSONEncoder().encode({"success": True, "value": []}) - matches = [ x for x in run_doc.get("params", {}).keys() if patt.match(x) ] + matches = [x for x in run_doc.get("params", {}).keys() if patt.match(x)] return JSONEncoder().encode({"success": True, "value": matches}) -@explgbk_blueprint.route("/lgbk//ws//daq_run_params", methods=["GET"]) +@explgbk_blueprint.route( + "/lgbk//ws//daq_run_params", methods=["GET"] +) def svc_get_get_daq_run_params(experiment_name, run_num): """ Return special parameters for the specified experiment and run. @@ -3176,12 +4782,18 @@ def svc_get_get_daq_run_params(experiment_name, run_num): Otherwise this returns params that have a standard prefix. This mostly applies to the LCLS DAQ. """ - std_prefixes = ["Calibrations/", "DAQ Detector Totals/", "DAQ_Detector_Totals/", "DAQ Detectors/", "DAQ_Detectors/"] + std_prefixes = [ + "Calibrations/", + "DAQ Detector Totals/", + "DAQ_Detector_Totals/", + "DAQ Detectors/", + "DAQ_Detectors/", + ] run_num_str = run_num try: run_num = int(run_num_str) except ValueError: - run_num = run_num_str # Cryo uses strings for run numbers. + run_num = run_num_str # Cryo uses strings for run numbers. run_doc = get_run_doc_for_run_num(experiment_name, run_num) if not run_doc: @@ -3195,6 +4807,7 @@ def svc_get_get_daq_run_params(experiment_name, run_num): return JSONEncoder().encode({"success": True, "value": matches}) + @explgbk_blueprint.route("/lgbk/ws/api_endpoints", methods=["GET"]) @context.security.authentication_required def svc_get_api_endpoints(): @@ -3204,58 +4817,80 @@ def svc_get_api_endpoints(): fnnames.append(v.__name__) api_endpoints_docs = {} for rule in current_app.url_map.iter_rules(): - if rule.endpoint != 'static': + if rule.endpoint != "static": if current_app.view_functions[rule.endpoint].__doc__: epname = current_app.view_functions[rule.endpoint].__name__ if epname not in api_endpoints_docs: api_endpoints_docs[epname] = [] - api_endpoints_docs[epname].append({ "endpoint": rule.rule, "methods": list(rule.methods - set(["HEAD", "OPTIONS"])), "docstring": current_app.view_functions[rule.endpoint].__doc__}) + api_endpoints_docs[epname].append( + { + "endpoint": rule.rule, + "methods": list(rule.methods - set(["HEAD", "OPTIONS"])), + "docstring": current_app.view_functions[rule.endpoint].__doc__, + } + ) sorted_api_endpoints = [] for fn in fnnames: if fn in api_endpoints_docs: sorted_api_endpoints.extend(api_endpoints_docs[fn]) return JSONEncoder().encode(sorted_api_endpoints) + @explgbk_blueprint.route("/lgbk/ws/projects", methods=["GET"]) @context.security.authentication_required def svc_get_projects(): - return JSONEncoder().encode({"success": True, "value": get_projects(context.security.get_current_user_id())}) + return JSONEncoder().encode( + {"success": True, "value": get_projects(context.security.get_current_user_id())} + ) + def user_in_project(wrapped_function): """ Check if user is in a project """ + @wraps(wrapped_function) def function_interceptor(*args, **kwargs): - project_id = kwargs.get('prjid', None) + project_id = kwargs.get("prjid", None) if project_id: - projectinfo = get_project_info(project_id) + projectinfo = get_project_info(project_id) if projectinfo: - if "uid:" + context.security.get_current_user_id() in projectinfo["players"]: + if ( + "uid:" + context.security.get_current_user_id() + in projectinfo["players"] + ): g.projectinfo = projectinfo return wrapped_function(*args, **kwargs) - logger.error("User " + context.security.get_current_user_id() + " does not have permissions to view project " + project_id) + logger.error( + "User " + + context.security.get_current_user_id() + + " does not have permissions to view project " + + project_id + ) abort(404) return None return function_interceptor + @explgbk_blueprint.route("/lgbk/ws/projects/", methods=["GET"]) @user_in_project def svc_get_project_info(prjid): - projectinfo = get_project_info(prjid) + projectinfo = get_project_info(prjid) return JSONEncoder().encode({"success": True, "value": projectinfo}) + @explgbk_blueprint.route("/lgbk/ws/projects/", methods=["POST"]) @context.security.authentication_required def svc_create_project(): prjinfo = request.json userid = context.security.get_current_user_id() - prjinfo["players"] = [ "uid:" + userid ] + prjinfo["players"] = ["uid:" + userid] prjinfo["owner"] = userid ret = create_project(prjinfo) return JSONEncoder().encode({"success": True, "value": ret}) + @explgbk_blueprint.route("/lgbk/ws/projects/", methods=["PUT"]) @user_in_project def svc_update_project_info(prjid): @@ -3263,19 +4898,22 @@ def svc_update_project_info(prjid): ret = update_project(prjid, prjinfo) return JSONEncoder().encode({"success": True, "value": ret}) + @explgbk_blueprint.route("/lgbk/ws/projects//grids", methods=["GET"]) @user_in_project def svc_get_project_grids(prjid): - projectinfo = get_project_info(prjid) + get_project_info(prjid) grids = get_project_grids(prjid) return JSONEncoder().encode({"success": True, "value": grids}) + @explgbk_blueprint.route("/lgbk/ws/projects//grids/", methods=["GET"]) @user_in_project def svc_get_project_grid(prjid, gridid): grid = get_project_grid(prjid, gridid) return JSONEncoder().encode({"success": True, "value": grid}) + @explgbk_blueprint.route("/lgbk/ws/projects//grids/", methods=["POST"]) @user_in_project def svc_add_grid_to_project(prjid): @@ -3283,6 +4921,7 @@ def svc_add_grid_to_project(prjid): (status, errormsg) = add_grid_to_project(prjid, griddetails) return JSONEncoder().encode({"success": status, "errormsg": errormsg}) + @explgbk_blueprint.route("/lgbk/ws/projects//grids/", methods=["PUT"]) @user_in_project def svc_update_project_grid(prjid, gridid): @@ -3290,19 +4929,26 @@ def svc_update_project_grid(prjid, gridid): (status, errormsg) = update_project_grid(prjid, gridid, griddetails) return JSONEncoder().encode({"success": status, "errormsg": errormsg}) -@explgbk_blueprint.route("/lgbk/ws/projects//grids//linksession", methods=["GET"]) + +@explgbk_blueprint.route( + "/lgbk/ws/projects//grids//linksession", methods=["GET"] +) @user_in_project def svc_link_experiment_project_grid(prjid, gridid): experiment_name = request.args["experiment_name"] (status, errormsg) = link_grid_to_experiment(prjid, gridid, experiment_name) return JSONEncoder().encode({"success": status, "errormsg": errormsg}) + @explgbk_blueprint.route("/lgbk/ws/projects//sessions", methods=["GET"]) @user_in_project def svc_get_project_sessions(prjid): - projectinfo = get_project_info(prjid) + projectinfo = get_project_info(prjid) sessions = projectinfo.get("sessions", {}) - experiments = { x["name"] : x for x in get_experiments_for_user(context.security.get_current_user_id())} + experiments = { + x["name"]: x + for x in get_experiments_for_user(context.security.get_current_user_id()) + } for sessionname, sessiondetails in sessions.items(): if sessionname in experiments: sessiondetails["expinfo"] = experiments[sessionname] diff --git a/pages.py b/explgbk/blueprints/pages.py similarity index 54% rename from pages.py rename to explgbk/blueprints/pages.py index 75b50e6..2e1352c 100755 --- a/pages.py +++ b/explgbk/blueprints/pages.py @@ -2,48 +2,77 @@ import json import logging import pkg_resources -import datetime import urllib.parse -import context +from explgbk import context -from flask import request, Blueprint, render_template, send_file, abort, make_response, jsonify, session, Response, redirect +from flask import ( + request, + Blueprint, + render_template, + send_file, + abort, + make_response, + jsonify, + Response, + redirect, +) -from dal.explgbk import get_current_sample_name, get_experiment_info, get_project_info -from services.explgbk import experiment_exists +from explgbk.dal.explgbk import ( + get_current_sample_name, + get_experiment_info, + get_project_info, +) +from explgbk.blueprints.api import experiment_exists -pages_blueprint = Blueprint('pages_api', __name__) +pages_blueprint = Blueprint("pages_api", __name__) logger = logging.getLogger(__name__) + def logAndAbort(error_msg, ret_status=500): logger.error(error_msg) return Response(error_msg, status=ret_status) + @pages_blueprint.route("/") def index(): return render_template("choose_experiment.html") + @pages_blueprint.route("/status") def status(): - return jsonify({"success": True, "mongo_version": context.logbookclient.server_info()['version']}) + return jsonify( + { + "success": True, + "mongo_version": context.logbookclient.server_info()["version"], + } + ) -@pages_blueprint.route('/js/') +@pages_blueprint.route("/js/") def send_js(path): pathparts = os.path.normpath(path).split(os.sep) - if pathparts[0] == 'python': + if pathparts[0] == "python": # This is code for gettting the JS file from the package data of the python module. - filepath = pkg_resources.resource_filename(pathparts[1], os.sep.join(pathparts[2:])) + filepath = pkg_resources.resource_filename( + pathparts[1], os.sep.join(pathparts[2:]) + ) if os.path.exists(filepath): # logger.debug("Found file %s as part of a python package resources", filepath) return send_file(filepath) - # $CONDA_PREFIX/lib/node_modules/jquery/dist/ filepath = os.path.join(os.getenv("CONDA_PREFIX"), "lib", "node_modules", path) if not os.path.exists(filepath): - filepath = os.path.join(os.getenv("CONDA_PREFIX"), "lib", "node_modules", pathparts[0], "dist", *pathparts[1:]) + filepath = os.path.join( + os.getenv("CONDA_PREFIX"), + "lib", + "node_modules", + pathparts[0], + "dist", + *pathparts[1:], + ) if os.path.exists(filepath): return send_file(filepath) else: @@ -51,11 +80,13 @@ def send_js(path): abort(404) return None + @pages_blueprint.route("/lgbk//templates/", methods=["GET"]) @experiment_exists def templates(experiment_name, path): return render_template(path, experiment_name=experiment_name) + @pages_blueprint.route("/lgbk/ops", methods=["GET"]) @context.security.authentication_required def operator_dashboard_default(): @@ -67,62 +98,105 @@ def operator_dashboard_default(): @context.security.authentication_required @context.security.authorization_required("ops_page") def operator_dashboard(tabname): - logged_in_user=context.security.get_current_user_id() - privileges = { x : context.security.check_privilege_for_experiment(x, None, None) for x in [ "ops_page", "switch", "experiment_create", "experiment_edit", "experiment_delete", "instrument_create", "manage_groups"]} - return render_template("ops.html", + logged_in_user = context.security.get_current_user_id() + privileges = { + x: context.security.check_privilege_for_experiment(x, None, None) + for x in [ + "ops_page", + "switch", + "experiment_create", + "experiment_edit", + "experiment_delete", + "instrument_create", + "manage_groups", + ] + } + return render_template( + "ops.html", logbook_site=context.LOGBOOK_SITE, logged_in_user=logged_in_user, tabname=tabname, pagepath=request.path, - privileges=json.dumps(privileges)) + privileges=json.dumps(privileges), + ) @pages_blueprint.route("/lgbk/experiments", methods=["GET"]) @context.security.authentication_required def choose_experiments(): - logged_in_user=context.security.get_current_user_id() - privileges = { x : context.security.check_privilege_for_experiment(x, None, None) for x in [ "read", "ops_page", "switch", "experiment_create", "experiment_edit"]} - return render_template("experiments.html", + logged_in_user = context.security.get_current_user_id() + privileges = { + x: context.security.check_privilege_for_experiment(x, None, None) + for x in ["read", "ops_page", "switch", "experiment_create", "experiment_edit"] + } + return render_template( + "experiments.html", logbook_site=context.LOGBOOK_SITE, logged_in_user=logged_in_user, pagepath=request.path, - logged_in_user_details=json.dumps(context.usergroups.get_userids_matching_pattern(logged_in_user)), - privileges=json.dumps(privileges)) + logged_in_user_details=json.dumps( + context.usergroups.get_userids_matching_pattern(logged_in_user) + ), + privileges=json.dumps(privileges), + ) + @pages_blueprint.route("/lgbk/projects/", methods=["GET"]) @context.security.authentication_required def projects(): - logged_in_user=context.security.get_current_user_id() - privileges = { x : context.security.check_privilege_for_experiment(x, None, None) for x in [ "read", "ops_page", "switch", "experiment_create", "experiment_edit"]} - return render_template("projects.html", + logged_in_user = context.security.get_current_user_id() + privileges = { + x: context.security.check_privilege_for_experiment(x, None, None) + for x in ["read", "ops_page", "switch", "experiment_create", "experiment_edit"] + } + return render_template( + "projects.html", logbook_site=context.LOGBOOK_SITE, logged_in_user=logged_in_user, pagepath=request.path, - logged_in_user_details=json.dumps(context.usergroups.get_userids_matching_pattern(logged_in_user)), - privileges=json.dumps(privileges)) + logged_in_user_details=json.dumps( + context.usergroups.get_userids_matching_pattern(logged_in_user) + ), + privileges=json.dumps(privileges), + ) + @pages_blueprint.route("/lgbk/projects//", methods=["GET"]) @context.security.authentication_required def project(project_id, tabname): - logged_in_user=context.security.get_current_user_id() - privileges = { x : context.security.check_privilege_for_experiment(x, None, None) for x in [ "read", "ops_page", "switch", "experiment_create", "experiment_edit"]} + logged_in_user = context.security.get_current_user_id() + privileges = { + x: context.security.check_privilege_for_experiment(x, None, None) + for x in ["read", "ops_page", "switch", "experiment_create", "experiment_edit"] + } project = get_project_info(project_id) if "uid:" + logged_in_user not in project["players"]: return logAndAbort("Permission denied", 403) - return render_template("project.html", + return render_template( + "project.html", project_id=project_id, project_name=project["name"], pagepath=request.path, tabname=tabname, logbook_site=context.LOGBOOK_SITE, logged_in_user=logged_in_user, - logged_in_user_details=json.dumps(context.usergroups.get_userids_matching_pattern(logged_in_user)), - privileges=json.dumps(privileges)) + logged_in_user_details=json.dumps( + context.usergroups.get_userids_matching_pattern(logged_in_user) + ), + privileges=json.dumps(privileges), + ) + @pages_blueprint.route("/lgbk/logout", methods=["GET"]) @context.security.authentication_required def logout(): - return make_response(redirect("https://vouch.slac.stanford.edu/logout?returnTo=" + urllib.parse.quote("https://pswww.slac.stanford.edu"))) + return make_response( + redirect( + "https://vouch.slac.stanford.edu/logout?returnTo=" + + urllib.parse.quote("https://pswww.slac.stanford.edu") + ) + ) + @pages_blueprint.route("/lgbk/docs/", methods=["GET"]) @context.security.authentication_required @@ -133,22 +207,42 @@ def docs(path): return send_file(doc_path) abort(404) + @pages_blueprint.route("/lgbk/help", methods=["GET"]) @context.security.authentication_required def lgbkhelp(): - logged_in_user=context.security.get_current_user_id() - privileges = { x : context.security.check_privilege_for_experiment(x, None, None) for x in [ "read", "ops_page", "switch", "experiment_create", "experiment_edit", "experiment_edit"]} - return render_template("help.html", + logged_in_user = context.security.get_current_user_id() + privileges = { + x: context.security.check_privilege_for_experiment(x, None, None) + for x in [ + "read", + "ops_page", + "switch", + "experiment_create", + "experiment_edit", + "experiment_edit", + ] + } + return render_template( + "help.html", logbook_site=context.LOGBOOK_SITE, logged_in_user=logged_in_user, pagepath=request.path, - logged_in_user_details=json.dumps(context.usergroups.get_userids_matching_pattern(logged_in_user)), - privileges=json.dumps(privileges)) + logged_in_user_details=json.dumps( + context.usergroups.get_userids_matching_pattern(logged_in_user) + ), + privileges=json.dumps(privileges), + ) def __parse_expiration_header__(request): expiration = request.headers.get("Webauth-Token-Expiration", "0") - return int(expiration.replace("t=", ""))//1000000 if expiration.startswith("t=") else int(expiration) + return ( + int(expiration.replace("t=", "")) // 1000000 + if expiration.startswith("t=") + else int(expiration) + ) + @pages_blueprint.route("/lgbk//", methods=["GET"]) @experiment_exists @@ -159,16 +253,36 @@ def exp_elog_legacy(experiment_name): # The hash never comes to the server; it's entirely a client side thing. return make_response(redirect("./info")) + @pages_blueprint.route("/lgbk//", methods=["GET"]) @experiment_exists @context.security.authentication_required @context.security.authorization_required("read") def exp_elog(experiment_name, tabname): - logged_in_user=context.security.get_current_user_id() + logged_in_user = context.security.get_current_user_id() exp_info = get_experiment_info(experiment_name) instrument_name = exp_info.get("instrument", None) if exp_info else None - privileges = { x : context.security.check_privilege_for_experiment(x, experiment_name, instrument_name) for x in [ "manage_groups", "delete", "edit", "post", "read", "experiment_create", "experiment_edit", "feedback_read", "feedback_write", "instrument_create", "ops_page", "switch" ]} - return render_template("lgbk.html", + privileges = { + x: context.security.check_privilege_for_experiment( + x, experiment_name, instrument_name + ) + for x in [ + "manage_groups", + "delete", + "edit", + "post", + "read", + "experiment_create", + "experiment_edit", + "feedback_read", + "feedback_write", + "instrument_create", + "ops_page", + "switch", + ] + } + return render_template( + "lgbk.html", experiment_name=experiment_name, instrument_name=instrument_name, tabname=tabname, @@ -178,18 +292,20 @@ def exp_elog(experiment_name, tabname): privileges=json.dumps(privileges), current_sample_name=get_current_sample_name(experiment_name), auth_expiration_time=__parse_expiration_header__(request), - logbook_site=context.LOGBOOK_SITE - ) + logbook_site=context.LOGBOOK_SITE, + ) + @pages_blueprint.route("/lgbk//elogs/", methods=["GET"]) @experiment_exists @context.security.authentication_required @context.security.authorization_required("read") def exp_elog_entry_only(experiment_name, entry_id): - logged_in_user=context.security.get_current_user_id() + logged_in_user = context.security.get_current_user_id() exp_info = get_experiment_info(experiment_name) instrument_name = exp_info.get("instrument", None) if exp_info else None - return render_template("elog_entry.html", + return render_template( + "elog_entry.html", experiment_name=experiment_name, instrument_name=instrument_name, pagepath=request.path, diff --git a/context.py b/explgbk/context.py similarity index 58% rename from context.py rename to explgbk/context.py index f177ffc..91a47cb 100755 --- a/context.py +++ b/explgbk/context.py @@ -10,35 +10,34 @@ from flask_authnz import FlaskAuthnz, MongoDBRoles, UserGroups from kafka import KafkaProducer -from kafka.errors import KafkaError -from dal.utils import JSONEncoder +from explgbk.dal.utils import JSONEncoder import jwt logger = logging.getLogger(__name__) -__author__ = 'mshankar@slac.stanford.edu' +__author__ = "mshankar@slac.stanford.edu" # Application context. app = None -MONGODB_HOST=os.environ.get('MONGODB_HOST', "localhost") -MONGODB_PORT=int(os.environ.get('MONGODB_PORT', 27017)) -MONGODB_HOSTS=os.environ.get("MONGODB_HOSTS", None) +MONGODB_HOST = os.environ.get("MONGODB_HOST", "localhost") +MONGODB_PORT = int(os.environ.get("MONGODB_PORT", 27017)) +MONGODB_HOSTS = os.environ.get("MONGODB_HOSTS", None) if not MONGODB_HOSTS: MONGODB_HOSTS = MONGODB_HOST + ":" + str(MONGODB_PORT) -MONGODB_URL=os.environ.get("MONGODB_URL", None) +MONGODB_URL = os.environ.get("MONGODB_URL", None) if not MONGODB_URL: MONGODB_URL = "mongodb://" + MONGODB_HOSTS + "/admin" -MONGODB_USERNAME=os.environ['MONGODB_USERNAME'] -MONGODB_PASSWORD=os.environ['MONGODB_PASSWORD'] +MONGODB_USERNAME = os.environ["MONGODB_USERNAME"] +MONGODB_PASSWORD = os.environ["MONGODB_PASSWORD"] # This identifies the current deployment site. # Functionality that depends on the deployment location is based off this variable. # For example, use LCLS for LCLS, Cryo for Cryo. -LOGBOOK_SITE = os.environ.get('LOGBOOK_SITE', 'test') +LOGBOOK_SITE = os.environ.get("LOGBOOK_SITE", "test") # Use this information to get proposal information from the questionnaire. # This is typically a ws-auth endpoint @@ -48,27 +47,46 @@ # Support for serving previews from the web server. Previews can get quite large and having python serve them is sometimes not practical. # Add run parms using ws/ext_preview. This preview_prefix will then be prepended to the path to serve the image. # A hash is added as part of the URL hashed with the PREVIEW_PREFIX_SHARED_SECRET -PREVIEW_PREFIX = os.environ.get('PREVIEW_PREFIX', '../../..') -PREVIEW_PREFIX_SHARED_SECRET = os.environ.get('PREVIEW_PREFIX_SHARED_SECRET', "SLACExpLgBk") +PREVIEW_PREFIX = os.environ.get("PREVIEW_PREFIX", "../../..") +PREVIEW_PREFIX_SHARED_SECRET = os.environ.get( + "PREVIEW_PREFIX_SHARED_SECRET", "SLACExpLgBk" +) # Set up the security manager -mongorolereaderclient = MongoClient(host=MONGODB_URL, username=MONGODB_USERNAME, password=MONGODB_PASSWORD, tz_aware=True, read_preference=ReadPreference.SECONDARY_PREFERRED) +mongorolereaderclient = MongoClient( + host=MONGODB_URL, + username=MONGODB_USERNAME, + password=MONGODB_PASSWORD, + tz_aware=True, + read_preference=ReadPreference.SECONDARY_PREFERRED, +) usergroups = UserGroups() roleslookup = MongoDBRoles(mongorolereaderclient, usergroups) security = FlaskAuthnz(roleslookup, "LogBook") -logbookclient = MongoClient(host=MONGODB_URL, username=MONGODB_USERNAME, password=MONGODB_PASSWORD, tz_aware=True, read_preference=ReadPreference.PRIMARY_PREFERRED) +logbookclient = MongoClient( + host=MONGODB_URL, + username=MONGODB_USERNAME, + password=MONGODB_PASSWORD, + tz_aware=True, + read_preference=ReadPreference.PRIMARY_PREFERRED, +) local_kafka_events = Queue() + class MyKafkaProducer(KafkaProducer): def __init__(self, *args, **kwargs): super(MyKafkaProducer, self).__init__(*args, **kwargs) + def send(self, topic, value, *args, **kwargs): - local_kafka_events.put({"topic": topic, "value": JSONEncoder().encode(value).encode('utf-8')}) + local_kafka_events.put( + {"topic": topic, "value": JSONEncoder().encode(value).encode("utf-8")} + ) super(MyKafkaProducer, self).send(topic, value, *args, **kwargs) + def __getKafkaProducer(): misc_params = {} if os.environ.get("SKIP_KAFKA_CONNECTION", False): @@ -76,7 +94,14 @@ def __getKafkaProducer(): else: # if LOGBOOK_SITE=="CryoEM": # misc_params["acks"] = 0 - return MyKafkaProducer(bootstrap_servers=os.environ.get("KAFKA_BOOTSTRAP_SERVER", "localhost:9092").split(","), value_serializer=lambda m: JSONEncoder().encode(m).encode('utf-8'), **misc_params) + return MyKafkaProducer( + bootstrap_servers=os.environ.get( + "KAFKA_BOOTSTRAP_SERVER", "localhost:9092" + ).split(","), + value_serializer=lambda m: JSONEncoder().encode(m).encode("utf-8"), + **misc_params, + ) + kafka_producer = __getKafkaProducer() @@ -92,36 +117,62 @@ def __getKafkaProducer(): instrument_scientists_run_table_defintions = {} run_table_secions_json = os.environ.get("RUNTABLE_SECTIONS_JSON", None) + def load_sections_json(): if run_table_secions_json and os.path.exists(run_table_secions_json): - logger.info("Loading run table instrument scientist descriptions from %s", run_table_secions_json) + logger.info( + "Loading run table instrument scientist descriptions from %s", + run_table_secions_json, + ) new_instrument_scientists_run_table_defintions = {} + def reverse_mapping_for_section(section): - return { x["name"]: {"section" : section["SECTION"], "title": section["TITLE"], "pv": x["name"]} for x in section["PARAMS"] } - with open(run_table_secions_json, 'r') as f: + return { + x["name"]: { + "section": section["SECTION"], + "title": section["TITLE"], + "pv": x["name"], + } + for x in section["PARAMS"] + } + + with open(run_table_secions_json, "r") as f: isdefs = json.load(f) for instrument, sections in isdefs.items(): new_instrument_scientists_run_table_defintions[instrument] = {} for section in sections: - new_instrument_scientists_run_table_defintions[instrument].update(reverse_mapping_for_section(section)) + new_instrument_scientists_run_table_defintions[instrument].update( + reverse_mapping_for_section(section) + ) global instrument_scientists_run_table_defintions instrument_scientists_run_table_defintions.clear() - instrument_scientists_run_table_defintions.update(new_instrument_scientists_run_table_defintions) + instrument_scientists_run_table_defintions.update( + new_instrument_scientists_run_table_defintions + ) + load_sections_json() # Cache some of the instrument definitions. instrument_definitions = {} + + def load_instrument_definitions(): global instrument_definitions instrument_definitions.clear() - instrument_definitions.update({ x["_id"]: x for x in logbookclient["site"]["instruments"].find() }) + instrument_definitions.update( + {x["_id"]: x for x in logbookclient["site"]["instruments"].find()} + ) + load_instrument_definitions() + def reload_named_caches(cache_name): if cache_name == "instrument_scientists_run_table_defintions": - logger.info("Reloading the instrument_scientists_run_table_defintions named cache") + logger.info( + "Reloading the instrument_scientists_run_table_defintions named cache" + ) load_sections_json() elif cache_name == "instrument_defintions": logger.info("Reloading the instrument_defintions named cache") @@ -130,7 +181,20 @@ def reload_named_caches(cache_name): def generateArpToken(userid, experiment_name, token_duration_in_mins=10): if "WFLOW_TRIG_ARP_PRIVATE_KEY" not in os.environ: - raise Exception("Please specify the ARP private key in the environment variable WFLOW_TRIG_ARP_PRIVATE_KEY") + raise Exception( + "Please specify the ARP private key in the environment variable WFLOW_TRIG_ARP_PRIVATE_KEY" + ) with open(os.environ["WFLOW_TRIG_ARP_PRIVATE_KEY"], "rb") as f: private_key = f.read() - return jwt.encode({"user": userid, "experiment_name": experiment_name, "expires": (datetime.datetime.now(tz=datetime.timezone.utc) + datetime.timedelta(minutes=token_duration_in_mins)).timestamp()}, private_key, algorithm="RS256") + return jwt.encode( + { + "user": userid, + "experiment_name": experiment_name, + "expires": ( + datetime.datetime.now(tz=datetime.timezone.utc) + + datetime.timedelta(minutes=token_duration_in_mins) + ).timestamp(), + }, + private_key, + algorithm="RS256", + ) diff --git a/explgbk/dal/__init__.py b/explgbk/dal/__init__.py new file mode 100755 index 0000000..077fffc --- /dev/null +++ b/explgbk/dal/__init__.py @@ -0,0 +1 @@ +__author__ = "mshankar@slac.stanford.edu" diff --git a/explgbk/dal/exp_cache.py b/explgbk/dal/exp_cache.py new file mode 100644 index 0000000..f9268c1 --- /dev/null +++ b/explgbk/dal/exp_cache.py @@ -0,0 +1,1029 @@ +import os +import json +import datetime +import pytz +import dateutil.relativedelta +import logging +import re + +import threading +import sched + +from pymongo import DESCENDING + +from kafka import KafkaConsumer + +from explgbk.context import ( + logbookclient, + usergroups, + kafka_producer, + local_kafka_events, + reload_named_caches, +) +from explgbk.dal.explgbk import ( + get_experiments_for_instrument, + get_poc_feedback_changes, + get_poc_feedback_document, +) + +__author__ = "mshankar@slac.stanford.edu" + +logger = logging.getLogger(__name__) + +all_experiment_names = set() +roles_with_post_privileges = [] + + +class PeriodicUpdates: + """ + Gather experiment names to be updated periodically in the future. + """ + + def __init__(self): + self.lock = threading.Lock() + self.experiment_names = set() + + def add(self, experiment_name): + with self.lock: + try: + self.experiment_names.add(experiment_name) + except Exception: + logger.exception( + "Exception adding %s to periodic updater", experiment_name + ) + + def getAndReset(self): + with self.lock: + ret = self.experiment_names + self.experiment_names = set() + return ret + + +periodic_updates = PeriodicUpdates() + + +def init_app(app): + if "experiments" not in list( + logbookclient["explgbk_cache"].list_collection_names() + ): + logbookclient["explgbk_cache"]["experiments"].create_index( + [ + ("name", "text"), + ("description", "text"), + ("instrument", "text"), + ("contact_info", "text"), + ("params.PNR", "text"), + ] + ) + if "operations" not in list(logbookclient["explgbk_cache"].list_collection_names()): + logbookclient["explgbk_cache"]["operations"].create_index( + [("name", DESCENDING)], unique=True + ) + logbookclient["explgbk_cache"]["operations"].insert_one( + { + "name": "explgbk_cache_rebuild", + "initiated": datetime.datetime.utcfromtimestamp(0.0), + "completed": datetime.datetime.utcfromtimestamp(0.0), + } + ) + global roles_with_post_privileges + roles_with_post_privileges = [ + x["name"] + for x in logbookclient["site"]["roles"].find( + {"app": "LogBook", "privileges": {"$in": ["post"]}}, {"name": 1, "_id": 0} + ) + ] + __load_experiment_names() + + scheduler = sched.scheduler() + __establish_kafka_consumers() + __establish_local_kafka_consumers__() + + def __periodic(scheduler, interval, action, actionargs=()): + # This is the function that runs periodically + scheduler.enter( + interval, 1, __periodic, (scheduler, interval, action, actionargs) + ) + last_rebuild = logbookclient["explgbk_cache"]["operations"].find_one( + {"name": "explgbk_cache_rebuild"} + ) + db_time_utc = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) + if (db_time_utc - last_rebuild["initiated"]).total_seconds() > interval: + action(*actionargs) + else: + logger.info( + "Skipping the periodic cache rebuild as we rebuilt the cache at %s within the specified interval %s", + last_rebuild["initiated"], + interval, + ) + + def __kickoff_cache_update_thread(): + # This runs in a background thread; the scheduler.run is blocking and will block forever. + __periodic(scheduler, 24 * 60 * 60, __update_experiments_info) + scheduler.run() + + __cache_update_thread = threading.Thread(target=__kickoff_cache_update_thread) + __cache_update_thread.start() + + def _non_immediate_update(): + try: + global periodic_updates + exps = periodic_updates.getAndReset() + logger.info( + "Processing the non immediate cache for %s experiments", len(exps) + ) + for exp in exps: + try: + update_single_experiment_info(exp) + except Exception: + logger.exception("Exception in periodic updater updating %s", exp) + except Exception: + logger.exception("Exception in periodic updater") + + def __non_immediate_periodic(scheduler, interval, action, actionargs=()): + scheduler.enter( + interval, + 1, + __non_immediate_periodic, + (scheduler, interval, action, actionargs), + ) + action(*actionargs) + + nonimmediatesched = sched.scheduler() + + def __kickoff_non_immediate_thread(): + __non_immediate_periodic(nonimmediatesched, 5 * 60, _non_immediate_update) + nonimmediatesched.run() + + __non_immediate_updater_thread = threading.Thread( + target=__kickoff_non_immediate_thread + ) + __non_immediate_updater_thread.start() + + +def reload_cache(experiment_name=None): + """ + Reload the experiment cache from the database. + Use only if you make changes directly in the database bypassing the app. + If you are using to recover from invalid cache issues; please do generate a bug report. + """ + if experiment_name: + logger.debug("Refreshing the cache for experiment %s", experiment_name) + if experiment_name in ["admin", "config", "local", "site"]: + return + update_single_experiment_info(experiment_name) + return + + __update_experiments_info() + + +def get_experiments(): + """ + Get a list of experiments from the database. + Returns basic information and also some info on the first and last runs. + """ + return list(logbookclient["explgbk_cache"]["experiments"].find({})) + + +def get_cached_experiment_info(experiment_id): + """ + Returns basic information and also some info on the first and last runs. + """ + return logbookclient["explgbk_cache"]["experiments"].find_one( + {"_id": experiment_id} + ) + + +def get_experiments_starting_in_time_frame(start_time, end_time): + """ + Get a list of experiments whose start_time is in the given time range. + """ + return list( + logbookclient["explgbk_cache"]["experiments"].find( + { + "$and": [ + {"start_time": {"$gte": start_time}}, + {"start_time": {"$lte": end_time}}, + ] + }, + {"name": 1, "start_time": 1}, + ) + ) + + +def get_sorted_experiments_ids(sort_criteria): + """ + Get only the experiment ids sorted according to the sort criteria. + Sort criteria is a JSON array of [[sort_attr, sort_direction]] + """ + return [ + x["_id"] + for x in logbookclient["explgbk_cache"]["experiments"] + .find({}, {"_id": 1, "name": 1}) + .sort(sort_criteria) + ] + + +def get_experiments_for_user(uid): + """ + Get a list of experiments for which the user has read access. + """ + sitedb = logbookclient["site"] + groups = usergroups.get_user_posix_groups(uid) + groups.append("uid:" + uid) + # See if the user has any global read privileges + global_read_roles_for_user = [ + x + for x in sitedb["roles"].find( + {"players": {"$in": groups}, "privileges": "read"} + ) + ] + if global_read_roles_for_user: + logger.debug( + "User %s has read privileges for all experiments from the site database", + uid, + ) + return get_experiments() + global_read_roles = set( + [ + (x["app"], x["name"]) + for x in sitedb["roles"].find( + {"privileges": "read"}, {"_id": 0, "app": 1, "name": 1} + ) + ] + ) + # Check for instrument level privileges + instrument_roles_for_user = [ + x + for x in sitedb["instruments"].aggregate( + [{"$match": {"roles.players": {"$in": groups}}}, {"$unwind": "$roles"}] + ) + ] + exp_for_uid = {} + for irole in instrument_roles_for_user: + if (irole["roles"]["app"], irole["roles"]["name"]) in global_read_roles: + logger.debug( + "User %s has read permission for instrument %s because of role %s/%s", + uid, + irole["_id"], + irole["roles"]["app"], + irole["roles"]["name"], + ) + for exp in get_experiments_for_instrument(irole["_id"]): + exp_for_uid[exp["_id"]] = exp + # Now for experiments for which the user is directly a collaborator + for exp in list( + logbookclient["explgbk_cache"]["experiments"].find({"players": {"$in": groups}}) + ): + exp_for_uid[exp["_id"]] = exp + return list(exp_for_uid.values()) + + +def get_direct_experiments_for_user(uid): + """ + Get a list of experiments for which the user is a direct collaborator. + This information typically comes in from the URAWI BTR. + """ + return list( + logbookclient["explgbk_cache"]["experiments"].find( + {"players": {"$in": ["uid:" + uid]}}, {"name": 1, "instrument": 1} + ) + ) + + +def get_cached_experiment_names(): + """ + Get the cached experiment names. Use for debugging. + """ + global all_experiment_names + return list(all_experiment_names) + + +def get_experiments_with_post_privileges(userid, active_exps): + """ + Get the list of experiments that the logged in user has post privileges for. + If the logged in user (or one of her groups) is in the site database, we return all experiments. + Else we query the experiment cache and return those. + """ + groups = usergroups.get_user_posix_groups(userid) + u_a_g = ["uid:" + userid] + groups + logger.debug("Looking for experiments with post privileges for %s", u_a_g) + site_roles = [ + x + for x in logbookclient["site"]["roles"].find( + { + "app": "LogBook", + "privileges": {"$in": ["post"]}, + "players": {"$in": u_a_g}, + } + ) + ] + if site_roles: + logger.debug("User %s has post privileges for all experiments") + postable_exps = get_experiments() + else: + postable_exps = list( + logbookclient["explgbk_cache"]["experiments"].find( + {"post_players": {"$in": u_a_g}} + ) + ) + # Sort + ret_exps = [ + { + attr: x.get(attr, None) + for attr in [ + "_id", + "name", + "instrument", + "description", + "start_time", + "end_time", + "posix_group", + "params", + ] + } + for x in postable_exps + ] + active_exp_names = [x["name"] for x in active_exps if "name" in x] + for exp in ret_exps: + if exp["name"] in active_exp_names: + exp["is_active"] = True + return ret_exps + + +def get_experiment_stats(): + """ + Get various computed/cached stats for the experiments + """ + return list(logbookclient["explgbk_cache"]["experiment_stats"].find({})) + + +def get_experiment_daily_data_breakdown(report_type, instrument): + """ + Run an aggregate on the daily data breakdown. + Data returned is in TB. + """ + if report_type == "file_sizes": + if not instrument or instrument == "ALL": + return list( + logbookclient["explgbk_cache"]["experiment_stats"].aggregate( + [ + {"$unwind": "$dataDailyBreakdown"}, + {"$replaceRoot": {"newRoot": "$dataDailyBreakdown"}}, + { + "$group": { + "_id": "$_id", + "total_size": { + "$sum": {"$divide": ["$total_size", 1024]} + }, + } + }, + {"$sort": {"_id": -1}}, + ] + ) + ) + else: + return list( + logbookclient["explgbk_cache"]["experiment_stats"].aggregate( + [ + { + "$lookup": { + "from": "experiments", + "localField": "_id", + "foreignField": "_id", + "as": "exp", + } + }, + {"$unwind": "$exp"}, + {"$match": {"exp.instrument": instrument}}, + {"$unwind": "$dataDailyBreakdown"}, + {"$replaceRoot": {"newRoot": "$dataDailyBreakdown"}}, + { + "$group": { + "_id": "$_id", + "total_size": { + "$sum": {"$divide": ["$total_size", 1024]} + }, + } + }, + {"$sort": {"_id": -1}}, + ] + ) + ) + elif report_type == "run_counts": + if not instrument or instrument == "ALL": + return list( + logbookclient["explgbk_cache"]["experiment_stats"].aggregate( + [ + {"$unwind": "$runDailyBreakdown"}, + {"$replaceRoot": {"newRoot": "$runDailyBreakdown"}}, + { + "$group": { + "_id": "$_id", + "total_runs": {"$sum": "$run_count"}, + } + }, + {"$sort": {"_id": -1}}, + ] + ) + ) + else: + return list( + logbookclient["explgbk_cache"]["experiment_stats"].aggregate( + [ + { + "$lookup": { + "from": "experiments", + "localField": "_id", + "foreignField": "_id", + "as": "exp", + } + }, + {"$unwind": "$exp"}, + {"$match": {"exp.instrument": instrument}}, + {"$unwind": "$runDailyBreakdown"}, + {"$replaceRoot": {"newRoot": "$runDailyBreakdown"}}, + { + "$group": { + "_id": "$_id", + "total_runs": {"$sum": "$run_count"}, + } + }, + {"$sort": {"_id": -1}}, + ] + ) + ) + + +def does_experiment_exist(experiment_name): + """ + Checks for the existence of the experiment_name. + This is meant mostly for validation of the experiment_name; we assume that this is going to be called many times. + So, we are avoiding a hit to the database by caching just the names themselves in memory. + """ + global all_experiment_names + if experiment_name in all_experiment_names: # Check the cache first. + return True + expdb = logbookclient[experiment_name] + collnames = list(expdb.list_collection_names()) + if "info" in collnames: + return True + + return False + + +def text_search_for_experiments(search_terms): + """ + Search the experiment cache for experiments matching the search terms. + Use search terms separated by spaces. The backslash escapes the space for literal searches. + Use the minus character to suppress a word. + """ + matching_entries = list( + logbookclient["explgbk_cache"]["experiments"].find( + {"$text": {"$search": search_terms}} + ) + ) + return sorted(matching_entries, key=lambda x: x["name"]) + + +def search_experiments_for_common_fields(search_term, sort_criteria): + """ + Search the experiment cache for experiments regex matching the search term in a subset of fields. + The fields searched are _id, name, contact_info, description + """ + patt = re.compile(search_term) + matching_entries = list( + logbookclient["explgbk_cache"]["experiments"] + .find( + { + "$or": [ + {"_id": {"$regex": patt}}, + {"name": {"$regex": patt}}, + {"contact_info": {"$regex": patt}}, + {"description": {"$regex": patt}}, + ] + }, + {"_id": 1, "name": 1}, + ) + .sort(sort_criteria) + ) + return matching_entries + + +def get_recently_updated_experiments(offset_secs): + """ + Return a list of experiment names which have run.last_run.begin_time gte specified offset + """ + offset_time = datetime.datetime.now() - datetime.timedelta(seconds=offset_secs) + return [ + x["name"] + for x in list( + logbookclient["explgbk_cache"]["experiments"].find( + {"last_run.begin_time": {"$gte": offset_time}}, {"name": 1} + ) + ) + ] + + +def get_all_param_names_matching_regex(rgx): + """ + Get all param names in all experiments matching the incoming regex. + """ + patt = re.compile(rgx) + apns = logbookclient["explgbk_cache"]["experiments"].distinct("all_param_names") + return [x for x in apns if patt.match(x)] + + +def get_experiments_proposal_mappings(): + """ + Get all the experiments with their PNR's if present. + """ + return list( + logbookclient["explgbk_cache"]["experiments"].find( + {}, {"name": 1, "params.PNR": 1, "instrument": 1} + ) + ) + + +def get_potentially_active_users(cutoff_date): + """ + Returns the set of users in experiment whose end date is after the specified date. + """ + ret = list( + logbookclient["explgbk_cache"]["experiments"].aggregate( + [ + {"$match": {"end_time": {"$gte": cutoff_date}}}, + {"$group": {"_id": 0, "players": {"$push": "$players"}}}, + {"$project": {"_id": 0, "players": 1}}, + { + "$project": { + "allusers": { + "$reduce": { + "input": "$players", + "initialValue": [], + "in": {"$concatArrays": ["$$value", "$$this"]}, + } + } + } + }, + {"$unwind": "$allusers"}, + {"$project": {"userid": "$allusers"}}, + {"$match": {"userid": {"$regex": "^uid:.*"}}}, + {"$group": {"_id": "$userid"}}, + { + "$project": { + "_id": 0, + "userid": { + "$replaceOne": { + "input": "$_id", + "find": "uid:", + "replacement": "", + } + }, + } + }, + {"$sort": {"userid": 1}}, + ] + ) + ) + return [x["userid"] for x in ret] + + +def __load_experiment_names(): + """We cache the list of experimemt names to speedup authz/other operations. + This reloads the cached list of experiment names from the explgbk_cache + """ + global all_experiment_names + all_experiment_names = set( + [ + x["name"] + for x in logbookclient["explgbk_cache"]["experiments"].find( + {}, {"name": 1, "_id": 0} + ) + ] + ) + + +def __update_experiments_info(): + """ + Since we are using an database per experiment, getting basic information that spans experiments can take some time. + We cache this information in a 'explgbk_cache' database. + We update this using Kafka; but we also periodically do a full reload of this information + """ + logger.info("Updating the experiment info cached in 'explgbk_cache'.") + database_names = sorted(list(logbookclient.list_database_names()), reverse=True) + db_time_utc = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) + logbookclient["explgbk_cache"]["operations"].update_one( + {"name": "explgbk_cache_rebuild"}, {"$set": {"initiated": db_time_utc}} + ) + for experiment_name in database_names: + if experiment_name in ["admin", "config", "local", "site"]: + continue + update_single_experiment_info(experiment_name) + db_time_utc = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) + logbookclient["explgbk_cache"]["operations"].update_one( + {"name": "explgbk_cache_rebuild"}, {"$set": {"completed": db_time_utc}} + ) + kafka_producer.send("explgbk_cache", {"cache_rebuild": True}) + + +def update_single_experiment_info(experiment_name, crud="Update"): + """ + Load a single experiment's info and return the info as a dict + """ + global all_experiment_names + if crud == "Delete": + all_experiment_names.remove(experiment_name) + logbookclient["explgbk_cache"]["experiments"].delete_one( + {"_id": experiment_name} + ) + logbookclient["explgbk_cache"]["experiment_stats"].delete_one( + {"_id": experiment_name} + ) + return + logger.debug( + "Gathering the experiment info cached in 'explgbk_cache' for experiment %s", + experiment_name, + ) + expdb = logbookclient[experiment_name] + collnames = list(expdb.list_collection_names()) + if "info" in collnames: + info = expdb["info"].find_one({}, {"latest_setup": 0}) + if "name" not in info or "instrument" not in info: + logger.error( + "Database %s has a info collection but the info object does not have an instrument or name. Note this could also be a timing issue if you are using secondaries", + experiment_name, + ) + return + all_experiment_names.add(experiment_name) + expinfo = {"_id": experiment_name} + roles = [x for x in expdb["roles"].find()] + all_players = set() + list(map(lambda x: all_players.update(x.get("players", [])), roles)) + expinfo["players"] = list(all_players) + post_players = set() + list( + map( + lambda x: post_players.update(x.get("players", [])), + expdb["roles"].find( + {"name": {"$in": roles_with_post_privileges}}, + {"_id": 0, "players": 1}, + ), + ) + ) + expinfo["post_players"] = list(post_players) + if "runs" in collnames: + run_count = expdb["runs"].count_documents({}) + expinfo["run_count"] = run_count + if run_count: + last_run = ( + expdb["runs"] + .find({}, {"num": 1, "begin_time": 1, "end_time": 1}) + .sort([("begin_time", -1)]) + .limit(1)[0] + ) + first_run = ( + expdb["runs"] + .find({}, {"num": 1, "begin_time": 1, "end_time": 1}) + .sort([("begin_time", 1)]) + .limit(1)[0] + ) + expinfo["first_run"] = { + "num": first_run["num"], + "begin_time": first_run["begin_time"], + "end_time": first_run["end_time"], + } + expinfo["last_run"] = { + "num": last_run["num"], + "begin_time": last_run["begin_time"], + "end_time": last_run["end_time"], + } + runDailyBreakdown = list( + expdb["runs"].aggregate( + [ + { + "$group": { + "_id": { + "$dateToParts": { + "date": { + "$convert": { + "input": "$begin_time", + "to": "date", + } + } + } + }, + "run_count": {"$sum": 1}, + } + }, + { + "$project": { + "_id.year": 1, + "_id.month": 1, + "_id.day": 1, + "run_count": 1, + } + }, + { + "$group": { + "_id": { + "$dateFromParts": { + "year": "$_id.year", + "month": "$_id.month", + "day": "$_id.day", + } + }, + "run_count": {"$sum": "$run_count"}, + } + }, + {"$sort": {"_id": 1}}, + ] + ) + ) + if runDailyBreakdown: + logbookclient["explgbk_cache"]["experiment_stats"].update_one( + {"_id": experiment_name}, + {"$set": {"runDailyBreakdown": runDailyBreakdown}}, + upsert=True, + ) + + expinfo["all_param_names"] = [ + x["_id"] + for x in expdb["runs"].aggregate( + [ + {"$project": {"params": {"$objectToArray": "$params"}}}, + {"$unwind": "$params"}, + {"$group": {"_id": "$params.k", "total": {"$sum": 1}}}, + {"$sort": {"_id": 1}}, + ] + ) + ] + else: + logger.debug("No runs in experiment " + experiment_name) + try: + if "file_catalog" in collnames: + + def asTime(val): + if not val: + return None + elif isinstance(val[0]["create_timestamp"], datetime.datetime): + return val[0]["create_timestamp"] + elif isinstance(val[0]["create_timestamp"], str): + return datetime.datetime.strptime( + val[0]["create_timestamp"], "%Y-%m-%dT%H:%M:%SZ" + ) + else: + return None + + f_file = asTime( + list( + expdb["file_catalog"] + .find({}) + .sort([("create_timestamp", -1)]) + .limit(1) + ) + ) + l_file = asTime( + list( + expdb["file_catalog"] + .find({}) + .sort([("create_timestamp", 1)]) + .limit(1) + ) + ) + if f_file and l_file: + attrs = ["years", "months", "days", "hours", "minutes"] + + def human_readable_diff(delta): + return [ + "%d %s" + % ( + getattr(delta, attr), + getattr(delta, attr) > 1 and attr or attr[:-1], + ) + for attr in attrs + if getattr(delta, attr) + ] + + expinfo["file_timestamps"] = { + "first_file_ts": f_file, + "last_file_ts": l_file, + "duration": (l_file - f_file).total_seconds(), + "hr_duration": human_readable_diff( + dateutil.relativedelta.relativedelta(f_file, l_file) + ), + } + + dataSummary = [ + x + for x in expdb["file_catalog"].aggregate( + [ + { + "$group": { + "_id": None, + "totalDataSize": { + "$sum": { + "$divide": [ + "$size", + 1024 * 1024 * 1024 * 1.0, + ] + } + }, + "totalFiles": {"$sum": 1}, + } + } + ] + ) + ] + if dataSummary: + expinfo["totalDataSize"] = dataSummary[0]["totalDataSize"] + expinfo["totalFiles"] = dataSummary[0]["totalFiles"] + dataDailyBreakdown = [ + x + for x in expdb["file_catalog"].aggregate( + [ + { + "$group": { + "_id": { + "$dateToParts": { + "date": { + "$convert": { + "input": "$create_timestamp", + "to": "date", + } + } + } + }, + "total_size": {"$sum": "$size"}, + } + }, + { + "$project": { + "_id.year": 1, + "_id.month": 1, + "_id.day": 1, + "total_size": 1, + } + }, + { + "$group": { + "_id": { + "$dateFromParts": { + "year": "$_id.year", + "month": "$_id.month", + "day": "$_id.day", + } + }, + "total_size": { + "$sum": { + "$divide": [ + "$total_size", + 1024 * 1024 * 1024, + ] + } + }, + } + }, + ], + allowDiskUse=True, + ) + ] + if dataDailyBreakdown: + logbookclient["explgbk_cache"]["experiment_stats"].update_one( + {"_id": experiment_name}, + {"$set": {"dataDailyBreakdown": dataDailyBreakdown}}, + upsert=True, + ) + except Exception: + logger.exception("Exception computing the file parameters") + + poc_feedback_changes = get_poc_feedback_changes(experiment_name) + if poc_feedback_changes: + poc_feedback_doc = get_poc_feedback_document(experiment_name) + expinfo["poc_feedback"] = { + "num_items": len(poc_feedback_changes), + "num_items_4_5": len( + list( + filter( + lambda kv: ( + kv[0] not in ["basic-scheduled", "basic-actual"] + and (kv[1] in ["4", "5"]) + ), + poc_feedback_doc.items(), + ) + ) + ), + "last_modified_by": poc_feedback_changes[-1]["modified_by"], + "last_modified_at": poc_feedback_changes[-1]["modified_at"], + } + + expinfo.update(info) + expinfo["_id"] = experiment_name + logbookclient["explgbk_cache"]["experiments"].replace_one( + {"_id": experiment_name}, expinfo, upsert=True + ) + + logger.info( + "Updated the experiment info cached in 'explgbk_cache' for experiment %s", + experiment_name, + ) + else: + logger.error( + "Database %s does not have a info collection. Note this could also be a timing issue if you are using secondaries", + experiment_name, + ) + + +def __establish_local_kafka_consumers__(): + """ + This processes from the local queue + """ + + def processMessage(msg): + try: + logger.debug("Kafka/local Message %s", msg) + info = json.loads(msg["value"]) + logger.debug("Kafka/local JSON %s", info) + message_type = msg["topic"] + if message_type == "explgbk_cache": + if info.get("named_cache", None): + logger.info("Reloading named cache %s", info["named_cache"]) + reload_named_caches(info["named_cache"]) + __load_experiment_names() + elif message_type in ["experiments", "roles", "samples"]: + if "experiment_name" in info: + experiment_name = info["experiment_name"] + logger.info( + "Got a Kafka/local message %s for experiment %s - building the cache entry", + message_type, + experiment_name, + ) + crud = ( + info.get("CRUD", "Update") + if message_type == "experiments" + else "Update" + ) + update_single_experiment_info(experiment_name, crud=crud) + else: + logger.error( + "Kafka/local message in immediate topics without an experiment name %s", + message_type, + ) + else: + logger.debug( + "Not re-building immediately for a non immediate topic %s", + message_type, + ) + global periodic_updates + if "experiment_name" in info: + experiment_name = info["experiment_name"] + periodic_updates.add(experiment_name) + else: + logger.error( + "Kafka/local message in non-immediate topics without an experiment name %s", + message_type, + ) + except Exception: + logger.exception("Exception processing Kafka/local message.") + + def worker(): + while True: + msg = local_kafka_events.get() + processMessage(msg) + local_kafka_events.task_done() + + local_msg_thread = threading.Thread(target=worker) + local_msg_thread.start() + + +def __establish_kafka_consumers(): + """ + Establish Kafka consumers that listen to new experiments and runs and updates the cache. + """ + + def subscribe_kafka(): + consumer = KafkaConsumer( + bootstrap_servers=os.environ.get( + "KAFKA_BOOTSTRAP_SERVER", "localhost:9092" + ).split(",") + ) + consumer.subscribe(["experiments", "explgbk_cache"]) + + for msg in consumer: + try: + logger.info("Message from Kafka in topic %s", msg.topic) + message_type = msg.topic + if message_type == "explgbk_cache": + info = json.loads(msg.value) + logger.debug("JSON from Kafka %s", info) + if info.get("named_cache", None): + logger.info("Reloading named cache %s", info["named_cache"]) + reload_named_caches(info["named_cache"]) + + __load_experiment_names() + except Exception: + logger.exception("Exception processing Kafka message.") + + # Create thread for kafka consumer + kafka_client_thread = threading.Thread(target=subscribe_kafka) + kafka_client_thread.start() diff --git a/dal/explgbk.py b/explgbk/dal/explgbk.py similarity index 51% rename from dal/explgbk.py rename to explgbk/dal/explgbk.py index 639e78b..ebea45f 100755 --- a/dal/explgbk.py +++ b/explgbk/dal/explgbk.py @@ -1,7 +1,8 @@ -''' +""" The model level business logic goes here. Most of the code here gets a connection to the database, executes a query and formats the results. -''' +""" + import os import json import datetime @@ -21,26 +22,36 @@ from bson import ObjectId # This (g) should be the only import from Flask; we use this as a thread local context variable. -from flask import g -from context import logbookclient, instrument_scientists_run_table_defintions, security, usergroups, imagestoreurl, \ - MAX_ATTACHMENT_SIZE, QUESTIONNAIRE_URL, LOGBOOK_SITE -from dal.run_control import get_current_run, start_run, end_run, is_run_closed -from dal.imagestores import parseImageStoreURL -from dal.utils import escape_chars_for_mongo, reverse_escape_chars_for_mongo +from explgbk.context import ( + logbookclient, + instrument_scientists_run_table_defintions, + security, + usergroups, + imagestoreurl, + MAX_ATTACHMENT_SIZE, + QUESTIONNAIRE_URL, + LOGBOOK_SITE, +) +from explgbk.dal.run_control import get_current_run, start_run, end_run, is_run_closed +from explgbk.dal.imagestores import parseImageStoreURL +from explgbk.dal.utils import reverse_escape_chars_for_mongo -PROJECTS_DB="lgbkprjs" +PROJECTS_DB = "lgbkprjs" -__author__ = 'mshankar@slac.stanford.edu' +__author__ = "mshankar@slac.stanford.edu" logger = logging.getLogger(__name__) + class LgbkException(Exception): """ Exception whose message gets passed to the end user as an error message """ + pass + def get_instruments(): """ Get the list of instruments from the site database. @@ -48,12 +59,15 @@ def get_instruments(): sitedb = logbookclient["site"] return [x for x in sitedb["instruments"].find().sort([("_id", 1)])] + def get_experiments_for_instrument(instrument): """ Get a list of experiments from the database for a instrument. Returns basic information and also some info on the first and last runs. """ - return list(logbookclient['explgbk_cache']['experiments'].find({"instrument": instrument})) + return list( + logbookclient["explgbk_cache"]["experiments"].find({"instrument": instrument}) + ) def get_experiment_info(experiment_name): @@ -62,14 +76,19 @@ def get_experiment_info(experiment_name): :param experiment_name - for example - diadaq13 :return: The info JSON document. """ - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED) - info = expdb['info'].find_one() + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED + ) + info = expdb["info"].find_one() if not info: - logger.error("Cannot find info for %s. Was the experiment deleted/renamed?", experiment_name) + logger.error( + "Cannot find info for %s. Was the experiment deleted/renamed?", + experiment_name, + ) return {"name": experiment_name} setup_oid = info.get("latest_setup", None) if setup_oid: - setup_doc = expdb['setup'].find_one({"_id": setup_oid}) + setup_doc = expdb["setup"].find_one({"_id": setup_oid}) if setup_doc: info["latest_setup"] = setup_doc return info @@ -85,11 +104,13 @@ def save_new_experiment_setup(experiment_name, setup_document, userid): expdb = logbookclient[experiment_name] setup_document["modified_by"] = userid setup_document["modified_at"] = datetime.datetime.utcnow() - latest_setup_id = expdb['setup'].insert_one(setup_document).inserted_id - expdb['info'].find_one_and_update({}, {'$set': {'latest_setup': latest_setup_id}}) + latest_setup_id = expdb["setup"].insert_one(setup_document).inserted_id + expdb["info"].find_one_and_update({}, {"$set": {"latest_setup": latest_setup_id}}) -def register_new_experiment(experiment_name, incoming_info, create_auto_roles=True, skip_initial_objects=False): +def register_new_experiment( + experiment_name, incoming_info, create_auto_roles=True, skip_initial_objects=False +): """ Registers a new experiment. In mongo, this mostly means creating the info object, the run number counter and various indices. @@ -100,30 +121,40 @@ def register_new_experiment(experiment_name, incoming_info, create_auto_roles=Tr expdb = logbookclient[experiment_name] info = {} info.update(incoming_info) - info["_id"] = experiment_name.replace(" ", "_") - info["name"] = experiment_name - info["start_time"] = datetime.datetime.strptime(info["start_time"], '%Y-%m-%dT%H:%M:%S.%fZ') - info["end_time"] = datetime.datetime.strptime(info["end_time"], '%Y-%m-%dT%H:%M:%S.%fZ') + info["_id"] = experiment_name.replace(" ", "_") + info["name"] = experiment_name + info["start_time"] = datetime.datetime.strptime( + info["start_time"], "%Y-%m-%dT%H:%M:%S.%fZ" + ) + info["end_time"] = datetime.datetime.strptime( + info["end_time"], "%Y-%m-%dT%H:%M:%S.%fZ" + ) - expdb['info'].insert_one(info) + expdb["info"].insert_one(info) # Create the run number counter - expdb["counters"].insert_one({'_id': "next_runnum", 'seq': 0}) + expdb["counters"].insert_one({"_id": "next_runnum", "seq": 0}) # Now create the various indices - expdb["runs"].create_index( [("num", DESCENDING)], unique=True) - expdb["elog"].create_index( [("root", ASCENDING)]) - expdb["elog"].create_index( [("parent", ASCENDING)]) - expdb["elog"].create_index( [("content", "text" ), ("title", "text" )]) - expdb["roles"].create_index( [("app", ASCENDING), ("name", ASCENDING)], unique=True) - expdb["run_param_descriptions"].create_index( [("param_name", DESCENDING)], unique=True) - expdb["setup"].create_index( [("modified_by", ASCENDING), ("modified_at", ASCENDING)], unique=True) - expdb["shifts"].create_index( [("name", ASCENDING)], unique=True) - expdb["shifts"].create_index( [("begin_time", ASCENDING)], unique=True) - expdb["file_catalog"].create_index( [("path", ASCENDING), ("run_num", DESCENDING)], unique=True) - expdb["file_catalog"].create_index( [("run_num", DESCENDING)]) - expdb["run_tables"].create_index( [("name", ASCENDING)], unique=True) - expdb["workflow_definitions"].create_index( [("name", ASCENDING)], unique=True) + expdb["runs"].create_index([("num", DESCENDING)], unique=True) + expdb["elog"].create_index([("root", ASCENDING)]) + expdb["elog"].create_index([("parent", ASCENDING)]) + expdb["elog"].create_index([("content", "text"), ("title", "text")]) + expdb["roles"].create_index([("app", ASCENDING), ("name", ASCENDING)], unique=True) + expdb["run_param_descriptions"].create_index( + [("param_name", DESCENDING)], unique=True + ) + expdb["setup"].create_index( + [("modified_by", ASCENDING), ("modified_at", ASCENDING)], unique=True + ) + expdb["shifts"].create_index([("name", ASCENDING)], unique=True) + expdb["shifts"].create_index([("begin_time", ASCENDING)], unique=True) + expdb["file_catalog"].create_index( + [("path", ASCENDING), ("run_num", DESCENDING)], unique=True + ) + expdb["file_catalog"].create_index([("run_num", DESCENDING)]) + expdb["run_tables"].create_index([("name", ASCENDING)], unique=True) + expdb["workflow_definitions"].create_index([("name", ASCENDING)], unique=True) if skip_initial_objects: logger.debug("Skipping creating initial objects") @@ -131,48 +162,86 @@ def register_new_experiment(experiment_name, incoming_info, create_auto_roles=Tr return (True, "") # Create a default shift - expdb["shifts"].insert_one( { "name" : "Default", - "begin_time" : datetime.datetime.utcnow(), - "end_time" : None, - "leader" : security.get_current_user_id(), - "description" : "Default shift created automatically during experiment registration", - "params" : {} - } ) + expdb["shifts"].insert_one( + { + "name": "Default", + "begin_time": datetime.datetime.utcnow(), + "end_time": None, + "leader": security.get_current_user_id(), + "description": "Default shift created automatically during experiment registration", + "params": {}, + } + ) if create_auto_roles: - leaderacc_roles = [ {"app" : "LogBook", "name": "Editor", "players": [ "uid:" + info["leader_account"]] } ] + leaderacc_roles = [ + { + "app": "LogBook", + "name": "Editor", + "players": ["uid:" + info["leader_account"]], + } + ] if LOGBOOK_SITE in ["LCLS"]: - logger.debug("LCLS does not want to give PI's the Manager role; so skipping for experiment %s", experiment_name) + logger.debug( + "LCLS does not want to give PI's the Manager role; so skipping for experiment %s", + experiment_name, + ) else: - leaderacc_roles.append({"app" : "LogBook", "name": "Manager", "players": [ "uid:" + info["leader_account"]] }) + leaderacc_roles.append( + { + "app": "LogBook", + "name": "Manager", + "players": ["uid:" + info["leader_account"]], + } + ) expdb["roles"].insert_many(leaderacc_roles) if "posix_group" in info and len(info["posix_group"]) > 1: - expdb["roles"].insert_one({"app" : "LogBook", "name": "Writer", "players": [ info["posix_group"]] }) - if security.get_current_user_id() != info["leader_account"] and not security.check_privilege_for_experiment("ops_page", None, None): - expdb["roles"].update_one({"app" : "LogBook", "name": "Editor"}, {"$addToSet": { "players": "uid:" + security.get_current_user_id() }}) - expdb["roles"].update_one({"app" : "LogBook", "name": "Manager"}, {"$addToSet": { "players": "uid:" + security.get_current_user_id() }}) - + expdb["roles"].insert_one( + {"app": "LogBook", "name": "Writer", "players": [info["posix_group"]]} + ) + if security.get_current_user_id() != info[ + "leader_account" + ] and not security.check_privilege_for_experiment("ops_page", None, None): + expdb["roles"].update_one( + {"app": "LogBook", "name": "Editor"}, + {"$addToSet": {"players": "uid:" + security.get_current_user_id()}}, + ) + expdb["roles"].update_one( + {"app": "LogBook", "name": "Manager"}, + {"$addToSet": {"players": "uid:" + security.get_current_user_id()}}, + ) if "initial_sample" in incoming_info and incoming_info["initial_sample"]: try: - create_sample(experiment_name, { - "name": incoming_info["initial_sample"], - "description": "The initial sample created as part of experiment creation." - }) + create_sample( + experiment_name, + { + "name": incoming_info["initial_sample"], + "description": "The initial sample created as part of experiment creation.", + }, + ) make_sample_current(experiment_name, incoming_info["initial_sample"]) start_run(experiment_name, "DATA") - except: - logger.exception("Exception creating an initial sample. Please create the sample manually.") - return (True, "Exception creating an initial sample. Please create the sample manually.") + except Exception: + logger.exception( + "Exception creating an initial sample. Please create the sample manually." + ) + return ( + True, + "Exception creating an initial sample. Please create the sample manually.", + ) try: - clone_system_template_run_tables_into_experiment(experiment_name, info["instrument"]) - except: + clone_system_template_run_tables_into_experiment( + experiment_name, info["instrument"] + ) + except Exception: logger.exception("Exception creating copies of template system run tables") return (True, "") + def update_existing_experiment(experiment_name, incoming_info): """ Update an existing experiment @@ -183,15 +252,25 @@ def update_existing_experiment(experiment_name, incoming_info): expdb = logbookclient[experiment_name] info = {} info.update(incoming_info) - info_id = experiment_name.replace(" ", "_") - info["name"] = experiment_name - info["start_time"] = datetime.datetime.strptime(info["start_time"], '%Y-%m-%dT%H:%M:%S.%fZ') - info["end_time"] = datetime.datetime.strptime(info["end_time"], '%Y-%m-%dT%H:%M:%S.%fZ') - - expdb['info'].update_one({}, { "$set": info }) + info["name"] = experiment_name + info["start_time"] = datetime.datetime.strptime( + info["start_time"], "%Y-%m-%dT%H:%M:%S.%fZ" + ) + info["end_time"] = datetime.datetime.strptime( + info["end_time"], "%Y-%m-%dT%H:%M:%S.%fZ" + ) + + expdb["info"].update_one({}, {"$set": info}) return (True, "") -def clone_experiment(experiment_name, source_experiment_name, incoming_info, copy_specs, skip_initial_objects=False): + +def clone_experiment( + experiment_name, + source_experiment_name, + incoming_info, + copy_specs, + skip_initial_objects=False, +): """ Registers a new experiment based on an existing experiment. We use the "info" of the existing experiment as a template for the new experiment. @@ -211,12 +290,17 @@ def clone_experiment(experiment_name, source_experiment_name, incoming_info, cop info = {} info.update(src_exp_db["info"].find_one()) info.update(incoming_info) - info["_id"] = experiment_name.replace(" ", "_") - info["name"] = experiment_name - if "experiment_name" in info: # Bug from previous releases? + info["_id"] = experiment_name.replace(" ", "_") + info["name"] = experiment_name + if "experiment_name" in info: # Bug from previous releases? del info["experiment_name"] - status, msg = register_new_experiment(experiment_name, info, create_auto_roles=False, skip_initial_objects=skip_initial_objects) + status, msg = register_new_experiment( + experiment_name, + info, + create_auto_roles=False, + skip_initial_objects=skip_initial_objects, + ) if not status: return (status, msg) @@ -226,17 +310,32 @@ def copy_collection_from_src_to_clone(collection_name): for coll, cp_select in copy_specs.items(): if cp_select: - logger.info("Copying over collection %s from source experiment %s to dest experiment %s", coll, source_experiment_name, experiment_name) + logger.info( + "Copying over collection %s from source experiment %s to dest experiment %s", + coll, + source_experiment_name, + experiment_name, + ) copy_collection_from_src_to_clone(coll) # Make sure the person doing the cloning has admin privileges in the expeirment. current_user = security.get_current_user_id() - expdb["roles"].update_one({"app" : "LogBook", "name": "Manager"}, {"$addToSet": {"players": "uid:" + current_user}}, upsert=True) - expdb["roles"].update_one({"app" : "LogBook", "name": "Editor"}, {"$addToSet": {"players": "uid:" + current_user}}, upsert=True) + expdb["roles"].update_one( + {"app": "LogBook", "name": "Manager"}, + {"$addToSet": {"players": "uid:" + current_user}}, + upsert=True, + ) + expdb["roles"].update_one( + {"app": "LogBook", "name": "Editor"}, + {"$addToSet": {"players": "uid:" + current_user}}, + upsert=True, + ) try: - clone_system_template_run_tables_into_experiment(experiment_name, info["instrument"]) - except: + clone_system_template_run_tables_into_experiment( + experiment_name, info["instrument"] + ) + except Exception: logger.exception("Exception creating copies of template system run tables") return (True, "") @@ -247,34 +346,48 @@ def rename_experiment(experiment_name, new_experiment_name): Renames an experiment with a new name. """ if new_experiment_name in list(logbookclient.list_database_names()): - return (False, "Experiment %s has already been registered" % new_experiment_name) - if experiment_name in [x["name"] for x in get_currently_active_experiments() if "name" in x]: + return ( + False, + "Experiment %s has already been registered" % new_experiment_name, + ) + if experiment_name in [ + x["name"] for x in get_currently_active_experiments() if "name" in x + ]: return (False, "Experiment %s is currently active" % experiment_name) src_exp_db = logbookclient[experiment_name] info = src_exp_db["info"].find_one() mods = {} - mods["start_time"] = info["start_time"].strftime('%Y-%m-%dT%H:%M:%S.%fZ') - mods["end_time"] = info["end_time"].strftime('%Y-%m-%dT%H:%M:%S.%fZ') + mods["start_time"] = info["start_time"].strftime("%Y-%m-%dT%H:%M:%S.%fZ") + mods["end_time"] = info["end_time"].strftime("%Y-%m-%dT%H:%M:%S.%fZ") - copy_specs = { x : True for x in src_exp_db.list_collection_names() } + copy_specs = {x: True for x in src_exp_db.list_collection_names()} for cn in ["info", "counters"]: copy_specs[cn] = False - status, msg = clone_experiment(new_experiment_name, experiment_name, mods, copy_specs, skip_initial_objects=True) + status, msg = clone_experiment( + new_experiment_name, + experiment_name, + mods, + copy_specs, + skip_initial_objects=True, + ) if not status: return (status, msg) new_exp_db = logbookclient[new_experiment_name] # Before dropping the src database, we should copy over the run counter. - run_counter = src_exp_db["counters"].find_one({'_id': "next_runnum"}) - new_exp_db["counters"].update_one({'_id': "next_runnum"}, {"$set": {'seq': run_counter['seq']}}) + run_counter = src_exp_db["counters"].find_one({"_id": "next_runnum"}) + new_exp_db["counters"].update_one( + {"_id": "next_runnum"}, {"$set": {"seq": run_counter["seq"]}} + ) logger.info("Dropping the experiment database for %s", experiment_name) logbookclient.drop_database(experiment_name) return (True, "") + def lock_unlock_experiment(experiment_name): """ Toggle an experiment's locked status. @@ -282,40 +395,58 @@ def lock_unlock_experiment(experiment_name): expdb = logbookclient[experiment_name] curr_is_locked = expdb["info"].find_one().get("is_locked", False) - expdb['info'].update_one({}, { "$set": {"is_locked": False if curr_is_locked else True } }) + expdb["info"].update_one( + {}, {"$set": {"is_locked": False if curr_is_locked else True}} + ) return (True, "") + def delete_experiment(experiment_name): """ Delete the experiment; we cannot recover from this without a backup. Make sure the experiment is not active and is actually an experiment. We also check to make sure that none of the attacments use http for an image store. """ - active_experiments = [ x.get("name", "Standby") for x in get_currently_active_experiments() ] + active_experiments = [ + x.get("name", "Standby") for x in get_currently_active_experiments() + ] if experiment_name in active_experiments: return (False, "Experiment %s is currently active" % experiment_name) expdb = logbookclient[experiment_name] - if not expdb["info"].find_one() or not "instrument" in expdb["info"].find_one(): + if not expdb["info"].find_one() or "instrument" not in expdb["info"].find_one(): return (False, "Is %s an experiment?" % experiment_name) - http_attachments = expdb["elog"].count_documents({"$or": [{"attachments.url": {"$regex": re.compile("^http://")}}, {"attachments.preview_url": {"$regex": re.compile("^http://")}}]}) + http_attachments = expdb["elog"].count_documents( + { + "$or": [ + {"attachments.url": {"$regex": re.compile("^http://")}}, + {"attachments.preview_url": {"$regex": re.compile("^http://")}}, + ] + } + ) if http_attachments: - return (False, "We have %s attachments in an external image store; please archive the experiment before deleting." % http_attachments) + return ( + False, + "We have %s attachments in an external image store; please archive the experiment before deleting." + % http_attachments, + ) logbookclient.drop_database(experiment_name) return (True, "") + def add_update_experiment_params(experiment_name, params): """ Add or update the experiment params. We expect a dict of key value pairs; we'll add params to all the keys. """ expdb = logbookclient[experiment_name] - updated_params = { "params." + k : v for k, v in params.items()} + updated_params = {"params." + k: v for k, v in params.items()} logger.debug(updated_params) expdb["info"].update_one({}, {"$set": updated_params}) return True, "" + def migrate_attachments_to_local_store(experiment_name): """ Move the attachments from an external image store to either GridFS or a tar.gz on the file system. @@ -324,19 +455,31 @@ def migrate_attachments_to_local_store(experiment_name): mg_imgstore = parseImageStoreURL("mongo://") expdb = logbookclient[experiment_name] failures = 0 - elog_entries = expdb["elog"].find({"$or": [{"attachments.url": {"$regex": re.compile("^http://")}}, {"attachments.preview_url": {"$regex": re.compile("^http://")}}]}) + elog_entries = expdb["elog"].find( + { + "$or": [ + {"attachments.url": {"$regex": re.compile("^http://")}}, + {"attachments.preview_url": {"$regex": re.compile("^http://")}}, + ] + } + ) for elog_entry in elog_entries: + def __migrate_to_mongo__(aurl, attachment): logger.debug("Migrating %s to mongo", aurl) try: - bio = parseImageStoreURL(aurl).return_url_contents(experiment_name, aurl) + bio = parseImageStoreURL(aurl).return_url_contents( + experiment_name, aurl + ) if not bio: logger.debug("Cannot get attachment contents for %s", aurl) return None - murl = mg_imgstore.store_file_and_return_url(experiment_name, attachment["name"], attachment["type"], bio) + murl = mg_imgstore.store_file_and_return_url( + experiment_name, attachment["name"], attachment["type"], bio + ) logger.debug("Migrated %s to mongo as %s", aurl, murl) return murl - except: + except Exception: logger.exception("Exception migrating attachment %s", aurl) return None @@ -344,18 +487,34 @@ def __migrate_to_mongo__(aurl, attachment): if attachment.get("url", "").startswith("http://"): murl = __migrate_to_mongo__(attachment["url"], attachment) if murl: - expdb["elog"].update_one({"_id": elog_entry["_id"], "attachments._id": attachment["_id"]}, {"$set": {"attachments.$.url": murl}}) + expdb["elog"].update_one( + { + "_id": elog_entry["_id"], + "attachments._id": attachment["_id"], + }, + {"$set": {"attachments.$.url": murl}}, + ) else: failures = failures + 1 if attachment.get("preview_url", "").startswith("http://"): murl = __migrate_to_mongo__(attachment["preview_url"], attachment) if murl: - expdb["elog"].update_one({"_id": elog_entry["_id"], "attachments._id": attachment["_id"]}, {"$set": {"attachments.$.preview_url": murl}}) + expdb["elog"].update_one( + { + "_id": elog_entry["_id"], + "attachments._id": attachment["_id"], + }, + {"$set": {"attachments.$.preview_url": murl}}, + ) else: failures = failures + 1 if failures: - return (False, "%s attachments were not migrated. Please see the server logs for more details" % failures) + return ( + False, + "%s attachments were not migrated. Please see the server logs for more details" + % failures, + ) return (True, "") @@ -375,9 +534,12 @@ def create_update_instrument(instrument_name, createp, incoming_info): if createp: sitedb.instruments.insert_one(incoming_info) else: - sitedb.instruments.find_one_and_update({"_id": instrument_name}, { "$set": incoming_info }) + sitedb.instruments.find_one_and_update( + {"_id": instrument_name}, {"$set": incoming_info} + ) return (True, "Instrument %s processed" % instrument_name) + def get_instrument_station_list(): """ Get a list of instrumens and end stations as a list. @@ -391,7 +553,7 @@ def get_instrument_station_list(): if num_stations: # Skip those that have a num_stations of 0. for station in range(num_stations): - ins_st_list.append({ "instrument": name, "station": station }) + ins_st_list.append({"instrument": name, "station": station}) return ins_st_list @@ -405,19 +567,31 @@ def get_currently_active_experiments(): ret = [] for qry in active_queries: logger.debug("Looking for active experiment for %s", qry) - for exp in sitedb["experiment_switch"].find(qry).sort([( "switch_time", -1 )]).limit(1): - exp_info = get_experiment_info(exp["experiment_name"]) if not exp.get("is_standby", False) else { "instrument": qry["instrument"], "is_standby": True } + for exp in ( + sitedb["experiment_switch"].find(qry).sort([("switch_time", -1)]).limit(1) + ): + exp_info = ( + get_experiment_info(exp["experiment_name"]) + if not exp.get("is_standby", False) + else {"instrument": qry["instrument"], "is_standby": True} + ) exp_info["station"] = qry["station"] exp_info["switch_time"] = exp["switch_time"] exp_info["requestor_uid"] = exp["requestor_uid"] - if 'name' in exp_info: - curr_sample = get_current_sample_name(exp_info['name']) + if "name" in exp_info: + curr_sample = get_current_sample_name(exp_info["name"]) if curr_sample: - exp_info['current_sample'] = curr_sample - curr_run = get_current_run(exp_info['name']) + exp_info["current_sample"] = curr_sample + curr_run = get_current_run(exp_info["name"]) if curr_run: - exp_info['current_run'] = { x : curr_run[x] for x in ['num', 'begin_time', 'end_time'] if x in curr_run and curr_run[x] } - cached_exp_info = logbookclient['explgbk_cache']['experiments'].find_one({"_id": exp_info['name']}) + exp_info["current_run"] = { + x: curr_run[x] + for x in ["num", "begin_time", "end_time"] + if x in curr_run and curr_run[x] + } + cached_exp_info = logbookclient["explgbk_cache"][ + "experiments" + ].find_one({"_id": exp_info["name"]}) if cached_exp_info: if "last_run" in cached_exp_info: exp_info["last_run"] = cached_exp_info["last_run"] @@ -427,74 +601,114 @@ def get_currently_active_experiments(): return ret + def get_active_experiment_name_for_instrument_station(instrument, station): sitedb = logbookclient["site"] origins = instrument - caseinsinslkp = { x["_id"].upper() : x["_id"] for x in sitedb["instruments"].find({}, {"_id": 1})} + caseinsinslkp = { + x["_id"].upper(): x["_id"] for x in sitedb["instruments"].find({}, {"_id": 1}) + } instrument = caseinsinslkp.get(instrument.upper(), None) if not instrument: logger.error("Cannot find instrument %s", origins) return None - for active_experiment in sitedb["experiment_switch"].find({ "instrument": instrument, "station": station }).sort([( "switch_time", -1 )]).limit(1): - if 'instrument' in active_experiment and active_experiment['instrument'] == instrument and active_experiment['station'] == int(station) and not active_experiment.get('is_standby', False): - return get_experiment_info(active_experiment['experiment_name']) + for active_experiment in ( + sitedb["experiment_switch"] + .find({"instrument": instrument, "station": station}) + .sort([("switch_time", -1)]) + .limit(1) + ): + if ( + "instrument" in active_experiment + and active_experiment["instrument"] == instrument + and active_experiment["station"] == int(station) + and not active_experiment.get("is_standby", False) + ): + return get_experiment_info(active_experiment["experiment_name"]) return None + def switch_experiment(instrument, station, experiment_name, userid): """ Switch the currently active experiment on the instrument. This mostly consists inserting an entry into the experiment_switch database. Also switch in/switch out the operator account for the instrument. """ - current_active_experiment = get_active_experiment_name_for_instrument_station(instrument, station) + current_active_experiment = get_active_experiment_name_for_instrument_station( + instrument, station + ) sitedb = logbookclient["site"] - sitedb.experiment_switch.insert_one({ - "experiment_name" : experiment_name, - "instrument" : instrument, - "station" : int(station), - "switch_time" : datetime.datetime.utcnow(), - "requestor_uid" : userid - }) - operator_uid = { x["_id"] : x for x in get_instruments()}[instrument].get("params", {}).get("operator_uid", None) + sitedb.experiment_switch.insert_one( + { + "experiment_name": experiment_name, + "instrument": instrument, + "station": int(station), + "switch_time": datetime.datetime.utcnow(), + "requestor_uid": userid, + } + ) + operator_uid = ( + {x["_id"]: x for x in get_instruments()}[instrument] + .get("params", {}) + .get("operator_uid", None) + ) if operator_uid: fq_role_name = "LogBook/Writer" if current_active_experiment: active_exp_name = current_active_experiment.get("name", None) - logger.info("Removing post privileges for uid:" + operator_uid + " from " + active_exp_name) - remove_collaborator_from_role(active_exp_name, "uid:" + operator_uid, fq_role_name) - logger.debug("Adding post privileges for uid:" + operator_uid + " to " + experiment_name) + logger.info( + "Removing post privileges for uid:" + + operator_uid + + " from " + + active_exp_name + ) + remove_collaborator_from_role( + active_exp_name, "uid:" + operator_uid, fq_role_name + ) + logger.debug( + "Adding post privileges for uid:" + operator_uid + " to " + experiment_name + ) add_collaborator_to_role(experiment_name, "uid:" + operator_uid, fq_role_name) else: - logger.debug("No operator_uid defined for %s; skipping auto-add of operator accounts", instrument) + logger.debug( + "No operator_uid defined for %s; skipping auto-add of operator accounts", + instrument, + ) return (True, "") + def instrument_standby(instrument, station, userid): """ Put the instrument into standby mode. This mostly creates an experiment switch entry with the is_standby flag set. """ sitedb = logbookclient["site"] - sitedb.experiment_switch.insert_one({ - "experiment_name" : "Standby", - "instrument" : instrument, - "station" : int(station), - "switch_time" : datetime.datetime.utcnow(), - "requestor_uid" : userid, - "is_standby": True - }) + sitedb.experiment_switch.insert_one( + { + "experiment_name": "Standby", + "instrument": instrument, + "station": int(station), + "switch_time": datetime.datetime.utcnow(), + "requestor_uid": userid, + "is_standby": True, + } + ) return (True, "") + def get_switch_history(instrument, station): """ Return the experiment switch history for an instrument/station. """ sitedb = logbookclient["site"] - ins_exps = { x["name"] : x for x in get_experiments_for_instrument(instrument) } + ins_exps = {x["name"]: x for x in get_experiments_for_instrument(instrument)} ret = [] - for x in sitedb.experiment_switch.find({"instrument": instrument, "station": station}).sort([("switch_time", -1)]): + for x in sitedb.experiment_switch.find( + {"instrument": instrument, "station": station} + ).sort([("switch_time", -1)]): expname = x.get("experiment_name", "") if expname != "Standby" and expname in ins_exps: x["description"] = ins_exps[expname]["description"] @@ -502,44 +716,82 @@ def get_switch_history(instrument, station): ret.append(x) return ret + def get_global_roles(): sitedb = logbookclient["site"] return [x for x in sitedb["roles"].find({"app": "LogBook"})] + def add_player_to_global_role(player, role): sitedb = logbookclient["site"] - sitedb["roles"].update_one({"app": "LogBook", "name": role}, {"$addToSet": {"players": player}}) + sitedb["roles"].update_one( + {"app": "LogBook", "name": role}, {"$addToSet": {"players": player}} + ) + def remove_player_from_global_role(player, role): sitedb = logbookclient["site"] - sitedb["roles"].update_one({"app": "LogBook", "name": role}, {"$pull": {"players": player}}) + sitedb["roles"].update_one( + {"app": "LogBook", "name": role}, {"$pull": {"players": player}} + ) + def add_player_to_instrument_role(instrument, player, role): sitedb = logbookclient["site"] - ins = sitedb["instruments"].find_one({ "_id": instrument }) - if not "roles" in ins: - sitedb["instruments"].update_one({ "_id": instrument }, {"$set": {"roles": [{"app": "LogBook", "name": role, "players": [ player ]}]}}) - return sitedb["instruments"].find_one({ "_id": instrument }) - rl = [ x for x in ins.get("roles", []) if x["app"] == "LogBook" and x["name"] == role ] + ins = sitedb["instruments"].find_one({"_id": instrument}) + if "roles" not in ins: + sitedb["instruments"].update_one( + {"_id": instrument}, + { + "$set": { + "roles": [{"app": "LogBook", "name": role, "players": [player]}] + } + }, + ) + return sitedb["instruments"].find_one({"_id": instrument}) + rl = [ + x for x in ins.get("roles", []) if x["app"] == "LogBook" and x["name"] == role + ] if rl: - sitedb["instruments"].update_one({ "_id": instrument, "roles.app": "LogBook", "roles.name": role}, {"$addToSet": {"roles.$.players": player}}) - return sitedb["instruments"].find_one({ "_id": instrument }) + sitedb["instruments"].update_one( + {"_id": instrument, "roles.app": "LogBook", "roles.name": role}, + {"$addToSet": {"roles.$.players": player}}, + ) + return sitedb["instruments"].find_one({"_id": instrument}) else: - sitedb["instruments"].update_one({ "_id": instrument}, {"$push": {"roles": {"app": "LogBook", "name": role, "players": [ player ]}}}) - return sitedb["instruments"].find_one({ "_id": instrument }) + sitedb["instruments"].update_one( + {"_id": instrument}, + {"$push": {"roles": {"app": "LogBook", "name": role, "players": [player]}}}, + ) + return sitedb["instruments"].find_one({"_id": instrument}) + def remove_player_from_instrument_role(instrument, player, role): sitedb = logbookclient["site"] - ins = sitedb["instruments"].find_one({ "_id": instrument }) - if not "roles" in ins: - return sitedb["instruments"].find_one({ "_id": instrument }) + ins = sitedb["instruments"].find_one({"_id": instrument}) + if "roles" not in ins: + return sitedb["instruments"].find_one({"_id": instrument}) - rl = [ x for x in ins.get("roles", []) if x["app"] == "LogBook" and x["name"] == role ] + rl = [ + x for x in ins.get("roles", []) if x["app"] == "LogBook" and x["name"] == role + ] if rl: - sitedb["instruments"].update_one({ "_id": instrument, "roles.app": "LogBook", "roles.name": role}, {"$pull": {"roles.$.players": player}}) - sitedb["instruments"].update_one({ "_id": instrument, "roles.app": "LogBook", "roles.name": role, "roles.players": {"$exists": True, "$size": 0}}, {"$pull": {"roles": {"app": "LogBook", "name": role}}}) + sitedb["instruments"].update_one( + {"_id": instrument, "roles.app": "LogBook", "roles.name": role}, + {"$pull": {"roles.$.players": player}}, + ) + sitedb["instruments"].update_one( + { + "_id": instrument, + "roles.app": "LogBook", + "roles.name": role, + "roles.players": {"$exists": True, "$size": 0}, + }, + {"$pull": {"roles": {"app": "LogBook", "name": role}}}, + ) + + return sitedb["instruments"].find_one({"_id": instrument}) - return sitedb["instruments"].find_one({ "_id": instrument }) def get_elog_entries(experiment_name, sample_name=None): """ @@ -548,19 +800,41 @@ def get_elog_entries(experiment_name, sample_name=None): """ expdb = logbookclient[experiment_name] if not sample_name or sample_name == "All Samples": - return [entry for entry in expdb['elog'].find().sort([("insert_time", 1)])] + return [entry for entry in expdb["elog"].find().sort([("insert_time", 1)])] else: # We start at samples for performance reasons; - return [x for x in expdb.samples.aggregate([ - { "$match": { "name": sample_name }}, - { "$lookup": { "from": "runs", "localField": "_id", "foreignField": "sample", "as": "run"}}, - { "$unwind": "$run" }, # lookup generates an array field, we convert to a list of docs with a single field instead. - { "$replaceRoot": { "newRoot": "$run" } }, - { "$lookup": { "from": "elog", "localField": "num", "foreignField": "run_num", "as": "elog"}}, - { "$unwind": "$elog" }, - { "$replaceRoot": { "newRoot": "$elog" } }, - { "$sort": { "insert_time": 1 }} - ])] + return [ + x + for x in expdb.samples.aggregate( + [ + {"$match": {"name": sample_name}}, + { + "$lookup": { + "from": "runs", + "localField": "_id", + "foreignField": "sample", + "as": "run", + } + }, + { + "$unwind": "$run" + }, # lookup generates an array field, we convert to a list of docs with a single field instead. + {"$replaceRoot": {"newRoot": "$run"}}, + { + "$lookup": { + "from": "elog", + "localField": "num", + "foreignField": "run_num", + "as": "elog", + } + }, + {"$unwind": "$elog"}, + {"$replaceRoot": {"newRoot": "$elog"}}, + {"$sort": {"insert_time": 1}}, + ] + ) + ] + def get_specific_elog_entry(experiment_name, id): """ @@ -568,18 +842,24 @@ def get_specific_elog_entry(experiment_name, id): For now, we have id based lookups. """ expdb = logbookclient[experiment_name] - return expdb['elog'].find_one({"_id": ObjectId(id)}) + return expdb["elog"].find_one({"_id": ObjectId(id)}) + def __get_root_and_parent_entries(experiment_name, matching_entries): """ Get the root and parent entries for all elog entries in matching entries. """ - logger.debug("Recursively expanding root and parent entries for %s", experiment_name) + logger.debug( + "Recursively expanding root and parent entries for %s", experiment_name + ) anyAdditions = False + def __addEntryIfNotPresent(matching_entry, idname): nonlocal anyAdditions if idname in matching_entry and matching_entry[idname] not in matching_entries: - matching_entries[matching_entry[idname]] = get_specific_elog_entry(experiment_name, matching_entry[idname]) + matching_entries[matching_entry[idname]] = get_specific_elog_entry( + experiment_name, matching_entry[idname] + ) anyAdditions = True for matching_entry in matching_entries.copy().values(): @@ -595,26 +875,38 @@ def search_elog_for_text(experiment_name, search_text): The sort order is important as the UI uses this to optimize tree-building. """ expdb = logbookclient[experiment_name] - matching_entries = {x["_id"] : x for x in expdb['elog'].find({ "$text": { "$search": search_text }})} + matching_entries = { + x["_id"]: x for x in expdb["elog"].find({"$text": {"$search": search_text}}) + } if not matching_entries: - matching_entries = {x["_id"] : x for x in expdb['elog'].find({ "content": { "$regex": re.compile(".*" + search_text + ".*", re.IGNORECASE)}})} + matching_entries = { + x["_id"]: x + for x in expdb["elog"].find( + { + "content": { + "$regex": re.compile(".*" + search_text + ".*", re.IGNORECASE) + } + } + ) + } # Recursively gather all root and parent entries while __get_root_and_parent_entries(experiment_name, matching_entries): pass - return list(sorted(matching_entries.values(), key=lambda x : x["insert_time"])) + return list(sorted(matching_entries.values(), key=lambda x: x["insert_time"])) + def get_elogs_for_run_num(experiment_name, run_num): """ Get elog entries belonging to run number run_num """ expdb = logbookclient[experiment_name] - matching_entries = {x["_id"] : x for x in expdb['elog'].find({ "run_num": run_num})} + matching_entries = {x["_id"]: x for x in expdb["elog"].find({"run_num": run_num})} # Recursively gather all root and parent entries while __get_root_and_parent_entries(experiment_name, matching_entries): pass - return list(sorted(matching_entries.values(), key=lambda x : x["insert_time"])) + return list(sorted(matching_entries.values(), key=lambda x: x["insert_time"])) def get_elogs_for_run_num_range(experiment_name, start_run_num, end_run_num): @@ -622,51 +914,73 @@ def get_elogs_for_run_num_range(experiment_name, start_run_num, end_run_num): Get elog entries for the run number range - start_run_num to end_run_num (both inclusive) """ expdb = logbookclient[experiment_name] - matching_entries = {x["_id"] : x for x in expdb['elog'].find({ "run_num": {"$gte": start_run_num, "$lte": end_run_num}})} + matching_entries = { + x["_id"]: x + for x in expdb["elog"].find( + {"run_num": {"$gte": start_run_num, "$lte": end_run_num}} + ) + } # Recursively gather all root and parent entries while __get_root_and_parent_entries(experiment_name, matching_entries): pass - return list(sorted(matching_entries.values(), key=lambda x : x["insert_time"])) + return list(sorted(matching_entries.values(), key=lambda x: x["insert_time"])) + def get_elog_entries_by_author(experiment_name, author): """ Get elog entries by the specified author """ expdb = logbookclient[experiment_name] - matching_entries = {x["_id"] : x for x in expdb['elog'].find({ "author": author })} + matching_entries = {x["_id"]: x for x in expdb["elog"].find({"author": author})} # Recursively gather all root and parent entries while __get_root_and_parent_entries(experiment_name, matching_entries): pass - return list(sorted(matching_entries.values(), key=lambda x : x["insert_time"])) + return list(sorted(matching_entries.values(), key=lambda x: x["insert_time"])) + def get_elog_entries_by_tag(experiment_name, tag): """ Get elog entries with the specified tag """ expdb = logbookclient[experiment_name] - matching_entries = {x["_id"] : x for x in expdb['elog'].find({ "tags": tag })} + matching_entries = {x["_id"]: x for x in expdb["elog"].find({"tags": tag})} # Recursively gather all root and parent entries while __get_root_and_parent_entries(experiment_name, matching_entries): pass - return list(sorted(matching_entries.values(), key=lambda x : x["insert_time"])) + return list(sorted(matching_entries.values(), key=lambda x: x["insert_time"])) + def get_run_numbers_with_tag(experiment_name, tag): """ Get elog entries with the specified tag """ expdb = logbookclient[experiment_name] - run_nums = set([ x["run_num"] for x in expdb['elog'].find({ "tags": tag, "run_num": {"$exists": 1} }, {"run_num": 1} ) ]) + run_nums = set( + [ + x["run_num"] + for x in expdb["elog"].find( + {"tags": tag, "run_num": {"$exists": 1}}, {"run_num": 1} + ) + ] + ) return sorted(list(run_nums)) + def get_tag_to_run_numbers(experiment_name): """ Get a dict of tag to the run number that have elog statements containing the tag """ expdb = logbookclient[experiment_name] - runs_with_tags = [ x for x in expdb['elog'].find({ "tags": {"$exists": 1}, "run_num": {"$exists": 1} }, {"run_num": 1, "tags": 1} ) ] + runs_with_tags = [ + x + for x in expdb["elog"].find( + {"tags": {"$exists": 1}, "run_num": {"$exists": 1}}, + {"run_num": 1, "tags": 1}, + ) + ] ret = {} for run_with_tags in runs_with_tags: for tag in run_with_tags["tags"]: @@ -675,12 +989,19 @@ def get_tag_to_run_numbers(experiment_name): ret[tag].append(run_with_tags["run_num"]) return ret + def get_tags_for_runs(experiment_name): """ Get a dict of run number to array of tags. """ expdb = logbookclient[experiment_name] - runs_with_tags = [ x for x in expdb['elog'].find({ "tags": {"$exists": 1}, "run_num": {"$exists": 1} }, {"run_num": 1, "tags": 1} ) ] + runs_with_tags = [ + x + for x in expdb["elog"].find( + {"tags": {"$exists": 1}, "run_num": {"$exists": 1}}, + {"run_num": 1, "tags": 1}, + ) + ] ret = {} for run_with_tags in runs_with_tags: for tag in run_with_tags["tags"]: @@ -692,32 +1013,35 @@ def get_tags_for_runs(experiment_name): ret[k] = sorted(set(v)) return ret + def get_elogs_for_specified_id(experiment_name, specified_id): """ Get the elog entries related to the entry with the specified id. """ specified_entry = get_specific_elog_entry(experiment_name, specified_id) if specified_entry: - matching_entries = { specified_entry["_id"]: specified_entry } + matching_entries = {specified_entry["_id"]: specified_entry} while __get_root_and_parent_entries(experiment_name, matching_entries): pass - return list(sorted(matching_entries.values(), key=lambda x : x["insert_time"])) + return list(sorted(matching_entries.values(), key=lambda x: x["insert_time"])) else: return [] + def get_elog_tree_for_specified_id(experiment_name, specified_id): """ Get all the children of the specified elog entry; the result includes the entry also. """ expdb = logbookclient[experiment_name] specified_entry = get_specific_elog_entry(experiment_name, specified_id) - ret = [ specified_entry ] + ret = [specified_entry] if specified_entry: ret.extend([x for x in expdb["elog"].find({"root": specified_entry["_id"]})]) - return list(sorted(ret, key=lambda x : x["insert_time"])) + return list(sorted(ret, key=lambda x: x["insert_time"])) else: return [] + def get_complete_elog_tree_for_specified_id(experiment_name, specified_id): """ Get all the parents and all the children of the specified elog entry; the result includes the entry also. @@ -725,27 +1049,42 @@ def get_complete_elog_tree_for_specified_id(experiment_name, specified_id): expdb = logbookclient[experiment_name] specified_entry = get_specific_elog_entry(experiment_name, ObjectId(specified_id)) if specified_entry: - if 'root' in specified_entry: - ret = {x["_id"] : x for x in expdb['elog'].find({ "root": specified_entry["root"]})} - ret[specified_entry["root"]] = get_specific_elog_entry(experiment_name, specified_entry["root"]) + if "root" in specified_entry: + ret = { + x["_id"]: x + for x in expdb["elog"].find({"root": specified_entry["root"]}) + } + ret[specified_entry["root"]] = get_specific_elog_entry( + experiment_name, specified_entry["root"] + ) else: - ret = {x["_id"] : x for x in expdb['elog'].find({ "root": specified_entry["_id"]})} + ret = { + x["_id"]: x + for x in expdb["elog"].find({"root": specified_entry["_id"]}) + } ret[specified_entry["_id"]] = specified_entry - return list(sorted(ret.values(), key=lambda x : x["insert_time"])) + return list(sorted(ret.values(), key=lambda x: x["insert_time"])) return [] + def get_elogs_for_date_range(experiment_name, start_date, end_date): """ Get the elog entries between the specified date range; >= start_date and <= end_date """ expdb = logbookclient[experiment_name] logger.debug("Looking for entries between %s and %s", start_date, end_date) - matching_entries = {x["_id"] : x for x in expdb['elog'].find({ "relevance_time": { "$gte": start_date, "$lte": end_date } })} + matching_entries = { + x["_id"]: x + for x in expdb["elog"].find( + {"relevance_time": {"$gte": start_date, "$lte": end_date}} + ) + } # Recursively gather all root and parent entries while __get_root_and_parent_entries(experiment_name, matching_entries): pass - return list(sorted(matching_entries.values(), key=lambda x : x["insert_time"])) + return list(sorted(matching_entries.values(), key=lambda x: x["insert_time"])) + def get_elog_entries_by_regex(experiment_name, regx): """ @@ -754,16 +1093,22 @@ def get_elog_entries_by_regex(experiment_name, regx): expdb = logbookclient[experiment_name] logger.debug("Looking for entries matching %s", regx) try: - matching_entries = {x["_id"] : x for x in expdb['elog'].find({"content": {"$regex": regx, "$options": "m"}})} + matching_entries = { + x["_id"]: x + for x in expdb["elog"].find({"content": {"$regex": regx, "$options": "m"}}) + } # Recursively gather all root and parent entries while __get_root_and_parent_entries(experiment_name, matching_entries): pass - return list(sorted(matching_entries.values(), key=lambda x : x["insert_time"])) - except: - logger.exception("Exception matching regex %s for experiment %s", regx, experiment_name) + return list(sorted(matching_entries.values(), key=lambda x: x["insert_time"])) + except Exception: + logger.exception( + "Exception matching regex %s for experiment %s", regx, experiment_name + ) return [] + def __upload_attachments_to_imagestore_and_return_urls(experiment_name, files): """ Given a list of file uploads, upload these to the imagestore, generate thumbnails and return a list of attachments objects. @@ -771,47 +1116,94 @@ def __upload_attachments_to_imagestore_and_return_urls(experiment_name, files): attachments = [] for file in files: filename = file[0] - filestorage = file[1] # http://werkzeug.pocoo.org/docs/0.14/datastructures/#werkzeug.datastructures.FileStorage - - imgurl = parseImageStoreURL(imagestoreurl).store_file_and_return_url(experiment_name, filename, filestorage.mimetype, filestorage.stream) - attachment = {"_id": ObjectId(), "name" : filename, "type": filestorage.mimetype, "url" : imgurl } + filestorage = file[ + 1 + ] # http://werkzeug.pocoo.org/docs/0.14/datastructures/#werkzeug.datastructures.FileStorage + + imgurl = parseImageStoreURL(imagestoreurl).store_file_and_return_url( + experiment_name, filename, filestorage.mimetype, filestorage.stream + ) + attachment = { + "_id": ObjectId(), + "name": filename, + "type": filestorage.mimetype, + "url": imgurl, + } # We get the data back from the image server; this is to make sure the content did make it there; also the stream is probably in an inconsistent state # Not the most efficient but the safest perhaps. - with parseImageStoreURL(imgurl).return_url_contents(experiment_name, imgurl) as imgget, tempfile.NamedTemporaryFile("w+b") as fd: - tfname, tf_thmbname = fd.name, fd.name+".png" + with ( + parseImageStoreURL(imgurl).return_url_contents( + experiment_name, imgurl + ) as imgget, + tempfile.NamedTemporaryFile("w+b") as fd, + ): + tfname, tf_thmbname = fd.name, fd.name + ".png" shutil.copyfileobj(imgget, fd, 1024) fd.flush() attachment_size = os.fstat(fd.fileno()).st_size if attachment_size > MAX_ATTACHMENT_SIZE: - raise LgbkException("We limit the size of attachments to %s M" % str(MAX_ATTACHMENT_SIZE/(1024*1024))) + raise LgbkException( + "We limit the size of attachments to %s M" + % str(MAX_ATTACHMENT_SIZE / (1024 * 1024)) + ) logger.info("Attachment size %s", attachment_size) - cp = subprocess.run(["convert", "-thumbnail", "128", tfname, tf_thmbname], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False, timeout=30) + cp = subprocess.run( + ["convert", "-thumbnail", "128", tfname, tf_thmbname], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + timeout=30, + ) logger.info(cp) if cp.returncode == 0 and os.path.exists(tf_thmbname): - with open(tf_thmbname, 'rb') as thmb_s: - thmb_imgurl = parseImageStoreURL(imagestoreurl).store_file_and_return_url(experiment_name, "preview_" + filename, "image/png", thmb_s) + with open(tf_thmbname, "rb") as thmb_s: + thmb_imgurl = parseImageStoreURL( + imagestoreurl + ).store_file_and_return_url( + experiment_name, "preview_" + filename, "image/png", thmb_s + ) attachment["preview_url"] = thmb_imgurl else: - logger.warn("Skipping generating a thumbnail for %s for experiment %s", filename, experiment_name) + logger.warn( + "Skipping generating a thumbnail for %s for experiment %s", + filename, + experiment_name, + ) attachments.append(attachment) return attachments -def post_new_log_entry(experiment_name, author, log_content, files, run_num=None, shift=None, root=None, parent=None, email_to=None, tags=None, title=None, post_to_elogs=None, jira_ticket=None): +def post_new_log_entry( + experiment_name, + author, + log_content, + files, + run_num=None, + shift=None, + root=None, + parent=None, + email_to=None, + tags=None, + title=None, + post_to_elogs=None, + jira_ticket=None, +): """ Create a new log entry. """ expdb = logbookclient[experiment_name] - attachments = __upload_attachments_to_imagestore_and_return_urls(experiment_name, files) + attachments = __upload_attachments_to_imagestore_and_return_urls( + experiment_name, files + ) now_ts = datetime.datetime.utcnow() elog_doc = { "relevance_time": now_ts, "insert_time": now_ts, "author": author, "content": log_content, - "content_type": "TEXT" + "content_type": "TEXT", } if attachments: elog_doc["attachments"] = attachments @@ -834,100 +1226,149 @@ def post_new_log_entry(experiment_name, author, log_content, files, run_num=None if jira_ticket: elog_doc["jira_ticket"] = jira_ticket - ins_id = expdb['elog'].insert_one(elog_doc).inserted_id - entry = expdb['elog'].find_one({"_id": ins_id}) + ins_id = expdb["elog"].insert_one(elog_doc).inserted_id + entry = expdb["elog"].find_one({"_id": ins_id}) return entry + def delete_elog_entry(experiment_name, entry_id, userid): """ Mark the elog entry specified by the entry_id as being deleted This is a logical delete; so we add a deleted_by and deleted_time """ expdb = logbookclient[experiment_name] - result = expdb['elog'].update_one({"_id": ObjectId(entry_id)}, {"$set": { "deleted_by": userid, "deleted_time": datetime.datetime.utcnow()}}) + result = expdb["elog"].update_one( + {"_id": ObjectId(entry_id)}, + {"$set": {"deleted_by": userid, "deleted_time": datetime.datetime.utcnow()}}, + ) if result.modified_count <= 0: return False - current_entry = expdb['elog'].find_one({"_id": ObjectId(entry_id)}) + current_entry = expdb["elog"].find_one({"_id": ObjectId(entry_id)}) if "post_to_elogs" in current_entry and current_entry["post_to_elogs"]: for post_to_elog in current_entry["post_to_elogs"]: logger.debug("Deleting instrument elog entry in %s", post_to_elog) - logbookclient[post_to_elog]["elog"].update_one({"src_id": current_entry["_id"], "src_expname": experiment_name}, {"$set": { "deleted_by": userid, "deleted_time": datetime.datetime.utcnow()}}) + logbookclient[post_to_elog]["elog"].update_one( + {"src_id": current_entry["_id"], "src_expname": experiment_name}, + { + "$set": { + "deleted_by": userid, + "deleted_time": datetime.datetime.utcnow(), + } + }, + ) return True -def modify_elog_entry(experiment_name, entry_id, userid, new_content, email_to, tags, files, title=None, run_num=None): + +def modify_elog_entry( + experiment_name, + entry_id, + userid, + new_content, + email_to, + tags, + files, + title=None, + run_num=None, +): """ Change the content for the specified elog entry. We have to retain the history of the change; so we clone the existing entry but make the clone a child of the existing entry. We also set the deleted_by/deleted_time for the clone; so it should show up as deleted. """ expdb = logbookclient[experiment_name] - attachments = __upload_attachments_to_imagestore_and_return_urls(experiment_name, files) - current_entry = expdb['elog'].find_one({"_id": ObjectId(entry_id)}) + attachments = __upload_attachments_to_imagestore_and_return_urls( + experiment_name, files + ) + current_entry = expdb["elog"].find_one({"_id": ObjectId(entry_id)}) hist_entry = copy.deepcopy(current_entry) del hist_entry["_id"] hist_entry["parent"] = current_entry["_id"] - hist_entry["root"] = current_entry["root"] if "root" in current_entry else current_entry["_id"] + hist_entry["root"] = ( + current_entry["root"] if "root" in current_entry else current_entry["_id"] + ) hist_entry["deleted_by"] = userid hist_entry["deleted_time"] = datetime.datetime.utcnow() hist_entry["relevance_time"] = hist_entry["deleted_time"] if "previous_version" in current_entry: hist_entry["previous_version"] = current_entry["previous_version"] - hist_result = expdb['elog'].insert_one(hist_entry) + hist_result = expdb["elog"].insert_one(hist_entry) if hist_result and hist_result.inserted_id: - modification = {"$set": { - "content": new_content, - "author": userid, - "tags": tags, - "previous_version": hist_result.inserted_id - }} + modification = { + "$set": { + "content": new_content, + "author": userid, + "tags": tags, + "previous_version": hist_result.inserted_id, + } + } if attachments: - modification["$push"] = { "attachments": { "$each": attachments }} + modification["$push"] = {"attachments": {"$each": attachments}} if email_to: - modification.setdefault("$addToSet", {})["email_to"] = { "$each": email_to } + modification.setdefault("$addToSet", {})["email_to"] = {"$each": email_to} if title: modification["$set"]["title"] = title if not hist_entry.get("run_num", None) == run_num: - if run_num == None: - modification["$unset"] = { "run_num" : 1 } + if run_num is None: + modification["$unset"] = {"run_num": 1} else: modification["$set"]["run_num"] = run_num - result = expdb['elog'].update_one({"_id": current_entry["_id"]}, modification) + result = expdb["elog"].update_one({"_id": current_entry["_id"]}, modification) if result.modified_count <= 0: return False if "post_to_elogs" in current_entry and current_entry["post_to_elogs"]: del modification["$set"]["previous_version"] for post_to_elog in current_entry["post_to_elogs"]: logger.debug("Updating instrument elog entry in %s", post_to_elog) - logbookclient[post_to_elog]["elog"].update_one({"src_id": current_entry["_id"], "src_expname": experiment_name}, modification) + logbookclient[post_to_elog]["elog"].update_one( + {"src_id": current_entry["_id"], "src_expname": experiment_name}, + modification, + ) return True return False + def post_related_elog_entry(related_experiment, src_experiment, src_elog_entry_id): - ''' + """ Copy an elog entry into a related elog; typically an instrument elog When we copy, we maintain the source experiment and source elog entry id to make it easier to tie children to the parent. - ''' + """ expdb = logbookclient[related_experiment] - src_expdb = logbookclient[src_experiment] src_elog_entry = get_specific_elog_entry(src_experiment, src_elog_entry_id) - if "root" not in src_elog_entry and "parent" not in src_elog_entry and related_experiment not in src_elog_entry.get("post_to_elogs", []): - expdb["elog"].update_one({"_id": src_elog_entry["_id"]}, {"$addToSet": {"post_to_elogs": related_experiment}}) + if ( + "root" not in src_elog_entry + and "parent" not in src_elog_entry + and related_experiment not in src_elog_entry.get("post_to_elogs", []) + ): + expdb["elog"].update_one( + {"_id": src_elog_entry["_id"]}, + {"$addToSet": {"post_to_elogs": related_experiment}}, + ) del src_elog_entry["_id"] if "post_to_elogs" in src_elog_entry: del src_elog_entry["post_to_elogs"] src_elog_entry["src_expname"] = src_experiment src_elog_entry["src_id"] = src_elog_entry_id + def __check_and_link__(attr): if attr in src_elog_entry: - rel_entry = expdb['elog'].find_one({"src_expname": src_experiment, "src_id": src_elog_entry[attr]}) + rel_entry = expdb["elog"].find_one( + {"src_expname": src_experiment, "src_id": src_elog_entry[attr]} + ) if not rel_entry: - logger.error("Cannot find related entry %s with id %s in %s", attr, src_elog_entry[attr], related_experiment) + logger.error( + "Cannot find related entry %s with id %s in %s", + attr, + src_elog_entry[attr], + related_experiment, + ) else: src_elog_entry[attr] = rel_entry["_id"] + __check_and_link__("root") __check_and_link__("parent") mg_imgstore = parseImageStoreURL("mongo://") + def __copy_attachment__(attch, attr): if attr in attch and attch[attr].startswith("mongo://"): aurl = attch[attr] @@ -936,112 +1377,172 @@ def __copy_attachment__(attch, attr): if not bio: logger.debug("Cannot get attachment contents for %s", aurl) return None - murl = mg_imgstore.store_file_and_return_url(related_experiment, attch["name"], attch["type"], bio) + murl = mg_imgstore.store_file_and_return_url( + related_experiment, attch["name"], attch["type"], bio + ) logger.debug("Copied %s to mongo as %s", aurl, murl) attch[attr] = murl return murl - except: + except Exception: logger.exception("Exception copying attachment %s", aurl) return None + for attch in src_elog_entry.get("attachments", []): __copy_attachment__(attch, "url") __copy_attachment__(attch, "preview_url") - ins_id = expdb['elog'].insert_one(src_elog_entry).inserted_id - entry = expdb['elog'].find_one({"_id": ins_id}) + ins_id = expdb["elog"].insert_one(src_elog_entry).inserted_id + entry = expdb["elog"].find_one({"_id": ins_id}) return entry + def get_related_instrument_elog_entries(experiment_name, entry_id): - ''' + """ Get the related instrument elog entries for a given elog entry if they are present. Returns a dict of experiment/elog name with the related entry. - ''' + """ current_entry = get_specific_elog_entry(experiment_name, entry_id) ret = {} - if current_entry and "post_to_elogs" in current_entry and current_entry["post_to_elogs"]: + if ( + current_entry + and "post_to_elogs" in current_entry + and current_entry["post_to_elogs"] + ): for post_to_elog in current_entry["post_to_elogs"]: - related_entry = logbookclient[post_to_elog]["elog"].find_one({"src_id": current_entry["_id"], "src_expname": experiment_name}) + related_entry = logbookclient[post_to_elog]["elog"].find_one( + {"src_id": current_entry["_id"], "src_expname": experiment_name} + ) if related_entry: ret[post_to_elog] = related_entry return ret + def get_elog_authors(experiment_name): - ''' + """ Get the distinct authors for the elog entries. - ''' + """ expdb = logbookclient[experiment_name] - return expdb['elog'].distinct("author") + return expdb["elog"].distinct("author") + def get_elog_tags(experiment_name): - ''' + """ Get the distinct tags for the elog entries. - ''' + """ sitedb = logbookclient["site"] - ins = sitedb["instruments"].find_one({ "_id": get_experiment_info(experiment_name)["instrument"] }) + ins = sitedb["instruments"].find_one( + {"_id": get_experiment_info(experiment_name)["instrument"]} + ) ins_tags = ins.get("params", {}).get("tags", "").split(" ") expdb = logbookclient[experiment_name] - return expdb['elog'].distinct("tags") + ins_tags + return expdb["elog"].distinct("tags") + ins_tags + def get_elog_emails(experiment_name): - ''' + """ Get all the email addresses that we sent elog messages to that are recorded in the db. - ''' + """ expdb = logbookclient[experiment_name] - emails = list({x for x in [y for x in [ x['email_to'] for x in filter(lambda x : x, list(expdb['elog'].find({}, { "email_to": 1, "_id": 0 }))) ] for y in x]}) + emails = list( + { + x + for x in [ + y + for x in [ + x["email_to"] + for x in filter( + lambda x: x, + list(expdb["elog"].find({}, {"email_to": 1, "_id": 0})), + ) + ] + for y in x + ] + } + ) # Add in site and instrument specific email mailing lists. - if logbookclient['site']['siteinfo'].find_one({}, {"_id": 0, "params.elog_mailing_lists": 1}): - emails.extend(logbookclient['site']['siteinfo'].find_one({}, {"_id": 0, "params.elog_mailing_lists": 1}).get('params', {}).get('elog_mailing_lists', [])) - ins = get_experiment_info(experiment_name)['instrument'] - if logbookclient['site']['instruments'].find_one({"_id": ins}, {"_id": 0, "params.elog_mailing_lists": 1}): - emails.extend(logbookclient['site']['instruments'].find_one({"_id": ins}, {"_id": 0, "params.elog_mailing_lists": 1}).get('params', {}).get('elog_mailing_lists', [])) + if logbookclient["site"]["siteinfo"].find_one( + {}, {"_id": 0, "params.elog_mailing_lists": 1} + ): + emails.extend( + logbookclient["site"]["siteinfo"] + .find_one({}, {"_id": 0, "params.elog_mailing_lists": 1}) + .get("params", {}) + .get("elog_mailing_lists", []) + ) + ins = get_experiment_info(experiment_name)["instrument"] + if logbookclient["site"]["instruments"].find_one( + {"_id": ins}, {"_id": 0, "params.elog_mailing_lists": 1} + ): + emails.extend( + logbookclient["site"]["instruments"] + .find_one({"_id": ins}, {"_id": 0, "params.elog_mailing_lists": 1}) + .get("params", {}) + .get("elog_mailing_lists", []) + ) return sorted(emails) + def get_elog_email_subscriptions(experiment_name): - ''' + """ The logbook will send email messages when new elog messages are posted to the elog. Get all the subscribers who have subscribed to email messages for this experiment. - ''' + """ expdb = logbookclient[experiment_name] return list(expdb.subscribers.find({})) + def get_elog_email_subscriptions_emails(experiment_name): - ''' + """ Get an array of email addresses of folks who have subscried to emails for this experiment. - ''' + """ expdb = logbookclient[experiment_name] - return [x['email_address'] for x in list(expdb.subscribers.find({}, {"email_address": 1}))] + return [ + x["email_address"] + for x in list(expdb.subscribers.find({}, {"email_address": 1})) + ] + def elog_email_subscribe(experiment_name, userid): - ''' + """ Add the specified user to the email subscriptions - ''' + """ expdb = logbookclient[experiment_name] - result = expdb.subscribers.insert_one({"_id": userid, "subscriber": userid, "email_address": userid + "@slac.stanford.edu", "subscribed_time": datetime.datetime.utcnow() }) + result = expdb.subscribers.insert_one( + { + "_id": userid, + "subscriber": userid, + "email_address": userid + "@slac.stanford.edu", + "subscribed_time": datetime.datetime.utcnow(), + } + ) return result.acknowledged + def elog_email_unsubscribe(experiment_name, userid): - ''' + """ Remove the specified user from the email subscriptions - ''' + """ expdb = logbookclient[experiment_name] result = expdb.subscribers.delete_one({"_id": userid}) return result.acknowledged + def get_site_naming_conventions(): - ''' + """ Get the naming conventions from the site config. Naming conventions are object/attribute documents; for example, experiment.name Each document has a placeholder, tooltip and validation_regex attribute. The placeholder is used as the HTML placeholder/example for the attribute. The validation regex will do some basic regex; however, it's probably very difficult to cover all cases with a regex. The tooltip should have enought detail to outlines the naming convention for operators. - ''' + """ sitedb = logbookclient["site"] s_config = sitedb["site_config"].find_one({}) if s_config: return s_config.get("naming_conventions", {}) return {} + def get_site_file_types(): """ For the file manager, we classify the files in the file catalog into types. @@ -1050,17 +1551,17 @@ def get_site_file_types(): Some users convert these into HDF5 (h5) files and primarily use those for analysis. Typically, users use one of these types of files. To capture this, we define filemanager_file_types in the site_config. - "filemanager_file_types" : { - "XTC" : { - "name" : "XTC", - "label" : "XTC files", - "tooltip" : "Large and smalldata xtc and xtc2 files", - "patterns" : [ - "^.*/xtc/.*.xtc$", - "^.*/xtc/.*.xtc2$" - ], - "selected" : true - }, + "filemanager_file_types" : { + "XTC" : { + "name" : "XTC", + "label" : "XTC files", + "tooltip" : "Large and smalldata xtc and xtc2 files", + "patterns" : [ + "^.*/xtc/.*.xtc$", + "^.*/xtc/.*.xtc2$" + ], + "selected" : true + }, The filemanager supports restoration of one or all of these file types. """ sitedb = logbookclient["site"] @@ -1069,13 +1570,16 @@ def get_site_file_types(): return s_config.get("filemanager_file_types", {}) return {} -def get_instrument_elogs(experiment_name, include_instrument_elogs=True, include_site_spanning_elogs=True): - ''' + +def get_instrument_elogs( + experiment_name, include_instrument_elogs=True, include_site_spanning_elogs=True +): + """ Get the associated elogs for experiment. This consists of the elog(s) for the instrument (instrument param elog in the instrument object). And global experiment_spanning_elogs logs (like the LCLS sample delivery elog) in the site's info object. Also can be specified on a per experiment basis as the experiment parameter xpost_elogs ( comma separated list ). - ''' + """ ret = [] exp_info = get_experiment_info(experiment_name) exp_x_post_elogs = exp_info.get("params", {}).get("xpost_elogs", None) @@ -1083,226 +1587,429 @@ def get_instrument_elogs(experiment_name, include_instrument_elogs=True, include ret.extend([x.strip() for x in exp_x_post_elogs.split(",")]) sitedb = logbookclient["site"] if include_instrument_elogs: - instrument_elog = sitedb["instruments"].find_one({"_id": exp_info["instrument"]}).get("params", {}).get("elog", None) + instrument_elog = ( + sitedb["instruments"] + .find_one({"_id": exp_info["instrument"]}) + .get("params", {}) + .get("elog", None) + ) if instrument_elog: ret.append(instrument_elog) if include_site_spanning_elogs: siteinfo = sitedb["site_config"].find_one() - if siteinfo and 'experiment_spanning_elogs' in siteinfo and siteinfo['experiment_spanning_elogs']: - ret.extend(siteinfo['experiment_spanning_elogs']) + if ( + siteinfo + and "experiment_spanning_elogs" in siteinfo + and siteinfo["experiment_spanning_elogs"] + ): + ret.extend(siteinfo["experiment_spanning_elogs"]) return ret + def get_experiment_files(experiment_name, sample_name=None): - ''' + """ Get the files for the given experiment - ''' + """ expdb = logbookclient[experiment_name] if not sample_name or sample_name == "All Samples": - return [file for file in expdb['file_catalog'].find().sort([("run_num", -1), ("create_timestamp", -1)])] + return [ + file + for file in expdb["file_catalog"] + .find() + .sort([("run_num", -1), ("create_timestamp", -1)]) + ] else: - return [x for x in expdb.samples.aggregate([ - { "$match": { "name": sample_name }}, - { "$lookup": { "from": "runs", "localField": "_id", "foreignField": "sample", "as": "run"}}, - { "$unwind": "$run" }, - { "$replaceRoot": { "newRoot": "$run" } }, - { "$lookup": { "from": "file_catalog", "localField": "num", "foreignField": "run_num", "as": "file_catalog"}}, - { "$unwind": "$file_catalog" }, - { "$replaceRoot": { "newRoot": "$file_catalog" } }, - { "$sort": { "run_num": -1 }} - ])] + return [ + x + for x in expdb.samples.aggregate( + [ + {"$match": {"name": sample_name}}, + { + "$lookup": { + "from": "runs", + "localField": "_id", + "foreignField": "sample", + "as": "run", + } + }, + {"$unwind": "$run"}, + {"$replaceRoot": {"newRoot": "$run"}}, + { + "$lookup": { + "from": "file_catalog", + "localField": "num", + "foreignField": "run_num", + "as": "file_catalog", + } + }, + {"$unwind": "$file_catalog"}, + {"$replaceRoot": {"newRoot": "$file_catalog"}}, + {"$sort": {"run_num": -1}}, + ] + ) + ] + def get_experiment_files_for_run(experiment_name, run_num): - ''' + """ Get the files for the given experiment for the specified run - ''' + """ expdb = logbookclient[experiment_name] - return [file for file in expdb['file_catalog'].find({"run_num": run_num}).sort([("run_num", -1), ("create_timestamp", -1)])] + return [ + file + for file in expdb["file_catalog"] + .find({"run_num": run_num}) + .sort([("run_num", -1), ("create_timestamp", -1)]) + ] + def get_experiment_files_for_run_for_live_mode(experiment_name, run_num): - ''' + """ Get a minimal set of information for psana live mode. Return only the path information for only the xtc/xtc2 files in the xtc folder ( and not it's children ). - ''' - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED) - ret = [file["path"] for file in expdb['file_catalog'].find({"run_num": run_num, "path": { "$regex": re.compile(".*/xtc/[^/]*[.](xtc|xtc2)$") }}, {"_id": -0, "path": 1}).sort([("path", 1)])] + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED + ) + ret = [ + file["path"] + for file in expdb["file_catalog"] + .find( + { + "run_num": run_num, + "path": {"$regex": re.compile(".*/xtc/[^/]*[.](xtc|xtc2)$")}, + }, + {"_id": -0, "path": 1}, + ) + .sort([("path", 1)]) + ] return ret -def get_experiment_files_for_run_for_live_mode_at_location(experiment_name, run_num, location): - ''' + +def get_experiment_files_for_run_for_live_mode_at_location( + experiment_name, run_num, location +): + """ Return some basic information on whether the run is complete and files are available at a location. - ''' - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED) + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED + ) run_doc = expdb.runs.find_one({"num": run_num}) ret = { "begin_time": run_doc["begin_time"], "end_time": run_doc.get("end_time", None), "is_closed": is_run_closed(experiment_name, run_num), - "files": [] + "files": [], } - files = expdb['file_catalog'].find({"run_num": run_num, "path": { "$regex": re.compile(".*/xtc/[^/]*[.](xtc|xtc2)$") }}, {"_id": -0, "path": 1, "locations": 1}).sort([("path", 1)]) + files = ( + expdb["file_catalog"] + .find( + { + "run_num": run_num, + "path": {"$regex": re.compile(".*/xtc/[^/]*[.](xtc|xtc2)$")}, + }, + {"_id": -0, "path": 1, "locations": 1}, + ) + .sort([("path", 1)]) + ) for file in files: - ret["files"].append({"path": file["path"], "is_present": "asof" in file.get("locations", {}).get(location, {}).keys()}) - - ret["all_present"] = ret["is_closed"] and all(map(lambda x : x["is_present"], ret["files"])) + ret["files"].append( + { + "path": file["path"], + "is_present": "asof" + in file.get("locations", {}).get(location, {}).keys(), + } + ) + + ret["all_present"] = ret["is_closed"] and all( + map(lambda x: x["is_present"], ret["files"]) + ) return ret + def get_experiment_files_for_live_mode_at_location(experiment_name, location): - ''' + """ A experiment-wide version of get_experiment_files_for_run_for_live_mode_at_location - ''' - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED) - aggr = expdb.runs.aggregate([ - { "$lookup": { "from": "file_catalog", "localField": "num", "foreignField": "run_num", "as": "files"}}, - { "$project": { "_id": 0, "num": 1, "begin_time" : 1, "end_time": 1, "files.path": 1, "files.locations." + location + ".asof": 1 } }, - { "$sort": {"num": -1} } - ]) + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED + ) + aggr = expdb.runs.aggregate( + [ + { + "$lookup": { + "from": "file_catalog", + "localField": "num", + "foreignField": "run_num", + "as": "files", + } + }, + { + "$project": { + "_id": 0, + "num": 1, + "begin_time": 1, + "end_time": 1, + "files.path": 1, + "files.locations." + location + ".asof": 1, + } + }, + {"$sort": {"num": -1}}, + ] + ) ret = [] for x in aggr: - robj = { "run_num": x["num"], "begin_time": x["begin_time"], "end_time": x.get("end_time", None), "is_closed": "end_time" in x and x["end_time"] != None, "files": [] } + robj = { + "run_num": x["num"], + "begin_time": x["begin_time"], + "end_time": x.get("end_time", None), + "is_closed": "end_time" in x and x["end_time"] is not None, + "files": [], + } for f in x["files"]: - robj["files"].append({"path": f["path"], "is_present": "asof" in f.get("locations", {}).get(location, {}).keys()}) - robj["all_present"] = robj["is_closed"] and all(map(lambda x : x["is_present"], robj["files"])) + robj["files"].append( + { + "path": f["path"], + "is_present": "asof" + in f.get("locations", {}).get(location, {}).keys(), + } + ) + robj["all_present"] = robj["is_closed"] and all( + map(lambda x: x["is_present"], robj["files"]) + ) ret.append(robj) return ret + def get_exp_file_counts_by_extension(experiment_name): """ Return the file counts based on file type that we know about. """ expdb = logbookclient[experiment_name] - extensions = [ x for x in expdb.file_catalog.aggregate([ - { "$project": { "extension": { "$first": { "$reverseArray": { "$split": [ "$path", "."] }}}}}, - { "$group": { "_id": "$extension", "count": { "$sum": 1 } }}, - { "$project": { "_id": 0, "extension": "$_id", "count": "$count" }} - ])] - return { x["extension"] : x["count"] for x in extensions } + extensions = [ + x + for x in expdb.file_catalog.aggregate( + [ + { + "$project": { + "extension": { + "$first": {"$reverseArray": {"$split": ["$path", "."]}} + } + } + }, + {"$group": {"_id": "$extension", "count": {"$sum": 1}}}, + {"$project": {"_id": 0, "extension": "$_id", "count": "$count"}}, + ] + ) + ] + return {x["extension"]: x["count"] for x in extensions} + def get_experiment_runs(experiment_name, include_run_params=False, sample_name=None): - ''' + """ Get the runs for the given experiment. Does not include the run parameters by default - ''' + """ expdb = logbookclient[experiment_name] - run_params_projection = { "params": False } if not include_run_params else { "some_non_existent_field": False } + run_params_projection = ( + {"params": False} + if not include_run_params + else {"some_non_existent_field": False} + ) if not sample_name or sample_name == "All Samples": - run_docs = [run for run in expdb['runs'].find(projection=run_params_projection).sort([("num", -1)])] + run_docs = [ + run + for run in expdb["runs"] + .find(projection=run_params_projection) + .sort([("num", -1)]) + ] else: # We start at samples for performance reasons; - run_docs = [x for x in expdb.samples.aggregate([ - { "$match": { "name": sample_name }}, - { "$lookup": { "from": "runs", "localField": "_id", "foreignField": "sample", "as": "run"}}, - { "$unwind": "$run" }, # lookup generates an array field, we convert to a list of docs with a single field instead. - { "$replaceRoot": { "newRoot": "$run" } }, - { "$project": run_params_projection }, - { "$sort": { "num": -1 }} - ])] + run_docs = [ + x + for x in expdb.samples.aggregate( + [ + {"$match": {"name": sample_name}}, + { + "$lookup": { + "from": "runs", + "localField": "_id", + "foreignField": "sample", + "as": "run", + } + }, + { + "$unwind": "$run" + }, # lookup generates an array field, we convert to a list of docs with a single field instead. + {"$replaceRoot": {"newRoot": "$run"}}, + {"$project": run_params_projection}, + {"$sort": {"num": -1}}, + ] + ) + ] for run_doc in run_docs: if "sample" in run_doc: - run_doc["sample"] = expdb['samples'].find_one({"_id": run_doc["sample"]}) + run_doc["sample"] = expdb["samples"].find_one({"_id": run_doc["sample"]}) return run_docs + def get_experiment_run_document(experiment_name, rnum): - ''' + """ Get the run document for the specified run. - ''' + """ expdb = logbookclient[experiment_name] - run_doc = expdb['runs'].find_one({"num": rnum}) + run_doc = expdb["runs"].find_one({"num": rnum}) if "sample" in run_doc: - run_doc["sample"] = expdb['samples'].find_one({"_id": run_doc["sample"]}) + run_doc["sample"] = expdb["samples"].find_one({"_id": run_doc["sample"]}) return run_doc + def get_all_run_tables(experiment_name, instrument): - ''' + """ Get specifications for both the default and user defined run tables. The default run tables are based on these items. * Find all run_param_descriptions that are summaries. These have their param names separated by the "/" character into the category and param name. We create a run table definition dynamically based on category. * The run tables defined in the site database are added to all experiments. These are ahead of the experiment specific ones; so the default run tables are the system wide run tables. * Finally, the experiment specific run tables are added. - ''' + """ expdb = logbookclient[experiment_name] sitedb = logbookclient["site"] allRunTables = [] + def mark_sys(x): x["is_system_run_table"] = True x["is_editable"] = False return x - system_run_tables = [ mark_sys(r) for r in sitedb["run_tables"].find({"$or": [ {"instrument": instrument}, {"instrument": { "$exists": False}}]})] + + system_run_tables = [ + mark_sys(r) + for r in sitedb["run_tables"].find( + {"$or": [{"instrument": instrument}, {"instrument": {"$exists": False}}]} + ) + ] allRunTables.extend(system_run_tables) - allRunTables.extend([x for x in expdb['run_tables'].find()]) + allRunTables.extend([x for x in expdb["run_tables"].find()]) - mimes = { "params." + x["param_name"] : x["type"] for x in sitedb['run_param_descriptions'].find({"type": { "$exists": True }})} - mimes.update({ "params." + x["param_name"] : x["type"] for x in expdb['run_param_descriptions'].find({"type": { "$exists": True }})}) + mimes = { + "params." + x["param_name"]: x["type"] + for x in sitedb["run_param_descriptions"].find({"type": {"$exists": True}}) + } + mimes.update( + { + "params." + x["param_name"]: x["type"] + for x in expdb["run_param_descriptions"].find({"type": {"$exists": True}}) + } + ) # The run table categories change with time. This is where we patch for versions of the instrument scientist source list. for rt in allRunTables: for coldef in rt["coldefs"]: - if coldef['type'].startswith('EPICS:'): - coldef['type'] = coldef['type'].replace('EPICS:', 'EPICS/') - coldef['mime_type'] = mimes.get(coldef['source'], "text") + if coldef["type"].startswith("EPICS:"): + coldef["type"] = coldef["type"].replace("EPICS:", "EPICS/") + coldef["mime_type"] = mimes.get(coldef["source"], "text") if "sort_index" not in rt: rt["sort_index"] = 100 # Sort by run table sort_index and then by name. - allRunTables = sorted(allRunTables, key=itemgetter('sort_index', 'name')) + allRunTables = sorted(allRunTables, key=itemgetter("sort_index", "name")) return allRunTables + def get_runtable_data(experiment_name, instrument, tableName, sampleName): - ''' + """ Get the data from the run tables for the given table. In addition to the basic run data, we add the sources for the given run table. This is mostly a matter of constructing the appropriate mongo filters. If sampleName is specified, we restrict the returned data to runs associated with the sample. Otherwise, we return all runs. - ''' - tableDef = next(x for x in get_all_run_tables(experiment_name, instrument) if x['name'] == tableName and not x.get("is_template", False)) - sources = { "num": 1, "begin_time": 1, "end_time": 1 } - sources.update({ x['source'] : 1 for x in tableDef['coldefs']}) - if tableDef.get("table_type", None) == "generatedtable" or tableDef.get("table_type", None) == "generatedscatter": - logger.debug("Getting run table data based on pattern matches '%s'", tableDef["patterns"]) - allsources = [ x["source"] for y in get_runtable_sources(experiment_name).values() for x in y ] + """ + tableDef = next( + x + for x in get_all_run_tables(experiment_name, instrument) + if x["name"] == tableName and not x.get("is_template", False) + ) + sources = {"num": 1, "begin_time": 1, "end_time": 1} + sources.update({x["source"]: 1 for x in tableDef["coldefs"]}) + if ( + tableDef.get("table_type", None) == "generatedtable" + or tableDef.get("table_type", None) == "generatedscatter" + ): + logger.debug( + "Getting run table data based on pattern matches '%s'", tableDef["patterns"] + ) + allsources = [ + x["source"] + for y in get_runtable_sources(experiment_name).values() + for x in y + ] ptrn = re.compile(tableDef["patterns"]) - sources.update({ x : 1 for x in allsources if ptrn.match(x.replace("params.", ""))}) + sources.update( + {x: 1 for x in allsources if ptrn.match(x.replace("params.", ""))} + ) query = {} if sampleName: logger.debug("Getting run table data for sample %s", sampleName) sample_doc = get_sample_for_experiment_by_name(experiment_name, sampleName) - query = { "sample": sample_doc["_id"] } if sample_doc else query - rtdata = [x for x in logbookclient[experiment_name]['runs'].find(query, sources).sort([("num", -1)])] # Use sources as a filter to find + query = {"sample": sample_doc["_id"]} if sample_doc else query + rtdata = [ + x + for x in logbookclient[experiment_name]["runs"] + .find(query, sources) + .sort([("num", -1)]) + ] # Use sources as a filter to find for rtd in rtdata: - if 'duration' in sources.keys() and 'end_time' in rtd and rtd['end_time'] and 'begin_time' in rtd: - rtd['duration'] = (rtd['end_time'] - rtd['begin_time']).total_seconds() - if 'begin_time' in rtd and rtd['begin_time']: - rtd['begin_time_epoch'] = int(rtd['begin_time'].timestamp()*1000) - if 'end_time' in rtd and rtd['end_time']: - rtd['end_time_epoch'] = int(rtd['end_time'].timestamp()*1000) - if 'sample' in sources.keys(): - samples = { x["_id"] : x for x in get_samples(experiment_name) } + if ( + "duration" in sources.keys() + and "end_time" in rtd + and rtd["end_time"] + and "begin_time" in rtd + ): + rtd["duration"] = (rtd["end_time"] - rtd["begin_time"]).total_seconds() + if "begin_time" in rtd and rtd["begin_time"]: + rtd["begin_time_epoch"] = int(rtd["begin_time"].timestamp() * 1000) + if "end_time" in rtd and rtd["end_time"]: + rtd["end_time_epoch"] = int(rtd["end_time"].timestamp() * 1000) + if "sample" in sources.keys(): + samples = {x["_id"]: x for x in get_samples(experiment_name)} + def __replace_with_sample_name(x): - if 'sample' in x: - x['sample'] = samples[x['sample']]['name'] + if "sample" in x: + x["sample"] = samples[x["sample"]]["name"] return x + rtdata = map(__replace_with_sample_name, rtdata) return rtdata + def get_run_param_descriptions(experiment_name): - ''' + """ Get the run param descriptions for this experiment. - ''' - return [x for x in logbookclient[experiment_name]['run_param_descriptions'].find({}).sort([("name", 1)])] + """ + return [ + x + for x in logbookclient[experiment_name]["run_param_descriptions"] + .find({}) + .sort([("name", 1)]) + ] + def add_update_run_param_descriptions(experiment_name, param_descs): - ''' + """ Add or update the run parameter descriptions for this experiment. - ''' + """ for k, v in param_descs.items(): - logbookclient[experiment_name]['run_param_descriptions'].update_one({"param_name": k}, {"$set": {"description": v}}, upsert=True) + logbookclient[experiment_name]["run_param_descriptions"].update_one( + {"param_name": k}, {"$set": {"description": v}}, upsert=True + ) return True + def get_runtable_sources(experiment_name): - ''' + """ Get the sources for user defined run tables. This is a combination of these items; not all of these are mutually exclusive. --> The attributes of the run itself @@ -1310,146 +2017,300 @@ def get_runtable_sources(experiment_name): --> The run_param_descriptions. --> The instrument leads maintain a per instrument list of EPICS variables in a JSON file external to the logbook. We combine all of these into a param name --> category+description - ''' + """ expdb = logbookclient[experiment_name] sitedb = logbookclient["site"] - instrument = expdb.info.find_one({})['instrument'] + instrument = expdb.info.find_one({})["instrument"] rtbl_sources = {} - rtbl_sources["Run Info"] = [{"label": "Begin Time", "description": "The start of the run", "source": "begin_time", "category": "Run Info"}, - {"label": "End time", "description": "The end of the run", "source": "end_time", "category": "Run Info"}, - {"label": "Number", "description": "The run number", "source": "num", "category": "Run Info"}, - {"label": "Type", "description": "The run type", "source": "type", "category": "Run Info"}, - {"label": "Sample", "description": "The sample associated with the run", "source": "sample", "category": "Run Info"}, - {"label": "Run Duration", "description": "The duration of the run", "source": "duration", "category": "Run Info"}] - rtbl_sources["Editables"] = [ { "label": x["_id"], "description": x["_id"], "source": "editable_params."+x["_id"]+".value", "category": "Editables" } for x in expdb.runs.aggregate([ - { "$project": { "editables": { "$objectToArray": "$editable_params" } } }, - { "$unwind": "$editables" }, - { "$group": { "_id": "$editables.k", "total": { "$sum": 1 } } } ])] - rtbl_sources["Calibrations"] = [{"label": "Calibrations/comment", "description": "Calibrations/comment", "source": "params.Calibrations/comment", "category": "Calibrations"}, - {"label": "Calibrations/dark", "description": "Calibrations/dark", "source": "params.Calibrations/dark", "category": "Calibrations"}, - {"label": "Calibrations/flat", "description": "Calibrations/flat", "source": "params.Calibrations/flat", "category": "Calibrations"}, - {"label": "Calibrations/geom", "description": "Calibrations/geom", "source": "params.Calibrations/geom", "category": "Calibrations"}] - rtbl_sources["Misc"] = [{"label": "Separator", "description": "A column separator", "source": "Separator", "category": "Misc"}] + rtbl_sources["Run Info"] = [ + { + "label": "Begin Time", + "description": "The start of the run", + "source": "begin_time", + "category": "Run Info", + }, + { + "label": "End time", + "description": "The end of the run", + "source": "end_time", + "category": "Run Info", + }, + { + "label": "Number", + "description": "The run number", + "source": "num", + "category": "Run Info", + }, + { + "label": "Type", + "description": "The run type", + "source": "type", + "category": "Run Info", + }, + { + "label": "Sample", + "description": "The sample associated with the run", + "source": "sample", + "category": "Run Info", + }, + { + "label": "Run Duration", + "description": "The duration of the run", + "source": "duration", + "category": "Run Info", + }, + ] + rtbl_sources["Editables"] = [ + { + "label": x["_id"], + "description": x["_id"], + "source": "editable_params." + x["_id"] + ".value", + "category": "Editables", + } + for x in expdb.runs.aggregate( + [ + {"$project": {"editables": {"$objectToArray": "$editable_params"}}}, + {"$unwind": "$editables"}, + {"$group": {"_id": "$editables.k", "total": {"$sum": 1}}}, + ] + ) + ] + rtbl_sources["Calibrations"] = [ + { + "label": "Calibrations/comment", + "description": "Calibrations/comment", + "source": "params.Calibrations/comment", + "category": "Calibrations", + }, + { + "label": "Calibrations/dark", + "description": "Calibrations/dark", + "source": "params.Calibrations/dark", + "category": "Calibrations", + }, + { + "label": "Calibrations/flat", + "description": "Calibrations/flat", + "source": "params.Calibrations/flat", + "category": "Calibrations", + }, + { + "label": "Calibrations/geom", + "description": "Calibrations/geom", + "source": "params.Calibrations/geom", + "category": "Calibrations", + }, + ] + rtbl_sources["Misc"] = [ + { + "label": "Separator", + "description": "A column separator", + "source": "Separator", + "category": "Misc", + } + ] + # Mongo currently does not support finding the leaves of documents if they have embedded fields; so we have to loop thru all the runs and do this in python. def join_key(current_key, new_key): return current_key + "." + new_key if current_key else new_key + def get_leaves_of_a_document(current_key, items, keyset): for k, v in items: if isinstance(v, dict): get_leaves_of_a_document(join_key(current_key, k), v.items(), keyset) else: keyset.add(join_key(current_key, k)) + param_names = set() - for run in expdb.runs.find({}, {"params" : 1}): + for run in expdb.runs.find({}, {"params": 1}): get_leaves_of_a_document(None, run["params"].items(), param_names) - param_descs = { x["param_name"] : { "label" : x["param_name"], "description": x["description"] if x["description"] else x["param_name"], "category": x['param_name'].split('/')[0] if '/' in x['param_name'] else "EPICS:Additional parameters" } for x in expdb.run_param_descriptions.find({})} - site_param_descs = { x["param_name"] : { "label" : x["param_name"], "description": x["description"] if "description" in x else x["param_name"], "category": x['category'] if 'category' in x else "EPICS:Additional parameters" } for x in sitedb.run_param_descriptions.find({})} + param_descs = { + x["param_name"]: { + "label": x["param_name"], + "description": x["description"] if x["description"] else x["param_name"], + "category": x["param_name"].split("/")[0] + if "/" in x["param_name"] + else "EPICS:Additional parameters", + } + for x in expdb.run_param_descriptions.find({}) + } + site_param_descs = { + x["param_name"]: { + "label": x["param_name"], + "description": x["description"] if "description" in x else x["param_name"], + "category": x["category"] + if "category" in x + else "EPICS:Additional parameters", + } + for x in sitedb.run_param_descriptions.find({}) + } # Update the category and description from the instrument_scientists_run_table_defintions if present param_names_with_categories = [] for param_name in sorted(param_names): unescaped_param_name = reverse_escape_chars_for_mongo(param_name) - if instrument in instrument_scientists_run_table_defintions and unescaped_param_name in instrument_scientists_run_table_defintions[instrument]: - param_names_with_categories.append({ - "label" : unescaped_param_name, - "category": "EPICS/" + instrument_scientists_run_table_defintions[instrument][unescaped_param_name]["title"], - "description": instrument_scientists_run_table_defintions[instrument][unescaped_param_name].get("description", param_name), - "source": "params." + param_name }) - elif 'HEADER' in instrument_scientists_run_table_defintions and unescaped_param_name in instrument_scientists_run_table_defintions['HEADER']: - param_names_with_categories.append({ - "label" : unescaped_param_name, - "category": "EPICS/" + instrument_scientists_run_table_defintions['HEADER'][unescaped_param_name]["title"], - "description": instrument_scientists_run_table_defintions['HEADER'][unescaped_param_name].get("description", param_name), - "source": "params." + param_name }) + if ( + instrument in instrument_scientists_run_table_defintions + and unescaped_param_name + in instrument_scientists_run_table_defintions[instrument] + ): + param_names_with_categories.append( + { + "label": unescaped_param_name, + "category": "EPICS/" + + instrument_scientists_run_table_defintions[instrument][ + unescaped_param_name + ]["title"], + "description": instrument_scientists_run_table_defintions[ + instrument + ][unescaped_param_name].get("description", param_name), + "source": "params." + param_name, + } + ) + elif ( + "HEADER" in instrument_scientists_run_table_defintions + and unescaped_param_name + in instrument_scientists_run_table_defintions["HEADER"] + ): + param_names_with_categories.append( + { + "label": unescaped_param_name, + "category": "EPICS/" + + instrument_scientists_run_table_defintions["HEADER"][ + unescaped_param_name + ]["title"], + "description": instrument_scientists_run_table_defintions["HEADER"][ + unescaped_param_name + ].get("description", param_name), + "source": "params." + param_name, + } + ) elif unescaped_param_name in param_descs: - param_names_with_categories.append({ - "label" : unescaped_param_name, - "category": param_descs[unescaped_param_name]['category'], - "description": param_descs[unescaped_param_name].get("description", param_name), - "source": "params." + param_name }) + param_names_with_categories.append( + { + "label": unescaped_param_name, + "category": param_descs[unescaped_param_name]["category"], + "description": param_descs[unescaped_param_name].get( + "description", param_name + ), + "source": "params." + param_name, + } + ) elif unescaped_param_name in site_param_descs: - param_names_with_categories.append({ - "label" : unescaped_param_name, - "category": site_param_descs[unescaped_param_name]['category'], - "description": site_param_descs[unescaped_param_name].get("description", param_name), - "source": "params." + param_name }) + param_names_with_categories.append( + { + "label": unescaped_param_name, + "category": site_param_descs[unescaped_param_name]["category"], + "description": site_param_descs[unescaped_param_name].get( + "description", param_name + ), + "source": "params." + param_name, + } + ) elif re.match("DAQ (.*)/(.*)", unescaped_param_name): - param_names_with_categories.append({ - "label" : unescaped_param_name, - "category": "DAQ", - "description": unescaped_param_name, - "source": "params." + param_name }) + param_names_with_categories.append( + { + "label": unescaped_param_name, + "category": "DAQ", + "description": unescaped_param_name, + "source": "params." + param_name, + } + ) else: - param_names_with_categories.append({ - "label" : param_name, - "category": "EPICS:Additional parameters", - "description": param_name, - "source": "params." + param_name }) + param_names_with_categories.append( + { + "label": param_name, + "category": "EPICS:Additional parameters", + "description": param_name, + "source": "params." + param_name, + } + ) # Got thru the param_names_with_categories and update the descriptions from the run_table param_descs for pnc in param_names_with_categories: - param_name = reverse_escape_chars_for_mongo(pnc["source"]).replace("params.", "") + param_name = reverse_escape_chars_for_mongo(pnc["source"]).replace( + "params.", "" + ) if param_name in param_descs: pnc["description"] = param_descs[param_name].get("description", param_name) elif param_name in site_param_descs: pnc["description"] = param_descs[param_name].get("description", param_name) - rtbl_sources.update({ x['category'] : [] for x in param_names_with_categories}) - [ rtbl_sources[x['category']].append(x) for x in param_names_with_categories ] + rtbl_sources.update({x["category"]: [] for x in param_names_with_categories}) + [rtbl_sources[x["category"]].append(x) for x in param_names_with_categories] return rtbl_sources def create_update_user_run_table_def(experiment_name, instrument, table_definition): - ''' + """ Create or update an existing user run table definition for an experiment We expect a fully formed table_definition here... - ''' + """ expdb = logbookclient[experiment_name] createp = "_id" not in table_definition.keys() rtbl_name = table_definition["name"] rtbl_id = None if createp else ObjectId(table_definition["_id"]) - all_rtbls = {x["name"] : x for x in get_all_run_tables(experiment_name, instrument)} + all_rtbls = {x["name"]: x for x in get_all_run_tables(experiment_name, instrument)} if createp and rtbl_name in all_rtbls.keys(): - return (False, f"We already have a run table definition with the same name {rtbl_name}", None) + return ( + False, + f"We already have a run table definition with the same name {rtbl_name}", + None, + ) if not createp and not expdb["run_tables"].find_one({"_id": rtbl_id}): - return (False, f"Cannot update a table definition that does not exist for name {rtbl_name}", None) + return ( + False, + f"Cannot update a table definition that does not exist for name {rtbl_name}", + None, + ) if not createp: rtbl_with_id = expdb["run_tables"].find_one({"_id": rtbl_id}) if rtbl_name in all_rtbls and all_rtbls[rtbl_name]["_id"] != rtbl_id: exn = rtbl_with_id["name"] - return (False, f"Cannot rename table {exn} to table {rtbl_name} that already exists ", None) + return ( + False, + f"Cannot rename table {exn} to table {rtbl_name} that already exists ", + None, + ) if createp: expdb["run_tables"].insert_one(table_definition) else: - table_definition["_id"] = rtbl_id # Replace string with ObjectId + table_definition["_id"] = rtbl_id # Replace string with ObjectId expdb["run_tables"].replace_one({"_id": rtbl_id}, table_definition) return (True, "", expdb["run_tables"].find_one({"name": table_definition["name"]})) + def delete_run_table(experiment_name, table_name): - ''' + """ Delete the specified run table for the experiment. - ''' + """ expdb = logbookclient[experiment_name] expdb["run_tables"].delete_one({"name": table_name}) return (True, "") + def delete_system_run_table(experiment_name, instrument, table_name): - ''' + """ Delete the specified system run table. We can't really tell if a particular name is a instrument specific run table or a global one just by the name alone. So, we first check to see if there is a instrument specific one; if so, we delete that. If not, we delete the global one. - ''' + """ sitedb = logbookclient["site"] - ins_rt_del = sitedb["run_tables"].delete_one({"name": table_name, "instrument": instrument}) + ins_rt_del = sitedb["run_tables"].delete_one( + {"name": table_name, "instrument": instrument} + ) if ins_rt_del.deleted_count <= 0: logger.debug("Could not find an instrument specific run table %s", table_name) sitedb["run_tables"].delete_one({"name": table_name}) return (True, "") -def clone_run_table_definition(experiment_name, existing_run_table_name, new_run_table_name): + +def clone_run_table_definition( + experiment_name, existing_run_table_name, new_run_table_name +): """ Clone an existing run table definition. """ @@ -1457,11 +2318,17 @@ def clone_run_table_definition(experiment_name, existing_run_table_name, new_run sitedb = logbookclient["site"] existing_run_table = expdb["run_tables"].find_one({"name": existing_run_table_name}) if not existing_run_table: - existing_run_table = sitedb["run_tables"].find_one({"name": existing_run_table_name}) + existing_run_table = sitedb["run_tables"].find_one( + {"name": existing_run_table_name} + ) new_run_table = expdb["run_tables"].find_one({"name": new_run_table_name}) system_run_table = sitedb["run_tables"].find_one({"name": new_run_table_name}) if not existing_run_table: - return (False, "Cannot find existing run table %s " % existing_run_table_name, None) + return ( + False, + "Cannot find existing run table %s " % existing_run_table_name, + None, + ) if new_run_table or system_run_table: return (False, "Run table %s already exists" % existing_run_table_name, None) new_run_table = existing_run_table @@ -1470,7 +2337,14 @@ def clone_run_table_definition(experiment_name, existing_run_table_name, new_run expdb["run_tables"].insert_one(new_run_table) return (True, "", expdb["run_tables"].find_one({"name": new_run_table_name})) -def replace_system_run_table_definition(experiment_name, existing_run_table_name, system_run_table_name, instrument=None, is_template=False): + +def replace_system_run_table_definition( + experiment_name, + existing_run_table_name, + system_run_table_name, + instrument=None, + is_template=False, +): """ Replace a system run table (defined in the site database) with a run table from this experiment. This is a means to edit system run tables using the info available from this experiment. @@ -1479,7 +2353,11 @@ def replace_system_run_table_definition(experiment_name, existing_run_table_name sitedb = logbookclient["site"] existing_run_table = expdb["run_tables"].find_one({"name": existing_run_table_name}) if not existing_run_table: - return (False, "Cannot find existing run table %s " % existing_run_table_name, None) + return ( + False, + "Cannot find existing run table %s " % existing_run_table_name, + None, + ) new_run_table = existing_run_table del new_run_table["_id"] new_run_table["name"] = system_run_table_name @@ -1487,9 +2365,15 @@ def replace_system_run_table_definition(experiment_name, existing_run_table_name new_run_table["is_template"] = True if instrument: new_run_table["instrument"] = instrument - sitedb["run_tables"].replace_one({"name": system_run_table_name, "instrument": instrument}, new_run_table, upsert=True) + sitedb["run_tables"].replace_one( + {"name": system_run_table_name, "instrument": instrument}, + new_run_table, + upsert=True, + ) else: - sitedb["run_tables"].replace_one({"name": system_run_table_name}, new_run_table, upsert=True) + sitedb["run_tables"].replace_one( + {"name": system_run_table_name}, new_run_table, upsert=True + ) expdb["run_tables"].delete_one({"name": existing_run_table_name}) return (True, "", sitedb["run_tables"].find_one({"name": system_run_table_name})) @@ -1502,8 +2386,23 @@ def clone_system_template_run_tables_into_experiment(experiment_name, instrument """ expdb = logbookclient[experiment_name] sitedb = logbookclient["site"] - template_run_tables = [ r for r in sitedb["run_tables"].find({"$and": [ {"$or": [ {"instrument": instrument}, {"instrument": { "$exists": False}}]}, {"is_template": True} ]})] - existing_run_tables = [ x["name"] for x in expdb["run_tables"].find() ] + template_run_tables = [ + r + for r in sitedb["run_tables"].find( + { + "$and": [ + { + "$or": [ + {"instrument": instrument}, + {"instrument": {"$exists": False}}, + ] + }, + {"is_template": True}, + ] + } + ) + ] + existing_run_tables = [x["name"] for x in expdb["run_tables"].find()] for tr in template_run_tables: if tr["name"] in existing_run_tables: logger.debug("Skipping existing run table %s", tr["name"]) @@ -1516,32 +2415,41 @@ def clone_system_template_run_tables_into_experiment(experiment_name, instrument expdb["run_tables"].insert_one(tr) return (True, "", list(sitedb["run_tables"].find())) + def update_editable_param_for_run(experiment_name, runnum, source, value, userid): - ''' + """ Update the specified editable parameter for the specified run for the experiment. :param experiment_name: :param runnum: :param source: Typically editable_params.Run Title or something like that. :param value: :param userid: - ''' + """ expdb = logbookclient[experiment_name] - if not source.startswith('editable_params.') and not source.startswith('params.Calibrations/'): + if not source.startswith("editable_params.") and not source.startswith( + "params.Calibrations/" + ): raise Exception("Cannot update anything else other than an editable param") - if source.startswith('editable_params.'): + if source.startswith("editable_params."): return expdb["runs"].find_one_and_update( {"num": runnum}, - {"$set": { source: { + { + "$set": { + source: { "value": value, "modified_by": userid, - "modified_time": datetime.datetime.utcnow() - }}}) - if source.startswith('params.Calibrations/'): + "modified_time": datetime.datetime.utcnow(), + } + } + }, + ) + if source.startswith("params.Calibrations/"): return expdb["runs"].find_one_and_update( - {"num": runnum}, - {"$set": { source: value }}) + {"num": runnum}, {"$set": {source: value}} + ) raise Exception("Update editable param called for unknown param type " + source) + def get_experiment_shifts(experiment_name): """ Get the shifts for an experiment. @@ -1549,37 +2457,46 @@ def get_experiment_shifts(experiment_name): expdb = logbookclient[experiment_name] shifts = list(expdb.shifts.find({}).sort([("begin_time", -1)])) - previous_end_time = datetime.datetime.now() + datetime.timedelta(days=10*365) + previous_end_time = datetime.datetime.now() + datetime.timedelta(days=10 * 365) for shift in shifts: - if 'end_time' in shift and shift['end_time']: - shift['logical_end_time'] = shift['end_time'] + if "end_time" in shift and shift["end_time"]: + shift["logical_end_time"] = shift["end_time"] else: - shift['logical_end_time'] = previous_end_time - previous_end_time = shift['begin_time'] + shift["logical_end_time"] = previous_end_time + previous_end_time = shift["begin_time"] return shifts + def get_specific_shift(experiment_name, id): """ Get the specified shift entry for the experiment. For now, we have id based lookups. """ expdb = logbookclient[experiment_name] - return expdb['shifts'].find_one({"_id": ObjectId(id)}) + return expdb["shifts"].find_one({"_id": ObjectId(id)}) + def get_shift_for_experiment_by_name(experiment_name, shift_name): """ Get the specified shift specified by shift_name for the experiment. """ expdb = logbookclient[experiment_name] - return expdb['shifts'].find_one({"name": shift_name}) + return expdb["shifts"].find_one({"name": shift_name}) + def get_latest_shift(experiment_name): """ Get's the latest shift as detemined by the shift begin time. """ expdb = logbookclient[experiment_name] - shifts = list(expdb.shifts.find({ "begin_time" : { "$lte": datetime.datetime.utcnow() }, "end_time": None }).sort([("begin_time", -1)]).limit(1)) + shifts = list( + expdb.shifts.find( + {"begin_time": {"$lte": datetime.datetime.utcnow()}, "end_time": None} + ) + .sort([("begin_time", -1)]) + .limit(1) + ) if shifts: return shifts[0] return None @@ -1591,31 +2508,37 @@ def close_shift_for_experiment(experiment_name, shift_name): For now, this mostly means setting the end time to the current time. """ expdb = logbookclient[experiment_name] - shift_doc = expdb['shifts'].find_one({"name": shift_name}) + shift_doc = expdb["shifts"].find_one({"name": shift_name}) if not shift_doc: return (False, "Cannot find the shift specified by shift name " % shift_name) - expdb['shifts'].find_one_and_update({"name": shift_name}, {"$set": { "end_time" : datetime.datetime.utcnow()}}) + expdb["shifts"].find_one_and_update( + {"name": shift_name}, {"$set": {"end_time": datetime.datetime.utcnow()}} + ) return (True, "") + def create_update_shift(experiment_name, shift_name, createp, info): """ Create or update the shift for the specified experiment. """ expdb = logbookclient[experiment_name] - shift_doc = expdb['shifts'].find_one({"name": shift_name}) + shift_doc = expdb["shifts"].find_one({"name": shift_name}) if shift_doc and createp: return (False, "Shift %s already exists" % shift_name) if not shift_doc and not createp: return (False, "Shift %s does not exist" % shift_name) - info['begin_time'] = datetime.datetime.strptime(info["begin_time"], '%Y-%m-%dT%H:%M:%S.%fZ') + info["begin_time"] = datetime.datetime.strptime( + info["begin_time"], "%Y-%m-%dT%H:%M:%S.%fZ" + ) if createp: - expdb['shifts'].insert_one(info) + expdb["shifts"].insert_one(info) else: - expdb['shifts'].find_one_and_update({"name": shift_name}, {"$set": info}) + expdb["shifts"].find_one_and_update({"name": shift_name}, {"$set": info}) return (True, "") + def get_samples(experiment_name): """ Get the defined samples for the experiment @@ -1626,13 +2549,16 @@ def get_samples(experiment_name): current_sample = expdb.current.find_one({"_id": "sample"}) if current_sample: current_sample_id = current_sample["sample"] + def set_current(x): if x["_id"] == current_sample_id: x["current"] = True return x + samples = list(map(set_current, samples)) return samples + def get_current_sample_name(experiment_name): """ Get the current sample name for the specified experiment. @@ -1640,7 +2566,12 @@ def get_current_sample_name(experiment_name): """ expdb = logbookclient[experiment_name] current_sample = expdb.current.find_one({"_id": "sample"}) - return expdb.samples.find_one({"_id": current_sample['sample']})['name'] if current_sample and 'sample' in current_sample else None + return ( + expdb.samples.find_one({"_id": current_sample["sample"]})["name"] + if current_sample and "sample" in current_sample + else None + ) + def get_sample_for_experiment_by_name(experiment_name, sample_name): """ @@ -1649,11 +2580,18 @@ def get_sample_for_experiment_by_name(experiment_name, sample_name): expdb = logbookclient[experiment_name] requested_sample = expdb.samples.find_one({"name": sample_name}) current_sample = expdb.current.find_one({"_id": "sample"}) - if current_sample and requested_sample and current_sample["sample"] == requested_sample["_id"]: + if ( + current_sample + and requested_sample + and current_sample["sample"] == requested_sample["_id"] + ): requested_sample["current"] = True return requested_sample -def create_sample(experiment_name, sampledetails, automatically_create_associated_run=False): + +def create_sample( + experiment_name, sampledetails, automatically_create_associated_run=False +): """ Create a new sample for an experiment. """ @@ -1661,82 +2599,97 @@ def create_sample(experiment_name, sampledetails, automatically_create_associate if "name" not in sampledetails or "description" not in sampledetails: return (False, "Please specify a sample name and description") sample_name = sampledetails["name"] - if expdb['samples'].find_one({"name": sample_name}): + if expdb["samples"].find_one({"name": sample_name}): return (False, "Sample %s already exists" % sample_name) validation, erromsg = validate_with_modal_params("samples", sampledetails) if not validation: return validation, erromsg - expdb['samples'].insert_one(sampledetails) + expdb["samples"].insert_one(sampledetails) if automatically_create_associated_run: current_run = get_current_run(experiment_name) if current_run and not is_run_closed(experiment_name, current_run["num"]): - return False, ("Cannot switch to and create a run if the current run %s is still open %s" % (current_run["num"], experiment_name)) + return False, ( + "Cannot switch to and create a run if the current run %s is still open %s" + % (current_run["num"], experiment_name) + ) make_sample_current(experiment_name, sample_name) start_run(experiment_name, "DATA") end_run(experiment_name) return (True, "") + def update_sample(experiment_name, sampleid, sampledetails): """ Update an existing sample for an experiment. """ expdb = logbookclient[experiment_name] sampledetails["_id"] = ObjectId(sampleid) - existing_sample = expdb['samples'].find_one({"_id": sampledetails["_id"]}) + existing_sample = expdb["samples"].find_one({"_id": sampledetails["_id"]}) if not existing_sample: return (False, "Sample %s does not exist" % sampledetails["_id"]) if "name" not in sampledetails or "description" not in sampledetails: - return (False, "Please specify a sample name and description") - sample_with_name = expdb['samples'].find_one({"name": sampledetails["name"]}) + return (False, "Please specify a sample name and description") + sample_with_name = expdb["samples"].find_one({"name": sampledetails["name"]}) if sample_with_name and existing_sample["_id"] != sample_with_name["_id"]: - return (False, "Cannot rename sample %s to one that already exists %s" % (existing_sample["_id"], sample_with_name["_id"])) + return ( + False, + "Cannot rename sample %s to one that already exists %s" + % (existing_sample["_id"], sample_with_name["_id"]), + ) validation, erromsg = validate_with_modal_params("samples", sampledetails) if not validation: return validation, erromsg - expdb['samples'].replace_one({"_id": existing_sample["_id"]}, sampledetails) + expdb["samples"].replace_one({"_id": existing_sample["_id"]}, sampledetails) return (True, "") + def clone_sample(experiment_name, existing_sample_name, new_sample_name): """ Clone an existing sample. """ expdb = logbookclient[experiment_name] - if not expdb['samples'].find_one({"name": existing_sample_name}): + if not expdb["samples"].find_one({"name": existing_sample_name}): return (False, "Sample %s does not exist" % existing_sample_name) - if expdb['samples'].find_one({"name": new_sample_name}): + if expdb["samples"].find_one({"name": new_sample_name}): return (False, "Sample %s already exists" % new_sample_name) - existing_sample_doc = expdb['samples'].find_one({"name": existing_sample_name}) + existing_sample_doc = expdb["samples"].find_one({"name": existing_sample_name}) del existing_sample_doc["_id"] existing_sample_doc["name"] = new_sample_name - expdb['samples'].insert_one(existing_sample_doc) + expdb["samples"].insert_one(existing_sample_doc) return (True, "") + def make_sample_current(experiment_name, sample_name): """ Make the sample specified by the sample_name as the current sample. """ expdb = logbookclient[experiment_name] - sample_doc = expdb['samples'].find_one({"name": sample_name}) + sample_doc = expdb["samples"].find_one({"name": sample_name}) if not sample_doc: return (False, "Sample %s does not exist" % sample_name) validation, erromsg = validate_with_modal_params("samples", sample_doc) if not validation: return validation, erromsg - expdb.current.find_one_and_update({"_id": "sample"}, {"$set": { "_id": "sample", "sample" : sample_doc["_id"] }} , upsert=True) + expdb.current.find_one_and_update( + {"_id": "sample"}, + {"$set": {"_id": "sample", "sample": sample_doc["_id"]}}, + upsert=True, + ) return (True, "") + def stop_current_sample(experiment_name, sample_name): """ Stop the sample specified by the sample_name if it is the current sample and set the current sample to null """ expdb = logbookclient[experiment_name] - sample_doc = expdb['samples'].find_one({"name": sample_name}) + sample_doc = expdb["samples"].find_one({"name": sample_name}) if not sample_doc: return (False, "Sample %s does not exist" % sample_name) @@ -1747,27 +2700,44 @@ def stop_current_sample(experiment_name, sample_name): expdb.current.delete_one({"_id": "sample"}) return (True, "") + def delete_sample_for_experiment(experiment_name, sample_name): - """ Delete the sample for an experiment. We only allow deletion of samples if there are no runs associated with the sample and it is not current""" + """Delete the sample for an experiment. We only allow deletion of samples if there are no runs associated with the sample and it is not current""" expdb = logbookclient[experiment_name] requested_sample = expdb.samples.find_one({"name": sample_name}) current_sample = expdb.current.find_one({"_id": "sample"}) if not requested_sample: return False, "Cannot find sample %s" % sample_name, None - if current_sample and requested_sample and current_sample["sample"] == requested_sample["_id"]: - return False, "Cannot delete sample %s as it is the current sample in the experiment" % sample_name, None - runs = get_experiment_runs(experiment_name, include_run_params=False, sample_name=sample_name) + if ( + current_sample + and requested_sample + and current_sample["sample"] == requested_sample["_id"] + ): + return ( + False, + "Cannot delete sample %s as it is the current sample in the experiment" + % sample_name, + None, + ) + runs = get_experiment_runs( + experiment_name, include_run_params=False, sample_name=sample_name + ) if runs and len(runs) > 0: - return False, "Cannot delete sample %s as it has %d runs associated with it" % (sample_name, len(runs)), None + return ( + False, + "Cannot delete sample %s as it has %d runs associated with it" + % (sample_name, len(runs)), + None, + ) logger.debug("Actually deleting sample") expdb["samples"].delete_one({"name": sample_name}) return True, "", None + def get_modal_param_definitions(modal_type): """ Get the site specific modal param definitions for the specified modal type. """ - sitedb = logbookclient["site"] modal_params_file = "static/json/{}/modals/{}.json".format(LOGBOOK_SITE, modal_type) logger.info("Looking for modal definition in %s", modal_params_file) param_defs = {} @@ -1784,6 +2754,7 @@ def get_modal_param_definitions(modal_type): return param_defs + def validate_with_modal_params(modal_type, business_obj): """ Validate a business object against any modal param definitions for this site. @@ -1792,23 +2763,33 @@ def validate_with_modal_params(modal_type, business_obj): modal_defs = get_modal_param_definitions(modal_type) if not modal_defs: return True, "" + def __get_nested_attr__(theobj, attrname): nameparts = attrname.split(".") for namepart in nameparts[:-1]: theobj = theobj.get(namepart, {}) return theobj.get(nameparts[-1], None) - for required_param in [ x["param_name"] for x in modal_defs["params"] if x.get("required", False) ]: + for required_param in [ + x["param_name"] for x in modal_defs["params"] if x.get("required", False) + ]: if not __get_nested_attr__(business_obj, required_param): logger.error("Missing %s in %s", required_param, business_obj) - return False, "One of the required parameters {} was not specified".format(required_param) - for num_param in [ x["param_name"] for x in modal_defs["params"] if x.get("param_type", "string") in ["int", "float"] ]: + return False, "One of the required parameters {} was not specified".format( + required_param + ) + for num_param in [ + x["param_name"] + for x in modal_defs["params"] + if x.get("param_type", "string") in ["int", "float"] + ]: thenumval = __get_nested_attr__(business_obj, num_param) if not isinstance(thenumval, int) and not isinstance(thenumval, float): logger.error("params.%s is not an int/float in %s", num_param, business_obj) return False, "The parameter {} is not an number".format(num_param) return True, "" + def change_sample_for_run(experiment_name, run_num, sample_name): """ Change the sample for the specified run. @@ -1822,29 +2803,48 @@ def change_sample_for_run(experiment_name, run_num, sample_name): expdb["runs"].update_one({"num": run_num}, {"$set": {"sample": sample_doc["_id"]}}) return True, "" + def register_file_for_experiment(experiment_name, info): """ Register a file for the experiment. """ expdb = logbookclient[experiment_name] - if expdb['file_catalog'].find_one({"path": info["path"], "run_num": info["run_num"]}): - expdb['file_catalog'].replace_one({"path": info["path"], "run_num": info["run_num"]}, info) + if expdb["file_catalog"].find_one( + {"path": info["path"], "run_num": info["run_num"]} + ): + expdb["file_catalog"].replace_one( + {"path": info["path"], "run_num": info["run_num"]}, info + ) else: - expdb['file_catalog'].insert_one(info) - return (True, expdb['file_catalog'].find_one({"path": info["path"], "run_num": info["run_num"]})) + expdb["file_catalog"].insert_one(info) + return ( + True, + expdb["file_catalog"].find_one( + {"path": info["path"], "run_num": info["run_num"]} + ), + ) + def file_available_at_location(experiment_name, run_num, file_path, location): """ Mark a file as being available at the specified location. """ expdb = logbookclient[experiment_name] - expdb['file_catalog'].update_one({"run_num": run_num, "path": file_path}, {"$set": {"locations."+location+".asof": datetime.datetime.utcnow()}}) - return expdb['file_catalog'].find_one({"run_num": run_num, "path": file_path}) + expdb["file_catalog"].update_one( + {"run_num": run_num, "path": file_path}, + {"$set": {"locations." + location + ".asof": datetime.datetime.utcnow()}}, + ) + return expdb["file_catalog"].find_one({"run_num": run_num, "path": file_path}) + def file_not_available_at_location(experiment_name, run_num, file_path, location): expdb = logbookclient[experiment_name] - expdb['file_catalog'].update_one({"run_num": run_num, "path": file_path}, {"$unset": {"locations."+location: 1}}) - return expdb['file_catalog'].find_one({"run_num": run_num, "path": file_path}) + expdb["file_catalog"].update_one( + {"run_num": run_num, "path": file_path}, + {"$unset": {"locations." + location: 1}}, + ) + return expdb["file_catalog"].find_one({"run_num": run_num, "path": file_path}) + def get_collaborators(experiment_name): """ @@ -1853,55 +2853,78 @@ def get_collaborators(experiment_name): """ expdb = logbookclient[experiment_name] roles = [x for x in expdb["roles"].find()] - all_players = set() # First, generate a set of all the players in the experiment. - list(map(lambda x : all_players.update(x.get('players', [])), roles)) - players2roles = { x : [] for x in all_players } + all_players = set() # First, generate a set of all the players in the experiment. + list(map(lambda x: all_players.update(x.get("players", [])), roles)) + players2roles = {x: [] for x in all_players} for role in roles: - for player in role.get('players', []): - players2roles[player].append("{0}/{1}".format(role['app'], role['name'])) + for player in role.get("players", []): + players2roles[player].append("{0}/{1}".format(role["app"], role["name"])) ret = [] for player in players2roles.keys(): is_group = False if player.startswith("uid:") else True - user_details = usergroups.get_userids_matching_pattern(player.replace("uid:", "")) if not is_group else None - ret.append({ - "uid": player, - "is_group": is_group, - "full_name": user_details[0].get('gecos', "N/A") if user_details else "N/A", - "uidNumber": user_details[0].get('uidNumber', "N/A") if user_details else "N/A", - "roles": players2roles[player] - }) + user_details = ( + usergroups.get_userids_matching_pattern(player.replace("uid:", "")) + if not is_group + else None + ) + ret.append( + { + "uid": player, + "is_group": is_group, + "full_name": user_details[0].get("gecos", "N/A") + if user_details + else "N/A", + "uidNumber": user_details[0].get("uidNumber", "N/A") + if user_details + else "N/A", + "roles": players2roles[player], + } + ) return sorted(ret, key=lambda x: x["uid"].replace("uid:", "")) + def get_role_object(experiment_name, role_fq_name): expdb = logbookclient[experiment_name] application_name, role_name = role_fq_name.split("/") roleobj = expdb["roles"].find_one({"app": application_name, "name": role_name}) return roleobj + def add_collaborator_to_role(experiment_name, uid, role_fq_name): expdb = logbookclient[experiment_name] application_name, role_name = role_fq_name.split("/") roleobj = expdb["roles"].find_one({"app": application_name, "name": role_name}) if not roleobj: - expdb["roles"].insert_one({"app": application_name, "name": role_name, "players": [ uid ]}) + expdb["roles"].insert_one( + {"app": application_name, "name": role_name, "players": [uid]} + ) return True if "players" not in roleobj: - result = expdb["roles"].update_one({"app": application_name, "name": role_name}, { "$set": { "players": [ uid ] }}) + result = expdb["roles"].update_one( + {"app": application_name, "name": role_name}, {"$set": {"players": [uid]}} + ) elif uid not in roleobj["players"]: - result = expdb["roles"].update_one({"app": application_name, "name": role_name}, {"$addToSet": { "players": uid }}) + result = expdb["roles"].update_one( + {"app": application_name, "name": role_name}, + {"$addToSet": {"players": uid}}, + ) else: return False return result.matched_count > 0 + def remove_collaborator_from_role(experiment_name, uid, role_fq_name): expdb = logbookclient[experiment_name] application_name, role_name = role_fq_name.split("/") roleobj = expdb["roles"].find_one({"app": application_name, "name": role_name}) if not roleobj or "players" not in roleobj or uid not in roleobj["players"]: return False - result = expdb["roles"].update_one({"app": application_name, "name": role_name}, { "$pull": { "players": uid }}) + result = expdb["roles"].update_one( + {"app": application_name, "name": role_name}, {"$pull": {"players": uid}} + ) return result.matched_count > 0 + def get_collaborators_list_for_experiment(experiment_name): """ Get all the collaborators for an experiment. @@ -1910,25 +2933,33 @@ def get_collaborators_list_for_experiment(experiment_name): Only the roles.players that being with uid: are returned (actual users, not groups). """ expdb = logbookclient[experiment_name] - roleobjs = [ x for x in expdb["roles"].find({}) ] + roleobjs = [x for x in expdb["roles"].find({})] ret = set() for roleobj in roleobjs: ret.update([x for x in roleobj.get("players", []) if x.startswith("uid:")]) # Remove the operator account if present instrument = get_experiment_info(experiment_name)["instrument"] - operator_uid = { x["_id"] : x for x in get_instruments()}[instrument].get("params", {}).get("operator_uid", None) + operator_uid = ( + {x["_id"]: x for x in get_instruments()}[instrument] + .get("params", {}) + .get("operator_uid", None) + ) if operator_uid and operator_uid in ret: ret.remove(operator_uid) return ret + def get_poc_feedback_changes(experiment_name): """ Gets a list of POC feedback items sorted by ascending modified date. To reconstruct the document, simply apply the changes to a dict in sequence. """ - expdb = logbookclient.get_database(experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED) + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.SECONDARY_PREFERRED + ) return list(expdb["poc_feedback"].find({}).sort([("modified_at", 1)])) + def get_poc_feedback_document(experiment_name): """ Reconstructs the current feedback document from a questionnaire like history of changes. @@ -1936,63 +2967,130 @@ def get_poc_feedback_document(experiment_name): poc_feedback_changes = get_poc_feedback_changes(experiment_name) poc_feedback_doc = {} exp_info = get_experiment_info(experiment_name) - poc_feedback_doc["basic-scheduled"] = math.ceil((exp_info["end_time"] - exp_info["start_time"]).total_seconds()/(8*3600)) + poc_feedback_doc["basic-scheduled"] = math.ceil( + (exp_info["end_time"] - exp_info["start_time"]).total_seconds() / (8 * 3600) + ) if poc_feedback_changes: - poc_feedback_doc.update({ x['name'] : x['value'] for x in get_poc_feedback_changes(experiment_name) }) - poc_feedback_doc['last_modified_by'] = poc_feedback_changes[-1]['modified_by'] - poc_feedback_doc['last_modified_at'] = poc_feedback_changes[-1]['modified_at'] + poc_feedback_doc.update( + {x["name"]: x["value"] for x in get_poc_feedback_changes(experiment_name)} + ) + poc_feedback_doc["last_modified_by"] = poc_feedback_changes[-1]["modified_by"] + poc_feedback_doc["last_modified_at"] = poc_feedback_changes[-1]["modified_at"] return poc_feedback_doc + def add_poc_feedback_item(experiment_name, item_name, item_value, modified_by): expdb = logbookclient[experiment_name] - expdb["poc_feedback"].insert_one({"name": item_name, "value": item_value, "modified_by": modified_by, "modified_at": datetime.datetime.utcnow()}) + expdb["poc_feedback"].insert_one( + { + "name": item_name, + "value": item_value, + "modified_by": modified_by, + "modified_at": datetime.datetime.utcnow(), + } + ) + def get_poc_feedback_experiments(): cachedb = logbookclient["explgbk_cache"] - return list(cachedb["experiments"].find({"poc_feedback.num_items": {"$gte": 1}}, {"_id": 0, "name": 1, "poc_feedback": 1, "instrument": 1, "params.PNR": 1, "last_run": 1 })) + return list( + cachedb["experiments"].find( + {"poc_feedback.num_items": {"$gte": 1}}, + { + "_id": 0, + "name": 1, + "poc_feedback": 1, + "instrument": 1, + "params.PNR": 1, + "last_run": 1, + }, + ) + ) def get_workflow_definitions(experiment_name): expdb = logbookclient[experiment_name] return list(expdb["workflow_definitions"].find({}).sort([("name", 1)])) + def get_dm_locations(experiment_name): sitedb = logbookclient["site"] expdb = logbookclient[experiment_name] siteinfo = sitedb["site_config"].find_one() ret = {} - if siteinfo and 'dm_locations' in siteinfo and siteinfo['dm_locations']: - ret.update({x["name"]: x for x in siteinfo['dm_locations'] if x.get("all_experiments", False)}) - exp_specific_dm_locations = expdb["info"].find_one().get("params", {}).get("dm_locations", "").split() + if siteinfo and "dm_locations" in siteinfo and siteinfo["dm_locations"]: + ret.update( + { + x["name"]: x + for x in siteinfo["dm_locations"] + if x.get("all_experiments", False) + } + ) + exp_specific_dm_locations = ( + expdb["info"].find_one().get("params", {}).get("dm_locations", "").split() + ) for exp_specific_dm_location in exp_specific_dm_locations: - if exp_specific_dm_location in [x["name"] for x in siteinfo['dm_locations']]: - ret[exp_specific_dm_location] = [x for x in siteinfo['dm_locations'] if x["name"] == exp_specific_dm_location][0] + if exp_specific_dm_location in [ + x["name"] for x in siteinfo["dm_locations"] + ]: + ret[exp_specific_dm_location] = [ + x + for x in siteinfo["dm_locations"] + if x["name"] == exp_specific_dm_location + ][0] return list(ret.values()) + def get_site_config(): sitedb = logbookclient["site"] siteinfo = sitedb["site_config"].find_one() return siteinfo + def get_workflow_triggers(experiment_name): - return [{"value": "MANUAL", "label": "Manually triggered"}, {"value": "START_OF_RUN", "label": "Start of a run"}, {"value": "END_OF_RUN", "label": "End of a run"}, {"value": "FIRST_FILE_TRANSFERRED", "label": "First file transfer"}, {"value": "ALL_FILES_TRANSFERRED", "label": "All files transferred"}, {"value": "ALL_NONREC_FILES_TRANSFERRED", "label": "All non-recorder files transferred"}, {"value": "RUN_PARAM_IS_VALUE", "label": "Run table param has value"}] + return [ + {"value": "MANUAL", "label": "Manually triggered"}, + {"value": "START_OF_RUN", "label": "Start of a run"}, + {"value": "END_OF_RUN", "label": "End of a run"}, + {"value": "FIRST_FILE_TRANSFERRED", "label": "First file transfer"}, + {"value": "ALL_FILES_TRANSFERRED", "label": "All files transferred"}, + { + "value": "ALL_NONREC_FILES_TRANSFERRED", + "label": "All non-recorder files transferred", + }, + {"value": "RUN_PARAM_IS_VALUE", "label": "Run table param has value"}, + ] + def create_update_wf_definition(experiment_name, wf_obj): expdb = logbookclient[experiment_name] if "_id" in wf_obj: logger.debug("Updating workflow definition %s", wf_obj["_id"]) - cur_wf_obj = expdb["workflow_definitions"].find_one({"_id": ObjectId(wf_obj["_id"])}) + cur_wf_obj = expdb["workflow_definitions"].find_one( + {"_id": ObjectId(wf_obj["_id"])} + ) if not cur_wf_obj: - return False, "Cannot find workflow definition with id %s " % wf_obj["_id"], None + return ( + False, + "Cannot find workflow definition with id %s " % wf_obj["_id"], + None, + ) wf_obj["_id"] = cur_wf_obj["_id"] - expdb["workflow_definitions"].replace_one({"_id": ObjectId(wf_obj["_id"])}, wf_obj) - return True, "", expdb["workflow_definitions"].find_one({"_id": ObjectId(wf_obj["_id"])}) + expdb["workflow_definitions"].replace_one( + {"_id": ObjectId(wf_obj["_id"])}, wf_obj + ) + return ( + True, + "", + expdb["workflow_definitions"].find_one({"_id": ObjectId(wf_obj["_id"])}), + ) else: wf_id = expdb["workflow_definitions"].insert_one(wf_obj).inserted_id return True, "", expdb["workflow_definitions"].find_one({"_id": wf_id}) + def delete_wf_definition(experiment_name, wf_obj_id): expdb = logbookclient[experiment_name] cur_wf_obj = expdb["workflow_definitions"].find_one({"_id": ObjectId(wf_obj_id)}) @@ -2000,32 +3098,55 @@ def delete_wf_definition(experiment_name, wf_obj_id): return False, "Cannot find workflow object with id %s " % wf_obj_id, None wfjobs = expdb["workflow_jobs"].count_documents({"def_id": cur_wf_obj["_id"]}) if wfjobs > 0: - return False, "Cannot delete workflow definition as we have %s jobs using this definition." % wfjobs, None + return ( + False, + "Cannot delete workflow definition as we have %s jobs using this definition." + % wfjobs, + None, + ) expdb["workflow_definitions"].delete_one({"_id": cur_wf_obj["_id"]}) return True, "", None + def get_workflow_jobs(experiment_name): expdb = logbookclient[experiment_name] - return [x for x in expdb["workflow_jobs"].aggregate([ - { "$lookup": { "from": "workflow_definitions", "localField": "def_id", "foreignField": "_id", "as": "def"}}, - { "$unwind": "$def" }, - { "$addFields": { "experiment": experiment_name }}, - { "$sort": { "run_num": -1, "name": 1}} - ])] + return [ + x + for x in expdb["workflow_jobs"].aggregate( + [ + { + "$lookup": { + "from": "workflow_definitions", + "localField": "def_id", + "foreignField": "_id", + "as": "def", + } + }, + {"$unwind": "$def"}, + {"$addFields": {"experiment": experiment_name}}, + {"$sort": {"run_num": -1, "name": 1}}, + ] + ) + ] + def get_workflow_job_doc(experiment_name, job_id): expdb = logbookclient[experiment_name] wf_doc = expdb["workflow_jobs"].find_one({"_id": ObjectId(job_id)}) - if wf_doc and 'def_id' in wf_doc: + if wf_doc and "def_id" in wf_doc: wf_doc["experiment"] = experiment_name - wf_doc["def"] = expdb["workflow_definitions"].find_one({"_id": wf_doc["def_id"]}) + wf_doc["def"] = expdb["workflow_definitions"].find_one( + {"_id": wf_doc["def_id"]} + ) return wf_doc + def create_wf_job(experiment_name, wf_job_doc): expdb = logbookclient[experiment_name] wf_id = expdb["workflow_jobs"].insert_one(wf_job_doc).inserted_id return True, "", get_workflow_job_doc(experiment_name, wf_id) + def delete_wf_job(experiment_name, job_id): expdb = logbookclient[experiment_name] wf_doc = expdb["workflow_jobs"].find_one({"_id": ObjectId(job_id)}) @@ -2034,25 +3155,27 @@ def delete_wf_job(experiment_name, job_id): expdb["workflow_jobs"].delete_one({"_id": ObjectId(job_id)}) return True, "", wf_doc + def update_wf_job(experiment_name, job_id, wf_updates): expdb = logbookclient[experiment_name] wf_doc = expdb["workflow_jobs"].find_one({"_id": ObjectId(job_id)}) if not wf_doc: return False, "Cannot find workflow job", None - expdb["workflow_jobs"].update_one({"_id": ObjectId(job_id)}, {"$set": wf_updates }) + expdb["workflow_jobs"].update_one({"_id": ObjectId(job_id)}, {"$set": wf_updates}) return True, "", get_workflow_job_doc(experiment_name, job_id) + def questionnaire_cache_refresh(experiment_name): """ The questionnaire caches URAWI and UPS proposal info. This call forces a cache refresh ( perhaps based on something from the UI ) """ if not QUESTIONNAIRE_URL: - return + return if not experiment_name: return expdb = logbookclient[experiment_name] - expinfo = expdb['info'].find_one() + expinfo = expdb["info"].find_one() if not expinfo: return thequesurl = QUESTIONNAIRE_URL @@ -2069,13 +3192,18 @@ def questionnaire_cache_refresh(experiment_name): if not additionalAuthToken: raise Exception("Please specify the auth token for the questionnaire") prts = additionalAuthToken.split("=") - cresp = requests.get(f"{thequesurl}/refresh_cache", params={"proposal_id": questionnaire_proposal_id}, auth=HTTPBasicAuth(prts[0], prts[1])) + cresp = requests.get( + f"{thequesurl}/refresh_cache", + params={"proposal_id": questionnaire_proposal_id}, + auth=HTTPBasicAuth(prts[0], prts[1]), + ) try: cresp.raise_for_status() - except: + except Exception: logger.exception("Exception refreshing questionnaire cache") logger.info("Refreshed the questionnaire cache for %s", questionnaire_proposal_id) + def get_ques_proposal_details(experiment_name, run_period=None, proposal_id=None): """ Get proposal details for the experiment from the questionnaire. @@ -2087,25 +3215,40 @@ def get_ques_proposal_details(experiment_name, run_period=None, proposal_id=None try: # Check to see if we have an info object with a PNR expdb = logbookclient[experiment_name] - expinfo = expdb['info'].find_one() + expinfo = expdb["info"].find_one() if expinfo: questionnaire_proposal_id = expinfo.get("params", {}).get("PNR", None) - questionnaire_run_period = expinfo.get("params", {}).get("run_period", questionnaire_run_period) + questionnaire_run_period = expinfo.get("params", {}).get( + "run_period", questionnaire_run_period + ) instrument = expinfo["instrument"] - insinfo = logbookclient["site"]["instruments"].find_one({"_id": instrument}) + insinfo = logbookclient["site"]["instruments"].find_one( + {"_id": instrument} + ) if insinfo.get("params", {}).get("questionnaire_ws_url", None): thequesurl = insinfo["params"]["questionnaire_ws_url"] - logger.debug("Overriding questionnaure ws_url from instrument %s", thequesurl) + logger.debug( + "Overriding questionnaure ws_url from instrument %s", thequesurl + ) else: # See if we can determine the instrument based on the first 3 chars of the experiment name if LOGBOOK_SITE in ["LCLS", "TestFac"]: instrument = experiment_name[0:3] - insinfo = logbookclient["site"]["instruments"].find_one({"_id": instrument}) + insinfo = logbookclient["site"]["instruments"].find_one( + {"_id": instrument} + ) if not insinfo: - insinfo = logbookclient["site"]["instruments"].find_one({"_id": instrument.upper()}) - if insinfo and insinfo.get("params", {}).get("questionnaire_ws_url", None): + insinfo = logbookclient["site"]["instruments"].find_one( + {"_id": instrument.upper()} + ) + if insinfo and insinfo.get("params", {}).get( + "questionnaire_ws_url", None + ): thequesurl = insinfo["params"]["questionnaire_ws_url"] - logger.debug("Overriding questionnaure ws_url from instrument %s", thequesurl) + logger.debug( + "Overriding questionnaure ws_url from instrument %s", + thequesurl, + ) if not questionnaire_proposal_id: # For LCLS and TestFac, we compose the experiment name by prefixing the instrument and appending the run period. @@ -2117,24 +3260,36 @@ def get_ques_proposal_details(experiment_name, run_period=None, proposal_id=None questionnaire_run_period = experiment_name[-2:] proposal_url = f"{thequesurl}/run{questionnaire_run_period}/{questionnaire_proposal_id}/entire" - logger.info("Getting questionnaire data for proposal using %s", proposal_url) + logger.info( + "Getting questionnaire data for proposal using %s", proposal_url + ) additionalAuthToken = os.environ.get("QUESTIONNAIRE_AUTH", None) if not additionalAuthToken: raise Exception("Please specify the auth token for the questionnaire") prts = additionalAuthToken.split("=") - resp = requests.get(proposal_url, auth=HTTPBasicAuth(prts[0], prts[1]), verify=False) + resp = requests.get( + proposal_url, auth=HTTPBasicAuth(prts[0], prts[1]), verify=False + ) resp.raise_for_status() ques_doc = resp.json() if LOGBOOK_SITE in ["LCLS", "TestFac"]: if ques_doc.get("instrument") == "CRIXS": - logger.debug("Mapping one off instrument for LCLS/UED for proposal %s", questionnaire_proposal_id) + logger.debug( + "Mapping one off instrument for LCLS/UED for proposal %s", + questionnaire_proposal_id, + ) ques_doc["instrument"] = "RIX" return ques_doc - except Exception as e: - logger.exception("Exception fetching data from URAWI using URL %s for %s", thequesurl, questionnaire_proposal_id) + except Exception: + logger.exception( + "Exception fetching data from URAWI using URL %s for %s", + thequesurl, + questionnaire_proposal_id, + ) return None return None + def import_users_from_URAWI(experiment_name, role_fq_name="LogBook/Writer"): """ Get users from URAWI and add them as collaborators into this experiment. @@ -2143,23 +3298,40 @@ def import_users_from_URAWI(experiment_name, role_fq_name="LogBook/Writer"): logger.debug(existing_collaborators) ques_doc = get_ques_proposal_details(experiment_name) if ques_doc: + if ques_doc.get("URAWI", {}).get("approved", "no").lower() != "yes": + logger.warning("Proposal is not approved in URAWI. Skipping importing collaborators.") + return + logger.info("The BTR for experiment %s has been approved. Importing collaborators from URAWI.", experiment_name) for coll in ques_doc.get("URAWI", {}).get("collaborators", []): for acc in coll.get("account", []): if "unixGroup" not in acc or acc.get("unixGroup", "") == "xs": - logger.debug("Skipping adding probable experiment related account %s", acc) + logger.debug( + "Skipping adding probable experiment related account %s", acc + ) continue accuid = "uid:" + acc["unixName"] if accuid in existing_collaborators: logger.debug("Collaborator %s is already in the system", accuid) continue add_collaborator_to_role(experiment_name, accuid, role_fq_name) - logger.info("Done adding collaborator %s to experiment %s", accuid, experiment_name) + logger.info( + "Done adding collaborator %s to experiment %s", + accuid, + experiment_name, + ) + def get_projects(user): """ Get the projects for a user. """ - return [ x for x in logbookclient[PROJECTS_DB]["projects"].find({"players": "uid:" + user}).sort([("name", 1)]) ] + return [ + x + for x in logbookclient[PROJECTS_DB]["projects"] + .find({"players": "uid:" + user}) + .sort([("name", 1)]) + ] + def get_project_info(prjid): """ @@ -2167,15 +3339,19 @@ def get_project_info(prjid): """ return logbookclient[PROJECTS_DB]["projects"].find_one({"_id": ObjectId(prjid)}) + def create_project(prjinfo): """ Create a new project """ if "_id" in prjinfo: - raise Exception("_id present in project info. Did you mean to edit the project?") + raise Exception( + "_id present in project info. Did you mean to edit the project?" + ) prjid = logbookclient[PROJECTS_DB]["projects"].insert_one(prjinfo).inserted_id return logbookclient[PROJECTS_DB]["projects"].find_one({"_id": ObjectId(prjid)}) + def update_project(prjid, prjinfo): """ Update an existing project @@ -2185,22 +3361,31 @@ def update_project(prjid, prjinfo): curr = logbookclient[PROJECTS_DB]["projects"].find_one({"_id": ObjectId(prjid)}) if not curr: raise Exception("Cannot find project with _id " + prjid) - logbookclient[PROJECTS_DB]["projects"].update_one({"_id": ObjectId(prjid)}, {"$set": prjinfo}) + logbookclient[PROJECTS_DB]["projects"].update_one( + {"_id": ObjectId(prjid)}, {"$set": prjinfo} + ) return logbookclient[PROJECTS_DB]["projects"].find_one({"_id": ObjectId(prjid)}) - get_project_samples, add_session_to_project, add_sample_to_project, update_project_sample, \ def get_project_grids(prjid): """ Get the project grids """ - return list(logbookclient[PROJECTS_DB]["grids"].find({"prjid": ObjectId(prjid)}).sort([("box", 1), ("number", 1)])) + return list( + logbookclient[PROJECTS_DB]["grids"] + .find({"prjid": ObjectId(prjid)}) + .sort([("box", 1), ("number", 1)]) + ) + def get_project_grid(prjid, gridid): """ Get the specified gridid in the project. """ - return logbookclient[PROJECTS_DB]["grids"].find_one({"_id": ObjectId(gridid), "prjid": ObjectId(prjid)}) + return logbookclient[PROJECTS_DB]["grids"].find_one( + {"_id": ObjectId(gridid), "prjid": ObjectId(prjid)} + ) + def add_grid_to_project(prjid, griddetails): """ @@ -2213,17 +3398,34 @@ def add_grid_to_project(prjid, griddetails): if not validation: return validation, erromsg - grid_with_number = logbookclient[PROJECTS_DB]["grids"].find_one({"prjid": ObjectId(prjid), "number": griddetails["number"]}) + grid_with_number = logbookclient[PROJECTS_DB]["grids"].find_one( + {"prjid": ObjectId(prjid), "number": griddetails["number"]} + ) if grid_with_number: - return (False, "A grid with the same grid number %s already exists in the project" % (griddetails["number"])) - - remapped_grid = logbookclient[PROJECTS_DB]["grids"].find_one({"prjid": ObjectId(prjid), "box": griddetails["box"], "boxposition": griddetails["boxposition"]}) + return ( + False, + "A grid with the same grid number %s already exists in the project" + % (griddetails["number"]), + ) + + remapped_grid = logbookclient[PROJECTS_DB]["grids"].find_one( + { + "prjid": ObjectId(prjid), + "box": griddetails["box"], + "boxposition": griddetails["boxposition"], + } + ) if remapped_grid: - return (False, "The grid box position %s in grid box %s is already mapped to grid number %s" % (griddetails["boxposition"], griddetails["box"], remapped_grid["number"])) + return ( + False, + "The grid box position %s in grid box %s is already mapped to grid number %s" + % (griddetails["boxposition"], griddetails["box"], remapped_grid["number"]), + ) logbookclient[PROJECTS_DB]["grids"].insert_one(griddetails) return True, "" + def update_project_grid(prjid, gridid, griddetails): """ Update a grid in the project @@ -2234,25 +3436,55 @@ def update_project_grid(prjid, gridid, griddetails): if not validation: return validation, erromsg - existing_grid = logbookclient[PROJECTS_DB]["grids"].find_one({"_id": ObjectId(gridid)}) + existing_grid = logbookclient[PROJECTS_DB]["grids"].find_one( + {"_id": ObjectId(gridid)} + ) - grid_with_number = logbookclient[PROJECTS_DB]["grids"].find_one({"prjid": ObjectId(prjid), "number": griddetails["number"]}) + grid_with_number = logbookclient[PROJECTS_DB]["grids"].find_one( + {"prjid": ObjectId(prjid), "number": griddetails["number"]} + ) if grid_with_number and existing_grid["_id"] != grid_with_number["_id"]: - return (False, "Cannot rename grid %s to one that already exists %s" % (existing_grid["_id"], grid_with_number["_id"])) - - remapped_grid = logbookclient[PROJECTS_DB]["grids"].find_one({"prjid": ObjectId(prjid), "box": griddetails["box"], "boxposition": griddetails["boxposition"]}) + return ( + False, + "Cannot rename grid %s to one that already exists %s" + % (existing_grid["_id"], grid_with_number["_id"]), + ) + + remapped_grid = logbookclient[PROJECTS_DB]["grids"].find_one( + { + "prjid": ObjectId(prjid), + "box": griddetails["box"], + "boxposition": griddetails["boxposition"], + } + ) if remapped_grid and existing_grid["_id"] != remapped_grid["_id"]: - return (False, "The grid box position %s in grid box %s is mapped to different grid number %s" % (griddetails["boxposition"], griddetails["box"], remapped_grid["number"])) - - logbookclient[PROJECTS_DB]["grids"].replace_one({"_id": griddetails["_id"]}, griddetails, upsert=True) + return ( + False, + "The grid box position %s in grid box %s is mapped to different grid number %s" + % (griddetails["boxposition"], griddetails["box"], remapped_grid["number"]), + ) + + logbookclient[PROJECTS_DB]["grids"].replace_one( + {"_id": griddetails["_id"]}, griddetails, upsert=True + ) return True, "" + def link_grid_to_experiment(prjid, gridid, experiment_name): """ Link an existing experiment with a grid """ - currently_mapped = logbookclient[PROJECTS_DB]["grids"].find_one({"exp_name": experiment_name}) + currently_mapped = logbookclient[PROJECTS_DB]["grids"].find_one( + {"exp_name": experiment_name} + ) if currently_mapped: - return (False, "The experiment %s is already mapped to a grid in an existing project %s" % (experiment_name, currently_mapped["prjid"])) - logbookclient[PROJECTS_DB]["grids"].update_one({"_id": ObjectId(gridid), "prjid": ObjectId(prjid)}, {"$set": {"exp_name": experiment_name}}) + return ( + False, + "The experiment %s is already mapped to a grid in an existing project %s" + % (experiment_name, currently_mapped["prjid"]), + ) + logbookclient[PROJECTS_DB]["grids"].update_one( + {"_id": ObjectId(gridid), "prjid": ObjectId(prjid)}, + {"$set": {"exp_name": experiment_name}}, + ) return True, "" diff --git a/dal/imagestores/__init__.py b/explgbk/dal/imagestores/__init__.py similarity index 77% rename from dal/imagestores/__init__.py rename to explgbk/dal/imagestores/__init__.py index df730b6..a792202 100644 --- a/dal/imagestores/__init__.py +++ b/explgbk/dal/imagestores/__init__.py @@ -4,6 +4,7 @@ from .tar import TarIS from .gridfs import GridFSIS + def parseImageStoreURL(imagestoreurl): if imagestoreurl.startswith("http://"): return SeaWeed(imagestoreurl) @@ -12,4 +13,6 @@ def parseImageStoreURL(imagestoreurl): elif imagestoreurl.startswith("tar://"): return TarIS() else: - raise Exception("Cannot initialize image store with unknown scheme " + imagestoreurl) + raise Exception( + "Cannot initialize image store with unknown scheme " + imagestoreurl + ) diff --git a/explgbk/dal/imagestores/gridfs.py b/explgbk/dal/imagestores/gridfs.py new file mode 100644 index 0000000..fbef65b --- /dev/null +++ b/explgbk/dal/imagestores/gridfs.py @@ -0,0 +1,32 @@ +from explgbk.dal.imagestores.imagestore import ImageStore +import logging +import re + +from bson import ObjectId +from gridfs import GridFS + +from explgbk.context import logbookclient + +logger = logging.getLogger(__name__) + + +class GridFSIS(ImageStore): + def store_file_and_return_url( + self, experiment_name, filename, mimetype, filecontents + ): + expdb = logbookclient[experiment_name] + fs = GridFS(expdb) + fid = fs.put(filecontents) + return "mongo://" + str(fid) + + def return_url_contents(self, experiment_name, remote_url): + mtch = re.match("mongo://([\w]*)/(.*)", remote_url) + if mtch: + expdb = logbookclient[mtch.group(1)] + fid = mtch.group(2) + else: + expdb = logbookclient[experiment_name] + fid = remote_url.replace("mongo://", "") + fs = GridFS(expdb) + out = fs.get(ObjectId(fid)) + return out diff --git a/dal/imagestores/imagestore.py b/explgbk/dal/imagestores/imagestore.py similarity index 83% rename from dal/imagestores/imagestore.py rename to explgbk/dal/imagestores/imagestore.py index aa790d4..c55df45 100644 --- a/dal/imagestores/imagestore.py +++ b/explgbk/dal/imagestores/imagestore.py @@ -1,8 +1,11 @@ import abc + class ImageStore(abc.ABC): @abc.abstractmethod - def store_file_and_return_url(self, experiment_name, filename, mimetype, filecontents): + def store_file_and_return_url( + self, experiment_name, filename, mimetype, filecontents + ): """ Store the file identified as filename as the specified filename; the contents of the file are in the file like filecontents object Return a URL that the store can later retrieve as a steamed response. diff --git a/dal/imagestores/seaweed.py b/explgbk/dal/imagestores/seaweed.py similarity index 52% rename from dal/imagestores/seaweed.py rename to explgbk/dal/imagestores/seaweed.py index 70bffc3..5cd7695 100644 --- a/dal/imagestores/seaweed.py +++ b/explgbk/dal/imagestores/seaweed.py @@ -1,4 +1,4 @@ -from dal.imagestores.imagestore import ImageStore +from explgbk.dal.imagestores.imagestore import ImageStore import logging import io @@ -6,20 +6,30 @@ logger = logging.getLogger(__name__) + class SeaWeed(ImageStore): def __init__(self, imagestoreurl): self.imagestoreurl = imagestoreurl - def store_file_and_return_url(self, experiment_name, filename, mimetype, filecontents): + def store_file_and_return_url( + self, experiment_name, filename, mimetype, filecontents + ): isloc = requests.post(self.imagestoreurl + "dir/assign").json() - imgurl = isloc['publicUrl'] + isloc['fid'] + imgurl = isloc["publicUrl"] + isloc["fid"] logger.info("Posting attachment %s to URL %s", filename, imgurl) - files = {'file': (filename, filecontents, mimetype, {'Content-Disposition' : 'inline; filename=%s' % filename})} + files = { + "file": ( + filename, + filecontents, + mimetype, + {"Content-Disposition": "inline; filename=%s" % filename}, + ) + } requests.post(imgurl, files=files) return imgurl def return_url_contents(self, experiment_name, remote_url): - resp = requests.get(remote_url, stream = True) + resp = requests.get(remote_url, stream=True) if not resp: return None return io.BytesIO(resp.content) diff --git a/explgbk/dal/imagestores/tar.py b/explgbk/dal/imagestores/tar.py new file mode 100644 index 0000000..01d9c91 --- /dev/null +++ b/explgbk/dal/imagestores/tar.py @@ -0,0 +1,63 @@ +from explgbk.dal.imagestores.imagestore import ImageStore +import os +import logging +import tarfile +import io + +from bson import ObjectId + +from explgbk.context import logbookclient, LOGBOOK_SITE + +logger = logging.getLogger(__name__) + + +class TarIS(ImageStore): + def __get_experiment_results_folder__(self, experiment_name): + if LOGBOOK_SITE == "LCLS": + expdb = logbookclient[experiment_name] + instrument = expdb["info"].find_one()["instrument"].lower() + results = os.path.join( + "/reg/d/psdm/", instrument, experiment_name, "results" + ) + return results + return None + + def store_file_and_return_url( + self, experiment_name, filename, mimetype, filecontents + ): + results_folder = self.__get_experiment_results_folder__(experiment_name) + if not ( + results_folder + and os.path.exists(results_folder) + and os.path.isdir(results_folder) + ): + raise Exception( + "Missing results folder for experiment %s %s" + % (experiment_name, results_folder) + ) + archive_folder = os.path.join(results_folder, "archive") + if not os.path.exists(archive_folder): + os.mkdir(archive_folder) + with tarfile.open(os.path.join(archive_folder, "attachments.tar"), "a") as t: + tinfo = tarfile.TarInfo(str(ObjectId())) + filecontents.seek(0, 2) + tinfo.size = filecontents.tell() + filecontents.seek(0, 0) + t.addfile(tinfo, filecontents) + return "tar://" + tinfo.name + + def return_url_contents(self, experiment_name, remote_url): + attachments_file = os.path.join( + self.__get_experiment_results_folder__(experiment_name), + "archive", + "attachments.tar", + ) + if not (attachments_file and os.path.exists(attachments_file)): + raise Exception( + "Missing attachments tar file for experiment %s %s" + % (experiment_name, attachments_file) + ) + + fid = remote_url.replace("tar://", "") + with tarfile.open(attachments_file, "r") as t: + return io.BytesIO(t.extractfile(fid).read()) diff --git a/explgbk/dal/run_control.py b/explgbk/dal/run_control.py new file mode 100644 index 0000000..dcafe09 --- /dev/null +++ b/explgbk/dal/run_control.py @@ -0,0 +1,255 @@ +""" +Run control business logic. +""" + +import datetime +import logging + + +from pymongo import DESCENDING, ReturnDocument, ReadPreference +from bson import ObjectId + +from explgbk.context import logbookclient +from explgbk.dal.utils import escape_chars_for_mongo + +__author__ = "mshankar@slac.stanford.edu" + +logger = logging.getLogger(__name__) + + +def start_run( + experiment_name, + run_type, + user_specified_run_number=None, + user_specified_start_time=None, + user_specified_sample=None, + params=None, +): + """ + Start a new run for the specified experiment + If the user_specified_run_number is not specified; we use the next_runnum autoincrement counter. + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) + if not user_specified_run_number: + next_run_num_doc = expdb["counters"].find_one_and_update( + {"_id": "next_runnum"}, + {"$inc": {"seq": 1}}, + return_document=ReturnDocument.AFTER, + ) + if not next_run_num_doc: + raise Exception( + "Could not update run number counter for experiment %s" + % experiment_name + ) + next_run_num = next_run_num_doc["seq"] + logger.info("Next run for experiment %s is %s", experiment_name, next_run_num) + else: + next_run_num = user_specified_run_number + logger.info( + "Next run for experiment %s is from the user %s", + experiment_name, + next_run_num, + ) + + begin_time = ( + user_specified_start_time + if user_specified_start_time + else datetime.datetime.utcnow() + ) + + run_doc = { + "num": next_run_num, + "type": run_type, + "begin_time": begin_time, + "end_time": None, + "params": {}, + "editable_params": {}, + } + if user_specified_sample: + user_sample = expdb.samples.find_one({"name": user_specified_sample}) + if user_sample: + run_doc["sample"] = user_sample["_id"] + else: + raise Exception( + "Could not find sample %s for experiment %s" + % (user_specified_sample, experiment_name) + ) + else: + current_sample = expdb.current.find_one({"_id": "sample"}) + if current_sample: + run_doc["sample"] = ObjectId(current_sample["sample"]) + if params: + run_doc["params"] = params + + expdb["runs"].insert_one(run_doc) + return expdb["runs"].find_one({"num": next_run_num}) + + +def get_current_run(experiment_name): + """ + Get the run document for the run with the maximum run number. + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) + current_run_doc = list(expdb.runs.find().sort([("num", DESCENDING)]).limit(1)) + return current_run_doc[0] if current_run_doc else None + + +def get_run_doc_for_run_num(experiment_name, run_num): + """ + Get the run document for the specified run number + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) + run_doc = expdb.runs.find_one({"num": run_num}) + if run_doc: + return run_doc + return None + + +def get_specified_run_params_for_all_runs(experiment_name, run_params): + """ + Get the specified run parameters for all runs in the experiment. + For now, this only includes the non-editable parameters submitted by the DAQ. + """ + expdb = logbookclient[experiment_name] + projection_op = {"num": 1} + for run_param in run_params: + projection_op["params." + escape_chars_for_mongo(run_param)] = 1 + return [x for x in expdb.runs.find({}, projection_op)] + + +def map_param_editable_to_run_nums(experiment_name, param_editable): + """ + Pass in a run param name or an editable name. + Returns a dict of param value to array of run numbers. + The method looks at both run params (as uploaded by the DAQ) and editable params (as set by the user) + Since this is a very user facing call, we let the editable win. + That is, if there is an editable param with the same name as a run param, we use the editable param as the source of the pivot. + """ + expdb = logbookclient[experiment_name] + + def __getval__( + rn, parts + ): # Should get you editable_params.TAG.value from { "num" : 23, "editable_params" : { "TAG" : { "value" : "Ravenclaw" } } } + ret = rn + for part in parts: + ret = ret[part] + return ret + + def __pivot__(rns, fqn): + parts = fqn.split(".") + ret = {} + for rn in rns: + val = __getval__(rn, parts) + if val not in ret.keys(): + ret[val] = [] + ret[val].append(rn["num"]) + return ret + + fqn = "editable_params." + escape_chars_for_mongo(param_editable) + ".value" + editables = list( + expdb.runs.find({fqn: {"$exists": 1}}, {"_id": 0, "num": 1, fqn: 1}) + ) + if editables and len(editables) > 0: + logger.debug("Found an editable param with name %s", fqn) + return __pivot__(editables, fqn) + + fqn = "params." + escape_chars_for_mongo(param_editable) + params = list(expdb.runs.find({fqn: {"$exists": 1}}, {"_id": 0, "num": 1, fqn: 1})) + if params and len(params) > 0: + logger.debug("Found an DAQ param with name %s", fqn) + return __pivot__(params, fqn) + + return {} + + +def get_run_nums_matching_params(experiment_name, query_document): + """ + Get an array of run numbers for all runs that have the specified value for the specified parameter. + This is a very simplistic query. + """ + expdb = logbookclient[experiment_name] + projection_op = {"num": 1} + query = { + "params." + escape_chars_for_mongo(k): v for k, v in query_document.items() + } + return [x["num"] for x in expdb.runs.find(query, projection_op)] + + +def get_run_nums_matching_editable_regex(experiment_name, param_name, incoming_regex): + """ + Get an array of run numbers for all runs that have an editable param matching the specified regex. + We do a case insensitive match. + """ + expdb = logbookclient[experiment_name] + projection_op = {"num": 1} + query = { + "editable_params." + escape_chars_for_mongo(param_name) + ".value": { + "$regex": incoming_regex, + "$options": "i", + } + } + return [x["num"] for x in expdb.runs.find(query, projection_op)] + + +def get_sample_for_run(experiment_name, run_num): + """ + Lookup the sample for the specified run + """ + expdb = logbookclient[experiment_name] + run_doc = expdb.runs.find_one({"num": run_num}) + if not run_doc: + return None + if "sample" not in run_doc: + return None + return expdb.samples.find_one({"_id": run_doc["sample"]}) + + +def end_run(experiment_name, user_specified_end_time=None): + """ + End the current run; this is mostly a matter of filling in the end time + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) + current_run_doc = get_current_run(experiment_name) + end_time = ( + user_specified_end_time + if user_specified_end_time + else datetime.datetime.utcnow() + ) + return expdb.runs.find_one_and_update( + {"num": current_run_doc["num"]}, + {"$set": {"end_time": end_time}}, + return_document=ReturnDocument.AFTER, + ) + + +def is_run_closed(experiment_name, run_num): + """ + Check if the specified run is closed + """ + expdb = logbookclient.get_database( + experiment_name, read_preference=ReadPreference.PRIMARY + ) + run_doc = expdb.runs.find_one({"num": run_num}) + if run_doc and run_doc.get("end_time", None): + return True + return False + + +def add_run_params(experiment_name, run_doc, run_params): + """ + Add run parameters to the specified run. + """ + expdb = logbookclient[experiment_name] + return expdb.runs.find_one_and_update( + {"num": run_doc["num"]}, + {"$set": run_params}, + return_document=ReturnDocument.AFTER, + ) diff --git a/dal/utils.py b/explgbk/dal/utils.py similarity index 89% rename from dal/utils.py rename to explgbk/dal/utils.py index d8539af..d22f8af 100644 --- a/dal/utils.py +++ b/explgbk/dal/utils.py @@ -1,6 +1,7 @@ -''' +""" Various small utilties. -''' +""" + import json import math import collections @@ -8,6 +9,7 @@ from bson import ObjectId from datetime import datetime + class JSONEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, ObjectId): @@ -37,24 +39,25 @@ def replaceInfNan(d): def escape_chars_for_mongo(attrname): - ''' + """ Mongo uses the '$' and '.' characters for query syntax. So, if your attributes have these characters, they get converted to dictionaires etc. EPICS variables use the '.' character quite a bit. We replace these with their unicode equivalents '.' gets replaced with U+FF0E '$' gets replaced with U+FF04 This will cause interesting query failures; but there does not seem to be a better choice. - For example, use something like so to find the param - db.runs.findOne({}, {"params.AMO:HFP:MMS:72\uFF0ERBV": 1}) - ''' - return attrname.replace(".", u"\uFF0E").replace("$", u"\uFF04") + For example, use something like so to find the param - db.runs.findOne({}, {"params.AMO:HFP:MMS:72\uff0eRBV": 1}) + """ + return attrname.replace(".", "\uff0e").replace("$", "\uff04") + def reverse_escape_chars_for_mongo(attrname): - ''' + """ Mongo uses the '$' and '.' characters for query syntax. So, if your attributes have these characters, they get converted to dictionaires etc. EPICS variables use the '.' character quite a bit. We replace these with their unicode equivalents '.' gets replaced with U+FF0E '$' gets replaced with U+FF04 This method undo'es the escape_chars_for_mongo method - ''' - return attrname.replace(u"\uFF0E", ".").replace(u"\uFF04", "$") + """ + return attrname.replace("\uff0e", ".").replace("\uff04", "$") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..137cf4e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "explgbk" +version = "0.1.0" +description = "LCLS, Cryo and UED data management logbook" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + {name = "SLAC", email = "mshankar@slac.stanford.edu"} +] +dependencies = [ + "pymongo==4.5.0", + "kafka-python==2.0.2", + "gunicorn==21.2.0", + "eventlet==0.36.1", + "Flask==2.3.3", + "Flask-SocketIO==5.3.6", + "requests==2.31.0", + "cachetools==5.3.1", + "pytz==2023.3.post1", + "python-dateutil==2.8.2", + "pyjwt[crypto]==2.8.0", + "seaweed==0.0.1", + "flask-authnz", + "flask-socket-util", +] + +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["explgbk"] + +[dependency-groups] +dev = [ + "ruff>=0.15.1", +] + +[tool.uv.sources] +flask-authnz = { git = "https://github.com/slaclab/flask_authnz", rev = "046fb9c46835be47f9387e878efd2b898a6d4025" } +flask-socket-util = { git = "https://github.com/slaclab/flask_socket_util", rev = "ad42186e1fb6d5406dffbc459e200bc490e5af91" } + +[tool.ruff] +exclude = ["modules"] diff --git a/runscripts/migration/LCLS/0002.py b/runscripts/migration/LCLS/0002.py deleted file mode 100644 index e0f74b6..0000000 --- a/runscripts/migration/LCLS/0002.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/python -""" -This migration script copies over the group membership from LDAP into the database. -This is how the newer experiments will look like. -We add the user into the database and publish a Kafka message. -The rolemgr then adds it to LDAP triggered by the Kafka message. -So, both database and LDAP have the group membership explicitly listed. -""" -import os -import logging -import argparse -import re - -from pymongo import MongoClient -from flask_authnz import UserGroups - - -MONGODB_HOST=os.environ.get('MONGODB_HOST', "localhost") -MONGODB_PORT=int(os.environ.get('MONGODB_PORT', 27017)) -MONGODB_USERNAME=os.environ['MONGODB_USERNAME'] -MONGODB_PASSWORD=os.environ['MONGODB_PASSWORD'] - -usergroups = UserGroups() -logbookclient = MongoClient(host=MONGODB_HOST, port=MONGODB_PORT, username=MONGODB_USERNAME, password=MONGODB_PASSWORD, tz_aware=True) - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Migrate group memberships from LDAP to the database') - parser.add_argument('--verbose', action="store_true") - parser.add_argument('--dryrun', action="store_true") - parser.add_argument('experiment', help='Specify experiment name; to run this for all experiments, use "all"') - args = parser.parse_args() - - if args.verbose: - logging.basicConfig(level=logging.DEBUG) - else: - logging.basicConfig(level=logging.INFO) - logger = logging.getLogger(__name__) - - experiments = [] - def __check_and_add__(experiment_name): - if experiment_name in ["admin", "config", "local", "site"]: - return - expdb = logbookclient[experiment_name] - collnames = list(expdb.list_collection_names()) - if 'info' in collnames and 'roles' in collnames: - info = expdb["info"].find_one({}, {"latest_setup": 0}) - if 'name' not in info or 'instrument' not in info: - logger.debug("Database %s has a info collection but the info object does not have an instrument or name", experiment_name) - return - if expdb["roles"].count_documents({"players": experiment_name}) <= 0: - logger.debug("Experiment %s has does not have an experiment specific role", experiment_name) - return - experiments.append(experiment_name) - if args.experiment != "all": - __check_and_add__(args.experiment) - else: - database_names = list(logbookclient.database_names()) - for experiment_name in database_names: - __check_and_add__(experiment_name) - - for experiment in experiments: - logger.debug("Migrating LDAP memberships for %s", experiment) - expdb = logbookclient[experiment] - for role in expdb["roles"].find({"players": experiment}): - logger.debug("Found role %s for experiment %s", role["name"], experiment) - try: - grps = usergroups.get_group_members(experiment) - except KeyError as ke: - if ke.args[0] == "memberUid": - logger.debug("Empty group membership for %s ", experiment) - continue - else: - raise ke - except Exception as ex: - raise ex - if isinstance(grps, list): - for uid in usergroups.get_group_members(experiment): - if len(uid) < 3: - logger.error("The experiment group %s probably has only one member; too small a userid %s", experiment, uid) - continue - logger.info("Adding user %s to role %s for experiment %s", "uid:"+uid, role["name"], experiment) - if args.dryrun: - continue - expdb["roles"].update_one({"app": role["app"], "name": role["name"]}, {"$addToSet": {"players": "uid:"+uid}}) - elif isinstance(grps, str): - uid = grps - if len(uid) < 3: - logger.error("The experiment group %s probably has only one member; too small a userid %s", experiment, uid) - continue - logger.info("Adding user %s to role %s for experiment %s", "uid:"+uid, role["name"], experiment) - if args.dryrun: - continue - expdb["roles"].update_one({"app": role["app"], "name": role["name"]}, {"$addToSet": {"players": "uid:"+uid}}) - else: - raise Exception("Unexpected type for roles %s", type(grps)) - - for role in expdb["roles"].find({"players": {"$regex": re.compile("^ps-.*")}}): - instr_players = [x for x in role["players"] if x.startswith("ps-")] - if not instr_players: - raise Exception("Mongo and python disagree.") - continue - logger.info("Removing instrument roles %s for role %s for experiment %s", ",".join(instr_players), role["name"], experiment) - if args.dryrun: - continue - expdb["roles"].update_one({"app": role["app"], "name": role["name"]}, {"$pull": {"players": {"$in": instr_players }}}) diff --git a/runscripts/backup.py b/scripts/backup.py similarity index 56% rename from runscripts/backup.py rename to scripts/backup.py index 4a0e37f..9201fae 100755 --- a/runscripts/backup.py +++ b/scripts/backup.py @@ -1,12 +1,12 @@ #!/usr/bin/env python -''' +""" Script for backing up the logbook/mongo. Note regarding the imagestores: We're deprecating support for external image stores; so only the mongo image stores are backed up. The file based image stores are expected to be backed up to tape as part of the experimental data backup. Backups are stored in a database per folder format with the filename of the backup based on the time the backup was initiated. -''' +""" import os import sys @@ -15,10 +15,9 @@ import argparse import subprocess import datetime -import shutil import pathlib -from pymongo import MongoClient, ASCENDING, DESCENDING +from pymongo import MongoClient DATETIME_FILE_NAME_FORMAT = "%Y_%m_%d_%H_%M_%S" @@ -28,6 +27,7 @@ logger = logging.getLogger(__name__) + def configureLogging(verbose): loglevel = logging.INFO @@ -38,21 +38,29 @@ def configureLogging(verbose): root.setLevel(loglevel) ch = logging.StreamHandler(sys.stdout) ch.setLevel(loglevel) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) ch.setFormatter(formatter) root.addHandler(ch) + def test_executable_exists(commands): """ Check to see if we can run the speficied command. """ try: - subprocess.run(commands, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + subprocess.run( + commands, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) return True - except: - logger.exception("Exception when checking to see if command exists %s", commands) + except Exception: + logger.exception( + "Exception when checking to see if command exists %s", commands + ) sys.exit(-1) + def find_latest_timestamp_in_folder(folder): list_of_files = glob.glob(os.path.join(folder, "*")) if list_of_files: @@ -60,11 +68,15 @@ def find_latest_timestamp_in_folder(folder): return datetime.datetime.fromtimestamp(os.path.getmtime(latest_file)) return None + def backup_database(args, database_name): logger.debug("Looking to backup for %s", database_name) db_backup_folder = os.path.join(args.backup_folder, database_name) pathlib.Path(db_backup_folder).mkdir(parents=True, exist_ok=True) - archive_file_name = os.path.join(db_backup_folder, datetime.datetime.now().strftime(DATETIME_FILE_NAME_FORMAT) + ".gz") + archive_file_name = os.path.join( + db_backup_folder, + datetime.datetime.now().strftime(DATETIME_FILE_NAME_FORMAT) + ".gz", + ) # We need to make a backup. Check to see if we need to delete an older backup if args.keep_backups != -1: @@ -75,47 +87,86 @@ def backup_database(args, database_name): list_of_files = glob.glob(os.path.join(db_backup_folder, "*.gz")) earliest_file = min(list_of_files, key=os.path.getmtime) if earliest_file: - logger.info("%s - Removing earliest file %s for database", database_name, earliest_file) + logger.info( + "%s - Removing earliest file %s for database", + database_name, + earliest_file, + ) os.remove(earliest_file) - uriwithdb = args.mongo_uri.split("?")[0] + database_name + "?" + args.mongo_uri.split("?")[1] + uriwithdb = ( + args.mongo_uri.split("?")[0] + + database_name + + "?" + + args.mongo_uri.split("?")[1] + ) logger.info("%s - New archive %s", database_name, archive_file_name) try: - mdargs = [ args.mongodump_path, + mdargs = [ + args.mongodump_path, "-vv", - "--uri", uriwithdb, + "--uri", + uriwithdb, "--gzip", - "--archive=" + archive_file_name - ] - mdp = subprocess.run(mdargs, check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding="utf-8") + "--archive=" + archive_file_name, + ] + mdp = subprocess.run( + mdargs, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + ) if mdp.returncode != 0: - logger.error("mongodump command - %s - returned non-zero error code %s", " ".join(mdargs), mdp.returncode) + logger.error( + "mongodump command - %s - returned non-zero error code %s", + " ".join(mdargs), + mdp.returncode, + ) logger.error(mdp.stdout) logger.error(mdp.stderr) else: logger.debug(mdp.stdout) logger.debug(mdp.stderr) - except: + except Exception: logger.exception("Exception dumping database to %s", archive_file_name) sys.exit(-1) if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbose", action='store_true', help="Turn on verbose logging") - parser.add_argument("--mongo_uri", help="The MongoURI to use; this seems the best way to specify some useful options including readPreference=secondary. This needs to be passed in a certain way for mongodump to work; that is, the path part should end with a / followed immediately by a ? with the options. For example, mongodb://backupuser:backuppassword@localhost:27017/?authSource=admin&readPreference=secondary", required=True) - parser.add_argument("--mongodump_path", help="Full path to the mongodump command. If not specified, we use mongodump from the PATH.", default="mongodump") - parser.add_argument("--keep_backups", help="Keep at least this many complete backups; older backups are deleted.", default=-1, type=int) - parser.add_argument("backup_folder", help="The folder containing the backups.") + parser.add_argument( + "-v", "--verbose", action="store_true", help="Turn on verbose logging" + ) + parser.add_argument( + "--mongo_uri", + help="The MongoURI to use; this seems the best way to specify some useful options including readPreference=secondary. This needs to be passed in a certain way for mongodump to work; that is, the path part should end with a / followed immediately by a ? with the options. For example, mongodb://backupuser:backuppassword@localhost:27017/?authSource=admin&readPreference=secondary", + required=True, + ) + parser.add_argument( + "--mongodump_path", + help="Full path to the mongodump command. If not specified, we use mongodump from the PATH.", + default="mongodump", + ) + parser.add_argument( + "--keep_backups", + help="Keep at least this many complete backups; older backups are deleted.", + default=-1, + type=int, + ) + parser.add_argument("backup_folder", help="The folder containing the backups.") args = parser.parse_args() configureLogging(args.verbose) test_executable_exists([args.mongodump_path, "--help"]) if not os.path.exists(args.backup_folder) or not os.path.isdir(args.backup_folder): - logger.error("The root folder for database backups %s does not seem to exist", args.backup_folder) + logger.error( + "The root folder for database backups %s does not seem to exist", + args.backup_folder, + ) sys.exit(-1) logbookclient = MongoClient(args.mongo_uri, tz_aware=True) @@ -123,7 +174,7 @@ def backup_database(args, database_name): logger.info("Gathering the list of databases") database_names = sorted(list(logbookclient.list_database_names())) for database_name in database_names: - if database_name in [ "config", "local" ]: + if database_name in ["config", "local"]: logger.info("Not backing up system database %s", database_name) continue backup_database(args, database_name) diff --git a/runscripts/migration/0001.mongo b/scripts/migration/0001.mongo similarity index 100% rename from runscripts/migration/0001.mongo rename to scripts/migration/0001.mongo diff --git a/runscripts/migration/0002.mongo b/scripts/migration/0002.mongo similarity index 100% rename from runscripts/migration/0002.mongo rename to scripts/migration/0002.mongo diff --git a/runscripts/migration/0003.mongo b/scripts/migration/0003.mongo similarity index 100% rename from runscripts/migration/0003.mongo rename to scripts/migration/0003.mongo diff --git a/runscripts/migration/0004.mongo b/scripts/migration/0004.mongo similarity index 100% rename from runscripts/migration/0004.mongo rename to scripts/migration/0004.mongo diff --git a/runscripts/migration/0005.mongo b/scripts/migration/0005.mongo similarity index 100% rename from runscripts/migration/0005.mongo rename to scripts/migration/0005.mongo diff --git a/runscripts/migration/0006.mongo b/scripts/migration/0006.mongo similarity index 100% rename from runscripts/migration/0006.mongo rename to scripts/migration/0006.mongo diff --git a/runscripts/migration/0007.mongo b/scripts/migration/0007.mongo similarity index 100% rename from runscripts/migration/0007.mongo rename to scripts/migration/0007.mongo diff --git a/runscripts/migration/0008.mongo b/scripts/migration/0008.mongo similarity index 100% rename from runscripts/migration/0008.mongo rename to scripts/migration/0008.mongo diff --git a/runscripts/migration/LCLS/0001.mongo b/scripts/migration/LCLS/0001.mongo similarity index 100% rename from runscripts/migration/LCLS/0001.mongo rename to scripts/migration/LCLS/0001.mongo diff --git a/scripts/migration/LCLS/0002.py b/scripts/migration/LCLS/0002.py new file mode 100644 index 0000000..6940f63 --- /dev/null +++ b/scripts/migration/LCLS/0002.py @@ -0,0 +1,157 @@ +#!/usr/bin/python +""" +This migration script copies over the group membership from LDAP into the database. +This is how the newer experiments will look like. +We add the user into the database and publish a Kafka message. +The rolemgr then adds it to LDAP triggered by the Kafka message. +So, both database and LDAP have the group membership explicitly listed. +""" + +import os +import logging +import argparse +import re + +from pymongo import MongoClient +from flask_authnz import UserGroups + + +MONGODB_HOST = os.environ.get("MONGODB_HOST", "localhost") +MONGODB_PORT = int(os.environ.get("MONGODB_PORT", 27017)) +MONGODB_USERNAME = os.environ["MONGODB_USERNAME"] +MONGODB_PASSWORD = os.environ["MONGODB_PASSWORD"] + +usergroups = UserGroups() +logbookclient = MongoClient( + host=MONGODB_HOST, + port=MONGODB_PORT, + username=MONGODB_USERNAME, + password=MONGODB_PASSWORD, + tz_aware=True, +) + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Migrate group memberships from LDAP to the database" + ) + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--dryrun", action="store_true") + parser.add_argument( + "experiment", + help='Specify experiment name; to run this for all experiments, use "all"', + ) + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + + experiments = [] + + def __check_and_add__(experiment_name): + if experiment_name in ["admin", "config", "local", "site"]: + return + expdb = logbookclient[experiment_name] + collnames = list(expdb.list_collection_names()) + if "info" in collnames and "roles" in collnames: + info = expdb["info"].find_one({}, {"latest_setup": 0}) + if "name" not in info or "instrument" not in info: + logger.debug( + "Database %s has a info collection but the info object does not have an instrument or name", + experiment_name, + ) + return + if expdb["roles"].count_documents({"players": experiment_name}) <= 0: + logger.debug( + "Experiment %s has does not have an experiment specific role", + experiment_name, + ) + return + experiments.append(experiment_name) + + if args.experiment != "all": + __check_and_add__(args.experiment) + else: + database_names = list(logbookclient.database_names()) + for experiment_name in database_names: + __check_and_add__(experiment_name) + + for experiment in experiments: + logger.debug("Migrating LDAP memberships for %s", experiment) + expdb = logbookclient[experiment] + for role in expdb["roles"].find({"players": experiment}): + logger.debug("Found role %s for experiment %s", role["name"], experiment) + try: + grps = usergroups.get_group_members(experiment) + except KeyError as ke: + if ke.args[0] == "memberUid": + logger.debug("Empty group membership for %s ", experiment) + continue + else: + raise ke + except Exception as ex: + raise ex + if isinstance(grps, list): + for uid in usergroups.get_group_members(experiment): + if len(uid) < 3: + logger.error( + "The experiment group %s probably has only one member; too small a userid %s", + experiment, + uid, + ) + continue + logger.info( + "Adding user %s to role %s for experiment %s", + "uid:" + uid, + role["name"], + experiment, + ) + if args.dryrun: + continue + expdb["roles"].update_one( + {"app": role["app"], "name": role["name"]}, + {"$addToSet": {"players": "uid:" + uid}}, + ) + elif isinstance(grps, str): + uid = grps + if len(uid) < 3: + logger.error( + "The experiment group %s probably has only one member; too small a userid %s", + experiment, + uid, + ) + continue + logger.info( + "Adding user %s to role %s for experiment %s", + "uid:" + uid, + role["name"], + experiment, + ) + if args.dryrun: + continue + expdb["roles"].update_one( + {"app": role["app"], "name": role["name"]}, + {"$addToSet": {"players": "uid:" + uid}}, + ) + else: + raise Exception("Unexpected type for roles %s", type(grps)) + + for role in expdb["roles"].find({"players": {"$regex": re.compile("^ps-.*")}}): + instr_players = [x for x in role["players"] if x.startswith("ps-")] + if not instr_players: + raise Exception("Mongo and python disagree.") + continue + logger.info( + "Removing instrument roles %s for role %s for experiment %s", + ",".join(instr_players), + role["name"], + experiment, + ) + if args.dryrun: + continue + expdb["roles"].update_one( + {"app": role["app"], "name": role["name"]}, + {"$pull": {"players": {"$in": instr_players}}}, + ) diff --git a/runscripts/migration/LCLS/0003.mongo b/scripts/migration/LCLS/0003.mongo similarity index 100% rename from runscripts/migration/LCLS/0003.mongo rename to scripts/migration/LCLS/0003.mongo diff --git a/runscripts/migration/LCLS/0004.mongo b/scripts/migration/LCLS/0004.mongo similarity index 100% rename from runscripts/migration/LCLS/0004.mongo rename to scripts/migration/LCLS/0004.mongo diff --git a/runscripts/migration/LCLS/add_missing_run_counters.mongo b/scripts/migration/LCLS/add_missing_run_counters.mongo similarity index 100% rename from runscripts/migration/LCLS/add_missing_run_counters.mongo rename to scripts/migration/LCLS/add_missing_run_counters.mongo diff --git a/runscripts/migration/LCLS/mec_tags_with_spaces.mongo b/scripts/migration/LCLS/mec_tags_with_spaces.mongo similarity index 100% rename from runscripts/migration/LCLS/mec_tags_with_spaces.mongo rename to scripts/migration/LCLS/mec_tags_with_spaces.mongo diff --git a/runscripts/migration/cryo/0001.mongo b/scripts/migration/cryo/0001.mongo similarity index 100% rename from runscripts/migration/cryo/0001.mongo rename to scripts/migration/cryo/0001.mongo diff --git a/runscripts/migration/cryo/0002.mongo b/scripts/migration/cryo/0002.mongo similarity index 100% rename from runscripts/migration/cryo/0002.mongo rename to scripts/migration/cryo/0002.mongo diff --git a/runscripts/migration/misc/spaces_in_tags.mongo b/scripts/migration/misc/spaces_in_tags.mongo similarity index 100% rename from runscripts/migration/misc/spaces_in_tags.mongo rename to scripts/migration/misc/spaces_in_tags.mongo diff --git a/runscripts/rundev.sh b/scripts/rundev.sh similarity index 77% rename from runscripts/rundev.sh rename to scripts/rundev.sh index 38455cf..fb1f80a 100755 --- a/runscripts/rundev.sh +++ b/scripts/rundev.sh @@ -18,19 +18,18 @@ fi export ACCESS_LOG_FORMAT='%(h)s %(l)s %({REMOTE-USER}i)s %(t)s "%(r)s" "%(q)s" %(s)s %(b)s %(D)s' # Of course, please change this port to the appropriate port in the 8000-1000 range. -# Also change start:app to your_service:app (this should make it easier to identify your service amongst the pile of gunicorns) +# Also change explgbk.app:app to your_service:app (this should make it easier to identify your service amongst the pile of gunicorns) # Add a proxy in the web servce to proxy this port onto the location for this service. export SERVER_IP_PORT=${SERVER_IP_PORT:-"0.0.0.0:5000"} -# Assume that the current directory for the process is this directory. -export PYTHONPATH="modules/flask_authnz:modules/flask_socket_util:${PYTHONPATH}" - +# With the new package structure, the explgbk package is properly importable +# No need to add to PYTHONPATH as it's a proper Python package export LOG_LEVEL=${LOG_LEVEL:-"INFO"} export RELOAD=${RELOAD:-""} export WORKER_CONFIG=${WORKER_CONFIG:-""} # The exec assumes you are calling this from supervisord. If you call this from the command line; your bash shell is proabably gone and you need to log in. -exec gunicorn start:app -b ${SERVER_IP_PORT} --worker-class eventlet ${WORKER_CONFIG} ${RELOAD} \ +exec gunicorn explgbk.app:app -b ${SERVER_IP_PORT} --worker-class eventlet ${WORKER_CONFIG} ${RELOAD} \ --log-level=${LOG_LEVEL} --capture-output --enable-stdio-inheritance \ --timeout 300 --graceful-timeout 1 \ --access-logfile - --access-logformat "${ACCESS_LOG_FORMAT}" diff --git a/services/__init__.py b/services/__init__.py index d2d6b66..077fffc 100755 --- a/services/__init__.py +++ b/services/__init__.py @@ -1 +1 @@ -__author__ = 'mshankar@slac.stanford.edu' +__author__ = "mshankar@slac.stanford.edu" diff --git a/start.py b/start.py deleted file mode 100755 index 6ce4857..0000000 --- a/start.py +++ /dev/null @@ -1,61 +0,0 @@ -from flask import Flask, current_app -import logging -import os -import sys -import json -from kafka import KafkaConsumer, TopicPartition -from threading import Thread - - -root = logging.getLogger() -root.setLevel(logging.getLevelName(os.environ.get("LOG_LEVEL", "INFO"))) -logging.getLogger('kafka').setLevel(logging.INFO) -logging.getLogger('engineio').setLevel(logging.WARN) -logging.getLogger('flask_authnz').setLevel(logging.WARN) -ch = logging.StreamHandler(sys.stdout) -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -ch.setFormatter(formatter) -root.addHandler(ch) - -logger = logging.getLogger(__name__) - -from context import app, security - -from pages import pages_blueprint - -from services.explgbk import explgbk_blueprint -from flask_socket_util import socket_service - -import dal.exp_cache - -__author__ = 'mshankar@slac.stanford.edu' - - -# Initialize application. -app = Flask("explgbk") -# Set the expiration for static files -app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 60*60; - -app.secret_key = "This is a secret key that is somewhat temporary." -app.debug = False - -@app.template_filter('json') -def jinga2_jsonfilter(value): - return json.dumps(value) - - - -# Register routes. -app.register_blueprint(pages_blueprint) -app.register_blueprint(explgbk_blueprint) - -socket_service.init_app(app, security, kafkatopics = ["experiments", "elog", "runs", "shifts", "samples", "file_catalog", "workflow_jobs"]) - -dal.exp_cache.init_app(app) - -logger.info("Server initialization complete") - -if __name__ == '__main__': - print("Please use gunicorn for development as well.") - sys.exit(-1) diff --git a/static/lgbk.css b/static/css/lgbk.css similarity index 100% rename from static/lgbk.css rename to static/css/lgbk.css diff --git a/static/lgbk.scss b/static/css/lgbk.scss similarity index 100% rename from static/lgbk.scss rename to static/css/lgbk.scss diff --git a/static/html/components/all.js b/static/js/all.js similarity index 100% rename from static/html/components/all.js rename to static/js/all.js diff --git a/static/html/components/customparam.js b/static/js/customparam.js similarity index 100% rename from static/html/components/customparam.js rename to static/js/customparam.js diff --git a/static/experiments.js b/static/js/experiments.js similarity index 100% rename from static/experiments.js rename to static/js/experiments.js diff --git a/static/html/components/fachkbx.js b/static/js/fachkbx.js similarity index 100% rename from static/html/components/fachkbx.js rename to static/js/fachkbx.js diff --git a/static/lgbk.js b/static/js/lgbk.js similarity index 100% rename from static/lgbk.js rename to static/js/lgbk.js diff --git a/static/spgntr.js b/static/js/spgntr.js similarity index 100% rename from static/spgntr.js rename to static/js/spgntr.js diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..6f852b3 --- /dev/null +++ b/uv.lock @@ -0,0 +1,844 @@ +version = 1 +revision = 3 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version < '3.13'", +] + +[[package]] +name = "bidict" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/6e/026678aa5a830e07cd9498a05d3e7e650a4f56a42f267a53d22bcda1bdc9/bidict-0.23.1.tar.gz", hash = "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71", size = 29093, upload-time = "2024-02-18T19:09:05.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/37/e8730c3587a65eb5645d4aba2d27aae48e8003614d6aaf15dda67f702f1f/bidict-0.23.1-py3-none-any.whl", hash = "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5", size = 32764, upload-time = "2024-02-18T19:09:04.156Z" }, +] + +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + +[[package]] +name = "cachetools" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/8b/8e2ebf5ee26c21504de5ea2fb29cc6ae612b35fd05f959cdb641feb94ec4/cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b", size = 27985, upload-time = "2023-05-27T20:44:00.567Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/c9/c8a7710f2cedcb1db9224fdd4d8307c9e48cbddc46c18b515fefc0f1abbe/cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590", size = 9288, upload-time = "2023-05-27T20:43:58.81Z" }, +] + +[[package]] +name = "certifi" +version = "2026.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/d7/516d984057745a6cd96575eea814fe1edd6646ee6efd552fb7b0921dec83/cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44", size = 184283, upload-time = "2025-09-08T23:22:08.01Z" }, + { url = "https://files.pythonhosted.org/packages/9e/84/ad6a0b408daa859246f57c03efd28e5dd1b33c21737c2db84cae8c237aa5/cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49", size = 180504, upload-time = "2025-09-08T23:22:10.637Z" }, + { url = "https://files.pythonhosted.org/packages/50/bd/b1a6362b80628111e6653c961f987faa55262b4002fcec42308cad1db680/cffi-2.0.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:53f77cbe57044e88bbd5ed26ac1d0514d2acf0591dd6bb02a3ae37f76811b80c", size = 208811, upload-time = "2025-09-08T23:22:12.267Z" }, + { url = "https://files.pythonhosted.org/packages/4f/27/6933a8b2562d7bd1fb595074cf99cc81fc3789f6a6c05cdabb46284a3188/cffi-2.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3e837e369566884707ddaf85fc1744b47575005c0a229de3327f8f9a20f4efeb", size = 216402, upload-time = "2025-09-08T23:22:13.455Z" }, + { url = "https://files.pythonhosted.org/packages/05/eb/b86f2a2645b62adcfff53b0dd97e8dfafb5c8aa864bd0d9a2c2049a0d551/cffi-2.0.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5eda85d6d1879e692d546a078b44251cdd08dd1cfb98dfb77b670c97cee49ea0", size = 203217, upload-time = "2025-09-08T23:22:14.596Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e0/6cbe77a53acf5acc7c08cc186c9928864bd7c005f9efd0d126884858a5fe/cffi-2.0.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9332088d75dc3241c702d852d4671613136d90fa6881da7d770a483fd05248b4", size = 203079, upload-time = "2025-09-08T23:22:15.769Z" }, + { url = "https://files.pythonhosted.org/packages/98/29/9b366e70e243eb3d14a5cb488dfd3a0b6b2f1fb001a203f653b93ccfac88/cffi-2.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc7de24befaeae77ba923797c7c87834c73648a05a4bde34b3b7e5588973a453", size = 216475, upload-time = "2025-09-08T23:22:17.427Z" }, + { url = "https://files.pythonhosted.org/packages/21/7a/13b24e70d2f90a322f2900c5d8e1f14fa7e2a6b3332b7309ba7b2ba51a5a/cffi-2.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cf364028c016c03078a23b503f02058f1814320a56ad535686f90565636a9495", size = 218829, upload-time = "2025-09-08T23:22:19.069Z" }, + { url = "https://files.pythonhosted.org/packages/60/99/c9dc110974c59cc981b1f5b66e1d8af8af764e00f0293266824d9c4254bc/cffi-2.0.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e11e82b744887154b182fd3e7e8512418446501191994dbf9c9fc1f32cc8efd5", size = 211211, upload-time = "2025-09-08T23:22:20.588Z" }, + { url = "https://files.pythonhosted.org/packages/49/72/ff2d12dbf21aca1b32a40ed792ee6b40f6dc3a9cf1644bd7ef6e95e0ac5e/cffi-2.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8ea985900c5c95ce9db1745f7933eeef5d314f0565b27625d9a10ec9881e1bfb", size = 218036, upload-time = "2025-09-08T23:22:22.143Z" }, + { url = "https://files.pythonhosted.org/packages/e2/cc/027d7fb82e58c48ea717149b03bcadcbdc293553edb283af792bd4bcbb3f/cffi-2.0.0-cp310-cp310-win32.whl", hash = "sha256:1f72fb8906754ac8a2cc3f9f5aaa298070652a0ffae577e0ea9bd480dc3c931a", size = 172184, upload-time = "2025-09-08T23:22:23.328Z" }, + { url = "https://files.pythonhosted.org/packages/33/fa/072dd15ae27fbb4e06b437eb6e944e75b068deb09e2a2826039e49ee2045/cffi-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:b18a3ed7d5b3bd8d9ef7a8cb226502c6bf8308df1525e1cc676c3680e7176739", size = 182790, upload-time = "2025-09-08T23:22:24.752Z" }, + { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, + { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, + { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, + { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, + { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, + { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320, upload-time = "2025-09-08T23:23:18.087Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487, upload-time = "2025-09-08T23:23:19.622Z" }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328, upload-time = "2025-09-08T23:23:44.61Z" }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650, upload-time = "2025-09-08T23:23:45.848Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687, upload-time = "2025-09-08T23:23:47.105Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773, upload-time = "2025-09-08T23:23:29.347Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013, upload-time = "2025-09-08T23:23:30.63Z" }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487, upload-time = "2025-09-08T23:23:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726, upload-time = "2025-09-08T23:23:41.742Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/13/69/33ddede1939fdd074bce5434295f38fae7136463422fe4fd3e0e89b98062/charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a", size = 129418, upload-time = "2025-10-14T04:42:32.879Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1f/b8/6d51fc1d52cbd52cd4ccedd5b5b2f0f6a11bbf6765c782298b0f3e808541/charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d", size = 209709, upload-time = "2025-10-14T04:40:11.385Z" }, + { url = "https://files.pythonhosted.org/packages/5c/af/1f9d7f7faafe2ddfb6f72a2e07a548a629c61ad510fe60f9630309908fef/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8", size = 148814, upload-time = "2025-10-14T04:40:13.135Z" }, + { url = "https://files.pythonhosted.org/packages/79/3d/f2e3ac2bbc056ca0c204298ea4e3d9db9b4afe437812638759db2c976b5f/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad", size = 144467, upload-time = "2025-10-14T04:40:14.728Z" }, + { url = "https://files.pythonhosted.org/packages/ec/85/1bf997003815e60d57de7bd972c57dc6950446a3e4ccac43bc3070721856/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8", size = 162280, upload-time = "2025-10-14T04:40:16.14Z" }, + { url = "https://files.pythonhosted.org/packages/3e/8e/6aa1952f56b192f54921c436b87f2aaf7c7a7c3d0d1a765547d64fd83c13/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d", size = 159454, upload-time = "2025-10-14T04:40:17.567Z" }, + { url = "https://files.pythonhosted.org/packages/36/3b/60cbd1f8e93aa25d1c669c649b7a655b0b5fb4c571858910ea9332678558/charset_normalizer-3.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313", size = 153609, upload-time = "2025-10-14T04:40:19.08Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/6a13396948b8fd3c4b4fd5bc74d045f5637d78c9675585e8e9fbe5636554/charset_normalizer-3.4.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e", size = 151849, upload-time = "2025-10-14T04:40:20.607Z" }, + { url = "https://files.pythonhosted.org/packages/b7/7a/59482e28b9981d105691e968c544cc0df3b7d6133152fb3dcdc8f135da7a/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93", size = 151586, upload-time = "2025-10-14T04:40:21.719Z" }, + { url = "https://files.pythonhosted.org/packages/92/59/f64ef6a1c4bdd2baf892b04cd78792ed8684fbc48d4c2afe467d96b4df57/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0", size = 145290, upload-time = "2025-10-14T04:40:23.069Z" }, + { url = "https://files.pythonhosted.org/packages/6b/63/3bf9f279ddfa641ffa1962b0db6a57a9c294361cc2f5fcac997049a00e9c/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84", size = 163663, upload-time = "2025-10-14T04:40:24.17Z" }, + { url = "https://files.pythonhosted.org/packages/ed/09/c9e38fc8fa9e0849b172b581fd9803bdf6e694041127933934184e19f8c3/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e", size = 151964, upload-time = "2025-10-14T04:40:25.368Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d1/d28b747e512d0da79d8b6a1ac18b7ab2ecfd81b2944c4c710e166d8dd09c/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db", size = 161064, upload-time = "2025-10-14T04:40:26.806Z" }, + { url = "https://files.pythonhosted.org/packages/bb/9a/31d62b611d901c3b9e5500c36aab0ff5eb442043fb3a1c254200d3d397d9/charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6", size = 155015, upload-time = "2025-10-14T04:40:28.284Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f3/107e008fa2bff0c8b9319584174418e5e5285fef32f79d8ee6a430d0039c/charset_normalizer-3.4.4-cp310-cp310-win32.whl", hash = "sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f", size = 99792, upload-time = "2025-10-14T04:40:29.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/66/e396e8a408843337d7315bab30dbf106c38966f1819f123257f5520f8a96/charset_normalizer-3.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d", size = 107198, upload-time = "2025-10-14T04:40:30.644Z" }, + { url = "https://files.pythonhosted.org/packages/b5/58/01b4f815bf0312704c267f2ccb6e5d42bcc7752340cd487bc9f8c3710597/charset_normalizer-3.4.4-cp310-cp310-win_arm64.whl", hash = "sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69", size = 100262, upload-time = "2025-10-14T04:40:32.108Z" }, + { url = "https://files.pythonhosted.org/packages/ed/27/c6491ff4954e58a10f69ad90aca8a1b6fe9c5d3c6f380907af3c37435b59/charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8", size = 206988, upload-time = "2025-10-14T04:40:33.79Z" }, + { url = "https://files.pythonhosted.org/packages/94/59/2e87300fe67ab820b5428580a53cad894272dbb97f38a7a814a2a1ac1011/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0", size = 147324, upload-time = "2025-10-14T04:40:34.961Z" }, + { url = "https://files.pythonhosted.org/packages/07/fb/0cf61dc84b2b088391830f6274cb57c82e4da8bbc2efeac8c025edb88772/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3", size = 142742, upload-time = "2025-10-14T04:40:36.105Z" }, + { url = "https://files.pythonhosted.org/packages/62/8b/171935adf2312cd745d290ed93cf16cf0dfe320863ab7cbeeae1dcd6535f/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc", size = 160863, upload-time = "2025-10-14T04:40:37.188Z" }, + { url = "https://files.pythonhosted.org/packages/09/73/ad875b192bda14f2173bfc1bc9a55e009808484a4b256748d931b6948442/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897", size = 157837, upload-time = "2025-10-14T04:40:38.435Z" }, + { url = "https://files.pythonhosted.org/packages/6d/fc/de9cce525b2c5b94b47c70a4b4fb19f871b24995c728e957ee68ab1671ea/charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381", size = 151550, upload-time = "2025-10-14T04:40:40.053Z" }, + { url = "https://files.pythonhosted.org/packages/55/c2/43edd615fdfba8c6f2dfbd459b25a6b3b551f24ea21981e23fb768503ce1/charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815", size = 149162, upload-time = "2025-10-14T04:40:41.163Z" }, + { url = "https://files.pythonhosted.org/packages/03/86/bde4ad8b4d0e9429a4e82c1e8f5c659993a9a863ad62c7df05cf7b678d75/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0", size = 150019, upload-time = "2025-10-14T04:40:42.276Z" }, + { url = "https://files.pythonhosted.org/packages/1f/86/a151eb2af293a7e7bac3a739b81072585ce36ccfb4493039f49f1d3cae8c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161", size = 143310, upload-time = "2025-10-14T04:40:43.439Z" }, + { url = "https://files.pythonhosted.org/packages/b5/fe/43dae6144a7e07b87478fdfc4dbe9efd5defb0e7ec29f5f58a55aeef7bf7/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4", size = 162022, upload-time = "2025-10-14T04:40:44.547Z" }, + { url = "https://files.pythonhosted.org/packages/80/e6/7aab83774f5d2bca81f42ac58d04caf44f0cc2b65fc6db2b3b2e8a05f3b3/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89", size = 149383, upload-time = "2025-10-14T04:40:46.018Z" }, + { url = "https://files.pythonhosted.org/packages/4f/e8/b289173b4edae05c0dde07f69f8db476a0b511eac556dfe0d6bda3c43384/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569", size = 159098, upload-time = "2025-10-14T04:40:47.081Z" }, + { url = "https://files.pythonhosted.org/packages/d8/df/fe699727754cae3f8478493c7f45f777b17c3ef0600e28abfec8619eb49c/charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224", size = 152991, upload-time = "2025-10-14T04:40:48.246Z" }, + { url = "https://files.pythonhosted.org/packages/1a/86/584869fe4ddb6ffa3bd9f491b87a01568797fb9bd8933f557dba9771beaf/charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a", size = 99456, upload-time = "2025-10-14T04:40:49.376Z" }, + { url = "https://files.pythonhosted.org/packages/65/f6/62fdd5feb60530f50f7e38b4f6a1d5203f4d16ff4f9f0952962c044e919a/charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016", size = 106978, upload-time = "2025-10-14T04:40:50.844Z" }, + { url = "https://files.pythonhosted.org/packages/7a/9d/0710916e6c82948b3be62d9d398cb4fcf4e97b56d6a6aeccd66c4b2f2bd5/charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1", size = 99969, upload-time = "2025-10-14T04:40:52.272Z" }, + { url = "https://files.pythonhosted.org/packages/f3/85/1637cd4af66fa687396e757dec650f28025f2a2f5a5531a3208dc0ec43f2/charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394", size = 208425, upload-time = "2025-10-14T04:40:53.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6a/04130023fef2a0d9c62d0bae2649b69f7b7d8d24ea5536feef50551029df/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25", size = 148162, upload-time = "2025-10-14T04:40:54.558Z" }, + { url = "https://files.pythonhosted.org/packages/78/29/62328d79aa60da22c9e0b9a66539feae06ca0f5a4171ac4f7dc285b83688/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef", size = 144558, upload-time = "2025-10-14T04:40:55.677Z" }, + { url = "https://files.pythonhosted.org/packages/86/bb/b32194a4bf15b88403537c2e120b817c61cd4ecffa9b6876e941c3ee38fe/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d", size = 161497, upload-time = "2025-10-14T04:40:57.217Z" }, + { url = "https://files.pythonhosted.org/packages/19/89/a54c82b253d5b9b111dc74aca196ba5ccfcca8242d0fb64146d4d3183ff1/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8", size = 159240, upload-time = "2025-10-14T04:40:58.358Z" }, + { url = "https://files.pythonhosted.org/packages/c0/10/d20b513afe03acc89ec33948320a5544d31f21b05368436d580dec4e234d/charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86", size = 153471, upload-time = "2025-10-14T04:40:59.468Z" }, + { url = "https://files.pythonhosted.org/packages/61/fa/fbf177b55bdd727010f9c0a3c49eefa1d10f960e5f09d1d887bf93c2e698/charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a", size = 150864, upload-time = "2025-10-14T04:41:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/05/12/9fbc6a4d39c0198adeebbde20b619790e9236557ca59fc40e0e3cebe6f40/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f", size = 150647, upload-time = "2025-10-14T04:41:01.754Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/6a9a593d52e3e8c5d2b167daf8c6b968808efb57ef4c210acb907c365bc4/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc", size = 145110, upload-time = "2025-10-14T04:41:03.231Z" }, + { url = "https://files.pythonhosted.org/packages/30/42/9a52c609e72471b0fc54386dc63c3781a387bb4fe61c20231a4ebcd58bdd/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf", size = 162839, upload-time = "2025-10-14T04:41:04.715Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/c0682bbf9f11597073052628ddd38344a3d673fda35a36773f7d19344b23/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15", size = 150667, upload-time = "2025-10-14T04:41:05.827Z" }, + { url = "https://files.pythonhosted.org/packages/e4/24/a41afeab6f990cf2daf6cb8c67419b63b48cf518e4f56022230840c9bfb2/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9", size = 160535, upload-time = "2025-10-14T04:41:06.938Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e5/6a4ce77ed243c4a50a1fecca6aaaab419628c818a49434be428fe24c9957/charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0", size = 154816, upload-time = "2025-10-14T04:41:08.101Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ef/89297262b8092b312d29cdb2517cb1237e51db8ecef2e9af5edbe7b683b1/charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26", size = 99694, upload-time = "2025-10-14T04:41:09.23Z" }, + { url = "https://files.pythonhosted.org/packages/3d/2d/1e5ed9dd3b3803994c155cd9aacb60c82c331bad84daf75bcb9c91b3295e/charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525", size = 107131, upload-time = "2025-10-14T04:41:10.467Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d9/0ed4c7098a861482a7b6a95603edce4c0d9db2311af23da1fb2b75ec26fc/charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3", size = 100390, upload-time = "2025-10-14T04:41:11.915Z" }, + { url = "https://files.pythonhosted.org/packages/97/45/4b3a1239bbacd321068ea6e7ac28875b03ab8bc0aa0966452db17cd36714/charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794", size = 208091, upload-time = "2025-10-14T04:41:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/7d/62/73a6d7450829655a35bb88a88fca7d736f9882a27eacdca2c6d505b57e2e/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed", size = 147936, upload-time = "2025-10-14T04:41:14.461Z" }, + { url = "https://files.pythonhosted.org/packages/89/c5/adb8c8b3d6625bef6d88b251bbb0d95f8205831b987631ab0c8bb5d937c2/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72", size = 144180, upload-time = "2025-10-14T04:41:15.588Z" }, + { url = "https://files.pythonhosted.org/packages/91/ed/9706e4070682d1cc219050b6048bfd293ccf67b3d4f5a4f39207453d4b99/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328", size = 161346, upload-time = "2025-10-14T04:41:16.738Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0d/031f0d95e4972901a2f6f09ef055751805ff541511dc1252ba3ca1f80cf5/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede", size = 158874, upload-time = "2025-10-14T04:41:17.923Z" }, + { url = "https://files.pythonhosted.org/packages/f5/83/6ab5883f57c9c801ce5e5677242328aa45592be8a00644310a008d04f922/charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894", size = 153076, upload-time = "2025-10-14T04:41:19.106Z" }, + { url = "https://files.pythonhosted.org/packages/75/1e/5ff781ddf5260e387d6419959ee89ef13878229732732ee73cdae01800f2/charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1", size = 150601, upload-time = "2025-10-14T04:41:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/d7/57/71be810965493d3510a6ca79b90c19e48696fb1ff964da319334b12677f0/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490", size = 150376, upload-time = "2025-10-14T04:41:21.398Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d5/c3d057a78c181d007014feb7e9f2e65905a6c4ef182c0ddf0de2924edd65/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44", size = 144825, upload-time = "2025-10-14T04:41:22.583Z" }, + { url = "https://files.pythonhosted.org/packages/e6/8c/d0406294828d4976f275ffbe66f00266c4b3136b7506941d87c00cab5272/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133", size = 162583, upload-time = "2025-10-14T04:41:23.754Z" }, + { url = "https://files.pythonhosted.org/packages/d7/24/e2aa1f18c8f15c4c0e932d9287b8609dd30ad56dbe41d926bd846e22fb8d/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3", size = 150366, upload-time = "2025-10-14T04:41:25.27Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5b/1e6160c7739aad1e2df054300cc618b06bf784a7a164b0f238360721ab86/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e", size = 160300, upload-time = "2025-10-14T04:41:26.725Z" }, + { url = "https://files.pythonhosted.org/packages/7a/10/f882167cd207fbdd743e55534d5d9620e095089d176d55cb22d5322f2afd/charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc", size = 154465, upload-time = "2025-10-14T04:41:28.322Z" }, + { url = "https://files.pythonhosted.org/packages/89/66/c7a9e1b7429be72123441bfdbaf2bc13faab3f90b933f664db506dea5915/charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac", size = 99404, upload-time = "2025-10-14T04:41:29.95Z" }, + { url = "https://files.pythonhosted.org/packages/c4/26/b9924fa27db384bdcd97ab83b4f0a8058d96ad9626ead570674d5e737d90/charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14", size = 107092, upload-time = "2025-10-14T04:41:31.188Z" }, + { url = "https://files.pythonhosted.org/packages/af/8f/3ed4bfa0c0c72a7ca17f0380cd9e4dd842b09f664e780c13cff1dcf2ef1b/charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2", size = 100408, upload-time = "2025-10-14T04:41:32.624Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/7051599bd493e62411d6ede36fd5af83a38f37c4767b92884df7301db25d/charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd", size = 207746, upload-time = "2025-10-14T04:41:33.773Z" }, + { url = "https://files.pythonhosted.org/packages/10/9a/97c8d48ef10d6cd4fcead2415523221624bf58bcf68a802721a6bc807c8f/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb", size = 147889, upload-time = "2025-10-14T04:41:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/979224a919a1b606c82bd2c5fa49b5c6d5727aa47b4312bb27b1734f53cd/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e", size = 143641, upload-time = "2025-10-14T04:41:36.116Z" }, + { url = "https://files.pythonhosted.org/packages/ba/33/0ad65587441fc730dc7bd90e9716b30b4702dc7b617e6ba4997dc8651495/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14", size = 160779, upload-time = "2025-10-14T04:41:37.229Z" }, + { url = "https://files.pythonhosted.org/packages/67/ed/331d6b249259ee71ddea93f6f2f0a56cfebd46938bde6fcc6f7b9a3d0e09/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191", size = 159035, upload-time = "2025-10-14T04:41:38.368Z" }, + { url = "https://files.pythonhosted.org/packages/67/ff/f6b948ca32e4f2a4576aa129d8bed61f2e0543bf9f5f2b7fc3758ed005c9/charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838", size = 152542, upload-time = "2025-10-14T04:41:39.862Z" }, + { url = "https://files.pythonhosted.org/packages/16/85/276033dcbcc369eb176594de22728541a925b2632f9716428c851b149e83/charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6", size = 149524, upload-time = "2025-10-14T04:41:41.319Z" }, + { url = "https://files.pythonhosted.org/packages/9e/f2/6a2a1f722b6aba37050e626530a46a68f74e63683947a8acff92569f979a/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e", size = 150395, upload-time = "2025-10-14T04:41:42.539Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2186cb2f2bbaea6338cad15ce23a67f9b0672929744381e28b0592676824/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c", size = 143680, upload-time = "2025-10-14T04:41:43.661Z" }, + { url = "https://files.pythonhosted.org/packages/7d/a5/bf6f13b772fbb2a90360eb620d52ed8f796f3c5caee8398c3b2eb7b1c60d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090", size = 162045, upload-time = "2025-10-14T04:41:44.821Z" }, + { url = "https://files.pythonhosted.org/packages/df/c5/d1be898bf0dc3ef9030c3825e5d3b83f2c528d207d246cbabe245966808d/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152", size = 149687, upload-time = "2025-10-14T04:41:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/a5/42/90c1f7b9341eef50c8a1cb3f098ac43b0508413f33affd762855f67a410e/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828", size = 160014, upload-time = "2025-10-14T04:41:47.631Z" }, + { url = "https://files.pythonhosted.org/packages/76/be/4d3ee471e8145d12795ab655ece37baed0929462a86e72372fd25859047c/charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec", size = 154044, upload-time = "2025-10-14T04:41:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6f/8f7af07237c34a1defe7defc565a9bc1807762f672c0fde711a4b22bf9c0/charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9", size = 99940, upload-time = "2025-10-14T04:41:49.946Z" }, + { url = "https://files.pythonhosted.org/packages/4b/51/8ade005e5ca5b0d80fb4aff72a3775b325bdc3d27408c8113811a7cbe640/charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c", size = 107104, upload-time = "2025-10-14T04:41:51.051Z" }, + { url = "https://files.pythonhosted.org/packages/da/5f/6b8f83a55bb8278772c5ae54a577f3099025f9ade59d0136ac24a0df4bde/charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2", size = 100743, upload-time = "2025-10-14T04:41:52.122Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4c/925909008ed5a988ccbb72dcc897407e5d6d3bd72410d69e051fc0c14647/charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f", size = 53402, upload-time = "2025-10-14T04:42:31.76Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cryptography" +version = "46.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/60/04/ee2a9e8542e4fa2773b81771ff8349ff19cdd56b7258a0cc442639052edb/cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", size = 750064, upload-time = "2026-02-10T19:18:38.255Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/81/b0bb27f2ba931a65409c6b8a8b358a7f03c0e46eceacddff55f7c84b1f3b/cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", size = 7176289, upload-time = "2026-02-10T19:17:08.274Z" }, + { url = "https://files.pythonhosted.org/packages/ff/9e/6b4397a3e3d15123de3b1806ef342522393d50736c13b20ec4c9ea6693a6/cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", size = 4275637, upload-time = "2026-02-10T19:17:10.53Z" }, + { url = "https://files.pythonhosted.org/packages/63/e7/471ab61099a3920b0c77852ea3f0ea611c9702f651600397ac567848b897/cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", size = 4424742, upload-time = "2026-02-10T19:17:12.388Z" }, + { url = "https://files.pythonhosted.org/packages/37/53/a18500f270342d66bf7e4d9f091114e31e5ee9e7375a5aba2e85a91e0044/cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", size = 4277528, upload-time = "2026-02-10T19:17:13.853Z" }, + { url = "https://files.pythonhosted.org/packages/22/29/c2e812ebc38c57b40e7c583895e73c8c5adb4d1e4a0cc4c5a4fdab2b1acc/cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", size = 4947993, upload-time = "2026-02-10T19:17:15.618Z" }, + { url = "https://files.pythonhosted.org/packages/6b/e7/237155ae19a9023de7e30ec64e5d99a9431a567407ac21170a046d22a5a3/cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", size = 4456855, upload-time = "2026-02-10T19:17:17.221Z" }, + { url = "https://files.pythonhosted.org/packages/2d/87/fc628a7ad85b81206738abbd213b07702bcbdada1dd43f72236ef3cffbb5/cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", size = 3984635, upload-time = "2026-02-10T19:17:18.792Z" }, + { url = "https://files.pythonhosted.org/packages/84/29/65b55622bde135aedf4565dc509d99b560ee4095e56989e815f8fd2aa910/cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", size = 4277038, upload-time = "2026-02-10T19:17:20.256Z" }, + { url = "https://files.pythonhosted.org/packages/bc/36/45e76c68d7311432741faf1fbf7fac8a196a0a735ca21f504c75d37e2558/cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", size = 4912181, upload-time = "2026-02-10T19:17:21.825Z" }, + { url = "https://files.pythonhosted.org/packages/6d/1a/c1ba8fead184d6e3d5afcf03d569acac5ad063f3ac9fb7258af158f7e378/cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", size = 4456482, upload-time = "2026-02-10T19:17:25.133Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e5/3fb22e37f66827ced3b902cf895e6a6bc1d095b5b26be26bd13c441fdf19/cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", size = 4405497, upload-time = "2026-02-10T19:17:26.66Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/9d58bb32b1121a8a2f27383fabae4d63080c7ca60b9b5c88be742be04ee7/cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", size = 4667819, upload-time = "2026-02-10T19:17:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ed/325d2a490c5e94038cdb0117da9397ece1f11201f425c4e9c57fe5b9f08b/cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", size = 3028230, upload-time = "2026-02-10T19:17:30.518Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5a/ac0f49e48063ab4255d9e3b79f5def51697fce1a95ea1370f03dc9db76f6/cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", size = 3480909, upload-time = "2026-02-10T19:17:32.083Z" }, + { url = "https://files.pythonhosted.org/packages/00/13/3d278bfa7a15a96b9dc22db5a12ad1e48a9eb3d40e1827ef66a5df75d0d0/cryptography-46.0.5-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2", size = 7119287, upload-time = "2026-02-10T19:17:33.801Z" }, + { url = "https://files.pythonhosted.org/packages/67/c8/581a6702e14f0898a0848105cbefd20c058099e2c2d22ef4e476dfec75d7/cryptography-46.0.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678", size = 4265728, upload-time = "2026-02-10T19:17:35.569Z" }, + { url = "https://files.pythonhosted.org/packages/dd/4a/ba1a65ce8fc65435e5a849558379896c957870dd64fecea97b1ad5f46a37/cryptography-46.0.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87", size = 4408287, upload-time = "2026-02-10T19:17:36.938Z" }, + { url = "https://files.pythonhosted.org/packages/f8/67/8ffdbf7b65ed1ac224d1c2df3943553766914a8ca718747ee3871da6107e/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee", size = 4270291, upload-time = "2026-02-10T19:17:38.748Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e5/f52377ee93bc2f2bba55a41a886fd208c15276ffbd2569f2ddc89d50e2c5/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981", size = 4927539, upload-time = "2026-02-10T19:17:40.241Z" }, + { url = "https://files.pythonhosted.org/packages/3b/02/cfe39181b02419bbbbcf3abdd16c1c5c8541f03ca8bda240debc467d5a12/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9", size = 4442199, upload-time = "2026-02-10T19:17:41.789Z" }, + { url = "https://files.pythonhosted.org/packages/c0/96/2fcaeb4873e536cf71421a388a6c11b5bc846e986b2b069c79363dc1648e/cryptography-46.0.5-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648", size = 3960131, upload-time = "2026-02-10T19:17:43.379Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d2/b27631f401ddd644e94c5cf33c9a4069f72011821cf3dc7309546b0642a0/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4", size = 4270072, upload-time = "2026-02-10T19:17:45.481Z" }, + { url = "https://files.pythonhosted.org/packages/f4/a7/60d32b0370dae0b4ebe55ffa10e8599a2a59935b5ece1b9f06edb73abdeb/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0", size = 4892170, upload-time = "2026-02-10T19:17:46.997Z" }, + { url = "https://files.pythonhosted.org/packages/d2/b9/cf73ddf8ef1164330eb0b199a589103c363afa0cf794218c24d524a58eab/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663", size = 4441741, upload-time = "2026-02-10T19:17:48.661Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/eee00b28c84c726fe8fa0158c65afe312d9c3b78d9d01daf700f1f6e37ff/cryptography-46.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826", size = 4396728, upload-time = "2026-02-10T19:17:50.058Z" }, + { url = "https://files.pythonhosted.org/packages/65/f4/6bc1a9ed5aef7145045114b75b77c2a8261b4d38717bd8dea111a63c3442/cryptography-46.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d", size = 4652001, upload-time = "2026-02-10T19:17:51.54Z" }, + { url = "https://files.pythonhosted.org/packages/86/ef/5d00ef966ddd71ac2e6951d278884a84a40ffbd88948ef0e294b214ae9e4/cryptography-46.0.5-cp314-cp314t-win32.whl", hash = "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a", size = 3003637, upload-time = "2026-02-10T19:17:52.997Z" }, + { url = "https://files.pythonhosted.org/packages/b7/57/f3f4160123da6d098db78350fdfd9705057aad21de7388eacb2401dceab9/cryptography-46.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4", size = 3469487, upload-time = "2026-02-10T19:17:54.549Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fa/a66aa722105ad6a458bebd64086ca2b72cdd361fed31763d20390f6f1389/cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", size = 7170514, upload-time = "2026-02-10T19:17:56.267Z" }, + { url = "https://files.pythonhosted.org/packages/0f/04/c85bdeab78c8bc77b701bf0d9bdcf514c044e18a46dcff330df5448631b0/cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", size = 4275349, upload-time = "2026-02-10T19:17:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/32/9b87132a2f91ee7f5223b091dc963055503e9b442c98fc0b8a5ca765fab0/cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", size = 4420667, upload-time = "2026-02-10T19:18:00.619Z" }, + { url = "https://files.pythonhosted.org/packages/a1/a6/a7cb7010bec4b7c5692ca6f024150371b295ee1c108bdc1c400e4c44562b/cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", size = 4276980, upload-time = "2026-02-10T19:18:02.379Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7c/c4f45e0eeff9b91e3f12dbd0e165fcf2a38847288fcfd889deea99fb7b6d/cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", size = 4939143, upload-time = "2026-02-10T19:18:03.964Z" }, + { url = "https://files.pythonhosted.org/packages/37/19/e1b8f964a834eddb44fa1b9a9976f4e414cbb7aa62809b6760c8803d22d1/cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", size = 4453674, upload-time = "2026-02-10T19:18:05.588Z" }, + { url = "https://files.pythonhosted.org/packages/db/ed/db15d3956f65264ca204625597c410d420e26530c4e2943e05a0d2f24d51/cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", size = 3978801, upload-time = "2026-02-10T19:18:07.167Z" }, + { url = "https://files.pythonhosted.org/packages/41/e2/df40a31d82df0a70a0daf69791f91dbb70e47644c58581d654879b382d11/cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", size = 4276755, upload-time = "2026-02-10T19:18:09.813Z" }, + { url = "https://files.pythonhosted.org/packages/33/45/726809d1176959f4a896b86907b98ff4391a8aa29c0aaaf9450a8a10630e/cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", size = 4901539, upload-time = "2026-02-10T19:18:11.263Z" }, + { url = "https://files.pythonhosted.org/packages/99/0f/a3076874e9c88ecb2ecc31382f6e7c21b428ede6f55aafa1aa272613e3cd/cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", size = 4452794, upload-time = "2026-02-10T19:18:12.914Z" }, + { url = "https://files.pythonhosted.org/packages/02/ef/ffeb542d3683d24194a38f66ca17c0a4b8bf10631feef44a7ef64e631b1a/cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", size = 4404160, upload-time = "2026-02-10T19:18:14.375Z" }, + { url = "https://files.pythonhosted.org/packages/96/93/682d2b43c1d5f1406ed048f377c0fc9fc8f7b0447a478d5c65ab3d3a66eb/cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", size = 4667123, upload-time = "2026-02-10T19:18:15.886Z" }, + { url = "https://files.pythonhosted.org/packages/45/2d/9c5f2926cb5300a8eefc3f4f0b3f3df39db7f7ce40c8365444c49363cbda/cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", size = 3010220, upload-time = "2026-02-10T19:18:17.361Z" }, + { url = "https://files.pythonhosted.org/packages/48/ef/0c2f4a8e31018a986949d34a01115dd057bf536905dca38897bacd21fac3/cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", size = 3467050, upload-time = "2026-02-10T19:18:18.899Z" }, + { url = "https://files.pythonhosted.org/packages/eb/dd/2d9fdb07cebdf3d51179730afb7d5e576153c6744c3ff8fded23030c204e/cryptography-46.0.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c", size = 3476964, upload-time = "2026-02-10T19:18:20.687Z" }, + { url = "https://files.pythonhosted.org/packages/e9/6f/6cc6cc9955caa6eaf83660b0da2b077c7fe8ff9950a3c5e45d605038d439/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a", size = 4218321, upload-time = "2026-02-10T19:18:22.349Z" }, + { url = "https://files.pythonhosted.org/packages/3e/5d/c4da701939eeee699566a6c1367427ab91a8b7088cc2328c09dbee940415/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356", size = 4381786, upload-time = "2026-02-10T19:18:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/ac/97/a538654732974a94ff96c1db621fa464f455c02d4bb7d2652f4edc21d600/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da", size = 4217990, upload-time = "2026-02-10T19:18:25.957Z" }, + { url = "https://files.pythonhosted.org/packages/ae/11/7e500d2dd3ba891197b9efd2da5454b74336d64a7cc419aa7327ab74e5f6/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257", size = 4381252, upload-time = "2026-02-10T19:18:27.496Z" }, + { url = "https://files.pythonhosted.org/packages/bc/58/6b3d24e6b9bc474a2dcdee65dfd1f008867015408a271562e4b690561a4d/cryptography-46.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", size = 3407605, upload-time = "2026-02-10T19:18:29.233Z" }, +] + +[[package]] +name = "dnspython" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, +] + +[[package]] +name = "eventlet" +version = "0.36.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython" }, + { name = "greenlet" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1b/df/f441947eef23192c9f179e46868ee8510a6f7b6627b76b88f07692f9c706/eventlet-0.36.1.tar.gz", hash = "sha256:d227fe76a63d9e6a6cef53beb8ad0b2dc40a5e7737c801f4b474cfae1db07bc5", size = 552863, upload-time = "2024-03-29T13:41:19.952Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/af/73efcf654d8875febc6599f5a3d1eed043c1ca34a9b12950208cbf710d2a/eventlet-0.36.1-py3-none-any.whl", hash = "sha256:e42d0f73b718e654c223a033b8692d1a94d778a6c1deb6c3d21442746f3f727f", size = 360472, upload-time = "2024-03-29T13:41:17.444Z" }, +] + +[[package]] +name = "explgbk" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "cachetools" }, + { name = "eventlet" }, + { name = "flask" }, + { name = "flask-authnz" }, + { name = "flask-socket-util" }, + { name = "flask-socketio" }, + { name = "gunicorn" }, + { name = "kafka-python" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "pymongo" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "requests" }, + { name = "seaweed" }, +] + +[package.dev-dependencies] +dev = [ + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "cachetools", specifier = "==5.3.1" }, + { name = "eventlet", specifier = "==0.36.1" }, + { name = "flask", specifier = "==2.3.3" }, + { name = "flask-authnz", git = "https://github.com/slaclab/flask_authnz?rev=046fb9c46835be47f9387e878efd2b898a6d4025" }, + { name = "flask-socket-util", git = "https://github.com/slaclab/flask_socket_util?rev=ad42186e1fb6d5406dffbc459e200bc490e5af91" }, + { name = "flask-socketio", specifier = "==5.3.6" }, + { name = "gunicorn", specifier = "==21.2.0" }, + { name = "kafka-python", specifier = "==2.0.2" }, + { name = "pyjwt", extras = ["crypto"], specifier = "==2.8.0" }, + { name = "pymongo", specifier = "==4.5.0" }, + { name = "python-dateutil", specifier = "==2.8.2" }, + { name = "pytz", specifier = "==2023.3.post1" }, + { name = "requests", specifier = "==2.31.0" }, + { name = "seaweed", specifier = "==0.0.1" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "ruff", specifier = ">=0.15.1" }] + +[[package]] +name = "flask" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blinker" }, + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/b7/4ace17e37abd9c21715dea5ee11774a25e404c486a7893fa18e764326ead/flask-2.3.3.tar.gz", hash = "sha256:09c347a92aa7ff4a8e7f3206795f30d826654baf38b873d0744cd571ca609efc", size = 672756, upload-time = "2023-08-21T19:52:35.012Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/56/26f0be8adc2b4257df20c1c4260ddd0aa396cf8e75d90ab2f7ff99bc34f9/flask-2.3.3-py3-none-any.whl", hash = "sha256:f69fcd559dc907ed196ab9df0e48471709175e696d6e698dd4dbe940f96ce66b", size = 96112, upload-time = "2023-08-21T19:52:33.115Z" }, +] + +[[package]] +name = "flask-authnz" +version = "0.0.24" +source = { git = "https://github.com/slaclab/flask_authnz?rev=046fb9c46835be47f9387e878efd2b898a6d4025#046fb9c46835be47f9387e878efd2b898a6d4025" } + +[[package]] +name = "flask-socket-util" +version = "0.0.13" +source = { git = "https://github.com/slaclab/flask_socket_util?rev=ad42186e1fb6d5406dffbc459e200bc490e5af91#ad42186e1fb6d5406dffbc459e200bc490e5af91" } +dependencies = [ + { name = "eventlet" }, + { name = "flask" }, + { name = "flask-socketio" }, +] + +[[package]] +name = "flask-socketio" +version = "5.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flask" }, + { name = "python-socketio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/b2/aa882384d130523d7d2d6eed33403aed68a438622df388d92171d7657960/Flask-SocketIO-5.3.6.tar.gz", hash = "sha256:bb8f9f9123ef47632f5ce57a33514b0c0023ec3696b2384457f0fcaa5b70501c", size = 17167, upload-time = "2023-09-05T09:35:25.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/c8/dc0be9e26272dc89342868ecc2d9ddb9e31002b4b8e49fdb754aa0f9ecbf/Flask_SocketIO-5.3.6-py3-none-any.whl", hash = "sha256:9e62d2131842878ae6bfdd7067dfc3be397c1f2b117ab1dc74e6fe74aad7a579", size = 18098, upload-time = "2023-09-05T09:35:23.601Z" }, +] + +[[package]] +name = "greenlet" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/99/1cd3411c56a410994669062bd73dd58270c00cc074cac15f385a1fd91f8a/greenlet-3.3.1.tar.gz", hash = "sha256:41848f3230b58c08bb43dee542e74a2a2e34d3c59dc3076cec9151aeeedcae98", size = 184690, upload-time = "2026-01-23T15:31:02.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/65/5b235b40581ad75ab97dcd8b4218022ae8e3ab77c13c919f1a1dfe9171fd/greenlet-3.3.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:04bee4775f40ecefcdaa9d115ab44736cd4b9c5fba733575bfe9379419582e13", size = 273723, upload-time = "2026-01-23T15:30:37.521Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ad/eb4729b85cba2d29499e0a04ca6fbdd8f540afd7be142fd571eea43d712f/greenlet-3.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e1457f4fed12a50e427988a07f0f9df53cf0ee8da23fab16e6732c2ec909d4", size = 574874, upload-time = "2026-01-23T16:00:54.551Z" }, + { url = "https://files.pythonhosted.org/packages/87/32/57cad7fe4c8b82fdaa098c89498ef85ad92dfbb09d5eb713adedfc2ae1f5/greenlet-3.3.1-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:070472cd156f0656f86f92e954591644e158fd65aa415ffbe2d44ca77656a8f5", size = 586309, upload-time = "2026-01-23T16:05:25.18Z" }, + { url = "https://files.pythonhosted.org/packages/66/66/f041005cb87055e62b0d68680e88ec1a57f4688523d5e2fb305841bc8307/greenlet-3.3.1-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1108b61b06b5224656121c3c8ee8876161c491cbe74e5c519e0634c837cf93d5", size = 597461, upload-time = "2026-01-23T16:15:51.943Z" }, + { url = "https://files.pythonhosted.org/packages/87/eb/8a1ec2da4d55824f160594a75a9d8354a5fe0a300fb1c48e7944265217e1/greenlet-3.3.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a300354f27dd86bae5fbf7002e6dd2b3255cd372e9242c933faf5e859b703fe", size = 586985, upload-time = "2026-01-23T15:32:47.968Z" }, + { url = "https://files.pythonhosted.org/packages/15/1c/0621dd4321dd8c351372ee8f9308136acb628600658a49be1b7504208738/greenlet-3.3.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e84b51cbebf9ae573b5fbd15df88887815e3253fc000a7d0ff95170e8f7e9729", size = 1547271, upload-time = "2026-01-23T16:04:18.977Z" }, + { url = "https://files.pythonhosted.org/packages/9d/53/24047f8924c83bea7a59c8678d9571209c6bfe5f4c17c94a78c06024e9f2/greenlet-3.3.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e0093bd1a06d899892427217f0ff2a3c8f306182b8c754336d32e2d587c131b4", size = 1613427, upload-time = "2026-01-23T15:33:44.428Z" }, + { url = "https://files.pythonhosted.org/packages/ff/07/ac9bf1ec008916d1a3373cae212884c1dcff4a4ba0d41127ce81a8deb4e9/greenlet-3.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:7932f5f57609b6a3b82cc11877709aa7a98e3308983ed93552a1c377069b20c8", size = 226100, upload-time = "2026-01-23T15:30:56.957Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e8/2e1462c8fdbe0f210feb5ac7ad2d9029af8be3bf45bd9fa39765f821642f/greenlet-3.3.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5fd23b9bc6d37b563211c6abbb1b3cab27db385a4449af5c32e932f93017080c", size = 274974, upload-time = "2026-01-23T15:31:02.891Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a8/530a401419a6b302af59f67aaf0b9ba1015855ea7e56c036b5928793c5bd/greenlet-3.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f51496a0bfbaa9d74d36a52d2580d1ef5ed4fdfcff0a73730abfbbbe1403dd", size = 577175, upload-time = "2026-01-23T16:00:56.213Z" }, + { url = "https://files.pythonhosted.org/packages/8e/89/7e812bb9c05e1aaef9b597ac1d0962b9021d2c6269354966451e885c4e6b/greenlet-3.3.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb0feb07fe6e6a74615ee62a880007d976cf739b6669cce95daa7373d4fc69c5", size = 590401, upload-time = "2026-01-23T16:05:26.365Z" }, + { url = "https://files.pythonhosted.org/packages/70/ae/e2d5f0e59b94a2269b68a629173263fa40b63da32f5c231307c349315871/greenlet-3.3.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:67ea3fc73c8cd92f42467a72b75e8f05ed51a0e9b1d15398c913416f2dafd49f", size = 601161, upload-time = "2026-01-23T16:15:53.456Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ae/8d472e1f5ac5efe55c563f3eabb38c98a44b832602e12910750a7c025802/greenlet-3.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39eda9ba259cc9801da05351eaa8576e9aa83eb9411e8f0c299e05d712a210f2", size = 590272, upload-time = "2026-01-23T15:32:49.411Z" }, + { url = "https://files.pythonhosted.org/packages/a8/51/0fde34bebfcadc833550717eade64e35ec8738e6b097d5d248274a01258b/greenlet-3.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2e7e882f83149f0a71ac822ebf156d902e7a5d22c9045e3e0d1daf59cee2cc9", size = 1550729, upload-time = "2026-01-23T16:04:20.867Z" }, + { url = "https://files.pythonhosted.org/packages/16/c9/2fb47bee83b25b119d5a35d580807bb8b92480a54b68fef009a02945629f/greenlet-3.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80aa4d79eb5564f2e0a6144fcc744b5a37c56c4a92d60920720e99210d88db0f", size = 1615552, upload-time = "2026-01-23T15:33:45.743Z" }, + { url = "https://files.pythonhosted.org/packages/1f/54/dcf9f737b96606f82f8dd05becfb8d238db0633dd7397d542a296fe9cad3/greenlet-3.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:32e4ca9777c5addcbf42ff3915d99030d8e00173a56f80001fb3875998fe410b", size = 226462, upload-time = "2026-01-23T15:36:50.422Z" }, + { url = "https://files.pythonhosted.org/packages/91/37/61e1015cf944ddd2337447d8e97fb423ac9bc21f9963fb5f206b53d65649/greenlet-3.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:da19609432f353fed186cc1b85e9440db93d489f198b4bdf42ae19cc9d9ac9b4", size = 225715, upload-time = "2026-01-23T15:33:17.298Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c8/9d76a66421d1ae24340dfae7e79c313957f6e3195c144d2c73333b5bfe34/greenlet-3.3.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7e806ca53acf6d15a888405880766ec84721aa4181261cd11a457dfe9a7a4975", size = 276443, upload-time = "2026-01-23T15:30:10.066Z" }, + { url = "https://files.pythonhosted.org/packages/81/99/401ff34bb3c032d1f10477d199724f5e5f6fbfb59816ad1455c79c1eb8e7/greenlet-3.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d842c94b9155f1c9b3058036c24ffb8ff78b428414a19792b2380be9cecf4f36", size = 597359, upload-time = "2026-01-23T16:00:57.394Z" }, + { url = "https://files.pythonhosted.org/packages/2b/bc/4dcc0871ed557792d304f50be0f7487a14e017952ec689effe2180a6ff35/greenlet-3.3.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20fedaadd422fa02695f82093f9a98bad3dab5fcda793c658b945fcde2ab27ba", size = 607805, upload-time = "2026-01-23T16:05:28.068Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/7a7ca57588dac3389e97f7c9521cb6641fd8b6602faf1eaa4188384757df/greenlet-3.3.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c620051669fd04ac6b60ebc70478210119c56e2d5d5df848baec4312e260e4ca", size = 622363, upload-time = "2026-01-23T16:15:54.754Z" }, + { url = "https://files.pythonhosted.org/packages/cf/05/821587cf19e2ce1f2b24945d890b164401e5085f9d09cbd969b0c193cd20/greenlet-3.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14194f5f4305800ff329cbf02c5fcc88f01886cadd29941b807668a45f0d2336", size = 609947, upload-time = "2026-01-23T15:32:51.004Z" }, + { url = "https://files.pythonhosted.org/packages/a4/52/ee8c46ed9f8babaa93a19e577f26e3d28a519feac6350ed6f25f1afee7e9/greenlet-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7b2fe4150a0cf59f847a67db8c155ac36aed89080a6a639e9f16df5d6c6096f1", size = 1567487, upload-time = "2026-01-23T16:04:22.125Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7c/456a74f07029597626f3a6db71b273a3632aecb9afafeeca452cfa633197/greenlet-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49f4ad195d45f4a66a0eb9c1ba4832bb380570d361912fa3554746830d332149", size = 1636087, upload-time = "2026-01-23T15:33:47.486Z" }, + { url = "https://files.pythonhosted.org/packages/34/2f/5e0e41f33c69655300a5e54aeb637cf8ff57f1786a3aba374eacc0228c1d/greenlet-3.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:cc98b9c4e4870fa983436afa999d4eb16b12872fab7071423d5262fa7120d57a", size = 227156, upload-time = "2026-01-23T15:34:34.808Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ab/717c58343cf02c5265b531384b248787e04d8160b8afe53d9eec053d7b44/greenlet-3.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:bfb2d1763d777de5ee495c85309460f6fd8146e50ec9d0ae0183dbf6f0a829d1", size = 226403, upload-time = "2026-01-23T15:31:39.372Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ab/d26750f2b7242c2b90ea2ad71de70cfcd73a948a49513188a0fc0d6fc15a/greenlet-3.3.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:7ab327905cabb0622adca5971e488064e35115430cec2c35a50fd36e72a315b3", size = 275205, upload-time = "2026-01-23T15:30:24.556Z" }, + { url = "https://files.pythonhosted.org/packages/10/d3/be7d19e8fad7c5a78eeefb2d896a08cd4643e1e90c605c4be3b46264998f/greenlet-3.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65be2f026ca6a176f88fb935ee23c18333ccea97048076aef4db1ef5bc0713ac", size = 599284, upload-time = "2026-01-23T16:00:58.584Z" }, + { url = "https://files.pythonhosted.org/packages/ae/21/fe703aaa056fdb0f17e5afd4b5c80195bbdab701208918938bd15b00d39b/greenlet-3.3.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a3ae05b3d225b4155bda56b072ceb09d05e974bc74be6c3fc15463cf69f33fd", size = 610274, upload-time = "2026-01-23T16:05:29.312Z" }, + { url = "https://files.pythonhosted.org/packages/06/00/95df0b6a935103c0452dad2203f5be8377e551b8466a29650c4c5a5af6cc/greenlet-3.3.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12184c61e5d64268a160226fb4818af4df02cfead8379d7f8b99a56c3a54ff3e", size = 624375, upload-time = "2026-01-23T16:15:55.915Z" }, + { url = "https://files.pythonhosted.org/packages/cb/86/5c6ab23bb3c28c21ed6bebad006515cfe08b04613eb105ca0041fecca852/greenlet-3.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6423481193bbbe871313de5fd06a082f2649e7ce6e08015d2a76c1e9186ca5b3", size = 612904, upload-time = "2026-01-23T15:32:52.317Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f3/7949994264e22639e40718c2daf6f6df5169bf48fb038c008a489ec53a50/greenlet-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:33a956fe78bbbda82bfc95e128d61129b32d66bcf0a20a1f0c08aa4839ffa951", size = 1567316, upload-time = "2026-01-23T16:04:23.316Z" }, + { url = "https://files.pythonhosted.org/packages/8d/6e/d73c94d13b6465e9f7cd6231c68abde838bb22408596c05d9059830b7872/greenlet-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b065d3284be43728dd280f6f9a13990b56470b81be20375a207cdc814a983f2", size = 1636549, upload-time = "2026-01-23T15:33:48.643Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b3/c9c23a6478b3bcc91f979ce4ca50879e4d0b2bd7b9a53d8ecded719b92e2/greenlet-3.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:27289986f4e5b0edec7b5a91063c109f0276abb09a7e9bdab08437525977c946", size = 227042, upload-time = "2026-01-23T15:33:58.216Z" }, + { url = "https://files.pythonhosted.org/packages/90/e7/824beda656097edee36ab15809fd063447b200cc03a7f6a24c34d520bc88/greenlet-3.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:2f080e028001c5273e0b42690eaf359aeef9cb1389da0f171ea51a5dc3c7608d", size = 226294, upload-time = "2026-01-23T15:30:52.73Z" }, + { url = "https://files.pythonhosted.org/packages/ae/fb/011c7c717213182caf78084a9bea51c8590b0afda98001f69d9f853a495b/greenlet-3.3.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:bd59acd8529b372775cd0fcbc5f420ae20681c5b045ce25bd453ed8455ab99b5", size = 275737, upload-time = "2026-01-23T15:32:16.889Z" }, + { url = "https://files.pythonhosted.org/packages/41/2e/a3a417d620363fdbb08a48b1dd582956a46a61bf8fd27ee8164f9dfe87c2/greenlet-3.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b31c05dd84ef6871dd47120386aed35323c944d86c3d91a17c4b8d23df62f15b", size = 646422, upload-time = "2026-01-23T16:01:00.354Z" }, + { url = "https://files.pythonhosted.org/packages/b4/09/c6c4a0db47defafd2d6bab8ddfe47ad19963b4e30f5bed84d75328059f8c/greenlet-3.3.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02925a0bfffc41e542c70aa14c7eda3593e4d7e274bfcccca1827e6c0875902e", size = 658219, upload-time = "2026-01-23T16:05:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/e2/89/b95f2ddcc5f3c2bc09c8ee8d77be312df7f9e7175703ab780f2014a0e781/greenlet-3.3.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3e0f3878ca3a3ff63ab4ea478585942b53df66ddde327b59ecb191b19dbbd62d", size = 671455, upload-time = "2026-01-23T16:15:57.232Z" }, + { url = "https://files.pythonhosted.org/packages/80/38/9d42d60dffb04b45f03dbab9430898352dba277758640751dc5cc316c521/greenlet-3.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34a729e2e4e4ffe9ae2408d5ecaf12f944853f40ad724929b7585bca808a9d6f", size = 660237, upload-time = "2026-01-23T15:32:53.967Z" }, + { url = "https://files.pythonhosted.org/packages/96/61/373c30b7197f9e756e4c81ae90a8d55dc3598c17673f91f4d31c3c689c3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aec9ab04e82918e623415947921dea15851b152b822661cce3f8e4393c3df683", size = 1615261, upload-time = "2026-01-23T16:04:25.066Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/ca534310343f5945316f9451e953dcd89b36fe7a19de652a1dc5a0eeef3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71c767cf281a80d02b6c1bdc41c9468e1f5a494fb11bc8688c360524e273d7b1", size = 1683719, upload-time = "2026-01-23T15:33:50.61Z" }, + { url = "https://files.pythonhosted.org/packages/52/cb/c21a3fd5d2c9c8b622e7bede6d6d00e00551a5ee474ea6d831b5f567a8b4/greenlet-3.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:96aff77af063b607f2489473484e39a0bbae730f2ea90c9e5606c9b73c44174a", size = 228125, upload-time = "2026-01-23T15:32:45.265Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8e/8a2db6d11491837af1de64b8aff23707c6e85241be13c60ed399a72e2ef8/greenlet-3.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:b066e8b50e28b503f604fa538adc764a638b38cf8e81e025011d26e8a627fa79", size = 227519, upload-time = "2026-01-23T15:31:47.284Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/cbbec49bacdcc9ec652a81d3efef7b59f326697e7edf6ed775a5e08e54c2/greenlet-3.3.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:3e63252943c921b90abb035ebe9de832c436401d9c45f262d80e2d06cc659242", size = 282706, upload-time = "2026-01-23T15:33:05.525Z" }, + { url = "https://files.pythonhosted.org/packages/86/2e/4f2b9323c144c4fe8842a4e0d92121465485c3c2c5b9e9b30a52e80f523f/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76e39058e68eb125de10c92524573924e827927df5d3891fbc97bd55764a8774", size = 651209, upload-time = "2026-01-23T16:01:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/d9/87/50ca60e515f5bb55a2fbc5f0c9b5b156de7d2fc51a0a69abc9d23914a237/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9f9d5e7a9310b7a2f416dd13d2e3fd8b42d803968ea580b7c0f322ccb389b97", size = 654300, upload-time = "2026-01-23T16:05:32.199Z" }, + { url = "https://files.pythonhosted.org/packages/7c/25/c51a63f3f463171e09cb586eb64db0861eb06667ab01a7968371a24c4f3b/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9721549a95db96689458a1e0ae32412ca18776ed004463df3a9299c1b257ab", size = 662574, upload-time = "2026-01-23T16:15:58.364Z" }, + { url = "https://files.pythonhosted.org/packages/1d/94/74310866dfa2b73dd08659a3d18762f83985ad3281901ba0ee9a815194fb/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92497c78adf3ac703b57f1e3813c2d874f27f71a178f9ea5887855da413cd6d2", size = 653842, upload-time = "2026-01-23T15:32:55.671Z" }, + { url = "https://files.pythonhosted.org/packages/97/43/8bf0ffa3d498eeee4c58c212a3905dd6146c01c8dc0b0a046481ca29b18c/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed6b402bc74d6557a705e197d47f9063733091ed6357b3de33619d8a8d93ac53", size = 1614917, upload-time = "2026-01-23T16:04:26.276Z" }, + { url = "https://files.pythonhosted.org/packages/89/90/a3be7a5f378fc6e84abe4dcfb2ba32b07786861172e502388b4c90000d1b/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:59913f1e5ada20fde795ba906916aea25d442abcc0593fba7e26c92b7ad76249", size = 1676092, upload-time = "2026-01-23T15:33:52.176Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2b/98c7f93e6db9977aaee07eb1e51ca63bd5f779b900d362791d3252e60558/greenlet-3.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:301860987846c24cb8964bdec0e31a96ad4a2a801b41b4ef40963c1b44f33451", size = 233181, upload-time = "2026-01-23T15:33:00.29Z" }, +] + +[[package]] +name = "gunicorn" +version = "21.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/89/acd9879fa6a5309b4bf16a5a8855f1e58f26d38e0c18ede9b3a70996b021/gunicorn-21.2.0.tar.gz", hash = "sha256:88ec8bff1d634f98e61b9f65bc4bf3cd918a90806c6f5c48bc5603849ec81033", size = 3632557, upload-time = "2023-07-19T11:46:46.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/2a/c3a878eccb100ccddf45c50b6b8db8cf3301a6adede6e31d48e8531cab13/gunicorn-21.2.0-py3-none-any.whl", hash = "sha256:3213aa5e8c24949e792bcacfc176fef362e7aac80b76c56f6b5122bf350722f0", size = 80176, upload-time = "2023-07-19T11:46:44.51Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "kafka-python" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/07/4c/2595fb5733c3ac01aef3dacce17ff07f7f3336d9f96548bcf723b9073e5c/kafka-python-2.0.2.tar.gz", hash = "sha256:04dfe7fea2b63726cd6f3e79a2d86e709d608d74406638c5da33a01d45a9d7e3", size = 265053, upload-time = "2020-09-30T07:24:03.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/68/dcb0db055309f680ab2931a3eeb22d865604b638acf8c914bedf4c1a0c8c/kafka_python-2.0.2-py2.py3-none-any.whl", hash = "sha256:2d92418c7cb1c298fa6c7f0fb3519b520d0d7526ac6cb7ae2a4fc65a51a94b6e", size = 246508, upload-time = "2020-09-30T07:24:01.49Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/4b/3541d44f3937ba468b75da9eebcae497dcf67adb65caa16760b0a6807ebb/markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559", size = 11631, upload-time = "2025-09-27T18:36:05.558Z" }, + { url = "https://files.pythonhosted.org/packages/98/1b/fbd8eed11021cabd9226c37342fa6ca4e8a98d8188a8d9b66740494960e4/markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419", size = 12057, upload-time = "2025-09-27T18:36:07.165Z" }, + { url = "https://files.pythonhosted.org/packages/40/01/e560d658dc0bb8ab762670ece35281dec7b6c1b33f5fbc09ebb57a185519/markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695", size = 22050, upload-time = "2025-09-27T18:36:08.005Z" }, + { url = "https://files.pythonhosted.org/packages/af/cd/ce6e848bbf2c32314c9b237839119c5a564a59725b53157c856e90937b7a/markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591", size = 20681, upload-time = "2025-09-27T18:36:08.881Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2a/b5c12c809f1c3045c4d580b035a743d12fcde53cf685dbc44660826308da/markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c", size = 20705, upload-time = "2025-09-27T18:36:10.131Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e3/9427a68c82728d0a88c50f890d0fc072a1484de2f3ac1ad0bfc1a7214fd5/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f", size = 21524, upload-time = "2025-09-27T18:36:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/bc/36/23578f29e9e582a4d0278e009b38081dbe363c5e7165113fad546918a232/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6", size = 20282, upload-time = "2025-09-27T18:36:12.573Z" }, + { url = "https://files.pythonhosted.org/packages/56/21/dca11354e756ebd03e036bd8ad58d6d7168c80ce1fe5e75218e4945cbab7/markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1", size = 20745, upload-time = "2025-09-27T18:36:13.504Z" }, + { url = "https://files.pythonhosted.org/packages/87/99/faba9369a7ad6e4d10b6a5fbf71fa2a188fe4a593b15f0963b73859a1bbd/markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa", size = 14571, upload-time = "2025-09-27T18:36:14.779Z" }, + { url = "https://files.pythonhosted.org/packages/d6/25/55dc3ab959917602c96985cb1253efaa4ff42f71194bddeb61eb7278b8be/markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8", size = 15056, upload-time = "2025-09-27T18:36:16.125Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9e/0a02226640c255d1da0b8d12e24ac2aa6734da68bff14c05dd53b94a0fc3/markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1", size = 13932, upload-time = "2025-09-27T18:36:17.311Z" }, + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492, upload-time = "2026-01-21T14:26:51.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/72/8259b2bccfe4673330cea843ab23f86858a419d8f1493f66d413a76c7e3b/PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de", size = 78313, upload-time = "2023-07-18T20:02:22.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/4f/e04a8067c7c96c364cef7ef73906504e2f40d690811c021e1a1901473a19/PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320", size = 22591, upload-time = "2023-07-18T20:02:21.561Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + +[[package]] +name = "pymongo" +version = "4.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/a6/eae874c4b686dd542e9425ba74a3945a0ebe1247e5801f83ab8b13dcfe59/pymongo-4.5.0.tar.gz", hash = "sha256:681f252e43b3ef054ca9161635f81b730f4d8cadd28b3f2b2004f5a72f853982", size = 848707, upload-time = "2023-08-22T13:26:01.72Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/ea/9c84561b42ea769b19f701959d838efd585c3a89019e41ad5d5513dedca8/pymongo-4.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2d4fa1b01fa7e5b7bb8d312e3542e211b320eb7a4e3d8dc884327039d93cb9e0", size = 529634, upload-time = "2023-08-22T13:23:14.059Z" }, + { url = "https://files.pythonhosted.org/packages/93/78/c0c510bcab3a2daa31a1839a30d16ff2e54f6d768702a248a47d127ced00/pymongo-4.5.0-cp310-cp310-manylinux1_i686.whl", hash = "sha256:dfcd2b9f510411de615ccedd47462dae80e82fdc09fe9ab0f0f32f11cf57eeb5", size = 686537, upload-time = "2023-08-22T13:23:16.139Z" }, + { url = "https://files.pythonhosted.org/packages/28/b1/779dee74f0631ddab0486114722a71e536dba27bba0d85ceb42cab1ece29/pymongo-4.5.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:3e33064f1984db412b34d51496f4ea785a9cff621c67de58e09fb28da6468a52", size = 689502, upload-time = "2023-08-22T13:23:17.636Z" }, + { url = "https://files.pythonhosted.org/packages/b4/bb/2e918b45a699faa0bf15599e37c0260423b6b327b9d8b7053ad28ae78743/pymongo-4.5.0-cp310-cp310-manylinux2014_i686.whl", hash = "sha256:33faa786cc907de63f745f587e9879429b46033d7d97a7b84b37f4f8f47b9b32", size = 686540, upload-time = "2023-08-22T13:23:19.221Z" }, + { url = "https://files.pythonhosted.org/packages/db/e8/1ae30654fdec29338a293b936a617bcdf68e810c9dca3708c8122b6620b4/pymongo-4.5.0-cp310-cp310-manylinux2014_ppc64le.whl", hash = "sha256:76a262c41c1a7cbb84a3b11976578a7eb8e788c4b7bfbd15c005fb6ca88e6e50", size = 703125, upload-time = "2023-08-22T13:23:21.557Z" }, + { url = "https://files.pythonhosted.org/packages/08/e2/014282c6605c09c704f39dd0a2419a705076667cb6a4b6de5be240383fa8/pymongo-4.5.0-cp310-cp310-manylinux2014_s390x.whl", hash = "sha256:0f4b125b46fe377984fbaecf2af40ed48b05a4b7676a2ff98999f2016d66b3ec", size = 694856, upload-time = "2023-08-22T13:23:23.055Z" }, + { url = "https://files.pythonhosted.org/packages/6a/25/682e092024d0697df7ddfb806309c8eb86dff554dcd32d0c6deb59fa0f54/pymongo-4.5.0-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:40d5f6e853ece9bfc01e9129b228df446f49316a4252bb1fbfae5c3c9dedebad", size = 691070, upload-time = "2023-08-22T13:23:25.239Z" }, + { url = "https://files.pythonhosted.org/packages/03/2d/1ea8cca6e7fffd217bac4b8b513026b032628bdef135ff0bbe9c3d98bb69/pymongo-4.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:152259f0f1a60f560323aacf463a3642a65a25557683f49cfa08c8f1ecb2395a", size = 670480, upload-time = "2023-08-22T13:23:27.414Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cc/047a8f0cd12167e6ae32df62ed3fd3a6b4a8ed25bcb7b90751aa46088ff2/pymongo-4.5.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6d64878d1659d2a5bdfd0f0a4d79bafe68653c573681495e424ab40d7b6d6d41", size = 683832, upload-time = "2023-08-22T13:23:29.519Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1d/c7f08d3898218d16ef5af6c858ac8b332ae9dfad30e7fab499a3f44b6958/pymongo-4.5.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1bb3a62395ffe835dbef3a1cbff48fbcce709c78bd1f52e896aee990928432b", size = 676960, upload-time = "2023-08-22T13:23:31.493Z" }, + { url = "https://files.pythonhosted.org/packages/4e/50/7e0e838892507d99a6b981824c73617cba567a56a802a518d49e12da77b6/pymongo-4.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe48f50fb6348511a3268a893bfd4ab5f263f5ac220782449d03cd05964d1ae7", size = 671344, upload-time = "2023-08-22T13:23:33.318Z" }, + { url = "https://files.pythonhosted.org/packages/4c/af/b50696624e65c7372918e384f4b62ab3b42b6627a07cfc75179dfcc94528/pymongo-4.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7591a3beea6a9a4fa3080d27d193b41f631130e3ffa76b88c9ccea123f26dc59", size = 659743, upload-time = "2023-08-22T13:23:35.311Z" }, + { url = "https://files.pythonhosted.org/packages/1c/c6/29a5b20c14e01455a686de834945d223657b515a3f5643004a7efd98371a/pymongo-4.5.0-cp310-cp310-win32.whl", hash = "sha256:3a7166d57dc74d679caa7743b8ecf7dc3a1235a9fd178654dddb2b2a627ae229", size = 462642, upload-time = "2023-08-22T13:23:36.863Z" }, + { url = "https://files.pythonhosted.org/packages/7a/88/a6b3661960518dbc656b67fefb85efff874d58ba0f3c31bda216d5726a02/pymongo-4.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:21b953da14549ff62ea4ae20889c71564328958cbdf880c64a92a48dda4c9c53", size = 468187, upload-time = "2023-08-22T13:23:38.574Z" }, + { url = "https://files.pythonhosted.org/packages/de/67/949da6f882723be8ca8ef63678d7f999b4d9c235c656c0376ea8b6c041d6/pymongo-4.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ead4f19d0257a756b21ac2e0e85a37a7245ddec36d3b6008d5bfe416525967dc", size = 529444, upload-time = "2023-08-22T13:23:40.067Z" }, + { url = "https://files.pythonhosted.org/packages/17/77/d607049092de6a467846912a3efe37a6e684b4e36b1bcce812c8410efec6/pymongo-4.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9aff6279e405dc953eeb540ab061e72c03cf38119613fce183a8e94f31be608f", size = 674130, upload-time = "2023-08-22T13:23:42.175Z" }, + { url = "https://files.pythonhosted.org/packages/68/c1/4948e472c408cdd9d7586015e7fcf23da512714cd3ad51e90a9eed1cac0d/pymongo-4.5.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4c8d6aa91d3e35016847cbe8d73106e3d1c9a4e6578d38e2c346bfe8edb3ca", size = 686920, upload-time = "2023-08-22T13:23:44.318Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/5b40ce26db3730b3728a9ff3d34414874bc90aeb81046fb26e3c1ffd8890/pymongo-4.5.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08819da7864f9b8d4a95729b2bea5fffed08b63d3b9c15b4fea47de655766cf5", size = 680739, upload-time = "2023-08-22T13:23:46.359Z" }, + { url = "https://files.pythonhosted.org/packages/77/c8/aa46a179d476a06630cf9a5463c5edc06b938fa8894b99194ebbdc775d76/pymongo-4.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a253b765b7cbc4209f1d8ee16c7287c4268d3243070bf72d7eec5aa9dfe2a2c2", size = 675095, upload-time = "2023-08-22T13:23:47.915Z" }, + { url = "https://files.pythonhosted.org/packages/83/3e/87ff6e82ec689af8a3b645fc3cba7ae35b8f7697a5f16cbf38aaedb4442f/pymongo-4.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8027c9063579083746147cf401a7072a9fb6829678076cd3deff28bb0e0f50c8", size = 663698, upload-time = "2023-08-22T13:23:49.486Z" }, + { url = "https://files.pythonhosted.org/packages/7d/57/70c0f761c9118e6e7d246feca5473983ddaec839d73b6ffa20d3816bbac7/pymongo-4.5.0-cp311-cp311-win32.whl", hash = "sha256:9d2346b00af524757576cc2406414562cced1d4349c92166a0ee377a2a483a80", size = 462641, upload-time = "2023-08-22T13:23:51.062Z" }, + { url = "https://files.pythonhosted.org/packages/4d/88/7b9bfad1ad6269e0b9e0bb39d093aabc27837da4ccb93391cda68f580984/pymongo-4.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:c3c3525ea8658ee1192cdddf5faf99b07ebe1eeaa61bf32821126df6d1b8072b", size = 468186, upload-time = "2023-08-22T13:23:52.964Z" }, + { url = "https://files.pythonhosted.org/packages/53/aa/771b2eafc9720ba14bac29429f10d735107130368b0d46364d01769372a1/pymongo-4.5.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:e5a27f348909235a106a3903fc8e70f573d89b41d723a500869c6569a391cff7", size = 528289, upload-time = "2023-08-22T13:23:54.989Z" }, + { url = "https://files.pythonhosted.org/packages/46/4e/3f805fd2bc1c378cf2359a5eeb89ce6ebce1deea14eaeb046a719338399c/pymongo-4.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9a9a39b7cac81dca79fca8c2a6479ef4c7b1aab95fad7544cc0e8fd943595a2", size = 689027, upload-time = "2023-08-22T13:23:57.116Z" }, + { url = "https://files.pythonhosted.org/packages/52/c7/631c27c7a30ab5d9217c6a3ec3c36c428627cba4b6a29b06fecb4a204faf/pymongo-4.5.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:496c9cbcb4951183d4503a9d7d2c1e3694aab1304262f831d5e1917e60386036", size = 698913, upload-time = "2023-08-22T13:23:59.071Z" }, + { url = "https://files.pythonhosted.org/packages/93/0b/461ceec310c6042f3b076cd8aa0acaeef3213f404df9b71f5ee0c9f3b611/pymongo-4.5.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23cc6d7eb009c688d70da186b8f362d61d5dd1a2c14a45b890bd1e91e9c451f2", size = 695520, upload-time = "2023-08-22T13:24:01.047Z" }, + { url = "https://files.pythonhosted.org/packages/46/c8/6e390e8e932b0c221ece274eb38b2a43dc50786472ea31c6d7dae9f006f9/pymongo-4.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fff7d17d30b2cd45afd654b3fc117755c5d84506ed25fda386494e4e0a3416e1", size = 686202, upload-time = "2023-08-22T13:24:03.458Z" }, + { url = "https://files.pythonhosted.org/packages/49/64/44d2010a264656d392fae75f9c6b13a9c84498dce3fc06aeb1a8d500f78a/pymongo-4.5.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6422b6763b016f2ef2beedded0e546d6aa6ba87910f9244d86e0ac7690f75c96", size = 675525, upload-time = "2023-08-22T13:24:05.507Z" }, + { url = "https://files.pythonhosted.org/packages/b1/8a/a3ccde2312ad9c35f134aee061fe31e69833da72118f0cbcd160a337b9e3/pymongo-4.5.0-cp312-cp312-win32.whl", hash = "sha256:77cfff95c1fafd09e940b3fdcb7b65f11442662fad611d0e69b4dd5d17a81c60", size = 463192, upload-time = "2023-08-22T13:24:06.979Z" }, + { url = "https://files.pythonhosted.org/packages/c0/59/900b1eacd90bf00980fd05abacfde216450684c5b34608c71850629661f4/pymongo-4.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:e57d859b972c75ee44ea2ef4758f12821243e99de814030f69a3decb2aa86807", size = 468368, upload-time = "2023-08-22T13:24:08.921Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/c4/13b4776ea2d76c115c1d1b84579f3764ee6d57204f6be27119f13a61d0a9/python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", size = 357324, upload-time = "2021-07-14T08:19:19.783Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/7a/87837f39d0296e723bb9b62bbb257d0355c7f6128853c78955f57342a56d/python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9", size = 247702, upload-time = "2021-07-14T08:19:18.161Z" }, +] + +[[package]] +name = "python-engineio" +version = "4.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "simple-websocket" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/12/bdef9dbeedbe2cdeba2a2056ad27b1fb081557d34b69a97f574843462cae/python_engineio-4.13.1.tar.gz", hash = "sha256:0a853fcef52f5b345425d8c2b921ac85023a04dfcf75d7b74696c61e940fd066", size = 92348, upload-time = "2026-02-06T23:38:06.12Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/54/0cce26da03a981f949bb8449c9778537f75f5917c172e1d2992ff25cb57d/python_engineio-4.13.1-py3-none-any.whl", hash = "sha256:f32ad10589859c11053ad7d9bb3c9695cdf862113bfb0d20bc4d890198287399", size = 59847, upload-time = "2026-02-06T23:38:04.861Z" }, +] + +[[package]] +name = "python-socketio" +version = "5.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "bidict" }, + { name = "python-engineio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/81/cf8284f45e32efa18d3848ed82cdd4dcc1b657b082458fbe01ad3e1f2f8d/python_socketio-5.16.1.tar.gz", hash = "sha256:f863f98eacce81ceea2e742f6388e10ca3cdd0764be21d30d5196470edf5ea89", size = 128508, upload-time = "2026-02-06T23:42:07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/c7/deb8c5e604404dbf10a3808a858946ca3547692ff6316b698945bb72177e/python_socketio-5.16.1-py3-none-any.whl", hash = "sha256:a3eb1702e92aa2f2b5d3ba00261b61f062cce51f1cfb6900bf3ab4d1934d2d35", size = 82054, upload-time = "2026-02-06T23:42:05.772Z" }, +] + +[[package]] +name = "pytz" +version = "2023.3.post1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/4f/7bf883f12ad496ecc9514cd9e267b29a68b3e9629661a2bbc24f80eff168/pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b", size = 316899, upload-time = "2023-09-05T01:56:58.535Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/4d/aaf7eff5deb402fd9a24a1449a8119f00d74ae9c2efa79f8ef9994261fc2/pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7", size = 502454, upload-time = "2023-09-05T01:56:55.916Z" }, +] + +[[package]] +name = "requests" +version = "2.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/be/10918a2eac4ae9f02f6cfe6414b7a155ccd8f7f9d4380d62fd5b955065c3/requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1", size = 110794, upload-time = "2023-05-22T15:12:44.175Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", size = 62574, upload-time = "2023-05-22T15:12:42.313Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/dc/4e6ac71b511b141cf626357a3946679abeba4cf67bc7cc5a17920f31e10d/ruff-0.15.1.tar.gz", hash = "sha256:c590fe13fb57c97141ae975c03a1aedb3d3156030cabd740d6ff0b0d601e203f", size = 4540855, upload-time = "2026-02-12T23:09:09.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/bf/e6e4324238c17f9d9120a9d60aa99a7daaa21204c07fcd84e2ef03bb5fd1/ruff-0.15.1-py3-none-linux_armv6l.whl", hash = "sha256:b101ed7cf4615bda6ffe65bdb59f964e9f4a0d3f85cbf0e54f0ab76d7b90228a", size = 10367819, upload-time = "2026-02-12T23:09:03.598Z" }, + { url = "https://files.pythonhosted.org/packages/b3/ea/c8f89d32e7912269d38c58f3649e453ac32c528f93bb7f4219258be2e7ed/ruff-0.15.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:939c995e9277e63ea632cc8d3fae17aa758526f49a9a850d2e7e758bfef46602", size = 10798618, upload-time = "2026-02-12T23:09:22.928Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0f/1d0d88bc862624247d82c20c10d4c0f6bb2f346559d8af281674cf327f15/ruff-0.15.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1d83466455fdefe60b8d9c8df81d3c1bbb2115cede53549d3b522ce2bc703899", size = 10148518, upload-time = "2026-02-12T23:08:58.339Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c8/291c49cefaa4a9248e986256df2ade7add79388fe179e0691be06fae6f37/ruff-0.15.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9457e3c3291024866222b96108ab2d8265b477e5b1534c7ddb1810904858d16", size = 10518811, upload-time = "2026-02-12T23:09:31.865Z" }, + { url = "https://files.pythonhosted.org/packages/c3/1a/f5707440e5ae43ffa5365cac8bbb91e9665f4a883f560893829cf16a606b/ruff-0.15.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:92c92b003e9d4f7fbd33b1867bb15a1b785b1735069108dfc23821ba045b29bc", size = 10196169, upload-time = "2026-02-12T23:09:17.306Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ff/26ddc8c4da04c8fd3ee65a89c9fb99eaa5c30394269d424461467be2271f/ruff-0.15.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fe5c41ab43e3a06778844c586251eb5a510f67125427625f9eb2b9526535779", size = 10990491, upload-time = "2026-02-12T23:09:25.503Z" }, + { url = "https://files.pythonhosted.org/packages/fc/00/50920cb385b89413f7cdb4bb9bc8fc59c1b0f30028d8bccc294189a54955/ruff-0.15.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66a6dd6df4d80dc382c6484f8ce1bcceb55c32e9f27a8b94c32f6c7331bf14fb", size = 11843280, upload-time = "2026-02-12T23:09:19.88Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6d/2f5cad8380caf5632a15460c323ae326f1e1a2b5b90a6ee7519017a017ca/ruff-0.15.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a4a42cbb8af0bda9bcd7606b064d7c0bc311a88d141d02f78920be6acb5aa83", size = 11274336, upload-time = "2026-02-12T23:09:14.907Z" }, + { url = "https://files.pythonhosted.org/packages/a3/1d/5f56cae1d6c40b8a318513599b35ea4b075d7dc1cd1d04449578c29d1d75/ruff-0.15.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ab064052c31dddada35079901592dfba2e05f5b1e43af3954aafcbc1096a5b2", size = 11137288, upload-time = "2026-02-12T23:09:07.475Z" }, + { url = "https://files.pythonhosted.org/packages/cd/20/6f8d7d8f768c93b0382b33b9306b3b999918816da46537d5a61635514635/ruff-0.15.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:5631c940fe9fe91f817a4c2ea4e81f47bee3ca4aa646134a24374f3c19ad9454", size = 11070681, upload-time = "2026-02-12T23:08:55.43Z" }, + { url = "https://files.pythonhosted.org/packages/9a/67/d640ac76069f64cdea59dba02af2e00b1fa30e2103c7f8d049c0cff4cafd/ruff-0.15.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:68138a4ba184b4691ccdc39f7795c66b3c68160c586519e7e8444cf5a53e1b4c", size = 10486401, upload-time = "2026-02-12T23:09:27.927Z" }, + { url = "https://files.pythonhosted.org/packages/65/3d/e1429f64a3ff89297497916b88c32a5cc88eeca7e9c787072d0e7f1d3e1e/ruff-0.15.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:518f9af03bfc33c03bdb4cb63fabc935341bb7f54af500f92ac309ecfbba6330", size = 10197452, upload-time = "2026-02-12T23:09:12.147Z" }, + { url = "https://files.pythonhosted.org/packages/78/83/e2c3bade17dad63bf1e1c2ffaf11490603b760be149e1419b07049b36ef2/ruff-0.15.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:da79f4d6a826caaea95de0237a67e33b81e6ec2e25fc7e1993a4015dffca7c61", size = 10693900, upload-time = "2026-02-12T23:09:34.418Z" }, + { url = "https://files.pythonhosted.org/packages/a1/27/fdc0e11a813e6338e0706e8b39bb7a1d61ea5b36873b351acee7e524a72a/ruff-0.15.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3dd86dccb83cd7d4dcfac303ffc277e6048600dfc22e38158afa208e8bf94a1f", size = 11227302, upload-time = "2026-02-12T23:09:36.536Z" }, + { url = "https://files.pythonhosted.org/packages/f6/58/ac864a75067dcbd3b95be5ab4eb2b601d7fbc3d3d736a27e391a4f92a5c1/ruff-0.15.1-py3-none-win32.whl", hash = "sha256:660975d9cb49b5d5278b12b03bb9951d554543a90b74ed5d366b20e2c57c2098", size = 10462555, upload-time = "2026-02-12T23:09:29.899Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5e/d4ccc8a27ecdb78116feac4935dfc39d1304536f4296168f91ed3ec00cd2/ruff-0.15.1-py3-none-win_amd64.whl", hash = "sha256:c820fef9dd5d4172a6570e5721704a96c6679b80cf7be41659ed439653f62336", size = 11599956, upload-time = "2026-02-12T23:09:01.157Z" }, + { url = "https://files.pythonhosted.org/packages/2a/07/5bda6a85b220c64c65686bc85bd0bbb23b29c62b3a9f9433fa55f17cda93/ruff-0.15.1-py3-none-win_arm64.whl", hash = "sha256:5ff7d5f0f88567850f45081fac8f4ec212be8d0b963e385c3f7d0d2eb4899416", size = 10874604, upload-time = "2026-02-12T23:09:05.515Z" }, +] + +[[package]] +name = "seaweed" +version = "0.0.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/95/ffc5d1e6e3059d6bd504cb01cfe416cb37bd658f374b4969a1794e7ed46a/seaweed-0.0.1-py3-none-any.whl", hash = "sha256:bf15560e83ce5e66931f5eb3bf20678bee2d1afd304bedd6bfcbb2359d165150", size = 1045, upload-time = "2020-02-16T10:58:06.282Z" }, +] + +[[package]] +name = "simple-websocket" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wsproto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/d4/bfa032f961103eba93de583b161f0e6a5b63cebb8f2c7d0c6e6efe1e3d2e/simple_websocket-1.1.0.tar.gz", hash = "sha256:7939234e7aa067c534abdab3a9ed933ec9ce4691b0713c78acb195560aa52ae4", size = 17300, upload-time = "2024-10-10T22:39:31.412Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/59/0782e51887ac6b07ffd1570e0364cf901ebc36345fea669969d2084baebb/simple_websocket-1.1.0-py3-none-any.whl", hash = "sha256:4af6069630a38ed6c561010f0e11a5bc0d4ca569b36306eb257cd9a192497c8c", size = 13842, upload-time = "2024-10-10T22:39:29.645Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + +[[package]] +name = "werkzeug" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/70/1469ef1d3542ae7c2c7b72bd5e3a4e6ee69d7978fa8a3af05a38eca5becf/werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67", size = 864754, upload-time = "2026-01-08T17:49:23.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/e4/8d97cca767bcc1be76d16fb76951608305561c6e056811587f36cb1316a8/werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc", size = 225025, upload-time = "2026-01-08T17:49:21.859Z" }, +] + +[[package]] +name = "wsproto" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c7/79/12135bdf8b9c9367b8701c2c19a14c913c120b882d50b014ca0d38083c2c/wsproto-1.3.2.tar.gz", hash = "sha256:b86885dcf294e15204919950f666e06ffc6c7c114ca900b060d6e16293528294", size = 50116, upload-time = "2025-11-20T18:18:01.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/f5/10b68b7b1544245097b2a1b8238f66f2fc6dcaeb24ba5d917f52bd2eed4f/wsproto-1.3.2-py3-none-any.whl", hash = "sha256:61eea322cdf56e8cc904bd3ad7573359a242ba65688716b0710a5eb12beab584", size = 24405, upload-time = "2025-11-20T18:18:00.454Z" }, +]