diff --git a/hydra_client/__init__.py b/hydra_client/__init__.py index f833b59..6571bca 100644 --- a/hydra_client/__init__.py +++ b/hydra_client/__init__.py @@ -19,7 +19,3 @@ from .connection import * from .exception import * from .output import * -from .templates import * -from .plugin import * - -from hydra_base.exceptions import HydraPluginError diff --git a/hydra_client/click.py b/hydra_client/click.py index 31ada0d..c8b4d23 100644 --- a/hydra_client/click.py +++ b/hydra_client/click.py @@ -1,9 +1,5 @@ -""" This module contains helper functions and utilities for generating plugin.xml files from a click CLI. -""" import click from pathlib import Path -from xml.etree import ElementTree as ET -from xml.dom.minidom import parseString import hydra_base import os @@ -28,69 +24,6 @@ def hydra_app_decorator(func): return hydra_app_decorator -def make_plugins(group, shell, docker_image=None): - """ Generator of plugin XML data from the hydra_pywr CLI. """ - for name, command in group.commands.items(): - - try: - hydra_app_category = command.hydra_app_category - except AttributeError: - hydra_app_category = False - - if not hydra_app_category: - continue - - # Create plugin data for each sub-command of the group. - data = make_plugin(command, hydra_app_category, shell, docker_image=docker_image) - # Convert the data to etree ElementTree - xml = plugin_to_xml(data) - yield name, xml - - -def make_plugin(command, category, shell, docker_image=None): - """ Make an individual plugin XML definition from a `click.Command`. """ - - name = command.hydra_app_name - if name is None: - name = command.short_help - - if name is None: - name = command.name - - plugin = { - 'plugin_name': name, - 'plugin_dir': '', - 'plugin_description': command.help, - 'plugin_category': category, - 'plugin_location': '', - 'plugin_nativelogextension': '.log', - 'plugin_nativeoutputextension': '.out', - 'smallicon': None, - 'largeicon': None, - 'plugin_epilog': command.epilog, - 'mandatory_args': [], - 'non_mandatory_args': [], - 'switches': [] - } - - for category, arg in make_args(command): - plugin[category].append(arg) - - if docker_image is None: - plugin.update({ - 'plugin_command': '{}'.format(command.name), - 'plugin_shell': shell, - }) - else: - plugin.update({ - 'plugin_command': '{} {}'.format(shell, command.name), - 'plugin_shell': 'docker', - 'plugin_docker_image': docker_image, - }) - - return plugin - - def make_args(command): """ Generate argument definitions for each parameter in command. """ for param in command.params: @@ -115,41 +48,3 @@ def make_args(command): arg['argtype'] = HYDRA_ARGTYPES[param.name] yield category, arg - - -def plugin_to_xml(data): - """ Convert plugin definition to ElementTree. """ - root = ET.Element('plugin_info') - - for key, value in data.items(): - e = ET.SubElement(root, key, ) - - if key in ('mandatory_args', 'non_mandatory_args', 'switches'): - for arg in value: - arg_element = ET.SubElement(e, 'arg') - for arg_key, arg_value in arg.items(): - arg_sub_element = ET.SubElement(arg_element, arg_key) - arg_sub_element.text = arg_value - else: - e.text = value - - return root - - -def write_plugins(plugins, app_name): - """ Write the generated plugins to XML files. """ - base_plugin_dir = Path(hydra_base.config.get('plugin', 'default_directory')) - base_plugin_dir = base_plugin_dir.joinpath(app_name) - - if not base_plugin_dir.exists(): - base_plugin_dir.mkdir(parents=True, exist_ok=True) - - for name, element in plugins: - plugin_path = os.path.join(base_plugin_dir, name) - - if not os.path.exists(plugin_path): - os.mkdir(plugin_path) - - with open(os.path.join(plugin_path, 'plugin.xml'), 'w') as fh: - reparsed = parseString(ET.tostring(element, 'utf-8')) - fh.write(reparsed.toprettyxml(indent="\t")) diff --git a/hydra_client/exception.py b/hydra_client/exception.py index 667eeb4..1dfc6f7 100644 --- a/hydra_client/exception.py +++ b/hydra_client/exception.py @@ -17,8 +17,8 @@ __all__ = ['RequestError'] -from hydra_base.exceptions import HydraPluginError +from hydra_base.exceptions import HydraError -class RequestError(HydraPluginError): +class RequestError(HydraError): pass diff --git a/hydra_client/output.py b/hydra_client/output.py index 211f8c5..9570890 100644 --- a/hydra_client/output.py +++ b/hydra_client/output.py @@ -15,66 +15,16 @@ # # -*- coding: utf-8 -*- -__all__ = ['create_xml_response', 'write_progress', 'write_output', - 'validate_plugin_xml'] +__all__ = ['write_progress', 'write_output'] import os import logging log = logging.getLogger(__name__) -from lxml import etree -from lxml.etree import XMLSyntaxError, ParseError - from hydra_base import config -from hydra_base.exceptions import HydraPluginError import sys -def create_xml_response(plugin_name, network_id, scenario_ids, - errors=[], warnings=[], message=None, files=[]): - """ - Build the XML string required at the end of each plugin, describing - the errors, warnings, messages and outputed files, if any of these - are relevant. - """ - - xml_string = """ - %(message)s - %(plugin_name)s - %(network_id)s - %(scenario_list)s - - %(error_list)s - - - %(warning_list)s - - - %(file_list)s - -""" - - scenario_string = "%s" - error_string = "%s" - warning_string = "%s" - file_string = "%s" - - if scenario_ids is None: - scenario_ids = [] - - xml_string = xml_string % dict( - plugin_name = plugin_name, - network_id = network_id, - scenario_list = "\n".join([scenario_string % scen_id - for scen_id in scenario_ids]), - message = message if message is not None else "", - error_list = "\n".join([error_string%error for error in errors]), - warning_list = "\n".join([warning_string%warning for warning in warnings]), - file_list = "\n".join([file_string % f for f in files]), - ) - - return xml_string - def write_progress(x, y): """ @@ -94,35 +44,3 @@ def write_output(text): msg = "!!Output %s" % (text,) print(msg) sys.stdout.flush() - - -def validate_plugin_xml(plugin_xml_file_path): - log.info('Validating plugin xml file (%s).' % plugin_xml_file_path) - - try: - with open(plugin_xml_file_path) as f: - plugin_xml = f.read() - except: - raise HydraPluginError("Couldn't find plugin.xml.") - - try: - plugin_xsd_path = os.path.expanduser(config.get('plugin', - 'plugin_xsd_path')) - log.info("Plugin Input xsd: %s", plugin_xsd_path) - xmlschema_doc = etree.parse(plugin_xsd_path) - xmlschema = etree.XMLSchema(xmlschema_doc) - xml_tree = etree.fromstring(plugin_xml) - except XMLSyntaxError as e: - raise HydraPluginError("There is an error in your XML syntax: %s" % e) - except ParseError as e: - raise HydraPluginError("There is an error in your XML: %s" % e) - except Exception as e: - log.exception(e) - raise HydraPluginError("An unknown error occurred with the plugin xsd: %s"%e.message) - - try: - xmlschema.assertValid(xml_tree) - except etree.DocumentInvalid as e: - raise HydraPluginError('Plugin validation failed: ' + e.message) - - log.info("Plugin XML OK") diff --git a/hydra_client/plugin.py b/hydra_client/plugin.py deleted file mode 100644 index 9efb37b..0000000 --- a/hydra_client/plugin.py +++ /dev/null @@ -1,118 +0,0 @@ -# (c) Copyright 2013, 2014, University of Manchester -# -# HydraLib is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# HydraPlatform is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with HydraPlatform. If not, see -# -# -*- coding: utf-8 -*- - -__all__ = ['JSONPlugin'] - -import re -import logging -log = logging.getLogger(__name__) - -from datetime import timedelta -from dateutil.relativedelta import relativedelta - -from hydra_base.lib import units -from hydra_base.util.hydra_dateutil import get_time_period, get_datetime -from hydra_base.exceptions import HydraPluginError - -from .connection import RemoteJSONConnection - - -class JSONPlugin(object): - - def connect(self, args): - self.session_id = args.session_id - self.server_url = args.server_url - self.app_name = self.__class__.__bases__[0].__name__ - - self.connection = RemoteJSONConnection(self.server_url, self.session_id, - self.app_name) - - if self.session_id is None: - self.session_id = self.connection.login() - - def parse_time_step(self, time_step, target='s'): - """ - Read in the time step and convert it to seconds. - """ - log.info("Parsing time step %s", time_step) - # export numerical value from string using regex - value = re.findall(r'\d+', time_step)[0] - valuelen = len(value) - - try: - value = float(value) - except: - HydraPluginError("Unable to extract number of time steps (%s) from time step %s" % (value, time_step)) - - units = time_step[valuelen:].strip() - - period = get_time_period(units) - - log.info("Time period is %s", period) - - converted_time_step = units.convert(value, period, target) - - log.info("Time period is %s %s", converted_time_step, period) - - return float(converted_time_step), value, period - - def get_time_axis(self, start_time, end_time, time_step, time_axis=None): - """ - Create a list of datetimes based on an start time, end time and - time step. If such a list is already passed in, then this is not - necessary. - - Often either the start_time, end_time, time_step is passed into an - app or the time_axis is passed in directly. This function returns a - time_axis in both situations. - """ - if time_axis is not None: - actual_dates_axis = [] - for t in time_axis: - #If the user has entered the time_axis with commas, remove them. - t = t.replace(',', '').strip() - if t == '': - continue - actual_dates_axis.append(get_datetime(t)) - return actual_dates_axis - - else: - if start_time is None: - raise HydraPluginError("A start time must be specified") - if end_time is None: - raise HydraPluginError("And end time must be specified") - if time_step is None: - raise HydraPluginError("A time-step must be specified") - - start_date = get_datetime(start_time) - end_date = get_datetime(end_time) - delta_t, value, units = self.parse_time_step(time_step) - - time_axis = [start_date] - - value = int(value) - while start_date < end_date: - #Months and years are a special case, so treat them differently - if(units.lower() == "mon"): - start_date = start_date + relativedelta(months=value) - elif (units.lower() == "yr"): - start_date = start_date + relativedelta(years=value) - else: - start_date += timedelta(seconds=delta_t) - time_axis.append(start_date) - return time_axis - diff --git a/hydra_client/templates.py b/hydra_client/templates.py deleted file mode 100644 index 0fe81b4..0000000 --- a/hydra_client/templates.py +++ /dev/null @@ -1,244 +0,0 @@ -# (c) Copyright 2013, 2014, University of Manchester -# -# HydraLib is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# HydraPlatform is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with HydraPlatform. If not, see -# -# -*- coding: utf-8 -*- - -__all__ = ['set_resource_types', 'validate_template', 'xsd_validate'] - -import os -import logging -log = logging.getLogger(__name__) - -from lxml import etree - -from hydra_base import config -from hydra_base import util -from hydra_base.exceptions import HydraPluginError - - -def set_resource_types(client, xml_template, network, - nodetype_dict, linktype_dict, - grouptype_dict, networktype): - log.info("Setting resource types") - - template = client.service.upload_template_xml(xml_template) - - type_ids = dict() - warnings = [] - - for type_name in nodetype_dict.keys(): - for tmpltype in template.types.TemplateType: - if tmpltype.name == type_name: - type_ids.update({tmpltype.name: tmpltype.id}) - break - - for type_name in linktype_dict.keys(): - for tmpltype in template.types.TemplateType: - if tmpltype.name == type_name: - type_ids.update({tmpltype.name: tmpltype.id}) - break - - for type_name in grouptype_dict.keys(): - for tmpltype in template.types.TemplateType: - if tmpltype.name == type_name: - type_ids.update({tmpltype.name: tmpltype.id}) - break - - for tmpltype in template.types.TemplateType: - if tmpltype.name == networktype: - type_ids.update({tmpltype.name: tmpltype.id}) - break - - args = client.factory.create('hyd:ResourceTypeDefArray') - if type_ids[networktype]: - args.ResourceTypeDef.append(dict( - ref_key='NETWORK', - ref_id=network.id, - type_id=type_ids[networktype], - )) - - if network.nodes: - for node in network.nodes.Node: - for typename, node_name_list in nodetype_dict.items(): - if type_ids[typename] and node.name in node_name_list: - args.ResourceTypeDef.append(dict( - ref_key='NODE', - ref_id=node.id, - type_id=type_ids[typename], - )) - else: - warnings.append("No nodes found when setting template types") - - if network.links: - for link in network.links.Link: - for typename, link_name_list in linktype_dict.items(): - if type_ids[typename] and link.name in link_name_list: - args.ResourceTypeDef.append(dict( - ref_key='LINK', - ref_id=link.id, - type_id=type_ids[typename], - )) - else: - warnings.append("No links found when setting template types") - - if network.resourcegroups: - for group in network.resourcegroups.ResourceGroup: - for typename, group_name_list in grouptype_dict.items(): - if type_ids[typename] and group.name in group_name_list: - args.ResourceTypeDef.append(dict( - ref_key='GROUP', - ref_id=group.id, - type_id=type_ids[typename], - )) - else: - warnings.append("No resourcegroups found when setting template types") - - client.service.assign_types_to_resources(args) - return warnings - -def xsd_validate(template_file): - """ - Validate a template against the xsd. - Return the xml tree if successful. - """ - - with open(template_file) as f: - xml_template = f.read() - - template_xsd_path = os.path.expanduser(config.get('templates', - 'template_xsd_path')) - log.info("Template xsd: %s", template_xsd_path) - xmlschema_doc = etree.parse(template_xsd_path) - xmlschema = etree.XMLSchema(xmlschema_doc) - xml_tree = etree.fromstring(xml_template) - - try: - xmlschema.assertValid(xml_tree) - except etree.DocumentInvalid as e: - raise HydraPluginError('Template validation failed: ' + e.message) - - log.info("Template XSD validation successful.") - - return xml_tree - -def validate_template(template_file, connection): - - log.info('Validating template file (%s).' % template_file) - - #Check for duplicate attributes on a single resource and for duplicate attribute names - #but with different capitalisation - warnings = [] - errors = [] - attribute_names = [] - - xml_tree = xsd_validate(template_file) - - - template_dict = {'name': xml_tree.find('template_name').text, - 'resources': {} - } - - attributes = [] - #A list of all the unique attributes in the template - #A unique attribute is name & dimension. - #If a duplicate name is found but with an inconsistent dimension, then an error is thrown. - unique_attributes = {} - - for r in xml_tree.find('resources'): - #keep track of resource attribute names to make sure there's no duplicates - resource_attr_names = [] - - resource_dict = {} - resource_name = r.find('name').text - resource_type = r.find('type').text - resource_dict['type'] = resource_type - resource_dict['name'] = resource_name - resource_dict['attributes'] = {} - for attr in r.findall("attribute"): - attr_dict = {} - attr_name = attr.find('name').text - attr_dict['name'] = attr_name - - #Check for inconsistent capitalisation - if attr_name not in attribute_names and attr_name.lower().replace(" ", "") in attribute_names: - warnings.append("A similar Attribute to %s is already specified in the template. Are you sure your spelling is correct?"%(attr_name)) - else: - attribute_names.append(attr_name.lower().replace(" ", "")) - - #Check for duplicate attribute names on a resource - if attr_name.lower() in resource_attr_names: - errors.append("Attribute %s specified multiple times on resource %s"%(attr_name, resource_name)) - else: - resource_attr_names.append(attr_name.lower()) - - attribute_names.append(attr_name) - - if attr.find('dimension') is not None and attr.find('dimension').text is not None: - dimension = attr.find('dimension').text - if dimension.lower() == 'dimensionless': - dimension = 'dimensionless' - attr_dict['dimension'] = dimension - else: - attr_dict['dimension'] = 'dimensionless' - - if unique_attributes.get(attr_name) is not None: - if unique_attributes[attr_name] != attr_dict['dimension']: - errors.append("Attribute %s has been defined twice in the template with different dimensions. " - "Please make them consistent or rename one of them."%(attr_name)) - else: - unique_attributes[attr_name] = attr_dict['dimension'] - - if attr.find('unit') is not None: - attr_dict['unit'] = attr.find('unit').text - if attr.find('is_var') is not None: - attr_dict['is_var'] = attr.find('is_var').text - if attr.find('data_type') is not None: - attr_dict['data_type'] = attr.find('data_type').text - - attributes.append({'name': attr_name, - 'dimen': attr_dict['dimension']}) - - restction_xml = attr.find("restrictions") - attr_dict['restrictions'] = \ - util.dataset_util.get_restriction_as_dict(restction_xml) - resource_dict['attributes'][attr_name] = attr_dict - - if template_dict['resources'].get(resource_type): - template_dict['resources'][resource_type][resource_name] = \ - resource_dict - else: - template_dict['resources'][resource_type] = {resource_name: - resource_dict} - - #Of the attributes in the template, get the ones that exist on the server - stored_attrs = connection.call('get_attributes', {'attrs': attributes}) - - attr_dict = {} - for a in stored_attrs: - if a: - attr_dict[(a['name'], a.get('dimen'))] = a['id'] - - log.info("Template attributes retrieved!") - - for rt in template_dict['resources'].values(): - for t in rt.values(): - for a in t['attributes'].values(): - a['id'] = attr_dict.get((a['name'], a.get('dimension'))) - - log.info("Template attributes updated with IDS") - - log.info("Template OK") - - return template_dict, warnings, errors diff --git a/setup.py b/setup.py index 4349166..f54e48f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ author_email='stephen.knox@manchester.ac.uk', url='https://github.com/hydraplatform/hydra-client-python', packages=find_packages(), - install_requires=['lxml', 'requests', 'cryptography'], + install_requires=['requests', 'cryptography'], entry_points=''' [console_scripts] hydra-cli=hydra_client.cli:start_cli