diff --git a/README.md b/README.md index bd8475d..68a944f 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ - [ ] Support OpenSSL as redirect method (for routing cleartext TCP sessions based on SNI header) - [ ] Support for a general TCP service for use with stunnel / haproxy / socat - [ ] Support for mosh -- [ ] Use a templating engine to generate SSH configuration and HTML files +- [ ] Use a templating engine to generate HTML files - [ ] Tests ;) - [ ] Host key management - [ ] Cross world via references diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..1db640f --- /dev/null +++ b/TODO.md @@ -0,0 +1,5 @@ +consider supporting fzf: + +```sh +cat .wssh_commands | fzf --multi --ansi --scroll-off=5 --height=10 --border=double --border-label-pos=-3:bottom --color=label:italic:black --tac --prompt=SSH\ to:\ --marker=✓ --layout=reverse --info=hidden --pointer=➜ +``` diff --git a/emuserema/__init__.py b/emuserema/__init__.py index 22b784f..77592c6 100644 --- a/emuserema/__init__.py +++ b/emuserema/__init__.py @@ -3,16 +3,33 @@ from os.path import expanduser from sys import exit, argv from emuserema.plugin_manager import PluginManager -from emuserema.services import AbstractService +from emuserema.services import AbstractService, SSHservice from emuserema.world import World from emuserema.yamlloader import EmuseremaYamlLoader from ruamel.yaml import dump -from emuserema.utils import traverse, get_default_directory +from emuserema.utils import traverse, get_default_directory, get_ansible_treevars, inherit from emuserema.services import * from emuserema.redirects import RedirectFactory # if yaml dumping is to be enabled: from sys import stdout +import logging +# create log +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) +# create console handler and set level to debug +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) + +# create formatter +formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s') + +# add formatter to ch +ch.setFormatter(formatter) + +# add ch to log +log.addHandler(ch) +log.debug("Emuserema starting") class Emuserema(object): def __init__(self, definitions_directory=None): @@ -26,13 +43,12 @@ def __init__(self, definitions_directory=None): self.services = {} self.worlds = {} + log.debug("Loading source data") self.load() + log.debug("Processing loaded data") self.process() - def __del__(self): - """Commit changes. For example new redirects to .redirects.yaml""" - self.commit() def service_builder(self, path, obj): """Iterate over YAML entries replacing them with objects based on those definitions.""" @@ -41,7 +57,11 @@ def service_builder(self, path, obj): if isinstance(obj, dict): if '_type' in obj: if self.service_factory.supported(obj): - service = self.service_factory.create(path, **obj) + service = self.service_factory.create( + path = path, + kwargs = obj, + emuserema = self + ) self.services[service.tag] = service return service else: @@ -86,67 +106,66 @@ def load(self): def process(self): """Iterate over definitions tree a numerous times""" + log.debug("Iterating data tree to build services") self.data = traverse(self.data, callback=self.service_builder) + log.debug("Iterating data tree to build service dependency tree") self.data = traverse(self.data, callback=self.via_resolver) + log.debug("Iterating data tree to create redirector rules") self.data = traverse(self.data, callback=self.redirector) + log.debug("Iterating data tree to create final worlds") self.data = traverse(self.data, callback=self.create_worlds) def dump(self): dump(self.data, stdout, default_flow_style=False) + def jsonServiceItemRenderer(self, service): + print(repr(service)) + if isinstance(service, SSHservice): + kw = { + "name": service.tag, + "path": service.path, + "address": None, + "metadata": {}, + "ansible": { + "groups": [], + "hostvars": {} + } + } + if 'HostName' in service.kwargs: + kw["address"] = service.kwargs['HostName'] + if "_ansible_groups" in service.kwargs: + kw['ansible']['groups'] = service.kwargs['_ansible_groups'] + if "_ansible_hostvars" in service.kwargs: + kw['ansible']['hostvars'] = service.kwargs['_ansible_hostvars'] + if '_metadata' in service.kwargs: + kw['metadata'] = service.kwargs['_metadata'] + return kw + return None + + def jsondump(self): + import json + ja = [] + for service in self.services: + if result := self.jsonServiceItemRenderer(self.services[service]): + ja.append(result) + print(json.dumps(ja)) + def render(self): config = self.yamlloader.loadyaml('config.yaml') renderer_plugins = PluginManager('emuserema.plugins.renderers', config=config['plugins']['renderers']) - renderer_plugins.run_selected(filter(lambda name: config['plugins']['renderers'][name]['enabled'] is True, - config['plugins']['renderers']), worlds=self.worlds, services=self.services) - + renderer_plugins.run_selected( + filter( + lambda name: config['plugins']['renderers'][name]['enabled'] is True, + config['plugins']['renderers'] + ), + worlds=self.worlds, services=self.services, emuserema=self + ) def populate_ansible_inventory(self, inventory=None, world='default'): """function to populate Ansible inventory on the spot""" - def get_ansible_treevars(data, data_path): - """Acquires the _ansible_treevars items from the main data dict""" - if len(data_path) > 1: - yield from get_ansible_treevars(data[data_path[0]], data_path[1:]) - #elif isinstance(data.get(data_path[0]), dict): - yield data.get(data_path[0]).get('_ansible_treevars') - - def inherit(ansible_treevars_items): - """Renders the actual inheritance that it gets back from get_ansible_treevars()""" - retval_dict = dict() - for ansible_treevars_item in ansible_treevars_items: - #a leaf with a value - if ansible_treevars_item: - print('item', ansible_treevars_item) - for ansible_treevars_dict in ansible_treevars_item: - for key, value in ansible_treevars_dict.items(): - if isinstance(value, str): - if key not in retval_dict: - retval_dict[key] = [value] - else: - retval_dict[key].append(value) - #retval_dict[key].insert(0, value) - elif isinstance(value, int): - if key not in retval_dict: - retval_dict[key] = [str(value)] - else: - retval_dict[key].append(str(value)) - #retval_dict[key].insert(0, value) - #currently we're only appending the list items, no nested lists are supported yet - elif isinstance(value, list): - if key not in retval_dict: - retval_dict[key] = value - else: - retval_dict[key] + value - #retval_dict[key].insert(0, value) - else: - raise NotImplementedError( - '_ansible_treevars item with a type of %s is unsupported.' % type(value)) - print('treevars items:', retval_dict) - for key, value in retval_dict.items(): - yield [key, value] world = self.worlds[world] for service in self.services: @@ -161,8 +180,7 @@ def inherit(ansible_treevars_items): # 'SSH_AUTH_SOCK=/home/e/.ssh/agents/%s /usr/bin/ssh' % service.world) # iterate over the data tree to get _ansible_treevars items - for key, value in inherit(get_ansible_treevars(self.data, service.path)): - print("setting treevars variable", key, value) + for key, value in inherit(get_ansible_treevars(self.data, service.path)).items(): #extra group memberships based on treevars if key == '_ansible_groups': for group in value: @@ -175,14 +193,17 @@ def inherit(ansible_treevars_items): inventory.set_variable(service.tag, key, value[0]) else: inventory.set_variable(service.tag, key, value) - if '_ansible_hostvars' in service.kwargs: - for key, value in service.kwargs['_ansible_hostvars'].items(): - inventory.set_variable(service.tag, key, value) - if '_ansible_groups' in service.kwargs: - for group in service.kwargs['_ansible_groups']: - group = group.lower().replace(' ', '_') - group = inventory.add_group(group) - inventory.add_child(group, service.tag) + for key, value in service.kwargs.items(): + if key == '_ansible_hostvars': + for key, value in service.kwargs['_ansible_hostvars'].items(): + inventory.set_variable(service.tag, key, value) + elif key == '_ansible_groups': + for group in service.kwargs['_ansible_groups']: + group = group.lower().replace(' ', '_') + group = inventory.add_group(group) + inventory.add_child(group, service.tag) + else: + inventory.set_variable(service.tag, 'emuserema_kwarg_'+key, value) #add one extra group with the service class name group = inventory.add_group('class_%s' % service.__class__.__name__) inventory.add_child(group, service.tag) @@ -190,21 +211,29 @@ def inherit(ansible_treevars_items): group = group.lower().replace(' ', '_') group = inventory.add_group(group) inventory.add_child(group, service.tag) + if 'rw' not in service.path: + group = inventory.add_group('__'.join(['nodename_stub', service.tag.split('.')[0]])) + inventory.add_child(group, service.tag) + # embed() pass def commit(self): + """Commit changes. For example new redirects to .redirects.yaml""" self.redirect_factory.commit() -def main(): - emuserema = Emuserema() - if len(argv) > 1: - if argv[1] == '-t': - print("Configuration parsing was successful.") - exit(0) - emuserema.render() - - -if __name__ == '__main__': - main() +##def main(): +## setup_logger() +## emuserema = Emuserema() +## if len(argv) > 1: +## if argv[1] == '-t': +## print("Configuration parsing was successful.") +## exit(0) +## emuserema.render() +## emuserema.commit() +## +## +##if __name__ == '__main__': +## main() +## \ No newline at end of file diff --git a/emuserema/__main__.py b/emuserema/__main__.py index 59908be..4c9f343 100755 --- a/emuserema/__main__.py +++ b/emuserema/__main__.py @@ -19,6 +19,9 @@ def main(args=None): parser.add_argument("-t", "--test", help="Only test configurations and exit.", action="store_true") + parser.add_argument("-j", "--json", + help="Dump parsed world configuration JSON.", + action="store_true") parser.add_argument("-d", "--dump", help="Dump parsed world configuration YAML.", action="store_true") @@ -33,14 +36,15 @@ def main(args=None): definitions_directory=args.config_dir ) + if args.json: + return emuserema.jsondump() if args.dump: - emuserema.dump() + return emuserema.dump() if args.test: print("Configuration parsing was successful.") exit(0) emuserema.render() - if __name__ == '__main__': main() diff --git a/emuserema/plugin_manager.py b/emuserema/plugin_manager.py index c95d980..bee0a62 100644 --- a/emuserema/plugin_manager.py +++ b/emuserema/plugin_manager.py @@ -10,6 +10,8 @@ class Plugin(object): def __init__(self, config={}): self.description = 'UNKNOWN' self._config = config + self.worlds = {} + self.services = {} self.config() if 'cleanup' in self._config and self._config['cleanup'] is True: self.cleanup() @@ -20,9 +22,14 @@ def cleanup(self): def config(self): raise NotImplementedError - def render(self, kwargs): + def render(self): raise NotImplementedError + def run(self, **kwargs): + self.services = kwargs['services'] + self.worlds = kwargs['worlds'] + self.render() + class PluginManager(object): def __init__(self, plugin_package, config): diff --git a/emuserema/plugins/renderers/openssh.py b/emuserema/plugins/renderers/openssh.py index f37db44..945631f 100644 --- a/emuserema/plugins/renderers/openssh.py +++ b/emuserema/plugins/renderers/openssh.py @@ -1,75 +1,42 @@ +"""OpenSSH renderer""" +from jinja2 import ChoiceLoader, PackageLoader, FileSystemLoader, Environment from emuserema.services import SSHservice from emuserema.plugin_manager import Plugin from emuserema.utils import makedir_getfd, cleanup_files class OpenSSHRenderer(Plugin): + """Plugin class that render SSH configurations""" def config(self): self.description = 'openssh client configuration files renderer' self.ssh_config_files = {} + self.jinja_env = Environment(loader=ChoiceLoader([ + PackageLoader('emuserema'), + FileSystemLoader(searchpath='templates', followlinks=True) + ])) + self.template = self.jinja_env.get_template('openssh/openssh.j2') def cleanup(self): + """Removes all files from the destination directories""" cleanup_files(self._config['output_dir']) - def render_openssh(self): - for key, service in self.services.items(): - #service=self.services[key] - if isinstance(service, SSHservice): - ssh_config_file = self.open_ssh_config_file(service.world) - print("\n###\n### %s\n###\n" % service.tag, file=ssh_config_file) - print("Host %s" % service.tag, file=ssh_config_file) - print("\n".join(service.pb), file=ssh_config_file) - #for k in self.render_kv(service): - # if type(k) == tuple: - # print("\t%s = %s" % (k[0], k[1]), file=ssh_config_file) - # elif type(k) == str: - # print("\t"+k, file=ssh_config_file) - - self.close_ssh_config_files() - - def render_kv(self, service): - rv = [] - for key, value in service.kwargs.items(): - if key[0] != '_': # Blindly copy all the non meta settings - rv.append((key, value)) - for bind_hostport, target_hostport in service.forwards['local'].items(): - if target_hostport[1]: - rv.append('#' + target_hostport[1]) - rv.append(('LocalForward', "%s %s" % (bind_hostport, target_hostport[0]))) - for bind_hostport, target_hostport in service.forwards['remote']: - if target_hostport[1]: - rv.append('#' + target_hostport[1]) - rv.append(('RemoteForward', "%s %s" % (bind_hostport, target_hostport[0]))) - for bind_hostport in service.forwards['dynamic']: - if target_hostport[1]: - rv.append('#' + target_hostport[1]) - rv.append(('DynamicForward', "%s" % (bind_hostport, target_hostport[0]))) - return rv - - def open_ssh_config_file(self, world_name): - if world_name not in self.ssh_config_files: - self.ssh_config_files[world_name] = makedir_getfd("%s/%s" % (self._config['output_dir'], world_name)) - try: - with open('templates/ssh_config_prefix', 'r') as preamble: - for line in preamble: - self.ssh_config_files[world_name].write(line) - except FileNotFoundError: - pass - - return self.ssh_config_files[world_name] - - def close_ssh_config_files(self): - for key, ssh_config_file in self.ssh_config_files.items(): - try: - with open('templates/ssh_config_postfix', 'r') as postfix: - for line in postfix: - ssh_config_file.write(line) - except FileNotFoundError: - pass - ssh_config_file.close() - - def run(self, **kwargs): - self.services = kwargs['services'] - self.worlds = kwargs['worlds'] - self.render_openssh() + def render(self): + """Actual function that renders the jinja templates""" + for world in self.worlds: + with makedir_getfd("%s/%s" % (self._config['output_dir'], + self.worlds[world].name)) as ssh_config_file: + print( + self.template.render( + services=list(map( + lambda service, w=world: self.worlds[w].services[service], + filter( + lambda service, w=world: + isinstance(self.worlds[w].services[service], SSHservice), + list(self.worlds[world].services.keys()) + ) + )), + world=self.worlds[world] + ), + file=ssh_config_file + ) diff --git a/emuserema/services.py b/emuserema/services.py index a8794a2..205247e 100644 --- a/emuserema/services.py +++ b/emuserema/services.py @@ -5,12 +5,15 @@ #FOR URLservice support: from urllib.parse import urlparse +from emuserema.utils import get_ansible_treevars, inherit, y7address ABC = ABCMeta('ABC', (object,), {'__slots__': ()}) +import logging +log = logging.getLogger(__name__) class AbstractService(ABC): - def __init__(self, path=[], **kwargs): + def __init__(self, path=[], kwargs={}, treevars=[]): self.pb = [] self.tag = path[-1] self.world = path[0] @@ -19,6 +22,7 @@ def __init__(self, path=[], **kwargs): #self.label="%s\t" % tag self.label = "" self.via = None + self.treevars = treevars self.config() @@ -38,7 +42,7 @@ def __repr__(self): class DummyService(AbstractService): - def __init__(self, path=[], **kwargs): + def __init__(self, path=[], kwargs={}, treevars=[]): self.pb = [] self.tag = path[-1] self.world = path[0] @@ -47,10 +51,13 @@ def __init__(self, path=[], **kwargs): #self.label="%s\t" % tag self.label = "" self.via = None + self.treevars = treevars self.config() def config(self): + if 'domain_item' in self.treevars and self.tag[-1] != '.': + self.tag = ".".join([self.tag] + self.treevars['domain_item']) pass def process_proxy(self, via_service, redirect_factory): @@ -60,6 +67,13 @@ def __repr__(self): return("<{}({!r})>".format(self.__class__.__name__, self.tag)) +class SSHserviceAttribute(object): + def __init__(self, key, value, comment): + self.key=key + self.value=value + self.comment=comment + + class SSHservice(AbstractService): #This is to support redirection of hosts that are SCB monitored and behind restricted SSH terminal server gateways @@ -85,15 +99,32 @@ def config(self): 'dynamic': {} } + if 'domain_item' in self.treevars and self.tag[-1]!='.': + self.tag = ".".join([self.tag]+self.treevars['domain_item']) + + #IP address or hostname was not specified in YAML, we add one if 'hostname' not in map(lambda x: x.lower(), self.kwargs.keys()): - self.kwargs['HostName'] = self.tag + if 'site_id' in self.treevars and '_ipoffset' in self.kwargs: + log.debug("adding default hostname for y7 host %r" % self.tag) + self.kwargs['HostName'] = y7address(self.treevars['site_id'][0], self.kwargs['_ipoffset'], 6) + else: + log.debug("adding default HostName entry for SSHservice %r" % self.tag) + self.kwargs['HostName'] = self.tag for key, value in self.kwargs.items(): if key[0]!='_': #Blindly copy all the non meta settings self.pb.append("\t%s = %s" % (key, self.kwargs[key])) else: key=key.lower() if key == '_via': - self.via = value + #non relative FQDN + if value[-1]=='.': + self.via = value[:-1] + #a relative host in our subdomain + elif 'domain_item' in self.treevars: + self.via = ".".join([value]+self.treevars['domain_item']) + else: + raise NotImplementedError('Unknown via host', self.path, self.tag, value, self.treevars) + elif key == '_forward': #print self.kwargs[key] if 'local' in self.kwargs[key]: @@ -155,7 +186,15 @@ class SSHoverSCBservice(SSHservice): class URLservice(AbstractService): def config(self): if '_via' in self.kwargs: - self.via = self.kwargs['_via'] + value = self.kwargs['_via'] + #non relative FQDN + if value[-1]=='.': + self.via = value[:-1] + #a relative host in our subdomain + elif 'domain_item' in self.treevars: + self.via = ".".join([value]+self.treevars['domain_item']) + else: + raise NotImplementedError('Unknown via host', value, self.tag) self.url = self.kwargs['url'] self.original_url = self.kwargs['url'] @@ -183,7 +222,15 @@ def process_proxy(self, via_service, redirect_factory): class RDPservice(AbstractService): def config(self): if '_via' in self.kwargs: - self.via = self.kwargs['_via'] + value = self.kwargs['_via'] + #non relative FQDN + if value[-1]=='.': + self.via = value[:-1] + #a relative host in our subdomain + elif 'domain_item' in self.treevars: + self.via = ".".join([value]+self.treevars['domain_item']) + else: + raise NotImplementedError('Unknown via host', value, self.tag) def process_proxy(self, via_service, redirect_factory): self._original_full_address=self.kwargs["full address"] @@ -196,7 +243,15 @@ def process_proxy(self, via_service, redirect_factory): class VNCservice(AbstractService): def config(self): if '_via' in self.kwargs: - self.via = self.kwargs['_via'] + value = self.kwargs['_via'] + #non relative FQDN + if value[-1]=='.': + self.via = value[:-1] + #a relative host in our subdomain + elif 'domain_item' in self.treevars: + self.via = ".".join([value]+self.treevars['domain_item']) + else: + raise NotImplementedError('Unknown via host', value, self.tag) def process_proxy(self, via_service, redirect_factory): bindip, bindport = redirect_factory.allocate(via_service, self) @@ -230,8 +285,13 @@ def __init__(self): 'vnc': VNCservice } - def create(self, path, **kwargs): - return self.builders[kwargs['_type']](path, **kwargs) + def create(self, path, kwargs, emuserema): + treevars = inherit(get_ansible_treevars(emuserema.data, path)) + return self.builders[kwargs['_type']]( + path = path, + kwargs = kwargs, + treevars = treevars + ) def supported(self, obj): return obj['_type'] in self.builders.keys() diff --git a/emuserema/templates/openssh/openssh.j2 b/emuserema/templates/openssh/openssh.j2 new file mode 100644 index 0000000..7824f3e --- /dev/null +++ b/emuserema/templates/openssh/openssh.j2 @@ -0,0 +1,11 @@ +# Services for world {{world.name}} +{% for service in services %} +### +### {{ service.tag }} +### + +Host {{ service.tag }} +{%-for pb in service.pb%} +{{pb-}} +{% endfor %} +{% endfor %} diff --git a/emuserema/utils.py b/emuserema/utils.py index 1e24082..a5bc696 100644 --- a/emuserema/utils.py +++ b/emuserema/utils.py @@ -3,9 +3,57 @@ from os import walk, path from shutil import rmtree from os import listdir, unlink +from netaddr import IPAddress + +IPv4_prefix = '44.128.0.0' +IPv6_prefix = 'fd4d:4045:e5e8::' #from __future__ import print_function +def y7address(site_id, offset, family): + if family == 6: + return str(IPAddress(IPv6_prefix) + site_id * 2**72 + int(str(offset), 16)) + elif family == 4: + return str(IPAddress(IPv4_prefix) + site_id * 1024 + offset) + +def get_ansible_treevars(data, data_path): + """Acquires the _ansible_treevars items from the main data dict""" + if len(data_path) > 1: + yield from get_ansible_treevars(data[data_path[0]], data_path[1:]) + #elif isinstance(data.get(data_path[0]), dict): + yield data.get(data_path[0]).get('_ansible_treevars') + +def inherit(ansible_treevars_items): + """Renders the actual inheritance that it gets back from get_ansible_treevars()""" + retval_dict = dict() + for ansible_treevars_item in ansible_treevars_items: + #a leaf with a value + if ansible_treevars_item: + for ansible_treevars_dict in ansible_treevars_item: + for key, value in ansible_treevars_dict.items(): + if isinstance(value, str): + if key not in retval_dict: + retval_dict[key] = [value] + else: + retval_dict[key].append(value) + #retval_dict[key].insert(0, value) + elif isinstance(value, int): + if key not in retval_dict: + retval_dict[key] = [value] + else: + retval_dict[key].append(value) + #retval_dict[key].insert(0, value) + #currently we're only appending the list items, no nested lists are supported yet + elif isinstance(value, list): + if key not in retval_dict: + retval_dict[key] = value + else: + retval_dict[key] + value + #retval_dict[key].insert(0, value) + else: + raise NotImplementedError( + '_ansible_treevars item with a type of %s is unsupported.' % type(value)) + return retval_dict def cleanup_dir(target): for root, dirs, files in walk(target, topdown=False): diff --git a/emuserema/yamlloader.py b/emuserema/yamlloader.py index b7e95b4..e7b1402 100644 --- a/emuserema/yamlloader.py +++ b/emuserema/yamlloader.py @@ -4,13 +4,57 @@ #from ruamel.yaml import YAML, version_info, dump, YAMLError, SafeLoader, RoundTripLoader #from ruamel.yaml import YAML, YAMLError, dump, SafeLoader import ruamel.yaml -from os.path import isfile -from jinja2 import Template, ChoiceLoader, PackageLoader, FileSystemLoader, Environment -from os import environ, uname - +from os.path import isfile, splitext, isdir +from jinja2 import ChoiceLoader, PackageLoader, FileSystemLoader, Environment +from os import environ, uname, listdir, access, X_OK +from subprocess import check_output +from toml import loads as toml_loads +from json import loads as json_loads +from csv import DictReader +from io import StringIO +import re +import logging + +log = logging.getLogger(__name__) + + +kv_regex = re.compile('^\s*([a-zA-Z_.]+)[\t :=]+(.*)') + +def csv_loads(s: str) -> dict: + """A simple interface to csvloader + + Args: + s (str): Input file as a string + + Returns: + str: Parsed CSV as dict + """ + + reader = DictReader(s.split("\n")) + rv={} + for row in reader: + #field with name 'nodename' is the dict key + rv[row.pop('nodename')] = row + return rv + +def get_file_or_exec_contents(path: str) -> str: + """gets file contents or executes it for its stdout + if file is executable + + Args: + path (str): path to file or executable + + Returns: + str: contents of file or stdout of executable ran + """ + if access(path, X_OK): + log.debug("file '%s' is executable, it will be executed and its output processed as file extensions suggests" % path) + return check_output(args=[path], shell=False).decode('utf-8') + with open(path, 'r') as infile: + return infile.read() class EmuseremaRelativeSafeLoader(ruamel.yaml.SafeLoader): - def __init__(self, definitions_directory=None): + def __init__(self, definitions_directory= None): self._definitions_directory = definitions_directory if ruamel.yaml.version_info < (0, 15): @@ -28,8 +72,8 @@ def construct_python_tuple(self, cons, node): # adapted from http://code.activestate.com/recipes/577613-yaml-include-support/ def yaml_include(self, cons, node): - path = node.value - if isfile(self._definitions_directory + '/' +path + '.j2'): + path = node.value + if isfile(self._definitions_directory + '/' + path + '.j2'): templatedata = self.loadyaml(path + '.j2.yaml') if not templatedata: templatedata = {} @@ -42,7 +86,7 @@ def yaml_include(self, cons, node): yaml = ruamel.yaml.YAML(typ=y.typ, pure=y.pure) yaml.composer.anchors = cons.composer.anchors return yaml.load(template.render(templatedata)) - with open(self._definitions_directory + '/' +path, 'r') as infile: + with open(self._definitions_directory + '/' + path, 'r') as infile: if ruamel.yaml.version_info < (0, 15): #yaml.composer.anchors = cons.composer.anchors return ruamel.yaml.safe_load(infile) @@ -51,6 +95,56 @@ def yaml_include(self, cons, node): yaml = ruamel.yaml.YAML(typ=y.typ, pure=y.pure) yaml.composer.anchors = cons.composer.anchors return yaml.load(infile) + + + def file_to_dict(self, yaml_cons, path): + name, ext = splitext(path) + rv = {} + if ext == '.toml': + log.debug('loading TOML format data from file %r' % path) + return toml_loads(s = get_file_or_exec_contents(path=path)) + elif ext == '.json': + log.debug('loading JSON format data from file %r' % path) + return json_loads(s = get_file_or_exec_contents(path=path)) + elif ext == '.yaml': + log.debug('loading YAML format data from file %r' % path) + y = yaml_cons.loader + yaml = ruamel.yaml.YAML(typ=y.typ, pure=y.pure) + yaml.composer.anchors = yaml_cons.composer.anchors + return yaml.load(get_file_or_exec_contents(path=path)) + elif ext == '.csv': + log.debug('loading CSV format data from file %r' % path) + return csv_loads(s = get_file_or_exec_contents(path=path)) + else: + log.warning("extension of file '%s' is unknown and will be read as a simple key-value store" % path) + with open(path, 'r') as file: + for index, line in enumerate(file): + match = kv_regex.match(line) + if (match): + group = match.groups() + if len(group) == 2: + rv[group[0]] = group[1] + else: + log.debug("simple key-value store entry in file '%s', line #%d did not match expected kv-regex: %r" % (path, index, line)) + else: + log.debug("simple key-value store entry in file '%s', line #%d did not match expected kv-regex: %r" % (path, index, line)) + return rv + + def file_include(self, cons, node): + path = node.value + if isdir(path): + rv = {} + for item in listdir(path): + if isfile(path + '/' + item): + name, ext = splitext(item) + rv[name] = self.file_to_dict(cons, path + '/' + item) + elif isdir(path): + log.debug("file_include() recursively descends into directory %r", path+'/'+item) + node.value = path + '/' + item + rv[item] = self.file_include(cons, node) + return rv + else: + return self.file_to_dict(cons, path + '/' + item) def loadyaml(self, filename): yamldata = None @@ -70,13 +164,12 @@ def loadyaml(self, filename): yamldata = self.yaml.load(stream, Loader=ruamel.yaml.Loader) else: yamldata = self.yaml.load(stream) - # print(dump(t, default_flow_style=False)) - # print type(t) except ruamel.yaml.YAMLError as exc: print(exc) return yamldata + class EmuseremaYamlLoader(object): def __init__(self, definitions_directory=None): self._definitions_directory = definitions_directory @@ -114,9 +207,11 @@ def my_compose_document(self): if ruamel.yaml.version_info < (0, 15): self.yaml.add_constructor('!include', loader.yaml_include) + self.yaml.add_constructor('!includedir', loader.file_include) self.yaml.add_constructor('tag:yaml.org,2002:python/tuple', loader.construct_python_tuple) else: self.yaml.Constructor.add_constructor('!include', loader.yaml_include) + self.yaml.Constructor.add_constructor('!includedir', loader.file_include) self.yaml.Constructor.add_constructor('tag:yaml.org,2002:python/tuple', loader.construct_python_tuple) def loadyaml(self, filename): @@ -137,13 +232,10 @@ def loadyaml(self, filename): yamldata = self.yaml.load(stream, Loader=ruamel.yaml.Loader) else: yamldata = self.yaml.load(stream) - # print(dump(t, default_flow_style=False)) - # print type(t) except ruamel.yaml.YAMLError as exc: print(exc) if not yamldata: - print(yamldata) raise ValueError("Failed to parse YAML Data") return yamldata diff --git a/setup.py b/setup.py index 097a75e..1f7d21c 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup name = 'emuserema' -version = '0.2' +version = '0.2.3' description = 'EMUSEREMA Multiprotocol Session and Redirect Manager' @@ -33,10 +33,10 @@ def readme(): package_data = { 'emuserema': ['templates/*'] }, - install_requires=[ - 'ruamel.yaml', - 'Jinja2', - ], +# install_requires=[ +# 'ruamel.yaml', +# 'Jinja2', +# ], scripts=[ 'bin/viemuserema', ],