caasp

#!/usr/bin/env python3

import argparse
import glob as gb
import json
import logging
import os
import os.path as op
import re
import readline
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import traceback
from cmd import Cmd
from contextlib import contextmanager
from datetime import datetime, timedelta
from tempfile import NamedTemporaryFile
from threading import Timer

if sys.version_info <= (3, 0):
    sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n")
    sys.exit(1)

CURR_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))

# default prefix for the VMs
VMS_PREFIX = 'caasp'

# default name for the admin node and nodes
VM_ADMIN_SUFFIX = '-admin'
VM_NODES_REGEX_SUFFIX = '-node.*'

# caaspctl executable in the VMs, and some important commands
CAASPCTL = '/tmp/caasp/caaspctl'
CAASPCTL_GEN_KUBECONFIG = 'kubeconfig'
CAASPCTL_GET_PILLAR = 'pillar get {key} {where}'
CAASPCTL_SET_PILLAR = 'pillar set {key} {value}'
CAASPCTL_SALT_SYNC = 'salt sync'
CAASPCTL_MINIONS_ACCEPT = 'minions accept {num}'
CAASPCTL_ENABLE_RW = 'rw 1'

CONTAINER_START_TIMEOUT = 300

# timeout for "virsh snapshot-create"
VIRSH_SNAPSHOT_TIMEOUT = 9000

# where the pillar.lst file is copied to
PILLARS_FILE_REM = '/tmp/caasp/pillar.lst'

# default terraform definition and state file
DEFAULT_TF = 'terraform.tf'
DEFAULT_TF_STATE = 'terraform.tfstate'
DEFAULT_TF_OUTPUTS = 'terraform-outputs.tf'

# default terraform profiles directory
DEFAULT_TFVARS_DIR = 'terraform'
DEFAULT_TF_DEVEL_PROFILE = os.path.join(DEFAULT_TFVARS_DIR, 'profile-devel.tf')

# default Terraform parallelist
DEFAULT_TF_PAR = 1

# directory where the kubeconfig can be fodun in the remote machines
DEFAULT_KUBECONFIG_REM = '/root/.kube/config'

# directory where Salt/manifests is copied to (in the Admin Node)
DEFAULT_SALT_REM = '/usr/share/salt/kubernetes/'
DEFAULT_MANIFESTS_REM = '/usr/share/caasp-container-manifests/'

# the path to admin-node-setup.sh
ADMIN_NODE_SETUP = '/usr/share/caasp-container-manifests/admin-node-setup.sh'

# default dirs for local and remote RPMs (for installations)
DEFAULT_RPMS_SRC = 'rpms'
DEFAULT_RPMS_DST = '/tmp/caasp-rpms'

# rsync arguments
# Notes: follows symlinks
DEFAULT_RSYNC_ARGS = "-avz -L --delete --delete-after --force"

# directory for images configuration
DEFAULT_IMGAGES_CFG_DIR = 'configs/images'

# default environments directory
DEFAULT_ENV_DIR = 'configs/environments'

# the default environment
DEFAULT_ENV = 'localhost'

# logging format
# https://docs.python.org/2/library/logging.html#logrecord-attributes
FORMAT = '# %(asctime)s [%(levelname)s] %(message)s'

CAASP_USER_DIR = os.path.expanduser('~/.caasp')
CAASP_LAST_RUN_ARGS = os.path.join(CAASP_USER_DIR, 'caasp.yaml')

# default ssh arguments
SSH_ARGS = """\
    -oStrictHostKeyChecking=no \
    -oUserKnownHostsFile=/dev/null \
    -oConnectTimeout=10 \
    -oLogLevel=ERROR \
"""

EXCLUDE_ARGS = """ \
    --exclude='*.tfstate*' \
    --exclude='.git*' \
    --exclude='.tox' \
    --exclude='README.md' \
    --exclude='*.sublime*' \
    --exclude='__pycache__' \
    --exclude='.idea' \
    --exclude='Jenkins*' \
    --exclude='.terraform' \
    --exclude='test_*.py' \
    --exclude='__init__.py' \
    --exclude='.pyc' \
"""

EXCLUDE_BINS_ARGS = """ \
    --exclude='*.tgz' \
    --exclude='*.qcow2' \
    --exclude='*.rpm' \
    --exclude='docker-image*.tar.gz' \
"""

# default ssh password
DEFAULT_SSH_PASS = "linux"

# the version for this
VERSION = "0.1"

# default copies to perform
DEFAULT_COPIES = {
    'admin': [
        ('resources/common/', '/tmp/caasp/'),
        ('resources/admin/', '/tmp/caasp/admin/')
    ],
    'nodes': [
        ('resources/common/', '/tmp/caasp/'),
        ('resources/nodes/', '/tmp/caasp/nodes/')
    ],
}

# terraform variable used for pointing to libvirt
TERRAFORM_LIBVIRT_URI_VAR = 'libvirt_uri'

# extra variables we want to save in the Terraform state file
TFSTATE_OUTPUTS = [
    TERRAFORM_LIBVIRT_URI_VAR,
    'img',
    'img_pool',
    'prefix',
    'nodes_count'
]

# variables in te terraform file for setting salt and manifests copies
TFVAR_SALT_DIR = 'salt_dir'
TFVAR_MANIFESTS_DIR = 'manifests_dir'

# default orchestration to run
DEFAULT_ORCHESTRATION = 'kubernetes'

# delay _after_ creating snapshot
DEFAULT_SNAP_DELAY = 30

# delay after creating the VMs
DEFAULT_TERRAFORM_APPLY_DELAY = 60

DATE_FMT = '%Y-%m-%d %H:%M:%S'

# commands to run in nodes before runing an orchestration
ORCH_PREPARE_NODES_SHELL_CMDS = [
    # it seems the original hostname is saved in /etc/hostname
    # so we must fix the hostname: it could be broken by libvirt's dnsmasq
    'cat /etc/hostname | xargs hostname',

    # synchronize the times in all the VMs by setting the
    # VMs' times here.
    'systemctl stop ntpd',
    'timedatectl set-ntp false',

    # it seems this is enough, but maybe we must
    # use virsh_sync_time)
    '/sbin/hwclock --hctosys',
    #"timedatectl set-time '{date}'",

    "date"
]

# caaspctl commands to run a orchestration
# there will be some replacements like {orch} and {orch_args}
ORCH_PREPARE_CAASPCTL_CMDS = [
    'pillar flush',
    'pillar load ' + PILLARS_FILE_REM,
    'pillar guess_dynamic'
]

K8S_DASHBOARD_HELM_CHART = 'stable/kubernetes-dashboard'
K8S_DASHBOARD_PORT = 8443
K8S_DASHBOARD_NAMESPACE = 'kube-system'
K8S_DASHBOARD_USER = 'cluster-admin'

# add the 'ca' and 'admin' Salt minions to the number of keys to wait for
SALT_KEYS_EXTRA_WAIT = 2

# directory for docker registry certificates
REGISTRY_CERTS_DIR = os.path.join(CURR_DIR, 'registry')

# configuration used for generating the certificates
REGISTRY_CERTS_CFG = '''
[req]
prompt                      = no
req_extensions              = v3_req
distinguished_name          = req_distinguished_name
[ req_distinguished_name ]
countryName                 = US
stateOrProvinceName         = Somewhere
localityName                = Somewhere
organizationName            = SUSE
commonName                  = {hostname}
[ v3_req ]
basicConstraints            = CA:FALSE
keyUsage                    = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName              = @alt_names
[alt_names]
DNS.1                       = {hostname}.local
DNS.2                       = {hostname}
DNS.3                       = {registry_hostname}
IP.1                        = {ip}
'''

REGISTRY_PORT = 5000
REGISTRY_DOCKER_IMAGE = 'registry'
REGISTRY_DOCKER_IMAGE_TAG = 2

SCRIPT_REGISTRY_CERTS = '''
openssl genrsa -out "{ca_key}" 4096
openssl req -new -x509 -days 1826 -key "{ca_key}" -out "{ca_crt}" -config "{cnf}"

openssl genrsa -out "{reg_key}" 2048
openssl req -new -key "{reg_key}" -out "{reg_csr}" -subj "/CN={hostname}" -config "{cnf}"
openssl x509 -req -CA "{ca_crt}" -CAkey "{ca_key}" -CAcreateserial -in "{reg_csr}" -out "{reg_crt}" -days 365 -extensions v3_req -extfile "{cnf}"
'''

REGISTRY_DOCKER_CMD = '''
docker run -d -p {registry_port}:{registry_port} \
  --restart=always --name registry \
  -v {registry_certs_dir}:/certs \
  -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/{basename_reg_crt} \
  -e REGISTRY_HTTP_TLS_KEY=/certs/{basename_reg_key} \
  -e REGISTRY_STORAGE_DELETE_ENABLED=true \
  {container_registry}:{container_registry_tag}
'''

# RC files that are automatically loaded on startup
# can be used for doing some actions or setting default values (ie, 'devel enable')
CAASP_RC_FILES = [
    '.caasp.rc',
    '.caasprc',
    'caasp.rc',
    'caasprc',
    '.caasp.rc.local',
    '.caasprc.local',
    'caasp.rc.local',
    'caasprc.local'
]

CAASP_RC_FILES_ABS = [
    '~/.caasp.rc',
    '~/.caasprc',
    '~/caasp.rc',
    '~/caasprc'
]

ENV_STAGE_RC_FILES = [
    '{stage}.rc',
    '{stage}.rc.local',
]

ENV_TFVARS_FILES = [
    'preset-local.tfvars',
    'preset.tfvars',
    'preset-local.tfvars.local',
    'preset.tfvars.local'
]

# directories where the Salt code and the manifests could be (relative to CURR_DIR)
MAYBE_DIRS_SALT = [
    './salt',
    '../salt',
    './k8s-salt',
    '../k8s-salt'
]

MAYBE_DIRS_MANIFESTS = [
    './caasp-container-manifests',
    '../caasp-container-manifests',
    './manifests',
    '../manifests'
]

# colors definitions
COLORS = {
    'HEADER': '\033[95m',
    'BLUE': '\033[94m',
    'GREEN': '\033[92m',
    'RED': '\033[91m',
    'ENDC': '\033[0m',
    'BOLD': '\033[1m',
    'UNDERLINE': '\033[4m'
}

PROMPT_COLORS = ['UNDERLINE', 'BLUE']

####################################################################
# Command line arguments
####################################################################

parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description='A utility for creating/managing a CaaSP cluster with Terraform and libvirt',
    epilog='''
Make sure you have Terraform installed and a valid libvirt instance running.

Then you can start by creating a cluster with 'cluster create'.
''')

parser.add_argument('args',
                    nargs=argparse.REMAINDER,
                    help=';-separated list of commands to run (get more info with "help")')

env_group = parser.add_argument_group(
    title='Environment and images',
    description='Environments for running the cluster')
env_group.add_argument('--env',
                       dest='env',
                       metavar='ENVIRONMENT',
                       default=DEFAULT_ENV,
                       help='environment to use')
env_group.add_argument('--env-dir',
                       dest='env_dir',
                       metavar='DIR',
                       default=DEFAULT_ENV_DIR,
                       help='environments directory')
env_group.add_argument('--last-env',
                       dest='last_env',
                       action='store_true',
                       default=False,
                       help='use the same environment used the last time')

img_group = parser.add_argument_group(
    title='Images',
    description='Images to use in the cluster')
img_group.add_argument('--image',
                       dest='image',
                       metavar='IMAGE',
                       default='',
                       help='image channel to use')

vms_group = parser.add_argument_group(
    title='Virtual machines',
    description='Arguments related to the virtual machines')
vms_group.add_argument('--vm-prefix',
                       dest='vm_prefix',
                       metavar='PREFIX',
                       default=VMS_PREFIX,
                       help='some distinctive prefix for the libvirt machines')
vms_group.add_argument('--vm-admin',
                       dest='vm_admin',
                       metavar='VM',
                       help='default name for the admin node (default: <PREFIX> + "-admin")')
vms_group.add_argument('--vm-nodes-regex',
                       dest='vm_nodes_regex',
                       metavar='REGEX',
                       help='regex for the recognizing nodes in the libvirt machines (default: <PREFIX> + "-node.*")')
vms_group.add_argument('--vm-caaspctl',
                       dest='vm_caaspctl',
                       metavar='EXE',
                       default=CAASPCTL,
                       help='caaspctl path in nodes')

tf_group = parser.add_argument_group(
    title='Terraform',
    description='Terraform low-level configuration')
tf_group.add_argument('--tf',
                      dest='tf',
                      metavar='TFFILE',
                      help='terraform tf file (default: <ENV>/terraform.tf)')
tf_group.add_argument('--tf-state',
                      dest='tf_state',
                      metavar='FILE',
                      help='Terraform state file (by default, will create a one in the environment directory)')
tf_group.add_argument('--tfvars-dir',
                      dest='tfvars_dir',
                      metavar='FILE',
                      default=DEFAULT_TFVARS_DIR,
                      help='default directory for looking for Terraform variables (.tfvars) files')
tf_group.add_argument('--tfvar',
                      dest='tfvar',
                      action='append',
                      metavar='VAR=VALUE',
                      default=[],
                      help='add a terraform variable.')
tf_group.add_argument('--tfvars',
                      dest='tfvars',
                      action='append',
                      metavar='FILE',
                      default=[],
                      help='add a terraform variables file.')
tf_group.add_argument('--tf-par',
                      dest='tf_par',
                      default=DEFAULT_TF_PAR,
                      metavar='NUM',
                      help='Terraform parallelism')

prio_group = parser.add_argument_group(
    title='Privileges/passwords',
    description='Password and privileges needed for running things')

prio_group.add_argument('--no-sudo-virsh',
                        dest='sudo_virsh',
                        default=True,
                        action='store_false',
                        help='do NOT use "sudo" for virsh')
prio_group.add_argument('--sudo-password',
                        dest='sudo_password',
                        metavar='PASS',
                        default='',
                        help='password for sudo commands')
prio_group.add_argument('--ssh-password',
                        dest='ssh_password',
                        metavar='PASS',
                        default=DEFAULT_SSH_PASS,
                        help='password for ssh')
prio_group.add_argument('--sshpass',
                        dest='sshpass',
                        metavar='EXE',
                        default='sshpass',
                        help='sshpass executable when using --ssh-password')
prio_group.add_argument('--ssh-multiplex',
                        dest='ssh_multiplex',
                        default=False,
                        action='store_true',
                        help='multiplex sessions on a ssh connection')

devel_group = parser.add_argument_group(
    title='Development options',
    description='Some options for developers')

devel_group.add_argument('--copy-salt-code',
                         dest='copy_salt_code',
                         default=False,
                         action='store_true',
                         help='copy the Salt code to the Admin Node')
devel_group.add_argument('--salt-src-dir',
                         dest='salt_dir',
                         metavar='DIR',
                         default='',
                         help='default Salt sources directory')
devel_group.add_argument('--salt-src-branch',
                         dest='salt_branch',
                         metavar='NAME',
                         default=None,
                         help='Salt code branch to copy to the Admin Node (NOTE: the local worktree will be used when specifing the current branch)')
devel_group.add_argument('--copy-manifests',
                         dest='copy_manifests',
                         default=False,
                         action='store_true',
                         help='copy the manifests to the Admin Node')
devel_group.add_argument('--manifests-dir',
                         dest='manifests_dir',
                         metavar='DIR',
                         default='',
                         help='default manifests directory')
devel_group.add_argument('--manifest-branch',
                         dest='manifests_branch',
                         metavar='NAME',
                         default=None,
                         help='manifests branch to copy to the Admin Node (NOTE: the local worktree will be used when specifing the current branch)')
devel_group.add_argument('--tf-copies',
                         dest='tf_copies',
                         default=False,
                         action='store_true',
                         help='perform an initial copy of the Salt/manifests with Terraform (copies will be performed anyway before orchestrations when "devel" mode is enabled)')
devel_group.add_argument('--tf-devel-profile',
                         dest='tf_devel_profile',
                         metavar='TF_FILE',
                         default=DEFAULT_TF_DEVEL_PROFILE,
                         help='Terraform development profile (.tf) file')

reg_group = parser.add_argument_group(
    title='Registry',
    description='Local Docker registry')

reg_group.add_argument('--reg-cert-dir',
                       dest='reg_certs_dir',
                       metavar='DIR',
                       default=REGISTRY_CERTS_DIR,
                       help='directory for the regstry certificates')
reg_group.add_argument('--reg-hostname',
                       dest='reg_hostname',
                       metavar='HOSTNAME',
                       default='registry.com',
                       help='exported registry name')


commands_group = parser.add_argument_group(
    title='Commands processing',
    description='How commands are processed from command line or from the loop')

commands_group.add_argument('--loop',
                            dest='loop',
                            default=False,
                            action='store_true',
                            help='the loop is only started when no commands are provided in command line. With this flag, the loop is started even when commands are provided as arguments')
commands_group.add_argument('--commands-pre',
                            dest='commands_pre',
                            default=False,
                            action='store_true',
                            help='process commands from arguments BEFORE processing scripts')
commands_group.add_argument('--exit-on-error',
                            dest='exit_on_err',
                            default=False,
                            action='store_true',
                            help='exit on any errors instead of just printing the error message')
commands_group.add_argument('--skip-rc-files',
                            dest='skip_rc_files',
                            default=False,
                            action='store_true',
                            help='do not load automatically the RC files')


script_group = parser.add_argument_group(
    title='Loading commands from scripts')

script_group.add_argument('--script',
                          dest='script',
                          action='append',
                          metavar='FILE',
                          default=[],
                          help='read commands from a script(s). can be provided multiple times for loading scripts in order.')
script_group.add_argument('--script-only',
                          dest='script_only',
                          default=True,
                          action='store_true',
                          help='quit after running the script')
script_group.add_argument('--script-begin',
                          dest='script_begin',
                          metavar='STAGE',
                          default='',
                          help='process the script after stage <STAGE>')
script_group.add_argument('--script-stage',
                          action='append',
                          dest='script_stage',
                          metavar='STAGE',
                          default=[],
                          help='process only the stage <STAGE> in the script (can be provided multiple times)')

files_group = parser.add_argument_group(
    title='Copies and Files',
    description='Local and remote paths necessary for performing copies from/to VMs')

files_group.add_argument('--kubeconfig',
                         dest='kubeconfig',
                         metavar='FILE',
                         default=os.environ.get('KUBECONFIG', 'kubeconfig'),
                         help='local kubeconfig (downloaded from the Admin Node after orchestrating)')
files_group.add_argument('--kubeconfig-remote',
                         dest='DEFAULT_KUBECONFIG_REM',
                         metavar='FILE',
                         default=DEFAULT_KUBECONFIG_REM,
                         help='the kubeconfig in the nodes of the cluster')
files_group.add_argument('--rpms',
                         dest='default_rpms_src',
                         metavar='DIR',
                         default=DEFAULT_RPMS_SRC,
                         help='default local directory for RPMs')
files_group.add_argument('--rpms-remote',
                         dest='default_rpms_dst',
                         metavar='DIR',
                         default=DEFAULT_RPMS_DST,
                         help='default remote directory for RPMs')

verbose_group = parser.add_argument_group(
    title='Logging/verbosity')

verbose_group.add_argument('--debug',
                           dest='debug',
                           default=False,
                           action='store_true',
                           help='use debug logging')

args = parser.parse_args()


loglevel = (logging.DEBUG if args.debug else logging.INFO)
log = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stderr,
                    format=FORMAT,
                    level=loglevel)

try:
    import coloredlogs

    # By default the install() function installs a handler on the root logger,
    # this means that log messages from your code and log messages from the
    # libraries that you use will all show up on the terminal.
    coloredlogs.install(fmt=FORMAT, level=loglevel)
except ImportError:
    log.debug('"coloredlogs" not available')

readline.set_completer_delims(' \t\n')

if args.ssh_multiplex:
    SSH_ARGS += " -oControlmaster=auto -oControlpath='/tmp/ssh-%r@%h:%p'"


####################################################################
# Aux
####################################################################


def str2bool(v):
    return v.lower() in ("yes", "true", "t", "1")


def replace_pattern(pat, replacer, line):
    for t in re.finditer(pat, line):
        txt = str(t.group())
        out = replacer(txt)
        line = line.replace(txt, out)
    return line


def reset_loglevel(level):
    '''
    Usage: reset_loglevel(logging.DEBUG)
    '''
    log.setLevel(level)
    for handler in log.handlers:
        handler.setLevel(level)


def get_local_ip_for(remote_ip='10.255.255.255'):
    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        # doesn't even have to be reachable
        s.connect((remote_ip, 1))
        IP = s.getsockname()[0]
    except:
        IP = '127.0.0.1'
    finally:
        s.close()
    return IP


def load_caasp_last_run():
    if not os.path.exists(CAASP_USER_DIR):
        log.debug('Creating %s', CAASP_USER_DIR)
        os.makedirs(CAASP_USER_DIR)

    if os.path.exists(CAASP_LAST_RUN_ARGS):
        import yaml

        log.debug('Loading %s', CAASP_LAST_RUN_ARGS)
        with open(CAASP_LAST_RUN_ARGS, 'r') as stream:
            try:
                return yaml.load(stream)
            except yaml.YAMLError as exc:
                log.error('could not load saved args: %s',exc)


def save_caasp_last_run(saved_args):
    if not os.path.exists(CAASP_USER_DIR):
        log.debug('Creating %s', CAASP_USER_DIR)
        os.makedirs(CAASP_USER_DIR)

    import yaml
    with open(CAASP_LAST_RUN_ARGS, 'w') as outfile:
        log.debug('Saving %s', CAASP_LAST_RUN_ARGS)
        yaml.dump(saved_args, outfile, default_flow_style=False)


def value_to_native(val):
    if isinstance(val, str):
        try:
            return int(val)
        except ValueError:
            pass

        if val.lower() in ["true", "yes", "on"]:
            return True
        elif val.lower() in ["false", "no", "off"]:
            return False

        val = os.path.expandvars(val)

        # in case it is a quoted string, remove them
        if val[0] in ['\'', '"']:
            return val[1:-1]

    return val


def which(program):
    def is_exe(fpath):
        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)

    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

    return None


def get_regular_container(name):
    for line in run('docker ps'):
        fields = re.split('\s{2,}', line.strip())
        cid = fields[0]
        cname = fields[-1]
        if name in cname:
            return cid


def notify(body='', summary='CaaSP', icon=None):
    '''
    (Try to) Send a desktop notification.

    Checkout the number of icons in /usr/share/icons/gnome/32x32/actions
    ie, 'up', 'down', 'start', 'finish'
    '''
    cmd = ['notify-send']
    cmd += ['--app-name=caasp']
    if icon:
        cmd += ['--icon=' + icon]

    cmd += [summary, body]
    try:
        subprocess.call(cmd)
    except Exception as e:
        log.debug('could not send notification: %s', e)


def expandvars(path):
    return re.sub(r'(?<!\\)\$[A-Za-z_][A-Za-z0-9_]*', '', os.path.expandvars(path))


def get_assign_from_str(line):
    res = {}

    if len(line) == 0:
        return res

    try:
        assert(isinstance(line, str))
        for assign in line.split(','):
            var = assign.split('=')
            variable = var[0].strip()
            value = var[1].strip()
            value = value_to_native(value)

            res[variable] = value
    except IndexError as e:
        log.debug('could not parse assignments in "%s": %s', line, e)

    return res


def create_link(orig, dest):
    ''' (re)create a symbolik link orig->dest '''
    if os.path.exists(orig):
        if os.path.islink(orig):
            log.debug('os: removing previous symbolic link %s', orig)
            os.remove(orig)

    log.debug('os: creating symbolic link from %s-> %s', orig, dest)
    try:
        os.symlink(dest, orig)
    except FileExistsError:
        log.debug('os: symbolic link already exists')


def on_color(color, txt):
    res = ''
    if isinstance(color, list):
        res += ''.join([COLORS[x] for x in color])
    else:
        res += COLORS[color]

    return res + txt + COLORS['ENDC']


def prompt(txt):
    return on_color(PROMPT_COLORS, '{} >'.format(txt)) + ' '


def is_ip(name):
    try:
        socket.inet_aton(name)
        return True
    except socket.error:
        return False


class cd:
    """Context manager for changing the current working directory"""

    def __init__(self, newPath):
        self.newPath = os.path.expanduser(newPath)

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, etype, value, traceback):
        os.chdir(self.savedPath)


def run(cmd, sudo=False, password=None, timeout=None):
    ''' Execute a command, yielding the stdout lines '''
    assert(isinstance(cmd, str))

    if sudo:
        if password:
            cmd = "echo %s | sudo -S %s" % (password, cmd)
        else:
            cmd = "sudo -S %s" % (cmd)

    log.debug('exec: running "%s"', cmd)
    popen = subprocess.Popen(
        cmd, shell=True, stdout=subprocess.PIPE, universal_newlines=True)
    if timeout:
        timer = Timer(timeout, popen.kill)

    try:
        if timeout:
            timer.start()

        for stdout_line in iter(popen.stdout.readline, ""):
            yield stdout_line

        popen.stdout.close()
        return_code = popen.wait()
        if return_code:
            raise subprocess.CalledProcessError(return_code, cmd)
    finally:
        if timeout:
            timer.cancel()


def run_interactive(cmd, sudo=False, password=None):
    ''' Execute an interactive command, returning the `retcode` '''
    assert(isinstance(cmd, str))

    if sudo:
        cmd = "echo %s | sudo -S %s" % (password, cmd)

    log.debug('exec: starting interactive command "%s"', cmd)
    return subprocess.call(cmd, shell=True)


def print_iterator(it, **kwargs):
    if it:
        for line in it:
            print(line, end='', **kwargs)


####################################################################
# Terraform
####################################################################


class TerraformError(Exception):
    pass


class InvalidMachineError(Exception):
    pass


def get_work_dir():
    if args.env:
        return os.path.abspath(os.path.join(args.env_dir, args.env))
    else:
        return os.path.abspath(args.env_dir)


def get_tfstate_filename(tfstate=None):
    if tfstate:
        return os.path.abspath(tfstate)

    if args.tf_state:
        return os.path.abspath(args.tf_state)

    return os.path.abspath(os.path.join(get_work_dir(), DEFAULT_TF_STATE))


def get_tfstatex_filename(tfstate=None):
    return get_tfstate_filename(tfstate) + 'x'


def tf_load_state(filename=None):
    '''
    Load the Terraform ".state" file for getting IP addresses
    for the VMs.
    '''
    res = {}

    filename = filename or get_tfstate_filename()

    if not os.path.exists(filename):
        log.warning('state: "%s" does not exist', filename)
        return res

    with open(filename, "r") as tfstate:
        try:
            json_data = tfstate.read()
            data = json.loads(json_data)
        except ValueError as e:
            log.error('state: parsing tfstate file: %s', e)
            sys.exit(1)

        for resource_name, resource_contents in data['modules'][0]['resources'].items():
            if re.search('libvirt_domain\..*', resource_name):
                try:
                    attrs = resource_contents['primary']['attributes']

                    name = attrs['name']
                    ipaddr = attrs['network_interface.0.addresses.0']

                    # if args.regex and not re.search(args.regex, name):
                    #    continue
                    # else:
                    res[name] = ipaddr
                except KeyError as e:
                    log.warning(
                        'state: cannot parse IP address for "%s" from %s: "%s" field not found', resource_name, filename, e)
                    res[name] = None

    return res


def tf_load_statex(vars={}, vars_files=[], tfstate=None):
    '''
    Get things stuff from the statex file.
    '''
    res = {
        'vars': vars,
        'vars_files': list(vars_files)
    }

    tfstatex_filename = get_tfstatex_filename(tfstate)

    if os.path.exists(tfstatex_filename):
        log.debug('statex: trying to load from "%s"', tfstatex_filename)
        with open(tfstatex_filename, 'r') as tfstatex:
            try:
                res_stored = json.load(tfstatex)
            except ValueError as e:
                log.error('statex: could not decode the JSON in %s: %s',
                          tfstatex_filename, e)
            else:
                log.debug('statex: file loaded successfully')
                res['vars'].update(res_stored['vars'])
                res['vars_files'] = list(res_stored['vars_files'])
    else:
        log.debug('statex: no file found at "%s"', tfstatex_filename)

    return res


def tf_save_statex(vars={}, vars_files=[], tfstate=None):
    '''
    Use a tfstatex file for saving thigs like:

      * the vars
      * vars_files

    we used for creating the cluster
    '''
    tfstatex_filename = get_tfstatex_filename(tfstate)

    log.debug('statex: saving file as %s', tfstatex_filename)
    with open(tfstatex_filename, 'w') as tfstatex:
        tfstatex_contents = {
            'vars': vars,
            'vars_files': list(vars_files)
        }
        json.dump(tfstatex_contents, tfstatex, indent=2)


def tf_get_extra_output(name, tf_state=None):
    '''
    Get an "output" variable from terraform
    '''
    res = ''
    cmd = 'terraform output'
    cmd += ' -state={}'.format(tf_state or get_tfstate_filename())
    cmd += ' ' + name

    try:
        log.debug('terraform: getting Terraform variable "%s"', name)
        res = subprocess.check_output(cmd, shell=True)
        res = res.decode('utf-8')
        res = res.rstrip()
    except Exception as e:
        log.debug(
            'terraform: could not obtain output variable "%s" from Terraform: %s', name, e)
    else:
        log.debug('terraform: obtained "%s"="%s"', name, res)

    return res


def run_tf(subcmd, extra='', vars={}, vars_files=[], outputs=[], tf_state=None, timeout=None):
    ''' Run a terraform command '''
    assert(isinstance(subcmd, str))

    workdir = get_work_dir()

    log.debug('terraform: using environment at %s', workdir)

    vars_str = ''
    var_files_str = ''

    if len(args.image) > 0:
        log.debug('terraform: trying to load presets for image %s...', args.image)
        try_file = os.path.abspath(os.path.join(DEFAULT_IMGAGES_CFG_DIR, "{}.tfvars".format(args.image)))
        if os.path.exists(try_file):
            log.debug('tfvars: adding image configuration file: %s', try_file)
            var_files_str += ' -var-file={}'.format(try_file)

    log.debug('terraform: trying to find presets files to autoload in %s...', workdir)
    for try_file in ENV_TFVARS_FILES:
        try_file = os.path.abspath(os.path.join(workdir, try_file))
        if os.path.exists(try_file) and try_file not in vars_files:
            log.debug('tfvars: adding file found: %s', try_file)
            var_files_str += ' -var-file={}'.format(try_file)

    # if we want to perform an initial copy of the Salt/Manifests with Terraform
    # (something that can be problematic because TF cannot copy, for example, symbolic links)
    if args.tf_copies:
        # ... then add the tfvars for salt_dir and manifest_dir
        if args.copy_salt_code and args.salt_dir:
            log.debug('terraform: adding variable "%s" for Salt: %s',
                      TFVAR_SALT_DIR, args.salt_dir)
            vars_str += ' -var \'{}={}\''.format(TFVAR_SALT_DIR, args.salt_dir)

        if args.copy_manifests and args.manifests_dir:
            log.debug('terraform: adding variable "%s" for manifests: %s',
                      TFVAR_MANIFESTS_DIR, args.manifests_dir)
            vars_str += ' -var \'{}={}\''.format(
                TFVAR_MANIFESTS_DIR, args.manifests_dir)

    # Add the tfvar/tfvars provided in command line
    for command_line_var in args.tfvar:
        k, v = command_line_var.split('=')
        log.debug('terraform: adding command line tfvar: "%s"="%s"', k, v)
        vars_str += ' -var \'{}={}\''.format(k.strip(), v.strip())

    if args.tfvars:
        log.debug('terraform: adding command line tfvars files: %s', args.tfvars)
        for f in args.tfvars:
            var_files_str += ' -var-file={}'.format(f)

    # Add the arguments to this function
    for f in vars_files:
        var_files_str += ' -var-file={}'.format(f)

    for k, v in vars.items():
        vars_str += ' -var \'{}={}\''.format(k, v)

    # Create a temporal terraform file where we add
    # some "output"s
    if outputs:
        filename = os.path.abspath(os.path.join(CURR_DIR, DEFAULT_TF_OUTPUTS))
        with open(filename, 'w') as tt:
            log.debug('terraform: creating temporal terraform file %s', filename)

            for name in outputs:
                contents = '''output "''' + name + \
                    '''" { value = "${var.''' + name + '''}" }'''
                log.debug('terraform: adding: %s', contents)
                tt.write(contents + '\n')

    state_file = tf_state or get_tfstate_filename()

    cmd = 'terraform ' + subcmd + ' ' + extra
    cmd += ' -state={state_file} {var_files_str} {vars_str}'.format(**locals())

    with cd(CURR_DIR):
        log.debug('terraform: running command: %s', cmd)
        yield from run('terraform init', timeout=timeout)
        yield from run(cmd, timeout=timeout)


def tf_cleanup(tfstate=None):
    '''
    Cleanup all the leftovers in the working directory
    '''
    log.debug('terraform: cleaning up things')
    try:
        filename = os.path.abspath(os.path.join(CURR_DIR, DEFAULT_TF_OUTPUTS))
        log.debug('terraform: removing outputs file %s', filename)
        os.remove(filename)
    except FileNotFoundError:
        pass

    tfstate = get_tfstate_filename(tfstate)
    tfstatex = get_tfstatex_filename(tfstate)
    for f in [tfstate, tfstatex]:
        try:
            log.debug('terraform: removing %s', f)
            os.remove(f)
        except Exception as e:
            log.debug('terraform: could not remove file: %s [ignored]', e)


def tf_create(vars={}, vars_files=[], outputs=[], **kwargs):
    '''
    Run a 'terraform apply'
    '''
    log.info('terraform: creating the Terraform cluster')

    tf_save_statex(vars, vars_files)

    cmd = 'apply -auto-approve'
    if len(args.tf_par) > 0:
        log.warning('terraform: using {} parallelism'.format(args.tf_par))
        cmd += ' -parallelism={}'.format(args.tf_par)

    try:
        yield from run_tf(cmd, vars=vars, vars_files=vars_files, outputs=outputs)
    except subprocess.CalledProcessError as e:
        raise TerraformError('terraform: construction error: {}\n'.format(e))


def tf_plan(**kwargs):
    log.info('terraform: planning the Terraform cluster')

    # try to use vars/vars_files form the statex file
    # if some vars/vars_files are defined in the current env,
    # they will be used instead
    statex = tf_load_statex(vars=kwargs.pop('vars', {}),
                            vars_files=kwargs.pop('vars_files', {}))
    try:
        yield from run_tf('plan',
                          vars=statex['vars'], vars_files=statex['vars_files'], **kwargs)
    except subprocess.CalledProcessError as e:
        raise TerraformError('terraform: planning error: {}\n'.format(e))


def tf_destroy(**kwargs):
    log.info('terraform: destroying cluster')

    # try to use vars/vars_files form the statex file
    # if some vars/vars_files are defined in the current env,
    # they will be used instead
    statex = tf_load_statex(vars=kwargs.pop('vars', {}),
                            vars_files=kwargs.pop('vars_files', {}))
    try:
        yield from run_tf('destroy', extra='-force',
                          vars=statex['vars'], vars_files=statex['vars_files'], **kwargs)
    except subprocess.CalledProcessError as e:
        raise TerraformError('terraform: destruction error: {}\n'.format(e))


def tf_refresh(**kwargs):
    log.info('terraform: refreshing state')

    # try to use vars/vars_files form the statex file
    # if some vars/vars_files are defined in the current env,
    # they will be used instead
    statex = tf_load_statex(vars=kwargs.pop('vars', {}),
                            vars_files=kwargs.pop('vars_files', {}))
    for _ in range(0, 5):
        try:
            yield from run_tf('refresh',
                              vars=statex['vars'], vars_files=statex['vars_files'], **kwargs)
        except subprocess.CalledProcessError as e:
            log.error('terraform: refresh error: %s', e)
            log.info('terraform: will try again...')
        else:
            break


def tf_vms_num(vms):
    ''' Get the number of VMs '''
    assert(isinstance(vms, dict))
    return len(vms)


def tf_vms_names(vms):
    ''' Get the list of VMs names '''
    assert(isinstance(vms, dict))
    return vms.keys()


def tf_vms_ip_for(vm_name, vms=None):
    ''' Get a list of IPs for a VM name '''
    if is_ip(vm_name):
        return vm_name

    vms = vms or tf_load_state()
    assert(isinstance(vms, dict))

    res = []
    for name in re.split(' |,', vm_name):
        try:
            res.append(vms[name])
        except KeyError as e:
            log.error('terraform: VM %s does not seem to exist', name)
            if vms.keys():
                log.error('terraform: valid names: %s', " ".join(vms.keys()))

    return res


def tf_vms_map(vms):
    ''' Get a list of <name> <IP> '''
    assert(isinstance(vms, dict))
    res = []
    for name, ip in vms.items():
        res.append("{} {}".format(name, ip))
    return res


def tf_vms_ips(vms):
    ''' Print all the IPs '''
    assert(isinstance(vms, dict))
    return vms.values()


def tf_vms_nodes(vms=None):
    ''' Get a map with all the nodes and their IPs. '''
    vms = vms or tf_load_state()
    assert(isinstance(vms, dict))

    # Get the regular expression used for identifying nodes
    if args.vm_nodes_regex:
        return args.vm_nodes_regex

    try:
        prefix = tf_get_extra_output('prefix')
    except Exception as e:
        log.debug('could not get prefix from terraform state file: %s', e)
        prefix = args.vm_prefix

    regex = prefix + VM_NODES_REGEX_SUFFIX

    return {name: ip for name, ip in vms.items()
            if re.search(regex, name)}


def tf_vms_admin():
    ''' Get a tuple with the admin node's name and IP '''
    if args.vm_admin:
        return args.vm_admin

    try:
        prefix = tf_get_extra_output('prefix')
    except Exception as e:
        log.debug('could not get prefix from terraform state file: %s', e)
        prefix = args.vm_prefix

    admin_name = prefix + VM_ADMIN_SUFFIX

    admin_ips = tf_vms_ip_for(admin_name)
    if not admin_ips:
        raise InvalidMachineError('cannot find IP for the Admin Node')

    return (admin_name, admin_ips[0])


def tf_find_for(vm, vms=None):
    ''' Get a full machine name for a (maybe partial) name'''
    if is_ip(vm):
        return vm

    vms = vms or tf_load_state()
    assert(isinstance(vms, dict))

    names = tf_vms_names(vms)
    for name in names:
        if vm in name:
            log.debug(
                'terraform: name "%s" matches "%s": running commands there', vm, name)
            return name

    raise CommandError('could not find matching VM name for {}'.format(vm))


def print_iterator_for_vm(fun, vm, *args, **kwargs):
    for line in fun(vm, *args, **kwargs):
        print('{vm}: {line}'.format(**locals()), end='', flush=True)


def map_vms_par(vms, fun, *args, **kwargs):
    '''
    Map a function (that yields lines) to all the VMs
    '''
    from multiprocessing import Pool
    from functools import partial

    with Pool(max(1, len(vms))) as p:
        return p.map(partial(print_iterator_for_vm, fun, *args, **kwargs), vms)


def map_all_vms_par(fun, *args, **kwargs):
    vms = tf_load_state()
    vm_names = tf_vms_names(vms)
    return map_vms_par(vm_names, fun, *args, **kwargs)


def map_vms(fun, *args, **kwargs):
    ''' Apply a function to all the VMs in the cluster '''
    vms = tf_load_state()
    for vm in tf_vms_names(vms):
        yield from fun(vm, *args, **kwargs)


####################################################################
# ssh
####################################################################


class SSHException(Exception):
    pass


def _get_sshpass_cmd(cmd):
    if args.ssh_password:
        return '{sshpass} -p "{password}" {cmd}'.format(
            sshpass=args.sshpass, password=args.ssh_password, cmd=cmd)
    else:
        return cmd


def _get_ip(vm):
    if is_ip(vm):
        return vm
    else:
        ips = tf_vms_ip_for(vm)
        if len(ips) > 0:
            return ips[0]
    return None


def run_ssh_cmd(vm, cmd, quiet=True, ignore_errors=False, timeout=None):
    '''
    Run a ssh command in a VM
    '''
    assert(isinstance(vm, str))

    ip = _get_ip(vm)
    if not ip:
        raise SSHException('could not obtain IP for {}'.format(vm))

    ssh_cmd = _get_sshpass_cmd('ssh') + ' ' + SSH_ARGS
    if quiet:
        ssh_cmd += ' -q'

    ssh_cmd = '{ssh_cmd} root@{ip} "{cmd}"'.format(**locals())

    log.debug('Running ssh command: %s', ssh_cmd)
    try:
        yield from run(ssh_cmd, timeout=timeout)
    except subprocess.CalledProcessError as e:
        if ignore_errors:
            log.error('could not run "%s" on %s: %s', cmd, vm, e)
        else:
            raise SSHException(
                '{} failed with return code {}'.format(cmd, e.returncode))


def run_ssh_session(vm, cmd='', quiet=True):
    '''
    Start a interactive SSH session to a machine
    '''
    assert(isinstance(vm, str))

    ip = _get_ip(vm)
    if not ip:
        raise SSHException('could not obtain IP for {}'.format(vm))

    ssh_cmd = _get_sshpass_cmd('ssh') + ' ' + SSH_ARGS

    if quiet:
        ssh_cmd += ' -q'

    ssh_cmd = '{ssh_cmd} root@{ip} {cmd}'.format(**locals())

    log.info('Starting ssh session at %s', vm)
    try:
        return run_interactive(ssh_cmd)
    except subprocess.CalledProcessError as e:
        raise SSHException(
            '{} failed with return code {}'.format(cmd, e.returncode))


def run_rsync(src, vm, dst, timeout=None, **kwargs):
    assert(isinstance(src, str))
    assert(isinstance(vm, str))
    assert(isinstance(dst, str))

    exclude = kwargs.get('exclude', EXCLUDE_ARGS + EXCLUDE_BINS_ARGS)

    if is_ip(vm):
        dst_addr = vm
    else:
        vms = tf_load_state()
        vm_name = tf_find_for(vm, vms=vms)
        dst_addrs = tf_vms_ip_for(vm_name, vms=vms)
        if not dst_addrs:
            raise SSHException('could not find a an address for {vm}'.format(
                **locals()))
        else:
            dst_addr = dst_addrs[0]

    ssh_cmd = 'ssh ' + SSH_ARGS

    rsync_cmd = _get_sshpass_cmd('rsync')
    rsync_args = kwargs.pop('rsync_args', DEFAULT_RSYNC_ARGS)
    rsync_opts = "{rsync_args} {exclude} -e '{ssh_cmd}'".format(**locals())

    log.info('copy: %s -> %s (%s):%s', src, vm, dst_addr, dst)
    cmd = '{rsync_cmd} {rsync_opts} {src} root@{dst_addr}:{dst}'.format(
        **locals())
    try:
        yield from run(cmd, timeout=timeout)
    except subprocess.CalledProcessError as e:
        raise SSHException(
            '{} failed with return code {}'.format(cmd, e.returncode))


def run_scp(src, dst, timeout=None, **kwargs):
    assert(isinstance(src, str))
    assert(isinstance(dst, str))

    scp_cmd = _get_sshpass_cmd('scp')
    scp_opts = "-q {ssh_args} ".format(ssh_args=SSH_ARGS)

    log.info('copy: %s -> %s', src, dst)
    cmd = '{scp_cmd} {scp_opts} {src} {dst}'.format(**locals())
    try:
        yield from run(cmd, timeout=timeout)
    except subprocess.CalledProcessError as e:
        raise SSHException('copy failed: {}'.format(e))


def run_caaspctl(vm, cmd):
    assert(isinstance(cmd, str))
    caaapctl_args = '' if not args.debug else '--debug'
    full_cmd = '{} {} {}'.format(args.vm_caaspctl, caaapctl_args, cmd)
    yield from run_ssh_cmd(vm, full_cmd)


def run_caaspctl_admin(cmd):
    assert(isinstance(cmd, str))
    yield from run_caaspctl(tf_vms_admin()[0], cmd)


def wait_ssh(vm=None, timeout=CONTAINER_START_TIMEOUT):
    '''
    Wait until we can ssh/scp into a machine
    '''
    vm = vm or tf_vms_admin()[0]
    timeout_limit = datetime.now() + timedelta(seconds=timeout)
    while datetime.now() <= timeout_limit:
        try:
            _ = list(run_ssh_cmd(vm, 'echo ping'))
        except Exception as e:
            log.debug('error when waiting for %s: %s', vm, e)
            time.sleep(5)
        else:
            log.debug('VM %s seems to be reachable', vm)
            return

    raise TimeoutError('timeout while waiting for "{}"'.format(vm))


def wait_container(cont, vm=None, timeout=CONTAINER_START_TIMEOUT):
    '''
    Wait for a container in the Admin node
    If the Admin Node is not available (ie, not reachable), we will insist
    '''
    vm = vm or tf_vms_admin()[0]
    timeout_limit = datetime.now() + timedelta(seconds=timeout)
    cmd = 'cont wait ' + cont
    while datetime.now() <= timeout_limit:
        try:
            yield from run_caaspctl(vm, cmd)
        except Exception as e:
            log.debug('error when waiting for %s: %s', cont, e)
            time.sleep(5)
        else:
            log.debug('container %s seems to be up in %s', cont, vm)
            return

    raise TimeoutError(
        'timeout while waiting for container "{}" at "{}"'.format(cont, vm))


def run_salt(where, cmd):
    assert(isinstance(where, str))
    assert(isinstance(cmd, str))
    yield from run_caaspctl_admin('salt \'{}\' {}'.format(where, cmd))


def run_ssh_nodes(cmd, **kwargs):
    ''' Run a command in all the nodes (non-admin machines) in the cluster '''
    assert(isinstance(cmd, str))
    map_vms_par(tf_vms_nodes(), run_ssh_cmd, cmd=cmd, **kwargs)


def run_ssh_all(cmd, **kwargs):
    ''' Run a command in all the VMs in the cluster '''
    assert(isinstance(cmd, str))
    map_all_vms_par(run_ssh_cmd, cmd=cmd, **kwargs)


def get_machine_id(vm):
    ''' Get the machine-id for a VM '''
    assert(isinstance(vm, str))
    real_vm = tf_find_for(vm)
    for line in run_ssh_cmd(real_vm, 'cat /etc/machine-id', quiet=True):
        mid = line.strip()
        if mid:
            return mid

    raise InvalidMachineError('could not find machine-id for {}'.format(vm))


def get_machine_ids(vms, ignore_errors=False):
    ''' Get a dictionary with all the machine-IDs '''
    assert(isinstance(vms, list))
    res = {}
    for node in vms:
        try:
            res[node] = get_machine_id(node)
        except Exception as e:
            if ignore_errors:
                log.warning('could not get machine-ID for %s', node)
            else:
                raise e
    return res


def get_cid(vm, name):
    ''' Get the container-id for a name in a VM '''
    assert(isinstance(vm, str))
    real_vm = tf_find_for(vm)
    for line in run_caaspctl(real_vm, 'cid ' + name):
        mid = line.strip()
        if mid:
            return mid

    raise InvalidMachineError('could not find container-id for {}'.format(vm))


def sync_hwtime(vm=None, **kwargs):
    '''
    Synchronize the clock between the VMs and the Host

    This is specially important after suspend/resume operations,
    as this can lead to some certificates beeing invalid.
    '''
    cmd = '/sbin/hwclock --hctosys'
    if vm:
        print_iterator(run_ssh_cmd(vm, cmd, **kwargs))
    else:
        run_ssh_all(cmd, **kwargs)


####################################################################
# VMs managements: resume/pause, snapshots...
####################################################################


def get_virsh_uri():
    try:
        return tf_get_extra_output(TERRAFORM_LIBVIRT_URI_VAR)
    except Exception as e:
        log.debug(
            'virsh: could not get libvirt URI from terraform state file: %s', e)
        return None


def run_virsh(command, timeout=None, **kwargs):
    ''' Run a virsh command '''
    assert(isinstance(command, str))

    cmd = 'virsh'
    uri = get_virsh_uri()
    if uri:
        log.debug('virsh: using %s', uri)
        cmd += ' --connect="{}"'.format(uri)

    cmd += '  --keepalive-interval=2'
    cmd += '  ' + command

    try:
        log.debug('virsh: running command: %s', cmd)
        yield from run(cmd, sudo=args.sudo_virsh, password=args.sudo_password, timeout=timeout)
    except subprocess.CalledProcessError as e:
        log.error('virsh: when running {}'.format(str(e)))


def virsh_status_for(vm):
    ''' Return the VM status: "paused", "running"... '''
    dominfo = run_virsh('dominfo {}'.format(vm))
    for line in dominfo:
        if 'State' in line:
            line_comps = line.split(':')
            return line_comps[1].strip()
    return 'unknown'


def virsh_get_snapshots_for_vm(vm):
    assert(isinstance(vm, str))
    snapshots = run_virsh('snapshot-list {}'.format(vm))
    for snapshot in list(snapshots)[2:]:
        names = snapshot.strip().split(' ')
        name = names[0]
        if name:
            yield (vm, name)


def virsh_get_snapshots_tree_for_vm(vm):
    assert(isinstance(vm, str))
    yield from run_virsh('snapshot-list --tree {}'.format(vm))


def virsh_create_snapshot_for_vm(vm, pre_suspend=True,
                                 stage='', description=''):
    assert(isinstance(vm, str))
    if stage or description:
        cmd = 'snapshot-create-as --domain "{}"'.format(vm)
        if stage:
            cmd += ' --name "{}"'.format(stage)
        if description:
            cmd += ' --description "{}"'.format(description)
    else:
        cmd = 'snapshot-create --atomic --quiesce --domain "{}"'.format(vm)

    if pre_suspend:
        yield from virsh_suspend_for_vm(vm)

    log.info('virsh: creating snapshot for %s (%d secs timeout)...',
             vm, VIRSH_SNAPSHOT_TIMEOUT)
    yield from run_virsh(cmd, timeout=VIRSH_SNAPSHOT_TIMEOUT)


def virsh_destroy_snapshots_for_vm(vm, only_snapshot=None, **kwargs):
    ''' Destroy all the snapshots for a VM '''
    assert(isinstance(vm, str))
    log.info('virsh: removing snapshots for %s', vm)
    if only_snapshot:
        log.info('virsh: on %s, dropping "%s"', vm, only_snapshot)
        yield from run_virsh('snapshot-delete {} {}'.format(
            vm, only_snapshot), ignore_error=True)
    else:
        for vm, snapshot in virsh_get_snapshots_for_vm(vm):
            log.info('virsh: removing snapshot %s for %s', snapshot, vm)
            yield from run_virsh('snapshot-delete --current --domain "{}"'.format(vm), ignore_error=True)


def virsh_suspend_for_vm(vm, **kwargs):
    assert(isinstance(vm, str))
    log.info('virsh: suspending %s', vm)

    status = virsh_status_for(vm)
    log.debug('virsh: rollback: current status: %s', status)

    if status == 'running':
        yield from run_virsh('suspend --domain {}'.format(vm), ignore_error=True)
    else:
        log.info('virsh: suspend: %s was not running', vm)


def virsh_resume_for_vm(vm, **kwargs):
    assert(isinstance(vm, str))
    log.info('virsh: resuming %s', vm)

    status = virsh_status_for(vm)
    log.debug('virsh: resume: current status: %s', status)

    if status == 'running':
        log.info('virsh: resume: %s was already running', vm)
    elif status == 'shut off':
        yield from virsh_power_on_for_vm(vm)
    else:
        yield from run_virsh('resume --domain {}'.format(vm), ignore_error=True)


def virsh_power_on_for_vm(vm, **kwargs):
    assert(isinstance(vm, str))
    log.info('virsh: resuming %s', vm)
    yield from run_virsh('start --domain {}'.format(vm), ignore_error=True)


def virsh_rollback_for_vm(vm, name=None, pre_suspend=True, post_running=True, **kwargs):
    '''
    Rollback the VM to the snapshot `name` (or the the current/latest one)
    '''
    assert(isinstance(vm, str))

    status = virsh_status_for(vm)
    log.debug('virsh: rollback: current status: %s', status)

    if status == 'shut off':
        yield from virsh_power_on_for_vm(vm)
        status = virsh_status_for(vm)

    if pre_suspend and status == 'running':
        log.info('virsh: suspending %s before rolling back', vm)
        yield from virsh_suspend_for_vm(vm)

    sargs = ''

    if not name:
        log.info('virsh: rolling back %s to latest snapshot', vm)
        sargs += ' --current'
    else:
        log.info('virsh: rolling back %s to "%s"', vm, name)
        sargs += ' ' + name

    if post_running:
        sargs += ' --running'

    yield from run_virsh('snapshot-revert --domain {vm} {sargs}'.format(**locals()), ignore_error=True)
    yield from run_virsh('snapshot-info --current {vm}'.format(**locals()), ignore_error=True)


def virsh_sync_time_form_vm(vm):
    assert(isinstance(vm, str))
    log.info('virsh: synchronizing time host<->%s', vm)
    cmd = ''' qemu-agent-command {} '{"execute":"guest-set-time"}' '''.format(
        vm)
    yield from run_virsh(cmd, ignore_error=True)


def virsh_reboot(vm):
    assert(isinstance(vm, str))
    log.info('virsh: rebooting %s', vm)
    yield from run_virsh('reboot {vm}'.format(**locals()), ignore_error=True)


####################################################################
# Copies
####################################################################

def _to_rsync_dir(d):
    if not d.endswith('/'):
        return d + '/'
    return d


RSYNC_STATS_PREFIXES = {
    'created': 'Number of created files: ',
    'deleted': 'Number of deleted files: ',
    'regular': 'Number of regular files transferred: '
}


def get_differences(src, vm, dst):
    '''
    print {% get_differences('../salt', 'admin', '/usr/share/kubernetes/salt') %}
    '''
    def line_proc(line, expr):
        if line.startswith(expr):
            rest_of_line = line[len(expr):]
            value = int(rest_of_line.split(' ')[0])
            return value
        return None

    log.debug('copy: calculating differences between %s and %s:%s', src, vm, dst)
    src = _to_rsync_dir(os.path.abspath(src))
    dst = _to_rsync_dir(dst)

    res = {}
    try:
        for line in run_rsync(src, vm, dst, rsync_args='-avun --delete --stats'):
            for category, expr in RSYNC_STATS_PREFIXES.items():
                value = line_proc(line, expr)
                if value is not None:
                    res[category] = value
    except SSHException as e:
        log.error(
            'copy: could not find differences between %s and %s:%s: %s', src, vm, dst, e)

    return res


def get_current_branch_in(dir):
    with cd(dir):
        for line in run('git rev-parse --abbrev-ref HEAD'):
            return line.strip()


def copy_code_to(code_src_dir, branch, code_remote_dir, vm, description):
    if not os.path.exists(code_src_dir):
        raise CommandError('could not find {} code at {}'.format(
            description, code_src_dir))

    log.info('copy: copying %s code (in %s) to the Admin node',
             description, code_src_dir)
    dst = code_remote_dir
    src = None
    tmp_src = None
    try:
        if branch:
            branch = branch.strip()
            salt_dir_branch = get_current_branch_in(code_src_dir)
            if branch == salt_dir_branch:
                log.warn('copy: same branch checked out in %s', code_src_dir)
            else:
                log.debug('copy: current branch in %s: %s',
                          code_src_dir, branch)

                src = tmp_src = tempfile.mkdtemp()
                log.info('copy: creating working tree in "%s" for %s branch "%s"',
                         tmp_src, description, branch)
                with cd(code_src_dir):
                    worktree_cmd = 'git worktree add {tmp_src} {branch}'.format(
                        **locals())
                    yield from run(worktree_cmd)

                worktree_branch = get_current_branch_in(tmp_src)
                log.info('copy: branch in the working tree: %s',
                         worktree_branch)
                if worktree_branch != branch:
                    raise CommandError(
                        'could not create a worktree for branch {}'.format(branch))

        if not src:
            log.debug('copy: will copy current checkout')
            src = code_src_dir

        src = _to_rsync_dir(os.path.abspath(src))
        dst = _to_rsync_dir(dst)
        try:
            for line in run_rsync(src, vm, dst):
                if args.debug:
                    yield line

        except SSHException as e:
            log.error('copy: could not copy %s to %s: %s', src, vm, e)
    finally:
        if tmp_src:
            log.info('copy: removing temporal directory %s', tmp_src)
            shutil.rmtree(tmp_src)

            with cd(code_src_dir):
                log.debug('copy: prunning old worktrees')
                worktree_cmd = 'git worktree prune'
                yield from run(worktree_cmd)


def copy_enable(**kwargs):
    log.info('copy: enabling rw filesystem in the Admin Node')
    try:
        yield from run_caaspctl_admin(CAASPCTL_ENABLE_RW)
    except:
        log.warning('copy: could not enable RW: copies could fail')


def copy_to_admin(**kwargs):
    '''Perform all the copies necessary to the admin node.'''
    log.info('copy: copying control scripts and files to the Admin node')
    admin_name, admin_ip = tf_vms_admin()
    assert is_ip(admin_ip)

    for src, dst in DEFAULT_COPIES['admin']:
        log.debug('copy: copying %s to %s', src, dst)
        src = _to_rsync_dir(os.path.abspath(src))
        dst = _to_rsync_dir(dst)
        try:
            for line in run_rsync(src, admin_name, dst):
                if args.debug:
                    yield line
        except SSHException as e:
            log.error('copy: could not copy %s to %s: %s', src, admin_name, e)


def copy_to_admin_code(skip_reboot=False, **kwargs):
    '''Perform all the copies necessary to the admin node.'''
    log.debug('copy: copying files to the Admin node')
    admin_name, admin_ip = tf_vms_admin()

    if args.copy_salt_code and args.salt_dir:
        yield from copy_code_to(args.salt_dir, args.salt_branch,
                                DEFAULT_SALT_REM, admin_name, 'Salt')

    if args.copy_manifests and args.manifests_dir:
        diffs = get_differences(
            args.manifests_dir, admin_name, DEFAULT_MANIFESTS_REM)
        if any(x > 0 for x in diffs.values()):
            log.info('copy: detected differences in the manifests: %s', diffs)
            yield from copy_code_to(args.manifests_dir, args.manifests_branch,
                                    DEFAULT_MANIFESTS_REM, admin_name, 'manifests')
            yield from run_ssh_cmd(admin_name, 'sh {}'.format(ADMIN_NODE_SETUP))

            if skip_reboot:
                log.warn(
                    'copy: skipping reboot. Manifests will not take effect...')
            else:
                log.info('copy: rebooting the Admin machine')
                yield from virsh_reboot(admin_name)
                time.sleep(10)
                log.info('copy: waiting until the Salt Master is back')
                yield from wait_container('salt')
        else:
            log.info('copy: it seems the Manifests do not need to be updated.')


def copy_to_node(vm, **kwargs):
    '''Perform all the copies necessary for a node.'''
    log.debug('copy: copying files to %s', vm)
    for src, dst in DEFAULT_COPIES['nodes']:
        src = _to_rsync_dir(os.path.abspath(src))
        dst = _to_rsync_dir(dst)
        try:
            for line in run_rsync(src, vm, dst):
                if args.debug:
                    yield line
        except SSHException as e:
            log.error('copy: could not copy %s to %s: %s', src, vm, e)


def copy_to_nodes(**kwargs):
    '''Perform all the copies necessary for the nodes.'''
    log.debug('copy: copying files to the Nodes')
    map_all_vms_par(copy_to_node)


def copy_file_from(vm, src, dst='./', orig_user='root'):
    '''Perform a SCP copy from a VM to the local machine.'''
    log.debug('copy: copying files from %s', src)
    vms = tf_load_state()
    vm_name = tf_find_for(vm, vms=vms)
    addrs = tf_vms_ip_for(vm_name, vms=vms)
    if not addrs:
        raise SSHException(
            'copy: could not find a an address for {vm}'.format(**locals()))

    addr = addrs[0]
    src_expr = '{orig_user}@{addr}:{src}'.format(**locals())
    dst_expr = '{dst}'.format(**locals())

    yield from run_scp(src=src_expr, dst=dst_expr)


def copy_file_to(src, vm, dst='', container=None, dst_user='root'):
    '''
    Perform a SCP copy from the local machine to a VM.
    '''
    log.debug('copy: copying file %s -> %s:%s', src, vm, dst)
    vms = tf_load_state()
    vm_name = tf_find_for(vm, vms=vms)
    addrs = tf_vms_ip_for(vm_name, vms=vms)
    if not addrs:
        raise SSHException(
            'copy: could not find a an address for {vm}'.format(**locals()))

    addr = addrs[0]
    src_expr = '{src}'.format(**locals())

    if container:
        temp_filename = '/tmp/temporal-file'
        if dst:
            temp_filename = os.path.join('/tmp', os.path.basename(src))

        dst_expr = '{dst_user}@{addr}:{temp_filename}'.format(**locals())
    else:
        dst_expr = '{dst_user}@{addr}:{dst}'.format(**locals())

    yield from run_scp(src=src_expr, dst=dst_expr)

    if container:
        cid = get_cid(addr, container)
        log.info('copy: %s:%s -> %s:%s:%s', addr,
                 temp_filename, addr, cid, dst)
        cmd = 'docker cp {temp_filename} {cid}:{dst}'.format(**locals())
        yield from(run_ssh_cmd(addr, cmd))

        log.info('copy: (removing temporal file %s:%s)', addr, temp_filename)
        cmd = 'rm -f {temp_filename}'.format(**locals())
        yield from(run_ssh_cmd(addr, cmd))


def install_rpms_in(vm, src=None, dst=None, forced=False):

    src = src or args.default_rpms_src
    dst = dst or args.default_rpms_dst

    src = _to_rsync_dir(os.path.abspath(src))
    dst = _to_rsync_dir(dst)

    log.info('copy: copying RPMs from %s in %s:%s', src, vm, dst)
    yield from run_rsync(src, vm, dst, exclude=EXCLUDE_ARGS)

    # install the RPMs
    log.info('copy: installing everything in %s', dst)
    zypper_command = 'install' if not forced else 'install_forced'
    cmd = 'caaspctl zypper {zypper_command} {dst}/*.rpm'.format(**locals())
    yield from run_ssh_cmd(vm, cmd)


def install_rpms_in_all(**kwargs):
    yield from map_vms(install_rpms_in, **kwargs)


def get_kubeconfig():
    yield from run_caaspctl_admin(CAASPCTL_GEN_KUBECONFIG)
    yield from copy_file_from(vm=tf_vms_admin()[0],
                              src=args.DEFAULT_KUBECONFIG_REM,
                              dst=args.kubeconfig)


def remove_kubeconfig():
    if os.path.exists(args.kubeconfig):
        os.remove(args.kubeconfig)


def set_pillar(key, value):
    ''' Set some pillar value (ie, api:server:external_fqdn) '''
    yield from run_caaspctl_admin(CAASPCTL_SET_PILLAR.format(**locals()))


########################
# registry
########################

def _reg_gen_certs(hostname=None, ip=None):
    ca_crt = os.path.join(args.reg_certs_dir, 'ca.pem')
    ca_key = os.path.join(args.reg_certs_dir, 'ca-key.pem')
    ca_csr = os.path.join(args.reg_certs_dir, 'ca.csr')

    reg_crt = os.path.join(args.reg_certs_dir, 'registry.pem')
    reg_key = os.path.join(args.reg_certs_dir, 'registry-key.pem')
    reg_csr = os.path.join(args.reg_certs_dir, 'registry.csr')

    registry_hostname = args.reg_hostname
    port = REGISTRY_PORT

    if not os.path.exists(args.reg_certs_dir):
        log.debug('Creating directory %s for registry`s certificates',
                  args.reg_certs_dir)
        os.makedirs(args.reg_certs_dir)

    log.debug('Removing old certificates')
    for filename in [reg_key, reg_crt, reg_csr]:
        if os.path.exists(filename):
            log.debug('Removing %s', filename)
            os.remove(filename)

    if not hostname:
        hostname = socket.gethostname()

    vms = tf_load_state()
    if not vms:
        raise CommandError('no VMs found')

    if not ip:
        random_vm = list(vms.items())[0]
        random_vm_ip = random_vm[1]
        ip = get_local_ip_for(random_vm_ip)
        if not ip:
            raise CommandError('could not guess default interface IP')
        log.debug('Using %s as the local IP in the certificates', ip)

    with NamedTemporaryFile(mode='w+', suffix='.cnf') as ntf:
        cnf = ntf.name
        contents = REGISTRY_CERTS_CFG.format(**locals())
        ntf.write(contents)
        ntf.flush()

        log.info('Generating Certificates in %s', args.reg_certs_dir)
        try:
            cmd = SCRIPT_REGISTRY_CERTS.format(**locals())
            log.debug('using script: %s', cmd)
            for line in run(cmd):
                yield line
        except Exception as e:
            log.error('command "%s" failed: %s', cmd, e)

        log.debug('cert: dumping information:')
        cmd = 'openssl x509 -in "{reg_crt}" -text -noout'.format(**locals())
        for line in run(cmd):
            log.debug('certificate: %s', line)

        log.info('Done. Certificates stored at %s', args.reg_certs_dir)

    log.info('Copying registry CA to the nodes')
    remote_dir = '/etc/docker/certs.d/{registry_hostname}:{port}/ca.crt'.format(
        **locals())

    for vm, vm_ip in vms.items():
        yield from run_ssh_cmd(vm, 'mkdir -p `dirname {remote_dir}`'.format(**locals()))
        yield from copy_file_to(ca_crt, vm, dst=remote_dir)
        yield from run_ssh_cmd(vm, 'caaspctl etchosts add {registry_hostname} {ip}'.format(**locals()))


def _reg_del_certs():
    ca_crt = os.path.join(args.reg_certs_dir, 'ca.pem')
    ca_key = os.path.join(args.reg_certs_dir, 'ca-key.pem')
    ca_csr = os.path.join(args.reg_certs_dir, 'ca.csr')

    reg_crt = os.path.join(args.reg_certs_dir, 'registry.pem')
    reg_key = os.path.join(args.reg_certs_dir, 'registry-key.pem')
    reg_csr = os.path.join(args.reg_certs_dir, 'registry.csr')

    log.debug('Removing old certificates')
    for filename in [reg_key, reg_crt, reg_csr, ca_crt, ca_key, ca_csr]:
        if os.path.exists(filename):
            log.debug('Removing %s', filename)
            os.remove(filename)


def _reg_refresh_nodes(ip=None):
    registry_hostname = args.reg_hostname

    vms = tf_load_state()
    if not vms:
        raise CommandError('no VMs found')

    if not ip:
        random_vm = list(vms.items())[0]
        random_vm_ip = random_vm[1]
        ip = get_local_ip_for(random_vm_ip)
        if not ip:
            raise CommandError('could not guess interface IP')
        log.debug('Using %s as the local IP in the certificates', ip)

    log.info('Updating registry references in the nodes')
    for vm, vm_ip in vms.items():
        yield from run_ssh_cmd(vm, 'caaspctl etchosts del {registry_hostname} ; caaspctl etchosts add {registry_hostname} {ip}'.format(**locals()))


def registry_start():
    '''
    Initialize the certificates and start the registry
    '''
    cid = get_regular_container(REGISTRY_DOCKER_IMAGE)
    if cid:
        log.error('the registry is already running')
        return

    reg_crt = os.path.join(args.reg_certs_dir, 'registry.pem')
    reg_key = os.path.join(args.reg_certs_dir, 'registry-key.pem')

    if not os.path.exists(reg_key):
        yield from _reg_gen_certs()
        if not os.path.exists(reg_key):
            raise CommandError(
                'no certificate found at {reg_key}'.format(**locals()))

    log.info('Starting the registry')
    registry_port = REGISTRY_PORT
    registry_certs_dir = args.reg_certs_dir
    basename_reg_crt = os.path.basename(reg_crt)
    basename_reg_key = os.path.basename(reg_key)
    container_registry = REGISTRY_DOCKER_IMAGE
    container_registry_tag = REGISTRY_DOCKER_IMAGE_TAG
    cmd = REGISTRY_DOCKER_CMD.format(**locals())
    yield from run(cmd)


def registry_stop():
    '''
    Stop the registry
    '''
    cid = get_regular_container(REGISTRY_DOCKER_IMAGE)
    if cid:
        yield from run('docker stop {cid}'.format(cid=cid))
        yield from run('docker rm {cid}'.format(cid=cid))
    else:
        log.info('No registry detected.')

    _reg_del_certs()


def registry_refresh():
    '''
    Refresh the registry
    '''
    yield from _reg_refresh_nodes()
    sync_hwtime()


def registry_pull(image_name, registry='localhost', port=REGISTRY_PORT):
    '''
    Pull an image from the Docker registry and import
    it to the local registry
    '''
    reg_image_name = '{registry}/{image_name}'.format(**locals())
    reg_image_name_port = '{registry}:{port}/{image_name}'.format(**locals())

    log.info('Pulling %s', image_name)
    yield from run('docker pull "{}"'.format(image_name))

    log.debug('Getting the hash for image %s', image_name)
    cmd = "docker images | grep " + image_name + \
        " | grep latest | awk '{ print $3 }'"
    output = list(run(cmd))
    image_hash = output[0].strip()

    log.info('Tagging %s as %s', image_hash, reg_image_name)
    yield from run('docker tag {image_hash} {reg_image_name_port}'.format(**locals()))

    log.info('Pushing to %s', reg_image_name_port)
    yield from run('docker push {reg_image_name_port}'.format(**locals()))

    log.info('Removing tag %s', reg_image_name_port)
    yield from run('docker rmi {reg_image_name_port}'.format(**locals()))


def registry_import(image_name, image_hash=None, registry='localhost', port=REGISTRY_PORT):
    '''
    Import an existing image to the local registry
    '''
    reg_image_name = '{registry}/{image_name}'.format(**locals())
    reg_image_name_port = '{registry}:{port}/{image_name}'.format(**locals())

    if not image_hash:
        name, tag = image_name.split(':')
        log.debug('Getting the hash for image %s', name)
        cmd = "docker images | grep " + name + \
            " | grep " + tag + " | awk '{ print $3 }'"
        output = list(run(cmd))
        image_hash = output[0].strip()

    log.info('Tagging %s as %s', image_hash, reg_image_name_port)
    yield from run('docker tag {image_hash} {reg_image_name_port}'.format(**locals()))

    log.info('Pushing to %s', reg_image_name_port)
    yield from run('docker push {reg_image_name_port}'.format(**locals()))

    log.info('Removing tag %s', reg_image_name_port)
    yield from run('docker rmi {reg_image_name_port}'.format(**locals()))


def registry_rmi(name):
    '''
    remove an image from the local registry
    '''
    port = REGISTRY_PORT
    url = 'https://127.0.0.1:{port}/v2/{name}/tags/list'.format(**locals())
    output = list(run("curl -s --no-proxy '' -k " + url))
    log.debug('output from %s: %s', name, output)

    import json
    contents = json.loads(str(output[0]))
    log.debug('tags in the registry for %s: %s', name, contents)
    if 'tags' in contents:
        for tag in contents['tags']:
            log.debug('getting digest for %s', tag)

            digest = ''
            url = 'https://127.0.0.1:{port}/v2/{name}/manifests/{tag}'.format(
                **locals())
            for line in run('curl -k -v --no-proxy "" -H "Accept:application/vnd.docker.distribution.manifest.v2+json" ' + url + ' 2>&1'):
                if 'docker-content-digest' in line:
                    line_comps = line.split(':')
                    digest = line_comps[-1].strip()

            if not digest:
                log.error('could not determine digest for %s', tag)
                continue
            else:
                log.debug('digest seems to be %s', digest)

            log.debug('removing "%s:%s/%s" in local registry',
                      name, tag, digest)
            url = 'https://127.0.0.1:{port}/v2/{name}/manifests/{digest}'.format(
                **locals())
            yield from run('curl -X DELETE -s -o /dev/null --no-proxy "" -H "Accept:application/vnd.docker.distribution.manifest.v2+json" -k ' + url)
    else:
        log.error('no tags in catalog for "%s"', name)


def registry_catalog():
    '''
    Dump the registry catalog contents
    '''
    url = 'https://127.0.0.1:5000/v2/_catalog'
    yield from run("curl -s --no-proxy '' -k " + url)


####################################################################
# Command-line processing
####################################################################


class CommandError(Exception):
    pass


class CmdBase(Cmd):

    def __init__(self, top=None):
        super().__init__()
        self.last_exc = None
        self.blocked = False
        self.current_script = ''
        self.top = top

    def abort(self):
        self.do_traceback('')
        log.critical(on_color('RED', 'aborting execution'))
        sys.exit(1)

    def command_line_args(self, cmd_args):
        line = ' '.join(cmd_args)
        for command in line.split(';'):
            self.onecmd(command)

    def is_interactive(self):
        if self.top:
            return self.top.is_interactive()
        else:
            return (self.stdin == sys.stdin)

    def onecmd(self, line):
        try:
            if not self.blocked or line == 'EOF' or line.startswith('stage'):
                return Cmd.onecmd(self, line)
            else:
                return False
        except subprocess.CalledProcessError as e:
            log.info(on_color('RED', 'Command error: ' + str(e)))
            if args.exit_on_err or not self.is_interactive():
                self.abort()
        except SSHException as e:
            if args.exit_on_err or not self.is_interactive():
                self.abort()
        except KeyboardInterrupt as e:
            log.info(on_color('RED', '[interrupted]'))
            if args.exit_on_err or not self.is_interactive():
                self.last_exc = sys.exc_info()
                self.abort()

    def cmdloop(self, intro=None):
        if self.intro:
            print(self.intro)

        while True:
            try:
                super().cmdloop(intro="")
                self.postloop()
                break
            except subprocess.CalledProcessError as e:
                log.info(on_color('RED', 'Command error: ' + str(e)))
                if args.exit_on_err or not self.is_interactive():
                    self.abort()
            except SSHException as e:
                if args.exit_on_err or not self.is_interactive():
                    self.abort()
            except KeyboardInterrupt as e:
                log.info(on_color('RED', '[interrupted]'))
                if args.exit_on_err or not self.is_interactive():
                    self.last_exc = sys.exc_info()
                    self.abort()
            except Exception as e:
                self.last_exc = sys.exc_info()

                if not self.is_interactive():
                    # we are running in batch mode
                    log.critical(
                        on_color('RED', 'exception catched in batch mode: %s'), e)
                    self.abort()

                log.critical(on_color('RED', 'exception catched !!! %s'), e)
                log.critical(
                    on_color('RED', 'get more details with "traceback".'))

    def precmd(self, line):
        if line.lstrip().startswith('#'):
            return ''

        if len(line.strip()) == 0:
            return line

        # replace all the `some-shell-command`
        def sh_replacer(text):
            cmd = text[1:-1]  # remove the ``
            log.debug('replacing %s by shell output', cmd)
            out = subprocess.check_output(
                cmd, stderr=subprocess.STDOUT, shell=True)
            return str(out.decode('utf-8').rstrip())
        line = replace_pattern(r"`.*`", sh_replacer, line)

        # replace all the {% some-python-code %}
        def python_replacer(text):
            code = text[2:-2]  # remove the {%%}
            log.debug('replacing %s by python evaluation', code)
            return str(self.eval(code))
        line = replace_pattern(r"\{\%.*\%\}", python_replacer, line)

        line = expandvars(line)

        return line

    def default(self, line):
        line = line.lstrip()

        if line.startswith('>'):
            # evaluate python code
            return self.do_eval(line[1:])

        if line.startswith('!'):
            # run a local command
            return self.do_shell(line[1:])

        if line.startswith('@'):
            # run a command in a VMs
            return self.do_sh_at(line[1:])

        if line == '..':
            return True

        super().default(line)

    def try_rc_files(self, rc_files, directory=None):
        if args.skip_rc_files:
            return

        directory = directory or CURR_DIR

        log.debug('rc files: trying to load from "%s"', directory)
        for maybe_rc_file in rc_files:
            maybe_rc_file = os.path.expandvars(maybe_rc_file)
            if not os.path.isabs(maybe_rc_file):
                maybe_rc_file = os.path.join(directory, maybe_rc_file)

            if os.path.exists(maybe_rc_file):
                try:
                    self.load_script(maybe_rc_file)
                    log.debug('rc files: %s loaded', maybe_rc_file)
                except Exception as e:
                    log.critical(
                        'rc files: could not read "%s": %s', maybe_rc_file, e)
                    sys.exit(1)

    def try_stage_rc_files(self, stage):
        if not args.skip_rc_files:
            env_dir = os.path.join(args.env_dir, args.env)
            if os.path.exists(env_dir):
                log.debug(
                    'rc files: trying to load all the "%s" resource files', stage)
                try_rc_files = [x.format(stage=stage)
                                for x in ENV_STAGE_RC_FILES]
                caasp_cmd.try_rc_files(try_rc_files, directory=env_dir)

    def load_script(self, script):
        if self.top:
            # loads must be performed at the top commands processor
            return self.top.load_script(script)

        log.info('rc files: loading "%s"', script)

        old_stdin = caasp_cmd.stdin
        old_prompt = caasp_cmd.prompt
        old_intro = caasp_cmd.intro
        old_use_rawinput = caasp_cmd.use_rawinput

        self.use_rawinput = False
        self.prompt = ''
        self.intro = ''
        self.current_script = os.path.abspath(script)

        try:
            with open(script, 'rt') as script_fd:
                self.stdin = script_fd
                self.cmdloop()
                script_fd.close()
        finally:
            # restore the previous settings
            self.stdin = old_stdin
            self.prompt = old_prompt
            self.intro = old_intro
            self.use_rawinput = old_use_rawinput
            self.current_script = ''

    def do_load(self, line):
        '''
        Load a script.
        '''
        filename = line
        if not os.path.isabs(filename):
            this_filename = os.path.realpath(__file__)
            this_dirname = os.path.dirname(this_filename)
            cur_script_dirname = os.path.dirname(self.current_script)

            log.debug('try to guess the real name of %s', filename)
            for i in [filename,
                      os.path.join(this_dirname, filename),
                      os.path.join(cur_script_dirname, filename)]:
                log.debug('trying %s', i)
                if os.path.exists(i):
                    filename = i
                    break

        if os.path.exists(filename):
            self.load_script(filename)
        else:
            log.error('could not load script at %s', filename)

    def complete_do_load(self, text, line, start_idx, end_idx):
        return _complete_path(text)

    def do_shell(self, line):
        '''
        Run a shell command in the local machine.
        Notes:

        * there is shortcut with the ! character (ie, "! ls -lisa")
        * a nodename NODE will be automatically expanded to any
          name that contains NODE.

        Usage:

        > sh ls /
        > sh cat README.txt
        > ! ls /
        '''
        if run_interactive(line) != 0:
            raise CommandError('command "{}" failed'.format(line))

    def do_sh_at(self, line):
        '''
        Run a command or get a shell with "ssh"

        * there is shortcut with the @ character followed by
          the node (ie, "@node-1 ls -lisa")
        * a nodename NODE will be automatically expanded to any
          name that contains NODE.

        > sh_at node-1 ls /
        > sh_at node-1
        > @node-1 ls -lisa
        '''
        args_components = line.split(' ')
        if not args_components:
            raise CommandError('wrong number of arguments')

        vm = tf_find_for(args_components[0])
        cmd = " ".join(args_components[1:])
        run_ssh_session(vm, cmd)

    def do_traceback(self, line):
        '''Get a traceback for the last exception. '''
        if self.last_exc:
            traceback.print_exception(*self.last_exc)
            self.last_exc = None

    def do_quiet(self, line):
        '''Set quiet mode.'''
        args.debug = False
        reset_loglevel(logging.INFO)

    def do_debug(self, line):
        '''Set debug mode.'''
        args.debug = True
        reset_loglevel(logging.DEBUG)

    def do_sleep(self, line):
        '''Sleep for some time.'''
        s = int(line)
        log.info('Sleeping for %s seconds...', s)
        time.sleep(s)
        log.info('... time to wake up!')

    def do_print(self, line):
        '''Quit.'''
        log.info(on_color('GREEN', line))

    def do_notify(self, line):
        '''
        Send a desktop notification.

        Usage:

        > notify We are doing something
        > notify body=Test 1 completed, summary=Tests
        '''
        try:
            kwargs = get_assign_from_str(line)
        except:
            kwargs = {}

        if kwargs:
            notify(**kwargs)
        else:
            notify(body=line)

    @contextmanager
    def _operation(self, stage=None, pre_msg=None, post_msg=None, icon=None):
        if pre_msg:
            notify(pre_msg, icon=icon)

        if stage:
            self.try_stage_rc_files('pre-' + stage)

        yield

        if stage:
            self.try_stage_rc_files('post-' + stage)

        if post_msg:
            notify(post_msg, icon=icon)

    def do_EOF(self, line):
        return True

    def do_quit(self, line):
        '''Quit.'''
        return self.do_EOF(line)

    def eval(self, code):
        gl = globals()
        return eval(code)  # , {'root': gl['caasp_cmd']}, {})

    def do_eval(self, line):
        '''
        Evaluate some Python code.
        '''
        print(self.eval(line))

    def emptyline(self):
        # ignore empty lines instead of repeating last command
        pass

    def do_stage(self, line):
        '''
        Mark the beginning of a new stage in a script.
        '''
        if not line:
            raise CommandError('no stage specified')

        stage = line.strip()

        if args.script_begin and str(stage) == str(args.script_begin):
            log.info(
                'stage: ************************ %s ************************', stage)
            log.debug('stage: unblocking input')
            self.blocked = False
        elif args.script_stage:
            if stage in args.script_stage:
                log.info(
                    'stage: ************************ %s ************************', stage)
                log.debug('stage: unblocking input')
                self.blocked = False
            else:
                log.debug(
                    'stage: %s not in the stages list: blocking input', stage)
                self.blocked = True
        else:
            log.debug('stage: "%s"', stage)
            log.debug('(waiting for stage "%s")', args.script_begin)

    def do_arg(self, line):
        '''
        Set some argument in the 'args' variable

        Usage:

        > arg sudo_virsh=False
        > arg kubeconfig=/my/file
        '''
        global args
        kwargs = get_assign_from_str(line)
        for k, v in kwargs.items():
            log.debug('Setting command line argument: "%s" = "%s"', k, v)
            args.__dict__[k] = v

    def check_kubeconfig(self):
        if not os.path.exists(args.kubeconfig):
            log.info('kubeconfig does not seem to exist: generating...')
            try:
                print_iterator(get_kubeconfig())
            except SSHException as e:
                log.error('kubeconfig generation error: %s', e)
                raise CommandError('could not obtain a kubeconfig')
            except KeyboardInterrupt as e:
                log.error('[Interrupted]')
                return

        if not os.path.exists(args.kubeconfig):
            raise CommandError('could not find a kubeconfig')

    def check_helm(self):
        helm_path = which('helm')
        if not helm_path:
            raise CommandError('no helm executable found')
        else:
            log.debug('helm is installed at %s', helm_path)

    def check_kubectl(self):
        kubectl_path = which('kubectl')
        if not kubectl_path:
            raise CommandError('no kubectl executable found')
        else:
            log.debug('kubectl is installed at %s', kubectl_path)

###################
# SSH
###################


class CaaSPSSH(CmdBase):

    prompt = prompt('caasp:ssh')

    def default(self, line):
        '''Run an interactive shell in some machine'''
        line_comps = line.split(' ')

        if len(line_comps) == 0:
            raise CommandError('wrong number of arguments')

        vm = tf_find_for(line_comps[0])

        if len(line_comps) > 1:
            cmd = " ".join(line_comps[1:])
            print_iterator(run_ssh_cmd(vm, cmd))
        else:
            run_ssh_session(vm)

    def do_run(self, line):
        '''Run a command in a VMs.'''
        args_components = line.split(' ')
        if len(args_components) < 2:
            raise CommandError('wrong number of arguments')

        vm = tf_find_for(args_components[0])
        cmd = " ".join(args_components[1:])

        with self._operation():
            print_iterator(run_ssh_cmd(vm, cmd))

    def complete_run(self, text, line, begidx, endidx):
        vms = tf_load_state()
        names = tf_vms_names(vms)
        if not text:
            completions = names[:]
        else:
            completions = [f for f in names if f.startswith(text)]
        return completions

    def do_admin(self, line):
        '''Run a command in the Admin node or start a interactive shell'''
        with self._operation():
            admin_name, admin_ip = tf_vms_admin()
            if line:
                print_iterator(run_ssh_cmd(admin_name, line))
            else:
                run_ssh_session(admin_name)

    def do_nodes(self, line):
        '''Run a command in the nodes (ie, all machines except the admin node) VMs.'''
        with self._operation():
            if len(line) == 0:
                raise CommandError('No command to run on the nodes')

            run_ssh_nodes(line, ignore_errors=True)

    def do_all(self, line):
        '''Run a command in all the VMs.'''
        if len(line) == 0:
            raise CommandError('No command to run')

        self.do_admin(line)
        self.do_nodes(line)

    def do_reboot_all(self, line):
        '''
        Reboot all the VMs.
        '''
        with self._operation():
            notify('Rebooting the cluster', icon='reload')
            run_ssh_all('{} reboot'.format(
                args.vm_caaspctl), ignore_errors=True)

    def do_wait(self, line):
        '''
        Wait for ssh connectivity to one (or more) machines
        '''
        if len(line) == 0:
            raise CommandError('No vm(s) provided')

        for vm_name in line.split(' '):
            vm = tf_find_for(vm_name)
            log.info('Waiting for ssh reachability to %s', vm)
            wait_ssh(vm)
            log.info('... %s seems to be reachable', vm)


def _complete_path(path):
    if op.isdir(path):
        return gb.glob(op.join(path, '*'))
    else:
        return gb.glob(path + '*')


###################
# Cluster
###################

class CaaSPCluster(CmdBase):

    prompt = prompt('caasp:cluster')

    def __init__(self, top=None):
        super().__init__(top)
        self.do_flush('')

    def do_flush(self, line):
        '''Flush the vars and vars files.'''
        self.vars = {}
        self.vars_files = []

    def do_tfvar(self, line):
        '''
        Set Terraform variable(s).

        Usage:

        > cluster tfvar some_var=some_value
        '''
        if line:
            self.vars.update(get_assign_from_str(line))
        else:
            # do not set anything: just dump the list of tfvars
            statex = tf_load_statex(vars=self.vars)
            if statex['vars']:
                log.info('terraform: current variables:')
                for k, v in statex['vars'].items():
                    log.info(' - "%s" = "%s"', k, v)

    def do_tfvars(self, line):
        '''
        Set Terraform variables from a file

        Usage:

        > cluster tfvars terraform/myvars.tfvars
        '''

        def _load_file(tried):
            if not os.path.exists(tried):
                return False

            tried = os.path.normpath(tried)
            tried = os.path.expandvars(tried)
            tried = os.path.abspath(tried)

            if tried not in self.vars_files:
                log.debug('terraform: will use vars from file "%s"', tried)
                self.vars_files.append(tried)

            return True

        if line:
            for f in line.split(' '):
                for tried in [f,
                              f + '.tfvars',
                              os.path.join(get_work_dir(), f),
                              os.path.join(get_work_dir(), f) + '.tfvars',
                              os.path.join(args.tfvars_dir, f),
                              os.path.join(args.tfvars_dir, f) + '.tfvars',
                              os.path.join('tfvars', f),
                              os.path.join('tfvars', f) + '.tfvars']:
                    if _load_file(tried):
                        break
        else:
            # do not set anything: just dump the list of tfvars
            statex = tf_load_statex(vars_files=self.vars_files)
            if statex['vars_files']:
                log.info('terraform: current variables files:')
                for tfvars_file in statex['vars_files']:
                    log.info(' - %s', tfvars_file)

    def complete_tfvars(self, text, line, start_idx, end_idx):
        return _complete_path(text)

    def do_tf_output(self, line):
        '''Get a Terraform output variable'''
        print(tf_get_extra_output(line))

    def do_create(self, line):
        '''Create the cluster with the help of Terraform. '''
        with self._operation('create', pre_msg='Creating cluster', post_msg='Cluster created', icon='up'):
            # dump the vars and vars files
            self.do_tfvar('')
            self.do_tfvars('')

            print_iterator(tf_create(vars=self.vars, vars_files=self.vars_files,
                                     outputs=TFSTATE_OUTPUTS), flush=True)

    def do_plan(self, line):
        '''Dump the Terraform plan. '''

        with self._operation('plan'):
            # dump the vars and vars files
            self.do_tfvar('')
            self.do_tfvars('')

            print_iterator(
                tf_plan(vars=self.vars, vars_files=self.vars_files), flush=True)

    def do_destroy(self, line):
        '''Destroy the cluster.'''
        tfstate = get_tfstate_filename()

        if not os.path.exists(tfstate):
            log.error(
                'no Terraform state file found: it seems there is nothing to destroy !!')
            tf_cleanup()
            return

        with self._operation('destroy',
                             pre_msg='Destroying the cluster',
                             post_msg='Cluster destroyed',
                             icon='down'):
            # dump the vars and vars files
            self.do_tfvar('')
            self.do_tfvars('')

            try:
                map_all_vms_par(virsh_destroy_snapshots_for_vm)
            except Exception as e:
                log.error('could not destroy snapshots in the cluster: %s', e)

            for i in range(0, 5):
                try:
                    print_iterator(tf_destroy(
                        vars=self.vars, vars_files=self.vars_files), flush=True)
                except TerraformError as e:
                    log.error('could not destroy the cluster: %s', e)
                    log.error('will try again...')
                else:
                    tf_cleanup()
                    remove_kubeconfig()
                    break

    def do_nuke(self, line):
        '''Force the destruction of the cluster by just forgetting about the state.'''
        tfstate = get_tfstate_filename()

        if not os.path.exists(tfstate):
            log.warning(
                'no Terraform state file found: it seems there is nothing to destroy !!')

        with self._operation('destroy',
                             pre_msg='Destroying the cluster',
                             post_msg='Cluster destroyed',
                             icon='down'):
            tf_cleanup()
            remove_kubeconfig()

    def do_refresh(self, line):
        '''Refresh the cluster state.'''
        with self._operation('refresh',
                             pre_msg='Refreshing the cluster',
                             post_msg='Cluster refreshed',
                             icon='reload'):
            print_iterator(tf_refresh())
            sync_hwtime()

    def do_ips(self, line):
        '''Get the list of VMs IPs.'''
        with self._operation():
            res = tf_load_state()
            print(" ".join(tf_vms_ips(res)))

    def do_ip(self, line):
        '''Get the Admin Node IP'''
        if not line:
            raise CommandError('no machine name provided')

        with self._operation():
            name = tf_find_for(line)
            ips = tf_vms_ip_for(name)
            if not ips:
                raise InvalidMachineError(
                    'could not find a valid IP for {}'.format(name))

            print(ips[0])

    def do_num(self, line):
        '''
        Get the number of VMs in the cluster.
        (as shown in the Terraform state file)
        '''
        with self._operation():
            res = tf_load_state()
            print(tf_vms_num(res))

    def do_names(self, line):
        '''
        Get the list of VMs names
        (as shown in the Terraform state file)
        '''
        with self._operation():
            res = tf_load_state()
            print(" ".join(tf_vms_names(res)))

    def do_ids(self, line):
        '''
        Get the list of VMs machine-ids
        '''
        with self._operation():
            res = tf_load_state()
            names = list(tf_vms_names(res))
            for node, node_id in get_machine_ids(names, ignore_errors=True).items():
                log.info('%s: %s', node, node_id)

    def do_status(self, line):
        '''
        Get the status of all the VMs
        '''
        with self._operation():
            res = tf_load_state()
            names = list(tf_vms_names(res))
            for vm in names:
                status = virsh_status_for(vm)
                log.info('%s: %s', vm, status)

    def do_map(self, line):
        '''
        Get a list of VMs as a map
        (as shown in the Terraform state file)
        '''
        with self._operation():
            res = tf_load_state()
            print(tf_vms_map(res))

    def do_snapshot(self, line):
        '''
        Create snapshots for all the VMs in the cluster.

        Optional arguments (must be comma separated):

        * stage: a short stage name
        * description: a long description of this snapshot

        Usage:

        > snapshot stage='post-apply', description='Snapshot after apply'
        '''
        with self._operation('snapshot',
                             pre_msg='Snapshotting the cluster',
                             post_msg='Cluster snapshotted',
                             icon='revert'):
            log.info('Creating snapshots')
            kwargs = get_assign_from_str(line)
            # suspend, snapshot and resume (all VMs at the same time)
            map_all_vms_par(virsh_create_snapshot_for_vm,
                            pre_suspend=True, **kwargs)
            map_all_vms_par(virsh_resume_for_vm)

    def do_snapshots(self, line):
        '''
        Get the list of snapshots for the VMs.
        '''
        with self._operation():
            for node, snap in map_vms(virsh_get_snapshots_for_vm):
                log.info('%s: %s', node, snap)

    def do_snapshots_tree(self, line):
        '''
        Get the tree of snapshots for the VMs.
        '''
        with self._operation():
            map_all_vms_par(virsh_get_snapshots_tree_for_vm)

    def do_snapshots_destroy(self, line):
        '''
        Detroy all the snapshots for the VMs.
        '''
        with self._operation():
            map_all_vms_par(virsh_destroy_snapshots_for_vm)

    def do_snapshots_drop(self, line):
        '''
        Detroy a particular snapshot in all the VMs

        Usage:

        > snapshots_drop post-orch
        '''
        name = line.strip()
        if not name:
            raise CommandError('no snapshot name provided')

        with self._operation():
            log.info('Dropping all the "%s" snapshots', name)
            map_all_vms_par(virsh_destroy_snapshots_for_vm, only_snapshot=name)

    def do_drop(self, line):
        '''
        Alias for `snapshots_drop`
        '''
        return self.do_snapshots_drop(line)

    def do_suspend(self, line):
        '''
        Suspend the VMs.
        '''
        with self._operation('suspend',
                             pre_msg='Suspending the cluster',
                             post_msg='Cluster suspended',
                             icon='down'):
            map_all_vms_par(virsh_suspend_for_vm)

    def do_rollback(self, line):
        '''
        Rolling back all the VMs.

        Optional arguments:

        * `running`: resume the machine after rolling back.
        * `name`: rollback to the snapshot named `name`
        * `vm`: rollback only a particular VM
        '''
        kwargs = {'running': True}
        kwargs.update(get_assign_from_str(line))

        with self._operation('rollback',
                             pre_msg='Rolling back the cluster',
                             post_msg='Cluster rolled back',
                             icon='undo'):

            vm = kwargs.pop('vm', None)
            post_running = kwargs.pop('running', True)
            if vm:
                log.info('Rolling back %s', vm)
                real_vm = tf_find_for(vm)
                print_iterator(virsh_rollback_for_vm(real_vm, post_running=post_running, **kwargs))
                # make sure the VMs are running: otherwise, refresh will fail
                print_iterator(virsh_resume_for_vm(real_vm))
            else:
                log.info('Rolling back all the VMs')
                map_all_vms_par(virsh_rollback_for_vm,
                                post_running=post_running, **kwargs)
                map_all_vms_par(virsh_resume_for_vm)
                print_iterator(tf_refresh())

            sync_hwtime(ignore_errors=True)

    def do_power_on(self, line):
        '''
        Power-on the VMs.
        '''
        with self._operation('power-on',
                             pre_msg='Powering on the cluster',
                             post_msg='Cluster powered up',
                             icon='up'):
            map_all_vms_par(virsh_power_on_for_vm)
            print_iterator(tf_refresh())
            sync_hwtime()

    def do_resume(self, line):
        '''
        Resume the VMs.
        '''
        with self._operation('resume',
                             pre_msg='Resuming the cluster',
                             post_msg='Cluster resumed',
                             icon='forward'):
            map_all_vms_par(virsh_resume_for_vm)
            print_iterator(tf_refresh())
            sync_hwtime()

    def do_reboot(self, line):
        '''
        Reboot a VM.

        Optional arguments:

        * wait_for=<CONT>: wait for reboot and a container to be up

        Usage:

        > cluster reboot node-1
        > cluster reboot admin wait_for=salt
        '''
        line_comps = line.split(' ')
        kwargs = get_assign_from_str(' '.join(line_comps[1:]))

        vm = tf_find_for(line[0])

        with self._operation('reboot',
                             pre_msg='Rebooting ' + vm,
                             post_msg=vm + ' rebooted',
                             icon='revert'):
            print_iterator(virsh_reboot(vm))
            wait_for = kwargs.get('wait_for', '')
            if wait_for:
                log.info('Waiting for "%s" container in "%s"', wait_for, vm)
                time.sleep(5)
                print_iterator(wait_container(wait_for, vm=vm))
                log.info(
                    'The "%s" container seems to be available in "%s"', wait_for, vm)

###################
# Salt stuff
###################


class CaaSPSalt(CmdBase):

    prompt = prompt('caasp:salt')

    def default(self, line):
        '''
        Run a raw Salt command in the Salt master in the Admin Node

        Usage:

        > salt -C 'G@roles:kube-master' test.ping
        '''
        with self._operation():
            print_iterator(copy_to_admin())
            run_ssh_session(tf_vms_admin()[0], 'caaspctl salt ' + line)

    def do_sync(self, line):
        '''Perform a Salt synchronization.'''
        with self._operation():
            print_iterator(copy_to_admin())
            print_iterator(run_caaspctl_admin(CAASPCTL_SALT_SYNC))
            sync_hwtime()

    def do_attach(self, line):
        '''Attach to the Salt Master output.'''
        with self._operation():
            print_iterator(copy_to_admin())
            run_ssh_session(tf_vms_admin()[0], 'caaspctl salt attach')

    def do_wait(self, line):
        '''
        Wait for Salt minions to be accepted (by default, as many clients as nodes VMs)
        It will really wait for +2 minions: the 'ca' and 'admin'

        Optional argumens:

        * num: number of Salt minions to wait for
        '''
        kwargs = get_assign_from_str(line)
        num = kwargs.get('num', 0)
        if num:
            wait_for = int(num)
        else:
            nodes = tf_vms_nodes()
            wait_for = len(nodes.keys())

        log.info('waiting for %s (+%d) Salt minions to be accepted',
                 wait_for, SALT_KEYS_EXTRA_WAIT)

        # add the 'ca' and 'admin' Salt minions
        wait_for += SALT_KEYS_EXTRA_WAIT

        with self._operation():
            try:
                wait_ssh()
                print_iterator(copy_to_admin())
                print_iterator(run_caaspctl_admin(
                    CAASPCTL_MINIONS_ACCEPT.format(num=wait_for)))
            except SSHException as e:
                log.critical('could not wait for Salt minions: %s', e)
                raise

    def do_set_pillar(self, line):
        '''
        Set a pillar.

        Usage:

        > set_pillar api:server:external_fqdn 192.168.122.4
        '''
        line_comps = line.split(' ')
        if len(line_comps) != 2:
            raise CommandError(
                'setting pillars requires two arguments: the key and the value')

        key, value = line_comps
        log.info('Setting the %s to %s', key, value)
        with self._operation():
            print_iterator(copy_to_admin())
            print_iterator(set_pillar(key, value))

    def do_get_pillar(self, line):
        '''
        Get some pillar(s) value(s)
        '''
        if line:
            line_comps = line.split(' ')
            if len(line_comps) != 2:
                raise CommandError(
                    'get pillar requires two arguments: the key and the value')

            key, where = line_comps
            log.info('Getting %s at %s', key, where)
        else:
            key, where = '', ''
            log.info('Getting all the pillars')

        with self._operation():
            print_iterator(copy_to_admin())
            print_iterator(run_caaspctl_admin(
                CAASPCTL_GET_PILLAR.format(**locals())))

    def do_apply(self, line):
        '''
        Apply some state at some machines

        Usage:

        > salt apply masters etc-hosts
        > salt apply "G@roles:ca" haproxy
        > salt apply 'G@caasp_etcd_member' etcd.remove-pre-stop-services
        '''
        with self._operation():
            print_iterator(copy_to_admin())
            print_iterator(copy_to_admin_code())
            print_iterator(run_caaspctl_admin('salt apply ' + line))

    def do_high(self, line):
        '''
        Apply a "highstate" on the machines that match <WHERE>

        Usage:

        > salt high minions high pillar='{"foo":"bar"}'
        > salt high *
        '''
        with self._operation():
            print_iterator(copy_to_admin())
            print_iterator(copy_to_admin_code())
            print_iterator(run_caaspctl_admin('salt high ' + line))

###################
# logs
###################


class CaaSPLogs(CmdBase):

    prompt = prompt('caasp:logs')

    def do_salt(self, line):
        '''Dump the Salt logs.'''
        with self._operation():
            print_iterator(copy_to_admin())
            print_iterator(run_caaspctl_admin('salt logs ' + line), flush=True)

    def do_events(self, line):
        '''Dump the Salt events.'''
        with self._operation():
            print_iterator(copy_to_admin())
            print_iterator(run_caaspctl_admin('db events ' + line), flush=True)


###################
# caaspctl
###################

class CaaSPCtl(CmdBase):

    prompt = prompt('caasp:ctl')

    def default(self, line):
        line = line.strip()
        if line:
            with self._operation():
                print_iterator(copy_to_admin())
                print_iterator(run_caaspctl_admin(line))

    def do_enter(self, line):
        with self._operation():
            print_iterator(copy_to_admin())
            run_ssh_session(tf_vms_admin()[0], args.vm_caaspctl)

    def do_sh(self, line):
        with self._operation():
            print_iterator(copy_to_admin())
            if line:
                print_iterator(run_caaspctl_admin(line))
            else:
                admin_name, admin_ip = tf_vms_admin()
                log.debug('Starting %s in the Admin node %s',
                          args.vm_caaspctl, admin_name)
                run_ssh_session(admin_name, args.vm_caaspctl)


###################
# Registry
###################


class CaaSPRegistry(CmdBase):

    prompt = prompt('caasp:registry')

    def do_start(self, line):
        '''
        Start a docker registry

        > registry start
        '''
        log.info('registry: starting')
        print_iterator(registry_start())
        log.info(
            'registry: the registry is up and running at port %d...', REGISTRY_PORT)

    def do_stop(self, line):
        '''
        Stop the docker registry

        Usage:

        > registry stop
        '''
        log.info('registry: stopping')
        print_iterator(registry_stop())
        log.info('registry: the registry is stopped...')

    def do_refresh(self, line):
        '''
        Refresh the node's vieew of the local registry

        Usage:

        > registry refresh
        '''
        log.info('registry: refreshing')
        print_iterator(registry_refresh())

    def do_pull(self, line):
        '''
        Pull an image from the Docker registry to the local registry

        Usage:

        > registry pull alpine
        '''
        line_comps = line.split(' ')
        if len(line_comps) != 1:
            raise CommandError('invalid number of arguments')

        name = line_comps[0]

        log.info('registry: pulling "%s" to the local registry', name)
        print_iterator(registry_pull(name))
        log.info('registry: image "%s" pulled to the local registry...', name)

    def do_import(self, line):
        '''
        Import an image to the local registry

        Usage:

        > registry import alpine 2c9365b8a85d
        '''
        line_comps = line.split(' ')

        log.info('registry: importing %s', line_comps[0])
        print_iterator(registry_import(*line_comps))
        log.info('registry: image %s imported...', line_comps[0])

    def do_rmi(self, line):
        '''
        Remove an image from the local registry
        '''
        if not line:
            raise CommandError('invalid number of arguments')

        log.info('registry: removing %s', line)
        print_iterator(registry_rmi(line))

    def do_catalog(self, line):
        '''
        '''
        log.info('registry: dumping catalog')
        print_iterator(registry_catalog())

###################
# Orchestrations
###################


class CaaSPOrch(CmdBase):

    prompt = prompt('caasp:orch')

    def _prepare_orchestration(self, orch, orch_args):
        notify('preparing', summary='pre-orchestration', icon='start')
        log.info('Preparing orchestration...'.format(**locals()))

        print_iterator(copy_enable())
        print_iterator(copy_to_admin())
        print_iterator(copy_to_admin_code())

        log.info('Running pre-orchestration commands...')

        date = datetime.now().strftime(DATE_FMT)
        date_fmt = DATE_FMT

        line = ' ; '.join(ORCH_PREPARE_NODES_SHELL_CMDS)
        line = line.format(**locals())

        try:
            log.debug('Running "%s" in all the machines', line)
            run_ssh_all(line)
        except Exception as e:
            log.error('could not run pre-orch commands: %s', e)

        notify('resources copied', summary='pre-orchestration', icon='start')

        for ctl_cmd in ORCH_PREPARE_CAASPCTL_CMDS:
            print_iterator(run_caaspctl_admin(ctl_cmd.format(**locals())))

    def _run_orchestration(self, orch, orch_args, prepare=True,
                           pre_orch_rcs=True,  post_orch_rcs=True):
        assert(orch)

        if pre_orch_rcs and not args.skip_rc_files:
            self.try_stage_rc_files('pre-orch')

        if prepare:
            self._prepare_orchestration(orch, orch_args)

        notify('starting', summary=orch + ' orchestration', icon='start')
        log.info('Starting {orch} orchestration...'.format(**locals()))

        ctl_cmd = 'orch {orch} {orch_args}'.format(**locals())
        print_iterator(run_caaspctl_admin(ctl_cmd))

        notify('finish', summary=orch + ' orchestration', icon='start')
        remove_kubeconfig()

        if post_orch_rcs and not args.skip_rc_files:
            self.try_stage_rc_files('post-orch')

    def default(self, line):
        if line:
            line_comps = line.split(' ')
            orch = line_comps[0]
            orch_args = ' '.join(line_comps[1:])
            self._run_orchestration(orch, orch_args)

    def do_boot(self, line):
        '''Run the bootstrap orchestration.'''
        self._run_orchestration(DEFAULT_ORCHESTRATION, line)

    def do_update(self, line):
        '''
        Run the update orchestration.

        Use 'fake=true' for doing a fake update by setting the update-is-needed
        flag in all the machines in the cluster
        '''
        kwargs = get_assign_from_str(line)

        prepare = True
        if str2bool(kwargs.get('fake', 'false')):
            self._run_orchestration('update_set_needed', '',
                                    prepare=prepare, post_orch_rcs=False)
            prepare = False

        log.info('updating the cluster')
        self._run_orchestration(
            'update', line, prepare=prepare, pre_orch_rcs=False)

    def do_update_set_needed(self, line):
        '''Set the update-is-needed flag in all the machines in the cluster.'''
        self._run_orchestration('update_set_needed', line)

    def do_rm(self, line):
        '''Run the removal orchestration.'''
        if len(line) < 1:
            raise CommandError(
                'the removal orchestration needs some node(s) as an argument')

        nodes = line.split(' ')
        ids = list(get_machine_ids(nodes).values())
        assert(ids)
        assert(all([isinstance(x, str) for x in ids]))

        orch_args = ' '.join(ids)
        log.info('Removing nodes: %s', ids)
        self._run_orchestration('rm', orch_args)

    def do_add(self, line):
        '''Run the addition orchestration.'''
        if len(line) < 1:
            raise CommandError(
                'the addition orchestration needs some node(s) as an argument')

        nodes = line.split(' ')
        ids = list(get_machine_ids(nodes).values())
        assert(ids)
        assert(all([isinstance(x, str) for x in ids]))

        log.info('Adding nodes: %s', ids)
        orch_args = ' '.join(ids)
        self._run_orchestration('add', orch_args)

    def do_kubeconfig(self, line):
        '''Get the kubeconfig for the latest orchestration.'''
        log.info('Getting kubeconfig')
        print_iterator(copy_to_admin())
        print_iterator(get_kubeconfig())


###################
# Copies
###################


class CaaSPCopies(CmdBase):

    prompt = prompt('caasp:copy')

    def do_all(self, line):
        '''Perform all the copies.'''
        print_iterator(copy_enable())
        print_iterator(copy_to_admin())
        print_iterator(copy_to_admin_code())
        print_iterator(copy_to_nodes())

    def do_admin(self, line):
        '''
        Perform all the copies necessary to the admin node.

        NOTE: this will reboot the Admin Node when copy for Manifests has been enabled.
        '''
        kwargs = get_assign_from_str(line)

        print_iterator(copy_enable(**kwargs))
        print_iterator(copy_to_admin(**kwargs))
        print_iterator(copy_to_admin_code(**kwargs))

    def do_nodes(self, line):
        '''Perform all the copies necessary for the nodes.'''
        kwargs = get_assign_from_str(line)

        print_iterator(copy_to_nodes(**kwargs))

    def do_to(self, line):
        '''
        Copy files to a machine.

        The destination can be:

        * a directory/full-path (as <dest>)
        * a directory/full-path inside a container (as <cont>:<dest>)

        When no file is specified, it will perform all the copies
        necessary for that machine. Copies will be different
        depending on the machine (admin/node).

        Usage:

        > copy to node-1
        > copy to admin
        > copy to node-2 file.txt
        > copy to node-2 file.txt /tmp/
        > copy to admin file.txt ldap:/tmp/file.txt
        '''
        if not line:
            raise CommandError('no VM provided')

        line_comps = line.split(' ')

        vm = line_comps[0]
        full_vm = tf_find_for(vm)

        orig = ''
        dest = ''
        container = None
        if len(line_comps) > 1:
            orig_expr = line_comps[1]
            orig = orig_expr.strip()
            if len(line_comps) > 2:
                dest_expr = line_comps[2].split(':')
                if len(dest_expr) == 1:
                    dest = dest_expr[0].strip()
                else:
                    container = dest_expr[0].strip()
                    dest = dest_expr[1].strip()

        if orig:
            print_iterator(copy_file_to(orig, full_vm, dest, container))
        else:
            if full_vm == tf_vms_admin()[0]:
                print_iterator(copy_to_admin())
                print_iterator(copy_to_admin_code())
            else:
                print_iterator(copy_to_node(full_vm))

    def do_file_from(self, line):
        '''
        Copy a file from a remote machine

        Usage:

        > copy_from node-1 /etc/hosts
        > copy_from node-1 /etc/hosts dst=/tmp/copied
        > copy_from node-1 /etc/hosts dst=/tmp/copied, orig_user=caasp
        '''
        if len(line) < 2:
            raise CommandError('invalid arguments for the file_from command')

        comps = line.split(' ')
        vm = comps[0]
        path = comps[1]

        kwargs = {}
        if len(comps) > 2:
            kwargs = get_assign_from_str(comps[2:])

        print_iterator(copy_file_from(vm=vm, src=path, **kwargs))

    def do_rpms_to(self, line):
        '''
        Copy and install all the RPMs in a directory to a
        machine in the cluster.

        Optional arguments:

        * `src`: source directory for all the rpms (default: ./rpms/)
        * `dst`: temporal directory where copy RPMs to
        * `forced`: forced the installation of packages

        Usage:

        > rpms_to node-1
        > rpms_to node-1 src=rpms, dst=/tmp/rpms
        > rpms_to node-1 src=rpms, forced=True
        '''
        if len(line) < 2:
            raise CommandError('invalid arguments for the rpms_to command')

        comps = line.split(' ')
        vm = tf_find_for(comps[0])
        kwargs = get_assign_from_str(' '.join(comps[1:]))

        log.info('Installing RPMS in %s', vm)
        print_iterator(copy_to_admin())
        print_iterator(install_rpms_in(vm, **kwargs))

    def do_rpms_to_all(self, line):
        '''
        Copy all the RPMs to all the machines

        Optional arguments:

        * `src`: source directory for all the rpms
        * `dst`: temporal directory where copy RPMs to
        * `forced`: forced the installation of packages

        Usage:

        > rpms_to_all
        > rpms_to_all src=rpms, dst=/tmp/rpms
        > rpms_to_all src=rpms, forced=True
        '''
        kwargs = get_assign_from_str(line.strip())
        log.info('Installing RPMS in all the machines')
        print_iterator(copy_to_admin())
        print_iterator(install_rpms_in_all(**kwargs))

###################
# Tests
###################


class CaaSPTest(CmdBase):

    prompt = prompt('caasp:test')

    def __init__(self, top=None):
        super().__init__(top)
        self.passed = set()
        self.failed = set()

    def _passed(self, line):
        log.info('PASSED: %s', line)
        self.passed.add(line)
        try:
            self.passed.remove(line)
        except KeyError:
            pass

    def _failed(self, line):
        log.error('FAILED: %s', line)
        self.failed.add(line)
        try:
            self.passed.remove(line)
        except KeyError:
            pass

    def do_cmd(self, line):
        '''
        Run a command.
        You can do something like

        > cmd [ $(ls | wc -l) = 21 ]
        > cmd grep "caasp" terraform.tf
        '''
        if run_interactive(line) != 0:
            self._failed(line)
        else:
            self._passed(line)

    def do_cmd_at(self, line):
        '''
        Run a test on a machine

        Usage:

        > cmd_at node-1 [ kubectl get nodes ]
        '''
        line_comps = line.split(' ')
        if len(line_comps) < 2:
            raise CommandError('wrong number of arguments for sh_on')

        vm = tf_find_for(line_comps[0])
        cmd = ' '.join(line_comps[1:])
        try:
            print_iterator(run_ssh_cmd(vm, cmd))
        except:
            self._failed(line)
        else:
            self._passed(line)

    def do_cmd_at_nodes(self, line):
        '''
        Run a test in the nodes (ie, all machines except the admin node) VMs.

        Usage:

        > cmd_at_nodes [ kubectl get nodes ]
        '''
        line_comps = line.split(' ')
        if len(line_comps) < 1:
            raise CommandError('wrong number of arguments for sh_on')

        nodes = tf_vms_nodes()
        for vm in nodes:
            try:
                print_iterator(run_ssh_cmd(vm, line))
            except:
                self._failed(line)
            else:
                self._passed(line)

    def do_summary(self, line):
        '''
        Print a summary of failures/passed
        '''
        if len(self.passed) > 0:
            for test in self.passed:
                log.info('PASSED: %s', test)

        if len(self.failed) > 0:
            for test in self.failed:
                log.info('FAILED: %s', test)

    def postloop(self):
        log.info('%s tests passed, %d failed',
                 len(self.passed), len(self.failed))


###################
# Main
###################

class CaaSPDevel(CmdBase):

    prompt = prompt('caasp:devel')

    def do_enable(self, line):
        '''
        Enable development mode
        When enabled, all the sources directories (Salt and Manifests) will be
        scheduled for being copied to the Admin Node. However, you can only enable
        this feature for some of the directories with the arguments.

        Usage:

        > # link the development profile and copy all the directories
        > devel enable
        > # we will copy only the Salt directory
        > devel enable salt=True
        '''
        def guess_dir(maybe_dirs, descr):
            log.info('devel: %s directory not provided: guessing...', descr)
            for maybe_dir in maybe_dirs:
                maybe_dir_abs = os.path.abspath(
                    os.path.join(CURR_DIR, maybe_dir))
                if os.path.exists(maybe_dir_abs):
                    log.info('devel: %s directory found at %s',
                             descr, maybe_dir_abs)
                    return maybe_dir_abs

            log.critical(
                'devel: %s directory neither provided nor found', descr)
            raise Exception(
                '{} directory neither provided nor found'.format(descr))

        if line:
            kwargs = get_assign_from_str(line)

            if kwargs.get('salt', False):
                args.copy_salt_code = True

            if kwargs.get('manifests', False):
                args.copy_manifests = True

        if args.copy_salt_code:
            log.debug('devel: will copy the Salt code')
            if not args.salt_dir:
                args.salt_dir = guess_dir(MAYBE_DIRS_SALT, 'Salt code')

            if args.salt_branch:
                log.info('devel: will use Salt branch "%s"', args.salt_branch)

        if args.copy_manifests:
            log.debug('devel: will copy the manifests')
            if not args.manifests_dir:
                args.manifests_dir = guess_dir(
                    MAYBE_DIRS_MANIFESTS, 'manifests')

            if args.manifests_branch:
                log.info('devel: will use manifests branch "%s"',
                         args.manifests_branch)

        local_devel = os.path.abspath(os.path.join(
            os.getcwd(), os.path.basename(args.tf_devel_profile)))
        try:
            devel_profile = os.path.abspath(args.tf_devel_profile)
            if not os.path.exists(local_devel):
                log.info('devel: creating symbolic link %s -> %s',
                         local_devel, devel_profile)
                create_link(local_devel, devel_profile)
            else:
                log.info('devel: devel profile symbolic link already exists')
        except FileExistsError:
            log.debug('devel: symbolic link already exists')

    def do_disable(self, line):
        '''
        Disable the development mode

        Usage:

        > devel disable
        '''
        local_devel = os.path.basename(args.tf_devel_profile)
        if os.path.exists(local_devel):
            log.info('devel: disabled')
            os.remove(local_devel)

        log.info('devel: disabling devel mode')
        args.copy_salt_code = False
        args.copy_manifests = False

    def do_branch(self, line):
        '''
        Use a specific branch for a component

        Components can be 'salt', 'manifests', etc...

        The special branch name 'RESET' will reset
        the branch to the current checkout.

        Usage:

        > devel branch salt release-2.1

        Example:

        > # Check we can upgrade from 2.0 to the current
        > # changes in my worktree:
        > dev branch salt release-2.0
        > orch boot
        > devel branch salt RESET
        > orch update
        '''
        line_comps = line.strip().split(' ')
        if len(line_comps) < 2:
            raise CommandError('insufficient arguments in branch command')

        component = line_comps[0].strip().lower()
        branch = line_comps[1].strip().lower()

        if component == 'salt':
            if not args.copy_salt_code:
                raise CommandError(
                    'Salt code is not going to be copied (has the devel mode been enabled?)')

            if not args.salt_dir:
                raise CommandError('Salt code has not been found/specified')

            if branch in ['reset', 'checkout', 'current', 'worktree']:
                log.info('setting the Salt branch to the current working tree')
                args.salt_branch = None
            else:
                log.info('setting the Salt branch to "%s"', branch)
                args.salt_branch = branch

        elif component in ['manifest', 'manifests']:
            if not args.copy_manifests:
                raise CommandError(
                    'manifests are not going to be copied (has the devel mode been enabled?)')

            if not args.manifests_dir:
                raise CommandError('manifests has not been found/specified')

            if branch in ['reset', 'checkout', 'current', 'worktree']:
                log.info(
                    'setting the manifests branch to the current working tree')
                args.manifests_branch = None
            else:
                log.info('setting the manifests branch to "%s"', branch)
                args.manifests_branch = branch

###################
# Main
###################


class CaaSP(CmdBase):
    """ CaaSP command line """

    prompt = prompt('caasp')
    intro = "CaaSP/libvirt cluster control tool.\n"

    def __init__(self):
        super().__init__()
        self.registry = CaaSPRegistry(self)
        self.cluster = CaaSPCluster(self)
        self.ssh = CaaSPSSH(self)
        self.salt = CaaSPSalt(self)
        self.orch = CaaSPOrch(self)
        self.ctl = CaaSPCtl(self)
        self.logs = CaaSPLogs(self)
        self.copies = CaaSPCopies(self)
        self.tests = CaaSPTest(self)
        self.devel = CaaSPDevel(self)

    def _subcommand(self, sub_cmd, line):
        if len(line) > 0:
            sub_cmd.onecmd(line)
        else:
            sub_cmd.cmdloop()

    def do_registry(self, line):
        '''Docker registry commands.'''
        self._subcommand(self.registry, line)

    def do_cluster(self, line):
        '''Manage the cluster.'''
        self._subcommand(self.cluster, line)

    def do_ssh(self, line):
        '''Run ssh commands in a VMs.'''
        self._subcommand(self.ssh, line)

    def do_salt(self, line):
        '''Salt commands.'''
        self._subcommand(self.salt, line)

    def do_orch(self, line):
        '''Orchestration commands.'''
        self._subcommand(self.orch, line)

    def do_ctl(self, line):
        '''Caaspctl commands in the Admin node.'''
        self._subcommand(self.ctl, line)

    def do_logs(self, line):
        '''Salt commands.'''
        self._subcommand(self.logs, line)

    def do_copy(self, line):
        '''Copies to the VMs.'''
        self._subcommand(self.copies, line)

    def do_test(self, line):
        '''Run some simple tests.'''
        self._subcommand(self.tests, line)

    def do_devel(self, line):
        '''Development tools.'''
        self._subcommand(self.devel, line)

    #
    # extra
    #

    def do_dashboard(self, line):
        '''
        Open the CaaSP Dashboard in a web browser
        '''
        admin_name, admin_ip = tf_vms_admin()
        dash_url = 'https://{admin_ip}'.format(**locals())
        log.info('Opening %s', dash_url)
        run_interactive('xdg-open {dash_url}'.format(**locals()))

    def do_helm(self, line):
        '''
        Initialize Helm
        '''
        admin_name, admin_ip = tf_vms_admin()
        log.info('Initializing Helm...')
        print_iterator(run_ssh_cmd(admin_name, 'helm init --client-only'))
        log.info('Updating Helm repos...')
        print_iterator(run_ssh_cmd(admin_name, 'helm repo update'))
        log.info(
            'Helm has been initialized... you can do things like `helm search mongo` now')

    def do_k8s_dashboard(self, line):
        '''
        Install and run the k8s dashboard

        NOTE: this requires

          * a local Helm client
          * Helm to have been initialized
        '''
        name = 'kubernetes-dashboard'
        port = K8S_DASHBOARD_PORT
        chart = K8S_DASHBOARD_HELM_CHART
        ns = K8S_DASHBOARD_NAMESPACE
        user = K8S_DASHBOARD_USER
        admin_name, admin_ip = tf_vms_admin()

        log.info('Checking if the dashboard is already installed...')
        self.check_kubeconfig()
        self.check_kubectl()
        self.check_helm()

        def get_dash_pod(**lkwargs):
            pod = None
            try:
                log.debug('Getting the k8s dashboard pod...')
                cmd = "kubectl get pods --namespace {ns} -l 'app={name}'".format(
                    **lkwargs)
                cmd += " -o jsonpath='{.items[0].metadata.name}'"
                for line in run(cmd):
                    pod = line
                    break

                if 'kubernetes-dashboard' not in pod:
                    pod = None
            except Exception as e:
                log.error('Could not get k8s dashboard pod: %s', e)
                pod = None

            if pod:
                log.debug('pod: %s', pod)

            return pod

        pod = get_dash_pod(**locals())
        if not pod:
            # TODO: we should check Helm has been initialized
            log.info('The dashboard was not installed... installing now')
            cmd = "helm install --replace --name {name} {chart}".format(
                **locals())
            cmd += " --namespace {ns}".format(**locals())
            cmd += " --set rbac.create=true"
            print_iterator(run(cmd))

            pod = get_dash_pod(**locals())
            if not pod:
                raise CommandError('no dashboard pod could be found')

        # 'caaspctl orch kubeconfig' generated a "admin-user" account and "cluster-admin" role
        # we "describe" that service account for getting the "token:"
        # see https://github.com/kubernetes/dashboard/wiki/Creating-sample-user
        log.info('Getting bearer token (used for logging into the Dashboard)')
        cmd = "kubectl -n {ns} describe secret $(kubectl -n {ns} get secret | grep {user} | cut -f1 -d' ') | grep token:".format(
            **locals())
        output = list(run(cmd))
        log.info('%s', output[0])

        log.info('Waiting for the Dashboard to be ready...')
        cmd = 'kubectl rollout status --namespace {ns} deployment/{name}'.format(
            **locals())
        print_iterator(run(cmd))

        k8s_dash_url = 'https://127.0.0.1:{port}'.format(**locals())

        log.info('Kubernetes dashboard will be available at %s in a few seconds...',
                 k8s_dash_url)
        cmd = 'kubectl --namespace {ns} port-forward {pod} {port}:8443 --pod-running-timeout=10m'.format(
            **locals())
        try:
            while True:
                # the kubectl port-forward breaks lots of times, so loop
                print_iterator(run(cmd))
                log.debug('... port-forward stopped: restarting')
        except KeyboardInterrupt:
            pass

    def do_kubectl(self, line):
        '''Run kubectl with the cluster.'''
        print_iterator(copy_to_admin())

        self.check_kubeconfig()
        self.check_kubectl()

        cmd = 'kubectl --kubeconfig={} {}'.format(args.kubeconfig, line)
        if run_interactive(cmd) != 0:
            log.error('kubectl failed')

    def do_version(self, line):
        '''Print the version.'''
        print(VERSION)

    def do_env(self, line):
        '''Set the environment.'''
        if line:
            env = line.strip()
            log.info('Setting environment: %s', env)
            args.env = env
        else:
            log.info('Current environment: %s', args.env)

#############################################################
# Main
#############################################################


if __name__ == '__main__':
    saved_args = load_caasp_last_run()
    if args.last_env:
        args.env = saved_args['env']

    try:
        caasp_cmd = CaaSP()

        if not args.skip_rc_files:
            log.debug('trying to load RC files')
            caasp_cmd.try_rc_files(
                CAASP_RC_FILES + CAASP_RC_FILES_ABS, directory=CURR_DIR)
            caasp_cmd.try_rc_files(CAASP_RC_FILES, directory=get_work_dir())

        if len(args.args) > 0 and args.commands_pre:
            caasp_cmd.command_line_args(args.args)

        if args.script:
            if args.script_begin:
                log.info('Will start execution at stage "%s"', args.script_begin)
                caasp_cmd.blocked = True

            for script in args.script:
                try:
                    caasp_cmd.load_script(script)
                except Exception as e:
                    log.critical('Could not read file %s: %s', script, e)
                    sys.exit(1)

            caasp_cmd.blocked = False

            if args.script_only:
                log.debug('we were running only scripts: exitting...')
                sys.exit(0)

        if len(args.args) > 0 and not args.commands_pre:
            caasp_cmd.command_line_args(args.args)

        if not args.args or args.loop:
            caasp_cmd.cmdloop()

    finally:
        save_caasp_last_run({'env': args.env})