Make Ansible Dir Configurable

tests/bgp/test_bgp_vnet.py

@@ -141,7 +141,7 @@ def restore_config_db(localhost, duthost):
 
 @pytest.fixture(scope="module", autouse=True)
 def setup_vnet(tbinfo, duthosts, rand_one_dut_hostname, ptfhost, localhost,
-              skip_test_module_over_backend_topologies):        # noqa F811
+              skip_test_module_over_backend_topologies, ansible_root):        # noqa F811
     duthost = duthosts[rand_one_dut_hostname]
 
     # backup config_db.json
@@ -202,7 +202,7 @@ def setup_vnet(tbinfo, duthosts, rand_one_dut_hostname, ptfhost, localhost,
         duthost.shell("sonic-clear nd")
         duthost.shell("sonic-clear fdb all")
 
-        with open("../ansible/vars/topo_{}.yml".format(tbinfo['topo']['name']), 'r') as fh:
+        with open(ansible_root.joinpath("vars/topo_{}.yml".format(tbinfo['topo']['name'])), 'r') as fh:
             g_vars['topo_properties'] = yaml.safe_load(fh)
 
         g_vars['props'] = g_vars['topo_properties']['configuration_properties']['common']

tests/common/devices/aos.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import pathlib
 
 from tests.common.devices.base import AnsibleHostBase
 import re
@@ -31,7 +32,9 @@ def __init__(self, ansible_adhoc, hostname, user, passwd, gather_facts=False):
 
     def _exec_jinja_template(self, task_name, jinja_template):
         inventory = 'lab'
-        ansible_root = '../ansible/'
+        ansible_root = str(pathlib.Path(os.getenv(
+            "ANSIBLE_CONFIG", pathlib.Path(__file__).joinpath("../../../ansible"))).resolve()) + "/"
+
         playbook_name = 'accton_os_cmd_exec.yml'
         jinja_name = 'accton_os_cmd_exec.j2'
         playbook_text = '- hosts: {}\n'.format(self.hostname) + \

tests/common/fixtures/conn_graph_facts.py

@@ -3,6 +3,7 @@
 import six
 import yaml
 import copy
+import pathlib
 
 
 @pytest.fixture(scope="module")
@@ -74,8 +75,9 @@ def get_graph_facts(duthost, localhost, hostnames):
     duthost - pytest fixture
     hostnames - can be either a single DUT or a list of multiple DUTs
     """
-    base_path = os.path.dirname(os.path.realpath(__file__))
-    lab_conn_graph_path = os.path.join(base_path, "../../../ansible/files/")
+    ansible_config_path = pathlib.Path(os.getenv("ANSIBLE_CONFIG",
+                                                 pathlib.Path(__file__).resolve().parent.joinpath("../../ansible")))
+    lab_conn_graph_path = os.path.join(ansible_config_path, "files/")
 
     inv_files = duthost.host.options["inventory_manager"]._sources
     graph_groups_file = os.path.join(lab_conn_graph_path, "graph_groups.yml")

tests/common/fixtures/pfc_asym.py

@@ -1,6 +1,7 @@
 import pytest
 import os
 import time
+import pathlib
 
 from netaddr import IPAddress
 from tests.common.helpers.generators import generate_ips
@@ -10,12 +11,13 @@
 PFC_FRAMES_NUMBER = 50000000
 PFC_QUEUE_INDEX = 0xff
 
-ANSIBLE_ROOT = os.path.normpath((os.path.join(__file__, "../../../ansible")))
 RUN_PLAYBOOK = os.path.realpath(os.path.join(os.path.dirname(__file__), "../../scripts/exec_template.yml"))
 
 OS_ROOT_DIR = "/root"
 TESTS_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "../.."))
-ANSIBLE_ROOT = os.path.realpath(os.path.join(TESTS_ROOT, "../ansible"))
+ANSIBLE_ROOT = pathlib.Path(os.getenv("ANSIBLE_CONFIG",
+                                      pathlib.Path(__file__).resolve().parent.joinpath("../../ansible")))
+
 
 ARP_RESPONDER = os.path.join(TESTS_ROOT, "scripts/arp_responder.py")
 ARP_RESPONDER_CONF = os.path.join(TESTS_ROOT, "templates/arp_responder.conf.j2")

tests/common/helpers/dut_utils.py

@@ -1,4 +1,5 @@
 import logging
+import pathlib
 import allure
 import os
 import jinja2
@@ -30,7 +31,8 @@
 CONSOLE_RECONNECT_BACKOFF_SECS = 12
 
 # Ansible config files
-LAB_CONNECTION_GRAPH_PATH = os.path.normpath((os.path.join(os.path.dirname(__file__), "../../../ansible/files")))
+LAB_CONNECTION_GRAPH_PATH = pathlib.Path(
+    os.getenv("ANSIBLE_CONFIG", pathlib.Path(__file__).resolve().parent.joinpath("../../ansible"))).joinpath("files")
 
 BASI_PATH = os.path.dirname(os.path.abspath(__file__))
 

tests/common/helpers/pfc_storm.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import pathlib
 import re
 import json
 
@@ -8,7 +9,8 @@
 from tests.common.devices.sonic import SonicHost
 
 TEMPLATES_DIR = os.path.realpath((os.path.join(os.path.dirname(__file__), "../../common/templates")))
-ANSIBLE_ROOT = os.path.realpath((os.path.join(os.path.dirname(__file__), "../../../ansible")))
+ANSIBLE_ROOT = pathlib.Path(os.getenv("ANSIBLE_CONFIG",
+                                      pathlib.Path(__file__).resolve().parent.joinpath("../../ansible")))
 RUN_PLAYBOOK = os.path.realpath(os.path.join(os.path.dirname(__file__), "../../scripts/exec_template.yml"))
 
 logger = logging.getLogger(__name__)

tests/common/plugins/conditional_mark/__init__.py

@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import pathlib
 import re
 import subprocess
 import yaml
@@ -24,8 +25,10 @@
 logger = logging.getLogger(__name__)
 
 DEFAULT_CONDITIONS_FILE = 'common/plugins/conditional_mark/tests_mark_conditions*.yaml'
-ASIC_NAME_PATH = '/../../../../ansible/group_vars/sonic/variables'
-ANSIBLE_LIBRARY_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../../ansible/library'))
+ANSIBLE_CONFIG_PATH = pathlib.Path(os.getenv("ANSIBLE_CONFIG",
+                                             pathlib.Path(__file__).resolve().parent.joinpath("../../../ansible")))
+ASIC_NAME_PATH = ANSIBLE_CONFIG_PATH.joinpath("group_vars/sonic/variables")
+ANSIBLE_LIBRARY_PATH = ANSIBLE_CONFIG_PATH.joinpath("library")
 MARK_CONDITIONS_CONSTANTS = {
     # Cisco platform prefixes for use in conditions like:
     #   platform.startswith(constants['CISCO_8122_PREFIX'])
@@ -133,9 +136,8 @@ def read_asic_name(hwsku):
         str or None: Return the asic generation name or None if something went wrong or nothing found in the file.
 
     '''
-    asic_name_file = os.path.dirname(__file__) + ASIC_NAME_PATH
     try:
-        with open(asic_name_file) as f:
+        with open(ASIC_NAME_PATH) as f:
             asic_name = yaml.safe_load(f)
 
         for key, value in list(asic_name.copy().items()):
@@ -168,7 +170,7 @@ def load_dut_basic_facts(inv_name, dut_name):
     results = {}
     logger.info('Getting dut basic facts: {}'.format(dut_name))
     try:
-        inv_full_path = os.path.join(os.path.dirname(__file__), '../../../../ansible', inv_name)
+        inv_full_path = ANSIBLE_CONFIG_PATH.joinpath(inv_name)
         ansible_cmd = (
             'ansible -M {} -m dut_basic_facts -i {} {} -o'
             .format(ANSIBLE_LIBRARY_PATH, inv_full_path, dut_name))
@@ -207,16 +209,13 @@ def get_basic_facts(session):
 
 
 def get_http_proxies(inv_name):
-    INV_ENV_FILE = '../../../../ansible/group_vars/{}/env.yml'.format(inv_name)
-    PUBLIC_ENV_FILE = '../../../../ansible/group_vars/all/env.yml'
-    base_path = os.path.dirname(__file__)
-    inv_env_path = os.path.join(base_path, INV_ENV_FILE)
-    public_env_path = os.path.join(base_path, PUBLIC_ENV_FILE)
+    INV_ENV_FILE = ANSIBLE_CONFIG_PATH.joinpath("group_vars/{}/env.yml".format(inv_name))
+    PUBLIC_ENV_FILE = ANSIBLE_CONFIG_PATH.joinpath("group_vars/all/env.yml")
     proxies = {}
 
-    if os.path.isfile(public_env_path):
+    if os.path.isfile(PUBLIC_ENV_FILE):
         try:
-            with open(public_env_path) as env_file:
+            with open(PUBLIC_ENV_FILE) as env_file:
                 proxy_env = yaml.safe_load(env_file)
                 if proxy_env is not None:
                     proxy = proxy_env.get("proxy_env", {})
@@ -225,19 +224,18 @@ def get_http_proxies(inv_name):
                 else:
                     proxies = {'http': '', 'https': ''}
         except Exception as e:
-            logger.error('Load proxy env from {} failed with error: {}'.format(public_env_path, repr(e)))
+            logger.error('Load proxy env from {} failed with error: {}'.format(PUBLIC_ENV_FILE, repr(e)))
 
-    if os.path.isfile(inv_env_path):
+    if os.path.isfile(INV_ENV_FILE):
         try:
-            with open(inv_env_path) as env_file:
+            with open(INV_ENV_FILE) as env_file:
                 proxy_env = yaml.safe_load(env_file)
                 if proxy_env is not None:
                     proxy = proxy_env.get("proxy_env", {})
                     http_proxy = proxy.get('http_proxy', '')
                     proxies = {'http': http_proxy, 'https': http_proxy}
         except Exception as e:
-            logger.error('Load proxy env from {} failed with error: {}'.format(inv_env_path, repr(e)))
-
+            logger.error('Load proxy env from {} failed with error: {}'.format(INV_ENV_FILE, repr(e)))
     return proxies
 
 

tests/common/sai_adhoc.py

@@ -9,6 +9,8 @@
 import argparse
 import yaml
 from yaml.loader import SafeLoader
+import os
+import pathlib
 
 logger = logging.getLogger(__name__)
 handler = logging.StreamHandler()
@@ -22,7 +24,9 @@ def get_info_helper(conf_name):
         param:
             conf_name : configuration name in testbed.yaml
     """
-    with open('./ansible/testbed.yaml') as f:
+    ansible_config_path = pathlib.Path(os.getenv("ANSIBLE_CONFIG",
+                                                 pathlib.Path(__file__).resolve().parent.joinpath("../ansible")))
+    with open(ansible_config_path.joinpath("testbed.yaml")) as f:
         testbed_infos = yaml.load(f, Loader=SafeLoader)
         for testbed_info in testbed_infos:
             if testbed_info['conf-name'] == conf_name:

tests/common/testbed.py

@@ -25,8 +25,8 @@ class TestbedInfo(object):
     TESTBED_FIELDS_RECOMMENDED = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf',
                                   'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut',
                                   'inv_name', 'auto_recover', 'is_smartswitch', 'comment')
-    TOPOLOGY_FILEPATH = "../../ansible/vars/"
-    NUT_TOPOLOGY_FILEPATH = "../../ansible/vars/nut_topos"
+    TOPOLOGY_FILEPATH = "../ansible/vars/"
+    NUT_TOPOLOGY_FILEPATH = "../ansible/vars/nut_topos"
 
     def __init__(self, testbed_file):
         if testbed_file.endswith(".csv"):
@@ -385,12 +385,12 @@ def parse_topo(self):
             tb["topo"]["type"] = self.get_testbed_type(topo)
 
             if topo.startswith("nut-"):
-                topo_dir = os.path.join(os.path.dirname(__file__), self.NUT_TOPOLOGY_FILEPATH)
+                topo_dir = os.path.join(os.path.dirname(self.testbed_filename), self.NUT_TOPOLOGY_FILEPATH)
                 topo_file = os.path.join(topo_dir, "{}.yml".format(topo))
                 with open(topo_file, 'r') as fh:
                     tb['topo']['properties'] = yaml.safe_load(fh)
             else:
-                topo_dir = os.path.join(os.path.dirname(__file__), self.TOPOLOGY_FILEPATH)
+                topo_dir = os.path.join(os.path.dirname(self.testbed_filename), self.TOPOLOGY_FILEPATH)
                 topo_file = os.path.join(topo_dir, "topo_{}.yml".format(topo))
                 with open(topo_file, 'r') as fh:
                     tb['topo']['properties'] = yaml.safe_load(fh)

tests/conftest.py

@@ -1051,7 +1051,7 @@ def ptfhosts(enhance_inventory, ansible_adhoc, tbinfo, duthost, request):
 
 
 @pytest.fixture(scope="module")
-def k8smasters(enhance_inventory, ansible_adhoc, request):
+def k8smasters(enhance_inventory, ansible_adhoc, request, ansible_root):
     """
     Shortcut fixture for getting Kubernetes master hosts
     """
@@ -1064,7 +1064,7 @@ def k8smasters(enhance_inventory, ansible_adhoc, request):
             k8s_inv_file = inv_file
     if not k8s_inv_file:
         pytest.skip("k8s inventory not found, skipping tests")
-    with open('../ansible/{}'.format(k8s_inv_file), 'r') as kinv:
+    with open(os.path.join(ansible_root, k8s_inv_file), 'r') as kinv:
         k8sinventory = yaml.safe_load(kinv)
         for hostname, attributes in list(k8sinventory[k8s_master_ansible_group]['hosts'].items()):
             if 'haproxy' in attributes:
@@ -1460,9 +1460,9 @@ def sonic():
 
 
 @pytest.fixture(scope='session')
-def pdu():
+def pdu(ansible_root):
     """ read and yield pdu configuration """
-    with open('../ansible/group_vars/pdu/pdu.yml') as stream:
+    with open(ansible_root.joinpath("group_vars/pdu/pdu.yml")) as stream:
         pdu = yaml.safe_load(stream)
         return pdu
 
@@ -1473,7 +1473,7 @@ def creds(duthost):
 
 
 @pytest.fixture(scope="session")
-def topo_bgp_routes(localhost, ptfhosts, tbinfo):
+def topo_bgp_routes(localhost, ptfhosts, tbinfo, ansible_root):
     bgp_routes = {}
     topo_name = tbinfo['topo']['name']
     servers_dut_interfaces = None
@@ -1490,7 +1490,7 @@ def topo_bgp_routes(localhost, ptfhosts, tbinfo):
             topo_name=topo_name,
             ptf_ip=ptf_ip,
             action='generate',
-            path="../ansible/",
+            path=str(ansible_root),
             log_path=log_path,
             dut_interfaces=servers_dut_interfaces.get(ptf_ip, '') if servers_dut_interfaces else '',
             verbose=False
@@ -4285,3 +4285,16 @@ def restore_counter_poll(rand_selected_dut):
         parsed_counterpoll_before,
         parsed_counterpoll_after
     )
+
+
+@pytest.fixture(scope="session")
+def ansible_root(request):
+    """
+    Returns the ansible directory.
+    """
+    ansible_config_path = os.getenv("ANSIBLE_CONFIG", None)
+    if ansible_config_path:
+        return pathlib.Path(ansible_config_path)
+    else:
+        tbfile = request.config.getoption("testbed_file")
+        return pathlib.Path(tbfile).parent

tests/run_tests.sh

@@ -33,6 +33,9 @@ function show_help_and_exit()
     echo "    -x             : print commands and their arguments as they are executed"
     echo "    -6             : IPv6-only management mode (use IPv6 for DUT mgmt connectivity)"
     echo "    -M             : run all 4 prober_type x neighbor_mode MUX_CABLE combos (dualtor/dualtor_io only)"
+    echo ""
+    echo "    environment variables:"
+    echo "    ANSIBLE_PARENT_DIR_OVERRIDE : root holding the 'ansible' directory (default: repo root)"
 
     exit $1
 }
@@ -103,6 +106,9 @@ function setup_environment()
     FULL_PATH=$(realpath ${SCRIPT})
     SCRIPT_PATH=$(dirname ${FULL_PATH})
     BASE_PATH=$(dirname ${SCRIPT_PATH})
+    # Root that holds the 'ansible' directory. Defaults to this repo checkout; set
+    # ANSIBLE_PARENT_DIR_OVERRIDE when the ansible tree lives somewhere else.
+    ANSIBLE_PARENT_DIR=${ANSIBLE_PARENT_DIR_OVERRIDE:-${BASE_PATH}}
     LOG_PATH="logs"
 
     AUTO_RECOVER="True"
@@ -112,13 +118,13 @@ function setup_environment()
     EXTRA_PARAMETERS=""
     FILE_LOG_LEVEL='debug'
     INCLUDE_FOLDERS=""
-    INVENTORY="${BASE_PATH}/ansible/lab,${BASE_PATH}/ansible/veos"
+    INVENTORY="${ANSIBLE_PARENT_DIR}/ansible/lab,${ANSIBLE_PARENT_DIR}/ansible/veos"
     KUBE_MASTER_ID="unset"
     OMIT_FILE_LOG="False"
     RETAIN_SUCCESS_LOG="False"
     SKIP_SCRIPTS=""
     SKIP_FOLDERS="ptftests acstests saitests scripts k8s sai_qualify"
-    TESTBED_FILE="${BASE_PATH}/ansible/testbed.yaml"
+    TESTBED_FILE="${ANSIBLE_PARENT_DIR}/ansible/testbed.yaml"
     TEST_CASES=""
     TEST_FILTER=""
     TEST_INPUT_ORDER="False"
@@ -129,11 +135,11 @@ function setup_environment()
     IPV6_ONLY_MGMT="False"
     MUX_COMBO_MODE="False"
 
-    export ANSIBLE_CONFIG=${BASE_PATH}/ansible
-    export ANSIBLE_LIBRARY=${BASE_PATH}/ansible/library/
-    export ANSIBLE_CONNECTION_PLUGINS=${BASE_PATH}/ansible/plugins/connection
-    export ANSIBLE_CLICONF_PLUGINS=${BASE_PATH}/ansible/cliconf_plugins
-    export ANSIBLE_TERMINAL_PLUGINS=${BASE_PATH}/ansible/terminal_plugins
+    export ANSIBLE_CONFIG=${ANSIBLE_PARENT_DIR}/ansible
+    export ANSIBLE_LIBRARY=${ANSIBLE_PARENT_DIR}/ansible/library/
+    export ANSIBLE_CONNECTION_PLUGINS=${ANSIBLE_PARENT_DIR}/ansible/plugins/connection
+    export ANSIBLE_CLICONF_PLUGINS=${ANSIBLE_PARENT_DIR}/ansible/cliconf_plugins
+    export ANSIBLE_TERMINAL_PLUGINS=${ANSIBLE_PARENT_DIR}/ansible/terminal_plugins
 
     # Kill pytest and ansible-playbook process
     pkill --signal 9 pytest
@@ -262,6 +268,7 @@ function run_debug_tests()
     echo "FULL_PATH:             ${FULL_PATH}"
     echo "SCRIPT_PATH:           ${SCRIPT_PATH}"
     echo "BASE_PATH:             ${BASE_PATH}"
+    echo "ANSIBLE_PARENT_DIR:    ${ANSIBLE_PARENT_DIR}"
 
     echo "ANSIBLE_CONFIG:        ${ANSIBLE_CONFIG}"
     echo "ANSIBLE_LIBRARY:       ${ANSIBLE_LIBRARY}"

tests/transceiver/attribute_parser/paths.py

@@ -15,8 +15,8 @@
 _REPO_ROOT = Path(__file__).resolve().parents[3]
 
 
-# Root directory (relative to repository root)
-REL_TRANSCEIVER_INV_DIR = os.path.join('ansible', 'files', 'transceiver', 'inventory')
+# Root directory (relative to ansible root)
+REL_TRANSCEIVER_INV_DIR = os.path.join('files', 'transceiver', 'inventory')
 
 # Subdirectories
 REL_ATTR_DIR = os.path.join(REL_TRANSCEIVER_INV_DIR, 'attributes')