diff --git a/docs/source/Reference/sr3.1.rst b/docs/source/Reference/sr3.1.rst index f3e2fcb4c..bb35263c2 100644 --- a/docs/source/Reference/sr3.1.rst +++ b/docs/source/Reference/sr3.1.rst @@ -112,13 +112,16 @@ It is actually building the effective configuration from: 2. admin.conf - 3. .conf (subscribe.conf, audit.conf, etc...) + 3. /default.inc 4. /.conf -Settings in an individual .conf file are read in after the default.conf -file, and so can override defaults. Options specified on -the command line override configuration files. +Settings in a default.inc include file (from the parent directory) +are read in after the default.conf and prior to /.conf. +So this allows to override the values for the component chosen. +Options specified in the configuration file override values in the components' default.inc. + +Options specified on the command line override configuration files. While one can manage configuration files using the *add*, *remove*, *list*, *edit*, *disable*, and *enable* actions, one can also do all diff --git a/docs/source/fr/Reference/sr3.1.rst b/docs/source/fr/Reference/sr3.1.rst index 77d5e47b5..9e749fc4e 100644 --- a/docs/source/fr/Reference/sr3.1.rst +++ b/docs/source/fr/Reference/sr3.1.rst @@ -1,12 +1,12 @@ ===== - SR3 + SR3 ===== ------------------ sr3 Sarracenia CLI ------------------ -:Manual section: 1 +:Manual section: 1 :Date: |today| :Version: |release| :Manual group: MetPX-Sarracenia @@ -112,14 +112,16 @@ la configuration se fait construire a partir de: 2. admin.conf - 3. .conf (subscribe.conf, audit.conf, etc...) + 3. /default.inc 4. /.conf -Les paramètres d'un fichier .conf sont lu après le fichier default.conf, -et les valeurs initiales choisi par défaut peuvent éventuellement être replacer. -Les options spécifiées sur la ligne de commande remplacent les options spécifiées dans le -fichier de configuration. + +Les paramètres du fichier d'inclusion default.inc (situé dans le répertoire parent ) +sont lus après default.conf et avant /.conf. +Cela permet donc de remplacer les valeurs par défaut du component sélectionné. +Les options spécifiées dans le fichier de configuration remplacent les valeurs du fichier default.inc du component. +Les options spécifiées en ligne de commande remplacent ceux des fichiers de configuration. Les fichiers de configurations peuvent être gérer en utilisant les actions *add*, *remove*, *list*, *edit*, *disable*, et *enable*. Il est également possible de faire diff --git a/sarracenia/config/__init__.py b/sarracenia/config/__init__.py index 8fdc53106..4e1c4b28c 100644 --- a/sarracenia/config/__init__.py +++ b/sarracenia/config/__init__.py @@ -976,6 +976,14 @@ def applyComponentDefaults( self, component ): if hasattr(component_module, 'default_options'): self.override(component_module.default_options) + def addComponentDefaultInc( self , component ): + """ + Added from issue 1196 + Parse and add component/default.inc options if the file exists. + """ + if os.path.exists(get_user_config_dir() + os.sep + component + os.sep + 'default.inc'): + self.parse_file(get_user_config_dir() + os.sep + component + os.sep + 'default.inc', component) + @property def admin(self): return self.__admin @@ -2890,6 +2898,8 @@ def one_config(component, config, action, isPost=False, hostDir=None): os.chdir(get_user_config_dir()) os.chdir(component) + cfg.addComponentDefaultInc( component ) + if config[-5:] != '.conf': fname = os.path.expanduser(config + '.conf') else: diff --git a/sarracenia/sr.py b/sarracenia/sr.py index b6f931398..387e54fd9 100755 --- a/sarracenia/sr.py +++ b/sarracenia/sr.py @@ -334,7 +334,9 @@ def _read_configs(self): 'action': self.options.action, 'directory': '${PWD}' }) + # Apply component defaults from source cfgbody.applyComponentDefaults( c ) + cfgbody.addComponentDefaultInc( c ) cfgbody.parse_file(cfg,c) cfgbody.finalize(c, cfg) self.configs[c][cbase]['options'] = cfgbody diff --git a/tests/sarracenia/sr_test.py b/tests/sarracenia/sr_test.py index c7e0722d5..b88336143 100644 --- a/tests/sarracenia/sr_test.py +++ b/tests/sarracenia/sr_test.py @@ -3,4 +3,358 @@ #from unittest.mock import Mock import sarracenia.config -import sarracenia.sr \ No newline at end of file +import sarracenia.sr + +import copy +import os +import shutil +import pytest +from pathlib import Path + +import sarracenia +import sarracenia.config +import sarracenia.sr + + +def _make_config_tree(tmp_path, component, default_content=None, config_content=None): + """ + Create: + + //default.inc + //test.conf + + Returns the component directory. + """ + component_dir = tmp_path + '/' + component + + Path(component_dir).mkdir(parents=True, exist_ok=True) + + if default_content is not None: + Path(component_dir + "/default.inc").write_text(default_content) + + if config_content is not None: + Path(component_dir + "/test.conf").write_text(config_content) + + return component_dir + + +def _remove_config_tree(tmp_path, component): + """ + Remove the entire temporary Sarracenia configuration tree. + """ + component_dir = tmp_path + "/" + component + + for filename in ["default.inc", "test.conf"]: + filepath = component_dir + "/" + filename + + if Path(filepath).exists(): + Path(filepath).unlink() + + if Path(component_dir).exists(): + Path(component_dir).rmdir() + + +def _make_global_state(tmp_path, components): + """ + Create a minimal sr_GlobalState suitable for exercising _read_configs() + without invoking the normal sr3 command-line startup machinery. + """ + state = sarracenia.sr.sr_GlobalState.__new__( + sarracenia.sr.sr_GlobalState + ) + + state.user_config_dir = str(tmp_path) + + state.components = components + + # _read_configs() uses options.action when building each cfgbody. + state.options = copy.deepcopy(sarracenia.config.default_config()) + state.options.action = "start" + + return state + +@pytest.mark.parametrize( + "component", + [ + "cpost", + "cpump", + "flow", + "poll", + "post", + "report", + "sarra", + "sender", + "subscribe", + "shovel", + "watch", + "winnow" + ], +) +def test_component_default_inc(component, monkeypatch): + """ + default.inc should be loaded for every supported flow component. + """ + + tmp_path = '/tmp/' + monkeypatch.setattr( + sarracenia.config, + "get_user_config_dir", + lambda: str(tmp_path) + ) + + _make_config_tree( + tmp_path, + component, + default_content="fileEvents create,modify\n", + config_content="exchange xs_Something\n", + ) + + try: + state = _make_global_state(tmp_path, [component]) + + state._read_configs() + + assert component in state.configs + assert "test" in state.configs[component] + + options = state.configs[component]["test"]["options"] + + assert options.exchange == "xs_Something" + assert options.fileEvents == {'modify', 'create'} + finally: + _remove_config_tree(tmp_path, component) + + +def test_component_default_inc_can_be_overridden_by_config(monkeypatch): + """ + Values from /default.inc are defaults and must be overridden + by values explicitly specified in the component configuration. + """ + tmp_path = '/tmp/' + monkeypatch.setattr( + sarracenia.config, + "get_user_config_dir", + lambda: str(tmp_path) + ) + + _make_config_tree( + tmp_path, + "poll", + default_content="retry_ttl 1d\n", + config_content="retry_ttl 1h\n", + ) + + try: + state = _make_global_state(tmp_path, ["poll"]) + + state._read_configs() + + options = state.configs["poll"]["test"]["options"] + + assert options.retry_ttl == 3600 + finally: + _remove_config_tree(tmp_path, "poll") + + +def test_component_default_inc_only_applies_to_its_component(monkeypatch): + """ + A component's default.inc must not leak into another component. + """ + tmp_path = '/tmp/' + monkeypatch.setattr( + sarracenia.config, + "get_user_config_dir", + lambda: str(tmp_path) + ) + + _make_config_tree( + tmp_path, + "poll", + default_content="retry_ttl 1h\n", + config_content="exchange poll_exchange\n", + ) + + _make_config_tree( + tmp_path, + "sarra", + default_content="retry_ttl 2h\n", + config_content="exchange sarra_exchange\n", + ) + + try: + state = _make_global_state(tmp_path, ["poll", "sarra"]) + + state._read_configs() + + assert state.configs["poll"]["test"]["options"].retry_ttl == 3600 + assert state.configs["sarra"]["test"]["options"].retry_ttl == 7200 + finally: + _remove_config_tree(tmp_path, "poll") + _remove_config_tree(tmp_path, "sarra") + +def test_component_default_inc_with_two_of_same_component(monkeypatch): + """ + A component's default.inc must not leak into another component. + """ + tmp_path = '/tmp/' + monkeypatch.setattr( + sarracenia.config, + "get_user_config_dir", + lambda: str(tmp_path) + ) + + _make_config_tree( + tmp_path, + "sarra", + default_content="retry_ttl 1h\n", + config_content="exchange sarra_exchange1\n", + ) + + Path(tmp_path + "sarra/test1.conf").write_text("exchange sarra_exchange2\n") + + try: + state = _make_global_state(tmp_path, ["sarra"]) + + state._read_configs() + + assert state.configs["sarra"]["test"]["options"].retry_ttl == 3600 + assert state.configs["sarra"]["test"]["options"].exchange == 'sarra_exchange1' + assert state.configs["sarra"]["test1"]["options"].retry_ttl == 3600 + assert state.configs["sarra"]["test1"]["options"].exchange == 'sarra_exchange2' + finally: + Path(tmp_path + "sarra/test1.conf").unlink() + _remove_config_tree(tmp_path, "sarra") + + +def test_default_inc_is_not_a_configuration(monkeypatch): + """ + default.inc is an include/default file and must not itself appear as a + configuration. + """ + tmp_path = '/tmp/' + monkeypatch.setattr( + sarracenia.config, + "get_user_config_dir", + lambda: str(tmp_path) + ) + + _make_config_tree( + tmp_path, + "poll", + default_content="exchange xs_Something\n", + config_content="accept .*\n", + ) + + try: + state = _make_global_state(tmp_path, ["poll"]) + + state._read_configs() + + assert "test" in state.configs["poll"] + assert "default" not in state.configs["poll"] + finally: + _remove_config_tree(tmp_path, "poll") + + +def test_default_inc_is_optional(monkeypatch): + """ + A component without a default.inc must continue to load normally. + """ + tmp_path = '/tmp/' + monkeypatch.setattr( + sarracenia.config, + "get_user_config_dir", + lambda: str(tmp_path) + ) + + _make_config_tree( + tmp_path, + "poll", + default_content=None, + config_content="exchange configured_exchange\n", + ) + + try: + state = _make_global_state(tmp_path, ["poll"]) + + state._read_configs() + + assert "test" in state.configs["poll"] + assert ( + state.configs["poll"]["test"]["options"].exchange + == "configured_exchange" + ) + finally: + _remove_config_tree(tmp_path, "poll") + + +def test_default_inc_nested_include(monkeypatch): + """ + A nested include file inside default.inc should parse properly + """ + tmp_path = '/tmp/' + monkeypatch.setattr( + sarracenia.config, + "get_user_config_dir", + lambda: str(tmp_path) + ) + + component_dir = tmp_path + "/poll" + Path(component_dir).mkdir() + + Path(component_dir + "/default.inc").write_text( + "include nested.inc\n" + ) + + Path(component_dir + "/nested.inc").write_text( + "retry_ttl 3h\n" + ) + + Path(component_dir + "/test.conf").write_text( + "exchange my_exchange\n" + ) + + try: + state = _make_global_state(tmp_path, ["poll"]) + state._read_configs() + + options = state.configs["poll"]["test"]["options"] + + assert options.retry_ttl == 10800 + assert options.exchange == "my_exchange" + finally: + Path(component_dir + "/nested.inc").unlink() + _remove_config_tree(tmp_path, "poll") + + +def test_default_inc_nested_include_can_be_overridden(): + """ + A nested include file inside default.inc should have its values ignored if the configuration overrides that value + """ + tmp_path = '/tmp' + component_dir = tmp_path + "/poll" + Path(component_dir).mkdir() + + Path(component_dir + "/default.inc").write_text( + "include nested.inc\n" + ) + + Path(component_dir + "/nested.inc").write_text( + "retry_ttl 3h\n" + ) + + Path(component_dir + "/test.conf").write_text( + "retry_ttl 30m\n" + ) + + try: + state = _make_global_state(tmp_path, ["poll"]) + state._read_configs() + + options = state.configs["poll"]["test"]["options"] + + assert options.retry_ttl == 1800 + finally: + Path(component_dir + "/nested.inc").unlink() + _remove_config_tree(tmp_path, "poll") +