From d820fb917b7c9cbc429e6a045762fef332c5f356 Mon Sep 17 00:00:00 2001 From: mak-454 Date: Sat, 30 Jul 2016 20:53:17 +0530 Subject: [PATCH 1/9] code changes to use single nfp.ini file --- gbpservice/nfp/bin/nfp.ini | 58 ++++++++++++ gbpservice/nfp/core/cfg.py | 12 ++- gbpservice/nfp/core/controller.py | 22 ++++- gbpservice/nfp/lib/transport.py | 9 -- gbpservice/nfp/proxy_agent/proxy/proxy.py | 104 +++++++++++++++------- 5 files changed, 163 insertions(+), 42 deletions(-) create mode 100644 gbpservice/nfp/bin/nfp.ini diff --git a/gbpservice/nfp/bin/nfp.ini b/gbpservice/nfp/bin/nfp.ini new file mode 100644 index 0000000000..c80b66bef1 --- /dev/null +++ b/gbpservice/nfp/bin/nfp.ini @@ -0,0 +1,58 @@ +[ORCHESTRATOR] +# Number of worker process to be spawned. +workers=1 +# Path to NFP modules in . format +# NFP core framework will load all .py files +# from this path as nfp modules +nfp_modules_path=gbpservice.nfp.orchestrator.modules +# To invoke OTC Apis +# It could be rpc/REST. +# rpc - where fip access unavaiable for OTC NFP controller +# REST - where fip access is available for OTC NFP controller +backend=rpc + +# Section describing params for backend=rpc +[RPC] +# Topic to send OTC request rpc on. +topic=nfp-proxy-agent-topic + +[PROXY_AGENT] +# Number of worker process to be spawned. +workers=1 +# Path to NFP modules in . format +# NFP core framework will load all .py files +# from this path as nfp modules +nfp_modules_path=gbpservice.nfp.proxy_agent.modules +# To invoke OTC Apis +# Proxy agent will always use unix_rest backend to +# communicate with proxy component inside namespace +backend=unix_rest + +[oslo_policy] +policy_file = /etc/neutron/policy.json + +[PROXY] +thread_pool_size= 10 +unix_bind_path= /var/run/uds_socket +max_connections=10 +nfp_controller_ip_address= 11.0.0.3 +worker_threads=100 +connect_max_wait_timeout=120 +idle_max_wait_timeout=120 +idle_min_wait_timeout=0.1 + +[NFP_CONTROLLER] +rest_server_address=120.0.0.3 +rest_server_port=8070 + +[CONFIG_ORCHESTRATOR] +# Number of worker process to be spawned. +workers=1 +# Path to NFP modules in . format +# NFP core framework will load all .py files +# from this path as nfp modules +nfp_modules_path=gbpservice.nfp.proxy_agent.modules +# To invoke OTC Apis +# Proxy agent will always use unix_rest backend to +# communicate with proxy component inside namespace +backend=unix_rest diff --git a/gbpservice/nfp/core/cfg.py b/gbpservice/nfp/core/cfg.py index 71463b2151..f3ba7c0321 100644 --- a/gbpservice/nfp/core/cfg.py +++ b/gbpservice/nfp/core/cfg.py @@ -24,9 +24,15 @@ default='gbpservice.nfp.core.test', help='Path for NFP modules.' 'All modules from this path are autloaded by framework' - ) + ), + oslo_config.StrOpt( + 'backend', + default='rpc', + help='Backend Support for communicationg with configurator.' + ), ] + es_openstack_opts = [ oslo_config.StrOpt('auth_host', default='localhost', @@ -54,7 +60,9 @@ def init(args, **kwargs): """Initialize the configuration. """ - oslo_config.CONF.register_opts(NFP_OPTS) + oslo_config.CONF.register_opts(NFP_OPTS, "orchestrator") + oslo_config.CONF.register_opts(NFP_OPTS, "proxy_agent") + oslo_config.CONF.register_opts(NFP_OPTS, "config_orchestrator") oslo_config.CONF.register_opts(es_openstack_opts, "keystone_authtoken") oslo_config.CONF.set_override('use_syslog', 'True') oslo_config.CONF.set_override('syslog_log_facility', 'local1') diff --git a/gbpservice/nfp/core/controller.py b/gbpservice/nfp/core/controller.py index b891762d3e..7c2598e197 100644 --- a/gbpservice/nfp/core/controller.py +++ b/gbpservice/nfp/core/controller.py @@ -571,9 +571,29 @@ def nfp_modules_post_init(conf, nfp_modules, nfp_controller): "nfp_module_post_init(), ignoring") % (identify(module)) LOG.debug(message) +def _module(args): + try: + index = args.index('--module') + module = args[index+1] + return module + except ValueError: + print "--module missing from cmd args" + sys.exit(-1) def main(): - conf = nfp_cfg.init(sys.argv[1:]) + args = sys.argv[1:] + module = _module(args) + args.remove('--module') + args.remove(module) + conf = nfp_cfg.init(args) + conf.module = module + module_opts = eval('conf.%s.keys' %(module))() + for module_opt in module_opts: + module_cfg_opt = eval("conf.%s._group._opts['%s']['opt']" %( + module, module_opt)) + module_cfg_opt_value = eval("conf.%s.%s" %(module, module_opt)) + conf.register_opt(module_cfg_opt) + conf.set_override(module_opt, module_cfg_opt_value) nfp_common.init() nfp_controller = NfpController(conf) # Load all nfp modules from path configured diff --git a/gbpservice/nfp/lib/transport.py b/gbpservice/nfp/lib/transport.py index 9d26143c54..2e6ab7b136 100644 --- a/gbpservice/nfp/lib/transport.py +++ b/gbpservice/nfp/lib/transport.py @@ -42,15 +42,6 @@ default='', help='Topic for rpc connection'), ] -OPTS = [ - cfg.StrOpt( - 'backend', - default='rpc', - help='Backend Support for communicationg with configurator.' - ), -] - -oslo_config.CONF.register_opts(OPTS) oslo_config.CONF.register_opts(rest_opts, "REST") oslo_config.CONF.register_opts(rpc_opts, "RPC") n_rpc.init(cfg.CONF) diff --git a/gbpservice/nfp/proxy_agent/proxy/proxy.py b/gbpservice/nfp/proxy_agent/proxy/proxy.py index 90566c7da2..287dc83fe2 100644 --- a/gbpservice/nfp/proxy_agent/proxy/proxy.py +++ b/gbpservice/nfp/proxy_agent/proxy/proxy.py @@ -17,14 +17,14 @@ import ConfigParser from gbpservice.nfp.core import log as nfp_logging import os -from oslo_config import cfg +from oslo_config import cfg as oslo_config from oslo_log import log as oslo_logging import socket import sys import time -oslo_logging.register_options(cfg.CONF) +oslo_logging.register_options(oslo_config.CONF) LOG = nfp_logging.getLogger(__name__) @@ -50,24 +50,16 @@ class ConnectionIdleTimeOut(Exception): class Configuration(object): - def __init__(self, filee): - config = ConfigParser.ConfigParser() - config.read(filee) - - self.thread_pool_size = config.getint('OPTIONS', 'thread_pool_size') - self.unix_bind_path = config.get('OPTIONS', 'unix_bind_path') - self.max_connections = config.getint('OPTIONS', 'max_connections') - self.worker_threads = config.getint('OPTIONS', 'worker_threads') - self.connect_max_wait_timeout = config.getfloat( - 'OPTIONS', 'connect_max_wait_timeout') - self.idle_max_wait_timeout = config.getfloat( - 'OPTIONS', 'idle_max_wait_timeout') - self.idle_min_wait_timeout = config.getfloat( - 'OPTIONS', 'idle_min_wait_timeout') - self.rest_server_address = config.get( - 'NFP_CONTROLLER', 'rest_server_address') - self.rest_server_port = config.getint( - 'NFP_CONTROLLER', 'rest_server_port') + def __init__(self, conf): + self.thread_pool_size = conf.proxy.thread_pool_size + self.unix_bind_path = conf.proxy.unix_bind_path + self.max_connections = conf.proxy.max_connections + self.worker_threads = conf.proxy.worker_threads + self.connect_max_wait_timeout = conf.proxy.connect_max_wait_timeout + self.idle_max_wait_timeout = conf.proxy.idle_max_wait_timeout + self.idle_min_wait_timeout = conf.proxy.idle_min_wait_timeout + self.rest_server_address = conf.nfp_controller.rest_server_address + self.rest_server_port = conf.nfp_controller.rest_server_port """ @@ -319,15 +311,67 @@ def new_client(self, unixsocket, address): pc = ProxyConnection(self.conf, unixsocket, tcpsocket) ConnQ.put(pc) - +PROXY_AGENT_OPTS = [ + oslo_config.IntOpt( + 'thread_pool_size', + default=10, + help='' + ), + oslo_config.IntOpt( + 'max_connections', + default=10, + help='' + ), + oslo_config.IntOpt( + 'worker_threads', + default=10, + help='' + ), + oslo_config.FloatOpt( + 'connect_max_wait_timeout', + default=10, + help='' + ), + oslo_config.FloatOpt( + 'idle_max_wait_timeout', + default=10, + help='' + ), + oslo_config.FloatOpt( + 'idle_min_wait_timeout', + default=10, + help='' + ), + oslo_config.StrOpt( + 'unix_bind_path', + default='gbpservice.nfp.core.test', + help='' + ), + oslo_config.StrOpt( + 'nfp_controller_ip_address', + default='11.0.0.3', + help='' + ), +] + +NFP_CONTROLLER_OPTS = [ + oslo_config.IntOpt( + 'rest_server_port', + default=8070, + help='' + ), + oslo_config.StrOpt( + 'rest_server_address', + default='11.0.0.3', + help='' + ), +] + + def main(argv): - cfg.CONF(args=sys.argv[1:]) - oslo_logging.setup(cfg.CONF, 'nfp') - parser = argparse.ArgumentParser() - parser.add_argument( - '-config-file', "--config-file", action="store", dest='config_file') - parser.add_argument( - '-log-file', "--log-file", action="store", dest='log_file') - args = parser.parse_args(sys.argv[1:]) - conf = Configuration(args.config_file) + oslo_config.CONF.register_opts(PROXY_AGENT_OPTS, 'proxy') + oslo_config.CONF.register_opts(NFP_CONTROLLER_OPTS, 'nfp_controller') + oslo_config.CONF(args=sys.argv[1:]) + oslo_logging.setup(oslo_config.CONF, 'nfp') + conf = Configuration(oslo_config.CONF) Proxy(conf).start() From 363dfda87ea9700f832f6135766dcf64ce908371 Mon Sep 17 00:00:00 2001 From: Rahul Shikhare Date: Fri, 5 Aug 2016 18:47:47 +0530 Subject: [PATCH 2/9] removed unused options from nfp.ini --- gbpservice/nfp/bin/nfp.ini | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/gbpservice/nfp/bin/nfp.ini b/gbpservice/nfp/bin/nfp.ini index c80b66bef1..b15d460fb8 100644 --- a/gbpservice/nfp/bin/nfp.ini +++ b/gbpservice/nfp/bin/nfp.ini @@ -11,11 +11,6 @@ nfp_modules_path=gbpservice.nfp.orchestrator.modules # REST - where fip access is available for OTC NFP controller backend=rpc -# Section describing params for backend=rpc -[RPC] -# Topic to send OTC request rpc on. -topic=nfp-proxy-agent-topic - [PROXY_AGENT] # Number of worker process to be spawned. workers=1 @@ -28,9 +23,6 @@ nfp_modules_path=gbpservice.nfp.proxy_agent.modules # communicate with proxy component inside namespace backend=unix_rest -[oslo_policy] -policy_file = /etc/neutron/policy.json - [PROXY] thread_pool_size= 10 unix_bind_path= /var/run/uds_socket @@ -40,8 +32,7 @@ worker_threads=100 connect_max_wait_timeout=120 idle_max_wait_timeout=120 idle_min_wait_timeout=0.1 - -[NFP_CONTROLLER] +#pecan controller ip address and port rest_server_address=120.0.0.3 rest_server_port=8070 @@ -55,4 +46,9 @@ nfp_modules_path=gbpservice.nfp.proxy_agent.modules # To invoke OTC Apis # Proxy agent will always use unix_rest backend to # communicate with proxy component inside namespace -backend=unix_rest +backend=rpc + +# Section describing params for backend=rpc +[RPC] +# Topic to send OTC request rpc on. +topic=nfp-proxy-agent-topic From c5cf6525d30552c0a6fc76592edb9e316c0a9fa6 Mon Sep 17 00:00:00 2001 From: mak-454 Date: Sat, 6 Aug 2016 13:02:23 +0530 Subject: [PATCH 3/9] removed unused configurations and addressed comments --- gbpservice/nfp/bin/nfp.ini | 14 +++--- gbpservice/nfp/proxy_agent/proxy/proxy.py | 52 +++++++++-------------- 2 files changed, 28 insertions(+), 38 deletions(-) diff --git a/gbpservice/nfp/bin/nfp.ini b/gbpservice/nfp/bin/nfp.ini index b15d460fb8..5f94ff364f 100644 --- a/gbpservice/nfp/bin/nfp.ini +++ b/gbpservice/nfp/bin/nfp.ini @@ -24,17 +24,21 @@ nfp_modules_path=gbpservice.nfp.proxy_agent.modules backend=unix_rest [PROXY] -thread_pool_size= 10 +# Path unix server should bind to unix_bind_path= /var/run/uds_socket +# Max number of client connections max_connections=10 -nfp_controller_ip_address= 11.0.0.3 +# Num of worker threads to handle connections worker_threads=100 +# Timeout for 'connect' operation connect_max_wait_timeout=120 +# Max time an idle channel is allowed to be open idle_max_wait_timeout=120 +# Min time to wait on channel to declare its idle. idle_min_wait_timeout=0.1 -#pecan controller ip address and port -rest_server_address=120.0.0.3 -rest_server_port=8070 +#NFP controllers ip address and port +nfp_controller_ip=172.16.0.3 +nfp_controller_port=8070 [CONFIG_ORCHESTRATOR] # Number of worker process to be spawned. diff --git a/gbpservice/nfp/proxy_agent/proxy/proxy.py b/gbpservice/nfp/proxy_agent/proxy/proxy.py index 287dc83fe2..a4d905f305 100644 --- a/gbpservice/nfp/proxy_agent/proxy/proxy.py +++ b/gbpservice/nfp/proxy_agent/proxy/proxy.py @@ -51,15 +51,14 @@ class ConnectionIdleTimeOut(Exception): class Configuration(object): def __init__(self, conf): - self.thread_pool_size = conf.proxy.thread_pool_size self.unix_bind_path = conf.proxy.unix_bind_path self.max_connections = conf.proxy.max_connections self.worker_threads = conf.proxy.worker_threads self.connect_max_wait_timeout = conf.proxy.connect_max_wait_timeout self.idle_max_wait_timeout = conf.proxy.idle_max_wait_timeout self.idle_min_wait_timeout = conf.proxy.idle_min_wait_timeout - self.rest_server_address = conf.nfp_controller.rest_server_address - self.rest_server_port = conf.nfp_controller.rest_server_port + self.rest_server_address = conf.proxy.nfp_controller_ip + self.rest_server_port = conf.proxy.nfp_controller_port """ @@ -311,66 +310,53 @@ def new_client(self, unixsocket, address): pc = ProxyConnection(self.conf, unixsocket, tcpsocket) ConnQ.put(pc) -PROXY_AGENT_OPTS = [ - oslo_config.IntOpt( - 'thread_pool_size', - default=10, - help='' - ), +PROXY_OPTS = [ oslo_config.IntOpt( 'max_connections', default=10, - help='' + help='Max num of client connections' ), oslo_config.IntOpt( 'worker_threads', default=10, - help='' + help='Number of threads to be used.' ), oslo_config.FloatOpt( 'connect_max_wait_timeout', default=10, - help='' + help='Max time to wait for connect.' ), oslo_config.FloatOpt( 'idle_max_wait_timeout', default=10, - help='' + help='Max time to keep idle channel open.' ), oslo_config.FloatOpt( 'idle_min_wait_timeout', default=10, - help='' + help='Minimum time to wait on idle channel.' ), oslo_config.StrOpt( 'unix_bind_path', - default='gbpservice.nfp.core.test', - help='' + default='/var/run/uds_socket', + help='Path for unix server to bind.' ), oslo_config.StrOpt( - 'nfp_controller_ip_address', - default='11.0.0.3', - help='' + 'nfp_controller_ip', + default='172.16.0.3', + help='NFP controllers ip address' ), + oslo_config.IntOpt( + 'nfp_controller_port', + default='8070', + help='NFP controllers port num' + ) ] -NFP_CONTROLLER_OPTS = [ - oslo_config.IntOpt( - 'rest_server_port', - default=8070, - help='' - ), - oslo_config.StrOpt( - 'rest_server_address', - default='11.0.0.3', - help='' - ), -] def main(argv): - oslo_config.CONF.register_opts(PROXY_AGENT_OPTS, 'proxy') - oslo_config.CONF.register_opts(NFP_CONTROLLER_OPTS, 'nfp_controller') + oslo_config.CONF.register_opts(PROXY_OPTS, 'proxy') oslo_config.CONF(args=sys.argv[1:]) oslo_logging.setup(oslo_config.CONF, 'nfp') conf = Configuration(oslo_config.CONF) From ae4d4a262aefd7da6361a6802419fb0c385e84ac Mon Sep 17 00:00:00 2001 From: Rahul Shikhare Date: Tue, 9 Aug 2016 14:51:38 +0530 Subject: [PATCH 4/9] modified controller.py to support for configurator and devestack/lib/nfp to start each module with single ini file --- devstack/lib/nfp | 23 ++++++++++------------- gbpservice/nfp/bin/nfp.ini | 2 +- gbpservice/nfp/core/cfg.py | 1 + gbpservice/nfp/core/controller.py | 29 ++++++++++++++++------------- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/devstack/lib/nfp b/devstack/lib/nfp index 88bc95157d..b2b04b0ac1 100644 --- a/devstack/lib/nfp +++ b/devstack/lib/nfp @@ -375,7 +375,7 @@ function namespace_create { else echo "New Neutron Port Created on Service management network with ip =$new_ip" fi - new_ip_cidr+="$new_ip/24" + new_ip_cidr="$new_ip/24" sleep 2 #get the ip address of new port eg : 11.0.0.6 and asign to namespace @@ -410,7 +410,7 @@ function namespace_create { echo "Fails reaching to $configurator_ip" fi - sudo ip netns exec nfp-proxy /usr/bin/nfp_proxy --config-file=/etc/nfp_proxy.ini + sudo ip netns exec nfp-proxy /usr/bin/nfp_proxy --config-file=/etc/nfp.ini } # copy_nfp_files_and_start_process() - Setup configuration and start processes @@ -418,11 +418,8 @@ function copy_nfp_files_and_start_process { cd $NFPSERVICE_DIR/gbpservice/nfp sudo cp -r bin/nfp /usr/bin/ sudo chmod +x /usr/bin/nfp - sudo rm -rf /etc/nfp_* - sudo cp -r bin/nfp_orchestrator.ini /etc/ - sudo cp -r bin/nfp_proxy_agent.ini /etc/ - [[ $NFP_DEVSTACK_MODE = advanced ]] && sudo cp -r ../contrib/nfp/bin/nfp_config_orch.ini /etc/ - sudo cp -r bin/nfp_proxy.ini /etc/nfp_proxy.ini + sudo rm -rf /etc/nfp.ini + sudo cp -r bin/nfp.ini /etc/ sudo cp -r bin/nfp_proxy /usr/bin/ if [[ $NFP_DEVSTACK_MODE = base ]]; then @@ -432,19 +429,19 @@ function copy_nfp_files_and_start_process { configurator_ip=$configurator_ip configurator_port=8070 fi - echo "Configuring proxy.ini .... with rest_server_address as $configurator_ip:$configurator_port" - sudo sed -i "s/rest_server_address=*.*/rest_server_address=$configurator_ip/g" /etc/nfp_proxy.ini - sudo sed -i "s/rest_server_port= *.*/rest_server_port=$configurator_port/g" /etc/nfp_proxy.ini + echo "Configuring nfp.ini .... with nfp_controller_ip as $configurator_ip:$configurator_port" + sudo sed -i "s/nfp_controller_ip=*.*/nfp_controller_ip=$configurator_ip/g" /etc/nfp.ini + sudo sed -i "s/nfp_controller_port= *.*/nfp_controller_port=$configurator_port/g" /etc/nfp.ini sed -i 's#source.*#source '$DEVSTACK_DIR'/openrc demo demo#g' $NFPSERVICE_DIR/devstack/exercises/nfp_service/*.sh source $DEVSTACK_DIR/functions-common echo "Starting nfp_orchestrator under screen named nfp_orchestrator" - run_process nfp_orchestrator "sudo /usr/bin/nfp --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/ml2_conf.ini --config-file /etc/nfp_orchestrator.ini --log-file $DEST/logs/nfp_orchestrator.log" + run_process nfp_orchestrator "sudo /usr/bin/nfp --module orchestrator --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/ml2_conf.ini --config-file /etc/nfp.ini --log-file $DEST/logs/nfp_orchestrator.log" sleep 4 echo "Starting nfp_proxy_agent under screen named nfp_proxy_agent" - run_process nfp_proxy_agent "sudo /usr/bin/nfp --config-file /etc/nfp_proxy_agent.ini --log-file $DEST/logs/nfp_proxy_agent.log" + run_process nfp_proxy_agent "sudo /usr/bin/nfp --module proxy_agent --config-file /etc/nfp.ini --log-file $DEST/logs/nfp_proxy_agent.log" sleep 4 echo "Starting nfp_proxy inside namespace named nfp-proxy, under screen named nfp_proxy" @@ -453,7 +450,7 @@ function copy_nfp_files_and_start_process { if [[ $NFP_DEVSTACK_MODE = advanced ]]; then echo "Starting nfp_config_orchestrator under screen named nfp_config_orchestrator" - run_process nfp_config_orchestrator "sudo /usr/bin/nfp --config-file /etc/nfp_config_orch.ini --config-file /etc/neutron/neutron.conf --log-file $DEST/logs/nfp_config_orchestrator.log" + run_process nfp_config_orchestrator "sudo /usr/bin/nfp --module config_orchestrator --config-file /etc/nfp.ini --config-file /etc/neutron/neutron.conf --log-file $DEST/logs/nfp_config_orchestrator.log" else cd pecan/api sudo python setup.py develop diff --git a/gbpservice/nfp/bin/nfp.ini b/gbpservice/nfp/bin/nfp.ini index 5f94ff364f..592bb14c71 100644 --- a/gbpservice/nfp/bin/nfp.ini +++ b/gbpservice/nfp/bin/nfp.ini @@ -46,7 +46,7 @@ workers=1 # Path to NFP modules in . format # NFP core framework will load all .py files # from this path as nfp modules -nfp_modules_path=gbpservice.nfp.proxy_agent.modules +nfp_modules_path=gbpservice.contrib.nfp.config_orchestrator.modules # To invoke OTC Apis # Proxy agent will always use unix_rest backend to # communicate with proxy component inside namespace diff --git a/gbpservice/nfp/core/cfg.py b/gbpservice/nfp/core/cfg.py index f3ba7c0321..beeb5f58cc 100644 --- a/gbpservice/nfp/core/cfg.py +++ b/gbpservice/nfp/core/cfg.py @@ -60,6 +60,7 @@ def init(args, **kwargs): """Initialize the configuration. """ + oslo_config.CONF.register_opts(NFP_OPTS) oslo_config.CONF.register_opts(NFP_OPTS, "orchestrator") oslo_config.CONF.register_opts(NFP_OPTS, "proxy_agent") oslo_config.CONF.register_opts(NFP_OPTS, "config_orchestrator") diff --git a/gbpservice/nfp/core/controller.py b/gbpservice/nfp/core/controller.py index 7c2598e197..2dedb3b3c2 100644 --- a/gbpservice/nfp/core/controller.py +++ b/gbpservice/nfp/core/controller.py @@ -577,23 +577,26 @@ def _module(args): module = args[index+1] return module except ValueError: - print "--module missing from cmd args" - sys.exit(-1) + print "--module missing from cmd args using default one" + return None def main(): args = sys.argv[1:] module = _module(args) - args.remove('--module') - args.remove(module) - conf = nfp_cfg.init(args) - conf.module = module - module_opts = eval('conf.%s.keys' %(module))() - for module_opt in module_opts: - module_cfg_opt = eval("conf.%s._group._opts['%s']['opt']" %( - module, module_opt)) - module_cfg_opt_value = eval("conf.%s.%s" %(module, module_opt)) - conf.register_opt(module_cfg_opt) - conf.set_override(module_opt, module_cfg_opt_value) + if module: + args.remove('--module') + args.remove(module) + conf = nfp_cfg.init(args) + conf.module = module + module_opts = eval('conf.%s.keys' %(module))() + for module_opt in module_opts: + module_cfg_opt = eval("conf.%s._group._opts['%s']['opt']" %( + module, module_opt)) + module_cfg_opt_value = eval("conf.%s.%s" %(module, module_opt)) + conf.register_opt(module_cfg_opt) + conf.set_override(module_opt, module_cfg_opt_value) + else: + conf = nfp_cfg.init(sys.argv[1:]) nfp_common.init() nfp_controller = NfpController(conf) # Load all nfp modules from path configured From 69d57f1bb2620f0764f17a3070666865e06b2b32 Mon Sep 17 00:00:00 2001 From: Rahul Shikhare Date: Tue, 9 Aug 2016 15:44:56 +0530 Subject: [PATCH 5/9] pep8 error fix --- gbpservice/nfp/core/cfg.py | 1 - gbpservice/nfp/core/controller.py | 14 ++++++++------ gbpservice/nfp/proxy_agent/proxy/proxy.py | 3 +-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gbpservice/nfp/core/cfg.py b/gbpservice/nfp/core/cfg.py index beeb5f58cc..227bb6f683 100644 --- a/gbpservice/nfp/core/cfg.py +++ b/gbpservice/nfp/core/cfg.py @@ -69,7 +69,6 @@ def init(args, **kwargs): oslo_config.CONF.set_override('syslog_log_facility', 'local1') oslo_config.CONF(args=args, project='nfp', version='%%(prog)s %s' % ('version'), - #version.version_info.release_string()), **kwargs) return oslo_config.CONF diff --git a/gbpservice/nfp/core/controller.py b/gbpservice/nfp/core/controller.py index 2dedb3b3c2..915675125e 100644 --- a/gbpservice/nfp/core/controller.py +++ b/gbpservice/nfp/core/controller.py @@ -278,7 +278,7 @@ def fork_child(self, wrap): message = ("Forked a new child: %d" "Parent Pipe: % s, Child Pipe: % s") % ( - proc.pid, str(parent_pipe), str(child_pipe)) + proc.pid, str(parent_pipe), str(child_pipe)) LOG.info(message) try: @@ -571,15 +571,17 @@ def nfp_modules_post_init(conf, nfp_modules, nfp_controller): "nfp_module_post_init(), ignoring") % (identify(module)) LOG.debug(message) + def _module(args): try: index = args.index('--module') - module = args[index+1] + module = args[index + 1] return module except ValueError: print "--module missing from cmd args using default one" return None + def main(): args = sys.argv[1:] module = _module(args) @@ -588,15 +590,15 @@ def main(): args.remove(module) conf = nfp_cfg.init(args) conf.module = module - module_opts = eval('conf.%s.keys' %(module))() + module_opts = eval('conf.%s.keys' % (module))() for module_opt in module_opts: - module_cfg_opt = eval("conf.%s._group._opts['%s']['opt']" %( + module_cfg_opt = eval("conf.%s._group._opts['%s']['opt']" % ( module, module_opt)) - module_cfg_opt_value = eval("conf.%s.%s" %(module, module_opt)) + module_cfg_opt_value = eval("conf.%s.%s" % (module, module_opt)) conf.register_opt(module_cfg_opt) conf.set_override(module_opt, module_cfg_opt_value) else: - conf = nfp_cfg.init(sys.argv[1:]) + conf = nfp_cfg.init(sys.argv[1:]) nfp_common.init() nfp_controller = NfpController(conf) # Load all nfp modules from path configured diff --git a/gbpservice/nfp/proxy_agent/proxy/proxy.py b/gbpservice/nfp/proxy_agent/proxy/proxy.py index a4d905f305..4ea83fe400 100644 --- a/gbpservice/nfp/proxy_agent/proxy/proxy.py +++ b/gbpservice/nfp/proxy_agent/proxy/proxy.py @@ -353,8 +353,7 @@ def new_client(self, unixsocket, address): ) ] - - + def main(argv): oslo_config.CONF.register_opts(PROXY_OPTS, 'proxy') oslo_config.CONF(args=sys.argv[1:]) From 01627a775fb87ca296082ecebf2088501b8baaf4 Mon Sep 17 00:00:00 2001 From: Rahul Shikhare Date: Wed, 10 Aug 2016 17:41:42 +0530 Subject: [PATCH 6/9] ConfigOptions fix --- .../contrib/nfp/bin/nfp_configurator.ini | 2 +- .../config_orchestrator/modules/__init__.py | 44 +++++++++++++++++++ gbpservice/nfp/core/cfg.py | 35 ++------------- gbpservice/nfp/core/controller.py | 31 +++++++------ .../nfp/orchestrator/modules/__init__.py | 27 ++++++++++++ 5 files changed, 91 insertions(+), 48 deletions(-) diff --git a/gbpservice/contrib/nfp/bin/nfp_configurator.ini b/gbpservice/contrib/nfp/bin/nfp_configurator.ini index a11a79d513..cc3f3fdb20 100644 --- a/gbpservice/contrib/nfp/bin/nfp_configurator.ini +++ b/gbpservice/contrib/nfp/bin/nfp_configurator.ini @@ -1,4 +1,4 @@ -[DEFAULT] +[CONFIGURATOR] policy_file=/etc/policy.json debug=False rabbit_password=guest diff --git a/gbpservice/contrib/nfp/config_orchestrator/modules/__init__.py b/gbpservice/contrib/nfp/config_orchestrator/modules/__init__.py index e69de29bb2..2ebd51de9e 100644 --- a/gbpservice/contrib/nfp/config_orchestrator/modules/__init__.py +++ b/gbpservice/contrib/nfp/config_orchestrator/modules/__init__.py @@ -0,0 +1,44 @@ +from oslo_config import cfg as oslo_config + +nfp_configurator_opts = [ + oslo_config.StrOpt('policy_file', + default='/etc/policy.json', + help='use credintial file path'), + oslo_config.StrOpt('rabbit_password', + default='guest', + help='RabbitMq server login password'), + oslo_config.StrOpt('rabbit_userid', + default='guest', + help='RabbiMq server User ID'), + oslo_config.StrOpt('rabbit_hosts', + default='127.0.0.1', + help='RabbitMq server IP address for multinode setup'), + oslo_config.IntOpt('rabbit_port', + default=5672, + help='RabbitMq server port number'), + oslo_config.FloatOpt('kombu_reconnect_delay', + default=1.0, help='Kombu reconnection delay'), + oslo_config.StrOpt('rabbit_host', + default='', + help='Kombu reconnection delay'), + oslo_config.StrOpt('control_exchange', + default='openstack', + help='RabbitMq control exchange name'), + oslo_config.BoolOpt('rabbit_use_ssl', + default=False, help='RabbitMq SSL Enbale'), + oslo_config.StrOpt('rabbit_virtual_host', + default='/', help='RabbitMq virtual host path'), + oslo_config.IntOpt('reportstate_interval', + default=10, help='Report stat interval'), + oslo_config.IntOpt('periodic_interval', + default=2, help='perodi interval'), + oslo_config.StrOpt('log_forward_ip_address', + default='', help='log collector host IP address'), + oslo_config.IntOpt('log_forward_port', + default=514, help='log collector port number'), + oslo_config.StrOpt('log_level', + default='debug', help='log level'), +] + +oslo_config.CONF.register_opts(nfp_configurator_opts, "configurator") + diff --git a/gbpservice/nfp/core/cfg.py b/gbpservice/nfp/core/cfg.py index 227bb6f683..b0c23dc6d1 100644 --- a/gbpservice/nfp/core/cfg.py +++ b/gbpservice/nfp/core/cfg.py @@ -13,6 +13,8 @@ from oslo_config import cfg as oslo_config +CONF = oslo_config.CONF + NFP_OPTS = [ oslo_config.IntOpt( 'workers', @@ -33,38 +35,9 @@ ] -es_openstack_opts = [ - oslo_config.StrOpt('auth_host', - default='localhost', - help='Openstack controller IP Address'), - oslo_config.StrOpt('admin_user', - help='Admin user name to create service VMs'), - oslo_config.StrOpt('admin_password', - help='Admin password to create service VMs'), - oslo_config.StrOpt('admin_tenant_name', - help='Admin tenant name to create service VMs'), - oslo_config.StrOpt('admin_tenant_id', - help='Admin tenant ID to create service VMs'), - oslo_config.StrOpt('auth_protocol', - default='http', help='Auth protocol used.'), - oslo_config.IntOpt('auth_port', - default='5000', help='Auth protocol used.'), - oslo_config.IntOpt('bind_port', - default='9696', help='Auth protocol used.'), - oslo_config.StrOpt('auth_version', - default='v2.0', help='Auth protocol used.'), - oslo_config.StrOpt('auth_uri', - default='', help='Auth URI.'), -] - - -def init(args, **kwargs): +def init(module, args, **kwargs): """Initialize the configuration. """ - oslo_config.CONF.register_opts(NFP_OPTS) - oslo_config.CONF.register_opts(NFP_OPTS, "orchestrator") - oslo_config.CONF.register_opts(NFP_OPTS, "proxy_agent") - oslo_config.CONF.register_opts(NFP_OPTS, "config_orchestrator") - oslo_config.CONF.register_opts(es_openstack_opts, "keystone_authtoken") + oslo_config.CONF.register_opts(NFP_OPTS, module) oslo_config.CONF.set_override('use_syslog', 'True') oslo_config.CONF.set_override('syslog_log_facility', 'local1') oslo_config.CONF(args=args, project='nfp', diff --git a/gbpservice/nfp/core/controller.py b/gbpservice/nfp/core/controller.py index 915675125e..5248985f89 100644 --- a/gbpservice/nfp/core/controller.py +++ b/gbpservice/nfp/core/controller.py @@ -578,27 +578,26 @@ def _module(args): module = args[index + 1] return module except ValueError: - print "--module missing from cmd args using default one" - return None + print "--module missing from cmd args" + sys.exit(-1) def main(): args = sys.argv[1:] module = _module(args) - if module: - args.remove('--module') - args.remove(module) - conf = nfp_cfg.init(args) - conf.module = module - module_opts = eval('conf.%s.keys' % (module))() - for module_opt in module_opts: - module_cfg_opt = eval("conf.%s._group._opts['%s']['opt']" % ( - module, module_opt)) - module_cfg_opt_value = eval("conf.%s.%s" % (module, module_opt)) - conf.register_opt(module_cfg_opt) - conf.set_override(module_opt, module_cfg_opt_value) - else: - conf = nfp_cfg.init(sys.argv[1:]) + args = sys.argv[1:] + module = _module(args) + args.remove('--module') + args.remove(module) + conf = nfp_cfg.init(module, args) + conf.module = module + module_opts = eval('conf.%s.keys' % (module))() + for module_opt in module_opts: + module_cfg_opt = eval("conf.%s._group._opts['%s']['opt']" % ( + module, module_opt)) + module_cfg_opt_value = eval("conf.%s.%s" % (module, module_opt)) + conf.register_opt(module_cfg_opt) + conf.set_override(module_opt, module_cfg_opt_value) nfp_common.init() nfp_controller = NfpController(conf) # Load all nfp modules from path configured diff --git a/gbpservice/nfp/orchestrator/modules/__init__.py b/gbpservice/nfp/orchestrator/modules/__init__.py index e69de29bb2..dd0ce1019a 100644 --- a/gbpservice/nfp/orchestrator/modules/__init__.py +++ b/gbpservice/nfp/orchestrator/modules/__init__.py @@ -0,0 +1,27 @@ +from oslo_config import cfg as oslo_config + +es_openstack_opts = [ + oslo_config.StrOpt('auth_host', + default='localhost', + help='Openstack controller IP Address'), + oslo_config.StrOpt('admin_user', + help='Admin user name to create service VMs'), + oslo_config.StrOpt('admin_password', + help='Admin password to create service VMs'), + oslo_config.StrOpt('admin_tenant_name', + help='Admin tenant name to create service VMs'), + oslo_config.StrOpt('admin_tenant_id', + help='Admin tenant ID to create service VMs'), + oslo_config.StrOpt('auth_protocol', + default='http', help='Auth protocol used.'), + oslo_config.IntOpt('auth_port', + default='5000', help='Auth protocol used.'), + oslo_config.IntOpt('bind_port', + default='9696', help='Auth protocol used.'), + oslo_config.StrOpt('auth_version', + default='v2.0', help='Auth protocol used.'), + oslo_config.StrOpt('auth_uri', + default='', help='Auth URI.'), +] + +oslo_config.CONF.register_opts(es_openstack_opts, "keystone_authtoken") From df36220218e4cbf58aa6bf8358fee96d56aa956a Mon Sep 17 00:00:00 2001 From: Rahul Shikhare Date: Wed, 10 Aug 2016 17:45:27 +0530 Subject: [PATCH 7/9] error fix --- .../config_orchestrator/modules/__init__.py | 44 ------------------- .../nfp/configurator/modules/__init__.py | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/gbpservice/contrib/nfp/config_orchestrator/modules/__init__.py b/gbpservice/contrib/nfp/config_orchestrator/modules/__init__.py index 2ebd51de9e..e69de29bb2 100644 --- a/gbpservice/contrib/nfp/config_orchestrator/modules/__init__.py +++ b/gbpservice/contrib/nfp/config_orchestrator/modules/__init__.py @@ -1,44 +0,0 @@ -from oslo_config import cfg as oslo_config - -nfp_configurator_opts = [ - oslo_config.StrOpt('policy_file', - default='/etc/policy.json', - help='use credintial file path'), - oslo_config.StrOpt('rabbit_password', - default='guest', - help='RabbitMq server login password'), - oslo_config.StrOpt('rabbit_userid', - default='guest', - help='RabbiMq server User ID'), - oslo_config.StrOpt('rabbit_hosts', - default='127.0.0.1', - help='RabbitMq server IP address for multinode setup'), - oslo_config.IntOpt('rabbit_port', - default=5672, - help='RabbitMq server port number'), - oslo_config.FloatOpt('kombu_reconnect_delay', - default=1.0, help='Kombu reconnection delay'), - oslo_config.StrOpt('rabbit_host', - default='', - help='Kombu reconnection delay'), - oslo_config.StrOpt('control_exchange', - default='openstack', - help='RabbitMq control exchange name'), - oslo_config.BoolOpt('rabbit_use_ssl', - default=False, help='RabbitMq SSL Enbale'), - oslo_config.StrOpt('rabbit_virtual_host', - default='/', help='RabbitMq virtual host path'), - oslo_config.IntOpt('reportstate_interval', - default=10, help='Report stat interval'), - oslo_config.IntOpt('periodic_interval', - default=2, help='perodi interval'), - oslo_config.StrOpt('log_forward_ip_address', - default='', help='log collector host IP address'), - oslo_config.IntOpt('log_forward_port', - default=514, help='log collector port number'), - oslo_config.StrOpt('log_level', - default='debug', help='log level'), -] - -oslo_config.CONF.register_opts(nfp_configurator_opts, "configurator") - diff --git a/gbpservice/contrib/nfp/configurator/modules/__init__.py b/gbpservice/contrib/nfp/configurator/modules/__init__.py index e69de29bb2..2ebd51de9e 100644 --- a/gbpservice/contrib/nfp/configurator/modules/__init__.py +++ b/gbpservice/contrib/nfp/configurator/modules/__init__.py @@ -0,0 +1,44 @@ +from oslo_config import cfg as oslo_config + +nfp_configurator_opts = [ + oslo_config.StrOpt('policy_file', + default='/etc/policy.json', + help='use credintial file path'), + oslo_config.StrOpt('rabbit_password', + default='guest', + help='RabbitMq server login password'), + oslo_config.StrOpt('rabbit_userid', + default='guest', + help='RabbiMq server User ID'), + oslo_config.StrOpt('rabbit_hosts', + default='127.0.0.1', + help='RabbitMq server IP address for multinode setup'), + oslo_config.IntOpt('rabbit_port', + default=5672, + help='RabbitMq server port number'), + oslo_config.FloatOpt('kombu_reconnect_delay', + default=1.0, help='Kombu reconnection delay'), + oslo_config.StrOpt('rabbit_host', + default='', + help='Kombu reconnection delay'), + oslo_config.StrOpt('control_exchange', + default='openstack', + help='RabbitMq control exchange name'), + oslo_config.BoolOpt('rabbit_use_ssl', + default=False, help='RabbitMq SSL Enbale'), + oslo_config.StrOpt('rabbit_virtual_host', + default='/', help='RabbitMq virtual host path'), + oslo_config.IntOpt('reportstate_interval', + default=10, help='Report stat interval'), + oslo_config.IntOpt('periodic_interval', + default=2, help='perodi interval'), + oslo_config.StrOpt('log_forward_ip_address', + default='', help='log collector host IP address'), + oslo_config.IntOpt('log_forward_port', + default=514, help='log collector port number'), + oslo_config.StrOpt('log_level', + default='debug', help='log level'), +] + +oslo_config.CONF.register_opts(nfp_configurator_opts, "configurator") + From a8e14393b77c0168c8a9e5b65e1fb45f4f0c7d0f Mon Sep 17 00:00:00 2001 From: Rahul Shikhare Date: Wed, 10 Aug 2016 17:53:54 +0530 Subject: [PATCH 8/9] typo error fix --- .../contrib/nfp/configurator/modules/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gbpservice/contrib/nfp/configurator/modules/__init__.py b/gbpservice/contrib/nfp/configurator/modules/__init__.py index 2ebd51de9e..0d7b944824 100644 --- a/gbpservice/contrib/nfp/configurator/modules/__init__.py +++ b/gbpservice/contrib/nfp/configurator/modules/__init__.py @@ -3,7 +3,7 @@ nfp_configurator_opts = [ oslo_config.StrOpt('policy_file', default='/etc/policy.json', - help='use credintial file path'), + help='use credentials file path'), oslo_config.StrOpt('rabbit_password', default='guest', help='RabbitMq server login password'), @@ -20,18 +20,18 @@ default=1.0, help='Kombu reconnection delay'), oslo_config.StrOpt('rabbit_host', default='', - help='Kombu reconnection delay'), + help='RabbitMq server IP address for single node setup'), oslo_config.StrOpt('control_exchange', default='openstack', help='RabbitMq control exchange name'), oslo_config.BoolOpt('rabbit_use_ssl', - default=False, help='RabbitMq SSL Enbale'), + default=False, help='RabbitMq SSL mode True/False'), oslo_config.StrOpt('rabbit_virtual_host', default='/', help='RabbitMq virtual host path'), oslo_config.IntOpt('reportstate_interval', - default=10, help='Report stat interval'), + default=10, help='Report stats interval'), oslo_config.IntOpt('periodic_interval', - default=2, help='perodi interval'), + default=2, help='perodic interval'), oslo_config.StrOpt('log_forward_ip_address', default='', help='log collector host IP address'), oslo_config.IntOpt('log_forward_port', From f7a7ea5646175d8fe0f962e5b18d4ffda89734b8 Mon Sep 17 00:00:00 2001 From: Rahul Shikhare Date: Wed, 10 Aug 2016 18:01:45 +0530 Subject: [PATCH 9/9] configurator run command changes --- gbpservice/contrib/nfp/configurator/modules/__init__.py | 4 ++-- gbpservice/contrib/nfp/configurator/run.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gbpservice/contrib/nfp/configurator/modules/__init__.py b/gbpservice/contrib/nfp/configurator/modules/__init__.py index 0d7b944824..bce94dcd02 100644 --- a/gbpservice/contrib/nfp/configurator/modules/__init__.py +++ b/gbpservice/contrib/nfp/configurator/modules/__init__.py @@ -12,7 +12,7 @@ help='RabbiMq server User ID'), oslo_config.StrOpt('rabbit_hosts', default='127.0.0.1', - help='RabbitMq server IP address for multinode setup'), + help='RabbitMq server IP address for multi node setup'), oslo_config.IntOpt('rabbit_port', default=5672, help='RabbitMq server port number'), @@ -31,7 +31,7 @@ oslo_config.IntOpt('reportstate_interval', default=10, help='Report stats interval'), oslo_config.IntOpt('periodic_interval', - default=2, help='perodic interval'), + default=2, help='periodic interval'), oslo_config.StrOpt('log_forward_ip_address', default='', help='log collector host IP address'), oslo_config.IntOpt('log_forward_port', diff --git a/gbpservice/contrib/nfp/configurator/run.sh b/gbpservice/contrib/nfp/configurator/run.sh index 73da244bcf..0289b5117a 100644 --- a/gbpservice/contrib/nfp/configurator/run.sh +++ b/gbpservice/contrib/nfp/configurator/run.sh @@ -1,7 +1,7 @@ #!/bin/bash service rabbitmq-server start -screen -dmS "configurator" /usr/bin/python2 /usr/bin/nfp --config-file=/etc/nfp_configurator.ini --log-file=/var/log/nfp/nfp_configurator.log +screen -dmS "configurator" /usr/bin/python2 /usr/bin/nfp --config-file=/etc/nfp_configurator.ini --module configurator --log-file=/var/log/nfp/nfp_configurator.log cd /usr/local/lib/python2.7/dist-packages/gbpservice/nfp/pecan/api/ python setup.py develop screen -dmS "pecan" pecan configurator_decider config.py --mode advanced