From a1074438fae25891044f147f37276e9d0a0657a3 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Thu, 12 Dec 2019 17:38:10 -0800 Subject: [PATCH 01/15] [config_mgmt.py]: Class in sonic_util config for Config Validation and for BreakOut Ports. Provided a class in sonic_util config for -- Config Validation, -- BreakOut Ports Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 715 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 715 insertions(+) create mode 100644 config/config_mgmt.py diff --git a/config/config_mgmt.py b/config/config_mgmt.py new file mode 100644 index 0000000000..8c2f1a65ee --- /dev/null +++ b/config/config_mgmt.py @@ -0,0 +1,715 @@ +#!/usr/bin/env python +# +# config_mgmt.py +# Provides a class for configuration validation and for Dynamic Port Breakout. + +try: + + # import from sonic-cfggen, re use this + from imp import load_source + load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') + from sonic_cfggen import deep_update, FormatConverter, sort_data + from swsssdk import ConfigDBConnector, SonicV2Connector, port_util + from pprint import PrettyPrinter, pprint + from json import dump, load, dumps, loads + from sys import path as sysPath + from os import path as osPath + from os import system + from datetime import datetime + from time import sleep as tsleep + + import sonic_yang + import syslog + import re + +except ImportError as e: + raise ImportError("%s - required module not found" % str(e)) + +# Globals +# This class may not need to know about YANG_DIR ?, sonic_yang shd use +# default dir. +YANG_DIR = "/usr/local/yang-models" +CONFIG_DB_JSON_FILE = '/etc/sonic/confib_db.json' +# TODO: Find a place for it on sonic switch. +DEFAULT_CONFIG_DB_JSON_FILE = '/etc/sonic/default_config_db.json' + +# Class to handle config managment for SONIC, this class will use PLY to verify +# config for the commands which are capable of change in config DB. + +class ConfigMgmt(): + + def __init__(self, source="configDB", debug=False, allowTablesWithOutYang=True): + + try: + self.configdbJsonIn = None + self.configdbJsonOut = None + self.allowTablesWithOutYang = allowTablesWithOutYang + + # logging vars + self.SYSLOG_IDENTIFIER = "ConfigMgmt" + self.DEBUG = debug + + self.sy = sonic_yang.SonicYang(YANG_DIR, debug=debug) + # load yang models + self.sy.loadYangModel() + # load jIn from config DB or from config DB json file. + if source.lower() == 'configdb': + self.readConfigDB() + # treat any other source as file input + else: + self.readConfigDBJson(source) + # this will crop config, xlate and load. + self.sy.loadData(self.configdbJsonIn) + + # Raise if tables without YANG models are not allowed but exist. + if not allowTablesWithOutYang and len(self.sy.tablesWithOutYang): + raise Exception('Config has tables without YANG models') + + except Exception as e: + print(e) + raise(Exception('ConfigMgmt Class creation failed')) + + return + + def __del__(self): + pass + + """ + Return tables loaded in config for which YANG model does not exist. + """ + def tablesWithOutYang(self): + + return self.sy.tablesWithOutYang + + """ + Explicit function to load config data in Yang Data Tree + """ + def loadData(self, configdbJson): + self.sy.loadData(configdbJson) + # Raise if tables without YANG models are not allowed but exist. + if not self.allowTablesWithOutYang and len(self.sy.tablesWithOutYang): + raise Exception('Config has tables without YANG models') + + return + + """ + Validate current Data Tree + """ + def validateConfigData(self): + + try: + self.sy.validate_data_tree() + except Exception as e: + self.sysLog(msg='Data Validation Failed') + return False + + print('Data Validation successful') + self.sysLog(msg='Data Validation successful') + return True + + """ + syslog Support + """ + def sysLog(self, debug=syslog.LOG_INFO, msg=None): + + # log debug only if enabled + if self.DEBUG == False and debug == syslog.LOG_DEBUG: + return + syslog.openlog(self.SYSLOG_IDENTIFIER) + syslog.syslog(debug, msg) + syslog.closelog() + + return + + def readConfigDBJson(self, source=CONFIG_DB_JSON_FILE): + + print('Reading data from {}'.format(source)) + self.configdbJsonIn = readJsonFile(source) + #print(type(self.configdbJsonIn)) + if not self.configdbJsonIn: + raise(Exception("Can not load config from config DB json file")) + self.sysLog(msg='Reading Input {}'.format(self.configdbJsonIn)) + + return + + """ + Get config from redis config DB + """ + def readConfigDB(self): + + print('Reading data from Redis configDb') + + # Read from config DB on sonic switch + db_kwargs = dict(); data = dict() + configdb = ConfigDBConnector(**db_kwargs) + configdb.connect() + deep_update(data, FormatConverter.db_to_output(configdb.get_config())) + self.configdbJsonIn = FormatConverter.to_serialized(data) + self.sysLog(syslog.LOG_DEBUG, 'Reading Input from ConfigDB {}'.\ + format(self.configdbJsonIn)) + + return + + def writeConfigDB(self, jDiff): + print('Writing in Config DB') + """ + On Sonic Switch + """ + db_kwargs = dict(); data = dict() + configdb = ConfigDBConnector(**db_kwargs) + configdb.connect(False) + deep_update(data, FormatConverter.to_deserialized(jDiff)) + data = sort_data(data) + self.sysLog(msg="Write in DB: {}".format(data)) + configdb.mod_config(FormatConverter.output_to_db(data)) + + return + +# End of Class ConfigMgmt + +""" + Config MGMT class for Dynamic Port Breakout(DPB). + This is derived from ConfigMgmt. +""" +class ConfigMgmtDPB(ConfigMgmt): + + def __init__(self, source="configDB", debug=False, allowTablesWithOutYang=True): + + try: + ConfigMgmt.__init__(self, source=source, debug=debug, \ + allowTablesWithOutYang=allowTablesWithOutYang) + self.oidKey = 'ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x' + + except Exception as e: + print(e) + raise(Exception('ConfigMgmtDPB Class creation failed')) + + return + + def __del__(self): + pass + + """ + Check if a key exists in ASIC DB or not. + """ + def _checkKeyinAsicDB(self, key, db): + + self.sysLog(msg='Check Key in Asic DB: {}'.format(key)) + try: + # chk key in ASIC DB + if db.exists('ASIC_DB', key): + return True + except Exception as e: + print(e) + raise(e) + + return False + + def _testRedisCli(self, key): + # To Debug + if self.DEBUG: + cmd = 'sudo redis-cli -n 1 hgetall "{}"'.format(key) + self.sysLog(syslog.LOG_DEBUG, "Running {}".format(cmd)) + print(cmd) + system(cmd) + return + + """ + Check ASIC DB for PORTs in port List + ports: List of ports + portMap: port to OID map. + Return: True, if all ports are not present. + """ + def _checkNoPortsInAsicDb(self, db, ports, portMap): + try: + # connect to ASIC DB, + db.connect(db.ASIC_DB) + for port in ports: + key = self.oidKey + portMap[port] + if self._checkKeyinAsicDB(key, db) == False: + # Test again via redis-cli + self._testRedisCli(key) + else: + return False + + except Exception as e: + print(e) + return False + + return True + + """ + Verify in the Asic DB that port are deleted, + Keep on trying till timeout period. + db = database, ports, portMap, timeout + """ + def _verifyAsicDB(self, db, ports, portMap, timeout): + + print("Verify Port Deletion from Asic DB, Wait...") + self.sysLog(msg="Verify Port Deletion from Asic DB, Wait...") + + try: + for waitTime in range(timeout): + self.sysLog(msg='Check Asic DB: {} try'.format(waitTime+1)) + # checkNoPortsInAsicDb will return True if all ports are not + # present in ASIC DB + if self._checkNoPortsInAsicDb(db, ports, portMap): + break + tsleep(1) + + # raise if timer expired + if waitTime + 1 == timeout: + print("!!! Critical Failure, Ports are not Deleted from \ + ASIC DB, Bail Out !!!") + self.sysLog(syslog.LOG_CRIT, "!!! Critical Failure, Ports \ + are not Deleted from ASIC DB, Bail Out !!!") + raise(Exception("Ports are present in ASIC DB after timeout")) + + except Exception as e: + print(e) + raise e + + return True + + def breakOutPort(self, delPorts=list(), addPorts= list(), portJson=dict(), \ + force=False, loadDefConfig=True): + + MAX_WAIT = 60 + try: + # delete Port and get the Config diff, deps and True/False + delConfigToLoad, deps, ret = self._deletePorts(ports=delPorts, \ + force=force) + # return dependencies if delete port fails + if ret == False: + return deps, ret + + # add Ports and get the config diff and True/False + addConfigtoLoad, ret = self._addPorts(ports=addPorts, \ + portJson=portJson, loadDefConfig=loadDefConfig) + # return if ret is False, Great thing, no change is done in Config + if ret == False: + return None, ret + + # Save Port OIDs Mapping Before Deleting Port + dataBase = SonicV2Connector(host="127.0.0.1") + if_name_map, if_oid_map = port_util.get_interface_oid_map(dataBase) + self.sysLog(syslog.LOG_DEBUG, 'if_name_map {}'.format(if_name_map)) + + # If we are here, then get ready to update the Config DB, Update + # deletion of Config first, then verify in Asic DB for port deletion, + # then update addition of ports in config DB. + self.writeConfigDB(delConfigToLoad) + # Verify in Asic DB, + self._verifyAsicDB(db=dataBase, ports=delPorts, portMap=if_name_map, \ + timeout=MAX_WAIT) + self.writeConfigDB(addConfigtoLoad) + + except Exception as e: + print(e) + return None, False + + return None, True + + """ + Delete all ports. + delPorts: list of port names. + force: if false return dependecies, else delete dependencies. + + Return: + WithOut Force: (xpath of dependecies, False) or (None, True) + With Force: (xpath of dependecies, False) or (None, True) + """ + def _deletePorts(self, ports=list(), force=False): + + configToLoad = None; deps = None + try: + self.sysLog(msg="delPorts ports:{} force:{}".format(ports, force)) + + print('\nStart Port Deletion') + deps = list() + + # Get all dependecies for ports + for port in ports: + xPathPort = self.sy.findXpathPortLeaf(port) + print('Find dependecies for port {}'.format(port)) + self.sysLog(msg='Find dependecies for port {}'.format(port)) + # print("Generated Xpath:" + xPathPort) + dep = self.sy.find_data_dependencies(str(xPathPort)) + if dep: + deps.extend(dep) + + # No further action with no force and deps exist + if force == False and deps: + return configToLoad, deps, False; + + # delets all deps, No topological sort is needed as of now, if deletion + # of deps fails, return immediately + elif deps and force: + for dep in deps: + self.sysLog(msg='Deleting {}'.format(dep)) + self.sy.deleteNode(str(dep)) + # mark deps as None now, + deps = None + + # all deps are deleted now, delete all ports now + for port in ports: + xPathPort = self.sy.findXpathPort(port) + print("Deleting Port: " + port) + self.sysLog(msg='Deleting Port:{}'.format(port)) + self.sy.deleteNode(str(xPathPort)) + + # Let`s Validate the tree now + if self.validateConfigData()==False: + return configToLoad, deps, False; + + # All great if we are here, Lets get the diff + self.configdbJsonOut = self.sy.getData() + # Update configToLoad + configToLoad = self._updateDiffConfigDB() + + except Exception as e: + print(e) + print("Port Deletion Failed") + return configToLoad, deps, False + + return configToLoad, deps, True + + """ + Add Ports and default config for ports to config DB, after validation of + data tree + + PortJson: Config DB Json Part of all Ports same as PORT Table of Config DB. + ports = list of ports + loadDefConfig: If loadDefConfig add default config as well. + + return: Sucess: True or Failure: False + """ + def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): + + configToLoad = None + try: + self.sysLog(msg='Start Port Addition') + self.sysLog(msg="addPorts ports:{} loadDefConfig:{}".\ + format(ports, loadDefConfig)) + self.sysLog(msg="addPorts Args portjson {}".format(portJson)) + + print('\nStart Port Addition') + # get default config if forced + defConfig = dict() + if loadDefConfig: + defConfig = self._getDefaultConfig(ports) + self.sysLog(msg='Default Config: {}'.format(defConfig)) + + # get the latest Data Tree, save this in input config, since this + # is our starting point now + self.configdbJsonIn = self.sy.getData() + + # Get the out dict as well, if not done already + if self.configdbJsonOut is None: + self.configdbJsonOut = self.sy.getData() + + # update portJson in configdbJsonOut PORT part + self.configdbJsonOut['PORT'].update(portJson['PORT']) + # merge new config with data tree, this is json level merge. + # We do not allow new table merge while adding default config. + if loadDefConfig: + print("Merge Default Config for {}".format(ports)) + self.sysLog(msg="Merge Default Config for {}".format(ports)) + self._mergeConfigs(self.configdbJsonOut, defConfig, True) + + # create a tree with merged config and validate, if validation is + # sucessful, then configdbJsonOut contains final and valid config. + self.sy.loadData(self.configdbJsonOut) + if self.validateConfigData()==False: + return configToLoad, False + + # All great if we are here, Let`s get the diff and update COnfig + configToLoad = self._updateDiffConfigDB() + + except Exception as e: + print(e) + print("Port Addition Failed") + return configToLoad, False + + return configToLoad, True + + """ + Merge second dict in first, Note both first and second dict will be changed + First Dict will have merged part D1 + D2 + Second dict will have D2 - D1 [unique keys in D2] + Unique keys in D2 will be merged in D1 only if uniqueKeys=True + """ + def _mergeConfigs(self, D1, D2, uniqueKeys=True): + + try: + def _mergeItems(it1, it2): + if isinstance(it1, list) and isinstance(it2, list): + it1.extend(it2) + elif isinstance(it1, dict) and isinstance(it2, dict): + self._mergeConfigs(it1, it2) + elif isinstance(it1, list) or isinstance(it2, list): + raise ("Can not merge Configs, List problem") + elif isinstance(it1, dict) or isinstance(it2, dict): + raise ("Can not merge Configs, Dict problem") + else: + #print("Do nothing") + # First Dict takes priority + pass + return + + for it in D1.keys(): + #print(it) + # D2 has the key + if D2.get(it): + _mergeItems(D1[it], D2[it]) + del D2[it] + # D2 does not have the keys + else: + pass + + # if uniqueKeys are needed, merge rest of the keys of D2 in D1 + if uniqueKeys: + D1.update(D2) + except Exce as e: + print("Merge Config failed") + print(e) + raise e + + return D1 + + """ + search Relevant Keys in Config using DFS, This function is mainly + used to search port related config in Default ConfigDbJson file. + In: Config to be searched + skeys: Keys to be searched in In Config i.e. search Keys. + Out: Contains the search result + """ + def _searchKeysInConfig(self, In, Out, skeys): + + found = False + if isinstance(In, dict): + for key in In.keys(): + #print("key:" + key) + for skey in skeys: + # pattern is very specific to current primary keys in + # config DB, may need to be updated later. + pattern = '^' + skey + '\|' + '|' + skey + '$' + \ + '|' + '^' + skey + '$' + #print(pattern) + reg = re.compile(pattern) + #print(reg) + if reg.search(key): + # In primary key, only 1 match can be found, so return + # print("Added key:" + key) + Out[key] = In[key] + found = True + break + # Put the key in Out by default, if not added already. + # Remove later, if subelements does not contain any port. + if Out.get(key) is None: + Out[key] = type(In[key])() + if self._searchKeysInConfig(In[key], Out[key], skeys) == False: + del Out[key] + else: + found = True + + elif isinstance(In, list): + for skey in skeys: + if skey in In: + found = True + Out.append(skey) + #print("Added in list:" + port) + + else: + # nothing for other keys + pass + + return found + + """ + This function returns the relavant keys in Input Config. + For Example: All Ports related Config in Config DB. + """ + def configWithKeys(self, configIn=dict(), keys=list()): + + configOut = dict() + try: + if len(configIn) and len(keys): + self._searchKeysInConfig(configIn, configOut, skeys=keys) + except Exception as e: + print("configWithKeys Failed, Error: {}".format(str(e))) + raise e + + return configOut + + """ + Create a defConfig for given Ports from Default Config File. + """ + def _getDefaultConfig(self, ports=list()): + + # function code + try: + print("Generating default config for {}".format(ports)) + defConfigIn = readJsonFile(DEFAULT_CONFIG_DB_JSON_FILE) + #print(defConfigIn) + defConfigOut = dict() + self._searchKeysInConfig(defConfigIn, defConfigOut, skeys=ports) + except Exception as e: + print("getDefaultConfig Failed, Error: {}".format(str(e))) + raise e + + return defConfigOut + + def _updateDiffConfigDB(self): + + # main code starts here + configToLoad = dict() + try: + # Get the Diff + print('Generate Final Config to write in DB') + configDBdiff = self._diffJson() + # Process diff and create Config which can be updated in Config DB + configToLoad = self._createConfigToLoad(configDBdiff, \ + self.configdbJsonIn, self.configdbJsonOut) + + except Exception as e: + print("Config Diff Generation failed") + print(e) + raise e + + return configToLoad + + """ + Create the config to write in Config DB from json diff + diff: diff in input config and output config. + inp: input config before delete/add ports. + outp: output config after delete/add ports. + """ + def _createConfigToLoad(self, diff, inp, outp): + + ### Internal Functions ### + """ + Handle deletes in diff dict + """ + def _deleteHandler(diff, inp, outp, config): + + # if output is dict, delete keys from config + if isinstance(inp, dict): + for key in diff: + #print(key) + # make sure keys from diff are present in inp but not in outp + # then delete it. + if key in inp and key not in outp: + # assign key to None(null), redis will delete entire key + config[key] = None + else: + # log such keys + print("Diff: Probably wrong key: {}".format(key)) + + elif isinstance(inp, list): + # just take list from output + # print("Delete from List: {} {} {}".format(inp, outp, list)) + #print(type(config)) + config.extend(outp) + + return + + """ + Handle inserts in diff dict + """ + def _insertHandler(diff, inp, outp, config): + + # if outp is a dict + if isinstance(outp, dict): + for key in diff: + #print(key) + # make sure keys are only in outp + if key not in inp and key in outp: + # assign key in config same as outp + config[key] = outp[key] + else: + # log such keys + print("Diff: Probably wrong key: {}".format(key)) + + elif isinstance(outp, list): + # just take list from output + # print("Delete from List: {} {} {}".format(inp, outp, list)) + config.extend(outp) + + return + + """ + Recursively iterate diff to generate config to write in configDB + """ + def _recurCreateConfig(diff, inp, outp, config): + + changed = False + # updates are represented by list in diff and as dict in outp\inp + # we do not allow updates right now + if isinstance(diff, list) and isinstance(outp, dict): + return changed + + idx = -1 + for key in diff: + #print(key) + idx = idx + 1 + if str(key) == '$delete': + _deleteHandler(diff[key], inp, outp, config) + changed = True + elif str(key) == '$insert': + _insertHandler(diff[key], inp, outp, config) + changed = True + else: + # insert in config by default, remove later if not needed + if isinstance(diff, dict): + # config should match with outp + config[key] = type(outp[key])() + if _recurCreateConfig(diff[key], inp[key], outp[key], \ + config[key]) == False: + del config[key] + else: + changed = True + elif isinstance(diff, list): + config.append(key) + if _recurCreateConfig(diff[idx], inp[idx], outp[idx], \ + config[-1]) == False: + del config[-1] + else: + changed = True + + return changed + + ### Function Code ### + try: + configToLoad = dict() + #import pdb; pdb.set_trace() + _recurCreateConfig(diff, inp, outp, configToLoad) + + except Exception as e: + print("Create Config to load in DB, Failed") + print(e) + raise e + + # if debug + #with open('configToLoad.json', 'w') as f: + # dump(configToLoad, f, indent=4) + + return configToLoad + + def _diffJson(self): + + from jsondiff import diff + return diff(self.configdbJsonIn, self.configdbJsonOut, syntax='symmetric') + +# end of class ConfigMgmtDPB + +# Helper Functions +def readJsonFile(fileName): + + try: + with open(fileName) as f: + result = load(f) + except Exception as e: + raise Exception(e) + + return result From d1af804b595ba0ffc5336e8e4d42251b0d1e3854 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Wed, 13 May 2020 14:08:37 -0700 Subject: [PATCH 02/15] [config_mgmt.py]: Categories public and private functions. Changes: -- Categories public and private functions. -- LGTM issues. --- config/config_mgmt.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 8c2f1a65ee..74d0c25e6a 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -10,12 +10,8 @@ load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') from sonic_cfggen import deep_update, FormatConverter, sort_data from swsssdk import ConfigDBConnector, SonicV2Connector, port_util - from pprint import PrettyPrinter, pprint - from json import dump, load, dumps, loads - from sys import path as sysPath - from os import path as osPath + from json import load from os import system - from datetime import datetime from time import sleep as tsleep import sonic_yang @@ -92,7 +88,7 @@ def loadData(self, configdbJson): return - """ + """ Validate current Data Tree """ def validateConfigData(self): @@ -394,8 +390,7 @@ def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): self.sysLog(msg="addPorts Args portjson {}".format(portJson)) print('\nStart Port Addition') - # get default config if forced - defConfig = dict() + if loadDefConfig: defConfig = self._getDefaultConfig(ports) self.sysLog(msg='Default Config: {}'.format(defConfig)) @@ -563,7 +558,6 @@ def _getDefaultConfig(self, ports=list()): def _updateDiffConfigDB(self): # main code starts here - configToLoad = dict() try: # Get the Diff print('Generate Final Config to write in DB') From 117a9eac21d338c6dd61483fa6ed147b7584b589 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Thu, 21 May 2020 10:39:28 -0700 Subject: [PATCH 03/15] [config_mgmt.py]: Address review comment V1. Changes: -- import order -- WithOut --> Without -- remove shebang --- config/config_mgmt.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 74d0c25e6a..bb240b571d 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -1,22 +1,24 @@ -#!/usr/bin/env python -# # config_mgmt.py # Provides a class for configuration validation and for Dynamic Port Breakout. try: - # import from sonic-cfggen, re use this - from imp import load_source - load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') - from sonic_cfggen import deep_update, FormatConverter, sort_data - from swsssdk import ConfigDBConnector, SonicV2Connector, port_util + import re + import syslog + from json import load from os import system from time import sleep as tsleep + from imp import load_source + # SONiC specific imports import sonic_yang - import syslog - import re + from swsssdk import ConfigDBConnector, SonicV2Connector, port_util + + # Using load_source to 'import /usr/local/bin/sonic-cfggen as sonic_cfggen' + # since /usr/local/bin/sonic-cfggen does not have .py extension. + load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen') + from sonic_cfggen import deep_update, FormatConverter, sort_data except ImportError as e: raise ImportError("%s - required module not found" % str(e)) @@ -34,12 +36,12 @@ class ConfigMgmt(): - def __init__(self, source="configDB", debug=False, allowTablesWithOutYang=True): + def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): try: self.configdbJsonIn = None self.configdbJsonOut = None - self.allowTablesWithOutYang = allowTablesWithOutYang + self.allowTablesWithoutYang = allowTablesWithoutYang # logging vars self.SYSLOG_IDENTIFIER = "ConfigMgmt" @@ -58,7 +60,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithOutYang=True): self.sy.loadData(self.configdbJsonIn) # Raise if tables without YANG models are not allowed but exist. - if not allowTablesWithOutYang and len(self.sy.tablesWithOutYang): + if not allowTablesWithoutYang and len(self.sy.tablesWithOutYang): raise Exception('Config has tables without YANG models') except Exception as e: @@ -73,7 +75,7 @@ def __del__(self): """ Return tables loaded in config for which YANG model does not exist. """ - def tablesWithOutYang(self): + def tablesWithoutYang(self): return self.sy.tablesWithOutYang @@ -83,7 +85,7 @@ def tablesWithOutYang(self): def loadData(self, configdbJson): self.sy.loadData(configdbJson) # Raise if tables without YANG models are not allowed but exist. - if not self.allowTablesWithOutYang and len(self.sy.tablesWithOutYang): + if not self.allowTablesWithoutYang and len(self.sy.tablesWithOutYang): raise Exception('Config has tables without YANG models') return @@ -169,11 +171,11 @@ def writeConfigDB(self, jDiff): """ class ConfigMgmtDPB(ConfigMgmt): - def __init__(self, source="configDB", debug=False, allowTablesWithOutYang=True): + def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): try: ConfigMgmt.__init__(self, source=source, debug=debug, \ - allowTablesWithOutYang=allowTablesWithOutYang) + allowTablesWithoutYang=allowTablesWithoutYang) self.oidKey = 'ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x' except Exception as e: @@ -557,7 +559,6 @@ def _getDefaultConfig(self, ports=list()): def _updateDiffConfigDB(self): - # main code starts here try: # Get the Diff print('Generate Final Config to write in DB') @@ -684,10 +685,6 @@ def _recurCreateConfig(diff, inp, outp, config): print(e) raise e - # if debug - #with open('configToLoad.json', 'w') as f: - # dump(configToLoad, f, indent=4) - return configToLoad def _diffJson(self): From cdf0047c97e7ad5faa9d4a378d79f747f3d5545d Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Wed, 17 Jun 2020 22:40:07 -0700 Subject: [PATCH 04/15] [config/config_mgmt.py]: Remove empty lines at start of functions. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index bb240b571d..aafbaed536 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -2,7 +2,6 @@ # Provides a class for configuration validation and for Dynamic Port Breakout. try: - import re import syslog @@ -37,7 +36,6 @@ class ConfigMgmt(): def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): - try: self.configdbJsonIn = None self.configdbJsonOut = None @@ -76,7 +74,6 @@ def __del__(self): Return tables loaded in config for which YANG model does not exist. """ def tablesWithoutYang(self): - return self.sy.tablesWithOutYang """ @@ -94,7 +91,6 @@ def loadData(self, configdbJson): Validate current Data Tree """ def validateConfigData(self): - try: self.sy.validate_data_tree() except Exception as e: @@ -109,7 +105,6 @@ def validateConfigData(self): syslog Support """ def sysLog(self, debug=syslog.LOG_INFO, msg=None): - # log debug only if enabled if self.DEBUG == False and debug == syslog.LOG_DEBUG: return @@ -120,7 +115,6 @@ def sysLog(self, debug=syslog.LOG_INFO, msg=None): return def readConfigDBJson(self, source=CONFIG_DB_JSON_FILE): - print('Reading data from {}'.format(source)) self.configdbJsonIn = readJsonFile(source) #print(type(self.configdbJsonIn)) @@ -134,9 +128,7 @@ def readConfigDBJson(self, source=CONFIG_DB_JSON_FILE): Get config from redis config DB """ def readConfigDB(self): - print('Reading data from Redis configDb') - # Read from config DB on sonic switch db_kwargs = dict(); data = dict() configdb = ConfigDBConnector(**db_kwargs) @@ -172,7 +164,6 @@ def writeConfigDB(self, jDiff): class ConfigMgmtDPB(ConfigMgmt): def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): - try: ConfigMgmt.__init__(self, source=source, debug=debug, \ allowTablesWithoutYang=allowTablesWithoutYang) @@ -191,7 +182,6 @@ def __del__(self): Check if a key exists in ASIC DB or not. """ def _checkKeyinAsicDB(self, key, db): - self.sysLog(msg='Check Key in Asic DB: {}'.format(key)) try: # chk key in ASIC DB @@ -242,10 +232,8 @@ def _checkNoPortsInAsicDb(self, db, ports, portMap): db = database, ports, portMap, timeout """ def _verifyAsicDB(self, db, ports, portMap, timeout): - print("Verify Port Deletion from Asic DB, Wait...") self.sysLog(msg="Verify Port Deletion from Asic DB, Wait...") - try: for waitTime in range(timeout): self.sysLog(msg='Check Asic DB: {} try'.format(waitTime+1)) @@ -271,7 +259,6 @@ def _verifyAsicDB(self, db, ports, portMap, timeout): def breakOutPort(self, delPorts=list(), addPorts= list(), portJson=dict(), \ force=False, loadDefConfig=True): - MAX_WAIT = 60 try: # delete Port and get the Config diff, deps and True/False @@ -318,7 +305,6 @@ def breakOutPort(self, delPorts=list(), addPorts= list(), portJson=dict(), \ With Force: (xpath of dependecies, False) or (None, True) """ def _deletePorts(self, ports=list(), force=False): - configToLoad = None; deps = None try: self.sysLog(msg="delPorts ports:{} force:{}".format(ports, force)) @@ -383,7 +369,6 @@ def _deletePorts(self, ports=list(), force=False): return: Sucess: True or Failure: False """ def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): - configToLoad = None try: self.sysLog(msg='Start Port Addition') @@ -437,7 +422,6 @@ def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): Unique keys in D2 will be merged in D1 only if uniqueKeys=True """ def _mergeConfigs(self, D1, D2, uniqueKeys=True): - try: def _mergeItems(it1, it2): if isinstance(it1, list) and isinstance(it2, list): @@ -482,7 +466,6 @@ def _mergeItems(it1, it2): Out: Contains the search result """ def _searchKeysInConfig(self, In, Out, skeys): - found = False if isinstance(In, dict): for key in In.keys(): @@ -528,7 +511,6 @@ def _searchKeysInConfig(self, In, Out, skeys): For Example: All Ports related Config in Config DB. """ def configWithKeys(self, configIn=dict(), keys=list()): - configOut = dict() try: if len(configIn) and len(keys): @@ -543,7 +525,6 @@ def configWithKeys(self, configIn=dict(), keys=list()): Create a defConfig for given Ports from Default Config File. """ def _getDefaultConfig(self, ports=list()): - # function code try: print("Generating default config for {}".format(ports)) @@ -558,7 +539,6 @@ def _getDefaultConfig(self, ports=list()): return defConfigOut def _updateDiffConfigDB(self): - try: # Get the Diff print('Generate Final Config to write in DB') @@ -581,13 +561,11 @@ def _updateDiffConfigDB(self): outp: output config after delete/add ports. """ def _createConfigToLoad(self, diff, inp, outp): - ### Internal Functions ### """ Handle deletes in diff dict """ def _deleteHandler(diff, inp, outp, config): - # if output is dict, delete keys from config if isinstance(inp, dict): for key in diff: @@ -613,7 +591,6 @@ def _deleteHandler(diff, inp, outp, config): Handle inserts in diff dict """ def _insertHandler(diff, inp, outp, config): - # if outp is a dict if isinstance(outp, dict): for key in diff: @@ -637,7 +614,6 @@ def _insertHandler(diff, inp, outp, config): Recursively iterate diff to generate config to write in configDB """ def _recurCreateConfig(diff, inp, outp, config): - changed = False # updates are represented by list in diff and as dict in outp\inp # we do not allow updates right now @@ -688,7 +664,6 @@ def _recurCreateConfig(diff, inp, outp, config): return configToLoad def _diffJson(self): - from jsondiff import diff return diff(self.configdbJsonIn, self.configdbJsonOut, syntax='symmetric') @@ -696,7 +671,6 @@ def _diffJson(self): # Helper Functions def readJsonFile(fileName): - try: with open(fileName) as f: result = load(f) From 7810ab0532ccc9cad398aec36e331d8993bca988 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Mon, 15 Jun 2020 22:50:09 -0700 Subject: [PATCH 05/15] [sonic-utilities-tests/config_mgmt_test.py]: Unit Tests for config_mgmt.py library. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- sonic-utilities-tests/config_mgmt_test.py | 475 ++++++++++++++++++++++ 1 file changed, 475 insertions(+) create mode 100644 sonic-utilities-tests/config_mgmt_test.py diff --git a/sonic-utilities-tests/config_mgmt_test.py b/sonic-utilities-tests/config_mgmt_test.py new file mode 100644 index 0000000000..8a370fa6a5 --- /dev/null +++ b/sonic-utilities-tests/config_mgmt_test.py @@ -0,0 +1,475 @@ +import imp +import os +import mock_tables.dbconnector +# import file under test i.e. config_mgmt.py +imp.load_source('config_mgmt', \ + os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py')) +import config_mgmt + +from unittest import TestCase +from mock import MagicMock, call +from json import dump + +class TestConfigMgmt(TestCase): + + def setUp(self): + config_mgmt.YANG_DIR = "../../sonic-yang-models/yang-models/" + config_mgmt.CONFIG_DB_JSON_FILE = "startConfigDb.json" + config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE = "portBreakOutConfigDb.json" + return + + def test_config_validation(self): + iConfig = dict(configDbJson) + self.writeJson(iConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cm = config_mgmt.ConfigMgmt(source=config_mgmt.CONFIG_DB_JSON_FILE) + assert cm.validateConfigData() == True + return + + def test_table_without_yang(self): + iConfig = dict(configDbJson) + unknown = {"unknown_table": {"ukey": "uvalue"}} + self.updateConfig(iConfig, unknown) + self.writeJson(iConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cm = config_mgmt.ConfigMgmt(source=config_mgmt.CONFIG_DB_JSON_FILE) + #assert "unknown_table" in cm.tablesWithoutYang() + return + + def test_search_keys(self): + iConfig = dict(configDbJson) + self.writeJson(iConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) + out = cmdpb.configWithKeys(portBreakOutConfigDbJson, \ + ["Ethernet8","Ethernet9"]) + assert "VLAN" not in out.keys() + assert "INTERFACE" not in out.keys() + for k in out['ACL_TABLE'].keys(): + # only ports must be chosen + len(out['ACL_TABLE'][k]) == 1 + out = cmdpb.configWithKeys(portBreakOutConfigDbJson, \ + ["Ethernet10","Ethernet11"]) + assert "INTERFACE" in out.keys() + for k in out['ACL_TABLE'].keys(): + # only ports must be chosen + len(out['ACL_TABLE'][k]) == 1 + return + + def test_break_out(self): + # prepare default config + self.writeJson(portBreakOutConfigDbJson, \ + config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE) + # prepare config dj json to start with + iConfig = dict(configDbJson) + #Ethernet8: start from 4x25G-->2x50G with -f -l + self.dpb_port8_4x25G_2x50G_f_l(iConfig) + #Ethernet8: move from 2x50G-->1x100G without force, list deps + self.dpb_port8_2x50G_1x100G(iConfig) + # Ethernet8: move from 2x50G-->1x100G with force, where deps exists + self.dpb_port8_2x50G_1x100G_f(iConfig) + # Ethernet8: move from 1x100G-->4x25G without force, no deps + self.dpb_port8_1x100G_4x25G(iConfig) + # Ethernet8: move from 4x25G-->1x100G with force, no deps + self.dpb_port8_4x25G_1x100G_f(iConfig) + # Ethernet8: move from 1x100G-->1x50G(2)+2x25G(2) with -f -l, + self.dpb_port8_1x100G_1x50G_2x25G_f_l(iConfig) + # Ethernet4: breakout from 4x25G to 2x50G with -f -l + self.dpb_port4_4x25G_2x50G_f_l(iConfig) + return + + def tearDown(self): + try: + os.remove(config_mgmt.CONFIG_DB_JSON_FILE) + os.remove(config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE) + except Exception as e: + pass + return + + ########### HELPER FUNCS ##################################### + def writeJson(self, d, file): + with open(file, 'w') as f: + dump(d, f, indent=4) + return + # config_mgmt.ConfigMgmtDPB class instance with mocked functions. Not using + # pytest fixture, because it is used in non test funcs. + def config_mgmt_dpb(self, iConfig): + # create object + self.writeJson(iConfig, config_mgmt.CONFIG_DB_JSON_FILE) + cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) + # mock funcs + cmdpb.writeConfigDB = MagicMock(return_value=True) + cmdpb._verifyAsicDB = MagicMock(return_value=True) + return cmdpb + + # Generate port to deleted, added and lanes and speed setting based on + # current and new mode. + def generate_args(self, portIdx, laneIdx, curMode, newMode): + # default params + pre = "Ethernet" + laneMap = {"4x25G": [1,1,1,1], "2x50G": [2,2], "1x100G":[4], \ + "1x50G(2)+2x25G(2)":[2,1,1], "2x25G(2)+1x50G(2)":[1,1,2]} + laneSpeed = 25000 + # generate dPorts + l = list(laneMap[curMode]); l.insert(0, 0); id = portIdx; dPorts = list() + for i in l[:-1]: + id = id + i + portName = portName = "{}{}".format(pre, id) + dPorts.append(portName) + # generate aPorts + l = list(laneMap[newMode]); l.insert(0, 0); id = portIdx; aPorts = list() + for i in l[:-1]: + id = id + i + portName = portName = "{}{}".format(pre, id) + aPorts.append(portName) + # generate pJson + l = laneMap[newMode]; pJson = {"PORT": {}}; li = laneIdx; pi = 0 + for i in l: + speed = laneSpeed*i + lanes = [str(li+j) for j in range(i)]; lanes = ','.join(lanes) + pJson['PORT'][aPorts[pi]] = {"speed": str(speed), "lanes": str(lanes)} + li = li+i; pi = pi + 1 + return dPorts, aPorts ,pJson + + # update the config to emulate continous breakingout a single port + def updateConfig(self, conf, uconf): + try: + for it in uconf.keys(): + # if conf has the key + if conf.get(it): + # if marked for deletion + if uconf[it] == None: + del conf[it] + else: + if isinstance(conf[it], list) and isinstance(uconf[it], list): + conf[it] = list(uconf[it]) + elif isinstance(conf[it], dict) and isinstance(uconf[it], dict): + self.updateConfig(conf[it], uconf[it]) + else: + conf[it] = uconf[it] + del uconf[it] + # update new keys in conf + conf.update(uconf) + except Exception as e: + print("update Config failed") + print(e) + raise e + return + + # Usual result check in many test is: Make sure dConfig and aConfig is + # pushed in order to configDb + def checkResult(self, cmdpb, dConfig, aConfig): + calls = [call(dConfig), call(aConfig)] + assert cmdpb.writeConfigDB.call_count == 2 + cmdpb.writeConfigDB.assert_has_calls(calls, any_order=False) + return + + # After breakout, update the config to emulate continous breakingout a + # single port + def postUpdateConfig(self, iConfig, dConfig, aConfig): + # update the iConfig with change + self.updateConfig(iConfig, dConfig) + self.updateConfig(iConfig, aConfig) + return + + def dpb_port8_1x100G_1x50G_2x25G_f_l(self, iConfig): + cmdpb = self.config_mgmt_dpb(iConfig) + # create ARGS + dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='1x100G', newMode='1x50G(2)+2x25G(2)') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + force=True, loadDefConfig=True) + # Expected Result dConfig and aConfig is pushed in order + dConfig = {u'PORT': {u'Ethernet8': None}} + aConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': {u'ports': ['Ethernet0', \ + 'Ethernet4', 'Ethernet8', 'Ethernet10']}, u'NO-NSW-PACL-TEST': {u'ports': \ + ['Ethernet11']}}, u'INTERFACE': {u'Ethernet11|2a04:1111:40:a709::1/126': \ + {u'scope': u'global', u'family': u'IPv6'}, u'Ethernet11': {}}, \ + u'VLAN_MEMBER': {u'Vlan100|Ethernet8': {u'tagging_mode': u'untagged'}, \ + u'Vlan100|Ethernet11': {u'tagging_mode': u'untagged'}}, u'PORT': \ + {'Ethernet8': {'speed': '50000', 'lanes': '73,74'}, 'Ethernet10': \ + {'speed': '25000', 'lanes': '75'}, 'Ethernet11': {'speed': '25000', 'lanes': '76'}}} + self.checkResult(cmdpb, dConfig, aConfig) + self.postUpdateConfig(iConfig, dConfig, aConfig) + return + + def dpb_port8_4x25G_1x100G_f(self, iConfig): + cmdpb = self.config_mgmt_dpb(iConfig) + # create ARGS + dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='4x25G', newMode='1x100G') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + force=False, loadDefConfig=False) + # Expected Result dConfig and aConfig is pushed in order + dConfig = {u'PORT': {u'Ethernet8': None, u'Ethernet9': None, \ + u'Ethernet10': None, u'Ethernet11': None}} + aConfig = pJson + self.checkResult(cmdpb, dConfig, aConfig) + self.postUpdateConfig(iConfig, dConfig, aConfig) + return + + def dpb_port8_1x100G_4x25G(self, iConfig): + cmdpb = self.config_mgmt_dpb(iConfig) + dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='1x100G', newMode='4x25G') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + force=False, loadDefConfig=False) + # Expected Result dConfig and aConfig is pushed in order + dConfig = {u'PORT': {u'Ethernet8': None}} + aConfig = pJson + self.checkResult(cmdpb, dConfig, aConfig) + self.postUpdateConfig(iConfig, dConfig, aConfig) + return + + def dpb_port8_2x50G_1x100G_f(self, iConfig): + cmdpb = self.config_mgmt_dpb(iConfig) + # create ARGS + dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='2x50G', newMode='1x100G') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + force=True, loadDefConfig=False) + # Expected Result dConfig and aConfig is pushed in order + dConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': {u'ports': ['Ethernet0', 'Ethernet4']}}, \ + u'VLAN_MEMBER': {u'Vlan100|Ethernet8': None}, u'PORT': {u'Ethernet8': None, \ + u'Ethernet10': None}} + aConfig = pJson + self.checkResult(cmdpb, dConfig, aConfig) + self.postUpdateConfig(iConfig, dConfig, aConfig) + + def dpb_port8_2x50G_1x100G(self, iConfig): + cmdpb = self.config_mgmt_dpb(iConfig) + # create ARGS + dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='2x50G', newMode='1x100G') + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + force=False, loadDefConfig=False) + # Expected Result + assert ret == False and len(deps) == 3 + assert cmdpb.writeConfigDB.call_count == 0 + return + + def dpb_port8_4x25G_2x50G_f_l(self, iConfig): + cmdpb = self.config_mgmt_dpb(iConfig) + # create ARGS + dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + curMode='4x25G', newMode='2x50G') + cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + force=True, loadDefConfig=True) + # Expected Result dConfig and aConfig is pushed in order + dConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': \ + {u'ports': ['Ethernet0', 'Ethernet4']}, u'NO-NSW-PACL-TEST': {u'ports': None}}, \ + u'INTERFACE': None, u'VLAN_MEMBER': {u'Vlan100|Ethernet8': None, \ + u'Vlan100|Ethernet11': None}, u'PORT': {u'Ethernet8': None, \ + u'Ethernet9': None, u'Ethernet10': None, u'Ethernet11': None}} + aConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': \ + {u'ports': ['Ethernet0', 'Ethernet4', 'Ethernet8', 'Ethernet10']}}, \ + u'VLAN_MEMBER': {u'Vlan100|Ethernet8': {u'tagging_mode': u'untagged'}}, \ + u'PORT': {'Ethernet8': {'speed': '50000', 'lanes': '73,74'}, \ + 'Ethernet10': {'speed': '50000', 'lanes': '75,76'}}} + assert cmdpb.writeConfigDB.call_count == 2 + self.checkResult(cmdpb, dConfig, aConfig) + self.postUpdateConfig(iConfig, dConfig, aConfig) + return + + def dpb_port4_4x25G_2x50G_f_l(self, iConfig): + cmdpb = self.config_mgmt_dpb(iConfig) + # create ARGS + dPorts, aPorts, pJson = self.generate_args(portIdx=4, laneIdx=69, \ + curMode='4x25G', newMode='2x50G') + cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + force=True, loadDefConfig=True) + # Expected Result dConfig and aConfig is pushed in order + dConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': \ + {u'ports': ['Ethernet0', 'Ethernet8', 'Ethernet10']}}, u'PORT': \ + {u'Ethernet4': None, u'Ethernet5': None, u'Ethernet6': None, \ + u'Ethernet7': None}} + aConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': \ + {u'ports': ['Ethernet0', 'Ethernet8', 'Ethernet10', 'Ethernet4']}}, \ + u'PORT': {'Ethernet4': {'speed': '50000', 'lanes': '69,70'}, \ + 'Ethernet6': {'speed': '50000', 'lanes': '71,72'}}} + self.checkResult(cmdpb, dConfig, aConfig) + self.postUpdateConfig(iConfig, dConfig, aConfig) + return + +###########GLOBAL Configs##################################### +configDbJson = { + "ACL_TABLE": { + "NO-NSW-PACL-TEST": { + "policy_desc": "NO-NSW-PACL-TEST", + "type": "L3", + "stage": "INGRESS", + "ports": [ + "Ethernet9", + "Ethernet11", + ] + }, + "NO-NSW-PACL-V4": { + "policy_desc": "NO-NSW-PACL-V4", + "type": "L3", + "stage": "INGRESS", + "ports": [ + "Ethernet0", + "Ethernet4", + "Ethernet8", + "Ethernet10" + ] + } + }, + "VLAN": { + "Vlan100": { + "admin_status": "up", + "description": "server_vlan", + "dhcp_servers": [ + "10.186.72.116" + ] + }, + }, + "VLAN_MEMBER": { + "Vlan100|Ethernet0": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet2": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet8": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet11": { + "tagging_mode": "untagged" + }, + }, + "INTERFACE": { + "Ethernet10": {}, + "Ethernet10|2a04:0000:40:a709::1/126": { + "scope": "global", + "family": "IPv6" + } + }, + "PORT": { + "Ethernet0": { + "alias": "Eth1/1", + "lanes": "65", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet1": { + "alias": "Eth1/2", + "lanes": "66", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet2": { + "alias": "Eth1/3", + "lanes": "67", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet3": { + "alias": "Eth1/4", + "lanes": "68", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet4": { + "alias": "Eth2/1", + "lanes": "69", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet5": { + "alias": "Eth2/2", + "lanes": "70", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet6": { + "alias": "Eth2/3", + "lanes": "71", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet7": { + "alias": "Eth2/4", + "lanes": "72", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet8": { + "alias": "Eth3/1", + "lanes": "73", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet9": { + "alias": "Eth3/2", + "lanes": "74", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet10": { + "alias": "Eth3/3", + "lanes": "75", + "description": "", + "speed": "25000", + "admin_status": "up" + }, + "Ethernet11": { + "alias": "Eth3/4", + "lanes": "76", + "description": "", + "speed": "25000", + "admin_status": "up" + } + } +} + +portBreakOutConfigDbJson = { + "ACL_TABLE": { + "NO-NSW-PACL-TEST": { + "ports": [ + "Ethernet9", + "Ethernet11", + ] + }, + "NO-NSW-PACL-V4": { + "policy_desc": "NO-NSW-PACL-V4", + "ports": [ + "Ethernet0", + "Ethernet4", + "Ethernet8", + "Ethernet10" + ] + } + }, + "VLAN": { + "Vlan100": { + "admin_status": "up", + "description": "server_vlan", + "dhcp_servers": [ + "10.186.72.116" + ] + } + }, + "VLAN_MEMBER": { + "Vlan100|Ethernet8": { + "tagging_mode": "untagged" + }, + "Vlan100|Ethernet11": { + "tagging_mode": "untagged" + } + }, + "INTERFACE": { + "Ethernet11": {}, + "Ethernet11|2a04:1111:40:a709::1/126": { + "scope": "global", + "family": "IPv6" + } + } +} From 1c8e032ad5a3a092ca9f097c965e35588234f1c6 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Mon, 22 Jun 2020 23:33:57 -0700 Subject: [PATCH 06/15] [config/config_mgmt.py]: Python DocString Comments. Changes: -- Python DocString Comments. -- Variable name changes. -- Exception while handling keys in diff. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 431 +++++++++++++++------ sonic-utilities-tests/config_mgmt_test.py | 449 +++++++++++++++++----- 2 files changed, 656 insertions(+), 224 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index aafbaed536..487b489eec 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -1,6 +1,7 @@ -# config_mgmt.py -# Provides a class for configuration validation and for Dynamic Port Breakout. - +''' +config_mgmt.py provides classes for configuration validation and for Dynamic +Port Breakout. +''' try: import re import syslog @@ -23,19 +24,30 @@ raise ImportError("%s - required module not found" % str(e)) # Globals -# This class may not need to know about YANG_DIR ?, sonic_yang shd use -# default dir. YANG_DIR = "/usr/local/yang-models" CONFIG_DB_JSON_FILE = '/etc/sonic/confib_db.json' # TODO: Find a place for it on sonic switch. DEFAULT_CONFIG_DB_JSON_FILE = '/etc/sonic/default_config_db.json' -# Class to handle config managment for SONIC, this class will use PLY to verify -# config for the commands which are capable of change in config DB. - class ConfigMgmt(): + ''' + Class to handle config managment for SONIC, this class will use sonic_yang + to verify config for the commands which are capable of change in config DB. + ''' def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): + ''' + Initialise the class, --read the config, --load in data tree. + + Parameters: + source (str): source for input config, default configDb else file. + debug (bool): verbose mode. + allowTablesWithoutYang (bool): allow tables without yang model in + config or not. + + Returns: + void + ''' try: self.configdbJsonIn = None self.configdbJsonOut = None @@ -70,16 +82,28 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): def __del__(self): pass - """ - Return tables loaded in config for which YANG model does not exist. - """ def tablesWithoutYang(self): + ''' + Return tables loaded in config for which YANG model does not exist. + + Parameters: + void + + Returns: + tablesWithoutYang (list): list of tables. + ''' return self.sy.tablesWithOutYang - """ - Explicit function to load config data in Yang Data Tree - """ def loadData(self, configdbJson): + ''' + Explicit function to load config data in Yang Data Tree. + + Parameters: + configdbJson (dict): dict similar to configDb. + + Returns: + void + ''' self.sy.loadData(configdbJson) # Raise if tables without YANG models are not allowed but exist. if not self.allowTablesWithoutYang and len(self.sy.tablesWithOutYang): @@ -87,10 +111,16 @@ def loadData(self, configdbJson): return - """ - Validate current Data Tree - """ def validateConfigData(self): + ''' + Validate current config data Tree. + + Parameters: + void + + Returns: + bool + ''' try: self.sy.validate_data_tree() except Exception as e: @@ -101,10 +131,17 @@ def validateConfigData(self): self.sysLog(msg='Data Validation successful') return True - """ - syslog Support - """ def sysLog(self, debug=syslog.LOG_INFO, msg=None): + ''' + Log the msg in syslog file. + + Parameters: + debug : syslog level + msg (str): msg to be logged. + + Returns: + void + ''' # log debug only if enabled if self.DEBUG == False and debug == syslog.LOG_DEBUG: return @@ -115,6 +152,15 @@ def sysLog(self, debug=syslog.LOG_INFO, msg=None): return def readConfigDBJson(self, source=CONFIG_DB_JSON_FILE): + ''' + Read the config from a Config File. + + Parameters: + source(str): config file name. + + Returns: + (void) + ''' print('Reading data from {}'.format(source)) self.configdbJsonIn = readJsonFile(source) #print(type(self.configdbJsonIn)) @@ -128,6 +174,15 @@ def readConfigDBJson(self, source=CONFIG_DB_JSON_FILE): Get config from redis config DB """ def readConfigDB(self): + ''' + Read the config in Config DB. Assign it in self.configdbJsonIn. + + Parameters: + (void) + + Returns: + (void) + ''' print('Reading data from Redis configDb') # Read from config DB on sonic switch db_kwargs = dict(); data = dict() @@ -141,10 +196,16 @@ def readConfigDB(self): return def writeConfigDB(self, jDiff): + ''' + Write the diff in Config DB. + + Parameters: + jDiff (dict): config to push in config DB. + + Returns: + void + ''' print('Writing in Config DB') - """ - On Sonic Switch - """ db_kwargs = dict(); data = dict() configdb = ConfigDBConnector(**db_kwargs) configdb.connect(False) @@ -157,13 +218,25 @@ def writeConfigDB(self, jDiff): # End of Class ConfigMgmt -""" - Config MGMT class for Dynamic Port Breakout(DPB). - This is derived from ConfigMgmt. -""" class ConfigMgmtDPB(ConfigMgmt): + ''' + Config MGMT class for Dynamic Port Breakout(DPB). This is derived from + ConfigMgmt. + ''' def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): + ''' + Initialise the class + + Parameters: + source (str): source for input config, default configDb else file. + debug (bool): verbose mode. + allowTablesWithoutYang (bool): allow tables without yang model in + config or not. + + Returns: + void + ''' try: ConfigMgmt.__init__(self, source=source, debug=debug, \ allowTablesWithoutYang=allowTablesWithoutYang) @@ -178,10 +251,17 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): def __del__(self): pass - """ - Check if a key exists in ASIC DB or not. - """ def _checkKeyinAsicDB(self, key, db): + ''' + Check if a key exists in ASIC DB or not. + + Parameters: + db (SonicV2Connector): database. + key (str): key in config DB, with table Seperator if applicable. + + Returns: + (bool): True, if given key is present. + ''' self.sysLog(msg='Check Key in Asic DB: {}'.format(key)) try: # chk key in ASIC DB @@ -194,6 +274,17 @@ def _checkKeyinAsicDB(self, key, db): return False def _testRedisCli(self, key): + ''' + Additional test funtion for _verifyAsicDB(). This funtions will fetch + ports information from Asic DB, using redis-cli. This is done only when + debugging is on. + + Parameters: + key (str): key in config DB, with table Seperator if applicable. + + Returns: + void + ''' # To Debug if self.DEBUG: cmd = 'sudo redis-cli -n 1 hgetall "{}"'.format(key) @@ -202,13 +293,18 @@ def _testRedisCli(self, key): system(cmd) return - """ - Check ASIC DB for PORTs in port List - ports: List of ports - portMap: port to OID map. - Return: True, if all ports are not present. - """ def _checkNoPortsInAsicDb(self, db, ports, portMap): + ''' + Check ASIC DB for PORTs in port List + + Parameters: + db (SonicV2Connector): database. + ports (list): List of ports + portMap (dict): port to OID map. + + Returns: + (bool): True, if all ports are not present. + ''' try: # connect to ASIC DB, db.connect(db.ASIC_DB) @@ -226,12 +322,20 @@ def _checkNoPortsInAsicDb(self, db, ports, portMap): return True - """ - Verify in the Asic DB that port are deleted, - Keep on trying till timeout period. - db = database, ports, portMap, timeout - """ def _verifyAsicDB(self, db, ports, portMap, timeout): + ''' + Verify in the Asic DB that port are deleted, Keep on trying till timeout + period. + + Parameters: + db (SonicV2Connector): database. + ports (list): port list to check in ASIC DB. + portMap (dict): oid<->port map. + timeout (int): timeout period + + Returns: + (bool) + ''' print("Verify Port Deletion from Asic DB, Wait...") self.sysLog(msg="Verify Port Deletion from Asic DB, Wait...") try: @@ -259,6 +363,20 @@ def _verifyAsicDB(self, db, ports, portMap, timeout): def breakOutPort(self, delPorts=list(), addPorts= list(), portJson=dict(), \ force=False, loadDefConfig=True): + ''' + This is the main function for port breakout. Exposed to caller. + + Parameters: + delPorts (list): ports to be deleted. + addPorts (list): ports to be added. + portJson (dict): Config DB json Part of all Ports, generated from + platform.json. + force (bool): if false return dependecies, else delete dependencies. + loadDefConfig: If loadDefConfig, add default config for ports as well. + + Returns: + (deps, ret) (tuple)[list, bool]: dependecies and success/failure. + ''' MAX_WAIT = 60 try: # delete Port and get the Config diff, deps and True/False @@ -295,16 +413,19 @@ def breakOutPort(self, delPorts=list(), addPorts= list(), portJson=dict(), \ return None, True - """ - Delete all ports. - delPorts: list of port names. - force: if false return dependecies, else delete dependencies. - - Return: - WithOut Force: (xpath of dependecies, False) or (None, True) - With Force: (xpath of dependecies, False) or (None, True) - """ def _deletePorts(self, ports=list(), force=False): + ''' + Delete ports and dependecies from data tree, validate and return resultant + config. + + Parameters: + ports (list): list of ports + force (bool): if false return dependecies, else delete dependencies. + + Returns: + (configToLoad, deps, ret) (tuple)[dict, list, bool]: config, dependecies + and success/fail. + ''' configToLoad = None; deps = None try: self.sysLog(msg="delPorts ports:{} force:{}".format(ports, force)) @@ -358,17 +479,20 @@ def _deletePorts(self, ports=list(), force=False): return configToLoad, deps, True - """ - Add Ports and default config for ports to config DB, after validation of - data tree - - PortJson: Config DB Json Part of all Ports same as PORT Table of Config DB. - ports = list of ports - loadDefConfig: If loadDefConfig add default config as well. - - return: Sucess: True or Failure: False - """ def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): + ''' + Add ports and default confug in data tree, validate and return resultant + config. + + Parameters: + ports (list): list of ports + portJson (dict): Config DB json Part of all Ports, generated from + platform.json. + loadDefConfig: If loadDefConfig, add default config for ports as well. + + Returns: + (configToLoad, ret) (tuple)[dict, bool] + ''' configToLoad = None try: self.sysLog(msg='Start Port Addition') @@ -415,13 +539,23 @@ def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): return configToLoad, True - """ - Merge second dict in first, Note both first and second dict will be changed - First Dict will have merged part D1 + D2 - Second dict will have D2 - D1 [unique keys in D2] - Unique keys in D2 will be merged in D1 only if uniqueKeys=True - """ def _mergeConfigs(self, D1, D2, uniqueKeys=True): + ''' + Merge D2 dict in D1 dict, Note both first and second dict will change. + First Dict will have merged part D1 + D2. Second dict will have D2 - D1 + i.e [unique keys in D2]. Unique keys in D2 will be merged in D1 only + if uniqueKeys=True. + Usage: This function can be used with 'config load' command to merge + new config with old. + + Parameters: + D1 (dict): Partial Config 1. + D2 (dict): Partial Config 2. + uniqueKeys (bool) + + Returns: + bool + ''' try: def _mergeItems(it1, it2): if isinstance(it1, list) and isinstance(it2, list): @@ -429,24 +563,19 @@ def _mergeItems(it1, it2): elif isinstance(it1, dict) and isinstance(it2, dict): self._mergeConfigs(it1, it2) elif isinstance(it1, list) or isinstance(it2, list): - raise ("Can not merge Configs, List problem") + raise Exception("Can not merge Configs, List problem") elif isinstance(it1, dict) or isinstance(it2, dict): - raise ("Can not merge Configs, Dict problem") + raise Exception("Can not merge Configs, Dict problem") else: - #print("Do nothing") # First Dict takes priority pass return for it in D1.keys(): - #print(it) # D2 has the key if D2.get(it): _mergeItems(D1[it], D2[it]) del D2[it] - # D2 does not have the keys - else: - pass # if uniqueKeys are needed, merge rest of the keys of D2 in D1 if uniqueKeys: @@ -458,29 +587,30 @@ def _mergeItems(it1, it2): return D1 - """ - search Relevant Keys in Config using DFS, This function is mainly - used to search port related config in Default ConfigDbJson file. - In: Config to be searched - skeys: Keys to be searched in In Config i.e. search Keys. - Out: Contains the search result - """ def _searchKeysInConfig(self, In, Out, skeys): + ''' + Search Relevant Keys in Input Config using DFS, This function is mainly + used to search ports related config in Default ConfigDbJson file. + + Parameters: + In (dict): Input Config to be searched + skeys (list): Keys to be searched in Input Config i.e. search Keys. + Out (dict): Contains the search result, i.e. Output Config with skeys. + + Returns: + found (bool): True if any of skeys is found else False. + ''' found = False if isinstance(In, dict): for key in In.keys(): - #print("key:" + key) for skey in skeys: # pattern is very specific to current primary keys in # config DB, may need to be updated later. pattern = '^' + skey + '\|' + '|' + skey + '$' + \ '|' + '^' + skey + '$' - #print(pattern) reg = re.compile(pattern) - #print(reg) if reg.search(key): # In primary key, only 1 match can be found, so return - # print("Added key:" + key) Out[key] = In[key] found = True break @@ -506,11 +636,18 @@ def _searchKeysInConfig(self, In, Out, skeys): return found - """ - This function returns the relavant keys in Input Config. - For Example: All Ports related Config in Config DB. - """ def configWithKeys(self, configIn=dict(), keys=list()): + ''' + This function returns the config with relavant keys in Input Config. + It calls _searchKeysInConfig. + + Parameters: + configIn (dict): Input Config + keys (list): Key list. + + Returns: + configOut (dict): Output Config containing only key related config. + ''' configOut = dict() try: if len(configIn) and len(keys): @@ -521,10 +658,17 @@ def configWithKeys(self, configIn=dict(), keys=list()): return configOut - """ - Create a defConfig for given Ports from Default Config File. - """ def _getDefaultConfig(self, ports=list()): + ''' + Create a default Config for given Port list from Default Config File. + It calls _searchKeysInConfig. + + Parameters: + ports (list): list of ports, for which default config must be fetched. + + Returns: + defConfigOut (dict): default Config for given Ports. + ''' # function code try: print("Generating default config for {}".format(ports)) @@ -539,6 +683,15 @@ def _getDefaultConfig(self, ports=list()): return defConfigOut def _updateDiffConfigDB(self): + ''' + Return ConfigDb format Diff b/w self.configdbJsonIn, self.configdbJsonOut + + Parameters: + void + + Returns: + configToLoad (dict): ConfigDb format Diff + ''' try: # Get the Diff print('Generate Final Config to write in DB') @@ -554,66 +707,76 @@ def _updateDiffConfigDB(self): return configToLoad - """ - Create the config to write in Config DB from json diff - diff: diff in input config and output config. - inp: input config before delete/add ports. - outp: output config after delete/add ports. - """ def _createConfigToLoad(self, diff, inp, outp): + ''' + Create the config to write in Config DB, i.e. compitible with mod_config() + This functions has 3 inner functions: + -- _deleteHandler: to handle delete in diff. See example below. + -- _insertHandler: to handle insert in diff. See example below. + -- _recurCreateConfig: recursively create this config. + + Parameters: + diff: jsondiff b/w 2 configs. + Example: + {u'VLAN': {u'Vlan100': {'members': {delete: [(95, 'Ethernet1')]}}, + u'Vlan777': {u'members': {insert: [(92, 'Ethernet2')]}}}, + 'PORT': {delete: {u'Ethernet1': {...}}}} + + inp: input config before delete/add ports, i.e. current config Db. + outp: output config after delete/add ports. i.e. config DB once diff + is applied. + + Returns: + configToLoad (dict): config in a format compitible with mod_Config(). + ''' + ### Internal Functions ### - """ - Handle deletes in diff dict - """ def _deleteHandler(diff, inp, outp, config): - # if output is dict, delete keys from config + ''' + Handle deletions in diff dict + ''' if isinstance(inp, dict): + # Example Case: diff = PORT': {delete: {u'Ethernet1': {...}}}} for key in diff: - #print(key) # make sure keys from diff are present in inp but not in outp - # then delete it. if key in inp and key not in outp: # assign key to None(null), redis will delete entire key config[key] = None else: - # log such keys - print("Diff: Probably wrong key: {}".format(key)) + # should not happen + raise Exception('Invalid deletion of {} in diff'.format(key)) elif isinstance(inp, list): - # just take list from output - # print("Delete from List: {} {} {}".format(inp, outp, list)) - #print(type(config)) + # Example case: {u'VLAN': {u'Vlan100': {'members': {delete: [(95, 'Ethernet1')]}} + # just take list from outputs config.extend(outp) - return - """ - Handle inserts in diff dict - """ def _insertHandler(diff, inp, outp, config): - # if outp is a dict + ''' + Handle inserts in diff dict + ''' if isinstance(outp, dict): + # Example Case: diff = PORT': {insert: {u'Ethernet1': {...}}}} for key in diff: - #print(key) # make sure keys are only in outp if key not in inp and key in outp: # assign key in config same as outp config[key] = outp[key] else: - # log such keys - print("Diff: Probably wrong key: {}".format(key)) + # should not happen + raise Exception('Invalid insertion of {} in diff'.format(key)) elif isinstance(outp, list): # just take list from output - # print("Delete from List: {} {} {}".format(inp, outp, list)) + # Example case: {u'VLAN': {u'Vlan100': {'members': {insert: [(95, 'Ethernet1')]}} config.extend(outp) - return - """ - Recursively iterate diff to generate config to write in configDB - """ def _recurCreateConfig(diff, inp, outp, config): + ''' + Recursively iterate diff to generate config to write in configDB + ''' changed = False # updates are represented by list in diff and as dict in outp\inp # we do not allow updates right now @@ -622,7 +785,6 @@ def _recurCreateConfig(diff, inp, outp, config): idx = -1 for key in diff: - #print(key) idx = idx + 1 if str(key) == '$delete': _deleteHandler(diff[key], inp, outp, config) @@ -633,7 +795,7 @@ def _recurCreateConfig(diff, inp, outp, config): else: # insert in config by default, remove later if not needed if isinstance(diff, dict): - # config should match with outp + # config should match type of outp config[key] = type(outp[key])() if _recurCreateConfig(diff[key], inp[key], outp[key], \ config[key]) == False: @@ -664,6 +826,20 @@ def _recurCreateConfig(diff, inp, outp, config): return configToLoad def _diffJson(self): + ''' + Return json diff between self.configdbJsonIn, self.configdbJsonOut dicts. + + Parameters: + void + + Returns: + (dict): json diff between self.configdbJsonIn, self.configdbJsonOut + dicts. + Example: + {u'VLAN': {u'Vlan100': {'members': {delete: [(95, 'Ethernet1')]}}, + u'Vlan777': {u'members': {insert: [(92, 'Ethernet2')]}}}, + 'PORT': {delete: {u'Ethernet1': {...}}}} + ''' from jsondiff import diff return diff(self.configdbJsonIn, self.configdbJsonOut, syntax='symmetric') @@ -671,6 +847,15 @@ def _diffJson(self): # Helper Functions def readJsonFile(fileName): + ''' + Read Json file. + + Parameters: + fileName (str): file + + Returns: + result (dict): json --> dict + ''' try: with open(fileName) as f: result = load(f) diff --git a/sonic-utilities-tests/config_mgmt_test.py b/sonic-utilities-tests/config_mgmt_test.py index 8a370fa6a5..ccd9c515ff 100644 --- a/sonic-utilities-tests/config_mgmt_test.py +++ b/sonic-utilities-tests/config_mgmt_test.py @@ -11,6 +11,9 @@ from json import dump class TestConfigMgmt(TestCase): + ''' + Test Class for config_mgmt.py + ''' def setUp(self): config_mgmt.YANG_DIR = "../../sonic-yang-models/yang-models/" @@ -19,24 +22,24 @@ def setUp(self): return def test_config_validation(self): - iConfig = dict(configDbJson) - self.writeJson(iConfig, config_mgmt.CONFIG_DB_JSON_FILE) + curConfig = dict(configDbJson) + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) cm = config_mgmt.ConfigMgmt(source=config_mgmt.CONFIG_DB_JSON_FILE) assert cm.validateConfigData() == True return def test_table_without_yang(self): - iConfig = dict(configDbJson) + curConfig = dict(configDbJson) unknown = {"unknown_table": {"ukey": "uvalue"}} - self.updateConfig(iConfig, unknown) - self.writeJson(iConfig, config_mgmt.CONFIG_DB_JSON_FILE) + self.updateConfig(curConfig, unknown) + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) cm = config_mgmt.ConfigMgmt(source=config_mgmt.CONFIG_DB_JSON_FILE) #assert "unknown_table" in cm.tablesWithoutYang() return def test_search_keys(self): - iConfig = dict(configDbJson) - self.writeJson(iConfig, config_mgmt.CONFIG_DB_JSON_FILE) + curConfig = dict(configDbJson) + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) out = cmdpb.configWithKeys(portBreakOutConfigDbJson, \ ["Ethernet8","Ethernet9"]) @@ -58,21 +61,21 @@ def test_break_out(self): self.writeJson(portBreakOutConfigDbJson, \ config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE) # prepare config dj json to start with - iConfig = dict(configDbJson) + curConfig = dict(configDbJson) #Ethernet8: start from 4x25G-->2x50G with -f -l - self.dpb_port8_4x25G_2x50G_f_l(iConfig) + self.dpb_port8_4x25G_2x50G_f_l(curConfig) #Ethernet8: move from 2x50G-->1x100G without force, list deps - self.dpb_port8_2x50G_1x100G(iConfig) + self.dpb_port8_2x50G_1x100G(curConfig) # Ethernet8: move from 2x50G-->1x100G with force, where deps exists - self.dpb_port8_2x50G_1x100G_f(iConfig) + self.dpb_port8_2x50G_1x100G_f(curConfig) # Ethernet8: move from 1x100G-->4x25G without force, no deps - self.dpb_port8_1x100G_4x25G(iConfig) + self.dpb_port8_1x100G_4x25G(curConfig) # Ethernet8: move from 4x25G-->1x100G with force, no deps - self.dpb_port8_4x25G_1x100G_f(iConfig) + self.dpb_port8_4x25G_1x100G_f(curConfig) # Ethernet8: move from 1x100G-->1x50G(2)+2x25G(2) with -f -l, - self.dpb_port8_1x100G_1x50G_2x25G_f_l(iConfig) + self.dpb_port8_1x100G_1x50G_2x25G_f_l(curConfig) # Ethernet4: breakout from 4x25G to 2x50G with -f -l - self.dpb_port4_4x25G_2x50G_f_l(iConfig) + self.dpb_port4_4x25G_2x50G_f_l(curConfig) return def tearDown(self): @@ -88,20 +91,48 @@ def writeJson(self, d, file): with open(file, 'w') as f: dump(d, f, indent=4) return - # config_mgmt.ConfigMgmtDPB class instance with mocked functions. Not using - # pytest fixture, because it is used in non test funcs. - def config_mgmt_dpb(self, iConfig): + + def config_mgmt_dpb(self, curConfig): + ''' + config_mgmt.ConfigMgmtDPB class instance with mocked functions. Not using + pytest fixture, because it is used in non test funcs. + + Parameter: + curConfig (dict): Config to start with. + + Return: + cmdpb (ConfigMgmtDPB): Class instance of ConfigMgmtDPB. + ''' # create object - self.writeJson(iConfig, config_mgmt.CONFIG_DB_JSON_FILE) + self.writeJson(curConfig, config_mgmt.CONFIG_DB_JSON_FILE) cmdpb = config_mgmt.ConfigMgmtDPB(source=config_mgmt.CONFIG_DB_JSON_FILE) # mock funcs cmdpb.writeConfigDB = MagicMock(return_value=True) cmdpb._verifyAsicDB = MagicMock(return_value=True) return cmdpb - # Generate port to deleted, added and lanes and speed setting based on - # current and new mode. def generate_args(self, portIdx, laneIdx, curMode, newMode): + ''' + Generate port to deleted, added and {lanes, speed} setting based on + current and new mode. + Example: + For generate_args(8, 73, '4x25G', '2x50G'): + output: + ( + ['Ethernet8', 'Ethernet9', 'Ethernet10', 'Ethernet11'], + ['Ethernet8', 'Ethernet10'], + {'Ethernet8': {'lanes': '73,74', 'speed': '50000'}, + 'Ethernet10': {'lanes': '75,76', 'speed': '50000'}}) + + Parameters: + portIdx (int): Port Index. + laneIdx (int): Lane Index. + curMode (str): current breakout mode of Port. + newMode (str): new breakout mode of Port. + + Return: + dPorts, aPorts ,pJson (tuple)[list, list, dict] + ''' # default params pre = "Ethernet" laneMap = {"4x25G": [1,1,1,1], "2x50G": [2,2], "1x100G":[4], \ @@ -128,8 +159,18 @@ def generate_args(self, portIdx, laneIdx, curMode, newMode): li = li+i; pi = pi + 1 return dPorts, aPorts ,pJson - # update the config to emulate continous breakingout a single port def updateConfig(self, conf, uconf): + ''' + update the config to emulate continous breakingout a single port. + + Parameters: + conf (dict): current config in config DB. + uconf (dict): config Diff to be pushed in config DB. + + Return: + void + conf will be updated with uconf, i.e. config diff. + ''' try: for it in uconf.keys(): # if conf has the key @@ -153,88 +194,219 @@ def updateConfig(self, conf, uconf): raise e return - # Usual result check in many test is: Make sure dConfig and aConfig is - # pushed in order to configDb - def checkResult(self, cmdpb, dConfig, aConfig): - calls = [call(dConfig), call(aConfig)] + def checkResult(self, cmdpb, delConfig, addConfig): + ''' + Usual result check in many test is: Make sure delConfig and addConfig is + pushed in order to configDb + + Parameters: + cmdpb (ConfigMgmtDPB): Class instance of ConfigMgmtDPB. + delConfig (dict): config Diff to be pushed in config DB while deletion + of ports. + addConfig (dict): config Diff to be pushed in config DB while addition + of ports. + + Return: + void + ''' + calls = [call(delConfig), call(addConfig)] assert cmdpb.writeConfigDB.call_count == 2 cmdpb.writeConfigDB.assert_has_calls(calls, any_order=False) return - # After breakout, update the config to emulate continous breakingout a - # single port - def postUpdateConfig(self, iConfig, dConfig, aConfig): - # update the iConfig with change - self.updateConfig(iConfig, dConfig) - self.updateConfig(iConfig, aConfig) + def postUpdateConfig(self, curConfig, delConfig, addConfig): + ''' + After breakout, update the config to emulate continous breakingout a + single port. + + Parameters: + curConfig (dict): current Config in config DB. + delConfig (dict): config Diff to be pushed in config DB while deletion + of ports. + addConfig (dict): config Diff to be pushed in config DB while addition + of ports. + + Return: + void + curConfig will be updated with delConfig and addConfig. + ''' + # update the curConfig with change + self.updateConfig(curConfig, delConfig) + self.updateConfig(curConfig, addConfig) return - def dpb_port8_1x100G_1x50G_2x25G_f_l(self, iConfig): - cmdpb = self.config_mgmt_dpb(iConfig) + def dpb_port8_1x100G_1x50G_2x25G_f_l(self, curConfig): + ''' + Breakout Port 8 1x100G->1x50G_2x25G with -f -l + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='1x100G', newMode='1x50G(2)+2x25G(2)') deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, force=True, loadDefConfig=True) - # Expected Result dConfig and aConfig is pushed in order - dConfig = {u'PORT': {u'Ethernet8': None}} - aConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': {u'ports': ['Ethernet0', \ - 'Ethernet4', 'Ethernet8', 'Ethernet10']}, u'NO-NSW-PACL-TEST': {u'ports': \ - ['Ethernet11']}}, u'INTERFACE': {u'Ethernet11|2a04:1111:40:a709::1/126': \ - {u'scope': u'global', u'family': u'IPv6'}, u'Ethernet11': {}}, \ - u'VLAN_MEMBER': {u'Vlan100|Ethernet8': {u'tagging_mode': u'untagged'}, \ - u'Vlan100|Ethernet11': {u'tagging_mode': u'untagged'}}, u'PORT': \ - {'Ethernet8': {'speed': '50000', 'lanes': '73,74'}, 'Ethernet10': \ - {'speed': '25000', 'lanes': '75'}, 'Ethernet11': {'speed': '25000', 'lanes': '76'}}} - self.checkResult(cmdpb, dConfig, aConfig) - self.postUpdateConfig(iConfig, dConfig, aConfig) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'PORT': { + u'Ethernet8': None + } + } + addConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet4', 'Ethernet8', 'Ethernet10'] + }, + u'NO-NSW-PACL-TEST': { + u'ports': ['Ethernet11'] + } + }, + u'INTERFACE': { + u'Ethernet11|2a04:1111:40:a709::1/126': { + u'scope': u'global', + u'family': u'IPv6' + }, + u'Ethernet11': {} + }, + u'VLAN_MEMBER': { + u'Vlan100|Ethernet8': { + u'tagging_mode': u'untagged' + }, + u'Vlan100|Ethernet11': { + u'tagging_mode': u'untagged' + } + }, + u'PORT': { + 'Ethernet8': { + 'speed': '50000', + 'lanes': '73,74' + }, + 'Ethernet10': { + 'speed': '25000', + 'lanes': '75' + }, + 'Ethernet11': { + 'speed': '25000', + 'lanes': '76' + } + } + } + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) return - def dpb_port8_4x25G_1x100G_f(self, iConfig): - cmdpb = self.config_mgmt_dpb(iConfig) + def dpb_port8_4x25G_1x100G_f(self, curConfig): + ''' + Breakout Port 8 4x25G->1x100G with -f + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='4x25G', newMode='1x100G') deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, force=False, loadDefConfig=False) - # Expected Result dConfig and aConfig is pushed in order - dConfig = {u'PORT': {u'Ethernet8': None, u'Ethernet9': None, \ - u'Ethernet10': None, u'Ethernet11': None}} - aConfig = pJson - self.checkResult(cmdpb, dConfig, aConfig) - self.postUpdateConfig(iConfig, dConfig, aConfig) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'PORT': { + u'Ethernet8': None, + u'Ethernet9': None, + u'Ethernet10': None, + u'Ethernet11': None + } + } + addConfig = pJson + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) return - def dpb_port8_1x100G_4x25G(self, iConfig): - cmdpb = self.config_mgmt_dpb(iConfig) + def dpb_port8_1x100G_4x25G(self, curConfig): + ''' + Breakout Port 8 1x100G->4x25G + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='1x100G', newMode='4x25G') deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, force=False, loadDefConfig=False) - # Expected Result dConfig and aConfig is pushed in order - dConfig = {u'PORT': {u'Ethernet8': None}} - aConfig = pJson - self.checkResult(cmdpb, dConfig, aConfig) - self.postUpdateConfig(iConfig, dConfig, aConfig) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'PORT': { + u'Ethernet8': None + } + } + addConfig = pJson + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) return - def dpb_port8_2x50G_1x100G_f(self, iConfig): - cmdpb = self.config_mgmt_dpb(iConfig) + def dpb_port8_2x50G_1x100G_f(self, curConfig): + ''' + Breakout Port 8 2x50G->1x100G with -f + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='2x50G', newMode='1x100G') deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, force=True, loadDefConfig=False) - # Expected Result dConfig and aConfig is pushed in order - dConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': {u'ports': ['Ethernet0', 'Ethernet4']}}, \ - u'VLAN_MEMBER': {u'Vlan100|Ethernet8': None}, u'PORT': {u'Ethernet8': None, \ - u'Ethernet10': None}} - aConfig = pJson - self.checkResult(cmdpb, dConfig, aConfig) - self.postUpdateConfig(iConfig, dConfig, aConfig) - - def dpb_port8_2x50G_1x100G(self, iConfig): - cmdpb = self.config_mgmt_dpb(iConfig) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet4'] + } + }, + u'VLAN_MEMBER': { + u'Vlan100|Ethernet8': None + }, + u'PORT': { + u'Ethernet8': None, + u'Ethernet10': None + } + } + addConfig = pJson + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) + + def dpb_port8_2x50G_1x100G(self, curConfig): + ''' + Breakout Port 8 2x50G->1x100G + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='2x50G', newMode='1x100G') @@ -245,47 +417,122 @@ def dpb_port8_2x50G_1x100G(self, iConfig): assert cmdpb.writeConfigDB.call_count == 0 return - def dpb_port8_4x25G_2x50G_f_l(self, iConfig): - cmdpb = self.config_mgmt_dpb(iConfig) + def dpb_port8_4x25G_2x50G_f_l(self, curConfig): + ''' + Breakout Port 8 4x25G->2x50G with -f -l + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='4x25G', newMode='2x50G') cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, force=True, loadDefConfig=True) - # Expected Result dConfig and aConfig is pushed in order - dConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': \ - {u'ports': ['Ethernet0', 'Ethernet4']}, u'NO-NSW-PACL-TEST': {u'ports': None}}, \ - u'INTERFACE': None, u'VLAN_MEMBER': {u'Vlan100|Ethernet8': None, \ - u'Vlan100|Ethernet11': None}, u'PORT': {u'Ethernet8': None, \ - u'Ethernet9': None, u'Ethernet10': None, u'Ethernet11': None}} - aConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': \ - {u'ports': ['Ethernet0', 'Ethernet4', 'Ethernet8', 'Ethernet10']}}, \ - u'VLAN_MEMBER': {u'Vlan100|Ethernet8': {u'tagging_mode': u'untagged'}}, \ - u'PORT': {'Ethernet8': {'speed': '50000', 'lanes': '73,74'}, \ - 'Ethernet10': {'speed': '50000', 'lanes': '75,76'}}} + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet4'] + }, + u'NO-NSW-PACL-TEST': { + u'ports': None + } + }, + u'INTERFACE': None, + u'VLAN_MEMBER': { + u'Vlan100|Ethernet8': None, + u'Vlan100|Ethernet11': None + }, + u'PORT': { + u'Ethernet8': None, + u'Ethernet9': None, + u'Ethernet10': None, + u'Ethernet11': None + } + } + addConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet4', 'Ethernet8', 'Ethernet10'] + } + }, + u'VLAN_MEMBER': { + u'Vlan100|Ethernet8': { + u'tagging_mode': u'untagged' + } + }, + u'PORT': { + 'Ethernet8': { + 'speed': '50000', + 'lanes': '73,74' + }, + 'Ethernet10': { + 'speed': '50000', + 'lanes': '75,76' + } + } + } assert cmdpb.writeConfigDB.call_count == 2 - self.checkResult(cmdpb, dConfig, aConfig) - self.postUpdateConfig(iConfig, dConfig, aConfig) + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) return - def dpb_port4_4x25G_2x50G_f_l(self, iConfig): - cmdpb = self.config_mgmt_dpb(iConfig) + def dpb_port4_4x25G_2x50G_f_l(self, curConfig): + ''' + Breakout Port 4 4x25G->2x50G with -f -l + + Parameters: + curConfig (dict): current Config in config DB. + + Return: + void + assert for success and failure. + ''' + cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS dPorts, aPorts, pJson = self.generate_args(portIdx=4, laneIdx=69, \ curMode='4x25G', newMode='2x50G') cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, force=True, loadDefConfig=True) - # Expected Result dConfig and aConfig is pushed in order - dConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': \ - {u'ports': ['Ethernet0', 'Ethernet8', 'Ethernet10']}}, u'PORT': \ - {u'Ethernet4': None, u'Ethernet5': None, u'Ethernet6': None, \ - u'Ethernet7': None}} - aConfig = {u'ACL_TABLE': {u'NO-NSW-PACL-V4': \ - {u'ports': ['Ethernet0', 'Ethernet8', 'Ethernet10', 'Ethernet4']}}, \ - u'PORT': {'Ethernet4': {'speed': '50000', 'lanes': '69,70'}, \ - 'Ethernet6': {'speed': '50000', 'lanes': '71,72'}}} - self.checkResult(cmdpb, dConfig, aConfig) - self.postUpdateConfig(iConfig, dConfig, aConfig) + # Expected Result delConfig and addConfig is pushed in order + delConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet8', 'Ethernet10'] + } + }, + u'PORT': { + u'Ethernet4': None, + u'Ethernet5': None, + u'Ethernet6': None, + u'Ethernet7': None + } + } + addConfig = { + u'ACL_TABLE': { + u'NO-NSW-PACL-V4': { + u'ports': ['Ethernet0', 'Ethernet8', 'Ethernet10', 'Ethernet4'] + } + }, + u'PORT': { + 'Ethernet4': { + 'speed': '50000', + 'lanes': '69,70' + }, + 'Ethernet6': { + 'speed': '50000', + 'lanes': '71,72' + } + } + } + self.checkResult(cmdpb, delConfig, addConfig) + self.postUpdateConfig(curConfig, delConfig, addConfig) return ###########GLOBAL Configs##################################### From 8579a0a773e8f5e3b772811322673889a5a76f67 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Tue, 23 Jun 2020 23:51:18 -0700 Subject: [PATCH 07/15] [sonic-utilities-tests/config_mgmt_test.py]: import mock_tables after creation of class instance. Changes: -- rebasing. -- import mock_tables after creation of class instance. -- add COUNTERS_LAG_NAME_MAP table in counters_db.json Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 1 - sonic-utilities-tests/config_mgmt_test.py | 4 ++-- sonic-utilities-tests/mock_tables/counters_db.json | 6 ++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 487b489eec..e45368d18e 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -815,7 +815,6 @@ def _recurCreateConfig(diff, inp, outp, config): ### Function Code ### try: configToLoad = dict() - #import pdb; pdb.set_trace() _recurCreateConfig(diff, inp, outp, configToLoad) except Exception as e: diff --git a/sonic-utilities-tests/config_mgmt_test.py b/sonic-utilities-tests/config_mgmt_test.py index ccd9c515ff..eabdde5eb5 100644 --- a/sonic-utilities-tests/config_mgmt_test.py +++ b/sonic-utilities-tests/config_mgmt_test.py @@ -1,6 +1,5 @@ import imp import os -import mock_tables.dbconnector # import file under test i.e. config_mgmt.py imp.load_source('config_mgmt', \ os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py')) @@ -16,7 +15,7 @@ class TestConfigMgmt(TestCase): ''' def setUp(self): - config_mgmt.YANG_DIR = "../../sonic-yang-models/yang-models/" + config_mgmt.YANG_DIR = "./../../../sonic-yang-models/yang-models/" config_mgmt.CONFIG_DB_JSON_FILE = "startConfigDb.json" config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE = "portBreakOutConfigDb.json" return @@ -109,6 +108,7 @@ def config_mgmt_dpb(self, curConfig): # mock funcs cmdpb.writeConfigDB = MagicMock(return_value=True) cmdpb._verifyAsicDB = MagicMock(return_value=True) + import mock_tables.dbconnector return cmdpb def generate_args(self, portIdx, laneIdx, curMode, newMode): diff --git a/sonic-utilities-tests/mock_tables/counters_db.json b/sonic-utilities-tests/mock_tables/counters_db.json index 2476837d71..2b2b600280 100644 --- a/sonic-utilities-tests/mock_tables/counters_db.json +++ b/sonic-utilities-tests/mock_tables/counters_db.json @@ -145,6 +145,12 @@ "Ethernet4": "oid:0x1000000000004", "Ethernet8": "oid:0x1000000000006" }, + "COUNTERS_LAG_NAME_MAP": { + "PortChannel0001": "oid:0x60000000005a1", + "PortChannel0002": "oid:0x60000000005a2", + "PortChannel0003": "oid:0x600000000063c", + "PortChannel0004": "oid:0x600000000063d" + }, "COUNTERS_DEBUG_NAME_PORT_STAT_MAP": { "DEBUG_0": "SAI_PORT_STAT_IN_DROP_REASON_RANGE_BASE", "DEBUG_2": "SAI_PORT_STAT_OUT_CONFIGURED_DROP_REASONS_1_DROPPED_PKTS" From 03cf997d78b2409fb38d4aa6ffe3cc4a9ef9a33d Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Wed, 24 Jun 2020 14:38:41 -0700 Subject: [PATCH 08/15] [setup.py]: Adding xmltodict and jsondiff as install_requirments in setup.py. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 2 +- setup.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index e45368d18e..71d1b42ef8 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -10,6 +10,7 @@ from os import system from time import sleep as tsleep from imp import load_source + from jsondiff import diff # SONiC specific imports import sonic_yang @@ -839,7 +840,6 @@ def _diffJson(self): u'Vlan777': {u'members': {insert: [(92, 'Ethernet2')]}}}, 'PORT': {delete: {u'Ethernet1': {...}}}} ''' - from jsondiff import diff return diff(self.configdbJsonIn, self.configdbJsonOut, syntax='symmetric') # end of class ConfigMgmtDPB diff --git a/setup.py b/setup.py index edffa77cd0..31b4099ea8 100644 --- a/setup.py +++ b/setup.py @@ -148,7 +148,9 @@ # - tabulate install_requires=[ 'click', - 'natsort' + 'natsort', + 'xmltodict', + 'jsondiff' ], setup_requires= [ 'pytest-runner' From 971c4185fe5c564ab75a14f6e9b98d91513ea61d Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Wed, 24 Jun 2020 23:47:23 -0700 Subject: [PATCH 09/15] [setup.py]: Specifing fix version of xmltodict and jsondiff. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 31b4099ea8..870fdfec8b 100644 --- a/setup.py +++ b/setup.py @@ -149,8 +149,8 @@ install_requires=[ 'click', 'natsort', - 'xmltodict', - 'jsondiff' + 'xmltodict==0.12.0', + 'jsondiff==1.2.0' ], setup_requires= [ 'pytest-runner' From eb870d5aeb9a21d6921037b9a23330e08cd71c1b Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Thu, 25 Jun 2020 10:37:17 -0700 Subject: [PATCH 10/15] [setup.py]: Remove jsondiff from setup.py since it is not found on allowed servers by Azure. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 870fdfec8b..068f8bfe7f 100644 --- a/setup.py +++ b/setup.py @@ -149,8 +149,7 @@ install_requires=[ 'click', 'natsort', - 'xmltodict==0.12.0', - 'jsondiff==1.2.0' + 'xmltodict==0.12.0' ], setup_requires= [ 'pytest-runner' From 2a8d16cf74cb7e18bac744b786f6dfbf2f74181f Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Thu, 25 Jun 2020 18:08:35 -0700 Subject: [PATCH 11/15] [config/config_mgmt.py]: Address latest review comments. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 142 ++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 82 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index 71d1b42ef8..dd5329ce0f 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -11,6 +11,7 @@ from time import sleep as tsleep from imp import load_source from jsondiff import diff + from sys import flags # SONiC specific imports import sonic_yang @@ -28,7 +29,7 @@ YANG_DIR = "/usr/local/yang-models" CONFIG_DB_JSON_FILE = '/etc/sonic/confib_db.json' # TODO: Find a place for it on sonic switch. -DEFAULT_CONFIG_DB_JSON_FILE = '/etc/sonic/default_config_db.json' +DEFAULT_CONFIG_DB_JSON_FILE = '/etc/sonic/port_breakout_config_db.json' class ConfigMgmt(): ''' @@ -75,7 +76,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): raise Exception('Config has tables without YANG models') except Exception as e: - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) raise(Exception('ConfigMgmt Class creation failed')) return @@ -128,11 +129,10 @@ def validateConfigData(self): self.sysLog(msg='Data Validation Failed') return False - print('Data Validation successful') - self.sysLog(msg='Data Validation successful') + self.sysLog(msg='Data Validation successful', doPrint=True) return True - def sysLog(self, debug=syslog.LOG_INFO, msg=None): + def sysLog(self, logLevel=syslog.LOG_INFO, msg=None, doPrint=False): ''' Log the msg in syslog file. @@ -144,10 +144,12 @@ def sysLog(self, debug=syslog.LOG_INFO, msg=None): void ''' # log debug only if enabled - if self.DEBUG == False and debug == syslog.LOG_DEBUG: + if self.DEBUG == False and logLevel == syslog.LOG_DEBUG: return + if flgas.interactive !=0 and doPrint == True: + print("{}".format(msg)) syslog.openlog(self.SYSLOG_IDENTIFIER) - syslog.syslog(debug, msg) + syslog.syslog(logLevel, msg) syslog.closelog() return @@ -162,9 +164,9 @@ def readConfigDBJson(self, source=CONFIG_DB_JSON_FILE): Returns: (void) ''' - print('Reading data from {}'.format(source)) + self.sysLog(msg='Reading data from {}'.format(source)) self.configdbJsonIn = readJsonFile(source) - #print(type(self.configdbJsonIn)) + #self.sysLog(msg=type(self.configdbJsonIn)) if not self.configdbJsonIn: raise(Exception("Can not load config from config DB json file")) self.sysLog(msg='Reading Input {}'.format(self.configdbJsonIn)) @@ -184,7 +186,7 @@ def readConfigDB(self): Returns: (void) ''' - print('Reading data from Redis configDb') + self.sysLog(doPrint=True, msg='Reading data from Redis configDb') # Read from config DB on sonic switch db_kwargs = dict(); data = dict() configdb = ConfigDBConnector(**db_kwargs) @@ -206,7 +208,7 @@ def writeConfigDB(self, jDiff): Returns: void ''' - print('Writing in Config DB') + self.sysLog(doPrint=True, msg='Writing in Config DB') db_kwargs = dict(); data = dict() configdb = ConfigDBConnector(**db_kwargs) configdb.connect(False) @@ -244,7 +246,7 @@ def __init__(self, source="configDB", debug=False, allowTablesWithoutYang=True): self.oidKey = 'ASIC_STATE:SAI_OBJECT_TYPE_PORT:oid:0x' except Exception as e: - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) raise(Exception('ConfigMgmtDPB Class creation failed')) return @@ -258,7 +260,7 @@ def _checkKeyinAsicDB(self, key, db): Parameters: db (SonicV2Connector): database. - key (str): key in config DB, with table Seperator if applicable. + key (str): key in ASIC DB, with table Seperator if applicable. Returns: (bool): True, if given key is present. @@ -269,31 +271,11 @@ def _checkKeyinAsicDB(self, key, db): if db.exists('ASIC_DB', key): return True except Exception as e: - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) raise(e) return False - def _testRedisCli(self, key): - ''' - Additional test funtion for _verifyAsicDB(). This funtions will fetch - ports information from Asic DB, using redis-cli. This is done only when - debugging is on. - - Parameters: - key (str): key in config DB, with table Seperator if applicable. - - Returns: - void - ''' - # To Debug - if self.DEBUG: - cmd = 'sudo redis-cli -n 1 hgetall "{}"'.format(key) - self.sysLog(syslog.LOG_DEBUG, "Running {}".format(cmd)) - print(cmd) - system(cmd) - return - def _checkNoPortsInAsicDb(self, db, ports, portMap): ''' Check ASIC DB for PORTs in port List @@ -311,14 +293,11 @@ def _checkNoPortsInAsicDb(self, db, ports, portMap): db.connect(db.ASIC_DB) for port in ports: key = self.oidKey + portMap[port] - if self._checkKeyinAsicDB(key, db) == False: - # Test again via redis-cli - self._testRedisCli(key) - else: + if self._checkKeyinAsicDB(key, db) == True: return False except Exception as e: - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) return False return True @@ -337,11 +316,11 @@ def _verifyAsicDB(self, db, ports, portMap, timeout): Returns: (bool) ''' - print("Verify Port Deletion from Asic DB, Wait...") - self.sysLog(msg="Verify Port Deletion from Asic DB, Wait...") + self.sysLog(doPrint=True, msg="Verify Port Deletion from Asic DB, Wait...") try: for waitTime in range(timeout): - self.sysLog(msg='Check Asic DB: {} try'.format(waitTime+1)) + self.sysLog(logLevel=syslog.LOG_DEBUG, msg='Check Asic DB: {} \ + try'.format(waitTime+1)) # checkNoPortsInAsicDb will return True if all ports are not # present in ASIC DB if self._checkNoPortsInAsicDb(db, ports, portMap): @@ -350,19 +329,18 @@ def _verifyAsicDB(self, db, ports, portMap, timeout): # raise if timer expired if waitTime + 1 == timeout: - print("!!! Critical Failure, Ports are not Deleted from \ - ASIC DB, Bail Out !!!") self.sysLog(syslog.LOG_CRIT, "!!! Critical Failure, Ports \ - are not Deleted from ASIC DB, Bail Out !!!") - raise(Exception("Ports are present in ASIC DB after timeout")) + are not Deleted from ASIC DB, Bail Out !!!", doPrint=True) + raise(Exception("Ports are present in ASIC DB after {} secs".\ + format(timeout))) except Exception as e: - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) raise e return True - def breakOutPort(self, delPorts=list(), addPorts= list(), portJson=dict(), \ + def breakOutPort(self, delPorts=list(), addPorts=list(), portJson=dict(), \ force=False, loadDefConfig=True): ''' This is the main function for port breakout. Exposed to caller. @@ -409,7 +387,7 @@ def breakOutPort(self, delPorts=list(), addPorts= list(), portJson=dict(), \ self.writeConfigDB(addConfigtoLoad) except Exception as e: - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) return None, False return None, True @@ -431,15 +409,14 @@ def _deletePorts(self, ports=list(), force=False): try: self.sysLog(msg="delPorts ports:{} force:{}".format(ports, force)) - print('\nStart Port Deletion') + self.sysLog(doPrint=True, msg='Start Port Deletion') deps = list() # Get all dependecies for ports for port in ports: xPathPort = self.sy.findXpathPortLeaf(port) - print('Find dependecies for port {}'.format(port)) - self.sysLog(msg='Find dependecies for port {}'.format(port)) - # print("Generated Xpath:" + xPathPort) + self.sysLog(doPrint=True, msg='Find dependecies for port {}'.\ + format(port)) dep = self.sy.find_data_dependencies(str(xPathPort)) if dep: deps.extend(dep) @@ -450,7 +427,7 @@ def _deletePorts(self, ports=list(), force=False): # delets all deps, No topological sort is needed as of now, if deletion # of deps fails, return immediately - elif deps and force: + elif deps: for dep in deps: self.sysLog(msg='Deleting {}'.format(dep)) self.sy.deleteNode(str(dep)) @@ -460,8 +437,7 @@ def _deletePorts(self, ports=list(), force=False): # all deps are deleted now, delete all ports now for port in ports: xPathPort = self.sy.findXpathPort(port) - print("Deleting Port: " + port) - self.sysLog(msg='Deleting Port:{}'.format(port)) + self.sysLog(doPrint=True, msg="Deleting Port: " + port) self.sy.deleteNode(str(xPathPort)) # Let`s Validate the tree now @@ -474,19 +450,19 @@ def _deletePorts(self, ports=list(), force=False): configToLoad = self._updateDiffConfigDB() except Exception as e: - print(e) - print("Port Deletion Failed") + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Port Deletion Failed") return configToLoad, deps, False return configToLoad, deps, True - def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): + def _addPorts(self, portJson=dict(), loadDefConfig=True): ''' Add ports and default confug in data tree, validate and return resultant config. Parameters: - ports (list): list of ports portJson (dict): Config DB json Part of all Ports, generated from platform.json. loadDefConfig: If loadDefConfig, add default config for ports as well. @@ -495,13 +471,11 @@ def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): (configToLoad, ret) (tuple)[dict, bool] ''' configToLoad = None + ports = portJson['PORT'].keys() try: - self.sysLog(msg='Start Port Addition') - self.sysLog(msg="addPorts ports:{} loadDefConfig:{}".\ - format(ports, loadDefConfig)) - self.sysLog(msg="addPorts Args portjson {}".format(portJson)) - - print('\nStart Port Addition') + self.sysLog(doPrint=True, msg='Start Port Addition') + self.sysLog(msg="addPorts Args portjson: {} loadDefConfig: {}".\ + format(portJson, loadDefConfig)) if loadDefConfig: defConfig = self._getDefaultConfig(ports) @@ -520,8 +494,8 @@ def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): # merge new config with data tree, this is json level merge. # We do not allow new table merge while adding default config. if loadDefConfig: - print("Merge Default Config for {}".format(ports)) - self.sysLog(msg="Merge Default Config for {}".format(ports)) + self.sysLog(doPrint=True, msg="Merge Default Config for {}".\ + format(ports)) self._mergeConfigs(self.configdbJsonOut, defConfig, True) # create a tree with merged config and validate, if validation is @@ -534,8 +508,9 @@ def _addPorts(self, ports=list(), portJson=dict(), loadDefConfig=True): configToLoad = self._updateDiffConfigDB() except Exception as e: - print(e) - print("Port Addition Failed") + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Port Addition Failed") return configToLoad, False return configToLoad, True @@ -582,8 +557,9 @@ def _mergeItems(it1, it2): if uniqueKeys: D1.update(D2) except Exce as e: - print("Merge Config failed") - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Merge Config failed") + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) raise e return D1 @@ -629,7 +605,6 @@ def _searchKeysInConfig(self, In, Out, skeys): if skey in In: found = True Out.append(skey) - #print("Added in list:" + port) else: # nothing for other keys @@ -654,7 +629,8 @@ def configWithKeys(self, configIn=dict(), keys=list()): if len(configIn) and len(keys): self._searchKeysInConfig(configIn, configOut, skeys=keys) except Exception as e: - print("configWithKeys Failed, Error: {}".format(str(e))) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="configWithKeys Failed, Error: {}".format(str(e))) raise e return configOut @@ -672,13 +648,13 @@ def _getDefaultConfig(self, ports=list()): ''' # function code try: - print("Generating default config for {}".format(ports)) + self.sysLog(doPrint=True, msg="Generating default config for {}".format(ports)) defConfigIn = readJsonFile(DEFAULT_CONFIG_DB_JSON_FILE) - #print(defConfigIn) defConfigOut = dict() self._searchKeysInConfig(defConfigIn, defConfigOut, skeys=ports) except Exception as e: - print("getDefaultConfig Failed, Error: {}".format(str(e))) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="getDefaultConfig Failed, Error: {}".format(str(e))) raise e return defConfigOut @@ -695,15 +671,16 @@ def _updateDiffConfigDB(self): ''' try: # Get the Diff - print('Generate Final Config to write in DB') + self.sysLog(msg='Generate Final Config to write in DB') configDBdiff = self._diffJson() # Process diff and create Config which can be updated in Config DB configToLoad = self._createConfigToLoad(configDBdiff, \ self.configdbJsonIn, self.configdbJsonOut) except Exception as e: - print("Config Diff Generation failed") - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Config Diff Generation failed") + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) raise e return configToLoad @@ -819,8 +796,9 @@ def _recurCreateConfig(diff, inp, outp, config): _recurCreateConfig(diff, inp, outp, configToLoad) except Exception as e: - print("Create Config to load in DB, Failed") - print(e) + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, \ + msg="Create Config to load in DB, Failed") + self.sysLog(doPrint=True, logLevel=syslog.LOG_ERR, msg=str(e)) raise e return configToLoad From a88ecc28f12128281634cdad513a7d42bcb581d6 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Fri, 26 Jun 2020 08:15:54 -0700 Subject: [PATCH 12/15] [sonic-utilities-tests/config_mgmt_test.py]: Remove addPorts Arg from breakOutPort API. Changes: -- Remove addPorts Arg from breakOutPort API. -- Change Test Code accordingly. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 11 ++++--- sonic-utilities-tests/config_mgmt_test.py | 36 +++++++++++------------ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index dd5329ce0f..f1177bf724 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -7,7 +7,6 @@ import syslog from json import load - from os import system from time import sleep as tsleep from imp import load_source from jsondiff import diff @@ -146,7 +145,7 @@ def sysLog(self, logLevel=syslog.LOG_INFO, msg=None, doPrint=False): # log debug only if enabled if self.DEBUG == False and logLevel == syslog.LOG_DEBUG: return - if flgas.interactive !=0 and doPrint == True: + if flags.interactive !=0 and doPrint == True: print("{}".format(msg)) syslog.openlog(self.SYSLOG_IDENTIFIER) syslog.syslog(logLevel, msg) @@ -340,8 +339,8 @@ def _verifyAsicDB(self, db, ports, portMap, timeout): return True - def breakOutPort(self, delPorts=list(), addPorts=list(), portJson=dict(), \ - force=False, loadDefConfig=True): + def breakOutPort(self, delPorts=list(), portJson=dict(), force=False, \ + loadDefConfig=True): ''' This is the main function for port breakout. Exposed to caller. @@ -366,8 +365,8 @@ def breakOutPort(self, delPorts=list(), addPorts=list(), portJson=dict(), \ return deps, ret # add Ports and get the config diff and True/False - addConfigtoLoad, ret = self._addPorts(ports=addPorts, \ - portJson=portJson, loadDefConfig=loadDefConfig) + addConfigtoLoad, ret = self._addPorts(portJson=portJson, \ + loadDefConfig=loadDefConfig) # return if ret is False, Great thing, no change is done in Config if ret == False: return None, ret diff --git a/sonic-utilities-tests/config_mgmt_test.py b/sonic-utilities-tests/config_mgmt_test.py index eabdde5eb5..d8704aa7d0 100644 --- a/sonic-utilities-tests/config_mgmt_test.py +++ b/sonic-utilities-tests/config_mgmt_test.py @@ -131,7 +131,7 @@ def generate_args(self, portIdx, laneIdx, curMode, newMode): newMode (str): new breakout mode of Port. Return: - dPorts, aPorts ,pJson (tuple)[list, list, dict] + dPorts, pJson (tuple)[list, dict] ''' # default params pre = "Ethernet" @@ -157,7 +157,7 @@ def generate_args(self, portIdx, laneIdx, curMode, newMode): lanes = [str(li+j) for j in range(i)]; lanes = ','.join(lanes) pJson['PORT'][aPorts[pi]] = {"speed": str(speed), "lanes": str(lanes)} li = li+i; pi = pi + 1 - return dPorts, aPorts ,pJson + return dPorts, pJson def updateConfig(self, conf, uconf): ''' @@ -248,9 +248,9 @@ def dpb_port8_1x100G_1x50G_2x25G_f_l(self, curConfig): ''' cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS - dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='1x100G', newMode='1x50G(2)+2x25G(2)') - deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=True, loadDefConfig=True) # Expected Result delConfig and addConfig is pushed in order delConfig = { @@ -314,9 +314,9 @@ def dpb_port8_4x25G_1x100G_f(self, curConfig): ''' cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS - dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='4x25G', newMode='1x100G') - deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=False, loadDefConfig=False) # Expected Result delConfig and addConfig is pushed in order delConfig = { @@ -344,9 +344,9 @@ def dpb_port8_1x100G_4x25G(self, curConfig): assert for success and failure. ''' cmdpb = self.config_mgmt_dpb(curConfig) - dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='1x100G', newMode='4x25G') - deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=False, loadDefConfig=False) # Expected Result delConfig and addConfig is pushed in order delConfig = { @@ -372,9 +372,9 @@ def dpb_port8_2x50G_1x100G_f(self, curConfig): ''' cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS - dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='2x50G', newMode='1x100G') - deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=True, loadDefConfig=False) # Expected Result delConfig and addConfig is pushed in order delConfig = { @@ -408,9 +408,9 @@ def dpb_port8_2x50G_1x100G(self, curConfig): ''' cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS - dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='2x50G', newMode='1x100G') - deps, ret = cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, + deps, ret = cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=False, loadDefConfig=False) # Expected Result assert ret == False and len(deps) == 3 @@ -430,10 +430,10 @@ def dpb_port8_4x25G_2x50G_f_l(self, curConfig): ''' cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS - dPorts, aPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ + dPorts, pJson = self.generate_args(portIdx=8, laneIdx=73, \ curMode='4x25G', newMode='2x50G') - cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, - force=True, loadDefConfig=True) + cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=True, \ + loadDefConfig=True) # Expected Result delConfig and addConfig is pushed in order delConfig = { u'ACL_TABLE': { @@ -496,10 +496,10 @@ def dpb_port4_4x25G_2x50G_f_l(self, curConfig): ''' cmdpb = self.config_mgmt_dpb(curConfig) # create ARGS - dPorts, aPorts, pJson = self.generate_args(portIdx=4, laneIdx=69, \ + dPorts, pJson = self.generate_args(portIdx=4, laneIdx=69, \ curMode='4x25G', newMode='2x50G') - cmdpb.breakOutPort(delPorts=dPorts, addPorts= aPorts, portJson=pJson, - force=True, loadDefConfig=True) + cmdpb.breakOutPort(delPorts=dPorts, portJson=pJson, force=True, \ + loadDefConfig=True) # Expected Result delConfig and addConfig is pushed in order delConfig = { u'ACL_TABLE': { From 2a4b9adf8c3b282164280fad9e1ff453e6115245 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Fri, 26 Jun 2020 10:00:06 -0700 Subject: [PATCH 13/15] [setup.py]: Removing xmltodict from install_requirements. this pkg will come from https://github.com/Azure/sonic-buildimage/pull/4740. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 068f8bfe7f..edffa77cd0 100644 --- a/setup.py +++ b/setup.py @@ -148,8 +148,7 @@ # - tabulate install_requires=[ 'click', - 'natsort', - 'xmltodict==0.12.0' + 'natsort' ], setup_requires= [ 'pytest-runner' From 4990ab5afe9a4881455a645933d36bbaf1ebb3fb Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Mon, 29 Jun 2020 12:21:39 -0700 Subject: [PATCH 14/15] [sonic-utilities-tests/config_mgmt_test.py]: Due to dependency on sonic_yang_models now we do not need relative path to yang models. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- sonic-utilities-tests/config_mgmt_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sonic-utilities-tests/config_mgmt_test.py b/sonic-utilities-tests/config_mgmt_test.py index d8704aa7d0..aec7f75e30 100644 --- a/sonic-utilities-tests/config_mgmt_test.py +++ b/sonic-utilities-tests/config_mgmt_test.py @@ -15,7 +15,6 @@ class TestConfigMgmt(TestCase): ''' def setUp(self): - config_mgmt.YANG_DIR = "./../../../sonic-yang-models/yang-models/" config_mgmt.CONFIG_DB_JSON_FILE = "startConfigDb.json" config_mgmt.DEFAULT_CONFIG_DB_JSON_FILE = "portBreakOutConfigDb.json" return From 6abb5bb2623bbb85962fb3f0fb0ad2fcb24bd7c2 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Thu, 2 Jul 2020 10:49:25 -0700 Subject: [PATCH 15/15] [config/config_mgmt.py]: Remove parameter from comment. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- config/config_mgmt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config_mgmt.py b/config/config_mgmt.py index f1177bf724..c9db79ea90 100644 --- a/config/config_mgmt.py +++ b/config/config_mgmt.py @@ -346,7 +346,6 @@ def breakOutPort(self, delPorts=list(), portJson=dict(), force=False, \ Parameters: delPorts (list): ports to be deleted. - addPorts (list): ports to be added. portJson (dict): Config DB json Part of all Ports, generated from platform.json. force (bool): if false return dependecies, else delete dependencies.