From ffd0bd18d1b99a13b66b9e1453174667ae0eeb3c Mon Sep 17 00:00:00 2001 From: abdosi <58047199+abdosi@users.noreply.github.com> Date: Wed, 13 May 2020 16:32:26 -0700 Subject: [PATCH] Changes to support acl-loader and mirror-session config commands for multi-npu platforms. (#908) * Changes to support acl-loader command for multi-npu platforms. Move multi-npu related utility functions from config/main.py to sonic-device-util.py so that it can be used by acl-loader/any other module. Updated Mirror Session add/remove for multi-npu platforms. Needed for Everflow ACL rule programming. * Address review comment to change comment from """ to # except for doc string --- acl_loader/main.py | 120 ++++++++++++++++++++++++++++++++++++++------- config/main.py | 117 +++++++++++++++---------------------------- 2 files changed, 143 insertions(+), 94 deletions(-) diff --git a/acl_loader/main.py b/acl_loader/main.py index 362f1f75ea..f9201846b3 100644 --- a/acl_loader/main.py +++ b/acl_loader/main.py @@ -6,11 +6,13 @@ import syslog import tabulate from natsort import natsorted +import sonic_device_util import openconfig_acl import pyangbind.lib.pybindJSON as pybindJSON from swsssdk import ConfigDBConnector from swsssdk import SonicV2Connector +from swsssdk import SonicDBConfig def info(msg): @@ -114,12 +116,39 @@ def __init__(self): self.tables_db_info = {} self.rules_db_info = {} self.rules_info = {} + # Load global db config. This call is no-op in single npu platforms + SonicDBConfig.load_sonic_global_db_config() self.sessions_db_info = {} self.configdb = ConfigDBConnector() self.configdb.connect() self.statedb = SonicV2Connector(host="127.0.0.1") self.statedb.connect(self.statedb.STATE_DB) + # For multi-npu architecture we will have both global and per front asic namespace. + # Global namespace will be used for Control plane ACL which are via IPTables. + # Per ASIC namespace will be used for Data and Everflow ACL's. + # Global Configdb will have all ACL information for both Ctrl and Data/Evereflow ACL's + # and will be used as souurce of truth for ACL modification to config DB which will be done to both Global DB and + # front asic namespace + + self.per_npu_configdb = {} + + # State DB are used for to get mirror Session monitor port. + # For multi-npu platforms each asic namespace can have different monitor port + # dependinding on which route to session destination ip. So for multi-npu + # platforms we get state db for all front asic namespace in addition to + + self.per_npu_statedb = {} + + # Getting all front asic namespace and correspding config and state DB connector + + namespaces = sonic_device_util.get_all_namespaces() + for front_asic_namespaces in namespaces['front_ns']: + self.per_npu_configdb[front_asic_namespaces] = ConfigDBConnector(use_unix_socket_path=True, namespace=front_asic_namespaces) + self.per_npu_configdb[front_asic_namespaces].connect() + self.per_npu_statedb[front_asic_namespaces] = SonicV2Connector(use_unix_socket_path=True, namespace=front_asic_namespaces) + self.per_npu_statedb[front_asic_namespaces].connect(self.per_npu_statedb[front_asic_namespaces].STATE_DB) + self.read_tables_info() self.read_rules_info() self.read_sessions_info() @@ -150,7 +179,14 @@ def read_policers_info(self): Read POLICER table from configuration database :return: """ - self.policers_db_info = self.configdb.get_table(self.POLICER) + + # For multi-npu platforms we will read from any one of front asic namespace + # config db as the information should be same across all config db + if self.per_npu_configdb: + namespace_configdb = (self.per_npu_configdb.values())[0] + self.policers_db_info = namespace_configdb.get_table(self.POLICER) + else: + self.policers_db_info = self.configdb.get_table(self.POLICER) def get_policers_db_info(self): return self.policers_db_info @@ -160,17 +196,30 @@ def read_sessions_info(self): Read MIRROR_SESSION table from configuration database :return: """ - self.sessions_db_info = self.configdb.get_table(self.CFG_MIRROR_SESSION_TABLE) + + # For multi-npu platforms we will read from any one of front asic namespace + # config db as the information should be same across all config db + if self.per_npu_configdb: + namespace_configdb = (self.per_npu_configdb.values())[0] + self.sessions_db_info = namespace_configdb.get_table(self.CFG_MIRROR_SESSION_TABLE) + else: + self.sessions_db_info = self.configdb.get_table(self.CFG_MIRROR_SESSION_TABLE) for key in self.sessions_db_info.keys(): - state_db_info = self.statedb.get_all(self.statedb.STATE_DB, "{}|{}".format(self.STATE_MIRROR_SESSION_TABLE, key)) - monitor_port = "" - if state_db_info: - status = state_db_info.get("status", "inactive") - monitor_port = state_db_info.get("monitor_port", "") + if self.per_npu_statedb: + # For multi-npu platforms we will read from all front asic name space + # statedb as the monitor port will be differnt for each asic + # and it's status also might be different (ideally should not happen) + # We will store them as dict of 'asic' : value + self.sessions_db_info[key]["status"] = {} + self.sessions_db_info[key]["monitor_port"] = {} + for namespace_key, namespace_statedb in self.per_npu_statedb.iteritems(): + state_db_info = namespace_statedb.get_all(self.statedb.STATE_DB, "{}|{}".format(self.STATE_MIRROR_SESSION_TABLE, key)) + self.sessions_db_info[key]["status"][namespace_key] = state_db_info.get("status", "inactive") if state_db_info else "error" + self.sessions_db_info[key]["monitor_port"][namespace_key] = state_db_info.get("monitor_port", "") if state_db_info else "" else: - status = "error" - self.sessions_db_info[key]["status"] = status - self.sessions_db_info[key]["monitor_port"] = monitor_port + state_db_info = self.statedb.get_all(self.statedb.STATE_DB, "{}|{}".format(self.STATE_MIRROR_SESSION_TABLE, key)) + self.sessions_db_info[key]["status"] = state_db_info.get("status", "inactive") if state_db_info else "error" + self.sessions_db_info[key]["monitor_port"] = state_db_info.get("monitor_port", "") if state_db_info else "" def get_sessions_db_info(self): return self.sessions_db_info @@ -309,7 +358,17 @@ def validate_actions(self, table_name, action_props): raise AclLoaderException("Table {} does not exist".format(table_name)) stage = self.tables_db_info[table_name].get("stage", Stage.INGRESS) - capability = self.statedb.get_all(self.statedb.STATE_DB, "{}|switch".format(self.SWITCH_CAPABILITY_TABLE)) + + # check if per npu state db is there then read using first state db + # else read from global statedb + if self.per_npu_statedb: + # For multi-npu we will read using anyone statedb connector for front asic namespace. + # Same information should be there in all state DB's + # as it is static information about switch capability + namespace_statedb = (self.per_npu_statedb.values())[0] + capability = namespace_statedb.get_all(self.statedb.STATE_DB, "{}|switch".format(self.SWITCH_CAPABILITY_TABLE)) + else: + capability = self.statedb.get_all(self.statedb.STATE_DB, "{}|switch".format(self.SWITCH_CAPABILITY_TABLE)) for action_key in dict(action_props): key = "{}|{}".format(self.ACL_ACTIONS_CAPABILITY_FIELD, stage.upper()) if key not in capability: @@ -518,9 +577,16 @@ def full_update(self): """ for key in self.rules_db_info.keys(): if self.current_table is None or self.current_table == key[0]: - self.configdb.mod_entry(self.ACL_RULE, key, None) + self.configdb.mod_entry(self.ACL_RULE, key, None) + # Program for per front asic namespace also if present + for namespace_configdb in self.per_npu_configdb.values(): + namespace_configdb.mod_entry(self.ACL_RULE, key, None) + self.configdb.mod_config({self.ACL_RULE: self.rules_info}) + # Program for per front asic namespace also if present + for namespace_configdb in self.per_npu_configdb.values(): + namespace_configdb.mod_config({self.ACL_RULE: self.rules_info}) def incremental_update(self): """ @@ -559,10 +625,17 @@ def incremental_update(self): # Remove all existing dataplane rules for key in current_dataplane_rules: self.configdb.mod_entry(self.ACL_RULE, key, None) + # Program for per-asic namespace also if present + for namespace_configdb in self.per_npu_configdb.values(): + namespace_configdb.mod_entry(self.ACL_RULE, key, None) + # Add all new dataplane rules for key in new_dataplane_rules: self.configdb.mod_entry(self.ACL_RULE, key, self.rules_info[key]) + # Program for per-asic namespace corresponding to front asic also if present. + for namespace_configdb in self.per_npu_configdb.values(): + namespace_configdb.mod_entry(self.ACL_RULE, key, self.rules_info[key]) added_controlplane_rules = new_controlplane_rules.difference(current_controlplane_rules) removed_controlplane_rules = current_controlplane_rules.difference(new_controlplane_rules) @@ -570,14 +643,25 @@ def incremental_update(self): for key in added_controlplane_rules: self.configdb.mod_entry(self.ACL_RULE, key, self.rules_info[key]) + # Program for per-asic namespace corresponding to front asic also if present. + # For control plane ACL it's not needed but to keep all db in sync program everywhere + for namespace_configdb in self.per_npu_configdb.values(): + namespace_configdb.mod_entry(self.ACL_RULE, key, self.rules_info[key]) for key in removed_controlplane_rules: self.configdb.mod_entry(self.ACL_RULE, key, None) + # Program for per-asic namespace corresponding to front asic also if present. + # For control plane ACL it's not needed but to keep all db in sync program everywhere + for namespace_configdb in self.per_npu_configdb.values(): + namespace_configdb.mod_entry(self.ACL_RULE, key, None) for key in existing_controlplane_rules: if cmp(self.rules_info[key], self.rules_db_info[key]) != 0: self.configdb.set_entry(self.ACL_RULE, key, self.rules_info[key]) - + # Program for per-asic namespace corresponding to front asic also if present. + # For control plane ACL it's not needed but to keep all db in sync program everywhere + for namespace_configdb in self.per_npu_configdb.values(): + namespace_configdb.set_entry(self.ACL_RULE, key, self.rules_info[key]) def delete(self, table=None, rule=None): """ @@ -589,8 +673,10 @@ def delete(self, table=None, rule=None): if not table or table == key[0]: if not rule or rule == key[1]: self.configdb.set_entry(self.ACL_RULE, key, None) - - + # Program for per-asic namespace corresponding to front asic also if present. + for namespace_configdb in self.per_npu_configdb.values(): + namespace_configdb.set_entry(self.ACL_RULE, key, None) + def show_table(self, table_name): """ Show ACL table configuration. @@ -626,7 +712,6 @@ def show_table(self, table_name): print(tabulate.tabulate(data, headers=header, tablefmt="simple", missingval="")) - def show_session(self, session_name): """ Show mirror session configuration. @@ -639,7 +724,8 @@ def show_session(self, session_name): for key, val in self.get_sessions_db_info().iteritems(): if session_name and key != session_name: continue - + # For multi-mpu platform status and monitor port will be dict() + # of 'asic-x':value data.append([key, val["status"], val["src_ip"], val["dst_ip"], val.get("gre_type", ""), val.get("dscp", ""), val.get("ttl", ""), val.get("queue", ""), val.get("policer", ""), diff --git a/config/main.py b/config/main.py index 063f94e518..8e110cd21d 100755 --- a/config/main.py +++ b/config/main.py @@ -117,7 +117,7 @@ def get_command(self, ctx, cmd_name): # Execute action on list of systemd services def execute_systemctl(list_of_services, action): - num_asic = _get_num_asic() + num_asic = sonic_device_util.get_num_npus() generated_services_list, generated_multi_instance_services = _get_sonic_generated_services(num_asic) if ((generated_services_list == []) and (generated_multi_instance_services == [])): @@ -156,44 +156,12 @@ def run_command(command, display_cmd=False, ignore_error=False): if proc.returncode != 0 and not ignore_error: sys.exit(proc.returncode) -# API to check if this is a multi-asic device or not. -def is_multi_asic(): - num_asics = _get_num_asic() - - if num_asics > 1: - return True - else: - return False - -"""In case of Multi-Asic platform, Each ASIC will have a linux network namespace created. - So we loop through the databases in different namespaces and depending on the sub_role - decide whether this is a front end ASIC/namespace or a back end one. -""" -def get_all_namespaces(): - front_ns = [] - back_ns = [] - num_asics = _get_num_asic() - - if is_multi_asic(): - for asic in range(num_asics): - namespace = "{}{}".format(NAMESPACE_PREFIX, asic) - config_db = ConfigDBConnector(use_unix_socket_path=True, namespace=namespace) - config_db.connect() - - metadata = config_db.get_table('DEVICE_METADATA') - if metadata['localhost']['sub_role'] == 'FrontEnd': - front_ns.append(namespace) - elif metadata['localhost']['sub_role'] == 'BackEnd': - back_ns.append(namespace) - - return {'front_ns': front_ns, 'back_ns': back_ns} - # Validate whether a given namespace name is valid in the device. def validate_namespace(namespace): - if not is_multi_asic(): + if not sonic_device_util.is_multi_npu(): return True - namespaces = get_all_namespaces() + namespaces = sonic_device_util.get_all_namespaces() if namespace in namespaces['front_ns'] + namespaces['back_ns']: return True else: @@ -494,32 +462,6 @@ def _clear_qos(): for qos_table in QOS_TABLE_NAMES: config_db.delete_table(qos_table) -def _get_hwsku(): - config_db = ConfigDBConnector() - config_db.connect() - metadata = config_db.get_table('DEVICE_METADATA') - return metadata['localhost']['hwsku'] - -def _get_platform(): - with open('/host/machine.conf') as machine_conf: - for line in machine_conf: - tokens = line.split('=') - if tokens[0].strip() == 'onie_platform' or tokens[0].strip() == 'aboot_platform': - return tokens[1].strip() - return '' - -def _get_num_asic(): - platform = _get_platform() - num_asic = 1 - asic_conf_file = os.path.join('/usr/share/sonic/device/', platform, ASIC_CONF_FILENAME) - if os.path.isfile(asic_conf_file): - with open(asic_conf_file) as conf_file: - for line in conf_file: - line_info = line.split('=') - if line_info[0].lower() == "num_asic": - num_asic = int(line_info[1]) - return num_asic - def _get_sonic_generated_services(num_asic): if not os.path.isfile(SONIC_GENERATED_SERVICE_PATH): return None @@ -644,11 +586,11 @@ def save(filename): """Export current config DB to a file on disk.\n : Names of configuration file(s) to save, separated by comma with no spaces in between """ - num_asic = _get_num_asic() + num_asic = sonic_device_util.get_num_npus() cfg_files = [] num_cfg_file = 1 - if is_multi_asic(): + if sonic_device_util.is_multi_npu(): num_cfg_file += num_asic # If the user give the filename[s], extract the file names. @@ -701,11 +643,11 @@ def load(filename, yes): if not yes: click.confirm(message, abort=True) - num_asic = _get_num_asic() + num_asic = sonic_device_util.get_num_npus() cfg_files = [] num_cfg_file = 1 - if is_multi_asic(): + if sonic_device_util.is_multi_npu(): num_cfg_file += num_asic # If the user give the filename[s], extract the file names. @@ -767,11 +709,11 @@ def reload(filename, yes, load_sysinfo): log_info("'reload' executing...") - num_asic = _get_num_asic() + num_asic = sonic_device_util.get_num_npus() cfg_files = [] num_cfg_file = 1 - if is_multi_asic(): + if sonic_device_util.is_multi_npu(): num_cfg_file += num_asic # If the user give the filename[s], extract the file names. @@ -1065,9 +1007,6 @@ def add(session_name, src_ip, dst_ip, dscp, ttl, gre_type, queue, policer): """ Add mirror session """ - config_db = ConfigDBConnector() - config_db.connect() - session_info = { "src_ip": src_ip, "dst_ip": dst_ip, @@ -1083,8 +1022,21 @@ def add(session_name, src_ip, dst_ip, dscp, ttl, gre_type, queue, policer): if queue is not None: session_info['queue'] = queue - - config_db.set_entry("MIRROR_SESSION", session_name, session_info) + + """ + For multi-npu platforms we need to program all front asic namespaces + """ + namespaces = sonic_device_util.get_all_namespaces() + if not namespaces['front_ns']: + config_db = ConfigDBConnector() + config_db.connect() + config_db.set_entry("MIRROR_SESSION", session_name, session_info) + else: + per_npu_configdb = {} + for front_asic_namespaces in namespaces['front_ns']: + per_npu_configdb[front_asic_namespaces] = ConfigDBConnector(use_unix_socket_path=True, namespace=front_asic_namespaces) + per_npu_configdb[front_asic_namespaces].connect() + per_npu_configdb[front_asic_namespaces].set_entry("MIRROR_SESSION", session_name, session_info) @mirror_session.command() @click.argument('session_name', metavar='', required=True) @@ -1092,10 +1044,21 @@ def remove(session_name): """ Delete mirror session """ - config_db = ConfigDBConnector() - config_db.connect() - config_db.set_entry("MIRROR_SESSION", session_name, None) + """ + For multi-npu platforms we need to program all front asic namespaces + """ + namespaces = sonic_device_util.get_all_namespaces() + if not namespaces['front_ns']: + config_db = ConfigDBConnector() + config_db.connect() + config_db.set_entry("MIRROR_SESSION", session_name, None) + else: + per_npu_configdb = {} + for front_asic_namespaces in namespaces['front_ns']: + per_npu_configdb[front_asic_namespaces] = ConfigDBConnector(use_unix_socket_path=True, namespace=front_asic_namespaces) + per_npu_configdb[front_asic_namespaces].connect() + per_npu_configdb[front_asic_namespaces].set_entry("MIRROR_SESSION", session_name, None) # # 'pfcwd' group ('config pfcwd ...') # @@ -1202,8 +1165,8 @@ def reload(): """Reload QoS configuration""" log_info("'qos reload' executing...") _clear_qos() - platform = _get_platform() - hwsku = _get_hwsku() + platform = sonic_device_util.get_platform() + hwsku = sonic_device_util.get_hwsku() buffer_template_file = os.path.join('/usr/share/sonic/device/', platform, hwsku, 'buffers.json.j2') if os.path.isfile(buffer_template_file): command = "{} -d -t {} >/tmp/buffers.json".format(SONIC_CFGGEN_PATH, buffer_template_file)