Merge branch 'sonic-net:master' into test3

config/main.py

@@ -20,13 +20,15 @@
 from jsonpointer import JsonPointerException
 from collections import OrderedDict
 from generic_config_updater.generic_updater import GenericUpdater, ConfigFormat, extract_scope
+from generic_config_updater.gu_common import HOST_NAMESPACE, GenericConfigUpdaterError
 from minigraph import parse_device_desc_xml, minigraph_encoder
 from natsort import natsorted
 from portconfig import get_child_ports
 from socket import AF_INET, AF_INET6
 from sonic_py_common import device_info, multi_asic
 from sonic_py_common.general import getstatusoutput_noshell
 from sonic_py_common.interface import get_interface_table_name, get_port_table_name, get_intf_longname
+from sonic_yang_cfg_generator import SonicYangCfgDbGenerator
 from utilities_common import util_base
 from swsscommon import swsscommon
 from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
@@ -1155,25 +1157,75 @@ def validate_gre_type(ctx, _, value):
         return gre_type_value
     except ValueError:
         raise click.UsageError("{} is not a valid GRE type".format(value))
-
+
+
+def multi_asic_save_config(db, filename):
+    """A function to save all asic's config to single file
+    """
+    all_current_config = {}
+    cfgdb_clients = db.cfgdb_clients
+
+    for ns, config_db in cfgdb_clients.items():
+        current_config = config_db.get_config()
+        sonic_cfggen.FormatConverter.to_serialized(current_config)
+        asic_name = "localhost" if ns == DEFAULT_NAMESPACE else ns
+        all_current_config[asic_name] = sort_dict(current_config)
+    click.echo("Integrate each ASIC's config into a single JSON file {}.".format(filename))
+    with open(filename, 'w') as file:
+        json.dump(all_current_config, file, indent=4)
+
 # Function to apply patch for a single ASIC.
 def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path):
     scope, changes = scope_changes
     # Replace localhost to DEFAULT_NAMESPACE which is db definition of Host
-    if scope.lower() == "localhost" or scope == "":
+    if scope.lower() == HOST_NAMESPACE or scope == "":
         scope = multi_asic.DEFAULT_NAMESPACE
-        
-    scope_for_log = scope if scope else "localhost"
+
+    scope_for_log = scope if scope else HOST_NAMESPACE
     try:
         # Call apply_patch with the ASIC-specific changes and predefined parameters
-        GenericUpdater(namespace=scope).apply_patch(jsonpatch.JsonPatch(changes), config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path)
+        GenericUpdater(namespace=scope).apply_patch(jsonpatch.JsonPatch(changes),
+                                                    config_format,
+                                                    verbose,
+                                                    dry_run,
+                                                    ignore_non_yang_tables,
+                                                    ignore_path)
         results[scope_for_log] = {"success": True, "message": "Success"}
         log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes}")
     except Exception as e:
         results[scope_for_log] = {"success": False, "message": str(e)}
         log.log_error(f"'apply-patch' executed failed for {scope_for_log} by {changes} due to {str(e)}")
 
 
+def validate_patch(patch):
+    try:
+        command = ["show", "runningconfiguration", "all"]
+        proc = subprocess.Popen(command, text=True, stdout=subprocess.PIPE)
+        all_running_config, returncode = proc.communicate()
+        if returncode:
+            log.log_notice(f"Fetch all runningconfiguration failed as output:{all_running_config}")
+            return False
+
+        # Structure validation and simulate apply patch.
+        all_target_config = patch.apply(json.loads(all_running_config))
+
+        # Verify target config by YANG models
+        target_config = all_target_config.pop(HOST_NAMESPACE) if multi_asic.is_multi_asic() else all_target_config
+        target_config.pop("bgpraw", None)
+        if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
+            return False
+
+        if multi_asic.is_multi_asic():
+            for asic in multi_asic.get_namespace_list():
+                target_config = all_target_config.pop(asic)
+                target_config.pop("bgpraw", None)
+                if not SonicYangCfgDbGenerator().validate_config_db_json(target_config):
+                    return False
+
+        return True
+    except Exception as e:
+        raise GenericConfigUpdaterError(f"Validate json patch: {patch} failed due to:{e}")
+
 # This is our main entrypoint - the main 'config' command
 @click.group(cls=clicommon.AbbreviationGroup, context_settings=CONTEXT_SETTINGS)
 @click.pass_context
@@ -1241,7 +1293,8 @@ def config(ctx):
 @click.option('-y', '--yes', is_flag=True, callback=_abort_if_false,
                 expose_value=False, prompt='Existing files will be overwritten, continue?')
 @click.argument('filename', required=False)
-def save(filename):
+@clicommon.pass_db
+def save(db, filename):
     """Export current config DB to a file on disk.\n
        <filename> : Names of configuration file(s) to save, separated by comma with no spaces in between
     """
@@ -1256,7 +1309,13 @@ def save(filename):
     if filename is not None:
         cfg_files = filename.split(',')
 
-        if len(cfg_files) != num_cfg_file:
+        # If only one filename is provided in multi-ASIC mode,
+        # save all ASIC configurations to that single file.
+        if len(cfg_files) == 1 and multi_asic.is_multi_asic():
+            filename = cfg_files[0]
+            multi_asic_save_config(db, filename)
+            return
+        elif len(cfg_files) != num_cfg_file:
             click.echo("Input {} config file(s) separated by comma for multiple files ".format(num_cfg_file))
             return
 
@@ -1381,6 +1440,9 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
             patch_as_json = json.loads(text)
             patch = jsonpatch.JsonPatch(patch_as_json)
 
+        if not validate_patch(patch):
+            raise GenericConfigUpdaterError(f"Failed validating patch:{patch}")
+
         results = {}
         config_format = ConfigFormat[format.upper()]
         # Initialize a dictionary to hold changes categorized by scope
@@ -1403,7 +1465,8 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
         # Empty case to force validate YANG model.
         if not changes_by_scope:
             asic_list = [multi_asic.DEFAULT_NAMESPACE]
-            asic_list.extend(multi_asic.get_namespace_list())
+            if multi_asic.is_multi_asic():
+                asic_list.extend(multi_asic.get_namespace_list())
             for asic in asic_list:
                 changes_by_scope[asic] = []
 
@@ -1416,7 +1479,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, ignore_non_yang_tables, i
 
         if failures:
             failure_messages = '\n'.join([f"- {failed_scope}: {results[failed_scope]['message']}" for failed_scope in failures])
-            raise Exception(f"Failed to apply patch on the following scopes:\n{failure_messages}")
+            raise GenericConfigUpdaterError(f"Failed to apply patch on the following scopes:\n{failure_messages}")
 
         log.log_notice(f"Patch applied successfully for {patch}.")
         click.secho("Patch applied successfully.", fg="cyan", underline=True)
@@ -1620,9 +1683,9 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart, force, file_form
             file_input = read_json_file(file)
 
             platform = file_input.get("DEVICE_METADATA", {}).\
-                get("localhost", {}).get("platform")
+                get(HOST_NAMESPACE, {}).get("platform")
             mac = file_input.get("DEVICE_METADATA", {}).\
-                get("localhost", {}).get("mac")
+                get(HOST_NAMESPACE, {}).get("mac")
 
             if not platform or not mac:
                 log.log_warning("Input file does't have platform or mac. platform: {}, mac: {}"
@@ -1995,8 +2058,8 @@ def override_config_table(db, input_config_db, dry_run):
         if multi_asic.is_multi_asic() and len(config_input):
             # Golden Config will use "localhost" to represent host name
             if ns == DEFAULT_NAMESPACE:
-                if "localhost" in config_input.keys():
-                    ns_config_input = config_input["localhost"]
+                if HOST_NAMESPACE in config_input.keys():
+                    ns_config_input = config_input[HOST_NAMESPACE]
                 else:
                     click.secho("Wrong config format! 'localhost' not found in host config! cannot override.. abort")
                     sys.exit(1)

generic_config_updater/generic_updater.py

@@ -2,7 +2,7 @@
 import jsonpointer
 import os
 from enum import Enum
-from .gu_common import GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \
+from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \
                        DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging
 from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \
                           TablesWithoutYangConfigSplitter, IgnorePathsFromYangConfigSplitter
@@ -16,28 +16,26 @@ def extract_scope(path):
     if not path:
         raise Exception("Wrong patch with empty path.")
 
-    try:
-        pointer = jsonpointer.JsonPointer(path)
-        parts = pointer.parts
-    except Exception as e:
-        raise Exception(f"Error resolving path: '{path}' due to {e}")
+    pointer = jsonpointer.JsonPointer(path)
+    parts = pointer.parts
 
     if not parts:
-        raise Exception("Wrong patch with empty path.")
+        raise GenericConfigUpdaterError("Wrong patch with empty path.")
     if parts[0].startswith("asic"):
         if not parts[0][len("asic"):].isnumeric():
-            raise Exception(f"Error resolving path: '{path}' due to incorrect ASIC number.")
+            raise GenericConfigUpdaterError(f"Error resolving path: '{path}' due to incorrect ASIC number.")
         scope = parts[0]
         remainder = "/" + "/".join(parts[1:])
-    elif parts[0] == "localhost":
-        scope = "localhost"
+    elif parts[0] == HOST_NAMESPACE:
+        scope = HOST_NAMESPACE
         remainder = "/" + "/".join(parts[1:])
     else:
         scope = ""
         remainder = path
 
     return scope, remainder
 
+
 class ConfigLock:
     def acquire_lock(self):
         # TODO: Implement ConfigLock
@@ -67,7 +65,7 @@ def __init__(self,
         self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier(namespace=self.namespace)
 
     def apply(self, patch, sort=True):
-        scope = self.namespace if self.namespace else 'localhost'
+        scope = self.namespace if self.namespace else HOST_NAMESPACE
         self.logger.log_notice(f"{scope}: Patch application starting.")
         self.logger.log_notice(f"{scope}: Patch: {patch}")
 
@@ -84,10 +82,10 @@ def apply(self, patch, sort=True):
         self.config_wrapper.validate_field_operation(old_config, target_config)
 
         # Validate target config does not have empty tables since they do not show up in ConfigDb
-        self.logger.log_notice(f"{scope}: alidating target config does not have empty tables, " \
-                               "since they do not show up in ConfigDb.")
+        self.logger.log_notice(f"""{scope}: validating target config does not have empty tables,
+                               since they do not show up in ConfigDb.""")
         empty_tables = self.config_wrapper.get_empty_tables(target_config)
-        if empty_tables: # if there are empty tables
+        if empty_tables:  # if there are empty tables
             empty_tables_txt = ", ".join(empty_tables)
             raise EmptyTableError(f"{scope}: given patch is not valid because it will result in empty tables " \
                              "which is not allowed in ConfigDb. " \
@@ -105,9 +103,6 @@ def apply(self, patch, sort=True):
         self.logger.log_notice(f"The {scope} patch was converted into {changes_len} " \
                           f"change{'s' if changes_len != 1 else ''}{':' if changes_len > 0 else '.'}")
 
-        for change in changes:
-            self.logger.log_notice(f"  * {change}")
-
         # Apply changes in order
         self.logger.log_notice(f"{scope}: applying {changes_len} change{'s' if changes_len != 1 else ''} " \
                                f"in order{':' if changes_len > 0 else '.'}")

generic_config_updater/gu_common.py

@@ -16,6 +16,7 @@
 SYSLOG_IDENTIFIER = "GenericConfigUpdater"
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 GCU_FIELD_OP_CONF_FILE = f"{SCRIPT_DIR}/gcu_field_operation_validators.conf.json"
+HOST_NAMESPACE = "localhost"
 
 class GenericConfigUpdaterError(Exception):
     pass

scripts/fabricstat

@@ -399,6 +399,49 @@ class FabricIsolation(FabricStat):
         print(tabulate(body, header, tablefmt='simple', stralign='right'))
         return
 
+class FabricRate(FabricStat):
+    def rate_print(self):
+        # Connect to database
+        self.db = multi_asic.connect_to_all_dbs_for_ns(self.namespace)
+        # Get the set of all fabric ports
+        port_keys = self.db.keys(self.db.STATE_DB, FABRIC_PORT_STATUS_TABLE_PREFIX + '*')
+        # Create a new dictionary. The keys are the local port values in integer format.
+        # Only fabric ports that have remote port data are added.
+        port_dict = {}
+        for port_key in port_keys:
+            port_data = self.db.get_all(self.db.STATE_DB, port_key)
+            port_number = int(port_key.replace("FABRIC_PORT_TABLE|PORT", ""))
+            port_dict.update({port_number: port_data})
+        # Create ordered table of fabric ports.
+        rxRate = 0
+        rxData = 0
+        txRate = 0
+        txData = 0
+        time = 0
+        local_time = ""
+        # RX data , Tx data , Time are for testing
+        asic = "asic0"
+        if self.namespace:
+            asic = self.namespace
+        header = ["ASIC", "Link ID", "Rx Data Mbps", "Tx Data Mbps"]
+        body = []
+        for port_number in sorted(port_dict.keys()):
+            port_data = port_dict[port_number]
+            if "OLD_RX_RATE_AVG" in port_data:
+                rxRate = port_data["OLD_RX_RATE_AVG"]
+            if "OLD_RX_DATA" in port_data:
+                rxData = port_data["OLD_RX_DATA"]
+            if "OLD_TX_RATE_AVG" in port_data:
+                txRate = port_data["OLD_TX_RATE_AVG"]
+            if "OLD_TX_DATA" in port_data:
+                txData = port_data["OLD_TX_DATA"]
+            if "LAST_TIME" in port_data:
+                time = int(port_data["LAST_TIME"])
+                local_time = datetime.fromtimestamp(time)
+            body.append((asic, port_number, rxRate, txRate));
+        click.echo()
+        click.echo(tabulate(body, header, tablefmt='simple', stralign='right'))
+
 def main():
     global cnstat_dir
     global cnstat_fqn_file_port
@@ -415,6 +458,8 @@ Examples:
     fabricstat -q -n asic0
     fabricstat -c
     fabricstat -c -n asic0
+    fabricstat -s
+    fabricstat -s -n asic0
     fabricstat -C
     fabricstat -D
 """)
@@ -425,6 +470,7 @@ Examples:
     parser.add_argument('-e', '--errors', action='store_true', help='Display errors')
     parser.add_argument('-c','--capacity',action='store_true', help='Display fabric capacity')
     parser.add_argument('-i','--isolation', action='store_true', help='Display fabric ports isolation status')
+    parser.add_argument('-s','--rate', action='store_true', help='Display fabric counters rate')
     parser.add_argument('-C','--clear', action='store_true', help='Copy & clear fabric counters')
     parser.add_argument('-D','--delete', action='store_true', help='Delete saved stats')
 
@@ -433,6 +479,7 @@ Examples:
     reachability = args.reachability
     capacity_status = args.capacity
     isolation_status = args.isolation
+    rate = args.rate
     namespace = args.namespace
     errors_only = args.errors
 
@@ -455,17 +502,21 @@ Examples:
 
     def nsStat(ns, errors_only):
         if queue:
-           stat = FabricQueueStat(ns)
+            stat = FabricQueueStat(ns)
         elif reachability:
-           stat = FabricReachability(ns)
-           stat.reachability_print()
-           return
+            stat = FabricReachability(ns)
+            stat.reachability_print()
+            return
         elif isolation_status:
-           stat = FabricIsolation(ns)
-           stat.isolation_print()
-           return
+            stat = FabricIsolation(ns)
+            stat.isolation_print()
+            return
+        elif rate:
+            stat = FabricRate(ns)
+            stat.rate_print()
+            return
         else:
-           stat = FabricPortStat(ns)
+            stat = FabricPortStat(ns)
         cnstat_dict = stat.get_cnstat_dict()
         if save_fresh_stats:
             stat.save_fresh_stats()

show/fabric.py

@@ -74,3 +74,13 @@ def queue(namespace):
     if namespace is not None:
         cmd += ['-n', str(namespace)]
     clicommon.run_command(cmd)
+
+
+@counters.command()
+@multi_asic_util.multi_asic_click_option_namespace
+def rate(namespace):
+    """Show fabric counters rate"""
+    cmd = ['fabricstat', '-s']
+    if namespace is not None:
+        cmd += ['-n', str(namespace)]
+    clicommon.run_command(cmd)

tests/config_save_output/all_config_db.json

@@ -0,0 +1,5 @@
+{
+    "localhost": {},
+    "asic0": {},
+    "asic1": {}
+}