[setup.py] Added an option for providing additional arguments to pytest

README.md

@@ -64,6 +64,12 @@ python3 setup.py bdist_wheel
 python3 setup.py test
 ```
 
+#### To pass additional pytest arguments
+
+```
+python3 setup.py test -a "-k 'TestAaa'"
+```
+
 
 ### sonic-utilities-data
 

doc/Command-Reference.md

@@ -150,6 +150,7 @@
   * [SONiC Package Manager](#sonic-package-manager)
   * [SONiC Installer](#sonic-installer)
 * [Troubleshooting Commands](#troubleshooting-commands)
+  * [Debug Dumps](#debug-dumps)
 * [Routing Stack](#routing-stack)
 * [Quagga BGP Show Commands](#Quagga-BGP-Show-Commands)
 * [ZTP Configuration And Show Commands](#ztp-configuration-and-show-commands)
@@ -9117,6 +9118,60 @@ If the SONiC system was running for quite some time `show techsupport` will prod
   admin@sonic:~$ show techsupport --since='hour ago' # Will collect syslog and core files for the last one hour
   ```
 
+### Debug Dumps
+
+In SONiC, there usually exists a set of tables related/relevant to a particular module. All of these might have to be looked at to confirm whether any configuration update is properly applied and propagated. This utility comes in handy because it prints a unified view of the redis-state for a given module
+
+- Usage:
+  ```
+  Usage: dump state [OPTIONS] MODULE IDENTIFIER	 
+  Dump the redis-state of the identifier for the module specified
+
+  Options:
+	  -s, --show            Display Modules Available
+	  -d, --db TEXT         Only dump from these Databases
+	  -t, --table           Print in tabular format  [default: False]
+	  -k, --key-map         Only fetch the keys matched, don't extract field-value dumps  [default: False]
+	  -v, --verbose         Prints any intermediate output to stdout useful for dev & troubleshooting  [default: False]
+	  -n, --namespace TEXT  Dump the redis-state for this namespace.  [default: DEFAULT_NAMESPACE]
+	  --help                Show this message and exit.
+  ```
+
+
+- Examples:
+  ```
+  root@sonic# dump state --show
+  Module    Identifier
+  --------  ------------
+  port      port_name
+  copp      trap_id		
+  ```
+
+  ```
+  admin@sonic:~$ dump state copp arp_req --key-map --db ASIC_DB
+  {
+	    "arp_req": {
+		"ASIC_DB": {
+		    "keys": [
+			"ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF_TRAP:oid:0x22000000000c5b",
+			"ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF_TRAP_GROUP:oid:0x11000000000c59",
+			"ASIC_STATE:SAI_OBJECT_TYPE_POLICER:oid:0x12000000000c5a",
+			"ASIC_STATE:SAI_OBJECT_TYPE_QUEUE:oid:0x15000000000626"
+		    ],
+		    "tables_not_found": [],
+		    "vidtorid": {
+			"oid:0x22000000000c5b": "oid:0x200000000022",
+			"oid:0x11000000000c59": "oid:0x300000011",
+			"oid:0x12000000000c5a": "oid:0x200000012",
+			"oid:0x15000000000626": "oid:0x12e0000040015"
+		    }
+		}
+	    }
+	}	
+  ```
+
+
+
 Go Back To [Beginning of the document](#) or [Beginning of this section](#troubleshooting-commands)
 
 ## Routing Stack

dump/main.py

@@ -0,0 +1,214 @@
+import os,sys,json,re
+import click
+from tabulate import tabulate
+
+sys.path.append(os.path.dirname(__file__))
+import plugins
+from dump.match_infra import RedisSource, JsonSource
+from sonic_py_common import multi_asic
+from utilities_common.constants import DEFAULT_NAMESPACE
+
+# Autocompletion Helper
+def get_available_modules(ctx, args, incomplete):
+    return [k for k in plugins.dump_modules.keys() if incomplete in k]
+
+# Display Modules Callback
+def show_modules(ctx, param, value):
+    if not value or ctx.resilient_parsing:
+        return
+    header = ["Module", "Identifier"]
+    display = []
+    for mod in plugins.dump_modules:
+        display.append((mod, plugins.dump_modules[mod].ARG_NAME))
+    click.echo(tabulate(display, header))
+    ctx.exit()
+
+@click.group()
+def dump():
+    pass
+
+@dump.command()
+@click.pass_context
+@click.argument('module', required=True, type=str, autocompletion=get_available_modules)
+@click.argument('identifier', required=True, type=str) 
+@click.option('--show', '-s', is_flag=True, default=False, help='Display Modules Available', is_eager=True, expose_value=False, callback=show_modules)
+@click.option('--db', '-d', multiple=True, help='Only dump from these Databases or the CONFIG_FILE')
+@click.option('--table', '-t', is_flag=True, default=False, help='Print in tabular format', show_default=True)
+@click.option('--key-map', '-k', is_flag=True, default=False, help="Only fetch the keys matched, don't extract field-value dumps", show_default=True)
+@click.option('--verbose', '-v', is_flag=True, default=False, help="Prints any intermediate output to stdout useful for dev & troubleshooting", show_default=True)
+@click.option('--namespace', '-n', default=DEFAULT_NAMESPACE, type=str, show_default=True, help='Dump the redis-state for this namespace.')  
+def state(ctx, module, identifier, db, table, key_map, verbose, namespace):
+    """
+    Dump the current state of the identifier for the specified module from Redis DB or CONFIG_FILE
+    """
+    if not multi_asic.is_multi_asic() and namespace != DEFAULT_NAMESPACE:
+        click.echo("Namespace option is not valid for a single-ASIC device")
+        ctx.exit()
+
+    if multi_asic.is_multi_asic() and (namespace != DEFAULT_NAMESPACE and namespace not in multi_asic.get_namespace_list()):
+        click.echo("Namespace option is not valid. Choose one of {}".format(multi_asic.get_namespace_list()))
+        ctx.exit()
+
+    if module not in plugins.dump_modules:
+        click.echo("No Matching Plugin has been Implemented")
+        ctx.exit()
+
+    if verbose:
+        os.environ["VERBOSE"] = "1"
+    else:
+        os.environ["VERBOSE"] = "0"
+
+    ctx.module = module
+    obj = plugins.dump_modules[module]()
+
+    if identifier == "all":
+        ids = obj.get_all_args(namespace)
+    else:
+        ids = identifier.split(",")
+
+    params = {}
+    collected_info = {}
+    params['namespace'] = namespace
+    for arg in ids: 
+        params[plugins.dump_modules[module].ARG_NAME] = arg 
+        collected_info[arg] = obj.execute(params)
+
+    if len(db) > 0:
+        collected_info = filter_out_dbs(db, collected_info)
+
+    vidtorid = extract_rid(collected_info, namespace)
+
+    if not key_map:
+        collected_info = populate_fv(collected_info, module, namespace)
+
+    for id in vidtorid.keys():
+        collected_info[id]["ASIC_DB"]["vidtorid"] = vidtorid[id]
+
+    print_dump(collected_info, table, module, identifier, key_map)
+
+    return 
+
+def extract_rid(info, ns):
+    r = RedisSource()
+    r.connect("ASIC_DB", ns)
+    vidtorid = {}
+    for arg in info.keys():
+        mp = get_v_r_map(r, info[arg])
+        if mp:
+            vidtorid[arg] = mp
+    return vidtorid
+
+def get_v_r_map(r, single_dict):
+    v_r_map = {}
+    asic_obj_ptrn = "ASIC_STATE:.*:oid:0x\w{1,14}"
+
+    if "ASIC_DB" in single_dict and 'keys' in single_dict["ASIC_DB"]:
+        for redis_key in single_dict["ASIC_DB"]['keys']:
+            if re.match(asic_obj_ptrn, redis_key):
+                matches = re.findall(r"oid:0x\w{1,14}", redis_key)
+                if matches:
+                   vid = matches[0]
+                   v_r_map[vid] =  r.hget("ASIC_DB", "VIDTORID", vid)
+                   if not v_r_map[vid]:
+                       v_r_map[vid] = "Real ID Not Found"
+    return v_r_map
+
+# Filter dbs which are not required
+def filter_out_dbs(db_list, collected_info):
+    args_ = list(collected_info.keys())
+    for arg in args_:
+        dbs = list(collected_info[arg].keys())
+        for db in dbs:
+            if db not in db_list:
+                del collected_info[arg][db]
+    return collected_info
+
+def populate_fv(info, module, namespace):
+
+    all_dbs = set()
+    for id in info.keys():
+        for db_name in info[id].keys():
+            all_dbs.add(db_name)
+
+    db_dict = {}  
+    for db_name in all_dbs:
+        if db_name is "CONFIG_FILE":
+            db_dict[db_name] = JsonSource()
+            db_dict[db_name].connect(plugins.dump_modules[module].CONFIG_FILE, namespace)
+        else:
+            db_dict[db_name] = RedisSource()
+            db_dict[db_name].connect(db_name, namespace)
+
+    final_info = {}
+    for id in info.keys():
+        final_info[id] = {}
+        for db_name in info[id].keys():
+            final_info[id][db_name] = {}
+            final_info[id][db_name]["keys"] = []
+            final_info[id][db_name]["tables_not_found"] = info[id][db_name]["tables_not_found"]
+            for key in info[id][db_name]["keys"]:
+                final_info[id][db_name]["keys"].append({key : db_dict[db_name].get(db_name, key)})
+
+    return final_info
+
+def get_dict_str(key_obj):
+    table = []
+    for pair in key_obj.items():
+        table.append(list(pair))
+    return tabulate(table, headers=["field", "value"], tablefmt="psql")
+
+def get_keys(dump):
+    keys = []
+    for key_ in dump:
+        if isinstance(key_, dict) and key_:
+            keys.append(list(key_.keys())[0])
+        else:
+            keys.append(key_)
+    return keys
+
+# print dump
+def print_dump(collected_info, table, module, identifier, key_map):
+    if not table:
+        click.echo(json.dumps(collected_info, indent=4))
+        return
+
+    top_header = [plugins.dump_modules[module].ARG_NAME, "DB_NAME", "DUMP"]
+    final_collection = []
+    for ids in collected_info.keys():
+        for db in collected_info[ids].keys():
+            total_info = ""
+
+            if collected_info[ids][db]["tables_not_found"]:
+                tabulate_fmt = []
+                for tab in collected_info[ids][db]["tables_not_found"]:
+                    tabulate_fmt.append([tab])
+                total_info += tabulate(tabulate_fmt, ["Tables Not Found"], tablefmt="grid")
+                total_info += "\n"
+
+            if not key_map:
+                values = []
+                hdrs = ["Keys", "field-value pairs"]
+                for key_obj in collected_info[ids][db]["keys"]:
+                    if isinstance(key_obj, dict) and key_obj:
+                        key = list(key_obj.keys())[0]
+                        values.append([key, get_dict_str(key_obj[key])])
+                total_info += str(tabulate(values, hdrs, tablefmt="grid"))
+            else:
+                temp = []
+                for key_ in collected_info[ids][db]["keys"]:
+                    temp.append([key_])
+                total_info += str(tabulate(temp, headers=["Keys Collected"], tablefmt="grid"))
+
+            total_info += "\n"
+            if "vidtorid" in collected_info[ids][db]:
+                temp = []
+                for pair in collected_info[ids][db]["vidtorid"].items():
+                    temp.append(list(pair))
+                total_info +=str(tabulate(temp, headers=["vid", "rid"], tablefmt="grid"))
+            final_collection.append([ids, db, total_info])
+
+    click.echo(tabulate(final_collection, top_header, tablefmt="grid"))
+    return
+
+if __name__ == '__main__':
+    dump()

scripts/route_check.py

@@ -500,7 +500,7 @@ def main():
     parser.add_argument('-m', "--mode", type=Level, choices=list(Level), default='ERR')
     parser.add_argument("-i", "--interval", type=int, default=0, help="Scan interval in seconds")
     parser.add_argument("-s", "--log_to_syslog", action="store_true", default=True, help="Write message to syslog")
-    args = parser.parse_args()
+    args = parser.parse_known_args()[0] # https://docs.python.org/3/library/argparse.html#partial-parsing
 
     set_level(args.mode, args.log_to_syslog)
 

setup.py

@@ -5,9 +5,21 @@
 # under scripts/. Consider stop using scripts and use console_scripts instead
 #
 # https://stackoverflow.com/questions/18787036/difference-between-entry-points-console-scripts-and-scripts-in-setup-py
-import fastentrypoints
+import fastentrypoints, sys
 
 from setuptools import setup
+from setuptools.command.test import test as TestCommand
+
+class PyTest(TestCommand):
+    user_options = [("pytest-args=", "a", "Arguments to pass to pytest")]
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ""
+    def run_tests(self):
+        import shlex
+        import pytest
+        errno = pytest.main(shlex.split(self.pytest_args))
+        sys.exit(errno)
 
 setup(
     name='sonic-utilities',
@@ -31,6 +43,8 @@
         'crm',
         'debug',
         'generic_config_updater',
+        'dump',
+        'dump.plugins',
         'pfcwd',
         'sfputil',
         'ssdutil',
@@ -71,7 +85,8 @@
                   'filter_fdb_input/*',
                   'pfcwd_input/*',
                   'wm_input/*',
-                  'ecn_input/*']
+                  'ecn_input/*',
+                  'dump_input/*']
     },
     scripts=[
         'scripts/aclshow',
@@ -141,6 +156,7 @@
             'counterpoll = counterpoll.main:cli',
             'crm = crm.main:cli',
             'debug = debug.main:cli',
+            'dump = dump.main:dump',
             'filter_fdb_entries = fdbutil.filter_fdb_entries:main',
             'pfcwd = pfcwd.main:cli',
             'sfputil = sfputil.main:cli',
@@ -216,5 +232,6 @@
         'Topic :: Utilities',
     ],
     keywords='sonic SONiC utilities command line cli CLI',
+    cmdclass={"pytest": PyTest},
     test_suite='setup.get_test_suite'
 )

sonic-utilities-data/bash_completion.d/dump

@@ -0,0 +1,8 @@
+_dump_completion() {
+    COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \
+                   COMP_CWORD=$COMP_CWORD \
+                   _DUMP_COMPLETE=complete $1 ) )
+    return 0
+}
+
+complete -F _dump_completion -o default dump