VER 1.1 - Improved Ctrl-C handling, better logging, commenting out no…

…des support and more. - Overhauled README with feature list, a screenshot, and usage information - Better formatted logging output - Improved categorisation of log messages, so that a reasonable amount of info is printed in non-verbose mode - Added signal handler to improve responsiveness to Ctrl-C - Fixed verbose flag (-v) so you don't have to specify a value after it - Added a flag to specify a custom node list (-f) - Added quiet flag (-q) for minimal logging output - Improved description text in the built-in help (-h)
officiallymarky · Jan 16, 2019 · ffa3771 · ffa3771
1 parent aefd22c
commit ffa3771
Show file tree

Hide file tree

Showing 4 changed files with 117 additions and 49 deletions.
diff --git a/README.md b/README.md
@@ -1,22 +1,61 @@
 # Steem node RPC scanner
 
-by @someguy123
+by [@someguy123](https://steemit.com/@someguy123)
 
-Version 1.0
+![Screenshot of RPC Scanner](https://i.imgur.com/B9EShPn.png)
+
+A fast and easy to use Python script which scans [Steem](https://www.steem.io) RPC nodes
+asynchronously using request-threads and Twisted's Reactor.
+
+**Features:**
+
+ - Colorized output for easy reading
+ - Tests a node's reliability during data collection, with multiple retries on error
+ - Reports the average response time, and average amount of retries needed for basic calls
+ - Detects a node's Steem version
+ - Show the node's last block number and block time
+ - Can determine whether a node is using Jussi, or if it's a raw steemd node
+ - Can scan a list of 10 nodes in as little as 20 seconds thanks to Twisted Reactor + request-threads 
 
 Python 3.7.0 or higher recommended
 
 # Install
 
 ```
+git clone https://github.com/Someguy123/steem-rpc-scanner.git
+cd steem-rpc-scanner
 python3 -m venv venv
 source venv/bin/activate
 pip3 install -r requirements.txt
 cp nodes.txt.example nodes.txt
-# normal run
+```
+
+# Usage
+
+For most people, the defaults are fine, so you can simply run:
+
+```
 ./app.py
-# verbose output
-./app.py -v
+```
+
+Add or delete nodes from `nodes.txt` line-by-line as needed. You can comment out nodes by placing `#` at the start of the line.
+
+Format: `https://steemd.privex.io` - can also specify a port in standard url format, e.g. `https://gtg.steem.house:8090`
+
+
+Full usage information (for most up to date usage, use `./app.py --help`)
+
+```
+usage: app.py [-h] [-v] [-q] [-f NODEFILE]
+
+Scan RPC nodes from a list of URLs to determine their last block, version,
+reliability, and response time.
+
+optional arguments:
+  -h, --help   show this help message and exit
+  -v           display debugging
+  -q           only show warnings or worse
+  -f NODEFILE  specify a custom file to read nodes from (default: nodes.txt)
 ```
 
 # License

diff --git a/app.py b/app.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # Steem node RPC scanner
 # by @someguy123
-# version 1.0
+# version 1.1
 # Python 3.7.0 or higher recommended
 from twisted.internet import defer
 from twisted.internet.defer import inlineCallbacks
@@ -12,25 +12,45 @@
 from colorama import Fore, Back
 import asyncio
 import argparse
+from privex.loghelper import LogHelper
 import logging
+import signal
 
-parser = argparse.ArgumentParser(description='Scan RPC nodes in nodes.txt')
-parser.add_argument('-v', metavar='verbose', dest='verbose', type=bool, 
-                    default=False, help='display debugging')
+parser = argparse.ArgumentParser(description='Scan RPC nodes from a list of URLs to determine their last block, version, reliability, and response time.')
+parser.add_argument('-v', dest='verbose', action='store_true', default=False, help='display debugging')
+parser.add_argument('-q', dest='quiet', action='store_true', default=False, help='only show warnings or worse')
+parser.add_argument('-f', dest='nodefile', default='nodes.txt', help='specify a custom file to read nodes from (default: nodes.txt)')
+parser.set_defaults(verbose=False, quiet=False)
 args = parser.parse_args()
 
+debug_level = logging.INFO
+
 verbose = args.verbose
 if verbose:
-    logging.basicConfig(level=logging.DEBUG)
+    print('Verbose mode enabled.')
+    debug_level = logging.DEBUG
+elif args.quiet:
+    debug_level = logging.WARNING
+else:
+    print("For more verbose logging (such as detailed scanning actions), use `./app.py -v`")
+    print("For less output, use -q for quiet mode (display only warnings and errors)")
+
+f = logging.Formatter('[%(asctime)s]: %(funcName)-14s : %(levelname)-8s:: %(message)s')
+lh = LogHelper(handler_level=debug_level, formatter=f)
+lh.add_console_handler()
+log = lh.get_logger()
+
 # s = requests.Session()
-s = AsyncSession(n=20)
+s = AsyncSession(n=50)
 
 RPC_TIMEOUT = 5
 MAX_TRIES = 5
 # nodes to be specified line by line. format: http://gtg.steem.house:8090
-NODE_LIST_FILE = "nodes.txt"
-NODE_LIST = open(NODE_LIST_FILE, 'r').readlines()
+# NODE_LIST_FILE = "nodes.txt"
+NODE_LIST = open(args.nodefile, 'r').readlines()
 NODE_LIST = [n.strip() for n in NODE_LIST]
+# Allow nodes to be commented out with # symbol
+NODE_LIST = [n for n in NODE_LIST if n[0] != '#']
 node_status = {}
 
 class ServerDead(BaseException):
@@ -44,17 +64,17 @@ def tryNode(self, reactor, host, method, params=[]):
             tn = yield self._tryNode(host, method, params)
             return tn
         except Exception as e:
-            logging.debug('caught in tryNode and raised')
+            log.debug('caught in tryNode and raised')
             raise e
 
     @defer.inlineCallbacks
     def _tryNode(self, host, method, params=[], tries=0):
         if tries >= MAX_TRIES:
-            logging.debug('SERVER IS DEAD')
+            log.debug('SERVER IS DEAD')
             raise ServerDead('{} did not respond properly after {} tries'.format(host, tries))
 
         try:
-            logging.info('{} {} attempt {}'.format(host, method, tries))
+            log.debug('{} {} attempt {}'.format(host, method, tries))
             start = time.time()
             tries += 1
             res = yield _rpc(host, method, params)
@@ -65,12 +85,12 @@ def _tryNode(self, host, method, params=[], tries=0):
             # if we made it this far, we're fine :)
             success = True
             results = [res, runtime, tries]
-            logging.debug(Fore.GREEN + '[{}] Successful request for {}'.format(host, method) + Fore.RESET)
+            log.debug(Fore.GREEN + '[{}] Successful request for {}'.format(host, method) + Fore.RESET)
             return tuple(results)
         except Exception as e:
             if 'HTTPError' in str(type(e)) and '426 Client Error' in str(e):
                 raise ServerDead('Server {} only supports websockets'.format(host))
-            logging.info('%s [%s] %s attempt %d failed. Message: %s %s %s', Fore.RED, method, host, tries, type(e), str(e), Fore.RESET)
+            log.info('%s [%s] %s attempt %d failed. Message: %s %s %s', Fore.RED, method, host, tries, type(e), str(e), Fore.RESET)
             dl = yield deferLater(self.reactor, 10, self._tryNode, host, method, params, tries)
             return dl
 
@@ -81,16 +101,16 @@ def identJussi(self, reactor, host):
             tn = yield self._identJussi(host)
             return tn
         except Exception as e:
-            logging.debug('caught in identJussi and raised')
+            log.debug('caught in identJussi and raised')
             raise e
 
     @defer.inlineCallbacks
     def _identJussi(self, host, tries=0):
         if tries >= MAX_TRIES:
-            logging.debug('[identJussi] SERVER IS DEAD')
+            log.debug('[identJussi] SERVER IS DEAD')
             raise ServerDead('{} did not respond properly after {} tries'.format(host, tries))
         try:
-            logging.info('{} identJussi attempt {}'.format(host, tries))
+            log.debug('{} identJussi attempt {}'.format(host, tries))
             start = time.time()
             tries += 1
             res = yield s.get(host)
@@ -108,12 +128,12 @@ def _identJussi(self, host, tries=0):
             # if we made it this far, we're fine :)
             success = True
             results = [srvtype, runtime, tries]
-            logging.debug(Fore.GREEN + '[{}] Successful request for identJussi'.format(host) + Fore.RESET)
+            log.debug(Fore.GREEN + '[{}] Successful request for identJussi'.format(host) + Fore.RESET)
             return tuple(results)
         except Exception as e:
             if 'HTTPError' in str(type(e)) and '426 Client Error' in str(e):
                 raise ServerDead('Server {} only supports websockets'.format(host))
-            logging.info('%s [identJussi] %s attempt %d failed. Message: %s %s %s', Fore.RED, host, tries, type(e), str(e), Fore.RESET)
+            log.debug('%s [identJussi] %s attempt %d failed. Message: %s %s %s', Fore.RED, host, tries, type(e), str(e), Fore.RESET)
             dl = yield deferLater(self.reactor, 10, self._identJussi, host, tries)
             return dl
 
@@ -128,15 +148,15 @@ def rpc(reactor, host, method, params=[]):
     :raises: ServerDead - tried too many times and failed
     """
     # tries = 0
-    logging.info(Fore.BLUE + 'Attempting method {method} on server {host}. Will try {tries} times'.format(
+    log.debug(Fore.BLUE + 'Attempting method {method} on server {host}. Will try {tries} times'.format(
         tries=MAX_TRIES, host=host, method=method
     ) + Fore.RESET)
     # d = defer.Deferred()
     np = NodePlug()
     try:
         d = yield np.tryNode(reactor, host, method, params)
     except ServerDead as e:
-        logging.debug('caught in rpc and raised')
+        log.debug('caught in rpc and raised')
         raise e
 
     # return tuple(results)
@@ -150,15 +170,15 @@ def identifyNode(reactor, host):
     :raises: ServerDead - tried too many times and failed
     """
     # tries = 0
-    logging.info(Fore.BLUE + 'Attempting method identifyNode on server {host}. Will try {tries} times'.format(
+    log.debug(Fore.BLUE + 'Attempting method identifyNode on server {host}. Will try {tries} times'.format(
         tries=MAX_TRIES, host=host
     ) + Fore.RESET)
     # d = defer.Deferred()
     np = NodePlug()
     try:
         d = yield np.identJussi(reactor, host)
     except ServerDead as e:
-        logging.debug('caught in identifyNode and raised')
+        log.debug('caught in identifyNode and raised')
         raise e
 
     # return tuple(results)
@@ -198,41 +218,41 @@ def scan_nodes(reactor):
     up_nodes = []
     nodes = NODE_LIST
     print('Scanning nodes... Please wait...')
+    print('{}[Stage 1 / 4] Identifying node types (jussi/appbase){}'.format(Fore.GREEN, Fore.RESET))
     for node in nodes:
         node_status[node] = dict(
             raw={}, timing={}, tries={},
             current_block='error', block_time='error', version='error',
             srvtype='err'
             )
         ident_nodes.append((node, identifyNode(reactor, node)))
-        logging.info('Identifying ', node)
+        log.info('Identifying %s', node)
     req_success = 0
-    print('{}[Stage 1 / 4] Identifying node types (jussi/appbase){}'.format(Fore.GREEN, Fore.RESET))
 
     for host, id_data in ident_nodes:
         ns = node_status[host]        
         try:
             c = yield id_data
             ident, ident_time, ident_tries = c
-            logging.info(Fore.GREEN + 'Successfully obtained server type' + Fore.RESET)
+            log.info(Fore.GREEN + 'Successfully obtained server type for node %s' + Fore.RESET, host)
 
             ns['srvtype'] = ident
             ns['timing']['ident'] = ident_time
             ns['tries']['ident'] = ident_tries
             if ns['srvtype'] == 'jussi':
-                logging.warning('Server {} is JUSSI'.format(host))
+                log.info('Server {} is JUSSI'.format(host))
                 up_nodes.append((host, ns['srvtype'], rpc(reactor, host, 'get_dynamic_global_properties')))
             if ns['srvtype'] == 'appbase':
-                logging.warning('Server {} is APPBASE (no jussi)'.format(host))
+                log.info('Server {} is APPBASE (no jussi)'.format(host))
                 up_nodes.append((host, ns['srvtype'], rpc(reactor, host, 'condenser_api.get_dynamic_global_properties')))
             req_success += 1
         except ServerDead as e:
-            logging.error(Fore.RED + '[ident jussi]' + str(e) + Fore.RESET)
+            log.error(Fore.RED + '[ident jussi]' + str(e) + Fore.RESET)
             if "only supports websockets" in str(e):
                 ns['err_reason'] = 'WS Only'
         except Exception as e:
-            logging.warning(Fore.RED + 'Unknown error occurred (ident jussi)...' + Fore.RESET)
-            logging.warning('[%s] %s', type(e), str(e))
+            log.warning(Fore.RED + 'Unknown error occurred (ident jussi)...' + Fore.RESET)
+            log.warning('[%s] %s', type(e), str(e))
 
     print('{}[Stage 2 / 4] Filtering out bad nodes{}'.format(Fore.GREEN, Fore.RESET))    
     for host, srvtype, blkdata in up_nodes:
@@ -246,13 +266,14 @@ def scan_nodes(reactor):
             if srvtype == 'appbase':
                 conf_nodes.append((host, rpc(reactor, host, 'condenser_api.get_config')))
                 prop_nodes.append((host, rpc(reactor, host, 'condenser_api.get_dynamic_global_properties')))
+            log.info(Fore.GREEN + 'Node %s seems fine' + Fore.RESET, host)
         except ServerDead as e:
-            logging.error(Fore.RED + '[badnodefilter]' + str(e) + Fore.RESET)
+            log.error(Fore.RED + '[badnodefilter]' + str(e) + Fore.RESET)
             if "only supports websockets" in str(e):
                 ns['err_reason'] = 'WS Only'
         except Exception as e:
-            logging.warning(Fore.RED + 'Unknown error occurred (badnodefilter)...' + Fore.RESET)
-            logging.warning('[%s] %s', type(e), str(e))
+            log.warning(Fore.RED + 'Unknown error occurred (badnodefilter)...' + Fore.RESET)
+            log.warning('[%s] %s', type(e), str(e))
 
     print('{}[Stage 3 / 4] Obtaining steemd versions {}'.format(Fore.GREEN, Fore.RESET))    
     for host, cfdata in conf_nodes:
@@ -261,20 +282,20 @@ def scan_nodes(reactor):
             # config, config_time, config_tries = rpc(node, 'get_config')
             c = yield cfdata
             config, config_time, config_tries = c
-            logging.info(Fore.GREEN + 'Successfully obtained config' + Fore.RESET)
+            log.info(Fore.GREEN + 'Successfully obtained config for node %s' + Fore.RESET, host)
 
             ns['raw']['config'] = config
             ns['timing']['config'] = config_time
             ns['tries']['config'] = config_tries
             ns['version'] = config.get('STEEM_BLOCKCHAIN_VERSION', config.get('STEEMIT_BLOCKCHAIN_VERSION', 'Unknown'))
             req_success += 1
         except ServerDead as e:
-            logging.error(Fore.RED + '[load config]' + str(e) + Fore.RESET)
+            log.error(Fore.RED + '[load config]' + str(e) + Fore.RESET)
             if "only supports websockets" in str(e):
                 ns['err_reason'] = 'WS Only'
         except Exception as e:
-            logging.warning(Fore.RED + 'Unknown error occurred (conf)...' + Fore.RESET)
-            logging.warning('[%s] %s', type(e), str(e))
+            log.warning(Fore.RED + 'Unknown error occurred (conf)...' + Fore.RESET)
+            log.warning('[%s] %s', type(e), str(e))
 
     print('{}[Stage 4 / 4] Checking current block / block time{}'.format(Fore.GREEN, Fore.RESET))    
     for host, prdata in prop_nodes:
@@ -283,7 +304,7 @@ def scan_nodes(reactor):
             # head_block_number
             # time (UTC)
             props, props_time, props_tries = yield prdata
-            logging.debug(Fore.GREEN + 'Successfully obtained props' + Fore.RESET)
+            log.debug(Fore.GREEN + 'Successfully obtained props' + Fore.RESET)
             ns['raw']['props'] = props
             ns['timing']['props'] = props_time
             ns['tries']['props'] = props_tries
@@ -292,13 +313,13 @@ def scan_nodes(reactor):
             req_success += 1
 
         except ServerDead as e:
-            logging.error(Fore.RED + '[load props]' + str(e) + Fore.RESET)
-            # logging.error(str(e))
+            log.error(Fore.RED + '[load props]' + str(e) + Fore.RESET)
+            # log.error(str(e))
             if "only supports websockets" in str(e):
                 ns['err_reason'] = 'WS Only'
         except Exception as e:
-            logging.warning(Fore.RED + 'Unknown error occurred (prop)...' + Fore.RESET)
-            logging.warning('[%s] %s', type(e), str(e))
+            log.warning(Fore.RED + 'Unknown error occurred (prop)...' + Fore.RESET)
+            log.warning('[%s] %s', type(e), str(e))
 
     print_nodes(node_status)
 
@@ -351,4 +372,7 @@ def print_nodes(list_nodes):
         ), Fore.RESET)
 
 if __name__ == "__main__":
+    # Make CTRL-C work properly with Twisted's Reactor
+    # https://stackoverflow.com/a/4126412/2648583
+    signal.signal(signal.SIGINT, signal.default_int_handler)
     react(scan_nodes)
diff --git a/nodes.txt.example b/nodes.txt.example
@@ -1,8 +1,12 @@
 https://steemd.privex.io
+https://direct.steemd.privex.io
 https://api.steemit.com
+https://steemd-appbase.steemit.com
+https://steemd.steemitstage.com
 https://rpc.buildteam.io
 https://gtg.steem.house:8090
 https://rpc.steemviz.com
 https://rpc.steemliberator.com
-https://rpc.curiesteem.com
-https://steemd.minnowsupportproject.org
+#https://rpc.curiesteem.com
+https://steemd.minnowsupportproject.org
+http://steemseed-fin.privex.io:8091
diff --git a/requirements.txt b/requirements.txt
@@ -14,3 +14,4 @@ six==1.11.0
 Twisted==18.7.0
 urllib3==1.23
 zope.interface==4.5.0
+privex-loghelper