From ffa3771958dfea8800afd251b3b2460badbb7bb9 Mon Sep 17 00:00:00 2001 From: "Chris (Someguy123)" Date: Wed, 16 Jan 2019 19:30:04 +0000 Subject: [PATCH] VER 1.1 - Improved Ctrl-C handling, better logging, commenting out nodes support and more. - Overhauled README with feature list, a screenshot, and usage information - Better formatted logging output - Improved categorisation of log messages, so that a reasonable amount of info is printed in non-verbose mode - Added signal handler to improve responsiveness to Ctrl-C - Fixed verbose flag (-v) so you don't have to specify a value after it - Added a flag to specify a custom node list (-f) - Added quiet flag (-q) for minimal logging output - Improved description text in the built-in help (-h) --- README.md | 49 ++++++++++++++++++--- app.py | 108 ++++++++++++++++++++++++++++------------------ nodes.txt.example | 8 +++- requirements.txt | 1 + 4 files changed, 117 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 0add918..38b4890 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,61 @@ # Steem node RPC scanner -by @someguy123 +by [@someguy123](https://steemit.com/@someguy123) -Version 1.0 +![Screenshot of RPC Scanner](https://i.imgur.com/B9EShPn.png) + +A fast and easy to use Python script which scans [Steem](https://www.steem.io) RPC nodes +asynchronously using request-threads and Twisted's Reactor. + +**Features:** + + - Colorized output for easy reading + - Tests a node's reliability during data collection, with multiple retries on error + - Reports the average response time, and average amount of retries needed for basic calls + - Detects a node's Steem version + - Show the node's last block number and block time + - Can determine whether a node is using Jussi, or if it's a raw steemd node + - Can scan a list of 10 nodes in as little as 20 seconds thanks to Twisted Reactor + request-threads Python 3.7.0 or higher recommended # Install ``` +git clone https://github.com/Someguy123/steem-rpc-scanner.git +cd steem-rpc-scanner python3 -m venv venv source venv/bin/activate pip3 install -r requirements.txt cp nodes.txt.example nodes.txt -# normal run +``` + +# Usage + +For most people, the defaults are fine, so you can simply run: + +``` ./app.py -# verbose output -./app.py -v +``` + +Add or delete nodes from `nodes.txt` line-by-line as needed. You can comment out nodes by placing `#` at the start of the line. + +Format: `https://steemd.privex.io` - can also specify a port in standard url format, e.g. `https://gtg.steem.house:8090` + + +Full usage information (for most up to date usage, use `./app.py --help`) + +``` +usage: app.py [-h] [-v] [-q] [-f NODEFILE] + +Scan RPC nodes from a list of URLs to determine their last block, version, +reliability, and response time. + +optional arguments: + -h, --help show this help message and exit + -v display debugging + -q only show warnings or worse + -f NODEFILE specify a custom file to read nodes from (default: nodes.txt) ``` # License diff --git a/app.py b/app.py index ef0a313..010d88d 100755 --- a/app.py +++ b/app.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Steem node RPC scanner # by @someguy123 -# version 1.0 +# version 1.1 # Python 3.7.0 or higher recommended from twisted.internet import defer from twisted.internet.defer import inlineCallbacks @@ -12,25 +12,45 @@ from colorama import Fore, Back import asyncio import argparse +from privex.loghelper import LogHelper import logging +import signal -parser = argparse.ArgumentParser(description='Scan RPC nodes in nodes.txt') -parser.add_argument('-v', metavar='verbose', dest='verbose', type=bool, - default=False, help='display debugging') +parser = argparse.ArgumentParser(description='Scan RPC nodes from a list of URLs to determine their last block, version, reliability, and response time.') +parser.add_argument('-v', dest='verbose', action='store_true', default=False, help='display debugging') +parser.add_argument('-q', dest='quiet', action='store_true', default=False, help='only show warnings or worse') +parser.add_argument('-f', dest='nodefile', default='nodes.txt', help='specify a custom file to read nodes from (default: nodes.txt)') +parser.set_defaults(verbose=False, quiet=False) args = parser.parse_args() +debug_level = logging.INFO + verbose = args.verbose if verbose: - logging.basicConfig(level=logging.DEBUG) + print('Verbose mode enabled.') + debug_level = logging.DEBUG +elif args.quiet: + debug_level = logging.WARNING +else: + print("For more verbose logging (such as detailed scanning actions), use `./app.py -v`") + print("For less output, use -q for quiet mode (display only warnings and errors)") + +f = logging.Formatter('[%(asctime)s]: %(funcName)-14s : %(levelname)-8s:: %(message)s') +lh = LogHelper(handler_level=debug_level, formatter=f) +lh.add_console_handler() +log = lh.get_logger() + # s = requests.Session() -s = AsyncSession(n=20) +s = AsyncSession(n=50) RPC_TIMEOUT = 5 MAX_TRIES = 5 # nodes to be specified line by line. format: http://gtg.steem.house:8090 -NODE_LIST_FILE = "nodes.txt" -NODE_LIST = open(NODE_LIST_FILE, 'r').readlines() +# NODE_LIST_FILE = "nodes.txt" +NODE_LIST = open(args.nodefile, 'r').readlines() NODE_LIST = [n.strip() for n in NODE_LIST] +# Allow nodes to be commented out with # symbol +NODE_LIST = [n for n in NODE_LIST if n[0] != '#'] node_status = {} class ServerDead(BaseException): @@ -44,17 +64,17 @@ def tryNode(self, reactor, host, method, params=[]): tn = yield self._tryNode(host, method, params) return tn except Exception as e: - logging.debug('caught in tryNode and raised') + log.debug('caught in tryNode and raised') raise e @defer.inlineCallbacks def _tryNode(self, host, method, params=[], tries=0): if tries >= MAX_TRIES: - logging.debug('SERVER IS DEAD') + log.debug('SERVER IS DEAD') raise ServerDead('{} did not respond properly after {} tries'.format(host, tries)) try: - logging.info('{} {} attempt {}'.format(host, method, tries)) + log.debug('{} {} attempt {}'.format(host, method, tries)) start = time.time() tries += 1 res = yield _rpc(host, method, params) @@ -65,12 +85,12 @@ def _tryNode(self, host, method, params=[], tries=0): # if we made it this far, we're fine :) success = True results = [res, runtime, tries] - logging.debug(Fore.GREEN + '[{}] Successful request for {}'.format(host, method) + Fore.RESET) + log.debug(Fore.GREEN + '[{}] Successful request for {}'.format(host, method) + Fore.RESET) return tuple(results) except Exception as e: if 'HTTPError' in str(type(e)) and '426 Client Error' in str(e): raise ServerDead('Server {} only supports websockets'.format(host)) - logging.info('%s [%s] %s attempt %d failed. Message: %s %s %s', Fore.RED, method, host, tries, type(e), str(e), Fore.RESET) + log.info('%s [%s] %s attempt %d failed. Message: %s %s %s', Fore.RED, method, host, tries, type(e), str(e), Fore.RESET) dl = yield deferLater(self.reactor, 10, self._tryNode, host, method, params, tries) return dl @@ -81,16 +101,16 @@ def identJussi(self, reactor, host): tn = yield self._identJussi(host) return tn except Exception as e: - logging.debug('caught in identJussi and raised') + log.debug('caught in identJussi and raised') raise e @defer.inlineCallbacks def _identJussi(self, host, tries=0): if tries >= MAX_TRIES: - logging.debug('[identJussi] SERVER IS DEAD') + log.debug('[identJussi] SERVER IS DEAD') raise ServerDead('{} did not respond properly after {} tries'.format(host, tries)) try: - logging.info('{} identJussi attempt {}'.format(host, tries)) + log.debug('{} identJussi attempt {}'.format(host, tries)) start = time.time() tries += 1 res = yield s.get(host) @@ -108,12 +128,12 @@ def _identJussi(self, host, tries=0): # if we made it this far, we're fine :) success = True results = [srvtype, runtime, tries] - logging.debug(Fore.GREEN + '[{}] Successful request for identJussi'.format(host) + Fore.RESET) + log.debug(Fore.GREEN + '[{}] Successful request for identJussi'.format(host) + Fore.RESET) return tuple(results) except Exception as e: if 'HTTPError' in str(type(e)) and '426 Client Error' in str(e): raise ServerDead('Server {} only supports websockets'.format(host)) - logging.info('%s [identJussi] %s attempt %d failed. Message: %s %s %s', Fore.RED, host, tries, type(e), str(e), Fore.RESET) + log.debug('%s [identJussi] %s attempt %d failed. Message: %s %s %s', Fore.RED, host, tries, type(e), str(e), Fore.RESET) dl = yield deferLater(self.reactor, 10, self._identJussi, host, tries) return dl @@ -128,7 +148,7 @@ def rpc(reactor, host, method, params=[]): :raises: ServerDead - tried too many times and failed """ # tries = 0 - logging.info(Fore.BLUE + 'Attempting method {method} on server {host}. Will try {tries} times'.format( + log.debug(Fore.BLUE + 'Attempting method {method} on server {host}. Will try {tries} times'.format( tries=MAX_TRIES, host=host, method=method ) + Fore.RESET) # d = defer.Deferred() @@ -136,7 +156,7 @@ def rpc(reactor, host, method, params=[]): try: d = yield np.tryNode(reactor, host, method, params) except ServerDead as e: - logging.debug('caught in rpc and raised') + log.debug('caught in rpc and raised') raise e # return tuple(results) @@ -150,7 +170,7 @@ def identifyNode(reactor, host): :raises: ServerDead - tried too many times and failed """ # tries = 0 - logging.info(Fore.BLUE + 'Attempting method identifyNode on server {host}. Will try {tries} times'.format( + log.debug(Fore.BLUE + 'Attempting method identifyNode on server {host}. Will try {tries} times'.format( tries=MAX_TRIES, host=host ) + Fore.RESET) # d = defer.Deferred() @@ -158,7 +178,7 @@ def identifyNode(reactor, host): try: d = yield np.identJussi(reactor, host) except ServerDead as e: - logging.debug('caught in identifyNode and raised') + log.debug('caught in identifyNode and raised') raise e # return tuple(results) @@ -198,6 +218,7 @@ def scan_nodes(reactor): up_nodes = [] nodes = NODE_LIST print('Scanning nodes... Please wait...') + print('{}[Stage 1 / 4] Identifying node types (jussi/appbase){}'.format(Fore.GREEN, Fore.RESET)) for node in nodes: node_status[node] = dict( raw={}, timing={}, tries={}, @@ -205,34 +226,33 @@ def scan_nodes(reactor): srvtype='err' ) ident_nodes.append((node, identifyNode(reactor, node))) - logging.info('Identifying ', node) + log.info('Identifying %s', node) req_success = 0 - print('{}[Stage 1 / 4] Identifying node types (jussi/appbase){}'.format(Fore.GREEN, Fore.RESET)) for host, id_data in ident_nodes: ns = node_status[host] try: c = yield id_data ident, ident_time, ident_tries = c - logging.info(Fore.GREEN + 'Successfully obtained server type' + Fore.RESET) + log.info(Fore.GREEN + 'Successfully obtained server type for node %s' + Fore.RESET, host) ns['srvtype'] = ident ns['timing']['ident'] = ident_time ns['tries']['ident'] = ident_tries if ns['srvtype'] == 'jussi': - logging.warning('Server {} is JUSSI'.format(host)) + log.info('Server {} is JUSSI'.format(host)) up_nodes.append((host, ns['srvtype'], rpc(reactor, host, 'get_dynamic_global_properties'))) if ns['srvtype'] == 'appbase': - logging.warning('Server {} is APPBASE (no jussi)'.format(host)) + log.info('Server {} is APPBASE (no jussi)'.format(host)) up_nodes.append((host, ns['srvtype'], rpc(reactor, host, 'condenser_api.get_dynamic_global_properties'))) req_success += 1 except ServerDead as e: - logging.error(Fore.RED + '[ident jussi]' + str(e) + Fore.RESET) + log.error(Fore.RED + '[ident jussi]' + str(e) + Fore.RESET) if "only supports websockets" in str(e): ns['err_reason'] = 'WS Only' except Exception as e: - logging.warning(Fore.RED + 'Unknown error occurred (ident jussi)...' + Fore.RESET) - logging.warning('[%s] %s', type(e), str(e)) + log.warning(Fore.RED + 'Unknown error occurred (ident jussi)...' + Fore.RESET) + log.warning('[%s] %s', type(e), str(e)) print('{}[Stage 2 / 4] Filtering out bad nodes{}'.format(Fore.GREEN, Fore.RESET)) for host, srvtype, blkdata in up_nodes: @@ -246,13 +266,14 @@ def scan_nodes(reactor): if srvtype == 'appbase': conf_nodes.append((host, rpc(reactor, host, 'condenser_api.get_config'))) prop_nodes.append((host, rpc(reactor, host, 'condenser_api.get_dynamic_global_properties'))) + log.info(Fore.GREEN + 'Node %s seems fine' + Fore.RESET, host) except ServerDead as e: - logging.error(Fore.RED + '[badnodefilter]' + str(e) + Fore.RESET) + log.error(Fore.RED + '[badnodefilter]' + str(e) + Fore.RESET) if "only supports websockets" in str(e): ns['err_reason'] = 'WS Only' except Exception as e: - logging.warning(Fore.RED + 'Unknown error occurred (badnodefilter)...' + Fore.RESET) - logging.warning('[%s] %s', type(e), str(e)) + log.warning(Fore.RED + 'Unknown error occurred (badnodefilter)...' + Fore.RESET) + log.warning('[%s] %s', type(e), str(e)) print('{}[Stage 3 / 4] Obtaining steemd versions {}'.format(Fore.GREEN, Fore.RESET)) for host, cfdata in conf_nodes: @@ -261,7 +282,7 @@ def scan_nodes(reactor): # config, config_time, config_tries = rpc(node, 'get_config') c = yield cfdata config, config_time, config_tries = c - logging.info(Fore.GREEN + 'Successfully obtained config' + Fore.RESET) + log.info(Fore.GREEN + 'Successfully obtained config for node %s' + Fore.RESET, host) ns['raw']['config'] = config ns['timing']['config'] = config_time @@ -269,12 +290,12 @@ def scan_nodes(reactor): ns['version'] = config.get('STEEM_BLOCKCHAIN_VERSION', config.get('STEEMIT_BLOCKCHAIN_VERSION', 'Unknown')) req_success += 1 except ServerDead as e: - logging.error(Fore.RED + '[load config]' + str(e) + Fore.RESET) + log.error(Fore.RED + '[load config]' + str(e) + Fore.RESET) if "only supports websockets" in str(e): ns['err_reason'] = 'WS Only' except Exception as e: - logging.warning(Fore.RED + 'Unknown error occurred (conf)...' + Fore.RESET) - logging.warning('[%s] %s', type(e), str(e)) + log.warning(Fore.RED + 'Unknown error occurred (conf)...' + Fore.RESET) + log.warning('[%s] %s', type(e), str(e)) print('{}[Stage 4 / 4] Checking current block / block time{}'.format(Fore.GREEN, Fore.RESET)) for host, prdata in prop_nodes: @@ -283,7 +304,7 @@ def scan_nodes(reactor): # head_block_number # time (UTC) props, props_time, props_tries = yield prdata - logging.debug(Fore.GREEN + 'Successfully obtained props' + Fore.RESET) + log.debug(Fore.GREEN + 'Successfully obtained props' + Fore.RESET) ns['raw']['props'] = props ns['timing']['props'] = props_time ns['tries']['props'] = props_tries @@ -292,13 +313,13 @@ def scan_nodes(reactor): req_success += 1 except ServerDead as e: - logging.error(Fore.RED + '[load props]' + str(e) + Fore.RESET) - # logging.error(str(e)) + log.error(Fore.RED + '[load props]' + str(e) + Fore.RESET) + # log.error(str(e)) if "only supports websockets" in str(e): ns['err_reason'] = 'WS Only' except Exception as e: - logging.warning(Fore.RED + 'Unknown error occurred (prop)...' + Fore.RESET) - logging.warning('[%s] %s', type(e), str(e)) + log.warning(Fore.RED + 'Unknown error occurred (prop)...' + Fore.RESET) + log.warning('[%s] %s', type(e), str(e)) print_nodes(node_status) @@ -351,4 +372,7 @@ def print_nodes(list_nodes): ), Fore.RESET) if __name__ == "__main__": + # Make CTRL-C work properly with Twisted's Reactor + # https://stackoverflow.com/a/4126412/2648583 + signal.signal(signal.SIGINT, signal.default_int_handler) react(scan_nodes) diff --git a/nodes.txt.example b/nodes.txt.example index 1f60073..632c9dc 100644 --- a/nodes.txt.example +++ b/nodes.txt.example @@ -1,8 +1,12 @@ https://steemd.privex.io +https://direct.steemd.privex.io https://api.steemit.com +https://steemd-appbase.steemit.com +https://steemd.steemitstage.com https://rpc.buildteam.io https://gtg.steem.house:8090 https://rpc.steemviz.com https://rpc.steemliberator.com -https://rpc.curiesteem.com -https://steemd.minnowsupportproject.org \ No newline at end of file +#https://rpc.curiesteem.com +https://steemd.minnowsupportproject.org +http://steemseed-fin.privex.io:8091 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 637fd6c..4958797 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ six==1.11.0 Twisted==18.7.0 urllib3==1.23 zope.interface==4.5.0 +privex-loghelper