From 725ed3dbfd94f1a5a81eefdcf66eca221fc0a751 Mon Sep 17 00:00:00 2001 From: "Chris (Someguy123)" Date: Wed, 3 Jul 2019 20:54:55 +0100 Subject: [PATCH] v1.5 - new health.py tool, for generating easily parsed lists of working nodes, or scanning individual nodes with unix return codes --- app.py | 7 +- health.py | 209 +++++++++++++++++++++++++++++++++++++++ requirements.txt | 3 +- rpcscanner/RPCScanner.py | 108 ++++++++++++++++---- rpcscanner/core.py | 13 ++- 5 files changed, 316 insertions(+), 24 deletions(-) create mode 100755 health.py diff --git a/app.py b/app.py index 5a930e0..c50cde6 100755 --- a/app.py +++ b/app.py @@ -8,12 +8,13 @@ Python 3.7.0 or higher recommended """ +from os.path import join from twisted.internet.defer import inlineCallbacks from twisted.internet.task import react from privex.loghelper import LogHelper from privex.helpers import ErrHelpParser -from rpcscanner import RPCScanner, settings +from rpcscanner import RPCScanner, settings, BASE_DIR, load_nodes import logging import signal @@ -55,8 +56,10 @@ @inlineCallbacks def scan(reactor): - rs = RPCScanner(reactor) + node_list = load_nodes(settings.node_file) + rs = RPCScanner(reactor, nodes=node_list) yield from rs.scan_nodes() + rs.print_nodes() if __name__ == "__main__": diff --git a/health.py b/health.py new file mode 100755 index 0000000..d416121 --- /dev/null +++ b/health.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +""" +Health check script - test individual nodes, with system return codes based on status. + +Designed for use in bash scripts +""" +import argparse +import sys +import textwrap +import signal +from datetime import datetime +from typing import Tuple + +from twisted.internet.defer import inlineCallbacks +from twisted.internet.task import react +from privex.helpers import ErrHelpParser, empty + +from rpcscanner import load_nodes, settings, RPCScanner +from rpcscanner.RPCScanner import NodeStatus + +MAX_SCORE = 20 +settings.plugins = True +settings.quiet = True + +help_text = textwrap.dedent('''\ + + This health check script has two modes: + + scan [node] - Scan an individual node, and return exit 0 if it's working, or 1 if it's not. + list [-d] - Return a list of working nodes from nodes.txt (-d returns more detailed status info, + which is whitespace separated) + + Examples: + + Scan Privex's RPC node, which will return 0 or 1, and output a small message detailing the status. + + $ ./health.py scan "https://direct.steemd.privex.io" + Node: https://direct.steemd.privex.io + Status: GOOD + Version: 0.20.11 + Block: 34343750 + Time: 2019-07-03T17:01:24 + Plugins: 4 / 4 + + Scan all nodes in nodes.txt and output the working ones in a plain list. + + $ ./health.py list + https://direct.steemd.privex.io + https://api.steemit.com + + Scan all nodes in nodes.txt and output the working ones in a plain list, with details sep by whitepsace. + + $ ./health.py list + Node Status Score Version Block Time Plugins + https://direct.steemd.privex.io GOOD 20 0.20.11 34343750 2019-07-03T17:01:24 4/4 + https://api.steemit.com GOOD 18 0.20.11 34343750 2019-07-03T17:01:24 4/4 + +''') + +parser = ErrHelpParser( + description="Someguy123's Steem Node Health Checker", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=help_text +) + +parser.add_argument('-s', dest='min_score', type=int, default=MAX_SCORE - 5, + help=f'Minimum score required before assuming a node is good (1 to {MAX_SCORE})') + +subparser = parser.add_subparsers() + + +def scan(opt): + if 1 > opt.min_score > MAX_SCORE: + return parser.error(f'Minimum score must be between 1 and {MAX_SCORE}') + # Make CTRL-C work properly with Twisted's Reactor + # https://stackoverflow.com/a/4126412/2648583 + signal.signal(signal.SIGINT, signal.default_int_handler) + react(_scan, (opt.node, opt.min_score,)) + + +def list_nodes(opt): + if 1 > opt.min_score > MAX_SCORE: + return parser.error(f'Minimum score must be between 1 and {MAX_SCORE}') + # Make CTRL-C work properly with Twisted's Reactor + # https://stackoverflow.com/a/4126412/2648583 + signal.signal(signal.SIGINT, signal.default_int_handler) + react(_list_nodes, (opt.detailed, opt.min_score,)) + + +p_scan = subparser.add_parser('scan', description='Scan an individual node') +p_scan.set_defaults(func=scan) +p_scan.add_argument('node', help='Steem Node with http(s):// prefix') + + +p_list = subparser.add_parser('list', description='Scan and output a plain text list of working nodes') +p_list.add_argument('-d', dest='detailed', action='store_true', default=False, + help='Return whitespace separated status information after the nodes in the list.') +p_list.set_defaults(func=list_nodes, detailed=False) + +args = parser.parse_args() + + +def iso_timestr(dt: datetime) -> str: + """Convert datetime object into ISO friendly ``2010-03-03Z21:16:45``""" + return str(dt.isoformat()).split('.')[0] + + +@inlineCallbacks +def _list_nodes(reactor, detailed, min_score): + node_list = load_nodes(settings.node_file) + rs = RPCScanner(reactor, nodes=node_list) + yield from rs.scan_nodes(True) + + if detailed: + print('(Detailed Mode. This msg and row header are sent to stderr for easy removal)', file=sys.stderr) + print('Node Status Score Version Block Time Plugins', file=sys.stderr) + for n in rs.node_objs: + score, _, status_name = score_node(min_score, n) + if score < min_score: + continue + + if detailed: + p_tr, p_tot = n.plugin_counts + dt = iso_timestr(n.block_time) if not empty(n.block_time) else 'Unknown' + print(f'{n.host} {status_name} {score} {n.version} {n.current_block} {dt} {p_tr}/{p_tot} ') + continue + print(n.host) + + +@inlineCallbacks +def _scan(reactor, node, min_score): + rs = RPCScanner(reactor, nodes=[node]) + yield from rs.scan_nodes(True) + + n = rs.get_node(node) + plug_tried, plug_total = n.plugin_counts + + score, return_code, status_name = score_node(min_score, n) + + if score == 0: + print("Node: {}\nStatus: DEAD".format(node)) + return sys.exit(1) + dt = iso_timestr(n.block_time) if not empty(n.block_time) else 'Unknown' + print(f""" +Node: {node} +Status: {status_name} +Version: {n.version} +Block: {n.current_block} +Time: {dt} +Plugins: {plug_tried} / {plug_total} +Retries: {n.total_tries} +Score: {score} (out of {MAX_SCORE}) +""") + return sys.exit(return_code) + + +def score_node(min_score: int, n: NodeStatus) -> Tuple[int, int, str]: + """ + Reviews the status information from a :py:class:`NodeStatus` object, and returns a numeric score, return code, + and human friendly status description as a tuple. + + Usage: + + >>> node = 'https://steemd.privex.io' + >>> rs = RPCScanner(reactor, nodes=[node]) + >>> n = rs.get_node(node) + >>> score, return_code, status_name = score_node(15, n) + >>> print(score, return_code, status_name) + 18 0 GOOD + + :param int min_score: Minimum score before a node is deemed "bad" + :param NodeStatus n: The NodeStatus object to use for scoring + :return tuple statusinfo: (score:int, return_code:int, status_name:str) + """ + # A node scores points based on whether it appears to be up, how many tries it took, and how many plugins work. + score, status = 0, n.status + plug_tried, plug_total = n.plugin_counts + + # If a node has a status of "dead", it immediately scores zero points and returns a bad status. + if status <= 0: + return 0, 1, 'DEAD' + elif status == 1: # Unstable nodes lose 5 points + score += MAX_SCORE - 5 + elif status >= 2: # Stable nodes start with full points + score += MAX_SCORE + + # Nodes lose half a score point for every retry needed + if n.total_tries > 0: score -= (n.total_tries / 2) + + # Nodes lose 2 points for each plugin that's responding incorrectly + if plug_tried < plug_total: score -= (plug_total - plug_tried) * 2 + + score = int(score) + return_code = 1 if score < min_score else 0 + status_name = 'BAD' if score < min_score else 'GOOD' + status_name = 'PERFECT' if score >= MAX_SCORE else status_name + return score, return_code, status_name + + +if __name__ == "__main__": + # Resolves the error "'Namespace' object has no attribute 'func' + # Taken from https://stackoverflow.com/a/54161510/2648583 + try: + func = args.func + func(args) + except AttributeError: + parser.error('Too few arguments') + +# parser.add_argument('node', help='') diff --git a/requirements.txt b/requirements.txt index 833d0d1..e0eaf0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,5 @@ Twisted>=18.7.0 urllib3==1.23 zope.interface>=4.5.0 privex-loghelper -privex-helpers>=1.1.0 \ No newline at end of file +privex-helpers>=1.1.0 +python-dateutil \ No newline at end of file diff --git a/rpcscanner/RPCScanner.py b/rpcscanner/RPCScanner.py index 0c4be43..43cfadd 100644 --- a/rpcscanner/RPCScanner.py +++ b/rpcscanner/RPCScanner.py @@ -1,11 +1,15 @@ -from os.path import join +from dataclasses import dataclass, field +from datetime import datetime +from typing import List, Tuple import twisted.internet.reactor import logging from colorama import Fore +from dateutil.parser import parse +from privex.helpers import empty from twisted.internet.defer import inlineCallbacks from rpcscanner.MethodTests import MethodTests -from rpcscanner.core import TEST_PLUGINS_LIST, BASE_DIR +from rpcscanner.core import TEST_PLUGINS_LIST from rpcscanner.rpc import rpc, identify_node from rpcscanner.exceptions import ServerDead from rpcscanner import settings @@ -13,29 +17,83 @@ log = logging.getLogger(__name__) +@dataclass +class NodeStatus: + host: str + raw: dict + timing: dict + tries: dict + plugins: list + err_reason: str = None + srvtype: str = 'Unknown' + current_block: int = None + block_time: datetime = None + version: str = None + + _statuses = { + 0: "Dead", + 1: "Unstable", + 2: "Online", + } + + @property + def status(self) -> int: + """Status of the node as a number from 0 to 2""" + return len(self.raw) + + @property + def status_human(self) -> str: + """Status of the node as a description, e.g. dead, unstable, online""" + return self._statuses[self.status] + + @property + def total_tries(self) -> int: + """How many retries were required to get the data for this node?""" + tries_total = 0 + for tries_type, tries in self.tries.items(): + tries_total += tries + return tries_total + + @property + def avg_tries(self) -> str: + """The average amount of tries required per API call to get a valid response, as a 2 DP formatted string""" + return '{:.2f}'.format(self.total_tries / len(self.tries)) + + @property + def plugin_counts(self) -> Tuple[int, int]: + """Returns as a tuple: how many plugins worked, and how many were tested""" + return len(self.plugins), len(TEST_PLUGINS_LIST) + + def __post_init__(self): + bt = self.block_time + if not empty(bt): + if type(bt) is str and bt.lower() == 'error': + self.block_time = None + return + self.block_time = parse(bt) + + class RPCScanner: - def __init__(self, reactor: twisted.internet.reactor): + def __init__(self, reactor: twisted.internet.reactor, nodes: list): self.conf_nodes = [] self.prop_nodes = [] self.reactor = reactor self.node_status = {} self.ident_nodes = [] self.up_nodes = [] - node_list = open(join(BASE_DIR, settings.node_file), 'r').readlines() - # nodes to be specified line by line. format: http://gtg.steem.house:8090 - # NODE_LIST_FILE = "nodes.txt" - node_list = [n.strip() for n in node_list] - # Allow nodes to be commented out with # symbol - node_list = [n for n in node_list if n[0] != '#'] - self.nodes = node_list + self.nodes = nodes self.req_success = 0 @inlineCallbacks - def scan_nodes(self): + def scan_nodes(self, quiet=False): + def p(*args): + if not quiet: + print(*args) + reactor = self.reactor - print('Scanning nodes... Please wait...') - print('{}[Stage 1 / 4] Identifying node types (jussi/appbase){}'.format(Fore.GREEN, Fore.RESET)) + p('Scanning nodes... Please wait...') + p('{}[Stage 1 / 4] Identifying node types (jussi/appbase){}'.format(Fore.GREEN, Fore.RESET)) for node in self.nodes: self.node_status[node] = dict( raw={}, timing={}, tries={}, plugins=[], @@ -46,18 +104,18 @@ def scan_nodes(self): yield from self.identify_nodes() - print('{}[Stage 2 / 4] Filtering out bad nodes{}'.format(Fore.GREEN, Fore.RESET)) + p('{}[Stage 2 / 4] Filtering out bad nodes{}'.format(Fore.GREEN, Fore.RESET)) yield from self.filter_badnodes() - print('{}[Stage 3 / 4] Obtaining steemd versions {}'.format(Fore.GREEN, Fore.RESET)) + p('{}[Stage 3 / 4] Obtaining steemd versions {}'.format(Fore.GREEN, Fore.RESET)) yield from self.scan_versions() - print('{}[Stage 4 / 4] Checking current block / block time{}'.format(Fore.GREEN, Fore.RESET)) + p('{}[Stage 4 / 4] Checking current block / block time{}'.format(Fore.GREEN, Fore.RESET)) yield from self.scan_block_info() if settings.plugins: - print('{}[Thorough Plugin Check] User specified --plugins. Now running thorough ' - 'plugin tests for alive nodes.{}'.format(Fore.GREEN, Fore.RESET)) + p('{}[Thorough Plugin Check] User specified --plugins. Now running thorough plugin tests for ' + 'alive nodes.{}'.format(Fore.GREEN, Fore.RESET)) for host, data in self.node_status.items(): status = len(data['raw']) if status == 0: @@ -69,7 +127,6 @@ def scan_nodes(self): pt = yield self.plugin_test(host, plugin, mt) log.info(f'{Fore.GREEN} (+) Finished plugin tests for node {host} ... {Fore.RESET}') - self.print_nodes() @inlineCallbacks def plugin_test(self, host: str, plugin_name: str, mt: MethodTests): @@ -188,6 +245,14 @@ def scan_versions(self): log.warning(Fore.RED + 'Unknown error occurred (conf)...' + Fore.RESET) log.warning('[%s] %s', type(e), str(e)) + @property + def node_objs(self) -> List[NodeStatus]: + return [NodeStatus(host=h, **n) for h, n in self.node_status.items()] + + def get_node(self, node: str) -> NodeStatus: + n = self.node_status[node] + return NodeStatus(host=node, **n) + def print_nodes(self): list_nodes = self.node_status print(Fore.BLUE, '(S) - SSL, (H) - HTTP : (A) - normal appbase (J) - jussi', Fore.RESET) @@ -246,4 +311,7 @@ def print_nodes(self): elif plg == ttl_plg: f_plugins = f'{Fore.GREEN}{f_plugins}' fmt_params.append(f'{f_plugins}{Fore.RESET}') - print(fmt_str.format(*fmt_params), Fore.RESET) \ No newline at end of file + print(fmt_str.format(*fmt_params), Fore.RESET) + + + diff --git a/rpcscanner/core.py b/rpcscanner/core.py index d4c8f63..00c68b4 100644 --- a/rpcscanner/core.py +++ b/rpcscanner/core.py @@ -1,7 +1,10 @@ # from dataclasses import dataclass -from os.path import dirname, abspath +from os.path import dirname, abspath, join import logging +from typing import List + +from rpcscanner import settings log = logging.getLogger(__name__) BASE_DIR = dirname(dirname(abspath(__file__))) @@ -20,6 +23,14 @@ ) +def load_nodes(file: str) -> List[str]: + # nodes to be specified line by line. format: http://gtg.steem.house:8090 + node_list = open(join(BASE_DIR, file), 'r').readlines() + node_list = [n.strip() for n in node_list] + # Allow nodes to be commented out with # symbol + return [n for n in node_list if n[0] != '#'] + + # @dataclass # class ScannerSettings: # verbose: bool = False