Add typing to all calls to self.stats (#4973)

* Add typing to all calls to ``self.stats``

All checkers inherit from a baseclass which has a ``stats`` attribute.
This attribute has a fairly unmanageable type, but the current typing includes all variations of the attribute.
Other changes not directly related to ``self.stats`` are due to ``mypy``warnings.
This incorporate the feedback received in #4954

* Add ``CheckerStatistic`` class to ``pylint/typing``

* Guard `typing.Counter` import

Co-authored-by: Marc Mueller <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

pylint/checkers/__init__.py

@@ -46,28 +46,35 @@
 
 """
 
+from typing import Iterable, List, Union
+
 from pylint.checkers.base_checker import BaseChecker, BaseTokenChecker
 from pylint.checkers.deprecated import DeprecatedMixin
 from pylint.checkers.mapreduce_checker import MapReduceMixin
+from pylint.typing import CheckerStats
 from pylint.utils import diff_string, register_plugins
 
 
-def table_lines_from_stats(stats, old_stats, columns):
+def table_lines_from_stats(
+    stats: CheckerStats,
+    old_stats: CheckerStats,
+    columns: Iterable[str],
+) -> List[str]:
     """get values listed in <columns> from <stats> and <old_stats>,
     and return a formated list of values, designed to be given to a
     ureport.Table object
     """
-    lines = []
+    lines: List[str] = []
     for m_type in columns:
-        new = stats[m_type]
-        old = old_stats.get(m_type)
+        new: Union[int, str] = stats[m_type]  # type: ignore
+        old: Union[int, str, None] = old_stats.get(m_type)  # type: ignore
         if old is not None:
             diff_str = diff_string(old, new)
         else:
             old, diff_str = "NC", "NC"
         new = f"{new:.3f}" if isinstance(new, float) else str(new)
         old = f"{old:.3f}" if isinstance(old, float) else str(old)
-        lines += (m_type.replace("_", " "), new, old, diff_str)
+        lines.extend((m_type.replace("_", " "), new, old, diff_str))
     return lines
 
 

pylint/checkers/base.py

@@ -66,7 +66,7 @@
 import itertools
 import re
 import sys
-from typing import Any, Iterator, Optional, Pattern
+from typing import Any, Dict, Iterator, Optional, Pattern, Union
 
 import astroid
 from astroid import nodes
@@ -81,6 +81,7 @@
     is_property_setter,
 )
 from pylint.reporters.ureports import nodes as reporter_nodes
+from pylint.typing import CheckerStats
 
 
 class NamingStyle:
@@ -386,36 +387,42 @@ def _has_abstract_methods(node):
     return len(utils.unimplemented_abstract_methods(node)) > 0
 
 
-def report_by_type_stats(sect, stats, old_stats):
+def report_by_type_stats(
+    sect,
+    stats: CheckerStats,
+    old_stats: CheckerStats,
+):
     """make a report of
 
     * percentage of different types documented
     * percentage of different types with a bad name
     """
     # percentage of different types documented and/or with a bad name
-    nice_stats = {}
+    nice_stats: Dict[str, Dict[str, str]] = {}
     for node_type in ("module", "class", "method", "function"):
         try:
-            total = stats[node_type]
+            total: int = stats[node_type]  # type: ignore
         except KeyError as e:
             raise exceptions.EmptyReportError() from e
         nice_stats[node_type] = {}
         if total != 0:
             try:
-                documented = total - stats["undocumented_" + node_type]
+                undocumented_node: int = stats["undocumented_" + node_type]  # type: ignore
+                documented = total - undocumented_node
                 percent = (documented * 100.0) / total
                 nice_stats[node_type]["percent_documented"] = f"{percent:.2f}"
             except KeyError:
                 nice_stats[node_type]["percent_documented"] = "NC"
             try:
-                percent = (stats["badname_" + node_type] * 100.0) / total
+                badname_node: int = stats["badname_" + node_type]  # type: ignore
+                percent = (badname_node * 100.0) / total
                 nice_stats[node_type]["percent_badname"] = f"{percent:.2f}"
             except KeyError:
                 nice_stats[node_type]["percent_badname"] = "NC"
     lines = ["type", "number", "old number", "difference", "%documented", "%badname"]
     for node_type in ("module", "class", "method", "function"):
         new = stats[node_type]
-        old = old_stats.get(node_type, None)
+        old: Optional[Union[str, int]] = old_stats.get(node_type, None)  # type: ignore
         if old is not None:
             diff_str = lint_utils.diff_string(old, new)
         else:
@@ -1082,7 +1089,7 @@ class BasicChecker(_BasicChecker):
 
     def __init__(self, linter):
         _BasicChecker.__init__(self, linter)
-        self.stats = None
+        self.stats: CheckerStats = {}
         self._tryfinallys = None
 
     def open(self):
@@ -1159,13 +1166,13 @@ def _check_using_constant_test(self, node, test):
 
     def visit_module(self, _: nodes.Module) -> None:
         """check module name, docstring and required arguments"""
-        self.stats["module"] += 1
+        self.stats["module"] += 1  # type: ignore
 
     def visit_classdef(self, _: nodes.ClassDef) -> None:
         """check module name, docstring and redefinition
         increment branch counter
         """
-        self.stats["class"] += 1
+        self.stats["class"] += 1  # type: ignore
 
     @utils.check_messages(
         "pointless-statement", "pointless-string-statement", "expression-not-assigned"
@@ -1304,7 +1311,7 @@ def visit_functiondef(self, node: nodes.FunctionDef) -> None:
         """check function name, docstring, arguments, redefinition,
         variable names, max locals
         """
-        self.stats["method" if node.is_method() else "function"] += 1
+        self.stats["method" if node.is_method() else "function"] += 1  # type: ignore
         self._check_dangerous_default(node)
 
     visit_asyncfunctiondef = visit_functiondef
@@ -2040,7 +2047,7 @@ def _raise_name_warning(
         )
 
         self.add_message(warning, node=node, args=args, confidence=confidence)
-        self.stats["badname_" + node_type] += 1
+        self.stats["badname_" + node_type] += 1  # type: ignore
 
     def _name_allowed_by_regex(self, name: str) -> bool:
         return name in self.config.good_names or any(

pylint/checkers/base_checker.py

@@ -24,6 +24,7 @@
 from pylint.exceptions import InvalidMessageError
 from pylint.interfaces import UNDEFINED, IRawChecker, ITokenChecker, implements
 from pylint.message.message_definition import MessageDefinition
+from pylint.typing import CheckerStats
 from pylint.utils import get_rst_section, get_rst_title
 
 
@@ -51,6 +52,7 @@ def __init__(self, linter=None):
             self.name = self.name.lower()
         OptionsProviderMixIn.__init__(self)
         self.linter = linter
+        self.stats: CheckerStats = {}
 
     def __gt__(self, other):
         """Permit to sort a list of Checker by name."""

pylint/checkers/design_analysis.py

@@ -35,6 +35,7 @@
 from pylint.checkers import BaseChecker
 from pylint.checkers.utils import check_messages
 from pylint.interfaces import IAstroidChecker
+from pylint.typing import CheckerStats
 
 MSGS = {  # pylint: disable=consider-using-namedtuple-or-dataclass
     "R0901": (
@@ -391,7 +392,7 @@ class MisdesignChecker(BaseChecker):
 
     def __init__(self, linter=None):
         BaseChecker.__init__(self, linter)
-        self.stats = None
+        self.stats: CheckerStats = {}
         self._returns = None
         self._branches = None
         self._stmts = None

pylint/checkers/imports.py

@@ -68,6 +68,7 @@
 from pylint.interfaces import IAstroidChecker
 from pylint.lint import PyLinter
 from pylint.reporters.ureports.nodes import Paragraph, VerbatimText, VNode
+from pylint.typing import CheckerStats
 from pylint.utils import IsortDriver, get_global_option
 
 
@@ -423,7 +424,7 @@ def __init__(
         self, linter: PyLinter = None
     ):  # pylint: disable=super-init-not-called # See https://github.com/PyCQA/pylint/issues/4941
         BaseChecker.__init__(self, linter)
-        self.stats: Dict[Any, Any] = {}
+        self.stats: CheckerStats = {}
         self.import_graph: collections.defaultdict = collections.defaultdict(set)
         self._imports_stack: List[Tuple[Any, Any]] = []
         self._first_non_import_node = None
@@ -839,9 +840,8 @@ def _add_imported_module(
                 self._module_pkg[context_name] = context_name.rsplit(".", 1)[0]
 
             # handle dependencies
-            importedmodnames = self.stats["dependencies"].setdefault(
-                importedmodname, set()
-            )
+            dependencies_stat: Dict[str, Union[Set]] = self.stats["dependencies"]  # type: ignore
+            importedmodnames = dependencies_stat.setdefault(importedmodname, set())
             if context_name not in importedmodnames:
                 importedmodnames.add(context_name)
 

pylint/checkers/raw_metrics.py

@@ -15,27 +15,32 @@
 # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
 
 import tokenize
-from typing import Any
+from typing import Any, Optional, Union
 
 from pylint.checkers import BaseTokenChecker
 from pylint.exceptions import EmptyReportError
 from pylint.interfaces import ITokenChecker
 from pylint.reporters.ureports.nodes import Table
+from pylint.typing import CheckerStats
 from pylint.utils import diff_string
 
 
-def report_raw_stats(sect, stats, old_stats):
+def report_raw_stats(
+    sect,
+    stats: CheckerStats,
+    old_stats: CheckerStats,
+):
     """calculate percentage of code / doc / comment / empty"""
-    total_lines = stats["total_lines"]
+    total_lines: int = stats["total_lines"]  # type: ignore
     if not total_lines:
         raise EmptyReportError()
     sect.description = f"{total_lines} lines have been analyzed"
     lines = ["type", "number", "%", "previous", "difference"]
     for node_type in ("code", "docstring", "comment", "empty"):
         key = node_type + "_lines"
-        total = stats[key]
+        total: int = stats[key]  # type: ignore
         percent = float(total * 100) / total_lines
-        old = old_stats.get(key, None)
+        old: Optional[Union[int, str]] = old_stats.get(key, None)  # type: ignore
         if old is not None:
             diff_str = diff_string(old, total)
         else:
@@ -66,7 +71,7 @@ class RawMetricsChecker(BaseTokenChecker):
 
     def __init__(self, linter):
         BaseTokenChecker.__init__(self, linter)
-        self.stats = None
+        self.stats: CheckerStats = {}
 
     def open(self):
         """init statistics"""

pylint/checkers/similar.py

@@ -73,6 +73,7 @@
 from pylint.checkers import BaseChecker, MapReduceMixin, table_lines_from_stats
 from pylint.interfaces import IRawChecker
 from pylint.reporters.ureports.nodes import Table
+from pylint.typing import CheckerStats
 from pylint.utils import decoding_stream
 
 DEFAULT_MIN_SIMILARITY_LINE = 4
@@ -721,7 +722,11 @@ def real_lines(self):
 }
 
 
-def report_similarities(sect, stats, old_stats):
+def report_similarities(
+    sect,
+    stats: CheckerStats,
+    old_stats: CheckerStats,
+):
     """make a layout with some stats about duplication"""
     lines = ["", "now", "previous", "difference"]
     lines += table_lines_from_stats(
@@ -804,7 +809,7 @@ def __init__(self, linter=None) -> None:
             ignore_imports=self.config.ignore_imports,
             ignore_signatures=self.config.ignore_signatures,
         )
-        self.stats = None
+        self.stats: CheckerStats = {}
 
     def set_option(self, optname, value, action=None, optdict=None):
         """method called to set an option (registered in the options list)

pylint/lint/parallel.py

@@ -3,10 +3,15 @@
 
 import collections
 import functools
+from typing import TYPE_CHECKING, Dict, List, Union
 
 from pylint import reporters
 from pylint.lint.utils import _patch_sys_path
 from pylint.message import Message
+from pylint.typing import CheckerStats
+
+if TYPE_CHECKING:
+    from typing import Counter  # typing.Counter added in Python 3.6.1
 
 try:
     import multiprocessing
@@ -30,20 +35,20 @@ def _get_new_args(message):
     return (message.msg_id, message.symbol, location, message.msg, message.confidence)
 
 
-def _merge_stats(stats):
-    merged = {}
-    by_msg = collections.Counter()
+def _merge_stats(stats: List[CheckerStats]):
+    merged: CheckerStats = {}
+    by_msg: "Counter[str]" = collections.Counter()
     for stat in stats:
-        message_stats = stat.pop("by_msg", {})
+        message_stats: Union["Counter[str]", Dict] = stat.pop("by_msg", {})  # type: ignore
         by_msg.update(message_stats)
 
         for key, item in stat.items():
             if key not in merged:
                 merged[key] = item
             elif isinstance(item, dict):
-                merged[key].update(item)
+                merged[key].update(item)  # type: ignore
             else:
-                merged[key] = merged[key] + item
+                merged[key] = merged[key] + item  # type: ignore
 
     merged["by_msg"] = by_msg
     return merged

pylint/lint/pylinter.py

@@ -31,6 +31,7 @@
 )
 from pylint.message import MessageDefinitionStore, MessagesHandlerMixIn
 from pylint.reporters.ureports import nodes as report_nodes
+from pylint.typing import CheckerStats
 from pylint.utils import ASTWalker, FileState, utils
 from pylint.utils.pragma_parser import (
     OPTION_PO,
@@ -502,7 +503,7 @@ def __init__(self, options=(), reporter=None, option_groups=(), pylintrc=None):
         self.file_state = FileState()
         self.current_name = None
         self.current_file = None
-        self.stats = None
+        self.stats: CheckerStats = {}
         self.fail_on_symbols = []
         # init options
         self._external_opts = options
@@ -729,8 +730,10 @@ def enable_fail_on_messages(self):
                         self.fail_on_symbols.append(msg.symbol)
 
     def any_fail_on_issues(self):
-        return self.stats is not None and any(
-            x in self.fail_on_symbols for x in self.stats["by_msg"]
+        return (
+            self.stats
+            and self.stats.get("by_msg") is not None
+            and any(x in self.fail_on_symbols for x in self.stats["by_msg"])
         )
 
     def disable_noerror_messages(self):