diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 310796e7ac6b14..9f9361a2fd569e 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -661,6 +661,76 @@ it with :func:`staticmethod`. For example:: You don't need to wrap with :func:`staticmethod` if you're setting the import callable on a configurator *instance*. +.. _configure-queue: + +Configuring QueueHandler and QueueListener +"""""""""""""""""""""""""""""""""""""""""" + +If you want to configure a :class:`~logging.handlers.QueueHandler`, noting that this +is normally used in conjunction with a :class:`~logging.handlers.QueueListener`, you +can configure both together. After the configuration, the ``QueueListener`` instance +will be available as the :attr:`~logging.handlers.QueueHandler.listener` attribute of +the created handler, and that in turn will be available to you using +:func:`~logging.getHandlerByName` and passing the name you have used for the +``QueueHandler`` in your configuration. The dictionary schema for configuring the pair +is shown in the example YAML snippet below. + +.. code-block:: yaml + + handlers: + qhand: + class: logging.handlers.QueueHandler + queue: my.module.queue_factory + listener: my.package.CustomListener + handlers: + - hand_name_1 + - hand_name_2 + ... + +The ``queue`` and ``listener`` keys are optional. + +If the ``queue`` key is present, the corresponding value can be one of the following: + +* An actual instance of :class:`queue.Queue` or a subclass thereof. This is of course + only possible if you are constructing or modifying the configuration dictionary in + code. + +* A string that resolves to a callable which, when called with no arguments, returns + the :class:`queue.Queue` instance to use. That callable could be a + :class:`queue.Queue` subclass or a function which returns a suitable queue instance, + such as ``my.module.queue_factory()``. + +* A dict with a ``'()'`` key which is constructed in the usual way as discussed in + :ref:`logging-config-dict-userdef`. The result of this construction should be a + :class:`queue.Queue` instance. + +If the ``queue`` key is absent, a standard unbounded :class:`queue.Queue` instance is +created and used. + +If the ``listener`` key is present, the corresponding value can be one of the following: + +* A subclass of :class:`logging.handlers.QueueListener`. This is of course only + possible if you are constructing or modifying the configuration dictionary in + code. + +* A string which resolves to a class which is a subclass of ``QueueListener``, such as + ``'my.package.CustomListener'``. + +* A dict with a ``'()'`` key which is constructed in the usual way as discussed in + :ref:`logging-config-dict-userdef`. The result of this construction should be a + callable with the same signature as the ``QueueListener`` initializer. + +If the ``listener`` key is absent, :class:`logging.handlers.QueueListener` is used. + +The values under the ``handlers`` key are the names of other handlers in the +configuration (not shown in the above snippet) which will be passed to the queue +listener. + +Any custom queue handler and listener classes will need to be defined with the same +initialization signatures as :class:`~logging.handlers.QueueHandler` and +:class:`~logging.handlers.QueueListener`. + +.. versionadded:: 3.12 .. _logging-config-fileformat: diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index f5ef80ea044c66..4bdd5508c67c4e 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -1051,7 +1051,13 @@ possible, while any potentially slow operations (such as sending an email via want to override this if you want to use blocking behaviour, or a timeout, or a customized queue implementation. + .. attribute:: listener + When created via configuration using :func:`~logging.config.dictConfig`, this + attribute will contain a :class:`QueueListener` instance for use with this + handler. Otherwise, it will be ``None``. + + .. versionadded:: 3.12 .. _queue-listener: diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index ac86bc8077ef74..cc83193c863933 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1164,6 +1164,19 @@ functions. This undocumented behaviour was considered a mistake, and was removed in Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. +.. function:: getHandlerByName(name) + + Returns a handler with the specified *name*, or ``None`` if there is no handler + with that name. + + .. versionadded:: 3.12 + +.. function:: getHandlerNames() + + Returns an immutable set of all known handler names. + + .. versionadded:: 3.12 + .. function:: makeLogRecord(attrdict) Creates and returns a new :class:`LogRecord` instance whose attributes are diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 20ab191bdb45ea..276845a9b62da8 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,7 +18,7 @@ Logging package for Python. Based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ @@ -38,7 +38,8 @@ 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'shutdown', 'warn', 'warning', 'getLogRecordFactory', 'setLogRecordFactory', - 'lastResort', 'raiseExceptions', 'getLevelNamesMapping'] + 'lastResort', 'raiseExceptions', 'getLevelNamesMapping', + 'getHandlerByName', 'getHandlerNames'] import threading @@ -885,6 +886,23 @@ def _addHandlerRef(handler): finally: _releaseLock() + +def getHandlerByName(name): + """ + Get a handler with the specified *name*, or None if there isn't one with + that name. + """ + return _handlers.get(name) + + +def getHandlerNames(): + """ + Return all known handler names as an immutable set. + """ + result = set(_handlers.keys()) + return frozenset(result) + + class Handler(Filterer): """ Handler instances dispatch logging events to specific destinations. diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 86a1e4eaf4cbc9..2b9d90c3ed5b79 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -1,4 +1,4 @@ -# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -19,15 +19,17 @@ is based on PEP 282 and comments thereto in comp.lang.python, and influenced by Apache's log4j system. -Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ import errno +import functools import io import logging import logging.handlers +import queue import re import struct import threading @@ -563,7 +565,7 @@ def configure(self): handler.name = name handlers[name] = handler except Exception as e: - if 'target not configured yet' in str(e.__cause__): + if ' not configured yet' in str(e.__cause__): deferred.append(name) else: raise ValueError('Unable to configure handler ' @@ -702,6 +704,21 @@ def add_filters(self, filterer, filters): except Exception as e: raise ValueError('Unable to add filter %r' % f) from e + def _configure_queue_handler(self, klass, **kwargs): + if 'queue' in kwargs: + q = kwargs['queue'] + else: + q = queue.Queue() # unbounded + rhl = kwargs.get('respect_handler_level', False) + if 'listener' in kwargs: + lklass = kwargs['listener'] + else: + lklass = logging.handlers.QueueListener + listener = lklass(q, *kwargs['handlers'], respect_handler_level=rhl) + handler = klass(q) + handler.listener = listener + return handler + def configure_handler(self, config): """Configure a handler from a dictionary.""" config_copy = dict(config) # for restoring in case of error @@ -721,26 +738,83 @@ def configure_handler(self, config): factory = c else: cname = config.pop('class') - klass = self.resolve(cname) - #Special case for handler which refers to another handler + if callable(cname): + klass = cname + else: + klass = self.resolve(cname) if issubclass(klass, logging.handlers.MemoryHandler) and\ 'target' in config: + # Special case for handler which refers to another handler try: - th = self.config['handlers'][config['target']] + tn = config['target'] + th = self.config['handlers'][tn] if not isinstance(th, logging.Handler): config.update(config_copy) # restore for deferred cfg raise TypeError('target not configured yet') config['target'] = th except Exception as e: - raise ValueError('Unable to set target handler ' - '%r' % config['target']) from e + raise ValueError('Unable to set target handler %r' % tn) from e + elif issubclass(klass, logging.handlers.QueueHandler): + # Another special case for handler which refers to other handlers + if 'handlers' not in config: + raise ValueError('No handlers specified for a QueueHandler') + if 'queue' in config: + qspec = config['queue'] + if not isinstance(qspec, queue.Queue): + if isinstance(qspec, str): + q = self.resolve(qspec) + if not callable(q): + raise TypeError('Invalid queue specifier %r' % qspec) + q = q() + elif isinstance(qspec, dict): + if '()' not in qspec: + raise TypeError('Invalid queue specifier %r' % qspec) + q = self.configure_custom(dict(qspec)) + else: + raise TypeError('Invalid queue specifier %r' % qspec) + config['queue'] = q + if 'listener' in config: + lspec = config['listener'] + if isinstance(lspec, type): + if not issubclass(lspec, logging.handlers.QueueListener): + raise TypeError('Invalid listener specifier %r' % lspec) + else: + if isinstance(lspec, str): + listener = self.resolve(lspec) + if isinstance(listener, type) and\ + not issubclass(listener, logging.handlers.QueueListener): + raise TypeError('Invalid listener specifier %r' % lspec) + elif isinstance(lspec, dict): + if '()' not in lspec: + raise TypeError('Invalid listener specifier %r' % lspec) + listener = self.configure_custom(dict(lspec)) + else: + raise TypeError('Invalid listener specifier %r' % lspec) + if not callable(listener): + raise TypeError('Invalid listener specifier %r' % lspec) + config['listener'] = listener + hlist = [] + try: + for hn in config['handlers']: + h = self.config['handlers'][hn] + if not isinstance(h, logging.Handler): + config.update(config_copy) # restore for deferred cfg + raise TypeError('Required handler %r ' + 'is not configured yet' % hn) + hlist.append(h) + except Exception as e: + raise ValueError('Unable to set required handler %r' % hn) from e + config['handlers'] = hlist elif issubclass(klass, logging.handlers.SMTPHandler) and\ 'mailhost' in config: config['mailhost'] = self.as_tuple(config['mailhost']) elif issubclass(klass, logging.handlers.SysLogHandler) and\ 'address' in config: config['address'] = self.as_tuple(config['address']) - factory = klass + if issubclass(klass, logging.handlers.QueueHandler): + factory = functools.partial(self._configure_queue_handler, klass) + else: + factory = klass props = config.pop('.', None) kwargs = {k: config[k] for k in config if valid_ident(k)} try: diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 78e919d195d974..b4c8a3ba9a1896 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1424,6 +1424,7 @@ def __init__(self, queue): """ logging.Handler.__init__(self) self.queue = queue + self.listener = None # will be set to listener if configured via dictConfig() def enqueue(self, record): """ diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 8af10d42981ce4..da298f258f5971 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1,4 +1,4 @@ -# Copyright 2001-2021 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2022 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -16,7 +16,7 @@ """Test harness for the logging module. Run all tests. -Copyright (C) 2001-2021 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved. """ import logging import logging.handlers @@ -29,6 +29,7 @@ import pathlib import pickle import io +import itertools import gc import json import os @@ -1211,6 +1212,9 @@ class ExceptionFormatter(logging.Formatter): def formatException(self, ei): return "Got a [%s]" % ei[0].__name__ +def closeFileHandler(h, fn): + h.close() + os.remove(fn) class ConfigFileTest(BaseTest): @@ -1594,10 +1598,6 @@ def test_config7_ok(self): def test_config8_ok(self): - def cleanup(h1, fn): - h1.close() - os.remove(fn) - with self.check_no_resource_warning(): fn = make_temp_file(".log", "test_logging-X-") @@ -1612,7 +1612,7 @@ def cleanup(h1, fn): self.apply_config(config8) handler = logging.root.handlers[0] - self.addCleanup(cleanup, handler, fn) + self.addCleanup(closeFileHandler, handler, fn) def test_logger_disabling(self): self.apply_config(self.disable_test) @@ -2233,6 +2233,21 @@ def handlerFunc(): class CustomHandler(logging.StreamHandler): pass +class CustomListener(logging.handlers.QueueListener): + pass + +class CustomQueue(queue.Queue): + pass + +def queueMaker(): + return queue.Queue() + +def listenerMaker(arg1, arg2, respect_handler_level=False): + def func(queue, *handlers, **kwargs): + kwargs.setdefault('respect_handler_level', respect_handler_level) + return CustomListener(queue, *handlers, **kwargs) + return func + class ConfigDictTest(BaseTest): """Reading logging config from a dictionary.""" @@ -2836,7 +2851,7 @@ class ConfigDictTest(BaseTest): }, } - out_of_order = { + bad_format = { "version": 1, "formatters": { "mySimpleFormatter": { @@ -2856,7 +2871,7 @@ class ConfigDictTest(BaseTest): "formatter": "mySimpleFormatter", "target": "fileGlobal", "level": "DEBUG" - } + } }, "loggers": { "mymodule": { @@ -2975,13 +2990,36 @@ class ConfigDictTest(BaseTest): } } + config_queue_handler = { + 'version': 1, + 'handlers' : { + 'h1' : { + 'class': 'logging.FileHandler', + }, + # key is before depended on handlers to test that deferred config works + 'ah' : { + 'class': 'logging.handlers.QueueHandler', + 'handlers': ['h1'] + }, + }, + "root": { + "level": "DEBUG", + "handlers": ["ah"] + } + } + def apply_config(self, conf): logging.config.dictConfig(conf) + def check_handler(self, name, cls): + h = logging.getHandlerByName(name) + self.assertIsInstance(h, cls) + def test_config0_ok(self): # A simple config which overrides the default settings. with support.captured_stdout() as output: self.apply_config(self.config0) + self.check_handler('hand1', logging.StreamHandler) logger = logging.getLogger() # Won't output anything logger.info(self.next_message()) @@ -3028,6 +3066,7 @@ def test_config4_ok(self): # A config specifying a custom formatter class. with support.captured_stdout() as output: self.apply_config(self.config4) + self.check_handler('hand1', logging.StreamHandler) #logger = logging.getLogger() try: raise RuntimeError() @@ -3056,6 +3095,7 @@ def test_config4a_ok(self): def test_config5_ok(self): self.test_config1_ok(config=self.config5) + self.check_handler('hand1', CustomHandler) def test_config6_failure(self): self.assertRaises(Exception, self.apply_config, self.config6) @@ -3075,6 +3115,7 @@ def test_config7_ok(self): self.assert_log_lines([]) with support.captured_stdout() as output: self.apply_config(self.config7) + self.check_handler('hand1', logging.StreamHandler) logger = logging.getLogger("compiler.parser") self.assertTrue(logger.disabled) logger = logging.getLogger("compiler.lexer") @@ -3104,6 +3145,7 @@ def test_config_8_ok(self): self.assert_log_lines([]) with support.captured_stdout() as output: self.apply_config(self.config8) + self.check_handler('hand1', logging.StreamHandler) logger = logging.getLogger("compiler.parser") self.assertFalse(logger.disabled) # Both will output a message @@ -3125,6 +3167,7 @@ def test_config_8_ok(self): def test_config_8a_ok(self): with support.captured_stdout() as output: self.apply_config(self.config1a) + self.check_handler('hand1', logging.StreamHandler) logger = logging.getLogger("compiler.parser") # See issue #11424. compiler-hyphenated sorts # between compiler and compiler.xyz and this @@ -3145,6 +3188,7 @@ def test_config_8a_ok(self): self.assert_log_lines([]) with support.captured_stdout() as output: self.apply_config(self.config8a) + self.check_handler('hand1', logging.StreamHandler) logger = logging.getLogger("compiler.parser") self.assertFalse(logger.disabled) # Both will output a message @@ -3168,6 +3212,7 @@ def test_config_8a_ok(self): def test_config_9_ok(self): with support.captured_stdout() as output: self.apply_config(self.config9) + self.check_handler('hand1', logging.StreamHandler) logger = logging.getLogger("compiler.parser") # Nothing will be output since both handler and logger are set to WARNING logger.info(self.next_message()) @@ -3186,6 +3231,7 @@ def test_config_9_ok(self): def test_config_10_ok(self): with support.captured_stdout() as output: self.apply_config(self.config10) + self.check_handler('hand1', logging.StreamHandler) logger = logging.getLogger("compiler.parser") logger.warning(self.next_message()) logger = logging.getLogger('compiler') @@ -3222,10 +3268,6 @@ def test_config14_ok(self): def test_config15_ok(self): - def cleanup(h1, fn): - h1.close() - os.remove(fn) - with self.check_no_resource_warning(): fn = make_temp_file(".log", "test_logging-X-") @@ -3247,7 +3289,7 @@ def cleanup(h1, fn): self.apply_config(config) handler = logging.root.handlers[0] - self.addCleanup(cleanup, handler, fn) + self.addCleanup(closeFileHandler, handler, fn) def setup_via_listener(self, text, verify=None): text = text.encode("utf-8") @@ -3281,6 +3323,7 @@ def setup_via_listener(self, text, verify=None): def test_listen_config_10_ok(self): with support.captured_stdout() as output: self.setup_via_listener(json.dumps(self.config10)) + self.check_handler('hand1', logging.StreamHandler) logger = logging.getLogger("compiler.parser") logger.warning(self.next_message()) logger = logging.getLogger('compiler') @@ -3375,11 +3418,11 @@ def verify_reverse(stuff): ('ERROR', '2'), ], pat=r"^[\w.]+ -> (\w+): (\d+)$") - def test_out_of_order(self): - self.assertRaises(ValueError, self.apply_config, self.out_of_order) + def test_bad_format(self): + self.assertRaises(ValueError, self.apply_config, self.bad_format) - def test_out_of_order_with_dollar_style(self): - config = copy.deepcopy(self.out_of_order) + def test_bad_format_with_dollar_style(self): + config = copy.deepcopy(self.bad_format) config['formatters']['mySimpleFormatter']['format'] = "${asctime} (${name}) ${levelname}: ${message}" self.apply_config(config) @@ -3387,6 +3430,8 @@ def test_out_of_order_with_dollar_style(self): self.assertIsInstance(handler.target, logging.Handler) self.assertIsInstance(handler.formatter._style, logging.StringTemplateStyle) + self.assertEqual(sorted(logging.getHandlerNames()), + ['bufferGlobal', 'fileGlobal']) def test_custom_formatter_class_with_validate(self): self.apply_config(self.custom_formatter_class_validate) @@ -3402,7 +3447,7 @@ def test_custom_formatter_class_with_validate2_with_wrong_fmt(self): config = self.custom_formatter_class_validate.copy() config['formatters']['form1']['style'] = "$" - # Exception should not be raise as we have configured 'validate' to False + # Exception should not be raised as we have configured 'validate' to False self.apply_config(config) handler = logging.getLogger("my_test_logger_custom_formatter").handlers[0] self.assertIsInstance(handler.formatter, ExceptionFormatter) @@ -3503,6 +3548,69 @@ class NotAFilter: pass {"version": 1, "root": {"level": "DEBUG", "filters": [filter_]}} ) + def do_queuehandler_configuration(self, qspec, lspec): + cd = copy.deepcopy(self.config_queue_handler) + fn = make_temp_file('.log', 'test_logging-cqh-') + cd['handlers']['h1']['filename'] = fn + if qspec is not None: + cd['handlers']['ah']['queue'] = qspec + if lspec is not None: + cd['handlers']['ah']['listener'] = lspec + qh = None + delay = 0.01 + try: + self.apply_config(cd) + qh = logging.getHandlerByName('ah') + self.assertEqual(sorted(logging.getHandlerNames()), ['ah', 'h1']) + self.assertIsNotNone(qh.listener) + qh.listener.start() + # Need to let the listener thread get started + time.sleep(delay) + logging.debug('foo') + logging.info('bar') + logging.warning('baz') + # Need to let the listener thread finish its work + time.sleep(delay) + with open(fn, encoding='utf-8') as f: + data = f.read().splitlines() + self.assertEqual(data, ['foo', 'bar', 'baz']) + finally: + if qh: + qh.listener.stop() + h = logging.getHandlerByName('h1') + if h: + self.addCleanup(closeFileHandler, h, fn) + else: + self.addCleanup(os.remove, fn) + + def test_config_queue_handler(self): + q = CustomQueue() + dq = { + '()': __name__ + '.CustomQueue', + 'maxsize': 10 + } + dl = { + '()': __name__ + '.listenerMaker', + 'arg1': None, + 'arg2': None, + 'respect_handler_level': True + } + qvalues = (None, __name__ + '.queueMaker', __name__ + '.CustomQueue', dq, q) + lvalues = (None, __name__ + '.CustomListener', dl, CustomListener) + for qspec, lspec in itertools.product(qvalues, lvalues): + self.do_queuehandler_configuration(qspec, lspec) + + # Some failure cases + qvalues = (None, 4, int, '', 'foo') + lvalues = (None, 4, int, '', 'bar') + for qspec, lspec in itertools.product(qvalues, lvalues): + if lspec is None and qspec is None: + continue + with self.assertRaises(ValueError) as ctx: + self.do_queuehandler_configuration(qspec, lspec) + msg = str(ctx.exception) + self.assertEqual(msg, "Unable to configure handler 'ah'") + class ManagerTest(BaseTest): def test_manager_loggerclass(self): diff --git a/Misc/NEWS.d/next/Library/2022-05-26-09-24-41.gh-issue-93162.W1VuhU.rst b/Misc/NEWS.d/next/Library/2022-05-26-09-24-41.gh-issue-93162.W1VuhU.rst new file mode 100644 index 00000000000000..4d916a1df5e065 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-26-09-24-41.gh-issue-93162.W1VuhU.rst @@ -0,0 +1,4 @@ +Add the ability for :func:`logging.config.dictConfig` to usefully configure +:class:`~logging.handlers.QueueHandler` and :class:`~logging.handlers.QueueListener` +as a pair, and add :func:`logging.getHandlerByName` and :func:`logging.getHandlerNames` +APIs to allow access to handlers by name.