From 904d15cd08a6f8a2692519faf208e12cdd41a469 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Sat, 5 Oct 2024 14:23:53 +0200
Subject: [PATCH 01/15] Add guide on how to handle missing extras

---
 .../handling-missing-extras-at-runtime.rst    | 65 +++++++++++++++++++
 1 file changed, 65 insertions(+)
 create mode 100644 source/guides/handling-missing-extras-at-runtime.rst

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
new file mode 100644
index 000000000..82463db9b
--- /dev/null
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -0,0 +1,65 @@
+.. _handling-missing-extras-at-runtime:
+
+============================================================
+Handling missing optional dependencies ("extras") at runtime
+============================================================
+
+If your package has :ref:`optional dependencies ("extras")
+<metadata_provides_extra>` which the package consumer hasn't installed, the
+default outcome is an ordinary `ModuleNotFoundError` exception being raised at
+the first attempted import of a missing module.
+
+This can make for a bad user experience, because there is no guidance about why
+the module is missing - users might think they've found a bug. If you're not
+careful, it can even make your package unusable without the extras installed,
+e.g. if your package is a library that imports the affected modules from the
+top-level module or from the application entry point if it's an application.
+
+As of the time of writing, there is no *great* way to handle this issue in
+the Python packaging ecosystem, but there are a few options that might be
+better than nothing:
+
+
+Overall approach
+================
+
+TODO General guidance about how to isolate imports in question
+
+TODO Optimistic vs pessimistic handling?
+
+
+Handling failing imports
+========================
+
+TODO example
+
+TODO mention it doesn't check versions, so a bit dangerous
+
+
+Using ``pkg_resources``
+=======================
+
+The now-deprecated ``pkg_resources`` package (part of the ``setuptools``
+distribution) provides a ``require`` function that you can use to check if a
+given optional dependency of your package is installed or not:
+
+
+.. code-block:: python
+
+   from pkg_resources import require, DistributionNotFound, VersionConflict
+
+   try:
+     require(["your-package-name[your-extra]"])
+   except DistributionNotFound:
+     ...  # handle package(s) not being installed at all
+   except VersionConflict:
+     ...  # handle version mismatches
+
+Unfortunately, no replacement for this functionality exists in
+``pkg_resources``'s successor packages yet
+(`packaging-problems #664 <packaging-problems #664>`_).
+
+
+------------------
+
+.. _packaging-problems-664: https://github.com/pypa/packaging-problems/issues/664

From c01232eae958f0485e90e3d2f7302e67a2c03582 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Sun, 6 Oct 2024 00:55:31 +0200
Subject: [PATCH 02/15] Include in TOC & fix links

---
 .../handling-missing-extras-at-runtime.rst    | 50 +++++++++++++++----
 source/guides/section-build-and-publish.rst   |  1 +
 2 files changed, 41 insertions(+), 10 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 82463db9b..7744174ec 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -31,17 +31,33 @@ TODO Optimistic vs pessimistic handling?
 Handling failing imports
 ========================
 
-TODO example
+The perhaps simplest option, which is also in line with the :term:`EAFP`
+principle, is to just import your optional dependency modules as normal and
+handle the relevant exceptions if the import fails:
+
+.. code-block:: python
+
+   try:
+     import your_optional_dependency
+   except ModuleNotFoundError:
+     ...  # handle missing dependency
+
+However, this can lead to difficult-to-debug errors when
+``your_optional_dependency`` *is* installed, but at the wrong version (e.g.
+because another installed package depends on it with a wider version
+requirement than specified by your extra).
 
-TODO mention it doesn't check versions, so a bit dangerous
 
+Using ``pkg_resources`` (deprecated)
+====================================
 
-Using ``pkg_resources``
-=======================
+The now-deprecated :ref:`pkg_resources <ResourceManager API>` package (part of
+the ``setuptools`` distribution) provides a ``require`` function that you can
+use to check if a given optional dependency of your package is installed or
+not:
 
-The now-deprecated ``pkg_resources`` package (part of the ``setuptools``
-distribution) provides a ``require`` function that you can use to check if a
-given optional dependency of your package is installed or not:
+.. :: TODO ask setuptools to add labels for pkg_resources & require, then link
+      properly
 
 
 .. code-block:: python
@@ -55,11 +71,25 @@ given optional dependency of your package is installed or not:
    except VersionConflict:
      ...  # handle version mismatches
 
-Unfortunately, no replacement for this functionality exists in
-``pkg_resources``'s successor packages yet
-(`packaging-problems #664 <packaging-problems #664>`_).
+Unfortunately, no drop-in replacement for this functionality exists in
+``pkg_resources``'s "official" successor packages yet
+(`packaging-problems #317 <packaging-problems-317_>`_).
+
+
+Using 3rd-party libraries
+=========================
+
+In response to the aforementioned lack of a replacement for
+``pkg_resources.require``, at least one 3rd party implementation of this
+functionality using only the ``packaging`` and ``importlib.metadata`` modules
+has been created (`packaging-problems #664 <packaging-problems-664_>`_) and
+made available in the 3rd-party `hbutils <https://pypi.org/project/hbutils/>`_
+package as ``hbutils.system.check_reqs``.
 
 
 ------------------
 
+.. _packaging-problems-317: https://github.com/pypa/packaging-problems/issues/317
+
 .. _packaging-problems-664: https://github.com/pypa/packaging-problems/issues/664
+
diff --git a/source/guides/section-build-and-publish.rst b/source/guides/section-build-and-publish.rst
index eb10c389f..29af0483e 100644
--- a/source/guides/section-build-and-publish.rst
+++ b/source/guides/section-build-and-publish.rst
@@ -16,3 +16,4 @@ Building and Publishing
    making-a-pypi-friendly-readme
    publishing-package-distribution-releases-using-github-actions-ci-cd-workflows
    modernize-setup-py-project
+   handling-missing-extras-at-runtime

From 6480078ff6b44739f6864beed4c027fe50c2270a Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sat, 5 Oct 2024 22:56:31 +0000
Subject: [PATCH 03/15] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 source/guides/handling-missing-extras-at-runtime.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 7744174ec..0730fd56c 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -92,4 +92,3 @@ package as ``hbutils.system.check_reqs``.
 .. _packaging-problems-317: https://github.com/pypa/packaging-problems/issues/317
 
 .. _packaging-problems-664: https://github.com/pypa/packaging-problems/issues/664
-

From de7893ec48a22e7835e0a471b0399a42eaaefa29 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Sun, 6 Oct 2024 01:07:54 +0200
Subject: [PATCH 04/15] Fix backticks

---
 source/guides/handling-missing-extras-at-runtime.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 0730fd56c..8edb38832 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -6,8 +6,8 @@ Handling missing optional dependencies ("extras") at runtime
 
 If your package has :ref:`optional dependencies ("extras")
 <metadata_provides_extra>` which the package consumer hasn't installed, the
-default outcome is an ordinary `ModuleNotFoundError` exception being raised at
-the first attempted import of a missing module.
+default outcome is an ordinary ``ModuleNotFoundError`` exception being raised
+at the first attempted import of a missing module.
 
 This can make for a bad user experience, because there is no guidance about why
 the module is missing - users might think they've found a bug. If you're not

From fdae83acb76bbece669136be485e9d2bb0091f12 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Sun, 6 Oct 2024 23:11:09 +0200
Subject: [PATCH 05/15] Add sections on how to handle missing extra

---
 .../handling-missing-extras-at-runtime.rst    | 144 ++++++++++++++++--
 1 file changed, 132 insertions(+), 12 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 8edb38832..c92b1381d 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -19,17 +19,14 @@ As of the time of writing, there is no *great* way to handle this issue in
 the Python packaging ecosystem, but there are a few options that might be
 better than nothing:
 
+Detecting missing extras
+========================
 
-Overall approach
-================
-
-TODO General guidance about how to isolate imports in question
-
-TODO Optimistic vs pessimistic handling?
-
+We first consider how to *detect* if an extra is missing, leaving what to do
+about it for the next section.
 
-Handling failing imports
-========================
+Trying to import and handling failure
+-------------------------------------
 
 The perhaps simplest option, which is also in line with the :term:`EAFP`
 principle, is to just import your optional dependency modules as normal and
@@ -48,8 +45,8 @@ because another installed package depends on it with a wider version
 requirement than specified by your extra).
 
 
-Using ``pkg_resources`` (deprecated)
-====================================
+Using ``pkg_resources``
+-----------------------
 
 The now-deprecated :ref:`pkg_resources <ResourceManager API>` package (part of
 the ``setuptools`` distribution) provides a ``require`` function that you can
@@ -77,7 +74,7 @@ Unfortunately, no drop-in replacement for this functionality exists in
 
 
 Using 3rd-party libraries
-=========================
+-------------------------
 
 In response to the aforementioned lack of a replacement for
 ``pkg_resources.require``, at least one 3rd party implementation of this
@@ -87,6 +84,129 @@ made available in the 3rd-party `hbutils <https://pypi.org/project/hbutils/>`_
 package as ``hbutils.system.check_reqs``.
 
 
+Handling missing extras
+=======================
+
+In each of the previous section's code snippets, we omitted what to actually do
+when a missing extra has been identified.
+
+The sensible answers to this questions are intimately linked to *where* in the
+code the missing extra detection and import of the optional dependencies should
+be performed, so we will look at our options for that as well.
+
+Import at module level, raise exception
+---------------------------------------
+
+If your package is a library and the feature that requires the extra is
+localized to a specific module or sub-package of your package, one option is to
+just raise a custom exception indicating which extra would be required:
+
+.. code-block:: python
+
+   @dataclass
+   class MissingExtra(Exception):
+     name: str
+
+   ...
+
+   # if extra not installed (see previous sections):
+   raise MissingExtra("your-extra")
+
+Library consumers will then have to either depend on your library with the
+extra enabled or handle the possibility that imports of this specific module
+fail (putting them in the same situation you were in). Because imports raising
+custom exceptions is highly unusual, you should make sure to document this in a
+**very** visible manner.
+
+If your package is an application, making *you* the module's consumer, and you
+want the application to work without the extra installed (i.e. the extra only
+provides optional functionality for the application), you've similarly "pushed"
+the problem of dealing with failing imports up one layer. At some point in the
+module dependency you'll have to switch to a different strategy, lest your
+application just crash with an exception on startup.
+
+
+Import at module level, replace with exception-raising dummies
+--------------------------------------------------------------
+
+An alternative is to delay raising the exception until an actual attempt is
+made to *use* the missing dependency. One way to do this is to assign "dummy"
+functions that do nothing but raise it to the would-be imported names in the
+event that the extra is missing:
+
+.. code-block:: python
+
+   # if extra installed (see previous sections):
+   import some_function from optional_dependency
+
+   ...
+
+   # if extra not installed (see previous sections):
+   def raise_missing_extra(*args, **kwargs):
+     raise MissingExtra("your-extra")
+
+   optional_dependency = raise_missing_extra
+
+Note that, if imports are not mere functions but also objects like classes that
+are subclassed from, the except shape of the dummy objects can get more
+involved depending on the expected usage, e.g.
+
+.. code-block:: python
+
+   class RaiseMissingExtra:
+     def __init__(self, *args, **kwargs):
+       raise MissingExtra("your-extra")
+
+which would in turn not be sufficient for a class with class methods that might
+be used without instantiating it, and so on.
+
+By delaying the exception until attempted usage, an application installed
+without the extra can start and run normally until the user tries to use
+functionality requiring the extra, at which point you can handle it (e.g.
+display an appropriate error message).
+
+TODO mention that 3rd party library that does this automatically
+
+Import at function/method level, raise exception
+------------------------------------------------
+
+Lastly, another way to delay exception raising until actual usage is to only
+perform the check for whether the extra is installed and the corresponding
+import when the functionality requiring it is actually used. E.g.:
+
+.. code-block:: python
+
+   def import_extra_func_if_avail():
+     # surround this with the appropriate checks / error handling:
+     ...
+     from your_optional_dependency import extra_func
+     ...
+
+     return extra_func
+
+   ...
+
+   def some_func_requiring_your_extra():
+     try:
+       some_function = import_extra_func_if_avail()
+     except MissingExtra:
+       ...  # handle missing extra
+
+While this solution is more robust than the one from the preceding subsection,
+it can take more effort to make it work with static type checking.
+
+Interaction with static type checking
+=====================================
+
+TODO either put here or directly in previous sections... not sure
+
+Other considerations
+====================
+
+TODO mention that you might want to provide a way for users to check
+     availability without performing another action for the last 2 methods
+
+
 ------------------
 
 .. _packaging-problems-317: https://github.com/pypa/packaging-problems/issues/317

From e0e790e92d885fdb7eec15cca6158cd5a6577b9e Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Sun, 6 Oct 2024 23:45:06 +0200
Subject: [PATCH 06/15] Fix typo

---
 source/guides/handling-missing-extras-at-runtime.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index c92b1381d..9cf0fa27e 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -148,8 +148,8 @@ event that the extra is missing:
    optional_dependency = raise_missing_extra
 
 Note that, if imports are not mere functions but also objects like classes that
-are subclassed from, the except shape of the dummy objects can get more
-involved depending on the expected usage, e.g.
+are subclassed from, the exact shape of the dummy objects can get more involved
+depending on the expected usage, e.g.
 
 .. code-block:: python
 

From 731aa07bb2fda0f5c95d6535138ca4f81f6e2bf1 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Sun, 6 Oct 2024 23:52:05 +0200
Subject: [PATCH 07/15] Cross-reference `ModuleNotFoundError` properly
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) <wk.cvs.github@sydorenko.org.ua>
---
 source/guides/handling-missing-extras-at-runtime.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 9cf0fa27e..7b10b5f84 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -6,7 +6,7 @@ Handling missing optional dependencies ("extras") at runtime
 
 If your package has :ref:`optional dependencies ("extras")
 <metadata_provides_extra>` which the package consumer hasn't installed, the
-default outcome is an ordinary ``ModuleNotFoundError`` exception being raised
+default outcome is an ordinary :exc:`ModuleNotFoundError` exception being raised
 at the first attempted import of a missing module.
 
 This can make for a bad user experience, because there is no guidance about why

From 444008c7b159ef33ba698e478d4aab0628e5d38e Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Mon, 7 Oct 2024 22:49:31 +0200
Subject: [PATCH 08/15] Make language clearer in intro

---
 source/guides/handling-missing-extras-at-runtime.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 7b10b5f84..1840a36a2 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -13,7 +13,7 @@ This can make for a bad user experience, because there is no guidance about why
 the module is missing - users might think they've found a bug. If you're not
 careful, it can even make your package unusable without the extras installed,
 e.g. if your package is a library that imports the affected modules from the
-top-level module or from the application entry point if it's an application.
+top-level module or if it's an application that imports them unconditionally.
 
 As of the time of writing, there is no *great* way to handle this issue in
 the Python packaging ecosystem, but there are a few options that might be

From 3bda5892c6c3c2cc958a44652c5efde6e42ddc64 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Mon, 7 Oct 2024 23:04:15 +0200
Subject: [PATCH 09/15] Add warning to pkg_resources section & fix link

---
 .../handling-missing-extras-at-runtime.rst    | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 1840a36a2..e4a425111 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -45,17 +45,20 @@ because another installed package depends on it with a wider version
 requirement than specified by your extra).
 
 
-Using ``pkg_resources``
------------------------
+Using ``pkg_resources`` (deprecated)
+------------------------------------
 
-The now-deprecated :ref:`pkg_resources <ResourceManager API>` package (part of
-the ``setuptools`` distribution) provides a ``require`` function that you can
-use to check if a given optional dependency of your package is installed or
-not:
+.. attention::
 
-.. :: TODO ask setuptools to add labels for pkg_resources & require, then link
-      properly
+   ``pkg_resources`` is **deprecated** and the PyPA **strongly discourages**
+   its use.
+   This method is included in this guide for completeness's sake and only until
+   functionality with a similar level of comfort exists in
+   ``importlib.metadata`` or ``packaging``.
 
+The now-deprecated `pkg_resources <pkg_resources_>`_ package (part of the
+``setuptools`` distribution) provides a ``require`` function that you can use
+to check if a given optional dependency of your package is installed or not:
 
 .. code-block:: python
 
@@ -209,6 +212,8 @@ TODO mention that you might want to provide a way for users to check
 
 ------------------
 
+.. _pkg_resources: https://setuptools.pypa.io/en/latest/pkg_resources.html
+
 .. _packaging-problems-317: https://github.com/pypa/packaging-problems/issues/317
 
 .. _packaging-problems-664: https://github.com/pypa/packaging-problems/issues/664

From d7ba10566d82f88619e047d99b29dd874d760fc6 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Mon, 7 Oct 2024 23:50:55 +0200
Subject: [PATCH 10/15] Extend hbutils section & put before pkg_resources

---
 .../handling-missing-extras-at-runtime.rst    | 53 ++++++++++++-------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index e4a425111..6840d528e 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -45,6 +45,37 @@ because another installed package depends on it with a wider version
 requirement than specified by your extra).
 
 
+Using ``importlib.metadata`` and ``packaging``
+----------------------------------------------
+
+As a safer alternative that does check whether the optional dependencies are
+installed at the correct versions, :py:mod:`importlib.metadata` and
+:ref:`packaging` can be used to iterate through the extra's requirements
+recursively and check whether all are installed in the current environment.
+
+This process is currently quite involved. An implementation can be found in
+`packaging-problems #664 <packaging-problems-664_>`_, which is also made
+available in the `hbutils <https://pypi.org/project/hbutils/>`_ package as
+``hbutils.system.check_reqs``.
+The possibility of offering a similar helper function in ``importlib.metadata``
+or ``packaging`` themselves is still being discussed
+(`packaging-problems #317 <packaging-problems-317_>`_).
+
+With ``check_reqs`` included in your codebase or imported from ``hbutils``,
+usage is as simple as:
+
+.. code-block:: python
+
+   extra_installed = check_reqs(["your-package[your-extra]"])
+
+In contrast to the method above, this is typically done in :term:`LBYL` style
+prior to importing the modules in question.
+In principle, it could also be done after the imports succeeded just to check
+the version, in which case the imports themselves would have to be wrapped in a
+``try``-``except`` block to handle the possibility of not being installed at
+all.
+
+
 Using ``pkg_resources`` (deprecated)
 ------------------------------------
 
@@ -53,12 +84,13 @@ Using ``pkg_resources`` (deprecated)
    ``pkg_resources`` is **deprecated** and the PyPA **strongly discourages**
    its use.
    This method is included in this guide for completeness's sake and only until
-   functionality with a similar level of comfort exists in
+   functionality with a similar level of convenience exists in
    ``importlib.metadata`` or ``packaging``.
 
 The now-deprecated `pkg_resources <pkg_resources_>`_ package (part of the
-``setuptools`` distribution) provides a ``require`` function that you can use
-to check if a given optional dependency of your package is installed or not:
+``setuptools`` distribution) provides a ``require`` function, which was the
+inspiration for ``check_reqs`` from the previous section. Its usage is quite
+similar to ``check_reqs`` but not identical:
 
 .. code-block:: python
 
@@ -71,21 +103,6 @@ to check if a given optional dependency of your package is installed or not:
    except VersionConflict:
      ...  # handle version mismatches
 
-Unfortunately, no drop-in replacement for this functionality exists in
-``pkg_resources``'s "official" successor packages yet
-(`packaging-problems #317 <packaging-problems-317_>`_).
-
-
-Using 3rd-party libraries
--------------------------
-
-In response to the aforementioned lack of a replacement for
-``pkg_resources.require``, at least one 3rd party implementation of this
-functionality using only the ``packaging`` and ``importlib.metadata`` modules
-has been created (`packaging-problems #664 <packaging-problems-664_>`_) and
-made available in the 3rd-party `hbutils <https://pypi.org/project/hbutils/>`_
-package as ``hbutils.system.check_reqs``.
-
 
 Handling missing extras
 =======================

From 2decc263eb22b9ca7ce7ac96c4ba73c2918c979b Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Tue, 8 Oct 2024 00:14:09 +0200
Subject: [PATCH 11/15] Add type checking considerations

---
 .../handling-missing-extras-at-runtime.rst    | 48 +++++++++++++++----
 1 file changed, 40 insertions(+), 8 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 6840d528e..ed5634af8 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -123,6 +123,8 @@ just raise a custom exception indicating which extra would be required:
 
 .. code-block:: python
 
+   from dataclasses import dataclass
+
    @dataclass
    class MissingExtra(Exception):
      name: str
@@ -196,29 +198,59 @@ import when the functionality requiring it is actually used. E.g.:
 
 .. code-block:: python
 
-   def import_extra_func_if_avail():
+   def import_extra_module_if_avail():
      # surround this with the appropriate checks / error handling:
      ...
-     from your_optional_dependency import extra_func
+     import your_optional_dependency
      ...
 
-     return extra_func
+     return your_optional_dependency
 
    ...
 
    def some_func_requiring_your_extra():
      try:
-       some_function = import_extra_func_if_avail()
+       optional_module = import_extra_module_if_avail()
      except MissingExtra:
        ...  # handle missing extra
 
+     # now you can use functionality from the optional dependency, e.g.:
+     optional_module.extra_func(...)
+
 While this solution is more robust than the one from the preceding subsection,
-it can take more effort to make it work with static type checking.
+it can take more effort to make it work with
+:term:`static type checking <static type checker>`:
+To correctly statically type a function returning a module, you'd have to
+introduce an "artificial" type representing the latter, e.g.
+
+.. code-block:: python
+
+   from typing import cast, Protocol
+
+   class YourOptionalModuleType(Protocol):
+     extra_func: Callable[...]
+     ...  # other objects you want to use
+
+   def some_func_requiring_your_extra() -> YourOptionalModuleType:
+     ...
+
+     return cast(YourOptionalModuleType, optional_module)
+
+An alternative would be to instead have functions that import and return only
+the objects you actually need:
+
+.. code-block:: python
+
+   def import_extra_func_if_avail() -> Callable[...]:
+     # surround this with the appropriate checks / error handling:
+     ...
+     from your_optional_dependency import extra_func
+     ...
+
+     return extra_func
 
-Interaction with static type checking
-=====================================
+But this can become verbose when you import a lot of names.
 
-TODO either put here or directly in previous sections... not sure
 
 Other considerations
 ====================

From f63711de4d08dd6ec93276e089c3a6d9dc58257d Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Fri, 11 Oct 2024 20:02:52 +0200
Subject: [PATCH 12/15] Mention generalimport library

---
 source/guides/handling-missing-extras-at-runtime.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index ed5634af8..8a16e9d78 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -187,7 +187,8 @@ without the extra can start and run normally until the user tries to use
 functionality requiring the extra, at which point you can handle it (e.g.
 display an appropriate error message).
 
-TODO mention that 3rd party library that does this automatically
+The `generalimport`_ library can automate this process by hooking into the
+import system.
 
 Import at function/method level, raise exception
 ------------------------------------------------
@@ -266,3 +267,5 @@ TODO mention that you might want to provide a way for users to check
 .. _packaging-problems-317: https://github.com/pypa/packaging-problems/issues/317
 
 .. _packaging-problems-664: https://github.com/pypa/packaging-problems/issues/664
+
+.. _generalimport: https://github.com/ManderaGeneral/generalimport

From c10905807762cc6a7037cfb84b46f1836c2cff7e Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Fri, 11 Oct 2024 20:29:01 +0200
Subject: [PATCH 13/15] Put modified hbutils script into guide verbatim

---
 .../handling-missing-extras-at-runtime.rst    | 68 ++++++++++++++-----
 1 file changed, 52 insertions(+), 16 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 8a16e9d78..969678caa 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -51,25 +51,59 @@ Using ``importlib.metadata`` and ``packaging``
 As a safer alternative that does check whether the optional dependencies are
 installed at the correct versions, :py:mod:`importlib.metadata` and
 :ref:`packaging` can be used to iterate through the extra's requirements
-recursively and check whether all are installed in the current environment.
-
-This process is currently quite involved. An implementation can be found in
-`packaging-problems #664 <packaging-problems-664_>`_, which is also made
-available in the `hbutils <https://pypi.org/project/hbutils/>`_ package as
-``hbutils.system.check_reqs``.
-The possibility of offering a similar helper function in ``importlib.metadata``
-or ``packaging`` themselves is still being discussed
-(`packaging-problems #317 <packaging-problems-317_>`_).
-
-With ``check_reqs`` included in your codebase or imported from ``hbutils``,
-usage is as simple as:
+recursively and check whether all are installed in the current environment
+(based on `code <hbutils-snippet_>`_ from the `hbutils`_ library):
 
 .. code-block:: python
 
+   # TODO Unless we get special permission, this snippet is Apache-2-licensed:
+   # https://github.com/HansBug/hbutils/blob/927b0757449a781ce8e30132f26b06089a24cd71/LICENSE
+
+   from collections.abc import Iterable
+   from importlib.metadata import PackageNotFoundError, distribution, metadata
+
+   from packaging.metadata import Metadata
+   from packaging.requirements import Requirement
+
+   def check_reqs(req_strs: Iterable[str]) -> bool:
+     return all(
+       _check_req_recursive(req)
+       for req_str in req_strs
+       if not (req := Requirement(req_str)).marker or req.marker.evaluate()
+     )
+
+   def _check_req_recursive(req: Requirement) -> bool:
+     try:
+       version = distribution(req.name).version
+     except PackageNotFoundError:
+       return False  # req not installed
+
+     if not req.specifier.contains(version):
+       return False  # req version does not match
+
+     req_metadata = Metadata.from_raw(metadata(req.name).json, validate=False)
+     for child_req in req_metadata.requires_dist or []:
+       for extra in req.extras:
+         if child_req.marker and child_req.marker.evaluate({"extra": extra}):
+           if not _check_req_recursive(child_req):
+             return False
+           break
+
+     return True
+
+
+   # Perform check, e.g.:
    extra_installed = check_reqs(["your-package[your-extra]"])
 
-In contrast to the method above, this is typically done in :term:`LBYL` style
-prior to importing the modules in question.
+TODO Either point out that this snippet doesn't actually check everything
+     (https://github.com/HansBug/hbutils/issues/109) or fix it.
+
+The possibility of offering a helper function similar to ``check_reqs`` in
+``importlib.metadata`` or ``packaging`` themselves is still being discussed
+(`packaging-problems #317 <packaging-problems-317_>`_).
+
+In contrast to the method above, this check is typically done in :term:`LBYL`
+style prior to importing the modules in question.
 In principle, it could also be done after the imports succeeded just to check
 the version, in which case the imports themselves would have to be wrapped in a
 ``try``-``except`` block to handle the possibility of not being installed at
@@ -262,10 +296,12 @@ TODO mention that you might want to provide a way for users to check
 
 ------------------
 
+.. _hbutils-snippet: https://github.com/HansBug/hbutils/blob/927b0757449a781ce8e30132f26b06089a24cd71/hbutils/system/python/package.py#L171-L242
+
+.. _hbutils: https://pypi.org/project/hbutils/
+
 .. _pkg_resources: https://setuptools.pypa.io/en/latest/pkg_resources.html
 
 .. _packaging-problems-317: https://github.com/pypa/packaging-problems/issues/317
 
-.. _packaging-problems-664: https://github.com/pypa/packaging-problems/issues/664
-
 .. _generalimport: https://github.com/ManderaGeneral/generalimport

From 6d0535df1c8bb2b89d1f0f46c5f4ed738ed1af69 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Fri, 11 Oct 2024 20:42:49 +0200
Subject: [PATCH 14/15] Replace detailed handling w/ general idea

---
 .../handling-missing-extras-at-runtime.rst    | 161 ++----------------
 1 file changed, 10 insertions(+), 151 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 969678caa..657dcd192 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -141,157 +141,18 @@ similar to ``check_reqs`` but not identical:
 Handling missing extras
 =======================
 
-In each of the previous section's code snippets, we omitted what to actually do
-when a missing extra has been identified.
+Where and how to embed the detection of missing extras in a package and what
+actions to take upon learning the outcome depends on the specifics of both the
+package and feature requiring the extra.
+Some common options are:
 
-The sensible answers to this questions are intimately linked to *where* in the
-code the missing extra detection and import of the optional dependencies should
-be performed, so we will look at our options for that as well.
+- Raise a custom exception that includes the name of the missing extra.
+- In applications, show an error message when an attempt is made to use the
+  feature that requires the extra.
+- In libraries, provide a function that lets library consumers query which
+  features are available.
 
-Import at module level, raise exception
----------------------------------------
-
-If your package is a library and the feature that requires the extra is
-localized to a specific module or sub-package of your package, one option is to
-just raise a custom exception indicating which extra would be required:
-
-.. code-block:: python
-
-   from dataclasses import dataclass
-
-   @dataclass
-   class MissingExtra(Exception):
-     name: str
-
-   ...
-
-   # if extra not installed (see previous sections):
-   raise MissingExtra("your-extra")
-
-Library consumers will then have to either depend on your library with the
-extra enabled or handle the possibility that imports of this specific module
-fail (putting them in the same situation you were in). Because imports raising
-custom exceptions is highly unusual, you should make sure to document this in a
-**very** visible manner.
-
-If your package is an application, making *you* the module's consumer, and you
-want the application to work without the extra installed (i.e. the extra only
-provides optional functionality for the application), you've similarly "pushed"
-the problem of dealing with failing imports up one layer. At some point in the
-module dependency you'll have to switch to a different strategy, lest your
-application just crash with an exception on startup.
-
-
-Import at module level, replace with exception-raising dummies
---------------------------------------------------------------
-
-An alternative is to delay raising the exception until an actual attempt is
-made to *use* the missing dependency. One way to do this is to assign "dummy"
-functions that do nothing but raise it to the would-be imported names in the
-event that the extra is missing:
-
-.. code-block:: python
-
-   # if extra installed (see previous sections):
-   import some_function from optional_dependency
-
-   ...
-
-   # if extra not installed (see previous sections):
-   def raise_missing_extra(*args, **kwargs):
-     raise MissingExtra("your-extra")
-
-   optional_dependency = raise_missing_extra
-
-Note that, if imports are not mere functions but also objects like classes that
-are subclassed from, the exact shape of the dummy objects can get more involved
-depending on the expected usage, e.g.
-
-.. code-block:: python
-
-   class RaiseMissingExtra:
-     def __init__(self, *args, **kwargs):
-       raise MissingExtra("your-extra")
-
-which would in turn not be sufficient for a class with class methods that might
-be used without instantiating it, and so on.
-
-By delaying the exception until attempted usage, an application installed
-without the extra can start and run normally until the user tries to use
-functionality requiring the extra, at which point you can handle it (e.g.
-display an appropriate error message).
-
-The `generalimport`_ library can automate this process by hooking into the
-import system.
-
-Import at function/method level, raise exception
-------------------------------------------------
-
-Lastly, another way to delay exception raising until actual usage is to only
-perform the check for whether the extra is installed and the corresponding
-import when the functionality requiring it is actually used. E.g.:
-
-.. code-block:: python
-
-   def import_extra_module_if_avail():
-     # surround this with the appropriate checks / error handling:
-     ...
-     import your_optional_dependency
-     ...
-
-     return your_optional_dependency
-
-   ...
-
-   def some_func_requiring_your_extra():
-     try:
-       optional_module = import_extra_module_if_avail()
-     except MissingExtra:
-       ...  # handle missing extra
-
-     # now you can use functionality from the optional dependency, e.g.:
-     optional_module.extra_func(...)
-
-While this solution is more robust than the one from the preceding subsection,
-it can take more effort to make it work with
-:term:`static type checking <static type checker>`:
-To correctly statically type a function returning a module, you'd have to
-introduce an "artificial" type representing the latter, e.g.
-
-.. code-block:: python
-
-   from typing import cast, Protocol
-
-   class YourOptionalModuleType(Protocol):
-     extra_func: Callable[...]
-     ...  # other objects you want to use
-
-   def some_func_requiring_your_extra() -> YourOptionalModuleType:
-     ...
-
-     return cast(YourOptionalModuleType, optional_module)
-
-An alternative would be to instead have functions that import and return only
-the objects you actually need:
-
-.. code-block:: python
-
-   def import_extra_func_if_avail() -> Callable[...]:
-     # surround this with the appropriate checks / error handling:
-     ...
-     from your_optional_dependency import extra_func
-     ...
-
-     return extra_func
-
-But this can become verbose when you import a lot of names.
-
-
-Other considerations
-====================
-
-TODO mention that you might want to provide a way for users to check
-     availability without performing another action for the last 2 methods
+... and probably more.
 
 
 ------------------
@@ -303,5 +164,3 @@ TODO mention that you might want to provide a way for users to check
 .. _pkg_resources: https://setuptools.pypa.io/en/latest/pkg_resources.html
 
 .. _packaging-problems-317: https://github.com/pypa/packaging-problems/issues/317
-
-.. _generalimport: https://github.com/ManderaGeneral/generalimport

From d933a545902367ef15b4796f07147a35046e0290 Mon Sep 17 00:00:00 2001
From: Shahriar Heidrich <smheidrich@weltenfunktion.de>
Date: Tue, 29 Oct 2024 00:32:05 +0100
Subject: [PATCH 15/15] Fix hbutils snippet for transitive non-extra deps

---
 .../handling-missing-extras-at-runtime.rst    | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/source/guides/handling-missing-extras-at-runtime.rst b/source/guides/handling-missing-extras-at-runtime.rst
index 657dcd192..2d2b5a2af 100644
--- a/source/guides/handling-missing-extras-at-runtime.rst
+++ b/source/guides/handling-missing-extras-at-runtime.rst
@@ -56,15 +56,17 @@ recursively and check whether all are installed in the current environment
 
 .. code-block:: python
 
-   # TODO Unless we get special permission, this snippet is Apache-2-licensed:
+   # Adapted from (see there for copyright & license):
    # https://github.com/HansBug/hbutils/blob/927b0757449a781ce8e30132f26b06089a24cd71/LICENSE
+   # SPDX-License-Identifier: Apache-2.0
 
    from collections.abc import Iterable
    from importlib.metadata import PackageNotFoundError, distribution, metadata
 
-   from packaging.metadata import Metadata
+   from packaging.metadata import Metadata, RawMetadata
    from packaging.requirements import Requirement
 
+
    def check_reqs(req_strs: Iterable[str]) -> bool:
      return all(
        _check_req_recursive(req)
@@ -72,6 +74,7 @@ recursively and check whether all are installed in the current environment
        if not (req := Requirement(req_str)).marker or req.marker.evaluate()
      )
 
+
    def _check_req_recursive(req: Requirement) -> bool:
      try:
        version = distribution(req.name).version
@@ -83,11 +86,16 @@ recursively and check whether all are installed in the current environment
 
      req_metadata = Metadata.from_raw(metadata(req.name).json, validate=False)
      for child_req in req_metadata.requires_dist or []:
-       for extra in req.extras:
-         if child_req.marker and child_req.marker.evaluate({"extra": extra}):
-           if not _check_req_recursive(child_req):
-             return False
-           break
+       # A dependency is only required to be present if ...
+       if (
+         not child_req.marker  # ... it doesn't have a marker
+         or child_req.marker.evaluate()  # ... its marker matches our env
+         or any(  # ... its marker matches our env given one of our extras
+           child_req.marker.evaluate({"extra": extra}) for extra in req.extras
+         )
+       ):
+         if not _check_req_recursive(child_req):
+           return False
 
      return True
 
@@ -95,9 +103,6 @@ recursively and check whether all are installed in the current environment
    # Perform check, e.g.:
    extra_installed = check_reqs(["your-package[your-extra]"])
 
-TODO Either point out that this snippet doesn't actually check everything
-     (https://github.com/HansBug/hbutils/issues/109) or fix it.
-
 The possibility of offering a helper function similar to ``check_reqs`` in
 ``importlib.metadata`` or ``packaging`` themselves is still being discussed
 (`packaging-problems #317 <packaging-problems-317_>`_).