API Reference building script update (#13587)

The namespaces like `langchain.agents.format_scratchpad` clogging the
API Reference sidebar.
This change removes those 3-level namespaces from sidebar (this issue
was discussed with @efriis )

---------

Co-authored-by: Erick Friis <[email protected]>

.gitignore

@@ -167,8 +167,7 @@ docs/node_modules/
 docs/.docusaurus/
 docs/.cache-loader/
 docs/_dist
-docs/api_reference/api_reference.rst
-docs/api_reference/experimental_api_reference.rst
+docs/api_reference/*api_reference.rst
 docs/api_reference/_build
 docs/api_reference/*/
 !docs/api_reference/_static/

docs/api_reference/create_api_rst.py

@@ -196,11 +196,13 @@ def _load_package_modules(
     return modules_by_namespace
 
 
-def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) -> str:
+def _construct_doc(
+    package_namespace: str, members_by_namespace: Dict[str, ModuleMembers]
+) -> str:
     """Construct the contents of the reference.rst file for the given package.
 
     Args:
-        pkg: The package name
+        package_namespace: The package top level namespace
         members_by_namespace: The members of the package, dict organized by top level
                               module contains a list of classes and functions
                               inside of the top level namespace.
@@ -210,7 +212,7 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
     """
     full_doc = f"""\
 =======================
-``{pkg}`` API Reference
+``{package_namespace}`` API Reference
 =======================
 
 """
@@ -222,13 +224,13 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
         functions = _members["functions"]
         if not (classes or functions):
             continue
-        section = f":mod:`{pkg}.{module}`"
+        section = f":mod:`{package_namespace}.{module}`"
         underline = "=" * (len(section) + 1)
         full_doc += f"""\
 {section}
 {underline}
 
-.. automodule:: {pkg}.{module}
+.. automodule:: {package_namespace}.{module}
     :no-members:
     :no-inherited-members:
 
@@ -238,7 +240,7 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
             full_doc += f"""\
 Classes
 --------------
-.. currentmodule:: {pkg}
+.. currentmodule:: {package_namespace}
 
 .. autosummary::
     :toctree: {module}
@@ -270,7 +272,7 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
             full_doc += f"""\
 Functions
 --------------
-.. currentmodule:: {pkg}
+.. currentmodule:: {package_namespace}
 
 .. autosummary::
     :toctree: {module}
@@ -282,57 +284,57 @@ def _construct_doc(pkg: str, members_by_namespace: Dict[str, ModuleMembers]) ->
     return full_doc
 
 
-def _document_langchain_experimental() -> None:
-    """Document the langchain_experimental package."""
-    # Generate experimental_api_reference.rst
-    exp_members = _load_package_modules(EXP_DIR)
-    exp_doc = ".. _experimental_api_reference:\n\n" + _construct_doc(
-        "langchain_experimental", exp_members
-    )
-    with open(EXP_WRITE_FILE, "w") as f:
-        f.write(exp_doc)
+def _build_rst_file(package_name: str = "langchain") -> None:
+    """Create a rst file for building of documentation.
 
+    Args:
+        package_name: Can be either "langchain" or "core" or "experimental".
+    """
+    package_members = _load_package_modules(_package_dir(package_name))
+    with open(_out_file_path(package_name), "w") as f:
+        f.write(
+            _doc_first_line(package_name)
+            + _construct_doc(package_namespace[package_name], package_members)
+        )
 
-def _document_langchain_core() -> None:
-    """Document the langchain_core package."""
-    # Generate core_api_reference.rst
-    core_members = _load_package_modules(CORE_DIR)
-    core_doc = ".. _core_api_reference:\n\n" + _construct_doc(
-        "langchain_core", core_members
-    )
-    with open(CORE_WRITE_FILE, "w") as f:
-        f.write(core_doc)
-
-
-def _document_langchain() -> None:
-    """Document the main langchain package."""
-    # load top level module members
-    lc_members = _load_package_modules(PKG_DIR)
-
-    # Add additional packages
-    tools = _load_package_modules(PKG_DIR, "tools")
-    agents = _load_package_modules(PKG_DIR, "agents")
-    schema = _load_package_modules(PKG_DIR, "schema")
-
-    lc_members.update(
-        {
-            "agents.output_parsers": agents["output_parsers"],
-            "agents.format_scratchpad": agents["format_scratchpad"],
-            "tools.render": tools["render"],
-        }
-    )
 
-    lc_doc = ".. _api_reference:\n\n" + _construct_doc("langchain", lc_members)
+package_namespace = {
+    "langchain": "langchain",
+    "experimental": "langchain_experimental",
+    "core": "langchain_core",
+}
+
+
+def _package_dir(package_name: str = "langchain") -> Path:
+    """Return the path to the directory containing the documentation."""
+    return ROOT_DIR / "libs" / package_name / package_namespace[package_name]
+
+
+def _out_file_path(package_name: str = "langchain") -> Path:
+    """Return the path to the file containing the documentation."""
+    name_prefix = {
+        "langchain": "",
+        "experimental": "experimental_",
+        "core": "core_",
+    }
+    return HERE / f"{name_prefix[package_name]}api_reference.rst"
+
 
-    with open(WRITE_FILE, "w") as f:
-        f.write(lc_doc)
+def _doc_first_line(package_name: str = "langchain") -> str:
+    """Return the path to the file containing the documentation."""
+    prefix = {
+        "langchain": "",
+        "experimental": "experimental",
+        "core": "core",
+    }
+    return f".. {prefix[package_name]}_api_reference:\n\n"
 
 
 def main() -> None:
-    """Generate the reference.rst file for each package."""
-    _document_langchain()
-    _document_langchain_experimental()
-    _document_langchain_core()
+    """Generate the api_reference.rst file for each package."""
+    _build_rst_file(package_name="core")
+    _build_rst_file(package_name="langchain")
+    _build_rst_file(package_name="experimental")
 
 
 if __name__ == "__main__":