feat(protoc): support using grpc_tools.protoc

cpcloud · Mar 19, 2023 · eb152b0 · eb152b0
1 parent 22e0209
commit eb152b0
Show file tree

Hide file tree

Showing 5 changed files with 143 additions and 8 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -158,7 +158,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           environment-file: environment.yml
 
-      - run: mamba install grpc-cpp
+      - run: mamba install grpc-cpp grpcio-tools
       - run: pip install .
       - run: pytest -ra
   conda-windows:
@@ -203,7 +203,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
           environment-file: environment.yml
 
-      - run: mamba install grpc-cpp pip
+      - run: mamba install grpc-cpp grpcio-tools pip
       - run: pip install .
       - run: pytest -ra
   tensorflow:

diff --git a/poetry.lock b/poetry.lock
diff --git a/protoletariat/fdsetgen.py b/protoletariat/fdsetgen.py
@@ -4,6 +4,7 @@
 import fnmatch
 import itertools
 import re
+import shlex
 import subprocess
 import tempfile
 from pathlib import Path
@@ -154,7 +155,7 @@ def generate_file_descriptor_set_bytes(self) -> bytes:
             filename = Path(f.name)
             subprocess.check_output(
                 [
-                    self.protoc_path,
+                    *shlex.split(self.protoc_path),
                     "--include_imports",
                     f"--descriptor_set_out={filename}",
                     *map("--proto_path={}".format, self.proto_paths),

diff --git a/protoletariat/tests/conftest.py b/protoletariat/tests/conftest.py
@@ -5,8 +5,10 @@
 import itertools
 import json
 import os
+import shlex
 import shutil
 import subprocess
+import sys
 import tempfile
 from functools import partial
 from pathlib import Path
@@ -138,6 +140,8 @@ def do_generate(self, cli: CliRunner, *, args: Iterable[str] = ()) -> Result:
 
 
 class ProtocFixture(ProtoletariatFixture):
+    protoc_exe = ("protoc",)
+
     def __init__(
         self,
         *,
@@ -161,7 +165,7 @@ def __init__(
 
     def do_generate(self, cli: CliRunner, *, args: Iterable[str] = ()) -> Result:
         protoc_args = [
-            "protoc",
+            *self.protoc_exe,
             "--proto_path",
             str(self.base_dir),
             "--python_out",
@@ -190,6 +194,8 @@ def do_generate(self, cli: CliRunner, *, args: Iterable[str] = ()) -> Result:
                 str(self.package_dir),
                 *args,
                 "protoc",
+                "--protoc-path",
+                shlex.join(self.protoc_exe),
                 "--proto-path",
                 str(self.base_dir),
                 *(str(filename) for filename, _ in self.proto_texts),
@@ -198,6 +204,10 @@ def do_generate(self, cli: CliRunner, *, args: Iterable[str] = ()) -> Result:
         )
 
 
+class GrpcIoToolsFixture(ProtocFixture):
+    protoc_exe = sys.executable, "-m", "grpc_tools.protoc"
+
+
 class RawFixture(ProtoletariatFixture):
     def __init__(
         self,
@@ -319,6 +329,10 @@ def basic_cli_texts(request: SubRequest) -> list[ProtoFile]:
             partial(ProtocFixture, package="basic_cli"),
             id="basic_cli_protoc",
         ),
+        pytest.param(
+            partial(GrpcIoToolsFixture, package="basic_cli"),
+            id="basic_cli_grpc_io_tools",
+        ),
         pytest.param(
             partial(RawFixture, package="basic_cli"),
             id="basic_cli_raw",
@@ -398,6 +412,10 @@ def thing_service_texts(request: SubRequest) -> list[ProtoFile]:
             partial(ProtocFixture, package="thing_service", grpc=True),
             id="thing_service_protoc",
         ),
+        pytest.param(
+            partial(GrpcIoToolsFixture, package="thing_service", grpc=True),
+            id="thing_service_grpc_io_tools",
+        ),
         pytest.param(
             partial(RawFixture, package="thing_service", grpc=True),
             id="thing_service_raw",
@@ -453,6 +471,9 @@ def nested_texts() -> list[ProtoFile]:
             id="nested_buf",
         ),
         pytest.param(partial(ProtocFixture, package="nested"), id="nested_protoc"),
+        pytest.param(
+            partial(GrpcIoToolsFixture, package="nested"), id="nested_grpc_io_tools"
+        ),
         pytest.param(partial(RawFixture, package="nested"), id="nested_raw"),
     ]
 )
@@ -512,6 +533,16 @@ def no_imports_service_texts(request: SubRequest) -> list[ProtoFile]:
             ),
             id="no_imports_service_protoc",
         ),
+        pytest.param(
+            partial(
+                GrpcIoToolsFixture,
+                package="no_imports_service",
+                grpc=True,
+                mypy=True,
+                mypy_grpc=True,
+            ),
+            id="no_imports_service_grpc_io_tools",
+        ),
         pytest.param(
             partial(
                 RawFixture,
@@ -601,6 +632,16 @@ def imports_service_texts(request: SubRequest) -> list[ProtoFile]:
             ),
             id="imports_service_protoc",
         ),
+        pytest.param(
+            partial(
+                GrpcIoToolsFixture,
+                package="imports_service",
+                grpc=True,
+                mypy=True,
+                mypy_grpc=True,
+            ),
+            id="imports_service_grpc_io_tools",
+        ),
         pytest.param(
             partial(
                 RawFixture,
@@ -659,6 +700,10 @@ def long_names_texts() -> list[ProtoFile]:
             partial(ProtocFixture, package="long_names", mypy=True),
             id="long_names_protoc",
         ),
+        pytest.param(
+            partial(GrpcIoToolsFixture, package="long_names", mypy=True),
+            id="long_names_grpc_io_tools",
+        ),
         pytest.param(
             partial(RawFixture, package="long_names", mypy=True),
             id="long_names_raw",
@@ -710,6 +755,10 @@ def ignored_import_texts(request: SubRequest) -> list[ProtoFile]:
             partial(ProtocFixture, package="ignored_imports"),
             id="ignored_imports_protoc",
         ),
+        pytest.param(
+            partial(GrpcIoToolsFixture, package="ignored_imports"),
+            id="ignored_imports_grpc_io_tools",
+        ),
         pytest.param(
             partial(RawFixture, package="ignored_imports"),
             id="ignored_imports_raw",

diff --git a/pyproject.toml b/pyproject.toml
@@ -16,13 +16,18 @@ classifiers = [
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
 ]
 
+[tool.poetry.scripts]
+protol = "protoletariat.__main__:main"
+
 [tool.poetry.dependencies]
 python = "^3.8"
 astunparse = { version = ">=1.6.3,<2", python = "<3.9" }
 click = ">=8,<9"
 protobuf = ">=3.19.1,<5"
+grpcio-tools = { version = ">=1.42.0,<2", optional = true }
 
 [tool.poetry.dev-dependencies]
 black = ">=22,<24"
@@ -35,8 +40,8 @@ pytest-randomly = ">=3.10.1,<4"
 ruff = ">=0.0.198,<1"
 types-protobuf = ">=4.21,<5"
 
-[tool.poetry.scripts]
-protol = "protoletariat.__main__:main"
+[tool.poetry.extras]
+grpcio-tools = ["grpcio-tools"]
 
 [tool.pytest.ini_options]
 xfail_strict = true