From 32b64572d1ebc40adc35a51fdb265a61f621f50f Mon Sep 17 00:00:00 2001 From: katelyn martin Date: Mon, 22 Jan 2024 18:45:21 -0500 Subject: [PATCH] =?UTF-8?q?pd:=20=F0=9F=94=A8=20rework=20RootCommand::star?= =?UTF-8?q?t=20auto-https=20logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` /!\ this is a work in progress and will /!\ /!\ be force pushed until ready for review /!\ ``` ## ๐Ÿ‘€ overview fixes #3627. this reorganizes the logic in pd's startup code related to automatically managed https functionality. ## ๐ŸŽจ background & motivation this PR, besides cleaning up the `rustls-acme`-related auto-https logic, is also interested in *creating a state-of-affairs that will dovetail into pr #3522*. in particular, this expression to start the GRPC serve given a bound listener... ```rust tokio::task::Builder::new() .name("grpc_server") .spawn(grpc_server.serve_with_incoming(tls_incoming)) .expect("failed to spawn grpc server") ``` ...should be adjusted so as to accept an `axum::Router`. other tertiary requirements: - when no `grpc_auto_https` flag has been configured, serve without using an ACME resolver - ALPN permit http/1.1 for grpc-web backwards compatibility. ### โš–๏ธ `rustls-acme` and `tokio-rustls-acme` quoth the #3627 description, citing an earlier comment: > In the ~year since this code was written, there may be better options. > `tokio-rustls-acme` seems promising \- for reference, the repositories for each live here, and here: - - after some comparison, i have come to the opinion that `rustls-acme` will still be adequate for our purposes. the latter is a fork of the former, but active development appears to have continued in the former, and i did not see any particular "_must-have_" features for us in the latter. ## ๐Ÿšฐ dropping down to `axum` (this part is lengthy...) as stated above, we want to switch to an `axum::Router`. this means that we won't be able to use the `AcmeConfig::incoming` function. the `rustls-acme` library provides some "low-level" examples we can check out: - - `low_level_axum` sounds promising, given that #3522 itself drops down from tonic into axum. a catch is that the "axum" example is referring to `axum-server`, a library that wraps `axum`, rather than `axum` itself. more on that momentarily. we also use `tonic` 0.10.2 in pd, and elsewhere in the penumbra monorepo. tonic isn't using hyper 1.x yet. this was being worked on in hyperium/tonic#1583, continued on in hyperium/tonic#1595, and tracked in hyperium/tonic#1579. that work also depends upon hyperium/hyper#3461. this is complicated by the fact that `axum-server` 0.6.0 depends on `hyper` 1.1.0. this is particularly relevant because of hyperium/hyper#3040. so, we must be mindful that there is now a `tower::Service` and a `hyper::Service`. `hyper-util` provides facilities to convert between the two, and other helper glue. additionally, see the "_upgrading"_ migration guide for more information on the differences between pre- and post-1.0 interfaces in `hyper`. - https://hyper.rs/guides/1/upgrading/ - at a high level we need to take our `tonic::Router`, and turn that into a `MakeService`, some object that can be used to _generate_ `Service` instances to process a request. this is additionally complicated by the fact that `axum-server` defines its _own_ `MakeService` trait. that trait is sealed, blanket implemented, and is a bound included in `axum_server::server::Server::serve`. ```rust // axum_server::serve impl Server { pub async fn serve(self, mut make_service: M) -> io::Result<()> where M: MakeService>, // ... { todo!() } } ``` the `SocketAddr` parameter above is of importance. `axum::Router::into_make_service` gives us `IntoMakeService` - ```rust impl Service for IntoMakeService where S: Clone, { type Response = S; type Error = Infallible; type Future = IntoMakeServiceFuture; ``` ...which means that we are faced with this error: ``` error[E0277]: the trait bound `Router: tower_service::Service>` is not satisfied --> crates/bin/pd/src/auto_https.rs:62:20 | 62 | .serve(make_svc) | ----- ^^^^^^^^ the trait `tower_service::Service>` is not implemented for `Router` | | | required by a bound introduced by this call | = help: the trait `tower_service::Service>` is implemented for `Router` = help: for that trait implementation, expected `axum::http::Request`, found `http::request::Request` = note: required for `IntoMakeService` to implement `axum_server::service::MakeService>` ``` `axum::Router::into_make_service` does not play well with `axum_server::Server`. ...and if we specify the new `hyper::body::incoming::Incoming` as the body type for the axum router, ``` error[E0599]: the method `into_make_service` exists for struct `Router<(), Incoming>`, but its trait bounds were not satisfied --> crates/bin/pd/src/auto_https.rs:57:31 | 57 | let make_svc = router.into_make_service(); | ^^^^^^^^^^^^^^^^^ method cannot be called on `Router<(), Incoming>` due to unsatisfied trait bounds | ::: /home/katie/.cargo/registry/src/index.crates.io-6f17d22bba15001f/hyper-1.1.0/src/body/incoming.rs:47:1 | 47 | pub struct Incoming { | ------------------- doesn't satisfy `hyper::body::Incoming: HttpBody` | = note: the following trait bounds were not satisfied: `hyper::body::Incoming: HttpBody` ``` `HttpBody` was the pre-1.0 trait, that is now `Body` post-1.0. in short, there are some incongruencies at play as different libraries catch up to the new breaking changes in `hyper`. ## ๐Ÿคจ ...so now what? TODO(kate): write up course of action for tomorrow. --- ๐ŸŸฅ NB: i worry about breakage due to untested load-bearing behavior during a migration like this. see "tertiary requirements", above. consider this a PR to be suspect of when deploying a new testnet. Refs: #3627 Refs: #3646 Refs: #3522 --- Cargo.lock | 308 +++++++++++++++++++++++++------- crates/bin/pd/Cargo.toml | 12 +- crates/bin/pd/src/auto_https.rs | 189 +++++++++++++++----- crates/bin/pd/src/main.rs | 64 ++----- 4 files changed, 412 insertions(+), 161 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe88e71f6d..fa910f45c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -602,7 +602,7 @@ checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ "async-channel 2.1.1", "async-executor", - "async-io 2.2.2", + "async-io 2.3.0", "async-lock 3.3.0", "blocking", "futures-lite 2.2.0", @@ -648,9 +648,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.2.2" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afaa937395a620e33dc6a742c593c01aced20aa376ffb0f628121198578ccc7" +checksum = "fb41eb19024a91746eba0773aa5e16036045bbf45733766661099e182ea6a744" dependencies = [ "async-lock 3.3.0", "cfg-if", @@ -658,7 +658,7 @@ dependencies = [ "futures-io", "futures-lite 2.2.0", "parking", - "polling 3.3.1", + "polling 3.3.2", "rustix 0.38.28", "slab", "tracing", @@ -719,7 +719,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e47d90f65a225c4527103a8d747001fc56e375203592b25ad103e1ca13124c5" dependencies = [ - "async-io 2.2.2", + "async-io 2.3.0", "async-lock 2.8.0", "atomic-waker", "cfg-if", @@ -870,9 +870,9 @@ dependencies = [ "bytes", "futures-util", "headers", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit 0.5.0", "memchr", @@ -901,9 +901,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit 0.7.3", "memchr", @@ -931,8 +931,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "mime", "tower-layer", "tower-service", @@ -947,8 +947,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -964,17 +964,40 @@ dependencies = [ "arc-swap", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "pin-project-lite", "rustls 0.20.9", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "tokio", "tokio-rustls 0.23.4", "tower-service", ] +[[package]] +name = "axum-server" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad46c3ec4e12f4a4b6835e173ba21c25e484c9d02b49770bf006ce5367c036" +dependencies = [ + "arc-swap", + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-util", + "pin-project-lite", + "rustls 0.21.10", + "rustls-pemfile 2.0.0", + "tokio", + "tokio-rustls 0.24.1", + "tower", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -1893,9 +1916,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.10.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ "generic-array", "subtle", @@ -2893,7 +2916,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.11", + "indexmap 2.1.0", + "slab", + "tokio", + "tokio-util 0.7.10", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31d030e59af851932b72ceebadf4a2b5986dba4c3b99dd2493f8273a0f151943" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.0.0", "indexmap 2.1.0", "slab", "tokio", @@ -2987,7 +3029,7 @@ dependencies = [ "base64 0.21.6", "bytes", "headers-core", - "http", + "http 0.2.11", "httpdate", "mime", "sha1 0.10.6", @@ -2999,7 +3041,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.11", ] [[package]] @@ -3101,6 +3143,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -3108,7 +3161,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.11", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.0.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +dependencies = [ + "bytes", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -3174,9 +3250,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -3188,6 +3264,25 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.2", + "http 1.0.0", + "http-body 1.0.0", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -3195,8 +3290,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.11", + "hyper 0.14.28", "rustls 0.21.10", "tokio", "tokio-rustls 0.24.1", @@ -3208,7 +3303,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -3221,12 +3316,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.28", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.0.0", + "http-body 1.0.0", + "hyper 1.1.0", + "pin-project-lite", + "socket2 0.5.5", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.59" @@ -4093,7 +4208,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953cbbb6f9ba4b9304f4df79b98cdc9d14071ed93065a9fca11c00c5d9181b66" dependencies = [ - "hyper", + "hyper 0.14.28", "indexmap 1.9.3", "ipnet", "metrics", @@ -4646,7 +4761,7 @@ dependencies = [ "ed25519-consensus", "futures", "hex", - "http-body", + "http-body 0.4.6", "ibc-proto", "ibc-types", "indicatif", @@ -4718,8 +4833,8 @@ dependencies = [ "ed25519-consensus", "futures", "hex", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "ibc-proto", "ibc-types", "metrics", @@ -4764,6 +4879,8 @@ dependencies = [ "async-stream 0.2.1", "async-trait", "atty", + "axum 0.6.20", + "axum-server 0.6.0", "base64 0.20.0", "bincode", "blake2b_simd 0.5.11", @@ -4780,7 +4897,9 @@ dependencies = [ "fs_extra", "futures", "hex", - "http", + "http 0.2.11", + "hyper 1.1.0", + "hyper-util", "ibc-proto", "ibc-types", "ics23", @@ -4818,6 +4937,7 @@ dependencies = [ "regex", "reqwest", "rocksdb", + "rustls 0.20.9", "rustls-acme", "serde", "serde_json", @@ -4830,6 +4950,7 @@ dependencies = [ "tendermint-proto", "tendermint-rpc", "tokio", + "tokio-rustls 0.25.0", "tokio-stream", "tokio-util 0.7.10", "toml 0.5.11", @@ -5781,7 +5902,7 @@ version = "0.64.1" dependencies = [ "anyhow", "axum 0.5.17", - "axum-server", + "axum-server 0.4.7", "bytes", "clap 3.2.25", "decaf377 0.5.0", @@ -5813,7 +5934,7 @@ dependencies = [ "chrono", "futures", "hex", - "http", + "http 0.2.11", "metrics", "pbjson-types", "penumbra-proto", @@ -5842,7 +5963,7 @@ version = "0.64.1" dependencies = [ "futures", "hex", - "http", + "http 0.2.11", "pin-project", "pin-project-lite", "sha2 0.9.9", @@ -6179,9 +6300,9 @@ dependencies = [ [[package]] name = "polling" -version = "3.3.1" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" +checksum = "545c980a3880efd47b2e262f6a4bb6daad6555cf3367aa9c4e52895f69537a41" dependencies = [ "cfg-if", "concurrent-queue", @@ -6852,10 +6973,10 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-rustls", "hyper-tls", "ipnet", @@ -6868,7 +6989,7 @@ dependencies = [ "pin-project-lite", "rustls 0.21.10", "rustls-native-certs", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -7075,10 +7196,24 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring 0.17.7", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" +dependencies = [ + "log", + "ring 0.17.7", + "rustls-pki-types", + "rustls-webpki 0.102.1", + "subtle", + "zeroize", +] + [[package]] name = "rustls-acme" version = "0.6.0" @@ -7088,6 +7223,7 @@ dependencies = [ "async-h1", "async-io 1.13.0", "async-trait", + "axum-server 0.4.7", "base64 0.13.1", "chrono", "futures", @@ -7103,6 +7239,8 @@ dependencies = [ "serde_json", "smol", "thiserror", + "tokio", + "tokio-util 0.7.10", "url", "webpki-roots 0.21.1", "x509-parser", @@ -7115,7 +7253,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "schannel", "security-framework", ] @@ -7129,6 +7267,22 @@ dependencies = [ "base64 0.21.6", ] +[[package]] +name = "rustls-pemfile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +dependencies = [ + "base64 0.21.6", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -7139,6 +7293,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b" +dependencies = [ + "ring 0.17.7", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -7828,9 +7993,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "subtle-encoding" @@ -7866,7 +8031,7 @@ dependencies = [ "decaf377 0.5.0", "futures", "hex", - "http-body", + "http-body 0.4.6", "metrics-tracing-context", "penumbra-asset", "penumbra-keys", @@ -8319,6 +8484,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.2", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -8429,16 +8605,16 @@ dependencies = [ "axum 0.6.20", "base64 0.21.6", "bytes", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.22", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", "prost", "rustls 0.21.10", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "tokio", "tokio-rustls 0.24.1", "tokio-stream", @@ -8470,9 +8646,9 @@ checksum = "0fddb2a37b247e6adcb9f239f4e5cefdcc5ed526141a416b943929f13aea2cce" dependencies = [ "base64 0.21.6", "bytes", - "http", - "http-body", - "hyper", + "http 0.2.11", + "http-body 0.4.6", + "hyper 0.14.28", "pin-project", "tokio-stream", "tonic", @@ -8547,8 +8723,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower", @@ -8567,8 +8743,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 0.2.11", + "http-body 0.4.6", "http-range-header", "pin-project-lite", "tower-layer", @@ -8778,9 +8954,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "universal-hash" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" dependencies = [ "generic-array", "subtle", diff --git a/crates/bin/pd/Cargo.toml b/crates/bin/pd/Cargo.toml index 925dd7b4a5..0d1a158f7a 100644 --- a/crates/bin/pd/Cargo.toml +++ b/crates/bin/pd/Cargo.toml @@ -122,10 +122,20 @@ console-subscriber = "0.2" metrics-tracing-context = "0.11.0" metrics-util = "0.13" clap = { version = "3", features = ["derive", "env"] } -rustls-acme = "0.6" atty = "0.2" fs_extra = "1.3.0" +## XXX(kate): auto-https dependencies +axum = { version = "0.6.20", features = ["tokio", "http2"] } +# tokio-rustls-acme = { version = "0.2.0", features = ["axum"] } +rustls = "0.20.9" +rustls-acme = { version = "0.6.0", features = ["axum"] } +tokio-rustls = "0.25.0" +# futures-rustls +axum-server = { version = "0.6.0", features = ["tls-rustls"] } +hyper-util = { version = "0.1.2", features = ["service", "server-auto"] } +hyper = { version = "1.1.0" } + [dev-dependencies] penumbra-proof-params = { path = "../../crypto/proof-params", features = [ "bundled-proving-keys", diff --git a/crates/bin/pd/src/auto_https.rs b/crates/bin/pd/src/auto_https.rs index 6f3ed67e61..c5c909db30 100644 --- a/crates/bin/pd/src/auto_https.rs +++ b/crates/bin/pd/src/auto_https.rs @@ -1,54 +1,159 @@ -use std::{ - pin::Pin, - task::{Context, Poll}, -}; +use std::net::SocketAddr; +use axum_server::{accept::DefaultAcceptor, service::TowerToHyperServiceFuture}; +use futures::Future; +use rustls_acme::{futures_rustls::TlsAcceptor, tokio::TokioIncoming, AcmeAcceptor}; +use tracing::{error_span, Instrument}; -use pin_project::pin_project; -use rustls_acme::futures_rustls::server::TlsStream; -use tokio::{ - io::{AsyncRead, AsyncWrite, ReadBuf}, - net::TcpStream, +use { + anyhow::Error, + rustls::ServerConfig, + std::fmt::Debug, + std::path::PathBuf, + std::sync::Arc, + tokio::task::JoinHandle, + rustls_acme::{axum::AxumAcceptor, caches::DirCache, AcmeConfig, AcmeState}, }; -use tokio_util::compat::Compat; -use tonic::transport::server::Connected; - -/// Wrapper type needed to convert between futures_io and tokio traits -#[pin_project] -pub struct Wrapper { - #[pin] - pub inner: Compat>>, -} -impl Connected for Wrapper { - type ConnectInfo = ::ConnectInfo; +/// Protocols supported by this server, in order of preference. +/// +/// See [rfc7301] for more info on ALPN. +/// +/// [rfc7301]: https://datatracker.ietf.org/doc/html/rfc7301 +// +// We also permit HTTP1.1 for backwards-compatibility, specifically for grpc-web. +const ALPN_PROTOCOLS: [&'static [u8]; 2] = [b"h2", b"http/1.1"]; - fn connect_info(&self) -> Self::ConnectInfo { - self.inner.get_ref().get_ref().0.get_ref().connect_info() - } +/// The location of the file-based certificate cache. +// NB: this must not be an absolute path see [Path::join]. +const CACHE_DIR: &'static str = "tokio_rustls_acme_cache"; + +/// If true, use the production Let's Encrypt environment. +/// +/// If false, the ACME resolver will use the [staging environment]. +/// +/// [staging environment]: https://letsencrypt.org/docs/staging-environment/ +const PRODUCTION_LETS_ENCRYPT: bool = true; + +pub struct AutoHttps { + pub home: PathBuf, + pub domain: Option, } -impl AsyncRead for Wrapper { - fn poll_read( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &mut ReadBuf<'_>, - ) -> Poll> { - self.project().inner.poll_read(cx, buf) +impl AutoHttps { + // XXX(kate): decide later if this should be a method, a free-standing function, or if it + // should move back to the main.rs entrypoint. draft here for now. + pub fn start_server(self, address: SocketAddr, router: axum::Router<(), hyper::body::Incoming>) -> impl Future> { + // let (_acceptor, acme_worker) = self.acceptor(); + let (rustls_config, acceptor, acme_worker) = self.server_config(); + let _ = acme_worker.map(tokio::spawn); + + // let _server = axum::Server::bind(&address) + // .serve(router.into_make_service()); + // let _server = axum_server::bind_rustls(address, rustls_config); + // use futures::TryFutureExt; + // _server.map_err(Error::from) + + use hyper_util::service::TowerToHyperService; + let make_svc = router.into_make_service(); + // let make_svc = TowerToHyperService::new(make_svc) + + axum_server::bind(address) + .acceptor(todo!()) + .serve(make_svc) + + // futures::future::ready(Ok(())) // XXX placeholder stub } -} -impl AsyncWrite for Wrapper { - fn poll_write( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - buf: &[u8], - ) -> Poll> { - self.project().inner.poll_write(cx, buf) + // XXX(kate): fixup doc comment. + /// Enable auto-https. + /// + /// Returns a handle to the worker task, and an [`axum::Acceptor`] + fn server_config(self) -> + ( + Arc, + AcmeAcceptor, + Option>>, + ) + { + let Self { home, domain } = self; + + let domains = match domain { + Some(d) => vec![d], + None => /*XXX(kate)*/ todo!(), + }; + + // Use a file-based cache located within the home directory. + let cache = home.join(CACHE_DIR); + let cache = DirCache::new(cache); + + // Create an ACME resolver. + let state = AcmeConfig::new(domains) + .cache(cache) + .directory_lets_encrypt(PRODUCTION_LETS_ENCRYPT) + .state(); + + // Define our server configuration, using the ACME state to resolve certificates. + let mut rustls_config = ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth() + .with_cert_resolver(state.resolver()); + rustls_config.alpn_protocols = Self::alpn_protocols(); + let rustls_config = Arc::new(rustls_config); + + // let acceptor = state.acceptor(); + let acceptor = state.acceptor(); + let worker = Self::acme_worker(state); + ( + rustls_config, + acceptor, + Some(worker) + ) } - fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.project().inner.poll_flush(cx) + + /// This function defines the task responsible for handling ACME events. + /// + /// This function will never return, unless an error is encountered. + #[tracing::instrument(level = "error", skip_all)] + async fn acme_worker( + mut state: AcmeState, + ) -> Result<(), anyhow::Error> + where + EC: Debug + 'static, + EA: Debug + 'static, + { + use futures::StreamExt; + loop { + match state.next().await { + Some(Ok(ok)) => tracing::debug!("received acme event: {:?}", ok), + Some(Err(err)) => tracing::error!("acme error: {:?}", err), + None => { + debug_assert!(false, "acme worker unexpectedly reached end-of-stream"); + tracing::error!("acme worker unexpectedly reached end-of-stream"); + anyhow::bail!("unexpected end-of-stream"); + } + } + } } - fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - self.project().inner.poll_shutdown(cx) + + /// Returns a vector of the protocols supported by this server. + /// + /// This is a convenience method to retrieve an owned copy of [`ALPN_PROTOCOLS`]. + fn alpn_protocols() -> Vec> { + ALPN_PROTOCOLS + .into_iter() + .map(<[u8]>::to_vec) + .collect() } } + + +// ------ >8 ------ +// pub enum AutoHttpsAcceptor { +// Acme { acceptor: AxumAcceptor }, +// DefaultAcceptor, +// } +// impl From for AutoHttpsAcceptor { +// fn from(acceptor: AxumAcceptor) -> Self { +// Self::Acme { acceptor } } +// } + diff --git a/crates/bin/pd/src/main.rs b/crates/bin/pd/src/main.rs index 8315d70efb..c62f40310b 100644 --- a/crates/bin/pd/src/main.rs +++ b/crates/bin/pd/src/main.rs @@ -3,6 +3,7 @@ #![recursion_limit = "512"] use std::{error::Error, net::SocketAddr, path::PathBuf}; +use axum_server::accept::{Accept, DefaultAcceptor}; use console_subscriber::ConsoleLayer; use metrics_tracing_context::{MetricsLayer, TracingContextLayer}; use metrics_util::layers::Stack; @@ -478,40 +479,15 @@ async fn main() -> anyhow::Result<()> { ))); } - let grpc_server = if let Some(domain) = grpc_auto_https { - use pd::auto_https::Wrapper; - use rustls_acme::{caches::DirCache, AcmeConfig}; - use tokio_stream::wrappers::TcpListenerStream; - use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt}; - - let mut acme_cache = pd_home.clone(); - acme_cache.push("rustls_acme_cache"); - - let bound_listener = TcpListener::bind(grpc_bind) - .await - .context(format!("Failed to bind HTTPS listener on {}", grpc_bind))?; - let listener = TcpListenerStream::new(bound_listener); - // Configure HTTP2 support for the TLS negotiation; we also permit HTTP1.1 - // for backwards-compatibility, specifically for grpc-web. - let alpn_config = vec!["h2".into(), "http/1.1".into()]; - let tls_incoming = AcmeConfig::new([domain.as_str()]) - .cache(DirCache::new(acme_cache)) - .directory_lets_encrypt(true) // Use the production LE environment - .incoming(listener.map_ok(|conn| conn.compat()), alpn_config) - .map_ok(|incoming| Wrapper { - inner: incoming.compat(), - }); - - tokio::task::Builder::new() - .name("grpc_server") - .spawn(grpc_server.serve_with_incoming(tls_incoming)) - .expect("failed to spawn grpc server") - } else { - tokio::task::Builder::new() - .name("grpc_server") - .spawn(grpc_server.serve(grpc_bind)) - .expect("failed to spawn grpc server") - }; + // Now we must drop down a layer of abstraction, from tonic to axum. + // + // Use auto-https, if enabled. + // + // TODO(katie): this is where we may hang additional routes upon this endpoint + // in the future. see #3646 for more information. + let router = grpc_server.into_router(); + let auto_https = pd::auto_https::AutoHttps { home: pd_home, domain: grpc_auto_https }; + let grpc_server = auto_https.start_server(grpc_bind, router); // Configure a Prometheus recorder and exporter. let (recorder, exporter) = PrometheusBuilder::new() @@ -559,24 +535,8 @@ async fn main() -> anyhow::Result<()> { } )?, - x = grpc_server => x?.map_err(|e| { - let mut msg = format!("grpc server on {} failed: {}", grpc_bind, e); - // Detect if we have a bind error. We need to unpack nested errors, from - // tonic -> hyper -> std. Otherwise, only "transport error" is reported, - // which isn't informative enough to take action. - if let Some(e) = e.source() { - if let Some(e) = e.source() { - if let Some(e) = e.downcast_ref::() { - if e.kind().to_string().contains("address in use") { - msg = format!("grpc bind socket already in use: {}", grpc_bind); - } - } - } - } - tracing::error!("{}", msg); - anyhow::anyhow!(msg) - } - )?, + // TODO(kate): restore friendly logging for bind errors. + x = grpc_server => x?, }; }