From e24b5cc459bcb54985abb5775bc0b063d9e75183 Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Mon, 18 Nov 2024 22:55:18 -0500 Subject: [PATCH] feat: new crate with io_uring backed ObjectStore --- Cargo.lock | 1262 +++++++++++++++++- Cargo.toml | 9 +- cyper-object-store/Cargo.toml | 37 + cyper-object-store/src/lib.rs | 131 ++ cyper-object-store/src/object_store.rs | 304 +++++ cyper-object-store/src/signer.rs | 207 +++ cyper-object-store/tests/integration_test.rs | 60 + 7 files changed, 1952 insertions(+), 58 deletions(-) create mode 100644 cyper-object-store/Cargo.toml create mode 100644 cyper-object-store/src/lib.rs create mode 100644 cyper-object-store/src/object_store.rs create mode 100644 cyper-object-store/src/signer.rs create mode 100644 cyper-object-store/tests/integration_test.rs diff --git a/Cargo.lock b/Cargo.lock index 2ca2e549a7..8f89c080a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -247,7 +247,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -391,6 +391,28 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "async-task" version = "4.7.1" @@ -429,6 +451,380 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e531658a0397d22365dfe26c3e1c0c8448bf6a3a2d8a098ded802f2b1261615" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09677244a9da92172c8dc60109b4a9658597d4d298b188dd0018b6a66b410ca4" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fea2f3a8bb3bd10932ae7ad59cc59f65f270fc9183a7e91f501dc5efbef7ee" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.1.0", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper 0.14.31", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.1.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -444,12 +840,40 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bench-vortex" version = "0.19.0" @@ -558,6 +982,25 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "bytestring" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" +dependencies = [ + "bytes", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -742,8 +1185,10 @@ dependencies = [ "compio-log", "compio-macros", "compio-net", + "compio-quic", "compio-runtime", "compio-signal", + "compio-tls", ] [[package]] @@ -806,6 +1251,7 @@ dependencies = [ "compio-buf", "futures-util", "paste", + "pin-project-lite", ] [[package]] @@ -847,6 +1293,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "compio-quic" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "969b10cb032385eb60163deac3609771300307038709396136228760e1e60cdb" +dependencies = [ + "compio-buf", + "compio-io", + "compio-log", + "compio-net", + "compio-runtime", + "flume", + "futures-util", + "libc", + "quinn-proto", + "rustc-hash", + "rustls 0.23.17", + "thiserror 1.0.69", + "windows-sys 0.52.0", +] + [[package]] name = "compio-runtime" version = "0.5.1" @@ -864,6 +1331,7 @@ dependencies = [ "once_cell", "os_pipe", "scoped-tls", + "slab", "smallvec", "socket2", "windows-sys 0.52.0", @@ -885,6 +1353,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "compio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "543293c36bf6f7945bcf813654674f7bbf457dfcd0e239a915a91ffa94091c7d" +dependencies = [ + "compio-buf", + "compio-io", + "native-tls", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -913,6 +1392,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const-random" version = "0.1.18" @@ -955,6 +1440,24 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -1069,6 +1572,28 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1100,6 +1625,66 @@ dependencies = [ "memchr", ] +[[package]] +name = "cyper" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4fef735ec7665da87d454a29612935383812fcae550062eb784346f681d0a00" +dependencies = [ + "async-stream", + "base64 0.22.1", + "compio", + "cyper-core", + "encoding_rs", + "futures-util", + "http 1.1.0", + "http-body-util", + "hyper 1.5.1", + "hyper-util", + "mime", + "send_wrapper", + "serde", + "serde_urlencoded", + "thiserror 1.0.69", + "url", +] + +[[package]] +name = "cyper-core" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761dca6e0a83611ad6438ab0ccaeb8e26531f77fc6592e9760181aedca87bd1e" +dependencies = [ + "cfg-if", + "compio", + "futures-util", + "hyper 1.5.1", + "hyper-util", + "send_wrapper", + "tower-service", +] + +[[package]] +name = "cyper-object-store" +version = "0.19.0" +dependencies = [ + "async-trait", + "aws-sigv4", + "bytes", + "chrono", + "compio", + "cyper", + "futures", + "hex", + "http 1.1.0", + "local_s3", + "object_store", + "sha2", + "tempfile", + "url", + "vortex-error", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -1279,7 +1864,7 @@ checksum = "f52c4012648b34853e40a2c6bcaa8772f837831019b68aca384fb38436dba162" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.22.1", "chrono", "datafusion-common", "datafusion-execution", @@ -1485,6 +2070,16 @@ dependencies = [ "strum", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -1502,6 +2097,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1561,12 +2157,44 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encode_unicode" version = "0.3.6" @@ -1663,6 +2291,16 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "filetime" version = "0.2.25" @@ -1898,18 +2536,48 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + [[package]] name = "h2" -version = "0.4.6" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http", + "http 1.1.0", "indexmap", "slab", "tokio", @@ -1973,6 +2641,25 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hex-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7685beb53fc20efc2605f32f5d51e9ba18b8ef237961d1760169d2290d3bee" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "homedir" version = "0.3.4" @@ -1985,6 +2672,17 @@ dependencies = [ "windows", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.1.0" @@ -1996,6 +2694,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -2003,7 +2712,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.1.0", ] [[package]] @@ -2014,8 +2723,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -2025,6 +2734,12 @@ version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humansize" version = "2.1.3" @@ -2042,17 +2757,42 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.0" +version = "0.14.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.7", + "http 1.1.0", + "http-body 1.0.1", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -2060,6 +2800,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.31", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.3" @@ -2067,14 +2823,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.1.0", + "hyper 1.5.1", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.17", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", ] @@ -2086,7 +2842,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.5.1", "hyper-util", "native-tls", "tokio", @@ -2103,9 +2859,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.1.0", + "http-body 1.0.1", + "hyper 1.5.1", "pin-project-lite", "socket2", "tokio", @@ -2376,9 +3132,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" [[package]] name = "jiff" @@ -2495,9 +3251,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.162" +version = "0.2.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libfuzzer-sys" @@ -2548,6 +3304,24 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +[[package]] +name = "local_s3" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e45e3ff0c7f782b63d18205d69e529f5aaa1de8e2835f5f76df8a724c2b788b1" +dependencies = [ + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "bytes", + "futures", + "hyper 1.5.1", + "hyper-util", + "s3s", + "s3s-fs", + "tokio", +] + [[package]] name = "lock_api" version = "0.4.12" @@ -2564,6 +3338,15 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.1", +] + [[package]] name = "lz4_flex" version = "0.11.3" @@ -2619,6 +3402,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -2684,6 +3473,25 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nugine-rust-utils" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dcd9cfa92246a9c7ca0671e00733c4e9d77ee1fa0ae08c9a181b7c8802aea2" +dependencies = [ + "simdutf8", +] + [[package]] name = "num" version = "0.4.3" @@ -2831,6 +3639,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "numeric_cast" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf70ee2d9b1737d1836c20d9f8f96ec3901b2bf92128439db13237ddce9173a5" + [[package]] name = "object" version = "0.36.5" @@ -2847,12 +3661,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6eb4c22c6154a1e759d7099f9ffad7cc5ef8245f9efbab4a41b92623079c82f3" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "futures", "humantime", - "hyper", + "hyper 1.5.1", "itertools 0.13.0", "md-5", "parking_lot", @@ -2861,7 +3675,7 @@ dependencies = [ "rand", "reqwest", "ring", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "serde", "serde_json", "snafu", @@ -2946,6 +3760,23 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "papergrid" version = "0.12.0" @@ -3000,7 +3831,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -3037,6 +3868,24 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -3123,6 +3972,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.31" @@ -3410,7 +4269,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.17", "socket2", "thiserror 2.0.3", "tokio", @@ -3428,7 +4287,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls", + "rustls 0.23.17", "rustls-pki-types", "slab", "thiserror 2.0.3", @@ -3577,18 +4436,18 @@ version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.7", + "http 1.1.0", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.5.1", + "hyper-rustls 0.27.3", "hyper-tls", "hyper-util", "ipnet", @@ -3600,9 +4459,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.23.17", + "rustls-native-certs 0.8.0", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -3611,7 +4470,7 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.0", "tokio-util", "tower-service", "url", @@ -3622,6 +4481,17 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.8" @@ -3690,9 +4560,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.40" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags 2.6.0", "errno", @@ -3703,18 +4573,42 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + +[[package]] +name = "rustls" +version = "0.23.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f1a745511c54ba6d4465e8d5dfbd81b45791756de28d4981af70d6dca128f1e" dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework", +] + [[package]] name = "rustls-native-certs" version = "0.8.0" @@ -3722,12 +4616,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -3746,6 +4649,16 @@ dependencies = [ "web-time", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -3769,6 +4682,82 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "s3s" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa54e3b4b4791c8c62291516997866b4f265c3fcbfdbcdd0b8da62896fba8bfa" +dependencies = [ + "arrayvec", + "async-trait", + "atoi", + "base64-simd", + "bytes", + "bytestring", + "chrono", + "crc32c", + "crc32fast", + "digest", + "futures", + "hex-simd", + "hmac", + "http-body 1.0.1", + "http-body-util", + "httparse", + "hyper 1.5.1", + "itoa", + "memchr", + "mime", + "nom", + "nugine-rust-utils", + "numeric_cast", + "pin-project-lite", + "quick-xml", + "serde", + "serde_urlencoded", + "sha1", + "sha2", + "smallvec", + "sync_wrapper", + "thiserror 1.0.69", + "time", + "tokio", + "tracing", + "transform-stream", + "urlencoding", + "zeroize", +] + +[[package]] +name = "s3s-fs" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b045f5ab67e8536d147ae48e038f6f533717fe5ddb0a68d0ba8f922ce7fb293f" +dependencies = [ + "async-trait", + "base64-simd", + "bytes", + "chrono", + "crc32c", + "futures", + "hex-simd", + "md-5", + "mime", + "nugine-rust-utils", + "numeric_cast", + "path-absolutize", + "s3s", + "serde_json", + "thiserror 1.0.69", + "time", + "tokio", + "tokio-util", + "tracing", + "tracing-error", + "transform-stream", + "uuid", +] + [[package]] name = "same-file" version = "1.0.6" @@ -3780,9 +4769,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -3799,6 +4788,30 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -3828,6 +4841,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +[[package]] +name = "send_wrapper" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73" +dependencies = [ + "futures-core", +] + [[package]] name = "seq-macro" version = "0.3.5" @@ -3909,6 +4931,37 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -3924,6 +4977,22 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simplelog" version = "0.12.2" @@ -4003,6 +5072,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "sqlparser" version = "0.51.0" @@ -4094,9 +5173,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -4242,6 +5321,16 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "thread_local" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "thrift" version = "0.17.0" @@ -4369,13 +5458,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls", + "rustls 0.23.17", "rustls-pki-types", "tokio", ] @@ -4433,6 +5532,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4458,6 +5558,36 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tracing-error" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + +[[package]] +name = "transform-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05034de7a8fcb11796a36478a2a8b16dca6772644dec5f49f709d5c66a38d359" +dependencies = [ + "futures-core", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -4482,9 +5612,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-segmentation" @@ -4527,6 +5657,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -5049,6 +6185,12 @@ dependencies = [ "zigzag", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -5414,8 +6556,8 @@ dependencies = [ "chrono", "futures-channel", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "js-sys", "matchit", "pin-project", @@ -5500,6 +6642,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xshell" version = "0.2.7" diff --git a/Cargo.toml b/Cargo.toml index 8c7a171c8f..11be9c8db2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "bench-vortex", + "cyper-object-store", "encodings/*", "fuzz", "pyvortex", @@ -62,16 +63,18 @@ arrow-ord = "53.0.0" arrow-schema = "53.0.0" arrow-select = "53.0.0" async-trait = "0.1" +aws-sigv4 = "1" bindgen = "0.70.0" bytes = "1.6.0" bzip2 = "0.4.4" cargo_metadata = "0.18.1" -chrono = "0.4.38" +chrono = "0.4" clap = "4.5.13" compio = "0.12" criterion = { version = "0.5.1", features = ["html_reports"] } croaring = "2.1.0" csv = "1.3.0" +cyper = "0.1" datafusion = { version = "43.0.0", default-features = false } datafusion-common = "43.0.0" datafusion-execution = "43.0.0" @@ -97,12 +100,15 @@ futures-util = "0.3" getrandom = "0.2.14" half = { version = "^2", features = ["std", "num-traits"] } hashbrown = "0.15.0" +hex = "0.4" homedir = "0.3.3" +http = "1" humansize = "2.1.3" indicatif = "0.17.8" itertools = "0.13.0" jiff = "0.1.8" libfuzzer-sys = "0.4" +local_s3 = "0.1" log = "0.4.21" mimalloc = "0.1.42" monoio = "0.2.3" @@ -129,6 +135,7 @@ seq-macro = "0.3.5" serde = "1.0.197" serde_json = "1.0.116" serde_test = "1.0.176" +sha2 = "0.10" simplelog = { version = "0.12.2", features = ["paris"] } static_assertions = "1" tar = "0.4" diff --git a/cyper-object-store/Cargo.toml b/cyper-object-store/Cargo.toml new file mode 100644 index 0000000000..831e632d97 --- /dev/null +++ b/cyper-object-store/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "cyper-object-store" +description = "An ObjectStore implementation on top of the cyper HTTP client and the compio runtime" +version.workspace = true +homepage.workspace = true +repository.workspace = true +authors.workspace = true +license.workspace = true +keywords.workspace = true +include.workspace = true +edition.workspace = true +rust-version.workspace = true +readme.workspace = true +categories.workspace = true + +[dependencies] +aws-sigv4 = { workspace = true } +async-trait = { workspace = true } +bytes = { workspace = true } +chrono = { workspace = true } +compio = { workspace = true, features = ["macros"] } +cyper = { workspace = true } +futures = { workspace = true } +hex = { workspace = true } +http = { workspace = true } +object_store = { workspace = true, features = ["aws"] } +sha2 = { workspace = true } +url = { workspace = true } +vortex-error = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } +local_s3 = { workspace = true } + + +[lints] +workspace = true diff --git a/cyper-object-store/src/lib.rs b/cyper-object-store/src/lib.rs new file mode 100644 index 0000000000..c8e9bbe712 --- /dev/null +++ b/cyper-object-store/src/lib.rs @@ -0,0 +1,131 @@ +//! An `ObjectStore` that runs IO operations on top of [compio]. +//! +//! `CyperS3` supports S3-compatible object storage services, and uses +//! an embedded compio runtime to implement object store requests. + +mod object_store; +mod signer; + +use std::fmt::Display; +use std::io; +use std::sync::Arc; +use std::time::SystemTime; + +use ::object_store::aws::AwsCredentialProvider; +use ::object_store::path::Path; +use ::object_store::GetRange; +use cyper::{Client, Response}; +use http::header::RANGE; +use signer::sign_request; +use url::Url; + +/// An `ObjectStore` implementation for S3-compatible services, where requests +/// execute in io_uring via the [compio] async runtime. +/// +/// `CyperS3` implements all methods for reading to S3-compatible object storage, +/// and can be used anywhere an `ObjectStore` or `Arc` is expected. +/// +/// The futures returned by `CyperS3` can only be executed within a compio context. +#[derive(Debug, Clone)] +pub struct CyperS3 { + client: Client, + base_url: Url, + config: S3Config, +} + +impl Display for CyperS3 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CyperS3").finish_non_exhaustive() + } +} + +/// S3 configuration parameters. +#[derive(Debug, Clone)] +pub struct S3Config { + pub bucket: Arc, + pub region: Arc, + pub endpoint: Option>, + pub credentials: AwsCredentialProvider, + pub virtual_host: bool, +} + +/// Infer a default endpoint URL from a bucket + region combo. +fn region_endpoint(bucket: &str, region: &str) -> io::Result { + let url = if region == "us-east-1" { + format!("https://{bucket}.s3.amazonaws.com") + } else { + format!("https://{bucket}.{region}.s3.amazonaws.com") + }; + Url::parse(url.as_str()) + .map_err(|parse_error| io::Error::new(io::ErrorKind::Other, parse_error)) +} + +impl CyperS3 { + /// Create a new S3 `ObjectStore` using a [cyper] client. + pub fn new(config: S3Config) -> io::Result { + let base_url = if let Some(ref endpoint) = config.endpoint { + Url::parse(endpoint.as_ref()) + .map_err(|parse_error| io::Error::new(io::ErrorKind::Other, parse_error))? + } else { + region_endpoint(config.bucket.as_ref(), config.region.as_ref())? + }; + + let client = cyper::ClientBuilder::new().build(); + + Ok(Self { + client, + base_url, + config, + }) + } +} + +// Create a new custom ObjectStoreError here instead. + +impl CyperS3 { + /// Get a set of bytes for the given object at the optional range. + /// + /// We want to return a GetResult instead here... + pub async fn get_byte_range( + &self, + path: &Path, + range: Option<&GetRange>, + ) -> io::Result { + let creds = self.config.credentials.get_credential().await?; + + // In virtual-hosting style, the hostname should start with the bucket name. + // If we are not in virtual-hosting style, we prefix HTTP request paths with the bucket name. + let url = if self.config.virtual_host { + self.base_url + .join(path.as_ref()) + .map_err(|parse_error| io::Error::new(io::ErrorKind::Other, parse_error))? + } else { + self.base_url + .join(format!("{}/{}", self.config.bucket.as_ref(), path.as_ref()).as_ref()) + .map_err(|parse_error| io::Error::new(io::ErrorKind::Other, parse_error))? + }; + + let mut request = self + .client + .get(url) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + if let Some(ref range) = range { + request = request + .header(RANGE, format!("{range}")) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + } + let request = request.build(); + let signed = sign_request( + request, + SystemTime::now(), + creds.key_id.as_str(), + creds.secret_key.as_str(), + self.config.region.as_ref(), + ); + + self.client + .execute(signed) + .await + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + } +} diff --git a/cyper-object-store/src/object_store.rs b/cyper-object-store/src/object_store.rs new file mode 100644 index 0000000000..068d506212 --- /dev/null +++ b/cyper-object-store/src/object_store.rs @@ -0,0 +1,304 @@ +use async_trait::async_trait; +use chrono::{DateTime, TimeZone, Utc}; +use cyper::Response; +use futures::stream::BoxStream; +use futures::{FutureExt, StreamExt}; +use http::header::{ + CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, CONTENT_LENGTH, + CONTENT_RANGE, CONTENT_TYPE, ETAG, LAST_MODIFIED, +}; +use http::{HeaderMap, StatusCode}; +use object_store::path::Path; +use object_store::{ + Attribute, Attributes, Error as ObjectStoreError, GetOptions, GetRange, GetResult, + GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOpts, + PutOptions, PutPayload, PutResult, +}; + +use crate::CyperS3; + +const STORE: &str = "CYPER_S3"; + +macro_rules! make_object_store_error { + ($err:expr) => { + ::object_store::Error::Generic { + store: STORE, + source: $err.into(), + } + }; +} + +#[async_trait] +impl ObjectStore for CyperS3 { + /// Save the provided `payload` to `location` with the given options + async fn put_opts( + &self, + _location: &Path, + _payload: PutPayload, + _opts: PutOptions, + ) -> Result { + Err(ObjectStoreError::NotSupported { + source: "operation not supported for CyperS3 object_store".into(), + }) + } + + async fn put_multipart_opts( + &self, + _location: &Path, + _options: PutMultipartOpts, + ) -> Result, ObjectStoreError> { + Err(ObjectStoreError::NotSupported { + source: "operation not supported for CyperS3 object_store".into(), + }) + } + + /// Implement the GET operation, optionally with a range header. + async fn get_opts( + &self, + location: &Path, + options: GetOptions, + ) -> Result { + let response = self + .get_byte_range(location, options.range.as_ref()) + .await + .map_err(|io_error| make_object_store_error!(io_error))?; + + match response.status() { + StatusCode::NOT_FOUND => Err(ObjectStoreError::NotFound { + path: location.to_string(), + source: "403 forbidden".into(), + }), + _ => get_result(location, options.range, response), + } + } + + async fn delete(&self, _location: &Path) -> Result<(), ObjectStoreError> { + Err(ObjectStoreError::NotSupported { + source: "operation not supported for CyperS3 object_store".into(), + }) + } + + fn list(&self, _prefix: Option<&Path>) -> BoxStream<'_, Result> { + futures::stream::once(async move { + Err(ObjectStoreError::NotSupported { + source: "operation not supported for CyperS3 object_store".into(), + }) + }) + .boxed() + } + + async fn list_with_delimiter( + &self, + _prefix: Option<&Path>, + ) -> Result { + Err(ObjectStoreError::NotSupported { + source: "operation not supported for CyperS3 object_store".into(), + }) + } + + async fn copy(&self, _from: &Path, _to: &Path) -> Result<(), ObjectStoreError> { + Err(ObjectStoreError::NotSupported { + source: "operation not supported for CyperS3 object_store".into(), + }) + } + + async fn copy_if_not_exists(&self, _from: &Path, _to: &Path) -> Result<(), ObjectStoreError> { + Err(ObjectStoreError::NotSupported { + source: "operation not supported for CyperS3 object_store".into(), + }) + } +} + +fn get_result( + location: &Path, + range: Option, + response: Response, +) -> Result { + let mut meta = header_meta(location, response.headers())?; + + // ensure that we receive the range we asked for + let range = if range.is_some() { + let val = response.headers().get(CONTENT_RANGE).ok_or_else(|| { + make_object_store_error!("Content-Range header required if range request made") + })?; + + let value = val.to_str().map_err(|err| make_object_store_error!(err))?; + let value = ContentRange::from_str(value) + .ok_or_else(|| make_object_store_error!("Invalid value for Content-Range header"))?; + let actual = value.range; + + // Update size to reflect full size of object (#5272) + meta.size = value.size; + + actual + } else { + 0..meta.size + }; + + macro_rules! parse_attributes { + ($headers:expr, $(($header:expr, $attr:expr, $err:expr)),*) => {{ + let mut attributes = Attributes::new(); + $( + if let Some(x) = $headers.get($header) { + let x = x.to_str().map_err(|_| make_object_store_error!($err))?; + attributes.insert($attr, x.to_string().into()); + } + )* + attributes + }} + } + + let mut attributes = parse_attributes!( + response.headers(), + ( + CACHE_CONTROL, + Attribute::CacheControl, + "Cache-Control header value invalid" + ), + ( + CONTENT_DISPOSITION, + Attribute::ContentDisposition, + "Content-Disposition header value invalid" + ), + ( + CONTENT_ENCODING, + Attribute::ContentEncoding, + "Content-Encoding header value invalid" + ), + ( + CONTENT_LANGUAGE, + Attribute::ContentLanguage, + "Content-Languge header value invalid" + ), + ( + CONTENT_TYPE, + Attribute::ContentType, + "Content-Type header value invalid" + ) + ); + + // Add attributes that match the user-defined metadata prefix (e.g. x-amz-meta-) + for (key, val) in response.headers() { + if let Some(suffix) = key + .as_str() + .strip_prefix(USER_DEFINED_METADATA_HEADER_PREFIX) + { + if let Ok(val_str) = val.to_str() { + attributes.insert( + Attribute::Metadata(suffix.to_string().into()), + val_str.to_string().into(), + ); + } else { + return Err(ObjectStoreError::Generic { + store: STORE, + source: format!("invalid metadata header {}", key.as_str()).into(), + }); + } + } + } + + let body_fut = response + .bytes() + .map(|res| res.map_err(|err| make_object_store_error!(err))) + .boxed(); + + Ok(GetResult { + range, + meta, + attributes, + payload: GetResultPayload::Stream(futures::stream::once(body_fut).boxed()), + }) +} + +use core::ops::Range; + +struct ContentRange { + /// The range of the object returned + range: Range, + /// The total size of the object being requested + size: usize, +} + +impl ContentRange { + /// Parse a content range of the form `bytes -/` + /// + /// + fn from_str(s: &str) -> Option { + let rem = s.trim().strip_prefix("bytes ")?; + let (range, size) = rem.split_once('/')?; + let size = size.parse().ok()?; + + let (start_s, end_s) = range.split_once('-')?; + + let start = start_s.parse().ok()?; + let end: usize = end_s.parse().ok()?; + + Some(Self { + size, + range: start..end + 1, + }) + } +} + +fn header_meta(location: &Path, headers: &HeaderMap) -> Result { + let last_modified = match headers.get(LAST_MODIFIED) { + Some(last_modified) => { + let last_modified = last_modified + .to_str() + .map_err(|err| make_object_store_error!(err))?; + DateTime::parse_from_rfc2822(last_modified) + .map_err(|err| make_object_store_error!(err))? + .with_timezone(&Utc) + } + None => Utc.timestamp_nanos(0), + }; + + let e_tag = match get_etag(headers) { + Ok(e_tag) => e_tag, + Err(e) => return Err(e), + }; + + let content_length = headers + .get(CONTENT_LENGTH) + .ok_or_else(|| make_object_store_error!("Content-Length header required"))?; + + let content_length = content_length + .to_str() + .map_err(|err| make_object_store_error!(err))?; + let size = content_length + .parse::() + .map_err(|parse_err| ObjectStoreError::Generic { + store: STORE, + source: parse_err.into(), + })?; + + let version = match headers.get(VERSION_HEADER) { + Some(v) => Some( + v.to_str() + .map_err(|err| make_object_store_error!(err))? + .to_string(), + ), + None => None, + }; + + Ok(ObjectMeta { + location: location.clone(), + last_modified, + version, + size, + e_tag, + }) +} + +fn get_etag(headers: &HeaderMap) -> Result, ObjectStoreError> { + match headers.get(ETAG) { + Some(etag) => etag + .to_str() + .map_err(|err| make_object_store_error!(err)) + .map(|etag| Some(etag.to_string())), + None => Ok(None), + } +} + +const VERSION_HEADER: &str = "x-amz-version-id"; +const USER_DEFINED_METADATA_HEADER_PREFIX: &str = "x-amz-meta-"; diff --git a/cyper-object-store/src/signer.rs b/cyper-object-store/src/signer.rs new file mode 100644 index 0000000000..24c72ba9bd --- /dev/null +++ b/cyper-object-store/src/signer.rs @@ -0,0 +1,207 @@ +#![allow(clippy::unwrap_used)] + +use std::collections::{BTreeMap, BTreeSet}; +use std::time::SystemTime; + +use aws_sigv4::sign::v4::{calculate_signature, generate_signing_key}; +use chrono::{DateTime, Utc}; +use cyper::Request; +use http::header::{AUTHORIZATION, HOST}; +use http::{HeaderName, HeaderValue}; +use sha2::{Digest, Sha256}; + +/// AWS signature algorithm used for signing +pub const SIGNATURE_ALGORITHM: &str = "AWS4-HMAC-SHA256"; + +/// SHA-256 hash digest of an empty string. +static EMPTY_CHECKSUM: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/// Service used for S3-compatible storage that uses the AWSV4 signature algorithm. +pub const S3_SERVICE: &str = "s3"; + +const X_AMAZON_CONTENT_SHA256: &str = "x-amz-content-sha256"; +const X_AMAZON_DATE: &str = "x-amz-date"; + +/// Sign request using the AWS v4 signature algorithm. +pub fn sign_request( + mut request: Request, + now: SystemTime, + key_id: &str, + secret_key: &str, + region: &str, +) -> Request { + // Set the Host header. + let host = &request.url()[url::Position::BeforeHost..url::Position::AfterPort]; + let host = host.to_string(); + request + .headers_mut() + .insert(HOST, HeaderValue::from_str(host.as_str()).unwrap()); + + // Set date header to current timestamp + let timestamp: DateTime = now.into(); + let timestamp = timestamp.format("%Y%m%dT%H%M%SZ").to_string(); + request.headers_mut().insert( + X_AMAZON_CONTENT_SHA256, + HeaderValue::from_str(EMPTY_CHECKSUM).unwrap(), + ); + request.headers_mut().insert( + X_AMAZON_DATE, + HeaderValue::from_str(timestamp.as_str()).unwrap(), + ); + + let (creq, signed_headers) = canonical_request(&request); + let string_to_sign = string_to_sign(creq.as_str(), now, region); + + let signing_key = generate_signing_key(secret_key, now, region, S3_SERVICE); + let sig = calculate_signature(signing_key, string_to_sign.as_bytes()); + + // Set the Authorization header to the provided value. + let authorization = + authorization_header(sig.as_str(), now, region, key_id, signed_headers.as_str()); + + request.headers_mut().insert( + AUTHORIZATION, + HeaderValue::from_str(authorization.as_str()).unwrap(), + ); + + request +} + +/// Generate an authorization header usable with an S3-compatible service that uses +/// the AWSV4 signature algorithm. +/// +/// See: https://docs.aws.amazon.com/images/IAM/latest/UserGuide/images/sigV4-using-auth-header.png +fn authorization_header( + signature: &str, + date: SystemTime, + region: &str, + key_id: &str, + signed_headers: &str, +) -> String { + let mut header_value = String::new(); + + // Signature algorithm + header_value.push_str(SIGNATURE_ALGORITHM); + header_value.push(' '); + + // Credential scope + let date_time: DateTime = date.into(); + header_value.push_str("Credential="); + header_value.push_str(key_id); + header_value.push('/'); + header_value.push_str(format!("{}", date_time.format("%Y%m%d")).as_str()); + header_value.push('/'); + header_value.push_str(region); + header_value.push('/'); + header_value.push_str("s3"); + header_value.push('/'); + header_value.push_str("aws4_request"); + + // Signed headers + header_value.push_str(", SignedHeaders="); + header_value.push_str(signed_headers); + + // Signature (HMAC of StringToSign) + header_value.push_str(", Signature="); + header_value.push_str(signature); + + header_value +} + +/// Generate the canonical string representation for an HTTP request, expected +/// as the input to the AWS V4 signing algorithm. +pub fn canonical_request(request: &Request) -> (String, String) { + let mut canonical_request = String::new(); + // Method + canonical_request.push_str(request.method().as_str()); + canonical_request.push('\n'); + + // Path + canonical_request.push_str(request.url().path()); + canonical_request.push('\n'); + + // Query string + canonical_request.push_str(request.url().query().unwrap_or_default()); + canonical_request.push('\n'); + + // Canonical headers + let mut canonical_headers = BTreeMap::new(); + let mut signed_headers = BTreeSet::new(); + for (name, value) in request.headers() { + if let Ok(value_str) = value.to_str() { + signed_headers.insert(canonical_header_name(name)); + canonical_headers.insert(canonical_header_name(name), value_str.to_string()); + } + } + for (name, value) in canonical_headers { + canonical_request.push_str(format!("{name}:{value}\n").as_str()); + } + canonical_request.push('\n'); + + // Signed Headers + let signed_headers = signed_headers.iter().cloned().collect::>().join(";"); + canonical_request.push_str(signed_headers.as_str()); + canonical_request.push('\n'); + + // TODO(aduffy): handle non-empty payloads + // Hash of payload bytes + canonical_request.push_str(EMPTY_CHECKSUM); + + (canonical_request, signed_headers) +} + +fn string_to_sign(canonical_request: &str, now: SystemTime, region: &str) -> String { + let mut to_sign = String::new(); + + to_sign.push_str("AWS4-HMAC-SHA256\n"); + + let timestamp: DateTime = now.into(); + to_sign.push_str(timestamp.format("%Y%m%dT%H%M%SZ").to_string().as_str()); + to_sign.push('\n'); + + let scope = format!("{}/{region}/s3/aws4_request\n", timestamp.format("%Y%m%d")); + to_sign.push_str(scope.as_str()); + + let hash = Sha256::digest(canonical_request); + to_sign.push_str(hex::encode(hash).as_str()); + + to_sign +} + +fn canonical_header_name(name: &HeaderName) -> String { + name.as_str().to_lowercase() +} + +#[cfg(test)] +mod tests { + use std::time::SystemTime; + + use cyper::Client; + use http::header::{AUTHORIZATION, RANGE}; + use http::HeaderValue; + + use super::sign_request; + + #[compio::test] + async fn test_signer() { + let client = Client::new(); + let request = client + .get("https://thisisjustfake.s3.amazonaws.com/ranges/ten") + .unwrap() + .header(RANGE, HeaderValue::from_str("bytes=2-4").unwrap()) + .unwrap() + .build(); + + let key_id = "key_id_goes_here"; + let secret_key = "icanttellyouitssecret"; + + let signed_request = sign_request( + request, + SystemTime::UNIX_EPOCH, + key_id, + secret_key, + "us-east-1", + ); + assert_eq!(signed_request.headers().get(AUTHORIZATION).unwrap(), "AWS4-HMAC-SHA256 Credential=key_id_goes_here/19700101/us-east-1/s3/aws4_request, SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=0c94c5091be6cd3193f1753414a4dfd6487bd0fab5a99d8f7bcd10e9c51db04d"); + } +} diff --git a/cyper-object-store/tests/integration_test.rs b/cyper-object-store/tests/integration_test.rs new file mode 100644 index 0000000000..703600ef27 --- /dev/null +++ b/cyper-object-store/tests/integration_test.rs @@ -0,0 +1,60 @@ +#![allow(clippy::tests_outside_test_module, clippy::panic_in_result_fn)] + +use std::io; +use std::sync::Arc; + +use bytes::Bytes; +use cyper_object_store::{CyperS3, S3Config}; +use local_s3::LocalS3; +use object_store::aws::AwsCredential; +use object_store::path::Path; +use object_store::{ObjectStore, StaticCredentialProvider}; + +const BUCKET: &str = "test-bucket"; +const KEY_ID: &str = "key1"; +const SECRET_KEY: &str = "secret1"; + +/// Integration test against a mock S3 service adapter. +#[compio::test] +async fn test_e2e() -> io::Result<()> { + let tempdir = tempfile::tempdir()?; + let path = tempdir.path().to_path_buf(); + + let local_s3 = LocalS3::new(path).with_credentials(KEY_ID, SECRET_KEY); + local_s3.create_bucket(BUCKET); + local_s3.put_object( + BUCKET, + "object1", + Bytes::from_static("0123456789".as_bytes()), + ); + + // Launch the S3 service. + local_s3.start().detach(); + + let static_creds = Arc::new(StaticCredentialProvider::new(AwsCredential { + key_id: KEY_ID.to_string(), + secret_key: SECRET_KEY.to_string(), + token: None, + })); + + // Make the object store work instead. + let object_store: Arc = Arc::new(CyperS3::new(S3Config { + bucket: BUCKET.into(), + credentials: static_creds, + region: "us-east-1".into(), + endpoint: Some("http://localhost:3030".into()), + virtual_host: false, + })?); + + let get_range = object_store.get_range(&Path::from("object1"), 0..4).await?; + assert_eq!(get_range.as_ref(), "0123".as_bytes()); + + let get_all = object_store + .get(&Path::from("object1")) + .await? + .bytes() + .await?; + assert_eq!(get_all.as_ref(), "0123456789".as_bytes()); + + Ok(()) +}