repo-sync-2024-08-28T18:19:29+0800 (#379)

* repo-sync-2024-08-28T18:19:29+0800 * Update arg_set.cc * Update item.cc * Update kdf_test.cc
secretflow · Aug 28, 2024 · 15ad722 · 15ad722
1 parent ddf3469
commit 15ad722
Show file tree

Hide file tree

Showing 48 changed files with 411 additions and 115 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -42,3 +42,10 @@ build:ubsan --features=ubsan
 test --keep_going
 test --test_output=errors
 test --test_timeout=360
+
+# make sure brpc is built with tongsuo in gm mode
+# this flag is used by the brpc project
+build:gm --define BRPC_WITH_TONGSUO=true
+
+# make sure yacl is built with tongsuo in gm mode
+build:gm --define YACL_WITH_TONGSUO=true
diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md
@@ -59,6 +59,7 @@ To build Yacl, at yacl's root directory, run the following
 bazel build //...
 bazel build //... -c opt        # build as optimized mode
 bazel build //... -c dbg        # build as debug mode
+bazel build //... --config gm   # build with gm mode
 ```
 
 To test Yacl

diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel
@@ -0,0 +1,19 @@
+# Copyright 2024 Ant Group Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+config_setting(
+    name = "gm",
+    define_values = {"YACL_WITH_TONGSUO": "true"},
+    visibility = ["//visibility:public"],
+)
diff --git a/bazel/hash_drbg.BUILD b/bazel/hash_drbg.BUILD
@@ -24,6 +24,6 @@ cc_library(
     copts = ["-Wno-parentheses"],
     visibility = ["//visibility:public"],
     deps = [
-        "@com_github_openssl_openssl//:openssl",
+        "@yacl//yacl/crypto:openssl_wrappers",
     ],
 )
diff --git a/bazel/patches/brpc.patch b/bazel/patches/brpc.patch
@@ -96,7 +96,7 @@ index 0bdaa4d3..b23881b3 100644
          "src/butil/time/time_mac.cc",
          "src/butil/mac/scoped_mach_port.cc",
      ],
-@@ -326,17 +326,17 @@ cc_library(
+@@ -326,13 +326,13 @@ cc_library(
      visibility = ["//visibility:public"],
      deps = [
          "@com_github_gflags_gflags//:gflags",
@@ -112,10 +112,6 @@ index 0bdaa4d3..b23881b3 100644
          "//conditions:default": [],
      }) + select({
          "//bazel/config:brpc_with_boringssl": ["@boringssl//:ssl", "@boringssl//:crypto"],
--        "//conditions:default": ["@openssl//:ssl", "@openssl//:crypto"],
-+        "//conditions:default": ["@com_github_openssl_openssl//:openssl"],
-     }),
- )
 
 diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel
 index 4471bad1..d9cc6bd6 100644

diff --git a/bazel/patches/brpc_crypto.patch b/bazel/patches/brpc_crypto.patch
@@ -0,0 +1,30 @@
+diff --git a/BUILD.bazel b/BUILD.bazel
+index b2a453e0..10eeee74 100644
+--- a/BUILD.bazel
++++ b/BUILD.bazel
+@@ -337,7 +337,8 @@ cc_library(
+         "//conditions:default": [],
+     }) + select({
+         "//bazel/config:brpc_with_boringssl": ["@boringssl//:ssl", "@boringssl//:crypto"],
+-        "//conditions:default": ["@openssl//:ssl", "@openssl//:crypto"],
++        "//bazel/config:brpc_with_tongsuo": ["@com_github_tongsuo_tongsuo//:tongsuo"],
++        "//conditions:default": ["@com_github_openssl_openssl//:openssl"],
+     }),
+ )
+
+diff --git a/bazel/config/BUILD.bazel b/bazel/config/BUILD.bazel
+index bed04d3b..8049ec3e 100644
+--- a/bazel/config/BUILD.bazel
++++ b/bazel/config/BUILD.bazel
+@@ -108,4 +108,10 @@ config_setting(
+     name = "brpc_with_boringssl",
+     define_values = {"BRPC_WITH_BORINGSSL": "true"},
+     visibility = ["//visibility:public"],
+-)
++)
++
++config_setting(
++    name = "brpc_with_tongsuo",
++    define_values = {"BRPC_WITH_TONGSUO": "true"},
++    visibility = ["//visibility:public"],
++)
diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl
@@ -34,8 +34,11 @@ def yacl_deps():
     _com_github_msgpack_msgpack()
     _com_github_greendow_hash_drbg()
 
-    # crypto related
+    # ssl
+    _com_github_tongsuo_tongsuo()
     _com_github_openssl_openssl()
+
+    # crypto related
     _com_github_blake3team_blake3()
     _com_github_libsodium()
     _com_github_libtom_libtommath()
@@ -92,6 +95,7 @@ def _com_github_brpc_brpc():
         patches = [
             "@yacl//bazel:patches/brpc.patch",
             "@yacl//bazel:patches/brpc_m1.patch",
+            "@yacl//bazel:patches/brpc_crypto.patch",
         ],
         urls = [
             "https://github.com/apache/brpc/archive/refs/tags/1.10.0.tar.gz",
@@ -177,6 +181,19 @@ def _com_github_openssl_openssl():
         build_file = "@yacl//bazel:openssl.BUILD",
     )
 
+def _com_github_tongsuo_tongsuo():
+    maybe(
+        http_archive,
+        name = "com_github_tongsuo_tongsuo",
+        sha256 = "57c2741750a699bfbdaa1bbe44a5733e9c8fc65d086c210151cfbc2bbd6fc975",
+        type = "tar.gz",
+        strip_prefix = "Tongsuo-8.4.0",
+        urls = [
+            "https://github.com/Tongsuo-Project/Tongsuo/archive/refs/tags/8.4.0.tar.gz",
+        ],
+        build_file = "@yacl//bazel:tongsuo.BUILD",
+    )
+
 def _com_github_fmtlib_fmt():
     maybe(
         http_archive,

diff --git a/bazel/tongsuo.BUILD b/bazel/tongsuo.BUILD
@@ -0,0 +1,67 @@
+# Copyright 2024 Ant Group Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("@yacl//bazel:yacl.bzl", "yacl_configure_make")
+
+filegroup(
+    name = "all_srcs",
+    srcs = glob(
+        include = ["**"],
+        exclude = ["*.bazel"],
+    ),
+)
+
+CONFIGURE_OPTIONS = [
+    # fixed openssl work dir for deterministic build.
+    "--openssldir=/tmp/openssl",
+    "--libdir=lib",
+    "no-legacy",
+    "no-weak-ssl-ciphers",
+    "no-tests",
+    "no-shared",
+    "no-ui-console",
+    "enable-ntls",  # for GM
+]
+
+MAKE_TARGETS = [
+    "build_programs",
+    "install_sw",
+]
+
+yacl_configure_make(
+    name = "tongsuo",
+    args = ["-j 4"],
+    configure_command = "Configure",
+    configure_in_place = True,
+    configure_options = CONFIGURE_OPTIONS,
+    env = select({
+        "@platforms//os:macos": {
+            "AR": "",
+        },
+        "//conditions:default": {
+            "MODULESDIR": "",
+        },
+    }),
+    lib_name = "tongsuo",
+    lib_source = ":all_srcs",
+    linkopts = ["-ldl"],
+    # Note that for Linux builds, libssl must come before libcrypto on the linker command-line.
+    # As such, libssl must be listed before libcrypto
+    out_static_libs = [
+        "libssl.a",
+        "libcrypto.a",
+    ],
+    targets = MAKE_TARGETS,
+    visibility = ["//visibility:public"],
+)
diff --git a/bazel/yacl.bzl b/bazel/yacl.bzl
@@ -68,7 +68,10 @@ def yacl_cc_binary(
         linkopts = [],
         **kargs):
     cc_binary(
-        copts = copts + _yacl_copts(),
+        copts = copts + _yacl_copts() + select({
+            "//bazel/config:gm": ["-DYACL_WITH_TONGSUO"],
+            "//conditions:default": [],
+        }),
         linkopts = linkopts + ["-ldl"],
         **kargs
     )
@@ -78,7 +81,10 @@ def yacl_cc_library(
         deps = [],
         **kargs):
     cc_library(
-        copts = _yacl_copts() + copts,
+        copts = _yacl_copts() + copts + select({
+            "//bazel/config:gm": ["-DYACL_WITH_TONGSUO"],
+            "//conditions:default": [],
+        }),
         deps = deps + [
             "@com_github_gabime_spdlog//:spdlog",
         ],
@@ -101,7 +107,10 @@ def yacl_cc_test(
         linkopts = [],
         **kwargs):
     cc_test(
-        copts = _yacl_copts() + copts,
+        copts = _yacl_copts() + copts + select({
+            "//bazel/config:gm": ["-DYACL_WITH_TONGSUO"],
+            "//conditions:default": [],
+        }),
         deps = deps + [
             "@com_google_googletest//:gtest_main",
         ],

diff --git a/yacl/crypto/BUILD.bazel b/yacl/crypto/BUILD.bazel
@@ -24,10 +24,11 @@ yacl_cc_library(
     deps = [
         "//yacl/base:byte_container_view",
         "//yacl/base:exception",
-        "//yacl/crypto/hash:hash_interface",
         "//yacl/utils:scope_guard",
-        "@com_github_openssl_openssl//:openssl",
-    ],
+    ] + select({
+        "//bazel/config:gm": ["@com_github_tongsuo_tongsuo//:tongsuo"],
+        "//conditions:default": ["@com_github_openssl_openssl//:openssl"],
+    }),
 )
 
 yacl_cc_library(
@@ -39,6 +40,7 @@ yacl_cc_library(
     deps = [
         ":openssl_wrappers",
         "//yacl:secparam",
+        "//yacl/crypto/hash:hash_interface",
         "//yacl/io/stream",
     ],
 )

diff --git a/yacl/crypto/aead/BUILD.bazel b/yacl/crypto/aead/BUILD.bazel
@@ -17,9 +17,9 @@ load("//bazel:yacl.bzl", "yacl_cc_library", "yacl_cc_test")
 package(default_visibility = ["//visibility:public"])
 
 yacl_cc_library(
-    name = "gcm_crypto",
-    srcs = ["gcm_crypto.cc"],
-    hdrs = ["gcm_crypto.h"],
+    name = "all_gcm",
+    srcs = ["all_gcm.cc"],
+    hdrs = ["all_gcm.h"],
     deps = [
         "//yacl:secparam",
         "//yacl/base:int128",
@@ -28,17 +28,17 @@ yacl_cc_library(
 )
 
 yacl_cc_test(
-    name = "gcm_crypto_test",
-    srcs = ["gcm_crypto_test.cc"],
+    name = "all_gcm_test",
+    srcs = ["all_gcm_test.cc"],
     deps = [
-        ":gcm_crypto",
+        ":all_gcm",
     ],
 )
 
 yacl_cc_library(
-    name = "sm4_mac",
-    srcs = ["sm4_mac.cc"],
-    hdrs = ["sm4_mac.h"],
+    name = "sm4_mte",
+    srcs = ["sm4_mte.cc"],
+    hdrs = ["sm4_mte.h"],
     deps = [
         "//yacl:secparam",
         "//yacl/crypto/block_cipher:symmetric_crypto",
@@ -48,9 +48,9 @@ yacl_cc_library(
 )
 
 yacl_cc_test(
-    name = "sm4_mac_test",
-    srcs = ["sm4_mac_test.cc"],
+    name = "sm4_mte_test",
+    srcs = ["sm4_mte_test.cc"],
     deps = [
-        ":sm4_mac",
+        ":sm4_mte",
     ],
 )
diff --git a/yacl/crypto/aead/gcm_crypto.cc → yacl/crypto/aead/all_gcm.cc b/yacl/crypto/aead/gcm_crypto.cc → yacl/crypto/aead/all_gcm.cc
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "yacl/crypto/aead/gcm_crypto.h"
+#include "yacl/crypto/aead/all_gcm.h"
 
 #include "yacl/crypto/openssl_wrappers.h"
 
@@ -21,14 +21,18 @@ namespace yacl::crypto {
 namespace {
 
 constexpr size_t kAesMacSize = 16;
+// constexpr size_t kSm4MacSize = 16;
 
 size_t GetMacSize(GcmCryptoSchema schema) {
   switch (schema) {
     case GcmCryptoSchema::AES128_GCM:
     case GcmCryptoSchema::AES256_GCM:
       return kAesMacSize;
-    // case GcmCryptoSchema::SM4_GCM:
-    //   return kAesMacSize;
+#ifdef YACL_WITH_TONGSUO
+    case GcmCryptoSchema::SM4_GCM:
+      return kAesMacSize;
+#endif
+
     default:
       YACL_THROW("Unknown crypto schema: {}", static_cast<int>(schema));
   }

diff --git a/yacl/crypto/aead/gcm_crypto.h → yacl/crypto/aead/all_gcm.h b/yacl/crypto/aead/gcm_crypto.h → yacl/crypto/aead/all_gcm.h
@@ -22,14 +22,16 @@
 #include "yacl/secparam.h"
 
 /* security parameter declaration */
-YACL_MODULE_DECLARE("aes_gcm", SecParam::C::k128, SecParam::S::INF);
+YACL_MODULE_DECLARE("all_gcm", SecParam::C::k128, SecParam::S::INF);
 
 namespace yacl::crypto {
 
 enum class GcmCryptoSchema : int {
   AES128_GCM, /* security level = 128 */
   AES256_GCM, /* security level = 256 */
-  // SM4_GCM /* TODO openssl 3.2 supports SM4 GCM */
+#ifdef YACL_WITH_TONGSUO
+  SM4_GCM /* NOTE only Yacl built with gm mode supports this feature */
+#endif
 };
 
 // -------------
@@ -72,11 +74,13 @@ class Aes256GcmCrypto : public GcmCrypto {
       : GcmCrypto(GcmCryptoSchema::AES256_GCM, key, iv) {}
 };
 
-// class Sm4GcmCrypto : public GcmCrypto {
-//  public:
-//   Sm4GcmCrypto(ByteContainerView key, ByteContainerView iv)
-//       : GcmCrypto(GcmCryptoSchema::SM4_GCM, key, iv) {}
-// };
+#ifdef YACL_WITH_TONGSUO
+class Sm4GcmCrypto : public GcmCrypto {
+ public:
+  Sm4GcmCrypto(ByteContainerView key, ByteContainerView iv)
+      : GcmCrypto(GcmCryptoSchema::SM4_GCM, key, iv) {}
+};
+#endif
 
 /* to a string which openssl recognizes */
 inline const char* ToString(GcmCryptoSchema scheme) {
@@ -85,8 +89,10 @@ inline const char* ToString(GcmCryptoSchema scheme) {
       return "aes-128-gcm";
     case GcmCryptoSchema::AES256_GCM:
       return "aes-256-gcm";
-    // case GcmCryptoSchema::SM4_GCM:
-    //   return "sm4-gcm";
+#ifdef YACL_WITH_TONGSUO
+    case GcmCryptoSchema::SM4_GCM:
+      return "sm4-gcm";
+#endif
     default:
       YACL_THROW("Unsupported gcm scheme: {}", static_cast<int>(scheme));
   }