From 4765f156a7b8dff8d420da1c7e03ff6532dfebbf Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 18 Sep 2023 13:47:35 -0400 Subject: [PATCH 1/9] fix: search additional protos in `BUILD.bazel` --- library_generation/generate_library.sh | 18 +++++- .../test/generate_library_integration_test.sh | 4 ++ library_generation/utilities.sh | 63 ++++++++++++++----- 3 files changed, 69 insertions(+), 16 deletions(-) diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index 812d1b1359..d43e0b13d6 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -25,6 +25,14 @@ case $key in protobuf_version="$2" shift ;; + --contains_iam_policy) + contains_iam_policy="$2" + shift + ;; + --contains_locations) + contains_locations="$2" + shift + ;; --grpc_version) grpc_version="$2" shift @@ -66,6 +74,14 @@ if [ -z "${grpc_version}" ]; then grpc_version=$(get_grpc_version "${gapic_generator_version}") fi +if [ -z "${contains_iam_policy}" ];then + contains_iam_policy="false" +fi + +if [ -z "${contains_locations}" ];then + contains_locations="false" +fi + if [ -z "${transport}" ]; then transport="grpc" fi @@ -117,7 +133,7 @@ fi "--plugin=protoc-gen-java_gapic=${script_dir}/gapic-generator-java-wrapper" \ "--java_gapic_out=metadata:${destination_path}/java_gapic_srcjar_raw.srcjar.zip" \ "--java_gapic_opt=$(get_gapic_opts)" \ -${proto_files} $(search_additional_protos) +${proto_files} $(search_additional_protos "${contains_iam_policy}" "${contains_locations}") unzip -o -q "${destination_path}/java_gapic_srcjar_raw.srcjar.zip" -d "${destination_path}" # Sync'\''d to the output file name in Writer.java. diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index 7b42402837..18eda4620b 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -68,6 +68,8 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do # parse GAPIC options from proto_path/BUILD.bazel pushd "${output_folder}" proto_build_file_path="${proto_path}/BUILD.bazel" + contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_build_file_path}") + contains_location=$(get_location_from_BUILD "${proto_build_file_path}") transport=$(get_transport_from_BUILD "${proto_build_file_path}") rest_numeric_enums=$(get_rest_numeric_enums_from_BUILD "${proto_build_file_path}") include_samples=$(get_include_samples_from_BUILD "${proto_build_file_path}") @@ -81,6 +83,8 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do --gapic_generator_version "${gapic_generator_version}" \ --protobuf_version "${protobuf_version}" \ --grpc_version "${grpc_version}" \ + --contains_iam_policy "${contains_iam_policy}" \ + --contains_locations "${contains_location}" \ --transport "${transport}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --include_samples "${include_samples}" diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 4ccdc42f32..1aa1f8bd69 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -18,7 +18,7 @@ __get_config_from_BUILD() { if_match=$5 result="${default}" - if grep -A 15 "${rule}" "${build_file}" | grep -q "${pattern}"; then + if grep -A 20 "${rule}" "${build_file}" | grep -q "${pattern}"; then result="${if_match}" fi echo "${result}" @@ -69,28 +69,19 @@ unzip_src_files() { rm -r -f "${destination_path}/${category}-${folder_name}/src/main/java/META-INF" } -find_additional_protos_in_yaml() { - local pattern=$1 - local find_result - find_result=$(grep --include=\*.yaml -rw "${proto_path}" -e "${pattern}") - if [ -n "${find_result}" ]; then - echo "${find_result}" - fi -} - # Apart from proto files in proto_path, additional protos are needed in order # to generate GAPIC client libraries. # In most cases, these protos should be within google/ directory, which is # pulled from googleapis as a prerequisite. -# Search additional protos in .yaml files. +# Search additional protos in BUILD.bazel. search_additional_protos() { + local contains_iam_policy=$1 + local contains_location=$2 additional_protos="google/cloud/common_resources.proto" # used by every library - iam_policy=$(find_additional_protos_in_yaml "name: '*google.iam.v1.IAMPolicy'*") - if [ -n "$iam_policy" ]; then + if [[ "${contains_iam_policy}" == "true" ]]; then additional_protos="$additional_protos google/iam/v1/iam_policy.proto" fi - locations=$(find_additional_protos_in_yaml "name: '*google.cloud.location.Locations'*") - if [ -n "${locations}" ]; then + if [[ "${contains_location}" == "true" ]]; then additional_protos="$additional_protos google/cloud/location/locations.proto" fi echo "${additional_protos}" @@ -253,6 +244,48 @@ get_version_from_WORKSPACE() { echo "${version}" } +get_iam_policy_from_BUILD() { + local build_file=$1 + local contains_iam_polcy + # search twice because it may be in two targets. + contains_iam_policy=$(__get_config_from_BUILD \ + "${build_file}" \ + "proto_library(" \ + "//google/iam/v1:iam_policy_proto" \ + "false" \ + "true" + ) + contains_iam_policy=$(__get_config_from_BUILD \ + "${build_file}" \ + "proto_library_with_info(" \ + "//google/iam/v1:iam_policy_proto" \ + "${contains_iam_policy}" \ + "true" + ) + echo "${contains_iam_polcy}" +} + +get_location_from_BUILD() { + local build_file=$1 + local contains_location + # search twice because it may be in two targets. + contains_location=$(__get_config_from_BUILD \ + "${build_file}" \ + "proto_library(" \ + "//google/cloud/location:location_proto" \ + "false" \ + "true" + ) + contains_location=$(__get_config_from_BUILD \ + "${build_file}" \ + "proto_library_with_info(" \ + "//google/cloud/location:location_proto" \ + "${contains_location}" \ + "true" + ) + echo "${contains_location}" +} + get_transport_from_BUILD() { local build_file=$1 local transport From 0ae9731e3b8d52063f8464883f088402b451b63b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 18 Sep 2023 14:24:05 -0400 Subject: [PATCH 2/9] add unit tests --- .../test/generate_library_unit_tests.sh | 76 +++++++++++++++++-- .../BUILD_iam_proto_library_match.bazel | 12 +++ ...LD_iam_proto_library_with_info_match.bazel | 12 +++ .../BUILD_iam_two_targets_match.bazel | 13 ++++ .../BUILD_iam_without_match.bazel | 11 +++ .../BUILD_location_proto_library_match.bazel | 12 +++ ...cation_proto_library_with_info_match.bazel | 12 +++ .../BUILD_location_two_targets_match.bazel | 13 ++++ .../BUILD_location_without_match.bazel | 11 +++ ...ditional_protos_common_resources_test.yaml | 10 --- .../search_additional_protos_iam_test.yaml | 6 -- ...h_additional_protos_iam_location_test.yaml | 6 -- ...earch_additional_protos_location_test.yaml | 5 -- library_generation/utilities.sh | 4 +- 14 files changed, 166 insertions(+), 37 deletions(-) create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_with_info_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_location_two_targets_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_location_without_match.bazel delete mode 100644 library_generation/test/resources/search_additional_protos/common_resources/search_additional_protos_common_resources_test.yaml delete mode 100644 library_generation/test/resources/search_additional_protos/iam/search_additional_protos_iam_test.yaml delete mode 100644 library_generation/test/resources/search_additional_protos/iam_location/search_additional_protos_iam_location_test.yaml delete mode 100644 library_generation/test/resources/search_additional_protos/location/search_additional_protos_location_test.yaml diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index b945ffffb1..97ace24aaa 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -42,34 +42,30 @@ get_protobuf_version_failed_with_invalid_generator_version_test() { } search_additional_protos_common_resources_test() { - local proto_path="${script_dir}/resources/search_additional_proto/common_resources" local addition_protos - addition_protos=$(search_additional_protos) + addition_protos=$(search_additional_protos "false" "false") assertEquals "google/cloud/common_resources.proto" "${addition_protos}" } search_additional_protos_iam_test() { - local proto_path="${script_dir}/resources/search_additional_protos/iam" local addition_protos - addition_protos=$(search_additional_protos) + addition_protos=$(search_additional_protos "true" "false") assertEquals \ "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto" \ "${addition_protos}" } search_additional_protos_location_test() { - local proto_path="${script_dir}/resources/search_additional_protos/location" local addition_protos - addition_protos=$(search_additional_protos) + addition_protos=$(search_additional_protos "false" "true") assertEquals \ "google/cloud/common_resources.proto google/cloud/location/locations.proto" \ "${addition_protos}" } search_additional_protos_iam_location_test() { - local proto_path="${script_dir}/resources/search_additional_protos/iam_location" local addition_protos - addition_protos=$(search_additional_protos) + addition_protos=$(search_additional_protos "true" "true") assertEquals \ "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto google/cloud/location/locations.proto" \ "${addition_protos}" @@ -227,6 +223,62 @@ generate_library_failed_with_invalid_grpc_version() { cleanup "${destination}" } +get_iam_policy_from_BUILD_without_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_without_match.bazel" + local contains_iam_policy + contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") + assertEquals "false" "${contains_iam_policy}" +} + +get_iam_policy_from_BUILD_proto_library_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel" + local contains_iam_policy + contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") + assertEquals "true" "${contains_iam_policy}" +} + +get_iam_policy_from_BUILD_proto_library_with_info_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel" + local contains_iam_policy + contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") + assertEquals "true" "${contains_iam_policy}" +} + +get_iam_policy_from_BUILD_two_targets_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel" + local contains_iam_policy + contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") + assertEquals "true" "${contains_iam_policy}" +} + +get_location_from_BUILD_without_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_location_without_match.bazel" + local contains_location + contains_location=$(get_location_from_BUILD "${proto_path}") + assertEquals "false" "${contains_location}" +} + +get_location_from_BUILD_proto_library_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_location_proto_library_match.bazel" + local contains_location + contains_location=$(get_location_from_BUILD "${proto_path}") + assertEquals "true" "${contains_location}" +} + +get_location_from_BUILD_proto_library_with_info_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_location_proto_library_with_info_match.bazel" + local contains_location + contains_location=$(get_location_from_BUILD "${proto_path}") + assertEquals "true" "${contains_location}" +} + +get_location_from_BUILD_two_targets_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_location_two_targets_match.bazel" + local contains_location + contains_location=$(get_location_from_BUILD "${proto_path}") + assertEquals "true" "${contains_location}" +} + get_transport_from_BUILD_grpc_rest_test() { local build_file="${script_dir}/resources/misc/BUILD_grpc_rest.bazel" local transport @@ -336,6 +388,14 @@ test_list=( generate_library_failed_with_invalid_generator_version generate_library_failed_with_invalid_protobuf_version generate_library_failed_with_invalid_grpc_version + get_iam_policy_from_BUILD_without_match_test + get_iam_policy_from_BUILD_proto_library_match_test + get_iam_policy_from_BUILD_proto_library_with_info_match_test + get_iam_policy_from_BUILD_two_targets_match_test + get_location_from_BUILD_without_match_test + get_location_from_BUILD_proto_library_match_test + get_location_from_BUILD_proto_library_with_info_match_test + get_location_from_BUILD_two_targets_match_test get_transport_from_BUILD_grpc_rest_test get_transport_from_BUILD_grpc_test get_transport_from_BUILD_rest_test diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel new file mode 100644 index 0000000000..4397a96283 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel @@ -0,0 +1,12 @@ +# this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh + +proto_library( + deps = [ + "//google/iam/v1:iam_policy_proto", + ] +) + +proto_library_with_info( + deps = [ + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel new file mode 100644 index 0000000000..132872f075 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel @@ -0,0 +1,12 @@ +# this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh + +proto_library( + deps = [ + ] +) + +proto_library_with_info( + deps = [ + "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel new file mode 100644 index 0000000000..0b7bc7d5b1 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel @@ -0,0 +1,13 @@ +# this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh + +proto_library( + deps = [ + "//google/iam/v1:iam_policy_proto", + ] +) + +proto_library_with_info( + deps = [ + "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel new file mode 100644 index 0000000000..55403ee23a --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel @@ -0,0 +1,11 @@ +# this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh + +proto_library( + deps = [ + ] +) + +proto_library_with_info( + deps = [ + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_match.bazel new file mode 100644 index 0000000000..39c3dde199 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_match.bazel @@ -0,0 +1,12 @@ +# this file is only used in testing `get_location_from_BUILD` in utilities.sh + +proto_library( + deps = [ + "//google/cloud/location:location_proto", + ] +) + +proto_library_with_info( + deps = [ + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_with_info_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_with_info_match.bazel new file mode 100644 index 0000000000..5d5141761e --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_with_info_match.bazel @@ -0,0 +1,12 @@ +# this file is only used in testing `get_location_from_BUILD` in utilities.sh + +proto_library( + deps = [ + ] +) + +proto_library_with_info( + deps = [ + "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_location_two_targets_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_location_two_targets_match.bazel new file mode 100644 index 0000000000..1730f271e2 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_location_two_targets_match.bazel @@ -0,0 +1,13 @@ +# this file is only used in testing `get_location_from_BUILD` in utilities.sh + +proto_library( + deps = [ + "//google/cloud/location:location_proto", + ] +) + +proto_library_with_info( + deps = [ + "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_location_without_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_location_without_match.bazel new file mode 100644 index 0000000000..e37d19e189 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_location_without_match.bazel @@ -0,0 +1,11 @@ +# this file is only used in testing `get_location_from_BUILD` in utilities.sh + +proto_library( + deps = [ + ] +) + +proto_library_with_info( + deps = [ + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/common_resources/search_additional_protos_common_resources_test.yaml b/library_generation/test/resources/search_additional_protos/common_resources/search_additional_protos_common_resources_test.yaml deleted file mode 100644 index fc5c8bfca5..0000000000 --- a/library_generation/test/resources/search_additional_protos/common_resources/search_additional_protos_common_resources_test.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# this file is only used in testing `search_additional_protos` in utilities.sh -apis: -- name: google.monitoring.v3.AlertPolicyService -- name: google.monitoring.v3.GroupService -- name: google.monitoring.v3.MetricService -- name: google.monitoring.v3.NotificationChannelService -- name: google.monitoring.v3.QueryService -- name: google.monitoring.v3.ServiceMonitoringService -- name: google.monitoring.v3.SnoozeService -- name: google.monitoring.v3.UptimeCheckService diff --git a/library_generation/test/resources/search_additional_protos/iam/search_additional_protos_iam_test.yaml b/library_generation/test/resources/search_additional_protos/iam/search_additional_protos_iam_test.yaml deleted file mode 100644 index cc946f1e11..0000000000 --- a/library_generation/test/resources/search_additional_protos/iam/search_additional_protos_iam_test.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# this file is only used in testing `search_additional_protos` in utilities.sh -apis: -- name: google.iam.v1.IAMPolicy -- name: google.pubsub.v1.Publisher -- name: google.pubsub.v1.SchemaService -- name: google.pubsub.v1.Subscriber diff --git a/library_generation/test/resources/search_additional_protos/iam_location/search_additional_protos_iam_location_test.yaml b/library_generation/test/resources/search_additional_protos/iam_location/search_additional_protos_iam_location_test.yaml deleted file mode 100644 index 0789b073e4..0000000000 --- a/library_generation/test/resources/search_additional_protos/iam_location/search_additional_protos_iam_location_test.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# this file is only used in testing `search_additional_protos` in utilities.sh -apis: -- name: google.cloud.alloydb.v1.AlloyDBAdmin -- name: google.cloud.location.Locations -- name: google.iam.v1.IAMPolicy -- name: google.longrunning.Operations diff --git a/library_generation/test/resources/search_additional_protos/location/search_additional_protos_location_test.yaml b/library_generation/test/resources/search_additional_protos/location/search_additional_protos_location_test.yaml deleted file mode 100644 index 73abf4dc7c..0000000000 --- a/library_generation/test/resources/search_additional_protos/location/search_additional_protos_location_test.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `search_additional_protos` in utilities.sh -apis: -- name: google.cloud.location.Locations -- name: google.firestore.v1.Firestore -- name: google.longrunning.Operations diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 1aa1f8bd69..88f7d31062 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -246,7 +246,7 @@ get_version_from_WORKSPACE() { get_iam_policy_from_BUILD() { local build_file=$1 - local contains_iam_polcy + local contains_iam_policy # search twice because it may be in two targets. contains_iam_policy=$(__get_config_from_BUILD \ "${build_file}" \ @@ -262,7 +262,7 @@ get_iam_policy_from_BUILD() { "${contains_iam_policy}" \ "true" ) - echo "${contains_iam_polcy}" + echo "${contains_iam_policy}" } get_location_from_BUILD() { From c6917cdf66a05613607e83fad17e169c0ee41888 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 18 Sep 2023 14:37:23 -0400 Subject: [PATCH 3/9] modify readme --- library_generation/README.md | 20 ++++++- .../test/generate_library_integration_test.sh | 4 +- .../test/generate_library_unit_tests.sh | 56 +++++++++---------- ...BUILD_locations_proto_library_match.bazel} | 0 ...tions_proto_library_with_info_match.bazel} | 0 ...> BUILD_locations_two_targets_match.bazel} | 0 ...el => BUILD_locations_without_match.bazel} | 0 library_generation/utilities.sh | 16 +++--- 8 files changed, 56 insertions(+), 40 deletions(-) rename library_generation/test/resources/search_additional_protos/{BUILD_location_proto_library_match.bazel => BUILD_locations_proto_library_match.bazel} (100%) rename library_generation/test/resources/search_additional_protos/{BUILD_location_proto_library_with_info_match.bazel => BUILD_locations_proto_library_with_info_match.bazel} (100%) rename library_generation/test/resources/search_additional_protos/{BUILD_location_two_targets_match.bazel => BUILD_locations_two_targets_match.bazel} (100%) rename library_generation/test/resources/search_additional_protos/{BUILD_location_without_match.bazel => BUILD_locations_without_match.bazel} (100%) diff --git a/library_generation/README.md b/library_generation/README.md index 1a9e77893f..4dd0bf44fa 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -91,14 +91,30 @@ Use `--grpc_version` to specify the value. Note that if specified, the version should be compatible with gapic-generator-java. +### contains_iam_policy (optional) +Whether to include `google/iam/v1/iam_policy.proto` into the library. +The value is either `true` or `false`. +The default value is `false`. + +Use `--contains_iam_policy` to specify the value. + +### contains_locations (optional) +Whether to include `google/cloud/location/locations.proto` into the library. +The value is either `true` or `false`. +The default value is `false`. + +Use `--contains_locations` to specify the value. + ### transport (optional) -One of GAPIC options passed to the generator. The value is either `grpc` or `grpc+rest`. +One of GAPIC options passed to the generator. +The value is either `grpc` or `grpc+rest`. The default value is `grpc`. Use `--transport` to specify the value. ### rest_numeric_enums (optional) -One of GAPIC options passed to the generator. The value is either `true` or `false`. +One of GAPIC options passed to the generator. +The value is either `true` or `false`. The default value is `true`. Use `--rest_numeric_enums` to specify the value. diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index 18eda4620b..6f1e0d84ef 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -69,7 +69,7 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do pushd "${output_folder}" proto_build_file_path="${proto_path}/BUILD.bazel" contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_build_file_path}") - contains_location=$(get_location_from_BUILD "${proto_build_file_path}") + contains_locations=$(get_locations_from_BUILD "${proto_build_file_path}") transport=$(get_transport_from_BUILD "${proto_build_file_path}") rest_numeric_enums=$(get_rest_numeric_enums_from_BUILD "${proto_build_file_path}") include_samples=$(get_include_samples_from_BUILD "${proto_build_file_path}") @@ -84,7 +84,7 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do --protobuf_version "${protobuf_version}" \ --grpc_version "${grpc_version}" \ --contains_iam_policy "${contains_iam_policy}" \ - --contains_locations "${contains_location}" \ + --contains_locations "${contains_locations}" \ --transport "${transport}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --include_samples "${include_samples}" diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index 97ace24aaa..523947b42c 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -55,7 +55,7 @@ search_additional_protos_iam_test() { "${addition_protos}" } -search_additional_protos_location_test() { +search_additional_protos_locations_test() { local addition_protos addition_protos=$(search_additional_protos "false" "true") assertEquals \ @@ -63,7 +63,7 @@ search_additional_protos_location_test() { "${addition_protos}" } -search_additional_protos_iam_location_test() { +search_additional_protos_iam_locations_test() { local addition_protos addition_protos=$(search_additional_protos "true" "true") assertEquals \ @@ -251,32 +251,32 @@ get_iam_policy_from_BUILD_two_targets_match_test() { assertEquals "true" "${contains_iam_policy}" } -get_location_from_BUILD_without_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_location_without_match.bazel" - local contains_location - contains_location=$(get_location_from_BUILD "${proto_path}") - assertEquals "false" "${contains_location}" +get_locations_from_BUILD_without_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_without_match.bazel" + local contains_locations + contains_locations=$(get_locations_from_BUILD "${proto_path}") + assertEquals "false" "${contains_locations}" } -get_location_from_BUILD_proto_library_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_location_proto_library_match.bazel" - local contains_location - contains_location=$(get_location_from_BUILD "${proto_path}") - assertEquals "true" "${contains_location}" +get_locations_from_BUILD_proto_library_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_proto_library_match.bazel" + local contains_locations + contains_locations=$(get_locations_from_BUILD "${proto_path}") + assertEquals "true" "${contains_locations}" } -get_location_from_BUILD_proto_library_with_info_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_location_proto_library_with_info_match.bazel" - local contains_location - contains_location=$(get_location_from_BUILD "${proto_path}") - assertEquals "true" "${contains_location}" +get_locations_from_BUILD_proto_library_with_info_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_proto_library_with_info_match.bazel" + local contains_locations + contains_locations=$(get_locations_from_BUILD "${proto_path}") + assertEquals "true" "${contains_locations}" } -get_location_from_BUILD_two_targets_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_location_two_targets_match.bazel" - local contains_location - contains_location=$(get_location_from_BUILD "${proto_path}") - assertEquals "true" "${contains_location}" +get_locations_from_BUILD_two_targets_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_two_targets_match.bazel" + local contains_locations + contains_locations=$(get_locations_from_BUILD "${proto_path}") + assertEquals "true" "${contains_locations}" } get_transport_from_BUILD_grpc_rest_test() { @@ -370,8 +370,8 @@ test_list=( get_protobuf_version_failed_with_invalid_generator_version_test search_additional_protos_common_resources_test search_additional_protos_iam_test - search_additional_protos_location_test - search_additional_protos_iam_location_test + search_additional_protos_locations_test + search_additional_protos_iam_locations_test get_gapic_opts_with_rest_test get_gapic_opts_without_rest_test remove_grpc_version_test @@ -392,10 +392,10 @@ test_list=( get_iam_policy_from_BUILD_proto_library_match_test get_iam_policy_from_BUILD_proto_library_with_info_match_test get_iam_policy_from_BUILD_two_targets_match_test - get_location_from_BUILD_without_match_test - get_location_from_BUILD_proto_library_match_test - get_location_from_BUILD_proto_library_with_info_match_test - get_location_from_BUILD_two_targets_match_test + get_locations_from_BUILD_without_match_test + get_locations_from_BUILD_proto_library_match_test + get_locations_from_BUILD_proto_library_with_info_match_test + get_locations_from_BUILD_two_targets_match_test get_transport_from_BUILD_grpc_rest_test get_transport_from_BUILD_grpc_test get_transport_from_BUILD_rest_test diff --git a/library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_match.bazel similarity index 100% rename from library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_match.bazel rename to library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_match.bazel diff --git a/library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_with_info_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_with_info_match.bazel similarity index 100% rename from library_generation/test/resources/search_additional_protos/BUILD_location_proto_library_with_info_match.bazel rename to library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_with_info_match.bazel diff --git a/library_generation/test/resources/search_additional_protos/BUILD_location_two_targets_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_two_targets_match.bazel similarity index 100% rename from library_generation/test/resources/search_additional_protos/BUILD_location_two_targets_match.bazel rename to library_generation/test/resources/search_additional_protos/BUILD_locations_two_targets_match.bazel diff --git a/library_generation/test/resources/search_additional_protos/BUILD_location_without_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel similarity index 100% rename from library_generation/test/resources/search_additional_protos/BUILD_location_without_match.bazel rename to library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 88f7d31062..3ebd832e02 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -76,12 +76,12 @@ unzip_src_files() { # Search additional protos in BUILD.bazel. search_additional_protos() { local contains_iam_policy=$1 - local contains_location=$2 + local contains_locations=$2 additional_protos="google/cloud/common_resources.proto" # used by every library if [[ "${contains_iam_policy}" == "true" ]]; then additional_protos="$additional_protos google/iam/v1/iam_policy.proto" fi - if [[ "${contains_location}" == "true" ]]; then + if [[ "${contains_locations}" == "true" ]]; then additional_protos="$additional_protos google/cloud/location/locations.proto" fi echo "${additional_protos}" @@ -265,25 +265,25 @@ get_iam_policy_from_BUILD() { echo "${contains_iam_policy}" } -get_location_from_BUILD() { +get_locations_from_BUILD() { local build_file=$1 - local contains_location + local contains_locations # search twice because it may be in two targets. - contains_location=$(__get_config_from_BUILD \ + contains_locations=$(__get_config_from_BUILD \ "${build_file}" \ "proto_library(" \ "//google/cloud/location:location_proto" \ "false" \ "true" ) - contains_location=$(__get_config_from_BUILD \ + contains_locations=$(__get_config_from_BUILD \ "${build_file}" \ "proto_library_with_info(" \ "//google/cloud/location:location_proto" \ - "${contains_location}" \ + "${contains_locations}" \ "true" ) - echo "${contains_location}" + echo "${contains_locations}" } get_transport_from_BUILD() { From e8eb12f21b1cd7d4aa15f21f083ea22d3f509580 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 18 Sep 2023 15:14:57 -0400 Subject: [PATCH 4/9] only search in proto_library_with_info --- .../test/generate_library_unit_tests.sh | 44 +++---------------- ...rary_match.bazel => BUILD_iam_match.bazel} | 7 +-- ...LD_iam_proto_library_with_info_match.bazel | 12 ----- .../BUILD_iam_two_targets_match.bazel | 13 ------ .../BUILD_iam_without_match.bazel | 5 --- ...atch.bazel => BUILD_locations_match.bazel} | 5 --- .../BUILD_locations_proto_library_match.bazel | 12 ----- .../BUILD_locations_two_targets_match.bazel | 13 ------ .../BUILD_locations_without_match.bazel | 5 --- library_generation/utilities.sh | 20 +-------- 10 files changed, 9 insertions(+), 127 deletions(-) rename library_generation/test/resources/search_additional_protos/{BUILD_iam_proto_library_match.bazel => BUILD_iam_match.bazel} (83%) delete mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel delete mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel rename library_generation/test/resources/search_additional_protos/{BUILD_locations_proto_library_with_info_match.bazel => BUILD_locations_match.bazel} (83%) delete mode 100644 library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_match.bazel delete mode 100644 library_generation/test/resources/search_additional_protos/BUILD_locations_two_targets_match.bazel diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index 523947b42c..ceca377c28 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -230,22 +230,8 @@ get_iam_policy_from_BUILD_without_match_test() { assertEquals "false" "${contains_iam_policy}" } -get_iam_policy_from_BUILD_proto_library_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel" - local contains_iam_policy - contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") - assertEquals "true" "${contains_iam_policy}" -} - -get_iam_policy_from_BUILD_proto_library_with_info_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel" - local contains_iam_policy - contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") - assertEquals "true" "${contains_iam_policy}" -} - -get_iam_policy_from_BUILD_two_targets_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel" +get_iam_policy_from_BUILD_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_match.bazel" local contains_iam_policy contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") assertEquals "true" "${contains_iam_policy}" @@ -258,22 +244,8 @@ get_locations_from_BUILD_without_match_test() { assertEquals "false" "${contains_locations}" } -get_locations_from_BUILD_proto_library_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_proto_library_match.bazel" - local contains_locations - contains_locations=$(get_locations_from_BUILD "${proto_path}") - assertEquals "true" "${contains_locations}" -} - -get_locations_from_BUILD_proto_library_with_info_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_proto_library_with_info_match.bazel" - local contains_locations - contains_locations=$(get_locations_from_BUILD "${proto_path}") - assertEquals "true" "${contains_locations}" -} - -get_locations_from_BUILD_two_targets_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_two_targets_match.bazel" +get_locations_from_BUILD_match_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_match.bazel" local contains_locations contains_locations=$(get_locations_from_BUILD "${proto_path}") assertEquals "true" "${contains_locations}" @@ -389,13 +361,9 @@ test_list=( generate_library_failed_with_invalid_protobuf_version generate_library_failed_with_invalid_grpc_version get_iam_policy_from_BUILD_without_match_test - get_iam_policy_from_BUILD_proto_library_match_test - get_iam_policy_from_BUILD_proto_library_with_info_match_test - get_iam_policy_from_BUILD_two_targets_match_test + get_iam_policy_from_BUILD_match_test get_locations_from_BUILD_without_match_test - get_locations_from_BUILD_proto_library_match_test - get_locations_from_BUILD_proto_library_with_info_match_test - get_locations_from_BUILD_two_targets_match_test + get_locations_from_BUILD_match_test get_transport_from_BUILD_grpc_rest_test get_transport_from_BUILD_grpc_test get_transport_from_BUILD_rest_test diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_match.bazel similarity index 83% rename from library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel rename to library_generation/test/resources/search_additional_protos/BUILD_iam_match.bazel index 4397a96283..f99b0c741c 100644 --- a/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_match.bazel +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_match.bazel @@ -1,12 +1,7 @@ # this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh -proto_library( - deps = [ - "//google/iam/v1:iam_policy_proto", - ] -) - proto_library_with_info( deps = [ + "//google/iam/v1:iam_policy_proto", ] ) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel deleted file mode 100644 index 132872f075..0000000000 --- a/library_generation/test/resources/search_additional_protos/BUILD_iam_proto_library_with_info_match.bazel +++ /dev/null @@ -1,12 +0,0 @@ -# this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh - -proto_library( - deps = [ - ] -) - -proto_library_with_info( - deps = [ - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel deleted file mode 100644 index 0b7bc7d5b1..0000000000 --- a/library_generation/test/resources/search_additional_protos/BUILD_iam_two_targets_match.bazel +++ /dev/null @@ -1,13 +0,0 @@ -# this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh - -proto_library( - deps = [ - "//google/iam/v1:iam_policy_proto", - ] -) - -proto_library_with_info( - deps = [ - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel index 55403ee23a..3765170fc9 100644 --- a/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel @@ -1,10 +1,5 @@ # this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh -proto_library( - deps = [ - ] -) - proto_library_with_info( deps = [ ] diff --git a/library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_with_info_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_match.bazel similarity index 83% rename from library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_with_info_match.bazel rename to library_generation/test/resources/search_additional_protos/BUILD_locations_match.bazel index 5d5141761e..2f00360b4d 100644 --- a/library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_with_info_match.bazel +++ b/library_generation/test/resources/search_additional_protos/BUILD_locations_match.bazel @@ -1,10 +1,5 @@ # this file is only used in testing `get_location_from_BUILD` in utilities.sh -proto_library( - deps = [ - ] -) - proto_library_with_info( deps = [ "//google/cloud/location:location_proto", diff --git a/library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_match.bazel deleted file mode 100644 index 39c3dde199..0000000000 --- a/library_generation/test/resources/search_additional_protos/BUILD_locations_proto_library_match.bazel +++ /dev/null @@ -1,12 +0,0 @@ -# this file is only used in testing `get_location_from_BUILD` in utilities.sh - -proto_library( - deps = [ - "//google/cloud/location:location_proto", - ] -) - -proto_library_with_info( - deps = [ - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_locations_two_targets_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_two_targets_match.bazel deleted file mode 100644 index 1730f271e2..0000000000 --- a/library_generation/test/resources/search_additional_protos/BUILD_locations_two_targets_match.bazel +++ /dev/null @@ -1,13 +0,0 @@ -# this file is only used in testing `get_location_from_BUILD` in utilities.sh - -proto_library( - deps = [ - "//google/cloud/location:location_proto", - ] -) - -proto_library_with_info( - deps = [ - "//google/cloud/location:location_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel index e37d19e189..f1d74b1261 100644 --- a/library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel +++ b/library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel @@ -1,10 +1,5 @@ # this file is only used in testing `get_location_from_BUILD` in utilities.sh -proto_library( - deps = [ - ] -) - proto_library_with_info( deps = [ ] diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 3ebd832e02..73dfaf4d5b 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -247,19 +247,11 @@ get_version_from_WORKSPACE() { get_iam_policy_from_BUILD() { local build_file=$1 local contains_iam_policy - # search twice because it may be in two targets. - contains_iam_policy=$(__get_config_from_BUILD \ - "${build_file}" \ - "proto_library(" \ - "//google/iam/v1:iam_policy_proto" \ - "false" \ - "true" - ) contains_iam_policy=$(__get_config_from_BUILD \ "${build_file}" \ "proto_library_with_info(" \ "//google/iam/v1:iam_policy_proto" \ - "${contains_iam_policy}" \ + "false" \ "true" ) echo "${contains_iam_policy}" @@ -268,19 +260,11 @@ get_iam_policy_from_BUILD() { get_locations_from_BUILD() { local build_file=$1 local contains_locations - # search twice because it may be in two targets. - contains_locations=$(__get_config_from_BUILD \ - "${build_file}" \ - "proto_library(" \ - "//google/cloud/location:location_proto" \ - "false" \ - "true" - ) contains_locations=$(__get_config_from_BUILD \ "${build_file}" \ "proto_library_with_info(" \ "//google/cloud/location:location_proto" \ - "${contains_locations}" \ + "false" \ "true" ) echo "${contains_locations}" From cf1cf50d4a90260473445969492ca11e53c5ad93 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Mon, 18 Sep 2023 15:28:39 -0400 Subject: [PATCH 5/9] fix showcase ci --- showcase/scripts/generate_showcase.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh index 6619f5801d..951992cc53 100755 --- a/showcase/scripts/generate_showcase.sh +++ b/showcase/scripts/generate_showcase.sh @@ -39,6 +39,8 @@ fi ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-java") +contains_iam_policy="true" +contains_locations="true" rest_numeric_enums="false" transport="grpc+rest" include_samples="false" @@ -49,6 +51,8 @@ bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \ --proto_path "schema/google/showcase/v1beta1" \ --destination_path "showcase-output" \ --gapic_generator_version "${ggj_version}" \ + --contains_iam_policy "${contains_iam_policy}" \ + --contains_locations "${contains_locations}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --include_samples "${include_samples}" \ --transport "${transport}" From ea7554bdb0332f19b3076f8453b1ada16b1aa156 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 19 Sep 2023 13:53:09 -0400 Subject: [PATCH 6/9] add option: gapic_additional_protos --- library_generation/README.md | 18 ++-- library_generation/generate_library.sh | 23 ++---- .../test/generate_library_integration_test.sh | 6 +- .../test/generate_library_unit_tests.sh | 82 ++++++------------- .../BUILD_common_resources.bazel | 6 ++ .../BUILD_iam_locations.bazel | 8 ++ .../BUILD_iam_match.bazel | 7 -- .../BUILD_iam_policy.bazel | 7 ++ .../BUILD_iam_without_match.bazel | 6 -- .../BUILD_locations.bazel | 7 ++ .../BUILD_locations_match.bazel | 7 -- .../BUILD_locations_without_match.bazel | 6 -- library_generation/utilities.sh | 58 +++++++------ 13 files changed, 103 insertions(+), 138 deletions(-) create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_common_resources.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_locations.bazel delete mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_policy.bazel delete mode 100644 library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel create mode 100644 library_generation/test/resources/search_additional_protos/BUILD_locations.bazel delete mode 100644 library_generation/test/resources/search_additional_protos/BUILD_locations_match.bazel delete mode 100644 library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel diff --git a/library_generation/README.md b/library_generation/README.md index 4dd0bf44fa..88a00a4744 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -91,19 +91,14 @@ Use `--grpc_version` to specify the value. Note that if specified, the version should be compatible with gapic-generator-java. -### contains_iam_policy (optional) -Whether to include `google/iam/v1/iam_policy.proto` into the library. -The value is either `true` or `false`. -The default value is `false`. - -Use `--contains_iam_policy` to specify the value. +### gapic_additional_protos (optional) +Additional protos that pass to the generator. +The default value is empty string, i.e., no additional proto. -### contains_locations (optional) -Whether to include `google/cloud/location/locations.proto` into the library. -The value is either `true` or `false`. -The default value is `false`. +Use `--gapic_additional_protos` to specify the value. -Use `--contains_locations` to specify the value. +Note that `google/cloud/common_resources.proto` is always pass to the generator, so +no need to specify this proto. ### transport (optional) One of GAPIC options passed to the generator. @@ -137,6 +132,7 @@ library_generation/generate_library.sh \ --gapic_generator_version 2.24.0 \ --protobuf_version 23.2 \ --grpc_version 1.55.1 \ +--gapic_additional_protos google/cloud/location/locations.proto\ --transport grpc+rest \ --rest_numeric_enums true \ --include_samples true diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index d43e0b13d6..f270041b9b 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -25,18 +25,14 @@ case $key in protobuf_version="$2" shift ;; - --contains_iam_policy) - contains_iam_policy="$2" - shift - ;; - --contains_locations) - contains_locations="$2" - shift - ;; --grpc_version) grpc_version="$2" shift ;; + --gapic_additional_protos) + gapic_additional_protos="$2" + shift + ;; --transport) transport="$2" shift @@ -64,7 +60,6 @@ done script_dir=$(dirname "$(readlink -f "$0")") source "${script_dir}"/utilities.sh output_folder="$(get_output_folder)" -# source utility functions if [ -z "${protobuf_version}" ]; then protobuf_version=$(get_protobuf_version "${gapic_generator_version}") @@ -74,12 +69,8 @@ if [ -z "${grpc_version}" ]; then grpc_version=$(get_grpc_version "${gapic_generator_version}") fi -if [ -z "${contains_iam_policy}" ];then - contains_iam_policy="false" -fi - -if [ -z "${contains_locations}" ];then - contains_locations="false" +if [ -z "${gapic_additional_protos}" ]; then + gapic_additional_protos="" fi if [ -z "${transport}" ]; then @@ -133,7 +124,7 @@ fi "--plugin=protoc-gen-java_gapic=${script_dir}/gapic-generator-java-wrapper" \ "--java_gapic_out=metadata:${destination_path}/java_gapic_srcjar_raw.srcjar.zip" \ "--java_gapic_opt=$(get_gapic_opts)" \ -${proto_files} $(search_additional_protos "${contains_iam_policy}" "${contains_locations}") +${proto_files} ${gapic_additional_protos} unzip -o -q "${destination_path}/java_gapic_srcjar_raw.srcjar.zip" -d "${destination_path}" # Sync'\''d to the output file name in Writer.java. diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index 6f1e0d84ef..96429c6e43 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -68,8 +68,7 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do # parse GAPIC options from proto_path/BUILD.bazel pushd "${output_folder}" proto_build_file_path="${proto_path}/BUILD.bazel" - contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_build_file_path}") - contains_locations=$(get_locations_from_BUILD "${proto_build_file_path}") + gapic_additional_protos=$(get_gapic_additional_protos_from_BUILD "${proto_build_file_path}") transport=$(get_transport_from_BUILD "${proto_build_file_path}") rest_numeric_enums=$(get_rest_numeric_enums_from_BUILD "${proto_build_file_path}") include_samples=$(get_include_samples_from_BUILD "${proto_build_file_path}") @@ -83,8 +82,7 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do --gapic_generator_version "${gapic_generator_version}" \ --protobuf_version "${protobuf_version}" \ --grpc_version "${grpc_version}" \ - --contains_iam_policy "${contains_iam_policy}" \ - --contains_locations "${contains_locations}" \ + --gapic_additional_protos "${gapic_additional_protos}" \ --transport "${transport}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --include_samples "${include_samples}" diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index ceca377c28..4af6da36ec 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -41,36 +41,6 @@ get_protobuf_version_failed_with_invalid_generator_version_test() { assertEquals 1 $((res)) } -search_additional_protos_common_resources_test() { - local addition_protos - addition_protos=$(search_additional_protos "false" "false") - assertEquals "google/cloud/common_resources.proto" "${addition_protos}" -} - -search_additional_protos_iam_test() { - local addition_protos - addition_protos=$(search_additional_protos "true" "false") - assertEquals \ - "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto" \ - "${addition_protos}" -} - -search_additional_protos_locations_test() { - local addition_protos - addition_protos=$(search_additional_protos "false" "true") - assertEquals \ - "google/cloud/common_resources.proto google/cloud/location/locations.proto" \ - "${addition_protos}" -} - -search_additional_protos_iam_locations_test() { - local addition_protos - addition_protos=$(search_additional_protos "true" "true") - assertEquals \ - "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto google/cloud/location/locations.proto" \ - "${addition_protos}" -} - get_gapic_opts_with_rest_test() { local proto_path="${script_dir}/resources/gapic_options" local transport="grpc" @@ -223,32 +193,32 @@ generate_library_failed_with_invalid_grpc_version() { cleanup "${destination}" } -get_iam_policy_from_BUILD_without_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_without_match.bazel" - local contains_iam_policy - contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") - assertEquals "false" "${contains_iam_policy}" +get_gapic_additional_protos_from_BUILD_common_resources_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_common_resources.bazel" + local addition_protos + addition_protos=$(get_gapic_additional_protos_from_BUILD "${proto_path}") + assertEquals "google/cloud/common_resources.proto" "${addition_protos}" } -get_iam_policy_from_BUILD_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_match.bazel" - local contains_iam_policy - contains_iam_policy=$(get_iam_policy_from_BUILD "${proto_path}") - assertEquals "true" "${contains_iam_policy}" +get_gapic_additional_protos_from_BUILD_iam_policy_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_policy.bazel" + local addition_protos + addition_protos=$(get_gapic_additional_protos_from_BUILD "${proto_path}") + assertEquals "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto" "${addition_protos}" } -get_locations_from_BUILD_without_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_without_match.bazel" - local contains_locations - contains_locations=$(get_locations_from_BUILD "${proto_path}") - assertEquals "false" "${contains_locations}" +get_gapic_additional_protos_from_BUILD_locations_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations.bazel" + local addition_protos + addition_protos=$(get_gapic_additional_protos_from_BUILD "${proto_path}") + assertEquals "google/cloud/common_resources.proto google/cloud/location/locations.proto" "${addition_protos}" } -get_locations_from_BUILD_match_test() { - local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations_match.bazel" - local contains_locations - contains_locations=$(get_locations_from_BUILD "${proto_path}") - assertEquals "true" "${contains_locations}" +get_gapic_additional_protos_from_BUILD_iam_locations_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_locations.bazel" + local addition_protos + addition_protos=$(get_gapic_additional_protos_from_BUILD "${proto_path}") + assertEquals "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto google/cloud/location/locations.proto" "${addition_protos}" } get_transport_from_BUILD_grpc_rest_test() { @@ -340,10 +310,6 @@ test_list=( get_grpc_version_failed_with_invalid_generator_version_test get_protobuf_version_succeed_with_valid_generator_version_test get_protobuf_version_failed_with_invalid_generator_version_test - search_additional_protos_common_resources_test - search_additional_protos_iam_test - search_additional_protos_locations_test - search_additional_protos_iam_locations_test get_gapic_opts_with_rest_test get_gapic_opts_without_rest_test remove_grpc_version_test @@ -360,10 +326,10 @@ test_list=( generate_library_failed_with_invalid_generator_version generate_library_failed_with_invalid_protobuf_version generate_library_failed_with_invalid_grpc_version - get_iam_policy_from_BUILD_without_match_test - get_iam_policy_from_BUILD_match_test - get_locations_from_BUILD_without_match_test - get_locations_from_BUILD_match_test + get_gapic_additional_protos_from_BUILD_common_resources_test + get_gapic_additional_protos_from_BUILD_iam_policy_test + get_gapic_additional_protos_from_BUILD_locations_test + get_gapic_additional_protos_from_BUILD_iam_locations_test get_transport_from_BUILD_grpc_rest_test get_transport_from_BUILD_grpc_test get_transport_from_BUILD_rest_test diff --git a/library_generation/test/resources/search_additional_protos/BUILD_common_resources.bazel b/library_generation/test/resources/search_additional_protos/BUILD_common_resources.bazel new file mode 100644 index 0000000000..45e3987adb --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_common_resources.bazel @@ -0,0 +1,6 @@ +# this file is only used in testing `get_gapic_additional_protos_from_BUILD` in utilities.sh + +proto_library_with_info( + deps = [ + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_locations.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_locations.bazel new file mode 100644 index 0000000000..e8241995e2 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_locations.bazel @@ -0,0 +1,8 @@ +# this file is only used in testing `get_gapic_additional_protos_from_BUILD` in utilities.sh + +proto_library_with_info( + deps = [ + "//google/iam/v1:iam_policy_proto", + "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_match.bazel deleted file mode 100644 index f99b0c741c..0000000000 --- a/library_generation/test/resources/search_additional_protos/BUILD_iam_match.bazel +++ /dev/null @@ -1,7 +0,0 @@ -# this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh - -proto_library_with_info( - deps = [ - "//google/iam/v1:iam_policy_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_policy.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_policy.bazel new file mode 100644 index 0000000000..81064a7eb4 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_policy.bazel @@ -0,0 +1,7 @@ +# this file is only used in testing `get_gapic_additional_protos_from_BUILD` in utilities.sh + +proto_library_with_info( + deps = [ + "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel deleted file mode 100644 index 3765170fc9..0000000000 --- a/library_generation/test/resources/search_additional_protos/BUILD_iam_without_match.bazel +++ /dev/null @@ -1,6 +0,0 @@ -# this file is only used in testing `get_iam_policy_from_BUILD` in utilities.sh - -proto_library_with_info( - deps = [ - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_locations.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations.bazel new file mode 100644 index 0000000000..07faa4ac95 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_locations.bazel @@ -0,0 +1,7 @@ +# this file is only used in testing `get_gapic_additional_protos_from_BUILD` in utilities.sh + +proto_library_with_info( + deps = [ + "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_locations_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_match.bazel deleted file mode 100644 index 2f00360b4d..0000000000 --- a/library_generation/test/resources/search_additional_protos/BUILD_locations_match.bazel +++ /dev/null @@ -1,7 +0,0 @@ -# this file is only used in testing `get_location_from_BUILD` in utilities.sh - -proto_library_with_info( - deps = [ - "//google/cloud/location:location_proto", - ] -) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel deleted file mode 100644 index f1d74b1261..0000000000 --- a/library_generation/test/resources/search_additional_protos/BUILD_locations_without_match.bazel +++ /dev/null @@ -1,6 +0,0 @@ -# this file is only used in testing `get_location_from_BUILD` in utilities.sh - -proto_library_with_info( - deps = [ - ] -) \ No newline at end of file diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 73dfaf4d5b..30a5fe133b 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -24,6 +24,32 @@ __get_config_from_BUILD() { echo "${result}" } +__get_iam_policy_from_BUILD() { + local build_file=$1 + local contains_iam_policy + contains_iam_policy=$(__get_config_from_BUILD \ + "${build_file}" \ + "proto_library_with_info(" \ + "//google/iam/v1:iam_policy_proto" \ + "false" \ + "true" + ) + echo "${contains_iam_policy}" +} + +__get_locations_from_BUILD() { + local build_file=$1 + local contains_locations + contains_locations=$(__get_config_from_BUILD \ + "${build_file}" \ + "proto_library_with_info(" \ + "//google/cloud/location:location_proto" \ + "false" \ + "true" + ) + echo "${contains_locations}" +} + # define utility functions extract_folder_name() { local destination_path=$1 @@ -244,30 +270,16 @@ get_version_from_WORKSPACE() { echo "${version}" } -get_iam_policy_from_BUILD() { +get_gapic_additional_protos_from_BUILD() { local build_file=$1 - local contains_iam_policy - contains_iam_policy=$(__get_config_from_BUILD \ - "${build_file}" \ - "proto_library_with_info(" \ - "//google/iam/v1:iam_policy_proto" \ - "false" \ - "true" - ) - echo "${contains_iam_policy}" -} - -get_locations_from_BUILD() { - local build_file=$1 - local contains_locations - contains_locations=$(__get_config_from_BUILD \ - "${build_file}" \ - "proto_library_with_info(" \ - "//google/cloud/location:location_proto" \ - "false" \ - "true" - ) - echo "${contains_locations}" + local gapic_additional_protos="google/cloud/common_resources.proto" + if [[ $(__get_iam_policy_from_BUILD "${build_file}") == "true" ]]; then + gapic_additional_protos="${gapic_additional_protos} google/iam/v1/iam_policy.proto" + fi + if [[ $(__get_locations_from_BUILD "${build_file}") == "true" ]]; then + gapic_additional_protos="${gapic_additional_protos} google/cloud/location/locations.proto" + fi + echo "${gapic_additional_protos}" } get_transport_from_BUILD() { From 34b6a742c83716454c13b9365aa20a038edc5883 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 19 Sep 2023 13:56:18 -0400 Subject: [PATCH 7/9] change showcase --- showcase/scripts/generate_showcase.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh index 951992cc53..1b25e3ae68 100755 --- a/showcase/scripts/generate_showcase.sh +++ b/showcase/scripts/generate_showcase.sh @@ -39,8 +39,7 @@ fi ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-java") -contains_iam_policy="true" -contains_locations="true" +gapic_additional_protos="google/iam/v1/iam_policy.proto google/cloud/location/locations.proto" rest_numeric_enums="false" transport="grpc+rest" include_samples="false" @@ -51,8 +50,7 @@ bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \ --proto_path "schema/google/showcase/v1beta1" \ --destination_path "showcase-output" \ --gapic_generator_version "${ggj_version}" \ - --contains_iam_policy "${contains_iam_policy}" \ - --contains_locations "${contains_locations}" \ + --gapic_additional_protos "${gapic_additional_protos}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --include_samples "${include_samples}" \ --transport "${transport}" From cda8f83f7ec35a0aaf8a1d53e0e8a68faf43d241 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 19 Sep 2023 14:07:36 -0400 Subject: [PATCH 8/9] change default value --- library_generation/README.md | 7 ++----- library_generation/generate_library.sh | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/library_generation/README.md b/library_generation/README.md index 88a00a4744..efb42b2d48 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -93,13 +93,10 @@ Note that if specified, the version should be compatible with gapic-generator-ja ### gapic_additional_protos (optional) Additional protos that pass to the generator. -The default value is empty string, i.e., no additional proto. +The default value is `google/cloud/common_resources.proto`. Use `--gapic_additional_protos` to specify the value. -Note that `google/cloud/common_resources.proto` is always pass to the generator, so -no need to specify this proto. - ### transport (optional) One of GAPIC options passed to the generator. The value is either `grpc` or `grpc+rest`. @@ -132,7 +129,7 @@ library_generation/generate_library.sh \ --gapic_generator_version 2.24.0 \ --protobuf_version 23.2 \ --grpc_version 1.55.1 \ ---gapic_additional_protos google/cloud/location/locations.proto\ +--gapic_additional_protos "google/cloud/common_resources.proto google/cloud/location/locations.proto" \ --transport grpc+rest \ --rest_numeric_enums true \ --include_samples true diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index f270041b9b..9863a6170e 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -70,7 +70,7 @@ if [ -z "${grpc_version}" ]; then fi if [ -z "${gapic_additional_protos}" ]; then - gapic_additional_protos="" + gapic_additional_protos="google/cloud/common_resources.proto" fi if [ -z "${transport}" ]; then From 319d107c0ec71afc829c043e032c8fd348b2575e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 19 Sep 2023 14:11:20 -0400 Subject: [PATCH 9/9] remove unused function --- library_generation/utilities.sh | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 30a5fe133b..2c90cb1554 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -95,24 +95,6 @@ unzip_src_files() { rm -r -f "${destination_path}/${category}-${folder_name}/src/main/java/META-INF" } -# Apart from proto files in proto_path, additional protos are needed in order -# to generate GAPIC client libraries. -# In most cases, these protos should be within google/ directory, which is -# pulled from googleapis as a prerequisite. -# Search additional protos in BUILD.bazel. -search_additional_protos() { - local contains_iam_policy=$1 - local contains_locations=$2 - additional_protos="google/cloud/common_resources.proto" # used by every library - if [[ "${contains_iam_policy}" == "true" ]]; then - additional_protos="$additional_protos google/iam/v1/iam_policy.proto" - fi - if [[ "${contains_locations}" == "true" ]]; then - additional_protos="$additional_protos google/cloud/location/locations.proto" - fi - echo "${additional_protos}" -} - # get gapic options from .yaml and .json files from proto_path. get_gapic_opts() { local gapic_config @@ -270,6 +252,11 @@ get_version_from_WORKSPACE() { echo "${version}" } +# Apart from proto files in proto_path, additional protos are needed in order +# to generate GAPIC client libraries. +# In most cases, these protos should be within google/ directory, which is +# pulled from googleapis as a prerequisite. +# Get additional protos in BUILD.bazel. get_gapic_additional_protos_from_BUILD() { local build_file=$1 local gapic_additional_protos="google/cloud/common_resources.proto"