From f921d555cca982fac23d19d9471728546da940c6 Mon Sep 17 00:00:00 2001 From: Thiago Nunes Date: Tue, 16 Mar 2021 14:03:12 +1100 Subject: [PATCH 1/7] samples: adds samples for CMEK Adds samples to create an encrypted database, to create an encrypted backup and to restore to an encrypted database. --- samples/install-without-bom/pom.xml | 3 + samples/snapshot/pom.xml | 3 + samples/snippets/pom.xml | 3 + .../CreateBackupWithEncryptionKey.java | 119 +++++++++++++++ .../CreateDatabaseWithEncryptionKey.java | 101 +++++++++++++ .../RestoreBackupWithEncryptionKey.java | 102 +++++++++++++ .../example/spanner/DatabaseIdGenerator.java | 37 +++++ .../com/example/spanner/EncryptionKeyIT.java | 135 ++++++++++++++++++ .../com/example/spanner/SampleRunner.java | 37 +++++ 9 files changed, 540 insertions(+) create mode 100644 samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java create mode 100644 samples/snippets/src/main/java/com/example/spanner/CreateDatabaseWithEncryptionKey.java create mode 100644 samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java create mode 100644 samples/snippets/src/test/java/com/example/spanner/DatabaseIdGenerator.java create mode 100644 samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java create mode 100644 samples/snippets/src/test/java/com/example/spanner/SampleRunner.java diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 0cbc50663cc..7b0b9b475b1 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -143,6 +143,9 @@ spanner-testing + us-central1 + spanner-test-keyring + spanner-test-key mysample quick-db diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 0dec641ccf2..631718d85fd 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -142,6 +142,9 @@ spanner-testing + us-central1 + spanner-test-keyring + spanner-test-key mysample quick-db diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index f16cf9abc73..8ae0250e70f 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -147,6 +147,9 @@ spanner-testing + us-central1 + spanner-test-keyring + spanner-test-key mysample quick-db diff --git a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java new file mode 100644 index 00000000000..828aa38e4da --- /dev/null +++ b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java @@ -0,0 +1,119 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.spanner; + +// [START spanner_create_backup_with_encryption_key] + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Backup; +import com.google.cloud.spanner.BackupId; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.SpannerExceptionFactory; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.encryption.EncryptionConfigs; +import com.google.cloud.spanner.encryption.CustomerManagedEncryption; +import com.google.spanner.admin.database.v1.CreateBackupMetadata; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.threeten.bp.LocalDateTime; +import org.threeten.bp.OffsetDateTime; + +public class CreateBackupWithEncryptionKey { + + static void createBackupWithEncryptionKey() throws InterruptedException { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project"; + String instanceId = "my-instance"; + String databaseId = "my-database"; + String backupId = "my-backup"; + String kmsKeyName = "my-key"; + + try (Spanner spanner = + SpannerOptions.newBuilder().setProjectId(projectId).build().getService()) { + DatabaseAdminClient adminClient = spanner.getDatabaseAdminClient(); + createBackupWithEncryptionKey( + adminClient, + projectId, + instanceId, + databaseId, + backupId, + kmsKeyName); + } + } + + static Void createBackupWithEncryptionKey(DatabaseAdminClient adminClient, + String projectId, String instanceId, String databaseId, String backupId, String kmsKeyName) + throws InterruptedException { + // Set expire time to 14 days from now. + final Timestamp expireTime = Timestamp.ofTimeMicroseconds(TimeUnit.MICROSECONDS.convert( + System.currentTimeMillis() + TimeUnit.DAYS.toMillis(14), TimeUnit.MILLISECONDS)); + final Backup backupToCreate = adminClient + .newBackupBuilder(BackupId.of(projectId, instanceId, backupId)) + .setDatabase(DatabaseId.of(projectId, instanceId, databaseId)) + .setExpireTime(expireTime) + .setEncryptionConfig(EncryptionConfigs.customerManagedEncryption(kmsKeyName)) + .build(); + final OperationFuture operation = adminClient + .createBackup(backupToCreate); + + Backup backup; + try { + System.out.println("Waiting for operation to complete..."); + backup = operation.get(1200, TimeUnit.SECONDS); + } catch (ExecutionException e) { + // If the operation failed during execution, expose the cause. + throw (SpannerException) e.getCause(); + } catch (InterruptedException e) { + // Throw when a thread is waiting, sleeping, or otherwise occupied, + // and the thread is interrupted, either before or during the activity. + throw SpannerExceptionFactory.propagateInterrupt(e); + } catch (TimeoutException e) { + // If the operation timed out propagates the timeout + throw SpannerExceptionFactory.propagateTimeout(e); + } + + // Waits for the backup to be ready + int maxWaitRetries = 10; + int currentRetry = 0; + while (currentRetry < maxWaitRetries && !backup.isReady()) { + System.out.println("Waiting for backup to be ready"); + Thread.sleep(10_000); + currentRetry++; + } + // Reload the metadata of the backup from the server. + backup = backup.reload(); + + System.out.printf( + "Backup %s of size %d bytes was created at %s using encryption key %s%n", + backup.getId().getName(), + backup.getSize(), + LocalDateTime.ofEpochSecond( + backup.getProto().getCreateTime().getSeconds(), + backup.getProto().getCreateTime().getNanos(), + OffsetDateTime.now().getOffset()), + ((CustomerManagedEncryption) backup.getEncryptionConfig()).getKmsKeyName() + ); + + return null; + } +} +// [END spanner_create_backup_with_encryption_key] diff --git a/samples/snippets/src/main/java/com/example/spanner/CreateDatabaseWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/CreateDatabaseWithEncryptionKey.java new file mode 100644 index 00000000000..8d535030b9a --- /dev/null +++ b/samples/snippets/src/main/java/com/example/spanner/CreateDatabaseWithEncryptionKey.java @@ -0,0 +1,101 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.spanner; + +// [START spanner_create_database_with_encryption_key] + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.SpannerExceptionFactory; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.encryption.EncryptionConfigs; +import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; +import java.util.Arrays; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class CreateDatabaseWithEncryptionKey { + + static void createDatabaseWithEncryptionKey() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project"; + String instanceId = "my-instance"; + String databaseId = "my-database"; + String kmsKeyName = "my-key"; + + try (Spanner spanner = + SpannerOptions.newBuilder().setProjectId(projectId).build().getService()) { + DatabaseAdminClient adminClient = spanner.getDatabaseAdminClient(); + createDatabaseWithEncryptionKey( + adminClient, + projectId, + instanceId, + databaseId, + kmsKeyName); + } + } + + static Void createDatabaseWithEncryptionKey(DatabaseAdminClient adminClient, + String projectId, String instanceId, String databaseId, String kmsKeyName) { + final Database databaseToCreate = adminClient + .newDatabaseBuilder(DatabaseId.of(projectId, instanceId, databaseId)) + .setEncryptionConfig(EncryptionConfigs.customerManagedEncryption(kmsKeyName)) + .build(); + final OperationFuture operation = adminClient + .createDatabase(databaseToCreate, Arrays.asList( + "CREATE TABLE Singers (" + + " SingerId INT64 NOT NULL," + + " FirstName STRING(1024)," + + " LastName STRING(1024)," + + " SingerInfo BYTES(MAX)" + + ") PRIMARY KEY (SingerId)", + "CREATE TABLE Albums (" + + " SingerId INT64 NOT NULL," + + " AlbumId INT64 NOT NULL," + + " AlbumTitle STRING(MAX)" + + ") PRIMARY KEY (SingerId, AlbumId)," + + " INTERLEAVE IN PARENT Singers ON DELETE CASCADE" + )); + try { + System.out.println("Waiting for operation to complete..."); + Database createdDatabase = operation.get(120, TimeUnit.SECONDS); + + System.out.printf( + "Database %s created with encryption key %s%n", + createdDatabase.getId(), + createdDatabase.getEncryptionConfig().getKmsKeyName() + ); + } catch (ExecutionException e) { + // If the operation failed during execution, expose the cause. + throw (SpannerException) e.getCause(); + } catch (InterruptedException e) { + // Throw when a thread is waiting, sleeping, or otherwise occupied, + // and the thread is interrupted, either before or during the activity. + throw SpannerExceptionFactory.propagateInterrupt(e); + } catch (TimeoutException e) { + // If the operation timed out propagates the timeout + throw SpannerExceptionFactory.propagateTimeout(e); + } + return null; + } +} +// [END spanner_create_database_with_encryption_key] diff --git a/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java new file mode 100644 index 00000000000..2ce2aa87a61 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java @@ -0,0 +1,102 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.spanner; + +// [START spanner_restore_backup_with_encryption_key] + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.cloud.spanner.BackupId; +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.Restore; +import com.google.cloud.spanner.RestoreInfo; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.SpannerExceptionFactory; +import com.google.cloud.spanner.SpannerOptions; +import com.google.cloud.spanner.encryption.EncryptionConfigs; +import com.google.spanner.admin.database.v1.RestoreDatabaseMetadata; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class RestoreBackupWithEncryptionKey { + + static void restoreBackupWithEncryptionKey() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "my-project"; + String instanceId = "my-instance"; + String databaseId = "my-database"; + String backupId = "my-backup"; + String kmsKeyName = "my-key"; + + try (Spanner spanner = + SpannerOptions.newBuilder().setProjectId(projectId).build().getService()) { + DatabaseAdminClient adminClient = spanner.getDatabaseAdminClient(); + restoreBackupWithEncryptionKey( + adminClient, + projectId, + instanceId, + backupId, + databaseId, + kmsKeyName); + } + } + + static Void restoreBackupWithEncryptionKey(DatabaseAdminClient adminClient, + String projectId, String instanceId, String backupId, String restoreId, String kmsKeyName) { + final Restore restore = adminClient + .newRestoreBuilder( + BackupId.of(projectId, instanceId, backupId), + DatabaseId.of(projectId, instanceId, restoreId)) + .setEncryptionConfig(EncryptionConfigs.customerManagedEncryption(kmsKeyName)) + .build(); + final OperationFuture operation = adminClient + .restoreDatabase(restore); + + Database database; + try { + System.out.println("Waiting for operation to complete..."); + database = operation.get(1600, TimeUnit.SECONDS); + } catch (ExecutionException e) { + // If the operation failed during execution, expose the cause. + throw (SpannerException) e.getCause(); + } catch (InterruptedException e) { + // Throw when a thread is waiting, sleeping, or otherwise occupied, + // and the thread is interrupted, either before or during the activity. + throw SpannerExceptionFactory.propagateInterrupt(e); + } catch (TimeoutException e) { + // If the operation timed out propagates the timeout + throw SpannerExceptionFactory.propagateTimeout(e); + } + + // Reload the metadata of the database from the server. + database = database.reload(); + final RestoreInfo restoreInfo = database.getRestoreInfo(); + + System.out.printf( + "Database %s restored to %s from backup %s using encryption key %s%n", + restoreInfo.getSourceDatabase(), + database.getId(), + restoreInfo.getBackup(), + database.getEncryptionConfig().getKmsKeyName() + ); + return null; + } +} +// [END spanner_restore_backup_with_encryption_key] diff --git a/samples/snippets/src/test/java/com/example/spanner/DatabaseIdGenerator.java b/samples/snippets/src/test/java/com/example/spanner/DatabaseIdGenerator.java new file mode 100644 index 00000000000..78fb2d201b5 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/spanner/DatabaseIdGenerator.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.spanner; + +import java.util.UUID; + +public class DatabaseIdGenerator { + + private static final int DATABASE_NAME_MAX_SIZE = 30; + private static final String BASE_DATABASE_ID = System.getProperty( + "spanner.sample.database", + "sampletest" + ); + + static String generateDatabaseId() { + final String databaseId = ( + BASE_DATABASE_ID + + "-" + + UUID.randomUUID().toString().replaceAll("-", "") + ).substring(0, DATABASE_NAME_MAX_SIZE); + return databaseId; + } +} diff --git a/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java b/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java new file mode 100644 index 00000000000..a9df29a29e1 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java @@ -0,0 +1,135 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.spanner; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.Spanner; +import com.google.cloud.spanner.SpannerOptions; +import java.util.ArrayList; +import java.util.List; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Integration tests for: - {@link CreateDatabaseWithEncryptionKey} - {@link + * CreateBackupWithEncryptionKey} - {@link RestoreBackupWithEncryptionKey} + */ +@RunWith(JUnit4.class) +public class EncryptionKeyIT { + + private static String projectId; + private static final String instanceId = System.getProperty("spanner.test.instance"); + private static DatabaseAdminClient databaseAdminClient; + private static List databasesToDrop; + private static List backupsToDrop; + private static Spanner spanner; + private static String key; + + @BeforeClass + public static void setUp() { + final SpannerOptions options = SpannerOptions + .newBuilder() + .setAutoThrottleAdministrativeRequests() + .build(); + projectId = options.getProjectId(); + spanner = options.getService(); + databaseAdminClient = spanner.getDatabaseAdminClient(); + databasesToDrop = new ArrayList<>(); + backupsToDrop = new ArrayList<>(); + + String keyLocation = System.getProperty("spanner.test.key.location"); + String keyRing = System.getProperty("spanner.test.key.ring"); + String keyName = System.getProperty("spanner.test.key.name"); + key = "projects/" + projectId + "/locations/" + keyLocation + "/keyRings/" + keyRing + + "/cryptoKeys/" + keyName; + } + + @AfterClass + public static void tearDown() { + for (String databaseId : databasesToDrop) { + try { + databaseAdminClient.deleteBackup(instanceId, databaseId); + } catch (Exception e) { + System.out.println("Failed to drop database " + databaseId + ", skipping..."); + } + } + for (String backupId : backupsToDrop) { + try { + databaseAdminClient.deleteBackup(instanceId, backupId); + } catch (Exception e) { + System.out.println("Failed to drop backup " + backupId + ", skipping..."); + } + } + spanner.close(); + } + + @Test + public void testEncryptedDatabaseAndBackupAndRestore() throws Exception { + final String databaseId = DatabaseIdGenerator.generateDatabaseId(); + final String backupId = DatabaseIdGenerator.generateDatabaseId(); + final String restoreId = DatabaseIdGenerator.generateDatabaseId(); + + databasesToDrop.add(databaseId); + backupsToDrop.add(backupId); + databasesToDrop.add(restoreId); + + String out = SampleRunner.runSample(() -> + CreateDatabaseWithEncryptionKey.createDatabaseWithEncryptionKey( + databaseAdminClient, + projectId, + instanceId, + databaseId, + key + )); + assertThat(out).contains( + "Database projects/" + projectId + "/instances/" + instanceId + "/databases/" + databaseId + + " created with encryption key " + key); + + out = SampleRunner.runSample(() -> + CreateBackupWithEncryptionKey.createBackupWithEncryptionKey( + databaseAdminClient, + projectId, + instanceId, + databaseId, + backupId, + key + )); + assertThat(out).containsMatch( + "Backup projects/" + projectId + "/instances/" + instanceId + "/backups/" + backupId + + " of size \\d+ bytes was created at (.*) using encryption key " + key); + + out = SampleRunner.runSample(() -> + RestoreBackupWithEncryptionKey.restoreBackupWithEncryptionKey( + databaseAdminClient, + projectId, + instanceId, + backupId, + restoreId, + key + )); + assertThat(out).contains( + "Database projects/" + projectId + "/instances/" + instanceId + "/databases/" + databaseId + + " restored to projects/" + projectId + "/instances/" + instanceId + "/databases/" + + restoreId + " from backup projects/" + projectId + "/instances/" + instanceId + + "/backups/" + backupId + " using encryption key " + key); + } +} diff --git a/samples/snippets/src/test/java/com/example/spanner/SampleRunner.java b/samples/snippets/src/test/java/com/example/spanner/SampleRunner.java new file mode 100644 index 00000000000..63c3f2d90f8 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/spanner/SampleRunner.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.spanner; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.concurrent.Callable; + +public class SampleRunner { + public static String runSample(Callable sample) throws Exception { + final PrintStream stdOut = System.out; + final ByteArrayOutputStream bout = new ByteArrayOutputStream(); + final PrintStream out = new PrintStream(bout); + System.setOut(out); + sample.call(); + System.setOut(stdOut); + return bout.toString(); + } + + public interface Sample { + void run(); + } +} From 7f3c2fb9c95d9284c27c9baa823af62bf12cc9c1 Mon Sep 17 00:00:00 2001 From: Thiago Nunes Date: Thu, 18 Mar 2021 12:47:50 +1100 Subject: [PATCH 2/7] samples: fix checkstyle violations --- .../java/com/example/spanner/CreateBackupWithEncryptionKey.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java index 828aa38e4da..02b16008e53 100644 --- a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java +++ b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java @@ -28,8 +28,8 @@ import com.google.cloud.spanner.SpannerException; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerOptions; -import com.google.cloud.spanner.encryption.EncryptionConfigs; import com.google.cloud.spanner.encryption.CustomerManagedEncryption; +import com.google.cloud.spanner.encryption.EncryptionConfigs; import com.google.spanner.admin.database.v1.CreateBackupMetadata; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; From 7034c3fa28375d4a190d6c402e2c59eab355732a Mon Sep 17 00:00:00 2001 From: Thiago Nunes Date: Fri, 19 Mar 2021 11:51:18 +1100 Subject: [PATCH 3/7] samples: addresses PR comments. --- .../spanner/CreateBackupWithEncryptionKey.java | 14 +++----------- .../spanner/CreateDatabaseWithEncryptionKey.java | 6 +++--- .../spanner/RestoreBackupWithEncryptionKey.java | 6 +++--- .../com/example/spanner/DatabaseIdGenerator.java | 3 +-- .../java/com/example/spanner/EncryptionKeyIT.java | 12 +++++------- .../java/com/example/spanner/SampleRunner.java | 7 +++---- 6 files changed, 18 insertions(+), 30 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java index 02b16008e53..39a433424a8 100644 --- a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java +++ b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java @@ -25,7 +25,6 @@ import com.google.cloud.spanner.DatabaseAdminClient; import com.google.cloud.spanner.DatabaseId; import com.google.cloud.spanner.Spanner; -import com.google.cloud.spanner.SpannerException; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.encryption.CustomerManagedEncryption; @@ -45,7 +44,8 @@ static void createBackupWithEncryptionKey() throws InterruptedException { String instanceId = "my-instance"; String databaseId = "my-database"; String backupId = "my-backup"; - String kmsKeyName = "my-key"; + String kmsKeyName = + "projects/" + projectId + "/locations//keyRings//cryptoKeys/"; try (Spanner spanner = SpannerOptions.newBuilder().setProjectId(projectId).build().getService()) { @@ -81,7 +81,7 @@ static Void createBackupWithEncryptionKey(DatabaseAdminClient adminClient, backup = operation.get(1200, TimeUnit.SECONDS); } catch (ExecutionException e) { // If the operation failed during execution, expose the cause. - throw (SpannerException) e.getCause(); + throw SpannerExceptionFactory.asSpannerException(e.getCause()); } catch (InterruptedException e) { // Throw when a thread is waiting, sleeping, or otherwise occupied, // and the thread is interrupted, either before or during the activity. @@ -91,14 +91,6 @@ static Void createBackupWithEncryptionKey(DatabaseAdminClient adminClient, throw SpannerExceptionFactory.propagateTimeout(e); } - // Waits for the backup to be ready - int maxWaitRetries = 10; - int currentRetry = 0; - while (currentRetry < maxWaitRetries && !backup.isReady()) { - System.out.println("Waiting for backup to be ready"); - Thread.sleep(10_000); - currentRetry++; - } // Reload the metadata of the backup from the server. backup = backup.reload(); diff --git a/samples/snippets/src/main/java/com/example/spanner/CreateDatabaseWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/CreateDatabaseWithEncryptionKey.java index 8d535030b9a..ea559006c39 100644 --- a/samples/snippets/src/main/java/com/example/spanner/CreateDatabaseWithEncryptionKey.java +++ b/samples/snippets/src/main/java/com/example/spanner/CreateDatabaseWithEncryptionKey.java @@ -23,7 +23,6 @@ import com.google.cloud.spanner.DatabaseAdminClient; import com.google.cloud.spanner.DatabaseId; import com.google.cloud.spanner.Spanner; -import com.google.cloud.spanner.SpannerException; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.encryption.EncryptionConfigs; @@ -40,7 +39,8 @@ static void createDatabaseWithEncryptionKey() { String projectId = "my-project"; String instanceId = "my-instance"; String databaseId = "my-database"; - String kmsKeyName = "my-key"; + String kmsKeyName = + "projects/" + projectId + "/locations//keyRings//cryptoKeys/"; try (Spanner spanner = SpannerOptions.newBuilder().setProjectId(projectId).build().getService()) { @@ -86,7 +86,7 @@ static Void createDatabaseWithEncryptionKey(DatabaseAdminClient adminClient, ); } catch (ExecutionException e) { // If the operation failed during execution, expose the cause. - throw (SpannerException) e.getCause(); + throw SpannerExceptionFactory.asSpannerException(e.getCause()); } catch (InterruptedException e) { // Throw when a thread is waiting, sleeping, or otherwise occupied, // and the thread is interrupted, either before or during the activity. diff --git a/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java index 2ce2aa87a61..09cbb4239db 100644 --- a/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java +++ b/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java @@ -26,7 +26,6 @@ import com.google.cloud.spanner.Restore; import com.google.cloud.spanner.RestoreInfo; import com.google.cloud.spanner.Spanner; -import com.google.cloud.spanner.SpannerException; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.encryption.EncryptionConfigs; @@ -43,7 +42,8 @@ static void restoreBackupWithEncryptionKey() { String instanceId = "my-instance"; String databaseId = "my-database"; String backupId = "my-backup"; - String kmsKeyName = "my-key"; + String kmsKeyName = + "projects/" + projectId + "/locations//keyRings//cryptoKeys/"; try (Spanner spanner = SpannerOptions.newBuilder().setProjectId(projectId).build().getService()) { @@ -75,7 +75,7 @@ static Void restoreBackupWithEncryptionKey(DatabaseAdminClient adminClient, database = operation.get(1600, TimeUnit.SECONDS); } catch (ExecutionException e) { // If the operation failed during execution, expose the cause. - throw (SpannerException) e.getCause(); + throw SpannerExceptionFactory.asSpannerException(e.getCause()); } catch (InterruptedException e) { // Throw when a thread is waiting, sleeping, or otherwise occupied, // and the thread is interrupted, either before or during the activity. diff --git a/samples/snippets/src/test/java/com/example/spanner/DatabaseIdGenerator.java b/samples/snippets/src/test/java/com/example/spanner/DatabaseIdGenerator.java index 78fb2d201b5..800db4d422a 100644 --- a/samples/snippets/src/test/java/com/example/spanner/DatabaseIdGenerator.java +++ b/samples/snippets/src/test/java/com/example/spanner/DatabaseIdGenerator.java @@ -27,11 +27,10 @@ public class DatabaseIdGenerator { ); static String generateDatabaseId() { - final String databaseId = ( + return ( BASE_DATABASE_ID + "-" + UUID.randomUUID().toString().replaceAll("-", "") ).substring(0, DATABASE_NAME_MAX_SIZE); - return databaseId; } } diff --git a/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java b/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java index a9df29a29e1..c2ab9de4a22 100644 --- a/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java +++ b/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java @@ -30,17 +30,17 @@ import org.junit.runners.JUnit4; /** - * Integration tests for: - {@link CreateDatabaseWithEncryptionKey} - {@link - * CreateBackupWithEncryptionKey} - {@link RestoreBackupWithEncryptionKey} + * Integration tests for: {@link CreateDatabaseWithEncryptionKey}, {@link + * CreateBackupWithEncryptionKey} and {@link RestoreBackupWithEncryptionKey} */ @RunWith(JUnit4.class) public class EncryptionKeyIT { private static String projectId; private static final String instanceId = System.getProperty("spanner.test.instance"); + private static final List databasesToDrop = new ArrayList<>(); + private static final List backupsToDrop = new ArrayList<>(); private static DatabaseAdminClient databaseAdminClient; - private static List databasesToDrop; - private static List backupsToDrop; private static Spanner spanner; private static String key; @@ -53,8 +53,6 @@ public static void setUp() { projectId = options.getProjectId(); spanner = options.getService(); databaseAdminClient = spanner.getDatabaseAdminClient(); - databasesToDrop = new ArrayList<>(); - backupsToDrop = new ArrayList<>(); String keyLocation = System.getProperty("spanner.test.key.location"); String keyRing = System.getProperty("spanner.test.key.ring"); @@ -67,7 +65,7 @@ public static void setUp() { public static void tearDown() { for (String databaseId : databasesToDrop) { try { - databaseAdminClient.deleteBackup(instanceId, databaseId); + databaseAdminClient.dropDatabase(instanceId, databaseId); } catch (Exception e) { System.out.println("Failed to drop database " + databaseId + ", skipping..."); } diff --git a/samples/snippets/src/test/java/com/example/spanner/SampleRunner.java b/samples/snippets/src/test/java/com/example/spanner/SampleRunner.java index 63c3f2d90f8..13adf0f66e0 100644 --- a/samples/snippets/src/test/java/com/example/spanner/SampleRunner.java +++ b/samples/snippets/src/test/java/com/example/spanner/SampleRunner.java @@ -20,6 +20,9 @@ import java.io.PrintStream; import java.util.concurrent.Callable; +/** + * Runs a sample and captures the output as a String. + */ public class SampleRunner { public static String runSample(Callable sample) throws Exception { final PrintStream stdOut = System.out; @@ -30,8 +33,4 @@ public static String runSample(Callable sample) throws Exception { System.setOut(stdOut); return bout.toString(); } - - public interface Sample { - void run(); - } } From fab597e08dfc61ca4c1d94ebb7b47a77066eb8dd Mon Sep 17 00:00:00 2001 From: Thiago Nunes Date: Fri, 19 Mar 2021 13:31:32 +1100 Subject: [PATCH 4/7] samples: fixes encryption key tests --- .../example/spanner/CreateBackupWithEncryptionKey.java | 2 +- .../test/java/com/example/spanner/EncryptionKeyIT.java | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java index 39a433424a8..bf097f74b1f 100644 --- a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java +++ b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java @@ -102,7 +102,7 @@ static Void createBackupWithEncryptionKey(DatabaseAdminClient adminClient, backup.getProto().getCreateTime().getSeconds(), backup.getProto().getCreateTime().getNanos(), OffsetDateTime.now().getOffset()), - ((CustomerManagedEncryption) backup.getEncryptionConfig()).getKmsKeyName() + backup.getEncryptionInfo().getKmsKeyVersion() ); return null; diff --git a/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java b/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java index c2ab9de4a22..13d84adf2b9 100644 --- a/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java +++ b/samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java @@ -21,6 +21,7 @@ import com.google.cloud.spanner.DatabaseAdminClient; import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerOptions; +import com.google.common.base.Preconditions; import java.util.ArrayList; import java.util.List; import org.junit.AfterClass; @@ -54,9 +55,10 @@ public static void setUp() { spanner = options.getService(); databaseAdminClient = spanner.getDatabaseAdminClient(); - String keyLocation = System.getProperty("spanner.test.key.location"); - String keyRing = System.getProperty("spanner.test.key.ring"); - String keyName = System.getProperty("spanner.test.key.name"); + String keyLocation = Preconditions + .checkNotNull(System.getProperty("spanner.test.key.location")); + String keyRing = Preconditions.checkNotNull(System.getProperty("spanner.test.key.ring")); + String keyName = Preconditions.checkNotNull(System.getProperty("spanner.test.key.name")); key = "projects/" + projectId + "/locations/" + keyLocation + "/keyRings/" + keyRing + "/cryptoKeys/" + keyName; } From 6e990767b911236d7ed087c7c05dc16304cd1468 Mon Sep 17 00:00:00 2001 From: Thiago Nunes Date: Wed, 24 Mar 2021 14:50:28 +1100 Subject: [PATCH 5/7] samples: prints user provided key in backup sample Prints out the user provided key in the encrypted backup sample, instead of printing out the Backup.encryption_info.kms_key_version. This should align with the key that we are printing on the other samples (instead of printing a key version). --- .../java/com/example/spanner/CreateBackupWithEncryptionKey.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java index bf097f74b1f..fb714a6fd67 100644 --- a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java +++ b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java @@ -102,7 +102,7 @@ static Void createBackupWithEncryptionKey(DatabaseAdminClient adminClient, backup.getProto().getCreateTime().getSeconds(), backup.getProto().getCreateTime().getNanos(), OffsetDateTime.now().getOffset()), - backup.getEncryptionInfo().getKmsKeyVersion() + kmsKeyName ); return null; From bd584d569fe5b7f8c8fa8d4dab06928e0288d5f5 Mon Sep 17 00:00:00 2001 From: Thiago Nunes Date: Wed, 24 Mar 2021 15:07:53 +1100 Subject: [PATCH 6/7] tests: verifies the key returned in create backup Verifies that the key used in the create backup is returned in the response correctly. --- .../google/cloud/spanner/it/ITBackupTest.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java index 6a2742c2d94..8de04fa3f0c 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java @@ -245,7 +245,7 @@ public void testBackups() throws InterruptedException, ExecutionException { .build())); // Verifies that the database encryption has been properly set - testDatabaseEncryption(db1); + testDatabaseEncryption(db1, keyName); // Create two backups in parallel. String backupId1 = testHelper.getUniqueBackupId() + "_bck1"; @@ -314,7 +314,7 @@ public void testBackups() throws InterruptedException, ExecutionException { // Verifies that backup version time is the specified one testBackupVersionTime(backup1, versionTime); // Verifies that backup encryption has been properly set - testBackupEncryption(backup1); + testBackupEncryption(backup1, keyName); // Insert some more data into db2 to get a timestamp from the server. Timestamp commitTs = @@ -374,7 +374,7 @@ public void testBackups() throws InterruptedException, ExecutionException { testGetBackup(db2, backupId2, expireTime); testUpdateBackup(backup1); testCreateInvalidExpirationDate(db1); - testRestore(backup1, op1, versionTime); + testRestore(backup1, versionTime, keyName); testDelete(backupId2); testCancelBackupOperation(db1); @@ -447,17 +447,17 @@ private void testBackupVersionTime(Backup backup, Timestamp versionTime) { logger.info("Done verifying backup version time for " + backup.getId()); } - private void testDatabaseEncryption(Database database) { + private void testDatabaseEncryption(Database database, String expectedKey) { logger.info("Verifying database encryption for " + database.getId()); assertThat(database.getEncryptionConfig()).isNotNull(); - assertThat(database.getEncryptionConfig().getKmsKeyName()).isEqualTo(keyName); + assertThat(database.getEncryptionConfig().getKmsKeyName()).isEqualTo(expectedKey); logger.info("Done verifying database encryption for " + database.getId()); } - private void testBackupEncryption(Backup backup) { + private void testBackupEncryption(Backup backup, String expectedKey) { logger.info("Verifying backup encryption for " + backup.getId()); assertThat(backup.getEncryptionInfo()).isNotNull(); - assertThat(backup.getEncryptionInfo().getKmsKeyVersion()).isNotNull(); + assertThat(backup.getEncryptionInfo().getKmsKeyVersion()).contains(expectedKey); logger.info("Done verifying backup encryption for " + backup.getId()); } @@ -620,8 +620,7 @@ private void testDelete(String backupId) throws InterruptedException { logger.info("Finished delete tests"); } - private void testRestore( - Backup backup, OperationFuture backupOp, Timestamp versionTime) + private void testRestore(Backup backup, Timestamp versionTime, String expectedKey) throws InterruptedException, ExecutionException { // Restore the backup to a new database. String restoredDb = testHelper.getUniqueDatabaseId(); @@ -636,7 +635,7 @@ private void testRestore( final Restore restore = dbAdminClient .newRestoreBuilder(backup.getId(), DatabaseId.of(projectId, instanceId, restoredDb)) - .setEncryptionConfig(EncryptionConfigs.customerManagedEncryption(keyName)) + .setEncryptionConfig(EncryptionConfigs.customerManagedEncryption(expectedKey)) .build(); restoreOperation = dbAdminClient.restoreDatabase(restore); restoreOperationName = restoreOperation.getName(); @@ -687,7 +686,7 @@ private void testRestore( Timestamp.fromProto( reloadedDatabase.getProto().getRestoreInfo().getBackupInfo().getVersionTime())) .isEqualTo(versionTime); - testDatabaseEncryption(reloadedDatabase); + testDatabaseEncryption(reloadedDatabase, expectedKey); // Restoring the backup to an existing database should fail. try { From 4a7b0d5bacc98e152fab907d00129e7193317aa7 Mon Sep 17 00:00:00 2001 From: Thiago Nunes Date: Thu, 25 Mar 2021 10:41:04 +1100 Subject: [PATCH 7/7] samples: addresses PR comments --- .../example/spanner/CreateBackupWithEncryptionKey.java | 4 ---- .../example/spanner/RestoreBackupWithEncryptionKey.java | 9 ++------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java index fb714a6fd67..b2a00ae2add 100644 --- a/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java +++ b/samples/snippets/src/main/java/com/example/spanner/CreateBackupWithEncryptionKey.java @@ -27,7 +27,6 @@ import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerOptions; -import com.google.cloud.spanner.encryption.CustomerManagedEncryption; import com.google.cloud.spanner.encryption.EncryptionConfigs; import com.google.spanner.admin.database.v1.CreateBackupMetadata; import java.util.concurrent.ExecutionException; @@ -91,9 +90,6 @@ static Void createBackupWithEncryptionKey(DatabaseAdminClient adminClient, throw SpannerExceptionFactory.propagateTimeout(e); } - // Reload the metadata of the backup from the server. - backup = backup.reload(); - System.out.printf( "Backup %s of size %d bytes was created at %s using encryption key %s%n", backup.getId().getName(), diff --git a/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java b/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java index 09cbb4239db..4635031c0a2 100644 --- a/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java +++ b/samples/snippets/src/main/java/com/example/spanner/RestoreBackupWithEncryptionKey.java @@ -24,7 +24,6 @@ import com.google.cloud.spanner.DatabaseAdminClient; import com.google.cloud.spanner.DatabaseId; import com.google.cloud.spanner.Restore; -import com.google.cloud.spanner.RestoreInfo; import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerOptions; @@ -85,15 +84,11 @@ static Void restoreBackupWithEncryptionKey(DatabaseAdminClient adminClient, throw SpannerExceptionFactory.propagateTimeout(e); } - // Reload the metadata of the database from the server. - database = database.reload(); - final RestoreInfo restoreInfo = database.getRestoreInfo(); - System.out.printf( "Database %s restored to %s from backup %s using encryption key %s%n", - restoreInfo.getSourceDatabase(), + database.getRestoreInfo().getSourceDatabase(), database.getId(), - restoreInfo.getBackup(), + database.getRestoreInfo().getBackup(), database.getEncryptionConfig().getKmsKeyName() ); return null;