From faa7e5dff17897b0432bc505b7ed24c33805f418 Mon Sep 17 00:00:00 2001 From: Arpan Mishra Date: Wed, 15 Nov 2023 13:55:06 +0530 Subject: [PATCH] =?UTF-8?q?feat:=20enable=20session=20leaks=20prevention?= =?UTF-8?q?=20by=20cleaning=20up=20long-running=20tra=E2=80=A6=20(#2655)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: enable session leaks prevention by cleaning up long-running transactions. * Update session-and-channel-pool-configuration.md Co-authored-by: Knut Olav Løite * Update session-and-channel-pool-configuration.md Co-authored-by: Knut Olav Løite * Update session-and-channel-pool-configuration.md Co-authored-by: Knut Olav Løite * Update session-and-channel-pool-configuration.md Co-authored-by: Knut Olav Løite * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Knut Olav Løite Co-authored-by: Owl Bot --- .../cloud/spanner/SessionPoolOptions.java | 7 +- .../cloud/spanner/SessionPoolOptionsTest.java | 2 +- session-and-channel-pool-configuration.md | 81 +++++++++++++++++++ 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPoolOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPoolOptions.java index cbea1495368..80d53a4d71f 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPoolOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPoolOptions.java @@ -357,7 +357,8 @@ static InactiveTransactionRemovalOptions.Builder newBuilder() { } static class Builder { - private ActionOnInactiveTransaction actionOnInactiveTransaction; + private ActionOnInactiveTransaction actionOnInactiveTransaction = + ActionOnInactiveTransaction.WARN; private Duration executionFrequency = Duration.ofMinutes(2); private double usedSessionsRatioThreshold = 0.95; private Duration idleTimeThreshold = Duration.ofMinutes(60L); @@ -598,7 +599,7 @@ public Builder setBlockIfPoolExhausted() { * * @return this builder for chaining */ - Builder setWarnIfInactiveTransactions() { + public Builder setWarnIfInactiveTransactions() { this.inactiveTransactionRemovalOptions = InactiveTransactionRemovalOptions.newBuilder() .setActionOnInactiveTransaction(ActionOnInactiveTransaction.WARN) @@ -617,7 +618,7 @@ Builder setWarnIfInactiveTransactions() { * * @return this builder for chaining */ - Builder setWarnAndCloseIfInactiveTransactions() { + public Builder setWarnAndCloseIfInactiveTransactions() { this.inactiveTransactionRemovalOptions = InactiveTransactionRemovalOptions.newBuilder() .setActionOnInactiveTransaction(ActionOnInactiveTransaction.WARN_AND_CLOSE) diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionPoolOptionsTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionPoolOptionsTest.java index 24e754f4050..23aa626f393 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionPoolOptionsTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionPoolOptionsTest.java @@ -129,7 +129,7 @@ public void verifyDefaultInactiveTransactionRemovalOptions() { InactiveTransactionRemovalOptions inactiveTransactionRemovalOptions = sessionPoolOptions.getInactiveTransactionRemovalOptions(); - assertFalse(sessionPoolOptions.warnInactiveTransactions()); + assertTrue(sessionPoolOptions.warnInactiveTransactions()); assertFalse(sessionPoolOptions.warnAndCloseInactiveTransactions()); assertFalse(sessionPoolOptions.closeInactiveTransactions()); assertEquals(0.95, inactiveTransactionRemovalOptions.getUsedSessionsRatioThreshold(), 0.0); diff --git a/session-and-channel-pool-configuration.md b/session-and-channel-pool-configuration.md index 838208aea8c..3a5bde5d3ab 100644 --- a/session-and-channel-pool-configuration.md +++ b/session-and-channel-pool-configuration.md @@ -281,3 +281,84 @@ This will cause the following to happen internally in the client library: 1. The `TransactionRunner` will automatically commit the transaction if the supplied user code finished without any errors. The `Commit` RPC that is invoked uses a thread from the default gRPC thread pool. + +### Session Leak +A `DatabaseClient` object of the Client Library has a limit on the number of maximum sessions. For example the +default value of `MaxSessions` in the Java Client Library is 400. You can configure these values at the time of +creating a `Spanner` instance by setting custom `SessionPoolOptions`. When all the sessions are checked +out of the session pool, every new transaction has to wait until a session is returned to the pool. +If a session is never returned to the pool (hence causing a session leak), the transactions will have to wait +indefinitely and your application will be blocked. + +#### Common Root Causes +The most common reason for session leaks in the Java client library are: +1. Not closing a `ResultSet` that is returned by `executeQuery`. Always put `ResultSet` objects in a try-with-resources block, or take other measures to ensure that the `ResultSet` is always closed. +2. Not closing a `ReadOnlyTransaction` when you no longer need it. Always put `ReadOnlyTransaction` objects in a try-with-resources block, or take other measures to ensure that the `ReadOnlyTransaction` is always closed. +3. Not closing a `TransactionManager` when you no longer need it. Always put `TransactionManager` objects in a try-with-resources block, or take other measures to ensure that the `TransactionManager` is always closed. + +As shown in the example below, the `try-with-resources` block releases the session after it is complete. +If you don't use `try-with-resources` block, unless you explicitly call the `close()` method on all resources +such as `ResultSet`, the session is not released back to the pool. + +```java +DatabaseClient client = + spanner.getDatabaseClient(DatabaseId.of("my-project", "my-instance", "my-database")); +try (ResultSet resultSet = + client.singleUse().executeQuery(Statement.of("select col1, col2 from my_table"))) { + while (resultSet.next()) { + // use the results. + } +} +``` + +#### Debugging and Resolving Session Leaks + +##### Logging +Enabled by default, the logging option shares warn logs when you have exhausted >95% of your session pool. +This could mean two things, either you need to increase the max sessions in your session pool (as the number +of queries run using the client side database object is greater than your session pool can serve) or you may +have a session leak. + +To help debug which transactions may be causing this session leak, the logs will also contain stack traces of +transactions which have been running longer than expected. The logs are pushed to a destination based on +how the log exporter is configured for the host application. + +``` java +final SessionPoolOptions sessionPoolOptions = + SessionPoolOptions.newBuilder().setWarnIfInactiveTransactions().build() + +final Spanner spanner = + SpannerOptions.newBuilder() + .setSessionPoolOption(sessionPoolOptions) + .build() + .getService(); +final DatabaseClient client = spanner.getDatabaseClient(databaseId); + +// Example Log message to warn presence of long running transactions +// Detected long-running session . To automatically remove long-running sessions, set SessionOption ActionOnInactiveTransaction +// to WARN_AND_CLOSE by invoking setWarnAndCloseIfInactiveTransactions() method. + +``` +##### Automatically clean inactive transactions +When the option to automatically clean inactive transactions is enabled, the client library will automatically spot +problematic transactions that are running for extremely long periods of time (thus causing session leaks) and close them. +The session will be removed from the pool and be replaced by a new session. To dig deeper into which transactions are being +closed, you can check the logs to see the stack trace of the transactions which might be causing these leaks and further +debug them. + +``` java +final SessionPoolOptions sessionPoolOptions = + SessionPoolOptions.newBuilder().setWarnAndCloseIfInactiveTransactions().build() + +final Spanner spanner = + SpannerOptions.newBuilder() + .setSessionPoolOption(sessionPoolOptions) + .build() + .getService(); +final DatabaseClient client = spanner.getDatabaseClient(databaseId); + +// Example Log message for when transaction is recycled +// Removing long-running session +``` + +