Implement Elasticsearch query timeout.

There's already Faraday or rack-timeout timeouts that will kick in and
kill the web request, but slow-running Elasticsearch queries from the
admin could run indefinitely inside Elasticsearch. This could compound
performance problems, since the admin interface would timeout, but
slow-running Elasticsearch queries could continue piling up in the
background.

We should make this timeout more configurable and try to better align
the Faraday timeout and rack-timeout (which now I'm not sure is even
working), but this is a quick fix to at least prevent background queries
from piling up and impacting system performance.

src/api-umbrella/web-app/app/controllers/api/v0/analytics_controller.rb

@@ -103,7 +103,7 @@ def generate_summary
       # This query can take a long time to run against PrestoDB, so set a long
       # timeout. But since we're only delivering cached results and refreshing
       # periodically in the background, this long timeout should be okay.
-      :query_timeout => "20m",
+      :query_timeout => 20 * 60, # 20 minutes
     })
 
     # Try to ignore some of the baseline monitoring traffic. Only include

src/api-umbrella/web-app/app/models/log_search/base.rb

@@ -31,6 +31,8 @@ def initialize(options = {})
       @end_time = Time.zone.now
     end
 
+    @options[:query_timeout] ||= 90
+
     @interval = options[:interval]
     @region = options[:region]
     @query = {}

src/api-umbrella/web-app/app/models/log_search/elastic_search.rb

@@ -37,6 +37,10 @@ def initialize(options = {})
       :ignore_unavailable => "missing",
       :allow_no_indices => true,
     }
+
+    if(@options[:query_timeout])
+      @query_options[:timeout] = "#{@options[:query_timeout]}s"
+    end
   end
 
   def result
@@ -52,6 +56,10 @@ def result
       query_options[:body].delete(:aggregations)
     end
     raw_result = @client.search(query_options)
+    if(raw_result["timed_out"])
+      # Don't return partial results.
+      raise "Elasticsearch request timed out"
+    end
     @result = LogResult.factory(self, raw_result)
   end
 

src/api-umbrella/web-app/app/models/log_search/kylin.rb

@@ -45,7 +45,7 @@ def execute_presto(sql)
       req.headers["X-Presto-Catalog"] = "hive"
       req.headers["X-Presto-Schema"] = "default"
       if(@options[:query_timeout])
-        req.headers["X-Presto-Session"] = "query_max_run_time=#{@options[:query_timeout]}"
+        req.headers["X-Presto-Session"] = "query_max_run_time=#{@options[:query_timeout]}s"
       end
       req.body = sql
     end