From a550e5465bd9263371ded778cea6b244fed673ac Mon Sep 17 00:00:00 2001
From: Jelte Fennema <github-tech@jeltef.nl>
Date: Tue, 14 Jul 2020 17:52:31 +0200
Subject: [PATCH] Use persistent HTTP client to speed up requests
I have to delete multiple blobs at once. This is quite anoying in Azure
Blob storage in general, since it does not support batch deletes. So I
need to send a request for each delete (multiple thousands in my case).
The problem is made much worse by this library not keeping its HTTP
connections open for successive requests. This commit fixes that by
using the `Net:HTTP::Persistent` adapter from Faraday.
This makes sending multiple requests much faster. On a bad connection it
also causes less errors, since a connection timeout can only happen
once. After connecting successfully the connection will continue
working as TCP will handle any hicups, instead of getting `ETIMEDOUT`
errors.
---
Gemfile | 1 +
common/lib/azure/storage/common/core/http_client.rb | 5 ++++-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/Gemfile b/Gemfile
index f6b7ff9..83c6b69 100644
--- a/Gemfile
+++ b/Gemfile
@@ -26,6 +26,7 @@
source "https://rubygems.org" do
gem "faraday", "~> 1.0", :require => false
gem "faraday_middleware", "~> 1.0.0.rc1", :require => false
+ gem "net-http-persistent", "~> 4.0", :require => false
gem "nokogiri", "~> 1.10.4", :require => false
gem "adal", "~> 1.0", :require => false
diff --git a/common/lib/azure/storage/common/core/http_client.rb b/common/lib/azure/storage/common/core/http_client.rb
index 28006cf..4fdc06f 100644
--- a/common/lib/azure/storage/common/core/http_client.rb
+++ b/common/lib/azure/storage/common/core/http_client.rb
@@ -72,7 +72,10 @@ module Azure::Storage::Common::Core
end || nil
Faraday.new(uri, ssl: ssl_options, proxy: proxy_options) do |conn|
conn.use FaradayMiddleware::FollowRedirects
- conn.adapter Faraday.default_adapter
+ conn.adapter :net_http_persistent, pool_size: 5 do |http|
+ # yields Net::HTTP::Persistent
+ http.idle_timeout = 100
+ end
end
end
end
--
GitLab