From 26da82c4b2b18948cfd317bc1e4ead6006377a90 Mon Sep 17 00:00:00 2001
From: Adrian Cochrane <adrian@openwork.nz>
Date: Fri, 17 Apr 2020 17:48:13 +1200
Subject: [PATCH] Add utility for concurrently fetching URLs.

---
 hurl.cabal               | 3 ++-
 src/Network/URI/Fetch.hs | 6 ++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/hurl.cabal b/hurl.cabal
index daf8fab..c0888e1 100644
--- a/hurl.cabal
+++ b/hurl.cabal
@@ -94,7 +94,8 @@ library
   
   -- Other library packages from which modules are imported.
   build-depends:       base >=4.9 && <=4.12, text >= 1.2 && <1.3,
-                       network-uri >=2.6 && <2.7, bytestring >= 0.10 && < 0.11
+                       network-uri >=2.6 && <2.7, bytestring >= 0.10 && < 0.11,
+                       async >= 2.1 && < 2.3
   
   -- Directories containing source files.
   hs-source-dirs:      src
diff --git a/src/Network/URI/Fetch.hs b/src/Network/URI/Fetch.hs
index 78b64f3..5287a37 100644
--- a/src/Network/URI/Fetch.hs
+++ b/src/Network/URI/Fetch.hs
@@ -12,6 +12,7 @@ import qualified Data.ByteString.Lazy as B
 import qualified Data.ByteString.Char8 as C8
 import           Network.URI.Charset
 import           Control.Exception
+import           Control.Concurrent.Async (forConcurrently)
 
 -- for about: URIs & port parsing, all standard lib
 import Data.Maybe (fromMaybe, listToMaybe)
@@ -98,6 +99,11 @@ fetchURL sess mimes uri = do
     (_, mime, resp) <- fetchURL' sess mimes uri
     return (mime, resp)
 
+-- | Concurrently fetch given URLs.
+fetchURLs :: Session -> [String] -> [URI] -> ((URI, String, Either Text ByteString) -> IO a) -> IO [(URI, a)]
+fetchURLs sess mimes uris cb =
+    forConcurrently uris (\u -> fetchURL' sess mimes u >>= cb) >>= return . zip uris
+
 -- | As per `fetchURL`, but also returns the redirected URI.
 fetchURL' :: Session -> [String] -> URI -> IO (URI, String, Either Text ByteString)
 fetchURL' session mimes uri@(URI {uriScheme = "about:", uriPath = ""}) =
-- 
2.30.2