~alcinnz/hurl

26da82c4b2b18948cfd317bc1e4ead6006377a90 — Adrian Cochrane 4 years ago a7c88c8
Add utility for concurrently fetching URLs.
2 files changed, 8 insertions(+), 1 deletions(-)

M hurl.cabal
M src/Network/URI/Fetch.hs
M hurl.cabal => hurl.cabal +2 -1
@@ 94,7 94,8 @@ library
  
  -- Other library packages from which modules are imported.
  build-depends:       base >=4.9 && <=4.12, text >= 1.2 && <1.3,
                       network-uri >=2.6 && <2.7, bytestring >= 0.10 && < 0.11
                       network-uri >=2.6 && <2.7, bytestring >= 0.10 && < 0.11,
                       async >= 2.1 && < 2.3
  
  -- Directories containing source files.
  hs-source-dirs:      src

M src/Network/URI/Fetch.hs => src/Network/URI/Fetch.hs +6 -0
@@ 12,6 12,7 @@ import qualified Data.ByteString.Lazy as B
import qualified Data.ByteString.Char8 as C8
import           Network.URI.Charset
import           Control.Exception
import           Control.Concurrent.Async (forConcurrently)

-- for about: URIs & port parsing, all standard lib
import Data.Maybe (fromMaybe, listToMaybe)


@@ 98,6 99,11 @@ fetchURL sess mimes uri = do
    (_, mime, resp) <- fetchURL' sess mimes uri
    return (mime, resp)

-- | Concurrently fetch given URLs.
fetchURLs :: Session -> [String] -> [URI] -> ((URI, String, Either Text ByteString) -> IO a) -> IO [(URI, a)]
fetchURLs sess mimes uris cb =
    forConcurrently uris (\u -> fetchURL' sess mimes u >>= cb) >>= return . zip uris

-- | As per `fetchURL`, but also returns the redirected URI.
fetchURL' :: Session -> [String] -> URI -> IO (URI, String, Either Text ByteString)
fetchURL' session mimes uri@(URI {uriScheme = "about:", uriPath = ""}) =