Asynchronous requests with
crul.
There are two interfaces to asynchronous requests in
crul:
The first option takes less thinking, less work, and is good solution when you just want to hit a bunch of URLs asynchronously.
The second option is ideal when you want to set curl options/headers on each request and/or want to do different types of HTTP methods on each request.
One thing to think about before using async is whether the data provider is okay with it. It’s possible that a data provider’s service may be brought down if you do too many async requests.
Build request object with 1 or more URLs
(cc <- Async$new(
urls = c(
'https://hb.opencpu.org/get?a=5',
'https://hb.opencpu.org/get?a=5&b=6',
'https://hb.opencpu.org/ip'
)
))
#> <crul async connection>
#> curl options:
#> proxies:
#> auth:
#> headers:
#> urls: (n: 3)
#> https://hb.opencpu.org/get?a=5
#> https://hb.opencpu.org/get?a=5&b=6
#> https://hb.opencpu.org/ipMake request with any HTTP method
(res <- cc$get())
#> async responses
#> status code - url (N=3; printing up to 10)
#> 200 - https://hb.opencpu.org/get?a=5
#> 200 - https://hb.opencpu.org/get?a=5&b=6
#> 200 - https://hb.opencpu.org/ipYou get back a list matching length of the number of input URLs
Access object variables and methods just as with
HttpClient results, here just one at a time.
res[[1]]$url
#> [1] "https://hb.opencpu.org/get?a=5"
res[[1]]$success()
#> [1] TRUE
res[[1]]$parse("UTF-8")
#> [1] "{\n \"args\": {\n \"a\": \"5\"\n }, \n \"headers\": {\n \"Accept\": \"application/json, text/xml, application/xml, */*\", \n \"Accept-Encoding\": \"gzip, deflate\", \n \"Connection\": \"close\", \n \"Host\": \"httpbin:8080\", \n \"User-Agent\": \"R (4.4.1 aarch64-apple-darwin20 aarch64 darwin20)\"\n }, \n \"origin\": \"172.18.0.2\", \n \"url\": \"http://httpbin:8080/get?a=5\"\n}\n"Or apply access/method calls across many results, e.g., parse all results
lapply(res, function(z) z$parse("UTF-8"))
#> [[1]]
#> [1] "{\n \"args\": {\n \"a\": \"5\"\n }, \n \"headers\": {\n \"Accept\": \"application/json, text/xml, application/xml, */*\", \n \"Accept-Encoding\": \"gzip, deflate\", \n \"Connection\": \"close\", \n \"Host\": \"httpbin:8080\", \n \"User-Agent\": \"R (4.4.1 aarch64-apple-darwin20 aarch64 darwin20)\"\n }, \n \"origin\": \"172.18.0.2\", \n \"url\": \"http://httpbin:8080/get?a=5\"\n}\n"
#>
#> [[2]]
#> [1] "{\n \"args\": {\n \"a\": \"5\", \n \"b\": \"6\"\n }, \n \"headers\": {\n \"Accept\": \"application/json, text/xml, application/xml, */*\", \n \"Accept-Encoding\": \"gzip, deflate\", \n \"Connection\": \"close\", \n \"Host\": \"httpbin:8080\", \n \"User-Agent\": \"R (4.4.1 aarch64-apple-darwin20 aarch64 darwin20)\"\n }, \n \"origin\": \"172.18.0.2\", \n \"url\": \"http://httpbin:8080/get?a=5&b=6\"\n}\n"
#>
#> [[3]]
#> [1] "{\n \"origin\": \"172.18.0.2\"\n}\n"req1 <- HttpRequest$new(
url = "https://hb.opencpu.org/get?a=5",
opts = list(
verbose = TRUE
)
)
req1$get()
#> <crul http request> get
#> url: https://hb.opencpu.org/get?a=5
#> curl options:
#> verbose: TRUE
#> proxies:
#> auth:
#> headers:
#> progress: FALSE
req2 <- HttpRequest$new(
url = "https://hb.opencpu.org/post?a=5&b=6"
)
req2$post(body = list(a = 5))
#> <crul http request> post
#> url: https://hb.opencpu.org/post?a=5&b=6
#> curl options:
#> proxies:
#> auth:
#> headers:
#> progress: FALSE
(res <- AsyncVaried$new(req1, req2))
#> <crul async varied connection>
#> requests: (n: 2)
#> get: https://hb.opencpu.org/get?a=5
#> post: https://hb.opencpu.org/post?a=5&b=6Make requests asynchronously
Parse all results
res$parse()
#> [1] "{\n \"args\": {\n \"a\": \"5\"\n }, \n \"headers\": {\n \"Accept\": \"application/json, text/xml, application/xml, */*\", \n \"Accept-Encoding\": \"gzip, deflate\", \n \"Connection\": \"close\", \n \"Host\": \"httpbin:8080\", \n \"User-Agent\": \"R (4.4.1 aarch64-apple-darwin20 aarch64 darwin20)\"\n }, \n \"origin\": \"172.18.0.2\", \n \"url\": \"http://httpbin:8080/get?a=5\"\n}\n"
#> [2] "{\n \"args\": {\n \"a\": \"5\", \n \"b\": \"6\"\n }, \n \"data\": \"\", \n \"files\": {}, \n \"form\": {\n \"a\": \"5\"\n }, \n \"headers\": {\n \"Accept\": \"application/json, text/xml, application/xml, */*\", \n \"Accept-Encoding\": \"gzip, deflate\", \n \"Connection\": \"close\", \n \"Content-Length\": \"149\", \n \"Content-Type\": \"multipart/form-data; boundary=------------------------eDgqnrhsvjXexjEFHyzvTd\", \n \"Host\": \"httpbin:8080\", \n \"User-Agent\": \"libcurl/8.6.0 r-curl/5.2.1 crul/1.5.0\"\n }, \n \"json\": null, \n \"origin\": \"172.18.0.2\", \n \"url\": \"http://httpbin:8080/post?a=5&b=6\"\n}\n"lapply(res$parse(), jsonlite::prettify)
#> [[1]]
#> {
#> "args": {
#> "a": "5"
#> },
#> "headers": {
#> "Accept": "application/json, text/xml, application/xml, */*",
#> "Accept-Encoding": "gzip, deflate",
#> "Connection": "close",
#> "Host": "httpbin:8080",
#> "User-Agent": "R (4.4.1 aarch64-apple-darwin20 aarch64 darwin20)"
#> },
#> "origin": "172.18.0.2",
#> "url": "http://httpbin:8080/get?a=5"
#> }
#>
#>
#> [[2]]
#> {
#> "args": {
#> "a": "5",
#> "b": "6"
#> },
#> "data": "",
#> "files": {
#>
#> },
#> "form": {
#> "a": "5"
#> },
#> "headers": {
#> "Accept": "application/json, text/xml, application/xml, */*",
#> "Accept-Encoding": "gzip, deflate",
#> "Connection": "close",
#> "Content-Length": "149",
#> "Content-Type": "multipart/form-data; boundary=------------------------eDgqnrhsvjXexjEFHyzvTd",
#> "Host": "httpbin:8080",
#> "User-Agent": "libcurl/8.6.0 r-curl/5.2.1 crul/1.5.0"
#> },
#> "json": null,
#> "origin": "172.18.0.2",
#> "url": "http://httpbin:8080/post?a=5&b=6"
#> }
#> Status codes