Merge pull request #42 from jameslamb/retry-logic

replaced single-shot HTTP requests with retry logic

DESCRIPTION

@@ -8,7 +8,8 @@ Authors@R: c(
       person("Andy", "Hine", email = "andyyhine@gmail.com", role = "ctb"),
       person("Scott", "Came", email = "scottcame10@gmail.com", role = "ctb"),
       person("David", "Severski", email = "davidski@deadheaven.com", role = "ctb",
-        comment = c(ORCID = "https://orcid.org/0000-0001-7867-0459"))
+        comment = c(ORCID = "https://orcid.org/0000-0001-7867-0459")),
+      person("James", "Lamb", email = "jaylamb20@gmail.com", role = "ctb")
     )
 Description: 'Apache Drill' is a low-latency distributed query engine designed to enable 
     data exploration and 'analytics' on both relational and non-relational 'datastores', 

R/dbi.r

@@ -1,4 +1,4 @@
-s_head <- purrr::safely(httr::HEAD)
+s_head <- purrr::safely(function(...){httr::RETRY(verb = "HEAD", ...)})
 
 #' Driver for Drill database.
 #'
@@ -165,27 +165,31 @@ setMethod(
 
     if (.progress) {
 
-      httr::POST(
+      httr::RETRY(
+        verb = "POST",
         url = res@drill_server,
         path = "/query.json",
         encode = "json",
         progress(),
         body = list(
           queryType = "SQL",
           query = res@statement
-        )
+        ),
+        terminate_on = c(403, 404)
       ) -> resp
 
     } else {
 
-      httr::POST(
+      httr::RETRY(
+        verb = "POST",
         url = res@drill_server,
         path = "/query.json",
         encode = "json",
         body = list(
           queryType = "SQL",
           query = res@statement
-        )
+        ),
+        terminate_on = c(403, 404)
       ) -> resp
 
     }
@@ -426,11 +430,13 @@ setMethod(
   'dbListFields',
   signature(conn='DrillResult', name='missing'),
   function(conn, name) {
-    httr::POST(
+    httr::RETRY(
+      verb = "POST",
       sprintf("%s/query.json", conn@drill_server),
       encode = "json",
       body = list(queryType="SQL", query=conn@statement
-      )
+      ),
+      terminate_on = c(403, 404)
     ) -> res
 
     # fatal query error on the Drill side so return no fields

R/query.r

@@ -13,7 +13,7 @@
 #'               ignored if \code{drill_con} is a \code{JDBCConnection} created by
 #'               \code{drill_jdbc()})
 #' @param .progress if \code{TRUE} (default if in an interactive session) then ask
-#'                  \code{httr::POST} to display a progress bar
+#'                  \code{httr::RETRY} to display a progress bar
 #' @references \href{https://drill.apache.org/docs/}{Drill documentation}
 #' @family Drill direct REST API Interface
 #' @export
@@ -40,23 +40,26 @@ drill_query <- function(drill_con, query, uplift=TRUE, .progress=interactive())
     drill_server <- make_server(drill_con)
 
     if (.progress) {
-      httr::POST(
+      httr::RETRY(
+        "POST",
         url = sprintf("%s/query.json", drill_server),
         encode = "json",
         httr::progress(),
         body = list(
           queryType = "SQL",
           query = query
-        )
+        ),
+        terminate_on = c(403, 404)
       ) -> res
     } else {
-      httr::POST(
+      httr::RETRY(
         url = sprintf("%s/query.json", drill_server),
         encode = "json",
         body = list(
           queryType = "SQL",
           query = query
-        )
+        ),
+        terminate_on = c(403, 404)
       ) -> res
     }
 

R/rest-api.r

@@ -1,4 +1,4 @@
-s_head <- purrr::safely(httr::HEAD)
+s_head <- purrr::safely(function(...){httr::RETRY(verb = "HEAD", ...)})
 
 #' Setup a Drill connection
 #'
@@ -61,7 +61,7 @@ drill_active <- function(drill_con) {
 #' }
 drill_status <- function(drill_con) {
   drill_server <- make_server(drill_con)
-  res <- httr::GET(sprintf("%s/status", drill_server))
+  res <- httr::RETRY("GET", sprintf("%s/status", drill_server), terminate_on = c(403, 404))
   httr::stop_for_status(res)
   cnt <- httr::content(res, as="text", encoding="UTF-8")
   cnt <- htmltools::HTML(cnt)
@@ -78,7 +78,7 @@ drill_status <- function(drill_con) {
 #' }
 drill_metrics <- function(drill_con) {
   drill_server <- make_server(drill_con)
-  res <- httr::GET(sprintf("%s/status/metrics", drill_server))
+  res <- httr::RETRY("GET", sprintf("%s/status/metrics", drill_server), terminate_on = c(403, 404))
   httr::stop_for_status(res)
   cnt <- httr::content(res, as="text", encoding="UTF-8")
   jsonlite::fromJSON(cnt, flatten=TRUE)
@@ -95,7 +95,7 @@ drill_metrics <- function(drill_con) {
 #' }
 drill_threads <- function(drill_con) {
   drill_server <- make_server(drill_con)
-  res <- httr::GET(sprintf("%s/status/threads", drill_server))
+  res <- httr::RETRY("GET", sprintf("%s/status/threads", drill_server), terminate_on = c(403, 404))
   httr::stop_for_status(res)
   cnt <- httr::content(res, as="text", encoding="UTF-8")
   cnt <- htmltools::HTML(sprintf("<pre>%s</pre>", cnt))
@@ -113,7 +113,7 @@ drill_threads <- function(drill_con) {
 #' }
 drill_profiles <- function(drill_con) {
   drill_server <- make_server(drill_con)
-  res <- httr::GET(sprintf("%s/profiles.json", drill_server))
+  res <- httr::RETRY("GET", sprintf("%s/profiles.json", drill_server), terminate_on = c(403, 404))
   httr::stop_for_status(res)
   cnt <- httr::content(res, as="text", encoding="UTF-8")
   jsonlite::fromJSON(cnt)
@@ -128,7 +128,7 @@ drill_profiles <- function(drill_con) {
 #' @export
 drill_profile <- function(drill_con, query_id) {
   drill_server <- make_server(drill_con)
-  res <- httr::GET(sprintf("%s/profiles/%s.json", drill_server, query_id))
+  res <- httr::RETRY("GET", sprintf("%s/profiles/%s.json", drill_server, query_id), terminate_on = c(403, 404))
   httr::stop_for_status(res)
   cnt <- httr::content(res, as="text", encoding="UTF-8")
   jsonlite::fromJSON(cnt)
@@ -143,7 +143,7 @@ drill_profile <- function(drill_con, query_id) {
 #' @export
 drill_cancel <- function(drill_con, query_id) {
   drill_server <- make_server(drill_con)
-  res <- httr::GET(sprintf("%s/profiles/cancel/%s", drill_server, query_id))
+  res <- httr::RETRY("GET", sprintf("%s/profiles/cancel/%s", drill_server, query_id), terminate_on = c(403, 404))
   httr::stop_for_status(res)
   message(httr::content(res, as="text", encoding="UTF-8"))
   invisible(TRUE)
@@ -195,9 +195,9 @@ drill_storage <- function(drill_con, plugin=NULL, as=c("tbl", "list", "raw")) {
   drill_server <- make_server(drill_con)
 
   if (is.null(plugin)) {
-    res <- httr::GET(sprintf("%s/storage.json", drill_server))
+    res <- httr::RETRY("GET", sprintf("%s/storage.json", drill_server), terminate_on = c(403, 404))
   } else {
-    res <- httr::GET(sprintf("%s/storage/%s.json", drill_server, plugin))
+    res <- httr::RETRY("GET", sprintf("%s/storage/%s.json", drill_server, plugin), terminate_on = c(403, 404))
   }
 
   httr::stop_for_status(res)
@@ -225,11 +225,13 @@ drill_mod_storage <- function(drill_con, name, config) {
 
   drill_server <- make_server(drill_con)
 
-  httr::POST(
+  httr::RETRY(
+    verb = "POST",
     url = sprintf("%s/storage/%s.json", drill_server, name),
     httr::content_type_json(),
     body = config,
-    encode = "raw"
+    encode = "raw",
+    terminate_on = c(403, 404)
   ) -> res
 
   httr::stop_for_status(res)
@@ -247,9 +249,11 @@ drill_rm_storage <- function(drill_con, name) {
 
   drill_server <- make_server(drill_con)
 
-  httr::DELETE(
+  httr::RETRY(
+    verb = "DELETE",
     url = sprintf("%s/storage/%s.json", drill_server, name),
-    httr::content_type_json()
+    httr::content_type_json(),
+    terminate_on = c(403, 404)
   ) -> res
 
   httr::stop_for_status(res)
@@ -271,7 +275,7 @@ drill_rm_storage <- function(drill_con, name) {
 #' }
 drill_options <- function(drill_con, pattern=NULL) {
   drill_server <- make_server(drill_con)
-  res <- httr::GET(sprintf("%s/options.json", drill_server))
+  res <- httr::RETRY("GET", sprintf("%s/options.json", drill_server), terminate_on = c(403, 404))
   httr::stop_for_status(res)
   cnt <- httr::content(res, as="text", encoding="UTF-8")
   jsonlite::fromJSON(cnt) %>%
@@ -291,7 +295,7 @@ drill_options <- function(drill_con, pattern=NULL) {
 #' }
 drill_stats <- function(drill_con) {
   drill_server <- make_server(drill_con)
-  res <- httr::GET(sprintf("%s/cluster.json", drill_server))
+  res <- httr::RETRY("GET", sprintf("%s/cluster.json", drill_server), terminate_on = c(403, 404))
   httr::stop_for_status(res)
   cnt <- httr::content(res, as="text", encoding="UTF-8")
   jsonlite::fromJSON(cnt)

R/schemas.R

@@ -2,7 +2,7 @@
 #'
 #' @param drill_con drill server connection object setup by \code{drill_connection()}
 #' @param .progress if \code{TRUE} (default if in an interactive session) then ask
-#'                  \code{httr::POST} to display a progress bar
+#'                  \code{httr::RETRY} to display a progress bar
 #' @references \href{https://drill.apache.org/docs/}{Drill documentation}
 #' @family Dill direct REST API Interface
 #' @export
@@ -16,7 +16,7 @@ drill_show_schemas <- function(drill_con, .progress=interactive()) {
 #' @param schema_name A unique name for a Drill schema. A schema in Drill is a configured
 #'                   storage plugin, such as hive, or a storage plugin and workspace.
 #' @param .progress if \code{TRUE} (default if in an interactive session) then ask
-#'                  \code{httr::POST} to display a progress bar
+#'                  \code{httr::RETRY} to display a progress bar
 #' @references \href{https://drill.apache.org/docs/}{Drill documentation}
 #' @family Dill direct REST API Interface
 #' @export
@@ -32,7 +32,7 @@ drill_use <- function(drill_con, schema_name, .progress=interactive()) {
 #' @param drill_con drill server connection object setup by \code{drill_connection()}
 #' @param schema_spec properly quoted "filesystem.directory_name" reference path
 #' @param .progress if \code{TRUE} (default if in an interactive session) then ask
-#'                  \code{httr::POST} to display a progress bar
+#'                  \code{httr::RETRY} to display a progress bar
 #' @export
 #' @references \href{https://drill.apache.org/docs/}{Drill documentation}
 #' @family Dill direct REST API Interface

R/utils.r

@@ -14,14 +14,16 @@ auth_drill <- function(ssl, host, port, username, password) {
 
   httr::set_config(config(ssl_verifypeer = 0L))
 
-  httr::POST(
+  httr::RETRY(
+    "POST",
     url = sprintf("%s://%s:%s", ifelse(ssl[1], "https", "http"), host, port),
     path = "/j_security_check",
     encode = "form",
     body = list(
       j_username = username,
       j_password = password
-    )
+    ),
+    terminate_on = c(403, 404)
   ) -> res
 
   httr::stop_for_status(res)