From 196ffee57185f1e2ba7ed79fb1627e376c476647 Mon Sep 17 00:00:00 2001
From: Samuel Gratzl <sam@sgratzl.com>
Date: Sat, 14 Aug 2021 12:45:25 -0400
Subject: [PATCH 1/6] feat: start with covidcast meta infos

---
 R/constants.R   |  2 +-
 R/covidcast.R   | 26 ++++++++++++++++++++++++++
 R/epidatacall.R | 21 +++------------------
 R/request.R     | 21 +++++++++++++++++++++
 4 files changed, 51 insertions(+), 19 deletions(-)
 create mode 100644 R/covidcast.R
 create mode 100644 R/request.R

diff --git a/R/constants.R b/R/constants.R
index 66480f4e..56068632 100644
--- a/R/constants.R
+++ b/R/constants.R
@@ -2,4 +2,4 @@
 
 version <- "1.0.0"
 http_headers <- httr::add_headers("User-Agent" = paste0("delphi_epidata", version))
-base_url <- "https://delphi.cmu.edu/epidata/"
+global_base_url <- "https://delphi.cmu.edu/epidata/"
diff --git a/R/covidcast.R b/R/covidcast.R
new file mode 100644
index 00000000..9881afec
--- /dev/null
+++ b/R/covidcast.R
@@ -0,0 +1,26 @@
+
+
+as_web_link <- function(obj) {
+
+}
+
+#'
+#' creates the covidcast epidata helper
+#'
+#' @param base_url optional alternative base url
+#' @importFrom httr RETRY stop_for_status content
+#' @importFrom jsonlite fromJSON
+#' @importFrom rlang abort
+#' @return TODO
+#'
+#' @export
+covidcast_epidata <- function(base_url = global_base_url) {
+    url <- join_url(base_url, 'covidcast/meta')
+    res <- do_request(url, list())
+
+    httr::stop_for_status(res)
+    r <- httr::content(res, "text", encoding = "UTF-8")
+    meta <- jsonlite::fromJSON(r, simplifyVector=FALSE)
+
+    meta
+}
\ No newline at end of file
diff --git a/R/epidatacall.R b/R/epidatacall.R
index 320926e8..d5a144bd 100644
--- a/R/epidatacall.R
+++ b/R/epidatacall.R
@@ -13,7 +13,7 @@ create_epidata_call <- function(endpoint, params, meta = NULL, only_supports_cla
     list(
       endpoint = endpoint,
       params = params,
-      base_url = base_url,
+      base_url = global_base_url,
       meta = meta,
       only_supports_classic = only_supports_classic
     ),
@@ -62,11 +62,7 @@ request_arguments <-
 
 full_url <- function(epidata_call) {
   stopifnot(inherits(epidata_call, "epidata_call"))
-  url <- epidata_call$base_url
-  if (url[length(url)] != "/") {
-    url <- paste0(url, "/")
-  }
-  paste0(url, epidata_call$endpoint)
+  join_url(epidata_call$base_url, epidata_call$endpoint)
 }
 
 #'
@@ -99,18 +95,7 @@ request_impl <- function(epidata_call, format_type, fields = NULL) {
   url <- full_url(epidata_call)
   params <- request_arguments(epidata_call, format_type, fields)
 
-  # don't retry in case of certain status codes
-  res <- httr::RETRY("GET", url,
-    query = params, http_headers,
-    terminate_on = c(400, 401, 403, 405, 414, 500)
-  )
-  if (res$status_code == 414) {
-    res <- httr::RETRY("POST", url,
-      body = params, encode = "form", http_headers,
-      terminate_on = c(400, 401, 403, 405, 414, 500)
-    )
-  }
-  res
+  do_request(url, params)
 }
 
 #'
diff --git a/R/request.R b/R/request.R
new file mode 100644
index 00000000..9cd13daa
--- /dev/null
+++ b/R/request.R
@@ -0,0 +1,21 @@
+join_url <- function(url, endpoint) {
+  if (url[length(url)] != "/") {
+    url <- paste0(url, "/")
+  }
+  paste0(url, endpoint)
+}
+
+do_request <- function(url, params) {
+  # don't retry in case of certain status codes
+  res <- httr::RETRY("GET", url,
+    query = params, http_headers,
+    terminate_on = c(400, 401, 403, 405, 414, 500)
+  )
+  if (res$status_code == 414) {
+    res <- httr::RETRY("POST", url,
+      body = params, encode = "form", http_headers,
+      terminate_on = c(400, 401, 403, 405, 414, 500)
+    )
+  }
+  res
+}

From 1eaf8e3ad017a5eacc0713459ee0bd0ba183d747 Mon Sep 17 00:00:00 2001
From: Samuel Gratzl <sam@sgratzl.com>
Date: Mon, 23 Aug 2021 09:59:16 -0400
Subject: [PATCH 2/6] feat: covidcast meta api

---
 R/covidcast.R           | 96 +++++++++++++++++++++++++++++++++++++----
 R/utils.R               |  2 +-
 vignettes/covidcast.Rmd | 49 +++++++++++++++++++++
 3 files changed, 138 insertions(+), 9 deletions(-)
 create mode 100644 vignettes/covidcast.Rmd

diff --git a/R/covidcast.R b/R/covidcast.R
index 9881afec..590647b0 100644
--- a/R/covidcast.R
+++ b/R/covidcast.R
@@ -1,9 +1,69 @@
 
+parse_signal <- function(signal) {
+  class(signal) <- c(class(signal), "covidcast_data_signal")
 
-as_web_link <- function(obj) {
+  #'
+  #' fetch covidcast data
+  #'
+  #' @param data_source data source to fetch
+  #' @param signals data source to fetch
+  #' @param time_type data source to fetch
+  #' @param time_values data source to fetch
+  #' @param geo_type geo_type to fetch
+  #' @param geo_values data source to fetch
+  #' @param as_of data source to fetch
+  #' @param issues data source to fetch
+  #' @param lag data source to fetch
+  #' @return an instance of epidata_call
+  signal$call <- function(geo_type,
+                          geo_values,
+                          time_values,
+                          ...) {
+    covidcast(
+      signal$source, signal$signal, signal$time_type, geo_type,
+      time_values, geo_values, ...
+    )
+  }
+  r <- list()
+  r[[signal$signal]] <- signal
+  r
+}
 
+parse_source <- function(source) {
+  class(source) <- c(class(source), "covidcast_data_source")
+  signals <- do.call(c, lapply(source$signals, parse_signal))
+  source$signals <- signals
+  signals_df <- as.data.frame(do.call(rbind, lapply(signals, function(x) {
+    sub <- x[c(
+      "source",
+      "signal",
+      "name",
+      "active",
+      "short_description",
+      "description",
+      "time_type",
+      "time_label",
+      "value_label",
+      "format",
+      "category",
+      "high_values_are",
+      "is_smoothed",
+      "is_weighted",
+      "is_cumulative",
+      "has_stderr",
+      "has_sample_size"
+    )]
+    sub$geo_types <- paste0(names(x$geo_types), collapse = ",")
+    sub
+  })))
+  rownames(signals_df) <- paste(signals_df$source, signals_df$signal, sep = ":")
+  source$signals_df <- signals_df
+  r <- list()
+  r[[source$source]] <- source
+  r
 }
 
+
 #'
 #' creates the covidcast epidata helper
 #'
@@ -15,12 +75,32 @@ as_web_link <- function(obj) {
 #'
 #' @export
 covidcast_epidata <- function(base_url = global_base_url) {
-    url <- join_url(base_url, 'covidcast/meta')
-    res <- do_request(url, list())
+  url <- join_url(base_url, "covidcast/meta")
+  res <- do_request(url, list())
 
-    httr::stop_for_status(res)
-    r <- httr::content(res, "text", encoding = "UTF-8")
-    meta <- jsonlite::fromJSON(r, simplifyVector=FALSE)
+  httr::stop_for_status(res)
+  r <- httr::content(res, "text", encoding = "UTF-8")
+  meta <- jsonlite::fromJSON(r, simplifyVector = FALSE)
 
-    meta
-}
\ No newline at end of file
+  sources <- do.call(c, lapply(meta, parse_source))
+  sources_df <- as.data.frame(do.call(rbind, lapply(sources, function(x) {
+    sub <- x[c(
+      "source", "name", "description", "reference_signal", "license"
+    )]
+    sub$signals <- paste0(x$signals_df$signal, collapse = ",")
+    sub
+  })))
+  rownames(sources_df) <- sources_df$source
+
+  structure(
+    list(
+      base_url = base_url,
+      sources = sources,
+      sources_df = sources_df,
+      signals_df = as.data.frame(do.call(rbind, lapply(sources, function(x) {
+        x$signals_df
+      })))
+    ),
+    class = "covidcast_epidata"
+  )
+}
diff --git a/R/utils.R b/R/utils.R
index d0211448..3b562377 100644
--- a/R/utils.R
+++ b/R/utils.R
@@ -5,7 +5,7 @@ format_item <- function(value) {
   if (inherits(value, "EpiRange")) {
     paste0(toString(value$from), "-", toString(value$to))
   } else if (is.list(value) &&
-    "from" %in% names(value) && "to" %in% names(value)) {
+    "from" %in% names(value) && "to" %in% names(value) && length(names(value)) == 2) {
     paste0(toString(value$from), "-", toString(value$to))
   } else {
     toString(value)
diff --git a/vignettes/covidcast.Rmd b/vignettes/covidcast.Rmd
new file mode 100644
index 00000000..63749014
--- /dev/null
+++ b/vignettes/covidcast.Rmd
@@ -0,0 +1,49 @@
+---
+title: "Delphi Epidata R API Client"
+author: "Delphi Group"
+date: "`r Sys.Date()`"
+output: rmarkdown::html_vignette
+vignette: >
+  %\VignetteIndexEntry{Delphi Epidata R API Client}
+  %\VignetteEngine{knitr::rmarkdown}
+  \usepackage[utf8]{inputenc}
+---
+
+```{r setup, include=FALSE}
+knitr::opts_chunk$set(echo = TRUE)
+```
+
+# Delphi Epidata R API Client
+
+```{r libraries}
+# devtools::install_url("https://github.com/cmu-delphi/delphi-epidata-r/releases/latest/download/delphi-epidata.tar.gz")
+# install.packages('delphi.epidata')
+library('delphi.epidata')
+library('magrittr')
+```
+
+
+```{r}
+covidcast_api <- covidcast_epidata()
+epicall <- covidcast_api$sources$`fb-survey`$signals$smoothed_cli$call("nation", "us", epirange(20210405, 20210410))
+```
+
+```{r}
+epicall %>% fetch_classic()
+```
+
+```{r}
+epicall %>% fetch_json()
+```
+
+```{r}
+epicall %>% fetch_csv()
+```
+
+```{r}
+epicall %>% fetch_df()
+```
+
+```{r}
+# epicall %>% with_base_url("https://staging.delphi.cmu.edu/epidata/") %>% fetch_df()
+```

From 8fdf14edbcbfe138e21c3faed09c799242785ec9 Mon Sep 17 00:00:00 2001
From: Samuel Gratzl <sam@sgratzl.com>
Date: Mon, 23 Aug 2021 11:37:38 -0400
Subject: [PATCH 3/6] feat: data frame on the fly and print

---
 R/covidcast.R   | 104 +++++++++++++++++++++++++++++++++++-------------
 R/epidatacall.R |   3 +-
 2 files changed, 78 insertions(+), 29 deletions(-)

diff --git a/R/covidcast.R b/R/covidcast.R
index 590647b0..9e75b4de 100644
--- a/R/covidcast.R
+++ b/R/covidcast.R
@@ -1,6 +1,7 @@
 
-parse_signal <- function(signal) {
-  class(signal) <- c(class(signal), "covidcast_data_signal")
+parse_signal <- function(signal, base_url) {
+  class(signal) <- c("covidcast_data_signal", class(signal))
+  signal$key <- paste(signal$source, signal$signal, sep = ":")
 
   #'
   #' fetch covidcast data
@@ -19,21 +20,37 @@ parse_signal <- function(signal) {
                           geo_values,
                           time_values,
                           ...) {
-    covidcast(
+    epicall <- covidcast(
       signal$source, signal$signal, signal$time_type, geo_type,
       time_values, geo_values, ...
     )
+    epicall$base_url <- base_url
+    epicall
   }
   r <- list()
   r[[signal$signal]] <- signal
   r
 }
 
-parse_source <- function(source) {
-  class(source) <- c(class(source), "covidcast_data_source")
-  signals <- do.call(c, lapply(source$signals, parse_signal))
+print.covidcast_data_signal <- function(signal, ...) {
+  print(signal$name)
+  print(signal$key)
+  print(signal$short_description)
+}
+
+parse_source <- function(source, base_url) {
+  class(source) <- c("covidcast_data_source", class(source))
+  signals <- do.call(c, lapply(source$signals, parse_signal, base_url = base_url))
+  class(signals) <- c("covidcast_data_signal_list", class(signals))
   source$signals <- signals
-  signals_df <- as.data.frame(do.call(rbind, lapply(signals, function(x) {
+  r <- list()
+  r[[source$source]] <- source
+  r
+}
+
+
+as.data.frame.covidcast_data_signal_list <- function(signals, ...) {
+  as.data.frame(do.call(rbind, lapply(signals, function(x) {
     sub <- x[c(
       "source",
       "signal",
@@ -55,15 +72,20 @@ parse_source <- function(source) {
     )]
     sub$geo_types <- paste0(names(x$geo_types), collapse = ",")
     sub
-  })))
-  rownames(signals_df) <- paste(signals_df$source, signals_df$signal, sep = ":")
-  source$signals_df <- signals_df
-  r <- list()
-  r[[source$source]] <- source
-  r
+  })), row.names = sapply(signals, function(x) {
+    x$key
+  }), ...)
 }
 
 
+print.covidcast_data_source <- function(source, ...) {
+  print(source$name, ...)
+  print(source$source, ...)
+  print(source$description, ...)
+  signals <- as.data.frame(source$signals)
+  print(signals[, c("signal", "name", "short_description")], ...)
+}
+
 #'
 #' creates the covidcast epidata helper
 #'
@@ -71,7 +93,7 @@ parse_source <- function(source) {
 #' @importFrom httr RETRY stop_for_status content
 #' @importFrom jsonlite fromJSON
 #' @importFrom rlang abort
-#' @return TODO
+#' @return an instance of covidcast_epidata
 #'
 #' @export
 covidcast_epidata <- function(base_url = global_base_url) {
@@ -82,25 +104,51 @@ covidcast_epidata <- function(base_url = global_base_url) {
   r <- httr::content(res, "text", encoding = "UTF-8")
   meta <- jsonlite::fromJSON(r, simplifyVector = FALSE)
 
-  sources <- do.call(c, lapply(meta, parse_source))
-  sources_df <- as.data.frame(do.call(rbind, lapply(sources, function(x) {
-    sub <- x[c(
-      "source", "name", "description", "reference_signal", "license"
-    )]
-    sub$signals <- paste0(x$signals_df$signal, collapse = ",")
-    sub
-  })))
-  rownames(sources_df) <- sources_df$source
+  sources <- do.call(c, lapply(meta, parse_source, base_url = base_url))
+  class(sources) <- c("covidcast_data_source_list", class(sources))
 
+  all_signals <- do.call(c, lapply(sources, function(x) {
+    l <- c(x$signals)
+    names(l) <- paste(x$source, names(l), sep = ":")
+    l
+  }))
+  class(all_signals) <- c("covidcast_data_signal_list", class(all_signals))
   structure(
     list(
-      base_url = base_url,
       sources = sources,
-      sources_df = sources_df,
-      signals_df = as.data.frame(do.call(rbind, lapply(sources, function(x) {
-        x$signals_df
-      })))
+      signals = all_signals
     ),
     class = "covidcast_epidata"
   )
 }
+
+
+as.data.frame.covidcast_data_source_list <- function(sources, ...) {
+  as.data.frame(do.call(rbind, lapply(sources, function(x) {
+    sub <- x[c(
+      "source", "name", "description", "reference_signal", "license"
+    )]
+    sub$signals <- paste0(sapply(x$signals, function(y) {
+      y$signal
+    }), collapse = ",")
+    sub
+  })), row.names = sapply(sources, function(x) {
+    x$source
+  }), ...)
+}
+
+print.covidcast_epidata <- function(epidata, ...) {
+  print("COVIDcast Epidata Fetcher")
+  print("Sources:")
+  sources <- as.data.frame(epidata$sources)
+  print(sources[1:5, c("source", "name")], ...)
+  if (nrow(sources) > 5) {
+    print(paste0((nrow(sources) - 5), " more..."))
+  }
+  print("Signals")
+  signals <- as.data.frame(epidata$signals)
+  print(signals[1:5, c("source", "signal", "name")], ...)
+  if (nrow(signals) > 5) {
+    print(paste0((nrow(signals) - 5), " more..."))
+  }
+}
diff --git a/R/epidatacall.R b/R/epidatacall.R
index d5a144bd..b106689f 100644
--- a/R/epidatacall.R
+++ b/R/epidatacall.R
@@ -1,7 +1,8 @@
 
 
 
-create_epidata_call <- function(endpoint, params, meta = NULL, only_supports_classic = FALSE) {
+create_epidata_call <- function(endpoint, params, meta = NULL,
+                                only_supports_classic = FALSE) {
   stopifnot(is.character(endpoint), length(endpoint) == 1)
   stopifnot(is.list(params))
   stopifnot(is.null(meta) || is.list(meta))

From 8106e7832609b5706f3c380faae20ff8d2102fce Mon Sep 17 00:00:00 2001
From: Samuel Gratzl <sam@sgratzl.com>
Date: Mon, 23 Aug 2021 11:38:58 -0400
Subject: [PATCH 4/6] fix: vignette title

---
 vignettes/covidcast.Rmd | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vignettes/covidcast.Rmd b/vignettes/covidcast.Rmd
index 63749014..2c708f0c 100644
--- a/vignettes/covidcast.Rmd
+++ b/vignettes/covidcast.Rmd
@@ -1,10 +1,10 @@
 ---
-title: "Delphi Epidata R API Client"
+title:"COVIDcast API Client"
 author: "Delphi Group"
 date: "`r Sys.Date()`"
 output: rmarkdown::html_vignette
 vignette: >
-  %\VignetteIndexEntry{Delphi Epidata R API Client}
+  %\VignetteIndexEntry{COVIDcast API Client}
   %\VignetteEngine{knitr::rmarkdown}
   \usepackage[utf8]{inputenc}
 ---

From 4b37caf557234f71f777e15deb9c675810b4ffea Mon Sep 17 00:00:00 2001
From: Samuel Gratzl <sam@sgratzl.com>
Date: Mon, 23 Aug 2021 11:44:12 -0400
Subject: [PATCH 5/6] refactor: change importFrom to where they belong

---
 R/epidatacall.R | 15 ++++-----------
 R/parse.R       |  6 ++++++
 R/request.R     |  3 +++
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/R/epidatacall.R b/R/epidatacall.R
index b106689f..1eafd5a4 100644
--- a/R/epidatacall.R
+++ b/R/epidatacall.R
@@ -105,9 +105,8 @@ request_impl <- function(epidata_call, format_type, fields = NULL) {
 #' @param epidata_call and instance of epidata_call
 #' @param fields filter fields
 #' @param disable_date_parsing disable automatic date parsing
-#' @importFrom httr RETRY stop_for_status content http_error
+#' @importFrom httr stop_for_status content http_error
 #' @importFrom jsonlite fromJSON
-#' @importFrom MMWRweek MMWRweek2Date
 #' @return parsed json message
 #'
 #' @export
@@ -135,9 +134,8 @@ fetch_classic <- function(epidata_call, fields = NULL, disable_date_parsing = FA
 #' @param epidata_call and instance of epidata_call
 #' @param fields filter fields
 #' @param disable_date_parsing disable automatic date parsing
-#' @importFrom httr RETRY stop_for_status content
+#' @importFrom httr stop_for_status content
 #' @importFrom jsonlite fromJSON
-#' @importFrom MMWRweek MMWRweek2Date
 #' @importFrom rlang abort
 #' @return parsed json message
 #'
@@ -163,7 +161,7 @@ fetch_json <- function(epidata_call, fields = NULL, disable_date_parsing = FALSE
 #'
 #' @param epidata_call and instance of epidata_call
 #' @param fields filter fields
-#' @importFrom httr RETRY stop_for_status content
+#' @importFrom httr stop_for_status content
 #' @importFrom rlang abort
 #' @return CSV text
 #'
@@ -224,8 +222,7 @@ info_to_type <- function(info, disable_date_parsing = FALSE) {
 #' @param fields filter fields
 #' @param disable_date_parsing disable automatic date parsing
 #' @importFrom readr read_csv
-#' @importFrom httr RETRY stop_for_status content
-#' @importFrom MMWRweek MMWRweek2Date
+#' @importFrom httr stop_for_status content
 #' @importFrom rlang abort
 #' @return tibble
 #'
@@ -279,10 +276,6 @@ fetch_tbl <- function(epidata_call, fields = NULL, disable_date_parsing = FALSE)
 #' @param epidata_call and instance of epidata_call
 #' @param fields filter fields
 #' @param disable_date_parsing disable automatic date parsing
-#' @importFrom readr read_csv
-#' @importFrom httr RETRY stop_for_status content
-#' @importFrom MMWRweek MMWRweek2Date
-#' @importFrom rlang abort
 #' @return data.frame
 #'
 #' @export
diff --git a/R/parse.R b/R/parse.R
index 56228aab..74a6b3da 100644
--- a/R/parse.R
+++ b/R/parse.R
@@ -3,6 +3,12 @@ parse_api_date <- function(value) {
   as.Date(as.character(value), format = "%Y%m%d")
 }
 
+
+
+#'
+#' parses a week
+#' @importFrom MMWRweek MMWRweek2Date
+#' @return a date
 parse_api_week <- function(value) {
   v <- as.integer(value)
   years <- floor(v / 100)
diff --git a/R/request.R b/R/request.R
index 9cd13daa..e05751da 100644
--- a/R/request.R
+++ b/R/request.R
@@ -5,6 +5,9 @@ join_url <- function(url, endpoint) {
   paste0(url, endpoint)
 }
 
+#'
+#' performs the request
+#' @importFrom httr RETRY
 do_request <- function(url, params) {
   # don't retry in case of certain status codes
   res <- httr::RETRY("GET", url,

From 5a2513aaeb708a04538407c5acfb204f857ea501 Mon Sep 17 00:00:00 2001
From: Samuel Gratzl <sam@sgratzl.com>
Date: Mon, 23 Aug 2021 11:45:16 -0400
Subject: [PATCH 6/6] fix: vignette title

---
 vignettes/covidcast.Rmd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vignettes/covidcast.Rmd b/vignettes/covidcast.Rmd
index 2c708f0c..23160de7 100644
--- a/vignettes/covidcast.Rmd
+++ b/vignettes/covidcast.Rmd
@@ -1,5 +1,5 @@
 ---
-title:"COVIDcast API Client"
+title: "COVIDcast API Client"
 author: "Delphi Group"
 date: "`r Sys.Date()`"
 output: rmarkdown::html_vignette