Merge pull request #1070 from cmu-delphi/release/delphi-epidata-0.4.5

Release Delphi Epidata 0.4.5

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.4
+current_version = 0.4.5
 commit = False
 tag = False
 

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 0.4.4
+version = 0.4.5
 
 [options]
 packages =

docs/api/covidcast-signals/covid-act-now.md

@@ -1,6 +1,6 @@
 ---
 title: COVID Act Now
-parent: Data Sources and Signals
+parent: Inactive Signals
 grand_parent: COVIDcast Main Endpoint
 ---
 

docs/api/covidcast-signals/quidel-inactive.md

@@ -0,0 +1,52 @@
+---
+title: Quidel
+parent: Inactive Signals
+grand_parent: COVIDcast Main Endpoint
+---
+
+# Quidel
+{: .no_toc}
+
+* **Source name:** `quidel`
+
+## Table of Contents
+{: .no_toc .text-delta}
+
+1. TOC
+{:toc}
+
+
+## COVID-19 Tests
+These signals are still active. Documentation is available on the [Quidel page](quidel.md).
+
+## Flu Tests
+
+* **Earliest issue available:** April 29, 2020
+* **Last issued:** May 19, 2020
+* **Number of data revisions since May 19, 2020:** 0
+* **Date of last change:** Never
+* **Available for:** msa, state (see [geography coding docs](../covidcast_geography.md))
+* **Time type:** day (see [date format docs](../covidcast_times.md))
+
+Data source based on flu lab tests, provided to us by Quidel, Inc. When a
+patient (whether at a doctor’s office, clinic, or hospital) has COVID-like
+symptoms, doctors may perform a flu test to rule out seasonal flu (influenza),
+because these two diseases have similar symptoms. Using this lab test data, we
+estimate the total number of flu tests per medical device (a measure of testing
+frequency), and the percentage of flu tests that are *negative* (since ruling
+out flu leaves open another cause---possibly covid---for the patient's
+symptoms), in a given location, on a given day.
+
+The number of flu tests conducted in individual counties can be quite small, so
+we do not report these signals at the county level.
+
+The flu test data is no longer updated as of May 19, 2020, as the number of flu
+tests conducted during the summer (outside of the normal flu season) is quite
+small. The data may be updated again when the Winter 2020 flu season begins.
+
+| Signal | Description |
+| --- | --- |
+| `raw_pct_negative` | The percentage of flu tests that are negative, suggesting the patient's illness has another cause, possibly COVID-19 <br/> **Earliest date available:** 2020-01-31 |
+| `smoothed_pct_negative` | Same as above, but smoothed in time <br/> **Earliest date available:** 2020-01-31 |
+| `raw_tests_per_device` | The average number of flu tests conducted by each testing device; measures volume of testing <br/> **Earliest date available:** 2020-01-31 |
+| `smoothed_tests_per_device` | Same as above, but smoothed in time <br/> **Earliest date available:** 2020-01-31 |

docs/api/covidcast-signals/quidel.md

@@ -157,34 +157,5 @@ not enough samples can be filled in from the parent state for smoothed signals s
 no data is reported for that area on that day; an API query for all reported geographic areas on
 that day will not include it.
 
-## Flu Tests
-
-* **Earliest issue available:** April 29, 2020
-* **Last issued:** May 19, 2020
-* **Number of data revisions since May 19, 2020:** 0
-* **Date of last change:** Never
-* **Available for:** msa, state (see [geography coding docs](../covidcast_geography.md))
-* **Time type:** day (see [date format docs](../covidcast_times.md))
-
-Data source based on flu lab tests, provided to us by Quidel, Inc. When a
-patient (whether at a doctor’s office, clinic, or hospital) has COVID-like
-symptoms, doctors may perform a flu test to rule out seasonal flu (influenza),
-because these two diseases have similar symptoms. Using this lab test data, we
-estimate the total number of flu tests per medical device (a measure of testing
-frequency), and the percentage of flu tests that are *negative* (since ruling
-out flu leaves open another cause---possibly covid---for the patient's
-symptoms), in a given location, on a given day.
-
-The number of flu tests conducted in individual counties can be quite small, so
-we do not report these signals at the county level.
-
-The flu test data is no longer updated as of May 19, 2020, as the number of flu
-tests conducted during the summer (outside of the normal flu season) is quite
-small. The data may be updated again when the Winter 2020 flu season begins.
-
-| Signal | Description |
-| --- | --- |
-| `raw_pct_negative` | The percentage of flu tests that are negative, suggesting the patient's illness has another cause, possibly COVID-19 <br/> **Earliest date available:** 2020-01-31 |
-| `smoothed_pct_negative` | Same as above, but smoothed in time <br/> **Earliest date available:** 2020-01-31 |
-| `raw_tests_per_device` | The average number of flu tests conducted by each testing device; measures volume of testing <br/> **Earliest date available:** 2020-01-31 |
-| `smoothed_tests_per_device` | Same as above, but smoothed in time <br/> **Earliest date available:** 2020-01-31 |
+## Flu Tests (inactive)
+These signals were updated until May 19, 2020. Documentation is still available on the [inactive Quidel page](quidel-inactive.md).

docs/api/covidcast-signals/safegraph.md

@@ -1,6 +1,6 @@
 ---
 title: USAFacts Cases and Deaths
-parent: Data Sources and Signals
+parent: Inactive Signals
 grand_parent: COVIDcast Main Endpoint
 ---
 

docs/api/covidcast-signals/usa-facts.md

@@ -73,6 +73,18 @@ def _insert_placeholder_set_four(self):
       for i in [4, 5, 6]
     ]
     self._insert_rows(rows)
+    return rows
+
+  def _insert_placeholder_set_five(self):
+    rows = [
+      self._make_placeholder_row(time_value=2000_01_01, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i)[0]
+      for i in [1, 2, 3]
+    ] + [
+      # different time_values, same issues
+      self._make_placeholder_row(time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03+i-3)[0]
+      for i in [4, 5, 6]
+    ]
+    self._insert_rows(rows)
     return rows 
 
   def test_round_trip(self):
@@ -237,6 +249,46 @@ def test_location_wildcard(self):
       'message': 'success',
     })
 
+  def test_time_values_wildcard(self):
+    """Select all time_values with a wildcard query."""
+
+    # insert placeholder data
+    rows = self._insert_placeholder_set_three()
+    expected_time_values = [
+      self.expected_from_row(r) for r in rows[:3]
+    ]
+
+    # make the request
+    response, _ = self.request_based_on_row(rows[0], time_values="*")
+
+    self.maxDiff = None
+    # assert that the right data came back
+    self.assertEqual(response, {
+      'result': 1,
+      'epidata': expected_time_values,
+      'message': 'success',
+    })
+
+  def test_issues_wildcard(self):
+    """Select all issues with a wildcard query."""
+
+    # insert placeholder data
+    rows = self._insert_placeholder_set_five()
+    expected_issues = [
+      self.expected_from_row(r) for r in rows[:3]
+    ]
+
+    # make the request
+    response, _ = self.request_based_on_row(rows[0], issues="*")
+
+    self.maxDiff = None
+    # assert that the right data came back
+    self.assertEqual(response, {
+      'result': 1,
+      'epidata': expected_issues,
+      'message': 'success',
+    })
+
   def test_signal_wildcard(self):
     """Select all signals with a wildcard query."""
 

integrations/server/test_covidcast.py

@@ -19,6 +19,7 @@ class Database:
   def __init__(self,
                connection,
                table_name=None,
+               hhs_dataset_id=None,
                columns_and_types=None,
                key_columns=None,
                additional_fields=None):
@@ -30,6 +31,8 @@ def __init__(self,
       An open connection to a database.
     table_name : str
       The name of the table which holds the dataset.
+    hhs_dataset_id : str
+      The 9-character healthdata.gov identifier for this dataset.
     columns_and_types : tuple[str, str, Callable]
       List of 3-tuples of (CSV header name, SQL column name, data type) for
       all the columns in the CSV file.
@@ -40,6 +43,7 @@ def __init__(self,
 
     self.connection = connection
     self.table_name = table_name
+    self.hhs_dataset_id = hhs_dataset_id
     self.publication_col_name = "issue" if table_name == 'covid_hosp_state_timeseries' else \
       'publication_date'
     self.columns_and_types = {
@@ -115,8 +119,8 @@ def contains_revision(self, revision):
         FROM
           `covid_hosp_meta`
         WHERE
-          `dataset_name` = %s AND `revision_timestamp` = %s
-      ''', (self.table_name, revision))
+          `hhs_dataset_id` = %s AND `revision_timestamp` = %s
+      ''', (self.hhs_dataset_id, revision))
       for (result,) in cursor:
         return bool(result)
 
@@ -138,14 +142,15 @@ def insert_metadata(self, publication_date, revision, meta_json):
         INSERT INTO
           `covid_hosp_meta` (
             `dataset_name`,
+            `hhs_dataset_id`,
             `publication_date`,
             `revision_timestamp`,
             `metadata_json`,
             `acquisition_datetime`
           )
         VALUES
-          (%s, %s, %s, %s, NOW())
-      ''', (self.table_name, publication_date, revision, meta_json))
+          (%s, %s, %s, %s, %s, NOW())
+      ''', (self.table_name, self.hhs_dataset_id, publication_date, revision, meta_json))
 
   def insert_dataset(self, publication_date, dataframe):
     """Add a dataset to the database.
@@ -232,7 +237,7 @@ def get_max_issue(self):
         from
           `covid_hosp_meta`
         WHERE
-          dataset_name = "{self.table_name}"
+          hhs_dataset_id = "{self.hhs_dataset_id}"
       ''')
       for (result,) in cursor:
         if result is not None:

src/acquisition/covid_hosp/common/database.py

@@ -169,19 +169,20 @@ def update_dataset(database, network, newer_than=None, older_than=None):
       # download the dataset and add it to the database
       dataset = Utils.merge_by_key_cols([network.fetch_dataset(url) for url, _ in revisions],
                                         db.KEY_COLS)
-      # add metadata to the database using the last revision seen.
-      last_url, last_index = revisions[-1]
-      metadata_json = metadata.loc[last_index].reset_index().to_json()
+      # add metadata to the database
+      all_metadata = []
+      for url, index in revisions:
+        all_metadata.append((url, metadata.loc[index].reset_index().to_json()))
       datasets.append((
         issue_int,
         dataset,
-        last_url,
-        metadata_json
+        all_metadata
       ))
     with database.connect() as db:
-      for issue_int, dataset, last_url, metadata_json in datasets:
+      for issue_int, dataset, all_metadata in datasets:
         db.insert_dataset(issue_int, dataset)
-        db.insert_metadata(issue_int, last_url, metadata_json)
+        for url, metadata_json in all_metadata:
+          db.insert_metadata(issue_int, url, metadata_json)
         print(f'successfully acquired {len(dataset)} rows')
 
       # note that the transaction is committed by exiting the `with` block

src/acquisition/covid_hosp/common/utils.py

@@ -2,6 +2,7 @@
 from delphi.epidata.acquisition.covid_hosp.common.database import Database as BaseDatabase
 from delphi.epidata.acquisition.covid_hosp.common.database import Columndef
 from delphi.epidata.acquisition.covid_hosp.common.utils import Utils
+from delphi.epidata.acquisition.covid_hosp.facility.network import Network
 
 
 class Database(BaseDatabase):
@@ -213,5 +214,6 @@ def __init__(self, *args, **kwargs):
         *args,
         **kwargs,
         table_name=Database.TABLE_NAME,
+        hhs_dataset_id=Network.DATASET_ID,
         key_columns=Database.KEY_COLS,
         columns_and_types=Database.ORDERED_CSV_COLUMNS)

src/acquisition/covid_hosp/facility/database.py

@@ -2,6 +2,7 @@
 from delphi.epidata.acquisition.covid_hosp.common.database import Database as BaseDatabase
 from delphi.epidata.acquisition.covid_hosp.common.database import Columndef
 from delphi.epidata.acquisition.covid_hosp.common.utils import Utils
+from delphi.epidata.acquisition.covid_hosp.state_daily.network import Network
 
 
 class Database(BaseDatabase):
@@ -223,6 +224,7 @@ def __init__(self, *args, **kwargs):
         *args,
         **kwargs,
         table_name=Database.TABLE_NAME,
+        hhs_dataset_id=Network.DATASET_ID,
         columns_and_types=Database.ORDERED_CSV_COLUMNS,
         key_columns=Database.KEY_COLS,
         additional_fields=[Columndef('D', 'record_type', None)])