From 983fed9229d3a0004d81de984efeecc8416f40f7 Mon Sep 17 00:00:00 2001 From: undefx Date: Sun, 19 Apr 2020 19:59:46 -0500 Subject: [PATCH 1/4] fix formatting --- integrations/server/test_covidcast_meta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 255b1ff33..905eb8149 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -43,8 +43,8 @@ def test_round_trip(self): # insert dummy data and accumulate expected results (in sort order) template = ''' - insert into covidcast values - (0, "%s", "%s", "%s", "%s", %d, "%s", %d, 0, 0, 0) + insert into covidcast values + (0, "%s", "%s", "%s", "%s", %d, "%s", %d, 0, 0, 0) ''' expected = [] for src in ('src1', 'src2'): From 62fa289977b63e8264f97132ec3196520702ea2f Mon Sep 17 00:00:00 2001 From: undefx Date: Sun, 19 Apr 2020 22:54:47 -0500 Subject: [PATCH 2/4] update signal name - change `fb_survey` to `fb-survey` - format spacing for consistency - remove unused if-main --- integrations/client/test_delphi_epidata.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 27ce566a4..b11c4e676 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -1,4 +1,6 @@ """Integration tests for delphi_epidata.py.""" + +# standard library import unittest # first party @@ -7,6 +9,7 @@ # py3tester coverage target __test_target__ = 'delphi.epidata.client.delphi_epidata' + class DelphiEpidataPythonClientTests(unittest.TestCase): """Tests the Python client.""" @@ -14,7 +17,14 @@ def test_covidcast_happy_path(self): """Test that the covidcast endpoint returns expected data.""" # Fetch data - res = Epidata.covidcast('fb_survey', 'ili', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001') + res = Epidata.covidcast( + 'fb-survey', + 'ili', + 'day', + 'county', + [20200401, Epidata.range(20200405, 20200414)], + '06001') + # Check result self.assertEqual(res['result'], 1) self.assertEqual(res['message'], 'success') @@ -27,12 +37,12 @@ def test_covidcast_happy_path(self): self.assertIn('stderr', item) self.assertIn('sample_size', item) - def test_covidcast_meta(self): """Test that the covidcast_meta endpoint returns expected data.""" # Fetch data res = Epidata.covidcast_meta() + # Check result self.assertEqual(res['result'], 1) self.assertEqual(res['message'], 'success') @@ -44,6 +54,3 @@ def test_covidcast_meta(self): self.assertIn('min_time', item) self.assertIn('max_time', item) self.assertIn('num_locations', item) - -if __name__ == '__main__': - unittest.main() From 37968f19778ff39604d5899bffbb081194bcbc8c Mon Sep 17 00:00:00 2001 From: undefx Date: Mon, 20 Apr 2020 14:23:50 -0500 Subject: [PATCH 3/4] client integration test uses local servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fixes #56 - point the python client at the local docker servers instead of prod - whitespace tweaks in client and test - unit: ✔ All 27 tests passed! - integration: existing tests pass --- integrations/client/test_delphi_epidata.py | 133 ++++++++++++++------- src/client/delphi_epidata.py | 2 +- 2 files changed, 91 insertions(+), 44 deletions(-) diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index b11c4e676..abb3d03c9 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -3,6 +3,9 @@ # standard library import unittest +# third party +import mysql.connector + # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -11,46 +14,90 @@ class DelphiEpidataPythonClientTests(unittest.TestCase): - """Tests the Python client.""" - - def test_covidcast_happy_path(self): - """Test that the covidcast endpoint returns expected data.""" - - # Fetch data - res = Epidata.covidcast( - 'fb-survey', - 'ili', - 'day', - 'county', - [20200401, Epidata.range(20200405, 20200414)], - '06001') - - # Check result - self.assertEqual(res['result'], 1) - self.assertEqual(res['message'], 'success') - self.assertGreater(len(res['epidata']), 0) - item = res['epidata'][0] - self.assertIn('geo_value', item) - self.assertIn('time_value', item) - self.assertIn('direction', item) - self.assertIn('value', item) - self.assertIn('stderr', item) - self.assertIn('sample_size', item) - - def test_covidcast_meta(self): - """Test that the covidcast_meta endpoint returns expected data.""" - - # Fetch data - res = Epidata.covidcast_meta() - - # Check result - self.assertEqual(res['result'], 1) - self.assertEqual(res['message'], 'success') - self.assertGreater(len(res['epidata']), 0) - item = res['epidata'][0] - self.assertIn('data_source', item) - self.assertIn('time_type', item) - self.assertIn('geo_type', item) - self.assertIn('min_time', item) - self.assertIn('max_time', item) - self.assertIn('num_locations', item) + """Tests the Python client.""" + + def setUp(self): + """Perform per-test setup.""" + + # connect to the `epidata` database and clear relevant tables + cnx = mysql.connector.connect( + user='user', + password='pass', + host='delphi_database_epidata', + database='epidata') + cur = cnx.cursor() + cur.execute('truncate table covidcast') + cnx.commit() + cur.close() + + # make connection and cursor available to test cases + self.cnx = cnx + self.cur = cnx.cursor() + + # use the local instance of the Epidata API + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + + def tearDown(self): + """Perform per-test teardown.""" + self.cur.close() + self.cnx.close() + + def test_covidcast(self): + """Test that the covidcast endpoint returns expected data.""" + + # insert dummy data + self.cur.execute(''' + insert into covidcast values + (0, 'src', 'sig', 'day', 'county', 20200414, '01234', 1.5, 2.5, 3.5, 4) + ''') + self.cnx.commit() + + # fetch data + response = Epidata.covidcast( + 'src', 'sig', 'day', 'county', 20200414, '01234') + + # check result + self.assertEqual(response, { + 'result': 1, + 'epidata': [{ + 'time_value': 20200414, + 'geo_value': '01234', + 'value': 1.5, + 'stderr': 2.5, + 'sample_size': 3.5, + 'direction': 4, + }], + 'message': 'success', + }) + + def test_covidcast_meta(self): + """Test that the covidcast_meta endpoint returns expected data.""" + + # insert dummy data + self.cur.execute(''' + insert into covidcast values + (0, 'src', 'sig', 'day', 'county', 20200414, '01234', 1.5, 2.5, 3.5, 4) + ''') + self.cnx.commit() + + # fetch data + response = Epidata.covidcast_meta() + + # check result + self.assertEqual(response, { + 'result': 1, + 'epidata': [{ + 'data_source': 'src', + 'signal': 'sig', + 'time_type': 'day', + 'geo_type': 'county', + 'min_time': 20200414, + 'max_time': 20200414, + 'num_locations': 1, + 'min_value': 1.5, + 'max_value': 1.5, + 'mean_value': 1.5, + 'stdev_value': 0, + }], + 'message': 'success', + }) diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index 0222a389b..b342292f2 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -87,7 +87,7 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): params['auth'] = auth # Make the API call return Epidata._request(params) - + # Fetch FluView clinical data @staticmethod def fluview_clinical(regions, epiweeks, issues=None, lag=None): From a0bb5e6c4f4e5713f0babcc5a6616fe78b891f6b Mon Sep 17 00:00:00 2001 From: undefx Date: Mon, 20 Apr 2020 14:29:41 -0500 Subject: [PATCH 4/4] Merge branch 'master' of https://github.com/dfarrow0/delphi-epidata --- docs/api/covidcast.md | 68 +++++++++++++++++++++-------- docs/api/covidcast_meta.md | 16 ++++--- src/client/delphi_epidata.R | 2 +- src/client/delphi_epidata.coffee | 2 +- src/client/delphi_epidata.js | 2 +- src/client/delphi_epidata.py | 2 +- src/client/packaging/pypi/README.md | 2 +- 7 files changed, 67 insertions(+), 27 deletions(-) diff --git a/docs/api/covidcast.md b/docs/api/covidcast.md index dcd596bf3..59099b0c2 100644 --- a/docs/api/covidcast.md +++ b/docs/api/covidcast.md @@ -10,7 +10,34 @@ General topics not specific to any particular data source are discussed in the ## Delphi's COVID-19 Surveillance Streams Data -... +Delphi's COVID-19 Surveillance Streams data includes the following data sources: +* `fb-survey`: Data signal based on CMU-run symptom surveys, advertised through Facebook. These surveys are +voluntary, and no individual survey responses are shared back to Facebook. Using this survey data, we estimate the +percentage of people in a given location, on a given day that have CLI (covid-like illness = fever, along with cough, +or shortness of breath, or difficulty breathing), and separately, that have ILI (influenza-like illness = fever, +along with cough or sore throat). +* `google-survey`: Data signal based on Google-run symptom surveys, through their Opinions Reward app (and similar +applications). These surveys are again voluntary. They are just one question long, and ask "Do you know someone +in your community who is sick (fever, along with cough, or shortness of breath, or difficulty breathing) right now?" +Using this survey data, we estimate the percentage of people in a given location, on a given day, that know somebody +who has CLI (covid-like illness = fever, along with cough, or shortness of breath, or difficulty breathing). +Note that this is tracking a different quantity than the surveys through Facebook, and (unsurprisingly) the estimates +here tend to be much larger. +* `quidel`: Data signal based on flu lab tests, provided to us by Quidel, Inc. When a patient (whether at a doctor’s +office, clinic, or hospital) has covid-like symptoms, standard practice currently is to perform a flu test to rule +out seasonal flu (influenza), because these two diseases have similar symptoms. Using this lab test data, we estimate +the percentage of flu tests that came back negative among all those that were performed in a given geography (state, +metro-area or county) and on a given day. While many of these patients may not have covid, the fraction of tests +negative for flu is positively correlated with the prevalence of covid. +* `ght`: Data signal based on Google searches, provided to us by Google Health Trends. Using this search data, we +estimate the volume of covid-related searches in a given location, on a given day. This signal is measured in +arbitrary units (its scale is meaningless). +* `doctor-visits`: Data based on outpatient visits, provided to us by a national health system. Using this outpatient +data, we estimate the percentage of covid-related doctor's visits in a given location, on a given day. + +The data is expected to be updated daily. You can use the [`covidcast_meta`](covidcast_meta.md) endpoint to get +summary information about the ranges of the different attributes for the different data sources currently in the data. + # The API @@ -24,13 +51,20 @@ See [this documentation](README.md) for details on specifying epiweeks, dates, a | Parameter | Description | Type | | --- | --- | --- | -| `data_source` | name of upstream data souce | string | -| `signal` | name of signal derived from upstream data | string | +| `data_source` | name of upstream data source (e.g., `fb-survey`, `google-survey`, `ght`, `quidel`, `doctor-visits`) | string | +| `signal` | name of signal derived from upstream data (see notes below) | string | | `time_type` | temporal resolution of the signal (e.g., `day`, `week`) | string | | `geo_type` | spatial resolution of the signal (e.g., `county`, `hrr`, `msa`, `dma`, `state`) | string | | `time_values` | time unit (e.g., date) over which underlying events happened | `list` of time values (e.g., 20200401) | | `geo_value` | unique code for each location, depending on `geo_type` (county -> FIPS 6-4 code, HRR -> HRR number, MSA -> CBSA code, DMA -> DMA code, state -> two-letter [state](../../labels/states.txt) code), or `*` for all | string | +Notes: +* `fb-survey` `signal` values include `cli`, `ili`, and `wili`. +* `google-survey` `signal` values include `cli`. +* `ght` `signal` values include `rawsearch` and `smoothedsearch`. +* `quidel` `signal` values include `raw_negativeprop` and `smooth_negativeprop`. +* `doctor-visits` `signal` values include `cli`. + ## Response | Field | Description | Type | @@ -47,8 +81,8 @@ See [this documentation](README.md) for details on specifying epiweeks, dates, a # Example URLs -### Delphi's COVID-19 Surveillance Streams from Facebook Survey ILI on 2020-04-06 - 2010-04-10 (county 06001) -https://delphi.cmu.edu/epidata/api.php?source=covidcast&data_source=fb_survey&signal=ili&time_type=day&geo_type=county&time_values=20200406-20200410&geo_value=06001 +### Delphi's COVID-19 Surveillance Streams from Facebook Survey CLI on 2020-04-06 - 2010-04-10 (county 06001) +https://delphi.cmu.edu/epidata/api.php?source=covidcast&data_source=fb-survey&signal=cli&time_type=day&geo_type=county&time_values=20200406-20200410&geo_value=06001 ```json { @@ -56,11 +90,11 @@ https://delphi.cmu.edu/epidata/api.php?source=covidcast&data_source=fb_survey&si "epidata": [ { "geo_value": "06001", - "time_value": 20200406, - "direction": 0, - "value": 0, - "stderr": null, - "sample_size": 417.2392 + "time_value": 20200407, + "direction": null, + "value": 1.1293550689064, + "stderr": 0.53185454111042, + "sample_size": 281.0245 }, ... ], @@ -68,13 +102,13 @@ https://delphi.cmu.edu/epidata/api.php?source=covidcast&data_source=fb_survey&si } ``` -## Delphi's COVID-19 Surveillance Streams from Facebook Survey ILI on 2020-04-06 (all counties) -https://delphi.cmu.edu/epidata/api.php?source=covidcast&data_source=fb_survey&signal=ili&time_type=day&geo_type=county&time_values=20200406&geo_value=* +## Delphi's COVID-19 Surveillance Streams from Facebook Survey CLI on 2020-04-06 (all counties) +https://delphi.cmu.edu/epidata/api.php?source=covidcast&data_source=fb-survey&signal=cli&time_type=day&geo_type=county&time_values=20200406&geo_value=* # Code Samples Libraries are available for [CoffeeScript](../../src/client/delphi_epidata.coffee), [JavaScript](../../src/client/delphi_epidata.js), [Python](../../src/client/delphi_epidata.py), and [R](../../src/client/delphi_epidata.R). -The following samples show how to import the library and fetch Delphi's COVID-19 Surveillance Streams from Facebook Survey ILI for county 06001 and days `20200401` and `20200405-20200414` (11 days total). +The following samples show how to import the library and fetch Delphi's COVID-19 Surveillance Streams from Facebook Survey CLI for county 06001 and days `20200401` and `20200405-20200414` (11 days total). ### CoffeeScript (in Node.js) @@ -84,7 +118,7 @@ The following samples show how to import the library and fetch Delphi's COVID-19 # Fetch data callback = (result, message, epidata) -> console.log(result, message, epidata?.length) -Epidata.covidcast(callback, 'fb_survey', 'ili', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001') +Epidata.covidcast(callback, 'fb-survey', 'cli', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001') ```` ### JavaScript (in a web browser) @@ -98,7 +132,7 @@ Epidata.covidcast(callback, 'fb_survey', 'ili', 'day', 'county', [20200401, Epid var callback = function(result, message, epidata) { console.log(result, message, epidata != null ? epidata.length : void 0); }; - Epidata.covidcast(callback, 'fb_survey', 'ili', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001'); + Epidata.covidcast(callback, 'fb-survey', 'cli', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001'); ```` @@ -115,7 +149,7 @@ Otherwise, place `delphi_epidata.py` from this repo next to your python script. # Import from delphi_epidata import Epidata # Fetch data -res = Epidata.covidcast('fb_survey', 'ili', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001') +res = Epidata.covidcast('fb-survey', 'cli', 'day', 'county', [20200401, Epidata.range(20200405, 20200414)], '06001') print(res['result'], res['message'], len(res['epidata'])) ```` @@ -125,6 +159,6 @@ print(res['result'], res['message'], len(res['epidata'])) # Import source('delphi_epidata.R') # Fetch data -res <- Epidata$covidcast('fb_survey', 'ili', 'day', 'county', list(20200401, Epidata$range(20200405, 20200414)), '06001') +res <- Epidata$covidcast('fb-survey', 'cli', 'day', 'county', list(20200401, Epidata$range(20200405, 20200414)), '06001') cat(paste(res$result, res$message, length(res$epidata), "\n")) ```` diff --git a/docs/api/covidcast_meta.md b/docs/api/covidcast_meta.md index 970a067d0..20fb87bb2 100644 --- a/docs/api/covidcast_meta.md +++ b/docs/api/covidcast_meta.md @@ -34,6 +34,10 @@ None. | `epidata[].min_time` | minimum time (e.g., 20200406) | integer | | `epidata[].max_time` | maximum time (e.g., 20200413) | integer | | `epidata[].num_locations` | number of locations | integer | +| `epidata[].min_value` | minimum value | float | +| `epidata[].max_value` | maximum value | float | +| `epidata[].mean_value` | mean of value | float | +| `epidata[].stdev_value` | standard deviation of value | float | | `message` | `success` or error message | string | # Example URLs @@ -45,15 +49,17 @@ https://delphi.cmu.edu/epidata/api.php?source=covidcast_meta "result": 1, "epidata": [ { - "data_source": "fb_survey", + "data_source": "doctor-visits", "signal": "cli", "time_type": "day", "geo_type": "county", - "min_time": 20200406, - "max_time": 20200413, - "num_locations": 555, + "min_time": 20200201, + "max_time": 20200418, + "num_locations": 1411, "min_value": 0, - "max_value": 5.0805650596568 + "max_value": 23.079023, + "mean_value": 0.42745842933726, + "stdev_value": 0.96461526722895 }, ... ], diff --git a/src/client/delphi_epidata.R b/src/client/delphi_epidata.R index 94b8a6362..d892e1e77 100644 --- a/src/client/delphi_epidata.R +++ b/src/client/delphi_epidata.R @@ -13,7 +13,7 @@ library(httr) Epidata <- (function() { # API base url - BASE_URL <- 'https://delphi.midas.cs.cmu.edu/epidata/api.php' + BASE_URL <- 'https://delphi.cmu.edu/epidata/api.php' # Helper function to cast values and/or ranges to strings .listitem <- function(value) { diff --git a/src/client/delphi_epidata.coffee b/src/client/delphi_epidata.coffee index 62eccff90..81bcc0950 100644 --- a/src/client/delphi_epidata.coffee +++ b/src/client/delphi_epidata.coffee @@ -18,7 +18,7 @@ unless $?.getJSON? class Epidata # API base url - BASE_URL = 'https://delphi.midas.cs.cmu.edu/epidata/api.php' + BASE_URL = 'https://delphi.cmu.edu/epidata/api.php' # Helper function to cast values and/or ranges to strings _listitem = (value) -> diff --git a/src/client/delphi_epidata.js b/src/client/delphi_epidata.js index b5ec380cf..d02362f79 100644 --- a/src/client/delphi_epidata.js +++ b/src/client/delphi_epidata.js @@ -23,7 +23,7 @@ Notes: function Epidata() {} - BASE_URL = 'https://delphi.midas.cs.cmu.edu/epidata/api.php'; + BASE_URL = 'https://delphi.cmu.edu/epidata/api.php'; _listitem = function(value) { if (value.hasOwnProperty('from') && value.hasOwnProperty('to')) { diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index b342292f2..3cd0af5a7 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -17,7 +17,7 @@ class Epidata: """An interface to DELPHI's Epidata API.""" # API base url - BASE_URL = 'https://delphi.midas.cs.cmu.edu/epidata/api.php' + BASE_URL = 'https://delphi.cmu.edu/epidata/api.php' # Helper function to cast values and/or ranges to strings @staticmethod diff --git a/src/client/packaging/pypi/README.md b/src/client/packaging/pypi/README.md index ccdc90379..86396a3e4 100644 --- a/src/client/packaging/pypi/README.md +++ b/src/client/packaging/pypi/README.md @@ -1,6 +1,6 @@ # Delphi Epidata API Client This package provides a programmatic interface to -[Delphi](https://delphi.midas.cs.cmu.edu/)'s epidemiological data ("epidata") +[Delphi](https://delphi.cmu.edu/)'s epidemiological data ("epidata") API. Source code and usage information can be found at [https://github.com/cmu-delphi/delphi-epidata](https://github.com/cmu-delphi/delphi-epidata).