diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 99008a0f1..b435b2b7c 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -10,9 +10,9 @@ # first party from delphi_utils import Nans -from delphi.epidata.client.delphi_epidata import Epidata import delphi.operations.secrets as secrets -import delphi.epidata.acquisition.covidcast.database as live +from delphi.epidata.client.delphi_epidata import Epidata +from delphi.epidata.acquisition.covidcast.database_meta import DatabaseMeta from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main # py3tester coverage target (equivalent to `import *`) @@ -97,7 +97,7 @@ def test_caching(self): self.cnx.commit() # make sure the live utility is serving something sensible - cvc_database = live.Database() + cvc_database = DatabaseMeta() cvc_database.connect() epidata1 = cvc_database.compute_covidcast_meta() cvc_database.disconnect(False) diff --git a/integrations/acquisition/covidcast/test_db.py b/integrations/acquisition/covidcast/test_db.py index 3cd7e91a7..5daf8d272 100644 --- a/integrations/acquisition/covidcast/test_db.py +++ b/integrations/acquisition/covidcast/test_db.py @@ -1,10 +1,11 @@ -import unittest - from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow, DBLoadStateException + +from delphi.epidata.acquisition.covidcast.database import DBLoadStateException +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase import delphi.operations.secrets as secrets + # all the Nans we use here are just one value, so this is a shortcut to it: nmv = Nans.NOT_MISSING.value @@ -31,8 +32,8 @@ def _find_matches_for_row(self, row): def test_insert_or_update_with_nonempty_load_table(self): # make rows - a_row = self._make_placeholder_row()[0] - another_row = self._make_placeholder_row(time_value=self.DEFAULT_TIME_VALUE+1, issue=self.DEFAULT_ISSUE+1)[0] + a_row = CovidcastRow(time_value=20200202) + another_row = CovidcastRow(time_value=20200203, issue=20200203) # insert one self._db.insert_or_update_bulk([a_row]) # put something into the load table @@ -61,7 +62,7 @@ def test_id_sync(self): latest_view = 'epimetric_latest_v' # add a data point - base_row, _ = self._make_placeholder_row() + base_row = CovidcastRow() self._insert_rows([base_row]) # ensure the primary keys match in the latest and history tables matches = self._find_matches_for_row(base_row) @@ -71,7 +72,7 @@ def test_id_sync(self): old_pk_id = matches[latest_view][pk_column] # add a reissue for said data point - next_row, _ = self._make_placeholder_row() + next_row = CovidcastRow() next_row.issue += 1 self._insert_rows([next_row]) # ensure the new keys also match diff --git a/integrations/acquisition/covidcast/test_delete_batch.py b/integrations/acquisition/covidcast/test_delete_batch.py index 915c9341b..15ae7e2e2 100644 --- a/integrations/acquisition/covidcast/test_delete_batch.py +++ b/integrations/acquisition/covidcast/test_delete_batch.py @@ -5,13 +5,10 @@ import unittest from os import path -# third party -import mysql.connector - # first party -from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow import delphi.operations.secrets as secrets +from delphi.epidata.acquisition.covidcast.database import Database +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow # py3tester coverage target (equivalent to `import *`) __test_target__ = 'delphi.epidata.acquisition.covidcast.database' diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 625d2859d..cfeb83bd4 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -1,26 +1,28 @@ """Integration tests for delphi_epidata.py.""" # standard library -import unittest import time -from unittest.mock import patch, MagicMock from json import JSONDecodeError +from unittest.mock import MagicMock, patch -# third party -from aiohttp.client_exceptions import ClientResponseError -import mysql.connector +# first party import pytest +from aiohttp.client_exceptions import ClientResponseError -# first party -from delphi_utils import Nans -from delphi.epidata.client.delphi_epidata import Epidata -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow +# third party +import delphi.operations.secrets as secrets from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_covidcast_meta_cache +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase -import delphi.operations.secrets as secrets +from delphi.epidata.client.delphi_epidata import Epidata +from delphi_utils import Nans + # py3tester coverage target __test_target__ = 'delphi.epidata.client.delphi_epidata' +# all the Nans we use here are just one value, so this is a shortcut to it: +nmv = Nans.NOT_MISSING.value +IGNORE_FIELDS = ["id", "direction_updated_timestamp", "value_updated_timestamp", "source", "time_type", "geo_type"] def fake_epidata_endpoint(func): """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses.""" @@ -30,9 +32,6 @@ def wrapper(*args): Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' return wrapper -# all the Nans we use here are just one value, so this is a shortcut to it: -nmv = Nans.NOT_MISSING.value - class DelphiEpidataPythonClientTests(CovidcastBase): """Tests the Python client.""" @@ -54,12 +53,12 @@ def test_covidcast(self): # insert placeholder data: three issues of one signal, one issue of another rows = [ - self._make_placeholder_row(issue=self.DEFAULT_ISSUE + i, value=i, lag=i)[0] + CovidcastRow(issue=20200202 + i, value=i, lag=i) for i in range(3) ] row_latest_issue = rows[-1] rows.append( - self._make_placeholder_row(signal="sig2")[0] + CovidcastRow(signal="sig2") ) self._insert_rows(rows) @@ -70,10 +69,11 @@ def test_covidcast(self): ) expected = [ - self.expected_from_row(row_latest_issue), - self.expected_from_row(rows[-1]) + row_latest_issue.as_dict(ignore_fields=IGNORE_FIELDS), + rows[-1].as_dict(ignore_fields=IGNORE_FIELDS) ] + self.assertEqual(response['epidata'], expected) # check result self.assertEqual(response, { 'result': 1, @@ -89,10 +89,10 @@ def test_covidcast(self): expected = [{ rows[0].signal: [ - self.expected_from_row(row_latest_issue, self.DEFAULT_MINUS + ['signal']), + row_latest_issue.as_dict(ignore_fields=IGNORE_FIELDS + ['signal']), ], rows[-1].signal: [ - self.expected_from_row(rows[-1], self.DEFAULT_MINUS + ['signal']), + rows[-1].as_dict(ignore_fields=IGNORE_FIELDS + ['signal']), ], }] @@ -109,12 +109,12 @@ def test_covidcast(self): **self.params_from_row(rows[0]) ) - expected = self.expected_from_row(row_latest_issue) + expected = [row_latest_issue.as_dict(ignore_fields=IGNORE_FIELDS)] # check result self.assertEqual(response_1, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) @@ -124,13 +124,13 @@ def test_covidcast(self): **self.params_from_row(rows[0], as_of=rows[1].issue) ) - expected = self.expected_from_row(rows[1]) + expected = [rows[1].as_dict(ignore_fields=IGNORE_FIELDS)] # check result self.maxDiff=None self.assertEqual(response_1a, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) @@ -141,8 +141,8 @@ def test_covidcast(self): ) expected = [ - self.expected_from_row(rows[0]), - self.expected_from_row(rows[1]) + rows[0].as_dict(ignore_fields=IGNORE_FIELDS), + rows[1].as_dict(ignore_fields=IGNORE_FIELDS) ] # check result @@ -158,12 +158,12 @@ def test_covidcast(self): **self.params_from_row(rows[0], lag=2) ) - expected = self.expected_from_row(row_latest_issue) + expected = [row_latest_issue.as_dict(ignore_fields=IGNORE_FIELDS)] # check result self.assertDictEqual(response_3, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) with self.subTest(name='long request'): @@ -223,16 +223,16 @@ def test_geo_value(self): # insert placeholder data: three counties, three MSAs N = 3 rows = [ - self._make_placeholder_row(geo_type="county", geo_value=str(i)*5, value=i)[0] + CovidcastRow(geo_type="county", geo_value=str(i)*5, value=i) for i in range(N) ] + [ - self._make_placeholder_row(geo_type="msa", geo_value=str(i)*5, value=i*10)[0] + CovidcastRow(geo_type="msa", geo_value=str(i)*5, value=i*10) for i in range(N) ] self._insert_rows(rows) counties = [ - self.expected_from_row(rows[i]) for i in range(N) + rows[i].as_dict(ignore_fields=IGNORE_FIELDS) for i in range(N) ] def fetch(geo): @@ -241,31 +241,31 @@ def fetch(geo): ) # test fetch all - r = fetch('*') - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], counties) + request = fetch('*') + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], counties) # test fetch a specific region - r = fetch('11111') - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [counties[1]]) + request = fetch('11111') + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], [counties[1]]) # test fetch a specific yet not existing region - r = fetch('55555') - self.assertEqual(r['message'], 'no results') + request = fetch('55555') + self.assertEqual(request['message'], 'no results') # test fetch a multiple regions - r = fetch(['11111', '22222']) - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [counties[1], counties[2]]) + request = fetch(['11111', '22222']) + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], [counties[1], counties[2]]) # test fetch a multiple regions in another variant - r = fetch(['00000', '22222']) - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [counties[0], counties[2]]) + request = fetch(['00000', '22222']) + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], [counties[0], counties[2]]) # test fetch a multiple regions but one is not existing - r = fetch(['11111', '55555']) - self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [counties[1]]) + request = fetch(['11111', '55555']) + self.assertEqual(request['message'], 'success') + self.assertEqual(request['epidata'], [counties[1]]) # test fetch a multiple regions but specify no region - r = fetch([]) - self.assertEqual(r['message'], 'no results') + request = fetch([]) + self.assertEqual(request['message'], 'no results') def test_covidcast_meta(self): """Test that the covidcast_meta endpoint returns expected data.""" @@ -275,7 +275,7 @@ def test_covidcast_meta(self): # 2nd issue: 1 11 21 # 3rd issue: 2 12 22 rows = [ - self._make_placeholder_row(time_value=self.DEFAULT_TIME_VALUE + t, issue=self.DEFAULT_ISSUE + i, value=t*10 + i)[0] + CovidcastRow(time_value=2020_02_02 + t, issue=2020_02_02 + i, value=t*10 + i) for i in range(3) for t in range(3) ] self._insert_rows(rows) @@ -299,14 +299,14 @@ def test_covidcast_meta(self): signal=rows[0].signal, time_type=rows[0].time_type, geo_type=rows[0].geo_type, - min_time=self.DEFAULT_TIME_VALUE, - max_time=self.DEFAULT_TIME_VALUE + 2, + min_time=2020_02_02, + max_time=2020_02_02 + 2, num_locations=1, min_value=2., mean_value=12., max_value=22., stdev_value=8.1649658, # population stdev, not sample, which is 10. - max_issue=self.DEFAULT_ISSUE + 2, + max_issue=2020_02_02 + 2, min_lag=0, max_lag=0, # we didn't set lag when inputting data ) @@ -322,10 +322,10 @@ def test_async_epidata(self): # insert placeholder data: three counties, three MSAs N = 3 rows = [ - self._make_placeholder_row(geo_type="county", geo_value=str(i)*5, value=i)[0] + CovidcastRow(geo_type="county", geo_value=str(i)*5, value=i) for i in range(N) ] + [ - self._make_placeholder_row(geo_type="msa", geo_value=str(i)*5, value=i*10)[0] + CovidcastRow(geo_type="msa", geo_value=str(i)*5, value=i*10) for i in range(N) ] self._insert_rows(rows) diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 86ce0c53d..c003f8d07 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -1,7 +1,7 @@ """Integration tests for the `covidcast` endpoint.""" # standard library -import json +from typing import Callable import unittest # third party @@ -10,12 +10,13 @@ # first party from delphi_utils import Nans +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase # use the local instance of the Epidata API +# TODO: should we still be using this? BASE_URL = 'http://delphi_web_epidata/epidata/api.php' - - +IGNORE_FIELDS = ["id", "direction_updated_timestamp", "value_updated_timestamp", "source", "time_type", "geo_type"] class CovidcastTests(CovidcastBase): """Tests the `covidcast` endpoint.""" @@ -24,28 +25,26 @@ def localSetUp(self): """Perform per-test setup.""" self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - def request_based_on_row(self, row, extract_response=lambda x: x.json(), **kwargs): + def request_based_on_row(self, row: CovidcastRow, extract_response: Callable = lambda x: x.json(), **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) response = requests.get(BASE_URL, params=params) response.raise_for_status() response = extract_response(response) - expected = self.expected_from_row(row) - - return response, expected + return response def _insert_placeholder_set_one(self): - row, settings = self._make_placeholder_row() + row = CovidcastRow() self._insert_rows([row]) return row def _insert_placeholder_set_two(self): rows = [ - self._make_placeholder_row(geo_type='county', geo_value=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0] + CovidcastRow(geo_type='county', geo_value=str(i)*5, value=i*1., stderr=i*10., sample_size=i*100.) for i in [1, 2, 3] ] + [ # geo value intended to overlap with counties above - self._make_placeholder_row(geo_type='msa', geo_value=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.)[0] + CovidcastRow(geo_type='msa', geo_value=str(i-3)*5, value=i*1., stderr=i*10., sample_size=i*100.) for i in [4, 5, 6] ] self._insert_rows(rows) @@ -53,11 +52,11 @@ def _insert_placeholder_set_two(self): def _insert_placeholder_set_three(self): rows = [ - self._make_placeholder_row(geo_type='county', geo_value='11111', time_value=2000_01_01+i, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=2-i)[0] + CovidcastRow(geo_type='county', geo_value='11111', time_value=2000_01_01+i, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=2-i) for i in [1, 2, 3] ] + [ # time value intended to overlap with 11111 above, with disjoint geo values - self._make_placeholder_row(geo_type='county', geo_value=str(i)*5, time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=5-i)[0] + CovidcastRow(geo_type='county', geo_value=str(i)*5, time_value=2000_01_01+i-3, value=i*1., stderr=i*10., sample_size=i*100., issue=2000_01_03, lag=5-i) for i in [4, 5, 6] ] self._insert_rows(rows) @@ -70,10 +69,13 @@ def test_round_trip(self): row = self._insert_placeholder_set_one() # make the request - response, expected = self.request_based_on_row(row) + response = self.request_based_on_row(row) + + expected = [row.as_dict(ignore_fields=IGNORE_FIELDS)] + self.assertEqual(response, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) @@ -130,32 +132,25 @@ def test_csv_format(self): # make the request # NB 'format' is a Python reserved word - response, _ = self.request_based_on_row( + response = self.request_based_on_row( row, extract_response=lambda resp: resp.text, **{'format':'csv'} ) - expected_response = ( - "geo_value,signal,time_value,direction,issue,lag,missing_value," + - "missing_stderr,missing_sample_size,value,stderr,sample_size\n" + - ",".join("" if x is None else str(x) for x in [ - row.geo_value, - row.signal, - row.time_value, - row.direction, - row.issue, - row.lag, - row.missing_value, - row.missing_stderr, - row.missing_sample_size, - row.value, - row.stderr, - row.sample_size - ]) + "\n" + + # TODO: This is a mess because of api.php. + column_order = [ + "geo_value", "signal", "time_value", "direction", "issue", "lag", "missing_value", + "missing_stderr", "missing_sample_size", "value", "stderr", "sample_size" + ] + expected = ( + row.api_compatibility_row_df + .assign(direction = None) + .to_csv(columns=column_order, index=False) ) # assert that the right data came back - self.assertEqual(response, expected_response) + self.assertEqual(response, expected) def test_raw_json_format(self): """Test generate raw json data.""" @@ -164,10 +159,12 @@ def test_raw_json_format(self): row = self._insert_placeholder_set_one() # make the request - response, expected = self.request_based_on_row(row, **{'format':'json'}) + response = self.request_based_on_row(row, **{'format':'json'}) + + expected = [row.as_dict(ignore_fields=IGNORE_FIELDS)] # assert that the right data came back - self.assertEqual(response, [expected]) + self.assertEqual(response, expected) def test_fields(self): """Test fields parameter""" @@ -176,7 +173,9 @@ def test_fields(self): row = self._insert_placeholder_set_one() # limit fields - response, expected = self.request_based_on_row(row, fields='time_value,geo_value') + response = self.request_based_on_row(row, fields='time_value,geo_value') + + expected = row.as_dict(ignore_fields=IGNORE_FIELDS) expected_all = { 'result': 1, 'epidata': [{ @@ -189,15 +188,14 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response, _ = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) # limit exclude fields: exclude all except time_value and geo_value - - response, _ = self.request_based_on_row(row, fields=( + response = self.request_based_on_row(row, fields=( '-value,-stderr,-sample_size,-direction,-issue,-lag,-signal,' + '-missing_value,-missing_stderr,-missing_sample_size' )) @@ -210,18 +208,15 @@ def test_location_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected_counties = [ - self.expected_from_row(r) for r in rows[:3] - ] - + expected = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]] # make the request - response, _ = self.request_based_on_row(rows[0], geo_value="*") + response = self.request_based_on_row(rows[0], geo_value="*") self.maxDiff = None # assert that the right data came back self.assertEqual(response, { 'result': 1, - 'epidata': expected_counties, + 'epidata': expected, 'message': 'success', }) @@ -230,35 +225,33 @@ def test_geo_value(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected_counties = [ - self.expected_from_row(r) for r in rows[:3] - ] + expected = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]] def fetch(geo_value): # make the request - response, _ = self.request_based_on_row(rows[0], geo_value=geo_value) + response = self.request_based_on_row(rows[0], geo_value=geo_value) return response # test fetch a specific region r = fetch('11111') self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [expected_counties[0]]) + self.assertEqual(r['epidata'], expected[0:1]) # test fetch a specific yet not existing region r = fetch('55555') self.assertEqual(r['message'], 'no results') # test fetch multiple regions r = fetch('11111,22222') self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [expected_counties[0], expected_counties[1]]) + self.assertEqual(r['epidata'], expected[0:2]) # test fetch multiple noncontiguous regions r = fetch('11111,33333') self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [expected_counties[0], expected_counties[2]]) + self.assertEqual(r['epidata'], [expected[0], expected[2]]) # test fetch multiple regions but one is not existing r = fetch('11111,55555') self.assertEqual(r['message'], 'success') - self.assertEqual(r['epidata'], [expected_counties[0]]) + self.assertEqual(r['epidata'], expected[0:1]) # test fetch empty region r = fetch('') self.assertEqual(r['message'], 'no results') @@ -268,12 +261,10 @@ def test_location_timeline(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected_timeseries = [ - self.expected_from_row(r) for r in rows[:3] - ] + expected_timeseries = [row.as_dict(ignore_fields=IGNORE_FIELDS) for row in rows[:3]] # make the request - response, _ = self.request_based_on_row(rows[0], time_values='20000101-20000105') + response = self.request_based_on_row(rows[0], time_values='20000101-20000105') # assert that the right data came back self.assertEqual(response, { @@ -299,15 +290,16 @@ def test_unique_key_constraint(self): def test_nullable_columns(self): """Missing values should be surfaced as null.""" - row, _ = self._make_placeholder_row( + row = CovidcastRow( stderr=None, sample_size=None, missing_stderr=Nans.OTHER.value, missing_sample_size=Nans.OTHER.value ) self._insert_rows([row]) # make the request - response, expected = self.request_based_on_row(row) - expected.update(stderr=None, sample_size=None) + response = self.request_based_on_row(row) + expected = row.as_dict(ignore_fields=IGNORE_FIELDS) + # expected.update(stderr=None, sample_size=None) # assert that the right data came back self.assertEqual(response, { @@ -321,18 +313,19 @@ def test_temporal_partitioning(self): # insert placeholder data rows = [ - self._make_placeholder_row(time_type=tt)[0] + CovidcastRow(time_type=tt) for tt in "hour day week month year".split() ] self._insert_rows(rows) # make the request - response, expected = self.request_based_on_row(rows[1], time_values="0-99999999") + response = self.request_based_on_row(rows[1], time_values="20000101-30010201") + expected = [rows[1].as_dict(ignore_fields=IGNORE_FIELDS)] # assert that the right data came back self.assertEqual(response, { 'result': 1, - 'epidata': [expected], + 'epidata': expected, 'message': 'success', }) @@ -343,37 +336,37 @@ def test_date_formats(self): rows = self._insert_placeholder_set_three() # make the request - response, expected = self.request_based_on_row(rows[0], time_values="20000102", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="20000102", geo_value="*") # assert that the right data came back self.assertEqual(len(response['epidata']), 2) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="2000-01-02", geo_value="*") # assert that the right data came back self.assertEqual(len(response['epidata']), 2) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="20000102,20000103", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="20000102,20000103", geo_value="*") # assert that the right data came back - self.assertEqual(len(response['epidata']), 4) + self.assertEqual(len(response['epidata']), 2 * 2) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02,2000-01-03", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="2000-01-02,2000-01-03", geo_value="*") # assert that the right data came back - self.assertEqual(len(response['epidata']), 4) + self.assertEqual(len(response['epidata']), 2 * 2) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="20000102-20000104", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="20000102-20000104", geo_value="*") # assert that the right data came back - self.assertEqual(len(response['epidata']), 6) + self.assertEqual(len(response['epidata']), 2 * 3) # make the request - response, expected = self.request_based_on_row(rows[0], time_values="2000-01-02:2000-01-04", geo_value="*") + response = self.request_based_on_row(rows[0], time_values="2000-01-02:2000-01-04", geo_value="*") # assert that the right data came back - self.assertEqual(len(response['epidata']), 6) + self.assertEqual(len(response['epidata']), 2 * 3) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 54974a874..aa96aae3d 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -1,30 +1,41 @@ """Integration tests for the custom `covidcast/*` endpoints.""" # standard library -from typing import Iterable, Dict, Any -import unittest +from copy import copy +from itertools import accumulate, chain +from typing import List, Sequence from io import StringIO -# from typing import Optional -from dataclasses import dataclass - # third party -import mysql.connector +from more_itertools import interleave_longest, windowed import requests import pandas as pd -from delphi_utils import Nans from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_cache - -from delphi.epidata.acquisition.covidcast.database import Database from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow, CovidcastRows, set_df_dtypes # use the local instance of the Epidata API BASE_URL = "http://delphi_web_epidata/epidata/covidcast" +BASE_URL_OLD = "http://delphi_web_epidata/epidata/api.php" -class CovidcastEndpointTests(CovidcastBase): +def _read_csv(txt: str) -> pd.DataFrame: + df = pd.read_csv(StringIO(txt), index_col=0).rename(columns={"data_source": "source"}) + df.time_value = pd.to_datetime(df.time_value).dt.strftime("%Y%m%d").astype(int) + df.issue = pd.to_datetime(df.issue).dt.strftime("%Y%m%d").astype(int) + df = set_df_dtypes(df, CovidcastRows()._DTYPES) + df.geo_value = df.geo_value.str.zfill(5) + return df + +def _diff_rows(rows: Sequence[float]): + return [float(x - y) if x is not None and y is not None else None for x, y in zip(rows[1:], rows[:-1])] + +def _smooth_rows(rows: Sequence[float]): + return [sum(e)/len(e) if None not in e else None for e in windowed(rows, 7)] + +class CovidcastEndpointTests(CovidcastBase): """Tests the `covidcast/*` endpoint.""" def localSetUp(self): @@ -32,19 +43,23 @@ def localSetUp(self): # reset the `covidcast_meta_cache` table (it should always have one row) self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') - def _fetch(self, endpoint="/", **params): + def _fetch(self, endpoint="/", is_compatibility=False, **params): # make the request - response = requests.get( - f"{BASE_URL}{endpoint}", - params=params, - ) + if is_compatibility: + url = BASE_URL_OLD + params.setdefault("endpoint", "covidcast") + if params.get("source"): + params.setdefault("data_source", params.get("source")) + else: + url = f"{BASE_URL}{endpoint}" + response = requests.get(url, params=params) response.raise_for_status() return response.json() def test_basic(self): """Request a signal from the / endpoint.""" - rows = [self._make_placeholder_row(time_value=20200401 + i, value=i)[0] for i in range(10)] + rows = [CovidcastRow(time_value=20200401 + i, value=i) for i in range(10)] first = rows[0] self._insert_rows(rows) @@ -53,53 +68,207 @@ def test_basic(self): self.assertEqual(out["result"], -1) with self.subTest("simple"): - out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + out = self._fetch("/", signal=first.signal_pair, geo=first.geo_pair, time="day:*") + self.assertEqual(len(out["epidata"]), len(rows)) + + with self.subTest("unknown signal"): + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_unknown", time_value=20200401 + i, value=i) for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + out = self._fetch("/", signal="jhu-csse:confirmed_unknown", geo=first.geo_pair, time="day:*") + out_values = [row["value"] for row in out["epidata"]] + expected_values = [float(row.value) for row in rows] + self.assertEqual(out_values, expected_values) + + def test_compatibility(self): + """Request at the /api.php endpoint.""" + rows = [CovidcastRow(source="src", signal="sig", time_value=20200401 + i, value=i) for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("simple"): + out = self._fetch("/", signal=first.signal_pair, geo=first.geo_pair, time="day:*") + self.assertEqual(len(out["epidata"]), len(rows)) + + with self.subTest("unknown signal"): + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_unknown", time_value=20200401 + i, value=i) for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + out = self._fetch("/", signal="jhu-csse:confirmed_unknown", geo=first.geo_pair, time="day:*") + out_values = [row["value"] for row in out["epidata"]] + expected_values = [float(row.value) for row in rows] + self.assertEqual(out_values, expected_values) + + # JIT tests + def test_derived_signals(self): + time_value_pairs = [(20200401 + i, i ** 2) for i in range(10)] + rows01 = [CovidcastRow(source="jhu-csse", signal="confirmed_cumulative_num", time_value=time_value, value=value, geo_value="01") for time_value, value in time_value_pairs] + rows02 = [CovidcastRow(source="jhu-csse", signal="confirmed_cumulative_num", time_value=time_value, value=2 * value, geo_value="02") for time_value, value in time_value_pairs] + first = rows01[0] + self._insert_rows(rows01 + rows02) + + with self.subTest("diffed signal"): + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num", geo=first.geo_pair, time="day:*") + assert out['result'] == -2 + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num", geo=first.geo_pair, time="day:20200401-20200410") + out_values = [row["value"] for row in out["epidata"]] + values = [value for _, value in time_value_pairs] + expected_values = _diff_rows(values) + self.assertAlmostEqual(out_values, expected_values) + + with self.subTest("diffed signal, multiple geos"): + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num", geo="county:01,02", time="day:20200401-20200410") + out_values = [row["value"] for row in out["epidata"]] + values1 = [value for _, value in time_value_pairs] + values2 = [2 * value for _, value in time_value_pairs] + expected_values = _diff_rows(values1) + _diff_rows(values2) + self.assertAlmostEqual(out_values, expected_values) + + with self.subTest("diffed signal, multiple geos using geo:*"): + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num", geo="county:*", time="day:20200401-20200410") + values1 = [value for _, value in time_value_pairs] + values2 = [2 * value for _, value in time_value_pairs] + expected_values = _diff_rows(values1) + _diff_rows(values2) + self.assertAlmostEqual(out_values, expected_values) + + with self.subTest("smooth diffed signal"): + out = self._fetch("/", signal="jhu-csse:confirmed_7dav_incidence_num", geo=first.geo_pair, time="day:20200401-20200410") + out_values = [row["value"] for row in out["epidata"]] + values = [value for _, value in time_value_pairs] + expected_values = _smooth_rows(_diff_rows(values)) + self.assertAlmostEqual(out_values, expected_values) + + with self.subTest("diffed signal and smoothed signal in one request"): + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num;jhu-csse:confirmed_7dav_incidence_num", geo=first.geo_pair, time="day:20200401-20200410") + out_values = [row["value"] for row in out["epidata"]] + values = [value for _, value in time_value_pairs] + expected_diff = _diff_rows(values) + expected_smoothed = _smooth_rows(expected_diff) + expected_values = list(interleave_longest(expected_smoothed, expected_diff)) + self.assertAlmostEqual(out_values, expected_values) + + time_value_pairs = [(20200401 + i, i ** 2) for i in chain(range(10), range(15, 20))] + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_cumulative_num", geo_value="03", time_value=time_value, value=value) for time_value, value in time_value_pairs] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("diffing with a time gap"): + # should fetch 1 extra day + out = self._fetch("/", signal="jhu-csse:confirmed_incidence_num", geo=first.geo_pair, time="day:20200401-20200420") + out_values = [row["value"] for row in out["epidata"]] + values = [value for _, value in time_value_pairs][:10] + [None] * 5 + [value for _, value in time_value_pairs][10:] + expected_values = _diff_rows(values) + self.assertAlmostEqual(out_values, expected_values) + + with self.subTest("smoothing and diffing with a time gap"): + # should fetch 1 extra day + out = self._fetch("/", signal="jhu-csse:confirmed_7dav_incidence_num", geo=first.geo_pair, time="day:20200401-20200420") + out_values = [row["value"] for row in out["epidata"]] + values = [value for _, value in time_value_pairs][:10] + [None] * 5 + [value for _, value in time_value_pairs][10:] + expected_values = _smooth_rows(_diff_rows(values)) + self.assertAlmostEqual(out_values, expected_values) + + def test_compatibility(self): + """Request at the /api.php endpoint.""" + rows = [CovidcastRow(source="src", signal="sig", time_value=20200401 + i, value=i) for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("simple"): + out = self._fetch(is_compatibility=True, source=first.source, signal=first.signal, geo=first.geo_pair, time="day:*") self.assertEqual(len(out["epidata"]), len(rows)) + def _diff_covidcast_rows(self, rows: List[CovidcastRow]) -> List[CovidcastRow]: + new_rows = list() + for x, y in zip(rows[1:], rows[:-1]): + new_row = copy(x) + new_row.value = x.value - y.value + new_rows.append(new_row) + return new_rows + def test_trend(self): """Request a signal from the /trend endpoint.""" num_rows = 30 - rows = [self._make_placeholder_row(time_value=20200401 + i, value=i)[0] for i in range(num_rows)] + rows = [CovidcastRow(time_value=20200401 + i, value=i) for i in range(num_rows)] first = rows[0] last = rows[-1] ref = rows[num_rows // 2] self._insert_rows(rows) - out = self._fetch("/trend", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20201212", basis=ref.time_value) + with self.subTest("no JIT"): + out = self._fetch("/trend", signal=first.signal_pair, geo=first.geo_pair, date=last.time_value, window="20200401-20201212", basis=ref.time_value) + + self.assertEqual(out["result"], 1) + self.assertEqual(len(out["epidata"]), 1) + trend = out["epidata"][0] + self.assertEqual(trend["geo_type"], last.geo_type) + self.assertEqual(trend["geo_value"], last.geo_value) + self.assertEqual(trend["signal_source"], last.source) + self.assertEqual(trend["signal_signal"], last.signal) + + self.assertEqual(trend["date"], last.time_value) + self.assertEqual(trend["value"], last.value) + + self.assertEqual(trend["basis_date"], ref.time_value) + self.assertEqual(trend["basis_value"], ref.value) + self.assertEqual(trend["basis_trend"], "increasing") + + self.assertEqual(trend["min_date"], first.time_value) + self.assertEqual(trend["min_value"], first.value) + self.assertEqual(trend["min_trend"], "increasing") + self.assertEqual(trend["max_date"], last.time_value) + self.assertEqual(trend["max_value"], last.value) + self.assertEqual(trend["max_trend"], "steady") + + num_rows = 30 + time_value_pairs = [(20200331, 0)] + [(20200401 + i, v) for i, v in enumerate(accumulate(range(num_rows)))] + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_cumulative_num", time_value=t, value=v) for t, v in time_value_pairs] + self._insert_rows(rows) + diffed_rows = self._diff_covidcast_rows(rows) + for row in diffed_rows: + row.signal = "confirmed_incidence_num" + first = diffed_rows[0] + last = diffed_rows[-1] + ref = diffed_rows[num_rows // 2] + with self.subTest("use JIT"): + out = self._fetch("/trend", signal="jhu-csse:confirmed_incidence_num", geo=first.geo_pair, date=last.time_value, window="20200401-20201212", basis=ref.time_value) + + self.assertEqual(out["result"], 1) + self.assertEqual(len(out["epidata"]), 1) + trend = out["epidata"][0] + self.assertEqual(trend["geo_type"], last.geo_type) + self.assertEqual(trend["geo_value"], last.geo_value) + self.assertEqual(trend["signal_source"], last.source) + self.assertEqual(trend["signal_signal"], last.signal) + + self.assertEqual(trend["date"], last.time_value) + self.assertEqual(trend["value"], last.value) + + self.assertEqual(trend["basis_date"], ref.time_value) + self.assertEqual(trend["basis_value"], ref.value) + self.assertEqual(trend["basis_trend"], "increasing") + + self.assertEqual(trend["min_date"], first.time_value) + self.assertEqual(trend["min_value"], first.value) + self.assertEqual(trend["min_trend"], "increasing") + self.assertEqual(trend["max_date"], last.time_value) + self.assertEqual(trend["max_value"], last.value) + self.assertEqual(trend["max_trend"], "steady") - self.assertEqual(out["result"], 1) - self.assertEqual(len(out["epidata"]), 1) - trend = out["epidata"][0] - self.assertEqual(trend["geo_type"], last.geo_type) - self.assertEqual(trend["geo_value"], last.geo_value) - self.assertEqual(trend["signal_source"], last.source) - self.assertEqual(trend["signal_signal"], last.signal) - - self.assertEqual(trend["date"], last.time_value) - self.assertEqual(trend["value"], last.value) - - self.assertEqual(trend["basis_date"], ref.time_value) - self.assertEqual(trend["basis_value"], ref.value) - self.assertEqual(trend["basis_trend"], "increasing") - - self.assertEqual(trend["min_date"], first.time_value) - self.assertEqual(trend["min_value"], first.value) - self.assertEqual(trend["min_trend"], "increasing") - self.assertEqual(trend["max_date"], last.time_value) - self.assertEqual(trend["max_value"], last.value) - self.assertEqual(trend["max_trend"], "steady") def test_trendseries(self): """Request a signal from the /trendseries endpoint.""" num_rows = 3 - rows = [self._make_placeholder_row(time_value=20200401 + i, value=num_rows - i)[0] for i in range(num_rows)] + rows = [CovidcastRow(time_value=20200401 + i, value=num_rows - i) for i in range(num_rows)] first = rows[0] last = rows[-1] self._insert_rows(rows) - out = self._fetch("/trendseries", signal=first.signal_pair(), geo=first.geo_pair(), date=last.time_value, window="20200401-20200410", basis=1) + out = self._fetch("/trendseries", signal=first.signal_pair, geo=first.geo_pair, date=last.time_value, window="20200401-20200410", basis=1) self.assertEqual(out["result"], 1) self.assertEqual(len(out["epidata"]), 3) @@ -127,6 +296,7 @@ def match_row(trend, row): self.assertEqual(trend["max_date"], first.time_value) self.assertEqual(trend["max_value"], first.value) self.assertEqual(trend["max_trend"], "steady") + with self.subTest("trend1"): trend = trends[1] match_row(trend, rows[1]) @@ -155,19 +325,78 @@ def match_row(trend, row): self.assertEqual(trend["max_value"], first.value) self.assertEqual(trend["max_trend"], "decreasing") + num_rows = 3 + time_value_pairs = [(20200331, 0)] + [(20200401 + i, v) for i, v in enumerate(accumulate([num_rows - i for i in range(num_rows)]))] + rows = [CovidcastRow(source="jhu-csse", signal="confirmed_cumulative_num", time_value=t, value=v) for t, v in time_value_pairs] + self._insert_rows(rows) + diffed_rows = self._diff_covidcast_rows(rows) + for row in diffed_rows: + row.signal = "confirmed_incidence_num" + first = diffed_rows[0] + last = diffed_rows[-1] + + out = self._fetch("/trendseries", signal="jhu-csse:confirmed_incidence_num", geo=first.geo_pair, date=last.time_value, window="20200401-20200410", basis=1) + + self.assertEqual(out["result"], 1) + self.assertEqual(len(out["epidata"]), 3) + trends = out["epidata"] + + with self.subTest("trend0, JIT"): + trend = trends[0] + match_row(trend, first) + self.assertEqual(trend["basis_date"], None) + self.assertEqual(trend["basis_value"], None) + self.assertEqual(trend["basis_trend"], "unknown") + + self.assertEqual(trend["min_date"], last.time_value) + self.assertEqual(trend["min_value"], last.value) + self.assertEqual(trend["min_trend"], "increasing") + self.assertEqual(trend["max_date"], first.time_value) + self.assertEqual(trend["max_value"], first.value) + self.assertEqual(trend["max_trend"], "steady") + + with self.subTest("trend1"): + trend = trends[1] + match_row(trend, diffed_rows[1]) + self.assertEqual(trend["basis_date"], first.time_value) + self.assertEqual(trend["basis_value"], first.value) + self.assertEqual(trend["basis_trend"], "decreasing") + + self.assertEqual(trend["min_date"], last.time_value) + self.assertEqual(trend["min_value"], last.value) + self.assertEqual(trend["min_trend"], "increasing") + self.assertEqual(trend["max_date"], first.time_value) + self.assertEqual(trend["max_value"], first.value) + self.assertEqual(trend["max_trend"], "decreasing") + + with self.subTest("trend2"): + trend = trends[2] + match_row(trend, last) + self.assertEqual(trend["basis_date"], diffed_rows[1].time_value) + self.assertEqual(trend["basis_value"], diffed_rows[1].value) + self.assertEqual(trend["basis_trend"], "decreasing") + + self.assertEqual(trend["min_date"], last.time_value) + self.assertEqual(trend["min_value"], last.value) + self.assertEqual(trend["min_trend"], "steady") + self.assertEqual(trend["max_date"], first.time_value) + self.assertEqual(trend["max_value"], first.value) + self.assertEqual(trend["max_trend"], "decreasing") + + def test_correlation(self): """Request a signal from the /correlation endpoint.""" num_rows = 30 - reference_rows = [self._make_placeholder_row(signal="ref", time_value=20200401 + i, value=i)[0] for i in range(num_rows)] + reference_rows = [CovidcastRow(signal="ref", time_value=20200401 + i, value=i) for i in range(num_rows)] first = reference_rows[0] self._insert_rows(reference_rows) - other_rows = [self._make_placeholder_row(signal="other", time_value=20200401 + i, value=i)[0] for i in range(num_rows)] + other_rows = [CovidcastRow(signal="other", time_value=20200401 + i, value=i) for i in range(num_rows)] other = other_rows[0] self._insert_rows(other_rows) max_lag = 3 - out = self._fetch("/correlation", reference=first.signal_pair(), others=other.signal_pair(), geo=first.geo_pair(), window="20200401-20201212", lag=max_lag) + out = self._fetch("/correlation", reference=first.signal_pair, others=other.signal_pair, geo=first.geo_pair, window="20200401-20201212", lag=max_lag) self.assertEqual(out["result"], 1) df = pd.DataFrame(out["epidata"]) self.assertEqual(len(df), max_lag * 2 + 1) # -...0...+ @@ -185,31 +414,75 @@ def test_correlation(self): def test_csv(self): """Request a signal from the /csv endpoint.""" - rows = [self._make_placeholder_row(time_value=20200401 + i, value=i)[0] for i in range(10)] - first = rows[0] - self._insert_rows(rows) - - response = requests.get( - f"{BASE_URL}/csv", - params=dict(signal=first.signal_pair(), start_day="2020-04-01", end_day="2020-12-12", geo_type=first.geo_type), + expected_columns = ["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "data_source"] + data = CovidcastRows.from_args( + time_value=pd.date_range("2020-04-01", "2020-04-10"), + value=range(10) ) - response.raise_for_status() - out = response.text - df = pd.read_csv(StringIO(out), index_col=0) - self.assertEqual(df.shape, (len(rows), 10)) - self.assertEqual(list(df.columns), ["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "data_source"]) + self._insert_rows(data.rows) + first = data.rows[0] + with self.subTest("no JIT"): + response = requests.get( + f"{BASE_URL}/csv", + params=dict(signal=first.signal_pair, start_day="2020-04-01", end_day="2020-04-10", geo_type=first.geo_type), + ) + response.raise_for_status() + out = response.text + df = pd.read_csv(StringIO(out), index_col=0) + + self.assertEqual(df.shape, (len(data.rows), 10)) + self.assertEqual(list(df.columns), expected_columns) + + data = CovidcastRows.from_args( + source=["jhu-csse"] * 11, + signal=["confirmed_cumulative_num"] * 11, + time_value=pd.date_range("2020-03-31", "2020-04-10"), + value=accumulate(range(11)), + ) + self._insert_rows(data.rows) + first = data.rows[0] + with self.subTest("use JIT"): + response = requests.get( + f"{BASE_URL}/csv", + params=dict(signal="jhu-csse:confirmed_cumulative_num", start_day="2020-04-01", end_day="2020-04-10", geo_type=first.geo_type), + ) + response.raise_for_status() + df = _read_csv(response.text) + expected_df = CovidcastRows.from_args( + source=["jhu-csse"] * 10, + signal=["confirmed_cumulative_num"] * 10, + time_value=pd.date_range("2020-04-01", "2020-04-10"), + value=list(accumulate(range(11)))[1:], + ).api_row_df[df.columns] + pd.testing.assert_frame_equal(df, expected_df) + + response = requests.get( + f"{BASE_URL}/csv", + params=dict(signal="jhu-csse:confirmed_incidence_num", start_day="2020-04-01", end_day="2020-04-10", geo_type=first.geo_type), + ) + response.raise_for_status() + df_diffed = _read_csv(response.text) + expected_df = CovidcastRows.from_args( + source=["jhu-csse"] * 10, + signal=["confirmed_incidence_num"] * 10, + time_value=pd.date_range("2020-04-01", "2020-04-10"), + value=range(1, 11), + stderr=[None] * 10, + sample_size=[None] * 10 + ).api_row_df[df_diffed.columns] + pd.testing.assert_frame_equal(df_diffed, expected_df) def test_backfill(self): """Request a signal from the /backfill endpoint.""" num_rows = 10 - issue_0 = [self._make_placeholder_row(time_value=20200401 + i, value=i, sample_size=1, lag=0, issue=20200401 + i)[0] for i in range(num_rows)] - issue_1 = [self._make_placeholder_row(time_value=20200401 + i, value=i + 1, sample_size=2, lag=1, issue=20200401 + i + 1)[0] for i in range(num_rows)] - last_issue = [self._make_placeholder_row(time_value=20200401 + i, value=i + 2, sample_size=3, lag=2, issue=20200401 + i + 2)[0] for i in range(num_rows)] # <-- the latest issues + issue_0 = [CovidcastRow(time_value=20200401 + i, value=i, sample_size=1, lag=0, issue=20200401 + i) for i in range(num_rows)] + issue_1 = [CovidcastRow(time_value=20200401 + i, value=i + 1, sample_size=2, lag=1, issue=20200401 + i + 1) for i in range(num_rows)] + last_issue = [CovidcastRow(time_value=20200401 + i, value=i + 2, sample_size=3, lag=2, issue=20200401 + i + 2) for i in range(num_rows)] # <-- the latest issues self._insert_rows([*issue_0, *issue_1, *last_issue]) first = issue_0[0] - out = self._fetch("/backfill", signal=first.signal_pair(), geo=first.geo_pair(), time="day:20200401-20201212", anchor_lag=3) + out = self._fetch("/backfill", signal=first.signal_pair, geo=first.geo_pair, time="day:20200401-20201212", anchor_lag=3) self.assertEqual(out["result"], 1) df = pd.DataFrame(out["epidata"]) self.assertEqual(len(df), 3 * num_rows) # num issues @@ -231,7 +504,7 @@ def test_meta(self): """Request a signal from the /meta endpoint.""" num_rows = 10 - rows = [self._make_placeholder_row(time_value=20200401 + i, value=i, source="fb-survey", signal="smoothed_cli")[0] for i in range(num_rows)] + rows = [CovidcastRow(time_value=20200401 + i, value=i, source="fb-survey", signal="smoothed_cli") for i in range(num_rows)] self._insert_rows(rows) first = rows[0] last = rows[-1] @@ -272,22 +545,22 @@ def test_coverage(self): num_geos_per_date = [10, 20, 30, 40, 44] dates = [20200401 + i for i in range(len(num_geos_per_date))] - rows = [self._make_placeholder_row(time_value=dates[i], value=i, geo_value=str(geo_value))[0] for i, num_geo in enumerate(num_geos_per_date) for geo_value in range(num_geo)] + rows = [CovidcastRow(time_value=dates[i], value=i, geo_value=str(geo_value)) for i, num_geo in enumerate(num_geos_per_date) for geo_value in range(num_geo)] self._insert_rows(rows) first = rows[0] with self.subTest("default"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type=first.geo_type, latest=dates[-1], format="json") + out = self._fetch("/coverage", signal=first.signal_pair, geo_type=first.geo_type, latest=dates[-1], format="json") self.assertEqual(len(out), len(num_geos_per_date)) self.assertEqual([o["time_value"] for o in out], dates) self.assertEqual([o["count"] for o in out], num_geos_per_date) with self.subTest("specify window"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type=first.geo_type, window=f"{dates[0]}-{dates[1]}", format="json") + out = self._fetch("/coverage", signal=first.signal_pair, geo_type=first.geo_type, window=f"{dates[0]}-{dates[1]}", format="json") self.assertEqual(len(out), 2) self.assertEqual([o["time_value"] for o in out], dates[:2]) self.assertEqual([o["count"] for o in out], num_geos_per_date[:2]) with self.subTest("invalid geo_type"): - out = self._fetch("/coverage", signal=first.signal_pair(), geo_type="doesnt_exist", format="json") + out = self._fetch("/coverage", signal=first.signal_pair, geo_type="doesnt_exist", format="json") self.assertEqual(len(out), 0) diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index d0aef6fe5..48777d4c1 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -1,22 +1,36 @@ """Integration tests for the `covidcast_meta` endpoint.""" # standard library -import unittest +from datetime import date +from itertools import chain +from typing import Iterable, Optional # third party -import mysql.connector +import numpy as np +import pandas as pd +import pytest import requests -#first party +# first party +import delphi.operations.secrets as secrets from delphi_utils import Nans from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import main as update_cache -import delphi.operations.secrets as secrets +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow +from delphi.epidata.acquisition.covidcast.database_meta import DatabaseMeta +from delphi.epidata.acquisition.covidcast.test_utils import CovidcastBase + # use the local instance of the Epidata API BASE_URL = 'http://delphi_web_epidata/epidata/api.php' -class CovidcastMetaTests(unittest.TestCase): +def _dicts_equal(d1: dict, d2: dict, ignore_keys: Optional[list] = None, rel: Optional[float] = None, abs: Optional[float] = None) -> bool: + """Compare dictionary values using floating point comparison for numeric values.""" + assert set(d1.keys()) == set(d2.keys()), "Dictionary keys should be the same." + return all(d1.get(key) == pytest.approx(d2.get(key), rel=rel, abs=abs, nan_ok=True) for key in d1.keys() if (ignore_keys and key not in ignore_keys)) + + +class TestCovidcastMeta(CovidcastBase): """Tests the `covidcast_meta` endpoint.""" src_sig_lookups = { @@ -47,55 +61,45 @@ class CovidcastMetaTests(unittest.TestCase): %d, %d) ''' - def setUp(self): + def localSetUp(self): """Perform per-test setup.""" - # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( - user='user', - password='pass', - host='delphi_database_epidata', - database='covid') - cur = cnx.cursor() - - # clear all tables - cur.execute("truncate table epimetric_load") - cur.execute("truncate table epimetric_full") - cur.execute("truncate table epimetric_latest") - cur.execute("truncate table geo_dim") - cur.execute("truncate table signal_dim") - # reset the `covidcast_meta_cache` table (it should always have one row) - cur.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') + # connect to the `epidata` database + self.db = DatabaseMeta(base_url="http://delphi_web_epidata/epidata") + self.db.connect(user="user", password="pass", host="delphi_database_epidata", database="covid") + + # TODO: Switch when delphi_epidata client is released. + self.db.delphi_epidata = False # populate dimension tables for (src,sig) in self.src_sig_lookups: - cur.execute(''' + self.db._cursor.execute(''' INSERT INTO `signal_dim` (`signal_key_id`, `source`, `signal`) VALUES (%d, '%s', '%s'); ''' % ( self.src_sig_lookups[(src,sig)], src, sig )) for (gt,gv) in self.geo_lookups: - cur.execute(''' + self.db._cursor.execute(''' INSERT INTO `geo_dim` (`geo_key_id`, `geo_type`, `geo_value`) VALUES (%d, '%s', '%s'); ''' % ( self.geo_lookups[(gt,gv)], gt, gv )) - cnx.commit() - cur.close() + self.db._connection.commit() # initialize counter for tables without non-autoincrement id self.id_counter = 666 - # make connection and cursor available to test cases - self.cnx = cnx - self.cur = cnx.cursor() - # use the local instance of the epidata database secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') - - def tearDown(self): + def localTearDown(self): """Perform per-test teardown.""" - self.cur.close() - self.cnx.close() + self.db._cursor.close() + self.db._connection.close() + + def _insert_rows(self, rows: Iterable[CovidcastRow]): + self.db.insert_or_update_bulk(list(rows)) + self.db.run_dbjobs() + self.db._connection.commit() + return rows def insert_placeholder_data(self): expected = [] @@ -122,13 +126,13 @@ def insert_placeholder_data(self): }) for tv in (1, 2): for gv, v in zip(('geo1', 'geo2'), (10, 20)): - self.cur.execute(self.template % ( + self.db._cursor.execute(self.template % ( self._get_id(), self.src_sig_lookups[(src,sig)], self.geo_lookups[(gt,gv)], tt, tv, v, tv, # re-use time value for issue Nans.NOT_MISSING, Nans.NOT_MISSING, Nans.NOT_MISSING )) - self.cnx.commit() + self.db._connection.commit() update_cache(args=None) return expected @@ -237,3 +241,62 @@ def fetch(**kwargs): self.assertEqual(len(res['epidata']), len(expected)) self.assertEqual(res['epidata'][0], {}) + def test_meta_values(self): + """This is an A/B test between the old meta compute approach and the new one which relies on an API call for JIT signals. + + It relies on synthetic data. + """ + + def get_rows_gen(df: pd.DataFrame, filter_nans: bool = False) -> Iterable[CovidcastRow]: + for row in df.itertuples(index=False): + row_dict = row._asdict() + if not filter_nans or (filter_nans and not any(map(pd.isna, row_dict.values()))): + yield CovidcastRow(**row_dict) + + start_date = date(2022, 4, 1) + end_date = date(2022, 6, 1) + n = (end_date - start_date).days + 1 + + # TODO: Build a more complex synthetic dataset here. + # fmt: off + cumulative_df = pd.DataFrame( + { + "source": ["jhu-csse"] * n + ["usa-facts"] * n, + "signal": ["confirmed_cumulative_num"] * n + ["confirmed_cumulative_num"] * (n // 2 - 1) + [np.nan] + ["confirmed_cumulative_num"] * (n // 2), + "time_value": chain(pd.date_range(start_date, end_date), pd.date_range(start_date, end_date)), + "issue": chain(pd.date_range(start_date, end_date), pd.date_range(start_date, end_date)), + "value": chain(range(n), range(n)) + } + ) + incidence_df = ( + cumulative_df.set_index(["source", "time_value"]) + .groupby("source") + .apply(lambda df: df.assign( + signal="confirmed_incidence_num", + value=df.value.diff(), + issue=[max(window) if window.size >= 2 else np.nan for window in df.issue.rolling(2)] + ) + ) + ).reset_index() + smoothed_incidence_df = ( + cumulative_df.set_index(["source", "time_value"]) + .groupby("source") + .apply(lambda df: df.assign( + signal="confirmed_7dav_incidence_num", + value=df.value.rolling(7).mean().diff(), + issue=[max(window) if window.size >= 7 else np.nan for window in df.issue.rolling(7)] + ) + ) + ).reset_index() + # fmt: on + + self._insert_rows(get_rows_gen(cumulative_df, filter_nans=True)) + self._insert_rows(get_rows_gen(incidence_df, filter_nans=True)) + self._insert_rows(get_rows_gen(smoothed_incidence_df, filter_nans=True)) + + meta_values = self.db.compute_covidcast_meta(jit=False) + meta_values2 = self.db.compute_covidcast_meta(jit=True) + + out = [_dicts_equal(x, y, ignore_keys=["max_lag"]) for x, y in zip(meta_values, meta_values2)] + + assert all(out) diff --git a/integrations/server/test_covidcast_meta_ab.py b/integrations/server/test_covidcast_meta_ab.py new file mode 100644 index 000000000..4eb229d6a --- /dev/null +++ b/integrations/server/test_covidcast_meta_ab.py @@ -0,0 +1,288 @@ +from datetime import datetime, timedelta +from functools import reduce +from math import inf +from numbers import Number +from pathlib import Path +from typing import Iterable, Optional, Union + +# third party +import numpy as np +import pandas as pd +import pytest +import pytest_check as check +import requests + +# first party +import delphi.operations.secrets as secrets +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow +from delphi.epidata.acquisition.covidcast.database_meta import DatabaseMeta + +# use the local instance of the Epidata API +BASE_URL = "http://delphi_web_epidata/epidata/api.php" +TEST_DATA_DIR = Path("repos/delphi/delphi-epidata/testdata/acquisition/covidcast/") + + +def _df_to_covidcastrows(df: pd.DataFrame) -> Iterable[CovidcastRow]: + """Iterates over the rows of a dataframe. + + The dataframe is expected to have many columns, see below for which. + """ + for _, row in df.iterrows(): + yield CovidcastRow( + source=row.data_source if "data_source" in df.columns else row.source, + signal=row.signal, + time_type=row.time_type, + geo_type=row.geo_type, + time_value=datetime.strptime(row.time_value, "%Y-%m-%d"), + geo_value=row.geo_value, + value=row.value, + stderr=row.stderr if not np.isnan(row.stderr) else None, + sample_size=row.sample_size if not np.isnan(row.sample_size) else None, + missing_value=row.missing_value, + missing_stderr=row.missing_stderr, + missing_sample_size=row.missing_sample_size, + issue=datetime.strptime(row.issue, "%Y-%m-%d"), + lag=row.lag, + ) + + +def _almost_equal(v1: Optional[Union[Number, str]], v2: Optional[Union[Number, str]], atol: float = 1e-08) -> bool: + if v1 is None and v2 is None: + return True + elif (v1 is None and v2 is not None) or (v1 is not None and v2 is None): + return False + else: + return np.allclose(v1, v2, atol=atol) if isinstance(v1, Number) and isinstance(v2, Number) else v1 == v2 + + +def _dicts_equal(d1: dict, d2: dict, ignore_keys: Optional[list] = None, atol: float = 1e-08) -> bool: + """Compare dictionary values using floating point comparison for numeric values.""" + assert set(d1.keys()) == set(d2.keys()) + return all(_almost_equal(d1.get(key), d2.get(key), atol=atol) for key in d1.keys() if (ignore_keys and key not in ignore_keys)) + + +class TestCovidcastMeta: + def setup_method(self): + """Perform per-test setup.""" + + # connect to the `epidata` database + self.db = DatabaseMeta(base_url="http://delphi_web_epidata/epidata") + self.db.connect(user="user", password="pass", host="delphi_database_epidata", database="covid") + + # TODO: Switch when delphi_epidata client is released. + self.db.delphi_epidata = False + + # clear all tables + self.db._cursor.execute("truncate table epimetric_load") + self.db._cursor.execute("truncate table epimetric_full") + self.db._cursor.execute("truncate table epimetric_latest") + self.db._cursor.execute("truncate table geo_dim") + self.db._cursor.execute("truncate table signal_dim") + self.db._connection.commit() + # reset the `covidcast_meta_cache` table (it should always have one row) + self.db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') + + self.db._connection.commit() + + # initialize counter for tables without non-autoincrement id + self.id_counter = 666 + + # use the local instance of the epidata database + secrets.db.host = "delphi_database_epidata" + secrets.db.epi = ("user", "pass") + + def teardown_method(self): + """Perform per-test teardown.""" + self.db._cursor.close() + self.db._connection.close() + + def _insert_rows(self, rows: Iterable[CovidcastRow]): + self.db.insert_or_update_bulk(list(rows)) + self.db.run_dbjobs() + self.db._connection.commit() + return rows + + def get_source_signal_from_db(self, source: str, signal: str) -> pd.DataFrame: + """Get the source signal data from the database.""" + sql = f"""SELECT c.signal, c.geo_type, c.geo_value, c.time_value, c.value FROM epimetric_latest_v c WHERE 1 = 1 AND c.`source` = '{source}' AND c.`signal` = '{signal}'""" + self.db._cursor.execute(sql) + df = ( + pd.DataFrame.from_records(self.db._cursor.fetchall(), columns=["signal", "geo_type", "geo_value", "time_value", "value"]) + .assign(time_value=lambda x: pd.to_datetime(x["time_value"], format="%Y%m%d")) + .set_index(["signal", "geo_value", "time_value"]) + .sort_index() + ) + return df + + def get_source_signal_from_api(self, source: str, signal: str) -> pd.DataFrame: + """Query the source signal data from the local API.""" + base_url = "http://delphi_web_epidata/epidata/covidcast/" + + def get_api_df(**params) -> pd.DataFrame: + return pd.DataFrame.from_records(requests.get(base_url, params=params).json()["epidata"]) + + ALLTIME = "19000101-20500101" + params = {"signal": f"{source}:{signal}", "geo": "state:*;county:*", "time": f"day:{ALLTIME}"} + df = get_api_df(**params).assign(geo_type="day", time_value=lambda x: pd.to_datetime(x["time_value"], format="%Y%m%d")).set_index(["signal", "geo_value", "time_value"]).sort_index() + return df + + def _insert_csv(self, filename: str): + with pd.read_csv(filename, chunksize=10_000) as reader: + for chunk_df in reader: + self._insert_rows(_df_to_covidcastrows(chunk_df)) + + @pytest.mark.skip("Archived.") + @pytest.mark.parametrize("test_data_filepath", TEST_DATA_DIR.glob("*.csv")) + def test_incidence(self, test_data_filepath): + """This is large-scale A/B test of the JIT system for the incidence signal. + + It was tested on large datasets and found to be correct. See #646. + + Uses live API data and compares: + - the results of the new JIT system to the API data + - the results of the new JIT system to the Pandas-derived data + """ + source = "usa-facts" if "usa-facts" in str(test_data_filepath) else "jhu-csse" + print(test_data_filepath) + self._insert_csv(test_data_filepath) + + # Here we load: + # test_data_full_df - the original CSV file with our test data + # db_pandas_incidence_df - the incidence data pulled from the database as cumulative, placed on a contiguous index (live data has gaps), and then diffed via Pandas + # api_incidence_df - the incidence data as returned by the API from JIT + test_data_full_df = ( + pd.read_csv(test_data_filepath) + .assign(time_value=lambda x: pd.to_datetime(x["time_value"]), geo_value=lambda x: x["geo_value"].astype(str)) + .set_index(["signal", "geo_value", "time_value"]) + .sort_index() + ) + db_pandas_incidence_df = ( + self.get_source_signal_from_db(source, "confirmed_cumulative_num") + # Place on a contiguous index + .groupby(["signal", "geo_value"]) + .apply(lambda x: x.reset_index().drop(columns=["signal", "geo_value"]).set_index("time_value").reindex(pd.date_range("2020-01-25", "2022-09-10"))) + .reset_index() + .rename(columns={"level_2": "time_value"}) + .set_index(["signal", "geo_value", "time_value"]) + # Diff + .groupby(["signal", "geo_value"]) + .apply(lambda x: x["value"].reset_index().drop(columns=["signal", "geo_value"]).set_index("time_value").diff()) + .reset_index() + .assign(signal="confirmed_incidence_num") + .set_index(["signal", "geo_value", "time_value"]) + ) + api_incidence_df = self.get_source_signal_from_api(source, "confirmed_incidence_num") + + # Join into one dataframe for easy comparison + test_data_full_df = test_data_full_df.join(db_pandas_incidence_df.value, rsuffix="_db_pandas") + test_data_full_df = test_data_full_df.join(api_incidence_df.value, rsuffix="_api_jit") + test_data_cumulative_df: pd.DataFrame = test_data_full_df.loc["confirmed_cumulative_num"] + test_data_incidence_df: pd.DataFrame = test_data_full_df.loc["confirmed_incidence_num"] + + # Test 1: show that Pandas-recomputed incidence (from cumulative) is identical to JIT incidence (up to 7 decimal places). + pandas_ne_jit = test_data_full_df[["value_db_pandas", "value_api_jit"]].dropna(how="any", axis=0) + pandas_ne_jit = pandas_ne_jit[pandas_ne_jit.value_db_pandas.sub(pandas_ne_jit.value_api_jit, fill_value=inf).abs().ge(1e-7)] + check.is_true(pandas_ne_jit.empty, "Check Pandas-JIT incidence match.") + if not pandas_ne_jit.empty: + print("Pandas-JIT incidence mismatch:") + print(pandas_ne_jit.to_string()) + + # Test 2: show that some JIT incidence values do not match live data. These are errors in the live data. + live_ne_jit = test_data_full_df[["value", "value_api_jit"]].dropna(how="any", axis=0) + live_ne_jit = live_ne_jit[live_ne_jit.value.sub(live_ne_jit.value_api_jit, fill_value=inf).abs().ge(1e-7)] + check.is_true(live_ne_jit.empty, "Check JIT-live match.") + if not live_ne_jit.empty: + print("JIT-live mismatch:") + print(live_ne_jit.to_string()) + + # Test 3: show that when JIT has a NAN, it is reasonable: the cumulative signal is either missing today or yesterday. + jit_nan_df = test_data_incidence_df[["value", "value_api_jit"]].query("value_api_jit.isna()") + jit_nan_df = reduce( + lambda x, y: pd.merge(x, y, how="outer", left_index=True, right_index=True), + ( + test_data_cumulative_df.filter(items=jit_nan_df.index.map(lambda x: (x[0], x[1] - timedelta(days=i))), axis=0)["value"].rename(f"value_{i}_days_past") + for i in range(2) + ), + ) + jit_nan_df = jit_nan_df[jit_nan_df.notna().all(axis=1)] + check.is_true(jit_nan_df.empty, "Check JIT NANs are reasonable.") + if not jit_nan_df.empty: + print("JIT NANs are unreasonable:") + print(jit_nan_df.to_string()) + + @pytest.mark.skip("Too slow.") + @pytest.mark.parametrize("test_data_filepath", TEST_DATA_DIR.glob("*.csv")) + def test_7dav_incidence(self, test_data_filepath): + """This is large-scale A/B test of the JIT system for the 7dav incidence signal. + + It was tested on large datasets and found to be correct. See #646. + + Uses live API data and compares: + - the results of the new JIT system to the API data + - the results of the new JIT system to the Pandas-derived data + """ + source = "usa-facts" if "usa-facts" in str(test_data_filepath) else "jhu-csse" + print(test_data_filepath) + self._insert_csv(test_data_filepath) + + # Here we load: + # test_data_full_df - the original CSV file with our test data + # db_pandas_incidence_df - the incidence data pulled from the database as cumulative, placed on a contiguous index (live data has gaps), and then diffed via Pandas + # api_incidence_df - the incidence data as returned by the API from JIT + test_data_full_df = ( + pd.read_csv(test_data_filepath) + .assign(time_value=lambda x: pd.to_datetime(x["time_value"]), geo_value=lambda x: x["geo_value"].astype(str)) + .set_index(["signal", "geo_value", "time_value"]) + .sort_index() + ) + db_pandas_7dav_incidence_df = ( + self.get_source_signal_from_db(source, "confirmed_cumulative_num") + .groupby(["signal", "geo_value"]) + .apply(lambda x: x.reset_index().drop(columns=["signal", "geo_value"]).set_index("time_value").reindex(pd.date_range("2020-01-25", "2022-09-10"))) + .reset_index() + .rename(columns={"level_2": "time_value"}) + .set_index(["signal", "geo_value", "time_value"]) + .groupby(["signal", "geo_value"]) + .apply(lambda x: x["value"].reset_index().drop(columns=["signal", "geo_value"]).set_index("time_value").diff().rolling(7).mean()) + .reset_index() + .assign(signal="confirmed_7dav_incidence_num") + .set_index(["signal", "geo_value", "time_value"]) + ) + api_7dav_incidence_df = self.get_source_signal_from_api(source, "confirmed_7dav_incidence_num") + + # Join into one dataframe for easy comparison + test_data_full_df = test_data_full_df.join(db_pandas_7dav_incidence_df.value, rsuffix="_db_pandas") + test_data_full_df = test_data_full_df.join(api_7dav_incidence_df.value, rsuffix="_api_jit") + test_data_cumulative_df: pd.DataFrame = test_data_full_df.loc["confirmed_cumulative_num"] + test_data_7dav_incidence_df: pd.DataFrame = test_data_full_df.loc["confirmed_7dav_incidence_num"] + + # Test 1: show that Pandas-recomputed incidence (from cumulative) is identical to JIT incidence (up to 7 decimal places). + pandas_ne_jit = test_data_full_df[["value_db_pandas", "value_api_jit"]].dropna(how="any", axis=0) + pandas_ne_jit = pandas_ne_jit[pandas_ne_jit.value_db_pandas.sub(pandas_ne_jit.value_api_jit, fill_value=inf).abs().ge(1e-7)] + check.is_true(pandas_ne_jit.empty, "Check Pandas-JIT incidence match.") + if not pandas_ne_jit.empty: + print("Pandas-JIT incidence mismatch:") + print(pandas_ne_jit.to_string()) + + # Test 2: show that some JIT incidence values do not match live data. These are errors in the live data. + live_ne_jit = test_data_7dav_incidence_df[["value", "value_api_jit"]].dropna(how="any", axis=0) + live_ne_jit = live_ne_jit[live_ne_jit.value.sub(live_ne_jit.value_api_jit, fill_value=inf).abs().ge(1e-7)] + check.is_true(live_ne_jit.empty, "Check JIT-live match.") + if not live_ne_jit.empty: + print("JIT-live mismatch:") + print(live_ne_jit.to_string()) + + # Test 3: show that when JIT has a NAN, it is reasonable: the cumulative signal is either missing today or yesterday. + jit_nan_df = test_data_7dav_incidence_df[["value", "value_api_jit"]].query("value_api_jit.isna()") + jit_nan_df = reduce( + lambda x, y: pd.merge(x, y, how="outer", left_index=True, right_index=True), + ( + test_data_cumulative_df.filter(items=jit_nan_df.index.map(lambda x: (x[0], x[1] - timedelta(days=i))), axis=0)["value"].rename(f"value_{i}_days_past") + for i in range(8) + ), + ) + jit_nan_df = jit_nan_df.dropna(how="any", axis=0) + check.is_true(jit_nan_df.empty, "Check JIT NANs are reasonable.") + if not jit_nan_df.empty: + print("JIT NANs are not reasonable:") + print(jit_nan_df.to_string()) diff --git a/requirements.txt b/requirements.txt index 945ac11ea..51abf274d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,5 @@ scipy==1.6.2 tenacity==7.0.0 newrelic epiweeks==2.1.2 +delphi_utils +more_itertools==8.4.0 \ No newline at end of file diff --git a/src/acquisition/covidcast/config.py b/src/acquisition/covidcast/config.py new file mode 100644 index 000000000..b84878ddf --- /dev/null +++ b/src/acquisition/covidcast/config.py @@ -0,0 +1,3 @@ +# TODO: Fill these in. +GEO_TYPES = ["county", "state", "hhs", "msa", "nation", "hrr"] +ALL_TIME = "19900101-20400101" diff --git a/src/acquisition/covidcast/covidcast_meta_cache_updater.py b/src/acquisition/covidcast/covidcast_meta_cache_updater.py index a46345b62..d5550312a 100644 --- a/src/acquisition/covidcast/covidcast_meta_cache_updater.py +++ b/src/acquisition/covidcast/covidcast_meta_cache_updater.py @@ -6,7 +6,7 @@ import time # first party -from delphi.epidata.acquisition.covidcast.database import Database +from .database_meta import DatabaseMeta from delphi.epidata.acquisition.covidcast.logger import get_structured_logger from delphi.epidata.client.delphi_epidata import Epidata @@ -19,7 +19,7 @@ def get_argument_parser(): return parser -def main(args, epidata_impl=Epidata, database_impl=Database): +def main(args, epidata_impl: Epidata = Epidata, database_impl: DatabaseMeta = DatabaseMeta): """Update the covidcast metadata cache. `args`: parsed command-line arguments diff --git a/src/acquisition/covidcast/covidcast_row.py b/src/acquisition/covidcast/covidcast_row.py new file mode 100644 index 000000000..af57b0b28 --- /dev/null +++ b/src/acquisition/covidcast/covidcast_row.py @@ -0,0 +1,269 @@ +from dataclasses import asdict, dataclass, field, fields +from datetime import date +from typing import Any, Dict, Iterable, List, Optional, Union, get_args, get_origin + +from delphi_utils import Nans +from numpy import isnan +from pandas import DataFrame, concat + +from .csv_importer import CsvImporter +from ...server.utils.dates import date_to_time_value, time_value_to_date + + +def _is_none(v: Optional[float]) -> bool: + return True if v is None or (v is not None and isnan(v)) else False + +@dataclass +class CovidcastRow: + """A container for (most of) the values of a single covidcast database row. + + Used for: + - inserting rows into the database + - creating test rows with default fields for testing + - created from many formats (dict, csv, df, kwargs) + - can be viewed in many formats (dict, csv, df) + + The rows are specified in 'v4_schema.sql'. + """ + + source: str = "src" + signal: str = "sig" + time_type: str = "day" + geo_type: str = "county" + time_value: int = 20200202 # Can be initialized with datetime.date + geo_value: str = "01234" + value: float = 10.0 + stderr: float = 10.0 + sample_size: float = 10.0 + missing_value: int = Nans.NOT_MISSING.value + missing_stderr: int = Nans.NOT_MISSING.value + missing_sample_size: int = Nans.NOT_MISSING.value + issue: Optional[int] = 20200202 # Can be initialized with datetime.date + lag: Optional[int] = 0 + id: Optional[int] = None + direction: Optional[int] = None + direction_updated_timestamp: int = 0 + value_updated_timestamp: int = 20200202 # Can be initialized with datetime.date + + def __post_init__(self): + # Convert time values to ints by default. + self.time_value = date_to_time_value(self.time_value) if isinstance(self.time_value, date) else self.time_value + self.issue = date_to_time_value(self.issue) if isinstance(self.issue, date) else self.issue + self.value_updated_timestamp = date_to_time_value(self.value_updated_timestamp) if isinstance(self.value_updated_timestamp, date) else self.value_updated_timestamp + + # These specify common views into this object: + # - 1. If this row was returned by an API request + self._api_row_ignore_fields = ["id", "direction", "direction_updated_timestamp", "value_updated_timestamp"] + # - 2. If this row was returned by an old API request (PHP server) + self._api_row_compatibility_ignore_fields = ["id", "direction", "direction_updated_timestamp", "value_updated_timestamp", "source"] + # - 3. If this row was returned by the database. + self._db_row_ignore_fields = [] + + def _sanity_check_fields(self, extra_checks: bool = True): + if self.issue and self.issue < self.time_value: + self.issue = self.time_value + + if self.issue: + self.lag = (time_value_to_date(self.issue) - time_value_to_date(self.time_value)).days + else: + self.lag = None + + # This sanity checking is already done in CsvImporter, but it's here so the testing class gets it too. + if _is_none(self.value) and self.missing_value == Nans.NOT_MISSING: + self.missing_value = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + + if _is_none(self.stderr) and self.missing_stderr == Nans.NOT_MISSING: + self.missing_stderr = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + + if _is_none(self.sample_size) and self.missing_sample_size == Nans.NOT_MISSING: + self.missing_sample_size = Nans.NOT_APPLICABLE.value if extra_checks else Nans.OTHER.value + + return self + + @staticmethod + def fromCsvRowValue(row_value: Optional[CsvImporter.RowValues], source: str, signal: str, time_type: str, geo_type: str, time_value: int, issue: int, lag: int): + if row_value is None: + return None + return CovidcastRow( + source, + signal, + time_type, + geo_type, + time_value, + row_value.geo_value, + row_value.value, + row_value.stderr, + row_value.sample_size, + row_value.missing_value, + row_value.missing_stderr, + row_value.missing_sample_size, + issue, + lag, + ) + + @staticmethod + def fromCsvRows(row_values: Iterable[Optional[CsvImporter.RowValues]], source: str, signal: str, time_type: str, geo_type: str, time_value: int, issue: int, lag: int): + # NOTE: returns a generator, as row_values is expected to be a generator + return (CovidcastRow.fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value, issue, lag) for row_value in row_values) + + @staticmethod + def from_json(json: Dict[str, Any]) -> "CovidcastRow": + return CovidcastRow( + source=json["source"], + signal=json["signal"], + time_type=json["time_type"], + geo_type=json["geo_type"], + geo_value=json["geo_value"], + issue=json["issue"], + lag=json["lag"], + value=json["value"], + stderr=json["stderr"], + sample_size=json["sample_size"], + missing_value=json["missing_value"], + missing_stderr=json["missing_stderr"], + missing_sample_size=json["missing_sample_size"], + ) + + def as_dict(self, ignore_fields: Optional[List[str]] = None) -> dict: + d = asdict(self) + if ignore_fields: + for key in ignore_fields: + del d[key] + return d + + def as_dataframe(self, ignore_fields: Optional[List[str]] = None) -> DataFrame: + return DataFrame.from_records([self.as_dict(ignore_fields=ignore_fields)]) + + @property + def api_row_df(self) -> DataFrame: + """Returns a dataframe view into the row with the fields returned by the API server.""" + return self.as_dataframe(ignore_fields=self._api_row_ignore_fields) + + @property + def api_compatibility_row_df(self) -> DataFrame: + """Returns a dataframe view into the row with the fields returned by the old API server (the PHP server).""" + return self.as_dataframe(ignore_fields=self._api_row_compatibility_ignore_fields) + + @property + def db_row_df(self) -> DataFrame: + """Returns a dataframe view into the row with the fields returned by an all-field database query.""" + return self.as_dataframe(ignore_fields=self._db_row_ignore_fields) + + @property + def signal_pair(self): + return f"{self.source}:{self.signal}" + + @property + def geo_pair(self): + return f"{self.geo_type}:{self.geo_value}" + + @property + def time_pair(self): + return f"{self.time_type}:{self.time_value}" + +# TODO: Deprecate this class in favor of functions over the List[CovidcastRow] datatype. +# All the inner variables of this class are derived from the CovidcastRow class. +@dataclass +class CovidcastRows: + rows: List[CovidcastRow] = field(default_factory=list) + + def __post_init__(self): + # These specify common views into this object: + # - 1. If this row was returned by an API request + self._api_row_ignore_fields = CovidcastRow()._api_row_ignore_fields + # - 2. If this row was returned by an old API request (PHP server) + self._api_row_compatibility_ignore_fields = CovidcastRow()._api_row_compatibility_ignore_fields + # - 3. If this row was returned by the database. + self._db_row_ignore_fields = CovidcastRow()._db_row_ignore_fields + + # Used to create a consistent DataFrame for tests. + dtypes = {field.name: field.type if get_origin(field.type) is not Union else get_args(field.type)[0] for field in fields(CovidcastRow)} + # Sometimes the int fields have None values, so we expand their scope using pandas.Int64DType. + self._DTYPES = {key: value if value is not int else "Int64" for key, value in dtypes.items()} + + @staticmethod + def from_args(sanity_check: bool = True, test_mode: bool = True, **kwargs: Dict[str, Iterable]): + """A convenience constructor. + + Handy for constructing batches of test cases. + + Example: + CovidcastRows.from_args(value=[1, 2, 3], time_value=[1, 2, 3]) will yield + CovidcastRows(rows=[CovidcastRow(value=1, time_value=1), CovidcastRow(value=2, time_value=2), CovidcastRow(value=3, time_value=3)]) + with all the defaults from CovidcastRow. + """ + # All the args must be fields of CovidcastRow. + assert set(kwargs.keys()) <= set(field.name for field in fields(CovidcastRow)) + + # If any iterables were passed instead of lists, convert them to lists. + kwargs = {key: list(value) for key, value in kwargs.items()} + + # All the arg values must be lists of the same length. + assert len(set(len(lst) for lst in kwargs.values())) == 1 + + return CovidcastRows(rows=[CovidcastRow(**_kwargs)._sanity_check_fields(extra_checks=test_mode) if sanity_check else CovidcastRow(**_kwargs) for _kwargs in transpose_dict(kwargs)]) + + @staticmethod + def from_records(records: Iterable[dict], sanity_check: bool = False): + """A convenience constructor. + + Default is different from from_args, because from_records is usually called on faux-API returns in tests, + where we don't want any values getting default filled in. + """ + records = list(records) + assert set().union(*[record.keys() for record in records]) <= set(field.name for field in fields(CovidcastRow)) + + return CovidcastRows(rows=[CovidcastRow(**record) if not sanity_check else CovidcastRow(**record)._sanity_check_fields() for record in records]) + + def as_dicts(self, ignore_fields: Optional[List[str]] = None) -> List[dict]: + return [row.as_dict(ignore_fields=ignore_fields) for row in self.rows] + + def as_dataframe(self, ignore_fields: Optional[List[str]] = None) -> DataFrame: + if ignore_fields is None: + ignore_fields = [] + columns = [field.name for field in fields(CovidcastRow) if field.name not in ignore_fields] + if self.rows: + df = concat([row.as_dataframe(ignore_fields=ignore_fields) for row in self.rows], ignore_index=True) + df = set_df_dtypes(df, self._DTYPES) + return df[columns] + else: + return DataFrame(columns=columns) + + @property + def api_row_df(self) -> DataFrame: + return self.as_dataframe(ignore_fields=self._api_row_ignore_fields) + + @property + def api_compatibility_row_df(self) -> DataFrame: + return self.as_dataframe(ignore_fields=self._api_row_compatibility_ignore_fields) + + @property + def db_row_df(self) -> DataFrame: + return self.as_dataframe(ignore_fields=self._db_row_ignore_fields) + + +def transpose_dict(d: Dict[Any, List[Any]]) -> List[Dict[Any, Any]]: + """Given a dictionary whose values are lists of the same length, turn it into a list of dictionaries whose values are the individual list entries. + + Example: + >>> transpose_dict(dict([["a", [2, 4, 6]], ["b", [3, 5, 7]], ["c", [10, 20, 30]]])) + [{"a": 2, "b": 3, "c": 10}, {"a": 4, "b": 5, "c": 20}, {"a": 6, "b": 7, "c": 30}] + """ + return [dict(zip(d.keys(), values)) for values in zip(*d.values())] + + +def set_df_dtypes(df: DataFrame, dtypes: Dict[str, Any]) -> DataFrame: + """Set the dataframe column datatypes. + df: pd.DataFrame + The dataframe to change. + dtypes: Dict[str, Any] + The keys of the dict are columns and the values are either types or Pandas + string aliases for types. Not all columns are required. + """ + assert all(isinstance(e, type) or isinstance(e, str) for e in dtypes.values()), "Values must be types or Pandas string aliases for types." + + df = df.copy() + for k, v in dtypes.items(): + if k in df.columns: + df[k] = df[k].astype(v) + return df diff --git a/src/acquisition/covidcast/csv_to_database.py b/src/acquisition/covidcast/csv_to_database.py index 34cbad663..0abe53f1f 100644 --- a/src/acquisition/covidcast/csv_to_database.py +++ b/src/acquisition/covidcast/csv_to_database.py @@ -7,7 +7,8 @@ # first party from delphi.epidata.acquisition.covidcast.csv_importer import CsvImporter -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow, DBLoadStateException +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow +from delphi.epidata.acquisition.covidcast.database import Database, DBLoadStateException from delphi.epidata.acquisition.covidcast.file_archiver import FileArchiver from delphi.epidata.acquisition.covidcast.logger import get_structured_logger diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 58631145a..86f78ac45 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -4,67 +4,14 @@ """ # third party -import json +from typing import Iterable, Sequence import mysql.connector -import numpy as np from math import ceil -from queue import Queue, Empty -import threading -from multiprocessing import cpu_count - -# first party import delphi.operations.secrets as secrets -from delphi.epidata.acquisition.covidcast.logger import get_structured_logger - -class CovidcastRow(): - """A container for all the values of a single covidcast row.""" - - @staticmethod - def fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value, issue, lag): - if row_value is None: return None - return CovidcastRow(source, signal, time_type, geo_type, time_value, - row_value.geo_value, - row_value.value, - row_value.stderr, - row_value.sample_size, - row_value.missing_value, - row_value.missing_stderr, - row_value.missing_sample_size, - issue, lag) - - @staticmethod - def fromCsvRows(row_values, source, signal, time_type, geo_type, time_value, issue, lag): - # NOTE: returns a generator, as row_values is expected to be a generator - return (CovidcastRow.fromCsvRowValue(row_value, source, signal, time_type, geo_type, time_value, issue, lag) - for row_value in row_values) - - def __init__(self, source, signal, time_type, geo_type, time_value, geo_value, value, stderr, - sample_size, missing_value, missing_stderr, missing_sample_size, issue, lag): - self.id = None - self.source = source - self.signal = signal - self.time_type = time_type - self.geo_type = geo_type - self.time_value = time_value - self.geo_value = geo_value # from CSV row - self.value = value # ... - self.stderr = stderr # ... - self.sample_size = sample_size # ... - self.missing_value = missing_value # ... - self.missing_stderr = missing_stderr # ... - self.missing_sample_size = missing_sample_size # from CSV row - self.direction_updated_timestamp = 0 - self.direction = None - self.issue = issue - self.lag = lag - - def signal_pair(self): - return f"{self.source}:{self.signal}" - - def geo_pair(self): - return f"{self.geo_type}:{self.geo_value}" +from .logger import get_structured_logger +from .covidcast_row import CovidcastRow class DBLoadStateException(Exception): @@ -74,29 +21,36 @@ class DBLoadStateException(Exception): class Database: """A collection of covidcast database operations.""" - DATABASE_NAME = 'covid' - - load_table = "epimetric_load" - # if you want to deal with foreign key ids: use table - # if you want to deal with source/signal names, geo type/values, etc: use view - latest_table = "epimetric_latest" - latest_view = latest_table + "_v" - history_table = "epimetric_full" - history_view = history_table + "_v" - # TODO: consider using class variables like this for dimension table names too - # TODO: also consider that for composite key tuples, like short_comp_key and long_comp_key as used in delete_batch() - - - def connect(self, connector_impl=mysql.connector): + def __init__(self): + self.load_table = "epimetric_load" + # if you want to deal with foreign key ids: use table + # if you want to deal with source/signal names, geo type/values, etc: use view + self.latest_table = "epimetric_latest" + self.latest_view = self.latest_table + "_v" + self.history_table = "epimetric_full" + self.history_view = self.history_table + "_v" + # TODO: consider using class variables like this for dimension table names too + # TODO: also consider that for composite key tuples, like short_comp_key and long_comp_key as used in delete_batch() + + self._connector_impl = mysql.connector + self._db_credential_user, self._db_credential_password = secrets.db.epi + self._db_host = secrets.db.host + self._db_database = 'covid' + + def connect(self, connector_impl=None, host=None, user=None, password=None, database=None): """Establish a connection to the database.""" + self._connector_impl = connector_impl if connector_impl is not None else self._connector_impl + self._db_host = host if host is not None else self._db_host + self._db_credential_user = user if user is not None else self._db_credential_user + self._db_credential_password = password if password is not None else self._db_credential_password + self._db_database = database if database is not None else self._db_database - u, p = secrets.db.epi - self._connector_impl = connector_impl self._connection = self._connector_impl.connect( - host=secrets.db.host, - user=u, - password=p, - database=Database.DATABASE_NAME) + host=self._db_host, + user=self._db_credential_user, + password=self._db_credential_password, + database=self._db_database + ) self._cursor = self._connection.cursor() def commit(self): @@ -116,8 +70,6 @@ def disconnect(self, commit): self._connection.commit() self._connection.close() - - def count_all_load_rows(self): self._cursor.execute(f'SELECT count(1) FROM `{self.load_table}`') for (num,) in self._cursor: @@ -153,10 +105,10 @@ def do_analyze(self): output = [self._cursor.column_names] + self._cursor.fetchall() get_structured_logger('do_analyze').info("ANALYZE results", results=str(output)) - def insert_or_update_bulk(self, cc_rows): + def insert_or_update_bulk(self, cc_rows: Iterable[CovidcastRow]): return self.insert_or_update_batch(cc_rows) - def insert_or_update_batch(self, cc_rows, batch_size=2**20, commit_partial=False, suppress_jobs=False): + def insert_or_update_batch(self, cc_rows: Sequence[CovidcastRow], batch_size: int = 2**20, commit_partial: bool = False, suppress_jobs: bool = False): """ Insert new rows into the load table and dispatch into dimension and fact tables. """ @@ -489,133 +441,3 @@ def split_list(lst, n): finally: self._cursor.execute(drop_tmp_table_sql) return total - - - def compute_covidcast_meta(self, table_name=None, n_threads=None): - """Compute and return metadata on all COVIDcast signals.""" - logger = get_structured_logger("compute_covidcast_meta") - - if table_name is None: - table_name = self.latest_view - - if n_threads is None: - logger.info("n_threads unspecified, automatically choosing based on number of detected cores...") - n_threads = max(1, cpu_count()*9//10) # aka number of concurrent db connections, which [sh|c]ould be ~<= 90% of the #cores available to SQL server - # NOTE: this may present a small problem if this job runs on different hardware than the db, - # which is why this value can be overriden by optional argument. - logger.info(f"using {n_threads} workers") - - srcsigs = Queue() # multi-consumer threadsafe! - sql = f'SELECT `source`, `signal` FROM `{table_name}` GROUP BY `source`, `signal` ORDER BY `source` ASC, `signal` ASC;' - self._cursor.execute(sql) - for source, signal in self._cursor: - srcsigs.put((source, signal)) - - inner_sql = f''' - SELECT - `source` AS `data_source`, - `signal`, - `time_type`, - `geo_type`, - MIN(`time_value`) AS `min_time`, - MAX(`time_value`) AS `max_time`, - COUNT(DISTINCT `geo_value`) AS `num_locations`, - MIN(`value`) AS `min_value`, - MAX(`value`) AS `max_value`, - ROUND(AVG(`value`),7) AS `mean_value`, - ROUND(STD(`value`),7) AS `stdev_value`, - MAX(`value_updated_timestamp`) AS `last_update`, - MAX(`issue`) as `max_issue`, - MIN(`lag`) as `min_lag`, - MAX(`lag`) as `max_lag` - FROM - `{table_name}` - WHERE - `source` = %s AND - `signal` = %s - GROUP BY - `time_type`, - `geo_type` - ORDER BY - `time_type` ASC, - `geo_type` ASC - ''' - - meta = [] - meta_lock = threading.Lock() - - def worker(): - name = threading.current_thread().name - logger.info("starting thread", thread=name) - # set up new db connection for thread - worker_dbc = Database() - worker_dbc.connect(connector_impl=self._connector_impl) - w_cursor = worker_dbc._cursor - try: - while True: - (source, signal) = srcsigs.get_nowait() # this will throw the Empty caught below - logger.info("starting pair", thread=name, pair=f"({source}, {signal})") - w_cursor.execute(inner_sql, (source, signal)) - with meta_lock: - meta.extend(list( - dict(zip(w_cursor.column_names, x)) for x in w_cursor - )) - srcsigs.task_done() - except Empty: - logger.info("no jobs left, thread terminating", thread=name) - finally: - worker_dbc.disconnect(False) # cleanup - - threads = [] - for n in range(n_threads): - t = threading.Thread(target=worker, name='MetacacheThread-'+str(n)) - t.start() - threads.append(t) - - srcsigs.join() - logger.info("jobs complete") - for t in threads: - t.join() - logger.info("all threads terminated") - - # sort the metadata because threaded workers dgaf - sorting_fields = "data_source signal time_type geo_type".split() - sortable_fields_fn = lambda x: [(field, x[field]) for field in sorting_fields] - prepended_sortables_fn = lambda x: sortable_fields_fn(x) + list(x.items()) - tuple_representation = list(map(prepended_sortables_fn, meta)) - tuple_representation.sort() - meta = list(map(dict, tuple_representation)) # back to dict form - - return meta - - - def update_covidcast_meta_cache(self, metadata): - """Updates the `covidcast_meta_cache` table.""" - - sql = ''' - UPDATE - `covidcast_meta_cache` - SET - `timestamp` = UNIX_TIMESTAMP(NOW()), - `epidata` = %s - ''' - epidata_json = json.dumps(metadata) - - self._cursor.execute(sql, (epidata_json,)) - - def retrieve_covidcast_meta_cache(self): - """Useful for viewing cache entries (was used in debugging)""" - - sql = ''' - SELECT `epidata` - FROM `covidcast_meta_cache` - ORDER BY `timestamp` DESC - LIMIT 1; - ''' - self._cursor.execute(sql) - cache_json = self._cursor.fetchone()[0] - cache = json.loads(cache_json) - cache_hash = {} - for entry in cache: - cache_hash[(entry['data_source'], entry['signal'], entry['time_type'], entry['geo_type'])] = entry - return cache_hash diff --git a/src/acquisition/covidcast/database_meta.py b/src/acquisition/covidcast/database_meta.py new file mode 100644 index 000000000..22f7a4d80 --- /dev/null +++ b/src/acquisition/covidcast/database_meta.py @@ -0,0 +1,448 @@ +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass, fields +from datetime import datetime +import json +from multiprocessing import cpu_count +from queue import Queue, Empty +import threading +from typing import Dict, Iterator, Optional, Tuple + +import pandas as pd +from requests import get + +# TODO: Switch to epidatpy when we release it https://github.com/cmu-delphi/delphi-epidata/issues/942. +# from epidatpy.request import Epidata, EpiRange + +from .logger import get_structured_logger +from .covidcast_row import CovidcastRow, set_df_dtypes +from .database import Database +from .config import GEO_TYPES, ALL_TIME +from ...server.endpoints.covidcast_utils.model import DataSignal, data_signals_by_key + +@dataclass +class MetaTableRow: + data_source: str + signal: str + time_type: str + geo_type: str + min_time: int + max_time: int + num_locations: int + min_value: float + max_value: float + mean_value: float + stdev_value: float + last_update: int + max_issue: int + min_lag: int + max_lag: int + + def as_df(self): + df = pd.DataFrame( + { + "data_source": self.data_source, + "signal": self.signal, + "time_type": self.time_type, + "geo_type": self.geo_type, + "min_time": self.min_time, + "max_time": self.max_time, + "num_locations": self.num_locations, + "min_value": self.min_value, + "max_value": self.max_value, + "mean_value": self.mean_value, + "stdev_value": self.stdev_value, + "last_update": self.last_update, + "max_issue": self.max_issue, + "min_lag": self.min_lag, + "max_lag": self.max_lag, + }, + index=[0], + ) + set_df_dtypes( + df, + dtypes={ + "data_source": str, + "signal": str, + "time_type": str, + "geo_type": str, + "min_time": int, + "max_time": int, + "num_locations": int, + "min_value": float, + "max_value": float, + "mean_value": float, + "stdev_value": float, + "last_update": int, + "max_issue": int, + "min_lag": int, + "max_lag": int, + }, + ) + return df + + def as_dict(self): + return asdict(self) + + @staticmethod + def _extract_fields(group_df): + if "source" in group_df.columns: + assert group_df["source"].unique().size == 1 + source = group_df["source"].iloc[0] + elif "data_source" in group_df.columns: + assert group_df["data_source"].unique().size == 1 + source = group_df["data_source"].iloc[0] + else: + raise ValueError("Source name not found in group_df.") + + if "signal" in group_df.columns: + assert group_df["signal"].unique().size == 1 + signal = group_df["signal"].iloc[0] + else: + raise ValueError("Signal name not found in group_df.") + + if "time_type" in group_df.columns: + assert group_df["time_type"].unique().size == 1 + time_type = group_df["time_type"].iloc[0] + else: + raise ValueError("Time type not found in group_df.") + + if "geo_type" in group_df.columns: + assert group_df["geo_type"].unique().size == 1 + geo_type = group_df["geo_type"].iloc[0] + else: + raise ValueError("Geo type not found in group_df.") + + if "value_updated_timestamp" in group_df.columns: + last_updated = max(group_df["value_updated_timestamp"]) + else: + last_updated = int(datetime.now().timestamp()) + + return source, signal, time_type, geo_type, last_updated + + @staticmethod + def from_group_df(group_df): + if group_df is None or group_df.empty: + raise ValueError("Empty group_df given.") + + source, signal, time_type, geo_type, last_updated = MetaTableRow._extract_fields(group_df) + + return MetaTableRow( + data_source=source, + signal=signal, + time_type=time_type, + geo_type=geo_type, + min_time=min(group_df["time_value"]), + max_time=max(group_df["time_value"]), + num_locations=len(group_df["geo_value"].unique()), + min_value=min(group_df["value"]), + max_value=max(group_df["value"]), + mean_value=group_df["value"].mean().round(7), + stdev_value=group_df["value"].std(ddof=0).round(7), + last_update=last_updated, + max_issue=max(group_df["issue"]), + min_lag=min(group_df["lag"]), + max_lag=max(group_df["lag"]), + ) + +class DatabaseMeta(Database): + # TODO: Verify the correct base_url for a local API server. + def __init__(self, base_url: str = "http://localhost/epidata") -> "DatabaseMeta": + Database.__init__(self) + self.epidata_base_url = base_url + # TODO: Switch to epidatpy when we release it https://github.com/cmu-delphi/delphi-epidata/issues/942. + self.delphi_epidata = False + + def compute_covidcast_meta(self, table_name: Optional[str] = None, jit: bool = False, parallel: bool = True, n_threads: Optional[int] = None): + """This wrapper is here for A/B testing the JIT and non-JIT metadata computation.""" + if jit: + return self.compute_covidcast_meta_jit(table_name, parallel, n_threads) + else: + return self.compute_covidcast_meta_non_jit(table_name, n_threads) + + def compute_covidcast_meta_non_jit(self, table_name: Optional[str] = None, n_threads: Optional[int] = None): + """This is the old method (not using JIT) to compute and return metadata on all COVIDcast signals. + + This is here for A/B testing and for safety, in case we need to revert JIT. + # TODO: Remove eventually. + """ + logger = get_structured_logger("compute_covidcast_meta") + + if table_name is None: + table_name = self.latest_view + + if n_threads is None: + # aka number of concurrent db connections, which [sh|c]ould be ~<= 90% of the #cores available to SQL server + n_threads = max(1, cpu_count() * 9 // 10) + + logger.info(f"using {n_threads} workers") + + srcsigs = Queue() # multi-consumer threadsafe! + sql = f"SELECT `source`, `signal` FROM `{table_name}` GROUP BY `source`, `signal` ORDER BY `source` ASC, `signal` ASC;" + self._cursor.execute(sql) + for source, signal in self._cursor: + srcsigs.put((source, signal)) + + inner_sql = f""" + SELECT + `source` AS `data_source`, + `signal`, + `time_type`, + `geo_type`, + MIN(`time_value`) AS `min_time`, + MAX(`time_value`) AS `max_time`, + COUNT(DISTINCT `geo_value`) AS `num_locations`, + MIN(`value`) AS `min_value`, + MAX(`value`) AS `max_value`, + ROUND(AVG(`value`),7) AS `mean_value`, + ROUND(STD(`value`),7) AS `stdev_value`, + MAX(`value_updated_timestamp`) AS `last_update`, + MAX(`issue`) as `max_issue`, + MIN(`lag`) as `min_lag`, + MAX(`lag`) as `max_lag` + FROM + `{table_name}` + WHERE + `source` = %s AND + `signal` = %s + GROUP BY + `time_type`, + `geo_type` + ORDER BY + `time_type` ASC, + `geo_type` ASC + """ + + meta = [] + meta_lock = threading.Lock() + + def worker(): + name = threading.current_thread().name + logger.info("starting thread", thread=name) + # set up new db connection for thread + worker_dbc = Database() + worker_dbc.connect(connector_impl=self._connector_impl, host=self._db_host, user=self._db_credential_user, password=self._db_credential_password, database=self._db_database) + w_cursor = worker_dbc._cursor + try: + while True: + (source, signal) = srcsigs.get_nowait() # this will throw the Empty caught below + logger.info("starting pair", thread=name, pair=f"({source}, {signal})") + w_cursor.execute(inner_sql, (source, signal)) + with meta_lock: + meta.extend(list(dict(zip(w_cursor.column_names, x)) for x in w_cursor)) + srcsigs.task_done() + except Empty: + logger.info("no jobs left, thread terminating", thread=name) + finally: + worker_dbc.disconnect(False) # cleanup + + threads = [] + for n in range(n_threads): + t = threading.Thread(target=worker, name="MetacacheThread-" + str(n)) + t.start() + threads.append(t) + + srcsigs.join() + logger.info("jobs complete") + for t in threads: + t.join() + logger.info("all threads terminated") + + # sort the metadata because threaded workers dgaf + sorting_fields = "data_source signal time_type geo_type".split() + sortable_fields_fn = lambda x: [(field, x[field]) for field in sorting_fields] + prepended_sortables_fn = lambda x: sortable_fields_fn(x) + list(x.items()) + tuple_representation = list(map(prepended_sortables_fn, meta)) + tuple_representation.sort() + meta = list(map(dict, tuple_representation)) # back to dict form + + return meta + + def get_source_sig_list(self, data_signal_table: Dict[Tuple[str, str], DataSignal] = data_signals_by_key, derived: bool = False) -> Iterator[Tuple[str]]: + """Return the source-signal pair names from the database. + + The derived flag determines whether the signals returned are derived or base signals. + """ + inner_sql = f"""SELECT `source`, `signal` FROM `epimetric_latest_v` WHERE `source` = %s AND `signal` = %s LIMIT 1""" + + lst = ((data_signal.source, data_signal.signal) for data_signal in data_signal_table.values() if data_signal.compute_from_base == derived) + for source, signal in lst: + self._cursor.execute(inner_sql, (source, signal)) + if self._cursor.fetchall(): + yield (source, signal) + + def compute_base_signal_meta(self, source: str, signal: str, table_name: str = None) -> pd.DataFrame: + """Compute the meta information for base signals. + + A base signal is a signal whose values do not depend on another signal. A derived signal is one whose values are obtained + through a transformation of a base signal, e.g. a 7 day average signal or an incidence (compared to cumulative) signal. + """ + if table_name is None: + table_name = self.latest_view + + inner_sql = f""" + SELECT + `source` AS `data_source`, + `signal`, + `time_type`, + `geo_type`, + MIN(`time_value`) AS `min_time`, + MAX(`time_value`) AS `max_time`, + COUNT(DISTINCT `geo_value`) AS `num_locations`, + MIN(`value`) AS `min_value`, + MAX(`value`) AS `max_value`, + ROUND(AVG(`value`),7) AS `mean_value`, + ROUND(STD(`value`),7) AS `stdev_value`, + MAX(`value_updated_timestamp`) AS `last_update`, + MAX(`issue`) as `max_issue`, + MIN(`lag`) as `min_lag`, + MAX(`lag`) as `max_lag` + FROM + `{table_name}` + WHERE + `source` = %s AND + `signal` = %s + GROUP BY + `time_type`, + `geo_type` + ORDER BY + `time_type` ASC, + `geo_type` ASC + """ + + self._cursor.execute(inner_sql, (source, signal)) + expected_columns = ["data_source", "signal", "time_type", "geo_type", "min_time", "max_time", "num_locations", "min_value", "max_value", "mean_value", "stdev_value", "last_update", "max_issue", "min_lag", "max_lag"] + base_signal_meta = pd.DataFrame(self._cursor.fetchall(), columns=expected_columns) + return base_signal_meta + + def compute_derived_signal_meta(self, source: str, signal: str, base_signal_meta: pd.DataFrame, data_signal_table: Dict[Tuple[str, str], DataSignal] = data_signals_by_key) -> pd.DataFrame: + """Compute the meta information for a derived signal. + + A derived signal is a transformation of a base signal. Since derived signals are not stored in the database, but are computed + on the fly by the API, we call the API here. It is assumed that we have already computed the meta information for the base + signals and passed that in base_signal_meta. The latter is needed to set the `last_updated` field. + """ + logger = get_structured_logger("get_derived_signal_meta") + + meta_table_columns = [field.name for field in fields(MetaTableRow)] + covidcast_response_columns = [field.name for field in fields(CovidcastRow) if field.name not in CovidcastRow()._api_row_ignore_fields] + + # We should be able to find the signal in our table. + data_signal = data_signal_table.get((source, signal)) + if not data_signal: + logger.warn(f"Could not find the requested derived signal {source}:{signal} in the data signal table. Returning no meta results.") + return pd.DataFrame(columns=meta_table_columns) + + # Request all the data for the derived signal. + # TODO: Use when epidatpy is released https://github.com/cmu-delphi/delphi-epidata/issues/942. + if self.delphi_epidata: + raise NotImplemented("Use the old epidata client for now.") + # TODO: Consider refactoring to combine multiple signal requests in one call. + all_time = EpiRange(19900101, 20400101) + epidata = Epidata.with_base_url(self.epidata_base_url) + api_response_df = pd.concat([epidata.covidcast(data_source=source, signals=signal, time_type=data_signal.time_type, geo_type=geo_type, time_values=all_time, geo_values="*").df() for geo_type in GEO_TYPES]) + else: + base_url = f"{self.epidata_base_url}/covidcast/" + params = {"data_source": source, "signals": signal, "time_type": data_signal.time_type, "time_values": ALL_TIME, "geo_values": "*"} + signal_data_dfs = [] + for geo_type in GEO_TYPES: + params.update({"geo_type": geo_type}) + response = get(base_url, params) + if response.status_code in [200]: + signal_data_dfs.append(pd.DataFrame.from_records(response.json()['epidata'], columns=covidcast_response_columns)) + else: + raise Exception(f"The API responded with an error when attempting to get data for the derived signal's {source}:{signal} meta computation. There may be an issue with the API server.") + + # Group the data by time_type and geo_type and find the statistical summaries for their values. + meta_rows = [MetaTableRow.from_group_df(group_df).as_df() for signal_data_df in signal_data_dfs for _, group_df in signal_data_df.groupby("time_type")] + if meta_rows: + meta_df = pd.concat(meta_rows) + else: + logger.warn(f"The meta computation for {source}:{signal} returned no summary statistics. There may be an issue with the API server or the database.") + return pd.DataFrame(columns=meta_table_columns) + + # Copy the value of 'last_updated' column from the base signal meta to the derived signal meta. + # TODO: Remove if/when we remove the 'last_updated' column. + meta_df = pd.merge( + meta_df.assign(parent_signal = data_signal.signal_basename), + base_signal_meta[["data_source", "signal", "time_type", "geo_type", "last_update"]], + left_on = ["data_source", "parent_signal", "time_type", "geo_type"], + right_on = ["data_source", "signal", "time_type", "geo_type"] + ).assign( + signal = lambda x: x["signal_x"], + last_update = lambda x: x["last_update_y"] + ) + + return meta_df[meta_table_columns] + + def compute_covidcast_meta_jit(self, table_name: Optional[str] = None, parallel: bool = True, n_threads: Optional[int] = None, data_signal_table: Dict[Tuple[str, str], DataSignal] = data_signals_by_key) -> Dict: + """Compute and return metadata on all non-WIP COVIDcast signals.""" + logger = get_structured_logger("compute_covidcast_meta") + + if table_name is None: + table_name = self.latest_view + + if parallel: + if n_threads is None: + # aka number of concurrent db connections, which [sh|c]ould be ~<= 90% of the #cores available to SQL server + n_threads = max(1, cpu_count() * 9 // 10) + logger.info(f"using {n_threads} workers") + + def worker_base(source: str, signal: str) -> pd.DataFrame: + # set up new db connection for thread + logger.info("starting thread", thread=f"{source}:{signal}") + worker_dbc = DatabaseMeta() + worker_dbc.connect(connector_impl=self._connector_impl, host=self._db_host, user=self._db_credential_user, password=self._db_credential_password, database=self._db_database) + df = worker_dbc.compute_base_signal_meta(source, signal, table_name) + worker_dbc.disconnect(False) + return df + + base_meta_dfs = [] + with ThreadPoolExecutor(max_workers=n_threads) as executor: + future = {executor.submit(worker_base, source, signal) for source, signal in self.get_source_sig_list(data_signal_table=data_signal_table, derived=False)} + for f in as_completed(future): + base_meta_dfs.append(f.result()) + + base_meta_df = pd.concat(base_meta_dfs) + + logger.info("jobs complete") + else: + # Here to illustrate the simple logic behind meta computations without the parallel boilerplate + base_meta_df = pd.concat([self.compute_base_signal_meta(source, signal, table_name) for source, signal in self.get_source_sig_list(data_signal_table=data_signal_table, derived=False)]) + + # Parallelization approach above doesn't work because urllib3.connection.HTTPConnection throws a "Failed to establish new connection" error. + # Even tried using requests.Session() to reuse the same connection, but that didn't work either. + derived_meta_df = pd.concat([self.compute_derived_signal_meta(source, signal, base_meta_df, data_signal_table=data_signal_table) for source, signal in self.get_source_sig_list(data_signal_table=data_signal_table, derived=True)]) + + meta_df = pd.concat([base_meta_df, derived_meta_df]).sort_values(by="data_source signal time_type geo_type".split()) + return meta_df.to_dict(orient="records") + + def update_covidcast_meta_cache(self, metadata): + """Updates the `covidcast_meta_cache` table.""" + + sql = """ + UPDATE + `covidcast_meta_cache` + SET + `timestamp` = UNIX_TIMESTAMP(NOW()), + `epidata` = %s + """ + epidata_json = json.dumps(metadata) + + self._cursor.execute(sql, (epidata_json,)) + + def retrieve_covidcast_meta_cache(self): + """Useful for viewing cache entries (was used in debugging)""" + + sql = """ + SELECT `epidata` + FROM `covidcast_meta_cache` + ORDER BY `timestamp` DESC + LIMIT 1; + """ + self._cursor.execute(sql) + cache_json = self._cursor.fetchone()[0] + cache = json.loads(cache_json) + cache_hash = {} + for entry in cache: + cache_hash[(entry["data_source"], entry["signal"], entry["time_type"], entry["geo_type"])] = entry + return cache_hash diff --git a/src/acquisition/covidcast/test_utils.py b/src/acquisition/covidcast/test_utils.py index 181dfac68..45f9fbfd0 100644 --- a/src/acquisition/covidcast/test_utils.py +++ b/src/acquisition/covidcast/test_utils.py @@ -1,7 +1,9 @@ +from typing import Sequence import unittest from delphi_utils import Nans -from delphi.epidata.acquisition.covidcast.database import Database, CovidcastRow +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRow +from delphi.epidata.acquisition.covidcast.database import Database import delphi.operations.secrets as secrets # all the Nans we use here are just one value, so this is a shortcut to it: @@ -31,36 +33,20 @@ def tearDown(self): # close and destroy conenction to the database self._db.disconnect(False) del self._db + self.localTearDown() - DEFAULT_TIME_VALUE=2000_01_01 - DEFAULT_ISSUE=2000_01_01 - def _make_placeholder_row(self, **kwargs): - settings = { - 'source': 'src', - 'signal': 'sig', - 'geo_type': 'state', - 'geo_value': 'pa', - 'time_type': 'day', - 'time_value': self.DEFAULT_TIME_VALUE, - 'value': 0.0, - 'stderr': 1.0, - 'sample_size': 2.0, - 'missing_value': nmv, - 'missing_stderr': nmv, - 'missing_sample_size': nmv, - 'issue': self.DEFAULT_ISSUE, - 'lag': 0 - } - settings.update(kwargs) - return (CovidcastRow(**settings), settings) + def localTearDown(self): + # stub; override in subclasses to perform custom teardown. + # runs after database changes have been committed + pass - def _insert_rows(self, rows): + def _insert_rows(self, rows: Sequence[CovidcastRow]): # inserts rows into the database using the full acquisition process, including 'dbjobs' load into history & latest tables n = self._db.insert_or_update_bulk(rows) print(f"{n} rows added to load table & dispatched to v4 schema") self._db._connection.commit() # NOTE: this isnt expressly needed for our test cases, but would be if using external access (like through client lib) to ensure changes are visible outside of this db session - def params_from_row(self, row, **kwargs): + def params_from_row(self, row: CovidcastRow, **kwargs): ret = { 'data_source': row.source, 'signals': row.signal, @@ -71,13 +57,3 @@ def params_from_row(self, row, **kwargs): } ret.update(kwargs) return ret - - DEFAULT_MINUS=['time_type', 'geo_type', 'source'] - def expected_from_row(self, row, minus=DEFAULT_MINUS): - expected = dict(vars(row)) - # remove columns commonly excluded from output - # nb may need to add source or *_type back in for multiplexed queries - for key in ['id', 'direction_updated_timestamp'] + minus: - del expected[key] - return expected - diff --git a/src/server/_config.py b/src/server/_config.py index 47688a8ef..7973f9872 100644 --- a/src/server/_config.py +++ b/src/server/_config.py @@ -9,6 +9,7 @@ MAX_RESULTS = int(10e6) MAX_COMPATIBILITY_RESULTS = int(3650) +MAX_SMOOTHER_WINDOW = 30 SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", "sqlite:///test.db") diff --git a/src/server/_params.py b/src/server/_params.py index fa4f63483..74ee8540d 100644 --- a/src/server/_params.py +++ b/src/server/_params.py @@ -1,10 +1,11 @@ -from math import inf import re from dataclasses import dataclass +from itertools import groupby +from math import inf from typing import List, Optional, Sequence, Tuple, Union from flask import request - +from more_itertools import flatten from ._exceptions import ValidationFailedException from .utils import days_in_range, weeks_in_range, guess_time_value_is_day @@ -92,9 +93,35 @@ def count(self) -> float: return inf if self.signal else 0 return len(self.signal) + def add_signal(self, signal: str) -> None: + if not isinstance(self.signal, bool): + self.signal.append(signal) + + def __hash__(self) -> int: + return hash((self.source, self.signal if self.signal is isinstance(self.signal, bool) else tuple(self.signal))) + + +def _combine_source_signal_pairs(source_signal_pairs: List[SourceSignalPair]) -> List[SourceSignalPair]: + """Combine SourceSignalPairs with the same source into a single SourceSignalPair object. + + Example: + [SourceSignalPair("src", ["sig1", "sig2"]), SourceSignalPair("src", ["sig2", "sig3"])] will be merged + into [SourceSignalPair("src", ["sig1", "sig2", "sig3])]. + """ + source_signal_pairs_grouped = groupby(sorted(source_signal_pairs, key=lambda x: x.source), lambda x: x.source) + source_signal_pairs_combined = [] + for source, group in source_signal_pairs_grouped: + group = list(group) + if any(x.signal == True for x in group): + combined_signals = True + else: + combined_signals = sorted(set(flatten(x.signal for x in group))) + source_signal_pairs_combined.append(SourceSignalPair(source, combined_signals)) + return source_signal_pairs_combined + def parse_source_signal_arg(key: str = "signal") -> List[SourceSignalPair]: - return [SourceSignalPair(source, signals) for [source, signals] in _parse_common_multi_arg(key)] + return _combine_source_signal_pairs([SourceSignalPair(source, signals) for [source, signals] in _parse_common_multi_arg(key)]) def parse_single_source_signal_arg(key: str) -> SourceSignalPair: @@ -110,6 +137,10 @@ class TimePair: time_type: str time_values: Union[bool, Sequence[Union[int, Tuple[int, int]]]] + def __post_init__(self): + if isinstance(self.time_values, list): + self.time_values = [(min(time_value), max(time_value)) if isinstance(time_value, tuple) else time_value for time_value in self.time_values] + @property def is_week(self) -> bool: return self.time_type == 'week' diff --git a/src/server/_validate.py b/src/server/_validate.py index 3b91e5570..af4f3e4d0 100644 --- a/src/server/_validate.py +++ b/src/server/_validate.py @@ -189,3 +189,13 @@ def push_range(first: str, last: str): values.append(parse_date(part)) # success, return the list return values + +def extract_bool(key: Union[str, Sequence[str]]) -> Optional[bool]: + s = _extract_value(key) + if not s: + return None + if s.lower() == "true": + return True + if s.lower() == "false": + return False + raise ValidationFailedException(f"{key}: not a boolean: {s}") diff --git a/src/server/endpoints/covidcast.py b/src/server/endpoints/covidcast.py index 4a636d891..842fd78a8 100644 --- a/src/server/endpoints/covidcast.py +++ b/src/server/endpoints/covidcast.py @@ -1,14 +1,18 @@ +from numbers import Number from typing import List, Optional, Union, Tuple, Dict, Any from itertools import groupby from datetime import date, timedelta +from bisect import bisect_right from epiweeks import Week from flask import Blueprint, request from flask.json import loads, jsonify -from bisect import bisect_right +from more_itertools import peekable +from numpy import nan from sqlalchemy import text from pandas import read_csv, to_datetime from .._common import is_compatibility_mode, db +from .._config import MAX_SMOOTHER_WINDOW from .._exceptions import ValidationFailedException, DatabaseErrorException from .._params import ( GeoPair, @@ -26,6 +30,7 @@ from .._query import QueryBuilder, execute_query, run_query, parse_row, filter_fields from .._printer import create_printer, CSVPrinter from .._validate import ( + extract_bool, extract_date, extract_dates, extract_integer, @@ -36,11 +41,13 @@ from .._pandas import as_pandas, print_pandas from .covidcast_utils import compute_trend, compute_trends, compute_correlations, compute_trend_value, CovidcastMetaEntry from ..utils import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, shift_week_value, week_value_to_week, guess_time_value_is_day, week_to_time_value -from .covidcast_utils.model import TimeType, count_signal_time_types, data_sources, create_source_signal_alias_mapper +from .covidcast_utils.model import TimeType, count_signal_time_types, data_sources, create_source_signal_alias_mapper, get_basename_signal_and_jit_generator, get_pad_length, pad_time_pairs, pad_time_window +from .covidcast_utils.smooth_diff import SmootherKernelValue # first argument is the endpoint name bp = Blueprint("covidcast", __name__) alias = None +JIT_COMPUTE = True latest_table = "epimetric_latest_v" history_table = "epimetric_full_v" @@ -81,12 +88,17 @@ def parse_time_pairs() -> List[TimePair]: # old version require_all("time_type", "time_values") time_values = extract_dates("time_values") + # TODO: Put a bound on the number of time_values? + # if time_values and len(time_values) > 30: + # raise ValidationFailedException("parameter value exceed: too many time pairs requested, consider using a timerange instead YYYYMMDD-YYYYMMDD") return [TimePair(time_type, time_values)] if ":" not in request.values.get("time", ""): raise ValidationFailedException("missing parameter: time or (time_type and time_values)") - return parse_time_arg() + time_pairs = parse_time_arg() + # TODO: Put a bound on the number of time_values? (see above) + return time_pairs def _handle_lag_issues_as_of(q: QueryBuilder, issues: Optional[List[Union[Tuple[int, int], int]]] = None, lag: Optional[int] = None, as_of: Optional[int] = None): @@ -111,50 +123,115 @@ def _handle_lag_issues_as_of(q: QueryBuilder, issues: Optional[List[Union[Tuple[ pass +def parse_transform_args(): + # The length of the window to smooth over. + smoother_window_length = extract_integer("smoother_window_length") + if smoother_window_length is None: + smoother_window_length = 7 + + # TODO: Add support for floats inputs here. + # The value to fill for missing date values. + pad_fill_value = extract_integer("pad_fill_value") + if pad_fill_value is None: + pad_fill_value = nan + + # The value to fill for None or nan values. + nan_fill_value = extract_integer("nans_fill_value") + if nan_fill_value is None: + nan_fill_value = nan + + smoother_args = { + "smoother_kernel": SmootherKernelValue.average, + "smoother_window_length": smoother_window_length if isinstance(smoother_window_length, Number) and smoother_window_length <= MAX_SMOOTHER_WINDOW else MAX_SMOOTHER_WINDOW, + "pad_fill_value": pad_fill_value if isinstance(pad_fill_value, Number) else nan, + "nans_fill_value": nan_fill_value if isinstance(nan_fill_value, Number) else nan + } + return smoother_args + + +def parse_jit_bypass(): + jit_bypass = extract_bool("jit_bypass") + if jit_bypass is None: + return False + else: + return jit_bypass + + @bp.route("/", methods=("GET", "POST")) def handle(): source_signal_pairs = parse_source_signal_pairs() source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) time_pairs = parse_time_pairs() geo_pairs = parse_geo_pairs() + jit_bypass = parse_jit_bypass() as_of = extract_date("as_of") issues = extract_dates("issues") lag = extract_integer("lag") + is_time_type_week = any(time_pair.time_type == "week" for time_pair in time_pairs) + is_time_value_true = any(isinstance(time_pair.time_values, bool) for time_pair in time_pairs) + + is_compatibility = is_compatibility_mode() + def alias_row(row): + if is_compatibility: + # old api returned fewer fields + remove_fields = ["geo_type", "source", "time_type"] + for field in remove_fields: + if field in row: + del row[field] + if is_compatibility or not alias_mapper or "source" not in row: + return row + row["source"] = alias_mapper(row["source"], row["signal"]) + return row # build query q = QueryBuilder(latest_table, "t") - fields_string = ["geo_value", "signal"] + fields_string = ["geo_type", "geo_value", "source", "signal", "time_type"] fields_int = ["time_value", "direction", "issue", "lag", "missing_value", "missing_stderr", "missing_sample_size"] fields_float = ["value", "stderr", "sample_size"] - is_compatibility = is_compatibility_mode() - if is_compatibility: - q.set_order("signal", "time_value", "geo_value", "issue") + + # TODO: JIT computations don't support time_value = *; there may be a clever way to implement this. + use_server_side_compute = not any((issues, lag, is_time_type_week, is_time_value_true)) and JIT_COMPUTE and not jit_bypass + if use_server_side_compute: + transform_args = parse_transform_args() + pad_length = get_pad_length(source_signal_pairs, transform_args.get("smoother_window_length")) + time_pairs = pad_time_pairs(time_pairs, pad_length) + source_signal_pairs, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs, transform_args=transform_args) + + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + transformed_rows = row_transform_generator(parsed_rows=parsed_rows, time_pairs=time_pairs, transform_args=transform_args) + for row in transformed_rows: + yield alias_row(row) else: - # transfer also the new detail columns - fields_string.extend(["source", "geo_type", "time_type"]) - q.set_order("source", "signal", "time_type", "time_value", "geo_type", "geo_value", "issue") + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + for row in parsed_rows: + yield alias_row(row) + + q.set_order("geo_type", "geo_value", "source", "signal", "time_type", "time_value", "issue") q.set_fields(fields_string, fields_int, fields_float) # basic query info # data type of each field # build the source, signal, time, and location (type and id) filters - q.where_source_signal_pairs("source", "signal", source_signal_pairs) q.where_geo_pairs("geo_type", "geo_value", geo_pairs) q.where_time_pairs("time_type", "time_value", time_pairs) _handle_lag_issues_as_of(q, issues, lag, as_of) - def transform_row(row, proxy): - if is_compatibility or not alias_mapper or "source" not in row: - return row - row["source"] = alias_mapper(row["source"], proxy["signal"]) - return row + p = create_printer() + + # execute first query + try: + r = run_query(p, (str(q), q.params)) + except Exception as e: + raise DatabaseErrorException(str(e)) - # send query - return execute_query(str(q), q.params, fields_string, fields_int, fields_float, transform=transform_row) + # now use a generator for sending the rows and execute all the other queries + return p(filter_fields(gen_transform(r))) def _verify_argument_time_type_matches(is_day_argument: bool, count_daily_signal: int, count_weekly_signal: int) -> None: @@ -171,12 +248,17 @@ def handle_trend(): daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) geo_pairs = parse_geo_pairs() + transform_args = parse_transform_args() + jit_bypass = parse_jit_bypass() time_window, is_day = parse_day_or_week_range_arg("window") time_value, is_also_day = parse_day_or_week_arg("date") + if is_day != is_also_day: raise ValidationFailedException("mixing weeks with day arguments") + _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) + basis_time_value = extract_date("basis") if basis_time_value is None: base_shift = extract_integer("basis_shift") @@ -184,12 +266,38 @@ def handle_trend(): base_shift = 7 basis_time_value = shift_time_value(time_value, -1 * base_shift) if is_day else shift_week_value(time_value, -1 * base_shift) + def gen_trend(rows): + for key, group in groupby(rows, lambda row: (row["geo_type"], row["geo_value"], row["source"], row["signal"])): + geo_type, geo_value, source, signal = key + if alias_mapper: + source = alias_mapper(source, signal) + trend = compute_trend(geo_type, geo_value, source, signal, time_value, basis_time_value, ((row["time_value"], row["value"]) for row in group)) + yield trend.asdict() + # build query q = QueryBuilder(latest_table, "t") fields_string = ["geo_type", "geo_value", "source", "signal"] fields_int = ["time_value"] fields_float = ["value"] + + use_server_side_compute = all((is_day, is_also_day)) and JIT_COMPUTE and not jit_bypass + if use_server_side_compute: + pad_length = get_pad_length(source_signal_pairs, transform_args.get("smoother_window_length")) + source_signal_pairs, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + time_window = pad_time_window(time_window, pad_length) + + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + transformed_rows = row_transform_generator(parsed_rows=parsed_rows, time_pairs=[TimePair("day", [time_window])], transform_args=transform_args) + for row in transformed_rows: + yield row + else: + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + for row in parsed_rows: + yield row + q.set_fields(fields_string, fields_int, fields_float) q.set_order("geo_type", "geo_value", "source", "signal", "time_value") @@ -202,13 +310,6 @@ def handle_trend(): p = create_printer() - def gen(rows): - for key, group in groupby((parse_row(row, fields_string, fields_int, fields_float) for row in rows), lambda row: (row["geo_type"], row["geo_value"], row["source"], row["signal"])): - geo_type, geo_value, source, signal = key - if alias_mapper: - source = alias_mapper(source, signal) - trend = compute_trend(geo_type, geo_value, source, signal, time_value, basis_time_value, ((row["time_value"], row["value"]) for row in group)) - yield trend.asdict() # execute first query try: @@ -217,7 +318,7 @@ def gen(rows): raise DatabaseErrorException(str(e)) # now use a generator for sending the rows and execute all the other queries - return p(filter_fields(gen(r))) + return p(filter_fields(gen_trend(gen_transform(r)))) @bp.route("/trendseries", methods=("GET", "POST")) @@ -227,19 +328,54 @@ def handle_trendseries(): daily_signals, weekly_signals = count_signal_time_types(source_signal_pairs) source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) geo_pairs = parse_geo_pairs() + transform_args = parse_transform_args() + jit_bypass = parse_jit_bypass() time_window, is_day = parse_day_or_week_range_arg("window") + _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) + basis_shift = extract_integer(("basis", "basis_shift")) if basis_shift is None: basis_shift = 7 + shifter = lambda x: shift_time_value(x, -basis_shift) + if not is_day: + shifter = lambda x: shift_week_value(x, -basis_shift) + + def gen_trend(rows): + for key, group in groupby(rows, lambda row: (row["geo_type"], row["geo_value"], row["source"], row["signal"])): + geo_type, geo_value, source, signal = key + if alias_mapper: + source = alias_mapper(source, signal) + trends = compute_trends(geo_type, geo_value, source, signal, shifter, ((row["time_value"], row["value"]) for row in group)) + for t in trends: + yield t.asdict() + # build query q = QueryBuilder(latest_table, "t") fields_string = ["geo_type", "geo_value", "source", "signal"] fields_int = ["time_value"] fields_float = ["value"] + + use_server_side_compute = is_day and JIT_COMPUTE and not jit_bypass + if use_server_side_compute: + pad_length = get_pad_length(source_signal_pairs, transform_args.get("smoother_window_length")) + source_signal_pairs, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + time_window = pad_time_window(time_window, pad_length) + + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + transformed_rows = row_transform_generator(parsed_rows=parsed_rows, time_pairs=[TimePair("day", [time_window])], transform_args=transform_args) + for row in transformed_rows: + yield row + else: + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + for row in parsed_rows: + yield row + q.set_fields(fields_string, fields_int, fields_float) q.set_order("geo_type", "geo_value", "source", "signal", "time_value") @@ -252,19 +388,6 @@ def handle_trendseries(): p = create_printer() - shifter = lambda x: shift_time_value(x, -basis_shift) - if not is_day: - shifter = lambda x: shift_week_value(x, -basis_shift) - - def gen(rows): - for key, group in groupby((parse_row(row, fields_string, fields_int, fields_float) for row in rows), lambda row: (row["geo_type"], row["geo_value"], row["source"], row["signal"])): - geo_type, geo_value, source, signal = key - if alias_mapper: - source = alias_mapper(source, signal) - trends = compute_trends(geo_type, geo_value, source, signal, shifter, ((row["time_value"], row["value"]) for row in group)) - for t in trends: - yield t.asdict() - # execute first query try: r = run_query(p, (str(q), q.params)) @@ -272,7 +395,7 @@ def gen(rows): raise DatabaseErrorException(str(e)) # now use a generator for sending the rows and execute all the other queries - return p(filter_fields(gen(r))) + return p(filter_fields(gen_trend(gen_transform(r)))) @bp.route("/correlation", methods=("GET", "POST")) @@ -355,10 +478,15 @@ def handle_export(): source_signal_pairs, alias_mapper = create_source_signal_alias_mapper(source_signal_pairs) start_day, is_day = parse_day_or_week_arg("start_day", 202001 if weekly_signals > 0 else 20200401) end_day, is_end_day = parse_day_or_week_arg("end_day", 202020 if weekly_signals > 0 else 20200901) + time_window = (start_day, end_day) if is_day != is_end_day: raise ValidationFailedException("mixing weeks with day arguments") + _verify_argument_time_type_matches(is_day, daily_signals, weekly_signals) + transform_args = parse_transform_args() + jit_bypass = parse_jit_bypass() + geo_type = request.args.get("geo_type", "county") geo_values = request.args.get("geo_values", "*") @@ -372,10 +500,30 @@ def handle_export(): # build query q = QueryBuilder(latest_table, "t") - q.set_fields(["geo_value", "signal", "time_value", "issue", "lag", "value", "stderr", "sample_size", "geo_type", "source"], [], []) + fields_string = ["geo_value", "signal", "geo_type", "source"] + fields_int = ["time_value", "issue", "lag"] + fields_float = ["value", "stderr", "sample_size"] + + use_server_side_compute = all([is_day, is_end_day]) and JIT_COMPUTE and not jit_bypass + if use_server_side_compute: + pad_length = get_pad_length(source_signal_pairs, transform_args.get("smoother_window_length")) + source_signal_pairs, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs) + time_window = pad_time_window(time_window, pad_length) + + def gen_transform(rows): + parsed_rows = (parse_row(row, fields_string, fields_int, fields_float) for row in rows) + transformed_rows = row_transform_generator(parsed_rows=parsed_rows, time_pairs=[TimePair("day", [time_window])], transform_args=transform_args) + for row in transformed_rows: + yield row + else: + def gen_transform(rows): + for row in rows: + yield row + + q.set_fields(fields_string, fields_int, fields_float) q.set_order("time_value", "geo_value") q.where_source_signal_pairs("source", "signal", source_signal_pairs) - q.where_time_pairs("time_type", "time_value", [TimePair("day" if is_day else "week", [(start_day, end_day)])]) + q.where_time_pairs("time_type", "time_value", [TimePair("day" if is_day else "week", [time_window])]) q.where_geo_pairs("geo_type", "geo_value", [GeoPair(geo_type, True if geo_values == "*" else geo_values)]) _handle_lag_issues_as_of(q, None, None, as_of) @@ -386,7 +534,7 @@ def handle_export(): filename = "covidcast-{source}-{signal}-{start_day}-to-{end_day}{as_of}".format(source=source, signal=signal, start_day=format_date(start_day), end_day=format_date(end_day), as_of=as_of_str) p = CSVPrinter(filename) - def parse_row(i, row): + def parse_csv_row(i, row): # '',geo_value,signal,{time_value,issue},lag,value,stderr,sample_size,geo_type,data_source return { "": i, @@ -402,10 +550,9 @@ def parse_row(i, row): "data_source": alias_mapper(row["source"], row["signal"]) if alias_mapper else row["source"], } - def gen(first_row, rows): - yield parse_row(0, first_row) + def gen_parse(rows): for i, row in enumerate(rows): - yield parse_row(i + 1, row) + yield parse_csv_row(i, row) # execute query try: @@ -414,14 +561,15 @@ def gen(first_row, rows): raise DatabaseErrorException(str(e)) # special case for no data to be compatible with the CSV server - first_row = next(r, None) + transformed_query = peekable(gen_transform(r)) + first_row = transformed_query.peek(None) if not first_row: return "No matching data found for signal {source}:{signal} " "at {geo} level from {start_day} to {end_day}, as of {as_of}.".format( source=source, signal=signal, geo=geo_type, start_day=format_date(start_day), end_day=format_date(end_day), as_of=(date.today().isoformat() if as_of is None else format_date(as_of)) ) # now use a generator for sending the rows and execute all the other queries - return p(gen(first_row, r)) + return p(gen_parse(transformed_query)) @bp.route("/backfill", methods=("GET", "POST")) diff --git a/src/server/endpoints/covidcast_utils/model.py b/src/server/endpoints/covidcast_utils/model.py index 28b398580..6f4df67af 100644 --- a/src/server/endpoints/covidcast_utils/model.py +++ b/src/server/endpoints/covidcast_utils/model.py @@ -1,12 +1,28 @@ from dataclasses import asdict, dataclass, field -from typing import Callable, Optional, Dict, List, Set, Tuple from enum import Enum +from functools import partial +from itertools import groupby, repeat, tee +from numbers import Number +from typing import Callable, Generator, Iterator, Optional, Dict, List, Set, Tuple, Union + from pathlib import Path import re +from more_itertools import flatten, interleave_longest, peekable import pandas as pd import numpy as np -from ..._params import SourceSignalPair +from delphi_utils.nancodes import Nans +from ..._params import SourceSignalPair, TimePair +from .smooth_diff import generate_smoothed_rows, generate_diffed_rows +from ...utils import shift_time_value, iterate_over_ints_and_ranges + + +IDENTITY: Callable = lambda rows, **kwargs: rows +DIFF: Callable = lambda rows, **kwargs: generate_diffed_rows(rows, **kwargs) +SMOOTH: Callable = lambda rows, **kwargs: generate_smoothed_rows(rows, **kwargs) +DIFF_SMOOTH: Callable = lambda rows, **kwargs: generate_smoothed_rows(generate_diffed_rows(rows, **kwargs), **kwargs) + +SignalTransforms = Dict[SourceSignalPair, SourceSignalPair] class HighValuesAre(str, Enum): @@ -21,6 +37,7 @@ class SignalFormat(str, Enum): fraction = "fraction" raw_count = "raw_count" raw = "raw" + count = "count" class SignalCategory(str, Enum): @@ -202,7 +219,7 @@ def _load_data_sources(): data_sources, data_sources_df = _load_data_sources() -data_source_by_id = {d.source: d for d in data_sources} +data_sources_by_id = {d.source: d for d in data_sources} def _load_data_signals(sources: List[DataSource]): @@ -231,12 +248,11 @@ def _load_data_signals(sources: List[DataSource]): data_signals_by_key = {d.key: d for d in data_signals} # also add the resolved signal version to the signal lookup for d in data_signals: - source = data_source_by_id.get(d.source) + source = data_sources_by_id.get(d.source) if source and source.uses_db_alias: data_signals_by_key[(source.db_source, d.signal)] = d - def get_related_signals(signal: DataSignal) -> List[DataSignal]: return [s for s in data_signals if s != signal and s.signal_basename == signal.signal_basename] @@ -266,7 +282,7 @@ def create_source_signal_alias_mapper(source_signals: List[SourceSignalPair]) -> alias_to_data_sources: Dict[str, List[DataSource]] = {} transformed_pairs: List[SourceSignalPair] = [] for pair in source_signals: - source = data_source_by_id.get(pair.source) + source = data_sources_by_id.get(pair.source) if not source or not source.uses_db_alias: transformed_pairs.append(pair) continue @@ -300,3 +316,298 @@ def map_row(source: str, signal: str) -> str: return signal_source.source return transformed_pairs, map_row + + +def _resolve_bool_source_signals(source_signals: Union[SourceSignalPair, List[SourceSignalPair]], data_sources_by_id: Dict[str, DataSource]) -> Union[SourceSignalPair, List[SourceSignalPair]]: + """Expand a request for all signals to an explicit list of signal names. + + Example: SourceSignalPair("jhu-csse", signal=True) would return SourceSignalPair("jhu-csse", []). + """ + if isinstance(source_signals, SourceSignalPair): + if source_signals.signal == True: + source = data_sources_by_id.get(source_signals.source) + if source: + return SourceSignalPair(source.source, [s.signal for s in source.signals]) + return source_signals + + if isinstance(source_signals, list): + return [_resolve_bool_source_signals(pair, data_sources_by_id) for pair in source_signals] + + raise TypeError("source_signals is not Union[SourceSignalPair, List[SourceSignalPair]].") + + +def _reindex_iterable(iterator: Iterator[Dict], time_pairs: Optional[List[TimePair]], fill_value: Optional[Number] = None) -> Iterator[Dict]: + """Produces an iterator that fills in gaps in the time window of another iterator. + + Used to produce an iterator with a contiguous time index for time series operations. + + We iterate over contiguous range of days made from time_pairs. If `iterator`, which is assumed to be sorted by its "time_value" key, + is missing a time_value in the range, a row is returned with the missing date and dummy fields. + """ + # Iterate as normal if time_pairs is empty or None. + if not time_pairs: + yield from iterator + return + + _iterator = peekable(iterator) + + # If the iterator is empty, we halt immediately. + try: + first_item = _iterator.peek() + except StopIteration: + return + + _default_item = first_item.copy() + _default_item.update({"stderr": None, "sample_size": None, "issue": None, "lag": None, "missing_stderr": Nans.NOT_APPLICABLE, "missing_sample_size": Nans.NOT_APPLICABLE}) + + # Non-trivial operations otherwise. + min_time_value = first_item.get("time_value") + for expected_time_value in get_day_range(time_pairs): + if expected_time_value < min_time_value: + continue + + try: + # This will stay the same until the peeked element is consumed. + new_item = _iterator.peek() + except StopIteration: + return + + if expected_time_value == new_item.get("time_value"): + # Get the value we just peeked. + yield next(_iterator) + else: + # Return a default row instead. + # Copy to avoid Python by-reference memory issues. + default_item = _default_item.copy() + default_item.update({"time_value": expected_time_value, "value": fill_value, "missing_value": Nans.NOT_MISSING if fill_value and not np.isnan(fill_value) else Nans.NOT_APPLICABLE}) + yield default_item + + +def _get_base_signal_transform(signal: Union[DataSignal, Tuple[str, str]], data_signals_by_key: Dict[Tuple[str, str], DataSignal] = data_signals_by_key) -> Callable: + """Given a DataSignal, return the transformation that needs to be applied to its base signal to derive the signal.""" + if isinstance(signal, DataSignal): + base_signal = data_signals_by_key.get((signal.source, signal.signal_basename)) + if signal.format not in [SignalFormat.raw, SignalFormat.raw_count, SignalFormat.count] or not signal.compute_from_base or not base_signal: + return IDENTITY + if signal.is_cumulative and signal.is_smoothed: + return SMOOTH + if not signal.is_cumulative and not signal.is_smoothed: + return DIFF if base_signal.is_cumulative else IDENTITY + if not signal.is_cumulative and signal.is_smoothed: + return DIFF_SMOOTH if base_signal.is_cumulative else SMOOTH + return IDENTITY + + if isinstance(signal, tuple): + if signal := data_signals_by_key.get(signal): + return _get_base_signal_transform(signal, data_signals_by_key) + return IDENTITY + + raise TypeError("signal must be either Tuple[str, str] or DataSignal.") + + +def get_transform_types( + source_signal_pairs: List[SourceSignalPair], + data_sources_by_id: Dict[str, DataSource] = data_sources_by_id, + data_signals_by_key: Dict[Tuple[str, str], DataSignal] = data_signals_by_key +) -> Set[Callable]: + """Return a collection of the unique transforms required for transforming a given source-signal pair list. + + Example: + SourceSignalPair("src", ["sig", "sig_smoothed", "sig_diff"]) would return {IDENTITY, SMOOTH, DIFF}. + + Used to pad the user DB query with extra days. + """ + source_signal_pairs = _resolve_bool_source_signals(source_signal_pairs, data_sources_by_id) + + transform_types = set() + for source_signal_pair in source_signal_pairs: + source_name = source_signal_pair.source + signal_names = source_signal_pair.signal + + if isinstance(signal_names, bool): + continue + + transform_types |= {_get_base_signal_transform((source_name, signal_name), data_signals_by_key=data_signals_by_key) for signal_name in signal_names} + + return transform_types + + +def get_pad_length( + source_signal_pairs: List[SourceSignalPair], + smoother_window_length: int, + data_sources_by_id: Dict[str, DataSource] = data_sources_by_id, + data_signals_by_key: Dict[Tuple[str, str], DataSignal] = data_signals_by_key, +): + """Returns the size of the extra date padding needed, depending on the transformations the source-signal pair list requires. + + If smoothing is required, we fetch an extra smoother_window_length - 1 days (6 by default). If both diffing and smoothing is required on the same signal, + then we fetch extra smoother_window_length days (7 by default). + + Used to pad the user DB query with extra days. + """ + transform_types = get_transform_types(source_signal_pairs, data_sources_by_id=data_sources_by_id, data_signals_by_key=data_signals_by_key) + pad_length = [0] + if DIFF_SMOOTH in transform_types: + pad_length.append(smoother_window_length) + if SMOOTH in transform_types: + pad_length.append(smoother_window_length - 1) + if DIFF in transform_types: + pad_length.append(1) + return max(pad_length) + + +def pad_time_pairs(time_pairs: List[TimePair], pad_length: int) -> List[TimePair]: + """Pads a list of TimePairs with another TimePair that extends the smallest time value by the pad_length, if needed. + + Assumes day time_type, since this function is only called for JIT computations which share the same assumption. + + Example: + [TimePair("day", [20210407])] with pad_length 6 would return [TimePair("day", [20210407]), TimePair("day", [(20210401, 20210407)])]. + """ + if pad_length < 0: + raise ValueError("pad_length should non-negative.") + if pad_length == 0: + return time_pairs.copy() + + extracted_non_bool_time_values = flatten(time_pair.time_values for time_pair in time_pairs if not isinstance(time_pair.time_values, bool)) + min_time = min(time_value if isinstance(time_value, int) else time_value[0] for time_value in extracted_non_bool_time_values) + + padded_time = TimePair("day", [(shift_time_value(min_time, -1 * pad_length), min_time)]) + + return time_pairs + [padded_time] + + +def pad_time_window(time_window: Tuple[int, int], pad_length: int) -> Tuple[int, int]: + """Extend a time window on the left by pad_length. + + Example: + (20210407, 20210413) with pad_length 6 would return (20210401, 20210413). + + Used to pad the user DB query with extra days. + """ + if pad_length < 0: + raise ValueError("pad_length should non-negative.") + if pad_length == 0: + return time_window + min_time, max_time = time_window + return (shift_time_value(min_time, -1 * pad_length), max_time) + + +def get_day_range(time_pairs: List[TimePair]) -> Iterator[int]: + """Iterate over a list of TimePair time_values, including the values contained in the ranges. + + Example: + [TimePair("day", [20210407, 20210408]), TimePair("day", [20210405, (20210408, 20210411)])] would iterate over + [20210405, 20210407, 20210408, 20210409, 20210410, 20210411]. + """ + time_values_flattened = [] + + for time_pair in time_pairs: + if time_pair.time_type != "day": + raise ValueError("get_day_range only supports 'day' time_type.") + + if isinstance(time_pair.time_values, int): + time_values_flattened.append(time_pair.time_values) + elif isinstance(time_pair.time_values, list): + time_values_flattened.extend(time_pair.time_values) + else: + raise ValueError("get_day_range only supports int or list time_values.") + + return iterate_over_ints_and_ranges(time_values_flattened) + + +def _generate_transformed_rows( + parsed_rows: Iterator[Dict], + time_pairs: Optional[List[TimePair]] = None, + transform_dict: Optional[SignalTransforms] = None, + transform_args: Optional[Dict] = None, + group_keyfunc: Optional[Callable] = None, + data_signals_by_key: Dict[Tuple[str, str], DataSignal] = data_signals_by_key, +) -> Iterator[Dict]: + """Applies time-series transformations to streamed rows from a database. + + Parameters: + parsed_rows: Iterator[Dict] + An iterator streaming rows from a database query. Assumed to be sorted by geo_type, geo_value, source, signal, time_type, and time_value. + time_pairs: Optional[List[TimePair]], default None + A list of TimePairs, which can be used to create a continguous time index for time-series operations. + The min and max dates in the TimePairs list is used. + transform_dict: Optional[SignalTransforms], default None + A dictionary mapping base sources to a list of their derived signals that the user wishes to query. + For example, transform_dict may be {("jhu-csse", "confirmed_cumulative_num): [("jhu-csse", "confirmed_incidence_num"), ("jhu-csse", "confirmed_7dav_incidence_num")]}. + transform_args: Optional[Dict], default None + A dictionary of keyword arguments for the transformer functions. + group_keyfunc: Optional[Callable], default None + The groupby function to use to order the streamed rows. Note that Python groupby does not do any sorting, so + parsed_rows are assumed to be sorted in accord with this groupby. + data_signals_by_key: Dict[Tuple[str, str], DataSignal], default data_signals_by_key + The dictionary of DataSignals which is used to find the base signal transforms. + + Yields: + transformed rows: Dict + The transformed rows returned in an interleaved fashion. Non-transformed rows have the IDENTITY operation applied. + """ + if not transform_args: + transform_args = dict() + if not transform_dict: + transform_dict = dict() + if not group_keyfunc: + group_keyfunc = lambda row: (row["geo_type"], row["geo_value"], row["source"], row["signal"]) + + for key, source_signal_geo_rows in groupby(parsed_rows, group_keyfunc): + _, _, base_source_name, base_signal_name = key + # Extract the list of derived signals; if a signal is not in the dictionary, then use the identity map. + derived_signal_transform_map: SourceSignalPair = transform_dict.get(SourceSignalPair(base_source_name, [base_signal_name]), SourceSignalPair(base_source_name, [base_signal_name])) + # Create a list of source-signal pairs along with the transformation required for the signal. + signal_names_and_transforms: List[Tuple[Tuple[str, str], Callable]] = [(signal, _get_base_signal_transform((base_source_name, signal), data_signals_by_key)) for signal in derived_signal_transform_map.signal] + # Put the current time series on a contiguous time index. + source_signal_geo_rows = _reindex_iterable(source_signal_geo_rows, time_pairs, fill_value=transform_args.get("pad_fill_value")) + # Create copies of the iterable, with smart memory usage. + source_signal_geo_rows_copies: Iterator[Iterator[Dict]] = tee(source_signal_geo_rows, len(signal_names_and_transforms)) + # Create a list of transformed group iterables, remembering their derived name as needed. + transformed_signals_iterator: Iterator[Tuple[str, Iterator[Dict]]] = (zip(repeat(derived_signal), transform(rows, **transform_args)) for (derived_signal, transform), rows in zip(signal_names_and_transforms, source_signal_geo_rows_copies)) + # Traverse through the transformed iterables in an interleaved fashion, which makes sure that only a small window + # of the original iterable (group) is stored in memory. + for derived_signal_name, row in interleave_longest(*transformed_signals_iterator): + row["signal"] = derived_signal_name + yield row + + +def get_basename_signal_and_jit_generator( + source_signal_pairs: List[SourceSignalPair], + transform_args: Optional[Dict[str, Union[str, int]]] = None, + data_sources_by_id: Dict[str, DataSource] = data_sources_by_id, + data_signals_by_key: Dict[Tuple[str, str], DataSignal] = data_signals_by_key, +) -> Tuple[List[SourceSignalPair], Generator]: + """From a list of SourceSignalPairs, return the base signals required to derive them and a transformation function to take a stream + of the base signals and return the transformed signals. + + Example: + SourceSignalPair("src", signal=["sig_base", "sig_smoothed"]) would return SourceSignalPair("src", signal=["sig_base"]) and a transformation function + that will take the returned database query for "sig_base" and return both the base time series and the smoothed time series. transform_dict in this case + would be {("src", "sig_base"): [("src", "sig_base"), ("src", "sig_smooth")]}. + """ + source_signal_pairs = _resolve_bool_source_signals(source_signal_pairs, data_sources_by_id) + base_signal_pairs: List[SourceSignalPair] = [] + transform_dict: SignalTransforms = dict() + + for pair in source_signal_pairs: + # Should only occur when the SourceSignalPair was unrecognized by _resolve_bool_source_signals. Useful for testing with fake signal names. + if isinstance(pair.signal, bool): + base_signal_pairs.append(pair) + continue + + signals = [] + for signal_name in pair.signal: + signal = data_signals_by_key.get((pair.source, signal_name)) + if not signal or not signal.compute_from_base: + transform_dict.setdefault(SourceSignalPair(source=pair.source, signal=[signal_name]), SourceSignalPair(source=pair.source, signal=[])).add_signal(signal_name) + signals.append(signal_name) + else: + transform_dict.setdefault(SourceSignalPair(source=pair.source, signal=[signal.signal_basename]), SourceSignalPair(source=pair.source, signal=[])).add_signal(signal_name) + signals.append(signal.signal_basename) + base_signal_pairs.append(SourceSignalPair(pair.source, signals)) + + row_transform_generator = partial(_generate_transformed_rows, transform_dict=transform_dict, transform_args=transform_args, data_signals_by_key=data_signals_by_key) + + return base_signal_pairs, row_transform_generator diff --git a/src/server/endpoints/covidcast_utils/smooth_diff.py b/src/server/endpoints/covidcast_utils/smooth_diff.py new file mode 100644 index 000000000..d4a986c97 --- /dev/null +++ b/src/server/endpoints/covidcast_utils/smooth_diff.py @@ -0,0 +1,179 @@ +from enum import Enum +from logging import getLogger +from numbers import Number +from typing import Dict, Iterable, List, Union + +from delphi_utils.nancodes import Nans +from more_itertools import windowed +from numpy import array, dot, isnan, nan, nan_to_num, ndarray + +from ...utils.dates import time_value_to_date + + +class SmootherKernelValue(str, Enum): + average = "average" + + +def generate_smoothed_rows( + rows: Iterable[Dict], + smoother_kernel: Union[List[Number], SmootherKernelValue] = SmootherKernelValue.average, + smoother_window_length: int = 7, + nan_fill_value: Number = nan, + **kwargs +) -> Iterable[Dict]: + """Generate smoothed row entries. + + There are roughly two modes of boundary handling: + * no padding, the windows start at length 1 on the left boundary and grow to size + smoother_window_length (achieved with pad_fill_value = None) + * value padding, smoother_window_length - 1 many fill_values are appended at the start of the + given date (achieved with any other pad_fill_value) + + Note that this function crucially relies on the assumption that the iterable rows + have been sorted by time_value. If this assumption is violated, the results will likely be + incoherent. + + Parameters + ---------- + rows: Iterable[Dict] + An iterable over the rows a database query returns. The rows are assumed to be + dicts containing the "geo_type", "geo_value", and "time_value" keys. Assumes the + rows have been sorted by geo and time_value beforehand. + smooth_kernel: Union[List[Number], SmootherKernelValue], default SmootherValue.average + Either a SmootherKernelValue or a custom list of numbers for weighted averaging. + smoother_window_length: int, default 7 + The length of the averaging window for the smoother. + nan_fill_value: Number, default nan + The value to use when encountering nans (e.g. None and numpy.nan types); uses nan by default. + **kwargs: + Container for non-shared parameters with other computation functions. + """ + # Validate params. + if not isinstance(smoother_window_length, int) or smoother_window_length < 1: + smoother_window_length = 7 + if isinstance(smoother_kernel, list): + smoother_window_length = len(smoother_kernel) + if not isinstance(nan_fill_value, Number): + nan_fill_value = nan + if not isinstance(smoother_kernel, (list, SmootherKernelValue)): + smoother_kernel = SmootherKernelValue.average + + for window in windowed(rows, smoother_window_length): # Iterable[List[Dict]] + # This occurs only if len(rows) < smoother_window_length. + if None in window: + continue + + new_value = _smoother(_get_validated_window_values(window, nan_fill_value), kernel=smoother_kernel) + # The database returns NULL values as None, so we stay consistent with that. + new_value = float(round(new_value, 7)) if not isnan(new_value) else None + if new_value and isnan(new_value): + breakpoint() + + new_item = _fill_remaining_row_values(window) + new_item.update({"value": new_value, "missing_value": Nans.NOT_MISSING if new_value is not None else Nans.NOT_APPLICABLE}) + + yield new_item + + +def generate_diffed_rows(rows: Iterable[Dict], nan_fill_value: Number = nan, **kwargs) -> Iterable[Dict]: + """Generate differences between row values. + + Note that this function crucially relies on the assumption that the iterable rows have been + sorted by time_value. If this assumption is violated, the results will likely be incoherent. + + rows: Iterable[Dict] + An iterable over the rows a database query returns. The rows are assumed to be dicts + containing the "geo_type", "geo_value", and "time_value" keys. Assumes the rows have been + sorted by geo and time_value beforehand. + nan_fill_value: Number, default nan + The value to use when encountering nans (e.g. None and numpy.nan types); uses nan by default. + **kwargs: + Container for non-shared parameters with other computation functions. + """ + if not isinstance(nan_fill_value, Number): + nan_fill_value = nan + + for window in windowed(rows, 2): + # This occurs only if len(rows) < 2. + if None in window: + continue + + first_value, second_value = _get_validated_window_values(window, nan_fill_value) + new_value = round(second_value - first_value, 7) + # The database returns NULL values as None, so we stay consistent with that. + new_value = float(new_value) if not isnan(new_value) else None + + new_item = _fill_remaining_row_values(window) + new_item.update({"value": new_value, "missing_value": Nans.NOT_MISSING if new_value is not None else Nans.NOT_APPLICABLE}) + + yield new_item + + +def _smoother(values: List[Number], kernel: Union[List[Number], SmootherKernelValue] = SmootherKernelValue.average) -> Number: + """Basic smoother. + + If kernel passed, uses the kernel as summation weights. If something is wrong, + defaults to the mean. + """ + + if kernel and isinstance(kernel, list): + kernel = array(kernel, copy=False) + values = array(values, copy=False) + smoothed_value = dot(values, kernel) + elif kernel and isinstance(kernel, SmootherKernelValue): + if kernel == SmootherKernelValue.average: + smoothed_value = array(values, copy=False).mean() + else: + raise ValueError("Unimplemented SmootherKernelValue.") + else: + raise ValueError("Kernel must be specified in _smoother.") + + return smoothed_value + + +def _get_validated_window_values(window: List[dict], nan_fill_value: Number) -> ndarray: + """Extracts and validates the values in the window, returning a list of floats. + + The main objective is to create a consistent nan type values from None or np.nan. We replace None with np.nan, so they can be filled. + + Assumes any None values were filtered out of window, so it is a list of Dict only. + """ + return nan_to_num([e.get("value") if e.get("value") is not None else nan for e in window], nan=nan_fill_value) + + +def _fill_remaining_row_values(window: Iterable[dict]) -> dict: + """Set a few default fields for the covidcast row.""" + logger = getLogger("gunicorn.error") + + # Start by defaulting to the field values of the last window member. + new_item = window[-1].copy() + + try: + issues = [e.get("issue") for e in window] + if None in issues: + new_issue = None + else: + new_issue = max(issues) + except (TypeError, ValueError): + logger.warn(f"There was an error computing an issue field for {new_item.get('source')}:{new_item.get('signal')}.") + new_issue = None + + try: + if new_issue is None: + new_lag = None + else: + new_lag = (time_value_to_date(new_issue) - time_value_to_date(new_item["time_value"])).days + except (TypeError, ValueError): + logger.warn(f"There was an error computing a lag field for {new_item.get('source')}:{new_item.get('signal')}.") + new_lag = None + + new_item.update({ + "issue": new_issue, + "lag": new_lag, + "stderr": None, + "sample_size": None, + "missing_stderr": Nans.NOT_APPLICABLE, + "missing_sample_size": Nans.NOT_APPLICABLE + }) + + return new_item diff --git a/src/server/endpoints/covidcast_utils/trend.py b/src/server/endpoints/covidcast_utils/trend.py index 43c4ac21b..9a2825208 100644 --- a/src/server/endpoints/covidcast_utils/trend.py +++ b/src/server/endpoints/covidcast_utils/trend.py @@ -42,6 +42,8 @@ def compute_trend(geo_type: str, geo_value: str, signal_source: str, signal_sign # find all needed rows for time, value in rows: + if value is None: + continue if time == current_time: t.value = value if time == basis_time: @@ -73,6 +75,8 @@ def compute_trends(geo_type: str, geo_value: str, signal_source: str, signal_sig lookup: Dict[int, float] = OrderedDict() # find all needed rows for time, value in rows: + if value is None: + continue lookup[time] = value if min_value is None or min_value > value: min_date = time diff --git a/src/server/utils/__init__.py b/src/server/utils/__init__.py index 3198779d0..bddb00c43 100644 --- a/src/server/utils/__init__.py +++ b/src/server/utils/__init__.py @@ -1 +1,17 @@ -from .dates import shift_time_value, date_to_time_value, time_value_to_iso, time_value_to_date, days_in_range, weeks_in_range, shift_week_value, week_to_time_value, week_value_to_week, guess_time_value_is_day, time_values_to_ranges, days_to_ranges, weeks_to_ranges +from .dates import ( + shift_time_value, + date_to_time_value, + time_value_to_iso, + time_value_to_date, + days_in_range, + weeks_in_range, + shift_week_value, + week_to_time_value, + week_value_to_week, + guess_time_value_is_day, + time_values_to_ranges, + days_to_ranges, + weeks_to_ranges, + iterate_over_range, + iterate_over_ints_and_ranges, +) diff --git a/src/server/utils/dates.py b/src/server/utils/dates.py index ef34a50b9..5a2fb0205 100644 --- a/src/server/utils/dates.py +++ b/src/server/utils/dates.py @@ -1,5 +1,6 @@ from typing import ( Callable, + Iterator, Optional, Sequence, Tuple, @@ -36,14 +37,12 @@ def guess_time_value_is_week(value: int) -> bool: def date_to_time_value(d: date) -> int: return int(d.strftime("%Y%m%d")) - def week_to_time_value(w: Week) -> int: return w.year * 100 + w.week def time_value_to_iso(value: int) -> str: return time_value_to_date(value).strftime("%Y-%m-%d") - def shift_time_value(time_value: int, days: int) -> int: if days == 0: return time_value @@ -140,3 +139,47 @@ def _to_ranges(values: Sequence[Union[Tuple[int, int], int]], value_to_date: Cal except Exception as e: logging.info('bad input to date ranges', input=values, exception=e) return values + +def iterate_over_range(start: int, end: int) -> Iterator[int]: + """Iterate over ints corresponding to dates in a time range. + + Left inclusive, right exclusive to mimic behavior of Python's built-in range. + """ + if start > end: + return + + current_date, final_date = time_value_to_date(start), time_value_to_date(end) + while current_date < final_date: + yield date_to_time_value(current_date) + current_date = current_date + timedelta(days=1) + +def iterate_over_ints_and_ranges(lst: Iterator[Union[int, Tuple[int, int]]], use_dates: bool = True) -> Iterator[int]: + """A generator that iterates over the unique values in a list of integers and ranges in ascending order. + + The tuples are assumed to be left- and right-inclusive. If use_dates is True, then the integers are interpreted as + YYYYMMDD dates. + + Examples: + >>> list(iterate_over_ints_and_ranges([(5, 8), 0], False)) + [0, 5, 6, 7, 8] + >>> list(iterate_over_ints_and_ranges([(5, 8), (4, 6), (3, 5)], False)) + [3, 4, 5, 6, 7, 8] + >>> list(iterate_over_ints_and_ranges([(7, 8), (5, 7), (3, 8), 8], False)) + [3, 4, 5, 6, 7, 8] + """ + lst = sorted((x, x) if isinstance(x, int) else x for x in lst) + if not lst: + return + + if use_dates: + increment = lambda x, y: date_to_time_value(time_value_to_date(x) + timedelta(days=y)) + range_handler = iterate_over_range + else: + increment = lambda x, y: x + y + range_handler = range + + biggest_seen = increment(lst[0][0], -1) + for a, b in lst: + for y in range_handler(max(a, increment(biggest_seen, 1)), increment(b, 1)): + yield y + biggest_seen = max(biggest_seen, b) diff --git a/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py b/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py index 40a242e22..90c9cb5ad 100644 --- a/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py +++ b/tests/acquisition/covidcast/test_covidcast_meta_cache_updater.py @@ -2,73 +2,62 @@ # standard library import argparse + import unittest from unittest.mock import MagicMock # third party -import pandas -from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import get_argument_parser, \ - main -# py3tester coverage target -__test_target__ = ( - 'delphi.epidata.acquisition.covidcast.' - 'covidcast_meta_cache_updater' -) +from delphi.epidata.acquisition.covidcast.covidcast_meta_cache_updater import get_argument_parser, main class UnitTests(unittest.TestCase): - """Basic unit tests.""" - - def test_get_argument_parser(self): - """Return a parser for command-line arguments.""" - - self.assertIsInstance(get_argument_parser(), argparse.ArgumentParser) + """Basic unit tests.""" - def test_main_successful(self): - """Run the main program successfully.""" + def test_get_argument_parser(self): + """Return a parser for command-line arguments.""" + self.assertIsInstance(get_argument_parser(), argparse.ArgumentParser) - api_response = { - 'result': 1, - 'message': 'yes', - 'epidata': [{'foo': 'bar'}], - } + def test_main_successful(self): + """Run the main program successfully.""" - args = MagicMock(log_file="log") - mock_epidata_impl = MagicMock() - mock_epidata_impl.covidcast_meta.return_value = api_response - mock_database = MagicMock() - mock_database.compute_covidcast_meta.return_value=api_response['epidata'] - fake_database_impl = lambda: mock_database + api_response = { + "result": 1, + "message": "yes", + "epidata": [{"foo": "bar"}], + } - main( - args, - epidata_impl=mock_epidata_impl, - database_impl=fake_database_impl) + args = MagicMock(log_file="log") + mock_epidata_impl = MagicMock() + mock_epidata_impl.covidcast_meta.return_value = api_response + mock_database = MagicMock() + mock_database.compute_covidcast_meta.return_value = api_response["epidata"] + fake_database_impl = lambda: mock_database - self.assertTrue(mock_database.connect.called) + main(args, epidata_impl=mock_epidata_impl, database_impl=fake_database_impl) - self.assertTrue(mock_database.update_covidcast_meta_cache.called) - actual_args = mock_database.update_covidcast_meta_cache.call_args[0] - expected_args = (api_response['epidata'],) - self.assertEqual(actual_args, expected_args) + self.assertTrue(mock_database.connect.called) - self.assertTrue(mock_database.disconnect.called) - self.assertTrue(mock_database.disconnect.call_args[0][0]) + self.assertTrue(mock_database.update_covidcast_meta_cache.called) + actual_args = mock_database.update_covidcast_meta_cache.call_args[0] + expected_args = (api_response["epidata"],) + self.assertEqual(actual_args, expected_args) - def test_main_failure(self): - """Run the main program with a query failure.""" + self.assertTrue(mock_database.disconnect.called) + self.assertTrue(mock_database.disconnect.call_args[0][0]) - api_response = { - 'result': -123, - 'message': 'no', - } + def test_main_failure(self): + """Run the main program with a query failure.""" + api_response = { + "result": -123, + "message": "no", + } - args = MagicMock(log_file="log") - mock_database = MagicMock() - mock_database.compute_covidcast_meta.return_value = list() - fake_database_impl = lambda: mock_database + args = MagicMock(log_file="log") + mock_database = MagicMock() + mock_database.compute_covidcast_meta.return_value = list() + fake_database_impl = lambda: mock_database - main(args, epidata_impl=None, database_impl=fake_database_impl) + main(args, epidata_impl=None, database_impl=fake_database_impl) - self.assertTrue(mock_database.compute_covidcast_meta.called) + self.assertTrue(mock_database.compute_covidcast_meta.called) diff --git a/tests/acquisition/covidcast/test_covidcast_row.py b/tests/acquisition/covidcast/test_covidcast_row.py new file mode 100644 index 000000000..969b521b9 --- /dev/null +++ b/tests/acquisition/covidcast/test_covidcast_row.py @@ -0,0 +1,90 @@ +import unittest + +from pandas import DataFrame, date_range +from pandas.testing import assert_frame_equal + +from delphi_utils.nancodes import Nans +from delphi.epidata.server.utils.dates import date_to_time_value +from delphi.epidata.acquisition.covidcast.covidcast_row import set_df_dtypes, transpose_dict, CovidcastRow, CovidcastRows + +class TestCovidcastRows(unittest.TestCase): + def test_transpose_dict(self): + assert transpose_dict(dict([["a", [2, 4, 6]], ["b", [3, 5, 7]], ["c", [10, 20, 30]]])) == [{"a": 2, "b": 3, "c": 10}, {"a": 4, "b": 5, "c": 20}, {"a": 6, "b": 7, "c": 30}] + + def test_CovidcastRow(self): + df = CovidcastRow(value=5.0).api_row_df + expected_df = DataFrame.from_records([{ + "source": "src", + "signal": "sig", + "time_type": "day", + "geo_type": "county", + "time_value": 20200202, + "geo_value": "01234", + "value": 5.0, + "stderr": 10.0, + "sample_size": 10.0, + "missing_value": Nans.NOT_MISSING, + "missing_stderr": Nans.NOT_MISSING, + "missing_sample_size": Nans.NOT_MISSING, + "issue": 20200202, + "lag": 0, + }]) + assert_frame_equal(df, expected_df) + + df = CovidcastRow(value=5.0).api_compatibility_row_df + expected_df = DataFrame.from_records([{ + "signal": "sig", + "time_type": "day", + "geo_type": "county", + "time_value": 20200202, + "geo_value": "01234", + "value": 5.0, + "stderr": 10.0, + "sample_size": 10.0, + "missing_value": Nans.NOT_MISSING, + "missing_stderr": Nans.NOT_MISSING, + "missing_sample_size": Nans.NOT_MISSING, + "issue": 20200202, + "lag": 0, + }]) + assert_frame_equal(df, expected_df) + + def test_CovidcastRows(self): + df = CovidcastRows.from_args(signal=["sig_base"] * 5 + ["sig_other"] * 5, time_value=date_range("2021-05-01", "2021-05-05").to_list() * 2, value=list(range(10))).api_row_df + expected_df = set_df_dtypes(DataFrame({ + "source": ["src"] * 10, + "signal": ["sig_base"] * 5 + ["sig_other"] * 5, + "time_type": ["day"] * 10, + "geo_type": ["county"] * 10, + "time_value": map(date_to_time_value, date_range("2021-05-01", "2021-05-5").to_list() * 2), + "geo_value": ["01234"] * 10, + "value": range(10), + "stderr": [10.0] * 10, + "sample_size": [10.0] * 10, + "missing_value": [Nans.NOT_MISSING] * 10, + "missing_stderr": [Nans.NOT_MISSING] * 10, + "missing_sample_size": [Nans.NOT_MISSING] * 10, + "issue": map(date_to_time_value, date_range("2021-05-01", "2021-05-5").to_list() * 2), + "lag": [0] * 10, + }), CovidcastRows()._DTYPES) + assert_frame_equal(df, expected_df) + + df = CovidcastRows.from_args( + signal=["sig_base"] * 5 + ["sig_other"] * 5, time_value=date_range("2021-05-01", "2021-05-05").to_list() * 2, value=list(range(10)) + ).api_compatibility_row_df + expected_df = set_df_dtypes(DataFrame({ + "signal": ["sig_base"] * 5 + ["sig_other"] * 5, + "time_type": ["day"] * 10, + "geo_type": ["county"] * 10, + "time_value": map(date_to_time_value, date_range("2021-05-01", "2021-05-5").to_list() * 2), + "geo_value": ["01234"] * 10, + "value": range(10), + "stderr": [10.0] * 10, + "sample_size": [10.0] * 10, + "missing_value": [Nans.NOT_MISSING] * 10, + "missing_stderr": [Nans.NOT_MISSING] * 10, + "missing_sample_size": [Nans.NOT_MISSING] * 10, + "issue": map(date_to_time_value, date_range("2021-05-01", "2021-05-5").to_list() * 2), + "lag": [0] * 10, + }), CovidcastRows()._DTYPES) + assert_frame_equal(df, expected_df) diff --git a/tests/acquisition/covidcast/test_database.py b/tests/acquisition/covidcast/test_database.py index b676e7413..c75e1bd8e 100644 --- a/tests/acquisition/covidcast/test_database.py +++ b/tests/acquisition/covidcast/test_database.py @@ -51,33 +51,6 @@ def test_disconnect_with_commit(self): self.assertTrue(connection.commit.called) self.assertTrue(connection.close.called) - def test_update_covidcast_meta_cache_query(self): - """Query to update the metadata cache looks sensible. - - NOTE: Actual behavior is tested by integration test. - """ - - args = ('epidata_json_str',) - mock_connector = MagicMock() - database = Database() - database.connect(connector_impl=mock_connector) - - database.update_covidcast_meta_cache(*args) - - connection = mock_connector.connect() - cursor = connection.cursor() - self.assertTrue(cursor.execute.called) - - sql, args = cursor.execute.call_args[0] - expected_args = ('"epidata_json_str"',) - self.assertEqual(args, expected_args) - - sql = sql.lower() - self.assertIn('update', sql) - self.assertIn('`covidcast_meta_cache`', sql) - self.assertIn('timestamp', sql) - self.assertIn('epidata', sql) - def test_insert_or_update_batch_exception_reraised(self): """Test that an exception is reraised""" mock_connector = MagicMock() diff --git a/tests/acquisition/covidcast/test_database_meta.py b/tests/acquisition/covidcast/test_database_meta.py new file mode 100644 index 000000000..78566d86a --- /dev/null +++ b/tests/acquisition/covidcast/test_database_meta.py @@ -0,0 +1,34 @@ +import unittest +from unittest.mock import MagicMock + +from delphi.epidata.acquisition.covidcast.database_meta import DatabaseMeta + +class UnitTests(unittest.TestCase): + """Basic unit tests.""" + + def test_update_covidcast_meta_cache_query(self): + """Query to update the metadata cache looks sensible. + + NOTE: Actual behavior is tested by integration test. + """ + + args = ('epidata_json_str',) + mock_connector = MagicMock() + database = DatabaseMeta() + database.connect(connector_impl=mock_connector) + + database.update_covidcast_meta_cache(*args) + + connection = mock_connector.connect() + cursor = connection.cursor() + self.assertTrue(cursor.execute.called) + + sql, args = cursor.execute.call_args[0] + expected_args = ('"epidata_json_str"',) + self.assertEqual(args, expected_args) + + sql = sql.lower() + self.assertIn('update', sql) + self.assertIn('`covidcast_meta_cache`', sql) + self.assertIn('timestamp', sql) + self.assertIn('epidata', sql) diff --git a/tests/server/endpoints/covidcast_utils/test_model.py b/tests/server/endpoints/covidcast_utils/test_model.py new file mode 100644 index 000000000..246f6702a --- /dev/null +++ b/tests/server/endpoints/covidcast_utils/test_model.py @@ -0,0 +1,557 @@ +import unittest +from itertools import chain +from numbers import Number +from typing import Iterable, List, Optional + +import pandas as pd +from more_itertools import interleave_longest, windowed +from pandas.testing import assert_frame_equal + +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRows +from delphi.epidata.server._params import SourceSignalPair, TimePair +from delphi.epidata.server.endpoints.covidcast_utils.model import ( + _generate_transformed_rows, + _get_base_signal_transform, + _reindex_iterable, + _resolve_bool_source_signals, + DataSignal, + DataSource, + DIFF_SMOOTH, + DIFF, + get_basename_signal_and_jit_generator, + get_day_range, + get_pad_length, + get_transform_types, + IDENTITY, + pad_time_pairs, + SMOOTH, +) +from delphi_utils.nancodes import Nans + +# fmt: off +DATA_SIGNALS_BY_KEY = { + ("src", "sig_diff"): DataSignal( + source="src", + signal="sig_diff", + signal_basename="sig_base", + name="src", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=False, + compute_from_base=True, + ), + ("src", "sig_smooth"): DataSignal( + source="src", + signal="sig_smooth", + signal_basename="sig_base", + name="src", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=True, + is_smoothed=True, + compute_from_base=True, + ), + ("src", "sig_diff_smooth"): DataSignal( + source="src", + signal="sig_diff_smooth", + signal_basename="sig_base", + name="src", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=False, + is_smoothed=True, + compute_from_base=True, + ), + ("src", "sig_base"): DataSignal( + source="src", + signal="sig_base", + signal_basename="sig_base", + name="src", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=True, + ), + ("src2", "sig_base"): DataSignal( + source="src2", + signal="sig_base", + signal_basename="sig_base", + name="sig_base", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=True, + ), + ("src2", "sig_diff_smooth"): DataSignal( + source="src2", + signal="sig_diff_smooth", + signal_basename="sig_base", + name="sig_smooth", + active=True, + short_description="", + description="", + time_label="", + value_label="", + is_cumulative=False, + is_smoothed=True, + compute_from_base=True, + ), +} + +DATA_SOURCES_BY_ID = { + "src": DataSource( + source="src", + db_source="src", + name="src", + description="", + reference_signal="sig_base", + signals=[DATA_SIGNALS_BY_KEY[key] for key in DATA_SIGNALS_BY_KEY if key[0] == "src"], + ), + "src2": DataSource( + source="src2", + db_source="src2", + name="src2", + description="", + reference_signal="sig_base", + signals=[DATA_SIGNALS_BY_KEY[key] for key in DATA_SIGNALS_BY_KEY if key[0] == "src2"], + ), +} +# fmt: on + + +def _diff_rows(rows: Iterable[Number]) -> List[Number]: + return [round(float(y - x), 8) if not (x is None or y is None) else None for x, y in windowed(rows, 2)] + + +def _smooth_rows(rows: Iterable[Number], window_length: int = 7, kernel: Optional[List[Number]] = None): + if not kernel: + kernel = [1.0 / window_length] * window_length + return [round(sum(x * y for x, y in zip(window, kernel)), 8) if None not in window else None for window in windowed(rows, len(kernel))] + + +def _reindex_windowed(lst: list, window_length: int) -> list: + return [max(window) if None not in window else None for window in windowed(lst, window_length)] + + +class TestModel(unittest.TestCase): + def test__resolve_bool_source_signals(self): + source_signal_pair = [SourceSignalPair(source="src", signal=True), SourceSignalPair(source="src", signal=["sig_unknown"])] + resolved_source_signal_pair = _resolve_bool_source_signals(source_signal_pair, DATA_SOURCES_BY_ID) + expected_source_signal_pair = [ + SourceSignalPair(source="src", signal=["sig_diff", "sig_smooth", "sig_diff_smooth", "sig_base"]), + SourceSignalPair(source="src", signal=["sig_unknown"]), + ] + assert resolved_source_signal_pair == expected_source_signal_pair + + def test__reindex_iterable(self): + # Trivial test. + time_pairs = [(20210503, 20210508)] + assert list(_reindex_iterable([], time_pairs)) == [] + + data = CovidcastRows.from_args(time_value=pd.date_range("2021-05-03", "2021-05-08").to_list()).api_row_df + for time_pairs in [[TimePair("day", [(20210503, 20210508)])], [], None]: + with self.subTest(f"Identity operations: {time_pairs}"): + df = CovidcastRows.from_records(_reindex_iterable(data.to_dict(orient="records"), time_pairs)).api_row_df + assert_frame_equal(df, data) + + data = CovidcastRows.from_args(time_value=pd.date_range("2021-05-03", "2021-05-08").to_list() + pd.date_range("2021-05-11", "2021-05-14").to_list()).api_row_df + with self.subTest("Non-trivial operations"): + time_pairs = [TimePair("day", [(20210501, 20210513)])] + + df = CovidcastRows.from_records(_reindex_iterable(data.to_dict(orient="records"), time_pairs)).api_row_df + expected_df = CovidcastRows.from_args( + time_value=pd.date_range("2021-05-03", "2021-05-13"), + issue=pd.date_range("2021-05-03", "2021-05-08").to_list() + [None] * 2 + pd.date_range("2021-05-11", "2021-05-13").to_list(), + lag=[0] * 6 + [None] * 2 + [0] * 3, + value=chain([10.0] * 6, [None] * 2, [10.0] * 3), + stderr=chain([10.0] * 6, [None] * 2, [10.0] * 3), + sample_size=chain([10.0] * 6, [None] * 2, [10.0] * 3), + ).api_row_df + assert_frame_equal(df, expected_df) + + df = CovidcastRows.from_records(_reindex_iterable(data.to_dict(orient="records"), time_pairs, fill_value=2.0)).api_row_df + expected_df = CovidcastRows.from_args( + time_value=pd.date_range("2021-05-03", "2021-05-13"), + issue=pd.date_range("2021-05-03", "2021-05-08").to_list() + [None] * 2 + pd.date_range("2021-05-11", "2021-05-13").to_list(), + lag=[0] * 6 + [None] * 2 + [0] * 3, + value=chain([10.0] * 6, [2.0] * 2, [10.0] * 3), + stderr=chain([10.0] * 6, [None] * 2, [10.0] * 3), + sample_size=chain([10.0] * 6, [None] * 2, [10.0] * 3), + ).api_row_df + assert_frame_equal(df, expected_df) + + def test__get_base_signal_transform(self): + assert _get_base_signal_transform(DATA_SIGNALS_BY_KEY[("src", "sig_smooth")], DATA_SIGNALS_BY_KEY) == SMOOTH + assert _get_base_signal_transform(DATA_SIGNALS_BY_KEY[("src", "sig_diff_smooth")], DATA_SIGNALS_BY_KEY) == DIFF_SMOOTH + assert _get_base_signal_transform(DATA_SIGNALS_BY_KEY[("src", "sig_diff")], DATA_SIGNALS_BY_KEY) == DIFF + assert _get_base_signal_transform(("src", "sig_diff"), DATA_SIGNALS_BY_KEY) == DIFF + assert _get_base_signal_transform(DATA_SIGNALS_BY_KEY[("src", "sig_base")], DATA_SIGNALS_BY_KEY) == IDENTITY + assert _get_base_signal_transform(("src", "sig_unknown"), DATA_SIGNALS_BY_KEY) == IDENTITY + + def test_get_transform_types(self): + source_signal_pairs = [SourceSignalPair(source="src", signal=True)] + transform_types = get_transform_types(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + expected_transform_types = {IDENTITY, DIFF, SMOOTH, DIFF_SMOOTH} + assert transform_types == expected_transform_types + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_diff"])] + transform_types = get_transform_types(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + expected_transform_types = {DIFF} + assert transform_types == expected_transform_types + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_smooth"])] + transform_types = get_transform_types(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + expected_transform_types = {SMOOTH} + assert transform_types == expected_transform_types + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_diff_smooth"])] + transform_types = get_transform_types(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + expected_transform_types = {DIFF_SMOOTH} + assert transform_types == expected_transform_types + + def test_get_pad_length(self): + source_signal_pairs = [SourceSignalPair(source="src", signal=True)] + pad_length = get_pad_length(source_signal_pairs, smoother_window_length=7, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + assert pad_length == 7 + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_diff"])] + pad_length = get_pad_length(source_signal_pairs, smoother_window_length=7, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + assert pad_length == 1 + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_smooth"])] + pad_length = get_pad_length(source_signal_pairs, smoother_window_length=5, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + assert pad_length == 4 + + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_diff_smooth"])] + pad_length = get_pad_length(source_signal_pairs, smoother_window_length=10, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + assert pad_length == 10 + + def test_pad_time_pairs(self): + # fmt: off + time_pairs = [ + TimePair("day", [20210817, (20210810, 20210815)]), + TimePair("day", True), + TimePair("day", [20210816]) + ] + expected_padded_time_pairs = [ + TimePair("day", [20210817, (20210810, 20210815)]), + TimePair("day", True), + TimePair("day", [20210816]), + TimePair("day", [(20210803, 20210810)]) + ] + assert pad_time_pairs(time_pairs, pad_length=7) == expected_padded_time_pairs + + time_pairs = [ + TimePair("day", [20210817, (20210810, 20210815)]), + TimePair("day", True), + TimePair("day", [20210816]), + TimePair("day", [20210809]) + ] + expected_padded_time_pairs = [ + TimePair("day", [20210817, (20210810, 20210815)]), + TimePair("day", True), + TimePair("day", [20210816]), + TimePair("day", [20210809]), + TimePair("day", [(20210801, 20210809)]), + ] + assert pad_time_pairs(time_pairs, pad_length=8) == expected_padded_time_pairs + + time_pairs = [ + TimePair("day", [20210817, (20210810, 20210815)]) + ] + expected_padded_time_pairs = [ + TimePair("day", [20210817, (20210810, 20210815)]) + ] + assert pad_time_pairs(time_pairs, pad_length=0) == expected_padded_time_pairs + # fmt: on + + def test__generate_transformed_rows(self): + # fmt: off + with self.subTest("diffed signal test"): + data = CovidcastRows.from_args( + signal=["sig_base"] * 5, + time_value=pd.date_range("2021-05-01", "2021-05-05"), + value=range(5) + ).api_row_df + transform_dict = {SourceSignalPair("src", ["sig_base"]): SourceSignalPair("src", ["sig_diff"])} + df = CovidcastRows.from_records(_generate_transformed_rows(data.to_dict(orient="records"), transform_dict=transform_dict, data_signals_by_key=DATA_SIGNALS_BY_KEY)).api_row_df + + expected_df = CovidcastRows.from_args( + signal=["sig_diff"] * 4, + time_value=pd.date_range("2021-05-02", "2021-05-05"), + value=[1.0] * 4, + stderr=[None] * 4, + sample_size=[None] * 4, + missing_stderr=[Nans.NOT_APPLICABLE] * 4, + missing_sample_size=[Nans.NOT_APPLICABLE] * 4, + ).api_row_df + + assert_frame_equal(df, expected_df) + + with self.subTest("smoothed and diffed signals on one base test"): + data = CovidcastRows.from_args( + signal=["sig_base"] * 10, + time_value=pd.date_range("2021-05-01", "2021-05-10"), + value=range(10), + stderr=range(10), + sample_size=range(10) + ).api_row_df + transform_dict = {SourceSignalPair("src", ["sig_base"]): SourceSignalPair("src", ["sig_diff", "sig_smooth"])} + df = CovidcastRows.from_records(_generate_transformed_rows(data.to_dict(orient="records"), transform_dict=transform_dict, data_signals_by_key=DATA_SIGNALS_BY_KEY)).api_row_df + + expected_df = CovidcastRows.from_args( + signal=interleave_longest(["sig_diff"] * 9, ["sig_smooth"] * 4), + time_value=interleave_longest(pd.date_range("2021-05-02", "2021-05-10"), pd.date_range("2021-05-07", "2021-05-10")), + value=interleave_longest(_diff_rows(data.value.to_list()), _smooth_rows(data.value.to_list())), + stderr=[None] * 13, + sample_size=[None] * 13, + ).api_row_df + + # Test no order. + idx = ["source", "signal", "time_value"] + assert_frame_equal(df.set_index(idx).sort_index(), expected_df.set_index(idx).sort_index()) + # Test order. + assert_frame_equal(df, expected_df) + + with self.subTest("smoothed and diffed signal on two non-continguous regions"): + data = CovidcastRows.from_args( + signal=["sig_base"] * 15, + time_value=chain(pd.date_range("2021-05-01", "2021-05-10"), pd.date_range("2021-05-16", "2021-05-20")), + value=range(15), + stderr=range(15), + sample_size=range(15), + ).api_row_df + transform_dict = {SourceSignalPair("src", ["sig_base"]): SourceSignalPair("src", ["sig_diff", "sig_smooth"])} + time_pairs = [TimePair("day", [(20210501, 20210520)])] + df = CovidcastRows.from_records( + _generate_transformed_rows(data.to_dict(orient="records"), time_pairs=time_pairs, transform_dict=transform_dict, data_signals_by_key=DATA_SIGNALS_BY_KEY) + ).api_row_df + + filled_values = data.value.to_list()[:10] + [None] * 5 + data.value.to_list()[10:] + filled_time_values = list(chain(pd.date_range("2021-05-01", "2021-05-10"), [None] * 5, pd.date_range("2021-05-16", "2021-05-20"))) + + expected_df = CovidcastRows.from_args( + signal=interleave_longest(["sig_diff"] * 19, ["sig_smooth"] * 14), + time_value=interleave_longest(pd.date_range("2021-05-02", "2021-05-20"), pd.date_range("2021-05-07", "2021-05-20")), + value=interleave_longest(_diff_rows(filled_values), _smooth_rows(filled_values)), + stderr=[None] * 33, + sample_size=[None] * 33, + issue=interleave_longest(_reindex_windowed(filled_time_values, 2), _reindex_windowed(filled_time_values, 7)), + ).api_row_df + # Test no order. + idx = ["source", "signal", "time_value"] + assert_frame_equal(df.set_index(idx).sort_index(), expected_df.set_index(idx).sort_index()) + # Test order. + assert_frame_equal(df, expected_df) + # fmt: on + + def test_get_basename_signals(self): + with self.subTest("none to transform"): + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_base"])] + basename_pairs, _ = get_basename_signal_and_jit_generator(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + expected_basename_pairs = [SourceSignalPair(source="src", signal=["sig_base"])] + assert basename_pairs == expected_basename_pairs + + with self.subTest("unrecognized signal"): + source_signal_pairs = [SourceSignalPair(source="src", signal=["sig_unknown"])] + basename_pairs, _ = get_basename_signal_and_jit_generator(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + expected_basename_pairs = [SourceSignalPair(source="src", signal=["sig_unknown"])] + assert basename_pairs == expected_basename_pairs + + with self.subTest("plain"): + source_signal_pairs = [ + SourceSignalPair(source="src", signal=["sig_diff", "sig_smooth", "sig_diff_smooth", "sig_base"]), + SourceSignalPair(source="src2", signal=["sig"]), + ] + basename_pairs, _ = get_basename_signal_and_jit_generator(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + expected_basename_pairs = [ + SourceSignalPair(source="src", signal=["sig_base", "sig_base", "sig_base", "sig_base"]), + SourceSignalPair(source="src2", signal=["sig"]), + ] + assert basename_pairs == expected_basename_pairs + + with self.subTest("resolve"): + source_signal_pairs = [SourceSignalPair(source="src", signal=True)] + basename_pairs, _ = get_basename_signal_and_jit_generator(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + expected_basename_pairs = [SourceSignalPair("src", ["sig_base"] * 4)] + assert basename_pairs == expected_basename_pairs + + with self.subTest("test base, diff, smooth"): + # fmt: off + data = CovidcastRows.from_args( + signal=["sig_base"] * 20 + ["sig_other"] * 5, + time_value=chain(pd.date_range("2021-05-01", "2021-05-10"), pd.date_range("2021-05-21", "2021-05-30"), pd.date_range("2021-05-01", "2021-05-05")), + value=chain(range(20), range(5)), + stderr=chain(range(20), range(5)), + sample_size=chain(range(20), range(5)), + ).api_row_df + source_signal_pairs = [SourceSignalPair("src", ["sig_base", "sig_diff", "sig_other", "sig_smooth"])] + _, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + time_pairs = [TimePair("day", [(20210501, 20210530)])] + df = CovidcastRows.from_records(row_transform_generator(data.to_dict(orient="records"), time_pairs=time_pairs)).api_row_df + + filled_values = list(chain(range(10), [None] * 10, range(10, 20))) + filled_time_values = list(chain(pd.date_range("2021-05-01", "2021-05-10"), [None] * 10, pd.date_range("2021-05-21", "2021-05-30"))) + + expected_df = CovidcastRows.from_args( + signal=["sig_base"] * 30 + ["sig_diff"] * 29 + ["sig_other"] * 5 + ["sig_smooth"] * 24, + time_value=chain( + pd.date_range("2021-05-01", "2021-05-30"), + pd.date_range("2021-05-02", "2021-05-30"), + pd.date_range("2021-05-01", "2021-05-05"), + pd.date_range("2021-05-07", "2021-05-30") + ), + value=chain( + filled_values, + _diff_rows(filled_values), + range(5), + _smooth_rows(filled_values) + ), + stderr=chain( + chain(range(10), [None] * 10, range(10, 20)), + chain([None] * 29), + range(5), + chain([None] * 24), + ), + sample_size=chain( + chain(range(10), [None] * 10, range(10, 20)), + chain([None] * 29), + range(5), + chain([None] * 24), + ), + issue=chain(filled_time_values, _reindex_windowed(filled_time_values, 2), pd.date_range("2021-05-01", "2021-05-05"), _reindex_windowed(filled_time_values, 7)), + ).api_row_df + # fmt: on + # Test no order. + idx = ["source", "signal", "time_value"] + assert_frame_equal(df.set_index(idx).sort_index(), expected_df.set_index(idx).sort_index()) + + with self.subTest("test base, diff, smooth; multiple geos"): + # fmt: off + data = CovidcastRows.from_args( + signal=["sig_base"] * 40, + geo_value=["ak"] * 20 + ["ca"] * 20, + time_value=chain(pd.date_range("2021-05-01", "2021-05-20"), pd.date_range("2021-05-01", "2021-05-20")), + value=chain(range(20), range(0, 40, 2)), + stderr=chain(range(20), range(0, 40, 2)), + sample_size=chain(range(20), range(0, 40, 2)), + ).api_row_df + source_signal_pairs = [SourceSignalPair("src", ["sig_base", "sig_diff", "sig_other", "sig_smooth"])] + _, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + df = CovidcastRows.from_records(row_transform_generator(data.to_dict(orient="records"))).api_row_df + + expected_df = CovidcastRows.from_args( + signal=["sig_base"] * 40 + ["sig_diff"] * 38 + ["sig_smooth"] * 28, + geo_value=["ak"] * 20 + ["ca"] * 20 + ["ak"] * 19 + ["ca"] * 19 + ["ak"] * 14 + ["ca"] * 14, + time_value=chain( + pd.date_range("2021-05-01", "2021-05-20"), + pd.date_range("2021-05-01", "2021-05-20"), + pd.date_range("2021-05-02", "2021-05-20"), + pd.date_range("2021-05-02", "2021-05-20"), + pd.date_range("2021-05-07", "2021-05-20"), + pd.date_range("2021-05-07", "2021-05-20"), + ), + value=chain( + chain(range(20), range(0, 40, 2)), + chain([1] * 19, [2] * 19), + chain([sum(x) / len(x) for x in windowed(range(20), 7)], + [sum(x) / len(x) for x in windowed(range(0, 40, 2), 7)]) + ), + stderr=chain( + chain(range(20), range(0, 40, 2)), + chain([None] * 38), + chain([None] * 28), + ), + sample_size=chain( + chain(range(20), range(0, 40, 2)), + chain([None] * 38), + chain([None] * 28), + ), + ).api_row_df + # fmt: on + # Test no order. + idx = ["source", "signal", "time_value"] + assert_frame_equal(df.set_index(idx).sort_index(), expected_df.set_index(idx).sort_index()) + + with self.subTest("resolve signals called"): + data = CovidcastRows.from_args( + signal=["sig_base"] * 20 + ["sig_other"] * 5, + time_value=chain(pd.date_range("2021-05-01", "2021-05-10"), pd.date_range("2021-05-21", "2021-05-30"), pd.date_range("2021-05-01", "2021-05-05")), + value=chain(range(20), range(5)), + stderr=chain(range(20), range(5)), + sample_size=chain(range(20), range(5)), + ).api_row_df + source_signal_pairs = [SourceSignalPair("src", True)] + _, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + time_pairs = [TimePair("day", [(20210501, 20210530)])] + df = CovidcastRows.from_records(row_transform_generator(data.to_dict(orient="records"), time_pairs=time_pairs)).api_row_df + + filled_values = list(chain(range(10), [None] * 10, range(10, 20))) + filled_time_values = list(chain(pd.date_range("2021-05-01", "2021-05-10"), [None] * 10, pd.date_range("2021-05-21", "2021-05-30"))) + + # fmt: off + expected_df = CovidcastRows.from_args( + signal=["sig_base"] * 30 + ["sig_diff"] * 29 + ["sig_diff_smooth"] * 23 + ["sig_other"] * 5 + ["sig_smooth"] * 24, + time_value=chain( + pd.date_range("2021-05-01", "2021-05-30"), + pd.date_range("2021-05-02", "2021-05-30"), + pd.date_range("2021-05-08", "2021-05-30"), + pd.date_range("2021-05-01", "2021-05-05"), + pd.date_range("2021-05-07", "2021-05-30"), + ), + value=chain( + filled_values, + _diff_rows(filled_values), + _smooth_rows(_diff_rows(filled_values)), + range(5), + _smooth_rows(filled_values) + ), + stderr=chain( + chain(range(10), [None] * 10, range(10, 20)), + chain([None] * 29), + chain([None] * 23), + range(5), + chain([None] * 24), + ), + sample_size=chain( + chain(range(10), [None] * 10, range(10, 20)), + chain([None] * 29), + chain([None] * 23), + range(5), + chain([None] * 24), + ), + issue=chain( + filled_time_values, + _reindex_windowed(filled_time_values, 2), + _reindex_windowed(filled_time_values, 8), + pd.date_range("2021-05-01", "2021-05-05"), + _reindex_windowed(filled_time_values, 7), + ), + ).api_row_df + # fmt: off + # Test no order. + idx = ["source", "signal", "time_value"] + assert_frame_equal(df.set_index(idx).sort_index(), expected_df.set_index(idx).sort_index()) + + with self.subTest("empty iterator"): + source_signal_pairs = [SourceSignalPair("src", ["sig_base", "sig_diff", "sig_smooth"])] + _, row_transform_generator = get_basename_signal_and_jit_generator(source_signal_pairs, data_sources_by_id=DATA_SOURCES_BY_ID, data_signals_by_key=DATA_SIGNALS_BY_KEY) + assert list(row_transform_generator({})) == [] + + def test_get_day_range(self): + assert list(get_day_range([TimePair("day", [20210817])])) == [20210817] + assert list(get_day_range([TimePair("day", [20210817, (20210810, 20210815)])])) == [20210810, 20210811, 20210812, 20210813, 20210814, 20210815, 20210817] + assert list(get_day_range([TimePair("day", [(20210801, 20210805)]), TimePair("day", [(20210803, 20210807)])])) == [20210801, 20210802, 20210803, 20210804, 20210805, 20210806, 20210807] diff --git a/tests/server/endpoints/covidcast_utils/test_smooth_diff.py b/tests/server/endpoints/covidcast_utils/test_smooth_diff.py new file mode 100644 index 000000000..5ce3e0b8a --- /dev/null +++ b/tests/server/endpoints/covidcast_utils/test_smooth_diff.py @@ -0,0 +1,163 @@ +from pandas import DataFrame, date_range +from pandas.testing import assert_frame_equal +from numpy import nan, isnan +from itertools import chain +from pytest import raises +import unittest + +from delphi.epidata.acquisition.covidcast.covidcast_row import CovidcastRows +from delphi.epidata.server.endpoints.covidcast_utils.smooth_diff import generate_diffed_rows, generate_smoothed_rows, _smoother +from .test_model import _diff_rows, _smooth_rows + + +class TestStreaming(unittest.TestCase): + def test__smoother(self): + assert _smoother(list(range(1, 7)), [1] * 6) == sum(range(1, 7)) + assert _smoother([1] * 6, list(range(1, 7))) == sum(range(1, 7)) + assert isnan(_smoother([1, nan, nan])) + with raises(TypeError, match=r"unsupported operand type*"): + _smoother([1, nan, None]) + + + def test_generate_smoothed_rows(self): + with self.subTest("an empty dataframe should return an empty dataframe"): + data = DataFrame({}) + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.to_dict(orient='records'))).api_row_df + expected_df = CovidcastRows(rows=[]).api_row_df + assert_frame_equal(smoothed_df, expected_df) + + with self.subTest("a dataframe with not enough entries to make a single smoothed value, should return an empty dataframe"): + data = CovidcastRows.from_args( + time_value=[20210501] * 6, + value=[1.0] * 6 + ).api_row_df + + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.to_dict(orient='records'))).api_row_df + expected_df = CovidcastRows(rows=[]).api_row_df + assert_frame_equal(smoothed_df, expected_df) + + data = CovidcastRows.from_args( + time_value=date_range("2021-05-01", "2021-05-13"), + value=chain(range(10), [None, 2., 1.]) + ).api_row_df + + with self.subTest("regular window, nan fill"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.to_dict(orient='records'))).api_row_df + + smoothed_values = _smooth_rows(data.value.to_list()) + reduced_time_values = data.time_value.to_list()[-len(smoothed_values):] + + expected_df = CovidcastRows.from_args( + time_value=reduced_time_values, + value=smoothed_values, + stderr=[None] * len(smoothed_values), + sample_size=[None] * len(smoothed_values), + ).api_row_df + + assert_frame_equal(smoothed_df, expected_df) + + with self.subTest("regular window, 0 fill"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.to_dict(orient='records'), nan_fill_value=0.)).api_row_df + + smoothed_values = _smooth_rows([v if v is not None and not isnan(v) else 0. for v in data.value.to_list()]) + reduced_time_values = data.time_value.to_list()[-len(smoothed_values):] + + expected_df = CovidcastRows.from_args( + time_value=reduced_time_values, + value=smoothed_values, + stderr=[None] * len(smoothed_values), + sample_size=[None] * len(smoothed_values), + ).api_row_df + + assert_frame_equal(smoothed_df, expected_df) + + with self.subTest("regular window, different window length"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.to_dict(orient='records'), smoother_window_length=8)).api_row_df + + smoothed_values = _smooth_rows(data.value.to_list(), window_length=8) + reduced_time_values = data.time_value.to_list()[-len(smoothed_values):] + + expected_df = CovidcastRows.from_args( + time_value=reduced_time_values, + value=smoothed_values, + stderr=[None] * len(smoothed_values), + sample_size=[None] * len(smoothed_values), + ).api_row_df + assert_frame_equal(smoothed_df, expected_df) + + with self.subTest("regular window, different kernel"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.to_dict(orient='records'), smoother_kernel=list(range(8)))).api_row_df + + smoothed_values = _smooth_rows(data.value.to_list(), kernel=list(range(8))) + reduced_time_values = data.time_value.to_list()[-len(smoothed_values):] + + expected_df = CovidcastRows.from_args( + time_value=reduced_time_values, + value=smoothed_values, + stderr=[None] * len(smoothed_values), + sample_size=[None] * len(smoothed_values), + ).api_row_df + assert_frame_equal(smoothed_df, expected_df) + + with self.subTest("conflicting smoother args validation, smoother kernel should overwrite window length"): + smoothed_df = CovidcastRows.from_records(generate_smoothed_rows(data.to_dict(orient='records'), smoother_kernel=[1/7.]*7, smoother_window_length=10)).api_row_df + + smoothed_values = _smooth_rows(data.value.to_list(), kernel=[1/7.]*7) + reduced_time_values = data.time_value.to_list()[-len(smoothed_values):] + + expected_df = CovidcastRows.from_args( + time_value=reduced_time_values, + value=smoothed_values, + stderr=[None] * len(smoothed_values), + sample_size=[None] * len(smoothed_values), + ).api_row_df + assert_frame_equal(smoothed_df, expected_df) + + + def test_generate_diffed_rows(self): + with self.subTest("an empty dataframe should return an empty dataframe"): + data = DataFrame({}) + diffs_df = CovidcastRows.from_records(generate_diffed_rows(data.to_dict(orient='records'))).api_row_df + expected_df = CovidcastRows(rows=[]).api_row_df + assert_frame_equal(diffs_df, expected_df) + + with self.subTest("a dataframe with not enough data to make one row should return an empty dataframe"): + data = CovidcastRows.from_args(time_value=[20210501], value=[1.0]).api_row_df + diffs_df = CovidcastRows.from_records(generate_diffed_rows(data.to_dict(orient='records'))).api_row_df + expected_df = CovidcastRows(rows=[]).api_row_df + assert_frame_equal(diffs_df, expected_df) + + data = CovidcastRows.from_args( + time_value=date_range("2021-05-01", "2021-05-10"), + value=chain(range(7), [None, 2., 1.]) + ).api_row_df + + with self.subTest("no fill"): + diffs_df = CovidcastRows.from_records(generate_diffed_rows(data.to_dict(orient='records'))).api_row_df + + diffed_values = _diff_rows(data.value.to_list()) + reduced_time_values = data.time_value.to_list()[-len(diffed_values):] + + expected_df = CovidcastRows.from_args( + time_value=reduced_time_values, + value=diffed_values, + stderr=[None] * len(diffed_values), + sample_size=[None] * len(diffed_values), + ).api_row_df + + assert_frame_equal(diffs_df, expected_df) + + with self.subTest("yes fill"): + diffs_df = CovidcastRows.from_records(generate_diffed_rows(data.to_dict(orient='records'), nan_fill_value=2.)).api_row_df + + diffed_values = _diff_rows([v if v is not None and not isnan(v) else 2. for v in data.value.to_list()]) + reduced_time_values = data.time_value.to_list()[-len(diffed_values):] + + expected_df = CovidcastRows.from_args( + time_value=reduced_time_values, + value=diffed_values, + stderr=[None] * len(diffed_values), + sample_size=[None] * len(diffed_values), + ).api_row_df + + assert_frame_equal(diffs_df, expected_df) diff --git a/tests/server/endpoints/test_covidcast.py b/tests/server/endpoints/test_covidcast.py index b7ecdc263..823f9126a 100644 --- a/tests/server/endpoints/test_covidcast.py +++ b/tests/server/endpoints/test_covidcast.py @@ -5,10 +5,6 @@ from flask import Response from delphi.epidata.server.main import app -from delphi.epidata.server._params import ( - GeoPair, - TimePair, -) # py3tester coverage target __test_target__ = "delphi.epidata.server.endpoints.covidcast" diff --git a/tests/server/test_params.py b/tests/server/test_params.py index fffea0043..d2299dd02 100644 --- a/tests/server/test_params.py +++ b/tests/server/test_params.py @@ -19,6 +19,7 @@ GeoPair, TimePair, SourceSignalPair, + _combine_source_signal_pairs ) from delphi.epidata.server._exceptions import ( ValidationFailedException, @@ -182,8 +183,7 @@ def test_parse_source_signal_arg(self): self.assertEqual( parse_source_signal_arg(), [ - SourceSignalPair("src1", ["sig1"]), - SourceSignalPair("src1", ["sig4"]), + SourceSignalPair("src1", ["sig1", "sig4"]), ], ) with self.subTest("multi list"): @@ -191,17 +191,17 @@ def test_parse_source_signal_arg(self): self.assertEqual( parse_source_signal_arg(), [ - SourceSignalPair("src1", ["sig1", "sig2"]), SourceSignalPair("county", ["sig5", "sig6"]), + SourceSignalPair("src1", ["sig1", "sig2"]), ], ) with self.subTest("hybrid"): - with app.test_request_context("/?signal=src2:*;src1:sig4;src3:sig5,sig6"): + with app.test_request_context("/?signal=src2:*;src1:sig4;src3:sig5,sig6;src1:sig5;src2:sig1"): self.assertEqual( parse_source_signal_arg(), [ + SourceSignalPair("src1", ["sig4", "sig5"]), SourceSignalPair("src2", True), - SourceSignalPair("src1", ["sig4"]), SourceSignalPair("src3", ["sig5", "sig6"]), ], ) @@ -357,3 +357,29 @@ def test_parse_day_arg(self): self.assertRaises(ValidationFailedException, parse_day_arg, "time") with app.test_request_context("/?time=week:20121010"): self.assertRaises(ValidationFailedException, parse_day_arg, "time") + + def test__combine_source_signal_pairs(self): + source_signal_pairs = [ + SourceSignalPair("src1", ["sig1", "sig2"]), + SourceSignalPair("src2", ["sig1"]), + SourceSignalPair("src1", ["sig1", "sig3"]), + SourceSignalPair("src3", ["sig1"]), + SourceSignalPair("src3", ["sig2"]), + SourceSignalPair("src3", ["sig1"]), + SourceSignalPair("src4", ["sig2"]), + SourceSignalPair("src4", True), + ] + expected_source_signal_pairs = [ + SourceSignalPair("src1", ["sig1", "sig2", "sig3"]), + SourceSignalPair("src2", ["sig1"]), + SourceSignalPair("src3", ["sig1", "sig2"]), + SourceSignalPair("src4", True), + ] + combined_pairs = _combine_source_signal_pairs(source_signal_pairs) + for i, x in enumerate(combined_pairs): + if isinstance(x, list): + sorted(x) == expected_source_signal_pairs[i] + if isinstance(x, bool): + x == expected_source_signal_pairs[i] + + assert _combine_source_signal_pairs(source_signal_pairs) == expected_source_signal_pairs diff --git a/tests/server/utils/test_dates.py b/tests/server/utils/test_dates.py index e825bbd9b..c450a9b5b 100644 --- a/tests/server/utils/test_dates.py +++ b/tests/server/utils/test_dates.py @@ -2,7 +2,7 @@ from datetime import date from epiweeks import Week -from delphi.epidata.server.utils.dates import time_value_to_date, date_to_time_value, shift_time_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, week_value_to_week, time_values_to_ranges +from delphi.epidata.server.utils.dates import time_value_to_date, date_to_time_value, shift_time_value, time_value_to_iso, days_in_range, weeks_in_range, week_to_time_value, week_value_to_week, time_values_to_ranges, iterate_over_range, iterate_over_ints_and_ranges class UnitTests(unittest.TestCase): @@ -59,3 +59,19 @@ def test_time_values_to_ranges(self): self.assertEqual(time_values_to_ranges([20210228, 20210301]), [(20210228, 20210301)]) # this becomes a range because these dates are indeed consecutive # individual weeks become a range (2020 is a rare year with 53 weeks) self.assertEqual(time_values_to_ranges([202051, 202052, 202053, 202101, 202102]), [(202051, 202102)]) + + def test_iterate_over_range(self): + self.assertEqual(list(iterate_over_range(20210801, 20210805)), [20210801, 20210802, 20210803, 20210804]) + self.assertEqual(list(iterate_over_range(20210801, 20210801)), []) + self.assertEqual(list(iterate_over_range(20210801, 20210701)), []) + + def test_iterate_over_ints_and_ranges(self): + assert list(iterate_over_ints_and_ranges([0, (5, 8)], use_dates=False)) == [0, 5, 6, 7, 8] + assert list(iterate_over_ints_and_ranges([(5, 8), (4, 6), (3, 5)], use_dates=False)) == [3, 4, 5, 6, 7, 8] + assert list(iterate_over_ints_and_ranges([(7, 8), (5, 7), (3, 8), 8], use_dates=False)) == [3, 4, 5, 6, 7, 8] + assert list(iterate_over_ints_and_ranges([2, (2, 3)], use_dates=False)) == [2, 3] + assert list(iterate_over_ints_and_ranges([20, 50, 25, (21, 25), 23, 30, 31, (24, 26)], use_dates=False)) == [20, 21, 22, 23, 24, 25, 26, 30, 31, 50] + + assert list(iterate_over_ints_and_ranges([20210817])) == [20210817] + assert list(iterate_over_ints_and_ranges([20210817, (20210810, 20210815)])) == [20210810, 20210811, 20210812, 20210813, 20210814, 20210815, 20210817] + assert list(iterate_over_ints_and_ranges([(20210801, 20210905), (20210815, 20210915)])) == list(iterate_over_range(20210801, 20210916)) # right-exclusive