From eb8f9b00d912242282625661e5c5065d622e506e Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 21:20:16 -0500 Subject: [PATCH 01/29] improve RFC 3339 datetime handling --- README.md | 8 ++++- stac_fastapi/api/setup.py | 1 + stac_fastapi/api/stac_fastapi/api/rfc3339.py | 30 +++++++++++++++++++ .../pgstac/tests/resources/test_item.py | 25 ++++++++-------- .../stac_fastapi/sqlalchemy/serializers.py | 9 +++--- .../sqlalchemy/tests/resources/test_item.py | 22 +++++++------- .../types/stac_fastapi/types/search.py | 24 +++++++-------- 7 files changed, 76 insertions(+), 43 deletions(-) create mode 100644 stac_fastapi/api/stac_fastapi/api/rfc3339.py diff --git a/README.md b/README.md index f5ef6a50c..37bac7e68 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,17 @@ docker-compose up app-sqlalchemy docker-compose up app-pgstac ``` -For local development it is often more convenient to run the application outside of docker-compose: +For local development it is often more convenient to run the application outside docker-compose: ```bash make docker-run ``` +Before commit, install the [pre-commit](https://pre-commit.com) hooks with: + +```shell +pre-commit install +``` + #### Note to Docker for Windows users You'll need to enable experimental features on Docker for Windows in order to run the docker-compose, due to the "--platform" flag that is required to allow the project to run on some Apple architectures. To do this, open Docker Desktop, go to settings, select "Docker Engine", and modify the configuration JSON to have `"experimental": true`. diff --git a/stac_fastapi/api/setup.py b/stac_fastapi/api/setup.py index 6de3585d1..637a336a5 100644 --- a/stac_fastapi/api/setup.py +++ b/stac_fastapi/api/setup.py @@ -11,6 +11,7 @@ "stac_pydantic==2.0.*", "brotli_asgi", "stac-fastapi.types", + "ciso8601~=2.2.0", ] extra_reqs = { diff --git a/stac_fastapi/api/stac_fastapi/api/rfc3339.py b/stac_fastapi/api/stac_fastapi/api/rfc3339.py new file mode 100644 index 000000000..087ad938a --- /dev/null +++ b/stac_fastapi/api/stac_fastapi/api/rfc3339.py @@ -0,0 +1,30 @@ +"""rfc3339.""" + +from datetime import datetime, timezone + +import ciso8601 + + +def parse_rfc3339(value: str) -> datetime: + """Doc.""" + return ciso8601.parse_rfc3339(value) + + +def rfc3339_str(value: datetime, use_z: bool = True) -> str: + """Doc.""" + if not value.tzinfo: + value = value.replace(tzinfo=timezone.utc) + + str_value = value.isoformat() + str_value = str_value.replace("+00:00", "Z") if use_z else str_value + return str_value + + +def now_in_utc() -> datetime: + """Doc.""" + return datetime.now(timezone.utc) + + +def now_as_rfc3339_str() -> str: + """Doc.""" + return rfc3339_str(now_in_utc()) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index dfcb2d120..4a4f05a66 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -1,6 +1,6 @@ import json import uuid -from datetime import datetime, timedelta +from datetime import timedelta from typing import Callable from urllib.parse import parse_qs, urljoin, urlparse @@ -9,9 +9,9 @@ from httpx import AsyncClient from shapely.geometry import Polygon from stac_pydantic import Collection, Item -from stac_pydantic.shared import DATETIME_RFC339 from starlette.requests import Request +from stac_fastapi.api.rfc3339 import parse_rfc3339, rfc3339_str from stac_fastapi.pgstac.models.links import CollectionLinks @@ -402,14 +402,14 @@ async def test_item_search_temporal_query_post( ) assert resp.status_code == 200 - item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(test_item["properties"]["datetime"]) print(item_date) item_date = item_date + timedelta(seconds=1) params = { "collections": [test_item["collection"]], "intersects": test_item["geometry"], - "datetime": item_date.strftime(DATETIME_RFC339), + "datetime": rfc3339_str(item_date), } resp = await app_client.post("/search", json=params) @@ -437,14 +437,15 @@ async def test_item_search_temporal_window_post( ) assert resp.status_code == 200 - item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) params = { "collections": [test_item["collection"]], - "datetime": f"{item_date_before.strftime(DATETIME_RFC339)}/{item_date_after.strftime(DATETIME_RFC339)}", + "datetime": f"{rfc3339_str(item_date_before)}/{rfc3339_str(item_date_after)}", } + resp = await app_client.post("/search", json=params) resp_json = resp.json() assert len(resp_json["features"]) == 1 @@ -482,7 +483,7 @@ async def test_item_search_temporal_open_window( async def test_item_search_sort_post(app_client, load_test_data, load_test_collection): """Test POST search with sorting (sort extension)""" first_item = load_test_data("test_item.json") - item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(first_item["properties"]["datetime"]) resp = await app_client.post( f"/collections/{first_item['collection']}/items", json=first_item ) @@ -491,7 +492,7 @@ async def test_item_search_sort_post(app_client, load_test_data, load_test_colle second_item = load_test_data("test_item.json") second_item["id"] = "another-item" another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.strftime(DATETIME_RFC339) + second_item["properties"]["datetime"] = rfc3339_str(another_item_date) resp = await app_client.post( f"/collections/{second_item['collection']}/items", json=second_item ) @@ -601,13 +602,13 @@ async def test_item_search_temporal_window_get( ) assert resp.status_code == 200 - item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) params = { "collections": test_item["collection"], - "datetime": f"{item_date_before.strftime(DATETIME_RFC339)}/{item_date_after.strftime(DATETIME_RFC339)}", + "datetime": f"{rfc3339_str(item_date_before)}/{rfc3339_str(item_date_after)}", } resp = await app_client.get("/search", params=params) resp_json = resp.json() @@ -619,7 +620,7 @@ async def test_item_search_temporal_window_get( async def test_item_search_sort_get(app_client, load_test_data, load_test_collection): """Test GET search with sorting (sort extension)""" first_item = load_test_data("test_item.json") - item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(first_item["properties"]["datetime"]) resp = await app_client.post( f"/collections/{first_item['collection']}/items", json=first_item ) @@ -628,7 +629,7 @@ async def test_item_search_sort_get(app_client, load_test_data, load_test_collec second_item = load_test_data("test_item.json") second_item["id"] = "another-item" another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.strftime(DATETIME_RFC339) + second_item["properties"]["datetime"] = rfc3339_str(another_item_date) resp = await app_client.post( f"/collections/{second_item['collection']}/items", json=second_item ) diff --git a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py index c1a04fbb0..9d4c6c6d7 100644 --- a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py +++ b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py @@ -1,13 +1,12 @@ """Serializers.""" import abc import json -from datetime import datetime from typing import TypedDict import attr import geoalchemy2 as ga -from stac_pydantic.shared import DATETIME_RFC339 +from stac_fastapi.api.rfc3339 import now_as_rfc3339_str, parse_rfc3339, rfc3339_str from stac_fastapi.sqlalchemy.models import database from stac_fastapi.types import stac as stac_types from stac_fastapi.types.config import Settings @@ -55,7 +54,7 @@ def db_to_stac(cls, db_model: database.Item, base_url: str) -> stac_types.Item: # Use getattr to accommodate extension namespaces field_value = getattr(db_model, field.split(":")[-1]) if field == "datetime": - field_value = field_value.strftime(DATETIME_RFC339) + field_value = rfc3339_str(field_value) properties[field] = field_value item_id = db_model.id collection_id = db_model.collection_id @@ -101,12 +100,12 @@ def stac_to_db( # Use getattr to accommodate extension namespaces field_value = stac_data["properties"][field] if field == "datetime": - field_value = datetime.strptime(field_value, DATETIME_RFC339) + field_value = parse_rfc3339(field_value) indexed_fields[field.split(":")[-1]] = field_value # TODO: Exclude indexed fields from the properties jsonb field to prevent duplication - now = datetime.utcnow().strftime(DATETIME_RFC339) + now = now_as_rfc3339_str() if "created" not in stac_data["properties"]: stac_data["properties"]["created"] = now stac_data["properties"]["updated"] = now diff --git a/stac_fastapi/sqlalchemy/tests/resources/test_item.py b/stac_fastapi/sqlalchemy/tests/resources/test_item.py index fe22ecfde..247eb787f 100644 --- a/stac_fastapi/sqlalchemy/tests/resources/test_item.py +++ b/stac_fastapi/sqlalchemy/tests/resources/test_item.py @@ -10,8 +10,8 @@ import pystac from pydantic.datetime_parse import parse_datetime from shapely.geometry import Polygon -from stac_pydantic.shared import DATETIME_RFC339 +from stac_fastapi.api.rfc3339 import parse_rfc3339, rfc3339_str from stac_fastapi.sqlalchemy.core import CoreCrudClient from stac_fastapi.types.core import LandingPageMixin @@ -419,13 +419,13 @@ def test_item_search_temporal_query_post(app_client, load_test_data): ) assert resp.status_code == 200 - item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(test_item["properties"]["datetime"]) item_date = item_date + timedelta(seconds=1) params = { "collections": [test_item["collection"]], "intersects": test_item["geometry"], - "datetime": f"../{item_date.strftime(DATETIME_RFC339)}", + "datetime": f"../{rfc3339_str(item_date)}", } resp = app_client.post("/search", json=params) resp_json = resp.json() @@ -440,14 +440,14 @@ def test_item_search_temporal_window_post(app_client, load_test_data): ) assert resp.status_code == 200 - item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) params = { "collections": [test_item["collection"]], "intersects": test_item["geometry"], - "datetime": f"{item_date_before.strftime(DATETIME_RFC339)}/{item_date_after.strftime(DATETIME_RFC339)}", + "datetime": f"{rfc3339_str(item_date_before)}/{rfc3339_str(item_date_after)}", } resp = app_client.post("/search", json=params) resp_json = resp.json() @@ -475,7 +475,7 @@ def test_item_search_temporal_open_window(app_client, load_test_data): def test_item_search_sort_post(app_client, load_test_data): """Test POST search with sorting (sort extension)""" first_item = load_test_data("test_item.json") - item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(first_item["properties"]["datetime"]) resp = app_client.post( f"/collections/{first_item['collection']}/items", json=first_item ) @@ -484,7 +484,7 @@ def test_item_search_sort_post(app_client, load_test_data): second_item = load_test_data("test_item.json") second_item["id"] = "another-item" another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.strftime(DATETIME_RFC339) + second_item["properties"]["datetime"] = rfc3339_str(another_item_date) resp = app_client.post( f"/collections/{second_item['collection']}/items", json=second_item ) @@ -563,14 +563,14 @@ def test_item_search_temporal_window_get(app_client, load_test_data): ) assert resp.status_code == 200 - item_date = datetime.strptime(test_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) params = { "collections": test_item["collection"], "bbox": ",".join([str(coord) for coord in test_item["bbox"]]), - "datetime": f"{item_date_before.strftime(DATETIME_RFC339)}/{item_date_after.strftime(DATETIME_RFC339)}", + "datetime": f"{rfc3339_str(item_date_before)}/{rfc3339_str(item_date_after)}", } resp = app_client.get("/search", params=params) resp_json = resp.json() @@ -580,7 +580,7 @@ def test_item_search_temporal_window_get(app_client, load_test_data): def test_item_search_sort_get(app_client, load_test_data): """Test GET search with sorting (sort extension)""" first_item = load_test_data("test_item.json") - item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339) + item_date = parse_rfc3339(first_item["properties"]["datetime"]) resp = app_client.post( f"/collections/{first_item['collection']}/items", json=first_item ) @@ -589,7 +589,7 @@ def test_item_search_sort_get(app_client, load_test_data): second_item = load_test_data("test_item.json") second_item["id"] = "another-item" another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = another_item_date.strftime(DATETIME_RFC339) + second_item["properties"]["datetime"] = rfc3339_str(another_item_date) resp = app_client.post( f"/collections/{second_item['collection']}/items", json=second_item ) diff --git a/stac_fastapi/types/stac_fastapi/types/search.py b/stac_fastapi/types/stac_fastapi/types/search.py index b7ca69f40..8b10ccb7e 100644 --- a/stac_fastapi/types/stac_fastapi/types/search.py +++ b/stac_fastapi/types/stac_fastapi/types/search.py @@ -97,31 +97,27 @@ class BaseSearchPostRequest(BaseModel): datetime: Optional[str] limit: Optional[conint(gt=0, le=10000)] = 10 - @property - def start_date(self) -> Optional[datetime]: + def _interval_date(self, position: int) -> Optional[datetime]: """Extract the start date from the datetime string.""" if not self.datetime: return values = self.datetime.split("/") - if len(values) == 1: + if len(values) != 2: return None - if values[0] == "..": + if values[position] in ["..", ""]: return None - return parse_datetime(values[0]) + return parse_datetime(values[position]) + + @property + def start_date(self) -> Optional[datetime]: + """Extract the start date from the datetime string.""" + return self._interval_date(self.datetime, 0) @property def end_date(self) -> Optional[datetime]: """Extract the end date from the datetime string.""" - if not self.datetime: - return - - values = self.datetime.split("/") - if len(values) == 1: - return parse_datetime(values[0]) - if values[1] == "..": - return None - return parse_datetime(values[1]) + return self._interval_date(self.datetime, 1) @validator("intersects") def validate_spatial(cls, v, values): From 7a751feab74e6f64862cc409b40f7c67769c0465 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 21:28:02 -0500 Subject: [PATCH 02/29] install build-essential --- .github/workflows/cicd.yaml | 1 + README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 336852f4b..1eb53cca9 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -109,6 +109,7 @@ jobs: test-docs: runs-on: ubuntu-latest steps: + - run: apt update && apt install build-essential - uses: actions/checkout@v2 - name: Test generating docs run: make docs \ No newline at end of file diff --git a/README.md b/README.md index 37bac7e68..cac659543 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ pip install -e stac_fastapi/pgstac ``` ## Local Development + Use docker-compose to deploy the application, migrate the database, and ingest some example data: ```bash docker-compose build From 15b6ff6ec9a8ca9a710f6820c0b4da874dbbe4dc Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 21:44:03 -0500 Subject: [PATCH 03/29] sudo, rearrange files --- .github/workflows/cicd.yaml | 2 +- .../pgstac/tests/resources/test_item.py | 2 +- .../stac_fastapi/sqlalchemy/serializers.py | 2 +- .../sqlalchemy/tests/resources/test_item.py | 2 +- .../stac_fastapi/types}/rfc3339.py | 23 +++++++++++++++++++ 5 files changed, 27 insertions(+), 4 deletions(-) rename stac_fastapi/{api/stac_fastapi/api => types/stac_fastapi/types}/rfc3339.py (52%) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 1eb53cca9..c8a7b0f1d 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -109,7 +109,7 @@ jobs: test-docs: runs-on: ubuntu-latest steps: - - run: apt update && apt install build-essential + - run: sudo apt update && sudo gapt install build-essential - uses: actions/checkout@v2 - name: Test generating docs run: make docs \ No newline at end of file diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 4a4f05a66..78cad0ec3 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -11,8 +11,8 @@ from stac_pydantic import Collection, Item from starlette.requests import Request -from stac_fastapi.api.rfc3339 import parse_rfc3339, rfc3339_str from stac_fastapi.pgstac.models.links import CollectionLinks +from stac_fastapi.types.rfc3339 import parse_rfc3339, rfc3339_str @pytest.mark.asyncio diff --git a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py index 9d4c6c6d7..2d5ecf2ca 100644 --- a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py +++ b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py @@ -6,11 +6,11 @@ import attr import geoalchemy2 as ga -from stac_fastapi.api.rfc3339 import now_as_rfc3339_str, parse_rfc3339, rfc3339_str from stac_fastapi.sqlalchemy.models import database from stac_fastapi.types import stac as stac_types from stac_fastapi.types.config import Settings from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links +from stac_fastapi.types.rfc3339 import now_as_rfc3339_str, parse_rfc3339, rfc3339_str @attr.s # type:ignore diff --git a/stac_fastapi/sqlalchemy/tests/resources/test_item.py b/stac_fastapi/sqlalchemy/tests/resources/test_item.py index 247eb787f..f31b8d92c 100644 --- a/stac_fastapi/sqlalchemy/tests/resources/test_item.py +++ b/stac_fastapi/sqlalchemy/tests/resources/test_item.py @@ -11,9 +11,9 @@ from pydantic.datetime_parse import parse_datetime from shapely.geometry import Polygon -from stac_fastapi.api.rfc3339 import parse_rfc3339, rfc3339_str from stac_fastapi.sqlalchemy.core import CoreCrudClient from stac_fastapi.types.core import LandingPageMixin +from stac_fastapi.types.rfc3339 import parse_rfc3339, rfc3339_str def test_create_and_delete_item(app_client, load_test_data): diff --git a/stac_fastapi/api/stac_fastapi/api/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py similarity index 52% rename from stac_fastapi/api/stac_fastapi/api/rfc3339.py rename to stac_fastapi/types/stac_fastapi/types/rfc3339.py index 087ad938a..f062a887c 100644 --- a/stac_fastapi/api/stac_fastapi/api/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -1,6 +1,7 @@ """rfc3339.""" from datetime import datetime, timezone +from typing import Optional import ciso8601 @@ -28,3 +29,25 @@ def now_in_utc() -> datetime: def now_as_rfc3339_str() -> str: """Doc.""" return rfc3339_str(now_in_utc()) + + +def parse_interval(value: str) -> Optional[(Optional[datetime], Optional[datetime])]: + """Extract a tuple of datetimes from an interval string.""" + if not value: + return + + values = value.split("/") + if len(values) != 2: + return None + + start = None + end = None + if not values[0] in ["..", ""]: + start = parse_rfc3339(values[0]) + if not values[1] in ["..", ""]: + end = parse_rfc3339(values[1]) + + if start is None and end is None: + return None + else: + return (start, end) From eec397171b24233d0767bf4c40484803d2d663df Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 21:45:00 -0500 Subject: [PATCH 04/29] gapt -> apt --- .github/workflows/cicd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index c8a7b0f1d..0faba5ace 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -109,7 +109,7 @@ jobs: test-docs: runs-on: ubuntu-latest steps: - - run: sudo apt update && sudo gapt install build-essential + - run: sudo apt update && sudo apt install build-essential - uses: actions/checkout@v2 - name: Test generating docs run: make docs \ No newline at end of file From 0c9b56ef5954df981b4b4587f39a2c2ec60674ff Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 21:46:53 -0500 Subject: [PATCH 05/29] install build-essential in docker img --- .github/workflows/cicd.yaml | 1 - Dockerfile.docs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 0faba5ace..336852f4b 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -109,7 +109,6 @@ jobs: test-docs: runs-on: ubuntu-latest steps: - - run: sudo apt update && sudo apt install build-essential - uses: actions/checkout@v2 - name: Test generating docs run: make docs \ No newline at end of file diff --git a/Dockerfile.docs b/Dockerfile.docs index cc60230d6..7e8a2a5c7 100644 --- a/Dockerfile.docs +++ b/Dockerfile.docs @@ -1,5 +1,6 @@ FROM python:3.8-slim +RUN apt update && apt install -y build-essential RUN python -m pip install --upgrade pip RUN python -m pip install mkdocs mkdocs-material pdocs From 17fb42df84579dafeccda0fa4d893e5a8514c4ca Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 21:50:52 -0500 Subject: [PATCH 06/29] move ciso8601 dep --- Dockerfile.docs | 2 ++ stac_fastapi/api/setup.py | 1 - stac_fastapi/types/setup.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile.docs b/Dockerfile.docs index 7e8a2a5c7..a7420f31f 100644 --- a/Dockerfile.docs +++ b/Dockerfile.docs @@ -1,6 +1,8 @@ FROM python:3.8-slim +# build-essential is required to build a wheel for ciso8601 RUN apt update && apt install -y build-essential + RUN python -m pip install --upgrade pip RUN python -m pip install mkdocs mkdocs-material pdocs diff --git a/stac_fastapi/api/setup.py b/stac_fastapi/api/setup.py index 637a336a5..6de3585d1 100644 --- a/stac_fastapi/api/setup.py +++ b/stac_fastapi/api/setup.py @@ -11,7 +11,6 @@ "stac_pydantic==2.0.*", "brotli_asgi", "stac-fastapi.types", - "ciso8601~=2.2.0", ] extra_reqs = { diff --git a/stac_fastapi/types/setup.py b/stac_fastapi/types/setup.py index 3baf6aecc..994015d85 100644 --- a/stac_fastapi/types/setup.py +++ b/stac_fastapi/types/setup.py @@ -10,6 +10,7 @@ "attrs", "pydantic[dotenv]", "stac_pydantic==2.0.*", + "ciso8601~=2.2.0", ] extra_reqs = { From 6666af54210ad73c2c6ca093a3cf1a7e40a8b5d0 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 22:08:37 -0500 Subject: [PATCH 07/29] fix typing for parse_interval function --- .../types/stac_fastapi/types/rfc3339.py | 6 ++-- .../types/stac_fastapi/types/search.py | 28 ++++++++----------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index f062a887c..65df89d29 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -1,7 +1,7 @@ """rfc3339.""" from datetime import datetime, timezone -from typing import Optional +from typing import Optional, tuple import ciso8601 @@ -31,7 +31,9 @@ def now_as_rfc3339_str() -> str: return rfc3339_str(now_in_utc()) -def parse_interval(value: str) -> Optional[(Optional[datetime], Optional[datetime])]: +def parse_interval( + value: str, +) -> Optional[tuple[Optional[datetime], Optional[datetime]]]: """Extract a tuple of datetimes from an interval string.""" if not value: return diff --git a/stac_fastapi/types/stac_fastapi/types/search.py b/stac_fastapi/types/stac_fastapi/types/search.py index 8b10ccb7e..781684bf4 100644 --- a/stac_fastapi/types/stac_fastapi/types/search.py +++ b/stac_fastapi/types/stac_fastapi/types/search.py @@ -5,7 +5,6 @@ import abc import operator -from datetime import datetime from enum import auto from types import DynamicClassAttribute from typing import Any, Callable, Dict, List, Optional, Union @@ -25,6 +24,8 @@ from stac_pydantic.shared import BBox from stac_pydantic.utils import AutoValueEnum +from stac_fastapi.types import parse_interval + # Be careful: https://github.com/samuelcolvin/pydantic/issues/1423#issuecomment-642797287 NumType = Union[float, int] @@ -97,28 +98,19 @@ class BaseSearchPostRequest(BaseModel): datetime: Optional[str] limit: Optional[conint(gt=0, le=10000)] = 10 - def _interval_date(self, position: int) -> Optional[datetime]: - """Extract the start date from the datetime string.""" - if not self.datetime: - return - - values = self.datetime.split("/") - if len(values) != 2: - return None - if values[position] in ["..", ""]: - return None - return parse_datetime(values[position]) - @property - def start_date(self) -> Optional[datetime]: + def start_date(self): """Extract the start date from the datetime string.""" - return self._interval_date(self.datetime, 0) + interval = parse_interval(self.datetime) + return interval[0] if interval else None @property - def end_date(self) -> Optional[datetime]: + def end_date(self): """Extract the end date from the datetime string.""" - return self._interval_date(self.datetime, 1) + interval = parse_interval(self.datetime) + return interval[1] if interval else None + @classmethod @validator("intersects") def validate_spatial(cls, v, values): """Check bbox and intersects are not both supplied.""" @@ -126,6 +118,7 @@ def validate_spatial(cls, v, values): raise ValueError("intersects and bbox parameters are mutually exclusive") return v + @classmethod @validator("bbox") def validate_bbox(cls, v: BBox): """Check order of supplied bbox coordinates.""" @@ -156,6 +149,7 @@ def validate_bbox(cls, v: BBox): return v + @classmethod @validator("datetime") def validate_datetime(cls, v): """Validate datetime.""" From 1238bc1ade9f86dbdd0e7873614a7fa8adf6059d Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 22:19:20 -0500 Subject: [PATCH 08/29] fix import of parse_interval --- stac_fastapi/types/stac_fastapi/types/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/types/stac_fastapi/types/search.py b/stac_fastapi/types/stac_fastapi/types/search.py index 781684bf4..9e98f61b2 100644 --- a/stac_fastapi/types/stac_fastapi/types/search.py +++ b/stac_fastapi/types/stac_fastapi/types/search.py @@ -24,7 +24,7 @@ from stac_pydantic.shared import BBox from stac_pydantic.utils import AutoValueEnum -from stac_fastapi.types import parse_interval +from stac_fastapi.types.rfc3339 import parse_interval # Be careful: https://github.com/samuelcolvin/pydantic/issues/1423#issuecomment-642797287 NumType = Union[float, int] From 36e5bdd27e2537f3cfe3305fcdbae8a4dab65069 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 22:25:31 -0500 Subject: [PATCH 09/29] more changes for datetimes --- stac_fastapi/types/stac_fastapi/types/rfc3339.py | 2 +- stac_fastapi/types/stac_fastapi/types/search.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index 65df89d29..86fa17532 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -1,7 +1,7 @@ """rfc3339.""" from datetime import datetime, timezone -from typing import Optional, tuple +from typing import Optional import ciso8601 diff --git a/stac_fastapi/types/stac_fastapi/types/search.py b/stac_fastapi/types/stac_fastapi/types/search.py index 9e98f61b2..df79d024a 100644 --- a/stac_fastapi/types/stac_fastapi/types/search.py +++ b/stac_fastapi/types/stac_fastapi/types/search.py @@ -5,6 +5,7 @@ import abc import operator +from datetime import datetime from enum import auto from types import DynamicClassAttribute from typing import Any, Callable, Dict, List, Optional, Union @@ -20,11 +21,10 @@ _GeometryBase, ) from pydantic import BaseModel, conint, validator -from pydantic.datetime_parse import parse_datetime from stac_pydantic.shared import BBox from stac_pydantic.utils import AutoValueEnum -from stac_fastapi.types.rfc3339 import parse_interval +from stac_fastapi.types.rfc3339 import parse_interval, parse_rfc3339 # Be careful: https://github.com/samuelcolvin/pydantic/issues/1423#issuecomment-642797287 NumType = Union[float, int] @@ -99,13 +99,13 @@ class BaseSearchPostRequest(BaseModel): limit: Optional[conint(gt=0, le=10000)] = 10 @property - def start_date(self): + def start_date(self) -> Optional[datetime]: """Extract the start date from the datetime string.""" interval = parse_interval(self.datetime) return interval[0] if interval else None @property - def end_date(self): + def end_date(self) -> Optional[datetime]: """Extract the end date from the datetime string.""" interval = parse_interval(self.datetime) return interval[1] if interval else None @@ -165,11 +165,11 @@ def validate_datetime(cls, v): dates.append(value) continue - parse_datetime(value) + parse_rfc3339(value) dates.append(value) if ".." not in dates: - if parse_datetime(dates[0]) > parse_datetime(dates[1]): + if parse_rfc3339(dates[0]) > parse_rfc3339(dates[1]): raise ValueError( "Invalid datetime range, must match format (begin_date, end_date)" ) From e36a8c95200eed38a5d88c426a930ad8e8c4b93f Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 22:44:21 -0500 Subject: [PATCH 10/29] get the types right --- stac_fastapi/types/stac_fastapi/types/rfc3339.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index 86fa17532..cbc51354a 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -1,7 +1,7 @@ """rfc3339.""" from datetime import datetime, timezone -from typing import Optional +from typing import Optional, Tuple import ciso8601 @@ -33,7 +33,7 @@ def now_as_rfc3339_str() -> str: def parse_interval( value: str, -) -> Optional[tuple[Optional[datetime], Optional[datetime]]]: +) -> Optional[Tuple[Optional[datetime], Optional[datetime]]]: """Extract a tuple of datetimes from an interval string.""" if not value: return From 94af4e7c535b330f3be55d7518da35b25a2db076 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 23:14:05 -0500 Subject: [PATCH 11/29] remove unnecessary parens --- stac_fastapi/types/stac_fastapi/types/rfc3339.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index cbc51354a..a62c57822 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -52,4 +52,4 @@ def parse_interval( if start is None and end is None: return None else: - return (start, end) + return start, end From 755220d9da90f7c2ef12a5e87215dae60691829c Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 23:41:03 -0500 Subject: [PATCH 12/29] remove classmethod --- stac_fastapi/types/stac_fastapi/types/rfc3339.py | 2 +- stac_fastapi/types/stac_fastapi/types/search.py | 5 +---- stac_fastapi/types/tests/test_rfc3339.py | 0 3 files changed, 2 insertions(+), 5 deletions(-) create mode 100644 stac_fastapi/types/tests/test_rfc3339.py diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index a62c57822..f49480c50 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -36,7 +36,7 @@ def parse_interval( ) -> Optional[Tuple[Optional[datetime], Optional[datetime]]]: """Extract a tuple of datetimes from an interval string.""" if not value: - return + return None values = value.split("/") if len(values) != 2: diff --git a/stac_fastapi/types/stac_fastapi/types/search.py b/stac_fastapi/types/stac_fastapi/types/search.py index df79d024a..7202392ad 100644 --- a/stac_fastapi/types/stac_fastapi/types/search.py +++ b/stac_fastapi/types/stac_fastapi/types/search.py @@ -110,7 +110,6 @@ def end_date(self) -> Optional[datetime]: interval = parse_interval(self.datetime) return interval[1] if interval else None - @classmethod @validator("intersects") def validate_spatial(cls, v, values): """Check bbox and intersects are not both supplied.""" @@ -118,7 +117,6 @@ def validate_spatial(cls, v, values): raise ValueError("intersects and bbox parameters are mutually exclusive") return v - @classmethod @validator("bbox") def validate_bbox(cls, v: BBox): """Check order of supplied bbox coordinates.""" @@ -149,7 +147,6 @@ def validate_bbox(cls, v: BBox): return v - @classmethod @validator("datetime") def validate_datetime(cls, v): """Validate datetime.""" @@ -161,7 +158,7 @@ def validate_datetime(cls, v): dates = [] for value in values: - if value == "..": + if value == ".." or value == "": dates.append(value) continue diff --git a/stac_fastapi/types/tests/test_rfc3339.py b/stac_fastapi/types/tests/test_rfc3339.py new file mode 100644 index 000000000..e69de29bb From 0f27bfc5bf642d98b50df99ca33f899caf3be58a Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 1 Mar 2022 23:52:53 -0500 Subject: [PATCH 13/29] add some interval tests --- stac_fastapi/types/tests/test_rfc3339.py | 88 ++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/stac_fastapi/types/tests/test_rfc3339.py b/stac_fastapi/types/tests/test_rfc3339.py index e69de29bb..d9da00528 100644 --- a/stac_fastapi/types/tests/test_rfc3339.py +++ b/stac_fastapi/types/tests/test_rfc3339.py @@ -0,0 +1,88 @@ +import pytest + +from stac_fastapi.types.rfc3339 import parse_interval, parse_rfc3339 + +invalid_datetimes = [ + "1985-04-12", # date only + "1937-01-01T12:00:27.87+0100", # invalid TZ format, no sep : + "37-01-01T12:00:27.87Z", # invalid year, must be 4 digits + "1985-12-12T23:20:50.52", # no TZ + "21985-12-12T23:20:50.52Z", # year must be 4 digits + "1985-13-12T23:20:50.52Z", # month > 12 + "1985-12-32T23:20:50.52Z", # day > 31 + "1985-12-01T25:20:50.52Z", # hour > 24 + "1985-12-01T00:60:50.52Z", # minute > 59 + "1985-12-01T00:06:61.52Z", # second > 60 + "1985-04-12T23:20:50.Z", # fractional sec . but no frac secs + "1985-04-12T23:20:50,Z", # fractional sec , but no frac secs + "1990-12-31T23:59:61Z", # second > 60 w/o fractional seconds + "1985-04-12T23:20:50,52Z", # comma as frac sec sep allowed in ISO8601 but not RFC3339 +] + +valid_datetimes = [ + "1985-04-12T23:20:50.52Z", + "1996-12-19T16:39:57-00:00", + "1996-12-19T16:39:57+00:00", + "1996-12-19T16:39:57-08:00", + "1996-12-19T16:39:57+08:00", + "1937-01-01T12:00:27.87+01:00", + "1985-04-12T23:20:50.52Z", + "1937-01-01T12:00:27.8710+01:00", + "1937-01-01T12:00:27.8+01:00", + "1937-01-01T12:00:27.8Z", + "2020-07-23T00:00:00.000+03:00", + "2020-07-23T00:00:00+03:00", + "1985-04-12t23:20:50.000z", + "2020-07-23T00:00:00Z", + "2020-07-23T00:00:00.0Z", + "2020-07-23T00:00:00.01Z", + "2020-07-23T00:00:00.012Z", + "2020-07-23T00:00:00.0123Z", + "2020-07-23T00:00:00.01234Z", + "2020-07-23T00:00:00.012345Z", + "2020-07-23T00:00:00.0123456Z", + "2020-07-23T00:00:00.01234567Z", + "2020-07-23T00:00:00.012345678Z", +] + +invalid_intervals = [ + "/" + "../" + "/.." + "../.." + "/1984-04-12T23:20:50.52Z/1985-04-12T23:20:50.52Z", # extra start / + "1984-04-12T23:20:50.52Z/1985-04-12T23:20:50.52Z/", # extra end / + "1986-04-12T23:20:50.52Z/1985-04-12T23:20:50.52Z", # start > end +] + +valid_intervals = [ + "../1985-04-12T23:20:50.52Z", + "1985-04-12T23:20:50.52Z/..", + "/1985-04-12T23:20:50.52Z", + "1985-04-12T23:20:50.52Z/", + "1985-04-12T23:20:50.52Z/1986-04-12T23:20:50.52Z", + "1985-04-12T23:20:50.52+01:00/1986-04-12T23:20:50.52+01:00", + "1985-04-12T23:20:50.52-01:00/1986-04-12T23:20:50.52-01:00", +] + + +@pytest.mark.parametrize("test_input", invalid_datetimes) +def test_parse_invalid_str_to_datetime(test_input): + with pytest.raises(ValueError): + parse_rfc3339(test_input) + + +@pytest.mark.parametrize("test_input", valid_datetimes) +def test_parse_valid_str_to_datetime(test_input): + assert parse_rfc3339(test_input) + + +# @pytest.mark.parametrize("test_input", invalid_intervals) +# def test_parse_invalid_interval_to_datetime(test_input): +# with pytest.raises(ValueError): +# parse_interval(test_input) + + +@pytest.mark.parametrize("test_input", valid_intervals) +def test_parse_valid_interval_to_datetime(test_input): + assert parse_interval(test_input) From 45747f5ec8eaa60194b61abf1302a29ac5ca0d6e Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 3 Mar 2022 08:22:32 -0500 Subject: [PATCH 14/29] replace rfc3339_str with pystac.utils.datetime_to_str --- stac_fastapi/pgstac/tests/resources/test_item.py | 13 +++++++------ .../stac_fastapi/sqlalchemy/serializers.py | 5 +++-- .../sqlalchemy/tests/resources/test_item.py | 13 +++++++------ stac_fastapi/types/stac_fastapi/types/rfc3339.py | 13 ++----------- stac_fastapi/types/tests/test_rfc3339.py | 11 +++++++---- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 78cad0ec3..f6c6c6809 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -7,12 +7,13 @@ import pystac import pytest from httpx import AsyncClient +from pystac.utils import datetime_to_str from shapely.geometry import Polygon from stac_pydantic import Collection, Item from starlette.requests import Request from stac_fastapi.pgstac.models.links import CollectionLinks -from stac_fastapi.types.rfc3339 import parse_rfc3339, rfc3339_str +from stac_fastapi.types.rfc3339 import parse_rfc3339 @pytest.mark.asyncio @@ -409,7 +410,7 @@ async def test_item_search_temporal_query_post( params = { "collections": [test_item["collection"]], "intersects": test_item["geometry"], - "datetime": rfc3339_str(item_date), + "datetime": datetime_to_str(item_date), } resp = await app_client.post("/search", json=params) @@ -443,7 +444,7 @@ async def test_item_search_temporal_window_post( params = { "collections": [test_item["collection"]], - "datetime": f"{rfc3339_str(item_date_before)}/{rfc3339_str(item_date_after)}", + "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}", } resp = await app_client.post("/search", json=params) @@ -492,7 +493,7 @@ async def test_item_search_sort_post(app_client, load_test_data, load_test_colle second_item = load_test_data("test_item.json") second_item["id"] = "another-item" another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = rfc3339_str(another_item_date) + second_item["properties"]["datetime"] = datetime_to_str(another_item_date) resp = await app_client.post( f"/collections/{second_item['collection']}/items", json=second_item ) @@ -608,7 +609,7 @@ async def test_item_search_temporal_window_get( params = { "collections": test_item["collection"], - "datetime": f"{rfc3339_str(item_date_before)}/{rfc3339_str(item_date_after)}", + "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}", } resp = await app_client.get("/search", params=params) resp_json = resp.json() @@ -629,7 +630,7 @@ async def test_item_search_sort_get(app_client, load_test_data, load_test_collec second_item = load_test_data("test_item.json") second_item["id"] = "another-item" another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = rfc3339_str(another_item_date) + second_item["properties"]["datetime"] = datetime_to_str(another_item_date) resp = await app_client.post( f"/collections/{second_item['collection']}/items", json=second_item ) diff --git a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py index 2d5ecf2ca..6a48a0552 100644 --- a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py +++ b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py @@ -5,12 +5,13 @@ import attr import geoalchemy2 as ga +from pystac.utils import datetime_to_str from stac_fastapi.sqlalchemy.models import database from stac_fastapi.types import stac as stac_types from stac_fastapi.types.config import Settings from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links -from stac_fastapi.types.rfc3339 import now_as_rfc3339_str, parse_rfc3339, rfc3339_str +from stac_fastapi.types.rfc3339 import now_as_rfc3339_str, parse_rfc3339 @attr.s # type:ignore @@ -54,7 +55,7 @@ def db_to_stac(cls, db_model: database.Item, base_url: str) -> stac_types.Item: # Use getattr to accommodate extension namespaces field_value = getattr(db_model, field.split(":")[-1]) if field == "datetime": - field_value = rfc3339_str(field_value) + field_value = datetime_to_str(field_value) properties[field] = field_value item_id = db_model.id collection_id = db_model.collection_id diff --git a/stac_fastapi/sqlalchemy/tests/resources/test_item.py b/stac_fastapi/sqlalchemy/tests/resources/test_item.py index f31b8d92c..6787a87b3 100644 --- a/stac_fastapi/sqlalchemy/tests/resources/test_item.py +++ b/stac_fastapi/sqlalchemy/tests/resources/test_item.py @@ -9,11 +9,12 @@ import pystac from pydantic.datetime_parse import parse_datetime +from pystac.utils import datetime_to_str from shapely.geometry import Polygon from stac_fastapi.sqlalchemy.core import CoreCrudClient from stac_fastapi.types.core import LandingPageMixin -from stac_fastapi.types.rfc3339 import parse_rfc3339, rfc3339_str +from stac_fastapi.types.rfc3339 import parse_rfc3339 def test_create_and_delete_item(app_client, load_test_data): @@ -425,7 +426,7 @@ def test_item_search_temporal_query_post(app_client, load_test_data): params = { "collections": [test_item["collection"]], "intersects": test_item["geometry"], - "datetime": f"../{rfc3339_str(item_date)}", + "datetime": f"../{datetime_to_str(item_date)}", } resp = app_client.post("/search", json=params) resp_json = resp.json() @@ -447,7 +448,7 @@ def test_item_search_temporal_window_post(app_client, load_test_data): params = { "collections": [test_item["collection"]], "intersects": test_item["geometry"], - "datetime": f"{rfc3339_str(item_date_before)}/{rfc3339_str(item_date_after)}", + "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}", } resp = app_client.post("/search", json=params) resp_json = resp.json() @@ -484,7 +485,7 @@ def test_item_search_sort_post(app_client, load_test_data): second_item = load_test_data("test_item.json") second_item["id"] = "another-item" another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = rfc3339_str(another_item_date) + second_item["properties"]["datetime"] = datetime_to_str(another_item_date) resp = app_client.post( f"/collections/{second_item['collection']}/items", json=second_item ) @@ -570,7 +571,7 @@ def test_item_search_temporal_window_get(app_client, load_test_data): params = { "collections": test_item["collection"], "bbox": ",".join([str(coord) for coord in test_item["bbox"]]), - "datetime": f"{rfc3339_str(item_date_before)}/{rfc3339_str(item_date_after)}", + "datetime": f"{datetime_to_str(item_date_before)}/{datetime_to_str(item_date_after)}", } resp = app_client.get("/search", params=params) resp_json = resp.json() @@ -589,7 +590,7 @@ def test_item_search_sort_get(app_client, load_test_data): second_item = load_test_data("test_item.json") second_item["id"] = "another-item" another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = rfc3339_str(another_item_date) + second_item["properties"]["datetime"] = datetime_to_str(another_item_date) resp = app_client.post( f"/collections/{second_item['collection']}/items", json=second_item ) diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index f49480c50..9068ba5b2 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -4,6 +4,7 @@ from typing import Optional, Tuple import ciso8601 +import pystac def parse_rfc3339(value: str) -> datetime: @@ -11,16 +12,6 @@ def parse_rfc3339(value: str) -> datetime: return ciso8601.parse_rfc3339(value) -def rfc3339_str(value: datetime, use_z: bool = True) -> str: - """Doc.""" - if not value.tzinfo: - value = value.replace(tzinfo=timezone.utc) - - str_value = value.isoformat() - str_value = str_value.replace("+00:00", "Z") if use_z else str_value - return str_value - - def now_in_utc() -> datetime: """Doc.""" return datetime.now(timezone.utc) @@ -28,7 +19,7 @@ def now_in_utc() -> datetime: def now_as_rfc3339_str() -> str: """Doc.""" - return rfc3339_str(now_in_utc()) + return pystac.utils.datetime_to_str(now_in_utc()) def parse_interval( diff --git a/stac_fastapi/types/tests/test_rfc3339.py b/stac_fastapi/types/tests/test_rfc3339.py index d9da00528..253604586 100644 --- a/stac_fastapi/types/tests/test_rfc3339.py +++ b/stac_fastapi/types/tests/test_rfc3339.py @@ -77,12 +77,15 @@ def test_parse_valid_str_to_datetime(test_input): assert parse_rfc3339(test_input) -# @pytest.mark.parametrize("test_input", invalid_intervals) -# def test_parse_invalid_interval_to_datetime(test_input): -# with pytest.raises(ValueError): -# parse_interval(test_input) +@pytest.mark.parametrize("test_input", invalid_intervals) +def test_parse_invalid_interval_to_datetime(test_input): + with pytest.raises(ValueError): + parse_interval(test_input) @pytest.mark.parametrize("test_input", valid_intervals) def test_parse_valid_interval_to_datetime(test_input): assert parse_interval(test_input) + + +# TODO: add tests for now and str functions From d933be7f81ca0af2ced9c1d29e9834dd7e677b93 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 3 Mar 2022 08:52:34 -0500 Subject: [PATCH 15/29] fix search datetime parameter parsing to support empty string as open end of interval --- README.md | 6 ++++++ .../types/stac_fastapi/types/search.py | 21 ++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index cac659543..ae2cdad0b 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,12 @@ Before commit, install the [pre-commit](https://pre-commit.com) hooks with: pre-commit install ``` +The pre-commit hooks can be run manually with: + +```shell +pre-commit run --all-files +``` + #### Note to Docker for Windows users You'll need to enable experimental features on Docker for Windows in order to run the docker-compose, due to the "--platform" flag that is required to allow the project to run on some Apple architectures. To do this, open Docker Desktop, go to settings, select "Docker Engine", and modify the configuration JSON to have `"experimental": true`. diff --git a/stac_fastapi/types/stac_fastapi/types/search.py b/stac_fastapi/types/stac_fastapi/types/search.py index 7202392ad..9086c5f0e 100644 --- a/stac_fastapi/types/stac_fastapi/types/search.py +++ b/stac_fastapi/types/stac_fastapi/types/search.py @@ -39,6 +39,7 @@ class Operator(str, AutoValueEnum): lte = auto() gt = auto() gte = auto() + # TODO: These are defined in the spec but aren't currently implemented by the api # startsWith = auto() # endsWith = auto() @@ -159,17 +160,21 @@ def validate_datetime(cls, v): dates = [] for value in values: if value == ".." or value == "": - dates.append(value) + dates.append("..") continue - parse_rfc3339(value) - dates.append(value) + # throws ValueError if invalid RFC 3339 string + dates.append(parse_rfc3339(value)) - if ".." not in dates: - if parse_rfc3339(dates[0]) > parse_rfc3339(dates[1]): - raise ValueError( - "Invalid datetime range, must match format (begin_date, end_date)" - ) + if dates[0] == ".." and dates[1] == "..": + raise ValueError( + "Invalid datetime range, both ends of range may not be open" + ) + + if ".." not in dates and dates[0] > dates[1]: + raise ValueError( + "Invalid datetime range, must match format (begin_date, end_date)" + ) return v From c9f943ca966fa27c97ab017cc650e2c23f054223 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 3 Mar 2022 09:02:58 -0500 Subject: [PATCH 16/29] rename methods --- .../pgstac/tests/resources/test_item.py | 12 ++-- .../stac_fastapi/sqlalchemy/serializers.py | 6 +- .../sqlalchemy/tests/resources/test_item.py | 12 ++-- .../types/stac_fastapi/types/rfc3339.py | 69 +++++++++++++------ .../types/stac_fastapi/types/search.py | 8 +-- stac_fastapi/types/tests/test_rfc3339.py | 10 +-- 6 files changed, 73 insertions(+), 44 deletions(-) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index f6c6c6809..0fd518c62 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -13,7 +13,7 @@ from starlette.requests import Request from stac_fastapi.pgstac.models.links import CollectionLinks -from stac_fastapi.types.rfc3339 import parse_rfc3339 +from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime @pytest.mark.asyncio @@ -403,7 +403,7 @@ async def test_item_search_temporal_query_post( ) assert resp.status_code == 200 - item_date = parse_rfc3339(test_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) print(item_date) item_date = item_date + timedelta(seconds=1) @@ -438,7 +438,7 @@ async def test_item_search_temporal_window_post( ) assert resp.status_code == 200 - item_date = parse_rfc3339(test_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) @@ -484,7 +484,7 @@ async def test_item_search_temporal_open_window( async def test_item_search_sort_post(app_client, load_test_data, load_test_collection): """Test POST search with sorting (sort extension)""" first_item = load_test_data("test_item.json") - item_date = parse_rfc3339(first_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"]) resp = await app_client.post( f"/collections/{first_item['collection']}/items", json=first_item ) @@ -603,7 +603,7 @@ async def test_item_search_temporal_window_get( ) assert resp.status_code == 200 - item_date = parse_rfc3339(test_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) @@ -621,7 +621,7 @@ async def test_item_search_temporal_window_get( async def test_item_search_sort_get(app_client, load_test_data, load_test_collection): """Test GET search with sorting (sort extension)""" first_item = load_test_data("test_item.json") - item_date = parse_rfc3339(first_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"]) resp = await app_client.post( f"/collections/{first_item['collection']}/items", json=first_item ) diff --git a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py index 6a48a0552..93590378e 100644 --- a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py +++ b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py @@ -11,7 +11,7 @@ from stac_fastapi.types import stac as stac_types from stac_fastapi.types.config import Settings from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links -from stac_fastapi.types.rfc3339 import now_as_rfc3339_str, parse_rfc3339 +from stac_fastapi.types.rfc3339 import now_to_rfc3339_str, rfc3339_str_to_datetime @attr.s # type:ignore @@ -101,12 +101,12 @@ def stac_to_db( # Use getattr to accommodate extension namespaces field_value = stac_data["properties"][field] if field == "datetime": - field_value = parse_rfc3339(field_value) + field_value = rfc3339_str_to_datetime(field_value) indexed_fields[field.split(":")[-1]] = field_value # TODO: Exclude indexed fields from the properties jsonb field to prevent duplication - now = now_as_rfc3339_str() + now = now_to_rfc3339_str() if "created" not in stac_data["properties"]: stac_data["properties"]["created"] = now stac_data["properties"]["updated"] = now diff --git a/stac_fastapi/sqlalchemy/tests/resources/test_item.py b/stac_fastapi/sqlalchemy/tests/resources/test_item.py index 6787a87b3..cb5262b32 100644 --- a/stac_fastapi/sqlalchemy/tests/resources/test_item.py +++ b/stac_fastapi/sqlalchemy/tests/resources/test_item.py @@ -14,7 +14,7 @@ from stac_fastapi.sqlalchemy.core import CoreCrudClient from stac_fastapi.types.core import LandingPageMixin -from stac_fastapi.types.rfc3339 import parse_rfc3339 +from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime def test_create_and_delete_item(app_client, load_test_data): @@ -420,7 +420,7 @@ def test_item_search_temporal_query_post(app_client, load_test_data): ) assert resp.status_code == 200 - item_date = parse_rfc3339(test_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) item_date = item_date + timedelta(seconds=1) params = { @@ -441,7 +441,7 @@ def test_item_search_temporal_window_post(app_client, load_test_data): ) assert resp.status_code == 200 - item_date = parse_rfc3339(test_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) @@ -476,7 +476,7 @@ def test_item_search_temporal_open_window(app_client, load_test_data): def test_item_search_sort_post(app_client, load_test_data): """Test POST search with sorting (sort extension)""" first_item = load_test_data("test_item.json") - item_date = parse_rfc3339(first_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"]) resp = app_client.post( f"/collections/{first_item['collection']}/items", json=first_item ) @@ -564,7 +564,7 @@ def test_item_search_temporal_window_get(app_client, load_test_data): ) assert resp.status_code == 200 - item_date = parse_rfc3339(test_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) @@ -581,7 +581,7 @@ def test_item_search_temporal_window_get(app_client, load_test_data): def test_item_search_sort_get(app_client, load_test_data): """Test GET search with sorting (sort extension)""" first_item = load_test_data("test_item.json") - item_date = parse_rfc3339(first_item["properties"]["datetime"]) + item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"]) resp = app_client.post( f"/collections/{first_item['collection']}/items", json=first_item ) diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index 9068ba5b2..0496fc35f 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -4,43 +4,72 @@ from typing import Optional, Tuple import ciso8601 -import pystac +from pystac.utils import datetime_to_str -def parse_rfc3339(value: str) -> datetime: - """Doc.""" - return ciso8601.parse_rfc3339(value) +def rfc3339_str_to_datetime(s: str) -> datetime: + """Convert a string conforming to RFC 3339 to a :class:`datetime.datetime`. + Uses :meth:`ciso8601.parse_rfc3339` under the hood. -def now_in_utc() -> datetime: - """Doc.""" - return datetime.now(timezone.utc) + Args: + s (str) : The string to convert to :class:`datetime.datetime`. + Returns: + str: The datetime represented by the ISO8601 (RFC 3339) formatted string. -def now_as_rfc3339_str() -> str: - """Doc.""" - return pystac.utils.datetime_to_str(now_in_utc()) + Raises: + ValueError: If the string is not a valid RFC 3339 string. + """ + return ciso8601.rfc3339_str_to_datetime(s) -def parse_interval( - value: str, +def str_to_interval( + interval: str, ) -> Optional[Tuple[Optional[datetime], Optional[datetime]]]: - """Extract a tuple of datetimes from an interval string.""" - if not value: - return None + """Extract a tuple of datetimes from an interval string. + + Interval strings are defined by + OGC API - Features Part 1 for the datetime query parameter value. These follow the + form '1985-04-12T23:20:50.52Z/1986-04-12T23:20:50.52Z', and allow either the start + or end (but not both) to be open-ended with '..' or ''. - values = value.split("/") + Args: + interval (str) : The interval string to convert to a :class:`datetime.datetime` + tuple. + + Raises: + ValueError: If the string is not a valid interval string. + """ + if not interval: + raise ValueError("Empty interval string is invalid.") + + values = interval.split("/") if len(values) != 2: - return None + raise ValueError( + f"Interval string '{interval}' contains more than one forward slash." + ) start = None end = None if not values[0] in ["..", ""]: - start = parse_rfc3339(values[0]) + start = rfc3339_str_to_datetime(values[0]) if not values[1] in ["..", ""]: - end = parse_rfc3339(values[1]) + end = rfc3339_str_to_datetime(values[1]) if start is None and end is None: - return None + raise ValueError("Double open-ended intervals are not allowed.") + if start is not None and end is not None and start > end: + raise ValueError("Start datetime cannot be before end datetime.") else: return start, end + + +def now_in_utc() -> datetime: + """Return a datetime value of now with the UTC timezone applied.""" + return datetime.now(timezone.utc) + + +def now_to_rfc3339_str() -> str: + """Return an RFC 3339 string representing now.""" + return datetime_to_str(now_in_utc()) diff --git a/stac_fastapi/types/stac_fastapi/types/search.py b/stac_fastapi/types/stac_fastapi/types/search.py index 9086c5f0e..f12c3c518 100644 --- a/stac_fastapi/types/stac_fastapi/types/search.py +++ b/stac_fastapi/types/stac_fastapi/types/search.py @@ -24,7 +24,7 @@ from stac_pydantic.shared import BBox from stac_pydantic.utils import AutoValueEnum -from stac_fastapi.types.rfc3339 import parse_interval, parse_rfc3339 +from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime, str_to_interval # Be careful: https://github.com/samuelcolvin/pydantic/issues/1423#issuecomment-642797287 NumType = Union[float, int] @@ -102,13 +102,13 @@ class BaseSearchPostRequest(BaseModel): @property def start_date(self) -> Optional[datetime]: """Extract the start date from the datetime string.""" - interval = parse_interval(self.datetime) + interval = str_to_interval(self.datetime) return interval[0] if interval else None @property def end_date(self) -> Optional[datetime]: """Extract the end date from the datetime string.""" - interval = parse_interval(self.datetime) + interval = str_to_interval(self.datetime) return interval[1] if interval else None @validator("intersects") @@ -164,7 +164,7 @@ def validate_datetime(cls, v): continue # throws ValueError if invalid RFC 3339 string - dates.append(parse_rfc3339(value)) + dates.append(rfc3339_str_to_datetime(value)) if dates[0] == ".." and dates[1] == "..": raise ValueError( diff --git a/stac_fastapi/types/tests/test_rfc3339.py b/stac_fastapi/types/tests/test_rfc3339.py index 253604586..c3013fe20 100644 --- a/stac_fastapi/types/tests/test_rfc3339.py +++ b/stac_fastapi/types/tests/test_rfc3339.py @@ -1,6 +1,6 @@ import pytest -from stac_fastapi.types.rfc3339 import parse_interval, parse_rfc3339 +from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime, str_to_interval invalid_datetimes = [ "1985-04-12", # date only @@ -69,23 +69,23 @@ @pytest.mark.parametrize("test_input", invalid_datetimes) def test_parse_invalid_str_to_datetime(test_input): with pytest.raises(ValueError): - parse_rfc3339(test_input) + rfc3339_str_to_datetime(test_input) @pytest.mark.parametrize("test_input", valid_datetimes) def test_parse_valid_str_to_datetime(test_input): - assert parse_rfc3339(test_input) + assert rfc3339_str_to_datetime(test_input) @pytest.mark.parametrize("test_input", invalid_intervals) def test_parse_invalid_interval_to_datetime(test_input): with pytest.raises(ValueError): - parse_interval(test_input) + str_to_interval(test_input) @pytest.mark.parametrize("test_input", valid_intervals) def test_parse_valid_interval_to_datetime(test_input): - assert parse_interval(test_input) + assert str_to_interval(test_input) # TODO: add tests for now and str functions From afb41769909e5805b8ff87751d17f2f84fe90d29 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Sat, 5 Mar 2022 18:48:59 -0500 Subject: [PATCH 17/29] fix accidental method name repacement for parse_rfc3339 --- stac_fastapi/types/stac_fastapi/types/rfc3339.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index 0496fc35f..0ba460034 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -21,7 +21,7 @@ def rfc3339_str_to_datetime(s: str) -> datetime: Raises: ValueError: If the string is not a valid RFC 3339 string. """ - return ciso8601.rfc3339_str_to_datetime(s) + return ciso8601.parse_rfc3339(s) def str_to_interval( From 7b00142e21f18c06f67c241e32b273da38e2491a Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Sat, 5 Mar 2022 18:58:39 -0500 Subject: [PATCH 18/29] update tests for double open ended temporal interval --- .../pgstac/tests/resources/test_item.py | 24 ++++++------------- stac_fastapi/types/tests/test_rfc3339.py | 18 ++++++++++++-- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 0fd518c62..dc32305ac 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -457,27 +457,17 @@ async def test_item_search_temporal_window_post( async def test_item_search_temporal_open_window( app_client, load_test_data, load_test_collection ): - """Test POST search with open spatio-temporal query (core)""" - test_item = load_test_data("test_item.json") - resp = await app_client.post( - f"/collections/{test_item['collection']}/items", json=test_item - ) - assert resp.status_code == 200 - - # Add second item with a different datetime. - second_test_item = load_test_data("test_item2.json") - resp = await app_client.post( - f"/collections/{test_item['collection']}/items", json=second_test_item - ) - assert resp.status_code == 200 - params = { - "collections": [test_item["collection"]], "datetime": "../..", } resp = await app_client.post("/search", json=params) - resp_json = resp.json() - assert len(resp_json["features"]) == 2 + assert resp.status_code == 400 + + params = { + "datetime": "/", + } + resp = await app_client.post("/search", json=params) + assert resp.status_code == 400 @pytest.mark.asyncio diff --git a/stac_fastapi/types/tests/test_rfc3339.py b/stac_fastapi/types/tests/test_rfc3339.py index c3013fe20..0a402699a 100644 --- a/stac_fastapi/types/tests/test_rfc3339.py +++ b/stac_fastapi/types/tests/test_rfc3339.py @@ -1,6 +1,13 @@ +from datetime import timezone + import pytest -from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime, str_to_interval +from stac_fastapi.types.rfc3339 import ( + now_in_utc, + now_to_rfc3339_str, + rfc3339_str_to_datetime, + str_to_interval, +) invalid_datetimes = [ "1985-04-12", # date only @@ -88,4 +95,11 @@ def test_parse_valid_interval_to_datetime(test_input): assert str_to_interval(test_input) -# TODO: add tests for now and str functions +def test_now_functions() -> None: + now1 = now_in_utc() + now2 = now_in_utc() + + assert now1 < now2 + assert now1.tzinfo == timezone.utc + + rfc3339_str_to_datetime(now_to_rfc3339_str()) From c9b83de6f084344a31d04a2ef4091a238106b837 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Sat, 5 Mar 2022 19:09:21 -0500 Subject: [PATCH 19/29] fix handling of empty string open-ended interval --- stac_fastapi/pgstac/stac_fastapi/pgstac/core.py | 2 +- stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/core.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py index 2e9d4b7cb..0acfa47ba 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py @@ -186,7 +186,7 @@ async def item_collection( Called with `GET /collections/{collection_id}/items` Args: - id: id of the collection. + collection_id: id of the collection. limit: number of items to return. token: pagination token. diff --git a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/core.py b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/core.py index a5dc1e411..cd1ca9eea 100644 --- a/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/core.py +++ b/stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/core.py @@ -349,13 +349,14 @@ def post_search( # Non-interval date ex. "2000-02-02T00:00:00.00Z" if len(dts) == 1: query = query.filter(self.item_table.datetime == dts[0]) - elif ".." not in search_request.datetime: + # is there a benefit to between instead of >= and <= ? + elif dts[0] not in ["", ".."] and dts[1] not in ["", ".."]: query = query.filter(self.item_table.datetime.between(*dts)) # All items after the start date - elif dts[0] != "..": + elif dts[0] not in ["", ".."]: query = query.filter(self.item_table.datetime >= dts[0]) # All items before the end date - elif dts[1] != "..": + elif dts[1] not in ["", ".."]: query = query.filter(self.item_table.datetime <= dts[1]) # Query fields From b9119b43916538d76b4de425d18305c1235f804c Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 31 Mar 2022 19:58:51 -0400 Subject: [PATCH 20/29] fix test that was successful with double-open-ended datetime interval to now fail --- stac_fastapi/sqlalchemy/tests/resources/test_item.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/stac_fastapi/sqlalchemy/tests/resources/test_item.py b/stac_fastapi/sqlalchemy/tests/resources/test_item.py index cb5262b32..f75a802bf 100644 --- a/stac_fastapi/sqlalchemy/tests/resources/test_item.py +++ b/stac_fastapi/sqlalchemy/tests/resources/test_item.py @@ -463,14 +463,9 @@ def test_item_search_temporal_open_window(app_client, load_test_data): ) assert resp.status_code == 200 - params = { - "collections": [test_item["collection"]], - "intersects": test_item["geometry"], - "datetime": "../..", - } - resp = app_client.post("/search", json=params) - resp_json = resp.json() - assert resp_json["features"][0]["id"] == test_item["id"] + for dt in ["/", "../", "/..", "../.."]: + resp = app_client.post("/search", json={"datetime": dt}) + assert resp.status_code == 400 def test_item_search_sort_post(app_client, load_test_data): From 9f400f413dfe9566c55d92331662b793aedd36c3 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 31 Mar 2022 20:53:25 -0400 Subject: [PATCH 21/29] replace ciso8601 with python-dateutil --- Dockerfile.docs | 5 +---- stac_fastapi/types/setup.py | 2 +- stac_fastapi/types/stac_fastapi/types/rfc3339.py | 6 +++--- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Dockerfile.docs b/Dockerfile.docs index a7420f31f..6d91ccca4 100644 --- a/Dockerfile.docs +++ b/Dockerfile.docs @@ -1,8 +1,5 @@ FROM python:3.8-slim -# build-essential is required to build a wheel for ciso8601 -RUN apt update && apt install -y build-essential - RUN python -m pip install --upgrade pip RUN python -m pip install mkdocs mkdocs-material pdocs @@ -10,7 +7,7 @@ COPY . /opt/src WORKDIR /opt/src -RUN python -m pip install -e \ +RUN python -m pip install \ stac_fastapi/api \ stac_fastapi/types \ stac_fastapi/extensions \ diff --git a/stac_fastapi/types/setup.py b/stac_fastapi/types/setup.py index 994015d85..3f70b74de 100644 --- a/stac_fastapi/types/setup.py +++ b/stac_fastapi/types/setup.py @@ -10,7 +10,7 @@ "attrs", "pydantic[dotenv]", "stac_pydantic==2.0.*", - "ciso8601~=2.2.0", + "python-dateutil~=2.7.0", ] extra_reqs = { diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index 0ba460034..f14b9df2b 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -3,14 +3,14 @@ from datetime import datetime, timezone from typing import Optional, Tuple -import ciso8601 +import dateutil from pystac.utils import datetime_to_str def rfc3339_str_to_datetime(s: str) -> datetime: """Convert a string conforming to RFC 3339 to a :class:`datetime.datetime`. - Uses :meth:`ciso8601.parse_rfc3339` under the hood. + Uses :meth:`dateutil.parser.isoparse` under the hood. Args: s (str) : The string to convert to :class:`datetime.datetime`. @@ -21,7 +21,7 @@ def rfc3339_str_to_datetime(s: str) -> datetime: Raises: ValueError: If the string is not a valid RFC 3339 string. """ - return ciso8601.parse_rfc3339(s) + return dateutil.parser.isoparse(s) def str_to_interval( From f325ebb2c6652b9c6b918bdf92edeb14ebf7e397 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 31 Mar 2022 20:55:29 -0400 Subject: [PATCH 22/29] Revert "replace ciso8601 with python-dateutil" This reverts commit 9f400f413dfe9566c55d92331662b793aedd36c3. --- Dockerfile.docs | 5 ++++- stac_fastapi/types/setup.py | 2 +- stac_fastapi/types/stac_fastapi/types/rfc3339.py | 6 +++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile.docs b/Dockerfile.docs index 6d91ccca4..a7420f31f 100644 --- a/Dockerfile.docs +++ b/Dockerfile.docs @@ -1,5 +1,8 @@ FROM python:3.8-slim +# build-essential is required to build a wheel for ciso8601 +RUN apt update && apt install -y build-essential + RUN python -m pip install --upgrade pip RUN python -m pip install mkdocs mkdocs-material pdocs @@ -7,7 +10,7 @@ COPY . /opt/src WORKDIR /opt/src -RUN python -m pip install \ +RUN python -m pip install -e \ stac_fastapi/api \ stac_fastapi/types \ stac_fastapi/extensions \ diff --git a/stac_fastapi/types/setup.py b/stac_fastapi/types/setup.py index 3f70b74de..994015d85 100644 --- a/stac_fastapi/types/setup.py +++ b/stac_fastapi/types/setup.py @@ -10,7 +10,7 @@ "attrs", "pydantic[dotenv]", "stac_pydantic==2.0.*", - "python-dateutil~=2.7.0", + "ciso8601~=2.2.0", ] extra_reqs = { diff --git a/stac_fastapi/types/stac_fastapi/types/rfc3339.py b/stac_fastapi/types/stac_fastapi/types/rfc3339.py index f14b9df2b..0ba460034 100644 --- a/stac_fastapi/types/stac_fastapi/types/rfc3339.py +++ b/stac_fastapi/types/stac_fastapi/types/rfc3339.py @@ -3,14 +3,14 @@ from datetime import datetime, timezone from typing import Optional, Tuple -import dateutil +import ciso8601 from pystac.utils import datetime_to_str def rfc3339_str_to_datetime(s: str) -> datetime: """Convert a string conforming to RFC 3339 to a :class:`datetime.datetime`. - Uses :meth:`dateutil.parser.isoparse` under the hood. + Uses :meth:`ciso8601.parse_rfc3339` under the hood. Args: s (str) : The string to convert to :class:`datetime.datetime`. @@ -21,7 +21,7 @@ def rfc3339_str_to_datetime(s: str) -> datetime: Raises: ValueError: If the string is not a valid RFC 3339 string. """ - return dateutil.parser.isoparse(s) + return ciso8601.parse_rfc3339(s) def str_to_interval( From a77d05e190ffe1351580916b2e5a3e8558b3f1e5 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 31 Mar 2022 22:07:03 -0400 Subject: [PATCH 23/29] add pystac dependency to types --- Dockerfile.docs | 3 +-- stac_fastapi/types/setup.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.docs b/Dockerfile.docs index a7420f31f..f145b311a 100644 --- a/Dockerfile.docs +++ b/Dockerfile.docs @@ -10,13 +10,12 @@ COPY . /opt/src WORKDIR /opt/src -RUN python -m pip install -e \ +RUN python -m pip install \ stac_fastapi/api \ stac_fastapi/types \ stac_fastapi/extensions \ stac_fastapi/sqlalchemy - CMD ["pdocs", \ "as_markdown", \ "--output_dir", \ diff --git a/stac_fastapi/types/setup.py b/stac_fastapi/types/setup.py index 994015d85..4eb9a8bc7 100644 --- a/stac_fastapi/types/setup.py +++ b/stac_fastapi/types/setup.py @@ -10,6 +10,7 @@ "attrs", "pydantic[dotenv]", "stac_pydantic==2.0.*", + "pystac==1.*", "ciso8601~=2.2.0", ] From d535c32cd975e21adab9f42040c8c59a88ba91ec Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 31 Mar 2022 22:15:49 -0400 Subject: [PATCH 24/29] add double-open-ended tests to pgstac tests --- stac_fastapi/pgstac/tests/resources/test_item.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index dc32305ac..f16bb8da6 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -457,17 +457,9 @@ async def test_item_search_temporal_window_post( async def test_item_search_temporal_open_window( app_client, load_test_data, load_test_collection ): - params = { - "datetime": "../..", - } - resp = await app_client.post("/search", json=params) - assert resp.status_code == 400 - - params = { - "datetime": "/", - } - resp = await app_client.post("/search", json=params) - assert resp.status_code == 400 + for dt in ["/", "../", "/..", "../.."]: + resp = app_client.post("/search", json={"datetime": dt}) + assert resp.status_code == 400 @pytest.mark.asyncio From b7a527c769df1a7601dd7528abe1b99b203d9eae Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 31 Mar 2022 22:27:22 -0400 Subject: [PATCH 25/29] skip mixed open-ended in pgstac --- stac_fastapi/pgstac/tests/resources/test_item.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index f16bb8da6..410734e49 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -457,7 +457,16 @@ async def test_item_search_temporal_window_post( async def test_item_search_temporal_open_window( app_client, load_test_data, load_test_collection ): - for dt in ["/", "../", "/..", "../.."]: + for dt in ["/", "../.."]: + resp = app_client.post("/search", json={"datetime": dt}) + assert resp.status_code == 400 + + +@pytest.mark.skip(reason="mixed double-open-ended doesn't fail") +async def test_item_search_temporal_open_window_mixed( + app_client, load_test_data, load_test_collection +): + for dt in ["../", "/.."]: resp = app_client.post("/search", json={"datetime": dt}) assert resp.status_code == 400 From c7b1a8e9718a49243fcdc2ae6b98b9c4bee1166a Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 31 Mar 2022 22:36:26 -0400 Subject: [PATCH 26/29] skip datetime interval empty string in pgstac --- stac_fastapi/pgstac/tests/resources/test_item.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 410734e49..53c89c65e 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -457,16 +457,17 @@ async def test_item_search_temporal_window_post( async def test_item_search_temporal_open_window( app_client, load_test_data, load_test_collection ): - for dt in ["/", "../.."]: + for dt in ["../.."]: resp = app_client.post("/search", json={"datetime": dt}) assert resp.status_code == 400 -@pytest.mark.skip(reason="mixed double-open-ended doesn't fail") +# when these work, incorporate them into test_item_search_temporal_open_window +@pytest.mark.skip(reason="empty string and mixed double-open-ended doesn't fail") async def test_item_search_temporal_open_window_mixed( app_client, load_test_data, load_test_collection ): - for dt in ["../", "/.."]: + for dt in ["/", "../", "/.."]: resp = app_client.post("/search", json={"datetime": dt}) assert resp.status_code == 400 From 11e55f76d8e0ae01b5f9cdf08912ffe7eec6c4b7 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Thu, 31 Mar 2022 22:44:37 -0400 Subject: [PATCH 27/29] maybe just await the test --- stac_fastapi/pgstac/tests/resources/test_item.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 53c89c65e..627338dbc 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -457,18 +457,8 @@ async def test_item_search_temporal_window_post( async def test_item_search_temporal_open_window( app_client, load_test_data, load_test_collection ): - for dt in ["../.."]: - resp = app_client.post("/search", json={"datetime": dt}) - assert resp.status_code == 400 - - -# when these work, incorporate them into test_item_search_temporal_open_window -@pytest.mark.skip(reason="empty string and mixed double-open-ended doesn't fail") -async def test_item_search_temporal_open_window_mixed( - app_client, load_test_data, load_test_collection -): - for dt in ["/", "../", "/.."]: - resp = app_client.post("/search", json={"datetime": dt}) + for dt in ["/", "../..", "../", "/.."]: + resp = await app_client.post("/search", json={"datetime": dt}) assert resp.status_code == 400 From 8a433e8242797245d61be5be0dc27720af69eacf Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Thu, 14 Apr 2022 14:19:49 -0500 Subject: [PATCH 28/29] Bump black version to avoid https://github.com/psf/black/issues/2964 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77f702265..3c4919547 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: language_version: python3.8 - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 22.3.0 hooks: - id: black args: ['--safe'] From 35357e952c45973d811d584e4adb896759cd411d Mon Sep 17 00:00:00 2001 From: Nathan Zimmerman Date: Thu, 14 Apr 2022 14:27:56 -0500 Subject: [PATCH 29/29] lint --- stac_fastapi/api/setup.py | 2 +- stac_fastapi/api/stac_fastapi/api/models.py | 1 - stac_fastapi/extensions/setup.py | 2 +- stac_fastapi/sqlalchemy/setup.py | 2 +- stac_fastapi/types/setup.py | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/api/setup.py b/stac_fastapi/api/setup.py index 6de3585d1..2b70d4292 100644 --- a/stac_fastapi/api/setup.py +++ b/stac_fastapi/api/setup.py @@ -40,7 +40,7 @@ "License :: OSI Approved :: MIT License", ], keywords="STAC FastAPI COG", - author=u"Arturo Engineering", + author="Arturo Engineering", author_email="engineering@arturo.ai", url="https://github.com/stac-utils/stac-fastapi", license="MIT", diff --git a/stac_fastapi/api/stac_fastapi/api/models.py b/stac_fastapi/api/stac_fastapi/api/models.py index 171fe7115..3e4e01fe3 100644 --- a/stac_fastapi/api/stac_fastapi/api/models.py +++ b/stac_fastapi/api/stac_fastapi/api/models.py @@ -160,7 +160,6 @@ class GeoJSONResponse(ORJSONResponse): media_type = "application/geo+json" - else: from starlette.responses import JSONResponse diff --git a/stac_fastapi/extensions/setup.py b/stac_fastapi/extensions/setup.py index b1646f861..7f5e28882 100644 --- a/stac_fastapi/extensions/setup.py +++ b/stac_fastapi/extensions/setup.py @@ -39,7 +39,7 @@ "License :: OSI Approved :: MIT License", ], keywords="STAC FastAPI COG", - author=u"Arturo Engineering", + author="Arturo Engineering", author_email="engineering@arturo.ai", url="https://github.com/stac-utils/stac-fastapi", license="MIT", diff --git a/stac_fastapi/sqlalchemy/setup.py b/stac_fastapi/sqlalchemy/setup.py index 9490eda89..d6a606a3c 100644 --- a/stac_fastapi/sqlalchemy/setup.py +++ b/stac_fastapi/sqlalchemy/setup.py @@ -47,7 +47,7 @@ "License :: OSI Approved :: MIT License", ], keywords="STAC FastAPI COG", - author=u"Arturo Engineering", + author="Arturo Engineering", author_email="engineering@arturo.ai", url="https://github.com/stac-utils/stac-fastapi", license="MIT", diff --git a/stac_fastapi/types/setup.py b/stac_fastapi/types/setup.py index 4eb9a8bc7..3d9f77ef9 100644 --- a/stac_fastapi/types/setup.py +++ b/stac_fastapi/types/setup.py @@ -40,7 +40,7 @@ "License :: OSI Approved :: MIT License", ], keywords="STAC FastAPI COG", - author=u"Arturo Engineering", + author="Arturo Engineering", author_email="engineering@arturo.ai", url="https://github.com/stac-utils/stac-fastapi", license="MIT",