From efc6e6404bd8920c5dc389f0069b5ee13e167e11 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 28 Apr 2022 15:03:33 -0400 Subject: [PATCH 01/26] Upgrade to pgstac 0.5.1 Initial changes to get most tests passing. --- docker-compose.yml | 2 +- stac_fastapi/pgstac/setup.py | 3 ++- stac_fastapi/pgstac/tests/conftest.py | 16 ++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 996bb6593..f0e42141f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,7 +62,7 @@ services: database: container_name: stac-db - image: ghcr.io/stac-utils/pgstac:v0.4.5 + image: ghcr.io/stac-utils/pgstac:v0.5.1 environment: - POSTGRES_USER=username - POSTGRES_PASSWORD=password diff --git a/stac_fastapi/pgstac/setup.py b/stac_fastapi/pgstac/setup.py index 726257435..f832bed38 100644 --- a/stac_fastapi/pgstac/setup.py +++ b/stac_fastapi/pgstac/setup.py @@ -13,6 +13,7 @@ "stac-fastapi.types", "stac-fastapi.api", "stac-fastapi.extensions", + "psycopg[binary]", "asyncpg", "buildpg", "brotli_asgi", @@ -26,7 +27,7 @@ "pytest-asyncio>=0.17", "pre-commit", "requests", - "pypgstac==0.4.5", + "pypgstac==0.5.1", "httpx", ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], diff --git a/stac_fastapi/pgstac/tests/conftest.py b/stac_fastapi/pgstac/tests/conftest.py index 170877a7d..04c4d7245 100644 --- a/stac_fastapi/pgstac/tests/conftest.py +++ b/stac_fastapi/pgstac/tests/conftest.py @@ -8,7 +8,8 @@ import pytest from fastapi.responses import ORJSONResponse from httpx import AsyncClient -from pypgstac import pypgstac +from pypgstac.db import PgstacDB +from pypgstac.migrate import Migrate from stac_pydantic import Collection, Item from stac_fastapi.api.app import StacApi @@ -63,7 +64,9 @@ async def pg(): val = await conn.fetchval("SELECT true") print(val) await conn.close() - version = await pypgstac.run_migration(dsn=settings.testing_connection_string) + db = PgstacDB(dsn=settings.testing_connection_string) + migrator = Migrate(db) + version = migrator.run_migration() print(f"PGStac Migrated to {version}") yield settings.testing_connection_string @@ -83,13 +86,14 @@ async def pgstac(pg): conn = await asyncpg.connect(dsn=settings.testing_connection_string) await conn.execute( """ - TRUNCATE pgstac.items CASCADE; - TRUNCATE pgstac.collections CASCADE; - TRUNCATE pgstac.searches CASCADE; - TRUNCATE pgstac.search_wheres CASCADE; + DROP SCHEMA IF EXISTS pgstac CASCADE; """ ) await conn.close() + db = PgstacDB(dsn=settings.testing_connection_string) + migrator = Migrate(db) + version = migrator.run_migration() + print(f"PGStac Migrated to {version}") @pytest.fixture(scope="session") From fed54e94b123c83a393e5de05dda7548daef1e13 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Tue, 19 Apr 2022 15:49:51 -0700 Subject: [PATCH 02/26] Add option to hydrate pgstac search results in API --- docker-compose.yml | 1 + .../pgstac/stac_fastapi/pgstac/config.py | 5 ++ .../pgstac/stac_fastapi/pgstac/core.py | 64 ++++++++++++++++--- .../stac_fastapi/pgstac/types/search.py | 52 ++++++++++++++- 4 files changed, 111 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f0e42141f..318da1fad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR - DB_MIN_CONN_SIZE=1 - DB_MAX_CONN_SIZE=1 + - NO_HYDRATE=true ports: - "8082:8082" volumes: diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py index ce60bfc65..ca23a7287 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py @@ -1,5 +1,6 @@ """Postgres API configuration.""" +from stac_fastapi.pgstac.types.search import PgStacSearchContent from stac_fastapi.types.config import ApiSettings @@ -13,6 +14,7 @@ class Settings(ApiSettings): postgres_host_writer: hostname for the writer connection. postgres_port: database port. postgres_dbname: database name. + no_hydrate: disable hydration of stac items within pgstac. """ postgres_user: str @@ -27,6 +29,9 @@ class Settings(ApiSettings): db_max_queries: int = 50000 db_max_inactive_conn_lifetime: float = 300 + no_hydrate: bool = False + search_content_class = PgStacSearchContent + testing: bool = False @property diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py index 4de102255..5d194e1f4 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py @@ -16,6 +16,7 @@ from stac_pydantic.shared import MimeTypes from starlette.requests import Request +from stac_fastapi.pgstac.config import Settings from stac_fastapi.pgstac.models.links import CollectionLinks, ItemLinks, PagingLinks from stac_fastapi.pgstac.types.search import PgstacSearch from stac_fastapi.types.core import AsyncBaseCoreClient @@ -24,6 +25,8 @@ NumType = Union[float, int] +settings = Settings() + @attr.s class CoreCrudClient(AsyncBaseCoreClient): @@ -103,8 +106,37 @@ async def get_collection(self, collection_id: str, **kwargs) -> Collection: return Collection(**collection) + async def get_base_item(self, collection_id: str, **kwargs: Any) -> Dict[str, Any]: + """Get the base item of a collection for use in rehydrating full item collection properties. + + Args: + collection: ID of the collection. + + Returns: + Item. + """ + item: Optional[Dict[str, Any]] + + request: Request = kwargs["request"] + pool = request.app.state.readpool + async with pool.acquire() as conn: + q, p = render( + """ + SELECT * FROM collection_base_item(:collection_id::text); + """, + collection_id=collection_id, + ) + item = await conn.fetchval(q, *p) + + if item is None: + raise NotFoundError(f"A base item for {collection_id} does not exist.") + + return item + async def _search_base( - self, search_request: PgstacSearch, **kwargs: Any + self, + search_request: PgstacSearch, + **kwargs: Any, ) -> ItemCollection: """Cross catalog search (POST). @@ -121,7 +153,8 @@ async def _search_base( request: Request = kwargs["request"] pool = request.app.state.readpool - # pool = kwargs["request"].app.state.readpool + search_request.conf = search_request.conf or {} + search_request.conf["nohydrate"] = settings.no_hydrate req = search_request.json(exclude_none=True, by_alias=True) try: @@ -143,27 +176,38 @@ async def _search_base( collection = ItemCollection(**items) cleaned_features: List[Item] = [] + exclude = search_request.fields.exclude + if exclude and len(exclude) == 0: + exclude = None + include = search_request.fields.include + if include and len(include) == 0: + include = None + + search_content = settings.search_content_class(client=self, request=request) + for feature in collection.get("features") or []: - feature = Item(**feature) + if settings.no_hydrate: + feature = await search_content.hydrate( + feature, + include, + exclude, + ) + else: + feature = Item(**feature) + if ( search_request.fields.exclude is None or "links" not in search_request.fields.exclude ): # TODO: feature.collection is not always included # This code fails if it's left outside of the fields expression - # I've fields extension updated test cases to always include feature.collection + # I've update fields extension test cases to always include feature.collection feature["links"] = await ItemLinks( collection_id=feature["collection"], item_id=feature["id"], request=request, ).get_links(extra_links=feature.get("links")) - exclude = search_request.fields.exclude - if exclude and len(exclude) == 0: - exclude = None - include = search_request.fields.include - if include and len(include) == 0: - include = None cleaned_features.append(feature) collection["features"] = cleaned_features diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py index 2b8abb9da..457bb2fb1 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py @@ -1,10 +1,13 @@ """stac_fastapi.types.search module.""" -from typing import Dict, Optional +from copy import deepcopy +from typing import Any, Dict, Optional, Set from pydantic import validator +from starlette.requests import Request from stac_fastapi.types.search import BaseSearchPostRequest +from stac_fastapi.types.stac import Item class PgstacSearch(BaseSearchPostRequest): @@ -24,3 +27,50 @@ def validate_query_uses_cql(cls, v, values): ) return v + + +class PgStacSearchContent: + """Perform post-search processing when settings prevent pgstac from doing so in the database.""" + + def __init__(self, client, request: Request): + """Set result state for a single search.""" + self.base_items = {} + self.client = client + self.request = request + + async def get_base_item(self, collection_id: str): + """Return the base item for the collection.""" + if collection_id not in self.base_items: + self.base_items[collection_id] = await self.client.get_base_item( + collection_id, + request=self.request, + ) + return self.base_items[collection_id] + + async def hydrate( + self, + item: Dict[str, Any], + include: Optional[Set] = None, + exclude: Optional[Set] = None, + ) -> Item: + """Hydrate item in-place with base_item properties, while honoring include/exclude sets.""" + + def merge(b: Dict[str, Any], i: Dict[str, Any]): + for key in b: + if key in i: + if isinstance(b[key], dict) and isinstance(i.get(key), dict): + # Recurse on dicts to merge values + merge(b[key], i[key]) + elif b[key] == i.get(key): + # Matching key/value is a no-op + pass + elif isinstance(b[key], list) and isinstance(i.get(key), list): + # Merge unequal lists + i[key].extend(b[key]) + else: + # Keys in base item that are not in item are simply copied over + i[key] = deepcopy(b[key]) + + base_item = await self.get_base_item(item["collection"]) + merge(base_item, item) + return item From d6d72c35971583c9d79c066cc6a06cac39934830 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Tue, 26 Apr 2022 19:43:48 -0400 Subject: [PATCH 03/26] Support fields extension in nohydrate mode --- docker-compose.yml | 2 +- .../pgstac/stac_fastapi/pgstac/config.py | 8 +- .../pgstac/stac_fastapi/pgstac/core.py | 7 +- .../stac_fastapi/pgstac/types/search.py | 97 ++++++++++++++++--- 4 files changed, 90 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 318da1fad..e6532fbd0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,7 @@ services: - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR - DB_MIN_CONN_SIZE=1 - DB_MAX_CONN_SIZE=1 - - NO_HYDRATE=true + - USE_API_HYDRATE=false ports: - "8082:8082" volumes: diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py index ca23a7287..452656135 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py @@ -1,6 +1,6 @@ """Postgres API configuration.""" -from stac_fastapi.pgstac.types.search import PgStacSearchContent +from stac_fastapi.pgstac.types.search import PgstacSearchContent from stac_fastapi.types.config import ApiSettings @@ -14,7 +14,7 @@ class Settings(ApiSettings): postgres_host_writer: hostname for the writer connection. postgres_port: database port. postgres_dbname: database name. - no_hydrate: disable hydration of stac items within pgstac. + use_api_hydrate: perform hydration of stac items within stac-fastapi. """ postgres_user: str @@ -29,8 +29,8 @@ class Settings(ApiSettings): db_max_queries: int = 50000 db_max_inactive_conn_lifetime: float = 300 - no_hydrate: bool = False - search_content_class = PgStacSearchContent + use_api_hydrate: bool = False + search_content_class: PgstacSearchContent = PgstacSearchContent testing: bool = False diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py index 5d194e1f4..ee7ed5c07 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py @@ -25,8 +25,6 @@ NumType = Union[float, int] -settings = Settings() - @attr.s class CoreCrudClient(AsyncBaseCoreClient): @@ -151,10 +149,11 @@ async def _search_base( items: Dict[str, Any] request: Request = kwargs["request"] + settings: Settings = request.app.state.settings pool = request.app.state.readpool search_request.conf = search_request.conf or {} - search_request.conf["nohydrate"] = settings.no_hydrate + search_request.conf["nohydrate"] = settings.use_api_hydrate req = search_request.json(exclude_none=True, by_alias=True) try: @@ -186,7 +185,7 @@ async def _search_base( search_content = settings.search_content_class(client=self, request=request) for feature in collection.get("features") or []: - if settings.no_hydrate: + if settings.use_api_hydrate: feature = await search_content.hydrate( feature, include, diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py index 457bb2fb1..c4d6b9562 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py @@ -29,8 +29,17 @@ def validate_query_uses_cql(cls, v, values): return v -class PgStacSearchContent: - """Perform post-search processing when settings prevent pgstac from doing so in the database.""" +class PgstacSearchContent: + """ + Perform post-search processing of items if pgstac is set to `nohydrate`. + + This includes: + - Hydrating the items with the collection's base_item properties + - Filtering the fields according to the fields extension's include/exclude sets + - Removing invalid assets + + Note that these actions are destructive to the item instances passed in. + """ def __init__(self, client, request: Request): """Set result state for a single search.""" @@ -38,20 +47,11 @@ def __init__(self, client, request: Request): self.client = client self.request = request - async def get_base_item(self, collection_id: str): - """Return the base item for the collection.""" - if collection_id not in self.base_items: - self.base_items[collection_id] = await self.client.get_base_item( - collection_id, - request=self.request, - ) - return self.base_items[collection_id] - async def hydrate( self, item: Dict[str, Any], - include: Optional[Set] = None, - exclude: Optional[Set] = None, + include: Set = None, + exclude: Set = None, ) -> Item: """Hydrate item in-place with base_item properties, while honoring include/exclude sets.""" @@ -66,11 +66,78 @@ def merge(b: Dict[str, Any], i: Dict[str, Any]): pass elif isinstance(b[key], list) and isinstance(i.get(key), list): # Merge unequal lists - i[key].extend(b[key]) + i[key].extend(deepcopy(b[key])) else: # Keys in base item that are not in item are simply copied over i[key] = deepcopy(b[key]) - base_item = await self.get_base_item(item["collection"]) + base_item = await self._get_base_item(item["collection"]) merge(base_item, item) + cleaned_item = self._filter_fields(item, include, exclude) + cleaned_item = self._remove_invalid_assets(cleaned_item) + + return Item(**cleaned_item) + + async def _get_base_item(self, collection_id: str): + """Return the base item for the collection and cache by collection id.""" + if collection_id not in self.base_items: + self.base_items[collection_id] = await self.client.get_base_item( + collection_id, + request=self.request, + ) + return self.base_items[collection_id] + + def _filter_fields( + self, + item: Item, + include: Set = None, + exclude: Set = None, + ): + """Preserve and remove fields as indicated by the fields extension include/exclude sets.""" + if not include and not exclude: + return item + + clean_item = {} + for key_path in include or []: + keys = key_path.split(".") + root = keys[0] + if root in item: + if isinstance(item[root], dict) and len(keys) > 1: + # Recurse on "includes" key paths notation for sub-keys + clean_item[root] = self._filter_fields( + item[root], include=[".".join(keys[1:])] + ) + else: + clean_item[root] = item[root] + + for key_path in exclude or []: + keys = key_path.split(".") + root = keys[0] + if root in item: + if isinstance(item[root], dict) and len(keys) > 1: + # Recurse on "excludes" key paths notation for sub-keys + clean_item[root] = self._filter_fields( + item[root], exclude=[".".join(keys[1:])] + ) + else: + # Item is marked for exclusion + clean_item = item + clean_item.pop(root, None) + else: + # Key is not in item, so preserve the item without modification + clean_item = item + + return clean_item + + def _remove_invalid_assets(self, item: Item): + """ + Remove invalid assets from the item. + + Pgstac may return assets without an href if assets aren't uniformly + distributed across all items. In this case, the asset without an href + is removed from the item. + """ + if "assets" in item: + item["assets"] = {k: v for k, v in item["assets"].items() if "href" in v} + return item From f7a689c749975a52405bce561af69fcc4dae8b03 Mon Sep 17 00:00:00 2001 From: Rob Emanuele Date: Thu, 28 Apr 2022 13:15:29 -0400 Subject: [PATCH 04/26] Updates to hydrate and filter functionality. This was done in a pairing session with @mmcfarland --- .../pgstac/stac_fastapi/pgstac/config.py | 8 +- .../pgstac/stac_fastapi/pgstac/core.py | 45 ++++--- .../pgstac/types/base_item_cache.py | 40 ++++++ .../stac_fastapi/pgstac/types/search.py | 119 +----------------- .../pgstac/stac_fastapi/pgstac/utils.py | 102 +++++++++++++++ 5 files changed, 178 insertions(+), 136 deletions(-) create mode 100644 stac_fastapi/pgstac/stac_fastapi/pgstac/types/base_item_cache.py create mode 100644 stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py index 452656135..dca66761c 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py @@ -1,6 +1,10 @@ """Postgres API configuration.""" -from stac_fastapi.pgstac.types.search import PgstacSearchContent +from typing import Type +from stac_fastapi.pgstac.types.base_item_cache import ( + BaseItemCache, + DefaultBaseItemCache, +) from stac_fastapi.types.config import ApiSettings @@ -30,7 +34,7 @@ class Settings(ApiSettings): db_max_inactive_conn_lifetime: float = 300 use_api_hydrate: bool = False - search_content_class: PgstacSearchContent = PgstacSearchContent + base_item_cache: Type[BaseItemCache] = DefaultBaseItemCache testing: bool = False diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py index ee7ed5c07..c4234eea3 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py @@ -19,6 +19,7 @@ from stac_fastapi.pgstac.config import Settings from stac_fastapi.pgstac.models.links import CollectionLinks, ItemLinks, PagingLinks from stac_fastapi.pgstac.types.search import PgstacSearch +from stac_fastapi.pgstac.utils import filter_fields, hydrate, remove_invalid_assets from stac_fastapi.types.core import AsyncBaseCoreClient from stac_fastapi.types.errors import InvalidQueryParameter, NotFoundError from stac_fastapi.types.stac import Collection, Collections, Item, ItemCollection @@ -104,7 +105,9 @@ async def get_collection(self, collection_id: str, **kwargs) -> Collection: return Collection(**collection) - async def get_base_item(self, collection_id: str, **kwargs: Any) -> Dict[str, Any]: + async def _get_base_item( + self, collection_id: str, request: Request + ) -> Dict[str, Any]: """Get the base item of a collection for use in rehydrating full item collection properties. Args: @@ -115,7 +118,6 @@ async def get_base_item(self, collection_id: str, **kwargs: Any) -> Dict[str, An """ item: Optional[Dict[str, Any]] - request: Request = kwargs["request"] pool = request.app.state.readpool async with pool.acquire() as conn: q, p = render( @@ -155,7 +157,7 @@ async def _search_base( search_request.conf = search_request.conf or {} search_request.conf["nohydrate"] = settings.use_api_hydrate req = search_request.json(exclude_none=True, by_alias=True) - + print("==req", req, flush=True) try: async with pool.acquire() as conn: q, p = render( @@ -173,7 +175,6 @@ async def _search_base( next: Optional[str] = items.pop("next", None) prev: Optional[str] = items.pop("prev", None) collection = ItemCollection(**items) - cleaned_features: List[Item] = [] exclude = search_request.fields.exclude if exclude and len(exclude) == 0: @@ -182,18 +183,11 @@ async def _search_base( if include and len(include) == 0: include = None - search_content = settings.search_content_class(client=self, request=request) - - for feature in collection.get("features") or []: - if settings.use_api_hydrate: - feature = await search_content.hydrate( - feature, - include, - exclude, - ) - else: - feature = Item(**feature) + async def _add_item_links(feature: Item) -> None: + """Add ItemLinks to the Item. + If the fields extension is excluding links, then don't add them. + """ if ( search_request.fields.exclude is None or "links" not in search_request.fields.exclude @@ -207,7 +201,26 @@ async def _search_base( request=request, ).get_links(extra_links=feature.get("links")) - cleaned_features.append(feature) + cleaned_features: List[Item] = [] + + if settings.use_api_hydrate: + + async def _get_base_item(collection_id: str) -> Dict[str, Any]: + return await self._get_base_item(collection_id, request) + + base_item_cache = settings.base_item_cache(fetch_base_item=_get_base_item) + + for feature in collection.get("features") or []: + feature = await hydrate(feature, base_item_cache=base_item_cache) + feature = filter_fields(feature, include, exclude) + remove_invalid_assets(feature) + await _add_item_links(feature) + + cleaned_features.append(feature) + else: + for feature in collection.get("features") or []: + await _add_item_links(feature) + cleaned_features.append(feature) collection["features"] = cleaned_features collection["links"] = await PagingLinks( diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/base_item_cache.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/base_item_cache.py new file mode 100644 index 000000000..b3ea66a9f --- /dev/null +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/base_item_cache.py @@ -0,0 +1,40 @@ +import abc +from typing import Any, Callable, Coroutine, Dict + + +class BaseItemCache(abc.ABC): + """ + A cache that returns a base item for a collection. + + If no base item is found in the cache, use the fetch_base_item function + to fetch the base item from pgstac. + """ + + def __init__( + self, fetch_base_item: Callable[[str], Coroutine[Any, Any, Dict[str, Any]]] + ): + self._fetch_base_item = fetch_base_item + + @abc.abstractmethod + async def get(self, collection_id: str) -> Dict[str, Any]: + """Return the base item for the collection and cache by collection id.""" + pass + + +class DefaultBaseItemCache(BaseItemCache): + """ + Implementation of the BaseItemCache that holds base items in a dict. + """ + + def __init__( + self, fetch_base_item: Callable[[str], Coroutine[Any, Any, Dict[str, Any]]] + ): + self._base_items = {} + super().__init__(fetch_base_item) + + async def get(self, collection_id: str): + if collection_id not in self._base_items: + self._base_items[collection_id] = await self._fetch_base_item( + collection_id, + ) + return self._base_items[collection_id] diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py index c4d6b9562..2b8abb9da 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py @@ -1,13 +1,10 @@ """stac_fastapi.types.search module.""" -from copy import deepcopy -from typing import Any, Dict, Optional, Set +from typing import Dict, Optional from pydantic import validator -from starlette.requests import Request from stac_fastapi.types.search import BaseSearchPostRequest -from stac_fastapi.types.stac import Item class PgstacSearch(BaseSearchPostRequest): @@ -27,117 +24,3 @@ def validate_query_uses_cql(cls, v, values): ) return v - - -class PgstacSearchContent: - """ - Perform post-search processing of items if pgstac is set to `nohydrate`. - - This includes: - - Hydrating the items with the collection's base_item properties - - Filtering the fields according to the fields extension's include/exclude sets - - Removing invalid assets - - Note that these actions are destructive to the item instances passed in. - """ - - def __init__(self, client, request: Request): - """Set result state for a single search.""" - self.base_items = {} - self.client = client - self.request = request - - async def hydrate( - self, - item: Dict[str, Any], - include: Set = None, - exclude: Set = None, - ) -> Item: - """Hydrate item in-place with base_item properties, while honoring include/exclude sets.""" - - def merge(b: Dict[str, Any], i: Dict[str, Any]): - for key in b: - if key in i: - if isinstance(b[key], dict) and isinstance(i.get(key), dict): - # Recurse on dicts to merge values - merge(b[key], i[key]) - elif b[key] == i.get(key): - # Matching key/value is a no-op - pass - elif isinstance(b[key], list) and isinstance(i.get(key), list): - # Merge unequal lists - i[key].extend(deepcopy(b[key])) - else: - # Keys in base item that are not in item are simply copied over - i[key] = deepcopy(b[key]) - - base_item = await self._get_base_item(item["collection"]) - merge(base_item, item) - cleaned_item = self._filter_fields(item, include, exclude) - cleaned_item = self._remove_invalid_assets(cleaned_item) - - return Item(**cleaned_item) - - async def _get_base_item(self, collection_id: str): - """Return the base item for the collection and cache by collection id.""" - if collection_id not in self.base_items: - self.base_items[collection_id] = await self.client.get_base_item( - collection_id, - request=self.request, - ) - return self.base_items[collection_id] - - def _filter_fields( - self, - item: Item, - include: Set = None, - exclude: Set = None, - ): - """Preserve and remove fields as indicated by the fields extension include/exclude sets.""" - if not include and not exclude: - return item - - clean_item = {} - for key_path in include or []: - keys = key_path.split(".") - root = keys[0] - if root in item: - if isinstance(item[root], dict) and len(keys) > 1: - # Recurse on "includes" key paths notation for sub-keys - clean_item[root] = self._filter_fields( - item[root], include=[".".join(keys[1:])] - ) - else: - clean_item[root] = item[root] - - for key_path in exclude or []: - keys = key_path.split(".") - root = keys[0] - if root in item: - if isinstance(item[root], dict) and len(keys) > 1: - # Recurse on "excludes" key paths notation for sub-keys - clean_item[root] = self._filter_fields( - item[root], exclude=[".".join(keys[1:])] - ) - else: - # Item is marked for exclusion - clean_item = item - clean_item.pop(root, None) - else: - # Key is not in item, so preserve the item without modification - clean_item = item - - return clean_item - - def _remove_invalid_assets(self, item: Item): - """ - Remove invalid assets from the item. - - Pgstac may return assets without an href if assets aren't uniformly - distributed across all items. In this case, the asset without an href - is removed from the item. - """ - if "assets" in item: - item["assets"] = {k: v for k, v in item["assets"].items() if "href" in v} - - return item diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py new file mode 100644 index 000000000..f0bb0b2b6 --- /dev/null +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py @@ -0,0 +1,102 @@ +"""stac-fastapi utility methods.""" +from copy import deepcopy +from typing import Any, Dict, Optional, Set, Union + +from stac_fastapi.pgstac.types.base_item_cache import BaseItemCache +from stac_fastapi.types.stac import Item + + +async def hydrate( + item: Union[Item, Dict[str, Any]], base_item_cache: BaseItemCache +) -> Item: + """Hydrate item in-place with base_item properties. + + This will not perform a deep copy; values of the original item will be referenced + in the return item. + """ + item = dict(item) + + # Merge will mutate i, but create deep copies of values in the base item + # This will prevent the base item values from being mutated, e.g. by + # filtering out fields in `filter_fields`. + def merge(b: Dict[str, Any], i: Dict[str, Any]): + for key in b: + if key in i: + if isinstance(b[key], dict) and isinstance(i.get(key), dict): + # Recurse on dicts to merge values + merge(b[key], i[key]) + elif b[key] == i.get(key): + # Matching key/value is a no-op + pass + elif isinstance(b[key], list) and isinstance(i.get(key), list): + # Merge unequal lists + i[key].extend(deepcopy(b[key])) + else: + # Keys in base item that are not in item are simply copied over + i[key] = deepcopy(b[key]) + + base_item = await base_item_cache.get(item["collection"]) + merge(base_item, item) + return Item(**item) + + +def filter_fields( + item: Union[Item, Dict[str, Any]], + include: Optional[Set[str]] = None, + exclude: Optional[Set[str]] = None, +) -> Item: + """Preserve and remove fields as indicated by the fields extension include/exclude sets. + + Returns a shallow copy of the Item with the fields filtered. + + This will not perform a deep copy; values of the original item will be referenced + in the return item. + """ + if not include and not exclude: + return Item(**item) + + item = dict(item) + + clean_item: Dict[str, Any] = {} + for key_path in include or []: + keys = key_path.split(".") + root = keys[0] + if root in item: + if isinstance(item[root], dict) and len(keys) > 1: + # Recurse on "includes" key paths notation for sub-keys + clean_item[root] = filter_fields( + item[root], include=set([".".join(keys[1:])]) + ) + else: + clean_item[root] = item[root] + + for key_path in exclude or []: + keys = key_path.split(".") + root = keys[0] + if root in item: + if isinstance(item[root], dict) and len(keys) > 1: + # Recurse on "excludes" key paths notation for sub-keys + clean_item[root] = filter_fields( + item[root], exclude=set([".".join(keys[1:])]) + ) + else: + # Item is marked for exclusion + clean_item = item + clean_item.pop(root, None) + else: + # Key is not in item, so preserve the item without modification + clean_item = item + + return Item(**clean_item) + + +def remove_invalid_assets(item: Item) -> None: + """ + Remove invalid assets from the item. This method mutates the Item. + + Pgstac may return assets without an href if assets aren't uniformly + distributed across all items. In this case, the asset without an href + is removed from the item. + """ + if "assets" in item: + item["assets"] = {k: v for k, v in item["assets"].items() if "href" in v} From adb763d77ed0d06d0f62dba171fbdb7cb2db8de3 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 28 Apr 2022 18:41:02 -0400 Subject: [PATCH 05/26] Fix fields extensions and reduce number of loops --- .../pgstac/stac_fastapi/pgstac/core.py | 2 +- .../pgstac/stac_fastapi/pgstac/utils.py | 82 +++++++++++-------- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py index c4234eea3..e56b45b93 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py @@ -157,7 +157,7 @@ async def _search_base( search_request.conf = search_request.conf or {} search_request.conf["nohydrate"] = settings.use_api_hydrate req = search_request.json(exclude_none=True, by_alias=True) - print("==req", req, flush=True) + try: async with pool.acquire() as conn: q, p = render( diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py index f0bb0b2b6..3794a4175 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py @@ -53,40 +53,58 @@ def filter_fields( in the return item. """ if not include and not exclude: - return Item(**item) - + return item + + # Build a shallow copy of included fields on item + def include_fields( + full_item: Dict[str, Any], fields: Optional[Set[str]] + ) -> Dict[str, Any]: + if not fields: + return full_item + + clean_item: Dict[str, Any] = {} + for key_path in fields or []: + keys = key_path.split(".") + root = keys[0] + if root in full_item: + if isinstance(full_item[root], dict) and len(keys) > 1: + # Recurse on "includes" key paths notation for sub-keys + clean_item[root] = include_fields( + full_item[root], fields=set([".".join(keys[1:])]) + ) + else: + clean_item[root] = full_item[root] + return clean_item + + # For an item built up for included fields, remove excluded fields + def exclude_fields( + clean_item: Dict[str, Any], fields: Optional[Set[str]] + ) -> Dict[str, Any]: + if not clean_item and not fields: + return item + + for key_path in fields or []: + keys = key_path.split(".") + root = keys[0] + if root in clean_item: + if isinstance(clean_item[root], dict) and len(keys) > 1: + # Recurse on "excludes" key path notation for sub-keys + clean_item[root] = exclude_fields( + clean_item[root], fields=set([".".join(keys[1:])]) + ) + # Remove root key entirely if it is now empty + if not clean_item[root]: + del clean_item[root] + else: + clean_item.pop(root, None) + + return clean_item + + # Coalesce incoming type to a dict item = dict(item) - clean_item: Dict[str, Any] = {} - for key_path in include or []: - keys = key_path.split(".") - root = keys[0] - if root in item: - if isinstance(item[root], dict) and len(keys) > 1: - # Recurse on "includes" key paths notation for sub-keys - clean_item[root] = filter_fields( - item[root], include=set([".".join(keys[1:])]) - ) - else: - clean_item[root] = item[root] - - for key_path in exclude or []: - keys = key_path.split(".") - root = keys[0] - if root in item: - if isinstance(item[root], dict) and len(keys) > 1: - # Recurse on "excludes" key paths notation for sub-keys - clean_item[root] = filter_fields( - item[root], exclude=set([".".join(keys[1:])]) - ) - else: - # Item is marked for exclusion - clean_item = item - clean_item.pop(root, None) - else: - # Key is not in item, so preserve the item without modification - clean_item = item - + clean_item = include_fields(item, include) + clean_item = exclude_fields(clean_item, exclude) return Item(**clean_item) From a45b0951f18c9ba109e084129c7ab2be4b1846be Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 28 Apr 2022 19:23:19 -0400 Subject: [PATCH 06/26] Tolerate missing required attributes with fields extension Use of the fields extension can result in the return of invalid stac items if excludes is used on required attributes. When injecting item links, don't attempt to build links for which needed attributes aren't available. When API Hydrate is enabled, the required attributes are preserved prior to filtering and are used in the link generation. --- .../pgstac/stac_fastapi/pgstac/core.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py index e56b45b93..195780ff5 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py @@ -183,21 +183,27 @@ async def _search_base( if include and len(include) == 0: include = None - async def _add_item_links(feature: Item) -> None: + async def _add_item_links( + feature: Item, + collection_id: Optional[str] = None, + item_id: Optional[str] = None, + ) -> None: """Add ItemLinks to the Item. If the fields extension is excluding links, then don't add them. + Also skip links if the item doesn't provide collection and item ids. """ + collection_id = feature.get("collection") or collection_id + item_id = feature.get("id") or item_id + if ( search_request.fields.exclude is None or "links" not in search_request.fields.exclude + and all([collection_id, item_id]) ): - # TODO: feature.collection is not always included - # This code fails if it's left outside of the fields expression - # I've update fields extension test cases to always include feature.collection feature["links"] = await ItemLinks( - collection_id=feature["collection"], - item_id=feature["id"], + collection_id=collection_id, + item_id=item_id, request=request, ).get_links(extra_links=feature.get("links")) @@ -212,9 +218,14 @@ async def _get_base_item(collection_id: str) -> Dict[str, Any]: for feature in collection.get("features") or []: feature = await hydrate(feature, base_item_cache=base_item_cache) + + # Grab ids needed for links that may be removed by the fields extension. + collection_id = feature.get("collection") + item_id = feature.get("id") + feature = filter_fields(feature, include, exclude) remove_invalid_assets(feature) - await _add_item_links(feature) + await _add_item_links(feature, collection_id, item_id) cleaned_features.append(feature) else: From c86e0d9ce449b7d5baca05affac881f8b537971f Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 28 Apr 2022 20:00:14 -0400 Subject: [PATCH 07/26] Run pgstac tests in db and api hydrate mode --- stac_fastapi/pgstac/tests/conftest.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/pgstac/tests/conftest.py b/stac_fastapi/pgstac/tests/conftest.py index 04c4d7245..13286e3d9 100644 --- a/stac_fastapi/pgstac/tests/conftest.py +++ b/stac_fastapi/pgstac/tests/conftest.py @@ -31,6 +31,7 @@ DATA_DIR = os.path.join(os.path.dirname(__file__), "data") settings = Settings(testing=True) +pgstac_api_hydrate_settings = Settings(testing=True, use_api_hydrate=True) @pytest.fixture(scope="session") @@ -96,9 +97,10 @@ async def pgstac(pg): print(f"PGStac Migrated to {version}") -@pytest.fixture(scope="session") -def api_client(pg): - print("creating client with settings") +# Run all the tests that use the api_client in both db hydrate and api hydrate mode +@pytest.fixture(params=[settings, pgstac_api_hydrate_settings], scope="session") +def api_client(request, pg): + print("creating client with settings, hydrate:", request.param.use_api_hydrate) extensions = [ TransactionExtension(client=TransactionsClient(), settings=settings), @@ -109,9 +111,8 @@ def api_client(pg): TokenPaginationExtension(), ] post_request_model = create_post_request_model(extensions, base_model=PgstacSearch) - api = StacApi( - settings=settings, + settings=request.param, extensions=extensions, client=CoreCrudClient(post_request_model=post_request_model), search_get_request_model=create_get_request_model(extensions), From 78224da807de7388dfc1311ec4a0bb4c8993a9cf Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Fri, 29 Apr 2022 13:08:33 -0400 Subject: [PATCH 08/26] Merge dicts within lists during hydration In practice, an asset on a base_item and an item may have mergable dicts (ie, raster bands). --- .../stac_fastapi/pgstac/types/errors.py | 7 + .../pgstac/stac_fastapi/pgstac/utils.py | 28 +- stac_fastapi/pgstac/tests/conftest.py | 22 ++ .../pgstac/tests/data/test2_collection.json | 271 ++++++++++++++++++ .../pgstac/tests/data/test2_item.json | 258 +++++++++++++++++ .../pgstac/tests/resources/test_item.py | 16 ++ 6 files changed, 594 insertions(+), 8 deletions(-) create mode 100644 stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py create mode 100644 stac_fastapi/pgstac/tests/data/test2_collection.json create mode 100644 stac_fastapi/pgstac/tests/data/test2_item.json diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py new file mode 100644 index 000000000..4cdef0f89 --- /dev/null +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py @@ -0,0 +1,7 @@ +"""stac_fastapi.pgstac.types.errors module.""" + + +class InvalidHydrationMergeValue(Exception): + """Raised when a merge value is invalid.""" + + pass diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py index 3794a4175..5d75156ab 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py @@ -1,8 +1,9 @@ """stac-fastapi utility methods.""" from copy import deepcopy -from typing import Any, Dict, Optional, Set, Union +from typing import Any, Dict, List, Optional, Set, Union from stac_fastapi.pgstac.types.base_item_cache import BaseItemCache +from stac_fastapi.pgstac.types.errors import InvalidHydrationMergeValue from stac_fastapi.types.stac import Item @@ -19,18 +20,29 @@ async def hydrate( # Merge will mutate i, but create deep copies of values in the base item # This will prevent the base item values from being mutated, e.g. by # filtering out fields in `filter_fields`. - def merge(b: Dict[str, Any], i: Dict[str, Any]): + def merge( + b: Dict[str, Any], i: Dict[str, Any], parent_key: Optional[List[str]] = [] + ): for key in b: if key in i: if isinstance(b[key], dict) and isinstance(i.get(key), dict): # Recurse on dicts to merge values - merge(b[key], i[key]) - elif b[key] == i.get(key): - # Matching key/value is a no-op - pass + merge(b[key], i[key], parent_key + [key]) elif isinstance(b[key], list) and isinstance(i.get(key), list): - # Merge unequal lists - i[key].extend(deepcopy(b[key])) + # Merge unequal lists, assume uniform types + if b[key] and isinstance(b[key][0], dict): + if len(b[key]) != len(i[key]): + error_path = ".".join(parent_key + [key]) + raise InvalidHydrationMergeValue( + f"Unequal list of dicts lengths at key item.{error_path}" + ) + + for bb, ii in zip(b[key], i[key]): + merge(bb, ii, parent_key + [key]) + else: + # Key exists on item but isn't a dict or list, keep item value + pass + else: # Keys in base item that are not in item are simply copied over i[key] = deepcopy(b[key]) diff --git a/stac_fastapi/pgstac/tests/conftest.py b/stac_fastapi/pgstac/tests/conftest.py index 13286e3d9..723bab66b 100644 --- a/stac_fastapi/pgstac/tests/conftest.py +++ b/stac_fastapi/pgstac/tests/conftest.py @@ -169,3 +169,25 @@ async def load_test_item(app_client, load_test_data, load_test_collection): ) assert resp.status_code == 200 return Item.parse_obj(resp.json()) + + +@pytest.fixture +async def load_test2_collection(app_client, load_test_data): + data = load_test_data("test2_collection.json") + resp = await app_client.post( + "/collections", + json=data, + ) + assert resp.status_code == 200 + return Collection.parse_obj(resp.json()) + + +@pytest.fixture +async def load_test2_item(app_client, load_test_data, load_test2_collection): + data = load_test_data("test2_item.json") + resp = await app_client.post( + "/collections/{coll.id}/items", + json=data, + ) + assert resp.status_code == 200 + return Item.parse_obj(resp.json()) diff --git a/stac_fastapi/pgstac/tests/data/test2_collection.json b/stac_fastapi/pgstac/tests/data/test2_collection.json new file mode 100644 index 000000000..32502a360 --- /dev/null +++ b/stac_fastapi/pgstac/tests/data/test2_collection.json @@ -0,0 +1,271 @@ +{ + "id": "test2-collection", + "type": "Collection", + "links": [ + { + "rel": "items", + "type": "application/geo+json", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/landsat-c2-l1/items" + }, + { + "rel": "parent", + "type": "application/json", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/" + }, + { + "rel": "root", + "type": "application/json", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/" + }, + { + "rel": "self", + "type": "application/json", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/landsat-c2-l1" + }, + { + "rel": "cite-as", + "href": "https://doi.org/10.5066/P9AF14YV", + "title": "Landsat 1-5 MSS Collection 2 Level-1" + }, + { + "rel": "license", + "href": "https://www.usgs.gov/core-science-systems/hdds/data-policy", + "title": "Public Domain" + }, + { + "rel": "describedby", + "href": "https://planetarycomputer.microsoft.com/dataset/landsat-c2-l1", + "title": "Human readable dataset overview and reference", + "type": "text/html" + } + ], + "title": "Landsat Collection 2 Level-1", + "assets": { + "thumbnail": { + "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/landsat-c2-l1-thumb.png", + "type": "image/png", + "roles": ["thumbnail"], + "title": "Landsat Collection 2 Level-1 thumbnail" + } + }, + "extent": { + "spatial": { + "bbox": [[-180, -90, 180, 90]] + }, + "temporal": { + "interval": [["1972-07-25T00:00:00Z", "2013-01-07T23:23:59Z"]] + } + }, + "license": "proprietary", + "keywords": ["Landsat", "USGS", "NASA", "Satellite", "Global", "Imagery"], + "providers": [ + { + "url": "https://landsat.gsfc.nasa.gov/", + "name": "NASA", + "roles": ["producer", "licensor"] + }, + { + "url": "https://www.usgs.gov/landsat-missions/landsat-collection-2-level-1-data", + "name": "USGS", + "roles": ["producer", "processor", "licensor"] + }, + { + "url": "https://planetarycomputer.microsoft.com", + "name": "Microsoft", + "roles": ["host"] + } + ], + "summaries": { + "gsd": [79], + "sci:doi": ["10.5066/P9AF14YV"], + "eo:bands": [ + { + "name": "B4", + "common_name": "green", + "description": "Visible green (Landsat 1-3 Band B4)", + "center_wavelength": 0.55, + "full_width_half_max": 0.1 + }, + { + "name": "B5", + "common_name": "red", + "description": "Visible red (Landsat 1-3 Band B5)", + "center_wavelength": 0.65, + "full_width_half_max": 0.1 + }, + { + "name": "B6", + "common_name": "nir08", + "description": "Near infrared (Landsat 1-3 Band B6)", + "center_wavelength": 0.75, + "full_width_half_max": 0.1 + }, + { + "name": "B7", + "common_name": "nir09", + "description": "Near infrared (Landsat 1-3 Band B7)", + "center_wavelength": 0.95, + "full_width_half_max": 0.3 + }, + { + "name": "B1", + "common_name": "green", + "description": "Visible green (Landsat 4-5 Band B1)", + "center_wavelength": 0.55, + "full_width_half_max": 0.1 + }, + { + "name": "B2", + "common_name": "red", + "description": "Visible red (Landsat 4-5 Band B2)", + "center_wavelength": 0.65, + "full_width_half_max": 0.1 + }, + { + "name": "B3", + "common_name": "nir08", + "description": "Near infrared (Landsat 4-5 Band B3)", + "center_wavelength": 0.75, + "full_width_half_max": 0.1 + }, + { + "name": "B4", + "common_name": "nir09", + "description": "Near infrared (Landsat 4-5 Band B4)", + "center_wavelength": 0.95, + "full_width_half_max": 0.3 + } + ], + "platform": [ + "landsat-1", + "landsat-2", + "landsat-3", + "landsat-4", + "landsat-5" + ], + "instruments": ["mss"], + "view:off_nadir": [0] + }, + "description": "Landsat Collection 2 Level-1 data, consisting of quantized and calibrated scaled Digital Numbers (DN) representing the multispectral image data. These [Level-1](https://www.usgs.gov/landsat-missions/landsat-collection-2-level-1-data) data can be [rescaled](https://www.usgs.gov/landsat-missions/using-usgs-landsat-level-1-data-product) to top of atmosphere (TOA) reflectance and/or radiance. Thermal band data can be rescaled to TOA brightness temperature.\\n\\nThis dataset represents the global archive of Level-1 data from [Landsat Collection 2](https://www.usgs.gov/core-science-systems/nli/landsat/landsat-collection-2) acquired by the [Multispectral Scanner System](https://landsat.gsfc.nasa.gov/multispectral-scanner-system/) onboard Landsat 1 through Landsat 5 from July 7, 1972 to January 7, 2013. Images are stored in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\\n", + "item_assets": { + "red": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "title": "Red Band", + "description": "Collection 2 Level-1 Red Band Top of Atmosphere Radiance", + "raster:bands": [ + { + "unit": "watt/steradian/square_meter/micrometer", + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 60 + } + ] + }, + "green": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "title": "Green Band", + "description": "Collection 2 Level-1 Green Band Top of Atmosphere Radiance", + "raster:bands": [ + { + "unit": "watt/steradian/square_meter/micrometer", + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 60 + } + ] + }, + "nir08": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "title": "Near Infrared Band 0.8", + "description": "Collection 2 Level-1 Near Infrared Band 0.8 Top of Atmosphere Radiance", + "raster:bands": [ + { + "unit": "watt/steradian/square_meter/micrometer", + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 60 + } + ] + }, + "nir09": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "title": "Near Infrared Band 0.9", + "description": "Collection 2 Level-1 Near Infrared Band 0.9 Top of Atmosphere Radiance", + "raster:bands": [ + { + "unit": "watt/steradian/square_meter/micrometer", + "nodata": 0, + "data_type": "uint8", + "spatial_resolution": 60 + } + ] + }, + "mtl.txt": { + "type": "text/plain", + "roles": ["metadata"], + "title": "Product Metadata File (txt)", + "description": "Collection 2 Level-1 Product Metadata File (txt)" + }, + "mtl.xml": { + "type": "application/xml", + "roles": ["metadata"], + "title": "Product Metadata File (xml)", + "description": "Collection 2 Level-1 Product Metadata File (xml)" + }, + "mtl.json": { + "type": "application/json", + "roles": ["metadata"], + "title": "Product Metadata File (json)", + "description": "Collection 2 Level-1 Product Metadata File (json)" + }, + "qa_pixel": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["cloud"], + "title": "Pixel Quality Assessment Band", + "description": "Collection 2 Level-1 Pixel Quality Assessment Band", + "raster:bands": [ + { + "unit": "bit index", + "nodata": 1, + "data_type": "uint16", + "spatial_resolution": 60 + } + ] + }, + "qa_radsat": { + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["saturation"], + "title": "Radiometric Saturation and Dropped Pixel Quality Assessment Band", + "description": "Collection 2 Level-1 Radiometric Saturation and Dropped Pixel Quality Assessment Band", + "raster:bands": [ + { + "unit": "bit index", + "data_type": "uint16", + "spatial_resolution": 60 + } + ] + }, + "thumbnail": { + "type": "image/jpeg", + "roles": ["thumbnail"], + "title": "Thumbnail image" + }, + "reduced_resolution_browse": { + "type": "image/jpeg", + "roles": ["overview"], + "title": "Reduced resolution browse image" + } + }, + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json" + ] +} diff --git a/stac_fastapi/pgstac/tests/data/test2_item.json b/stac_fastapi/pgstac/tests/data/test2_item.json new file mode 100644 index 000000000..62fa2521a --- /dev/null +++ b/stac_fastapi/pgstac/tests/data/test2_item.json @@ -0,0 +1,258 @@ +{ + "id": "test2-item", + "bbox": [-84.7340712, 30.8344014, -82.3892149, 32.6891482], + "type": "Feature", + "links": [ + { + "rel": "collection", + "type": "application/json", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/landsat-c2-l1" + }, + { + "rel": "parent", + "type": "application/json", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/landsat-c2-l1" + }, + { + "rel": "root", + "type": "application/json", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/" + }, + { + "rel": "self", + "type": "application/geo+json", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/landsat-c2-l1/items/LM05_L1GS_018038_19901223_02_T2" + }, + { + "rel": "cite-as", + "href": "https://doi.org/10.5066/P9AF14YV", + "title": "Landsat 1-5 MSS Collection 2 Level-1" + }, + { + "rel": "via", + "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l1/items/LM05_L1GS_018038_19901223_20200827_02_T2", + "type": "application/json", + "title": "USGS STAC Item" + }, + { + "rel": "preview", + "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/data/item/map?collection=landsat-c2-l1&item=LM05_L1GS_018038_19901223_02_T2", + "title": "Map of item", + "type": "text/html" + } + ], + "assets": { + "red": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_B2.TIF", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "title": "Red Band (B2)", + "eo:bands": [ + { + "name": "B2", + "common_name": "red", + "description": "Landsat 4-5 Band B2", + "center_wavelength": 0.65, + "full_width_half_max": 0.1 + } + ], + "description": "Collection 2 Level-1 Red Band Top of Atmosphere Radiance", + "raster:bands": [ + { + "unit": "watt/steradian/square_meter/micrometer", + "scale": 0.66024, + "nodata": 0, + "offset": 2.03976, + "data_type": "uint8", + "spatial_resolution": 60 + } + ] + }, + "green": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_B1.TIF", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "title": "Green Band (B1)", + "eo:bands": [ + { + "name": "B1", + "common_name": "green", + "description": "Landsat 4-5 Band B1", + "center_wavelength": 0.55, + "full_width_half_max": 0.1 + } + ], + "description": "Collection 2 Level-1 Green Band Top of Atmosphere Radiance", + "raster:bands": [ + { + "unit": "watt/steradian/square_meter/micrometer", + "scale": 0.88504, + "nodata": 0, + "offset": 1.51496, + "data_type": "uint8", + "spatial_resolution": 60 + } + ] + }, + "nir08": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_B3.TIF", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "title": "Near Infrared Band 0.8 (B3)", + "eo:bands": [ + { + "name": "B3", + "common_name": "nir08", + "description": "Landsat 4-5 Band B3", + "center_wavelength": 0.75, + "full_width_half_max": 0.1 + } + ], + "description": "Collection 2 Level-1 Near Infrared Band 0.7 Top of Atmosphere Radiance", + "raster:bands": [ + { + "unit": "watt/steradian/square_meter/micrometer", + "scale": 0.55866, + "nodata": 0, + "offset": 4.34134, + "data_type": "uint8", + "spatial_resolution": 60 + } + ] + }, + "nir09": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_B4.TIF", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["data"], + "title": "Near Infrared Band 0.9 (B4)", + "eo:bands": [ + { + "name": "B4", + "common_name": "nir09", + "description": "Landsat 4-5 Band B4", + "center_wavelength": 0.95, + "full_width_half_max": 0.3 + } + ], + "description": "Collection 2 Level-1 Near Infrared Band 0.9 Top of Atmosphere Radiance", + "raster:bands": [ + { + "unit": "watt/steradian/square_meter/micrometer", + "scale": 0.46654, + "nodata": 0, + "offset": 1.03346, + "data_type": "uint8", + "spatial_resolution": 60 + } + ] + }, + "mtl.txt": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_MTL.txt", + "type": "text/plain", + "roles": ["metadata"], + "title": "Product Metadata File (txt)", + "description": "Collection 2 Level-1 Product Metadata File (txt)" + }, + "mtl.xml": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_MTL.xml", + "type": "application/xml", + "roles": ["metadata"], + "title": "Product Metadata File (xml)", + "description": "Collection 2 Level-1 Product Metadata File (xml)" + }, + "mtl.json": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_MTL.json", + "type": "application/json", + "roles": ["metadata"], + "title": "Product Metadata File (json)", + "description": "Collection 2 Level-1 Product Metadata File (json)" + }, + "qa_pixel": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_QA_PIXEL.TIF", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["cloud"], + "title": "Pixel Quality Assessment Band (QA_PIXEL)", + "description": "Collection 2 Level-1 Pixel Quality Assessment Band", + "raster:bands": [ + { + "unit": "bit index", + "nodata": 1, + "data_type": "uint16", + "spatial_resolution": 60 + } + ] + }, + "qa_radsat": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_QA_RADSAT.TIF", + "type": "image/tiff; application=geotiff; profile=cloud-optimized", + "roles": ["saturation"], + "title": "Radiometric Saturation and Dropped Pixel Quality Assessment Band (QA_RADSAT)", + "description": "Collection 2 Level-1 Radiometric Saturation and Dropped Pixel Quality Assessment Band", + "raster:bands": [ + { + "unit": "bit index", + "data_type": "uint16", + "spatial_resolution": 60 + } + ] + }, + "thumbnail": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_thumb_small.jpeg", + "type": "image/jpeg", + "roles": ["thumbnail"], + "title": "Thumbnail image" + }, + "reduced_resolution_browse": { + "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1990/018/038/LM05_L1GS_018038_19901223_20200827_02_T2/LM05_L1GS_018038_19901223_20200827_02_T2_thumb_large.jpeg", + "type": "image/jpeg", + "roles": ["overview"], + "title": "Reduced resolution browse image" + } + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [-84.3264316, 32.6891482], + [-84.7340712, 31.1114869], + [-82.8283452, 30.8344014], + [-82.3892149, 32.4079117], + [-84.3264316, 32.6891482] + ] + ] + }, + "collection": "test2-collection", + "properties": { + "gsd": 79, + "created": "2022-03-31T16:51:57.476085Z", + "sci:doi": "10.5066/P9AF14YV", + "datetime": "1990-12-23T15:26:35.581000Z", + "platform": "landsat-5", + "proj:epsg": 32617, + "proj:shape": [3525, 3946], + "description": "Landsat Collection 2 Level-1", + "instruments": ["mss"], + "eo:cloud_cover": 23, + "proj:transform": [60, 0, 140790, 0, -60, 3622110], + "view:off_nadir": 0, + "landsat:wrs_row": "038", + "landsat:scene_id": "LM50180381990357AAA03", + "landsat:wrs_path": "018", + "landsat:wrs_type": "2", + "view:sun_azimuth": 147.23255058, + "landsat:correction": "L1GS", + "view:sun_elevation": 27.04507311, + "landsat:cloud_cover_land": 28, + "landsat:collection_number": "02", + "landsat:collection_category": "T2" + }, + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/raster/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json", + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json" + ] +} diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 8e9e7de1e..19d1d5338 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -1323,3 +1323,19 @@ async def test_filter_cql2text(app_client, load_test_data, load_test_collection) resp_json = resp.json() print(resp_json) assert len(resp.json()["features"]) == 0 + + +async def test_item_merge_raster_bands( + app_client, load_test2_item, load_test2_collection +): + resp = await app_client.get("/collections/test2-collection/items/test2-item") + resp_json = resp.json() + red_bands = resp_json["assets"]["red"]["raster:bands"] + + # The merged item should have merged the band dicts from base and item + # into a single dict + assert len(red_bands) == 1 + # The merged item should have the full 6 bands + assert len(red_bands[0].keys()) == 6 + # The merged item should have kept the item value rather than the base value + assert red_bands[0]["offset"] == 2.03976 From e60fa7d49ab7e1a492f1aa1b85159f1a70264129 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Fri, 29 Apr 2022 13:37:42 -0400 Subject: [PATCH 09/26] Add note on settings in readme --- docker-compose.yml | 2 +- stac_fastapi/pgstac/README.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e6532fbd0..475df15c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,7 @@ services: - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR - DB_MIN_CONN_SIZE=1 - DB_MAX_CONN_SIZE=1 - - USE_API_HYDRATE=false + - USE_API_HYDRATE=${USE_API_HYDRATE:-false} ports: - "8082:8082" volumes: diff --git a/stac_fastapi/pgstac/README.md b/stac_fastapi/pgstac/README.md index 787ff4fd2..7961ad260 100644 --- a/stac_fastapi/pgstac/README.md +++ b/stac_fastapi/pgstac/README.md @@ -46,7 +46,12 @@ pip install -e \ stac_fastapi/pgstac[dev,server] ``` +### Settings + +To configure PGStac stac-fastapi to [hydrate search result items in the API](https://github.com/stac-utils/pgstac#runtime-configurations), set the `USE_API_HYDRATE` environment variable to `true` or explicitly set the option in the PGStac Settings object. + ### Migrations + PGStac is an external project and the may be used by multiple front ends. For Stac FastAPI development, a docker image (which is pulled as part of the docker-compose) is available at bitner/pgstac:[version] that has the full database already set up for PGStac. From 7bda2ba0e7918009935c79130aeae571c8c15492 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Fri, 29 Apr 2022 14:07:38 -0400 Subject: [PATCH 10/26] Pass request to base_item_cache This will be used by implementors who need app state which is stored on request. --- .../pgstac/stac_fastapi/pgstac/core.py | 4 ++- .../pgstac/types/base_item_cache.py | 27 ++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py index 195780ff5..201fec770 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py @@ -214,7 +214,9 @@ async def _add_item_links( async def _get_base_item(collection_id: str) -> Dict[str, Any]: return await self._get_base_item(collection_id, request) - base_item_cache = settings.base_item_cache(fetch_base_item=_get_base_item) + base_item_cache = settings.base_item_cache( + fetch_base_item=_get_base_item, request=request + ) for feature in collection.get("features") or []: feature = await hydrate(feature, base_item_cache=base_item_cache) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/base_item_cache.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/base_item_cache.py index b3ea66a9f..9b92e759d 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/base_item_cache.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/base_item_cache.py @@ -1,6 +1,9 @@ +"""base_item_cache classes for pgstac fastapi.""" import abc from typing import Any, Callable, Coroutine, Dict +from starlette.requests import Request + class BaseItemCache(abc.ABC): """ @@ -11,9 +14,19 @@ class BaseItemCache(abc.ABC): """ def __init__( - self, fetch_base_item: Callable[[str], Coroutine[Any, Any, Dict[str, Any]]] + self, + fetch_base_item: Callable[[str], Coroutine[Any, Any, Dict[str, Any]]], + request: Request, ): + """ + Initialize the base item cache. + + Args: + fetch_base_item: A function that fetches the base item for a collection. + request: The request object containing app state that may be used by caches. + """ self._fetch_base_item = fetch_base_item + self._request = request @abc.abstractmethod async def get(self, collection_id: str) -> Dict[str, Any]: @@ -22,17 +35,19 @@ async def get(self, collection_id: str) -> Dict[str, Any]: class DefaultBaseItemCache(BaseItemCache): - """ - Implementation of the BaseItemCache that holds base items in a dict. - """ + """Implementation of the BaseItemCache that holds base items in a dict.""" def __init__( - self, fetch_base_item: Callable[[str], Coroutine[Any, Any, Dict[str, Any]]] + self, + fetch_base_item: Callable[[str], Coroutine[Any, Any, Dict[str, Any]]], + request: Request, ): + """Initialize the base item cache.""" self._base_items = {} - super().__init__(fetch_base_item) + super().__init__(fetch_base_item, request) async def get(self, collection_id: str): + """Return the base item for the collection and cache by collection id.""" if collection_id not in self._base_items: self._base_items[collection_id] = await self._fetch_base_item( collection_id, From 35d7a1219fb35d5aa2356f400814f46234fdbab5 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Wed, 4 May 2022 19:01:35 -0400 Subject: [PATCH 11/26] Upgrade pypgstac and use included hydrate function The hydrate function was improved and moved to pypgstac so it could be used in other projects outside of stac-fastapi. It was developed with a corresponding dehydrate function to ensure parity between the two. The version of pypgstac is unpublished and pinned to a draft commit at the point and will be upgraded subsequently. --- stac_fastapi/pgstac/setup.py | 7 ++- .../pgstac/stac_fastapi/pgstac/core.py | 7 ++- .../pgstac/stac_fastapi/pgstac/utils.py | 62 +------------------ 3 files changed, 11 insertions(+), 65 deletions(-) diff --git a/stac_fastapi/pgstac/setup.py b/stac_fastapi/pgstac/setup.py index f832bed38..0ca369e70 100644 --- a/stac_fastapi/pgstac/setup.py +++ b/stac_fastapi/pgstac/setup.py @@ -18,16 +18,21 @@ "buildpg", "brotli_asgi", "pygeofilter @ git+https://github.com/geopython/pygeofilter@v0.1.1#egg=pygeofilter", + # TODO: "pypgstac==0.5.2", + "pypgstac @ git+https://github.com/stac-utils/pgstac@dc11b66539aac5aa37b81fb09b23b79a185a29d7#egg=pypgstac&subdirectory=pypgstac", ] extra_reqs = { "dev": [ + # TODO: replace with pypgstac[psycopg] after pypgstac is published + "psycopg[binary]==3.0.*", + "psycopg-pool==3.1.*", + # ==== "pytest", "pytest-cov", "pytest-asyncio>=0.17", "pre-commit", "requests", - "pypgstac==0.5.1", "httpx", ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py index 201fec770..811afbba6 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/core.py @@ -12,6 +12,7 @@ from pydantic import ValidationError from pygeofilter.backends.cql2_json import to_cql2 from pygeofilter.parsers.cql2_text import parse as parse_cql2_text +from pypgstac.hydration import hydrate from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes from starlette.requests import Request @@ -19,7 +20,7 @@ from stac_fastapi.pgstac.config import Settings from stac_fastapi.pgstac.models.links import CollectionLinks, ItemLinks, PagingLinks from stac_fastapi.pgstac.types.search import PgstacSearch -from stac_fastapi.pgstac.utils import filter_fields, hydrate, remove_invalid_assets +from stac_fastapi.pgstac.utils import filter_fields from stac_fastapi.types.core import AsyncBaseCoreClient from stac_fastapi.types.errors import InvalidQueryParameter, NotFoundError from stac_fastapi.types.stac import Collection, Collections, Item, ItemCollection @@ -219,14 +220,14 @@ async def _get_base_item(collection_id: str) -> Dict[str, Any]: ) for feature in collection.get("features") or []: - feature = await hydrate(feature, base_item_cache=base_item_cache) + base_item = await base_item_cache.get(feature.get("collection")) + feature = hydrate(base_item, feature) # Grab ids needed for links that may be removed by the fields extension. collection_id = feature.get("collection") item_id = feature.get("id") feature = filter_fields(feature, include, exclude) - remove_invalid_assets(feature) await _add_item_links(feature, collection_id, item_id) cleaned_features.append(feature) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py index 5d75156ab..cabe9c315 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py @@ -1,57 +1,9 @@ """stac-fastapi utility methods.""" -from copy import deepcopy -from typing import Any, Dict, List, Optional, Set, Union +from typing import Any, Dict, Optional, Set, Union -from stac_fastapi.pgstac.types.base_item_cache import BaseItemCache -from stac_fastapi.pgstac.types.errors import InvalidHydrationMergeValue from stac_fastapi.types.stac import Item -async def hydrate( - item: Union[Item, Dict[str, Any]], base_item_cache: BaseItemCache -) -> Item: - """Hydrate item in-place with base_item properties. - - This will not perform a deep copy; values of the original item will be referenced - in the return item. - """ - item = dict(item) - - # Merge will mutate i, but create deep copies of values in the base item - # This will prevent the base item values from being mutated, e.g. by - # filtering out fields in `filter_fields`. - def merge( - b: Dict[str, Any], i: Dict[str, Any], parent_key: Optional[List[str]] = [] - ): - for key in b: - if key in i: - if isinstance(b[key], dict) and isinstance(i.get(key), dict): - # Recurse on dicts to merge values - merge(b[key], i[key], parent_key + [key]) - elif isinstance(b[key], list) and isinstance(i.get(key), list): - # Merge unequal lists, assume uniform types - if b[key] and isinstance(b[key][0], dict): - if len(b[key]) != len(i[key]): - error_path = ".".join(parent_key + [key]) - raise InvalidHydrationMergeValue( - f"Unequal list of dicts lengths at key item.{error_path}" - ) - - for bb, ii in zip(b[key], i[key]): - merge(bb, ii, parent_key + [key]) - else: - # Key exists on item but isn't a dict or list, keep item value - pass - - else: - # Keys in base item that are not in item are simply copied over - i[key] = deepcopy(b[key]) - - base_item = await base_item_cache.get(item["collection"]) - merge(base_item, item) - return Item(**item) - - def filter_fields( item: Union[Item, Dict[str, Any]], include: Optional[Set[str]] = None, @@ -118,15 +70,3 @@ def exclude_fields( clean_item = include_fields(item, include) clean_item = exclude_fields(clean_item, exclude) return Item(**clean_item) - - -def remove_invalid_assets(item: Item) -> None: - """ - Remove invalid assets from the item. This method mutates the Item. - - Pgstac may return assets without an href if assets aren't uniformly - distributed across all items. In this case, the asset without an href - is removed from the item. - """ - if "assets" in item: - item["assets"] = {k: v for k, v in item["assets"].items() if "href" in v} From 951dd8feb862630a906b038b297ae483373d9d26 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 5 May 2022 10:55:39 -0400 Subject: [PATCH 12/26] Improve fields extension implementation Correctly supports deeply nested property keys in both include and exclude, as well as improves variable naming, comments, and test cases. --- .../pgstac/stac_fastapi/pgstac/utils.py | 107 ++++++++++++------ .../pgstac/tests/resources/test_item.py | 92 +++++++++++++++ 2 files changed, 167 insertions(+), 32 deletions(-) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py index cabe9c315..eef1b8b73 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py @@ -19,54 +19,97 @@ def filter_fields( if not include and not exclude: return item - # Build a shallow copy of included fields on item + # Build a shallow copy of included fields on an item, or a sub-tree of an item def include_fields( - full_item: Dict[str, Any], fields: Optional[Set[str]] + source: Dict[str, Any], fields: Optional[Set[str]] ) -> Dict[str, Any]: if not fields: - return full_item + return source clean_item: Dict[str, Any] = {} for key_path in fields or []: - keys = key_path.split(".") - root = keys[0] - if root in full_item: - if isinstance(full_item[root], dict) and len(keys) > 1: - # Recurse on "includes" key paths notation for sub-keys - clean_item[root] = include_fields( - full_item[root], fields=set([".".join(keys[1:])]) + key_path_parts = key_path.split(".") + key_root = key_path_parts[0] + if key_root in source: + if isinstance(source[key_root], dict) and len(key_path_parts) > 1: + # The root of this key path on the item is a dict, and the + # key path indicates a sub-key to be included. Walk the dict + # from the root key and get the full nested value to include. + value = include_fields( + source[key_root], fields=set([".".join(key_path_parts[1:])]) ) + + if isinstance(clean_item.get(key_root), dict): + # A previously specified key and sub-keys may have been included + # already, so do a deep merge update if the root key already exists. + dict_deep_update(clean_item[key_root], value) + else: + # The root key does not exist, so add it. Fields + # extension only allows nested referencing on dicts, so + # this won't overwrite anything. + clean_item[key_root] = value else: - clean_item[root] = full_item[root] + # The item value to include is not a dict, or, it is a dict but the + # key path is for the whole value, not a sub-key. Include the entire + # value in the cleaned item. + clean_item[key_root] = source[key_root] + else: + # The key, or root key of a multi-part key, is not present in the item, + # so it is ignored + pass return clean_item - # For an item built up for included fields, remove excluded fields - def exclude_fields( - clean_item: Dict[str, Any], fields: Optional[Set[str]] - ) -> Dict[str, Any]: - if not clean_item and not fields: - return item - + # For an item built up for included fields, remove excluded fields. This + # modifies `source` in place. + def exclude_fields(source: Dict[str, Any], fields: Optional[Set[str]]) -> None: for key_path in fields or []: - keys = key_path.split(".") - root = keys[0] - if root in clean_item: - if isinstance(clean_item[root], dict) and len(keys) > 1: - # Recurse on "excludes" key path notation for sub-keys - clean_item[root] = exclude_fields( - clean_item[root], fields=set([".".join(keys[1:])]) + key_path_part = key_path.split(".") + key_root = key_path_part[0] + if key_root in source: + if isinstance(source[key_root], dict) and len(key_path_part) > 1: + # Walk the nested path of this key to remove the leaf-key + exclude_fields( + source[key_root], fields=set([".".join(key_path_part[1:])]) ) - # Remove root key entirely if it is now empty - if not clean_item[root]: - del clean_item[root] + # If, after removing the leaf-key, the root is now an empty + # dict, remove it entirely + if not source[key_root]: + del source[key_root] else: - clean_item.pop(root, None) - - return clean_item + # The key's value is not a dict, or there is no sub-key to remove. The + # entire key can be removed from the source. + source.pop(key_root, None) + else: + # The key to remove does not exist on the source, so it is ignored + pass # Coalesce incoming type to a dict item = dict(item) clean_item = include_fields(item, include) - clean_item = exclude_fields(clean_item, exclude) + + # If, after including all the specified fields, there are no included properties, + # return the full item. + if not clean_item: + return Item(**item) + + exclude_fields(clean_item, exclude) + return Item(**clean_item) + + +def dict_deep_update(merge_to: Dict[str, Any], merge_from: Dict[str, Any]) -> None: + """Perform a deep update of two dicts. + + merge_to is updated in-place with the values from merge_from. + merge_from values take precedence over existing values in merge_to. + """ + for k, v in merge_from.items(): + if ( + k in merge_to + and isinstance(merge_to[k], dict) + and isinstance(merge_from[k], dict) + ): + dict_deep_update(merge_to[k], merge_from[k]) + else: + merge_to[k] = v diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 19d1d5338..7c201adf9 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -1087,6 +1087,98 @@ async def test_field_extension_exclude_default_includes( assert "geometry" not in resp_json["features"][0] +async def test_field_extension_include_multiple_subkeys( + app_client, load_test_item, load_test_collection +): + """Test that multiple subkeys of an object field are included""" + body = {"fields": {"include": ["properties.width", "properties.height"]}} + + resp = await app_client.post("/search", json=body) + assert resp.status_code == 200 + resp_json = resp.json() + + resp_prop_keys = resp_json["features"][0]["properties"].keys() + assert set(resp_prop_keys) == set(["width", "height"]) + + +async def test_field_extension_include_multiple_deeply_nested_subkeys( + app_client, load_test_item, load_test_collection +): + """Test that multiple deeply nested subkeys of an object field are included""" + body = {"fields": {"include": ["assets.ANG.type", "assets.ANG.href"]}} + + resp = await app_client.post("/search", json=body) + assert resp.status_code == 200 + resp_json = resp.json() + + resp_assets = resp_json["features"][0]["assets"] + assert set(resp_assets.keys()) == set(["ANG"]) + assert set(resp_assets["ANG"].keys()) == set(["type", "href"]) + + +async def test_field_extension_exclude_multiple_deeply_nested_subkeys( + app_client, load_test_item, load_test_collection +): + """Test that multiple deeply nested subkeys of an object field are excluded""" + body = {"fields": {"exclude": ["assets.ANG.type", "assets.ANG.href"]}} + + resp = await app_client.post("/search", json=body) + assert resp.status_code == 200 + resp_json = resp.json() + + resp_assets = resp_json["features"][0]["assets"] + assert len(resp_assets.keys()) > 0 + assert "type" not in resp_assets["ANG"] + assert "href" not in resp_assets["ANG"] + + +async def test_field_extension_exclude_deeply_nested_included_subkeys( + app_client, load_test_item, load_test_collection +): + """Test that deeply nested keys of a nested object that was included are excluded""" + body = { + "fields": { + "include": ["assets.ANG.type", "assets.ANG.href"], + "exclude": ["assets.ANG.href"], + } + } + + resp = await app_client.post("/search", json=body) + assert resp.status_code == 200 + resp_json = resp.json() + + resp_assets = resp_json["features"][0]["assets"] + assert "type" in resp_assets["ANG"] + assert "href" not in resp_assets["ANG"] + + +async def test_field_extension_exclude_links( + app_client, load_test_item, load_test_collection +): + """Links have special injection behavior, ensure they can be excluded with the fields extension""" + body = {"fields": {"exclude": ["links"]}} + + resp = await app_client.post("/search", json=body) + assert resp.status_code == 200 + resp_json = resp.json() + + assert "links" not in resp_json["features"][0] + + +async def test_field_extension_include_only_non_existant_field( + app_client, load_test_item, load_test_collection +): + """Including only a non-existant field should return the full item""" + body = {"fields": {"include": ["non_existant_field"]}} + + resp = await app_client.post("/search", json=body) + assert resp.status_code == 200 + resp_json = resp.json() + + assert len(resp_json["features"][0].keys()) > 0 + assert "properties" in resp_json["features"][0] + + async def test_search_intersects_and_bbox(app_client): """Test POST search intersects and bbox are mutually exclusive (core)""" bbox = [-118, 34, -117, 35] From bf3acd50e398dda2956e38ac4830675bead746c1 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 5 May 2022 12:23:47 -0400 Subject: [PATCH 13/26] Remove unused error type --- stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py deleted file mode 100644 index 4cdef0f89..000000000 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/types/errors.py +++ /dev/null @@ -1,7 +0,0 @@ -"""stac_fastapi.pgstac.types.errors module.""" - - -class InvalidHydrationMergeValue(Exception): - """Raised when a merge value is invalid.""" - - pass From 9be6586ba68b834ed7756eddcb68dd2de6fa9cd5 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 May 2022 18:24:47 +0000 Subject: [PATCH 14/26] adjust tests for changes in api --- stac_fastapi/api/stac_fastapi/api/errors.py | 2 +- .../pgstac/tests/clients/test_postgres.py | 18 +++++++--- stac_fastapi/pgstac/tests/conftest.py | 26 ++++++++++---- .../pgstac/tests/resources/test_item.py | 36 ++++++++++--------- 4 files changed, 54 insertions(+), 28 deletions(-) diff --git a/stac_fastapi/api/stac_fastapi/api/errors.py b/stac_fastapi/api/stac_fastapi/api/errors.py index 95058d4c9..29df3b9ab 100644 --- a/stac_fastapi/api/stac_fastapi/api/errors.py +++ b/stac_fastapi/api/stac_fastapi/api/errors.py @@ -23,7 +23,7 @@ DEFAULT_STATUS_CODES = { NotFoundError: status.HTTP_404_NOT_FOUND, ConflictError: status.HTTP_409_CONFLICT, - ForeignKeyError: status.HTTP_422_UNPROCESSABLE_ENTITY, + ForeignKeyError: status.HTTP_424_FAILED_DEPENDENCY, DatabaseError: status.HTTP_424_FAILED_DEPENDENCY, Exception: status.HTTP_500_INTERNAL_SERVER_ERROR, InvalidQueryParameter: status.HTTP_400_BAD_REQUEST, diff --git a/stac_fastapi/pgstac/tests/clients/test_postgres.py b/stac_fastapi/pgstac/tests/clients/test_postgres.py index f08952e64..3a7a20455 100644 --- a/stac_fastapi/pgstac/tests/clients/test_postgres.py +++ b/stac_fastapi/pgstac/tests/clients/test_postgres.py @@ -1,6 +1,6 @@ import uuid from typing import Callable - +import orjson from stac_pydantic import Collection, Item # from tests.conftest import MockStarletteRequest @@ -58,14 +58,22 @@ async def test_create_item(app_client, load_test_data: Callable, load_test_colle ) assert resp.status_code == 200 + gresp = await app_client.get(f"/collections/{coll.id}/items") + print (gresp.json()['context']) + post_item = Item.parse_obj(resp.json()) assert in_item.dict(exclude={"links"}) == post_item.dict(exclude={"links"}) resp = await app_client.get(f"/collections/{coll.id}/items/{post_item.id}") assert resp.status_code == 200 + get_item = Item.parse_obj(resp.json()) - assert in_item.dict(exclude={"links"}) == get_item.dict(exclude={"links"}) + print(get_item.dict(exclude={"links"})['geometry']) + print(in_item.dict(exclude={"links"})['geometry']) + print(get_item.dict(exclude={"links"})['bbox']) + print(in_item.dict(exclude={"links"})['bbox']) + assert in_item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) async def test_update_item(app_client, load_test_collection, load_test_item): @@ -74,14 +82,16 @@ async def test_update_item(app_client, load_test_collection, load_test_item): item.properties.description = "Update Test" + print(item.json()) resp = await app_client.put(f"/collections/{coll.id}/items", content=item.json()) assert resp.status_code == 200 resp = await app_client.get(f"/collections/{coll.id}/items/{item.id}") assert resp.status_code == 200 - + print(resp.json()) get_item = Item.parse_obj(resp.json()) - assert item.dict(exclude={"links"}) == get_item.dict(exclude={"links"}) + print(get_item) + assert item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) assert get_item.properties.description == "Update Test" diff --git a/stac_fastapi/pgstac/tests/conftest.py b/stac_fastapi/pgstac/tests/conftest.py index 723bab66b..188729499 100644 --- a/stac_fastapi/pgstac/tests/conftest.py +++ b/stac_fastapi/pgstac/tests/conftest.py @@ -68,6 +68,7 @@ async def pg(): db = PgstacDB(dsn=settings.testing_connection_string) migrator = Migrate(db) version = migrator.run_migration() + db.close() print(f"PGStac Migrated to {version}") yield settings.testing_connection_string @@ -75,8 +76,15 @@ async def pg(): print("Getting rid of test database") os.environ["postgres_dbname"] = os.environ["orig_postgres_dbname"] conn = await asyncpg.connect(dsn=settings.writer_connection_string) - await conn.execute("DROP DATABASE pgstactestdb;") - await conn.close() + try: + await conn.execute("DROP DATABASE pgstactestdb;") + await conn.close() + except: + try: + await conn.execute("DROP DATABASE pgstactestdb WITH (force);") + await conn.close() + except: + pass @pytest.fixture(autouse=True) @@ -91,9 +99,9 @@ async def pgstac(pg): """ ) await conn.close() - db = PgstacDB(dsn=settings.testing_connection_string) - migrator = Migrate(db) - version = migrator.run_migration() + with PgstacDB(dsn=settings.testing_connection_string) as db: + migrator = Migrate(db) + version = migrator.run_migration() print(f"PGStac Migrated to {version}") @@ -123,8 +131,9 @@ def api_client(request, pg): return api -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") async def app(api_client): + print('Creating app Fixture') time.time() app = api_client.app await connect_to_db(app) @@ -133,9 +142,12 @@ async def app(api_client): await close_db_connection(app) + print('Closed Pools.') -@pytest.fixture(scope="session") + +@pytest.fixture(scope="function") async def app_client(app): + print("creating app_client") async with AsyncClient(app=app, base_url="http://test") as c: yield c diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 7c201adf9..c816df4b3 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -70,13 +70,13 @@ async def test_create_item(app_client, load_test_data: Callable, load_test_colle assert resp.status_code == 200 post_item = Item.parse_obj(resp.json()) - assert in_item.dict(exclude={"links"}) == post_item.dict(exclude={"links"}) + assert in_item.dict(exclude={"links","bbox"}) == post_item.dict(exclude={"links","bbox"}) resp = await app_client.get(f"/collections/{coll.id}/items/{post_item.id}") assert resp.status_code == 200 get_item = Item.parse_obj(resp.json()) - assert in_item.dict(exclude={"links"}) == get_item.dict(exclude={"links"}) + assert in_item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) async def test_fetches_valid_item( @@ -93,7 +93,7 @@ async def test_fetches_valid_item( assert resp.status_code == 200 post_item = Item.parse_obj(resp.json()) - assert in_item.dict(exclude={"links"}) == post_item.dict(exclude={"links"}) + assert in_item.dict(exclude={"links","bbox"}) == post_item.dict(exclude={"links","bbox"}) resp = await app_client.get(f"/collections/{coll.id}/items/{post_item.id}") @@ -122,7 +122,7 @@ async def test_update_item( assert resp.status_code == 200 get_item = Item.parse_obj(resp.json()) - assert item.dict(exclude={"links"}) == get_item.dict(exclude={"links"}) + assert item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) assert get_item.properties.description == "Update Test" @@ -204,6 +204,8 @@ async def test_create_item_missing_collection( item["collection"] = None resp = await app_client.post(f"/collections/{coll.id}/items", json=item) + print(resp.status_code) + print(resp.content) assert resp.status_code == 424 @@ -386,7 +388,6 @@ async def test_item_search_temporal_query_post( item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) print(item_date) - item_date = item_date + timedelta(seconds=1) params = { "collections": [test_item["collection"]], @@ -659,6 +660,7 @@ async def test_item_search_properties_field( assert resp.status_code == 200 second_test_item = load_test_data("test_item2.json") + second_test_item["properties"]["eo:cloud_cover"]=5 resp = await app_client.post( f"/collections/{test_item['collection']}/items", json=second_test_item ) @@ -669,6 +671,8 @@ async def test_item_search_properties_field( resp = await app_client.post("/search", json=params) assert resp.status_code == 200 resp_json = resp.json() + for feature in resp_json["features"]: + print(feature['properties']['eo:cloud_cover']) assert len(resp_json["features"]) == 1 @@ -1165,18 +1169,18 @@ async def test_field_extension_exclude_links( assert "links" not in resp_json["features"][0] -async def test_field_extension_include_only_non_existant_field( - app_client, load_test_item, load_test_collection -): - """Including only a non-existant field should return the full item""" - body = {"fields": {"include": ["non_existant_field"]}} +# async def test_field_extension_include_only_non_existant_field( +# app_client, load_test_item, load_test_collection +# ): +# """Including only a non-existant field should return the full item""" +# body = {"fields": {"include": ["non_existant_field"]}} - resp = await app_client.post("/search", json=body) - assert resp.status_code == 200 - resp_json = resp.json() +# resp = await app_client.post("/search", json=body) +# assert resp.status_code == 200 +# resp_json = resp.json() - assert len(resp_json["features"][0].keys()) > 0 - assert "properties" in resp_json["features"][0] +# assert len(resp_json["features"][0].keys()) > 0 +# assert "properties" in resp_json["features"][0] async def test_search_intersects_and_bbox(app_client): @@ -1242,7 +1246,7 @@ async def test_preserves_extra_link( ) assert response_item.status_code == 200 item = response_item.json() - + print(item['links']) extra_link = [link for link in item["links"] if link["rel"] == "preview"] assert extra_link assert extra_link[0]["href"] == expected_href From fe0a8a8805e2c3bdec4059ecd672195bcf1ea2bd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 May 2022 18:33:30 +0000 Subject: [PATCH 15/26] remove print statements --- .../pgstac/tests/clients/test_postgres.py | 9 ----- .../pgstac/tests/resources/test_item.py | 38 ------------------- 2 files changed, 47 deletions(-) diff --git a/stac_fastapi/pgstac/tests/clients/test_postgres.py b/stac_fastapi/pgstac/tests/clients/test_postgres.py index 3a7a20455..064577872 100644 --- a/stac_fastapi/pgstac/tests/clients/test_postgres.py +++ b/stac_fastapi/pgstac/tests/clients/test_postgres.py @@ -1,6 +1,5 @@ import uuid from typing import Callable -import orjson from stac_pydantic import Collection, Item # from tests.conftest import MockStarletteRequest @@ -59,7 +58,6 @@ async def test_create_item(app_client, load_test_data: Callable, load_test_colle assert resp.status_code == 200 gresp = await app_client.get(f"/collections/{coll.id}/items") - print (gresp.json()['context']) post_item = Item.parse_obj(resp.json()) assert in_item.dict(exclude={"links"}) == post_item.dict(exclude={"links"}) @@ -69,10 +67,6 @@ async def test_create_item(app_client, load_test_data: Callable, load_test_colle assert resp.status_code == 200 get_item = Item.parse_obj(resp.json()) - print(get_item.dict(exclude={"links"})['geometry']) - print(in_item.dict(exclude={"links"})['geometry']) - print(get_item.dict(exclude={"links"})['bbox']) - print(in_item.dict(exclude={"links"})['bbox']) assert in_item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) @@ -82,15 +76,12 @@ async def test_update_item(app_client, load_test_collection, load_test_item): item.properties.description = "Update Test" - print(item.json()) resp = await app_client.put(f"/collections/{coll.id}/items", content=item.json()) assert resp.status_code == 200 resp = await app_client.get(f"/collections/{coll.id}/items/{item.id}") assert resp.status_code == 200 - print(resp.json()) get_item = Item.parse_obj(resp.json()) - print(get_item) assert item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) assert get_item.properties.description == "Update Test" diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index c816df4b3..6fb1759d8 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -133,7 +133,6 @@ async def test_delete_item( item = load_test_item resp = await app_client.delete(f"/collections/{coll.id}/items/{item.id}") - print(resp.content) assert resp.status_code == 200 resp = await app_client.get(f"/collections/{coll.id}/items/{item.id}") @@ -188,11 +187,9 @@ async def test_delete_missing_item( item = load_test_item resp = await app_client.delete(f"/collections/{coll.id}/items/{item.id}") - print(resp.content) assert resp.status_code == 200 resp = await app_client.delete(f"/collections/{coll.id}/items/{item.id}") - print(resp.content) assert resp.status_code == 404 @@ -204,8 +201,6 @@ async def test_create_item_missing_collection( item["collection"] = None resp = await app_client.post(f"/collections/{coll.id}/items", json=item) - print(resp.status_code) - print(resp.content) assert resp.status_code == 424 @@ -247,9 +242,6 @@ async def test_pagination(app_client, load_test_data, load_test_collection): resp = await app_client.get(f"/collections/{coll.id}/items", params={"limit": 3}) assert resp.status_code == 200 first_page = resp.json() - for feature in first_page["features"]: - print(feature["id"], feature["properties"]["datetime"]) - print(f"first page links {first_page['links']}") assert len(first_page["features"]) == 3 nextlink = [ @@ -264,14 +256,10 @@ async def test_pagination(app_client, load_test_data, load_test_collection): "test-item18", ] - print(f"Next {nextlink}") resp = await app_client.get(nextlink) assert resp.status_code == 200 second_page = resp.json() - for feature in second_page["features"]: - print(feature["id"], feature["properties"]["datetime"]) - print(f"second page links {second_page['links']}") assert len(first_page["features"]) == 3 nextlink = [ @@ -285,7 +273,6 @@ async def test_pagination(app_client, load_test_data, load_test_collection): ].pop() assert prevlink is not None - print(nextlink, prevlink) assert [f["id"] for f in second_page["features"]] == [ "test-item17", @@ -296,9 +283,6 @@ async def test_pagination(app_client, load_test_data, load_test_collection): resp = await app_client.get(prevlink) assert resp.status_code == 200 back_page = resp.json() - for feature in back_page["features"]: - print(feature["id"], feature["properties"]["datetime"]) - print(back_page["links"]) assert len(back_page["features"]) == 3 assert [f["id"] for f in back_page["features"]] == [ "test-item20", @@ -387,7 +371,6 @@ async def test_item_search_temporal_query_post( assert resp.status_code == 200 item_date = rfc3339_str_to_datetime(test_item["properties"]["datetime"]) - print(item_date) params = { "collections": [test_item["collection"]], @@ -396,7 +379,6 @@ async def test_item_search_temporal_query_post( } resp = await app_client.post("/search", json=params) - print(resp.content) resp_json = resp.json() assert len(resp_json["features"]) == 1 assert resp_json["features"][0]["id"] == test_item["id"] @@ -642,7 +624,6 @@ async def test_item_search_properties_jsonb( # EPSG is a JSONB key params = {"query": {"proj:epsg": {"gt": test_item["properties"]["proj:epsg"] - 1}}} - print(params) resp = await app_client.post("/search", json=params) assert resp.status_code == 200 resp_json = resp.json() @@ -667,12 +648,9 @@ async def test_item_search_properties_field( assert resp.status_code == 200 params = {"query": {"eo:cloud_cover": {"eq": 0}}} - print(params) resp = await app_client.post("/search", json=params) assert resp.status_code == 200 resp_json = resp.json() - for feature in resp_json["features"]: - print(feature['properties']['eo:cloud_cover']) assert len(resp_json["features"]) == 1 @@ -794,7 +772,6 @@ async def test_item_search_get_filter_extension_cql2( ], }, } - print(params) resp = await app_client.post("/search", json=params) resp_json = resp.json() @@ -850,9 +827,7 @@ async def test_item_search_get_filter_extension_cql2_with_query_fails( }, "query": {"eo:cloud_cover": {"eq": 0}}, } - print(params) resp = await app_client.post("/search", json=params) - print(resp.content) assert resp.status_code == 400 @@ -885,7 +860,6 @@ async def test_pagination_item_collection( assert resp.status_code == 200 ids.append(uid) - print(ids) # Paginate through all 5 items with a limit of 1 (expecting 5 requests) page = await app_client.get( @@ -897,7 +871,6 @@ async def test_pagination_item_collection( idx += 1 page_data = page.json() item_ids.append(page_data["features"][0]["id"]) - print(idx, item_ids) nextlink = [ link["href"] for link in page_data["links"] if link["rel"] == "next" ] @@ -935,7 +908,6 @@ async def test_pagination_post(app_client, load_test_data, load_test_collection) "filter": {"op": "in", "args": [{"property": "id"}, ids]}, "limit": 1, } - print(f"REQUEST BODY: {request_body}") page = await app_client.post("/search", json=request_body) idx = 0 item_ids = [] @@ -943,7 +915,6 @@ async def test_pagination_post(app_client, load_test_data, load_test_collection) idx += 1 page_data = page.json() item_ids.append(page_data["features"][0]["id"]) - print(f"PAGING: {page_data['links']}") next_link = list(filter(lambda l: l["rel"] == "next", page_data["links"])) if not next_link: break @@ -955,7 +926,6 @@ async def test_pagination_post(app_client, load_test_data, load_test_collection) assert False # Our limit is 1 so we expect len(ids) number of requests before we run out of pages - print(idx, ids) assert idx == len(ids) # Confirm we have paginated through all items @@ -988,7 +958,6 @@ async def test_pagination_token_idempotent( }, ) page_data = page.json() - print(f"LINKS: {page_data['links']}") next_link = list(filter(lambda l: l["rel"] == "next", page_data["links"])) # Confirm token is idempotent @@ -1043,7 +1012,6 @@ async def test_field_extension_post(app_client, load_test_data, load_test_collec resp = await app_client.post("/search", json=body) resp_json = resp.json() - print(resp_json) assert "B1" not in resp_json["features"][0]["assets"].keys() assert not set(resp_json["features"][0]["properties"]) - { "orientation", @@ -1246,7 +1214,6 @@ async def test_preserves_extra_link( ) assert response_item.status_code == 200 item = response_item.json() - print(item['links']) extra_link = [link for link in item["links"] if link["rel"] == "preview"] assert extra_link assert extra_link[0]["href"] == expected_href @@ -1331,10 +1298,8 @@ async def test_item_search_get_filter_extension_cql2_2( ], }, } - print(json.dumps(params)) resp = await app_client.post("/search", json=params) resp_json = resp.json() - print(resp_json) assert resp.status_code == 200 assert len(resp_json.get("features")) == 0 @@ -1363,7 +1328,6 @@ async def test_item_search_get_filter_extension_cql2_2( } resp = await app_client.post("/search", json=params) resp_json = resp.json() - print(resp_json) assert len(resp.json()["features"]) == 1 assert ( resp_json["features"][0]["properties"]["proj:epsg"] @@ -1406,7 +1370,6 @@ async def test_filter_cql2text(app_client, load_test_data, load_test_collection) params = {"filter": filter, "filter-lang": "cql2-text"} resp = await app_client.get("/search", params=params) resp_json = resp.json() - print(resp_json) assert len(resp.json()["features"]) == 1 assert ( resp_json["features"][0]["properties"]["proj:epsg"] @@ -1417,7 +1380,6 @@ async def test_filter_cql2text(app_client, load_test_data, load_test_collection) params = {"filter": filter, "filter-lang": "cql2-text"} resp = await app_client.get("/search", params=params) resp_json = resp.json() - print(resp_json) assert len(resp.json()["features"]) == 0 From 97e88d6e56defaabfb648b7e81aceca842a7f079 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 12 May 2022 18:41:43 +0000 Subject: [PATCH 16/26] add bbox back to items in tests --- stac_fastapi/pgstac/tests/clients/test_postgres.py | 4 ++-- stac_fastapi/pgstac/tests/resources/test_item.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stac_fastapi/pgstac/tests/clients/test_postgres.py b/stac_fastapi/pgstac/tests/clients/test_postgres.py index 064577872..357228268 100644 --- a/stac_fastapi/pgstac/tests/clients/test_postgres.py +++ b/stac_fastapi/pgstac/tests/clients/test_postgres.py @@ -67,7 +67,7 @@ async def test_create_item(app_client, load_test_data: Callable, load_test_colle assert resp.status_code == 200 get_item = Item.parse_obj(resp.json()) - assert in_item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) + assert in_item.dict(exclude={"links"}) == get_item.dict(exclude={"links"}) async def test_update_item(app_client, load_test_collection, load_test_item): @@ -82,7 +82,7 @@ async def test_update_item(app_client, load_test_collection, load_test_item): resp = await app_client.get(f"/collections/{coll.id}/items/{item.id}") assert resp.status_code == 200 get_item = Item.parse_obj(resp.json()) - assert item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) + assert item.dict(exclude={"links"}) == get_item.dict(exclude={"links"}) assert get_item.properties.description == "Update Test" diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index 6fb1759d8..d11a86c6c 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -70,13 +70,13 @@ async def test_create_item(app_client, load_test_data: Callable, load_test_colle assert resp.status_code == 200 post_item = Item.parse_obj(resp.json()) - assert in_item.dict(exclude={"links","bbox"}) == post_item.dict(exclude={"links","bbox"}) + assert in_item.dict(exclude={"links"}) == post_item.dict(exclude={"links"}) resp = await app_client.get(f"/collections/{coll.id}/items/{post_item.id}") assert resp.status_code == 200 get_item = Item.parse_obj(resp.json()) - assert in_item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) + assert in_item.dict(exclude={"links"}) == get_item.dict(exclude={"links"}) async def test_fetches_valid_item( @@ -93,7 +93,7 @@ async def test_fetches_valid_item( assert resp.status_code == 200 post_item = Item.parse_obj(resp.json()) - assert in_item.dict(exclude={"links","bbox"}) == post_item.dict(exclude={"links","bbox"}) + assert in_item.dict(exclude={"links"}) == post_item.dict(exclude={"links"}) resp = await app_client.get(f"/collections/{coll.id}/items/{post_item.id}") @@ -122,7 +122,7 @@ async def test_update_item( assert resp.status_code == 200 get_item = Item.parse_obj(resp.json()) - assert item.dict(exclude={"links","bbox"}) == get_item.dict(exclude={"links","bbox"}) + assert item.dict(exclude={"links"}) == get_item.dict(exclude={"links"}) assert get_item.properties.description == "Update Test" From 7648261840b2ca659b0b7035cae4fe5848618b71 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 12 May 2022 16:05:13 -0400 Subject: [PATCH 17/26] Upgrade pgstac --- docker-compose.yml | 2 +- stac_fastapi/pgstac/setup.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 475df15c8..7a2926702 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: database: container_name: stac-db - image: ghcr.io/stac-utils/pgstac:v0.5.1 + image: ghcr.io/stac-utils/pgstac:v0.6.0 environment: - POSTGRES_USER=username - POSTGRES_PASSWORD=password diff --git a/stac_fastapi/pgstac/setup.py b/stac_fastapi/pgstac/setup.py index 0ca369e70..c941125db 100644 --- a/stac_fastapi/pgstac/setup.py +++ b/stac_fastapi/pgstac/setup.py @@ -18,16 +18,12 @@ "buildpg", "brotli_asgi", "pygeofilter @ git+https://github.com/geopython/pygeofilter@v0.1.1#egg=pygeofilter", - # TODO: "pypgstac==0.5.2", - "pypgstac @ git+https://github.com/stac-utils/pgstac@dc11b66539aac5aa37b81fb09b23b79a185a29d7#egg=pypgstac&subdirectory=pypgstac", + "pypgstac==0.6.0", ] extra_reqs = { "dev": [ - # TODO: replace with pypgstac[psycopg] after pypgstac is published - "psycopg[binary]==3.0.*", - "psycopg-pool==3.1.*", - # ==== + "pypgstac[psycopg]==0.6.0", "pytest", "pytest-cov", "pytest-asyncio>=0.17", From 6d6149224422898418f73ba4c457c80c7735b2bb Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 12 May 2022 16:11:42 -0400 Subject: [PATCH 18/26] Fix conformance test fixtures --- stac_fastapi/pgstac/tests/resources/test_conformance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/pgstac/tests/resources/test_conformance.py b/stac_fastapi/pgstac/tests/resources/test_conformance.py index 2b7cd1d36..b080c4b8a 100644 --- a/stac_fastapi/pgstac/tests/resources/test_conformance.py +++ b/stac_fastapi/pgstac/tests/resources/test_conformance.py @@ -4,12 +4,12 @@ import pytest -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") async def response(app_client): return await app_client.get("/") -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") async def response_json(response) -> Dict: return response.json() From 3f03f0bf71b8d20d3ae9f593e3abc24ea558432f Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 12 May 2022 16:26:40 -0400 Subject: [PATCH 19/26] Fix sqlalchemy test with new status for FK error --- stac_fastapi/sqlalchemy/tests/resources/test_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/sqlalchemy/tests/resources/test_item.py b/stac_fastapi/sqlalchemy/tests/resources/test_item.py index f75a802bf..2f671de68 100644 --- a/stac_fastapi/sqlalchemy/tests/resources/test_item.py +++ b/stac_fastapi/sqlalchemy/tests/resources/test_item.py @@ -194,7 +194,7 @@ def test_create_item_missing_collection(app_client, load_test_data): resp = app_client.post( f"/collections/{test_item['collection']}/items", json=test_item ) - assert resp.status_code == 422 + assert resp.status_code == 424 def test_update_item_already_exists(app_client, load_test_data): From ff2fe8643f015a0a6e64f21ccaec1ed93900b821 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 12 May 2022 16:27:35 -0400 Subject: [PATCH 20/26] Align fields ext behavior for invalid includes --- .../pgstac/stac_fastapi/pgstac/utils.py | 4 ++-- .../pgstac/tests/resources/test_item.py | 23 ++++++++----------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py index eef1b8b73..4a0ce4c72 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/utils.py @@ -89,9 +89,9 @@ def exclude_fields(source: Dict[str, Any], fields: Optional[Set[str]]) -> None: clean_item = include_fields(item, include) # If, after including all the specified fields, there are no included properties, - # return the full item. + # return just id and collection. if not clean_item: - return Item(**item) + return Item({"id": item.get(id), "collection": item.get("collection")}) exclude_fields(clean_item, exclude) diff --git a/stac_fastapi/pgstac/tests/resources/test_item.py b/stac_fastapi/pgstac/tests/resources/test_item.py index d11a86c6c..40b4b514a 100644 --- a/stac_fastapi/pgstac/tests/resources/test_item.py +++ b/stac_fastapi/pgstac/tests/resources/test_item.py @@ -256,7 +256,6 @@ async def test_pagination(app_client, load_test_data, load_test_collection): "test-item18", ] - resp = await app_client.get(nextlink) assert resp.status_code == 200 second_page = resp.json() @@ -641,7 +640,7 @@ async def test_item_search_properties_field( assert resp.status_code == 200 second_test_item = load_test_data("test_item2.json") - second_test_item["properties"]["eo:cloud_cover"]=5 + second_test_item["properties"]["eo:cloud_cover"] = 5 resp = await app_client.post( f"/collections/{test_item['collection']}/items", json=second_test_item ) @@ -860,7 +859,6 @@ async def test_pagination_item_collection( assert resp.status_code == 200 ids.append(uid) - # Paginate through all 5 items with a limit of 1 (expecting 5 requests) page = await app_client.get( f"/collections/{test_item['collection']}/items", params={"limit": 1} @@ -1137,18 +1135,17 @@ async def test_field_extension_exclude_links( assert "links" not in resp_json["features"][0] -# async def test_field_extension_include_only_non_existant_field( -# app_client, load_test_item, load_test_collection -# ): -# """Including only a non-existant field should return the full item""" -# body = {"fields": {"include": ["non_existant_field"]}} +async def test_field_extension_include_only_non_existant_field( + app_client, load_test_item, load_test_collection +): + """Including only a non-existant field should return the full item""" + body = {"fields": {"include": ["non_existant_field"]}} -# resp = await app_client.post("/search", json=body) -# assert resp.status_code == 200 -# resp_json = resp.json() + resp = await app_client.post("/search", json=body) + assert resp.status_code == 200 + resp_json = resp.json() -# assert len(resp_json["features"][0].keys()) > 0 -# assert "properties" in resp_json["features"][0] + assert list(resp_json["features"][0].keys()) == ["id", "collection", "links"] async def test_search_intersects_and_bbox(app_client): From 6ccd15e815d252fa431aebdb116e8479ad373721 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 12 May 2022 16:57:13 -0400 Subject: [PATCH 21/26] Lint --- stac_fastapi/pgstac/stac_fastapi/pgstac/config.py | 1 + stac_fastapi/pgstac/tests/clients/test_postgres.py | 3 +-- stac_fastapi/pgstac/tests/conftest.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py b/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py index dca66761c..5312fcd08 100644 --- a/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py +++ b/stac_fastapi/pgstac/stac_fastapi/pgstac/config.py @@ -1,6 +1,7 @@ """Postgres API configuration.""" from typing import Type + from stac_fastapi.pgstac.types.base_item_cache import ( BaseItemCache, DefaultBaseItemCache, diff --git a/stac_fastapi/pgstac/tests/clients/test_postgres.py b/stac_fastapi/pgstac/tests/clients/test_postgres.py index 357228268..b337566b7 100644 --- a/stac_fastapi/pgstac/tests/clients/test_postgres.py +++ b/stac_fastapi/pgstac/tests/clients/test_postgres.py @@ -1,5 +1,6 @@ import uuid from typing import Callable + from stac_pydantic import Collection, Item # from tests.conftest import MockStarletteRequest @@ -57,8 +58,6 @@ async def test_create_item(app_client, load_test_data: Callable, load_test_colle ) assert resp.status_code == 200 - gresp = await app_client.get(f"/collections/{coll.id}/items") - post_item = Item.parse_obj(resp.json()) assert in_item.dict(exclude={"links"}) == post_item.dict(exclude={"links"}) diff --git a/stac_fastapi/pgstac/tests/conftest.py b/stac_fastapi/pgstac/tests/conftest.py index 188729499..1a88d979d 100644 --- a/stac_fastapi/pgstac/tests/conftest.py +++ b/stac_fastapi/pgstac/tests/conftest.py @@ -79,11 +79,11 @@ async def pg(): try: await conn.execute("DROP DATABASE pgstactestdb;") await conn.close() - except: + except Exception: try: await conn.execute("DROP DATABASE pgstactestdb WITH (force);") await conn.close() - except: + except Exception: pass @@ -133,7 +133,7 @@ def api_client(request, pg): @pytest.fixture(scope="function") async def app(api_client): - print('Creating app Fixture') + print("Creating app Fixture") time.time() app = api_client.app await connect_to_db(app) @@ -142,7 +142,7 @@ async def app(api_client): await close_db_connection(app) - print('Closed Pools.') + print("Closed Pools.") @pytest.fixture(scope="function") From 43586982c69e2d0f33f6c59fd8054bb725bb8d8f Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 12 May 2022 17:10:01 -0400 Subject: [PATCH 22/26] Changelog --- CHANGES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2355d3534..76561628d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,7 @@ * Bulk Transactions object Items iterator now returns the Item objects rather than the string IDs of the Item objects ([#355](https://github.com/stac-utils/stac-fastapi/issues/355)) * docker-compose now runs uvicorn with hot-reloading enabled +* Bump version of PGStac to 0.6.0 that includes support for hydrating results in the API backed ([#397](https://github.com/stac-utils/stac-fastapi/pull/397)) ### Removed @@ -27,7 +28,8 @@ * Fixes issues (and adds tests) for issues caused by regression in pgstac ([#345](https://github.com/stac-utils/stac-fastapi/issues/345) * Update error response payloads to match the API spec. ([#361](https://github.com/stac-utils/stac-fastapi/pull/361)) * Fixed stray `/` before the `#` in several extension conformance class strings ([383](https://github.com/stac-utils/stac-fastapi/pull/383)) -* SQLAlchemy backend bulk item insert now works ([#356]https://github.com/stac-utils/stac-fastapi/issues/356)) +* SQLAlchemy backend bulk item insert now works ([#356](https://github.com/stac-utils/stac-fastapi/issues/356)) +* PGStac Backend has stricter implementation of Fields Extension syntax ([#397](https://github.com/stac-utils/stac-fastapi/pull/397)) ## [2.3.0] From 9a70a029d6203edb80fdfb865dd089bc6ae9a506 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 12 May 2022 18:06:16 -0400 Subject: [PATCH 23/26] Remove psycopg install dependency --- stac_fastapi/pgstac/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stac_fastapi/pgstac/setup.py b/stac_fastapi/pgstac/setup.py index c941125db..19d957dfa 100644 --- a/stac_fastapi/pgstac/setup.py +++ b/stac_fastapi/pgstac/setup.py @@ -13,7 +13,6 @@ "stac-fastapi.types", "stac-fastapi.api", "stac-fastapi.extensions", - "psycopg[binary]", "asyncpg", "buildpg", "brotli_asgi", From f6da7b13eb31f8ea466ddee07e2bbfed69b1ec26 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Fri, 13 May 2022 11:27:10 -0400 Subject: [PATCH 24/26] Relax dependency version of pgstac to 0.6.* series --- stac_fastapi/pgstac/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/pgstac/setup.py b/stac_fastapi/pgstac/setup.py index 19d957dfa..a5d2eee41 100644 --- a/stac_fastapi/pgstac/setup.py +++ b/stac_fastapi/pgstac/setup.py @@ -17,12 +17,12 @@ "buildpg", "brotli_asgi", "pygeofilter @ git+https://github.com/geopython/pygeofilter@v0.1.1#egg=pygeofilter", - "pypgstac==0.6.0", + "pypgstac==0.6.*", ] extra_reqs = { "dev": [ - "pypgstac[psycopg]==0.6.0", + "pypgstac[psycopg]==0.6.*", "pytest", "pytest-cov", "pytest-asyncio>=0.17", From a9a3f5df935af0fe7dea88ef63628b84eb1ca879 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Fri, 13 May 2022 19:56:13 -0400 Subject: [PATCH 25/26] Update dev environment to pgstac 0.6.2 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7a2926702..c9337a593 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: database: container_name: stac-db - image: ghcr.io/stac-utils/pgstac:v0.6.0 + image: ghcr.io/stac-utils/pgstac:v0.6.2 environment: - POSTGRES_USER=username - POSTGRES_PASSWORD=password From 88d66750c44188e35301fda84e4cc46820dae048 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Fri, 13 May 2022 20:20:23 -0400 Subject: [PATCH 26/26] Changelog fix --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 76561628d..43a437221 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,7 @@ * Bulk Transactions object Items iterator now returns the Item objects rather than the string IDs of the Item objects ([#355](https://github.com/stac-utils/stac-fastapi/issues/355)) * docker-compose now runs uvicorn with hot-reloading enabled -* Bump version of PGStac to 0.6.0 that includes support for hydrating results in the API backed ([#397](https://github.com/stac-utils/stac-fastapi/pull/397)) +* Bump version of PGStac to 0.6.2 that includes support for hydrating results in the API backed ([#397](https://github.com/stac-utils/stac-fastapi/pull/397)) ### Removed