diff --git a/elasticsearch/_async/client/__init__.py b/elasticsearch/_async/client/__init__.py index 365f1b1df..4a4ea9840 100644 --- a/elasticsearch/_async/client/__init__.py +++ b/elasticsearch/_async/client/__init__.py @@ -469,7 +469,7 @@ async def bulk(self, body, index=None, doc_type=None, params=None, headers=None) body=body, ) - @query_params() + @query_params(body_params=["scroll_id"]) async def clear_scroll(self, body=None, scroll_id=None, params=None, headers=None): """ Explicitly clears the search context for a scroll. @@ -481,7 +481,7 @@ async def clear_scroll(self, body=None, scroll_id=None, params=None, headers=Non :arg scroll_id: A comma-separated list of scroll IDs to clear """ if scroll_id in SKIP_IN_PATH and body in SKIP_IN_PATH: - raise ValueError("You need to supply scroll_id or body.") + raise ValueError("Empty value passed for a required argument 'scroll_id'.") elif scroll_id and not body: body = {"scroll_id": [scroll_id]} elif scroll_id: @@ -1496,7 +1496,12 @@ async def scripts_painless_execute(self, body=None, params=None, headers=None): body=body, ) - @query_params("rest_total_hits_as_int", "scroll") + @query_params( + "rest_total_hits_as_int", + "scroll", + "scroll_id", + body_params=["scroll", "scroll_id"], + ) async def scroll(self, body=None, scroll_id=None, params=None, headers=None): """ Allows to retrieve a large numbers of results from a single search request. @@ -1506,13 +1511,13 @@ async def scroll(self, body=None, scroll_id=None, params=None, headers=None): :arg body: The scroll ID if not passed by URL or query parameter. :arg scroll_id: The scroll ID - :arg rest_total_hits_as_int: Indicates whether hits.total should - be rendered as an integer or an object in the rest search response - :arg scroll: Specify how long a consistent view of the index - should be maintained for scrolled search + :arg rest_total_hits_as_int: If true, the API response’s + hit.total property is returned as an integer. If false, the API + response’s hit.total property is returned as an object. + :arg scroll: Period to retain the search context for scrolling. """ if scroll_id in SKIP_IN_PATH and body in SKIP_IN_PATH: - raise ValueError("You need to supply scroll_id or body.") + raise ValueError("Empty value passed for a required argument 'scroll_id'.") elif scroll_id and not body: body = {"scroll_id": scroll_id} elif scroll_id: @@ -2297,7 +2302,25 @@ async def terms_enum(self, index, body=None, params=None, headers=None): body=body, ) - @query_params("exact_bounds", "extent", "grid_precision", "grid_type", "size") + @query_params( + "exact_bounds", + "extent", + "grid_precision", + "grid_type", + "size", + body_params=[ + "aggs", + "exact_bounds", + "extent", + "fields", + "grid_precision", + "grid_type", + "query", + "runtime_mappings", + "size", + "sort", + ], + ) async def search_mvt( self, index, field, zoom, x, y, body=None, params=None, headers=None ): @@ -2319,17 +2342,52 @@ async def search_mvt( :arg x: X coordinate for the vector tile to search :arg y: Y coordinate for the vector tile to search :arg body: Search request body. - :arg exact_bounds: If false, the meta layer's feature is the - bounding box of the tile. If true, the meta layer's feature is a - bounding box resulting from a `geo_bounds` aggregation. - :arg extent: Size, in pixels, of a side of the vector tile. - Default: 4096 + :arg aggs: Sub-aggregations for the geotile_grid. + + Supports the following aggregation types: + - avg + - cardinality + - max + - min + - sum + :arg exact_bounds: If false, the meta layer’s feature is the + bounding box of the tile. + If true, the meta layer’s feature is a bounding box resulting from a + geo_bounds aggregation. The aggregation runs on values that + intersect + the // tile with wrap_longitude set to false. The resulting + bounding box may be larger than the vector tile. + :arg extent: Size, in pixels, of a side of the tile. Vector + tiles are square with equal sides. + :arg fields: Fields to return in the `hits` layer. Supports + wildcards (`*`). + This parameter does not support fields with array values. Fields with + array + values may return inconsistent results. :arg grid_precision: Additional zoom levels available through - the aggs layer. Accepts 0-8. Default: 8 + the aggs layer. For example, if is 7 + and grid_precision is 8, you can zoom in up to level 15. Accepts 0-8. If + 0, results + don’t include the aggs layer. :arg grid_type: Determines the geometry type for features in the - aggs layer. Valid choices: grid, point Default: grid + aggs layer. In the aggs layer, + each feature represents a geotile_grid cell. If 'grid' each feature is a + Polygon + of the cells bounding box. If 'point' each feature is a Point that is + the centroid + of the cell. + :arg query: Query DSL used to filter documents for the search. + :arg runtime_mappings: Defines one or more runtime fields in the + search request. These fields take + precedence over mapped fields with the same name. :arg size: Maximum number of features to return in the hits - layer. Accepts 0-10000. Default: 10000 + layer. Accepts 0-10000. + If 0, results don’t include the hits layer. + :arg sort: Sorts features in the hits layer. By default, the API + calculates a bounding + box for each feature. It sorts features based on this box’s diagonal + length, + from longest to shortest. """ for param in (index, field, zoom, x, y): if param in SKIP_IN_PATH: diff --git a/elasticsearch/_async/client/__init__.pyi b/elasticsearch/_async/client/__init__.pyi index c9899d56b..e40e7a7a4 100644 --- a/elasticsearch/_async/client/__init__.pyi +++ b/elasticsearch/_async/client/__init__.pyi @@ -244,8 +244,8 @@ class AsyncElasticsearch(object): async def clear_scroll( self, *, - body: Optional[Union[Mapping[str, Any], str]] = ..., - scroll_id: Optional[Any] = ..., + body: Optional[Mapping[str, Any]] = ..., + scroll_id: Optional[Union[List[str], str]] = ..., pretty: Optional[bool] = ..., human: Optional[bool] = ..., error_trace: Optional[bool] = ..., @@ -834,9 +834,9 @@ class AsyncElasticsearch(object): self, *, body: Optional[Mapping[str, Any]] = ..., - scroll_id: Optional[Any] = ..., rest_total_hits_as_int: Optional[bool] = ..., - scroll: Optional[Any] = ..., + scroll: Optional[Union[int, str]] = ..., + scroll_id: Optional[str] = ..., pretty: Optional[bool] = ..., human: Optional[bool] = ..., error_trace: Optional[bool] = ..., @@ -1229,17 +1229,35 @@ class AsyncElasticsearch(object): async def search_mvt( self, *, - index: Any, - field: Any, - zoom: Any, - x: Any, - y: Any, + index: Union[List[str], str], + field: str, + zoom: int, + x: int, + y: int, body: Optional[Mapping[str, Any]] = ..., + aggs: Optional[Mapping[str, Mapping[str, Any]]] = ..., exact_bounds: Optional[bool] = ..., - extent: Optional[Any] = ..., - grid_precision: Optional[Any] = ..., - grid_type: Optional[Any] = ..., - size: Optional[Any] = ..., + extent: Optional[int] = ..., + fields: Optional[Union[List[str], str]] = ..., + grid_precision: Optional[int] = ..., + grid_type: Optional[Union[Literal["grid", "point"], str]] = ..., + query: Optional[Mapping[str, Any]] = ..., + runtime_mappings: Optional[Mapping[str, Mapping[str, Any]]] = ..., + size: Optional[int] = ..., + sort: Optional[ + Union[ + List[ + Union[ + Mapping[str, Any], + Union[Literal["asc", "desc", "_doc"], str], + str, + ] + ], + Union[ + Mapping[str, Any], Union[Literal["asc", "desc", "_doc"], str], str + ], + ] + ] = ..., pretty: Optional[bool] = ..., human: Optional[bool] = ..., error_trace: Optional[bool] = ..., @@ -1252,4 +1270,4 @@ class AsyncElasticsearch(object): api_key: Optional[Union[str, Tuple[str, str]]] = ..., params: Optional[MutableMapping[str, Any]] = ..., headers: Optional[MutableMapping[str, str]] = ..., - ) -> Union[Dict[str, Any], bytes]: ... + ) -> bytes: ... diff --git a/elasticsearch/_async/helpers.py b/elasticsearch/_async/helpers.py index 4b2505315..2abc39c64 100644 --- a/elasticsearch/_async/helpers.py +++ b/elasticsearch/_async/helpers.py @@ -71,7 +71,7 @@ async def _process_bulk_chunk( try: # send the actual request - resp = await client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs) + resp = await client.bulk(*args, body="\n".join(bulk_actions) + "\n", **kwargs) except TransportError as e: gen = _process_bulk_chunk_error( error=e, @@ -391,14 +391,14 @@ async def async_scan( ), ) resp = await client.scroll( - body={"scroll_id": scroll_id, "scroll": scroll}, **scroll_kwargs + scroll_id=scroll_id, scroll=scroll, **scroll_kwargs ) scroll_id = resp.get("_scroll_id") finally: if scroll_id and clear_scroll: await client.clear_scroll( - body={"scroll_id": [scroll_id]}, + scroll_id=scroll_id, **transport_kwargs, ignore=(404,), params={"__elastic_client_meta": (("h", "s"),)}, diff --git a/elasticsearch/client/__init__.py b/elasticsearch/client/__init__.py index 5483a25ad..2a61a9cc8 100644 --- a/elasticsearch/client/__init__.py +++ b/elasticsearch/client/__init__.py @@ -467,7 +467,7 @@ def bulk(self, body, index=None, doc_type=None, params=None, headers=None): body=body, ) - @query_params() + @query_params(body_params=["scroll_id"]) def clear_scroll(self, body=None, scroll_id=None, params=None, headers=None): """ Explicitly clears the search context for a scroll. @@ -479,7 +479,7 @@ def clear_scroll(self, body=None, scroll_id=None, params=None, headers=None): :arg scroll_id: A comma-separated list of scroll IDs to clear """ if scroll_id in SKIP_IN_PATH and body in SKIP_IN_PATH: - raise ValueError("You need to supply scroll_id or body.") + raise ValueError("Empty value passed for a required argument 'scroll_id'.") elif scroll_id and not body: body = {"scroll_id": [scroll_id]} elif scroll_id: @@ -1486,7 +1486,12 @@ def scripts_painless_execute(self, body=None, params=None, headers=None): body=body, ) - @query_params("rest_total_hits_as_int", "scroll") + @query_params( + "rest_total_hits_as_int", + "scroll", + "scroll_id", + body_params=["scroll", "scroll_id"], + ) def scroll(self, body=None, scroll_id=None, params=None, headers=None): """ Allows to retrieve a large numbers of results from a single search request. @@ -1496,13 +1501,13 @@ def scroll(self, body=None, scroll_id=None, params=None, headers=None): :arg body: The scroll ID if not passed by URL or query parameter. :arg scroll_id: The scroll ID - :arg rest_total_hits_as_int: Indicates whether hits.total should - be rendered as an integer or an object in the rest search response - :arg scroll: Specify how long a consistent view of the index - should be maintained for scrolled search + :arg rest_total_hits_as_int: If true, the API response’s + hit.total property is returned as an integer. If false, the API + response’s hit.total property is returned as an object. + :arg scroll: Period to retain the search context for scrolling. """ if scroll_id in SKIP_IN_PATH and body in SKIP_IN_PATH: - raise ValueError("You need to supply scroll_id or body.") + raise ValueError("Empty value passed for a required argument 'scroll_id'.") elif scroll_id and not body: body = {"scroll_id": scroll_id} elif scroll_id: @@ -2285,7 +2290,25 @@ def terms_enum(self, index, body=None, params=None, headers=None): body=body, ) - @query_params("exact_bounds", "extent", "grid_precision", "grid_type", "size") + @query_params( + "exact_bounds", + "extent", + "grid_precision", + "grid_type", + "size", + body_params=[ + "aggs", + "exact_bounds", + "extent", + "fields", + "grid_precision", + "grid_type", + "query", + "runtime_mappings", + "size", + "sort", + ], + ) def search_mvt( self, index, field, zoom, x, y, body=None, params=None, headers=None ): @@ -2307,17 +2330,52 @@ def search_mvt( :arg x: X coordinate for the vector tile to search :arg y: Y coordinate for the vector tile to search :arg body: Search request body. - :arg exact_bounds: If false, the meta layer's feature is the - bounding box of the tile. If true, the meta layer's feature is a - bounding box resulting from a `geo_bounds` aggregation. - :arg extent: Size, in pixels, of a side of the vector tile. - Default: 4096 + :arg aggs: Sub-aggregations for the geotile_grid. + + Supports the following aggregation types: + - avg + - cardinality + - max + - min + - sum + :arg exact_bounds: If false, the meta layer’s feature is the + bounding box of the tile. + If true, the meta layer’s feature is a bounding box resulting from a + geo_bounds aggregation. The aggregation runs on values that + intersect + the // tile with wrap_longitude set to false. The resulting + bounding box may be larger than the vector tile. + :arg extent: Size, in pixels, of a side of the tile. Vector + tiles are square with equal sides. + :arg fields: Fields to return in the `hits` layer. Supports + wildcards (`*`). + This parameter does not support fields with array values. Fields with + array + values may return inconsistent results. :arg grid_precision: Additional zoom levels available through - the aggs layer. Accepts 0-8. Default: 8 + the aggs layer. For example, if is 7 + and grid_precision is 8, you can zoom in up to level 15. Accepts 0-8. If + 0, results + don’t include the aggs layer. :arg grid_type: Determines the geometry type for features in the - aggs layer. Valid choices: grid, point Default: grid + aggs layer. In the aggs layer, + each feature represents a geotile_grid cell. If 'grid' each feature is a + Polygon + of the cells bounding box. If 'point' each feature is a Point that is + the centroid + of the cell. + :arg query: Query DSL used to filter documents for the search. + :arg runtime_mappings: Defines one or more runtime fields in the + search request. These fields take + precedence over mapped fields with the same name. :arg size: Maximum number of features to return in the hits - layer. Accepts 0-10000. Default: 10000 + layer. Accepts 0-10000. + If 0, results don’t include the hits layer. + :arg sort: Sorts features in the hits layer. By default, the API + calculates a bounding + box for each feature. It sorts features based on this box’s diagonal + length, + from longest to shortest. """ for param in (index, field, zoom, x, y): if param in SKIP_IN_PATH: diff --git a/elasticsearch/client/__init__.pyi b/elasticsearch/client/__init__.pyi index 1b9836219..f6f1cb32a 100644 --- a/elasticsearch/client/__init__.pyi +++ b/elasticsearch/client/__init__.pyi @@ -244,8 +244,8 @@ class Elasticsearch(object): def clear_scroll( self, *, - body: Optional[Union[Mapping[str, Any], str]] = ..., - scroll_id: Optional[Any] = ..., + body: Optional[Mapping[str, Any]] = ..., + scroll_id: Optional[Union[List[str], str]] = ..., pretty: Optional[bool] = ..., human: Optional[bool] = ..., error_trace: Optional[bool] = ..., @@ -834,9 +834,9 @@ class Elasticsearch(object): self, *, body: Optional[Mapping[str, Any]] = ..., - scroll_id: Optional[Any] = ..., rest_total_hits_as_int: Optional[bool] = ..., - scroll: Optional[Any] = ..., + scroll: Optional[Union[int, str]] = ..., + scroll_id: Optional[str] = ..., pretty: Optional[bool] = ..., human: Optional[bool] = ..., error_trace: Optional[bool] = ..., @@ -1229,17 +1229,35 @@ class Elasticsearch(object): def search_mvt( self, *, - index: Any, - field: Any, - zoom: Any, - x: Any, - y: Any, + index: Union[List[str], str], + field: str, + zoom: int, + x: int, + y: int, body: Optional[Mapping[str, Any]] = ..., + aggs: Optional[Mapping[str, Mapping[str, Any]]] = ..., exact_bounds: Optional[bool] = ..., - extent: Optional[Any] = ..., - grid_precision: Optional[Any] = ..., - grid_type: Optional[Any] = ..., - size: Optional[Any] = ..., + extent: Optional[int] = ..., + fields: Optional[Union[List[str], str]] = ..., + grid_precision: Optional[int] = ..., + grid_type: Optional[Union[Literal["grid", "point"], str]] = ..., + query: Optional[Mapping[str, Any]] = ..., + runtime_mappings: Optional[Mapping[str, Mapping[str, Any]]] = ..., + size: Optional[int] = ..., + sort: Optional[ + Union[ + List[ + Union[ + Mapping[str, Any], + Union[Literal["asc", "desc", "_doc"], str], + str, + ] + ], + Union[ + Mapping[str, Any], Union[Literal["asc", "desc", "_doc"], str], str + ], + ] + ] = ..., pretty: Optional[bool] = ..., human: Optional[bool] = ..., error_trace: Optional[bool] = ..., @@ -1252,4 +1270,4 @@ class Elasticsearch(object): api_key: Optional[Union[str, Tuple[str, str]]] = ..., params: Optional[MutableMapping[str, Any]] = ..., headers: Optional[MutableMapping[str, str]] = ..., - ) -> Union[Dict[str, Any], bytes]: ... + ) -> bytes: ... diff --git a/elasticsearch/helpers/actions.py b/elasticsearch/helpers/actions.py index e6e7906be..063eead9b 100644 --- a/elasticsearch/helpers/actions.py +++ b/elasticsearch/helpers/actions.py @@ -237,7 +237,7 @@ def _process_bulk_chunk( try: # send the actual request - resp = client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs) + resp = client.bulk(*args, body="\n".join(bulk_actions) + "\n", **kwargs) except TransportError as e: gen = _process_bulk_chunk_error( error=e, @@ -602,15 +602,13 @@ def scan( shards_total, ), ) - resp = client.scroll( - body={"scroll_id": scroll_id, "scroll": scroll}, **scroll_kwargs - ) + resp = client.scroll(scroll_id=scroll_id, scroll=scroll, **scroll_kwargs) scroll_id = resp.get("_scroll_id") finally: if scroll_id and clear_scroll: client.clear_scroll( - body={"scroll_id": [scroll_id]}, + scroll_id=scroll_id, ignore=(404,), params={"__elastic_client_meta": (("h", "s"),)}, **transport_kwargs diff --git a/test_elasticsearch/test_async/test_server/test_helpers.py b/test_elasticsearch/test_async/test_server/test_helpers.py index ce5abf223..4a01a51c0 100644 --- a/test_elasticsearch/test_async/test_server/test_helpers.py +++ b/test_elasticsearch/test_async/test_server/test_helpers.py @@ -68,11 +68,13 @@ async def test_actions_remain_unchanged(self, async_client): async def test_all_documents_get_inserted(self, async_client): docs = [{"answer": x, "_id": x} for x in range(100)] - async for ok, item in helpers.async_streaming_bulk( - async_client, docs, index="test-index", refresh=True - ): - assert ok + with warnings.catch_warnings(record=True) as w: + async for ok, item in helpers.async_streaming_bulk( + async_client, docs, index="test-index", refresh=True + ): + assert ok + assert w == [] assert 100 == (await async_client.count(index="test-index"))["count"] assert {"answer": 42} == (await async_client.get(index="test-index", id=42))[ "_source" diff --git a/test_elasticsearch/test_async/test_server/test_mapbox_vector_tile.py b/test_elasticsearch/test_async/test_server/test_mapbox_vector_tile.py index 4398c4e29..5a20f9315 100644 --- a/test_elasticsearch/test_async/test_server/test_mapbox_vector_tile.py +++ b/test_elasticsearch/test_async/test_server/test_mapbox_vector_tile.py @@ -164,15 +164,13 @@ async def test_mapbox_vector_tile_response(mvt_setup): x=4207, y=2692, field="location", - body={ - "grid_precision": 2, - "fields": ["name", "price"], - "query": {"term": {"included": True}}, - "aggs": { - "min_price": {"min": {"field": "price"}}, - "max_price": {"max": {"field": "price"}}, - "avg_price": {"avg": {"field": "price"}}, - }, + grid_precision=2, + fields=["name", "price"], + query={"term": {"included": True}}, + aggs={ + "min_price": {"min": {"field": "price"}}, + "max_price": {"max": {"field": "price"}}, + "avg_price": {"avg": {"field": "price"}}, }, ) assert isinstance(resp, bytes) diff --git a/test_elasticsearch/test_client/test_overrides.py b/test_elasticsearch/test_client/test_overrides.py index 689a3b4c9..6290b4966 100644 --- a/test_elasticsearch/test_client/test_overrides.py +++ b/test_elasticsearch/test_client/test_overrides.py @@ -162,3 +162,25 @@ def test_indices_put_mapping(self): def test_tasks_get(self): with pytest.warns(DeprecationWarning): self.client.tasks.get() + + def test_scroll(self): + self.client.scroll( + scroll_id="scroll-id", scroll="5m", rest_total_hits_as_int=True + ) + calls = self.client.transport.calls + assert calls == { + ("POST", "/_search/scroll"): [ + ( + {"rest_total_hits_as_int": b"true"}, + {}, + {"scroll": "5m", "scroll_id": "scroll-id"}, + ) + ] + } + + def test_clear_scroll(self): + self.client.clear_scroll(scroll_id="scroll-id") + calls = self.client.transport.calls + assert calls == { + ("DELETE", "/_search/scroll"): [({}, {}, {"scroll_id": "scroll-id"})] + } diff --git a/test_elasticsearch/test_server/test_helpers.py b/test_elasticsearch/test_server/test_helpers.py index 9861ffa4d..49d4690f7 100644 --- a/test_elasticsearch/test_server/test_helpers.py +++ b/test_elasticsearch/test_server/test_helpers.py @@ -73,11 +73,13 @@ def test_actions_remain_unchanged(self): def test_all_documents_get_inserted(self): docs = [{"answer": x, "_id": x} for x in range(100)] - for ok, item in helpers.streaming_bulk( - self.client, docs, index="test-index", refresh=True - ): - self.assertTrue(ok) + with warnings.catch_warnings(record=True) as w: + for ok, item in helpers.streaming_bulk( + self.client, docs, index="test-index", refresh=True + ): + self.assertTrue(ok) + self.assertEqual(w, []) self.assertEqual(100, self.client.count(index="test-index")["count"]) self.assertEqual( {"answer": 42}, self.client.get(index="test-index", id=42)["_source"] diff --git a/test_elasticsearch/test_server/test_mapbox_vector_tile.py b/test_elasticsearch/test_server/test_mapbox_vector_tile.py index aeee0db2b..036b2fb47 100644 --- a/test_elasticsearch/test_server/test_mapbox_vector_tile.py +++ b/test_elasticsearch/test_server/test_mapbox_vector_tile.py @@ -174,15 +174,13 @@ def test_mapbox_vector_tile_response(mvt_setup, connection_class): x=4207, y=2692, field="location", - body={ - "grid_precision": 2, - "fields": ["name", "price"], - "query": {"term": {"included": True}}, - "aggs": { - "min_price": {"min": {"field": "price"}}, - "max_price": {"max": {"field": "price"}}, - "avg_price": {"avg": {"field": "price"}}, - }, + grid_precision=2, + fields=["name", "price"], + query={"term": {"included": True}}, + aggs={ + "min_price": {"min": {"field": "price"}}, + "max_price": {"max": {"field": "price"}}, + "avg_price": {"avg": {"field": "price"}}, }, ) assert isinstance(resp, bytes) diff --git a/test_elasticsearch/test_server/test_rest_api_spec.py b/test_elasticsearch/test_server/test_rest_api_spec.py index dce8d49f6..d8e31a2b2 100644 --- a/test_elasticsearch/test_server/test_rest_api_spec.py +++ b/test_elasticsearch/test_server/test_rest_api_spec.py @@ -114,6 +114,9 @@ APIS_WITH_BODY_FIELDS = { "search", + "search_mvt", + "scroll", + "clear_scroll", "update", "indices.create", }