diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/Pipeline.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/Pipeline.java index 3405a5408..c8cb17cc0 100644 --- a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/Pipeline.java +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/Pipeline.java @@ -42,10 +42,16 @@ import com.google.cloud.firestore.pipeline.stages.Limit; import com.google.cloud.firestore.pipeline.stages.Offset; import com.google.cloud.firestore.pipeline.stages.RemoveFields; +import com.google.cloud.firestore.pipeline.stages.Replace; +import com.google.cloud.firestore.pipeline.stages.Sample; +import com.google.cloud.firestore.pipeline.stages.SampleOptions; import com.google.cloud.firestore.pipeline.stages.Select; import com.google.cloud.firestore.pipeline.stages.Sort; import com.google.cloud.firestore.pipeline.stages.Stage; import com.google.cloud.firestore.pipeline.stages.StageUtils; +import com.google.cloud.firestore.pipeline.stages.Union; +import com.google.cloud.firestore.pipeline.stages.Unnest; +import com.google.cloud.firestore.pipeline.stages.UnnestOptions; import com.google.cloud.firestore.pipeline.stages.Where; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableList; @@ -54,6 +60,7 @@ import com.google.firestore.v1.ExecutePipelineRequest; import com.google.firestore.v1.ExecutePipelineResponse; import com.google.firestore.v1.StructuredPipeline; +import com.google.firestore.v1.Value; import com.google.protobuf.ByteString; import io.opencensus.trace.AttributeValue; import io.opencensus.trace.Tracing; @@ -586,6 +593,297 @@ public Pipeline sort(Ordering... orders) { return append(new Sort(orders)); } + /** + * Fully overwrites all fields in a document with those coming from a nested map. + * + *

This stage allows you to emit a map value as a document. Each key of the map becomes a field + * on the document that contains the corresponding value. + * + *

Example: + * + *

{@code
+   * // Input.
+   * // {
+   * //  "name": "John Doe Jr.",
+   * //  "parents": {
+   * //    "father": "John Doe Sr.",
+   * //    "mother": "Jane Doe"
+   * // }
+   *
+   * // Emit parents as document.
+   * firestore.pipeline().collection("people").replace("parents");
+   *
+   * // Output
+   * // {
+   * //  "father": "John Doe Sr.",
+   * //  "mother": "Jane Doe"
+   * // }
+   * }
+ * + * @param fieldName The name of the field containing the nested map. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + @BetaApi + public Pipeline replace(String fieldName) { + return replace(Field.of(fieldName)); + } + + /** + * Fully overwrites all fields in a document with those coming from a nested map. + * + *

This stage allows you to emit a map value as a document. Each key of the map becomes a field + * on the document that contains the corresponding value. + * + *

Example: + * + *

{@code
+   * // Input.
+   * // {
+   * //  "name": "John Doe Jr.",
+   * //  "parents": {
+   * //    "father": "John Doe Sr.",
+   * //    "mother": "Jane Doe"
+   * // }
+   *
+   * // Emit parents as document.
+   * firestore.pipeline().collection("people").replace(Field.of("parents"));
+   *
+   * // Output
+   * // {
+   * //  "father": "John Doe Sr.",
+   * //  "mother": "Jane Doe"
+   * // }
+   * }
+ * + * @param field The {@link Selectable} field containing the nested map. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + @BetaApi + public Pipeline replace(Selectable field) { + return append(new Replace(field)); + } + + /** + * Performs a pseudo-random sampling of the documents from the previous stage. + * + *

This stage will filter documents pseudo-randomly. The 'limit' parameter specifies the number + * of documents to emit from this stage, but if there are fewer documents from previous stage than + * the 'limit' parameter, then no filtering will occur and all documents will pass through. + * + *

Example: + * + *

{@code
+   * // Sample 10 books, if available.
+   * firestore.pipeline().collection("books")
+   *     .sample(10);
+   * }
+ * + * @param limit The number of documents to emit, if possible. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + @BetaApi + public Pipeline sample(int limit) { + SampleOptions options = SampleOptions.docLimit(limit); + return sample(options); + } + + /** + * Performs a pseudo-random sampling of the documents from the previous stage. + * + *

This stage will filter documents pseudo-randomly. The 'options' parameter specifies how + * sampling will be performed. See {@code SampleOptions} for more information. + * + *

Examples: + * + *

{@code
+   * // Sample 10 books, if available.
+   * firestore.pipeline().collection("books")
+   *     .sample(SampleOptions.docLimit(10));
+   *
+   * // Sample 50% of books.
+   * firestore.pipeline().collection("books")
+   *     .sample(SampleOptions.percentage(0.5));
+   * }
+ * + * @param options The {@code SampleOptions} specifies how sampling is performed. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + @BetaApi + public Pipeline sample(SampleOptions options) { + return append(new Sample(options)); + } + + /** + * Performs union of all documents from two pipelines, including duplicates. + * + *

This stage will pass through documents from previous stage, and also pass through documents + * from previous stage of the `other` {@code Pipeline} given in parameter. The order of documents + * emitted from this stage is undefined. + * + *

Example: + * + *

{@code
+   * // Emit documents from books collection and magazines collection.
+   * firestore.pipeline().collection("books")
+   *     .union(firestore.pipeline().collection("magazines"));
+   * }
+ * + * @param other The other {@code Pipeline} that is part of union. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + @BetaApi + public Pipeline union(Pipeline other) { + return append(new Union(other)); + } + + /** + * Produces a document for each element in array found in previous stage document. + * + *

For each previous stage document, this stage will emit zero or more augmented documents. The + * input array found in the previous stage document field specified by the `fieldName` parameter, + * will for each input array element produce an augmented document. The input array element will + * augment the previous stage document by replacing the field specified by `fieldName` parameter + * with the element value. + * + *

In other words, the field containing the input array will be removed from the augmented + * document and replaced by the corresponding array element. + * + *

Example: + * + *

{@code
+   * // Input:
+   * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": [ "comedy", "space", "adventure" ], ... }
+   *
+   * // Emit a book document for each tag of the book.
+   * firestore.pipeline().collection("books")
+   *     .unnest("tags");
+   *
+   * // Output:
+   * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": "comedy", ... }
+   * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": "space", ... }
+   * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": "adventure", ... }
+   * }
+ * + * @param fieldName The name of the field containing the array. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + @BetaApi + public Pipeline unnest(String fieldName) { + // return unnest(Field.of(fieldName)); + return append(new Unnest(Field.of(fieldName))); + } + + // /** + // * Produces a document for each element in array found in previous stage document. + // * + // *

For each previous stage document, this stage will emit zero or more augmented documents. + // * The input array found in the specified by {@code Selectable} expression parameter, will for + // * each input array element produce an augmented document. The input array element will augment + // * the previous stage document by assigning the {@code Selectable} alias the element value. + // * + // *

Example: + // * + // *

{@code
+  //  * // Input:
+  //  * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": [ "comedy", "space",
+  // "adventure" ], ... }
+  //  *
+  //  * // Emit a book document for each tag of the book.
+  //  * firestore.pipeline().collection("books")
+  //  *     .unnest(Field.of("tags").as("tag"));
+  //  *
+  //  * // Output:
+  //  * // { "title": "The Hitchhiker's Guide to the Galaxy", "tag": "comedy", "tags": [ "comedy",
+  // "space", "adventure" ], ... }
+  //  * // { "title": "The Hitchhiker's Guide to the Galaxy", "tag": "space", "tags": [ "comedy",
+  // "space", "adventure" ], ... }
+  //  * // { "title": "The Hitchhiker's Guide to the Galaxy", "tag": "adventure", "tags": [
+  // "comedy", "space", "adventure" ], ... }
+  //  * }
+ // * + // * @param field The expression that evaluates to the input array. + // * @return A new {@code Pipeline} object with this stage appended to the stage list. + // */ + // @BetaApi + // public Pipeline unnest(Selectable field) { + // return append(new Unnest(field)); + // } + + /** + * Produces a document for each element in array found in previous stage document. + * + *

For each previous stage document, this stage will emit zero or more augmented documents. The + * input array found in the previous stage document field specified by the `fieldName` parameter, + * will for each input array element produce an augmented document. The input array element will + * augment the previous stage document by replacing the field specified by `fieldName` parameter + * with the element value. + * + *

In other words, the field containing the input array will be removed from the augmented + * document and replaced by the corresponding array element. + * + *

Example: + * + *

{@code
+   * // Input:
+   * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": [ "comedy", "space", "adventure" ], ... }
+   *
+   * // Emit a book document for each tag of the book.
+   * firestore.pipeline().collection("books")
+   *     .unnest("tags", UnnestOptions.indexField("tagIndex"));
+   *
+   * // Output:
+   * // { "title": "The Hitchhiker's Guide to the Galaxy", "tagIndex": 0, "tags": "comedy", ... }
+   * // { "title": "The Hitchhiker's Guide to the Galaxy", "tagIndex": 1, "tags": "space", ... }
+   * // { "title": "The Hitchhiker's Guide to the Galaxy", "tagIndex": 2, "tags": "adventure", ... }
+   * }
+ * + * @param fieldName The name of the field containing the array. + * @param options The {@code UnnestOptions} options. + * @return A new {@code Pipeline} object with this stage appended to the stage list. + */ + @BetaApi + public Pipeline unnest(String fieldName, UnnestOptions options) { + // return unnest(Field.of(fieldName), options); + return append(new Unnest(Field.of(fieldName), options)); + } + + // /** + // * Produces a document for each element in array found in previous stage document. + // * + // *

For each previous stage document, this stage will emit zero or more augmented documents. + // * The input array found in the specified by {@code Selectable} expression parameter, will for + // * each input array element produce an augmented document. The input array element will augment + // * the previous stage document by assigning the {@code Selectable} alias the element value. + // * + // *

Example: + // * + // *

{@code
+  //  * // Input:
+  //  * // { "title": "The Hitchhiker's Guide to the Galaxy", "tags": [ "comedy", "space",
+  // "adventure" ], ... }
+  //  *
+  //  * // Emit a book document for each tag of the book.
+  //  * firestore.pipeline().collection("books")
+  //  *     .unnest(Field.of("tags").as("tag"), UnnestOptions.indexField("tagIndex"));
+  //  *
+  //  * // Output:
+  //  * // { "title": "The Hitchhiker's Guide to the Galaxy", "tagIndex": 0, "tag": "comedy",
+  // "tags": [ "comedy", "space", "adventure" ], ... }
+  //  * // { "title": "The Hitchhiker's Guide to the Galaxy", "tagIndex": 1, "tag": "space", "tags":
+  // [ "comedy", "space", "adventure" ], ... }
+  //  * // { "title": "The Hitchhiker's Guide to the Galaxy", "tagIndex": 2, "tag": "adventure",
+  // "tags": [ "comedy", "space", "adventure" ], ... }
+  //  * }
+ // * + // * @param field The expression that evaluates to the input array. + // * @param options The {@code UnnestOptions} options. + // * @return A new {@code Pipeline} object with this stage appended to the stage list. + // */ + // @BetaApi + // public Pipeline unnest(Selectable field, UnnestOptions options) { + // return append(new Unnest(field, options)); + // } + /** * Adds a generic stage to the pipeline. * @@ -648,7 +946,7 @@ public Pipeline genericStage(String name, List params) { */ @BetaApi public ApiFuture> execute() { - return execute(null, null); + return execute((ByteString) null, (com.google.protobuf.Timestamp) null); } /** @@ -701,6 +999,22 @@ public void execute(ApiStreamObserver observer) { executeInternal(null, null, observer); } + // @BetaApi + // public void execute(ApiStreamObserver observer, PipelineOptions options) { + // throw new RuntimeException("Not Implemented"); + // } + // + // @BetaApi + // public ApiFuture> explain() { + // throw new RuntimeException("Not Implemented"); + // } + // + // @BetaApi + // public void explain(ApiStreamObserver observer, PipelineExplainOptions options) + // { + // throw new RuntimeException("Not Implemented"); + // } + ApiFuture> execute( @Nullable final ByteString transactionId, @Nullable com.google.protobuf.Timestamp readTime) { SettableApiFuture> futureResult = SettableApiFuture.create(); @@ -767,12 +1081,18 @@ public void onError(Throwable t) { }); } + @InternalApi private com.google.firestore.v1.Pipeline toProto() { return com.google.firestore.v1.Pipeline.newBuilder() .addAllStages(stages.transform(StageUtils::toStageProto)) .build(); } + @InternalApi + public com.google.firestore.v1.Value toProtoValue() { + return Value.newBuilder().setPipelineValue(toProto()).build(); + } + private void pipelineInternalStream( ExecutePipelineRequest request, PipelineResultObserver resultObserver) { ResponseObserver observer = diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/PipelineUtils.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/PipelineUtils.java index edf2d994a..2b12c6e90 100644 --- a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/PipelineUtils.java +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/PipelineUtils.java @@ -34,7 +34,6 @@ import com.google.cloud.firestore.pipeline.expressions.Expr; import com.google.cloud.firestore.pipeline.expressions.ExprWithAlias; import com.google.cloud.firestore.pipeline.expressions.Field; -import com.google.cloud.firestore.pipeline.expressions.Fields; import com.google.cloud.firestore.pipeline.expressions.FilterCondition; import com.google.cloud.firestore.pipeline.expressions.Selectable; import com.google.common.collect.Lists; @@ -173,11 +172,6 @@ public static Map selectablesToMap(Selectable... selectables) { } else if (proj instanceof AccumulatorTarget) { AccumulatorTarget aggregatorProj = (AccumulatorTarget) proj; projMap.put(aggregatorProj.getFieldName(), aggregatorProj.getAccumulator()); - } else if (proj instanceof Fields) { - Fields fieldsProj = (Fields) proj; - if (fieldsProj.getFields() != null) { - fieldsProj.getFields().forEach(f -> projMap.put(f.getPath().getEncodedPath(), f)); - } } else if (proj instanceof ExprWithAlias) { ExprWithAlias exprWithAlias = (ExprWithAlias) proj; projMap.put(exprWithAlias.getAlias(), exprWithAlias.getExpr()); diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/Expr.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/Expr.java index 39f21c3d9..20e8825bb 100644 --- a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/Expr.java +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/Expr.java @@ -1902,6 +1902,6 @@ default Ordering descending() { */ @BetaApi default Selectable as(String alias) { - return new ExprWithAlias(this, alias); + return new ExprWithAlias<>(this, alias); } } diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/ExprWithAlias.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/ExprWithAlias.java index d2384c675..c248937c3 100644 --- a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/ExprWithAlias.java +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/ExprWithAlias.java @@ -19,7 +19,7 @@ import com.google.api.core.InternalApi; @InternalApi -public final class ExprWithAlias implements Selectable { +public final class ExprWithAlias implements Expr, Selectable { private final String alias; private final T expr; @@ -39,4 +39,9 @@ public String getAlias() { public T getExpr() { return expr; } + + @Override + public Selectable as(String alias) { + return new ExprWithAlias<>(this.expr, alias); + } } diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/Fields.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/Fields.java deleted file mode 100644 index 2d3c19ef0..000000000 --- a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/Fields.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.firestore.pipeline.expressions; - -import com.google.api.core.BetaApi; -import com.google.api.core.InternalApi; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Represents a selection of multiple {@link Field} instances. - * - *

This class is used to conveniently specify multiple fields for operations like selecting - * fields in a pipeline. - * - *

Example: - * - *

{@code
- * // Select the 'name', 'email', and 'age' fields
- * Fields selectedFields = Fields.of("name", "email", "age");
- *
- * firestore.pipeline().collection("users")
- *     .select(selectedFields)
- *     .execute();
- * }
- */ -@BetaApi -public final class Fields implements Expr, Selectable { - private final List fields; - - private Fields(List fs) { - this.fields = fs; - } - - /** - * Creates a {@code Fields} instance containing the specified fields. - * - * @param f1 The first field to include. - * @param f Additional fields to include. - * @return A new {@code Fields} instance containing the specified fields. - */ - @BetaApi - public static Fields of(String f1, String... f) { - List fields = Arrays.stream(f).map(Field::of).collect(Collectors.toList()); - fields.add(0, Field.of(f1)); // Add f1 at the beginning - return new Fields(fields); - } - - /** - * Creates a {@code Fields} instance representing a selection of all fields. - * - *

This is equivalent to not specifying any fields in a select operation, resulting in all - * fields being included in the output. - * - * @return A new {@code Fields} instance representing all fields. - */ - @BetaApi - public static Fields ofAll() { - return new Fields(Collections.singletonList(Field.of(""))); - } - - /** - * Returns the list of {@link Field} instances contained in this {@code Fields} object. - * - * @return The list of fields. - */ - @InternalApi - public List getFields() { - return fields; - } -} diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/FunctionUtils.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/FunctionUtils.java index c425095ed..4f1e85cec 100644 --- a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/FunctionUtils.java +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/expressions/FunctionUtils.java @@ -29,6 +29,8 @@ public final class FunctionUtils { public static Value exprToValue(Expr expr) { if (expr == null) { return Constant.of((String) null).toProto(); + } else if (expr instanceof ExprWithAlias) { + return exprToValue(((ExprWithAlias) expr).getExpr()); } else if (expr instanceof Constant) { return ((Constant) expr).toProto(); } else if (expr instanceof Field) { diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/AbstractStage.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/AbstractStage.java new file mode 100644 index 000000000..0d46ab7e2 --- /dev/null +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/AbstractStage.java @@ -0,0 +1,30 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.firestore.pipeline.stages; + +import com.google.firestore.v1.Value; + +/** + * Parent to all stages. + * + *

This class is package private to prevent public access to these methods. Methods in this class + * support internal polymorphic processing, that would otherwise require conditional processing + * based on type. This should eliminate `instanceof` usage with respect to `Stage` implementations. + */ +abstract class AbstractStage implements Stage { + abstract Value getProtoArgs(); +} diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Replace.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Replace.java new file mode 100644 index 000000000..3d9ab9cfc --- /dev/null +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Replace.java @@ -0,0 +1,66 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.firestore.pipeline.stages; + +import static com.google.cloud.firestore.PipelineUtils.encodeValue; + +import com.google.api.core.InternalApi; +import com.google.cloud.firestore.pipeline.expressions.Selectable; +import com.google.firestore.v1.MapValue; +import com.google.firestore.v1.Value; +import javax.annotation.Nonnull; + +public class Replace extends AbstractStage { + + private static final String name = "unnest"; + private final Selectable field; + private final Mode mode; + + public enum Mode { + FULL_REPLACE(Value.newBuilder().setStringValue("full_replace").build()), + MERGE_PREFER_NEXT(Value.newBuilder().setStringValue("merge_prefer_nest").build()), + MERGE_PREFER_PARENT(Value.newBuilder().setStringValue("merge_prefer_parent").build()); + + public final Value value; + + Mode(Value value) { + this.value = value; + } + } + + public Replace(@Nonnull Selectable field) { + this(field, Mode.FULL_REPLACE); + } + + public Replace(@Nonnull Selectable field, @Nonnull Mode mode) { + this.field = field; + this.mode = mode; + } + + @InternalApi + public String getName() { + return name; + } + + @Override + Value getProtoArgs() { + MapValue.Builder builder = MapValue.newBuilder(); + builder.putFields("map", encodeValue(field)); + builder.putFields("mode", mode.value); + return Value.newBuilder().setMapValue(builder).build(); + } +} diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Sample.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Sample.java new file mode 100644 index 000000000..739badcb1 --- /dev/null +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Sample.java @@ -0,0 +1,46 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.firestore.pipeline.stages; + +import com.google.api.core.InternalApi; +import com.google.firestore.v1.Value; + +public final class Sample extends AbstractStage { + + private static final String name = "sample"; + private final SampleOptions options; + + @InternalApi + public Sample(SampleOptions options) { + this.options = options; + } + + @InternalApi + public String getName() { + return name; + } + + @InternalApi + public SampleOptions getOptions() { + return options; + } + + @Override + Value getProtoArgs() { + return options.getProtoArgs(); + } +} diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/SampleOptions.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/SampleOptions.java new file mode 100644 index 000000000..607675c83 --- /dev/null +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/SampleOptions.java @@ -0,0 +1,62 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.firestore.pipeline.stages; + +import static com.google.cloud.firestore.PipelineUtils.encodeValue; + +import com.google.firestore.v1.MapValue; +import com.google.firestore.v1.Value; + +public class SampleOptions { + + private final Number n; + private final Mode mode; + + private SampleOptions(Number n, Mode mode) { + this.n = n; + this.mode = mode; + } + + public enum Mode { + DOCUMENTS(Value.newBuilder().setStringValue("documents").build()), + PERCENT(Value.newBuilder().setStringValue("percent").build()); + + public final Value value; + + Mode(Value value) { + this.value = value; + } + } + + public static SampleOptions percentage(double percentage) { + return new SampleOptions(percentage, Mode.PERCENT); + } + + public static SampleOptions docLimit(int limit) { + return new SampleOptions(limit, Mode.DOCUMENTS); + } + + Value getProtoArgs() { + return Value.newBuilder() + .setMapValue( + MapValue.newBuilder() + .putFields("n", encodeValue(n)) + .putFields("mode", mode.value) + .build()) + .build(); + } +} diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/StageUtils.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/StageUtils.java index e342e5954..e65eacb6c 100644 --- a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/StageUtils.java +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/StageUtils.java @@ -29,7 +29,13 @@ public final class StageUtils { @InternalApi public static com.google.firestore.v1.Pipeline.Stage toStageProto(Stage stage) { - if (stage instanceof Collection) { + if (stage instanceof AbstractStage) { + AbstractStage abstractStage = (AbstractStage) stage; + return com.google.firestore.v1.Pipeline.Stage.newBuilder() + .setName(abstractStage.getName()) + .addArgs(abstractStage.getProtoArgs()) + .build(); + } else if (stage instanceof Collection) { Collection collectionStage = (Collection) stage; return com.google.firestore.v1.Pipeline.Stage.newBuilder() .setName(collectionStage.getName()) diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Union.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Union.java new file mode 100644 index 000000000..4eb25da6f --- /dev/null +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Union.java @@ -0,0 +1,44 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.firestore.pipeline.stages; + +import com.google.api.core.InternalApi; +import com.google.cloud.firestore.Pipeline; +import com.google.firestore.v1.MapValue; +import com.google.firestore.v1.Value; + +public class Union extends AbstractStage { + + private static final String name = "union"; + private final Pipeline other; + + public Union(Pipeline other) { + this.other = other; + } + + @InternalApi + public String getName() { + return name; + } + + @Override + Value getProtoArgs() { + return Value.newBuilder() + .setMapValue(MapValue.newBuilder().putFields("other", other.toProtoValue())) + .build(); + } +} diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Unnest.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Unnest.java new file mode 100644 index 000000000..71f4e03cd --- /dev/null +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/Unnest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.firestore.pipeline.stages; + +import static com.google.cloud.firestore.PipelineUtils.encodeValue; + +import com.google.api.core.InternalApi; +import com.google.cloud.firestore.pipeline.expressions.Field; +import com.google.firestore.v1.MapValue; +import com.google.firestore.v1.Value; +import javax.annotation.Nonnull; + +public class Unnest extends AbstractStage { + + private static final String name = "unnest"; + private final Field field; + private final UnnestOptions options; + + public Unnest(Field field) { + this.field = field; + this.options = null; + } + + public Unnest(@Nonnull Field field, @Nonnull UnnestOptions options) { + this.field = field; + this.options = options; + } + + @InternalApi + public String getName() { + return name; + } + + @Override + Value getProtoArgs() { + MapValue.Builder builder = MapValue.newBuilder(); + builder.putFields("field", encodeValue(field)); + if (options != null) { + builder.putFields("index_field", encodeValue(options.indexField)); + } + return Value.newBuilder().setMapValue(builder).build(); + } +} diff --git a/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/UnnestOptions.java b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/UnnestOptions.java new file mode 100644 index 000000000..3ca12d792 --- /dev/null +++ b/google-cloud-firestore/src/main/java/com/google/cloud/firestore/pipeline/stages/UnnestOptions.java @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.firestore.pipeline.stages; + +import javax.annotation.Nonnull; + +public class UnnestOptions { + + final String indexField; + + public static UnnestOptions indexField(@Nonnull String indexField) { + return new UnnestOptions(indexField); + } + + private UnnestOptions(String indexField) { + this.indexField = indexField; + } +} diff --git a/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITPipelineTest.java b/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITPipelineTest.java index b08683aab..550d04fa1 100644 --- a/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITPipelineTest.java +++ b/google-cloud-firestore/src/test/java/com/google/cloud/firestore/it/ITPipelineTest.java @@ -49,7 +49,12 @@ import com.google.cloud.firestore.pipeline.expressions.Field; import com.google.cloud.firestore.pipeline.expressions.Function; import com.google.cloud.firestore.pipeline.stages.Aggregate; +import com.google.cloud.firestore.pipeline.stages.SampleOptions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -64,6 +69,7 @@ @RunWith(JUnit4.class) public class ITPipelineTest extends ITBaseTest { private CollectionReference collection; + private Map> bookDocs; public CollectionReference testCollectionWithDocs(Map> docs) throws ExecutionException, InterruptedException, TimeoutException { @@ -84,10 +90,10 @@ public void setup() throws Exception { return; } - Map> bookDocs = - map( + bookDocs = + ImmutableMap.of( "book1", - map( + ImmutableMap.of( "title", "The Hitchhiker's Guide to the Galaxy", "author", @@ -99,11 +105,11 @@ public void setup() throws Exception { "rating", 4.2, "tags", - Lists.newArrayList("comedy", "space", "adventure"), + ImmutableList.of("comedy", "space", "adventure"), "awards", - map("hugo", true, "nebula", false)), + ImmutableMap.of("hugo", true, "nebula", false)), "book2", - map( + ImmutableMap.of( "title", "Pride and Prejudice", "author", @@ -115,11 +121,11 @@ public void setup() throws Exception { "rating", 4.5, "tags", - Lists.newArrayList("classic", "social commentary", "love"), + ImmutableList.of("classic", "social commentary", "love"), "awards", - map("none", true)), + ImmutableMap.of("none", true)), "book3", - map( + ImmutableMap.of( "title", "One Hundred Years of Solitude", "author", @@ -131,11 +137,11 @@ public void setup() throws Exception { "rating", 4.3, "tags", - Lists.newArrayList("family", "history", "fantasy"), + ImmutableList.of("family", "history", "fantasy"), "awards", - map("nobel", true, "nebula", false)), + ImmutableMap.of("nobel", true, "nebula", false)), "book4", - map( + ImmutableMap.of( "title", "The Lord of the Rings", "author", @@ -147,11 +153,11 @@ public void setup() throws Exception { "rating", 4.7, "tags", - Lists.newArrayList("adventure", "magic", "epic"), + ImmutableList.of("adventure", "magic", "epic"), "awards", - map("hugo", false, "nebula", false)), + ImmutableMap.of("hugo", false, "nebula", false)), "book5", - map( + ImmutableMap.of( "title", "The Handmaid's Tale", "author", @@ -163,11 +169,11 @@ public void setup() throws Exception { "rating", 4.1, "tags", - Lists.newArrayList("feminism", "totalitarianism", "resistance"), + ImmutableList.of("feminism", "totalitarianism", "resistance"), "awards", - map("arthur c. clarke", true, "booker prize", false)), + ImmutableMap.of("arthur c. clarke", true, "booker prize", false)), "book6", - map( + ImmutableMap.of( "title", "Crime and Punishment", "author", @@ -179,11 +185,11 @@ public void setup() throws Exception { "rating", 4.3, "tags", - Lists.newArrayList("philosophy", "crime", "redemption"), + ImmutableList.of("philosophy", "crime", "redemption"), "awards", - map("none", true)), + ImmutableMap.of("none", true)), "book7", - map( + ImmutableMap.of( "title", "To Kill a Mockingbird", "author", @@ -195,11 +201,11 @@ public void setup() throws Exception { "rating", 4.2, "tags", - Lists.newArrayList("racism", "injustice", "coming-of-age"), + ImmutableList.of("racism", "injustice", "coming-of-age"), "awards", - map("pulitzer", true)), + ImmutableMap.of("pulitzer", true)), "book8", - map( + ImmutableMap.of( "title", "1984", "author", @@ -211,11 +217,11 @@ public void setup() throws Exception { "rating", 4.2, "tags", - Lists.newArrayList("surveillance", "totalitarianism", "propaganda"), + ImmutableList.of("surveillance", "totalitarianism", "propaganda"), "awards", - map("prometheus", true)), + ImmutableMap.of("prometheus", true)), "book9", - map( + ImmutableMap.of( "title", "The Great Gatsby", "author", @@ -227,11 +233,11 @@ public void setup() throws Exception { "rating", 4.0, "tags", - Lists.newArrayList("wealth", "american dream", "love"), + ImmutableList.of("wealth", "american dream", "love"), "awards", - map("none", true)), + ImmutableMap.of("none", true)), "book10", - map( + ImmutableMap.of( "title", "Dune", "author", @@ -243,9 +249,9 @@ public void setup() throws Exception { "rating", 4.6, "tags", - Lists.newArrayList("politics", "desert", "ecology"), + ImmutableList.of("politics", "desert", "ecology"), "awards", - map("hugo", true, "nebula", true))); + ImmutableMap.of("hugo", true, "nebula", true))); collection = testCollectionWithDocs(bookDocs); } @@ -1099,4 +1105,61 @@ public void testPipelineInTransactions() throws Exception { assertThat(data(result)) .isEqualTo(Lists.newArrayList(map("title", "The Hitchhiker's Guide to the Galaxy"))); } + + @Test + public void testReplace() throws Exception { + List results = collection.pipeline().replace("awards").execute().get(); + + List> list = + bookDocs.values().stream() + .map( + book -> { + HashMap awards = (HashMap) book.get("awards"); + HashMap map = Maps.newHashMap(book); + // Remove "awards" field. + map.remove("awards"); + // Life nested "awards". + map.putAll(awards); + return map; + }) + .collect(Collectors.toList()); + + assertThat(data(results)).containsExactly(list); + } + + @Test + public void testSampleLimit() throws Exception { + List results = collection.pipeline().sample(3).execute().get(); + + assertThat(results).hasSize(3); + } + + @Test + public void testSamplePercentage() throws Exception { + List results = + collection.pipeline().sample(SampleOptions.percentage(0.6)).execute().get(); + + assertThat(results).hasSize(6); + } + + @Test + public void testUnion() throws Exception { + List results = + collection.pipeline().union(collection.pipeline()).execute().get(); + + assertThat(results).hasSize(20); + } + + @Test + public void testUnnest() throws Exception { + List results = + collection + .pipeline() + .where(eq(Field.of("title"), "The Hitchhiker's Guide to the Galaxy")) + .unnest("tags") + .execute() + .get(); + + assertThat(results).hasSize(3); + } } diff --git a/grpc-google-cloud-firestore-v1/clirr-ignored-differences.xml b/grpc-google-cloud-firestore-v1/clirr-ignored-differences.xml index fc73daacd..3cced6b5b 100644 --- a/grpc-google-cloud-firestore-v1/clirr-ignored-differences.xml +++ b/grpc-google-cloud-firestore-v1/clirr-ignored-differences.xml @@ -1,4 +1,9 @@ + + 7012 + com/google/firestore/v1/FirestoreGrpc$AsyncService + void executePipeline(com.google.firestore.v1.ExecutePipelineRequest, io.grpc.stub.StreamObserver) + diff --git a/proto-google-cloud-firestore-v1/clirr-ignored-differences.xml b/proto-google-cloud-firestore-v1/clirr-ignored-differences.xml index 7355f0619..bca500b0c 100644 --- a/proto-google-cloud-firestore-v1/clirr-ignored-differences.xml +++ b/proto-google-cloud-firestore-v1/clirr-ignored-differences.xml @@ -161,4 +161,49 @@ com/google/firestore/v1/StructuredQuery$FindNearestOrBuilder boolean hasDistanceThreshold() + + 7012 + com/google/firestore/v1/ValueOrBuilder + java.lang.String getFieldReferenceValue() + + + 7012 + com/google/firestore/v1/ValueOrBuilder + com.google.protobuf.ByteString getFieldReferenceValueBytes() + + + 7012 + com/google/firestore/v1/ValueOrBuilder + com.google.firestore.v1.Function getFunctionValue() + + + 7012 + com/google/firestore/v1/ValueOrBuilder + com.google.firestore.v1.FunctionOrBuilder getFunctionValueOrBuilder() + + + 7012 + com/google/firestore/v1/ValueOrBuilder + com.google.firestore.v1.Pipeline getPipelineValue() + + + 7012 + com/google/firestore/v1/ValueOrBuilder + com.google.firestore.v1.PipelineOrBuilder getPipelineValueOrBuilder() + + + 7012 + com/google/firestore/v1/ValueOrBuilder + boolean hasFieldReferenceValue() + + + 7012 + com/google/firestore/v1/ValueOrBuilder + boolean hasFunctionValue() + + + 7012 + com/google/firestore/v1/ValueOrBuilder + boolean hasPipelineValue() +