From 1df80fd16d2039005b025b0433c4985bd9b51cc3 Mon Sep 17 00:00:00 2001 From: gloriacai01 Date: Tue, 15 Jul 2025 10:04:49 -0400 Subject: [PATCH 1/4] add scope + fragmentids --- .../app/datapipelines/v1/data_pipelines.proto | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/proto/viam/app/datapipelines/v1/data_pipelines.proto b/proto/viam/app/datapipelines/v1/data_pipelines.proto index 5ae73e714..ff883fa2b 100644 --- a/proto/viam/app/datapipelines/v1/data_pipelines.proto +++ b/proto/viam/app/datapipelines/v1/data_pipelines.proto @@ -64,6 +64,23 @@ message DataPipeline { optional viam.app.data.v1.TabularDataSourceType data_source_type = 9; } +enum PipelineScopeType { + PIPELINE_SCOPE_TYPE_UNSPECIFIED = 0; + + // PIPELINE_SCOPE_TYPE_ALL_MACHINES specifies that the pipeline should run for each machine in the organization. + PIPELINE_SCOPE_TYPE_ALL_MACHINES = 1; + + // PIPELINE_SCOPE_TYPE_ALL_MACHINES_WITH_FRAGMENTS specifies that the pipeline should run for each machine with at least one of the specified fragments. + PIPELINE_SCOPE_TYPE_ALL_MACHINES_WITH_FRAGMENTS = 2; +} + +message PipelineScope { + PipelineScopeType scope = 1; + + // Required when using SCOPE_ALL_MACHINES_WITH_FRAGMENTS. When specified, the pipeline will only be run on machines with at least one of these specified fragments. If scope is SCOPE_ALL_MACHINES, this field will have no effect. + repeated string fragment_ids = 2; +} + message GetDataPipelineRequest { // The ID of the data pipeline to retrieve. string id = 1; @@ -90,7 +107,8 @@ message CreateDataPipelineRequest { string name = 2; // A MongoDB aggregation pipeline as a list of BSON documents, where - // each document is one stage in the pipeline. + // each document is one stage in the pipeline. The pipeline will be run + // for the defined scope. repeated bytes mql_binary = 3; // A cron expression representing the expected execution schedule in UTC (note this also @@ -102,6 +120,9 @@ message CreateDataPipelineRequest { // The type of data source for the pipeline. If not specified, default is standard data storage. optional viam.app.data.v1.TabularDataSourceType data_source_type = 6; + + // Defines the set of resources that the pipeline should execute against. + PipelineScope scope = 7; } message CreateDataPipelineResponse { From 63852731e26f6b9dd1c1230f1a1812f92e9a7bfa Mon Sep 17 00:00:00 2001 From: gloriacai01 Date: Tue, 15 Jul 2025 10:07:19 -0400 Subject: [PATCH 2/4] update comment --- proto/viam/app/datapipelines/v1/data_pipelines.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proto/viam/app/datapipelines/v1/data_pipelines.proto b/proto/viam/app/datapipelines/v1/data_pipelines.proto index ff883fa2b..2b35464d3 100644 --- a/proto/viam/app/datapipelines/v1/data_pipelines.proto +++ b/proto/viam/app/datapipelines/v1/data_pipelines.proto @@ -100,7 +100,8 @@ message ListDataPipelinesResponse { } message CreateDataPipelineRequest { - // The associated Viam organization ID. + // The associated Viam organization ID. This organization will also be billed for + // data usage. string organization_id = 1; // A unique identifier at the org level. From b3f973dffa8ef4fff6c2d715e8ab6df733d56c9c Mon Sep 17 00:00:00 2001 From: gloriacai01 Date: Tue, 15 Jul 2025 11:56:12 -0400 Subject: [PATCH 3/4] oopsie --- proto/viam/app/datapipelines/v1/data_pipelines.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/viam/app/datapipelines/v1/data_pipelines.proto b/proto/viam/app/datapipelines/v1/data_pipelines.proto index 2b35464d3..5f694e5a7 100644 --- a/proto/viam/app/datapipelines/v1/data_pipelines.proto +++ b/proto/viam/app/datapipelines/v1/data_pipelines.proto @@ -77,7 +77,7 @@ enum PipelineScopeType { message PipelineScope { PipelineScopeType scope = 1; - // Required when using SCOPE_ALL_MACHINES_WITH_FRAGMENTS. When specified, the pipeline will only be run on machines with at least one of these specified fragments. If scope is SCOPE_ALL_MACHINES, this field will have no effect. + // Required when using PIPELINE_SCOPE_TYPE_ALL_MACHINES_WITH_FRAGMENTS. When specified, the pipeline will only be run on machines with at least one of these specified fragments. If scope is PIPELINE_SCOPE_TYPE_ALL_MACHINES, this field will have no effect. repeated string fragment_ids = 2; } From 42424fe4d0f4ec3c82ba7c7d7657c97b9a59dd1b Mon Sep 17 00:00:00 2001 From: gloriacai01 <55797301+gloriacai01@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:24:47 -0400 Subject: [PATCH 4/4] Update proto/viam/app/datapipelines/v1/data_pipelines.proto Co-authored-by: Katie Peters --- proto/viam/app/datapipelines/v1/data_pipelines.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/viam/app/datapipelines/v1/data_pipelines.proto b/proto/viam/app/datapipelines/v1/data_pipelines.proto index 5f694e5a7..9b4a4955c 100644 --- a/proto/viam/app/datapipelines/v1/data_pipelines.proto +++ b/proto/viam/app/datapipelines/v1/data_pipelines.proto @@ -77,7 +77,7 @@ enum PipelineScopeType { message PipelineScope { PipelineScopeType scope = 1; - // Required when using PIPELINE_SCOPE_TYPE_ALL_MACHINES_WITH_FRAGMENTS. When specified, the pipeline will only be run on machines with at least one of these specified fragments. If scope is PIPELINE_SCOPE_TYPE_ALL_MACHINES, this field will have no effect. + // Required when using PIPELINE_SCOPE_TYPE_ALL_MACHINES_WITH_FRAGMENTS. When defined, the pipeline will only be run on machines with at least one of the specified fragments. If scope is PIPELINE_SCOPE_TYPE_ALL_MACHINES, this field will have no effect. repeated string fragment_ids = 2; }