From dd5b6ce7f4ad65dc69e6bbad6326edcb2179fa9c Mon Sep 17 00:00:00 2001 From: Martin Taillefer Date: Fri, 16 Dec 2016 11:18:30 -0800 Subject: [PATCH] Migrate mixer API and config to this repo. This clearly separates mixer's API and configuration surface from the implementation. This is designed to make it possible to replace the mixer in a fully compatible way. --- README.md | 23 ++-- mixer/api/v1/attributes.proto | 106 ++++++++++++++++++ mixer/api/v1/check.proto | 37 ++++++ mixer/api/v1/quota.proto | 72 ++++++++++++ mixer/api/v1/report.proto | 37 ++++++ mixer/api/v1/service.proto | 37 ++++++ mixer/config/v1/attribute_descriptor.proto | 75 +++++++++++++ mixer/config/v1/log_entry_descriptor.proto | 29 +++++ mixer/config/v1/metric_descriptor.proto | 59 ++++++++++ .../v1/monitored_resource_descriptor.proto | 31 +++++ mixer/config/v1/principal_descriptor.proto | 28 +++++ mixer/config/v1/quota_descriptor.proto | 74 ++++++++++++ 12 files changed, 592 insertions(+), 16 deletions(-) create mode 100644 mixer/api/v1/attributes.proto create mode 100644 mixer/api/v1/check.proto create mode 100644 mixer/api/v1/quota.proto create mode 100644 mixer/api/v1/report.proto create mode 100644 mixer/api/v1/service.proto create mode 100644 mixer/config/v1/attribute_descriptor.proto create mode 100644 mixer/config/v1/log_entry_descriptor.proto create mode 100644 mixer/config/v1/metric_descriptor.proto create mode 100644 mixer/config/v1/monitored_resource_descriptor.proto create mode 100644 mixer/config/v1/principal_descriptor.proto create mode 100644 mixer/config/v1/quota_descriptor.proto diff --git a/README.md b/README.md index 647e7ec5d18..7778b890e24 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,15 @@ -# istioapi -API, Config Schema Definitions and Standard Vocabulary definitions for the istio project +# Istio APIs and Common Configuration Definitions +This repo defines component-level APIs and common configuration formats for the Istio +platform. These definitions are specified using the [protobuf](https://github.com/google/protobuf) +syntax. -Usually definitions are in the form on .proto files. -In the context of config, Proto is used as a schema description language. - -## -All other repositories can depend on istioapi -This repository *will not* depend on any other repos +All other Istio repositories can take a dependency on the api +repository. This repository *will not* depend on any other repos -## We may check-in generated .pb.go and .pb.cc files here. - -## API definitions -istio/api - -## Schema definitions -istio/config - ## Standard vocabulary + All components of an Istio installation operate on a shared vocabulary of attributes. A standard vocabulary of attributes including it meaning is available in this repo. diff --git a/mixer/api/v1/attributes.proto b/mixer/api/v1/attributes.proto new file mode 100644 index 00000000000..eba094a0de8 --- /dev/null +++ b/mixer/api/v1/attributes.proto @@ -0,0 +1,106 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +import "google/protobuf/timestamp.proto"; + +// An instance of this is delivered to the mixer with every +// API call. +// +// The general idea is to leverage the stateful gRPC streams from the +// proxy to the mixer to keep to a minimum the 'attribute chatter'. +// Only delta attributes are sent over, multiple concurrent attribute +// contexts can be used to avoid thrashing, and attribute indices are used to +// keep the wire protocol maximally efficient. +// +// Producing this message is the responsibility of the mixer's client +// library which is linked into different proxy implementations. +// +// The processing order for this state in the mixer is: +// +// * Any new dictionary is applied +// +// * The requested attribute context is looked up. If no such context has been defined, a +// new context is automatically created and initialized to the empty state. When a gRPC +// stream is first created, there are no attribute contexts for the stream. +// +// * If reset_context is true, then the attribute context is reset to the +// empty state. +// +// * All attribute changes are applied to the attribute context. +// +// * All deleted attributes are removed from the attribute context. +// +message Attributes { + // A dictionary that provides a mapping of shorthand index values to + // attribute names. + // + // This is intended to leverage the stateful gRPC stream from the + // proxy to the mixer. This dictionary is sent over only when a + // stream to the mixer is first established and when the proxy's + // config changes and different attributes may be produced. + // + // Once a dictionary has been sent over, it stays in effect until + // a new dictionary is sent to replace it. The first request sent on a + // stream must include a dictionary, otherwise the mixer can't process + // any attribute updates. + // + // Dictionaries are independent of the attribute context and are thus global + // for the current gRPC stream. + map dictionary = 1; + + // The attribute context against which to operate. + // + // The mixer keeps different contexts live for any proxy gRPC stream. This + // allows the proxy to maintain multiple concurrent 'bags of attributes' + // within the mixer. + // + // If the proxy doesn't want to leverage multiple contexts, it just passes + // 0 here for every request. + // + // The proxy is configured to use a maximum number of attribute contexts in order + // to prevent an explosion of contexts in the mixer's memory space. + // + // TODO: Consider removing support for this feature. The proxy can achieve + // the same thing using multiple gRPC streams. The benefit of using streams + // would be that the mixer would be in control of the maximum number of streams + // it allows, whereas with the current model the proxy could overwhelm the + // mixer by creating too many contexts. + int32 attribute_context = 2; + + // When true, resets the current attribute context to the empty state before + // applying any incoming attributes. + // + // Resetting contexts is useful to constrain the amount of resources used by + // the mixer. The proxy needs to intelligently manage a pool of contexts. + // It may be useful to reset a context when certain big events happen, such + // as when an HTTP2 connection into the proxy terminates. + bool reset_context = 3; + + // Attributes being updated within the specified attribute context. These maps + // add and/or overwrite the context's current set of attributes. + map string_attributes = 4; + map int64_attributes = 5; + map double_attributes = 6; + map bool_attributes = 7; + map timestamp_attributes = 8; + map bytes_attributes = 9; + + // Attributes that should be removed from the specified attribute context. Deleting + // attributes which aren't currently in the attribute context is not considered an error. + repeated int32 deleted_attributes = 10; +} diff --git a/mixer/api/v1/check.proto b/mixer/api/v1/check.proto new file mode 100644 index 00000000000..3aa15d9adc8 --- /dev/null +++ b/mixer/api/v1/check.proto @@ -0,0 +1,37 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +import "google/rpc/status.proto"; +import "mixer/api/v1/attribute.proto"; + +// Used to verify preconditions before performing an action. +message CheckRequest { + // Index within the stream for this request, used to match to responses + int64 request_index = 1; + + // The attributes to use for this request + Attributes attribute_update = 2; +} + +message CheckResponse { + // Index of the request this response is associated with + int64 request_index = 1; + + // Indicates whether or not the preconditions succeeded + google.rpc.Status result = 2; +} diff --git a/mixer/api/v1/quota.proto b/mixer/api/v1/quota.proto new file mode 100644 index 00000000000..e4445c5ae0a --- /dev/null +++ b/mixer/api/v1/quota.proto @@ -0,0 +1,72 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +import "google/rpc/status.proto"; +import "mixer/api/v1/attribute.proto"; + +message QuotaRequest { + // Index within the stream for this request, used to match to responses + int64 request_index = 1; + + // The attributes to use for this request + Attributes attribute_update = 2; + + // what kind of quota operation to perform + OperationKind kind = 3; + + // Used for deduplicating quota allocation/free calls in the case of + // failed RPCs are retries. This should be a UUID per call, where the same + // UUID is used for retries of the same quota allocation or release call. + string deduplication_id = 4; + + enum OperationKind { + QUOTA_MODE_UNSPECIFIED = 0; + + // Allocate the specified amount, fail if insufficient resources available. + ALLOC_NORMAL = 1; + + // Allocate from 0 to the specified amount, never fails. + ALLOC_BEST_EFFORT = 2; + + // Release from 0 to the specified amount, never fails. + RELEASE_BEST_EFFORT = 3; + + // Return the current content of the quota value cell. + QUERY = 4; + } +} + +// Used to update quota values before and/or after performing an action. +// +// A common use case for quotas is to implement rate limits within a service. +// Quotas can also be used to impose upper limits on the number of specific +// resources exposed by a service to its consumers. +message QuotaResponse { + // Index of the request this response is associated with + int64 request_index = 1; + + // Results, one for each operation in the request. + // This map is indexed by the quota_operation_id of the individual quota operations. + oneof result { + // The effective amount of quota + uint64 effective_amount = 2; + + // An error indication in case the quota operation failed + google.rpc.Status error = 3; + } +} diff --git a/mixer/api/v1/report.proto b/mixer/api/v1/report.proto new file mode 100644 index 00000000000..3155070a9b2 --- /dev/null +++ b/mixer/api/v1/report.proto @@ -0,0 +1,37 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +import "google/rpc/status.proto"; +import "mixer/api/v1/attribute.proto"; + +// Used to report telemetry after performing an action. +message ReportRequest { + // Index within the stream for this request, used to match to responses + int64 request_index = 1; + + // The attributes to use for this request + Attributes attribute_update = 2; +} + +message ReportResponse { + // Index of the request this response is associated with + int64 request_index = 1; + + // Indicates whether the report was processed or not + google.rpc.Status result = 2; +} diff --git a/mixer/api/v1/service.proto b/mixer/api/v1/service.proto new file mode 100644 index 00000000000..1bdecf656dd --- /dev/null +++ b/mixer/api/v1/service.proto @@ -0,0 +1,37 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +import "api/mixer/v1/check.proto"; +import "api/mixer/v1/report.proto"; +import "api/mixer/v1/quota.proto"; + +// The Istio Mixer API +service Mixer { + // Checks preconditions before performing an operation. + // The preconditions enforced depend on the set of supplied attributes + // and the active configuration. + rpc Check(stream CheckRequest) returns (stream CheckResponse) {} + + // Reports telemetry, such as logs and metrics. + // The reported information depends on the set of supplied attributes + // and the active configuration. + rpc Report(stream ReportRequest) returns (stream ReportResponse) {} + + // Quota allocates and releases quota. + rpc Quota(stream QuotaRequest) returns (stream QuotaResponse) {} +} diff --git a/mixer/config/v1/attribute_descriptor.proto b/mixer/config/v1/attribute_descriptor.proto new file mode 100644 index 00000000000..8e7cdfa22e0 --- /dev/null +++ b/mixer/config/v1/attribute_descriptor.proto @@ -0,0 +1,75 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +// The definition of an attribute type. +// +// An attribute carries a piece of information describing ingress or egress traffic, along +// with the environment the traffic takes place in. +// +// This descriptor represents the schema for an individual attribute type. +message AttributeDescriptor { + // The name of this descriptor, referenced from individual attribute instances and other + // descriptors. + // + // The format of this name is: + // + // Name = IDENT { "." IDENT } ; + // + // Where `IDENT` must match the regular expression `[a-z][a-z0-9]+`. + // + // Attribute descriptor names must be unique within a single Istio deployment. There is a well + // known set of attributes which have succinct names. Attributes not on this list should be + // name with a component-specific suffix such as request_count-my_component + string name = 1; + + // The type of data carried by attributes + ValueType value_type = 2; + + // An optional human-readable description of the attribute's purpose. + string description = 3; + + // Types of supported attribute values. + enum ValueType { + // An undiscriminated variable-length string. + STRING = 0; + + // An undiscriminated 64-bit signed integer. + INT64 = 1; + + // An undiscriminated 64-bit floating-point value. + DOUBLE = 2; + + // An undiscriminated boolean value. + BOOL = 3; + + // A point in time. + TIMESTAMP = 4; + + // An IP address, encoded as bytes. + IP_ADDRESS = 5; + + // An email address, encoded as a string. + EMAIL_ADDRESS = 6; + + // A URI, encoded as a string. + URI = 7; + + // A DNS name, encoded as a string. + DNS_NAME = 8; + } +} diff --git a/mixer/config/v1/log_entry_descriptor.proto b/mixer/config/v1/log_entry_descriptor.proto new file mode 100644 index 00000000000..c474849af08 --- /dev/null +++ b/mixer/config/v1/log_entry_descriptor.proto @@ -0,0 +1,29 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +// Defines the format of a single log entry. +message LogEntryDescriptor { + // The name of this descriptor. + string name = 1; + + // The set of attributes that are necessary to describe a log entry of this type. + repeated string attributes = 2; + + // The monitored resource to associated with log entries of this type. + string monitored_resource_descriptor = 3; +} diff --git a/mixer/config/v1/metric_descriptor.proto b/mixer/config/v1/metric_descriptor.proto new file mode 100644 index 00000000000..e1326ee8f17 --- /dev/null +++ b/mixer/config/v1/metric_descriptor.proto @@ -0,0 +1,59 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +// Defines a metric type and its schema. +// +// A given metric is dimensioned by a set of attributes. A given metric holds a +// unique value for potentially any combination of these attributes. +message MetricDescriptor { + // The name of this descriptor. + string name = 1; + + // The name of the attribute that supplies the value for metrics + // of this type. + string value_attribute = 2; + + // The set of attributes that are necessary to describe a specific value cell + // for a metric of this type. + repeated string dimension_attributes = 3; + + // The monitored resource to associate with metrics of this type. + string monitored_resource_descriptor = 4; + + // Whether the metric records instantaneous values, changes to a value, etc. + MetricKind metric_kind = 5; + + // An optional description of the metric, which can be used in documentation. + string description = 6; + + // An optional concise name for the metric, which can be displayed in user interfaces. + // Use sentence case without an ending period, for example "Request count". + string display_name = 7; + + // The kind of measurement. It describes how the data is reported. + enum MetricKind { + // Do not use this default value. + METRIC_KIND_UNSPECIFIED = 0; + + // An instantaneous measurement of a value. + GAUGE = 1; + + // A count of occurrences over an interval, always a positive integer + COUNTER = 2; + } +} diff --git a/mixer/config/v1/monitored_resource_descriptor.proto b/mixer/config/v1/monitored_resource_descriptor.proto new file mode 100644 index 00000000000..1b4f8a0bbb2 --- /dev/null +++ b/mixer/config/v1/monitored_resource_descriptor.proto @@ -0,0 +1,31 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +// An object that describes the schema of a 'MonitoredResource' +// using a name and a set of attributes. +message MonitoredResourceDescriptor { + // The name of this descriptor + string name = 1; + + // The set of attributes that are necessary to describe a specific monitored resource. + repeated string attributes = 3; + + // An optional detailed description of the monitored resource descriptor that might + // be used in documentation. + string description = 2; +} diff --git a/mixer/config/v1/principal_descriptor.proto b/mixer/config/v1/principal_descriptor.proto new file mode 100644 index 00000000000..d660b005f65 --- /dev/null +++ b/mixer/config/v1/principal_descriptor.proto @@ -0,0 +1,28 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +// Defines a a security principal. +// +// A principal is described by a set of attributes. +message PrincipalDescriptor { + // The name of this descriptor. + string name = 1; + + // The set of attributes which are used to describe a specific security principal. + repeated string attributes = 2; +} diff --git a/mixer/config/v1/quota_descriptor.proto b/mixer/config/v1/quota_descriptor.proto new file mode 100644 index 00000000000..cc74f26185e --- /dev/null +++ b/mixer/config/v1/quota_descriptor.proto @@ -0,0 +1,74 @@ +// Copyright 2016 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.mixer.v1; + +// Configuration state for a particular quota. +// +// Quotas are similar to metrics, except that they are mutated through method +// calls and there are limits on the allowed values. +// The descriptor below lets you define a quota and indicate the maximum +// amount values of this quota are allowed to hold. +// +// A given quota is described by a set of attribtues. These attributes represent +// the different dimensions to associate with the quota. A given quota holds a +// unique value for potentially any combination of these attributes. +// +// The quota kind controls the general behavior of the quota. An allocation +// quota is only adjusted through explicit method calls. A rate limit quota's +// values are reset to 0 automatically at a fixed interval. +message QuotaDescriptor { + // The name of this descriptor. + string name = 1; + + // The name of the attribute that supplies the amount for a given + // quota allocation or release operation. + string amount_attribute = 2; + + // The set of attributes that are necessary to describe a specific value cell + // for a quota of this type. + repeated string dimension_attributes = 3; + + // An optional description of the quota, which can be used in documentation. + string description = 4; + + // A optional concise name for the quota, which can be displayed in user interfaces. + // Use sentence case without an ending period, for example "Request count". + string display_name = 5; + + // The default imposed maximum amount for values of this quota. + int64 max_amount = 6; + + // The type of quota behavior expected, defaults to ALLOCATION_LIMIT if not specified + QuotaKind kind = 7; + + // Whether the quota's current value is tracked precisely or not. + // Precisely tracked quotas only allow a relatively modest level of + // scaling, whereas imprecise quotas can support a nearly unbounded + // level of scaling at the cost of potentially having inaccuracies. + bool precise = 8; + + enum QuotaKind { + // Quota values are always explicitly manipulated via the Quota method. + ALLOCATION_LIMIT = 0; + + // Quota limit expresses a maximum amount over a rolling time interval + PER_SECOND_LIMIT = 1; + + // Quota limit expresses a maximum amount over a rolling time interval + PER_MINUTE_LIMIT = 1; + } +}