diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index fbbe984415865..44ed9f0ca3e5d 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -840,6 +840,11 @@ const ( // Enables a StatefulSet to start from an arbitrary non zero ordinal StatefulSetStartOrdinal featuregate.Feature = "StatefulSetStartOrdinal" + + // owner: @serathius + // Allow API server to encode collections item by item, instead of all at once. + StreamingCollectionEncodingToJSON featuregate.Feature = "StreamingCollectionEncodingToJSON" + // owner: @robscott // kep: https://kep.k8s.io/2433 // alpha: v1.21 diff --git a/pkg/registry/core/rest/storage_core_generic.go b/pkg/registry/core/rest/storage_core_generic.go index 193b5b98f473e..0f97dbef55127 100644 --- a/pkg/registry/core/rest/storage_core_generic.go +++ b/pkg/registry/core/rest/storage_core_generic.go @@ -25,11 +25,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/generic" "k8s.io/apiserver/pkg/registry/rest" genericapiserver "k8s.io/apiserver/pkg/server" serverstorage "k8s.io/apiserver/pkg/server/storage" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" restclient "k8s.io/client-go/rest" @@ -69,6 +72,14 @@ func (c *GenericConfig) NewRESTStorage(apiResourceConfigSource serverstorage.API NegotiatedSerializer: legacyscheme.Codecs, } + opts := []serializer.CodecFactoryOptionsMutator{} + if utilfeature.DefaultFeatureGate.Enabled(features.StreamingCollectionEncodingToJSON) { + opts = append(opts, serializer.WithStreamingCollectionEncodingToJSON()) + } + if len(opts) != 0 { + apiGroupInfo.NegotiatedSerializer = serializer.NewCodecFactory(legacyscheme.Scheme, opts...) + } + eventStorage, err := eventstore.NewREST(restOptionsGetter, uint64(c.EventTTL.Seconds())) if err != nil { return genericapiserver.APIGroupInfo{}, err diff --git a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go index 7a586b979a7d0..564d39e2df1fd 100644 --- a/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go +++ b/staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/customresource_handler.go @@ -25,6 +25,8 @@ import ( "sync/atomic" "time" + "k8s.io/apiserver/pkg/features" + utilfeature "k8s.io/apiserver/pkg/util/feature" "sigs.k8s.io/structured-merge-diff/v4/fieldpath" apiextensionshelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers" @@ -826,6 +828,7 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd clusterScoped := crd.Spec.Scope == apiextensionsv1.ClusterScoped // CRDs explicitly do not support protobuf, but some objects returned by the API server do + streamingCollections := utilfeature.DefaultFeatureGate.Enabled(features.StreamingCollectionEncodingToJSON) negotiatedSerializer := unstructuredNegotiatedSerializer{ typer: typer, creator: creator, @@ -839,10 +842,11 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd MediaTypeType: "application", MediaTypeSubType: "json", EncodesAsText: true, - Serializer: json.NewSerializer(json.DefaultMetaFactory, creator, typer, false), - PrettySerializer: json.NewSerializer(json.DefaultMetaFactory, creator, typer, true), + Serializer: json.NewSerializerWithOptions(json.DefaultMetaFactory, creator, typer, json.SerializerOptions{StreamingCollectionsEncoding: streamingCollections}), + PrettySerializer: json.NewSerializerWithOptions(json.DefaultMetaFactory, creator, typer, json.SerializerOptions{Pretty: true}), StrictSerializer: json.NewSerializerWithOptions(json.DefaultMetaFactory, creator, typer, json.SerializerOptions{ - Strict: true, + Strict: true, + StreamingCollectionsEncoding: streamingCollections, }), StreamSerializer: &runtime.StreamSerializerInfo{ EncodesAsText: true, @@ -936,7 +940,11 @@ func (r *crdHandler) getOrCreateServingInfoFor(uid types.UID, name string) (*crd scaleScope := *requestScopes[v.Name] scaleConverter := scale.NewScaleConverter() scaleScope.Subresource = "scale" - scaleScope.Serializer = serializer.NewCodecFactory(scaleConverter.Scheme()) + var opts []serializer.CodecFactoryOptionsMutator + if utilfeature.DefaultFeatureGate.Enabled(features.StreamingCollectionEncodingToJSON) { + opts = append(opts, serializer.WithStreamingCollectionEncodingToJSON()) + } + scaleScope.Serializer = serializer.NewCodecFactory(scaleConverter.Scheme(), opts...) scaleScope.Kind = autoscalingv1.SchemeGroupVersion.WithKind("Scale") scaleScope.Namer = handlers.ContextBasedNaming{ Namer: meta.NewAccessor(), diff --git a/staging/src/k8s.io/apimachinery/pkg/api/meta/help.go b/staging/src/k8s.io/apimachinery/pkg/api/meta/help.go index 1fdd32c4ba3e0..468afd0e9ee09 100644 --- a/staging/src/k8s.io/apimachinery/pkg/api/meta/help.go +++ b/staging/src/k8s.io/apimachinery/pkg/api/meta/help.go @@ -221,6 +221,9 @@ func extractList(obj runtime.Object, allocNew bool) ([]runtime.Object, error) { if err != nil { return nil, err } + if items.IsNil() { + return nil, nil + } list := make([]runtime.Object, items.Len()) if len(list) == 0 { return list, nil diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/codec_factory.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/codec_factory.go index ff98208420465..96e15fef01463 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/codec_factory.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/codec_factory.go @@ -52,7 +52,7 @@ type serializerType struct { func newSerializersForScheme(scheme *runtime.Scheme, mf json.MetaFactory, options CodecFactoryOptions) []serializerType { jsonSerializer := json.NewSerializerWithOptions( mf, scheme, scheme, - json.SerializerOptions{Yaml: false, Pretty: false, Strict: options.Strict}, + json.SerializerOptions{Yaml: false, Pretty: false, Strict: options.Strict, StreamingCollectionsEncoding: options.StreamingCollectionsEncodingToJSON}, ) jsonSerializerType := serializerType{ AcceptContentTypes: []string{runtime.ContentTypeJSON}, @@ -60,9 +60,17 @@ func newSerializersForScheme(scheme *runtime.Scheme, mf json.MetaFactory, option FileExtensions: []string{"json"}, EncodesAsText: true, Serializer: jsonSerializer, - - Framer: json.Framer, - StreamSerializer: jsonSerializer, + StrictSerializer: json.NewSerializerWithOptions( + mf, scheme, scheme, + json.SerializerOptions{Yaml: false, Pretty: false, Strict: true, StreamingCollectionsEncoding: options.StreamingCollectionsEncodingToJSON}, + ), + + Framer: json.Framer, + StreamSerializer: &runtime.StreamSerializerInfo{ + EncodesAsText: true, + Serializer: jsonSerializer, + Framer: json.Framer, + }, } if options.Pretty { jsonSerializerType.PrettySerializer = json.NewSerializerWithOptions( @@ -136,6 +144,10 @@ type CodecFactoryOptions struct { Strict bool // Pretty includes a pretty serializer along with the non-pretty one Pretty bool + + StreamingCollectionsEncodingToJSON bool + + serializers []func(runtime.ObjectCreater, runtime.ObjectTyper) runtime.SerializerInfo } // CodecFactoryOptionsMutator takes a pointer to an options struct and then modifies it. @@ -162,6 +174,19 @@ func DisableStrict(options *CodecFactoryOptions) { options.Strict = false } +// WithSerializer configures a serializer to be supported in addition to the default serializers. +func WithSerializer(f func(runtime.ObjectCreater, runtime.ObjectTyper) runtime.SerializerInfo) CodecFactoryOptionsMutator { + return func(options *CodecFactoryOptions) { + options.serializers = append(options.serializers, f) + } +} + +func WithStreamingCollectionEncodingToJSON() CodecFactoryOptionsMutator { + return func(options *CodecFactoryOptions) { + options.StreamingCollectionsEncodingToJSON = true + } +} + // NewCodecFactory provides methods for retrieving serializers for the supported wire formats // and conversion wrappers to define preferred internal and external versions. In the future, // as the internal version is used less, callers may instead use a defaulting serializer and diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/collections.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/collections.go new file mode 100644 index 0000000000000..9836fadad0321 --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/collections.go @@ -0,0 +1,229 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "encoding/json" + "fmt" + "io" + "sort" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/conversion" + "k8s.io/apimachinery/pkg/util/sets" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +func streamEncodeCollections(obj runtime.Object, w io.Writer) (bool, error) { + list, ok := obj.(*unstructured.UnstructuredList) + if ok { + return true, streamingEncodeUnstructuredList(w, list) + } + if _, ok := obj.(json.Marshaler); ok { + return false, nil + } + typeMeta, listMeta, items, err := getListMeta(obj) + if err == nil { + return true, streamingEncodeList(w, typeMeta, listMeta, items) + } + return false, nil +} + +// getListMeta implements list extraction logic for json stream serialization. +// +// Reason for a custom logic instead of reusing accessors from meta package: +// * Validate json tags to prevent incompatibility with json standard package. +// * ListMetaAccessor doesn't distinguish empty from nil value. +// * TypeAccessort reparsing "apiVersion" and serializing it with "{group}/{version}" +func getListMeta(list runtime.Object) (metav1.TypeMeta, metav1.ListMeta, []runtime.Object, error) { + listValue, err := conversion.EnforcePtr(list) + if err != nil { + return metav1.TypeMeta{}, metav1.ListMeta{}, nil, err + } + listType := listValue.Type() + if listType.NumField() != 3 { + return metav1.TypeMeta{}, metav1.ListMeta{}, nil, fmt.Errorf("expected ListType to have 3 fields") + } + // TypeMeta + typeMeta, ok := listValue.Field(0).Interface().(metav1.TypeMeta) + if !ok { + return metav1.TypeMeta{}, metav1.ListMeta{}, nil, fmt.Errorf("expected TypeMeta field to have TypeMeta type") + } + if listType.Field(0).Tag.Get("json") != ",inline" { + return metav1.TypeMeta{}, metav1.ListMeta{}, nil, fmt.Errorf(`expected TypeMeta json field tag to be ",inline"`) + } + // ListMeta + listMeta, ok := listValue.Field(1).Interface().(metav1.ListMeta) + if !ok { + return metav1.TypeMeta{}, metav1.ListMeta{}, nil, fmt.Errorf("expected ListMeta field to have ListMeta type") + } + if listType.Field(1).Tag.Get("json") != "metadata,omitempty" { + return metav1.TypeMeta{}, metav1.ListMeta{}, nil, fmt.Errorf(`expected ListMeta json field tag to be "metadata,omitempty"`) + } + // Items + items, err := meta.ExtractList(list) + if err != nil { + return metav1.TypeMeta{}, metav1.ListMeta{}, nil, err + } + if listType.Field(2).Tag.Get("json") != "items" { + return metav1.TypeMeta{}, metav1.ListMeta{}, nil, fmt.Errorf(`expected Items json field tag to be "items"`) + } + return typeMeta, listMeta, items, nil +} + +func streamingEncodeList(w io.Writer, typeMeta metav1.TypeMeta, listMeta metav1.ListMeta, items []runtime.Object) error { + // Start + if _, err := w.Write([]byte(`{`)); err != nil { + return err + } + + // TypeMeta + if typeMeta.Kind != "" { + if err := encodeKeyValuePair(w, "kind", typeMeta.Kind, []byte(",")); err != nil { + return err + } + } + if typeMeta.APIVersion != "" { + if err := encodeKeyValuePair(w, "apiVersion", typeMeta.APIVersion, []byte(",")); err != nil { + return err + } + } + + // ListMeta + if err := encodeKeyValuePair(w, "metadata", listMeta, []byte(",")); err != nil { + return err + } + + // Items + if err := encodeItemsObjectSlice(w, items); err != nil { + return err + } + + // End + _, err := w.Write([]byte("}\n")) + return err +} + +func encodeItemsObjectSlice(w io.Writer, items []runtime.Object) (err error) { + if items == nil { + err := encodeKeyValuePair(w, "items", nil, nil) + return err + } + _, err = w.Write([]byte(`"items":[`)) + if err != nil { + return err + } + suffix := []byte(",") + for i, item := range items { + if i == len(items)-1 { + suffix = nil + } + err := encodeValue(w, item, suffix) + if err != nil { + return err + } + } + _, err = w.Write([]byte("]")) + if err != nil { + return err + } + return err +} + +func streamingEncodeUnstructuredList(w io.Writer, list *unstructured.UnstructuredList) error { + _, err := w.Write([]byte(`{`)) + if err != nil { + return err + } + keys := sets.List(sets.KeySet(list.Object)) + if _, exists := list.Object["items"]; !exists { + keys = append(keys, "items") + } + sort.Strings(keys) + + suffix := []byte(",") + for i, key := range keys { + if i == len(keys)-1 { + suffix = nil + } + if key == "items" { + err = encodeItemsUnstructuredSlice(w, list.Items, suffix) + } else { + err = encodeKeyValuePair(w, key, list.Object[key], suffix) + } + if err != nil { + return err + } + } + _, err = w.Write([]byte("}\n")) + return err +} + +func encodeItemsUnstructuredSlice(w io.Writer, items []unstructured.Unstructured, suffix []byte) (err error) { + _, err = w.Write([]byte(`"items":[`)) + if err != nil { + return err + } + comma := []byte(",") + for i, item := range items { + if i == len(items)-1 { + comma = nil + } + err := encodeValue(w, item.Object, comma) + if err != nil { + return err + } + } + _, err = w.Write([]byte("]")) + if err != nil { + return err + } + if len(suffix) > 0 { + _, err = w.Write(suffix) + } + return err +} + +func encodeKeyValuePair(w io.Writer, key string, value any, suffix []byte) (err error) { + err = encodeValue(w, key, []byte(":")) + if err != nil { + return err + } + err = encodeValue(w, value, suffix) + if err != nil { + return err + } + return err +} + +func encodeValue(w io.Writer, value any, suffix []byte) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + _, err = w.Write(data) + if err != nil { + return err + } + if len(suffix) > 0 { + _, err = w.Write(suffix) + } + return err +} diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/collections_test.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/collections_test.go new file mode 100644 index 0000000000000..03ec4d88f2236 --- /dev/null +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/collections_test.go @@ -0,0 +1,794 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package json + +import ( + "bytes" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + fuzz "github.com/google/gofuzz" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + testapigroupv1 "k8s.io/apimachinery/pkg/apis/testapigroup/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestCollectionsEncoding(t *testing.T) { + t.Run("Normal", func(t *testing.T) { + testCollectionsEncoding(t, NewSerializerWithOptions(DefaultMetaFactory, nil, nil, SerializerOptions{}), false) + }) + t.Run("Streaming", func(t *testing.T) { + testCollectionsEncoding(t, NewSerializerWithOptions(DefaultMetaFactory, nil, nil, SerializerOptions{StreamingCollectionsEncoding: true}), true) + }) +} + +// testCollectionsEncoding should provide comprehensive tests to validate streaming implementation of encoder. +func testCollectionsEncoding(t *testing.T, s *Serializer, streamingEnabled bool) { + var buf writeCountingBuffer + var remainingItems int64 = 1 + // As defined in KEP-5116 we it should include the following scenarios: + // Context: https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/5116-streaming-response-encoding#unit-tests + for _, tc := range []struct { + name string + in runtime.Object + cannotStream bool + expect string + }{ + // Preserving the distinction between integers and floating-point numbers + { + name: "Struct with floats", + in: &StructWithFloatsList{ + Items: []StructWithFloats{ + { + Int: 1, + Float32: float32(1), + Float64: 1.1, + }, + }, + }, + expect: "{\"metadata\":{},\"items\":[{\"metadata\":{\"creationTimestamp\":null},\"Int\":1,\"Float32\":1,\"Float64\":1.1}]}\n", + }, + { + name: "Unstructured object float", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "int": 1, + "float32": float32(1), + "float64": 1.1, + }, + }, + expect: "{\"float32\":1,\"float64\":1.1,\"int\":1,\"items\":[]}\n", + }, + { + name: "Unstructured items float", + in: &unstructured.UnstructuredList{ + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "int": 1, + "float32": float32(1), + "float64": 1.1, + }, + }, + }, + }, + expect: "{\"items\":[{\"float32\":1,\"float64\":1.1,\"int\":1}]}\n", + }, + // Handling structs with duplicate field names (JSON tag names) without producing duplicate keys in the encoded output + { + name: "StructWithDuplicatedTags", + in: &StructWithDuplicatedTagsList{ + Items: []StructWithDuplicatedTags{ + { + Key1: "key1", + Key2: "key2", + }, + }, + }, + expect: "{\"metadata\":{},\"items\":[{\"metadata\":{\"creationTimestamp\":null}}]}\n", + }, + // Encoding Go strings containing invalid UTF-8 sequences without error + { + name: "UnstructuredList object invalid UTF-8 ", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "key": "\x80", // first byte is a continuation byte + }, + }, + expect: "{\"items\":[],\"key\":\"\\ufffd\"}\n", + }, + { + name: "UnstructuredList items invalid UTF-8 ", + in: &unstructured.UnstructuredList{ + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "key": "\x80", // first byte is a continuation byte + }, + }, + }, + }, + expect: "{\"items\":[{\"key\":\"\\ufffd\"}]}\n", + }, + // Preserving the distinction between absent, present-but-null, and present-and-empty states for slices and maps + { + name: "CarpList items nil", + in: &testapigroupv1.CarpList{ + Items: nil, + }, + expect: "{\"metadata\":{},\"items\":null}\n", + }, + { + name: "CarpList slice nil", + in: &testapigroupv1.CarpList{ + Items: []testapigroupv1.Carp{ + { + Status: testapigroupv1.CarpStatus{ + Conditions: nil, + }, + }, + }, + }, + expect: "{\"metadata\":{},\"items\":[{\"metadata\":{\"creationTimestamp\":null},\"spec\":{},\"status\":{}}]}\n", + }, + { + name: "CarpList map nil", + in: &testapigroupv1.CarpList{ + Items: []testapigroupv1.Carp{ + { + Spec: testapigroupv1.CarpSpec{ + NodeSelector: nil, + }, + }, + }, + }, + expect: "{\"metadata\":{},\"items\":[{\"metadata\":{\"creationTimestamp\":null},\"spec\":{},\"status\":{}}]}\n", + }, + { + name: "UnstructuredList items nil", + in: &unstructured.UnstructuredList{ + Items: nil, + }, + expect: "{\"items\":[]}\n", + }, + { + name: "UnstructuredList items slice nil", + in: &unstructured.UnstructuredList{ + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "slice": ([]string)(nil), + }, + }, + }, + }, + expect: "{\"items\":[{\"slice\":null}]}\n", + }, + { + name: "UnstructuredList items map nil", + in: &unstructured.UnstructuredList{ + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "map": (map[string]string)(nil), + }, + }, + }, + }, + expect: "{\"items\":[{\"map\":null}]}\n", + }, + { + name: "UnstructuredList object nil", + in: &unstructured.UnstructuredList{ + Object: nil, + }, + expect: "{\"items\":[]}\n", + }, + { + name: "UnstructuredList object slice nil", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "slice": ([]string)(nil), + }, + }, + expect: "{\"items\":[],\"slice\":null}\n", + }, + { + name: "UnstructuredList object map nil", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "map": (map[string]string)(nil), + }, + }, + expect: "{\"items\":[],\"map\":null}\n", + }, + { + name: "CarpList items empty", + in: &testapigroupv1.CarpList{ + Items: []testapigroupv1.Carp{}, + }, + expect: "{\"metadata\":{},\"items\":[]}\n", + }, + { + name: "CarpList slice empty", + in: &testapigroupv1.CarpList{ + Items: []testapigroupv1.Carp{ + { + Status: testapigroupv1.CarpStatus{ + Conditions: []testapigroupv1.CarpCondition{}, + }, + }, + }, + }, + expect: "{\"metadata\":{},\"items\":[{\"metadata\":{\"creationTimestamp\":null},\"spec\":{},\"status\":{}}]}\n", + }, + { + name: "CarpList map empty", + in: &testapigroupv1.CarpList{ + Items: []testapigroupv1.Carp{ + { + Spec: testapigroupv1.CarpSpec{ + NodeSelector: map[string]string{}, + }, + }, + }, + }, + expect: "{\"metadata\":{},\"items\":[{\"metadata\":{\"creationTimestamp\":null},\"spec\":{},\"status\":{}}]}\n", + }, + { + name: "UnstructuredList items empty", + in: &unstructured.UnstructuredList{ + Items: []unstructured.Unstructured{}, + }, + expect: "{\"items\":[]}\n", + }, + { + name: "UnstructuredList items slice empty", + in: &unstructured.UnstructuredList{ + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "slice": []string{}, + }, + }, + }, + }, + expect: "{\"items\":[{\"slice\":[]}]}\n", + }, + { + name: "UnstructuredList items map empty", + in: &unstructured.UnstructuredList{ + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "map": map[string]string{}, + }, + }, + }, + }, + expect: "{\"items\":[{\"map\":{}}]}\n", + }, + { + name: "UnstructuredList object empty", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{}, + }, + expect: "{\"items\":[]}\n", + }, + { + name: "UnstructuredList object slice empty", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "slice": []string{}, + }, + }, + expect: "{\"items\":[],\"slice\":[]}\n", + }, + { + name: "UnstructuredList object map empty", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "map": map[string]string{}, + }, + }, + expect: "{\"items\":[],\"map\":{}}\n", + }, + // Handling structs implementing MarshallJSON method, especially built-in collection types. + { + name: "List with MarshallJSON cannot be streamed", + in: &ListWithMarshalJSONList{}, + expect: "\"marshallJSON\"\n", + cannotStream: true, + }, + { + name: "Struct with MarshallJSON", + in: &StructWithMarshalJSONList{ + Items: []StructWithMarshalJSON{ + {}, + }, + }, + expect: "{\"metadata\":{},\"items\":[\"marshallJSON\"]}\n", + }, + // Handling raw bytes. + { + name: "Struct with raw bytes", + in: &StructWithRawBytesList{ + Items: []StructWithRawBytes{ + { + Slice: []byte{0x01, 0x02, 0x03}, + Array: [3]byte{0x01, 0x02, 0x03}, + }, + }, + }, + expect: "{\"metadata\":{},\"items\":[{\"metadata\":{\"creationTimestamp\":null},\"Slice\":\"AQID\",\"Array\":[1,2,3]}]}\n", + }, + { + name: "UnstructuredList object raw bytes", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{ + "slice": []byte{0x01, 0x02, 0x03}, + "array": [3]byte{0x01, 0x02, 0x03}, + }, + }, + expect: "{\"array\":[1,2,3],\"items\":[],\"slice\":\"AQID\"}\n", + }, + { + name: "UnstructuredList items raw bytes", + in: &unstructured.UnstructuredList{ + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "slice": []byte{0x01, 0x02, 0x03}, + "array": [3]byte{0x01, 0x02, 0x03}, + }, + }, + }, + }, + expect: "{\"items\":[{\"array\":[1,2,3],\"slice\":\"AQID\"}]}\n", + }, + // Other scenarios: + { + name: "List just kind", + in: &testapigroupv1.CarpList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + }, + }, + expect: "{\"kind\":\"List\",\"metadata\":{},\"items\":null}\n", + }, + { + name: "List just apiVersion", + in: &testapigroupv1.CarpList{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + }, + }, + expect: "{\"apiVersion\":\"v1\",\"metadata\":{},\"items\":null}\n", + }, + { + name: "List no elements", + in: &testapigroupv1.CarpList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + ListMeta: metav1.ListMeta{ + ResourceVersion: "2345", + }, + Items: []testapigroupv1.Carp{}, + }, + expect: "{\"kind\":\"List\",\"apiVersion\":\"v1\",\"metadata\":{\"resourceVersion\":\"2345\"},\"items\":[]}\n", + }, + { + name: "List one element with continue", + in: &testapigroupv1.CarpList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + ListMeta: metav1.ListMeta{ + ResourceVersion: "2345", + Continue: "abc", + RemainingItemCount: &remainingItems, + }, + Items: []testapigroupv1.Carp{ + {TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Carp"}, ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "default", + }}, + }, + }, + expect: "{\"kind\":\"List\",\"apiVersion\":\"v1\",\"metadata\":{\"resourceVersion\":\"2345\",\"continue\":\"abc\",\"remainingItemCount\":1},\"items\":[{\"kind\":\"Carp\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"pod\",\"namespace\":\"default\",\"creationTimestamp\":null},\"spec\":{},\"status\":{}}]}\n", + }, + { + name: "List two elements", + in: &testapigroupv1.CarpList{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + ListMeta: metav1.ListMeta{ + ResourceVersion: "2345", + }, + Items: []testapigroupv1.Carp{ + {TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Carp"}, ObjectMeta: metav1.ObjectMeta{ + Name: "pod", + Namespace: "default", + }}, + {TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Carp"}, ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default2", + }}, + }, + }, + expect: `{"kind":"List","apiVersion":"v1","metadata":{"resourceVersion":"2345"},"items":[{"kind":"Carp","apiVersion":"v1","metadata":{"name":"pod","namespace":"default","creationTimestamp":null},"spec":{},"status":{}},{"kind":"Carp","apiVersion":"v1","metadata":{"name":"pod2","namespace":"default2","creationTimestamp":null},"spec":{},"status":{}}]} +`, + }, + { + name: "List with extra field cannot be streamed", + in: &ListWithAdditionalFields{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + ListMeta: metav1.ListMeta{ + ResourceVersion: "2345", + }, + Items: []testapigroupv1.Carp{}, + }, + cannotStream: true, + expect: "{\"kind\":\"List\",\"apiVersion\":\"v1\",\"metadata\":{\"resourceVersion\":\"2345\"},\"items\":[],\"AdditionalField\":0}\n", + }, + { + name: "Not a collection cannot be streamed", + in: &testapigroupv1.Carp{ + TypeMeta: metav1.TypeMeta{ + Kind: "List", + APIVersion: "v1", + }, + }, + cannotStream: true, + expect: "{\"kind\":\"List\",\"apiVersion\":\"v1\",\"metadata\":{\"creationTimestamp\":null},\"spec\":{},\"status\":{}}\n", + }, + { + name: "UnstructuredList empty", + in: &unstructured.UnstructuredList{}, + expect: "{\"items\":[]}\n", + }, + { + name: "UnstructuredList just kind", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{"kind": "List"}, + }, + expect: "{\"items\":[],\"kind\":\"List\"}\n", + }, + { + name: "UnstructuredList just apiVersion", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{"apiVersion": "v1"}, + }, + expect: "{\"apiVersion\":\"v1\",\"items\":[]}\n", + }, + { + name: "UnstructuredList no elements", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{"kind": "List", "apiVersion": "v1", "metadata": map[string]interface{}{"resourceVersion": "2345"}}, + Items: []unstructured.Unstructured{}, + }, + expect: "{\"apiVersion\":\"v1\",\"items\":[],\"kind\":\"List\",\"metadata\":{\"resourceVersion\":\"2345\"}}\n", + }, + { + name: "UnstructuredList one element with continue", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{"kind": "List", "apiVersion": "v1", "metadata": map[string]interface{}{ + "resourceVersion": "2345", + "continue": "abc", + "remainingItemCount": "1", + }}, + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Carp", + "metadata": map[string]interface{}{ + "name": "pod", + "namespace": "default", + }, + }, + }, + }, + }, + expect: "{\"apiVersion\":\"v1\",\"items\":[{\"apiVersion\":\"v1\",\"kind\":\"Carp\",\"metadata\":{\"name\":\"pod\",\"namespace\":\"default\"}}],\"kind\":\"List\",\"metadata\":{\"continue\":\"abc\",\"remainingItemCount\":\"1\",\"resourceVersion\":\"2345\"}}\n", + }, + { + name: "UnstructuredList two elements", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{"kind": "List", "apiVersion": "v1", "metadata": map[string]interface{}{ + "resourceVersion": "2345", + }}, + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Carp", + "metadata": map[string]interface{}{ + "name": "pod", + "namespace": "default", + }, + }, + }, + { + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Carp", + "metadata": map[string]interface{}{ + "name": "pod2", + "namespace": "default", + }, + }, + }, + }, + }, + expect: "{\"apiVersion\":\"v1\",\"items\":[{\"apiVersion\":\"v1\",\"kind\":\"Carp\",\"metadata\":{\"name\":\"pod\",\"namespace\":\"default\"}},{\"apiVersion\":\"v1\",\"kind\":\"Carp\",\"metadata\":{\"name\":\"pod2\",\"namespace\":\"default\"}}],\"kind\":\"List\",\"metadata\":{\"resourceVersion\":\"2345\"}}\n", + }, + { + name: "UnstructuredList conflict on items", + in: &unstructured.UnstructuredList{ + Object: map[string]interface{}{"items": []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "name": "pod", + }, + }, + }, + }, + Items: []unstructured.Unstructured{ + { + Object: map[string]interface{}{ + "name": "pod2", + }, + }, + }, + }, + expect: "{\"items\":[{\"name\":\"pod2\"}]}\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + buf.Reset() + if err := s.Encode(tc.in, &buf); err != nil { + t.Fatalf("unexpected error: %v", err) + } + t.Logf("encoded: %s", buf.String()) + if diff := cmp.Diff(buf.String(), tc.expect); diff != "" { + t.Errorf("not matching:\n%s", diff) + } + expectStreaming := !tc.cannotStream && streamingEnabled + if expectStreaming && buf.writeCount <= 1 { + t.Errorf("expected streaming but Write was called only: %d", buf.writeCount) + } + if !expectStreaming && buf.writeCount > 1 { + t.Errorf("expected non-streaming but Write was called more than once: %d", buf.writeCount) + } + }) + } +} + +type StructWithFloatsList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + Items []StructWithFloats `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +func (l *StructWithFloatsList) DeepCopyObject() runtime.Object { + return nil +} + +type StructWithFloats struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + Int int + Float32 float32 + Float64 float64 +} + +func (s *StructWithFloats) DeepCopyObject() runtime.Object { + return nil +} + +type StructWithDuplicatedTagsList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + Items []StructWithDuplicatedTags `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +func (l *StructWithDuplicatedTagsList) DeepCopyObject() runtime.Object { + return nil +} + +type StructWithDuplicatedTags struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + + Key1 string `json:"key"` + Key2 string `json:"key"` //nolint:govet +} + +func (s *StructWithDuplicatedTags) DeepCopyObject() runtime.Object { + return nil +} + +type ListWithMarshalJSONList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + Items []string `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +func (l *ListWithMarshalJSONList) DeepCopyObject() runtime.Object { + return nil +} + +func (l *ListWithMarshalJSONList) MarshalJSON() ([]byte, error) { + return []byte(`"marshallJSON"`), nil +} + +type StructWithMarshalJSONList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + Items []StructWithMarshalJSON `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +func (s *StructWithMarshalJSONList) DeepCopyObject() runtime.Object { + return nil +} + +type StructWithMarshalJSON struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` +} + +func (l *StructWithMarshalJSON) DeepCopyObject() runtime.Object { + return nil +} + +func (l *StructWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(`"marshallJSON"`), nil +} + +type StructWithRawBytesList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + Items []StructWithRawBytes `json:"items" protobuf:"bytes,2,rep,name=items"` +} + +func (s *StructWithRawBytesList) DeepCopyObject() runtime.Object { + return nil +} + +type StructWithRawBytes struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + Slice []byte + Array [3]byte +} + +func (s *StructWithRawBytes) DeepCopyObject() runtime.Object { + return nil +} + +type ListWithAdditionalFields struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` + Items []testapigroupv1.Carp `json:"items" protobuf:"bytes,2,rep,name=items"` + AdditionalField int +} + +func (s *ListWithAdditionalFields) DeepCopyObject() runtime.Object { + return nil +} + +type writeCountingBuffer struct { + writeCount int + bytes.Buffer +} + +func (b *writeCountingBuffer) Write(data []byte) (int, error) { + b.writeCount++ + return b.Buffer.Write(data) +} + +func (b *writeCountingBuffer) Reset() { + b.writeCount = 0 + b.Buffer.Reset() +} + +func TestFuzzCollectionsEncoding(t *testing.T) { + disableFuzzFieldsV1 := func(field *metav1.FieldsV1, c fuzz.Continue) {} + fuzzUnstructuredList := func(list *unstructured.UnstructuredList, c fuzz.Continue) { + list.Object = map[string]interface{}{ + "kind": "List", + "apiVersion": "v1", + c.RandString(): c.RandString(), + c.RandString(): c.RandUint64(), + c.RandString(): c.RandBool(), + "metadata": map[string]interface{}{ + "resourceVersion": fmt.Sprintf("%d", c.RandUint64()), + "continue": c.RandString(), + "remainingItemCount": fmt.Sprintf("%d", c.RandUint64()), + c.RandString(): c.RandString(), + }} + c.Fuzz(&list.Items) + } + fuzzMap := func(kvs map[string]interface{}, c fuzz.Continue) { + kvs[c.RandString()] = c.RandBool() + kvs[c.RandString()] = c.RandUint64() + kvs[c.RandString()] = c.RandString() + } + f := fuzz.New().Funcs(disableFuzzFieldsV1, fuzzUnstructuredList, fuzzMap) + streamingBuffer := &bytes.Buffer{} + normalSerializer := NewSerializerWithOptions(DefaultMetaFactory, nil, nil, SerializerOptions{StreamingCollectionsEncoding: false}) + normalBuffer := &bytes.Buffer{} + t.Run("CarpList", func(t *testing.T) { + for i := 0; i < 1000; i++ { + list := &testapigroupv1.CarpList{} + f.Fuzz(list) + streamingBuffer.Reset() + normalBuffer.Reset() + ok, err := streamEncodeCollections(list, streamingBuffer) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ok { + t.Fatalf("expected streaming encoder to encode %T", list) + } + if err := normalSerializer.Encode(list, normalBuffer); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(normalBuffer.String(), streamingBuffer.String()); diff != "" { + t.Logf("normal: %s", normalBuffer.String()) + t.Logf("streaming: %s", streamingBuffer.String()) + t.Errorf("not matching:\n%s", diff) + } + } + }) + t.Run("UnstructuredList", func(t *testing.T) { + for i := 0; i < 1000; i++ { + list := &unstructured.UnstructuredList{} + f.Fuzz(list) + streamingBuffer.Reset() + normalBuffer.Reset() + ok, err := streamEncodeCollections(list, streamingBuffer) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !ok { + t.Fatalf("expected streaming encoder to encode %T", list) + } + if err := normalSerializer.Encode(list, normalBuffer); err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(normalBuffer.String(), streamingBuffer.String()); diff != "" { + t.Logf("normal: %s", normalBuffer.String()) + t.Logf("streaming: %s", streamingBuffer.String()) + t.Errorf("not matching:\n%s", diff) + } + } + }) +} diff --git a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go index 1ae4a32eb720c..24f66a10174b0 100644 --- a/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go +++ b/staging/src/k8s.io/apimachinery/pkg/runtime/serializer/json/json.go @@ -36,7 +36,7 @@ import ( // is not nil, the object has the group, version, and kind fields set. // Deprecated: use NewSerializerWithOptions instead. func NewSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper, pretty bool) *Serializer { - return NewSerializerWithOptions(meta, creater, typer, SerializerOptions{false, pretty, false}) + return NewSerializerWithOptions(meta, creater, typer, SerializerOptions{false, pretty, false, false}) } // NewYAMLSerializer creates a YAML serializer that handles encoding versioned objects into the proper YAML form. If typer @@ -44,7 +44,7 @@ func NewSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtim // matches JSON, and will error if constructs are used that do not serialize to JSON. // Deprecated: use NewSerializerWithOptions instead. func NewYAMLSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper) *Serializer { - return NewSerializerWithOptions(meta, creater, typer, SerializerOptions{true, false, false}) + return NewSerializerWithOptions(meta, creater, typer, SerializerOptions{true, false, false, false}) } // NewSerializerWithOptions creates a JSON/YAML serializer that handles encoding versioned objects into the proper JSON/YAML @@ -93,6 +93,9 @@ type SerializerOptions struct { // Strict: configures the Serializer to return strictDecodingError's when duplicate fields are present decoding JSON or YAML. // Note that enabling this option is not as performant as the non-strict variant, and should not be used in fast paths. Strict bool + + // StreamingCollectionsEncoding enables encoding collection, one item at the time, drastically reducing memory needed. + StreamingCollectionsEncoding bool } // Serializer handles encoding versioned objects into the proper JSON form @@ -242,6 +245,15 @@ func (s *Serializer) doEncode(obj runtime.Object, w io.Writer) error { _, err = w.Write(data) return err } + if s.options.StreamingCollectionsEncoding { + ok, err := streamEncodeCollections(obj, w) + if err != nil { + return err + } + if ok { + return nil + } + } encoder := json.NewEncoder(w) return encoder.Encode(obj) } diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go index acd8f0357aaf8..acc95bdc65e59 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go @@ -156,6 +156,9 @@ const ( // (usually the entire object), and if the size is smaller no gzipping will be performed // if the client requests it. defaultGzipThresholdBytes = 128 * 1024 + // Use the length of the first write of streaming implementations. + // TODO: Update when streaming proto is implemented + firstWriteStreamingThresholdBytes = 1 ) // negotiateContentEncoding returns a supported client-requested content encoding for the @@ -191,14 +194,53 @@ type deferredResponseWriter struct { statusCode int contentEncoding string - hasWritten bool - hw http.ResponseWriter - w io.Writer + hasBuffered bool + buffer []byte + hasWritten bool + hw http.ResponseWriter + w io.Writer ctx context.Context } func (w *deferredResponseWriter) Write(p []byte) (n int, err error) { + switch { + case w.hasWritten: + // already written, cannot buffer + return w.unbufferedWrite(p) + + case w.contentEncoding != "gzip": + // non-gzip, no need to buffer + return w.unbufferedWrite(p) + + case !w.hasBuffered && len(p) > defaultGzipThresholdBytes: + // not yet buffered, first write is long enough to trigger gzip, no need to buffer + return w.unbufferedWrite(p) + + case !w.hasBuffered && len(p) > firstWriteStreamingThresholdBytes: + // not yet buffered, first write is longer than expected for streaming scenarios that would require buffering, no need to buffer + return w.unbufferedWrite(p) + + default: + if !w.hasBuffered { + w.hasBuffered = true + // Start at 80 bytes to avoid rapid reallocation of the buffer. + // The minimum size of a 0-item serialized list object is 80 bytes: + // {"kind":"List","apiVersion":"v1","metadata":{"resourceVersion":"1"},"items":[]}\n + w.buffer = make([]byte, 0, max(80, len(p))) + } + w.buffer = append(w.buffer, p...) + var err error + if len(w.buffer) > defaultGzipThresholdBytes { + // we've accumulated enough to trigger gzip, write and clear buffer + _, err = w.unbufferedWrite(w.buffer) + w.buffer = nil + } + return len(p), err + } +} + +func (w *deferredResponseWriter) unbufferedWrite(p []byte) (n int, err error) { ctx := w.ctx span := tracing.SpanFromContext(ctx) // This Step usually wraps in-memory object serialization. @@ -244,11 +286,17 @@ func (w *deferredResponseWriter) Write(p []byte) (n int, err error) { return w.w.Write(p) } -func (w *deferredResponseWriter) Close() error { +func (w *deferredResponseWriter) Close() (err error) { if !w.hasWritten { - return nil + if !w.hasBuffered { + return nil + } + // never reached defaultGzipThresholdBytes, no need to do the gzip writer cleanup + _, err := w.unbufferedWrite(w.buffer) + w.buffer = nil + return err } - var err error + switch t := w.w.(type) { case *gzip.Writer: err = t.Close() diff --git a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers_test.go b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers_test.go index 2fdcfc58ad0c0..c92d4847524a9 100644 --- a/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers_test.go +++ b/staging/src/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers_test.go @@ -19,6 +19,7 @@ package responsewriters import ( "bytes" "compress/gzip" + "context" "encoding/hex" "encoding/json" "errors" @@ -38,8 +39,12 @@ import ( "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + testapigroupv1 "k8s.io/apimachinery/pkg/apis/testapigroup/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + jsonserializer "k8s.io/apimachinery/pkg/runtime/serializer/json" + rand2 "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" @@ -374,6 +379,261 @@ func TestSerializeObject(t *testing.T) { } } +func TestDeferredResponseWriter_Write(t *testing.T) { + smallChunk := bytes.Repeat([]byte("b"), defaultGzipThresholdBytes-1) + largeChunk := bytes.Repeat([]byte("b"), defaultGzipThresholdBytes+1) + + tests := []struct { + name string + chunks [][]byte + expectGzip bool + expectHeaders http.Header + }{ + { + name: "no writes", + chunks: nil, + expectGzip: false, + expectHeaders: http.Header{}, + }, + { + name: "one empty write", + chunks: [][]byte{{}}, + expectGzip: false, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + }, + }, + { + name: "one single byte write", + chunks: [][]byte{{'{'}}, + expectGzip: false, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + }, + }, + { + name: "one small chunk write", + chunks: [][]byte{smallChunk}, + expectGzip: false, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + }, + }, + { + name: "two small chunk writes", + chunks: [][]byte{smallChunk, smallChunk}, + expectGzip: false, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + }, + }, + { + name: "one single byte and one small chunk write", + chunks: [][]byte{{'{'}, smallChunk}, + expectGzip: false, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + }, + }, + { + name: "two single bytes and one small chunk write", + chunks: [][]byte{{'{'}, {'{'}, smallChunk}, + expectGzip: true, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + "Content-Encoding": []string{"gzip"}, + "Vary": []string{"Accept-Encoding"}, + }, + }, + { + name: "one large chunk writes", + chunks: [][]byte{largeChunk}, + expectGzip: true, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + "Content-Encoding": []string{"gzip"}, + "Vary": []string{"Accept-Encoding"}, + }, + }, + { + name: "two large chunk writes", + chunks: [][]byte{largeChunk, largeChunk}, + expectGzip: true, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + "Content-Encoding": []string{"gzip"}, + "Vary": []string{"Accept-Encoding"}, + }, + }, + { + name: "one small chunk and one large chunk write", + chunks: [][]byte{smallChunk, largeChunk}, + expectGzip: false, + expectHeaders: http.Header{ + "Content-Type": []string{"text/plain"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockResponseWriter := httptest.NewRecorder() + + drw := &deferredResponseWriter{ + mediaType: "text/plain", + statusCode: 200, + contentEncoding: "gzip", + hw: mockResponseWriter, + ctx: context.Background(), + } + + fullPayload := []byte{} + + for _, chunk := range tt.chunks { + n, err := drw.Write(chunk) + + if err != nil { + t.Fatalf("unexpected error while writing chunk: %v", err) + } + if n != len(chunk) { + t.Errorf("write is not complete, expected: %d bytes, written: %d bytes", len(chunk), n) + } + + fullPayload = append(fullPayload, chunk...) + } + + err := drw.Close() + if err != nil { + t.Fatalf("unexpected error when closing deferredResponseWriter: %v", err) + } + + res := mockResponseWriter.Result() + + if res.StatusCode != http.StatusOK { + t.Fatalf("status code is not writtend properly, expected: 200, got: %d", res.StatusCode) + } + if !reflect.DeepEqual(res.Header, tt.expectHeaders) { + t.Fatal(cmp.Diff(tt.expectHeaders, res.Header)) + } + + resBytes, err := io.ReadAll(res.Body) + if err != nil { + t.Fatalf("unexpected error occurred while reading response body: %v", err) + } + + if tt.expectGzip { + gr, err := gzip.NewReader(bytes.NewReader(resBytes)) + if err != nil { + t.Fatalf("failed to create gzip reader: %v", err) + } + + decompressed, err := io.ReadAll(gr) + if err != nil { + t.Fatalf("failed to decompress: %v", err) + } + + if !bytes.Equal(fullPayload, decompressed) { + t.Errorf("payload mismatch, expected: %s, got: %s", fullPayload, decompressed) + } + } else { + if !bytes.Equal(fullPayload, resBytes) { + t.Errorf("payload mismatch, expected: %s, got: %s", fullPayload, resBytes) + } + } + }) + } +} + +func benchmarkChunkingGzip(b *testing.B, count int, chunk []byte) { + mockResponseWriter := httptest.NewRecorder() + mockResponseWriter.Body = nil + + drw := &deferredResponseWriter{ + mediaType: "text/plain", + statusCode: 200, + contentEncoding: "gzip", + hw: mockResponseWriter, + ctx: context.Background(), + } + b.ResetTimer() + for i := 0; i < count; i++ { + n, err := drw.Write(chunk) + if err != nil { + b.Fatalf("unexpected error while writing chunk: %v", err) + } + if n != len(chunk) { + b.Errorf("write is not complete, expected: %d bytes, written: %d bytes", len(chunk), n) + } + } + err := drw.Close() + if err != nil { + b.Fatalf("unexpected error when closing deferredResponseWriter: %v", err) + } + res := mockResponseWriter.Result() + if res.StatusCode != http.StatusOK { + b.Fatalf("status code is not writtend properly, expected: 200, got: %d", res.StatusCode) + } +} + +func BenchmarkChunkingGzip(b *testing.B) { + tests := []struct { + count int + size int + }{ + { + count: 100, + size: 1_000, + }, + { + count: 100, + size: 100_000, + }, + { + count: 1_000, + size: 100_000, + }, + { + count: 1_000, + size: 1_000_000, + }, + { + count: 10_000, + size: 100_000, + }, + { + count: 100_000, + size: 10_000, + }, + { + count: 1, + size: 100_000, + }, + { + count: 1, + size: 1_000_000, + }, + { + count: 1, + size: 10_000_000, + }, + { + count: 1, + size: 100_000_000, + }, + { + count: 1, + size: 1_000_000_000, + }, + } + + for _, t := range tests { + b.Run(fmt.Sprintf("Count=%d/Size=%d", t.count, t.size), func(b *testing.B) { + chunk := []byte(rand2.String(t.size)) + benchmarkChunkingGzip(b, t.count, chunk) + }) + } +} + func randTime(t *time.Time, r *rand.Rand) { *t = time.Unix(r.Int63n(1000*365*24*60*60), r.Int63()) } @@ -550,3 +810,80 @@ func gzipContent(data []byte, level int) []byte { } return buf.Bytes() } + +func TestStreamingGzipIntegration(t *testing.T) { + largeChunk := bytes.Repeat([]byte("b"), defaultGzipThresholdBytes+1) + tcs := []struct { + name string + serializer runtime.Encoder + object runtime.Object + expectGzip bool + expectStreaming bool + }{ + { + name: "JSON, small object, default -> no gzip", + serializer: jsonserializer.NewSerializerWithOptions(jsonserializer.DefaultMetaFactory, nil, nil, jsonserializer.SerializerOptions{}), + object: &testapigroupv1.CarpList{}, + expectGzip: false, + expectStreaming: false, + }, + { + name: "JSON, small object, streaming -> no gzip", + serializer: jsonserializer.NewSerializerWithOptions(jsonserializer.DefaultMetaFactory, nil, nil, jsonserializer.SerializerOptions{StreamingCollectionsEncoding: true}), + object: &testapigroupv1.CarpList{}, + expectGzip: false, + expectStreaming: true, + }, + { + name: "JSON, large object, default -> gzip", + serializer: jsonserializer.NewSerializerWithOptions(jsonserializer.DefaultMetaFactory, nil, nil, jsonserializer.SerializerOptions{}), + object: &testapigroupv1.CarpList{TypeMeta: metav1.TypeMeta{Kind: string(largeChunk)}}, + expectGzip: true, + expectStreaming: false, + }, + { + name: "JSON, large object, streaming -> gzip", + serializer: jsonserializer.NewSerializerWithOptions(jsonserializer.DefaultMetaFactory, nil, nil, jsonserializer.SerializerOptions{StreamingCollectionsEncoding: true}), + object: &testapigroupv1.CarpList{TypeMeta: metav1.TypeMeta{Kind: string(largeChunk)}}, + expectGzip: true, + expectStreaming: true, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + mockResponseWriter := httptest.NewRecorder() + drw := &deferredResponseWriter{ + mediaType: "text/plain", + statusCode: 200, + contentEncoding: "gzip", + hw: mockResponseWriter, + ctx: context.Background(), + } + counter := &writeCounter{Writer: drw} + err := tc.serializer.Encode(tc.object, counter) + if err != nil { + t.Fatal(err) + } + encoding := mockResponseWriter.Header().Get("Content-Encoding") + if (encoding == "gzip") != tc.expectGzip { + t.Errorf("Expect gzip: %v, got: %q", tc.expectGzip, encoding) + } + if counter.writeCount < 1 { + t.Fatalf("Expect at least 1 write") + } + if (counter.writeCount > 1) != tc.expectStreaming { + t.Errorf("Expect streaming: %v, got write count: %d", tc.expectStreaming, counter.writeCount) + } + }) + } +} + +type writeCounter struct { + writeCount int + io.Writer +} + +func (b *writeCounter) Write(data []byte) (int, error) { + b.writeCount++ + return b.Writer.Write(data) +} diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index e524e0c6474c1..b76a0af56ea54 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -218,6 +218,10 @@ const ( // document. StorageVersionHash featuregate.Feature = "StorageVersionHash" + // owner: @serathius + // Allow API server to encode collections item by item, instead of all at once. + StreamingCollectionEncodingToJSON featuregate.Feature = "StreamingCollectionEncodingToJSON" + // owner: @aramase, @enj, @nabokihms // kep: https://kep.k8s.io/3331 // alpha: v1.29 @@ -328,6 +332,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS StructuredAuthenticationConfiguration: {Default: false, PreRelease: featuregate.Alpha}, + StreamingCollectionEncodingToJSON: {Default: true, PreRelease: featuregate.Beta}, + StructuredAuthorizationConfiguration: {Default: false, PreRelease: featuregate.Alpha}, UnauthenticatedHTTP2DOSMitigation: {Default: true, PreRelease: featuregate.Beta}, diff --git a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go index 4b1ca59471297..b01fe282a02c2 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go +++ b/staging/src/k8s.io/apiserver/pkg/server/genericapiserver.go @@ -979,6 +979,13 @@ func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupV // NewDefaultAPIGroupInfo returns an APIGroupInfo stubbed with "normal" values // exposed for easier composition from other packages func NewDefaultAPIGroupInfo(group string, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo { + opts := []serializer.CodecFactoryOptionsMutator{} + if utilfeature.DefaultFeatureGate.Enabled(features.StreamingCollectionEncodingToJSON) { + opts = append(opts, serializer.WithStreamingCollectionEncodingToJSON()) + } + if len(opts) != 0 { + codecs = serializer.NewCodecFactory(scheme, opts...) + } return APIGroupInfo{ PrioritizedVersions: scheme.PrioritizedVersionsForGroup(group), VersionedResourcesStorageMap: map[string]map[string]rest.Storage{}, diff --git a/staging/src/k8s.io/kms/internal/plugins/_mock/go.work.sum b/staging/src/k8s.io/kms/internal/plugins/_mock/go.work.sum new file mode 100644 index 0000000000000..1e8555df00ad2 --- /dev/null +++ b/staging/src/k8s.io/kms/internal/plugins/_mock/go.work.sum @@ -0,0 +1,122 @@ +cel.dev/expr v0.15.0 h1:O1jzfJCQBfL5BFoYktaxwIhuttaQPsVWerH9/EEKx0w= +cel.dev/expr v0.15.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= +cel.dev/expr v0.16.0 h1:yloc84fytn4zmJX2GU3TkXGsaieaV7dQ057Qs4sIG2Y= +cel.dev/expr v0.16.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= +cel.dev/expr v0.16.1 h1:NR0+oFYzR1CqLFhTAqg3ql59G9VfN8fKq1TCHJ6gq1g= +cel.dev/expr v0.16.1/go.mod h1:AsGA5zb3WruAEQeQng1RZdGEXmBj0jvMWh6l5SnNuC8= +cel.dev/expr v0.20.0 h1:OunBvVCfvpWlt4dN7zg3FM6TDkzOePe1+foGJ9AXeeI= +cel.dev/expr v0.20.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.26.0 h1:f2Qw/Ehhimh5uO1fayV0QIW7DShEQqhtUfhYc+cBPlw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.26.0/go.mod h1:2bIszWvQRlJVmJLiuLhukLImRjKPcYdzzsx6darK02A= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b h1:ga8SEFjZ60pxLcmhnThWgvH2wg8376yUJmPhEH4H3kw= +github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20 h1:N+3sFI5GUjRKBi+i0TxYVST9h4Ie192jJWpHvthBBgg= +github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 h1:QVw89YDxXxEe+l8gU8ETbOasdwEV+avkR75ZzsVV9WI= +github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42 h1:Om6kYQYDUk5wWbT0t0q6pvyM49i9XZAv9dDrkDA7gjk= +github.com/cncf/xds/go v0.0.0-20250121191232-2f005788dc42/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= +github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= +github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= +github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= +github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= +github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.opentelemetry.io/contrib/detectors/gcp v1.34.0 h1:JRxssobiPg23otYU5SbWtQC//snGVIM3Tx6QRzlQBao= +go.opentelemetry.io/contrib/detectors/gcp v1.34.0/go.mod h1:cV4BMFcscUR/ckqLkbfQmF0PRsq8w/lMGzdbCSveBHo= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE= +golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a h1:nwKuGPlUAt+aR+pcrkfFRrTU1BVrSmYyYMxYbUIVHr0= +google.golang.org/genproto/googleapis/api v0.0.0-20250218202821-56aae31c358a/go.mod h1:3kWAYMk1I75K4vykHtKt2ycnOgpA6974V7bREqbsenU= diff --git a/test/featuregates_linter/test_data/versioned_feature_list.yaml b/test/featuregates_linter/test_data/versioned_feature_list.yaml new file mode 100644 index 0000000000000..0db254cb4203e --- /dev/null +++ b/test/featuregates_linter/test_data/versioned_feature_list.yaml @@ -0,0 +1,1546 @@ +- name: AdmissionWebhookMatchConditions + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.28" + - default: true + lockToDefault: true + preRelease: GA + version: "1.30" +- name: AggregatedDiscoveryEndpoint + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" + - default: true + lockToDefault: true + preRelease: GA + version: "1.30" +- name: AllowDNSOnlyNodeCSR + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.0" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.31" +- name: AllowInsecureKubeletCertificateSigningRequests + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.0" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.31" +- name: AllowOverwriteTerminationGracePeriodSeconds + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.0" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.32" +- name: AllowParsingUserUIDFromCertAuth + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.33" +- name: AllowServiceLBStatusOnNonLB + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.0" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.29" + - default: false + lockToDefault: true + preRelease: Deprecated + version: "1.32" +- name: AllowUnsafeMalformedObjectDeletion + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: AnonymousAuthConfigurableEndpoints + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.31" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: AnyVolumeDataSource + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.18" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.24" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" +- name: APIResponseCompression + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.8" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.16" +- name: APIServerIdentity + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.20" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.26" +- name: APIServerTracing + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.22" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" +- name: APIServingWithRoutine + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" +- name: AppArmor + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.4" + - default: true + lockToDefault: true + preRelease: GA + version: "1.31" +- name: AuthorizeNodeWithSelectors + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.31" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: AuthorizeWithSelectors + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.31" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: BtreeWatchCache + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" +- name: CBORServingAndStorage + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: CloudControllerManagerWebhook + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" +- name: ClusterTrustBundle + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" +- name: ClusterTrustBundleProjection + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" +- name: ComponentFlagz + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: ComponentSLIs + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: ComponentStatusz + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: ConcurrentWatchObjectDecode + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: ConsistentListFromCache + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: ContainerCheckpoint + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.25" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" +- name: CoordinatedLeaderElection + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.31" +- name: CPUCFSQuotaPeriod + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.12" +- name: CPUManager + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.8" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.10" + - default: true + lockToDefault: true + preRelease: GA + version: "1.26" +- name: CPUManagerPolicyAlphaOptions + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" +- name: CPUManagerPolicyBetaOptions + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.23" +- name: CPUManagerPolicyOptions + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.22" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.23" +- name: CRDValidationRatcheting + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" +- name: CronJobsScheduledAnnotation + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.28" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: CrossNamespaceVolumeDataSource + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" +- name: CSIMigrationPortworx + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.25" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" +- name: CSIVolumeHealth + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.21" +- name: CustomResourceFieldSelectors + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: DeploymentPodReplacementPolicy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: DevicePluginCDIDevices + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.29" + - default: true + lockToDefault: true + preRelease: GA + version: "1.31" +- name: DisableAllocatorDualWrite + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.31" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.33" +- name: DisableCPUQuotaWithExclusiveCPUs + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.33" +- name: DisableNodeKubeProxyVersion + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.31" +- name: DRAAdminAccess + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: DRAResourceClaimDeviceStatus + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: DynamicResourceAllocation + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: ElasticIndexedJob + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" + - default: true + lockToDefault: true + preRelease: GA + version: "1.31" +- name: EventedPLEG + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" +- name: ExecProbeTimeout + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.20" +- name: ExternalServiceAccountTokenSigner + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: GracefulNodeShutdown + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.20" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.21" +- name: GracefulNodeShutdownBasedOnPodPriority + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.24" +- name: HonorPVReclaimPolicy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: HPAScaleToZero + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.16" +- name: ImageMaximumGCAge + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" +- name: ImageVolume + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.31" +- name: InPlacePodVerticalScaling + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" +- name: InPlacePodVerticalScalingAllocatedStatus + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: InPlacePodVerticalScalingExclusiveCPUs + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: InTreePluginPortworxUnregister + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" +- name: JobBackoffLimitPerIndex + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.29" +- name: JobManagedBy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: JobPodReplacementPolicy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.29" +- name: JobSuccessPolicy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: KMSv1 + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.0" + - default: true + lockToDefault: false + preRelease: Deprecated + version: "1.28" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.29" +- name: KubeletCgroupDriverFromCRI + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: KubeletCrashLoopBackOffMax + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: KubeletFineGrainedAuthz + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.33" +- name: KubeletInUserNamespace + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.22" +- name: KubeletPodResourcesDynamicResources + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" +- name: KubeletPodResourcesGet + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" +- name: KubeletRegistrationGetOnExistsOnly + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.0" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.32" +- name: KubeletSeparateDiskGC + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: KubeletTracing + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.25" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" +- name: LegacySidecarContainers + versionedSpecs: + - default: true + lockToDefault: false + preRelease: GA + version: "1.0" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.33" +- name: ListFromCacheSnapshot + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.33" +- name: LoadBalancerIPMode + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: LocalStorageCapacityIsolationFSQuotaMonitoring + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.15" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: LogarithmicScaleDown + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.21" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.22" + - default: true + lockToDefault: true + preRelease: GA + version: "1.31" +- name: MatchLabelKeysInPodAffinity + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: MatchLabelKeysInPodTopologySpread + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.25" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" +- name: MaxUnavailableStatefulSet + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.24" +- name: MemoryManager + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.21" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.22" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: MemoryQoS + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.22" +- name: MultiCIDRServiceAllocator + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.31" + - default: true + lockToDefault: false + preRelease: GA + version: "1.33" +- name: MutatingAdmissionPolicy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: NFTablesProxyMode + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" +- name: NodeInclusionPolicyInPodTopologySpread + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.25" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.26" +- name: NodeLogQuery + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.30" +- name: NodeSwap + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.22" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" +- name: OpenAPIEnums + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.24" +- name: OrderedNamespaceDeletion + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.33" +- name: PersistentVolumeLastPhaseTransitionTime + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.29" + - default: true + lockToDefault: true + preRelease: GA + version: "1.31" +- name: PodAndContainerStatsFromCRI + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" +- name: PodDeletionCost + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.21" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.22" +- name: PodDisruptionConditions + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.25" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.26" + - default: true + lockToDefault: true + preRelease: GA + version: "1.31" +- name: PodIndexLabel + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.28" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: PodLevelResources + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: PodLifecycleSleepAction + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" +- name: PodLifecycleSleepActionAllowZero + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: PodLogsQuerySplitStreams + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: PodReadyToStartContainersCondition + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.29" +- name: PodSchedulingReadiness + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" + - default: true + lockToDefault: true + preRelease: GA + version: "1.30" +- name: PortForwardWebsockets + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: ProcMountType + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.12" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: QOSReserved + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.11" +- name: RecoverVolumeExpansionFailure + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: RecursiveReadOnlyMounts + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: RelaxedDNSSearchValidation + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.33" +- name: RelaxedEnvironmentVariableValidation + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: ReloadKubeletServerCertificateFile + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: RemainingItemCount + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.15" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.16" + - default: true + lockToDefault: true + preRelease: GA + version: "1.29" +- name: RemoteRequestHeaderUID + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: ResilientWatchCacheInitialization + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: ResourceHealthStatus + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.31" +- name: RetryGenerateName + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: RotateKubeletServerCertificate + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.7" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.12" +- name: RuntimeClassInImageCriAPI + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" +- name: SchedulerAsyncPreemption + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: SchedulerQueueingHints + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: SELinuxChangePolicy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: SELinuxMount + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" +- name: SELinuxMountReadWriteOncePod + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.25" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.27" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.28" +- name: SeparateCacheWatchRPC + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.28" + - default: false + lockToDefault: false + preRelease: Deprecated + version: "1.33" +- name: SeparateTaintEvictionController + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.29" +- name: ServiceAccountNodeAudienceRestriction + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.32" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.33" +- name: ServiceAccountTokenJTI + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: ServiceAccountTokenNodeBinding + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" +- name: ServiceAccountTokenNodeBindingValidation + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: ServiceAccountTokenPodNodeInfo + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: ServiceTrafficDistribution + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: SidecarContainers + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.29" + - default: true + lockToDefault: true + preRelease: GA + version: "1.33" +- name: SizeMemoryBackedVolumes + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.20" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.22" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: StatefulSetAutoDeletePVC + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.23" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: StatefulSetStartOrdinal + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.27" + - default: true + lockToDefault: true + preRelease: GA + version: "1.31" +- name: StorageNamespaceIndex + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: false + preRelease: Deprecated + version: "1.33" +- name: StorageVersionAPI + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.20" +- name: StorageVersionHash + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.14" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.15" +- name: StorageVersionMigrator + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.30" +- name: StreamingCollectionEncodingToJSON + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.33" +- name: StrictCostEnforcementForVAP + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: StrictCostEnforcementForWebhooks + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: StructuredAuthenticationConfiguration + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" +- name: StructuredAuthorizationConfiguration + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" + - default: true + lockToDefault: true + preRelease: GA + version: "1.32" +- name: SupplementalGroupsPolicy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.31" +- name: SystemdWatchdog + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: TopologyAwareHints + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.21" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.23" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.24" +- name: TopologyManagerPolicyAlphaOptions + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" +- name: TopologyManagerPolicyBetaOptions + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.26" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.28" +- name: TopologyManagerPolicyOptions + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.26" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.28" + - default: true + lockToDefault: false + preRelease: GA + version: "1.32" +- name: TranslateStreamCloseWebsocketRequests + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.30" +- name: UnauthenticatedHTTP2DOSMitigation + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.25" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.29" +- name: UnknownVersionInteroperabilityProxy + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.28" +- name: UserNamespacesPodSecurityStandards + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" +- name: UserNamespacesSupport + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.25" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.30" +- name: VolumeAttributesClass + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.29" + - default: false + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: VolumeCapacityPriority + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.21" +- name: WatchCacheInitializationPostStartHook + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.31" +- name: WatchFromStorageWithoutResourceVersion + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Beta + version: "1.27" + - default: false + lockToDefault: true + preRelease: Deprecated + version: "1.33" +- name: WatchList + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.27" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.32" +- name: WindowsCPUAndMemoryAffinity + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: WindowsGracefulNodeShutdown + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.32" +- name: WindowsHostNetwork + versionedSpecs: + - default: true + lockToDefault: false + preRelease: Alpha + version: "1.26" +- name: WinDSR + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.14" +- name: WinOverlay + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.14" + - default: true + lockToDefault: false + preRelease: Beta + version: "1.20"