diff --git a/.github/workflows/verify_generated_files.yml b/.github/workflows/verify_generated_files.yml index aa85fcff5..b99a9a060 100644 --- a/.github/workflows/verify_generated_files.yml +++ b/.github/workflows/verify_generated_files.yml @@ -7,6 +7,7 @@ on: - '**.go' - '**go.mod' - '**go.sum' + - 'config/**' tags-ignore: - 'v*' pull_request: @@ -14,6 +15,7 @@ on: - '**.go' - '**go.mod' - '**go.sum' + - 'config/**' jobs: verify-imports: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 4dcd3618f..00de9c196 100644 --- a/Makefile +++ b/Makefile @@ -146,9 +146,12 @@ defaults: gofmt -w $(DEFAULTS_TEST_FILE) +# this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests -manifests: controller-gen ## Generate RBAC objects. +manifests: controller-gen kustomize ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." +# $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml +# $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml && make split_yaml FILE=config/crd/mcad.yaml .PHONY: fmt fmt: ## Run go fmt against code. @@ -192,19 +195,16 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl apply -f - git restore config/* .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - git restore config/* .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} $(KUSTOMIZE) build config/${ENV} | kubectl apply -f - git restore config/* @@ -224,6 +224,7 @@ $(LOCALBIN): ## Tool Binaries KUSTOMIZE ?= $(LOCALBIN)/kustomize +YQ ?= $(LOCALBIN)/yq CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest OPENSHIFT-GOIMPORTS ?= $(LOCALBIN)/openshift-goimports @@ -284,7 +285,6 @@ validate-bundle: install-operator-sdk .PHONY: bundle bundle: defaults manifests kustomize install-operator-sdk ## Generate bundle manifests and metadata, then validate generated files. $(OPERATOR_SDK) generate kustomize manifests -q - $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/metadata/annotations/containerImage", "value": "$(IMG)" }]' --kind ClusterServiceVersion cd config/manifests && $(KUSTOMIZE) edit add patch --patch '[{"op":"add", "path":"/spec/replaces", "value": "codeflare-operator.$(PREVIOUS_VERSION)" }]' --kind ClusterServiceVersion @@ -388,3 +388,32 @@ verify-imports: openshift-goimports ## Run import verifications. .PHONY: scorecard-bundle scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. $(OPERATOR_SDK) scorecard bundle + + +FILE ?= input.yaml # Default value, it isn't a file, but the make cmds fill hang for longer without it +temp_dir := temp_split +output_dir := 'config/crd/' + +.PHONY: check_yq +check_yq: + @command -v wget >/dev/null 2>&1 || (echo "Installing wget..."; apt-get install -y wget) + @command -v $(YQ) >/dev/null 2>&1 || (echo "Installing yq..."; wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64.tar.gz -O - |\ + tar xz && mv yq_linux_amd64 $(YQ)) + +# this works on a MacOS by replacing awk with gawk +.PHONY: split_yaml +split_yaml: + @$(MAKE) check_yq + @mkdir -p $(temp_dir) + @awk '/apiVersion: /{if (x>0) close("$(temp_dir)/section_" x ".yaml"); x++}{print > "$(temp_dir)/section_"x".yaml"}' $(FILE) + @$(MAKE) process_sections + +.PHONY: process_sections +process_sections: + @mkdir -p $(output_dir) + @for section_file in $(temp_dir)/section_*; do \ + metadata_name=$$(YQ e '.metadata.name' $$section_file); \ + file_name=$$(echo $$metadata_name | awk -F'.' '{print $$2"."$$3"_"$$1".yaml"}'); \ + mv $$section_file $(output_dir)/$$file_name; \ + done + @rm -r $(temp_dir) diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 81d71c1e1..73a576238 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,5 +2,8 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- mcad +- quota.codeflare_quotasubtrees.yaml +- workload.codeflare_appwrappers.yaml +- workload.codeflare_schedulingspecs.yaml + #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index 8ea96b055..e016d77f8 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v0.0.0 # kpt-set: ${MCAD_CRD} +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.36.0 diff --git a/config/crd/quota.codeflare_quotasubtrees.yaml b/config/crd/quota.codeflare_quotasubtrees.yaml new file mode 100644 index 000000000..44bb5e0a6 --- /dev/null +++ b/config/crd/quota.codeflare_quotasubtrees.yaml @@ -0,0 +1,122 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: quotasubtrees.quota.codeflare.dev +spec: + group: quota.codeflare.dev + names: + kind: QuotaSubtree + listKind: QuotaSubtreeList + plural: quotasubtrees + singular: quotasubtree + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: QuotaSubtree is a specification for a quota subtree resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: QuotaSubtreeSpec is the spec for a resource plan + properties: + children: + items: + description: Child is the spec for a QuotaSubtree resource + properties: + name: + type: string + namespace: + type: string + path: + type: string + quotas: + description: Quota is the spec for a QuotaSubtree resource + properties: + disabled: + type: boolean + hardLimit: + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: array + parent: + type: string + parentNamespace: + type: string + type: object + status: + description: QuotaSubtreeStatus is the status for a QuotaSubtree resource + properties: + children: + items: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + type: array + totalAllocation: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child + resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + required: + - children + - totalAllocation + type: object + required: + - spec + type: object + served: true + storage: true +--- diff --git a/config/crd/workload.codeflare_appwrappers.yaml b/config/crd/workload.codeflare_appwrappers.yaml new file mode 100644 index 000000000..e1d260411 --- /dev/null +++ b/config/crd/workload.codeflare_appwrappers.yaml @@ -0,0 +1,765 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: appwrappers.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: AppWrapper + listKind: AppWrapperList + plural: appwrappers + singular: appwrapper + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Definition of AppWrapper class + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AppWrapperSpec describes how the App Wrapper will look like. + properties: + priority: + format: int32 + type: integer + priorityslope: + format: float + type: number + resources: + description: a collection of AppWrapperResource + properties: + GenericItems: + items: + description: AppWrapperGenericResource is App Wrapper aggregation + resource + properties: + allocated: + description: The number of allocated replicas from this + resource type + format: int32 + type: integer + completionstatus: + description: 'Optional field that drives completion status + of this AppWrapper. This field within an item of an AppWrapper + determines the full state of the AppWrapper. The completionstatus + field contains a list of conditions that make the associate + item considered completed, for instance: - completion + conditions could be "Complete" or "Failed". The associated + item''s level .status.conditions[].type field is monitored + for any one of these conditions. Once all items with this + option is set and the conditionstatus is met the entire + AppWrapper state will be changed to one of the valid AppWrapper + completion state. Note: - this is an AND operation for + all items where this option is set. See the list of AppWrapper + states for a list of valid complete states.' + type: string + custompodresources: + description: Optional section that specifies resource requirements + for non-standard k8s resources, follows same format as + that of standard k8s resources. + items: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, + quantity) pairs. + type: object + replicas: + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'todo: replace with Containers []Container + Contain v1.ResourceRequirements' + type: object + required: + - replicas + - requests + type: object + type: array + generictemplate: + description: The template for the resource; it is now a + raw text because we don't know for what resource it should + be instantiated + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + minavailable: + description: The minimal available pods to run for this + AppWrapper; the default value is nil + format: int32 + type: integer + priority: + description: The priority of this resource + format: int32 + type: integer + priorityslope: + description: The increasing rate of priority value for this + resource + format: float + type: number + replicas: + description: Replicas is the number of desired replicas + format: int32 + type: integer + type: object + type: array + type: object + schedulingSpec: + description: SchedSpec specifies the parameters used for scheduling + generic items wrapped inside AppWrappers. It defines the policy + for requeuing jobs based on the number of running pods. + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed + state. Requeuing is triggered when the number of running/completed + pods is not equal to this value. When not specified, requeuing + is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on + waiting time. Values in this field control how often the pod + check should happen, and if requeuing has reached its maximum + number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time + between requeuing checks. The values available are 'exponential', + 'linear', or 'none'. For example, 'exponential' growth would + double the 'timeInSeconds' value every time a requeuing + event is triggered. If the string value is misspelled or + not one of the possible options, the growth behavior is + defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. + Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing + event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it + may grow as requeuing events happen. + type: integer + type: object + type: object + selector: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + service: + description: AppWrapperService is App Wrapper service definition + properties: + spec: + description: ServiceSpec describes the attributes that a user + creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: allocateLoadBalancerNodePorts defines if NodePorts + will be automatically allocated for services with type LoadBalancer. Default + is "true". It may be set to "false" if the cluster load-balancer + does not rely on NodePorts. If the caller requests specific + NodePorts (by specifying a value), those requests will be + respected, regardless of this field. This field may only + be set for services with type LoadBalancer and will be cleared + if the type is changed to any other type. + type: boolean + clusterIP: + description: 'clusterIP is the IP address of the service and + is usually assigned randomly. If an address is specified + manually, is in-range (as per system configuration), and + is not in use, it will be allocated to the service; otherwise + creation of the service will fail. This field may not be + changed through updates unless the type field is also being + changed to ExternalName (which requires this field to be + blank) or the type field is being changed from ExternalName + (in which case this field may optionally be specified, as + describe above). Valid values are "None", empty string + (""), or a valid IP address. Setting this to "None" makes + a "headless service" (no virtual IP), which is useful when + direct endpoint connections are preferred and proxying is + not required. Only applies to types ClusterIP, NodePort, + and LoadBalancer. If this field is specified when creating + a Service of type ExternalName, creation will fail. This + field will be wiped when updating a Service to type ExternalName. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + clusterIPs: + description: "ClusterIPs is a list of IP addresses assigned + to this service, and are usually assigned randomly. If + an address is specified manually, is in-range (as per system + configuration), and is not in use, it will be allocated + to the service; otherwise creation of the service will fail. + This field may not be changed through updates unless the + type field is also being changed to ExternalName (which + requires this field to be empty) or the type field is being + changed from ExternalName (in which case this field may + optionally be specified, as describe above). Valid values + are \"None\", empty string (\"\"), or a valid IP address. + \ Setting this to \"None\" makes a \"headless service\" + (no virtual IP), which is useful when direct endpoint connections + are preferred and proxying is not required. Only applies + to types ClusterIP, NodePort, and LoadBalancer. If this + field is specified when creating a Service of type ExternalName, + creation will fail. This field will be wiped when updating + a Service to type ExternalName. If this field is not specified, + it will be initialized from the clusterIP field. If this + field is specified, clients must ensure that clusterIPs[0] + and clusterIP have the same value. \n This field may hold + a maximum of two entries (dual-stack IPs, in either order). + These IPs must correspond to the values of the ipFamilies + field. Both clusterIPs and ipFamilies are governed by the + ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: externalIPs is a list of IP addresses for which + nodes in the cluster will also accept traffic for this service. These + IPs are not managed by Kubernetes. The user is responsible + for ensuring that traffic arrives at a node with this IP. A + common example is external load-balancers that are not part + of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that discovery + mechanisms will return as an alias for this service (e.g. + a DNS CNAME record). No proxying will be involved. Must + be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) + and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy describes how nodes distribute + service traffic they receive on one of the Service's "externally-facing" + addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). + If set to "Local", the proxy will configure the service + in a way that assumes that external load balancers will + take care of balancing the service traffic between nodes, + and so each node will deliver traffic only to the node-local + endpoints of the service, without masquerading the client + source IP. (Traffic mistakenly sent to a node with no endpoints + will be dropped.) The default value, "Cluster", uses the + standard behavior of routing to all endpoints evenly (possibly + modified by topology and other features). Note that traffic + sent to an External IP or LoadBalancer IP from within the + cluster will always get "Cluster" semantics, but clients + sending to a NodePort from within the cluster may need to + take traffic policy into account when picking a node. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck + nodePort for the service. This only applies when type is + set to LoadBalancer and externalTrafficPolicy is set to + Local. If a value is specified, is in-range, and is not + in use, it will be used. If not specified, a value will + be automatically allocated. External systems (e.g. load-balancers) + can use this port to determine if a given node holds endpoints + for this service or not. If this field is specified when + creating a Service which does not need it, creation will + fail. This field will be wiped when updating a Service to + no longer need it (e.g. changing type). This field cannot + be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes distribute + service traffic they receive on the ClusterIP. If set to + "Local", the proxy will assume that pods only want to talk + to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The + default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology + and other features). + type: string + ipFamilies: + description: "IPFamilies is a list of IP families (e.g. IPv4, + IPv6) assigned to this service. This field is usually assigned + automatically based on cluster configuration and the ipFamilyPolicy + field. If this field is specified manually, the requested + family is available in the cluster, and ipFamilyPolicy allows + it, it will be used; otherwise creation of the service will + fail. This field is conditionally mutable: it allows for + adding or removing a secondary IP family, but it does not + allow changing the primary IP family of the Service. Valid + values are \"IPv4\" and \"IPv6\". This field only applies + to Services of types ClusterIP, NodePort, and LoadBalancer, + and does apply to \"headless\" services. This field will + be wiped when updating a Service to type ExternalName. \n + This field may hold a maximum of two entries (dual-stack + families, in either order). These families must correspond + to the values of the clusterIPs field, if specified. Both + clusterIPs and ipFamilies are governed by the ipFamilyPolicy + field." + items: + description: IPFamily represents the IP Family (IPv4 or + IPv6). This type is used to express the family of an IP + expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness + requested or required by this Service. If there is no value + provided, then this field will be set to SingleStack. Services + can be "SingleStack" (a single IP family), "PreferDualStack" + (two IP families on dual-stack configured clusters or a + single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise + fail). The ipFamilies and clusterIPs fields depend on the + value of this field. This field will be wiped when updating + a service to type ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load balancer + implementation this Service belongs to. If specified, the + value of this field must be a label-style identifier, with + an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". + Unprefixed names are reserved for end-users. This field + can only be set when the Service type is 'LoadBalancer'. + If not set, the default load balancer implementation is + used, today this is typically done through the cloud provider + integration, but should apply for any default implementation. + If set, it is assumed that a load balancer implementation + is watching for Services with a matching class. Any default + load balancer implementation (e.g. cloud providers) should + ignore Services that set this field. This field can only + be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped + when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer. + This feature depends on whether the underlying cloud-provider + supports specifying the loadBalancerIP when a load balancer + is created. This field will be ignored if the cloud-provider + does not support the feature. Deprecated: This field was + under-specified and its meaning varies across implementations, + and it cannot support dual-stack. As of Kubernetes v1.24, + users are encouraged to use implementation-specific annotations + when available. This field may be removed in a future API + version.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, + this will restrict traffic through the cloud-provider load-balancer + will be restricted to the specified client IPs. This field + will be ignored if the cloud-provider does not support the + feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this service. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's + port. + properties: + appProtocol: + description: The application protocol for this port. + This field follows standard Kubernetes label syntax. + Un-prefixed names are reserved for IANA standard service + names (as per RFC-6335 and https://www.iana.org/assignments/service-names). + Non-standard protocols should use prefixed names such + as mycompany.com/my-custom-protocol. + type: string + name: + description: The name of this port within the service. + This must be a DNS_LABEL. All ports within a ServiceSpec + must have unique names. When considering the endpoints + for a Service, this must match the 'name' field in + the EndpointPort. Optional if only one ServicePort + is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this service + is exposed when type is NodePort or LoadBalancer. Usually + assigned by the system. If a value is specified, in-range, + and not in use it will be used, otherwise the operation + will fail. If not specified, a port will be allocated + if this Service requires one. If this field is specified + when creating a Service which does not need it, creation + will fail. This field will be wiped when updating + a Service to no longer need it (e.g. changing type + from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports + "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access on + the pods targeted by the service. Number must be in + the range 1 to 65535. Name must be an IANA_SVC_NAME. + If this is a string, it will be looked up as a named + port in the target Pod''s container ports. If this + is not specified, the value of the ''port'' field + is used (an identity map). This field is ignored for + services with clusterIP=None, and should be omitted + or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: publishNotReadyAddresses indicates that any agent + which deals with endpoints for this Service should disregard + any indications of ready/not-ready. The primary use case + for setting this field is for a StatefulSet's Headless Service + to propagate SRV DNS records for its Pods for the purpose + of peer discovery. The Kubernetes controllers that generate + Endpoints and EndpointSlice resources for Services interpret + this to mean that all endpoints are considered "ready" even + if the Pods themselves are not. Agents which consume only + Kubernetes generated endpoints through the Endpoints or + EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys + and values matching this selector. If empty or not present, + the service is assumed to have an external process managing + its endpoints, which Kubernetes will not modify. Only applies + to types ClusterIP, NodePort, and LoadBalancer. Ignored + if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain + session affinity. Enable client IP based session affinity. + Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client + IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds + of ClientIP type session sticky time. The value + must be >0 && <=86400(for 1 day) if ServiceAffinity + == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. + Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, + NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal + IP address for load-balancing to endpoints. Endpoints are + determined by the selector or if that is not specified, + by manual construction of an Endpoints object or EndpointSlice + objects. If clusterIP is "None", no virtual IP is allocated + and the endpoints are published as a set of endpoints rather + than a virtual IP. "NodePort" builds on ClusterIP and allocates + a port on every node which routes to the same endpoints + as the clusterIP. "LoadBalancer" builds on NodePort and + creates an external load-balancer (if supported in the current + cloud) which routes to the same endpoints as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + required: + - spec + type: object + required: + - resources + type: object + status: + description: AppWrapperStatus represents the current state of a AppWrapper + properties: + Succeeded: + description: The number of resources which reached phase Succeeded. + format: int32 + type: integer + canrun: + description: Can run? + type: boolean + conditions: + description: Represents the latest available observations of the AppWrapper's + current condition. + items: + description: AppWrapperCondition describes the state of an AppWrapper + at a certain point. + properties: + lastTransitionMicroTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateMicroTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of AppWrapper condition. + type: string + required: + - status + - type + type: object + type: array + controllerfirstdispatchtimestamp: + description: Microsecond level timestamp when controller first dispatches + the AppWrapper + format: date-time + type: string + controllerfirsttimestamp: + description: Microsecond level timestamp when controller first sees + QueueJob (by Informer) + format: date-time + type: string + failed: + description: The number of resources which reached phase Failed. + format: int32 + type: integer + filterignore: + description: Tell Informer to ignore this update message (do not generate + a controller event) + type: boolean + isdispatched: + description: Is Dispatched? + type: boolean + local: + description: Indicate if message is a duplicate (for Informer to recognize + duplicate messages) + type: boolean + message: + type: string + numberOfRequeueings: + description: Field to keep track of how many times a requeuing event + has been triggered + type: integer + pending: + description: The number of pending pods. + format: int32 + type: integer + pendingpodconditions: + description: Represents the latest available observations of pods + belonging to the AppWrapper. + items: + properties: + conditions: + items: + description: PodCondition contains details for the current + condition of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can + be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + podname: + type: string + type: object + type: array + queuejobstate: + description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, + ... + type: string + requeueingTimeInSeconds: + description: Field to keep track of total number of seconds spent + in requeueing + type: integer + running: + format: int32 + type: integer + sender: + description: Indicate sender of this message (extremely useful for + debugging) + type: string + state: + description: State - Pending, Running, Failed, Deleted + type: string + systempriority: + description: System defined Priority + format: float + type: number + template: + description: The minimal available resources to run for this AppWrapper + (is this different from the MinAvailable from JobStatus) + format: int32 + type: integer + totalcpu: + description: The number of CPU consumed by all pods belonging to the + AppWrapper. + format: int32 + type: integer + totalgpu: + description: The total number of GPUs consumed by all pods belonging + to the AppWrapper. + format: int32 + type: integer + totalmemory: + description: The amount of memory consumed by all pods belonging to + the AppWrapper. + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- diff --git a/config/crd/workload.codeflare_schedulingspecs.yaml b/config/crd/workload.codeflare_schedulingspecs.yaml new file mode 100644 index 000000000..fc44a4326 --- /dev/null +++ b/config/crd/workload.codeflare_schedulingspecs.yaml @@ -0,0 +1,102 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: schedulingspecs.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: SchedulingSpec + listKind: SchedulingSpecList + plural: schedulingspecs + singular: schedulingspec + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. + Requeuing is triggered when the number of running/completed pods + is not equal to this value. When not specified, requeuing is disabled + and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting + time. Values in this field control how often the pod check should + happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between + requeuing checks. The values available are 'exponential', 'linear', + or 'none'. For example, 'exponential' growth would double the + 'timeInSeconds' value every time a requeuing event is triggered. + If the string value is misspelled or not one of the possible + options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users + cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once + this value is reached (e.g., 'numRequeuings = maxNumRequeuings', + no more requeuing checks are performed and the generic items + are stopped and removed from the cluster (AppWrapper remains + deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing + event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions + are checked. This value is specified by the user, but it may + grow as requeuing events happen. + type: integer + type: object + type: object + required: + - metadata + type: object + served: true + storage: true