diff --git a/Makefile b/Makefile index e7bd3787a..2e6ab468d 100644 --- a/Makefile +++ b/Makefile @@ -148,10 +148,11 @@ defaults: # this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \) .PHONY: manifests -manifests: controller-gen kustomize ## Generate RBAC objects. +manifests: controller-gen kustomize install-yq ## Generate RBAC objects. $(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..." -# $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml -# $(KUSTOMIZE) build config/crd/mcad > config/crd/mcad.yaml && make split_yaml FILE=config/crd/mcad.yaml + $(SED) -i -E "s|(- )\${MCAD_REPO}.*|\1\${MCAD_CRD}|" config/crd/mcad/kustomization.yaml + $(KUSTOMIZE) build config/crd/mcad | $(YQ) -s '"crd-" + .spec.names.singular' --no-doc + mv crd-*.yml config/crd .PHONY: fmt fmt: ## Run go fmt against code. @@ -236,6 +237,7 @@ SED ?= /usr/bin/sed KUSTOMIZE_VERSION ?= v4.5.4 CODEGEN_VERSION ?= v0.27.2 CONTROLLER_TOOLS_VERSION ?= v0.9.2 +YQ_VERSION ?= v4.30.8 ## latest version that works with go1.19 OPERATOR_SDK_VERSION ?= v1.27.0 GH_CLI_VERSION ?= 2.30.0 @@ -261,6 +263,11 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar $(CONTROLLER_GEN): $(LOCALBIN) test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) +.PHONY: install-yq +install-yq: $(YQ) ## Download yq locally if necessary +$(YQ): $(LOCALBIN) + test -s $(LOCALBIN)/yq || GOBIN=$(LOCALBIN) go install github.com/mikefarah/yq/v4@$(YQ_VERSION) + .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) @@ -388,32 +395,3 @@ verify-imports: openshift-goimports ## Run import verifications. .PHONY: scorecard-bundle scorecard-bundle: install-operator-sdk ## Run scorecard tests on bundle image. $(OPERATOR_SDK) scorecard bundle - - -FILE ?= input.yaml # Default value, it isn't a file, but the make cmds fill hang for longer without it -temp_dir := temp_split -output_dir := 'config/crd/' - -.PHONY: check_yq -check_yq: - @command -v wget >/dev/null 2>&1 || (echo "Installing wget..."; apt-get install -y wget) - @command -v $(YQ) >/dev/null 2>&1 || (echo "Installing yq..."; wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64.tar.gz -O - |\ - tar xz && mv yq_linux_amd64 $(YQ)) - -# this works on a MacOS by replacing awk with gawk -.PHONY: split_yaml -split_yaml: - @$(MAKE) check_yq - @mkdir -p $(temp_dir) - @awk '/apiVersion: /{if (x>0) close("$(temp_dir)/section_" x ".yaml"); x++}{print > "$(temp_dir)/section_"x".yaml"}' $(FILE) - @$(MAKE) process_sections - -.PHONY: process_sections -process_sections: - @mkdir -p $(output_dir) - @for section_file in $(temp_dir)/section_*; do \ - metadata_name=$$(YQ e '.metadata.name' $$section_file); \ - file_name=$$(echo $$metadata_name | awk -F'.' '{print $$2"."$$3"_"$$1".yaml"}'); \ - mv $$section_file $(output_dir)/$$file_name; \ - done - @rm -r $(temp_dir) diff --git a/config/crd/crd-appwrapper.yml b/config/crd/crd-appwrapper.yml new file mode 100644 index 000000000..9140857a0 --- /dev/null +++ b/config/crd/crd-appwrapper.yml @@ -0,0 +1,462 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: appwrappers.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: AppWrapper + listKind: AppWrapperList + plural: appwrappers + singular: appwrapper + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Definition of AppWrapper class + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AppWrapperSpec describes how the App Wrapper will look like. + properties: + priority: + format: int32 + type: integer + priorityslope: + format: float + type: number + resources: + description: a collection of AppWrapperResource + properties: + GenericItems: + items: + description: AppWrapperGenericResource is App Wrapper aggregation resource + properties: + allocated: + description: The number of allocated replicas from this resource type + format: int32 + type: integer + completionstatus: + description: 'Optional field that drives completion status of this AppWrapper. This field within an item of an AppWrapper determines the full state of the AppWrapper. The completionstatus field contains a list of conditions that make the associate item considered completed, for instance: - completion conditions could be "Complete" or "Failed". The associated item''s level .status.conditions[].type field is monitored for any one of these conditions. Once all items with this option is set and the conditionstatus is met the entire AppWrapper state will be changed to one of the valid AppWrapper completion state. Note: - this is an AND operation for all items where this option is set. See the list of AppWrapper states for a list of valid complete states.' + type: string + custompodresources: + description: Optional section that specifies resource requirements for non-standard k8s resources, follows same format as that of standard k8s resources. + items: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) pairs. + type: object + replicas: + type: integer + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'todo: replace with Containers []Container Contain v1.ResourceRequirements' + type: object + required: + - replicas + - requests + type: object + type: array + generictemplate: + description: The template for the resource; it is now a raw text because we don't know for what resource it should be instantiated + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + minavailable: + description: The minimal available pods to run for this AppWrapper; the default value is nil + format: int32 + type: integer + priority: + description: The priority of this resource + format: int32 + type: integer + priorityslope: + description: The increasing rate of priority value for this resource + format: float + type: number + replicas: + description: Replicas is the number of desired replicas + format: int32 + type: integer + type: object + type: array + type: object + schedulingSpec: + description: SchedSpec specifies the parameters used for scheduling generic items wrapped inside AppWrappers. It defines the policy for requeuing jobs based on the number of running pods. + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. + type: integer + type: object + type: object + selector: + description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + service: + description: AppWrapperService is App Wrapper service definition + properties: + spec: + description: ServiceSpec describes the attributes that a user creates on a service. + properties: + allocateLoadBalancerNodePorts: + description: allocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer. Default is "true". It may be set to "false" if the cluster load-balancer does not rely on NodePorts. If the caller requests specific NodePorts (by specifying a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. + type: boolean + clusterIP: + description: 'clusterIP is the IP address of the service and is usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be blank) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are "None", empty string (""), or a valid IP address. Setting this to "None" makes a "headless service" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + clusterIPs: + description: "ClusterIPs is a list of IP addresses assigned to this service, and are usually assigned randomly. If an address is specified manually, is in-range (as per system configuration), and is not in use, it will be allocated to the service; otherwise creation of the service will fail. This field may not be changed through updates unless the type field is also being changed to ExternalName (which requires this field to be empty) or the type field is being changed from ExternalName (in which case this field may optionally be specified, as describe above). Valid values are \"None\", empty string (\"\"), or a valid IP address. Setting this to \"None\" makes a \"headless service\" (no virtual IP), which is useful when direct endpoint connections are preferred and proxying is not required. Only applies to types ClusterIP, NodePort, and LoadBalancer. If this field is specified when creating a Service of type ExternalName, creation will fail. This field will be wiped when updating a Service to type ExternalName. If this field is not specified, it will be initialized from the clusterIP field. If this field is specified, clients must ensure that clusterIPs[0] and clusterIP have the same value. \n This field may hold a maximum of two entries (dual-stack IPs, in either order). These IPs must correspond to the values of the ipFamilies field. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" + items: + type: string + type: array + x-kubernetes-list-type: atomic + externalIPs: + description: externalIPs is a list of IP addresses for which nodes in the cluster will also accept traffic for this service. These IPs are not managed by Kubernetes. The user is responsible for ensuring that traffic arrives at a node with this IP. A common example is external load-balancers that are not part of the Kubernetes system. + items: + type: string + type: array + externalName: + description: externalName is the external reference that discovery mechanisms will return as an alias for this service (e.g. a DNS CNAME record). No proxying will be involved. Must be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: externalTrafficPolicy describes how nodes distribute service traffic they receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure the service in a way that assumes that external load balancers will take care of balancing the service traffic between nodes, and so each node will deliver traffic only to the node-local endpoints of the service, without masquerading the client source IP. (Traffic mistakenly sent to a node with no endpoints will be dropped.) The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). Note that traffic sent to an External IP or LoadBalancer IP from within the cluster will always get "Cluster" semantics, but clients sending to a NodePort from within the cluster may need to take traffic policy into account when picking a node. + type: string + healthCheckNodePort: + description: healthCheckNodePort specifies the healthcheck nodePort for the service. This only applies when type is set to LoadBalancer and externalTrafficPolicy is set to Local. If a value is specified, is in-range, and is not in use, it will be used. If not specified, a value will be automatically allocated. External systems (e.g. load-balancers) can use this port to determine if a given node holds endpoints for this service or not. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type). This field cannot be updated once set. + format: int32 + type: integer + internalTrafficPolicy: + description: InternalTrafficPolicy describes how nodes distribute service traffic they receive on the ClusterIP. If set to "Local", the proxy will assume that pods only want to talk to endpoints of the service on the same node as the pod, dropping the traffic if there are no local endpoints. The default value, "Cluster", uses the standard behavior of routing to all endpoints evenly (possibly modified by topology and other features). + type: string + ipFamilies: + description: "IPFamilies is a list of IP families (e.g. IPv4, IPv6) assigned to this service. This field is usually assigned automatically based on cluster configuration and the ipFamilyPolicy field. If this field is specified manually, the requested family is available in the cluster, and ipFamilyPolicy allows it, it will be used; otherwise creation of the service will fail. This field is conditionally mutable: it allows for adding or removing a secondary IP family, but it does not allow changing the primary IP family of the Service. Valid values are \"IPv4\" and \"IPv6\". This field only applies to Services of types ClusterIP, NodePort, and LoadBalancer, and does apply to \"headless\" services. This field will be wiped when updating a Service to type ExternalName. \n This field may hold a maximum of two entries (dual-stack families, in either order). These families must correspond to the values of the clusterIPs field, if specified. Both clusterIPs and ipFamilies are governed by the ipFamilyPolicy field." + items: + description: IPFamily represents the IP Family (IPv4 or IPv6). This type is used to express the family of an IP expressed by a type (e.g. service.spec.ipFamilies). + type: string + type: array + x-kubernetes-list-type: atomic + ipFamilyPolicy: + description: IPFamilyPolicy represents the dual-stack-ness requested or required by this Service. If there is no value provided, then this field will be set to SingleStack. Services can be "SingleStack" (a single IP family), "PreferDualStack" (two IP families on dual-stack configured clusters or a single IP family on single-stack clusters), or "RequireDualStack" (two IP families on dual-stack configured clusters, otherwise fail). The ipFamilies and clusterIPs fields depend on the value of this field. This field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: loadBalancerClass is the class of the load balancer implementation this Service belongs to. If specified, the value of this field must be a label-style identifier, with an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load balancer implementation is used, today this is typically done through the cloud provider integration, but should apply for any default implementation. If set, it is assumed that a load balancer implementation is watching for Services with a matching class. Any default load balancer implementation (e.g. cloud providers) should ignore Services that set this field. This field can only be set when creating or updating a Service to type 'LoadBalancer'. Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerIP: + description: 'Only applies to Service Type: LoadBalancer. This feature depends on whether the underlying cloud-provider supports specifying the loadBalancerIP when a load balancer is created. This field will be ignored if the cloud-provider does not support the feature. Deprecated: This field was under-specified and its meaning varies across implementations, and it cannot support dual-stack. As of Kubernetes v1.24, users are encouraged to use implementation-specific annotations when available. This field may be removed in a future API version.' + type: string + loadBalancerSourceRanges: + description: 'If specified and supported by the platform, this will restrict traffic through the cloud-provider load-balancer will be restricted to the specified client IPs. This field will be ignored if the cloud-provider does not support the feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' + items: + type: string + type: array + ports: + description: 'The list of ports that are exposed by this service. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + items: + description: ServicePort contains information on service's port. + properties: + appProtocol: + description: The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and https://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. + type: string + name: + description: The name of this port within the service. This must be a DNS_LABEL. All ports within a ServiceSpec must have unique names. When considering the endpoints for a Service, this must match the 'name' field in the EndpointPort. Optional if only one ServicePort is defined on this service. + type: string + nodePort: + description: 'The port on each node on which this service is exposed when type is NodePort or LoadBalancer. Usually assigned by the system. If a value is specified, in-range, and not in use it will be used, otherwise the operation will fail. If not specified, a port will be allocated if this Service requires one. If this field is specified when creating a Service which does not need it, creation will fail. This field will be wiped when updating a Service to no longer need it (e.g. changing type from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' + format: int32 + type: integer + port: + description: The port that will be exposed by this service. + format: int32 + type: integer + protocol: + default: TCP + description: The IP protocol for this port. Supports "TCP", "UDP", and "SCTP". Default is TCP. + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Number or name of the port to access on the pods targeted by the service. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. If this is a string, it will be looked up as a named port in the target Pod''s container ports. If this is not specified, the value of the ''port'' field is used (an identity map). This field is ignored for services with clusterIP=None, and should be omitted or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + - protocol + x-kubernetes-list-type: map + publishNotReadyAddresses: + description: publishNotReadyAddresses indicates that any agent which deals with endpoints for this Service should disregard any indications of ready/not-ready. The primary use case for setting this field is for a StatefulSet's Headless Service to propagate SRV DNS records for its Pods for the purpose of peer discovery. The Kubernetes controllers that generate Endpoints and EndpointSlice resources for Services interpret this to mean that all endpoints are considered "ready" even if the Pods themselves are not. Agents which consume only Kubernetes generated endpoints through the Endpoints or EndpointSlice resources can safely assume this behavior. + type: boolean + selector: + additionalProperties: + type: string + description: 'Route service traffic to pods with label keys and values matching this selector. If empty or not present, the service is assumed to have an external process managing its endpoints, which Kubernetes will not modify. Only applies to types ClusterIP, NodePort, and LoadBalancer. Ignored if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' + type: object + x-kubernetes-map-type: atomic + sessionAffinity: + description: 'Supports "ClientIP" and "None". Used to maintain session affinity. Enable client IP based session affinity. Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations of Client IP based session affinity. + properties: + timeoutSeconds: + description: timeoutSeconds specifies the seconds of ClientIP type session sticky time. The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: 'type determines how the Service is exposed. Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal IP address for load-balancing to endpoints. Endpoints are determined by the selector or if that is not specified, by manual construction of an Endpoints object or EndpointSlice objects. If clusterIP is "None", no virtual IP is allocated and the endpoints are published as a set of endpoints rather than a virtual IP. "NodePort" builds on ClusterIP and allocates a port on every node which routes to the same endpoints as the clusterIP. "LoadBalancer" builds on NodePort and creates an external load-balancer (if supported in the current cloud) which routes to the same endpoints as the clusterIP. "ExternalName" aliases this service to the specified externalName. Several other fields do not apply to ExternalName services. More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' + type: string + type: object + required: + - spec + type: object + required: + - resources + type: object + status: + description: AppWrapperStatus represents the current state of a AppWrapper + properties: + Succeeded: + description: The number of resources which reached phase Succeeded. + format: int32 + type: integer + canrun: + description: Can run? + type: boolean + conditions: + description: Represents the latest available observations of the AppWrapper's current condition. + items: + description: AppWrapperCondition describes the state of an AppWrapper at a certain point. + properties: + lastTransitionMicroTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + lastUpdateMicroTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human-readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of AppWrapper condition. + type: string + required: + - status + - type + type: object + type: array + controllerfirstdispatchtimestamp: + description: Microsecond level timestamp when controller first dispatches the AppWrapper + format: date-time + type: string + controllerfirsttimestamp: + description: Microsecond level timestamp when controller first sees QueueJob (by Informer) + format: date-time + type: string + failed: + description: The number of resources which reached phase Failed. + format: int32 + type: integer + filterignore: + description: Tell Informer to ignore this update message (do not generate a controller event) + type: boolean + isdispatched: + description: Is Dispatched? + type: boolean + local: + description: Indicate if message is a duplicate (for Informer to recognize duplicate messages) + type: boolean + message: + type: string + numberOfRequeueings: + description: Field to keep track of how many times a requeuing event has been triggered + type: integer + pending: + description: The number of pending pods. + format: int32 + type: integer + pendingpodconditions: + description: Represents the latest available observations of pods belonging to the AppWrapper. + items: + properties: + conditions: + items: + description: PodCondition contains details for the current condition of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details about last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + podname: + type: string + type: object + type: array + queuejobstate: + description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, ... + type: string + requeueingTimeInSeconds: + description: Field to keep track of total number of seconds spent in requeueing + type: integer + running: + format: int32 + type: integer + sender: + description: Indicate sender of this message (extremely useful for debugging) + type: string + state: + description: State - Pending, Running, Failed, Deleted + type: string + systempriority: + description: System defined Priority + format: float + type: number + template: + description: The minimal available resources to run for this AppWrapper (is this different from the MinAvailable from JobStatus) + format: int32 + type: integer + totalcpu: + description: The number of CPU consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + totalgpu: + description: The total number of GPUs consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + totalmemory: + description: The amount of memory consumed by all pods belonging to the AppWrapper. + format: int32 + type: integer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/crd-quotasubtree.yml b/config/crd/crd-quotasubtree.yml new file mode 100644 index 000000000..924aaacd7 --- /dev/null +++ b/config/crd/crd-quotasubtree.yml @@ -0,0 +1,115 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: quotasubtrees.quota.codeflare.dev +spec: + group: quota.codeflare.dev + names: + kind: QuotaSubtree + listKind: QuotaSubtreeList + plural: quotasubtrees + singular: quotasubtree + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: QuotaSubtree is a specification for a quota subtree resource + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: QuotaSubtreeSpec is the spec for a resource plan + properties: + children: + items: + description: Child is the spec for a QuotaSubtree resource + properties: + name: + type: string + namespace: + type: string + path: + type: string + quotas: + description: Quota is the spec for a QuotaSubtree resource + properties: + disabled: + type: boolean + hardLimit: + type: boolean + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: array + parent: + type: string + parentNamespace: + type: string + type: object + status: + description: QuotaSubtreeStatus is the status for a QuotaSubtree resource + properties: + children: + items: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + type: array + totalAllocation: + description: ResourceAllocation is the spec for the child status + properties: + allocated: + description: ResourceAllocationStatus is the spec for the child resource usage + properties: + requests: + additionalProperties: + type: string + type: object + type: object + name: + type: string + namespace: + type: string + path: + type: string + type: object + required: + - children + - totalAllocation + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/config/crd/crd-schedulingspec.yml b/config/crd/crd-schedulingspec.yml new file mode 100644 index 000000000..373dc89d8 --- /dev/null +++ b/config/crd/crd-schedulingspec.yml @@ -0,0 +1,80 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: schedulingspecs.workload.codeflare.dev +spec: + group: workload.codeflare.dev + names: + kind: SchedulingSpec + listKind: SchedulingSpecList + plural: schedulingspecs + singular: schedulingspec + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + dispatchDuration: + description: Wall clock duration time of appwrapper in seconds. + properties: + expected: + type: integer + limit: + type: integer + overrun: + type: boolean + type: object + minAvailable: + description: Expected number of pods in running and/or completed state. Requeuing is triggered when the number of running/completed pods is not equal to this value. When not specified, requeuing is disabled and no check is performed. + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + requeuing: + description: Specification of the requeuing strategy based on waiting time. Values in this field control how often the pod check should happen, and if requeuing has reached its maximum number of times. + properties: + growthType: + default: exponential + description: Growth strategy to increase the waiting time between requeuing checks. The values available are 'exponential', 'linear', or 'none'. For example, 'exponential' growth would double the 'timeInSeconds' value every time a requeuing event is triggered. If the string value is misspelled or not one of the possible options, the growth behavior is defaulted to 'none'. + type: string + initialTimeInSeconds: + description: Value to keep track of the initial wait time. Users cannot set this as it is taken from 'timeInSeconds'. + type: integer + maxNumRequeuings: + default: 0 + description: Maximum number of requeuing events allowed. Once this value is reached (e.g., 'numRequeuings = maxNumRequeuings', no more requeuing checks are performed and the generic items are stopped and removed from the cluster (AppWrapper remains deployed). + type: integer + maxTimeInSeconds: + default: 0 + description: Maximum waiting time for requeuing checks. + type: integer + numRequeuings: + default: 0 + description: Field to keep track of how many times a requeuing event has been triggered. + type: integer + timeInSeconds: + default: 300 + description: Initial waiting time before requeuing conditions are checked. This value is specified by the user, but it may grow as requeuing events happen. + type: integer + type: object + type: object + required: + - metadata + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 73a576238..faaff445f 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,8 +2,8 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- quota.codeflare_quotasubtrees.yaml -- workload.codeflare_appwrappers.yaml -- workload.codeflare_schedulingspecs.yaml +- crd-appwrapper.yml +- crd-quotasubtree.yml +- crd-schedulingspec.yml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/mcad/kustomization.yaml b/config/crd/mcad/kustomization.yaml index e016d77f8..e53d2dd94 100644 --- a/config/crd/mcad/kustomization.yaml +++ b/config/crd/mcad/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.36.0 +- github.com/project-codeflare/multi-cluster-app-dispatcher/config/crd?ref=v1.37.1 diff --git a/config/crd/quota.codeflare_quotasubtrees.yaml b/config/crd/quota.codeflare_quotasubtrees.yaml deleted file mode 100644 index 44bb5e0a6..000000000 --- a/config/crd/quota.codeflare_quotasubtrees.yaml +++ /dev/null @@ -1,122 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: quotasubtrees.quota.codeflare.dev -spec: - group: quota.codeflare.dev - names: - kind: QuotaSubtree - listKind: QuotaSubtreeList - plural: quotasubtrees - singular: quotasubtree - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: QuotaSubtree is a specification for a quota subtree resource - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: QuotaSubtreeSpec is the spec for a resource plan - properties: - children: - items: - description: Child is the spec for a QuotaSubtree resource - properties: - name: - type: string - namespace: - type: string - path: - type: string - quotas: - description: Quota is the spec for a QuotaSubtree resource - properties: - disabled: - type: boolean - hardLimit: - type: boolean - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - type: object - type: object - type: array - parent: - type: string - parentNamespace: - type: string - type: object - status: - description: QuotaSubtreeStatus is the status for a QuotaSubtree resource - properties: - children: - items: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child - resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - type: array - totalAllocation: - description: ResourceAllocation is the spec for the child status - properties: - allocated: - description: ResourceAllocationStatus is the spec for the child - resource usage - properties: - requests: - additionalProperties: - type: string - type: object - type: object - name: - type: string - namespace: - type: string - path: - type: string - type: object - required: - - children - - totalAllocation - type: object - required: - - spec - type: object - served: true - storage: true ---- diff --git a/config/crd/workload.codeflare_appwrappers.yaml b/config/crd/workload.codeflare_appwrappers.yaml deleted file mode 100644 index e1d260411..000000000 --- a/config/crd/workload.codeflare_appwrappers.yaml +++ /dev/null @@ -1,765 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: appwrappers.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: AppWrapper - listKind: AppWrapperList - plural: appwrappers - singular: appwrapper - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - description: Definition of AppWrapper class - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: AppWrapperSpec describes how the App Wrapper will look like. - properties: - priority: - format: int32 - type: integer - priorityslope: - format: float - type: number - resources: - description: a collection of AppWrapperResource - properties: - GenericItems: - items: - description: AppWrapperGenericResource is App Wrapper aggregation - resource - properties: - allocated: - description: The number of allocated replicas from this - resource type - format: int32 - type: integer - completionstatus: - description: 'Optional field that drives completion status - of this AppWrapper. This field within an item of an AppWrapper - determines the full state of the AppWrapper. The completionstatus - field contains a list of conditions that make the associate - item considered completed, for instance: - completion - conditions could be "Complete" or "Failed". The associated - item''s level .status.conditions[].type field is monitored - for any one of these conditions. Once all items with this - option is set and the conditionstatus is met the entire - AppWrapper state will be changed to one of the valid AppWrapper - completion state. Note: - this is an AND operation for - all items where this option is set. See the list of AppWrapper - states for a list of valid complete states.' - type: string - custompodresources: - description: Optional section that specifies resource requirements - for non-standard k8s resources, follows same format as - that of standard k8s resources. - items: - properties: - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: ResourceList is a set of (resource name, - quantity) pairs. - type: object - replicas: - type: integer - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'todo: replace with Containers []Container - Contain v1.ResourceRequirements' - type: object - required: - - replicas - - requests - type: object - type: array - generictemplate: - description: The template for the resource; it is now a - raw text because we don't know for what resource it should - be instantiated - type: object - x-kubernetes-embedded-resource: true - x-kubernetes-preserve-unknown-fields: true - minavailable: - description: The minimal available pods to run for this - AppWrapper; the default value is nil - format: int32 - type: integer - priority: - description: The priority of this resource - format: int32 - type: integer - priorityslope: - description: The increasing rate of priority value for this - resource - format: float - type: number - replicas: - description: Replicas is the number of desired replicas - format: int32 - type: integer - type: object - type: array - type: object - schedulingSpec: - description: SchedSpec specifies the parameters used for scheduling - generic items wrapped inside AppWrappers. It defines the policy - for requeuing jobs based on the number of running pods. - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed - state. Requeuing is triggered when the number of running/completed - pods is not equal to this value. When not specified, requeuing - is disabled and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on - waiting time. Values in this field control how often the pod - check should happen, and if requeuing has reached its maximum - number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time - between requeuing checks. The values available are 'exponential', - 'linear', or 'none'. For example, 'exponential' growth would - double the 'timeInSeconds' value every time a requeuing - event is triggered. If the string value is misspelled or - not one of the possible options, the growth behavior is - defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. - Users cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once - this value is reached (e.g., 'numRequeuings = maxNumRequeuings', - no more requeuing checks are performed and the generic items - are stopped and removed from the cluster (AppWrapper remains - deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing - event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions - are checked. This value is specified by the user, but it - may grow as requeuing events happen. - type: integer - type: object - type: object - selector: - description: A label selector is a label query over a set of resources. - The result of matchLabels and matchExpressions are ANDed. An empty - label selector matches all objects. A null label selector matches - no objects. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - service: - description: AppWrapperService is App Wrapper service definition - properties: - spec: - description: ServiceSpec describes the attributes that a user - creates on a service. - properties: - allocateLoadBalancerNodePorts: - description: allocateLoadBalancerNodePorts defines if NodePorts - will be automatically allocated for services with type LoadBalancer. Default - is "true". It may be set to "false" if the cluster load-balancer - does not rely on NodePorts. If the caller requests specific - NodePorts (by specifying a value), those requests will be - respected, regardless of this field. This field may only - be set for services with type LoadBalancer and will be cleared - if the type is changed to any other type. - type: boolean - clusterIP: - description: 'clusterIP is the IP address of the service and - is usually assigned randomly. If an address is specified - manually, is in-range (as per system configuration), and - is not in use, it will be allocated to the service; otherwise - creation of the service will fail. This field may not be - changed through updates unless the type field is also being - changed to ExternalName (which requires this field to be - blank) or the type field is being changed from ExternalName - (in which case this field may optionally be specified, as - describe above). Valid values are "None", empty string - (""), or a valid IP address. Setting this to "None" makes - a "headless service" (no virtual IP), which is useful when - direct endpoint connections are preferred and proxying is - not required. Only applies to types ClusterIP, NodePort, - and LoadBalancer. If this field is specified when creating - a Service of type ExternalName, creation will fail. This - field will be wiped when updating a Service to type ExternalName. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - clusterIPs: - description: "ClusterIPs is a list of IP addresses assigned - to this service, and are usually assigned randomly. If - an address is specified manually, is in-range (as per system - configuration), and is not in use, it will be allocated - to the service; otherwise creation of the service will fail. - This field may not be changed through updates unless the - type field is also being changed to ExternalName (which - requires this field to be empty) or the type field is being - changed from ExternalName (in which case this field may - optionally be specified, as describe above). Valid values - are \"None\", empty string (\"\"), or a valid IP address. - \ Setting this to \"None\" makes a \"headless service\" - (no virtual IP), which is useful when direct endpoint connections - are preferred and proxying is not required. Only applies - to types ClusterIP, NodePort, and LoadBalancer. If this - field is specified when creating a Service of type ExternalName, - creation will fail. This field will be wiped when updating - a Service to type ExternalName. If this field is not specified, - it will be initialized from the clusterIP field. If this - field is specified, clients must ensure that clusterIPs[0] - and clusterIP have the same value. \n This field may hold - a maximum of two entries (dual-stack IPs, in either order). - These IPs must correspond to the values of the ipFamilies - field. Both clusterIPs and ipFamilies are governed by the - ipFamilyPolicy field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies" - items: - type: string - type: array - x-kubernetes-list-type: atomic - externalIPs: - description: externalIPs is a list of IP addresses for which - nodes in the cluster will also accept traffic for this service. These - IPs are not managed by Kubernetes. The user is responsible - for ensuring that traffic arrives at a node with this IP. A - common example is external load-balancers that are not part - of the Kubernetes system. - items: - type: string - type: array - externalName: - description: externalName is the external reference that discovery - mechanisms will return as an alias for this service (e.g. - a DNS CNAME record). No proxying will be involved. Must - be a lowercase RFC-1123 hostname (https://tools.ietf.org/html/rfc1123) - and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: externalTrafficPolicy describes how nodes distribute - service traffic they receive on one of the Service's "externally-facing" - addresses (NodePorts, ExternalIPs, and LoadBalancer IPs). - If set to "Local", the proxy will configure the service - in a way that assumes that external load balancers will - take care of balancing the service traffic between nodes, - and so each node will deliver traffic only to the node-local - endpoints of the service, without masquerading the client - source IP. (Traffic mistakenly sent to a node with no endpoints - will be dropped.) The default value, "Cluster", uses the - standard behavior of routing to all endpoints evenly (possibly - modified by topology and other features). Note that traffic - sent to an External IP or LoadBalancer IP from within the - cluster will always get "Cluster" semantics, but clients - sending to a NodePort from within the cluster may need to - take traffic policy into account when picking a node. - type: string - healthCheckNodePort: - description: healthCheckNodePort specifies the healthcheck - nodePort for the service. This only applies when type is - set to LoadBalancer and externalTrafficPolicy is set to - Local. If a value is specified, is in-range, and is not - in use, it will be used. If not specified, a value will - be automatically allocated. External systems (e.g. load-balancers) - can use this port to determine if a given node holds endpoints - for this service or not. If this field is specified when - creating a Service which does not need it, creation will - fail. This field will be wiped when updating a Service to - no longer need it (e.g. changing type). This field cannot - be updated once set. - format: int32 - type: integer - internalTrafficPolicy: - description: InternalTrafficPolicy describes how nodes distribute - service traffic they receive on the ClusterIP. If set to - "Local", the proxy will assume that pods only want to talk - to endpoints of the service on the same node as the pod, - dropping the traffic if there are no local endpoints. The - default value, "Cluster", uses the standard behavior of - routing to all endpoints evenly (possibly modified by topology - and other features). - type: string - ipFamilies: - description: "IPFamilies is a list of IP families (e.g. IPv4, - IPv6) assigned to this service. This field is usually assigned - automatically based on cluster configuration and the ipFamilyPolicy - field. If this field is specified manually, the requested - family is available in the cluster, and ipFamilyPolicy allows - it, it will be used; otherwise creation of the service will - fail. This field is conditionally mutable: it allows for - adding or removing a secondary IP family, but it does not - allow changing the primary IP family of the Service. Valid - values are \"IPv4\" and \"IPv6\". This field only applies - to Services of types ClusterIP, NodePort, and LoadBalancer, - and does apply to \"headless\" services. This field will - be wiped when updating a Service to type ExternalName. \n - This field may hold a maximum of two entries (dual-stack - families, in either order). These families must correspond - to the values of the clusterIPs field, if specified. Both - clusterIPs and ipFamilies are governed by the ipFamilyPolicy - field." - items: - description: IPFamily represents the IP Family (IPv4 or - IPv6). This type is used to express the family of an IP - expressed by a type (e.g. service.spec.ipFamilies). - type: string - type: array - x-kubernetes-list-type: atomic - ipFamilyPolicy: - description: IPFamilyPolicy represents the dual-stack-ness - requested or required by this Service. If there is no value - provided, then this field will be set to SingleStack. Services - can be "SingleStack" (a single IP family), "PreferDualStack" - (two IP families on dual-stack configured clusters or a - single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise - fail). The ipFamilies and clusterIPs fields depend on the - value of this field. This field will be wiped when updating - a service to type ExternalName. - type: string - loadBalancerClass: - description: loadBalancerClass is the class of the load balancer - implementation this Service belongs to. If specified, the - value of this field must be a label-style identifier, with - an optional prefix, e.g. "internal-vip" or "example.com/internal-vip". - Unprefixed names are reserved for end-users. This field - can only be set when the Service type is 'LoadBalancer'. - If not set, the default load balancer implementation is - used, today this is typically done through the cloud provider - integration, but should apply for any default implementation. - If set, it is assumed that a load balancer implementation - is watching for Services with a matching class. Any default - load balancer implementation (e.g. cloud providers) should - ignore Services that set this field. This field can only - be set when creating or updating a Service to type 'LoadBalancer'. - Once set, it can not be changed. This field will be wiped - when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerIP: - description: 'Only applies to Service Type: LoadBalancer. - This feature depends on whether the underlying cloud-provider - supports specifying the loadBalancerIP when a load balancer - is created. This field will be ignored if the cloud-provider - does not support the feature. Deprecated: This field was - under-specified and its meaning varies across implementations, - and it cannot support dual-stack. As of Kubernetes v1.24, - users are encouraged to use implementation-specific annotations - when available. This field may be removed in a future API - version.' - type: string - loadBalancerSourceRanges: - description: 'If specified and supported by the platform, - this will restrict traffic through the cloud-provider load-balancer - will be restricted to the specified client IPs. This field - will be ignored if the cloud-provider does not support the - feature." More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/' - items: - type: string - type: array - ports: - description: 'The list of ports that are exposed by this service. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - items: - description: ServicePort contains information on service's - port. - properties: - appProtocol: - description: The application protocol for this port. - This field follows standard Kubernetes label syntax. - Un-prefixed names are reserved for IANA standard service - names (as per RFC-6335 and https://www.iana.org/assignments/service-names). - Non-standard protocols should use prefixed names such - as mycompany.com/my-custom-protocol. - type: string - name: - description: The name of this port within the service. - This must be a DNS_LABEL. All ports within a ServiceSpec - must have unique names. When considering the endpoints - for a Service, this must match the 'name' field in - the EndpointPort. Optional if only one ServicePort - is defined on this service. - type: string - nodePort: - description: 'The port on each node on which this service - is exposed when type is NodePort or LoadBalancer. Usually - assigned by the system. If a value is specified, in-range, - and not in use it will be used, otherwise the operation - will fail. If not specified, a port will be allocated - if this Service requires one. If this field is specified - when creating a Service which does not need it, creation - will fail. This field will be wiped when updating - a Service to no longer need it (e.g. changing type - from NodePort to ClusterIP). More info: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport' - format: int32 - type: integer - port: - description: The port that will be exposed by this service. - format: int32 - type: integer - protocol: - default: TCP - description: The IP protocol for this port. Supports - "TCP", "UDP", and "SCTP". Default is TCP. - type: string - targetPort: - anyOf: - - type: integer - - type: string - description: 'Number or name of the port to access on - the pods targeted by the service. Number must be in - the range 1 to 65535. Name must be an IANA_SVC_NAME. - If this is a string, it will be looked up as a named - port in the target Pod''s container ports. If this - is not specified, the value of the ''port'' field - is used (an identity map). This field is ignored for - services with clusterIP=None, and should be omitted - or set equal to the ''port'' field. More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service' - x-kubernetes-int-or-string: true - required: - - port - type: object - type: array - x-kubernetes-list-map-keys: - - port - - protocol - x-kubernetes-list-type: map - publishNotReadyAddresses: - description: publishNotReadyAddresses indicates that any agent - which deals with endpoints for this Service should disregard - any indications of ready/not-ready. The primary use case - for setting this field is for a StatefulSet's Headless Service - to propagate SRV DNS records for its Pods for the purpose - of peer discovery. The Kubernetes controllers that generate - Endpoints and EndpointSlice resources for Services interpret - this to mean that all endpoints are considered "ready" even - if the Pods themselves are not. Agents which consume only - Kubernetes generated endpoints through the Endpoints or - EndpointSlice resources can safely assume this behavior. - type: boolean - selector: - additionalProperties: - type: string - description: 'Route service traffic to pods with label keys - and values matching this selector. If empty or not present, - the service is assumed to have an external process managing - its endpoints, which Kubernetes will not modify. Only applies - to types ClusterIP, NodePort, and LoadBalancer. Ignored - if type is ExternalName. More info: https://kubernetes.io/docs/concepts/services-networking/service/' - type: object - x-kubernetes-map-type: atomic - sessionAffinity: - description: 'Supports "ClientIP" and "None". Used to maintain - session affinity. Enable client IP based session affinity. - Must be ClientIP or None. Defaults to None. More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies' - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations - of session affinity. - properties: - clientIP: - description: clientIP contains the configurations of Client - IP based session affinity. - properties: - timeoutSeconds: - description: timeoutSeconds specifies the seconds - of ClientIP type session sticky time. The value - must be >0 && <=86400(for 1 day) if ServiceAffinity - == "ClientIP". Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - type: - description: 'type determines how the Service is exposed. - Defaults to ClusterIP. Valid options are ExternalName, ClusterIP, - NodePort, and LoadBalancer. "ClusterIP" allocates a cluster-internal - IP address for load-balancing to endpoints. Endpoints are - determined by the selector or if that is not specified, - by manual construction of an Endpoints object or EndpointSlice - objects. If clusterIP is "None", no virtual IP is allocated - and the endpoints are published as a set of endpoints rather - than a virtual IP. "NodePort" builds on ClusterIP and allocates - a port on every node which routes to the same endpoints - as the clusterIP. "LoadBalancer" builds on NodePort and - creates an external load-balancer (if supported in the current - cloud) which routes to the same endpoints as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types' - type: string - type: object - required: - - spec - type: object - required: - - resources - type: object - status: - description: AppWrapperStatus represents the current state of a AppWrapper - properties: - Succeeded: - description: The number of resources which reached phase Succeeded. - format: int32 - type: integer - canrun: - description: Can run? - type: boolean - conditions: - description: Represents the latest available observations of the AppWrapper's - current condition. - items: - description: AppWrapperCondition describes the state of an AppWrapper - at a certain point. - properties: - lastTransitionMicroTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - lastUpdateMicroTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human-readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of AppWrapper condition. - type: string - required: - - status - - type - type: object - type: array - controllerfirstdispatchtimestamp: - description: Microsecond level timestamp when controller first dispatches - the AppWrapper - format: date-time - type: string - controllerfirsttimestamp: - description: Microsecond level timestamp when controller first sees - QueueJob (by Informer) - format: date-time - type: string - failed: - description: The number of resources which reached phase Failed. - format: int32 - type: integer - filterignore: - description: Tell Informer to ignore this update message (do not generate - a controller event) - type: boolean - isdispatched: - description: Is Dispatched? - type: boolean - local: - description: Indicate if message is a duplicate (for Informer to recognize - duplicate messages) - type: boolean - message: - type: string - numberOfRequeueings: - description: Field to keep track of how many times a requeuing event - has been triggered - type: integer - pending: - description: The number of pending pods. - format: int32 - type: integer - pendingpodconditions: - description: Represents the latest available observations of pods - belonging to the AppWrapper. - items: - properties: - conditions: - items: - description: PodCondition contains details for the current - condition of this pod. - properties: - lastProbeTime: - description: Last time we probed the condition. - format: date-time - type: string - lastTransitionTime: - description: Last time the condition transitioned from - one status to another. - format: date-time - type: string - message: - description: Human-readable message indicating details - about last transition. - type: string - reason: - description: Unique, one-word, CamelCase reason for the - condition's last transition. - type: string - status: - description: 'Status is the status of the condition. Can - be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - type: - description: 'Type is the type of the condition. More - info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' - type: string - required: - - status - - type - type: object - type: array - podname: - type: string - type: object - type: array - queuejobstate: - description: State of QueueJob - Init, Queueing, HeadOfLine, Rejoining, - ... - type: string - requeueingTimeInSeconds: - description: Field to keep track of total number of seconds spent - in requeueing - type: integer - running: - format: int32 - type: integer - sender: - description: Indicate sender of this message (extremely useful for - debugging) - type: string - state: - description: State - Pending, Running, Failed, Deleted - type: string - systempriority: - description: System defined Priority - format: float - type: number - template: - description: The minimal available resources to run for this AppWrapper - (is this different from the MinAvailable from JobStatus) - format: int32 - type: integer - totalcpu: - description: The number of CPU consumed by all pods belonging to the - AppWrapper. - format: int32 - type: integer - totalgpu: - description: The total number of GPUs consumed by all pods belonging - to the AppWrapper. - format: int32 - type: integer - totalmemory: - description: The amount of memory consumed by all pods belonging to - the AppWrapper. - format: int32 - type: integer - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- diff --git a/config/crd/workload.codeflare_schedulingspecs.yaml b/config/crd/workload.codeflare_schedulingspecs.yaml deleted file mode 100644 index fc44a4326..000000000 --- a/config/crd/workload.codeflare_schedulingspecs.yaml +++ /dev/null @@ -1,102 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null - name: schedulingspecs.workload.codeflare.dev -spec: - group: workload.codeflare.dev - names: - kind: SchedulingSpec - listKind: SchedulingSpecList - plural: schedulingspecs - singular: schedulingspec - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - dispatchDuration: - description: Wall clock duration time of appwrapper in seconds. - properties: - expected: - type: integer - limit: - type: integer - overrun: - type: boolean - type: object - minAvailable: - description: Expected number of pods in running and/or completed state. - Requeuing is triggered when the number of running/completed pods - is not equal to this value. When not specified, requeuing is disabled - and no check is performed. - type: integer - nodeSelector: - additionalProperties: - type: string - type: object - requeuing: - description: Specification of the requeuing strategy based on waiting - time. Values in this field control how often the pod check should - happen, and if requeuing has reached its maximum number of times. - properties: - growthType: - default: exponential - description: Growth strategy to increase the waiting time between - requeuing checks. The values available are 'exponential', 'linear', - or 'none'. For example, 'exponential' growth would double the - 'timeInSeconds' value every time a requeuing event is triggered. - If the string value is misspelled or not one of the possible - options, the growth behavior is defaulted to 'none'. - type: string - initialTimeInSeconds: - description: Value to keep track of the initial wait time. Users - cannot set this as it is taken from 'timeInSeconds'. - type: integer - maxNumRequeuings: - default: 0 - description: Maximum number of requeuing events allowed. Once - this value is reached (e.g., 'numRequeuings = maxNumRequeuings', - no more requeuing checks are performed and the generic items - are stopped and removed from the cluster (AppWrapper remains - deployed). - type: integer - maxTimeInSeconds: - default: 0 - description: Maximum waiting time for requeuing checks. - type: integer - numRequeuings: - default: 0 - description: Field to keep track of how many times a requeuing - event has been triggered. - type: integer - timeInSeconds: - default: 300 - description: Initial waiting time before requeuing conditions - are checked. This value is specified by the user, but it may - grow as requeuing events happen. - type: integer - type: object - type: object - required: - - metadata - type: object - served: true - storage: true