diff --git a/Gopkg.lock b/Gopkg.lock index 4a997dba..386a8094 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -50,6 +50,14 @@ revision = "f140a6486e521aad38f5917de355cbf147cc0496" version = "v1.0.0" +[[projects]] + digest = "1:653085796b580bbe8a516d4ccdac890f57bc568bb1a68da02eefca8d087ea130" + name = "github.com/google/uuid" + packages = ["."] + pruneopts = "NUT" + revision = "0e4e31197428a347842d152773b4cace4645ca25" + version = "v1.1.2" + [[projects]] digest = "1:06a7dadb7b760767341ffb6c8d377238d68a1226f2b21b5d497d2e3f6ecf6b4e" name = "github.com/googleapis/gnostic" @@ -105,6 +113,14 @@ revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" +[[projects]] + digest = "1:d0e34331d084e65cfb94d335bf285a94d3b068549c50f8e2c8d96c82897ebf9e" + name = "github.com/pborman/uuid" + packages = ["."] + pruneopts = "NUT" + revision = "5b6091a6a160ee5ce12917b21ab96acec2a4fdc0" + version = "v1.2.1" + [[projects]] digest = "1:14715f705ff5dfe0ffd6571d7d201dd8e921030f8070321a79380d8ca4ec1a24" name = "github.com/pkg/errors" @@ -312,7 +328,7 @@ version = "kubernetes-1.14.2" [[projects]] - digest = "1:0db508f16c0059bbc1dc063fd5917c48e64693f5756a6e313a2f39821d81b327" + digest = "1:f9bde8a163863c37fbf2bb896e8972c94b4a00330fc73c17a40a3ce26a5f7831" name = "k8s.io/apimachinery" packages = [ "pkg/api/errors", @@ -349,6 +365,7 @@ "pkg/util/runtime", "pkg/util/sets", "pkg/util/strategicpatch", + "pkg/util/uuid", "pkg/util/validation", "pkg/util/validation/field", "pkg/util/wait", @@ -618,7 +635,9 @@ "k8s.io/apimachinery/pkg/runtime/serializer", "k8s.io/apimachinery/pkg/types", "k8s.io/apimachinery/pkg/util/errors", + "k8s.io/apimachinery/pkg/util/net", "k8s.io/apimachinery/pkg/util/runtime", + "k8s.io/apimachinery/pkg/util/uuid", "k8s.io/apimachinery/pkg/util/wait", "k8s.io/apimachinery/pkg/watch", "k8s.io/client-go/discovery", diff --git a/doc/user-manual.md b/doc/user-manual.md index 6538a23b..750944f1 100644 --- a/doc/user-manual.md +++ b/doc/user-manual.md @@ -3,7 +3,7 @@ ## Index - [Framework Interop](#FrameworkInterop) - [Framework ExecutionType](#FrameworkExecutionType) - - [Container EnvironmentVariable](#ContainerEnvironmentVariable) + - [Predefined Container EnvironmentVariable](#PredefinedContainerEnvironmentVariable) - [Pod Failure Classification](#PodFailureClassification) - [Predefined CompletionCode](#PredefinedCompletionCode) - [CompletionStatus](#CompletionStatus) @@ -475,8 +475,10 @@ spec: 3. [Get Framework](#GET_Framework), and archive it into a DataBase first. 4. [Delete Framework](#DELETE_Framework), then the Framework will be deleted. -## Container EnvironmentVariable -[Container EnvironmentVariable](../pkg/apis/frameworkcontroller/v1/constants.go) +## Predefined Container EnvironmentVariable +[Predefined Container EnvironmentVariable](../pkg/apis/frameworkcontroller/v1/constants.go) + +[Framework Example](../example/framework/basic/batchstatefulfailed.yaml) ## Pod Failure Classification You can specify how to classify and summarize Pod failures by the [PodFailureSpec](../pkg/apis/frameworkcontroller/v1/config.go). @@ -842,7 +844,7 @@ Besides these general [Framework ConsistencyGuarantees](#ConsistencyGuarantees), To safely run large scale Framework, i.e. the total task number in a single Framework is greater than 300, you just need to enable the [LargeFrameworkCompression](../pkg/apis/frameworkcontroller/v1/config.go). However, you may also need to decompress the Framework by yourself. ## Framework and Pod History -By leveraging the [LogObjectSnapshot](../pkg/apis/frameworkcontroller/v1/config.go), external systems, such as [Fluentd](https://www.fluentd.org) and [ElasticSearch](https://www.elastic.co/products/elasticsearch), can collect and process Framework and Pod history snapshots even if it was retried or deleted, such as for persistence, metrics conversion, visualization, alerting, acting, analysis, etc. +By leveraging the [LogObjectSnapshot](../pkg/apis/frameworkcontroller/v1/config.go), external systems, such as [Fluentd](https://www.fluentd.org) and [ElasticSearch](https://www.elastic.co/products/elasticsearch), can collect and process Framework, Task and Pod history snapshots even if it was retried or deleted, such as for persistence, metrics conversion, visualization, alerting, acting, analysis, etc. ## Framework and Task State Machine ### Framework State Machine @@ -894,7 +896,7 @@ The default behavior is to achieve all the [ConsistencyGuarantees](#ConsistencyG For example, [drain the Node](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node) before delete it is acceptable. - *The Task instance can be universally located by its [TaskAttemptInstanceUID](../pkg/apis/frameworkcontroller/v1/types.go) or [PodUID](../pkg/apis/frameworkcontroller/v1/types.go).* + *The Task running instance can be universally located by its [TaskAttemptInstanceUID](../pkg/apis/frameworkcontroller/v1/types.go) or [PodUID](../pkg/apis/frameworkcontroller/v1/types.go).* *To avoid the Pod is stuck in deleting forever, such as if its Node is down forever, leverage the same approach as [Delete StatefulSet Pod only after the Pod termination has been confirmed](https://kubernetes.io/docs/tasks/run-application/force-delete-stateful-set-pod/#delete-pods) manually or by your [Cloud Controller Manager](https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#running-cloud-controller-manager).* @@ -911,7 +913,7 @@ The default behavior is to achieve all the [ConsistencyGuarantees](#ConsistencyG 4. Do not change the [OwnerReferences](https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#owners-and-dependents) of the managed ConfigMap and Pods. - *The Framework instance can be universally located by its [FrameworkAttemptInstanceUID](../pkg/apis/frameworkcontroller/v1/types.go) or [ConfigMapUID](../pkg/apis/frameworkcontroller/v1/types.go).* + *The Framework running instance can be universally located by its [FrameworkAttemptInstanceUID](../pkg/apis/frameworkcontroller/v1/types.go) or [ConfigMapUID](../pkg/apis/frameworkcontroller/v1/types.go).* ### Framework Availability According to the [CAP theorem](https://en.wikipedia.org/wiki/CAP_theorem), in the presence of a network partition, you cannot achieve both consistency and availability at the same time in any distributed system. So you have to make a trade-off between the [Framework Consistency](#FrameworkConsistency) and the [Framework Availability](#FrameworkAvailability). diff --git a/example/framework/basic/batchstatefulfailed.yaml b/example/framework/basic/batchstatefulfailed.yaml index df14d0a9..09044e7b 100644 --- a/example/framework/basic/batchstatefulfailed.yaml +++ b/example/framework/basic/batchstatefulfailed.yaml @@ -27,21 +27,30 @@ spec: - name: ubuntu image: ubuntu:trusty # To locate a specific Task during its whole lifecycle regardless of - # any retry: + # any retry and rescale: # Consistent Identity: - # PodNamespace = {FrameworkNamespace} - # PodName = {FrameworkName}-{TaskRoleName}-{TaskIndex} + # PodNamespace = {FrameworkNamespace} + # PodName = {FrameworkName}-{TaskRoleName}-{TaskIndex} # Consistent Environment Variable Value: - # ${FC_FRAMEWORK_NAMESPACE}, - # ${FC_FRAMEWORK_NAME}, ${FC_TASKROLE_NAME}, ${FC_TASK_INDEX}, - # ${FC_CONFIGMAP_NAME}, ${FC_POD_NAME} + # ${FC_FRAMEWORK_NAMESPACE} + # ${FC_FRAMEWORK_NAME} + # ${FC_TASKROLE_NAME} + # ${FC_TASK_INDEX} # - # To locate a specific execution attempt of a specific Task: - # Attempt Specific Environment Variable Value: - # ${FC_FRAMEWORK_ATTEMPT_ID}, ${FC_TASK_ATTEMPT_ID} + # To locate a specific Task instance, in case the Task is deleted then + # added by rescale with a different Task instance: + # Environment Variable Value: + # ${FC_TASK_UID} # - # To locate a specific execution attempt instance of a specific Task: - # Attempt Instance Specific Environment Variable Value: - # ${FC_FRAMEWORK_ATTEMPT_INSTANCE_UID}, ${FC_CONFIGMAP_UID} - # ${FC_TASK_ATTEMPT_INSTANCE_UID}, ${FC_POD_UID} + # To locate a specific execution attempt of a specific Task instance: + # Environment Variable Value: + # ${FC_TASK_UID} + # ${FC_TASK_ATTEMPT_ID} + # + # To locate a specific execution attempt instance of a specific Task + # instance, in case the attempt instance, i.e. the Pod instance is + # created but not observed by FrameworkController, then it is deleted + # and created later with a different attempt instance: + # Environment Variable Value: + # ${FC_TASK_ATTEMPT_INSTANCE_UID} command: ["sh", "-c", "printenv && sleep 60 && exit 1"] diff --git a/pkg/apis/frameworkcontroller/v1/config.go b/pkg/apis/frameworkcontroller/v1/config.go index a47df52c..c11607a3 100644 --- a/pkg/apis/frameworkcontroller/v1/config.go +++ b/pkg/apis/frameworkcontroller/v1/config.go @@ -117,7 +117,7 @@ type Config struct { // analysis, etc. // Notes: // 1. The snapshot is logged to stderr and can be extracted by the regular - // expression ": ObjectSnapshot: (.+)". + // expression ": ObjectSnapshot: (.+)", see LogMarkerObjectSnapshot. // 2. To determine the type of the snapshot, using object.apiVersion and // object.kind. // 3. The same snapshot may be logged more than once in some rare cases, so @@ -149,16 +149,20 @@ type Config struct { type LogObjectSnapshot struct { Framework LogFrameworkSnapshot `yaml:"framework"` + Task LogTaskSnapshot `yaml:"task"` Pod LogPodSnapshot `yaml:"pod"` } type LogFrameworkSnapshot struct { - OnTaskRetry *bool `yaml:"onTaskRetry"` OnFrameworkRetry *bool `yaml:"onFrameworkRetry"` - OnFrameworkRescale *bool `yaml:"onFrameworkRescale"` OnFrameworkDeletion *bool `yaml:"onFrameworkDeletion"` } +type LogTaskSnapshot struct { + OnTaskRetry *bool `yaml:"onTaskRetry"` + OnTaskDeletion *bool `yaml:"onTaskDeletion"` +} + type LogPodSnapshot struct { OnPodDeletion *bool `yaml:"onPodDeletion"` } @@ -254,18 +258,18 @@ func NewConfig() *Config { if c.FrameworkMaxRetryDelaySecForTransientConflictFailed == nil { c.FrameworkMaxRetryDelaySecForTransientConflictFailed = common.PtrInt64(15 * 60) } - if c.LogObjectSnapshot.Framework.OnTaskRetry == nil { - c.LogObjectSnapshot.Framework.OnTaskRetry = common.PtrBool(true) - } if c.LogObjectSnapshot.Framework.OnFrameworkRetry == nil { c.LogObjectSnapshot.Framework.OnFrameworkRetry = common.PtrBool(true) } - if c.LogObjectSnapshot.Framework.OnFrameworkRescale == nil { - c.LogObjectSnapshot.Framework.OnFrameworkRescale = common.PtrBool(true) - } if c.LogObjectSnapshot.Framework.OnFrameworkDeletion == nil { c.LogObjectSnapshot.Framework.OnFrameworkDeletion = common.PtrBool(true) } + if c.LogObjectSnapshot.Task.OnTaskRetry == nil { + c.LogObjectSnapshot.Task.OnTaskRetry = common.PtrBool(true) + } + if c.LogObjectSnapshot.Task.OnTaskDeletion == nil { + c.LogObjectSnapshot.Task.OnTaskDeletion = common.PtrBool(true) + } if c.LogObjectSnapshot.Pod.OnPodDeletion == nil { c.LogObjectSnapshot.Pod.OnPodDeletion = common.PtrBool(true) } diff --git a/pkg/apis/frameworkcontroller/v1/constants.go b/pkg/apis/frameworkcontroller/v1/constants.go index 819af5d9..1638d9eb 100644 --- a/pkg/apis/frameworkcontroller/v1/constants.go +++ b/pkg/apis/frameworkcontroller/v1/constants.go @@ -38,6 +38,7 @@ const ( FrameworkPlural = "frameworks" FrameworkCRDName = FrameworkPlural + "." + GroupName FrameworkKind = "Framework" + TaskKind = "Task" ConfigMapKind = "ConfigMap" PodKind = "Pod" ObjectUIDFieldPath = "metadata.uid" @@ -56,9 +57,12 @@ const ( AnnotationKeyConfigMapName = "FC_CONFIGMAP_NAME" AnnotationKeyPodName = "FC_POD_NAME" + AnnotationKeyFrameworkUID = "FC_FRAMEWORK_UID" AnnotationKeyFrameworkAttemptID = "FC_FRAMEWORK_ATTEMPT_ID" AnnotationKeyFrameworkAttemptInstanceUID = "FC_FRAMEWORK_ATTEMPT_INSTANCE_UID" AnnotationKeyConfigMapUID = "FC_CONFIGMAP_UID" + AnnotationKeyTaskRoleUID = "FC_TASKROLE_UID" + AnnotationKeyTaskUID = "FC_TASK_UID" AnnotationKeyTaskAttemptID = "FC_TASK_ATTEMPT_ID" // Predefined Labels @@ -79,9 +83,12 @@ const ( EnvNameConfigMapName = AnnotationKeyConfigMapName EnvNamePodName = AnnotationKeyPodName + EnvNameFrameworkUID = AnnotationKeyFrameworkUID EnvNameFrameworkAttemptID = AnnotationKeyFrameworkAttemptID EnvNameFrameworkAttemptInstanceUID = AnnotationKeyFrameworkAttemptInstanceUID EnvNameConfigMapUID = AnnotationKeyConfigMapUID + EnvNameTaskRoleUID = AnnotationKeyTaskRoleUID + EnvNameTaskUID = AnnotationKeyTaskUID EnvNameTaskAttemptID = AnnotationKeyTaskAttemptID EnvNameTaskAttemptInstanceUID = "FC_TASK_ATTEMPT_INSTANCE_UID" EnvNamePodUID = "FC_POD_UID" @@ -98,9 +105,22 @@ const ( PlaceholderTaskIndex = AnnotationKeyTaskIndex PlaceholderConfigMapName = AnnotationKeyConfigMapName PlaceholderPodName = AnnotationKeyPodName + + // For LogObjectSnapshot + // All snapshots are logged in format: + // {AnyLogMessage}{ObjectSnapshotTrigger}{LogMarkerObjectSnapshot}{JsonObjectSnapshot} + LogMarkerObjectSnapshot = ": ObjectSnapshot: " + LogMarkerOnFrameworkRetry ObjectSnapshotTrigger = ": OnFrameworkRetry" + LogMarkerOnFrameworkDeletion ObjectSnapshotTrigger = ": OnFrameworkDeletion" + LogMarkerOnTaskRetry ObjectSnapshotTrigger = ": OnTaskRetry" + LogMarkerOnTaskDeletion ObjectSnapshotTrigger = ": OnTaskDeletion" + LogMarkerOnPodDeletion ObjectSnapshotTrigger = ": OnPodDeletion" ) +type ObjectSnapshotTrigger string + var FrameworkGroupVersionKind = SchemeGroupVersion.WithKind(FrameworkKind) +var TaskGroupVersionKind = SchemeGroupVersion.WithKind(TaskKind) var ConfigMapGroupVersionKind = core.SchemeGroupVersion.WithKind(ConfigMapKind) var PodGroupVersionKind = core.SchemeGroupVersion.WithKind(PodKind) diff --git a/pkg/apis/frameworkcontroller/v1/funcs.go b/pkg/apis/frameworkcontroller/v1/funcs.go index 0baa209e..e59a5283 100644 --- a/pkg/apis/frameworkcontroller/v1/funcs.go +++ b/pkg/apis/frameworkcontroller/v1/funcs.go @@ -28,6 +28,7 @@ import ( core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/klog" "sort" "strconv" @@ -57,22 +58,30 @@ func SplitConfigMapName(configMapName string) (frameworkName string) { return parts[0] } -func GetPodName(frameworkName string, taskRoleName string, taskIndex int32) string { +func GetTaskName(frameworkName string, taskRoleName string, taskIndex int32) string { return strings.Join([]string{frameworkName, taskRoleName, fmt.Sprint(taskIndex)}, "-") } -func SplitPodName(podName string) (frameworkName string, taskRoleName string, taskIndex int32) { - parts := strings.Split(podName, "-") +func SplitTaskName(taskName string) (frameworkName string, taskRoleName string, taskIndex int32) { + parts := strings.Split(taskName, "-") if len(parts) != 3 { - panic(fmt.Errorf("Failed to SplitPodName %v", podName)) + panic(fmt.Errorf("Failed to SplitTaskName %v", taskName)) } i, err := strconv.ParseInt(parts[2], 10, 32) if err != nil { - panic(fmt.Errorf("Failed to SplitPodName %v: %v", podName, err)) + panic(fmt.Errorf("Failed to SplitTaskName %v: %v", taskName, err)) } return parts[0], parts[1], int32(i) } +func GetPodName(frameworkName string, taskRoleName string, taskIndex int32) string { + return GetTaskName(frameworkName, taskRoleName, taskIndex) +} + +func SplitPodName(podName string) (frameworkName string, taskRoleName string, taskIndex int32) { + return SplitTaskName(podName) +} + func GetFrameworkAttemptInstanceUID(frameworkAttemptID int32, configMapUID *types.UID) *types.UID { return common.PtrUIDStr(fmt.Sprintf("%v_%v", frameworkAttemptID, *configMapUID)) } @@ -115,26 +124,6 @@ func SplitTaskAttemptInstanceUID(taskAttemptInstanceUID *types.UID) ( return int32(i), common.PtrUIDStr(parts[1]) } -func getObjectSnapshotLogTail(obj interface{}) string { - return ": ObjectSnapshot: " + common.ToJson(obj) -} - -func GetFrameworkSnapshotLogTail(f *Framework) string { - if f.GroupVersionKind().Empty() { - f = f.DeepCopy() - f.SetGroupVersionKind(FrameworkGroupVersionKind) - } - return getObjectSnapshotLogTail(f) -} - -func GetPodSnapshotLogTail(pod *core.Pod) string { - if pod.GroupVersionKind().Empty() { - pod = pod.DeepCopy() - pod.SetGroupVersionKind(PodGroupVersionKind) - } - return getObjectSnapshotLogTail(pod) -} - func GetAllContainerStatuses(pod *core.Pod) []core.ContainerStatus { // All Container names in a Pod must be different, so we can still identify // a Container even after the InitContainers is merged with the AppContainers. @@ -209,6 +198,74 @@ func NewCompletedTaskTriggeredCompletionStatus( } } +/////////////////////////////////////////////////////////////////////////////////////// +// LogObjectSnapshot Methods +// Before calling them, ensure the object snapshot has been persisted, so it is +// safe to also expose it as one history snapshot here. +/////////////////////////////////////////////////////////////////////////////////////// +func getObjectSnapshotLogTail(obj interface{}) string { + return LogMarkerObjectSnapshot + common.ToJson(obj) +} + +func getFrameworkSnapshotLogTail(f *Framework) string { + if f.GroupVersionKind().Empty() { + f = f.DeepCopy() + f.SetGroupVersionKind(FrameworkGroupVersionKind) + } + return getObjectSnapshotLogTail(f) +} + +func getTaskSnapshotLogTail(task *Task) string { + if task.GroupVersionKind().Empty() { + task = task.DeepCopy() + task.SetGroupVersionKind(TaskGroupVersionKind) + } + return getObjectSnapshotLogTail(task) +} + +func getPodSnapshotLogTail(pod *core.Pod) string { + if pod.GroupVersionKind().Empty() { + pod = pod.DeepCopy() + pod.SetGroupVersionKind(PodGroupVersionKind) + } + return getObjectSnapshotLogTail(pod) +} + +func (s *LogFrameworkSnapshot) GetLogTailOnFrameworkRetry(f *Framework) string { + if *s.OnFrameworkRetry { + return string(LogMarkerOnFrameworkRetry) + getFrameworkSnapshotLogTail(f) + } + return "" +} + +func (s *LogFrameworkSnapshot) GetLogTailOnFrameworkDeletion(f *Framework) string { + if *s.OnFrameworkDeletion { + return string(LogMarkerOnFrameworkDeletion) + getFrameworkSnapshotLogTail(f) + } + return "" +} + +func (s *LogTaskSnapshot) GetLogTailOnTaskRetry(task *Task) string { + if *s.OnTaskRetry { + return string(LogMarkerOnTaskRetry) + getTaskSnapshotLogTail(task) + } + return "" +} + +func (s *LogTaskSnapshot) GetLogTailOnTaskDeletion(task *Task) string { + if *s.OnTaskDeletion { + return string(LogMarkerOnTaskDeletion) + getTaskSnapshotLogTail(task) + } + return "" +} + +func (s *LogPodSnapshot) GetLogTailOnPodDeletion(pod *core.Pod) string { + if *s.OnPodDeletion { + return string(LogMarkerOnPodDeletion) + getPodSnapshotLogTail(pod) + } + return "" +} + /////////////////////////////////////////////////////////////////////////////////////// // Interfaces /////////////////////////////////////////////////////////////////////////////////////// @@ -222,7 +279,7 @@ func (f *Framework) Key() string { return f.Namespace + "/" + f.Name } -// Return nil if and only if TaskRoleSpec is deleted while the TaskRole's +// Return nil if and only if the TaskRoleSpec is deleted while the TaskRole's // TaskRoleStatus still exist due to graceful deletion. func (f *Framework) GetTaskRoleSpec(taskRoleName string) *TaskRoleSpec { for _, taskRole := range f.Spec.TaskRoles { @@ -233,7 +290,16 @@ func (f *Framework) GetTaskRoleSpec(taskRoleName string) *TaskRoleSpec { return nil } -// Panic if and only if TaskRoleSpec is deleted while the TaskRole's +// Return nil if and only if its TaskRole's TaskRoleSpec is deleted while its +// TaskStatus still exist due to graceful deletion. +func (f *Framework) GetTaskSpec(taskRoleName string) *TaskSpec { + if taskRole := f.GetTaskRoleSpec(taskRoleName); taskRole != nil { + return &taskRole.Task + } + return nil +} + +// Panic if and only if the TaskRoleSpec is deleted while the TaskRole's // TaskRoleStatus still exist due to graceful deletion. func (f *Framework) TaskRoleSpec(taskRoleName string) *TaskRoleSpec { if taskRole := f.GetTaskRoleSpec(taskRoleName); taskRole != nil { @@ -310,6 +376,15 @@ func (f *Framework) GetTaskRoleStatus(taskRoleName string) *TaskRoleStatus { return nil } +func (f *Framework) GetTaskStatus(taskRoleName string, taskIndex int32) *TaskStatus { + if taskRoleStatus := f.GetTaskRoleStatus(taskRoleName); taskRoleStatus != nil { + if 0 <= taskIndex && taskIndex < int32(len(taskRoleStatus.TaskStatuses)) { + return taskRoleStatus.TaskStatuses[taskIndex] + } + } + return nil +} + func (f *Framework) TaskRoleStatus(taskRoleName string) *TaskRoleStatus { if taskRoleStatus := f.GetTaskRoleStatus(taskRoleName); taskRoleStatus != nil { return taskRoleStatus @@ -317,18 +392,11 @@ func (f *Framework) TaskRoleStatus(taskRoleName string) *TaskRoleStatus { panic(fmt.Errorf("[%v]: TaskRole is not found in Status", taskRoleName)) } -func (f *Framework) GetTaskStatus(taskRoleName string, taskIndex int32) *TaskStatus { +func (f *Framework) TaskStatus(taskRoleName string, taskIndex int32) *TaskStatus { taskRoleStatus := f.TaskRoleStatus(taskRoleName) if 0 <= taskIndex && taskIndex < int32(len(taskRoleStatus.TaskStatuses)) { return taskRoleStatus.TaskStatuses[taskIndex] } - return nil -} - -func (f *Framework) TaskStatus(taskRoleName string, taskIndex int32) *TaskStatus { - if taskStatus := f.GetTaskStatus(taskRoleName, taskIndex); taskStatus != nil { - return taskStatus - } panic(fmt.Errorf("[%v][%v]: Task is not found in Status", taskRoleName, taskIndex)) } @@ -446,6 +514,7 @@ func (f *Framework) IsAnyTaskRunning(ignoreDeletionPending bool) bool { } func (f *Framework) NewConfigMap() *core.ConfigMap { + frameworkUIDStr := string(f.UID) frameworkAttemptIDStr := fmt.Sprint(f.FrameworkAttemptID()) cm := &core.ConfigMap{ @@ -463,6 +532,7 @@ func (f *Framework) NewConfigMap() *core.ConfigMap { cm.Annotations[AnnotationKeyFrameworkNamespace] = f.Namespace cm.Annotations[AnnotationKeyFrameworkName] = f.Name cm.Annotations[AnnotationKeyConfigMapName] = cm.Name + cm.Annotations[AnnotationKeyFrameworkUID] = frameworkUIDStr cm.Annotations[AnnotationKeyFrameworkAttemptID] = frameworkAttemptIDStr cm.Labels = map[string]string{} @@ -471,14 +541,19 @@ func (f *Framework) NewConfigMap() *core.ConfigMap { return cm } +// Before calling it, ensure its TaskRole's TaskRoleSpec exists. func (f *Framework) NewPod(cm *core.ConfigMap, taskRoleName string, taskIndex int32) *core.Pod { // Deep copy Task.Pod before modify it taskPodJson := common.ToJson(f.TaskRoleSpec(taskRoleName).Task.Pod) + taskRoleStatus := f.TaskRoleStatus(taskRoleName) taskStatus := f.TaskStatus(taskRoleName, taskIndex) taskIndexStr := fmt.Sprint(taskIndex) + frameworkUIDStr := string(f.UID) frameworkAttemptIDStr := fmt.Sprint(f.FrameworkAttemptID()) frameworkAttemptInstanceUIDStr := string(*f.FrameworkAttemptInstanceUID()) configMapUIDStr := string(*f.ConfigMapUID()) + taskRoleUIDStr := string(taskRoleStatus.InstanceUID) + taskUIDStr := string(taskStatus.InstanceUID) taskAttemptIDStr := fmt.Sprint(taskStatus.TaskAttemptID()) taskAttemptInstanceUIDReferStr := string(*GetTaskAttemptInstanceUID( taskStatus.TaskAttemptID(), @@ -526,9 +601,12 @@ func (f *Framework) NewPod(cm *core.ConfigMap, taskRoleName string, taskIndex in pod.Annotations[AnnotationKeyTaskIndex] = taskIndexStr pod.Annotations[AnnotationKeyConfigMapName] = f.ConfigMapName() pod.Annotations[AnnotationKeyPodName] = pod.Name + pod.Annotations[AnnotationKeyFrameworkUID] = frameworkUIDStr pod.Annotations[AnnotationKeyFrameworkAttemptID] = frameworkAttemptIDStr pod.Annotations[AnnotationKeyFrameworkAttemptInstanceUID] = frameworkAttemptInstanceUIDStr pod.Annotations[AnnotationKeyConfigMapUID] = configMapUIDStr + pod.Annotations[AnnotationKeyTaskRoleUID] = taskRoleUIDStr + pod.Annotations[AnnotationKeyTaskUID] = taskUIDStr pod.Annotations[AnnotationKeyTaskAttemptID] = taskAttemptIDStr if pod.Labels == nil { @@ -545,9 +623,12 @@ func (f *Framework) NewPod(cm *core.ConfigMap, taskRoleName string, taskIndex in {Name: EnvNameTaskIndex, Value: taskIndexStr}, {Name: EnvNameConfigMapName, Value: f.ConfigMapName()}, {Name: EnvNamePodName, Value: pod.Name}, + {Name: EnvNameFrameworkUID, Value: frameworkUIDStr}, {Name: EnvNameFrameworkAttemptID, Value: frameworkAttemptIDStr}, {Name: EnvNameFrameworkAttemptInstanceUID, Value: frameworkAttemptInstanceUIDStr}, {Name: EnvNameConfigMapUID, Value: configMapUIDStr}, + {Name: EnvNameTaskRoleUID, Value: taskRoleUIDStr}, + {Name: EnvNameTaskUID, Value: taskUIDStr}, {Name: EnvNameTaskAttemptID, Value: taskAttemptIDStr}, {Name: EnvNamePodUID, ValueFrom: ObjectUIDEnvVarSource}, {Name: EnvNameTaskAttemptInstanceUID, Value: taskAttemptInstanceUIDReferStr}, @@ -579,12 +660,69 @@ func (f *Framework) NewPod(cm *core.ConfigMap, taskRoleName string, taskIndex in return pod } +// Mock a standalone Task from an embedded Task. +// Before calling it, ensure the embedded Task has been persisted, so the standalone +// Task can be considered to have ever existed with the same ResourceVersion as its +// Framework. +func (f *Framework) MockTask(taskRoleName string, taskIndex int32, taskDeleting bool) *Task { + taskSpec := f.GetTaskSpec(taskRoleName) + taskRoleStatus := f.TaskRoleStatus(taskRoleName) + taskStatus := f.TaskStatus(taskRoleName, taskIndex) + taskIndexStr := fmt.Sprint(taskIndex) + frameworkUIDStr := string(f.UID) + frameworkAttemptIDStr := fmt.Sprint(f.FrameworkAttemptID()) + frameworkAttemptInstanceUIDStr := string(*f.FrameworkAttemptInstanceUID()) + configMapUIDStr := string(*f.ConfigMapUID()) + taskRoleUIDStr := string(taskRoleStatus.InstanceUID) + + task := &Task{ + TypeMeta: meta.TypeMeta{}, + ObjectMeta: meta.ObjectMeta{}, + Spec: taskSpec, + Status: taskStatus, + } + + // Mock Task + task.SetGroupVersionKind(TaskGroupVersionKind) + task.Name = GetTaskName(f.Name, taskRoleName, taskIndex) + task.Namespace = f.Namespace + task.UID = taskStatus.InstanceUID + task.ResourceVersion = f.ResourceVersion + task.Generation = f.Generation + task.CreationTimestamp = taskStatus.StartTime + if taskDeleting { + task.DeletionTimestamp = common.PtrNow() + task.DeletionGracePeriodSeconds = common.PtrInt64(0) + } + + task.Annotations = map[string]string{} + task.Annotations[AnnotationKeyFrameworkNamespace] = f.Namespace + task.Annotations[AnnotationKeyFrameworkName] = f.Name + task.Annotations[AnnotationKeyTaskRoleName] = taskRoleName + task.Annotations[AnnotationKeyTaskIndex] = taskIndexStr + task.Annotations[AnnotationKeyConfigMapName] = f.ConfigMapName() + task.Annotations[AnnotationKeyFrameworkUID] = frameworkUIDStr + task.Annotations[AnnotationKeyFrameworkAttemptID] = frameworkAttemptIDStr + task.Annotations[AnnotationKeyFrameworkAttemptInstanceUID] = frameworkAttemptInstanceUIDStr + task.Annotations[AnnotationKeyConfigMapUID] = configMapUIDStr + task.Annotations[AnnotationKeyTaskRoleUID] = taskRoleUIDStr + + task.Labels = map[string]string{} + task.Labels[LabelKeyFrameworkName] = f.Name + task.Labels[LabelKeyTaskRoleName] = taskRoleName + task.Labels[LabelKeyTaskIndex] = taskIndexStr + + return task +} + func (f *Framework) NewFrameworkStatus() *FrameworkStatus { + now := meta.Now() return &FrameworkStatus{ - StartTime: meta.Now(), + StartTime: now, + RunTime: nil, CompletionTime: nil, State: FrameworkAttemptCreationPending, - TransitionTime: meta.Now(), + TransitionTime: now, RetryPolicyStatus: RetryPolicyStatus{ TotalRetriedCount: 0, AccountableRetriedCount: 0, @@ -596,9 +734,10 @@ func (f *Framework) NewFrameworkStatus() *FrameworkStatus { func (f *Framework) NewFrameworkAttemptStatus( frameworkAttemptID int32) FrameworkAttemptStatus { + now := meta.Now() return FrameworkAttemptStatus{ ID: frameworkAttemptID, - StartTime: meta.Now(), + StartTime: now, RunTime: nil, CompletionTime: nil, InstanceUID: nil, @@ -613,22 +752,38 @@ func (f *Framework) NewFrameworkAttemptStatus( func (f *Framework) NewTaskRoleStatuses() []*TaskRoleStatus { trss := []*TaskRoleStatus{} for _, taskRole := range f.Spec.TaskRoles { - trs := TaskRoleStatus{Name: taskRole.Name, TaskStatuses: []*TaskStatus{}} - for taskIndex := int32(0); taskIndex < taskRole.TaskNumber; taskIndex++ { - trs.TaskStatuses = append(trs.TaskStatuses, f.NewTaskStatus(taskRole.Name, taskIndex)) - } - trss = append(trss, &trs) + trss = append(trss, f.NewTaskRoleStatus(taskRole.Name, taskRole.TaskNumber)) } return trss } +func (f *Framework) NewTaskRoleStatus(taskRoleName string, taskNumber int32) *TaskRoleStatus { + return &TaskRoleStatus{ + Name: taskRoleName, + InstanceUID: uuid.NewUUID(), + PodGracefulDeletionTimeoutSec: nil, + TaskStatuses: f.NewTaskStatuses(taskRoleName, taskNumber), + } +} + +func (f *Framework) NewTaskStatuses(taskRoleName string, taskNumber int32) []*TaskStatus { + tss := []*TaskStatus{} + for taskIndex := int32(0); taskIndex < taskNumber; taskIndex++ { + tss = append(tss, f.NewTaskStatus(taskRoleName, taskIndex)) + } + return tss +} + func (f *Framework) NewTaskStatus(taskRoleName string, taskIndex int32) *TaskStatus { + now := meta.Now() return &TaskStatus{ Index: taskIndex, - StartTime: meta.Now(), + InstanceUID: uuid.NewUUID(), + StartTime: now, + RunTime: nil, CompletionTime: nil, State: TaskAttemptCreationPending, - TransitionTime: meta.Now(), + TransitionTime: now, DeletionPending: false, RetryPolicyStatus: RetryPolicyStatus{ TotalRetriedCount: 0, @@ -641,9 +796,10 @@ func (f *Framework) NewTaskStatus(taskRoleName string, taskIndex int32) *TaskSta func (f *Framework) NewTaskAttemptStatus( taskRoleName string, taskIndex int32, taskAttemptID int32) TaskAttemptStatus { + now := meta.Now() return TaskAttemptStatus{ ID: taskAttemptID, - StartTime: meta.Now(), + StartTime: now, RunTime: nil, CompletionTime: nil, InstanceUID: nil, @@ -740,6 +896,9 @@ func (f *Framework) TransitionFrameworkState(dstState FrameworkState) { now := common.PtrNow() if dstState == FrameworkAttemptRunning { f.Status.AttemptStatus.RunTime = now + if f.Status.RunTime == nil { + f.Status.RunTime = now + } } if dstState == FrameworkAttemptCompleted { f.Status.AttemptStatus.CompletionTime = now @@ -768,6 +927,9 @@ func (f *Framework) TransitionTaskState( now := common.PtrNow() if dstState == TaskAttemptRunning { taskStatus.AttemptStatus.RunTime = now + if taskStatus.RunTime == nil { + taskStatus.RunTime = now + } } if dstState == TaskAttemptCompleted { taskStatus.AttemptStatus.CompletionTime = now diff --git a/pkg/apis/frameworkcontroller/v1/types.go b/pkg/apis/frameworkcontroller/v1/types.go index 73f715da..ebabb4fa 100644 --- a/pkg/apis/frameworkcontroller/v1/types.go +++ b/pkg/apis/frameworkcontroller/v1/types.go @@ -35,22 +35,10 @@ type FrameworkList struct { Items []Framework `json:"items"` } -// +genclient -// +genclient:noStatus -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - ////////////////////////////////////////////////////////////////////////////////////////////////// -// A Framework represents an application with a set of Tasks: -// 1. Executed by Kubernetes Pod -// 2. Partitioned to different heterogeneous TaskRoles which share the same lifecycle -// 3. Ordered in the same homogeneous TaskRole by TaskIndex -// 4. With consistent identity {FrameworkName}-{TaskRoleName}-{TaskIndex} as PodName -// 5. With fine grained ExecutionType to Start/Stop the whole Framework -// 6. With fine grained RetryPolicy for each Task and the whole Framework -// 7. With fine grained FrameworkAttemptCompletionPolicy for each TaskRole -// 8. With PodGracefulDeletionTimeoutSec for each Task to tune Consistency vs Availability -// 9. With fine grained Status for each TaskAttempt/Task, each TaskRole and the whole -// FrameworkAttempt/Framework +// Framework +// A Framework represents an application with a set of Tasks. +// See README.md. // // Notes: // 1. Status field should only be modified by FrameworkController, and @@ -60,6 +48,9 @@ type FrameworkList struct { // This can help to avoid unintended modification, such as users may unintendedly modify // the status when updating the spec. ////////////////////////////////////////////////////////////////////////////////////////////////// +// +genclient +// +genclient:noStatus +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type Framework struct { meta.TypeMeta `json:",inline"` meta.ObjectMeta `json:"metadata"` @@ -68,7 +59,7 @@ type Framework struct { } ////////////////////////////////////////////////////////////////////////////////////////////////// -// Spec +// Framework.Spec ////////////////////////////////////////////////////////////////////////////////////////////////// type FrameworkSpec struct { Description string `json:"description"` @@ -178,8 +169,8 @@ const ( // create fails but succeeds on remote and then followed by an external delete. // So, an attempt identified by its attempt id may be associated with multiple // attempt instances over time, i.e. multiple instances may be run for the -// attempt over time, however, at most one instance is exposed into ApiServer -// over time. +// attempt over time, however, at most one instance is running at any point in +// time and at most one instance is exposed into ApiServer over time. // So, the actual retried attempt instances may exceed the RetryPolicySpec in // rare cases, however, the RetryPolicyStatus will never exceed the RetryPolicySpec. // 2. Resort to other spec to control other kind of RetryPolicy: @@ -230,7 +221,7 @@ type CompletionPolicySpec struct { } ////////////////////////////////////////////////////////////////////////////////////////////////// -// Status +// Framework.Status // It is used to: // 1. Aggregate the ground truth from other related objects, such as Pod.Status. // 2. Maintain the Framework owned ground truth, such as PodUID. @@ -257,6 +248,7 @@ type CompletionPolicySpec struct { ////////////////////////////////////////////////////////////////////////////////////////////////// type FrameworkStatus struct { StartTime meta.Time `json:"startTime"` + RunTime *meta.Time `json:"runTime"` CompletionTime *meta.Time `json:"completionTime"` State FrameworkState `json:"state"` TransitionTime meta.Time `json:"transitionTime"` @@ -279,6 +271,7 @@ type FrameworkAttemptStatus struct { // FrameworkAttemptInstanceUID = {FrameworkAttemptID}_{ConfigMapUID} // It is ordered by FrameworkAttemptID and can universally locate the // FrameworkAttemptInstance. + // See RetryPolicySpec. InstanceUID *types.UID `json:"instanceUID"` // A FrameworkAttemptInstance is represented by a ConfigMap object: // ConfigMapName = {FrameworkName}-attempt @@ -293,8 +286,20 @@ type FrameworkAttemptStatus struct { type TaskRoleStatus struct { // TaskRoleName + // It can only locate the TaskRole within a specific Framework, i.e. it cannot + // universally locate the TaskRole and cannot locate the TaskRoleInstance even + // within a specific Framework. Name string `json:"name"` + // Current associated TaskRoleInstanceUID: + // TaskRoleInstanceUID = {TaskRole.UID} + // It is totally generated by FrameworkController to universally locate the + // TaskRoleInstance. + // One TaskRole may be associated with multiple instances over time, such as + // due to ScaleDown and ScaleUp, however, at most one instance is associated + // with the TaskRole at any point in time. + InstanceUID types.UID `json:"instanceUID"` + // Effective and Backup PodGracefulDeletionTimeoutSec: // It is the immediate backup of corresponding field in TaskRoleSpec.TaskSpec, // in case the TaskRoleSpec is directly deleted later while the TaskRole's @@ -306,10 +311,23 @@ type TaskRoleStatus struct { } type TaskStatus struct { - // TaskIndex + // TaskIndex = The Task array index within its TaskRole. + // It can only locate the Task within a specific TaskRole, i.e. it cannot + // universally locate the Task and cannot locate the TaskInstance even within + // a specific Task. Index int32 `json:"index"` + // Current associated TaskInstance: + // TaskInstanceUID = {Task.UID} + // It is totally generated by FrameworkController to universally locate the + // TaskInstance. + // One Task may be associated with multiple instances over time, such as due to + // ScaleDown and ScaleUp, however, at most one instance is associated with the + // Task at any point in time. + InstanceUID types.UID `json:"instanceUID"` + StartTime meta.Time `json:"startTime"` + RunTime *meta.Time `json:"runTime"` CompletionTime *meta.Time `json:"completionTime"` State TaskState `json:"state"` TransitionTime meta.Time `json:"transitionTime"` @@ -338,6 +356,7 @@ type TaskAttemptStatus struct { // TaskAttemptInstanceUID = {TaskAttemptID}_{PodUID} // It is ordered by TaskAttemptID and can universally locate the // TaskAttemptInstance. + // See RetryPolicySpec. InstanceUID *types.UID `json:"instanceUID"` // A TaskAttemptInstance is represented by a Pod object: // PodName = {FrameworkName}-{TaskRoleName}-{TaskIndex} @@ -620,3 +639,24 @@ const ( // [FinalState] TaskCompleted TaskState = "Completed" ) + +////////////////////////////////////////////////////////////////////////////////////////////////// +// Task +// All information of a Task is already embedded in its Framework object. +// So, the below Task object is not a real CRD, instead it is used to: +// 1. Only expose a specific Task object instead of its whole Framework object. +// Such as LogObjectSnapshot for a specific Task when the Task will be retried. +////////////////////////////////////////////////////////////////////////////////////////////////// +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type Task struct { + meta.TypeMeta `json:",inline"` + // Enough information is provided in the Task's metadata to help locate its + // corresponding Framework with the same ResourceVersion, and locate itself + // in the Framework. + // See MockTask. + meta.ObjectMeta `json:"metadata"` + // Task.Spec can be nil if and only if its TaskRole's TaskRoleSpec is deleted + // while its TaskStatus still exist due to graceful deletion. + Spec *TaskSpec `json:"spec"` + Status *TaskStatus `json:"status"` +} diff --git a/pkg/apis/frameworkcontroller/v1/zz_generated.deepcopy.go b/pkg/apis/frameworkcontroller/v1/zz_generated.deepcopy.go index 57b875b4..0034c4ed 100644 --- a/pkg/apis/frameworkcontroller/v1/zz_generated.deepcopy.go +++ b/pkg/apis/frameworkcontroller/v1/zz_generated.deepcopy.go @@ -448,6 +448,10 @@ func (in *FrameworkSpec) DeepCopy() *FrameworkSpec { func (in *FrameworkStatus) DeepCopyInto(out *FrameworkStatus) { *out = *in in.StartTime.DeepCopyInto(&out.StartTime) + if in.RunTime != nil { + in, out := &in.RunTime, &out.RunTime + *out = (*in).DeepCopy() + } if in.CompletionTime != nil { in, out := &in.CompletionTime, &out.CompletionTime *out = (*in).DeepCopy() @@ -497,21 +501,11 @@ func (in *Int32Range) DeepCopy() *Int32Range { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LogFrameworkSnapshot) DeepCopyInto(out *LogFrameworkSnapshot) { *out = *in - if in.OnTaskRetry != nil { - in, out := &in.OnTaskRetry, &out.OnTaskRetry - *out = new(bool) - **out = **in - } if in.OnFrameworkRetry != nil { in, out := &in.OnFrameworkRetry, &out.OnFrameworkRetry *out = new(bool) **out = **in } - if in.OnFrameworkRescale != nil { - in, out := &in.OnFrameworkRescale, &out.OnFrameworkRescale - *out = new(bool) - **out = **in - } if in.OnFrameworkDeletion != nil { in, out := &in.OnFrameworkDeletion, &out.OnFrameworkDeletion *out = new(bool) @@ -534,6 +528,7 @@ func (in *LogFrameworkSnapshot) DeepCopy() *LogFrameworkSnapshot { func (in *LogObjectSnapshot) DeepCopyInto(out *LogObjectSnapshot) { *out = *in in.Framework.DeepCopyInto(&out.Framework) + in.Task.DeepCopyInto(&out.Task) in.Pod.DeepCopyInto(&out.Pod) return } @@ -569,6 +564,32 @@ func (in *LogPodSnapshot) DeepCopy() *LogPodSnapshot { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogTaskSnapshot) DeepCopyInto(out *LogTaskSnapshot) { + *out = *in + if in.OnTaskRetry != nil { + in, out := &in.OnTaskRetry, &out.OnTaskRetry + *out = new(bool) + **out = **in + } + if in.OnTaskDeletion != nil { + in, out := &in.OnTaskDeletion, &out.OnTaskDeletion + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogTaskSnapshot. +func (in *LogTaskSnapshot) DeepCopy() *LogTaskSnapshot { + if in == nil { + return nil + } + out := new(LogTaskSnapshot) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MatchedContainer) DeepCopyInto(out *MatchedContainer) { *out = *in @@ -768,6 +789,42 @@ func (in *RetryPolicyStatus) DeepCopy() *RetryPolicyStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Task) DeepCopyInto(out *Task) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Spec != nil { + in, out := &in.Spec, &out.Spec + *out = new(TaskSpec) + (*in).DeepCopyInto(*out) + } + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(TaskStatus) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Task. +func (in *Task) DeepCopy() *Task { + if in == nil { + return nil + } + out := new(Task) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Task) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TaskAttemptCompletionStatus) DeepCopyInto(out *TaskAttemptCompletionStatus) { *out = *in @@ -926,6 +983,10 @@ func (in *TaskSpec) DeepCopy() *TaskSpec { func (in *TaskStatus) DeepCopyInto(out *TaskStatus) { *out = *in in.StartTime.DeepCopyInto(&out.StartTime) + if in.RunTime != nil { + in, out := &in.RunTime, &out.RunTime + *out = (*in).DeepCopy() + } if in.CompletionTime != nil { in, out := &in.CompletionTime, &out.CompletionTime *out = (*in).DeepCopy() diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 0f9054e4..cbe82313 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -289,11 +289,8 @@ func (c *FrameworkController) updateFrameworkObj(oldObj, newObj interface{}) { func (c *FrameworkController) deleteFrameworkObj(obj interface{}) { f := internal.ToFramework(obj) - logSfx := "" - if *c.cConfig.LogObjectSnapshot.Framework.OnFrameworkDeletion { - logSfx = ci.GetFrameworkSnapshotLogTail(f) - } - c.enqueueFrameworkObj(f, "Framework Deleted "+string(f.UID)+logSfx) + c.enqueueFrameworkObj(f, "Framework Deleted "+string(f.UID)+ + c.cConfig.LogObjectSnapshot.Framework.GetLogTailOnFrameworkDeletion(f)) } func (c *FrameworkController) addConfigMapObj(obj interface{}) { @@ -341,11 +338,8 @@ func (c *FrameworkController) updatePodObj(oldObj, newObj interface{}) { func (c *FrameworkController) deletePodObj(obj interface{}) { pod := internal.ToPod(obj) - logSfx := "" - if *c.cConfig.LogObjectSnapshot.Pod.OnPodDeletion { - logSfx = ci.GetPodSnapshotLogTail(pod) - } - c.enqueuePodObj(pod, "Framework Pod Deleted "+string(pod.UID)+logSfx) + c.enqueuePodObj(pod, "Framework Pod Deleted "+string(pod.UID)+ + c.cConfig.LogObjectSnapshot.Pod.GetLogTailOnPodDeletion(pod)) } func (c *FrameworkController) getConfigMapOwner(cm *core.ConfigMap) *ci.Framework { @@ -783,14 +777,12 @@ func (c *FrameworkController) syncFrameworkScale( klog.Infof("[%v][%v]: syncFrameworkScale: ScaleUp: Goal: %v -> %v", f.Key(), taskRoleName, nil, taskCountSpec) - trs := ci.TaskRoleStatus{Name: taskRoleName, TaskStatuses: []*ci.TaskStatus{}} - for taskIndex := int32(0); taskIndex < taskCountSpec; taskIndex++ { - trs.TaskStatuses = - append(trs.TaskStatuses, f.NewTaskStatus(taskRoleName, taskIndex)) + f.Status.AttemptStatus.TaskRoleStatuses = + append(f.Status.AttemptStatus.TaskRoleStatuses, + f.NewTaskRoleStatus(taskRoleName, taskCountSpec)) + if taskCountSpec > 0 { producedNewPendingTask = true } - f.Status.AttemptStatus.TaskRoleStatuses = - append(f.Status.AttemptStatus.TaskRoleStatuses, &trs) } else { taskCountStatus := int32(len(taskRoleStatus.TaskStatuses)) if taskCountStatus < taskCountSpec { @@ -905,15 +897,18 @@ func (c *FrameworkController) compactFrameworkScale( } // Start deletion - logSfx := "" - if *c.cConfig.LogObjectSnapshot.Framework.OnFrameworkRescale { - // Ensure the FrameworkSnapshot is exposed before the deletion. - logSfx = ci.GetFrameworkSnapshotLogTail(f) + klog.Infof("[%v][%v]: compactFrameworkScale: ScaleDown: Deletion: %v -> %v", + f.Key(), taskRoleName, taskCountStatus, common.SprintPtrInt32(newTaskCountStatus)) + + for taskIndex := taskCountStatus - 1; taskIndex >= taskIndexDeleteStart; taskIndex-- { + klog.Info(fmt.Sprintf( + "[%v][%v][%v]: compactFrameworkScale: ScaleDown: Deletion", + f.Key(), taskRoleName, taskIndex) + + c.cConfig.LogObjectSnapshot.Task.GetLogTailOnTaskDeletion( + f.MockTask(taskRoleName, taskIndex, true))) + + taskRoleStatus.TaskStatuses[taskIndex] = nil } - klog.Info(fmt.Sprintf( - "[%v][%v]: compactFrameworkScale: ScaleDown: Deletion: %v -> %v", - f.Key(), taskRoleName, taskCountStatus, - common.SprintPtrInt32(newTaskCountStatus)) + logSfx) if newTaskCountStatus == nil { taskRoleLastIndex := len(*taskRoleStatuses) - 1 @@ -921,9 +916,6 @@ func (c *FrameworkController) compactFrameworkScale( (*taskRoleStatuses)[taskRoleLastIndex] = nil *taskRoleStatuses = (*taskRoleStatuses)[:taskRoleLastIndex] } else { - for taskIndex := taskCountStatus - 1; taskIndex >= *newTaskCountStatus; taskIndex-- { - taskRoleStatus.TaskStatuses[taskIndex] = nil - } taskRoleStatus.TaskStatuses = taskRoleStatus.TaskStatuses[:*newTaskCountStatus] } } @@ -943,14 +935,11 @@ func (c *FrameworkController) compactFrameworkScale( if taskStatus.DeletionPending && taskStatus.State == ci.TaskCompleted { // Replace the Completed DeletionPending Task with new instance - logSfx := "" - if *c.cConfig.LogObjectSnapshot.Framework.OnFrameworkRescale { - // Ensure the FrameworkSnapshot is exposed before the deletion. - logSfx = ci.GetFrameworkSnapshotLogTail(f) - } klog.Info(fmt.Sprintf( "[%v][%v][%v]: compactFrameworkScale: ScaleDown: Replacement", - f.Key(), taskRoleName, taskIndex) + logSfx) + f.Key(), taskRoleName, taskIndex) + + c.cConfig.LogObjectSnapshot.Task.GetLogTailOnTaskDeletion( + f.MockTask(taskRoleName, taskIndex, true))) taskRoleStatus.TaskStatuses[taskIndex] = f.NewTaskStatus(taskRoleName, taskIndex) @@ -1012,14 +1001,11 @@ func (c *FrameworkController) syncFrameworkState(f *ci.Framework) (err error) { } // deleteFramework - logSfx := "" - if *c.cConfig.LogObjectSnapshot.Framework.OnFrameworkDeletion { - // Ensure the FrameworkSnapshot is exposed before the deletion. - logSfx = ci.GetFrameworkSnapshotLogTail(f) - } klog.Info(logPfx + fmt.Sprintf("Framework will be deleted due to "+ "FrameworkCompletedRetainSec %v is expired", - common.SecToDuration(c.cConfig.FrameworkCompletedRetainSec)) + logSfx) + common.SecToDuration(c.cConfig.FrameworkCompletedRetainSec)) + + c.cConfig.LogObjectSnapshot.Framework.GetLogTailOnFrameworkDeletion(f)) + return c.deleteFramework(f, true) } @@ -1177,13 +1163,8 @@ func (c *FrameworkController) syncFrameworkState(f *ci.Framework) (err error) { } // retryFramework - logSfx := "" - if *c.cConfig.LogObjectSnapshot.Framework.OnFrameworkRetry { - // The completed FrameworkAttempt has been persisted, so it is safe to - // also expose it as one history snapshot. - logSfx = ci.GetFrameworkSnapshotLogTail(f) - } - klog.Info(logPfx + "Framework will be retried" + logSfx) + klog.Info(logPfx + "Framework will be retried" + + c.cConfig.LogObjectSnapshot.Framework.GetLogTailOnFrameworkRetry(f)) f.Status.RetryPolicyStatus.TotalRetriedCount++ if retryDecision.IsAccountable { @@ -1856,13 +1837,9 @@ func (c *FrameworkController) syncTaskState( } // retryTask - logSfx := "" - if *c.cConfig.LogObjectSnapshot.Framework.OnTaskRetry { - // The completed TaskAttempt has been persisted, so it is safe to also - // expose it as one history snapshot. - logSfx = ci.GetFrameworkSnapshotLogTail(f) - } - klog.Info(logPfx + "Task will be retried" + logSfx) + klog.Info(logPfx + "Task will be retried" + + c.cConfig.LogObjectSnapshot.Task.GetLogTailOnTaskRetry( + f.MockTask(taskRoleName, taskIndex, false))) taskStatus.RetryPolicyStatus.TotalRetriedCount++ if retryDecision.IsAccountable { @@ -2037,8 +2014,8 @@ func (c *FrameworkController) handlePodGracefulDeletion( f *ci.Framework, taskRoleName string, taskIndex int32, pod *core.Pod) error { logPfx := fmt.Sprintf("[%v][%v][%v]: handlePodGracefulDeletion: ", f.Key(), taskRoleName, taskIndex) - taskStatus := f.TaskRoleStatus(taskRoleName) - timeoutSec := taskStatus.PodGracefulDeletionTimeoutSec + taskRoleStatus := f.TaskRoleStatus(taskRoleName) + timeoutSec := taskRoleStatus.PodGracefulDeletionTimeoutSec if pod.DeletionTimestamp == nil { return nil diff --git a/vendor/github.com/google/uuid/CONTRIBUTORS b/vendor/github.com/google/uuid/CONTRIBUTORS new file mode 100644 index 00000000..b4bb97f6 --- /dev/null +++ b/vendor/github.com/google/uuid/CONTRIBUTORS @@ -0,0 +1,9 @@ +Paul Borman +bmatsuo +shawnps +theory +jboverfelt +dsymonds +cd1 +wallclockbuilder +dansouza diff --git a/vendor/github.com/google/uuid/LICENSE b/vendor/github.com/google/uuid/LICENSE new file mode 100644 index 00000000..5dc68268 --- /dev/null +++ b/vendor/github.com/google/uuid/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/google/uuid/dce.go b/vendor/github.com/google/uuid/dce.go new file mode 100644 index 00000000..fa820b9d --- /dev/null +++ b/vendor/github.com/google/uuid/dce.go @@ -0,0 +1,80 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" + "fmt" + "os" +) + +// A Domain represents a Version 2 domain +type Domain byte + +// Domain constants for DCE Security (Version 2) UUIDs. +const ( + Person = Domain(0) + Group = Domain(1) + Org = Domain(2) +) + +// NewDCESecurity returns a DCE Security (Version 2) UUID. +// +// The domain should be one of Person, Group or Org. +// On a POSIX system the id should be the users UID for the Person +// domain and the users GID for the Group. The meaning of id for +// the domain Org or on non-POSIX systems is site defined. +// +// For a given domain/id pair the same token may be returned for up to +// 7 minutes and 10 seconds. +func NewDCESecurity(domain Domain, id uint32) (UUID, error) { + uuid, err := NewUUID() + if err == nil { + uuid[6] = (uuid[6] & 0x0f) | 0x20 // Version 2 + uuid[9] = byte(domain) + binary.BigEndian.PutUint32(uuid[0:], id) + } + return uuid, err +} + +// NewDCEPerson returns a DCE Security (Version 2) UUID in the person +// domain with the id returned by os.Getuid. +// +// NewDCESecurity(Person, uint32(os.Getuid())) +func NewDCEPerson() (UUID, error) { + return NewDCESecurity(Person, uint32(os.Getuid())) +} + +// NewDCEGroup returns a DCE Security (Version 2) UUID in the group +// domain with the id returned by os.Getgid. +// +// NewDCESecurity(Group, uint32(os.Getgid())) +func NewDCEGroup() (UUID, error) { + return NewDCESecurity(Group, uint32(os.Getgid())) +} + +// Domain returns the domain for a Version 2 UUID. Domains are only defined +// for Version 2 UUIDs. +func (uuid UUID) Domain() Domain { + return Domain(uuid[9]) +} + +// ID returns the id for a Version 2 UUID. IDs are only defined for Version 2 +// UUIDs. +func (uuid UUID) ID() uint32 { + return binary.BigEndian.Uint32(uuid[0:4]) +} + +func (d Domain) String() string { + switch d { + case Person: + return "Person" + case Group: + return "Group" + case Org: + return "Org" + } + return fmt.Sprintf("Domain%d", int(d)) +} diff --git a/vendor/github.com/google/uuid/doc.go b/vendor/github.com/google/uuid/doc.go new file mode 100644 index 00000000..5b8a4b9a --- /dev/null +++ b/vendor/github.com/google/uuid/doc.go @@ -0,0 +1,12 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package uuid generates and inspects UUIDs. +// +// UUIDs are based on RFC 4122 and DCE 1.1: Authentication and Security +// Services. +// +// A UUID is a 16 byte (128 bit) array. UUIDs may be used as keys to +// maps or compared directly. +package uuid diff --git a/vendor/github.com/google/uuid/hash.go b/vendor/github.com/google/uuid/hash.go new file mode 100644 index 00000000..b1746163 --- /dev/null +++ b/vendor/github.com/google/uuid/hash.go @@ -0,0 +1,53 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "crypto/md5" + "crypto/sha1" + "hash" +) + +// Well known namespace IDs and UUIDs +var ( + NameSpaceDNS = Must(Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceURL = Must(Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceOID = Must(Parse("6ba7b812-9dad-11d1-80b4-00c04fd430c8")) + NameSpaceX500 = Must(Parse("6ba7b814-9dad-11d1-80b4-00c04fd430c8")) + Nil UUID // empty UUID, all zeros +) + +// NewHash returns a new UUID derived from the hash of space concatenated with +// data generated by h. The hash should be at least 16 byte in length. The +// first 16 bytes of the hash are used to form the UUID. The version of the +// UUID will be the lower 4 bits of version. NewHash is used to implement +// NewMD5 and NewSHA1. +func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID { + h.Reset() + h.Write(space[:]) + h.Write(data) + s := h.Sum(nil) + var uuid UUID + copy(uuid[:], s) + uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4) + uuid[8] = (uuid[8] & 0x3f) | 0x80 // RFC 4122 variant + return uuid +} + +// NewMD5 returns a new MD5 (Version 3) UUID based on the +// supplied name space and data. It is the same as calling: +// +// NewHash(md5.New(), space, data, 3) +func NewMD5(space UUID, data []byte) UUID { + return NewHash(md5.New(), space, data, 3) +} + +// NewSHA1 returns a new SHA1 (Version 5) UUID based on the +// supplied name space and data. It is the same as calling: +// +// NewHash(sha1.New(), space, data, 5) +func NewSHA1(space UUID, data []byte) UUID { + return NewHash(sha1.New(), space, data, 5) +} diff --git a/vendor/github.com/google/uuid/marshal.go b/vendor/github.com/google/uuid/marshal.go new file mode 100644 index 00000000..14bd3407 --- /dev/null +++ b/vendor/github.com/google/uuid/marshal.go @@ -0,0 +1,38 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import "fmt" + +// MarshalText implements encoding.TextMarshaler. +func (uuid UUID) MarshalText() ([]byte, error) { + var js [36]byte + encodeHex(js[:], uuid) + return js[:], nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (uuid *UUID) UnmarshalText(data []byte) error { + id, err := ParseBytes(data) + if err != nil { + return err + } + *uuid = id + return nil +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (uuid UUID) MarshalBinary() ([]byte, error) { + return uuid[:], nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (uuid *UUID) UnmarshalBinary(data []byte) error { + if len(data) != 16 { + return fmt.Errorf("invalid UUID (got %d bytes)", len(data)) + } + copy(uuid[:], data) + return nil +} diff --git a/vendor/github.com/google/uuid/node.go b/vendor/github.com/google/uuid/node.go new file mode 100644 index 00000000..d651a2b0 --- /dev/null +++ b/vendor/github.com/google/uuid/node.go @@ -0,0 +1,90 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "sync" +) + +var ( + nodeMu sync.Mutex + ifname string // name of interface being used + nodeID [6]byte // hardware for version 1 UUIDs + zeroID [6]byte // nodeID with only 0's +) + +// NodeInterface returns the name of the interface from which the NodeID was +// derived. The interface "user" is returned if the NodeID was set by +// SetNodeID. +func NodeInterface() string { + defer nodeMu.Unlock() + nodeMu.Lock() + return ifname +} + +// SetNodeInterface selects the hardware address to be used for Version 1 UUIDs. +// If name is "" then the first usable interface found will be used or a random +// Node ID will be generated. If a named interface cannot be found then false +// is returned. +// +// SetNodeInterface never fails when name is "". +func SetNodeInterface(name string) bool { + defer nodeMu.Unlock() + nodeMu.Lock() + return setNodeInterface(name) +} + +func setNodeInterface(name string) bool { + iname, addr := getHardwareInterface(name) // null implementation for js + if iname != "" && addr != nil { + ifname = iname + copy(nodeID[:], addr) + return true + } + + // We found no interfaces with a valid hardware address. If name + // does not specify a specific interface generate a random Node ID + // (section 4.1.6) + if name == "" { + ifname = "random" + randomBits(nodeID[:]) + return true + } + return false +} + +// NodeID returns a slice of a copy of the current Node ID, setting the Node ID +// if not already set. +func NodeID() []byte { + defer nodeMu.Unlock() + nodeMu.Lock() + if nodeID == zeroID { + setNodeInterface("") + } + nid := nodeID + return nid[:] +} + +// SetNodeID sets the Node ID to be used for Version 1 UUIDs. The first 6 bytes +// of id are used. If id is less than 6 bytes then false is returned and the +// Node ID is not set. +func SetNodeID(id []byte) bool { + if len(id) < 6 { + return false + } + defer nodeMu.Unlock() + nodeMu.Lock() + copy(nodeID[:], id) + ifname = "user" + return true +} + +// NodeID returns the 6 byte node id encoded in uuid. It returns nil if uuid is +// not valid. The NodeID is only well defined for version 1 and 2 UUIDs. +func (uuid UUID) NodeID() []byte { + var node [6]byte + copy(node[:], uuid[10:]) + return node[:] +} diff --git a/vendor/github.com/google/uuid/node_js.go b/vendor/github.com/google/uuid/node_js.go new file mode 100644 index 00000000..24b78edc --- /dev/null +++ b/vendor/github.com/google/uuid/node_js.go @@ -0,0 +1,12 @@ +// Copyright 2017 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js + +package uuid + +// getHardwareInterface returns nil values for the JS version of the code. +// This remvoves the "net" dependency, because it is not used in the browser. +// Using the "net" library inflates the size of the transpiled JS code by 673k bytes. +func getHardwareInterface(name string) (string, []byte) { return "", nil } diff --git a/vendor/github.com/google/uuid/node_net.go b/vendor/github.com/google/uuid/node_net.go new file mode 100644 index 00000000..0cbbcddb --- /dev/null +++ b/vendor/github.com/google/uuid/node_net.go @@ -0,0 +1,33 @@ +// Copyright 2017 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !js + +package uuid + +import "net" + +var interfaces []net.Interface // cached list of interfaces + +// getHardwareInterface returns the name and hardware address of interface name. +// If name is "" then the name and hardware address of one of the system's +// interfaces is returned. If no interfaces are found (name does not exist or +// there are no interfaces) then "", nil is returned. +// +// Only addresses of at least 6 bytes are returned. +func getHardwareInterface(name string) (string, []byte) { + if interfaces == nil { + var err error + interfaces, err = net.Interfaces() + if err != nil { + return "", nil + } + } + for _, ifs := range interfaces { + if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) { + return ifs.Name, ifs.HardwareAddr + } + } + return "", nil +} diff --git a/vendor/github.com/google/uuid/sql.go b/vendor/github.com/google/uuid/sql.go new file mode 100644 index 00000000..f326b54d --- /dev/null +++ b/vendor/github.com/google/uuid/sql.go @@ -0,0 +1,59 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "database/sql/driver" + "fmt" +) + +// Scan implements sql.Scanner so UUIDs can be read from databases transparently +// Currently, database types that map to string and []byte are supported. Please +// consult database-specific driver documentation for matching types. +func (uuid *UUID) Scan(src interface{}) error { + switch src := src.(type) { + case nil: + return nil + + case string: + // if an empty UUID comes from a table, we return a null UUID + if src == "" { + return nil + } + + // see Parse for required string format + u, err := Parse(src) + if err != nil { + return fmt.Errorf("Scan: %v", err) + } + + *uuid = u + + case []byte: + // if an empty UUID comes from a table, we return a null UUID + if len(src) == 0 { + return nil + } + + // assumes a simple slice of bytes if 16 bytes + // otherwise attempts to parse + if len(src) != 16 { + return uuid.Scan(string(src)) + } + copy((*uuid)[:], src) + + default: + return fmt.Errorf("Scan: unable to scan type %T into UUID", src) + } + + return nil +} + +// Value implements sql.Valuer so that UUIDs can be written to databases +// transparently. Currently, UUIDs map to strings. Please consult +// database-specific driver documentation for matching types. +func (uuid UUID) Value() (driver.Value, error) { + return uuid.String(), nil +} diff --git a/vendor/github.com/google/uuid/time.go b/vendor/github.com/google/uuid/time.go new file mode 100644 index 00000000..e6ef06cd --- /dev/null +++ b/vendor/github.com/google/uuid/time.go @@ -0,0 +1,123 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" + "sync" + "time" +) + +// A Time represents a time as the number of 100's of nanoseconds since 15 Oct +// 1582. +type Time int64 + +const ( + lillian = 2299160 // Julian day of 15 Oct 1582 + unix = 2440587 // Julian day of 1 Jan 1970 + epoch = unix - lillian // Days between epochs + g1582 = epoch * 86400 // seconds between epochs + g1582ns100 = g1582 * 10000000 // 100s of a nanoseconds between epochs +) + +var ( + timeMu sync.Mutex + lasttime uint64 // last time we returned + clockSeq uint16 // clock sequence for this run + + timeNow = time.Now // for testing +) + +// UnixTime converts t the number of seconds and nanoseconds using the Unix +// epoch of 1 Jan 1970. +func (t Time) UnixTime() (sec, nsec int64) { + sec = int64(t - g1582ns100) + nsec = (sec % 10000000) * 100 + sec /= 10000000 + return sec, nsec +} + +// GetTime returns the current Time (100s of nanoseconds since 15 Oct 1582) and +// clock sequence as well as adjusting the clock sequence as needed. An error +// is returned if the current time cannot be determined. +func GetTime() (Time, uint16, error) { + defer timeMu.Unlock() + timeMu.Lock() + return getTime() +} + +func getTime() (Time, uint16, error) { + t := timeNow() + + // If we don't have a clock sequence already, set one. + if clockSeq == 0 { + setClockSequence(-1) + } + now := uint64(t.UnixNano()/100) + g1582ns100 + + // If time has gone backwards with this clock sequence then we + // increment the clock sequence + if now <= lasttime { + clockSeq = ((clockSeq + 1) & 0x3fff) | 0x8000 + } + lasttime = now + return Time(now), clockSeq, nil +} + +// ClockSequence returns the current clock sequence, generating one if not +// already set. The clock sequence is only used for Version 1 UUIDs. +// +// The uuid package does not use global static storage for the clock sequence or +// the last time a UUID was generated. Unless SetClockSequence is used, a new +// random clock sequence is generated the first time a clock sequence is +// requested by ClockSequence, GetTime, or NewUUID. (section 4.2.1.1) +func ClockSequence() int { + defer timeMu.Unlock() + timeMu.Lock() + return clockSequence() +} + +func clockSequence() int { + if clockSeq == 0 { + setClockSequence(-1) + } + return int(clockSeq & 0x3fff) +} + +// SetClockSequence sets the clock sequence to the lower 14 bits of seq. Setting to +// -1 causes a new sequence to be generated. +func SetClockSequence(seq int) { + defer timeMu.Unlock() + timeMu.Lock() + setClockSequence(seq) +} + +func setClockSequence(seq int) { + if seq == -1 { + var b [2]byte + randomBits(b[:]) // clock sequence + seq = int(b[0])<<8 | int(b[1]) + } + oldSeq := clockSeq + clockSeq = uint16(seq&0x3fff) | 0x8000 // Set our variant + if oldSeq != clockSeq { + lasttime = 0 + } +} + +// Time returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in +// uuid. The time is only defined for version 1 and 2 UUIDs. +func (uuid UUID) Time() Time { + time := int64(binary.BigEndian.Uint32(uuid[0:4])) + time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32 + time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48 + return Time(time) +} + +// ClockSequence returns the clock sequence encoded in uuid. +// The clock sequence is only well defined for version 1 and 2 UUIDs. +func (uuid UUID) ClockSequence() int { + return int(binary.BigEndian.Uint16(uuid[8:10])) & 0x3fff +} diff --git a/vendor/github.com/google/uuid/util.go b/vendor/github.com/google/uuid/util.go new file mode 100644 index 00000000..5ea6c737 --- /dev/null +++ b/vendor/github.com/google/uuid/util.go @@ -0,0 +1,43 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "io" +) + +// randomBits completely fills slice b with random data. +func randomBits(b []byte) { + if _, err := io.ReadFull(rander, b); err != nil { + panic(err.Error()) // rand should never fail + } +} + +// xvalues returns the value of a byte as a hexadecimal digit or 255. +var xvalues = [256]byte{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +} + +// xtob converts hex characters x1 and x2 into a byte. +func xtob(x1, x2 byte) (byte, bool) { + b1 := xvalues[x1] + b2 := xvalues[x2] + return (b1 << 4) | b2, b1 != 255 && b2 != 255 +} diff --git a/vendor/github.com/google/uuid/uuid.go b/vendor/github.com/google/uuid/uuid.go new file mode 100644 index 00000000..524404cc --- /dev/null +++ b/vendor/github.com/google/uuid/uuid.go @@ -0,0 +1,245 @@ +// Copyright 2018 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "strings" +) + +// A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC +// 4122. +type UUID [16]byte + +// A Version represents a UUID's version. +type Version byte + +// A Variant represents a UUID's variant. +type Variant byte + +// Constants returned by Variant. +const ( + Invalid = Variant(iota) // Invalid UUID + RFC4122 // The variant specified in RFC4122 + Reserved // Reserved, NCS backward compatibility. + Microsoft // Reserved, Microsoft Corporation backward compatibility. + Future // Reserved for future definition. +) + +var rander = rand.Reader // random function + +// Parse decodes s into a UUID or returns an error. Both the standard UUID +// forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the +// Microsoft encoding {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} and the raw hex +// encoding: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx. +func Parse(s string) (UUID, error) { + var uuid UUID + switch len(s) { + // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36: + + // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36 + 9: + if strings.ToLower(s[:9]) != "urn:uuid:" { + return uuid, fmt.Errorf("invalid urn prefix: %q", s[:9]) + } + s = s[9:] + + // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} + case 36 + 2: + s = s[1:] + + // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + case 32: + var ok bool + for i := range uuid { + uuid[i], ok = xtob(s[i*2], s[i*2+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + } + return uuid, nil + default: + return uuid, fmt.Errorf("invalid UUID length: %d", len(s)) + } + // s is now at least 36 bytes long + // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + return uuid, errors.New("invalid UUID format") + } + for i, x := range [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34} { + v, ok := xtob(s[x], s[x+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + uuid[i] = v + } + return uuid, nil +} + +// ParseBytes is like Parse, except it parses a byte slice instead of a string. +func ParseBytes(b []byte) (UUID, error) { + var uuid UUID + switch len(b) { + case 36: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36 + 9: // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if !bytes.Equal(bytes.ToLower(b[:9]), []byte("urn:uuid:")) { + return uuid, fmt.Errorf("invalid urn prefix: %q", b[:9]) + } + b = b[9:] + case 36 + 2: // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} + b = b[1:] + case 32: // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + var ok bool + for i := 0; i < 32; i += 2 { + uuid[i/2], ok = xtob(b[i], b[i+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + } + return uuid, nil + default: + return uuid, fmt.Errorf("invalid UUID length: %d", len(b)) + } + // s is now at least 36 bytes long + // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + if b[8] != '-' || b[13] != '-' || b[18] != '-' || b[23] != '-' { + return uuid, errors.New("invalid UUID format") + } + for i, x := range [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34} { + v, ok := xtob(b[x], b[x+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + uuid[i] = v + } + return uuid, nil +} + +// MustParse is like Parse but panics if the string cannot be parsed. +// It simplifies safe initialization of global variables holding compiled UUIDs. +func MustParse(s string) UUID { + uuid, err := Parse(s) + if err != nil { + panic(`uuid: Parse(` + s + `): ` + err.Error()) + } + return uuid +} + +// FromBytes creates a new UUID from a byte slice. Returns an error if the slice +// does not have a length of 16. The bytes are copied from the slice. +func FromBytes(b []byte) (uuid UUID, err error) { + err = uuid.UnmarshalBinary(b) + return uuid, err +} + +// Must returns uuid if err is nil and panics otherwise. +func Must(uuid UUID, err error) UUID { + if err != nil { + panic(err) + } + return uuid +} + +// String returns the string form of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +// , or "" if uuid is invalid. +func (uuid UUID) String() string { + var buf [36]byte + encodeHex(buf[:], uuid) + return string(buf[:]) +} + +// URN returns the RFC 2141 URN form of uuid, +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, or "" if uuid is invalid. +func (uuid UUID) URN() string { + var buf [36 + 9]byte + copy(buf[:], "urn:uuid:") + encodeHex(buf[9:], uuid) + return string(buf[:]) +} + +func encodeHex(dst []byte, uuid UUID) { + hex.Encode(dst, uuid[:4]) + dst[8] = '-' + hex.Encode(dst[9:13], uuid[4:6]) + dst[13] = '-' + hex.Encode(dst[14:18], uuid[6:8]) + dst[18] = '-' + hex.Encode(dst[19:23], uuid[8:10]) + dst[23] = '-' + hex.Encode(dst[24:], uuid[10:]) +} + +// Variant returns the variant encoded in uuid. +func (uuid UUID) Variant() Variant { + switch { + case (uuid[8] & 0xc0) == 0x80: + return RFC4122 + case (uuid[8] & 0xe0) == 0xc0: + return Microsoft + case (uuid[8] & 0xe0) == 0xe0: + return Future + default: + return Reserved + } +} + +// Version returns the version of uuid. +func (uuid UUID) Version() Version { + return Version(uuid[6] >> 4) +} + +func (v Version) String() string { + if v > 15 { + return fmt.Sprintf("BAD_VERSION_%d", v) + } + return fmt.Sprintf("VERSION_%d", v) +} + +func (v Variant) String() string { + switch v { + case RFC4122: + return "RFC4122" + case Reserved: + return "Reserved" + case Microsoft: + return "Microsoft" + case Future: + return "Future" + case Invalid: + return "Invalid" + } + return fmt.Sprintf("BadVariant%d", int(v)) +} + +// SetRand sets the random number generator to r, which implements io.Reader. +// If r.Read returns an error when the package requests random data then +// a panic will be issued. +// +// Calling SetRand with nil sets the random number generator to the default +// generator. +func SetRand(r io.Reader) { + if r == nil { + rander = rand.Reader + return + } + rander = r +} diff --git a/vendor/github.com/google/uuid/version1.go b/vendor/github.com/google/uuid/version1.go new file mode 100644 index 00000000..46310962 --- /dev/null +++ b/vendor/github.com/google/uuid/version1.go @@ -0,0 +1,44 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" +) + +// NewUUID returns a Version 1 UUID based on the current NodeID and clock +// sequence, and the current time. If the NodeID has not been set by SetNodeID +// or SetNodeInterface then it will be set automatically. If the NodeID cannot +// be set NewUUID returns nil. If clock sequence has not been set by +// SetClockSequence then it will be set automatically. If GetTime fails to +// return the current NewUUID returns nil and an error. +// +// In most cases, New should be used. +func NewUUID() (UUID, error) { + var uuid UUID + now, seq, err := GetTime() + if err != nil { + return uuid, err + } + + timeLow := uint32(now & 0xffffffff) + timeMid := uint16((now >> 32) & 0xffff) + timeHi := uint16((now >> 48) & 0x0fff) + timeHi |= 0x1000 // Version 1 + + binary.BigEndian.PutUint32(uuid[0:], timeLow) + binary.BigEndian.PutUint16(uuid[4:], timeMid) + binary.BigEndian.PutUint16(uuid[6:], timeHi) + binary.BigEndian.PutUint16(uuid[8:], seq) + + nodeMu.Lock() + if nodeID == zeroID { + setNodeInterface("") + } + copy(uuid[10:], nodeID[:]) + nodeMu.Unlock() + + return uuid, nil +} diff --git a/vendor/github.com/google/uuid/version4.go b/vendor/github.com/google/uuid/version4.go new file mode 100644 index 00000000..c110465d --- /dev/null +++ b/vendor/github.com/google/uuid/version4.go @@ -0,0 +1,43 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import "io" + +// New creates a new random UUID or panics. New is equivalent to +// the expression +// +// uuid.Must(uuid.NewRandom()) +func New() UUID { + return Must(NewRandom()) +} + +// NewRandom returns a Random (Version 4) UUID. +// +// The strength of the UUIDs is based on the strength of the crypto/rand +// package. +// +// A note about uniqueness derived from the UUID Wikipedia entry: +// +// Randomly generated UUIDs have 122 random bits. One's annual risk of being +// hit by a meteorite is estimated to be one chance in 17 billion, that +// means the probability is about 0.00000000006 (6 × 10−11), +// equivalent to the odds of creating a few tens of trillions of UUIDs in a +// year and having one duplicate. +func NewRandom() (UUID, error) { + return NewRandomFromReader(rander) +} + +// NewRandomFromReader returns a UUID based on bytes read from a given io.Reader. +func NewRandomFromReader(r io.Reader) (UUID, error) { + var uuid UUID + _, err := io.ReadFull(r, uuid[:]) + if err != nil { + return Nil, err + } + uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4 + uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 + return uuid, nil +} diff --git a/vendor/github.com/pborman/uuid/CONTRIBUTORS b/vendor/github.com/pborman/uuid/CONTRIBUTORS new file mode 100644 index 00000000..b382a04e --- /dev/null +++ b/vendor/github.com/pborman/uuid/CONTRIBUTORS @@ -0,0 +1 @@ +Paul Borman diff --git a/vendor/github.com/pborman/uuid/LICENSE b/vendor/github.com/pborman/uuid/LICENSE new file mode 100644 index 00000000..5dc68268 --- /dev/null +++ b/vendor/github.com/pborman/uuid/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/pborman/uuid/dce.go b/vendor/github.com/pborman/uuid/dce.go new file mode 100644 index 00000000..50a0f2d0 --- /dev/null +++ b/vendor/github.com/pborman/uuid/dce.go @@ -0,0 +1,84 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" + "fmt" + "os" +) + +// A Domain represents a Version 2 domain +type Domain byte + +// Domain constants for DCE Security (Version 2) UUIDs. +const ( + Person = Domain(0) + Group = Domain(1) + Org = Domain(2) +) + +// NewDCESecurity returns a DCE Security (Version 2) UUID. +// +// The domain should be one of Person, Group or Org. +// On a POSIX system the id should be the users UID for the Person +// domain and the users GID for the Group. The meaning of id for +// the domain Org or on non-POSIX systems is site defined. +// +// For a given domain/id pair the same token may be returned for up to +// 7 minutes and 10 seconds. +func NewDCESecurity(domain Domain, id uint32) UUID { + uuid := NewUUID() + if uuid != nil { + uuid[6] = (uuid[6] & 0x0f) | 0x20 // Version 2 + uuid[9] = byte(domain) + binary.BigEndian.PutUint32(uuid[0:], id) + } + return uuid +} + +// NewDCEPerson returns a DCE Security (Version 2) UUID in the person +// domain with the id returned by os.Getuid. +// +// NewDCEPerson(Person, uint32(os.Getuid())) +func NewDCEPerson() UUID { + return NewDCESecurity(Person, uint32(os.Getuid())) +} + +// NewDCEGroup returns a DCE Security (Version 2) UUID in the group +// domain with the id returned by os.Getgid. +// +// NewDCEGroup(Group, uint32(os.Getgid())) +func NewDCEGroup() UUID { + return NewDCESecurity(Group, uint32(os.Getgid())) +} + +// Domain returns the domain for a Version 2 UUID or false. +func (uuid UUID) Domain() (Domain, bool) { + if v, _ := uuid.Version(); v != 2 { + return 0, false + } + return Domain(uuid[9]), true +} + +// Id returns the id for a Version 2 UUID or false. +func (uuid UUID) Id() (uint32, bool) { + if v, _ := uuid.Version(); v != 2 { + return 0, false + } + return binary.BigEndian.Uint32(uuid[0:4]), true +} + +func (d Domain) String() string { + switch d { + case Person: + return "Person" + case Group: + return "Group" + case Org: + return "Org" + } + return fmt.Sprintf("Domain%d", int(d)) +} diff --git a/vendor/github.com/pborman/uuid/doc.go b/vendor/github.com/pborman/uuid/doc.go new file mode 100644 index 00000000..727d7616 --- /dev/null +++ b/vendor/github.com/pborman/uuid/doc.go @@ -0,0 +1,13 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The uuid package generates and inspects UUIDs. +// +// UUIDs are based on RFC 4122 and DCE 1.1: Authentication and Security +// Services. +// +// This package is a partial wrapper around the github.com/google/uuid package. +// This package represents a UUID as []byte while github.com/google/uuid +// represents a UUID as [16]byte. +package uuid diff --git a/vendor/github.com/pborman/uuid/hash.go b/vendor/github.com/pborman/uuid/hash.go new file mode 100644 index 00000000..a0420c1e --- /dev/null +++ b/vendor/github.com/pborman/uuid/hash.go @@ -0,0 +1,53 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "crypto/md5" + "crypto/sha1" + "hash" +) + +// Well known Name Space IDs and UUIDs +var ( + NameSpace_DNS = Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + NameSpace_URL = Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8") + NameSpace_OID = Parse("6ba7b812-9dad-11d1-80b4-00c04fd430c8") + NameSpace_X500 = Parse("6ba7b814-9dad-11d1-80b4-00c04fd430c8") + NIL = Parse("00000000-0000-0000-0000-000000000000") +) + +// NewHash returns a new UUID derived from the hash of space concatenated with +// data generated by h. The hash should be at least 16 byte in length. The +// first 16 bytes of the hash are used to form the UUID. The version of the +// UUID will be the lower 4 bits of version. NewHash is used to implement +// NewMD5 and NewSHA1. +func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID { + h.Reset() + h.Write(space) + h.Write([]byte(data)) + s := h.Sum(nil) + uuid := make([]byte, 16) + copy(uuid, s) + uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4) + uuid[8] = (uuid[8] & 0x3f) | 0x80 // RFC 4122 variant + return uuid +} + +// NewMD5 returns a new MD5 (Version 3) UUID based on the +// supplied name space and data. +// +// NewHash(md5.New(), space, data, 3) +func NewMD5(space UUID, data []byte) UUID { + return NewHash(md5.New(), space, data, 3) +} + +// NewSHA1 returns a new SHA1 (Version 5) UUID based on the +// supplied name space and data. +// +// NewHash(sha1.New(), space, data, 5) +func NewSHA1(space UUID, data []byte) UUID { + return NewHash(sha1.New(), space, data, 5) +} diff --git a/vendor/github.com/pborman/uuid/marshal.go b/vendor/github.com/pborman/uuid/marshal.go new file mode 100644 index 00000000..35b89352 --- /dev/null +++ b/vendor/github.com/pborman/uuid/marshal.go @@ -0,0 +1,85 @@ +// Copyright 2016 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "errors" + "fmt" + + guuid "github.com/google/uuid" +) + +// MarshalText implements encoding.TextMarshaler. +func (u UUID) MarshalText() ([]byte, error) { + if len(u) != 16 { + return nil, nil + } + var js [36]byte + encodeHex(js[:], u) + return js[:], nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (u *UUID) UnmarshalText(data []byte) error { + if len(data) == 0 { + return nil + } + id := Parse(string(data)) + if id == nil { + return errors.New("invalid UUID") + } + *u = id + return nil +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (u UUID) MarshalBinary() ([]byte, error) { + return u[:], nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (u *UUID) UnmarshalBinary(data []byte) error { + if len(data) == 0 { + return nil + } + if len(data) != 16 { + return fmt.Errorf("invalid UUID (got %d bytes)", len(data)) + } + var id [16]byte + copy(id[:], data) + *u = id[:] + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (u Array) MarshalText() ([]byte, error) { + var js [36]byte + encodeHex(js[:], u[:]) + return js[:], nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (u *Array) UnmarshalText(data []byte) error { + id, err := guuid.ParseBytes(data) + if err != nil { + return err + } + *u = Array(id) + return nil +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (u Array) MarshalBinary() ([]byte, error) { + return u[:], nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (u *Array) UnmarshalBinary(data []byte) error { + if len(data) != 16 { + return fmt.Errorf("invalid UUID (got %d bytes)", len(data)) + } + copy(u[:], data) + return nil +} diff --git a/vendor/github.com/pborman/uuid/node.go b/vendor/github.com/pborman/uuid/node.go new file mode 100644 index 00000000..e524e010 --- /dev/null +++ b/vendor/github.com/pborman/uuid/node.go @@ -0,0 +1,50 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + guuid "github.com/google/uuid" +) + +// NodeInterface returns the name of the interface from which the NodeID was +// derived. The interface "user" is returned if the NodeID was set by +// SetNodeID. +func NodeInterface() string { + return guuid.NodeInterface() +} + +// SetNodeInterface selects the hardware address to be used for Version 1 UUIDs. +// If name is "" then the first usable interface found will be used or a random +// Node ID will be generated. If a named interface cannot be found then false +// is returned. +// +// SetNodeInterface never fails when name is "". +func SetNodeInterface(name string) bool { + return guuid.SetNodeInterface(name) +} + +// NodeID returns a slice of a copy of the current Node ID, setting the Node ID +// if not already set. +func NodeID() []byte { + return guuid.NodeID() +} + +// SetNodeID sets the Node ID to be used for Version 1 UUIDs. The first 6 bytes +// of id are used. If id is less than 6 bytes then false is returned and the +// Node ID is not set. +func SetNodeID(id []byte) bool { + return guuid.SetNodeID(id) +} + +// NodeID returns the 6 byte node id encoded in uuid. It returns nil if uuid is +// not valid. The NodeID is only well defined for version 1 and 2 UUIDs. +func (uuid UUID) NodeID() []byte { + if len(uuid) != 16 { + return nil + } + node := make([]byte, 6) + copy(node, uuid[10:]) + return node +} diff --git a/vendor/github.com/pborman/uuid/sql.go b/vendor/github.com/pborman/uuid/sql.go new file mode 100644 index 00000000..929c3847 --- /dev/null +++ b/vendor/github.com/pborman/uuid/sql.go @@ -0,0 +1,68 @@ +// Copyright 2015 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "database/sql/driver" + "errors" + "fmt" +) + +// Scan implements sql.Scanner so UUIDs can be read from databases transparently +// Currently, database types that map to string and []byte are supported. Please +// consult database-specific driver documentation for matching types. +func (uuid *UUID) Scan(src interface{}) error { + switch src.(type) { + case string: + // if an empty UUID comes from a table, we return a null UUID + if src.(string) == "" { + return nil + } + + // see uuid.Parse for required string format + parsed := Parse(src.(string)) + + if parsed == nil { + return errors.New("Scan: invalid UUID format") + } + + *uuid = parsed + case []byte: + b := src.([]byte) + + // if an empty UUID comes from a table, we return a null UUID + if len(b) == 0 { + return nil + } + + // assumes a simple slice of bytes if 16 bytes + // otherwise attempts to parse + if len(b) == 16 { + parsed := make([]byte, 16) + copy(parsed, b) + *uuid = UUID(parsed) + } else { + u := Parse(string(b)) + + if u == nil { + return errors.New("Scan: invalid UUID format") + } + + *uuid = u + } + + default: + return fmt.Errorf("Scan: unable to scan type %T into UUID", src) + } + + return nil +} + +// Value implements sql.Valuer so that UUIDs can be written to databases +// transparently. Currently, UUIDs map to strings. Please consult +// database-specific driver documentation for matching types. +func (uuid UUID) Value() (driver.Value, error) { + return uuid.String(), nil +} diff --git a/vendor/github.com/pborman/uuid/time.go b/vendor/github.com/pborman/uuid/time.go new file mode 100644 index 00000000..7286824d --- /dev/null +++ b/vendor/github.com/pborman/uuid/time.go @@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "encoding/binary" + + guuid "github.com/google/uuid" +) + +// A Time represents a time as the number of 100's of nanoseconds since 15 Oct +// 1582. +type Time = guuid.Time + +// GetTime returns the current Time (100s of nanoseconds since 15 Oct 1582) and +// clock sequence as well as adjusting the clock sequence as needed. An error +// is returned if the current time cannot be determined. +func GetTime() (Time, uint16, error) { return guuid.GetTime() } + +// ClockSequence returns the current clock sequence, generating one if not +// already set. The clock sequence is only used for Version 1 UUIDs. +// +// The uuid package does not use global static storage for the clock sequence or +// the last time a UUID was generated. Unless SetClockSequence a new random +// clock sequence is generated the first time a clock sequence is requested by +// ClockSequence, GetTime, or NewUUID. (section 4.2.1.1) sequence is generated +// for +func ClockSequence() int { return guuid.ClockSequence() } + +// SetClockSequence sets the clock sequence to the lower 14 bits of seq. Setting to +// -1 causes a new sequence to be generated. +func SetClockSequence(seq int) { guuid.SetClockSequence(seq) } + +// Time returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in +// uuid. It returns false if uuid is not valid. The time is only well defined +// for version 1 and 2 UUIDs. +func (uuid UUID) Time() (Time, bool) { + if len(uuid) != 16 { + return 0, false + } + time := int64(binary.BigEndian.Uint32(uuid[0:4])) + time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32 + time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48 + return Time(time), true +} + +// ClockSequence returns the clock sequence encoded in uuid. It returns false +// if uuid is not valid. The clock sequence is only well defined for version 1 +// and 2 UUIDs. +func (uuid UUID) ClockSequence() (int, bool) { + if len(uuid) != 16 { + return 0, false + } + return int(binary.BigEndian.Uint16(uuid[8:10])) & 0x3fff, true +} diff --git a/vendor/github.com/pborman/uuid/util.go b/vendor/github.com/pborman/uuid/util.go new file mode 100644 index 00000000..255b5e24 --- /dev/null +++ b/vendor/github.com/pborman/uuid/util.go @@ -0,0 +1,32 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +// xvalues returns the value of a byte as a hexadecimal digit or 255. +var xvalues = [256]byte{ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, +} + +// xtob converts the the first two hex bytes of x into a byte. +func xtob(x string) (byte, bool) { + b1 := xvalues[x[0]] + b2 := xvalues[x[1]] + return (b1 << 4) | b2, b1 != 255 && b2 != 255 +} diff --git a/vendor/github.com/pborman/uuid/uuid.go b/vendor/github.com/pborman/uuid/uuid.go new file mode 100644 index 00000000..33700042 --- /dev/null +++ b/vendor/github.com/pborman/uuid/uuid.go @@ -0,0 +1,162 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "io" + + guuid "github.com/google/uuid" +) + +// Array is a pass-by-value UUID that can be used as an effecient key in a map. +type Array [16]byte + +// UUID converts uuid into a slice. +func (uuid Array) UUID() UUID { + return uuid[:] +} + +// String returns the string representation of uuid, +// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx. +func (uuid Array) String() string { + return guuid.UUID(uuid).String() +} + +// A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC +// 4122. +type UUID []byte + +// A Version represents a UUIDs version. +type Version = guuid.Version + +// A Variant represents a UUIDs variant. +type Variant = guuid.Variant + +// Constants returned by Variant. +const ( + Invalid = guuid.Invalid // Invalid UUID + RFC4122 = guuid.RFC4122 // The variant specified in RFC4122 + Reserved = guuid.Reserved // Reserved, NCS backward compatibility. + Microsoft = guuid.Microsoft // Reserved, Microsoft Corporation backward compatibility. + Future = guuid.Future // Reserved for future definition. +) + +var rander = rand.Reader // random function + +// New returns a new random (version 4) UUID as a string. It is a convenience +// function for NewRandom().String(). +func New() string { + return NewRandom().String() +} + +// Parse decodes s into a UUID or returns nil. See github.com/google/uuid for +// the formats parsed. +func Parse(s string) UUID { + gu, err := guuid.Parse(s) + if err == nil { + return gu[:] + } + return nil +} + +// ParseBytes is like Parse, except it parses a byte slice instead of a string. +func ParseBytes(b []byte) (UUID, error) { + gu, err := guuid.ParseBytes(b) + if err == nil { + return gu[:], nil + } + return nil, err +} + +// Equal returns true if uuid1 and uuid2 are equal. +func Equal(uuid1, uuid2 UUID) bool { + return bytes.Equal(uuid1, uuid2) +} + +// Array returns an array representation of uuid that can be used as a map key. +// Array panics if uuid is not valid. +func (uuid UUID) Array() Array { + if len(uuid) != 16 { + panic("invalid uuid") + } + var a Array + copy(a[:], uuid) + return a +} + +// String returns the string form of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +// , or "" if uuid is invalid. +func (uuid UUID) String() string { + if len(uuid) != 16 { + return "" + } + var buf [36]byte + encodeHex(buf[:], uuid) + return string(buf[:]) +} + +// URN returns the RFC 2141 URN form of uuid, +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, or "" if uuid is invalid. +func (uuid UUID) URN() string { + if len(uuid) != 16 { + return "" + } + var buf [36 + 9]byte + copy(buf[:], "urn:uuid:") + encodeHex(buf[9:], uuid) + return string(buf[:]) +} + +func encodeHex(dst []byte, uuid UUID) { + hex.Encode(dst[:], uuid[:4]) + dst[8] = '-' + hex.Encode(dst[9:13], uuid[4:6]) + dst[13] = '-' + hex.Encode(dst[14:18], uuid[6:8]) + dst[18] = '-' + hex.Encode(dst[19:23], uuid[8:10]) + dst[23] = '-' + hex.Encode(dst[24:], uuid[10:]) +} + +// Variant returns the variant encoded in uuid. It returns Invalid if +// uuid is invalid. +func (uuid UUID) Variant() Variant { + if len(uuid) != 16 { + return Invalid + } + switch { + case (uuid[8] & 0xc0) == 0x80: + return RFC4122 + case (uuid[8] & 0xe0) == 0xc0: + return Microsoft + case (uuid[8] & 0xe0) == 0xe0: + return Future + default: + return Reserved + } +} + +// Version returns the version of uuid. It returns false if uuid is not +// valid. +func (uuid UUID) Version() (Version, bool) { + if len(uuid) != 16 { + return 0, false + } + return Version(uuid[6] >> 4), true +} + +// SetRand sets the random number generator to r, which implements io.Reader. +// If r.Read returns an error when the package requests random data then +// a panic will be issued. +// +// Calling SetRand with nil sets the random number generator to the default +// generator. +func SetRand(r io.Reader) { + guuid.SetRand(r) +} diff --git a/vendor/github.com/pborman/uuid/version1.go b/vendor/github.com/pborman/uuid/version1.go new file mode 100644 index 00000000..7af948da --- /dev/null +++ b/vendor/github.com/pborman/uuid/version1.go @@ -0,0 +1,23 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + guuid "github.com/google/uuid" +) + +// NewUUID returns a Version 1 UUID based on the current NodeID and clock +// sequence, and the current time. If the NodeID has not been set by SetNodeID +// or SetNodeInterface then it will be set automatically. If the NodeID cannot +// be set NewUUID returns nil. If clock sequence has not been set by +// SetClockSequence then it will be set automatically. If GetTime fails to +// return the current NewUUID returns nil. +func NewUUID() UUID { + gu, err := guuid.NewUUID() + if err == nil { + return UUID(gu[:]) + } + return nil +} diff --git a/vendor/github.com/pborman/uuid/version4.go b/vendor/github.com/pborman/uuid/version4.go new file mode 100644 index 00000000..767dd0c3 --- /dev/null +++ b/vendor/github.com/pborman/uuid/version4.go @@ -0,0 +1,26 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import guuid "github.com/google/uuid" + +// NewRandom returns a Random (Version 4) UUID or panics. +// +// The strength of the UUIDs is based on the strength of the crypto/rand +// package. +// +// A note about uniqueness derived from the UUID Wikipedia entry: +// +// Randomly generated UUIDs have 122 random bits. One's annual risk of being +// hit by a meteorite is estimated to be one chance in 17 billion, that +// means the probability is about 0.00000000006 (6 × 10−11), +// equivalent to the odds of creating a few tens of trillions of UUIDs in a +// year and having one duplicate. +func NewRandom() UUID { + if gu, err := guuid.NewRandom(); err == nil { + return UUID(gu[:]) + } + return nil +} diff --git a/vendor/k8s.io/apimachinery/pkg/util/uuid/uuid.go b/vendor/k8s.io/apimachinery/pkg/util/uuid/uuid.go new file mode 100644 index 00000000..bf478223 --- /dev/null +++ b/vendor/k8s.io/apimachinery/pkg/util/uuid/uuid.go @@ -0,0 +1,43 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package uuid + +import ( + "sync" + + "github.com/pborman/uuid" + + "k8s.io/apimachinery/pkg/types" +) + +var uuidLock sync.Mutex +var lastUUID uuid.UUID + +func NewUUID() types.UID { + uuidLock.Lock() + defer uuidLock.Unlock() + result := uuid.NewUUID() + // The UUID package is naive and can generate identical UUIDs if the + // time interval is quick enough. + // The UUID uses 100 ns increments so it's short enough to actively + // wait for a new value. + for uuid.Equal(lastUUID, result) == true { + result = uuid.NewUUID() + } + lastUUID = result + return types.UID(result.String()) +}