Skip to content

Fix EndTime nil pointer when persisting metrics #2259

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jun 18, 2021
Merged

Fix EndTime nil pointer when persisting metrics #2259

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
92cde98
Fetch metrics from prometheus based on current time
vishalbollu Jun 17, 2021
173c7ef
Set endtime in status based annotation as a backup
vishalbollu Jun 17, 2021
9d3881f
Merge branch 'master' into set-endtime-incomplete-jobs
vishalbollu Jun 17, 2021
1448d2a
Merge branch 'master' into set-endtime-incomplete-jobs
vishalbollu Jun 18, 2021
9b57e65
Use end time when available
vishalbollu Jun 18, 2021
a3e48c8
Merge branch 'master' into set-endtime-incomplete-jobs
vishalbollu Jun 18, 2021
c453593
Merge branch 'master' into set-endtime-incomplete-jobs
Jun 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion pkg/crds/controllers/batch/batchjob_controller_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import (
kbatch "k8s.io/api/batch/v1"
kcore "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand Down Expand Up @@ -462,6 +463,18 @@ func (r *BatchJobReconciler) updateStatus(ctx context.Context, batchJob *batch.B
if worker != nil {
batchJob.Status.EndTime = worker.Status.CompletionTime // assign right away, because it's a pointer

if batchJob.Status.EndTime == nil {
completedTimestampStr, completedTimestampExists := batchJob.Annotations[_completedTimestampAnnotation]
if completedTimestampExists {
ts, err := time.Parse(time.RFC3339, completedTimestampStr)
if err != nil {
return errors.Wrap(err, "failed to parse completed timestamp string")
}
completedTime := v1.NewTime(ts)
batchJob.Status.EndTime = &completedTime
}
}

if worker.Status.Failed == batchJob.Spec.Workers {
batchJobStatus := status.JobWorkerError
for _, condition := range worker.Status.Conditions {
Expand Down Expand Up @@ -668,14 +681,20 @@ func getTotalBatchCount(r *BatchJobReconciler, batchJob batch.BatchJob) (int, er
}

func getMetrics(r *BatchJobReconciler, batchJob batch.BatchJob) (metrics.BatchMetrics, error) {
endTime := time.Now()
if batchJob.Status.EndTime != nil {
endTime = batchJob.Status.EndTime.Time
}

jobMetrics, err := batch.GetMetrics(r.Prometheus, spec.JobKey{
ID: batchJob.Name,
APIName: batchJob.Spec.APIName,
Kind: userconfig.BatchAPIKind,
}, batchJob.Status.EndTime.Time)
}, endTime)
if err != nil {
return metrics.BatchMetrics{}, err
}

return jobMetrics, nil
}

Expand Down