Skip to content

Commit 38183b3

Browse files
easyCZeasyCZ
committed
[usage] Validate workspace instance records for usage
1 parent 86b2b9e commit 38183b3

File tree

4 files changed

+169
-9
lines changed

4 files changed

+169
-9
lines changed

‎components/usage/pkg/controller/reconciler.go

Lines changed: 100 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"fmt"
1010
"github.com/gitpod-io/gitpod/common-go/log"
1111
"github.com/gitpod-io/gitpod/usage/pkg/db"
12+
"github.com/google/uuid"
1213
"gorm.io/gorm"
1314
"time"
1415
)
@@ -23,12 +24,20 @@ func (f ReconcilerFunc) Reconcile() error {
2324
return f()
2425
}
2526

27+
type UsageReconciler struct {
28+
conn *gorm.DB
29+
}
30+
2631
func NewUsageReconciler(conn *gorm.DB) *UsageReconciler {
2732
return &UsageReconciler{conn: conn}
2833
}
2934

30-
type UsageReconciler struct {
31-
conn *gorm.DB
35+
type UsageReconcileStatus struct {
36+
StartTime time.Time
37+
EndTime time.Time
38+
39+
WorkspaceInstances int
40+
InvalidWorkspaceInstances int
3241
}
3342

3443
func (u *UsageReconciler) Reconcile() error {
@@ -38,16 +47,100 @@ func (u *UsageReconciler) Reconcile() error {
3847
startOfCurrentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
3948
startOfNextMonth := startOfCurrentMonth.AddDate(0, 1, 0)
4049

41-
return u.reconcile(ctx, startOfCurrentMonth, startOfNextMonth)
50+
status, err := u.ReconcileTimeRange(ctx, startOfCurrentMonth, startOfNextMonth)
51+
if err != nil {
52+
return err
53+
}
54+
log.WithField("usage_reconcile_status", status).Info("Reconcile completed.")
55+
return nil
4256
}
4357

44-
func (u *UsageReconciler) reconcile(ctx context.Context, from, to time.Time) error {
58+
func (u *UsageReconciler) ReconcileTimeRange(ctx context.Context, from, to time.Time) (*UsageReconcileStatus, error) {
4559
log.Infof("Gathering usage data from %s to %s", from, to)
46-
instances, err := db.ListWorkspaceInstancesInRange(ctx, u.conn, from, to)
60+
status := &UsageReconcileStatus{
61+
StartTime: from,
62+
EndTime: to,
63+
}
64+
instances, invalidInstances, err := u.loadWorkspaceInstances(ctx, from, to)
4765
if err != nil {
48-
return fmt.Errorf("failed to list instances: %w", err)
66+
return nil, fmt.Errorf("failed to load workspace instances: %w", err)
4967
}
68+
status.WorkspaceInstances = len(instances)
69+
status.InvalidWorkspaceInstances = len(invalidInstances)
5070

71+
if len(invalidInstances) > 0 {
72+
log.WithField("invalid_workspace_instances", invalidInstances).Errorf("Detected %d invalid instances. These will be skipped in the current run.", len(invalidInstances))
73+
}
74+
75+
log.WithField("workspace_instances", instances).Debug("Successfully loaded workspace instances.")
76+
return status, nil
77+
}
78+
79+
func (u *UsageReconciler) loadWorkspaceInstances(ctx context.Context, from, to time.Time) ([]db.WorkspaceInstance, []invalidWorkspaceInstance, error) {
80+
log.Infof("Gathering usage data from %s to %s", from, to)
81+
instances, err := db.ListWorkspaceInstancesInRange(ctx, u.conn, from, to)
82+
if err != nil {
83+
return nil, nil, fmt.Errorf("failed to list instances from db: %w", err)
84+
}
5185
log.Infof("Identified %d instances between %s and %s", len(instances), from, to)
52-
return nil
86+
87+
valid, invalid := validateInstances(instances)
88+
trimmed := trimStartStopTime(valid, from, to)
89+
return trimmed, invalid, nil
90+
}
91+
92+
type invalidWorkspaceInstance struct {
93+
reason string
94+
workspaceInstanceID uuid.UUID
95+
}
96+
97+
func validateInstances(instances []db.WorkspaceInstance) (valid []db.WorkspaceInstance, invalid []invalidWorkspaceInstance) {
98+
for _, i := range instances {
99+
// i is a pointer to the current element, we need to assign it to ensure we're copying the value, not the current pointer.
100+
instance := i
101+
102+
// Each instance must have a start time, without it, we do not have a baseline for usage computation.
103+
if !instance.CreationTime.IsSet() {
104+
invalid = append(invalid, invalidWorkspaceInstance{
105+
reason: "missing creation time",
106+
workspaceInstanceID: instance.ID,
107+
})
108+
continue
109+
}
110+
111+
start := instance.CreationTime.Time()
112+
113+
// Currently running instances do not have a stopped time set, so we ignore these.
114+
if instance.StoppedTime.IsSet() {
115+
stop := instance.StoppedTime.Time()
116+
if stop.Before(start) {
117+
invalid = append(invalid, invalidWorkspaceInstance{
118+
reason: "stop time is before start time",
119+
workspaceInstanceID: instance.ID,
120+
})
121+
continue
122+
}
123+
}
124+
125+
valid = append(valid, instance)
126+
}
127+
return valid, invalid
128+
}
129+
130+
// trimStartStopTime ensures that start time or stop time of an instance is never outside of specified start or stop time range.
131+
func trimStartStopTime(instances []db.WorkspaceInstance, maximumStart, minimumStop time.Time) []db.WorkspaceInstance {
132+
var updated []db.WorkspaceInstance
133+
134+
for _, instance := range instances {
135+
if instance.StartedTime.Time().Before(maximumStart) {
136+
instance.StartedTime = db.NewVarcharTime(maximumStart)
137+
}
138+
139+
if instance.StoppedTime.Time().After(minimumStop) {
140+
instance.StoppedTime = db.NewVarcharTime(minimumStop)
141+
}
142+
143+
updated = append(updated, instance)
144+
}
145+
return updated
53146
}

‎components/usage/pkg/controller/reconciler_test.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
// Copyright (c) 2022 Gitpod GmbH. All rights reserved.
2+
// Licensed under the GNU Affero General Public License (AGPL).
3+
// See License-AGPL.txt in the project root for license information.
4+
5+
package controller
6+
7+
import (
8+
"context"
9+
"github.com/gitpod-io/gitpod/usage/pkg/db"
10+
"github.com/google/uuid"
11+
"github.com/stretchr/testify/require"
12+
"testing"
13+
"time"
14+
)
15+
16+
func TestUsageReconciler_Reconcile(t *testing.T) {
17+
conn := db.ConnectForTests(t)
18+
workspaceID := "gitpodio-gitpod-gyjr82jkfnd"
19+
instanceStatus := []byte(`{"phase": "stopped", "conditions": {"deployed": false, "pullingImages": false, "serviceExists": false}}`)
20+
startOfMay := time.Date(2022, 05, 1, 0, 00, 00, 00, time.UTC)
21+
startOfJune := time.Date(2022, 06, 1, 0, 00, 00, 00, time.UTC)
22+
instances := []db.WorkspaceInstance{
23+
{
24+
ID: uuid.New(),
25+
WorkspaceID: workspaceID,
26+
CreationTime: db.NewVarcharTime(time.Date(2022, 05, 1, 00, 00, 00, 00, time.UTC)),
27+
StoppedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
28+
Status: instanceStatus,
29+
},
30+
// No creation time, invalid record
31+
{
32+
ID: uuid.New(),
33+
WorkspaceID: workspaceID,
34+
StoppedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
35+
Status: instanceStatus,
36+
},
37+
}
38+
39+
tx := conn.Create(instances)
40+
require.NoError(t, tx.Error)
41+
42+
reconciler := NewUsageReconciler(conn)
43+
44+
status, err := reconciler.ReconcileTimeRange(context.Background(), startOfMay, startOfJune)
45+
require.NoError(t, err)
46+
require.Equal(t, &UsageReconcileStatus{
47+
StartTime: startOfMay,
48+
EndTime: startOfJune,
49+
WorkspaceInstances: 1,
50+
InvalidWorkspaceInstances: 1,
51+
}, status)
52+
}

‎components/usage/pkg/db/workspace_instance.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,9 @@ func ListWorkspaceInstancesInRange(ctx context.Context, conn *gorm.DB, from, to
5959
Where(
6060
conn.Where("stoppedTime >= ?", from).Or("stoppedTime = ?", ""),
6161
).
62-
Where("creationTime < ?", to).
63-
Where("creationTime != ?", "").
62+
Where(
63+
conn.Where("creationTime < ?", to).Or("creationTime = ?", ""),
64+
).
6465
Find(&instances)
6566
if tx.Error != nil {
6667
return nil, fmt.Errorf("failed to list workspace instances: %w", tx.Error)

‎components/usage/pkg/db/workspace_instance_test.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,20 @@ func TestListWorkspaceInstancesInRange(t *testing.T) {
150150
StoppedTime: db.NewVarcharTime(time.Date(2022, 06, 1, 1, 0, 0, 0, time.UTC)),
151151
Status: status,
152152
},
153+
// Stopped in May, no creation time, should be retrieved but this is a poor data quality record.
154+
{
155+
ID: uuid.New(),
156+
WorkspaceID: workspaceID,
157+
StoppedTime: db.NewVarcharTime(time.Date(2022, 05, 1, 1, 0, 0, 0, time.UTC)),
158+
Status: status,
159+
},
160+
// Started in April, no stop time, still running
161+
{
162+
ID: uuid.New(),
163+
WorkspaceID: workspaceID,
164+
CreationTime: db.NewVarcharTime(time.Date(2022, 04, 31, 23, 00, 00, 00, time.UTC)),
165+
Status: status,
166+
},
153167
}
154168
invalid := []*db.WorkspaceInstance{
155169
// Start of June

0 commit comments

Comments
 (0)