Skip to content

[pvc] fix prebuilds when using pvc #12746

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 12, 2022
Merged

[pvc] fix prebuilds when using pvc #12746

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 45 additions & 5 deletions components/content-service/pkg/layer/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,12 @@ func (s *Provider) GetContentLayerPVC(ctx context.Context, owner, workspaceID st
if err != nil {
return nil, nil, err
}
layerReady, err := workspaceReadyLayerPVC(csapi.WorkspaceInitFromBackup)
if err != nil {
return nil, nil, err
}

l = []Layer{*layer}
l = []Layer{*layer, *layerReady}
return l, manifest, nil
}

Expand All @@ -274,8 +278,12 @@ func (s *Provider) GetContentLayerPVC(ctx context.Context, owner, workspaceID st
if err != nil {
return nil, nil, err
}
layerReady, err := workspaceReadyLayerPVC(csapi.WorkspaceInitFromBackup)
if err != nil {
return nil, nil, err
}

l = []Layer{*layer}
l = []Layer{*layer, *layerReady}
return l, manifest, nil
}

Expand All @@ -287,8 +295,12 @@ func (s *Provider) GetContentLayerPVC(ctx context.Context, owner, workspaceID st
if err != nil {
return nil, nil, err
}
layerReady, err := workspaceReadyLayerPVC(csapi.WorkspaceInitFromBackup)
if err != nil {
return nil, nil, err
}

l = []Layer{*layer}
l = []Layer{*layer, *layerReady}
return l, manifest, nil
}
return s.getSnapshotContentLayer(ctx, gis)
Expand All @@ -299,8 +311,13 @@ func (s *Provider) GetContentLayerPVC(ctx context.Context, owner, workspaceID st
if err != nil {
return nil, nil, err
}
layerReady, err := workspaceReadyLayerPVC(csapi.WorkspaceInitFromPrebuild)
if err != nil {
return nil, nil, err
}

l = []Layer{*layer, *layerReady}

l = []Layer{*layer}
return l, manifest, nil
}
l, manifest, err = s.getPrebuildContentLayer(ctx, pis, true)
Expand Down Expand Up @@ -341,7 +358,11 @@ func (s *Provider) GetContentLayerPVC(ctx context.Context, owner, workspaceID st
if err != nil {
return nil, nil, err
}
return []Layer{*layer}, nil, nil
layerReady, err := workspaceReadyLayerPVC(csapi.WorkspaceInitFromOther)
if err != nil {
return nil, nil, err
}
return []Layer{*layer, *layerReady}, nil, nil
}
if initializer.GetBackup() != nil {
// We were asked to restore a backup and have tried above. We've failed to restore the backup,
Expand Down Expand Up @@ -508,13 +529,16 @@ git log --pretty=%H -n 1 > /.workspace/prestophookdata/git_log_2.txt
cp /workspace/.gitpod/prebuild-log* /.workspace/prestophookdata/
`

var pvcEnabledFile = `PVC`

// version of this function for persistent volume claim feature
// we cannot use /workspace folder as when mounting /workspace folder through PVC
// it will mask anything that was in container layer, hence we are using /.workspace instead here
func contentDescriptorToLayerPVC(cdesc []byte) (*Layer, error) {
layers := []fileInLayer{
{&tar.Header{Typeflag: tar.TypeDir, Name: "/.workspace", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
{&tar.Header{Typeflag: tar.TypeDir, Name: "/.workspace/.gitpod", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
{&tar.Header{Typeflag: tar.TypeReg, Name: "/.workspace/.gitpod/pvc", Uid: 0, Gid: 0, Mode: 0775, Size: int64(len(pvcEnabledFile))}, []byte(pvcEnabledFile)},
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this allows us to reliably check in supervisor if workspace is using pvc or not, instead of relying on content.json being present, which is not the case everytime (restoring from prebuild will not generate any content.json for example).

{&tar.Header{Typeflag: tar.TypeReg, Name: "/.supervisor/prestophook.sh", Uid: 0, Gid: 0, Mode: 0775, Size: int64(len(prestophookScript))}, []byte(prestophookScript)},
}
if len(cdesc) > 0 {
Expand All @@ -523,6 +547,22 @@ func contentDescriptorToLayerPVC(cdesc []byte) (*Layer, error) {
return layerFromContent(layers...)
}

func workspaceReadyLayerPVC(src csapi.WorkspaceInitSource) (*Layer, error) {
msg := csapi.WorkspaceReadyMessage{
Source: src,
}
ctnt, err := json.Marshal(msg)
if err != nil {
return nil, err
}

return layerFromContent(
fileInLayer{&tar.Header{Typeflag: tar.TypeDir, Name: "/.workspace", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
fileInLayer{&tar.Header{Typeflag: tar.TypeDir, Name: "/.workspace/.gitpod", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755}, nil},
fileInLayer{&tar.Header{Typeflag: tar.TypeReg, Name: "/.workspace/.gitpod/ready", Uid: initializer.GitpodUID, Gid: initializer.GitpodGID, Mode: 0755, Size: int64(len(ctnt))}, []byte(ctnt)},
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this allows us to place ready file.
why we cannot use /workspace folder? because when using pvc that folder is masked by pvc.
also, on top of that, when restoring from prebuild for example, prebuild volume snapshot will contain ready file in there from the time when prebuild was generated.
so this allows us to completely separate pvc and non pvc approach.

)
}

func workspaceReadyLayer(src csapi.WorkspaceInitSource) (*Layer, error) {
msg := csapi.WorkspaceReadyMessage{
Source: src,
Expand Down
9 changes: 7 additions & 2 deletions components/supervisor/pkg/supervisor/supervisor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1327,9 +1327,14 @@ func startContentInit(ctx context.Context, cfg *Config, wg *sync.WaitGroup, cst

fn := "/workspace/.gitpod/content.json"
fnReady := "/workspace/.gitpod/ready"
if _, err := os.Stat("/.workspace/.gitpod/content.json"); !os.IsNotExist(err) {
if _, err := os.Stat("/.workspace/.gitpod/pvc"); !os.IsNotExist(err) {
fn = "/.workspace/.gitpod/content.json"
log.Info("Detected content.json in /.workspace folder, assuming PVC feature enabled")
fnReady = "/.workspace/.gitpod/ready"
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we need to wait for ready file that comes from PVC layer, we should not use ready file that comes from /workspace since that file will be backed up into volume snapshot when it is created from pvc.
so this makes a clear separation between pvc and non-pvc code path.

log.Info("Detected pvc file in /.workspace folder, assuming PVC feature enabled")
} else if _, err := os.Stat("/.workspace/.gitpod/content.json"); !os.IsNotExist(err) {
// todo(pavel): remove this after gen65 has shipped
fn = "/.workspace/.gitpod/content.json"
log.Info("[deprecated] Detected content.json in /.workspace folder, assuming PVC feature enabled")
}

var contentFile *os.File
Expand Down