|
5 | 5 | package storage |
6 | 6 |
|
7 | 7 | import ( |
8 | | - "archive/tar" |
9 | 8 | "context" |
10 | | - "crypto/rand" |
11 | | - "crypto/tls" |
12 | | - "flag" |
13 | | - "io" |
14 | | - "io/ioutil" |
15 | | - "net/http" |
16 | | - "os" |
17 | 9 | "strings" |
18 | 10 | "testing" |
19 | | - "time" |
20 | 11 |
|
21 | 12 | gcp_storage "cloud.google.com/go/storage" |
22 | 13 | "github.com/fsouza/fake-gcs-server/fakestorage" |
23 | | - "github.com/testcontainers/testcontainers-go" |
24 | | - "github.com/testcontainers/testcontainers-go/wait" |
25 | 14 | "google.golang.org/api/option" |
26 | | - "k8s.io/apimachinery/pkg/api/resource" |
27 | 15 |
|
28 | | - csapi "github.com/gitpod-io/gitpod/content-service/api" |
29 | 16 | "github.com/gitpod-io/gitpod/content-service/api/config" |
30 | 17 | "github.com/gitpod-io/gitpod/content-service/pkg/archive" |
31 | 18 | ) |
@@ -66,230 +53,3 @@ func TestObjectAccessToNonExistentObj(t *testing.T) { |
66 | 53 | t.Errorf("gcloud storage reported object found despite it being non-existent") |
67 | 54 | } |
68 | 55 | } |
69 | | - |
70 | | -var runWithDocker = flag.Bool("with-docker", false, "run fake-gcs-server in docker") |
71 | | - |
72 | | -func TestObjectUpload(t *testing.T) { |
73 | | - tests := []struct { |
74 | | - Desc string |
75 | | - Name string |
76 | | - Stage config.Stage |
77 | | - Owner string |
78 | | - WorkspaceID string |
79 | | - InstanceID string |
80 | | - Payload func() (string, error) |
81 | | - }{ |
82 | | - {"valid 1M backup", DefaultBackup, config.StageDevStaging, "fake-owner", "fake-workspace", "fake-instance", fakeTarPayload("1Mi")}, |
83 | | - {"valid 100M backup", DefaultBackup, config.StageDevStaging, "fake-owner", "fake-workspace", "fake-instance", fakeTarPayload("100Mi")}, |
84 | | - {"valid 1GB backup", DefaultBackup, config.StageDevStaging, "fake-owner", "fake-workspace", "fake-instance", fakeTarPayload("1Gi")}, |
85 | | - } |
86 | | - |
87 | | - for _, test := range tests { |
88 | | - t.Run(test.Desc, func(t *testing.T) { |
89 | | - ctx, cancel := context.WithCancel(context.Background()) |
90 | | - defer cancel() |
91 | | - |
92 | | - var err error |
93 | | - var client *gcp_storage.Client |
94 | | - |
95 | | - // use docker to run the fake GCS server |
96 | | - // or start an embed server otherwise |
97 | | - switch *runWithDocker { |
98 | | - case true: |
99 | | - var gcsContainer *fakeGCSContainer |
100 | | - gcsContainer, err = setupFakeStorage(ctx) |
101 | | - if err != nil { |
102 | | - break |
103 | | - } |
104 | | - |
105 | | - defer gcsContainer.Terminate(ctx) |
106 | | - |
107 | | - client, err = gcp_storage.NewClient(ctx, |
108 | | - option.WithEndpoint(gcsContainer.URI), |
109 | | - option.WithoutAuthentication(), |
110 | | - option.WithHTTPClient(&http.Client{ |
111 | | - Transport: &http.Transport{ |
112 | | - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, |
113 | | - }, |
114 | | - }), |
115 | | - ) |
116 | | - case false: |
117 | | - // the downside of using the fake server as library is the amount or |
118 | | - // memory used by the test binary (this makes too hard the use of pprof) |
119 | | - var server *fakestorage.Server |
120 | | - server, err = fakestorage.NewServerWithOptions(fakestorage.Options{ |
121 | | - Writer: os.Stdout, |
122 | | - }) |
123 | | - if err != nil { |
124 | | - break |
125 | | - } |
126 | | - |
127 | | - defer server.Stop() |
128 | | - |
129 | | - server.CreateBucketWithOpts(fakestorage.CreateBucketOpts{ |
130 | | - Name: gcpBucketName(test.Stage, test.Owner), |
131 | | - }) |
132 | | - |
133 | | - client = server.Client() |
134 | | - } |
135 | | - |
136 | | - if err != nil { |
137 | | - t.Fatalf("failed to create GCS client: %v", err) |
138 | | - } |
139 | | - |
140 | | - storage := DirectGCPStorage{ |
141 | | - Stage: test.Stage, |
142 | | - Username: test.Owner, |
143 | | - WorkspaceName: test.WorkspaceID, |
144 | | - InstanceID: test.InstanceID, |
145 | | - GCPConfig: config.GCPConfig{ |
146 | | - Region: "fake-region", |
147 | | - Project: "fake-project", |
148 | | - }, |
149 | | - // replace internal (valid) client with fake one |
150 | | - client: client, |
151 | | - } |
152 | | - |
153 | | - storage.ObjectAccess = storage.defaultObjectAccess |
154 | | - |
155 | | - bucket := storage.client.Bucket(gcpBucketName(test.Stage, test.Owner)) |
156 | | - err = bucket.Create(context.Background(), "fake-project", nil) |
157 | | - if err != nil { |
158 | | - t.Fatal(err) |
159 | | - } |
160 | | - |
161 | | - t.Log("creating fake tar file...") |
162 | | - payloadPath, err := test.Payload() |
163 | | - if err != nil { |
164 | | - t.Fatalf("error creating test file: %v", err) |
165 | | - } |
166 | | - defer os.Remove(payloadPath) |
167 | | - |
168 | | - _, _, err = storage.Upload(ctx, payloadPath, test.Name, WithContentType(csapi.ContentTypeManifest)) |
169 | | - if err != nil { |
170 | | - t.Fatalf("error uploading file: %v", err) |
171 | | - } |
172 | | - |
173 | | - dst, err := ioutil.TempDir("", "gcs-download") |
174 | | - if err != nil { |
175 | | - t.Fatalf("error creating temporal file: %v", err) |
176 | | - } |
177 | | - defer os.RemoveAll(dst) |
178 | | - |
179 | | - found, err := storage.Download(ctx, dst, test.Name, []archive.IDMapping{}) |
180 | | - if err != nil { |
181 | | - t.Fatalf("error downloading file: %v", err) |
182 | | - } |
183 | | - |
184 | | - if !found { |
185 | | - t.Fatal("backup not found") |
186 | | - } |
187 | | - |
188 | | - files, err := ioutil.ReadDir(dst) |
189 | | - if err != nil { |
190 | | - t.Fatalf("error reading content of directory: %v", err) |
191 | | - } |
192 | | - |
193 | | - if len(files) != 1 { |
194 | | - t.Fatalf("expected 1 file but found %v", len(files)) |
195 | | - } |
196 | | - }) |
197 | | - } |
198 | | -} |
199 | | - |
200 | | -func fakeTarPayload(size string) func() (string, error) { |
201 | | - return func() (string, error) { |
202 | | - payload, err := ioutil.TempFile("", "test-payload-") |
203 | | - if err != nil { |
204 | | - return "", err |
205 | | - } |
206 | | - |
207 | | - defer os.Remove(payload.Name()) |
208 | | - |
209 | | - q, err := resource.ParseQuantity(size) |
210 | | - if err != nil { |
211 | | - return "", err |
212 | | - } |
213 | | - |
214 | | - _, err = io.CopyN(payload, rand.Reader, q.Value()) |
215 | | - if err != nil { |
216 | | - return "", err |
217 | | - } |
218 | | - |
219 | | - tarFile, err := ioutil.TempFile("", "tar-payload-") |
220 | | - if err != nil { |
221 | | - return "", err |
222 | | - } |
223 | | - |
224 | | - defer tarFile.Close() |
225 | | - |
226 | | - tarWriter := tar.NewWriter(tarFile) |
227 | | - defer tarWriter.Close() |
228 | | - |
229 | | - err = addFileToTar(payload.Name(), tarWriter) |
230 | | - if err != nil { |
231 | | - return "", err |
232 | | - } |
233 | | - |
234 | | - return tarFile.Name(), nil |
235 | | - } |
236 | | -} |
237 | | - |
238 | | -func addFileToTar(filePath string, tarWriter *tar.Writer) error { |
239 | | - file, err := os.Open(filePath) |
240 | | - if err != nil { |
241 | | - return err |
242 | | - } |
243 | | - defer file.Close() |
244 | | - |
245 | | - stat, err := file.Stat() |
246 | | - if err != nil { |
247 | | - return err |
248 | | - } |
249 | | - |
250 | | - err = tarWriter.WriteHeader(&tar.Header{ |
251 | | - Name: file.Name(), |
252 | | - Size: stat.Size(), |
253 | | - Mode: int64(stat.Mode()), |
254 | | - ModTime: stat.ModTime(), |
255 | | - }) |
256 | | - if err != nil { |
257 | | - return err |
258 | | - } |
259 | | - |
260 | | - _, err = io.Copy(tarWriter, file) |
261 | | - if err != nil { |
262 | | - return err |
263 | | - } |
264 | | - |
265 | | - return nil |
266 | | -} |
267 | | - |
268 | | -type fakeGCSContainer struct { |
269 | | - testcontainers.Container |
270 | | - |
271 | | - URI string |
272 | | -} |
273 | | - |
274 | | -func setupFakeStorage(ctx context.Context) (*fakeGCSContainer, error) { |
275 | | - req := testcontainers.ContainerRequest{ |
276 | | - Image: "fsouza/fake-gcs-server", |
277 | | - ExposedPorts: []string{"4443/tcp"}, |
278 | | - NetworkMode: "host", |
279 | | - Entrypoint: []string{"/bin/fake-gcs-server", "-data=/data", "-backend=filesystem", "-public-host=localhost:4443"}, |
280 | | - WaitingFor: wait.ForLog("server started at").WithPollInterval(1 * time.Second), |
281 | | - } |
282 | | - |
283 | | - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ |
284 | | - ContainerRequest: req, |
285 | | - Started: true, |
286 | | - }) |
287 | | - if err != nil { |
288 | | - return nil, err |
289 | | - } |
290 | | - |
291 | | - return &fakeGCSContainer{ |
292 | | - Container: container, |
293 | | - URI: "https://localhost:4443/storage/v1/", |
294 | | - }, nil |
295 | | -} |
0 commit comments