Skip to content

Commit 465dd17

Browse files
authored
feat(cloudreve): s3 policy support (#8245)
* feat(cloudreve): s3 policy support * fix(cloudreve): correct potential off-by-one error in `etags` initialization
1 parent a630428 commit 465dd17

File tree

3 files changed

+87
-5
lines changed

3 files changed

+87
-5
lines changed

drivers/cloudreve/driver.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,8 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File
162162
switch r.Policy.Type {
163163
case "onedrive":
164164
err = d.upOneDrive(ctx, stream, u, up)
165+
case "s3":
166+
err = d.upS3(ctx, stream, u, up)
165167
case "remote": // 从机存储
166168
err = d.upRemote(ctx, stream, u, up)
167169
case "local": // 本机存储

drivers/cloudreve/types.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@ type Policy struct {
2121
}
2222

2323
type UploadInfo struct {
24-
SessionID string `json:"sessionID"`
25-
ChunkSize int `json:"chunkSize"`
26-
Expires int `json:"expires"`
27-
UploadURLs []string `json:"uploadURLs"`
28-
Credential string `json:"credential,omitempty"`
24+
SessionID string `json:"sessionID"`
25+
ChunkSize int `json:"chunkSize"`
26+
Expires int `json:"expires"`
27+
UploadURLs []string `json:"uploadURLs"`
28+
Credential string `json:"credential,omitempty"` // local
29+
CompleteURL string `json:"completeURL,omitempty"` // s3
2930
}
3031

3132
type DirectoryResp struct {

drivers/cloudreve/util.go

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,3 +312,82 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u
312312
}
313313
return nil
314314
}
315+
316+
func (d *Cloudreve) upS3(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error {
317+
var finish int64 = 0
318+
var chunk int = 0
319+
var etags []string
320+
DEFAULT := int64(u.ChunkSize)
321+
for finish < stream.GetSize() {
322+
if utils.IsCanceled(ctx) {
323+
return ctx.Err()
324+
}
325+
utils.Log.Debugf("[Cloudreve-S3] upload: %d", finish)
326+
var byteSize = DEFAULT
327+
left := stream.GetSize() - finish
328+
if left < DEFAULT {
329+
byteSize = left
330+
}
331+
byteData := make([]byte, byteSize)
332+
n, err := io.ReadFull(stream, byteData)
333+
utils.Log.Debug(err, n)
334+
if err != nil {
335+
return err
336+
}
337+
req, err := http.NewRequest("PUT", u.UploadURLs[chunk],
338+
driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData)))
339+
if err != nil {
340+
return err
341+
}
342+
req = req.WithContext(ctx)
343+
req.ContentLength = byteSize
344+
finish += byteSize
345+
res, err := base.HttpClient.Do(req)
346+
if err != nil {
347+
return err
348+
}
349+
_ = res.Body.Close()
350+
etags = append(etags, res.Header.Get("ETag"))
351+
up(float64(finish) * 100 / float64(stream.GetSize()))
352+
chunk++
353+
}
354+
355+
// s3LikeFinishUpload
356+
// https://github.com/cloudreve/frontend/blob/b485bf297974cbe4834d2e8e744ae7b7e5b2ad39/src/component/Uploader/core/api/index.ts#L204-L252
357+
bodyBuilder := &strings.Builder{}
358+
bodyBuilder.WriteString("<CompleteMultipartUpload>")
359+
for i, etag := range etags {
360+
bodyBuilder.WriteString(fmt.Sprintf(
361+
`<Part><PartNumber>%d</PartNumber><ETag>%s</ETag></Part>`,
362+
i+1, // PartNumber 从 1 开始
363+
etag,
364+
))
365+
}
366+
bodyBuilder.WriteString("</CompleteMultipartUpload>")
367+
req, err := http.NewRequest(
368+
"POST",
369+
u.CompleteURL,
370+
strings.NewReader(bodyBuilder.String()),
371+
)
372+
if err != nil {
373+
return err
374+
}
375+
req.Header.Set("Content-Type", "application/xml")
376+
req.Header.Set("User-Agent", d.getUA())
377+
res, err := base.HttpClient.Do(req)
378+
if err != nil {
379+
return err
380+
}
381+
defer res.Body.Close()
382+
if res.StatusCode != http.StatusOK {
383+
body, _ := io.ReadAll(res.Body)
384+
return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body))
385+
}
386+
387+
// 上传成功发送回调请求
388+
err = d.request(http.MethodGet, "/callback/s3/"+u.SessionID, nil, nil)
389+
if err != nil {
390+
return err
391+
}
392+
return nil
393+
}

0 commit comments

Comments
 (0)