8000 fix(115): 20GB file upload restriction (#7452) · AlistGo/alist@b803b00 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit b803b00

Browse files
foxxorcatfoxxorcat
authored
fix(115): 20GB file upload restriction (#7452)
* fix(115): multipart upload error * feat(115): Modify default page size * fix(115): Replace temporary repair scheme
1 parent 64ceb5a commit b803b00

File tree

4 files changed

+38
-35
lines changed

4 files changed

+38
-35
lines changed

‎drivers/115/meta.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ type Addition struct {
99
Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"`
1010
QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"`
1111
QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"`
12-
PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"`
12+
PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"`
1313
LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"`
1414
driver.RootID
1515
}

‎drivers/115/types.go

+17-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
package _115
22

33
import (
4+
"time"
5+
46
"github.com/SheltonZhu/115driver/pkg/driver"
57
"github.com/alist-org/alist/v3/internal/model"
68
"github.com/alist-org/alist/v3/pkg/utils"
7-
"time"
89
)
910

1011
var _ model.Obj = (*FileObj)(nil)
@@ -20,3 +21,18 @@ func (f *FileObj) CreateTime() time.Time {
2021
func (f *FileObj) GetHash() utils.HashInfo {
2122
return utils.NewHashInfo(utils.SHA1, f.Sha1)
2223
}
24+
25+
type UploadResult struct {
26+
driver.BasicResp
27+
Data struct {
28+
PickCode string `json:"pick_code"`
29+
FileSize int `json:"file_size"`
30+
FileID string `json:"file_id"`
31+
ThumbURL string `json:"thumb_url"`
32+
Sha1 string `json:"sha1"`
33+
Aid int `json:"aid"`
34+
FileName string `json:"file_name"`
35+
Cid string `json:"cid"`
36+
IsVideo int `json:"is_video"`
37+
} `json:"data"`
38+
}

‎drivers/115/util.go

+19-32
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"io"
1111
"net/http"
1212
"net/url"
13-
"path/filepath"
1413
"strconv"
1514
"strings"
1615
"sync"
@@ -254,6 +253,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i
254253
ossClient *oss.Client
255254
bucket *oss.Bucket
256255
ossToken *driver115.UploadOSSTokenResp
256+
bodyBytes []byte
257257
err error
258258
)
259259

@@ -268,12 +268,14 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i
268268
f(options)
269269
}
270270
}
271+
// oss 启用Sequential必须按顺序上传
272+
options.ThreadsNum = 1
271273

272274
if ossToken, err = d.client.GetOSSToken(); err != nil {
273275
return err
274276
}
275277

276-
if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret); err != nil {
278+
if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil {
277279
return err
278280
}
279281

@@ -294,6 +296,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i
294296
if imur, err = bucket.InitiateMultipartUpload(params.Object,
295297
oss.SetHeader(driver115.OssSecurityTokenHeaderName, ossToken.SecurityToken),
296298
oss.UserAgentHeader(driver115.OSSUserAgent),
299+
oss.EnableSha1(), oss.Sequential(),
297300
); err != nil {
298301
return err
299302
}
@@ -337,8 +340,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i
337340
continue
338341
}
339342

340-
b := bytes.NewBuffer(buf)
341-
if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil {
343+
if part, err = bucket.UploadPart(imur, bytes.NewBuffer(buf), chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil {
342344
break
343345
}
344346
}
@@ -373,14 +375,20 @@ LOOP:
373375
}
374376
}
375377

376-
// EOF错误是xml的Unmarshal导致的，响应其实是json格式，所以实际上上传是成功的
377-
if _, err = bucket.CompleteMultipartUpload(imur, parts, driver115.OssOption(params, ossToken)...); err != nil && !errors.Is(err, io.EOF) {
378-
// 当文件名含有 &< 这两个字符之一时响应的xml解析会出现错误，实际上上传是成功的
379-
if filename := filepath.Base(stream.GetName()); !strings.ContainsAny(filename, "&<") {
380-
return err
381-
}
378+
// 不知道啥原因，oss那边分片上传不计算sha1，导致115服务器校验错误
379+
// params.Callback.Callback = strings.ReplaceAll(params.Callback.Callback, "${sha1}", params.SHA1)
380+
if _, err := bucket.CompleteMultipartUpload(imur, parts, append(
381+
driver115.OssOption(params, ossToken),
382+
oss.CallbackResult(&bodyBytes),
383+
)...); err != nil {
384+
return err
382385
}
383-
return d.checkUploadStatus(dirID, params.SHA1)
386+
387+
var uploadResult UploadResult
388+
if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil {
389+
return err
390+
}
391+
return uploadResult.Err(string(bodyBytes))
384392
}
385393

386394
func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) {
@@ -389,27 +397,6 @@ func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) {
389397
}
390398
}
391399

392-
func (d *Pan115) checkUploadStatus(dirID, sha1 string) error {
393-
// 验证上传是否成功
394-
req := d.client.NewRequest().ForceContentType("application/json;charset=UTF-8")
395-
opts := []driver115.GetFileOptions{
396-
driver115.WithOrder(driver115.FileOrderByTime),
397-
driver115.WithShowDirEnable(false),
398-
driver115.WithAsc(false),
399-
driver115.WithLimit(500),
400-
}
401-
fResp, err := driver115.GetFiles(req, dirID, opts...)
402-
if err != nil {
403-
return err
404-
}
405-
for _, fileInfo := range fResp.Files {
406-
if fileInfo.Sha1 == sha1 {
407-
return nil
408-
}
409-
}
410-
return driver115.ErrUploadFailed
411-
}
412-
413400
func SplitFile(fileSize int64) (chunks []oss.FileChunk, err error) {
414401
for i := int64(1); i < 10; i++ {
415402
if fileSize < i*utils.GB { // 文件大小小于iGB时分为i*1000片

‎drivers/115_share/meta.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ type Addition struct {
99
Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"`
1010
QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"`
1111
QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"`
12-
PageSize int64 `json:"page_size" type:"number" default:"20" help:"list api per page size of 115 driver"`
12+
PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"`
1313
LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"`
1414
ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"`
1515
ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"`

0 commit comments

Comments
 (0)
0