8000 feat(baidu_netdisk): support dynamical slice size for low bandwith up… · anwen-anyi/alist@2570707 · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Commit 2570707

Browse files
ljcbabyljcbaby
authored
feat(baidu_netdisk): support dynamical slice size for low bandwith upload case (AlistGo#7965)
* 动态分片尺寸 * 补充严格测试结果
1 parent 4145734 commit 2570707

File tree

3 files changed

+72
-21
lines changed

3 files changed

+72
-21
lines changed

‎drivers/baidu_netdisk/driver.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,15 +189,15 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F
189189
}
190190

191191
streamSize := stream.GetSize()
192-
sliceSize := d.getSliceSize()
192+
sliceSize := d.getSliceSize(streamSize)
193193
count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1))
194194
lastBlockSize := streamSize % sliceSize
195195
if streamSize > 0 && lastBlockSize == 0 {
196196
lastBlockSize = sliceSize
197197
}
198198

199199
//cal md5 for first 256k data
200-
const SliceSize int64 = 256 * 1024
200+
const SliceSize int64 = 256 * utils.KB
201201
// cal md5
202202
blockList := make([]string, 0, count)
203203
byteSize := sliceSize

‎drivers/baidu_netdisk/meta.go

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,17 @@ import (
88
type Addition struct {
99
RefreshToken string `json:"refresh_token" required:"true"`
1010
driver.RootPath
11-
OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"`
12-
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
13-
DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"`
14-
ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"`
15-
ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"`
16-
CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"`
17-
AccessToken string
18-
UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"`
19-
UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"`
20-
CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"`
11+
OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"`
12+
OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
13+
DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"`
14+
ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"`
15+
ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"`
16+
CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"`
17+
AccessToken string
18+
UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"`
19+
UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"`
20+
CustomUploadPartSize int64 10000 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"`
21+
LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"`
2122
}
2223

2324
var config = driver.Config{

‎drivers/baidu_netdisk/util.go

Lines changed: 59 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) {
136136
return res, nil
137137
}
138138

139-
func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model.Link, error) {
139+
func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Link, error) {
140140
var resp DownloadResp
141141
params := map[string]string{
142142
"method": "filemetas",
@@ -164,7 +164,7 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model
164164
}, nil
165165
}
166166

167-
func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Link, error) {
167+
func (d *BaiduNetdisk) linkCrack(file model.Obj, _ model.LinkArgs) (*model.Link, error) {
168168
var resp DownloadResp2
169169
param := map[string]string{
170170
"target": fmt.Sprintf("[\"%s\"]", file.GetPath()),
@@ -230,22 +230,72 @@ func joinTime(form map[string]string, ctime, mtime int64) {
230230

231231
const (
232232
DefaultSliceSize int64 = 4 * utils.MB
233-
VipSliceSize = 16 * utils.MB
234-
SVipSliceSize = 32 * utils.MB
233+
VipSliceSize int64 = 16 * utils.MB
234+
SVipSliceSize int64 = 32 * utils.MB
235+
236+
MaxSliceNum = 2048 // 文档写的是 6D4E 1024/没写 ，但实际测试是 2048
237+
SliceStep int64 = 1 * utils.MB
235238
)
236239

237-
func (d *BaiduNetdisk) getSliceSize() int64 {
240+
func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 {
241+
// 非会员固定为 4MB
242+
if d.vipType == 0 {
243+
if d.CustomUploadPartSize != 0 {
244+
log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize")
245+
}
246+
if filesize > MaxSliceNum*DefaultSliceSize {
247+
log.Warnf("File size(%d) is too large, may cause upload failure", filesize)
248+
}
249+
250+
return DefaultSliceSize
251+
}
252+
238253
if d.CustomUploadPartSize != 0 {
254+
if d.CustomUploadPartSize < DefaultSliceSize {
255+
log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize)
256+
return DefaultSliceSize
257+
}
258+
259+
if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize {
260+
log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize)
261+
return VipSliceSize
262+
}
263+
264+
if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize {
265+
log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize)
266+
return SVipSliceSize
267+
}
268+
239269
return d.CustomUploadPartSize
240270
}
271+
272+
maxSliceSize := DefaultSliceSize
273+
241274
switch d.vipType {
242275
case 1:
243-
return VipSliceSize
276+
maxSliceSize = VipSliceSize
244277
case 2:
245-
return SVipSliceSize
246-
default:
247-
return DefaultSliceSize
278+
maxSliceSize = SVipSliceSize
279+
}
280+
281+
// upload on low bandwidth
282+
if d.LowBandwithUploadMode {
283+
size := DefaultSliceSize
284+
285+
for size <= maxSliceSize {
286+
if filesize <= MaxSliceNum*size {
287+
return size
288+
}
289+
290+
size += SliceStep
291+
}
248292
}
293+
294+
if filesize > MaxSliceNum*maxSliceSize {
295+
log.Warnf("File size(%d) is too large, may cause upload failure", filesize)
296+
}
297+
298+
return maxSliceSize
249299
}
250300

251301
// func encodeURIComponent(str string) string {

0 commit comments

Comments
 (0)
0