Mirror
/
Cloudreve
mirror of https://github.com/cloudreve/Cloudreve.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

Feat: put / get file in qiniu policy

pull/247/head
HFO4 6 years ago
parent
7fbbcefc57
commit
90f82100cf
8 changed files with 158 additions and 92 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 27
      models/policy.go
  2. 3
      models/policy_test.go
  3. 5
      pkg/filesystem/archive.go
  4. 42
      pkg/filesystem/qiniu/handler_test.go
  5. 135
      pkg/filesystem/qiniu/handller.go
  6. 6
      pkg/filesystem/remote/handler.go
  7. 20
      pkg/request/request.go
  8. 12
      pkg/request/request_test.go

27
models/policy.go
View File

@ -126,18 +126,23 @@ func (policy *Policy) GenerateFileName(uid uint, origin string) string {
}
// 部分存储策略可以使用{origin}代表原始文件名
switch policy.Type {
case "qiniu":
// 七牛会将$(fname)自动替换为原始文件名
replaceTable["{originname}"] = "$(fname)"
case "local", "remote":
if origin == "" {
// 如果上游未传回原始文件名，则使用占位符，让云存储端替换
switch policy.Type {
case "qiniu":
// 七牛会将$(fname)自动替换为原始文件名
replaceTable["{originname}"] = "$(fname)"
case "local", "remote":
replaceTable["{originname}"] = origin
case "oss":
// OSS会将${filename}自动替换为原始文件名
replaceTable["{originname}"] = "${filename}"
case "upyun":
// Upyun会将{filename}{.suffix}自动替换为原始文件名
replaceTable["{originname}"] = "{filename}{.suffix}"
}
} else {
replaceTable["{originname}"] = origin
case "oss":
// OSS会将${filename}自动替换为原始文件名
replaceTable["{originname}"] = "${filename}"
case "upyun":
// Upyun会将{filename}{.suffix}自动替换为原始文件名
replaceTable["{originname}"] = "{filename}{.suffix}"
}
fileRule = util.Replace(replaceTable, fileRule)

3
models/policy_test.go
View File

@ -123,7 +123,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
testPolicy.Type = "qiniu"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123$(fname)", testPolicy.GenerateFileName(1, "123.txt"))
asserts.Equal("1123123.txt", testPolicy.GenerateFileName(1, "123.txt"))
testPolicy.Type = "oss"
testPolicy.FileNameRule = "{uid}123{originname}"
@ -132,6 +132,7 @@ func TestPolicy_GenerateFileName(t *testing.T) {
testPolicy.Type = "upyun"
testPolicy.FileNameRule = "{uid}123{originname}"
asserts.Equal("1123{filename}{.suffix}", testPolicy.GenerateFileName(1, ""))
}
func TestPolicy_IsDirectlyPreview(t *testing.T) {

5
pkg/filesystem/archive.go
View File

@ -112,7 +112,10 @@ func (fs *FileSystem) doCompress(ctx context.Context, file *model.File, folder *
}
// 获取文件内容
fileToZip, err := fs.Handler.Get(ctx, file.SourceName)
fileToZip, err := fs.Handler.Get(
context.WithValue(ctx, fsctx.FileModelCtx, *file),
file.SourceName,
)
if err != nil {
util.Log().Debug("Open%s，%s", file.Name, err)
return

42
pkg/filesystem/qiniu/handler_test.go
View File

@ -1,42 +0,0 @@
package qiniu
import (
"context"
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/cache"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/stretchr/testify/assert"
"testing"
)
func TestHandler_Token(t *testing.T) {
asserts := assert.New(t)
handler := Handler{
Policy: &model.Policy{
MaxSize: 10,
OptionsSerialized: model.PolicyOption{
MimeType: "ss",
},
AccessKey: "ak",
SecretKey: "sk",
Server: "http://test.com",
},
}
ctx := context.Background()
// 成功
{
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
ctx = context.WithValue(ctx, fsctx.SavePathCtx, "/123")
_, err := handler.Token(ctx, 10, "123")
asserts.NoError(err)
}
// 上下文无存储路径
{
ctx = context.Background()
cache.Set("setting_siteURL", "http://test.cloudreve.org", 0)
_, err := handler.Token(ctx, 10, "123")
asserts.Error(err)
}
}

135
pkg/filesystem/qiniu/handller.go
View File

@ -7,10 +7,12 @@ import (
model "github.com/HFO4/cloudreve/models"
"github.com/HFO4/cloudreve/pkg/filesystem/fsctx"
"github.com/HFO4/cloudreve/pkg/filesystem/response"
"github.com/HFO4/cloudreve/pkg/request"
"github.com/HFO4/cloudreve/pkg/serializer"
"github.com/qiniu/api.v7/v7/auth/qbox"
"github.com/qiniu/api.v7/v7/storage"
"io"
"net/http"
"net/url"
"time"
)
@ -22,43 +24,118 @@ type Handler struct {
// Get 获取文件
func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, errors.New("未实现")
// 给文件名加上随机参数以强制拉取
path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
path,
url.URL{},
int64(model.GetIntSetting("preview_timeout", 60)),
false,
0,
)
if err != nil {
return nil, err
}
// 获取文件数据流
client := request.HTTPClient{}
resp, err := client.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithHeader(
http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}},
),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试自主获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler Handler) Put(ctx context.Context, file io.ReadCloser, dst string, size uint64) error {
return errors.New("未实现")
//// 凭证生成
//putPolicy := storage.PutPolicy{
// Scope: "cloudrevetest",
//}
//mac := auth.New("YNzTBBpDUq4EEiFV0-vyJCZCJ0LvUEI0_WvxtEXE", "Clm9d9M2CH7pZ8vm049ZlGZStQxrRQVRTjU_T5_0")
//upToken := putPolicy.UploadToken(mac)
//
//cfg := storage.Config{}
//// 空间对应的机房
//cfg.Zone = &storage.ZoneHuadong
//formUploader := storage.NewFormUploader(&cfg)
//ret := storage.PutRet{}
//putExtra := storage.PutExtra{
// Params: map[string]string{},
//}
//
//defer file.Close()
//
//err := formUploader.Put(ctx, &ret, upToken, dst, file, int64(size), &putExtra)
//if err != nil {
// fmt.Println(err)
// return err
//}
//fmt.Println(ret.Key, ret.Hash)
//return nil
defer file.Close()
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_credential_timeout", 3600)
// 生成上传策略
putPolicy := storage.PutPolicy{
// 指定为覆盖策略
Scope: fmt.Sprintf("%s:%s", handler.Policy.BucketName, dst),
SaveKey: dst,
ForceSaveKey: true,
FsizeLimit: int64(size),
}
// 是否开启了MIMEType限制
if handler.Policy.OptionsSerialized.MimeType != "" {
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
}
// 生成上传凭证
token, err := handler.getUploadCredential(ctx, putPolicy, int64(credentialTTL))
if err != nil {
return err
}
// 创建上传表单
cfg := storage.Config{}
formUploader := storage.NewFormUploader(&cfg)
ret := storage.PutRet{}
putExtra := storage.PutExtra{
Params: map[string]string{},
}
// 开始上传
err = formUploader.Put(ctx, &ret, token.Token, dst, file, int64(size), &putExtra)
if err != nil {
return err
}
return nil
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
// 返回未删除的文件
func (handler Handler) Delete(ctx context.Context, files []string) ([]string, error) {
return []string{}, errors.New("未实现")
// TODO 大于一千个文件需要分批发送
deleteOps := make([]string, 0, len(files))
for _, key := range files {
deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
}
mac := qbox.NewMac(handler.Policy.AccessKey, handler.Policy.SecretKey)
cfg := storage.Config{
UseHTTPS: true,
}
bucketManager := storage.NewBucketManager(mac, &cfg)
rets, err := bucketManager.Batch(deleteOps)
// 处理删除结果
if err != nil {
failed := make([]string, 0, len(rets))
for k, ret := range rets {
if ret.Code != 200 {
failed = append(failed, files[k])
}
}
return failed, errors.New("删除失败")
}
return []string{}, nil
}
// Thumb 获取文件缩略图

6
pkg/filesystem/remote/handler.go
View File

@ -76,6 +76,12 @@ func (handler Handler) Get(ctx context.Context, path string) (response.RSCloser,
}
resp.SetFirstFakeChunk()
// 尝试获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}

20
pkg/request/request.go
View File

@ -191,7 +191,6 @@ func (resp *Response) DecodeResponse() (*serializer.Response, error) {
// NopRSCloser 实现不完整seeker
type NopRSCloser struct {
body io.ReadCloser
size int64
status *rscStatus
}
@ -199,6 +198,8 @@ type rscStatus struct {
// http.ServeContent 会读取一小块以决定内容类型，
// 但是响应body无法实现seek，所以此项为真时第一个read会返回假数据
IgnoreFirst bool
Size int64
}
// GetRSCloser 返回带有空seeker的RSCloser，供http.ServeContent使用
@ -208,9 +209,10 @@ func (resp *Response) GetRSCloser() (*NopRSCloser, error) {
}
return &NopRSCloser{
body: resp.Response.Body,
size: resp.Response.ContentLength,
status: &rscStatus{},
body: resp.Response.Body,
status: &rscStatus{
Size: resp.Response.ContentLength,
},
}, resp.Err
}
@ -220,11 +222,13 @@ func (instance NopRSCloser) SetFirstFakeChunk() {
instance.status.IgnoreFirst = true
}
// SetContentLength 设置数据流大小
func (instance NopRSCloser) SetContentLength(size int64) {
instance.status.Size = size
}
// Read 实现 NopRSCloser reader
func (instance NopRSCloser) Read(p []byte) (n int, err error) {
if instance.status.IgnoreFirst {
return 0, io.EOF
}
return instance.body.Read(p)
}
@ -244,7 +248,7 @@ func (instance NopRSCloser) Seek(offset int64, whence int) (int64, error) {
case io.SeekStart:
return 0, nil
case io.SeekEnd:
return instance.size, nil
return instance.status.Size, nil
}
}
return 0, errors.New("未实现")

12
pkg/request/request_test.go
View File

@ -211,3 +211,15 @@ func TestResponse_DecodeResponse(t *testing.T) {
asserts.Equal(0, response.Code)
}
}
func TestNopRSCloser_SetFirstFakeChunk(t *testing.T) {
asserts := assert.New(t)
rsc := NopRSCloser{
status: &rscStatus{},
}
rsc.SetFirstFakeChunk()
asserts.True(rsc.status.IgnoreFirst)
rsc.SetContentLength(20)
asserts.EqualValues(20, rsc.status.Size)
}
Write Preview
Loading…
Cancel
Save