gh_mirrors/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-01-31 14:08:06 +08:00
Code

feat: 增加阿里云盘备份账号 (#6961)

Browse Source
This commit is contained in:
ssongliu 2024-11-06 17:37:14 +08:00 committed by GitHub
parent b721050c4d
commit 9e82e2837d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 1014 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
agent/constant/backup.go
View File

@ -14,6 +14,7 @@ const (
WebDAV = "WebDAV"
Local = "LOCAL"
UPYUN = "UPYUN"
ALIYUN = "ALIYUN"
OneDriveRedirectURI = "http://localhost/login/authorized"
)

1
agent/go.mod
View File

@ -19,6 +19,7 @@ require (
github.com/go-gormigrate/gormigrate/v2 v2.1.2
github.com/go-playground/validator/v10 v10.22.0
github.com/go-redis/redis v6.15.9+incompatible
github.com/go-resty/resty/v2 v2.11.0
github.com/go-sql-driver/mysql v1.8.1
github.com/goh-chunlin/go-onedrive v1.1.1
github.com/google/uuid v1.6.0

17
agent/go.sum
View File

@ -279,6 +279,8 @@ github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4
github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8=
github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
@ -830,6 +832,7 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -869,6 +872,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -906,6 +910,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -926,6 +933,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -977,12 +985,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -993,11 +1005,15 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -1045,6 +1061,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

504
agent/utils/cloud_storage/client/ali.go Normal file
View File

@ -0,0 +1,504 @@
package client
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path"
"strings"
"github.com/go-resty/resty/v2"
)
type aliClient struct {
token string
driveID string
}
func NewALIClient(vars map[string]interface{}) (*aliClient, error) {
refresh_token := loadParamFromVars("refresh_token", vars)
drive_id := loadParamFromVars("drive_id", vars)
token, err := loadToken(refresh_token)
if err != nil {
return nil, err
}
return &aliClient{token: token, driveID: drive_id}, nil
}
func (a aliClient) ListBuckets() ([]interface{}, error) {
return nil, nil
}
func (a aliClient) Exist(pathItem string) (bool, error) {
pathItem = path.Join("root", pathItem)
if _, err := a.loadFileWithName(pathItem); err != nil {
return false, err
}
return true, nil
}
func (a aliClient) Size(pathItem string) (int64, error) {
pathItem = path.Join("root", pathItem)
fileInfo, err := a.loadFileWithName(pathItem)
if err != nil {
return 0, err
}
return int64(fileInfo.Size), nil
}
func (a aliClient) Delete(pathItem string) (bool, error) {
pathItem = path.Join("root", pathItem)
fileInfo, err := a.loadFileWithName(pathItem)
if err != nil {
return false, err
}
client := resty.New()
data := map[string]interface{}{
"drive_id": a.driveID,
"file_id": fileInfo.FileID,
}
url := "https://api.alipan.com/v2/file/delete"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return false, err
}
if resp.StatusCode() != 204 {
return false, fmt.Errorf("delete file %s failed, err: %v", pathItem, string(resp.Body()))
}
return true, nil
}
func (a aliClient) Upload(src, target string) (bool, error) {
target = path.Join("/root", target)
parentID := "root"
var err error
if path.Dir(target) != "/root" {
parentID, err = a.mkdirWithPath(path.Dir(target))
if err != nil {
return false, err
}
}
file, err := os.Open(src)
if err != nil {
return false, err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return false, err
}
data := map[string]interface{}{
"drive_id": a.driveID,
"part_info_list": makePartInfoList(fileInfo.Size()),
"parent_file_id": parentID,
"name": path.Base(src),
"type": "file",
"size": fileInfo.Size(),
"check_name_mode": "auto_rename",
}
client := resty.New()
url := "https://api.alipan.com/v2/file/create"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return false, err
}
var createResp createFileResp
if err := json.Unmarshal(resp.Body(), &createResp); err != nil {
return false, err
}
for _, part := range createResp.PartInfoList {
err = a.uploadPart(part.UploadURL, io.LimitReader(file, 1024*1024*1024))
if err != nil {
return false, err
}
}
if err := a.completeUpload(createResp.UploadID, createResp.FileID); err != nil {
return false, err
}
return true, nil
}
func (a aliClient) Download(src, target string) (bool, error) {
src = path.Join("/root", src)
fileInfo, err := a.loadFileWithName(src)
if err != nil {
return false, err
}
client := resty.New()
if fileInfo.Size > 100*1024*1024 {
return false, fmt.Errorf("The translation file %s exceeds 100MB, please download it through the client.", src)
}
data := map[string]interface{}{
"drive_id": a.driveID,
"file_id": fileInfo.FileID,
}
url := "https://api.aliyundrive.com/v2/file/get_download_url"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return false, err
}
if resp.StatusCode() != 200 {
return false, fmt.Errorf("download file %s failed, err: %v", src, string(resp.Body()))
}
var respItem downloadResp
if err := json.Unmarshal(resp.Body(), &respItem); err != nil {
return false, err
}
if err := a.handleDownload(respItem.URL, target); err != nil {
return false, err
}
return true, nil
}
func (a *aliClient) ListObjects(src string) ([]string, error) {
if len(src) == 0 || src == "root" || src == "/root" {
src = "root"
} else {
src = path.Join("/root", src)
}
fileInfos, err := a.loadDirWithPath(src)
if err != nil {
return nil, err
}
var names []string
for _, item := range fileInfos {
names = append(names, item.Name)
}
return names, nil
}
func (a aliClient) loadFileWithName(pathItem string) (fileInfo, error) {
pathItems := strings.Split(pathItem, "/")
var (
fileInfos []fileInfo
err error
)
parentID := "root"
for i := 0; i < len(pathItems); i++ {
if len(pathItems[i]) == 0 {
continue
}
fileInfos, err = a.loadFileWithParentID(parentID)
if err != nil {
return fileInfo{}, err
}
isEnd := false
if i == len(pathItems)-2 {
isEnd = true
}
exist := false
for _, item := range fileInfos {
if item.Name == pathItems[i+1] {
if isEnd {
return item, nil
} else {
parentID = item.FileID
exist = true
}
}
}
if !exist {
return fileInfo{}, errors.New("no such file or dir")
}
}
return fileInfo{}, errors.New("no such file or dir")
}
func (a aliClient) loadDirWithPath(path string) ([]fileInfo, error) {
pathItems := strings.Split(path, "/")
var (
fileInfos []fileInfo
err error
)
parentID := "root"
for i := 0; i < len(pathItems); i++ {
if len(pathItems[i]) == 0 {
continue
}
fileInfos, err = a.loadFileWithParentID(parentID)
if err != nil {
return fileInfos, err
}
if i == len(pathItems)-1 {
return fileInfos, nil
}
exist := false
for _, item := range fileInfos {
if item.Name == pathItems[i+1] {
parentID = item.FileID
exist = true
}
}
if !exist {
return nil, errors.New("no such file or dir")
}
}
return fileInfos, errors.New("no such file or dir")
}
func (a aliClient) loadFileWithParentID(parentID string) ([]fileInfo, error) {
client := resty.New()
data := map[string]interface{}{
"drive_id": a.driveID,
"fields": "*",
"limit": 100,
"parent_file_id": parentID,
}
url := "https://api.aliyundrive.com/adrive/v3/file/list"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return nil, err
}
if resp.StatusCode() != 200 {
return nil, fmt.Errorf("load file list failed, code: %v, err: %v", resp.StatusCode(), string(resp.Body()))
}
var fileResp fileResp
if err := json.Unmarshal(resp.Body(), &fileResp); err != nil {
return nil, err
}
return fileResp.Items, nil
}
func (a aliClient) mkdirWithPath(target string) (string, error) {
pathItems := strings.Split(target, "/")
var (
fileInfos []fileInfo
err error
)
parentID := "root"
for i := 0; i < len(pathItems); i++ {
if len(pathItems[i]) == 0 {
continue
}
fileInfos, err = a.loadFileWithParentID(parentID)
if err != nil {
return "", err
}
isEnd := false
if i == len(pathItems)-2 {
isEnd = true
}
exist := false
for _, item := range fileInfos {
if item.Name == pathItems[i+1] {
parentID = item.FileID
if isEnd {
return item.FileID, nil
} else {
exist = true
}
}
}
if !exist {
parentID, err = a.mkdir(parentID, pathItems[i+1])
if err != nil {
return parentID, err
}
if isEnd {
return parentID, nil
}
}
}
return "", errors.New("mkdir failed.")
}
func (a aliClient) mkdir(parentID, name string) (string, error) {
client := resty.New()
data := map[string]interface{}{
"drive_id": a.driveID,
"name": name,
"type": "folder",
"limit": 100,
"parent_file_id": parentID,
}
url := "https://api.aliyundrive.com/adrive/v2/file/createWithFolders"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return "", err
}
if resp.StatusCode() != 201 {
return "", fmt.Errorf("mkdir %s failed, code: %v, err: %v", name, resp.StatusCode(), string(resp.Body()))
}
var mkdirResp mkdirResp
if err := json.Unmarshal(resp.Body(), &mkdirResp); err != nil {
return "", err
}
return mkdirResp.FileID, nil
}
type fileResp struct {
Items []fileInfo `json:"items"`
}
type fileInfo struct {
FileID string `json:"file_id"`
Name string `json:"name"`
Size int `json:"size"`
}
type mkdirResp struct {
FileID string `json:"file_id"`
}
type partInfo struct {
PartNumber int `json:"part_number"`
UploadURL string `json:"upload_url"`
InternalUploadURL string `json:"internal_upload_url"`
ContentType string `json:"content_type"`
}
func makePartInfoList(size int64) []*partInfo {
var res []*partInfo
maxPartSize := int64(1024 * 1024 * 1024)
partInfoNum := int(size / maxPartSize)
if size%maxPartSize > 0 {
partInfoNum += 1
}
for i := 0; i < partInfoNum; i++ {
res = append(res, &partInfo{PartNumber: i + 1})
}
return res
}
type createFileResp struct {
Type string `json:"type"`
RapidUpload bool `json:"rapid_upload"`
DomainId string `json:"domain_id"`
DriveId string `json:"drive_id"`
FileName string `json:"file_name"`
EncryptMode string `json:"encrypt_mode"`
Location string `json:"location"`
UploadID string `json:"upload_id"`
FileID string `json:"file_id"`
PartInfoList []*partInfo `json:"part_info_list,omitempty"`
}
func (a aliClient) uploadPart(uri string, reader io.Reader) error {
req, err := http.NewRequest(http.MethodPut, uri, reader)
if err != nil {
return err
}
client := &http.Client{}
response, err := client.Do(req)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("handle upload park file with url failed, code: %v", response.StatusCode)
}
return nil
}
type downloadResp struct {
URL string `json:"url"`
}
func (a aliClient) handleDownload(uri string, target string) error {
req, err := http.NewRequest(http.MethodGet, uri, nil)
if err != nil {
return err
}
req.Header.Add("Authorization", a.token)
req.Header.Add("origin", "https://www.aliyundrive.com")
req.Header.Add("referer", "https://www.aliyundrive.com/")
client := &http.Client{}
response, err := client.Do(req)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("handle download with url failed, code: %v", response.StatusCode)
}
if _, err := os.Stat(path.Dir(target)); err != nil {
_ = os.MkdirAll(path.Dir(target), os.ModePerm)
}
out, err := os.Create(target)
if err != nil {
return err
}
defer out.Close()
if _, err = io.Copy(out, response.Body); err != nil {
return err
}
return nil
}
func (a *aliClient) completeUpload(uploadID, fileID string) error {
client := resty.New()
data := map[string]interface{}{
"drive_id": a.driveID,
"upload_id": uploadID,
"file_id": fileID,
}
url := "https://api.aliyundrive.com/v2/file/complete"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return err
}
if resp.StatusCode() != 200 {
return fmt.Errorf("complete upload failed, err: %v", string(resp.Body()))
}
return nil
}
type tokenResp struct {
AccessToken string `json:"access_token"`
}
func loadToken(refresh_token string) (string, error) {
client := resty.New()
data := map[string]interface{}{
"grant_type": "refresh_token",
"refresh_token": refresh_token,
}
url := "https://api.aliyundrive.com/token/refresh"
resp, err := client.R().
SetBody(data).
Post(url)
if err != nil {
return "", fmt.Errorf("load account token failed, err: %v", err)
}
if resp.StatusCode() != 200 {
return "", fmt.Errorf("load account token failed, code: %v", resp.StatusCode())
}
var respItem tokenResp
if err := json.Unmarshal(resp.Body(), &respItem); err != nil {
return "", err
}
return respItem.AccessToken, nil
}

2
agent/utils/cloud_storage/cloud_storage_client.go
View File

@ -38,6 +38,8 @@ func NewCloudStorageClient(backupType string, vars map[string]interface{}) (Clou
return client.NewOneDriveClient(vars)
case constant.UPYUN:
return client.NewUpClient(vars)
case constant.ALIYUN:
return client.NewALIClient(vars)
default:
return nil, constant.ErrNotSupportType
}

4
core/app/model/backup.go
View File

@ -4,8 +4,8 @@ import "time"
type BackupAccount struct {
BaseModel
Name string `gorm:"unique;not null" json:"name"`
Type string `gorm:"unique;not null" json:"type"`
Name string `gorm:"not null" json:"name"`
Type string `gorm:"not null" json:"type"`
Bucket string `json:"bucket"`
AccessKey string `json:"accessKey"`
Credential string `json:"credential"`

103
core/app/service/backup.go
View File

@ -156,12 +156,13 @@ func (u *BackupService) SearchWithPage(req dto.SearchPageWithType) (int64, inter
item.Credential = base64.StdEncoding.EncodeToString([]byte(item.Credential))
}
if account.Type == constant.OneDrive {
if account.Type == constant.OneDrive || account.Type == constant.ALIYUN {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(item.Vars), &varMap); err != nil {
continue
}
delete(varMap, "refresh_token")
delete(varMap, "drive_id")
itemVars, _ := json.Marshal(varMap)
item.Vars = string(itemVars)
}
@ -500,33 +501,75 @@ func StartRefreshOneDriveToken(backup *model.BackupAccount) error {
}
func (u *BackupService) Run() {
var backupItem model.BackupAccount
_ = global.DB.Where("`type` = ?", "OneDrive").First(&backupItem)
if backupItem.ID == 0 {
return
}
global.LOG.Info("start to refresh token of OneDrive ...")
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil {
global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err)
return
}
refreshToken, err := client.RefreshToken("refresh_token", "refreshToken", varMap)
varMap["refresh_status"] = constant.StatusSuccess
varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
if err != nil {
varMap["refresh_status"] = constant.StatusFailed
varMap["refresh_msg"] = err.Error()
global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err)
return
}
varMap["refresh_token"] = refreshToken
varsItem, _ := json.Marshal(varMap)
_ = global.DB.Model(&model.BackupAccount{}).
Where("id = ?", backupItem.ID).
Updates(map[string]interface{}{
"vars": varsItem,
}).Error
global.LOG.Info("Successfully refreshed OneDrive token.")
refreshOneDrive()
refreshALIYUN()
}
func refreshOneDrive() {
var backups []model.BackupAccount
_ = global.DB.Where("`type` = ?", "OneDrive").Find(&backups)
for _, backupItem := range backups {
if backupItem.ID == 0 {
return
}
global.LOG.Infof("start to refresh token of OneDrive %s ...", backupItem.Name)
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil {
global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err)
return
}
refreshToken, err := client.RefreshToken("refresh_token", "refreshToken", varMap)
varMap["refresh_status"] = constant.StatusSuccess
varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
if err != nil {
varMap["refresh_status"] = constant.StatusFailed
varMap["refresh_msg"] = err.Error()
global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err)
return
}
varMap["refresh_token"] = refreshToken
varsItem, _ := json.Marshal(varMap)
_ = global.DB.Model(&model.BackupAccount{}).
Where("id = ?", backupItem.ID).
Updates(map[string]interface{}{
"vars": varsItem,
}).Error
global.LOG.Info("Successfully refreshed OneDrive token.")
}
}
func refreshALIYUN() {
var backups []model.BackupAccount
_ = global.DB.Where("`type` = ?", "ALIYUN").Find(&backups)
for _, backupItem := range backups {
global.LOG.Infof("start to refresh token of ALIYUN %s ...", backupItem.Name)
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil {
global.LOG.Errorf("Failed to refresh ALIYUN token, please retry, err: %v", err)
return
}
if _, ok := varMap["refresh_token"]; !ok {
global.LOG.Error("no such refresh token find in db")
return
}
refreshToken, err := client.RefreshALIToken(fmt.Sprintf("%v", varMap["refresh_token"]))
varMap["refresh_status"] = constant.StatusSuccess
varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
if err != nil {
varMap["refresh_status"] = constant.StatusFailed
varMap["refresh_msg"] = err.Error()
global.LOG.Errorf("Failed to refresh ALIYUN token, please retry, err: %v", err)
return
}
varMap["refresh_token"] = refreshToken
varsItem, _ := json.Marshal(varMap)
_ = global.DB.Model(&model.BackupAccount{}).
Where("id = ?", backupItem.ID).
Updates(map[string]interface{}{
"vars": varsItem,
}).Error
global.LOG.Info("Successfully refreshed ALIYUN token.")
}
}

1
core/constant/common.go
View File

@ -29,5 +29,6 @@ const (
WebDAV = "WebDAV"
Local = "LOCAL"
UPYUN = "UPYUN"
ALIYUN = "ALIYUN"
OneDriveRedirectURI = "http://localhost/login/authorized"
)

1
core/init/migration/migrate.go
View File

@ -16,6 +16,7 @@ func Init() {
migrations.InitHost,
migrations.InitTerminalSetting,
migrations.InitAppLauncher,
migrations.InitBackup,
})
if err := m.Migrate(); err != nil {
global.LOG.Error(err)

7
core/init/migration/migrations/init.go
View File

@ -240,3 +240,10 @@ var InitAppLauncher = &gormigrate.Migration{
return tx.AutoMigrate(&model.AppLauncher{})
},
}
var InitBackup = &gormigrate.Migration{
ID: "20241107-init-backup",
Migrate: func(tx *gorm.DB) error {
return tx.AutoMigrate(&model.BackupAccount{})
},
}

304
core/utils/cloud_storage/client/ali.go Normal file
View File

@ -0,0 +1,304 @@
package client
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path"
"strings"
"github.com/go-resty/resty/v2"
)
type aliClient struct {
token string
driveID string
}
func NewALIClient(vars map[string]interface{}) (*aliClient, error) {
refresh_token := loadParamFromVars("refresh_token", vars)
drive_id := loadParamFromVars("drive_id", vars)
token, err := RefreshALIToken(refresh_token)
if err != nil {
return nil, err
}
return &aliClient{token: token, driveID: drive_id}, nil
}
func (a aliClient) ListBuckets() ([]interface{}, error) {
return nil, nil
}
func (a aliClient) Upload(src, target string) (bool, error) {
target = path.Join("/root", target)
parentID := "root"
var err error
if path.Dir(target) != "/root" {
parentID, err = a.mkdirWithPath(path.Dir(target))
if err != nil {
return false, err
}
}
file, err := os.Open(src)
if err != nil {
return false, err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return false, err
}
data := map[string]interface{}{
"drive_id": a.driveID,
"part_info_list": makePartInfoList(fileInfo.Size()),
"parent_file_id": parentID,
"name": path.Base(src),
"type": "file",
"size": fileInfo.Size(),
"check_name_mode": "auto_rename",
}
client := resty.New()
url := "https://api.alipan.com/v2/file/create"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return false, err
}
var createResp createFileResp
if err := json.Unmarshal(resp.Body(), &createResp); err != nil {
return false, err
}
for _, part := range createResp.PartInfoList {
err = a.uploadPart(part.UploadURL, io.LimitReader(file, 1024*1024*1024))
if err != nil {
return false, err
}
}
if err := a.completeUpload(createResp.UploadID, createResp.FileID); err != nil {
return false, err
}
return true, nil
}
func (a aliClient) loadFileWithParentID(parentID string) ([]fileInfo, error) {
client := resty.New()
data := map[string]interface{}{
"drive_id": a.driveID,
"fields": "*",
"limit": 100,
"parent_file_id": parentID,
}
url := "https://api.alipan.com/v2/file/list"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return nil, err
}
if resp.StatusCode() != 200 {
return nil, fmt.Errorf("load file list failed, code: %v, err: %v", resp.StatusCode(), string(resp.Body()))
}
var fileResp fileResp
if err := json.Unmarshal(resp.Body(), &fileResp); err != nil {
return nil, err
}
return fileResp.Items, nil
}
func (a aliClient) mkdirWithPath(target string) (string, error) {
pathItems := strings.Split(target, "/")
var (
fileInfos []fileInfo
err error
)
parentID := "root"
for i := 0; i < len(pathItems); i++ {
if len(pathItems[i]) == 0 {
continue
}
fileInfos, err = a.loadFileWithParentID(parentID)
if err != nil {
return "", err
}
isEnd := false
if i == len(pathItems)-2 {
isEnd = true
}
exist := false
for _, item := range fileInfos {
if item.Name == pathItems[i+1] {
parentID = item.FileID
if isEnd {
return item.FileID, nil
} else {
exist = true
}
}
}
if !exist {
parentID, err = a.mkdir(parentID, pathItems[i+1])
if err != nil {
return parentID, err
}
if isEnd {
return parentID, nil
}
}
}
return "", errors.New("mkdir failed.")
}
func (a aliClient) mkdir(parentID, name string) (string, error) {
client := resty.New()
data := map[string]interface{}{
"drive_id": a.driveID,
"name": name,
"type": "folder",
"limit": 100,
"parent_file_id": parentID,
}
url := "https://api.alipan.com/v2/file/create"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return "", err
}
if resp.StatusCode() != 201 {
return "", fmt.Errorf("mkdir %s failed, code: %v, err: %v", name, resp.StatusCode(), string(resp.Body()))
}
var mkdirResp mkdirResp
if err := json.Unmarshal(resp.Body(), &mkdirResp); err != nil {
return "", err
}
return mkdirResp.FileID, nil
}
type fileResp struct {
Items []fileInfo `json:"items"`
}
type fileInfo struct {
FileID string `json:"file_id"`
Name string `json:"name"`
Size int `json:"size"`
}
type mkdirResp struct {
FileID string `json:"file_id"`
}
type partInfo struct {
PartNumber int `json:"part_number"`
UploadURL string `json:"upload_url"`
InternalUploadURL string `json:"internal_upload_url"`
ContentType string `json:"content_type"`
}
func makePartInfoList(size int64) []*partInfo {
var res []*partInfo
maxPartSize := int64(1024 * 1024 * 1024)
partInfoNum := int(size / maxPartSize)
if size%maxPartSize > 0 {
partInfoNum += 1
}
for i := 0; i < partInfoNum; i++ {
res = append(res, &partInfo{PartNumber: i + 1})
}
return res
}
type createFileResp struct {
Type string `json:"type"`
RapidUpload bool `json:"rapid_upload"`
DomainId string `json:"domain_id"`
DriveId string `json:"drive_id"`
FileName string `json:"file_name"`
EncryptMode string `json:"encrypt_mode"`
Location string `json:"location"`
UploadID string `json:"upload_id"`
FileID string `json:"file_id"`
PartInfoList []*partInfo `json:"part_info_list,omitempty"`
}
func (a aliClient) uploadPart(uri string, reader io.Reader) error {
req, err := http.NewRequest(http.MethodPut, uri, reader)
if err != nil {
return err
}
client := &http.Client{}
response, err := client.Do(req)
if err != nil {
return err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return fmt.Errorf("handle upload park file with url failed, code: %v", response.StatusCode)
}
return nil
}
func (a *aliClient) completeUpload(uploadID, fileID string) error {
client := resty.New()
data := map[string]interface{}{
"drive_id": a.driveID,
"upload_id": uploadID,
"file_id": fileID,
}
url := "https://api.aliyundrive.com/v2/file/complete"
resp, err := client.R().
SetHeader("Authorization", a.token).
SetBody(data).
Post(url)
if err != nil {
return err
}
if resp.StatusCode() != 200 {
return fmt.Errorf("complete upload failed, err: %v", string(resp.Body()))
}
return nil
}
type tokenResp struct {
AccessToken string `json:"access_token"`
}
func RefreshALIToken(refresh_token string) (string, error) {
client := resty.New()
data := map[string]interface{}{
"grant_type": "refresh_token",
"refresh_token": refresh_token,
}
url := "https://api.aliyundrive.com/token/refresh"
resp, err := client.R().
SetBody(data).
Post(url)
if err != nil {
return "", fmt.Errorf("load account token failed, err: %v", err)
}
if resp.StatusCode() != 200 {
return "", fmt.Errorf("load account token failed, code: %v", resp.StatusCode())
}
var respItem tokenResp
if err := json.Unmarshal(resp.Body(), &respItem); err != nil {
return "", err
}
return respItem.AccessToken, nil
}

32
core/utils/cloud_storage/client/onedrive.go
View File

@ -80,38 +80,6 @@ func (o oneDriveClient) Upload(src, target string) (bool, error) {
return isOk, err
}
func (o oneDriveClient) Download(src, target string) (bool, error) {
src = "/" + strings.TrimPrefix(src, "/")
req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil)
if err != nil {
return false, fmt.Errorf("new request for file id failed, err: %v", err)
}
var driveItem *odsdk.DriveItem
if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil {
return false, fmt.Errorf("do request for file id failed, err: %v", err)
}
resp, err := http.Get(driveItem.DownloadURL)
if err != nil {
return false, err
}
defer resp.Body.Close()
out, err := os.Create(target)
if err != nil {
return false, err
}
defer out.Close()
buffer := make([]byte, 2*1024*1024)
_, err = io.CopyBuffer(out, resp.Body, buffer)
if err != nil {
return false, err
}
return true, nil
}
func (o *oneDriveClient) loadIDByPath(path string) (string, error) {
pathItem := "root:" + path
if path == "/" {

63
core/utils/cloud_storage/client/oss.go
View File

@ -1,8 +1,6 @@
package client
import (
"fmt"
osssdk "github.com/aliyun/aliyun-oss-go-sdk/oss"
)
@ -41,40 +39,6 @@ func (o ossClient) ListBuckets() ([]interface{}, error) {
return result, err
}
func (o ossClient) Exist(path string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return false, err
}
return bucket.IsObjectExist(path)
}
func (o ossClient) Size(path string) (int64, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return 0, err
}
lor, err := bucket.ListObjectsV2(osssdk.Prefix(path))
if err != nil {
return 0, err
}
if len(lor.Objects) == 0 {
return 0, fmt.Errorf("no such file %s", path)
}
return lor.Objects[0].Size, nil
}
func (o ossClient) Delete(path string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return false, err
}
if err := bucket.DeleteObject(path); err != nil {
return false, err
}
return true, nil
}
func (o ossClient) Upload(src, target string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
@ -89,30 +53,3 @@ func (o ossClient) Upload(src, target string) (bool, error) {
}
return true, nil
}
func (o ossClient) Download(src, target string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return false, err
}
if err := bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, "")); err != nil {
return false, err
}
return true, nil
}
func (o *ossClient) ListObjects(prefix string) ([]string, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return nil, err
}
lor, err := bucket.ListObjectsV2(osssdk.Prefix(prefix))
if err != nil {
return nil, err
}
var result []string
for _, obj := range lor.Objects {
result = append(result, obj.Key)
}
return result, nil
}

84
core/utils/cloud_storage/client/s3.go
View File

@ -4,7 +4,6 @@ import (
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
@ -53,49 +52,6 @@ func (s s3Client) ListBuckets() ([]interface{}, error) {
return result, nil
}
func (s s3Client) Exist(path string) (bool, error) {
svc := s3.New(&s.Sess)
if _, err := svc.HeadObject(&s3.HeadObjectInput{
Bucket: &s.bucket,
Key: &path,
}); err != nil {
if aerr, ok := err.(awserr.RequestFailure); ok {
if aerr.StatusCode() == 404 {
return false, nil
}
} else {
return false, aerr
}
}
return true, nil
}
func (s *s3Client) Size(path string) (int64, error) {
svc := s3.New(&s.Sess)
file, err := svc.GetObject(&s3.GetObjectInput{
Bucket: &s.bucket,
Key: &path,
})
if err != nil {
return 0, err
}
return *file.ContentLength, nil
}
func (s s3Client) Delete(path string) (bool, error) {
svc := s3.New(&s.Sess)
if _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(path)}); err != nil {
return false, err
}
if err := svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
}); err != nil {
return false, err
}
return true, nil
}
func (s s3Client) Upload(src, target string) (bool, error) {
fileInfo, err := os.Stat(src)
if err != nil {
@ -121,43 +77,3 @@ func (s s3Client) Upload(src, target string) (bool, error) {
}
return true, nil
}
func (s s3Client) Download(src, target string) (bool, error) {
if _, err := os.Stat(target); err != nil {
if os.IsNotExist(err) {
os.Remove(target)
} else {
return false, err
}
}
file, err := os.Create(target)
if err != nil {
return false, err
}
defer file.Close()
downloader := s3manager.NewDownloader(&s.Sess)
if _, err = downloader.Download(file, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(src),
}); err != nil {
os.Remove(target)
return false, err
}
return true, nil
}
func (s *s3Client) ListObjects(prefix string) ([]string, error) {
svc := s3.New(&s.Sess)
var result []string
outputs, err := svc.ListObjects(&s3.ListObjectsInput{
Bucket: &s.bucket,
Prefix: &prefix,
})
if err != nil {
return result, err
}
for _, item := range outputs.Contents {
result = append(result, *item.Key)
}
return result, nil
}

18
core/utils/cloud_storage/client/sftp.go
View File

@ -108,21 +108,3 @@ func (s sftpClient) ListBuckets() ([]interface{}, error) {
var result []interface{}
return result, nil
}
func (s sftpClient) Delete(filePath string) error {
sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return err
}
defer client.Close()
defer sshClient.Close()
if err := client.Remove(filePath); err != nil {
return err
}
return nil
}

59
core/utils/cloud_storage/client/webdav.go
View File

@ -3,7 +3,6 @@ package client
import (
"crypto/tls"
"fmt"
"io"
"net/http"
"os"
"path"
@ -61,61 +60,3 @@ func (s webDAVClient) ListBuckets() ([]interface{}, error) {
var result []interface{}
return result, nil
}
func (s webDAVClient) Download(src, target string) (bool, error) {
srcPath := path.Join(s.Bucket, src)
info, err := s.client.Stat(srcPath)
if err != nil {
return false, err
}
targetStat, err := os.Stat(target)
if err == nil {
if info.Size() == targetStat.Size() {
return true, nil
}
}
file, err := os.Create(target)
if err != nil {
return false, err
}
defer file.Close()
reader, _ := s.client.ReadStream(srcPath)
if _, err := io.Copy(file, reader); err != nil {
return false, err
}
return true, err
}
func (s webDAVClient) Exist(pathItem string) (bool, error) {
if _, err := s.client.Stat(path.Join(s.Bucket, pathItem)); err != nil {
return false, err
}
return true, nil
}
func (s webDAVClient) Size(pathItem string) (int64, error) {
file, err := s.client.Stat(path.Join(s.Bucket, pathItem))
if err != nil {
return 0, err
}
return file.Size(), nil
}
func (s webDAVClient) Delete(pathItem string) (bool, error) {
if err := s.client.Remove(path.Join(s.Bucket, pathItem)); err != nil {
return false, err
}
return true, nil
}
func (s webDAVClient) ListObjects(prefix string) ([]string, error) {
files, err := s.client.ReadDir(path.Join(s.Bucket, prefix))
if err != nil {
return nil, err
}
var result []string
for _, file := range files {
result = append(result, file.Name())
}
return result, nil
}

2
core/utils/cloud_storage/cloud_storage_client.go
View File

@ -32,6 +32,8 @@ func NewCloudStorageClient(backupType string, vars map[string]interface{}) (Clou
return client.NewOneDriveClient(vars)
case constant.UPYUN:
return client.NewUpClient(vars)
case constant.ALIYUN:
return client.NewALIClient(vars)
default:
return nil, constant.ErrNotSupportType
}

1
frontend/src/api/interface/backup.ts
View File

@ -50,6 +50,7 @@ export namespace Backup {
downloadAccountID: number;
fileDir: string;
fileName: string;
size: number;
}
export interface ForBucket {
type: string;

8
frontend/src/lang/modules/en.ts
View File

@ -1452,6 +1452,14 @@ const message = {
WebDAV: 'WebDAV',
WebDAVAlist: 'WebDAV connect Alist can refer to the official documentation',
UPYUN: 'UPYUN',
ALIYUN: 'Aliyun Drive',
ALIYUNHelper:
'The current maximum limit for non-client downloads on Aliyun Drive is 100 MB. Exceeding this limit requires downloading through the client.',
ALIYUNRecover:
'The current maximum limit for non-client downloads on Aliyun Drive is 100 MB. Exceeding this limit requires downloading through the client to the local device, then synchronizing the snapshot for recovery.',
analysis: 'Analysis',
analysisHelper:
'Paste the entire token content to automatically parse the required parts. For specific operations, please refer to the official documentation.',
serviceName: 'Service Name',
operator: 'Operator',
OneDrive: 'Microsoft OneDrive',

6
frontend/src/lang/modules/tw.ts
View File

@ -1365,6 +1365,12 @@ const message = {
WebDAV: 'WebDAV',
WebDAVAlist: 'WebDAV 連接 Alist 可參考官方文檔',
UPYUN: '又拍雲',
ALIYUN: '阿里雲盤',
ALIYUNHelper: '當前阿里雲盤非客戶端下載最大限制為 100 MB，超過限制需要通過客戶端下載',
ALIYUNRecover:
'當前阿里雲盤非客戶端下載最大限制為 100 MB，超過限制需要通過客戶端下載到本地後，同步快照進行恢復',
analysis: '解析',
analysisHelper: '粘貼整個 token 內容，自動解析所需部分，具體操作可參考官方文檔',
serviceName: '服務名稱',
operator: '操作員',
OneDrive: '微軟 OneDrive',

6
frontend/src/lang/modules/zh.ts
View File

@ -1367,6 +1367,12 @@ const message = {
WebDAV: 'WebDAV',
WebDAVAlist: 'WebDAV 连接 Alist 可参考官方文档',
UPYUN: '又拍云',
ALIYUN: '阿里云盘',
ALIYUNHelper: '当前阿里云盘非客户端下载最大限制为 100 MB，超过限制需要通过客户端下载',
ALIYUNRecover:
'当前阿里云盘非客户端下载最大限制为 100 MB，超过限制需要通过客户端下载到本地后，同步快照进行恢复',
analysis: '解析',
analysisHelper: '粘贴整个 token 内容，自动解析所需部分，具体操作可参考官方文档',
serviceName: '服务名称',
operator: '操作员',
OneDrive: '微软 OneDrive',

5
frontend/src/views/cronjob/backup/index.vue
View File

@ -53,6 +53,7 @@ import { computeSize, dateFormat, downloadFile } from '@/utils/util';
import i18n from '@/lang';
import { downloadBackupRecord, searchBackupRecordsByCronjob } from '@/api/modules/backup';
import { Backup } from '@/api/interface/backup';
import { MsgError } from '@/utils/message';
const selects = ref<any>([]);
const loading = ref();
@ -102,6 +103,10 @@ const search = async () => {
};
const onDownload = async (row: Backup.RecordInfo) => {
if (row.accountType === 'ALIYUN' && row.size < 100 * 1024 * 1024) {
MsgError(i18n.global.t('setting.ALIYUNHelper'));
return;
}
let params = {
downloadAccountID: row.downloadAccountID,
fileDir: row.fileDir,

60
frontend/src/views/setting/backup-account/operate/index.vue
View File

@ -17,7 +17,9 @@
<el-option :label="$t('setting.SFTP')" value="SFTP"></el-option>
<el-option :label="$t('setting.WebDAV')" value="WebDAV"></el-option>
<el-option :label="$t('setting.UPYUN')" value="UPYUN"></el-option>
<el-option :label="$t('setting.ALIYUN')" value="ALIYUN"></el-option>
</el-select>
<span v-if="isALIYUNYUN()" class="input-help">{{ $t('setting.ALIYUNHelper') }}</span>
</el-form-item>
<el-form-item v-if="hasAccessKey()" label="Access Key ID" prop="accessKey" :rules="Rules.requiredInput">
<el-input v-model.trim="dialogData.rowData!.accessKey" />
@ -95,10 +97,7 @@
<el-input type="password" clearable show-password v-model.trim="dialogData.rowData!.credential" />
</el-form-item>
</div>
<el-form-item
v-if="dialogData.rowData!.type !== 'LOCAL' && dialogData.rowData!.type !== 'OneDrive'"
prop="rememberAuth"
>
<el-form-item v-if="hasRemember()" prop="rememberAuth">
<el-checkbox v-model="dialogData.rowData!.rememberAuth">
{{ $t('terminal.rememberPassword') }}
</el-checkbox>
@ -233,6 +232,40 @@
v-model.number="dialogData.rowData!.varsJson['timeout']"
></el-input-number>
</el-form-item>
<div v-if="isALIYUNYUN()">
<el-form-item label="Token" prop="varsJson.token">
<div class="!w-full">
<el-input
style="width: calc(100% - 80px)"
:rows="3"
type="textarea"
clearable
v-model.trim="dialogData.rowData!.varsJson['token']"
/>
<el-button class="append-button" @click="loadFromToken()">
{{ $t('setting.analysis') }}
</el-button>
<span class="input-help">
{{ $t('setting.analysisHelper') }}
<el-link
style="font-size: 12px; margin-left: 5px"
icon="Position"
@click="toDoc(true)"
type="primary"
>
{{ $t('firewall.quickJump') }}
</el-link>
</span>
</div>
</el-form-item>
<el-form-item label="Drive ID" prop="varsJson.drive_id" :rules="Rules.requiredInput">
<el-input v-model.trim="dialogData.rowData!.varsJson['drive_id']" />
</el-form-item>
<el-form-item label="Refresh Token" prop="varsJson.refresh_token" :rules="Rules.requiredInput">
<el-input v-model.trim="dialogData.rowData!.varsJson['refresh_token']" />
</el-form-item>
</div>
<div v-if="dialogData.rowData!.type === 'OneDrive'">
<el-form-item>
<el-radio-group v-model="dialogData.rowData!.varsJson['isCN']" @change="changeFrom">
@ -437,10 +470,26 @@ function callback(error: any) {
}
}
const loadFromToken = () => {
const obj = JSON.parse(dialogData.value.rowData!.varsJson['token']);
dialogData.value.rowData!.varsJson['drive_id'] = obj.default_drive_id;
dialogData.value.rowData!.varsJson['refresh_token'] = obj.refresh_token;
};
const hasRemember = () => {
return (
dialogData.value.rowData!.type !== 'LOCAL' &&
dialogData.value.rowData!.type !== 'OneDrive' &&
dialogData.value.rowData!.type !== 'ALIYUN'
);
};
const isUPYUN = () => {
let itemType = dialogData.value.rowData!.type;
return itemType === 'UPYUN';
};
const isALIYUNYUN = () => {
let itemType = dialogData.value.rowData!.type;
return itemType === 'ALIYUN';
};
const hasAccessKey = () => {
let itemType = dialogData.value.rowData!.type;
return itemType === 'COS' || itemType === 'KODO' || itemType === 'MINIO' || itemType === 'OSS' || itemType === 'S3';
@ -558,6 +607,9 @@ const onSubmit = async (formEl: FormInstance | undefined) => {
}
dialogData.value.rowData!.varsJson['endpointItem'] = undefined;
}
if (isALIYUNYUN()) {
dialogData.value.rowData!.varsJson['token'] = undefined;
}
dialogData.value.rowData.vars = JSON.stringify(dialogData.value.rowData!.varsJson);
loading.value = true;
if (dialogData.value.title === 'create') {

6
frontend/src/views/setting/snapshot/index.vue
View File

@ -196,7 +196,7 @@ import RecoverStatus from '@/views/setting/snapshot/status/index.vue';
import SnapshotImport from '@/views/setting/snapshot/import/index.vue';
import SnapshotCreate from '@/views/setting/snapshot/create/index.vue';
import SnapRecover from '@/views/setting/snapshot/recover/index.vue';
import { MsgSuccess } from '@/utils/message';
import { MsgError, MsgSuccess } from '@/utils/message';
import { loadOsInfo } from '@/api/modules/dashboard';
const loading = ref(false);
@ -291,6 +291,10 @@ const onChange = async (info: any) => {
};
const onRecover = async (row: any) => {
if (row.defaultDownload.indexOf('ALIYUN') !== -1 && row.size > 100 * 1024 * 1024) {
MsgError(i18n.global.t('setting.ALIYUNRecover'));
return;
}
loading.value = true;
await loadOsInfo()
.then((res) => {

7
go.mod
View File

@ -13,6 +13,7 @@ require (
github.com/glebarez/sqlite v1.11.0
github.com/go-gormigrate/gormigrate/v2 v2.1.2
github.com/go-playground/validator/v10 v10.22.0
github.com/go-resty/resty/v2 v2.15.3
github.com/goh-chunlin/go-onedrive v1.1.1
github.com/golang-jwt/jwt/v4 v4.5.0
github.com/google/uuid v1.6.0
@ -37,7 +38,7 @@ require (
github.com/tencentyun/cos-go-sdk-v5 v0.7.54
github.com/upyun/go-sdk v2.1.0+incompatible
github.com/xlzd/gotp v0.1.0
golang.org/x/crypto v0.24.0
golang.org/x/crypto v0.25.0
golang.org/x/oauth2 v0.18.0
golang.org/x/sys v0.22.0
golang.org/x/term v0.22.0
@ -119,9 +120,9 @@ require (
golang.org/x/arch v0.8.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/image v0.13.0 // indirect
golang.org/x/net v0.26.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/time v0.6.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.34.1 // indirect

14
go.sum
View File

@ -136,6 +136,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk=
github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao=
github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-resty/resty/v2 v2.15.3 h1:bqff+hcqAflpiF591hhJzNdkRsFhlB96CYfBwSFvql8=
github.com/go-resty/resty/v2 v2.15.3/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
@ -413,8 +415,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -487,8 +489,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -572,8 +574,8 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=