From 9e82e2837d78684516af8562b84aaa42be6db743 Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:37:14 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E4=BA=91=E7=9B=98=E5=A4=87=E4=BB=BD=E8=B4=A6=E5=8F=B7=20(#6961?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- agent/constant/backup.go | 1 + agent/go.mod | 1 + agent/go.sum | 17 + agent/utils/cloud_storage/client/ali.go | 504 ++++++++++++++++++ .../cloud_storage/cloud_storage_client.go | 2 + core/app/model/backup.go | 4 +- core/app/service/backup.go | 103 ++-- core/constant/common.go | 1 + core/init/migration/migrate.go | 1 + core/init/migration/migrations/init.go | 7 + core/utils/cloud_storage/client/ali.go | 304 +++++++++++ core/utils/cloud_storage/client/onedrive.go | 32 -- core/utils/cloud_storage/client/oss.go | 63 --- core/utils/cloud_storage/client/s3.go | 84 --- core/utils/cloud_storage/client/sftp.go | 18 - core/utils/cloud_storage/client/webdav.go | 59 -- .../cloud_storage/cloud_storage_client.go | 2 + frontend/src/api/interface/backup.ts | 1 + frontend/src/lang/modules/en.ts | 8 + frontend/src/lang/modules/tw.ts | 6 + frontend/src/lang/modules/zh.ts | 6 + frontend/src/views/cronjob/backup/index.vue | 5 + .../setting/backup-account/operate/index.vue | 60 ++- frontend/src/views/setting/snapshot/index.vue | 6 +- go.mod | 7 +- go.sum | 14 +- 26 files changed, 1014 insertions(+), 302 deletions(-) create mode 100644 agent/utils/cloud_storage/client/ali.go create mode 100644 core/utils/cloud_storage/client/ali.go diff --git a/agent/constant/backup.go b/agent/constant/backup.go index 8636226b1..a2d884148 100644 --- a/agent/constant/backup.go +++ b/agent/constant/backup.go @@ -14,6 +14,7 @@ const ( WebDAV = "WebDAV" Local = "LOCAL" UPYUN = "UPYUN" + ALIYUN = "ALIYUN" OneDriveRedirectURI = "http://localhost/login/authorized" ) diff --git a/agent/go.mod b/agent/go.mod index a1d586f74..b05a31d89 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -19,6 +19,7 @@ require ( github.com/go-gormigrate/gormigrate/v2 v2.1.2 github.com/go-playground/validator/v10 v10.22.0 github.com/go-redis/redis v6.15.9+incompatible + github.com/go-resty/resty/v2 v2.11.0 github.com/go-sql-driver/mysql v1.8.1 github.com/goh-chunlin/go-onedrive v1.1.1 github.com/google/uuid v1.6.0 diff --git a/agent/go.sum b/agent/go.sum index d5f32354c..edf91b53b 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -279,6 +279,8 @@ github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4 github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= @@ -830,6 +832,7 @@ golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -869,6 +872,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -906,6 +910,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -926,6 +933,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -977,12 +985,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -993,11 +1005,15 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1045,6 +1061,7 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/agent/utils/cloud_storage/client/ali.go b/agent/utils/cloud_storage/client/ali.go new file mode 100644 index 000000000..1b2fa6c22 --- /dev/null +++ b/agent/utils/cloud_storage/client/ali.go @@ -0,0 +1,504 @@ +package client + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + + "github.com/go-resty/resty/v2" +) + +type aliClient struct { + token string + driveID string +} + +func NewALIClient(vars map[string]interface{}) (*aliClient, error) { + refresh_token := loadParamFromVars("refresh_token", vars) + drive_id := loadParamFromVars("drive_id", vars) + + token, err := loadToken(refresh_token) + if err != nil { + return nil, err + } + return &aliClient{token: token, driveID: drive_id}, nil +} + +func (a aliClient) ListBuckets() ([]interface{}, error) { + return nil, nil +} + +func (a aliClient) Exist(pathItem string) (bool, error) { + pathItem = path.Join("root", pathItem) + if _, err := a.loadFileWithName(pathItem); err != nil { + return false, err + } + return true, nil +} + +func (a aliClient) Size(pathItem string) (int64, error) { + pathItem = path.Join("root", pathItem) + fileInfo, err := a.loadFileWithName(pathItem) + if err != nil { + return 0, err + } + return int64(fileInfo.Size), nil +} + +func (a aliClient) Delete(pathItem string) (bool, error) { + pathItem = path.Join("root", pathItem) + fileInfo, err := a.loadFileWithName(pathItem) + if err != nil { + return false, err + } + client := resty.New() + data := map[string]interface{}{ + "drive_id": a.driveID, + "file_id": fileInfo.FileID, + } + url := "https://api.alipan.com/v2/file/delete" + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return false, err + } + if resp.StatusCode() != 204 { + return false, fmt.Errorf("delete file %s failed, err: %v", pathItem, string(resp.Body())) + } + return true, nil +} + +func (a aliClient) Upload(src, target string) (bool, error) { + target = path.Join("/root", target) + parentID := "root" + var err error + if path.Dir(target) != "/root" { + parentID, err = a.mkdirWithPath(path.Dir(target)) + if err != nil { + return false, err + } + } + file, err := os.Open(src) + if err != nil { + return false, err + } + defer file.Close() + fileInfo, err := file.Stat() + if err != nil { + return false, err + } + data := map[string]interface{}{ + "drive_id": a.driveID, + "part_info_list": makePartInfoList(fileInfo.Size()), + "parent_file_id": parentID, + "name": path.Base(src), + "type": "file", + "size": fileInfo.Size(), + "check_name_mode": "auto_rename", + } + client := resty.New() + url := "https://api.alipan.com/v2/file/create" + + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return false, err + } + + var createResp createFileResp + if err := json.Unmarshal(resp.Body(), &createResp); err != nil { + return false, err + } + for _, part := range createResp.PartInfoList { + err = a.uploadPart(part.UploadURL, io.LimitReader(file, 1024*1024*1024)) + if err != nil { + return false, err + } + } + + if err := a.completeUpload(createResp.UploadID, createResp.FileID); err != nil { + return false, err + } + return true, nil +} + +func (a aliClient) Download(src, target string) (bool, error) { + src = path.Join("/root", src) + fileInfo, err := a.loadFileWithName(src) + if err != nil { + return false, err + } + client := resty.New() + if fileInfo.Size > 100*1024*1024 { + return false, fmt.Errorf("The translation file %s exceeds 100MB, please download it through the client.", src) + } + data := map[string]interface{}{ + "drive_id": a.driveID, + "file_id": fileInfo.FileID, + } + url := "https://api.aliyundrive.com/v2/file/get_download_url" + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return false, err + } + if resp.StatusCode() != 200 { + return false, fmt.Errorf("download file %s failed, err: %v", src, string(resp.Body())) + } + var respItem downloadResp + if err := json.Unmarshal(resp.Body(), &respItem); err != nil { + return false, err + } + if err := a.handleDownload(respItem.URL, target); err != nil { + return false, err + } + return true, nil +} + +func (a *aliClient) ListObjects(src string) ([]string, error) { + if len(src) == 0 || src == "root" || src == "/root" { + src = "root" + } else { + src = path.Join("/root", src) + } + fileInfos, err := a.loadDirWithPath(src) + if err != nil { + return nil, err + } + var names []string + for _, item := range fileInfos { + names = append(names, item.Name) + } + return names, nil +} + +func (a aliClient) loadFileWithName(pathItem string) (fileInfo, error) { + pathItems := strings.Split(pathItem, "/") + var ( + fileInfos []fileInfo + err error + ) + parentID := "root" + for i := 0; i < len(pathItems); i++ { + if len(pathItems[i]) == 0 { + continue + } + fileInfos, err = a.loadFileWithParentID(parentID) + if err != nil { + return fileInfo{}, err + } + isEnd := false + if i == len(pathItems)-2 { + isEnd = true + } + exist := false + for _, item := range fileInfos { + if item.Name == pathItems[i+1] { + if isEnd { + return item, nil + } else { + parentID = item.FileID + exist = true + } + } + } + if !exist { + return fileInfo{}, errors.New("no such file or dir") + } + + } + return fileInfo{}, errors.New("no such file or dir") +} + +func (a aliClient) loadDirWithPath(path string) ([]fileInfo, error) { + pathItems := strings.Split(path, "/") + var ( + fileInfos []fileInfo + err error + ) + parentID := "root" + for i := 0; i < len(pathItems); i++ { + if len(pathItems[i]) == 0 { + continue + } + fileInfos, err = a.loadFileWithParentID(parentID) + if err != nil { + return fileInfos, err + } + if i == len(pathItems)-1 { + return fileInfos, nil + } + exist := false + for _, item := range fileInfos { + if item.Name == pathItems[i+1] { + parentID = item.FileID + exist = true + } + } + if !exist { + return nil, errors.New("no such file or dir") + } + } + return fileInfos, errors.New("no such file or dir") +} + +func (a aliClient) loadFileWithParentID(parentID string) ([]fileInfo, error) { + client := resty.New() + data := map[string]interface{}{ + "drive_id": a.driveID, + "fields": "*", + "limit": 100, + "parent_file_id": parentID, + } + url := "https://api.aliyundrive.com/adrive/v3/file/list" + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return nil, err + } + if resp.StatusCode() != 200 { + return nil, fmt.Errorf("load file list failed, code: %v, err: %v", resp.StatusCode(), string(resp.Body())) + } + var fileResp fileResp + if err := json.Unmarshal(resp.Body(), &fileResp); err != nil { + return nil, err + } + return fileResp.Items, nil +} + +func (a aliClient) mkdirWithPath(target string) (string, error) { + pathItems := strings.Split(target, "/") + var ( + fileInfos []fileInfo + err error + ) + parentID := "root" + for i := 0; i < len(pathItems); i++ { + if len(pathItems[i]) == 0 { + continue + } + fileInfos, err = a.loadFileWithParentID(parentID) + if err != nil { + return "", err + } + isEnd := false + if i == len(pathItems)-2 { + isEnd = true + } + exist := false + for _, item := range fileInfos { + if item.Name == pathItems[i+1] { + parentID = item.FileID + if isEnd { + return item.FileID, nil + } else { + exist = true + } + } + } + if !exist { + parentID, err = a.mkdir(parentID, pathItems[i+1]) + if err != nil { + return parentID, err + } + if isEnd { + return parentID, nil + } + } + } + return "", errors.New("mkdir failed.") +} + +func (a aliClient) mkdir(parentID, name string) (string, error) { + client := resty.New() + data := map[string]interface{}{ + "drive_id": a.driveID, + "name": name, + "type": "folder", + "limit": 100, + "parent_file_id": parentID, + } + url := "https://api.aliyundrive.com/adrive/v2/file/createWithFolders" + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return "", err + } + if resp.StatusCode() != 201 { + return "", fmt.Errorf("mkdir %s failed, code: %v, err: %v", name, resp.StatusCode(), string(resp.Body())) + } + var mkdirResp mkdirResp + if err := json.Unmarshal(resp.Body(), &mkdirResp); err != nil { + return "", err + } + return mkdirResp.FileID, nil +} + +type fileResp struct { + Items []fileInfo `json:"items"` +} +type fileInfo struct { + FileID string `json:"file_id"` + Name string `json:"name"` + Size int `json:"size"` +} + +type mkdirResp struct { + FileID string `json:"file_id"` +} + +type partInfo struct { + PartNumber int `json:"part_number"` + UploadURL string `json:"upload_url"` + InternalUploadURL string `json:"internal_upload_url"` + ContentType string `json:"content_type"` +} + +func makePartInfoList(size int64) []*partInfo { + var res []*partInfo + maxPartSize := int64(1024 * 1024 * 1024) + partInfoNum := int(size / maxPartSize) + if size%maxPartSize > 0 { + partInfoNum += 1 + } + + for i := 0; i < partInfoNum; i++ { + res = append(res, &partInfo{PartNumber: i + 1}) + } + + return res +} + +type createFileResp struct { + Type string `json:"type"` + RapidUpload bool `json:"rapid_upload"` + DomainId string `json:"domain_id"` + DriveId string `json:"drive_id"` + FileName string `json:"file_name"` + EncryptMode string `json:"encrypt_mode"` + Location string `json:"location"` + UploadID string `json:"upload_id"` + FileID string `json:"file_id"` + PartInfoList []*partInfo `json:"part_info_list,omitempty"` +} + +func (a aliClient) uploadPart(uri string, reader io.Reader) error { + req, err := http.NewRequest(http.MethodPut, uri, reader) + if err != nil { + return err + } + client := &http.Client{} + response, err := client.Do(req) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("handle upload park file with url failed, code: %v", response.StatusCode) + } + + return nil +} + +type downloadResp struct { + URL string `json:"url"` +} + +func (a aliClient) handleDownload(uri string, target string) error { + req, err := http.NewRequest(http.MethodGet, uri, nil) + if err != nil { + return err + } + req.Header.Add("Authorization", a.token) + req.Header.Add("origin", "https://www.aliyundrive.com") + req.Header.Add("referer", "https://www.aliyundrive.com/") + client := &http.Client{} + response, err := client.Do(req) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("handle download with url failed, code: %v", response.StatusCode) + } + if _, err := os.Stat(path.Dir(target)); err != nil { + _ = os.MkdirAll(path.Dir(target), os.ModePerm) + } + out, err := os.Create(target) + if err != nil { + return err + } + defer out.Close() + if _, err = io.Copy(out, response.Body); err != nil { + return err + } + + return nil +} + +func (a *aliClient) completeUpload(uploadID, fileID string) error { + client := resty.New() + data := map[string]interface{}{ + "drive_id": a.driveID, + "upload_id": uploadID, + "file_id": fileID, + } + + url := "https://api.aliyundrive.com/v2/file/complete" + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return err + } + if resp.StatusCode() != 200 { + return fmt.Errorf("complete upload failed, err: %v", string(resp.Body())) + } + + return nil +} + +type tokenResp struct { + AccessToken string `json:"access_token"` +} + +func loadToken(refresh_token string) (string, error) { + client := resty.New() + data := map[string]interface{}{ + "grant_type": "refresh_token", + "refresh_token": refresh_token, + } + + url := "https://api.aliyundrive.com/token/refresh" + resp, err := client.R(). + SetBody(data). + Post(url) + + if err != nil { + return "", fmt.Errorf("load account token failed, err: %v", err) + } + if resp.StatusCode() != 200 { + return "", fmt.Errorf("load account token failed, code: %v", resp.StatusCode()) + } + var respItem tokenResp + if err := json.Unmarshal(resp.Body(), &respItem); err != nil { + return "", err + } + return respItem.AccessToken, nil +} diff --git a/agent/utils/cloud_storage/cloud_storage_client.go b/agent/utils/cloud_storage/cloud_storage_client.go index 5cc59f1c3..731753eac 100644 --- a/agent/utils/cloud_storage/cloud_storage_client.go +++ b/agent/utils/cloud_storage/cloud_storage_client.go @@ -38,6 +38,8 @@ func NewCloudStorageClient(backupType string, vars map[string]interface{}) (Clou return client.NewOneDriveClient(vars) case constant.UPYUN: return client.NewUpClient(vars) + case constant.ALIYUN: + return client.NewALIClient(vars) default: return nil, constant.ErrNotSupportType } diff --git a/core/app/model/backup.go b/core/app/model/backup.go index 375c08a92..58fd1f3bc 100644 --- a/core/app/model/backup.go +++ b/core/app/model/backup.go @@ -4,8 +4,8 @@ import "time" type BackupAccount struct { BaseModel - Name string `gorm:"unique;not null" json:"name"` - Type string `gorm:"unique;not null" json:"type"` + Name string `gorm:"not null" json:"name"` + Type string `gorm:"not null" json:"type"` Bucket string `json:"bucket"` AccessKey string `json:"accessKey"` Credential string `json:"credential"` diff --git a/core/app/service/backup.go b/core/app/service/backup.go index 59204f4ed..ddf6e9c4a 100644 --- a/core/app/service/backup.go +++ b/core/app/service/backup.go @@ -156,12 +156,13 @@ func (u *BackupService) SearchWithPage(req dto.SearchPageWithType) (int64, inter item.Credential = base64.StdEncoding.EncodeToString([]byte(item.Credential)) } - if account.Type == constant.OneDrive { + if account.Type == constant.OneDrive || account.Type == constant.ALIYUN { varMap := make(map[string]interface{}) if err := json.Unmarshal([]byte(item.Vars), &varMap); err != nil { continue } delete(varMap, "refresh_token") + delete(varMap, "drive_id") itemVars, _ := json.Marshal(varMap) item.Vars = string(itemVars) } @@ -500,33 +501,75 @@ func StartRefreshOneDriveToken(backup *model.BackupAccount) error { } func (u *BackupService) Run() { - var backupItem model.BackupAccount - _ = global.DB.Where("`type` = ?", "OneDrive").First(&backupItem) - if backupItem.ID == 0 { - return - } - global.LOG.Info("start to refresh token of OneDrive ...") - varMap := make(map[string]interface{}) - if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil { - global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err) - return - } - refreshToken, err := client.RefreshToken("refresh_token", "refreshToken", varMap) - varMap["refresh_status"] = constant.StatusSuccess - varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout) - if err != nil { - varMap["refresh_status"] = constant.StatusFailed - varMap["refresh_msg"] = err.Error() - global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err) - return - } - varMap["refresh_token"] = refreshToken - - varsItem, _ := json.Marshal(varMap) - _ = global.DB.Model(&model.BackupAccount{}). - Where("id = ?", backupItem.ID). - Updates(map[string]interface{}{ - "vars": varsItem, - }).Error - global.LOG.Info("Successfully refreshed OneDrive token.") + refreshOneDrive() + refreshALIYUN() +} + +func refreshOneDrive() { + var backups []model.BackupAccount + _ = global.DB.Where("`type` = ?", "OneDrive").Find(&backups) + for _, backupItem := range backups { + if backupItem.ID == 0 { + return + } + global.LOG.Infof("start to refresh token of OneDrive %s ...", backupItem.Name) + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil { + global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err) + return + } + refreshToken, err := client.RefreshToken("refresh_token", "refreshToken", varMap) + varMap["refresh_status"] = constant.StatusSuccess + varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout) + if err != nil { + varMap["refresh_status"] = constant.StatusFailed + varMap["refresh_msg"] = err.Error() + global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err) + return + } + varMap["refresh_token"] = refreshToken + + varsItem, _ := json.Marshal(varMap) + _ = global.DB.Model(&model.BackupAccount{}). + Where("id = ?", backupItem.ID). + Updates(map[string]interface{}{ + "vars": varsItem, + }).Error + global.LOG.Info("Successfully refreshed OneDrive token.") + } +} + +func refreshALIYUN() { + var backups []model.BackupAccount + _ = global.DB.Where("`type` = ?", "ALIYUN").Find(&backups) + for _, backupItem := range backups { + global.LOG.Infof("start to refresh token of ALIYUN %s ...", backupItem.Name) + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil { + global.LOG.Errorf("Failed to refresh ALIYUN token, please retry, err: %v", err) + return + } + if _, ok := varMap["refresh_token"]; !ok { + global.LOG.Error("no such refresh token find in db") + return + } + refreshToken, err := client.RefreshALIToken(fmt.Sprintf("%v", varMap["refresh_token"])) + varMap["refresh_status"] = constant.StatusSuccess + varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout) + if err != nil { + varMap["refresh_status"] = constant.StatusFailed + varMap["refresh_msg"] = err.Error() + global.LOG.Errorf("Failed to refresh ALIYUN token, please retry, err: %v", err) + return + } + varMap["refresh_token"] = refreshToken + + varsItem, _ := json.Marshal(varMap) + _ = global.DB.Model(&model.BackupAccount{}). + Where("id = ?", backupItem.ID). + Updates(map[string]interface{}{ + "vars": varsItem, + }).Error + global.LOG.Info("Successfully refreshed ALIYUN token.") + } } diff --git a/core/constant/common.go b/core/constant/common.go index 10f2a9c7e..b165927bc 100644 --- a/core/constant/common.go +++ b/core/constant/common.go @@ -29,5 +29,6 @@ const ( WebDAV = "WebDAV" Local = "LOCAL" UPYUN = "UPYUN" + ALIYUN = "ALIYUN" OneDriveRedirectURI = "http://localhost/login/authorized" ) diff --git a/core/init/migration/migrate.go b/core/init/migration/migrate.go index 958b7d57d..1e8f36de2 100644 --- a/core/init/migration/migrate.go +++ b/core/init/migration/migrate.go @@ -16,6 +16,7 @@ func Init() { migrations.InitHost, migrations.InitTerminalSetting, migrations.InitAppLauncher, + migrations.InitBackup, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/core/init/migration/migrations/init.go b/core/init/migration/migrations/init.go index 68e73534b..ab0a42c24 100644 --- a/core/init/migration/migrations/init.go +++ b/core/init/migration/migrations/init.go @@ -240,3 +240,10 @@ var InitAppLauncher = &gormigrate.Migration{ return tx.AutoMigrate(&model.AppLauncher{}) }, } + +var InitBackup = &gormigrate.Migration{ + ID: "20241107-init-backup", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate(&model.BackupAccount{}) + }, +} diff --git a/core/utils/cloud_storage/client/ali.go b/core/utils/cloud_storage/client/ali.go new file mode 100644 index 000000000..b6d62e812 --- /dev/null +++ b/core/utils/cloud_storage/client/ali.go @@ -0,0 +1,304 @@ +package client + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + + "github.com/go-resty/resty/v2" +) + +type aliClient struct { + token string + driveID string +} + +func NewALIClient(vars map[string]interface{}) (*aliClient, error) { + refresh_token := loadParamFromVars("refresh_token", vars) + drive_id := loadParamFromVars("drive_id", vars) + + token, err := RefreshALIToken(refresh_token) + if err != nil { + return nil, err + } + return &aliClient{token: token, driveID: drive_id}, nil +} + +func (a aliClient) ListBuckets() ([]interface{}, error) { + return nil, nil +} + +func (a aliClient) Upload(src, target string) (bool, error) { + target = path.Join("/root", target) + parentID := "root" + var err error + if path.Dir(target) != "/root" { + parentID, err = a.mkdirWithPath(path.Dir(target)) + if err != nil { + return false, err + } + } + file, err := os.Open(src) + if err != nil { + return false, err + } + defer file.Close() + fileInfo, err := file.Stat() + if err != nil { + return false, err + } + data := map[string]interface{}{ + "drive_id": a.driveID, + "part_info_list": makePartInfoList(fileInfo.Size()), + "parent_file_id": parentID, + "name": path.Base(src), + "type": "file", + "size": fileInfo.Size(), + "check_name_mode": "auto_rename", + } + client := resty.New() + url := "https://api.alipan.com/v2/file/create" + + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return false, err + } + + var createResp createFileResp + if err := json.Unmarshal(resp.Body(), &createResp); err != nil { + return false, err + } + for _, part := range createResp.PartInfoList { + err = a.uploadPart(part.UploadURL, io.LimitReader(file, 1024*1024*1024)) + if err != nil { + return false, err + } + } + + if err := a.completeUpload(createResp.UploadID, createResp.FileID); err != nil { + return false, err + } + return true, nil +} + +func (a aliClient) loadFileWithParentID(parentID string) ([]fileInfo, error) { + client := resty.New() + data := map[string]interface{}{ + "drive_id": a.driveID, + "fields": "*", + "limit": 100, + "parent_file_id": parentID, + } + url := "https://api.alipan.com/v2/file/list" + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return nil, err + } + if resp.StatusCode() != 200 { + return nil, fmt.Errorf("load file list failed, code: %v, err: %v", resp.StatusCode(), string(resp.Body())) + } + var fileResp fileResp + if err := json.Unmarshal(resp.Body(), &fileResp); err != nil { + return nil, err + } + return fileResp.Items, nil +} + +func (a aliClient) mkdirWithPath(target string) (string, error) { + pathItems := strings.Split(target, "/") + var ( + fileInfos []fileInfo + err error + ) + parentID := "root" + for i := 0; i < len(pathItems); i++ { + if len(pathItems[i]) == 0 { + continue + } + fileInfos, err = a.loadFileWithParentID(parentID) + if err != nil { + return "", err + } + isEnd := false + if i == len(pathItems)-2 { + isEnd = true + } + exist := false + for _, item := range fileInfos { + if item.Name == pathItems[i+1] { + parentID = item.FileID + if isEnd { + return item.FileID, nil + } else { + exist = true + } + } + } + if !exist { + parentID, err = a.mkdir(parentID, pathItems[i+1]) + if err != nil { + return parentID, err + } + if isEnd { + return parentID, nil + } + } + } + return "", errors.New("mkdir failed.") +} + +func (a aliClient) mkdir(parentID, name string) (string, error) { + client := resty.New() + data := map[string]interface{}{ + "drive_id": a.driveID, + "name": name, + "type": "folder", + "limit": 100, + "parent_file_id": parentID, + } + url := "https://api.alipan.com/v2/file/create" + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return "", err + } + if resp.StatusCode() != 201 { + return "", fmt.Errorf("mkdir %s failed, code: %v, err: %v", name, resp.StatusCode(), string(resp.Body())) + } + var mkdirResp mkdirResp + if err := json.Unmarshal(resp.Body(), &mkdirResp); err != nil { + return "", err + } + return mkdirResp.FileID, nil +} + +type fileResp struct { + Items []fileInfo `json:"items"` +} +type fileInfo struct { + FileID string `json:"file_id"` + Name string `json:"name"` + Size int `json:"size"` +} + +type mkdirResp struct { + FileID string `json:"file_id"` +} + +type partInfo struct { + PartNumber int `json:"part_number"` + UploadURL string `json:"upload_url"` + InternalUploadURL string `json:"internal_upload_url"` + ContentType string `json:"content_type"` +} + +func makePartInfoList(size int64) []*partInfo { + var res []*partInfo + maxPartSize := int64(1024 * 1024 * 1024) + partInfoNum := int(size / maxPartSize) + if size%maxPartSize > 0 { + partInfoNum += 1 + } + + for i := 0; i < partInfoNum; i++ { + res = append(res, &partInfo{PartNumber: i + 1}) + } + + return res +} + +type createFileResp struct { + Type string `json:"type"` + RapidUpload bool `json:"rapid_upload"` + DomainId string `json:"domain_id"` + DriveId string `json:"drive_id"` + FileName string `json:"file_name"` + EncryptMode string `json:"encrypt_mode"` + Location string `json:"location"` + UploadID string `json:"upload_id"` + FileID string `json:"file_id"` + PartInfoList []*partInfo `json:"part_info_list,omitempty"` +} + +func (a aliClient) uploadPart(uri string, reader io.Reader) error { + req, err := http.NewRequest(http.MethodPut, uri, reader) + if err != nil { + return err + } + client := &http.Client{} + response, err := client.Do(req) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("handle upload park file with url failed, code: %v", response.StatusCode) + } + + return nil +} + +func (a *aliClient) completeUpload(uploadID, fileID string) error { + client := resty.New() + data := map[string]interface{}{ + "drive_id": a.driveID, + "upload_id": uploadID, + "file_id": fileID, + } + + url := "https://api.aliyundrive.com/v2/file/complete" + resp, err := client.R(). + SetHeader("Authorization", a.token). + SetBody(data). + Post(url) + if err != nil { + return err + } + if resp.StatusCode() != 200 { + return fmt.Errorf("complete upload failed, err: %v", string(resp.Body())) + } + + return nil +} + +type tokenResp struct { + AccessToken string `json:"access_token"` +} + +func RefreshALIToken(refresh_token string) (string, error) { + client := resty.New() + data := map[string]interface{}{ + "grant_type": "refresh_token", + "refresh_token": refresh_token, + } + + url := "https://api.aliyundrive.com/token/refresh" + resp, err := client.R(). + SetBody(data). + Post(url) + + if err != nil { + return "", fmt.Errorf("load account token failed, err: %v", err) + } + if resp.StatusCode() != 200 { + return "", fmt.Errorf("load account token failed, code: %v", resp.StatusCode()) + } + var respItem tokenResp + if err := json.Unmarshal(resp.Body(), &respItem); err != nil { + return "", err + } + return respItem.AccessToken, nil +} diff --git a/core/utils/cloud_storage/client/onedrive.go b/core/utils/cloud_storage/client/onedrive.go index 47349fd07..956230352 100644 --- a/core/utils/cloud_storage/client/onedrive.go +++ b/core/utils/cloud_storage/client/onedrive.go @@ -80,38 +80,6 @@ func (o oneDriveClient) Upload(src, target string) (bool, error) { return isOk, err } -func (o oneDriveClient) Download(src, target string) (bool, error) { - src = "/" + strings.TrimPrefix(src, "/") - req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil) - if err != nil { - return false, fmt.Errorf("new request for file id failed, err: %v", err) - } - var driveItem *odsdk.DriveItem - if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { - return false, fmt.Errorf("do request for file id failed, err: %v", err) - } - - resp, err := http.Get(driveItem.DownloadURL) - if err != nil { - return false, err - } - defer resp.Body.Close() - - out, err := os.Create(target) - if err != nil { - return false, err - } - defer out.Close() - buffer := make([]byte, 2*1024*1024) - - _, err = io.CopyBuffer(out, resp.Body, buffer) - if err != nil { - return false, err - } - - return true, nil -} - func (o *oneDriveClient) loadIDByPath(path string) (string, error) { pathItem := "root:" + path if path == "/" { diff --git a/core/utils/cloud_storage/client/oss.go b/core/utils/cloud_storage/client/oss.go index e4fed15d7..e0b96701a 100644 --- a/core/utils/cloud_storage/client/oss.go +++ b/core/utils/cloud_storage/client/oss.go @@ -1,8 +1,6 @@ package client import ( - "fmt" - osssdk "github.com/aliyun/aliyun-oss-go-sdk/oss" ) @@ -41,40 +39,6 @@ func (o ossClient) ListBuckets() ([]interface{}, error) { return result, err } -func (o ossClient) Exist(path string) (bool, error) { - bucket, err := o.client.Bucket(o.bucketStr) - if err != nil { - return false, err - } - return bucket.IsObjectExist(path) -} - -func (o ossClient) Size(path string) (int64, error) { - bucket, err := o.client.Bucket(o.bucketStr) - if err != nil { - return 0, err - } - lor, err := bucket.ListObjectsV2(osssdk.Prefix(path)) - if err != nil { - return 0, err - } - if len(lor.Objects) == 0 { - return 0, fmt.Errorf("no such file %s", path) - } - return lor.Objects[0].Size, nil -} - -func (o ossClient) Delete(path string) (bool, error) { - bucket, err := o.client.Bucket(o.bucketStr) - if err != nil { - return false, err - } - if err := bucket.DeleteObject(path); err != nil { - return false, err - } - return true, nil -} - func (o ossClient) Upload(src, target string) (bool, error) { bucket, err := o.client.Bucket(o.bucketStr) if err != nil { @@ -89,30 +53,3 @@ func (o ossClient) Upload(src, target string) (bool, error) { } return true, nil } - -func (o ossClient) Download(src, target string) (bool, error) { - bucket, err := o.client.Bucket(o.bucketStr) - if err != nil { - return false, err - } - if err := bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, "")); err != nil { - return false, err - } - return true, nil -} - -func (o *ossClient) ListObjects(prefix string) ([]string, error) { - bucket, err := o.client.Bucket(o.bucketStr) - if err != nil { - return nil, err - } - lor, err := bucket.ListObjectsV2(osssdk.Prefix(prefix)) - if err != nil { - return nil, err - } - var result []string - for _, obj := range lor.Objects { - result = append(result, obj.Key) - } - return result, nil -} diff --git a/core/utils/cloud_storage/client/s3.go b/core/utils/cloud_storage/client/s3.go index 9fc7ebadd..ed1118c54 100644 --- a/core/utils/cloud_storage/client/s3.go +++ b/core/utils/cloud_storage/client/s3.go @@ -4,7 +4,6 @@ import ( "os" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" @@ -53,49 +52,6 @@ func (s s3Client) ListBuckets() ([]interface{}, error) { return result, nil } -func (s s3Client) Exist(path string) (bool, error) { - svc := s3.New(&s.Sess) - if _, err := svc.HeadObject(&s3.HeadObjectInput{ - Bucket: &s.bucket, - Key: &path, - }); err != nil { - if aerr, ok := err.(awserr.RequestFailure); ok { - if aerr.StatusCode() == 404 { - return false, nil - } - } else { - return false, aerr - } - } - return true, nil -} - -func (s *s3Client) Size(path string) (int64, error) { - svc := s3.New(&s.Sess) - file, err := svc.GetObject(&s3.GetObjectInput{ - Bucket: &s.bucket, - Key: &path, - }) - if err != nil { - return 0, err - } - return *file.ContentLength, nil -} - -func (s s3Client) Delete(path string) (bool, error) { - svc := s3.New(&s.Sess) - if _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(path)}); err != nil { - return false, err - } - if err := svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(path), - }); err != nil { - return false, err - } - return true, nil -} - func (s s3Client) Upload(src, target string) (bool, error) { fileInfo, err := os.Stat(src) if err != nil { @@ -121,43 +77,3 @@ func (s s3Client) Upload(src, target string) (bool, error) { } return true, nil } - -func (s s3Client) Download(src, target string) (bool, error) { - if _, err := os.Stat(target); err != nil { - if os.IsNotExist(err) { - os.Remove(target) - } else { - return false, err - } - } - file, err := os.Create(target) - if err != nil { - return false, err - } - defer file.Close() - downloader := s3manager.NewDownloader(&s.Sess) - if _, err = downloader.Download(file, &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(src), - }); err != nil { - os.Remove(target) - return false, err - } - return true, nil -} - -func (s *s3Client) ListObjects(prefix string) ([]string, error) { - svc := s3.New(&s.Sess) - var result []string - outputs, err := svc.ListObjects(&s3.ListObjectsInput{ - Bucket: &s.bucket, - Prefix: &prefix, - }) - if err != nil { - return result, err - } - for _, item := range outputs.Contents { - result = append(result, *item.Key) - } - return result, nil -} diff --git a/core/utils/cloud_storage/client/sftp.go b/core/utils/cloud_storage/client/sftp.go index b9e1defd3..167ba0415 100644 --- a/core/utils/cloud_storage/client/sftp.go +++ b/core/utils/cloud_storage/client/sftp.go @@ -108,21 +108,3 @@ func (s sftpClient) ListBuckets() ([]interface{}, error) { var result []interface{} return result, nil } - -func (s sftpClient) Delete(filePath string) error { - sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) - if err != nil { - return err - } - client, err := sftp.NewClient(sshClient) - if err != nil { - return err - } - defer client.Close() - defer sshClient.Close() - - if err := client.Remove(filePath); err != nil { - return err - } - return nil -} diff --git a/core/utils/cloud_storage/client/webdav.go b/core/utils/cloud_storage/client/webdav.go index c1013357c..b237787c1 100644 --- a/core/utils/cloud_storage/client/webdav.go +++ b/core/utils/cloud_storage/client/webdav.go @@ -3,7 +3,6 @@ package client import ( "crypto/tls" "fmt" - "io" "net/http" "os" "path" @@ -61,61 +60,3 @@ func (s webDAVClient) ListBuckets() ([]interface{}, error) { var result []interface{} return result, nil } - -func (s webDAVClient) Download(src, target string) (bool, error) { - srcPath := path.Join(s.Bucket, src) - info, err := s.client.Stat(srcPath) - if err != nil { - return false, err - } - targetStat, err := os.Stat(target) - if err == nil { - if info.Size() == targetStat.Size() { - return true, nil - } - } - file, err := os.Create(target) - if err != nil { - return false, err - } - defer file.Close() - reader, _ := s.client.ReadStream(srcPath) - if _, err := io.Copy(file, reader); err != nil { - return false, err - } - return true, err -} - -func (s webDAVClient) Exist(pathItem string) (bool, error) { - if _, err := s.client.Stat(path.Join(s.Bucket, pathItem)); err != nil { - return false, err - } - return true, nil -} - -func (s webDAVClient) Size(pathItem string) (int64, error) { - file, err := s.client.Stat(path.Join(s.Bucket, pathItem)) - if err != nil { - return 0, err - } - return file.Size(), nil -} - -func (s webDAVClient) Delete(pathItem string) (bool, error) { - if err := s.client.Remove(path.Join(s.Bucket, pathItem)); err != nil { - return false, err - } - return true, nil -} - -func (s webDAVClient) ListObjects(prefix string) ([]string, error) { - files, err := s.client.ReadDir(path.Join(s.Bucket, prefix)) - if err != nil { - return nil, err - } - var result []string - for _, file := range files { - result = append(result, file.Name()) - } - return result, nil -} diff --git a/core/utils/cloud_storage/cloud_storage_client.go b/core/utils/cloud_storage/cloud_storage_client.go index 912f3cc8f..8ca5ce041 100644 --- a/core/utils/cloud_storage/cloud_storage_client.go +++ b/core/utils/cloud_storage/cloud_storage_client.go @@ -32,6 +32,8 @@ func NewCloudStorageClient(backupType string, vars map[string]interface{}) (Clou return client.NewOneDriveClient(vars) case constant.UPYUN: return client.NewUpClient(vars) + case constant.ALIYUN: + return client.NewALIClient(vars) default: return nil, constant.ErrNotSupportType } diff --git a/frontend/src/api/interface/backup.ts b/frontend/src/api/interface/backup.ts index 01cd406a8..ed77d1742 100644 --- a/frontend/src/api/interface/backup.ts +++ b/frontend/src/api/interface/backup.ts @@ -50,6 +50,7 @@ export namespace Backup { downloadAccountID: number; fileDir: string; fileName: string; + size: number; } export interface ForBucket { type: string; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index b096a07d9..a376260dd 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1452,6 +1452,14 @@ const message = { WebDAV: 'WebDAV', WebDAVAlist: 'WebDAV connect Alist can refer to the official documentation', UPYUN: 'UPYUN', + ALIYUN: 'Aliyun Drive', + ALIYUNHelper: + 'The current maximum limit for non-client downloads on Aliyun Drive is 100 MB. Exceeding this limit requires downloading through the client.', + ALIYUNRecover: + 'The current maximum limit for non-client downloads on Aliyun Drive is 100 MB. Exceeding this limit requires downloading through the client to the local device, then synchronizing the snapshot for recovery.', + analysis: 'Analysis', + analysisHelper: + 'Paste the entire token content to automatically parse the required parts. For specific operations, please refer to the official documentation.', serviceName: 'Service Name', operator: 'Operator', OneDrive: 'Microsoft OneDrive', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index 686845418..559426110 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -1365,6 +1365,12 @@ const message = { WebDAV: 'WebDAV', WebDAVAlist: 'WebDAV 連接 Alist 可參考官方文檔', UPYUN: '又拍雲', + ALIYUN: '阿里雲盤', + ALIYUNHelper: '當前阿里雲盤非客戶端下載最大限制為 100 MB，超過限制需要通過客戶端下載', + ALIYUNRecover: + '當前阿里雲盤非客戶端下載最大限制為 100 MB，超過限制需要通過客戶端下載到本地後，同步快照進行恢復', + analysis: '解析', + analysisHelper: '粘貼整個 token 內容，自動解析所需部分，具體操作可參考官方文檔', serviceName: '服務名稱', operator: '操作員', OneDrive: '微軟 OneDrive', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 3de1445ff..f7c35eb47 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1367,6 +1367,12 @@ const message = { WebDAV: 'WebDAV', WebDAVAlist: 'WebDAV 连接 Alist 可参考官方文档', UPYUN: '又拍云', + ALIYUN: '阿里云盘', + ALIYUNHelper: '当前阿里云盘非客户端下载最大限制为 100 MB，超过限制需要通过客户端下载', + ALIYUNRecover: + '当前阿里云盘非客户端下载最大限制为 100 MB，超过限制需要通过客户端下载到本地后，同步快照进行恢复', + analysis: '解析', + analysisHelper: '粘贴整个 token 内容，自动解析所需部分，具体操作可参考官方文档', serviceName: '服务名称', operator: '操作员', OneDrive: '微软 OneDrive', diff --git a/frontend/src/views/cronjob/backup/index.vue b/frontend/src/views/cronjob/backup/index.vue index 7f8736323..42db0724b 100644 --- a/frontend/src/views/cronjob/backup/index.vue +++ b/frontend/src/views/cronjob/backup/index.vue @@ -53,6 +53,7 @@ import { computeSize, dateFormat, downloadFile } from '@/utils/util'; import i18n from '@/lang'; import { downloadBackupRecord, searchBackupRecordsByCronjob } from '@/api/modules/backup'; import { Backup } from '@/api/interface/backup'; +import { MsgError } from '@/utils/message'; const selects = ref([]); const loading = ref(); @@ -102,6 +103,10 @@ const search = async () => { }; const onDownload = async (row: Backup.RecordInfo) => { + if (row.accountType === 'ALIYUN' && row.size < 100 * 1024 * 1024) { + MsgError(i18n.global.t('setting.ALIYUNHelper')); + return; + } let params = { downloadAccountID: row.downloadAccountID, fileDir: row.fileDir, diff --git a/frontend/src/views/setting/backup-account/operate/index.vue b/frontend/src/views/setting/backup-account/operate/index.vue index 21b34bd44..60cca0fb9 100644 --- a/frontend/src/views/setting/backup-account/operate/index.vue +++ b/frontend/src/views/setting/backup-account/operate/index.vue @@ -17,7 +17,9 @@ + + {{ $t('setting.ALIYUNHelper') }} @@ -95,10 +97,7 @@ - + {{ $t('terminal.rememberPassword') }} @@ -233,6 +232,40 @@ v-model.number="dialogData.rowData!.varsJson['timeout']" > +
+ +
+ + + {{ $t('setting.analysis') }} + + + {{ $t('setting.analysisHelper') }} + + {{ $t('firewall.quickJump') }} + + +
+ + + + + + + +
+
@@ -437,10 +470,26 @@ function callback(error: any) { } } +const loadFromToken = () => { + const obj = JSON.parse(dialogData.value.rowData!.varsJson['token']); + dialogData.value.rowData!.varsJson['drive_id'] = obj.default_drive_id; + dialogData.value.rowData!.varsJson['refresh_token'] = obj.refresh_token; +}; +const hasRemember = () => { + return ( + dialogData.value.rowData!.type !== 'LOCAL' && + dialogData.value.rowData!.type !== 'OneDrive' && + dialogData.value.rowData!.type !== 'ALIYUN' + ); +}; const isUPYUN = () => { let itemType = dialogData.value.rowData!.type; return itemType === 'UPYUN'; }; +const isALIYUNYUN = () => { + let itemType = dialogData.value.rowData!.type; + return itemType === 'ALIYUN'; +}; const hasAccessKey = () => { let itemType = dialogData.value.rowData!.type; return itemType === 'COS' || itemType === 'KODO' || itemType === 'MINIO' || itemType === 'OSS' || itemType === 'S3'; @@ -558,6 +607,9 @@ const onSubmit = async (formEl: FormInstance | undefined) => { } dialogData.value.rowData!.varsJson['endpointItem'] = undefined; } + if (isALIYUNYUN()) { + dialogData.value.rowData!.varsJson['token'] = undefined; + } dialogData.value.rowData.vars = JSON.stringify(dialogData.value.rowData!.varsJson); loading.value = true; if (dialogData.value.title === 'create') { diff --git a/frontend/src/views/setting/snapshot/index.vue b/frontend/src/views/setting/snapshot/index.vue index 1fcb26152..dbee75def 100644 --- a/frontend/src/views/setting/snapshot/index.vue +++ b/frontend/src/views/setting/snapshot/index.vue @@ -196,7 +196,7 @@ import RecoverStatus from '@/views/setting/snapshot/status/index.vue'; import SnapshotImport from '@/views/setting/snapshot/import/index.vue'; import SnapshotCreate from '@/views/setting/snapshot/create/index.vue'; import SnapRecover from '@/views/setting/snapshot/recover/index.vue'; -import { MsgSuccess } from '@/utils/message'; +import { MsgError, MsgSuccess } from '@/utils/message'; import { loadOsInfo } from '@/api/modules/dashboard'; const loading = ref(false); @@ -291,6 +291,10 @@ const onChange = async (info: any) => { }; const onRecover = async (row: any) => { + if (row.defaultDownload.indexOf('ALIYUN') !== -1 && row.size > 100 * 1024 * 1024) { + MsgError(i18n.global.t('setting.ALIYUNRecover')); + return; + } loading.value = true; await loadOsInfo() .then((res) => { diff --git a/go.mod b/go.mod index 7d7bec108..3b0ef8e1f 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/glebarez/sqlite v1.11.0 github.com/go-gormigrate/gormigrate/v2 v2.1.2 github.com/go-playground/validator/v10 v10.22.0 + github.com/go-resty/resty/v2 v2.15.3 github.com/goh-chunlin/go-onedrive v1.1.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 @@ -37,7 +38,7 @@ require ( github.com/tencentyun/cos-go-sdk-v5 v0.7.54 github.com/upyun/go-sdk v2.1.0+incompatible github.com/xlzd/gotp v0.1.0 - golang.org/x/crypto v0.24.0 + golang.org/x/crypto v0.25.0 golang.org/x/oauth2 v0.18.0 golang.org/x/sys v0.22.0 golang.org/x/term v0.22.0 @@ -119,9 +120,9 @@ require ( golang.org/x/arch v0.8.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/image v0.13.0 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.34.1 // indirect diff --git a/go.sum b/go.sum index 86c7e4c97..658c1bac4 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.8.0/go.mod h1:9JhgTzTaE31GZDpH/HSvHiRJrJ3iKAgqqH0Bl/Ocjdk= github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao= github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-resty/resty/v2 v2.15.3 h1:bqff+hcqAflpiF591hhJzNdkRsFhlB96CYfBwSFvql8= +github.com/go-resty/resty/v2 v2.15.3/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= @@ -413,8 +415,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -487,8 +489,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -572,8 +574,8 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=