gh_mirrors/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-01-31 22:18:07 +08:00
Code

feat: 备份列表显示文件备份文件大小 (#3134)

Browse Source 
Refs #2968 #1922
This commit is contained in:
ssongliu 2023-12-01 17:50:09 +08:00 committed by GitHub
parent 4daed1a3de
commit 64ad829e3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 491 additions and 587 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/app/dto/backup.go
View File

@ -52,6 +52,7 @@ type BackupRecords struct {
BackupType string `json:"backupType"` BackupType string `json:"backupType"`
FileDir string `json:"fileDir"` FileDir string `json:"fileDir"`
FileName string `json:"fileName"` FileName string `json:"fileName"`
Size int64 `json:"size"`
} }
type DownloadRecord struct { type DownloadRecord struct {

45
backend/app/service/backup.go
View File

@ -81,15 +81,46 @@ func (u *BackupService) SearchRecordsWithPage(search dto.RecordSearch) (int64, [
commonRepo.WithByType(search.Type), commonRepo.WithByType(search.Type),
backupRepo.WithByDetailName(search.DetailName), backupRepo.WithByDetailName(search.DetailName),
) )
var dtobas []dto.BackupRecords
for _, group := range records { var datas []dto.BackupRecords
clientMap := make(map[string]loadSizeHelper)
for i := 0; i < len(records); i++ {
var item dto.BackupRecords var item dto.BackupRecords
if err := copier.Copy(&item, &group); err != nil { if err := copier.Copy(&item, &records[i]); err != nil {
return 0, nil, errors.WithMessage(constant.ErrStructTransform, err.Error()) return 0, nil, errors.WithMessage(constant.ErrStructTransform, err.Error())
} }
dtobas = append(dtobas, item) itemPath := path.Join(records[i].FileDir, records[i].FileName)
if records[i].Source == "LOCAL" {
fileInfo, _ := os.Stat(itemPath)
item.Size = fileInfo.Size()
datas = append(datas, item)
continue
}
if _, ok := clientMap[records[i].Source]; !ok {
backup, err := backupRepo.Get(commonRepo.WithByType(records[i].Source))
if err != nil {
global.LOG.Errorf("load backup model %s from db failed, err: %v", records[i].Source, err)
return total, datas, err
}
client, err := u.NewClient(&backup)
if err != nil {
global.LOG.Errorf("load backup client %s from db failed, err: %v", records[i].Source, err)
return total, datas, err
}
item.Size, _ = client.Size(path.Join(strings.TrimLeft(backup.BackupPath, "/"), itemPath))
datas = append(datas, item)
clientMap[records[i].Source] = loadSizeHelper{backupPath: strings.TrimLeft(backup.BackupPath, "/"), client: client}
continue
}
item.Size, _ = clientMap[records[i].Source].client.Size(path.Join(clientMap[records[i].Source].backupPath, itemPath))
datas = append(datas, item)
} }
return total, dtobas, err return total, datas, err
}
type loadSizeHelper struct {
backupPath string
client cloud_storage.CloudStorageClient
} }
func (u *BackupService) LoadOneDriveInfo() (string, error) { func (u *BackupService) LoadOneDriveInfo() (string, error) {
@ -139,9 +170,7 @@ func (u *BackupService) DownloadRecord(info dto.DownloadRecord) (string, error)
} }
srcPath := fmt.Sprintf("%s/%s", info.FileDir, info.FileName) srcPath := fmt.Sprintf("%s/%s", info.FileDir, info.FileName)
if len(backup.BackupPath) != 0 { if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/") srcPath = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), srcPath)
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
srcPath = itemPath + srcPath
} }
if exist, _ := backClient.Exist(srcPath); exist { if exist, _ := backClient.Exist(srcPath); exist {
isOK, err := backClient.Download(srcPath, targetPath) isOK, err := backClient.Download(srcPath, targetPath)

20
backend/app/service/cronjob_helper.go
View File

@ -157,9 +157,7 @@ func (u *CronjobService) handleBackup(cronjob *model.Cronjob, startTime time.Tim
return "", err return "", err
} }
if len(backup.BackupPath) != 0 { if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/") itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir)
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
itemFileDir = itemPath + itemFileDir
} }
if _, err = client.Upload(backupDir+"/"+fileName, itemFileDir+"/"+fileName); err != nil { if _, err = client.Upload(backupDir+"/"+fileName, itemFileDir+"/"+fileName); err != nil {
return "", err return "", err
@ -193,9 +191,7 @@ func (u *CronjobService) HandleRmExpired(backType, backupPath, localDir string,
fileItem := file fileItem := file
if cronjob.KeepLocal { if cronjob.KeepLocal {
if len(backupPath) != 0 { if len(backupPath) != 0 {
itemPath := strings.TrimPrefix(backupPath, "/") fileItem = path.Join(strings.TrimPrefix(backupPath, "/") + strings.TrimPrefix(file, localDir+"/"))
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
fileItem = itemPath + strings.TrimPrefix(file, localDir+"/")
} else { } else {
fileItem = strings.TrimPrefix(file, localDir+"/") fileItem = strings.TrimPrefix(file, localDir+"/")
} }
@ -332,9 +328,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, backup model.Back
}() }()
} }
if len(backup.BackupPath) != 0 { if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/") itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir)
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
itemFileDir = itemPath + itemFileDir
} }
if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
return paths, err return paths, err
@ -475,9 +469,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, backup model.BackupAcc
}() }()
} }
if len(backup.BackupPath) != 0 { if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/") itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir)
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
itemFileDir = itemPath + itemFileDir
} }
if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
return paths, err return paths, err
@ -551,9 +543,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, backup model.Backu
}() }()
} }
if len(backup.BackupPath) != 0 { if len(backup.BackupPath) != 0 {
itemPath := strings.TrimPrefix(backup.BackupPath, "/") itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir)
itemPath = strings.TrimSuffix(itemPath, "/") + "/"
itemFileDir = itemPath + itemFileDir
} }
if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
return paths, err return paths, err

133
backend/utils/cloud_storage/client/cos.go
View File

@ -7,44 +7,24 @@ import (
"net/url" "net/url"
"os" "os"
"github.com/1Panel-dev/1Panel/backend/constant"
cosSDK "github.com/tencentyun/cos-go-sdk-v5" cosSDK "github.com/tencentyun/cos-go-sdk-v5"
) )
type cosClient struct { type cosClient struct {
region string scType string
accessKey string client *cosSDK.Client
secretKey string clientWithBucket *cosSDK.Client
scType string
Vars map[string]interface{}
client *cosSDK.Client
} }
func NewCosClient(vars map[string]interface{}) (*cosClient, error) { func NewCosClient(vars map[string]interface{}) (*cosClient, error) {
var accessKey string region := loadParamFromVars("region", true, vars)
var secretKey string accessKey := loadParamFromVars("accessKey", true, vars)
var scType string secretKey := loadParamFromVars("secretKey", true, vars)
var region string bucket := loadParamFromVars("bucket", true, vars)
if _, ok := vars["region"]; ok { scType := loadParamFromVars("scType", true, vars)
region = vars["region"].(string) if len(scType) == 0 {
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["accessKey"]; ok {
accessKey = vars["accessKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["scType"]; ok {
scType = vars["scType"].(string)
} else {
scType = "Standard" scType = "Standard"
} }
if _, ok := vars["secretKey"]; ok {
secretKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
u, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", region)) u, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", region))
b := &cosSDK.BaseURL{BucketURL: u} b := &cosSDK.BaseURL{BucketURL: u}
@ -55,11 +35,23 @@ func NewCosClient(vars map[string]interface{}) (*cosClient, error) {
}, },
}) })
return &cosClient{Vars: vars, client: client, accessKey: accessKey, secretKey: secretKey, scType: scType, region: region}, nil if len(bucket) != 0 {
u2, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", bucket, region))
b2 := &cosSDK.BaseURL{BucketURL: u2}
clientWithBucket := cosSDK.NewClient(b2, &http.Client{
Transport: &cosSDK.AuthorizationTransport{
SecretID: accessKey,
SecretKey: secretKey,
},
})
return &cosClient{client: client, clientWithBucket: clientWithBucket, scType: scType}, nil
}
return &cosClient{client: client, clientWithBucket: nil, scType: scType}, nil
} }
func (cos cosClient) ListBuckets() ([]interface{}, error) { func (c cosClient) ListBuckets() ([]interface{}, error) {
buckets, _, err := cos.client.Service.Get(context.Background()) buckets, _, err := c.client.Service.Get(context.Background())
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -70,34 +62,33 @@ func (cos cosClient) ListBuckets() ([]interface{}, error) {
return datas, nil return datas, nil
} }
func (cos cosClient) Exist(path string) (bool, error) { func (c cosClient) Exist(path string) (bool, error) {
client, err := cos.newClientWithBucket() exist, err := c.clientWithBucket.Object.IsExist(context.Background(), path)
if err != nil {
return false, err
}
exist, err := client.Object.IsExist(context.Background(), path)
if err != nil { if err != nil {
return false, err return false, err
} }
return exist, nil return exist, nil
} }
func (cos cosClient) Delete(path string) (bool, error) { func (c cosClient) Size(path string) (int64, error) {
client, err := cos.newClientWithBucket() data, _, err := c.clientWithBucket.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: path})
if err != nil { if err != nil {
return false, err return 0, err
} }
if _, err := client.Object.Delete(context.Background(), path); err != nil { if len(data.Contents) == 0 {
return 0, fmt.Errorf("no such file %s", path)
}
return data.Contents[0].Size, nil
}
func (c cosClient) Delete(path string) (bool, error) {
if _, err := c.clientWithBucket.Object.Delete(context.Background(), path); err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (cos cosClient) Upload(src, target string) (bool, error) { func (c cosClient) Upload(src, target string) (bool, error) {
client, err := cos.newClientWithBucket()
if err != nil {
return false, err
}
fileInfo, err := os.Stat(src) fileInfo, err := os.Stat(src)
if err != nil { if err != nil {
return false, err return false, err
@ -107,22 +98,22 @@ func (cos cosClient) Upload(src, target string) (bool, error) {
OptIni: &cosSDK.InitiateMultipartUploadOptions{ OptIni: &cosSDK.InitiateMultipartUploadOptions{
ACLHeaderOptions: nil, ACLHeaderOptions: nil,
ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{ ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{
XCosStorageClass: cos.scType, XCosStorageClass: c.scType,
}, },
}, },
PartSize: 200, PartSize: 200,
} }
if _, _, err := client.Object.MultiUpload( if _, _, err := c.clientWithBucket.Object.MultiUpload(
context.Background(), target, src, opt, context.Background(), target, src, opt,
); err != nil { ); err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
if _, err := client.Object.PutFromFile(context.Background(), target, src, &cosSDK.ObjectPutOptions{ if _, err := c.clientWithBucket.Object.PutFromFile(context.Background(), target, src, &cosSDK.ObjectPutOptions{
ACLHeaderOptions: nil, ACLHeaderOptions: nil,
ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{ ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{
XCosStorageClass: cos.scType, XCosStorageClass: c.scType,
}, },
}); err != nil { }); err != nil {
return false, err return false, err
@ -130,31 +121,15 @@ func (cos cosClient) Upload(src, target string) (bool, error) {
return true, nil return true, nil
} }
func (cos cosClient) Download(src, target string) (bool, error) { func (c cosClient) Download(src, target string) (bool, error) {
client, err := cos.newClientWithBucket() if _, err := c.clientWithBucket.Object.Download(context.Background(), src, target, &cosSDK.MultiDownloadOptions{}); err != nil {
if err != nil {
return false, err
}
if _, err := client.Object.Download(context.Background(), src, target, &cosSDK.MultiDownloadOptions{}); err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (cos *cosClient) GetBucket() (string, error) { func (c cosClient) ListObjects(prefix string) ([]string, error) {
if _, ok := cos.Vars["bucket"]; ok { datas, _, err := c.clientWithBucket.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: prefix})
return cos.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (cos cosClient) ListObjects(prefix string) ([]string, error) {
client, err := cos.newClientWithBucket()
if err != nil {
return nil, err
}
datas, _, err := client.Bucket.Get(context.Background(), &cosSDK.BucketGetOptions{Prefix: prefix})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -165,19 +140,3 @@ func (cos cosClient) ListObjects(prefix string) ([]string, error) {
} }
return result, nil return result, nil
} }
func (cos cosClient) newClientWithBucket() (*cosSDK.Client, error) {
bucket, err := cos.GetBucket()
if err != nil {
return nil, err
}
u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", bucket, cos.region))
b := &cosSDK.BaseURL{BucketURL: u}
client := cosSDK.NewClient(b, &http.Client{
Transport: &cosSDK.AuthorizationTransport{
SecretID: cos.accessKey,
SecretKey: cos.secretKey,
},
})
return client, nil
}

20
backend/utils/cloud_storage/client/helper.go Normal file
View File

@ -0,0 +1,20 @@
package client
import (
"fmt"
"github.com/1Panel-dev/1Panel/backend/global"
)
func loadParamFromVars(key string, isString bool, vars map[string]interface{}) string {
if _, ok := vars[key]; !ok {
if key != "bucket" {
global.LOG.Errorf("load param %s from vars failed, err: not exist!", key)
}
return ""
}
if isString {
return vars[key].(string)
}
return fmt.Sprintf("%v", vars[key].(float64))
}

91
backend/utils/cloud_storage/client/kodo.go
View File

@ -4,45 +4,35 @@ import (
"context" "context"
"time" "time"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/1Panel-dev/1Panel/backend/utils/files" "github.com/1Panel-dev/1Panel/backend/utils/files"
"github.com/qiniu/go-sdk/v7/auth" "github.com/qiniu/go-sdk/v7/auth"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage" "github.com/qiniu/go-sdk/v7/storage"
) )
type kodoClient struct { type kodoClient struct {
accessKey string bucket string
secretKey string domain string
Vars map[string]interface{} auth *auth.Credentials
client *storage.BucketManager client *storage.BucketManager
} }
func NewKodoClient(vars map[string]interface{}) (*kodoClient, error) { func NewKodoClient(vars map[string]interface{}) (*kodoClient, error) {
var accessKey string accessKey := loadParamFromVars("accessKey", true, vars)
var secretKey string secretKey := loadParamFromVars("secretKey", true, vars)
if _, ok := vars["accessKey"]; ok { bucket := loadParamFromVars("bucket", true, vars)
accessKey = vars["accessKey"].(string) domain := loadParamFromVars("domain", true, vars)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["secretKey"]; ok {
secretKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
conn := qbox.NewMac(accessKey, secretKey) conn := auth.New(accessKey, secretKey)
cfg := storage.Config{ cfg := storage.Config{
UseHTTPS: false, UseHTTPS: false,
} }
bucketManager := storage.NewBucketManager(conn, &cfg) bucketManager := storage.NewBucketManager(conn, &cfg)
return &kodoClient{Vars: vars, client: bucketManager, accessKey: accessKey, secretKey: secretKey}, nil return &kodoClient{client: bucketManager, auth: conn, bucket: bucket, domain: domain}, nil
} }
func (kodo kodoClient) ListBuckets() ([]interface{}, error) { func (k kodoClient) ListBuckets() ([]interface{}, error) {
buckets, err := kodo.client.Buckets(true) buckets, err := k.client.Buckets(true)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -53,38 +43,33 @@ func (kodo kodoClient) ListBuckets() ([]interface{}, error) {
return datas, nil return datas, nil
} }
func (kodo kodoClient) Exist(path string) (bool, error) { func (k kodoClient) Exist(path string) (bool, error) {
bucket, err := kodo.GetBucket() if _, err := k.client.Stat(k.bucket, path); err != nil {
if err != nil {
return false, err
}
if _, err := kodo.client.Stat(bucket, path); err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (kodo kodoClient) Delete(path string) (bool, error) { func (k kodoClient) Size(path string) (int64, error) {
bucket, err := kodo.GetBucket() file, err := k.client.Stat(k.bucket, path)
if err != nil { if err != nil {
return false, err return 0, err
} }
if err := kodo.client.Delete(bucket, path); err != nil { return file.Fsize, nil
}
func (k kodoClient) Delete(path string) (bool, error) {
if err := k.client.Delete(k.bucket, path); err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (kodo kodoClient) Upload(src, target string) (bool, error) { func (k kodoClient) Upload(src, target string) (bool, error) {
bucket, err := kodo.GetBucket()
if err != nil {
return false, err
}
putPolicy := storage.PutPolicy{ putPolicy := storage.PutPolicy{
Scope: bucket, Scope: k.bucket,
} }
mac := qbox.NewMac(kodo.accessKey, kodo.secretKey) upToken := putPolicy.UploadToken(k.auth)
upToken := putPolicy.UploadToken(mac)
cfg := storage.Config{UseHTTPS: true, UseCdnDomains: false} cfg := storage.Config{UseHTTPS: true, UseCdnDomains: false}
resumeUploader := storage.NewResumeUploaderV2(&cfg) resumeUploader := storage.NewResumeUploaderV2(&cfg)
ret := storage.PutRet{} ret := storage.PutRet{}
@ -95,14 +80,9 @@ func (kodo kodoClient) Upload(src, target string) (bool, error) {
return true, nil return true, nil
} }
func (kodo kodoClient) Download(src, target string) (bool, error) { func (k kodoClient) Download(src, target string) (bool, error) {
mac := auth.New(kodo.accessKey, kodo.secretKey)
if _, ok := kodo.Vars["domain"]; !ok {
return false, constant.ErrInvalidParams
}
domain := kodo.Vars["domain"].(string)
deadline := time.Now().Add(time.Second * 3600).Unix() deadline := time.Now().Add(time.Second * 3600).Unix()
privateAccessURL := storage.MakePrivateURL(mac, domain, src, deadline) privateAccessURL := storage.MakePrivateURL(k.auth, k.domain, src, deadline)
fo := files.NewFileOp() fo := files.NewFileOp()
if err := fo.DownloadFile(privateAccessURL, target); err != nil { if err := fo.DownloadFile(privateAccessURL, target); err != nil {
@ -111,24 +91,11 @@ func (kodo kodoClient) Download(src, target string) (bool, error) {
return true, nil return true, nil
} }
func (kodo *kodoClient) GetBucket() (string, error) { func (k kodoClient) ListObjects(prefix string) ([]string, error) {
if _, ok := kodo.Vars["bucket"]; ok {
return kodo.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (kodo kodoClient) ListObjects(prefix string) ([]string, error) {
bucket, err := kodo.GetBucket()
if err != nil {
return nil, constant.ErrInvalidParams
}
var result []string var result []string
marker := "" marker := ""
for { for {
entries, _, nextMarker, hashNext, err := kodo.client.ListFiles(bucket, prefix, "", marker, 1000) entries, _, nextMarker, hashNext, err := k.client.ListFiles(k.bucket, prefix, "", marker, 1000)
if err != nil { if err != nil {
return nil, err return nil, err
} }

146
backend/utils/cloud_storage/client/minio.go
View File

@ -14,29 +14,15 @@ import (
) )
type minIoClient struct { type minIoClient struct {
Vars map[string]interface{} bucket string
client *minio.Client client *minio.Client
} }
func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) { func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) {
var endpoint string endpoint := loadParamFromVars("endpoint", true, vars)
var accessKeyID string accessKeyID := loadParamFromVars("accessKey", true, vars)
var secretAccessKey string secretAccessKey := loadParamFromVars("secretKey", true, vars)
if _, ok := vars["endpoint"]; ok { bucket := loadParamFromVars("bucket", true, vars)
endpoint = vars["endpoint"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["accessKey"]; ok {
accessKeyID = vars["accessKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["secretKey"]; ok {
secretAccessKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
ssl := strings.Split(endpoint, ":")[0] ssl := strings.Split(endpoint, ":")[0]
if len(ssl) == 0 || (ssl != "https" && ssl != "http") { if len(ssl) == 0 || (ssl != "https" && ssl != "http") {
return nil, constant.ErrInvalidParams return nil, constant.ErrInvalidParams
@ -59,14 +45,11 @@ func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &minIoClient{ return &minIoClient{bucket: bucket, client: client}, nil
client: client,
Vars: vars,
}, nil
} }
func (minIo minIoClient) ListBuckets() ([]interface{}, error) { func (m minIoClient) ListBuckets() ([]interface{}, error) {
buckets, err := minIo.client.ListBuckets(context.Background()) buckets, err := m.client.ListBuckets(context.Background())
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -77,49 +60,44 @@ func (minIo minIoClient) ListBuckets() ([]interface{}, error) {
return result, err return result, err
} }
func (minIo minIoClient) Exist(path string) (bool, error) { func (m minIoClient) Exist(path string) (bool, error) {
if _, ok := minIo.Vars["bucket"]; ok { if _, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{}); err != nil {
_, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.GetObjectOptions{}) return false, err
if err != nil {
return false, err
}
return true, nil
} else {
return false, constant.ErrInvalidParams
} }
return true, nil
} }
func (minIo minIoClient) Delete(path string) (bool, error) { func (m minIoClient) Size(path string) (int64, error) {
if _, ok := minIo.Vars["bucket"]; ok { obj, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{})
object, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.GetObjectOptions{}) if err != nil {
if err != nil { return 0, err
return false, err
}
info, err := object.Stat()
if err != nil {
return false, err
}
err = minIo.client.RemoveObject(context.Background(), minIo.Vars["bucket"].(string), path, minio.RemoveObjectOptions{
GovernanceBypass: true,
VersionID: info.VersionID,
})
if err != nil {
return false, err
}
return true, nil
} else {
return false, constant.ErrInvalidParams
} }
file, err := obj.Stat()
if err != nil {
return 0, err
}
return file.Size, nil
} }
func (minIo minIoClient) Upload(src, target string) (bool, error) { func (m minIoClient) Delete(path string) (bool, error) {
var bucket string object, err := m.client.GetObject(context.Background(), m.bucket, path, minio.GetObjectOptions{})
if _, ok := minIo.Vars["bucket"]; ok { if err != nil {
bucket = minIo.Vars["bucket"].(string) return false, err
} else {
return false, constant.ErrInvalidParams
} }
info, err := object.Stat()
if err != nil {
return false, err
}
if err = m.client.RemoveObject(context.Background(), m.bucket, path, minio.RemoveObjectOptions{
GovernanceBypass: true,
VersionID: info.VersionID,
}); err != nil {
return false, err
}
return true, nil
}
func (m minIoClient) Upload(src, target string) (bool, error) {
file, err := os.Open(src) file, err := os.Open(src)
if err != nil { if err != nil {
return false, err return false, err
@ -130,52 +108,36 @@ func (minIo minIoClient) Upload(src, target string) (bool, error) {
if err != nil { if err != nil {
return false, err return false, err
} }
_, err = minIo.client.PutObject(context.Background(), bucket, target, file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) _, err = m.client.PutObject(context.Background(), m.bucket, target, file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"})
if err != nil { if err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (minIo minIoClient) Download(src, target string) (bool, error) { func (m minIoClient) Download(src, target string) (bool, error) {
if _, ok := minIo.Vars["bucket"]; ok { object, err := m.client.GetObject(context.Background(), m.bucket, src, minio.GetObjectOptions{})
object, err := minIo.client.GetObject(context.Background(), minIo.Vars["bucket"].(string), src, minio.GetObjectOptions{})
if err != nil {
return false, err
}
localFile, err := os.Create(target)
if err != nil {
return false, err
}
if _, err = io.Copy(localFile, object); err != nil {
return false, err
}
return true, nil
} else {
return false, constant.ErrInvalidParams
}
}
func (minIo *minIoClient) GetBucket() (string, error) {
if _, ok := minIo.Vars["bucket"]; ok {
return minIo.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (minIo minIoClient) ListObjects(prefix string) ([]string, error) {
bucket, err := minIo.GetBucket()
if err != nil { if err != nil {
return nil, constant.ErrInvalidParams return false, err
} }
localFile, err := os.Create(target)
if err != nil {
return false, err
}
if _, err = io.Copy(localFile, object); err != nil {
return false, err
}
return true, nil
}
func (m minIoClient) ListObjects(prefix string) ([]string, error) {
opts := minio.ListObjectsOptions{ opts := minio.ListObjectsOptions{
Recursive: true, Recursive: true,
Prefix: prefix, Prefix: prefix,
} }
var result []string var result []string
for object := range minIo.client.ListObjects(context.Background(), bucket, opts) { for object := range m.client.ListObjects(context.Background(), m.bucket, opts) {
if object.Err != nil { if object.Err != nil {
continue continue
} }

94
backend/utils/cloud_storage/client/onedrive.go
View File

@ -23,17 +23,11 @@ import (
) )
type oneDriveClient struct { type oneDriveClient struct {
Vars map[string]interface{}
client odsdk.Client client odsdk.Client
} }
func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) { func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) {
token := "" token := loadParamFromVars("accessToken", true, vars)
if _, ok := vars["accessToken"]; ok {
token = vars["accessToken"].(string)
} else {
return nil, constant.ErrInvalidParams
}
ctx := context.Background() ctx := context.Background()
newToken, err := refreshToken(token) newToken, err := refreshToken(token)
@ -51,13 +45,13 @@ func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) {
return &oneDriveClient{client: *client}, nil return &oneDriveClient{client: *client}, nil
} }
func (onedrive oneDriveClient) ListBuckets() ([]interface{}, error) { func (o oneDriveClient) ListBuckets() ([]interface{}, error) {
return nil, nil return nil, nil
} }
func (onedrive oneDriveClient) Exist(path string) (bool, error) { func (o oneDriveClient) Exist(path string) (bool, error) {
path = "/" + strings.TrimPrefix(path, "/") path = "/" + strings.TrimPrefix(path, "/")
fileID, err := onedrive.loadIDByPath(path) fileID, err := o.loadIDByPath(path)
if err != nil { if err != nil {
return false, err return false, err
} }
@ -65,26 +59,50 @@ func (onedrive oneDriveClient) Exist(path string) (bool, error) {
return len(fileID) != 0, nil return len(fileID) != 0, nil
} }
func (onedrive oneDriveClient) Delete(path string) (bool, error) { func (o oneDriveClient) Size(path string) (int64, error) {
path = "/" + strings.TrimPrefix(path, "/") path = "/" + strings.TrimPrefix(path, "/")
req, err := onedrive.client.NewRequest("DELETE", fmt.Sprintf("me/drive/root:%s", path), nil) pathItem := "root:" + path
if path == "/" {
pathItem = "root"
}
req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil)
if err != nil {
return 0, fmt.Errorf("new request for file id failed, err: %v", err)
}
var driveItem myDriverItem
if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil {
return 0, fmt.Errorf("do request for file id failed, err: %v", err)
}
return driveItem.Size, nil
}
type myDriverItem struct {
Name string `json:"name"`
Id string `json:"id"`
Size int64 `json:"size"`
}
func (o oneDriveClient) Delete(path string) (bool, error) {
path = "/" + strings.TrimPrefix(path, "/")
req, err := o.client.NewRequest("DELETE", fmt.Sprintf("me/drive/root:%s", path), nil)
if err != nil { if err != nil {
return false, fmt.Errorf("new request for delete file failed, err: %v \n", err) return false, fmt.Errorf("new request for delete file failed, err: %v \n", err)
} }
if err := onedrive.client.Do(context.Background(), req, false, nil); err != nil { if err := o.client.Do(context.Background(), req, false, nil); err != nil {
return false, fmt.Errorf("do request for delete file failed, err: %v \n", err) return false, fmt.Errorf("do request for delete file failed, err: %v \n", err)
} }
return true, nil return true, nil
} }
func (onedrive oneDriveClient) Upload(src, target string) (bool, error) { func (o oneDriveClient) Upload(src, target string) (bool, error) {
target = "/" + strings.TrimPrefix(target, "/") target = "/" + strings.TrimPrefix(target, "/")
if _, err := onedrive.loadIDByPath(path.Dir(target)); err != nil { if _, err := o.loadIDByPath(path.Dir(target)); err != nil {
if !strings.Contains(err.Error(), "itemNotFound") { if !strings.Contains(err.Error(), "itemNotFound") {
return false, err return false, err
} }
if err := onedrive.createFolder(path.Dir(target)); err != nil { if err := o.createFolder(path.Dir(target)); err != nil {
return false, fmt.Errorf("create dir before upload failed, err: %v", err) return false, fmt.Errorf("create dir before upload failed, err: %v", err)
} }
} }
@ -105,7 +123,7 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
fileName := fileInfo.Name() fileName := fileInfo.Name()
fileSize := fileInfo.Size() fileSize := fileInfo.Size()
folderID, err := onedrive.loadIDByPath(path.Dir(target)) folderID, err := o.loadIDByPath(path.Dir(target))
if err != nil { if err != nil {
return false, err return false, err
} }
@ -119,13 +137,13 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
DeferCommit bool `json:"deferCommit"` DeferCommit bool `json:"deferCommit"`
}{sessionCreationRequestInside, false} }{sessionCreationRequestInside, false}
sessionCreationReq, err := onedrive.client.NewRequest("POST", apiURL, sessionCreationRequest) sessionCreationReq, err := o.client.NewRequest("POST", apiURL, sessionCreationRequest)
if err != nil { if err != nil {
return false, err return false, err
} }
var sessionCreationResp *NewUploadSessionCreationResponse var sessionCreationResp *NewUploadSessionCreationResponse
err = onedrive.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp) err = o.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp)
if err != nil { if err != nil {
return false, fmt.Errorf("session creation failed %w", err) return false, fmt.Errorf("session creation failed %w", err)
} }
@ -149,11 +167,11 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
bufferLast := buffer[:length] bufferLast := buffer[:length]
buffer = bufferLast buffer = bufferLast
} }
sessionFileUploadReq, err := onedrive.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer)) sessionFileUploadReq, err := o.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer))
if err != nil { if err != nil {
return false, err return false, err
} }
if err := onedrive.client.Do(ctx, sessionFileUploadReq, false, &fileUploadResp); err != nil { if err := o.client.Do(ctx, sessionFileUploadReq, false, &fileUploadResp); err != nil {
return false, err return false, err
} }
} }
@ -164,14 +182,14 @@ func (onedrive oneDriveClient) Upload(src, target string) (bool, error) {
return true, nil return true, nil
} }
func (onedrive oneDriveClient) Download(src, target string) (bool, error) { func (o oneDriveClient) Download(src, target string) (bool, error) {
src = "/" + strings.TrimPrefix(src, "/") src = "/" + strings.TrimPrefix(src, "/")
req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil) req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil)
if err != nil { if err != nil {
return false, fmt.Errorf("new request for file id failed, err: %v", err) return false, fmt.Errorf("new request for file id failed, err: %v", err)
} }
var driveItem *odsdk.DriveItem var driveItem *odsdk.DriveItem
if err := onedrive.client.Do(context.Background(), req, false, &driveItem); err != nil { if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil {
return false, fmt.Errorf("do request for file id failed, err: %v", err) return false, fmt.Errorf("do request for file id failed, err: %v", err)
} }
@ -196,19 +214,19 @@ func (onedrive oneDriveClient) Download(src, target string) (bool, error) {
return true, nil return true, nil
} }
func (onedrive *oneDriveClient) ListObjects(prefix string) ([]string, error) { func (o *oneDriveClient) ListObjects(prefix string) ([]string, error) {
prefix = "/" + strings.TrimPrefix(prefix, "/") prefix = "/" + strings.TrimPrefix(prefix, "/")
folderID, err := onedrive.loadIDByPath(prefix) folderID, err := o.loadIDByPath(prefix)
if err != nil { if err != nil {
return nil, err return nil, err
} }
req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/items/%s/children", folderID), nil) req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/items/%s/children", folderID), nil)
if err != nil { if err != nil {
return nil, fmt.Errorf("new request for list failed, err: %v", err) return nil, fmt.Errorf("new request for list failed, err: %v", err)
} }
var driveItems *odsdk.OneDriveDriveItemsResponse var driveItems *odsdk.OneDriveDriveItemsResponse
if err := onedrive.client.Do(context.Background(), req, false, &driveItems); err != nil { if err := o.client.Do(context.Background(), req, false, &driveItems); err != nil {
return nil, fmt.Errorf("do request for list failed, err: %v", err) return nil, fmt.Errorf("do request for list failed, err: %v", err)
} }
@ -219,17 +237,17 @@ func (onedrive *oneDriveClient) ListObjects(prefix string) ([]string, error) {
return itemList, nil return itemList, nil
} }
func (onedrive *oneDriveClient) loadIDByPath(path string) (string, error) { func (o *oneDriveClient) loadIDByPath(path string) (string, error) {
pathItem := "root:" + path pathItem := "root:" + path
if path == "/" { if path == "/" {
pathItem = "root" pathItem = "root"
} }
req, err := onedrive.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil) req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil)
if err != nil { if err != nil {
return "", fmt.Errorf("new request for file id failed, err: %v", err) return "", fmt.Errorf("new request for file id failed, err: %v", err)
} }
var driveItem *odsdk.DriveItem var driveItem *odsdk.DriveItem
if err := onedrive.client.Do(context.Background(), req, false, &driveItem); err != nil { if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil {
return "", fmt.Errorf("do request for file id failed, err: %v", err) return "", fmt.Errorf("do request for file id failed, err: %v", err)
} }
return driveItem.Id, nil return driveItem.Id, nil
@ -269,18 +287,18 @@ func refreshToken(oldToken string) (string, error) {
return accessToken, nil return accessToken, nil
} }
func (onedrive *oneDriveClient) createFolder(parent string) error { func (o *oneDriveClient) createFolder(parent string) error {
if _, err := onedrive.loadIDByPath(path.Dir(parent)); err != nil { if _, err := o.loadIDByPath(path.Dir(parent)); err != nil {
if !strings.Contains(err.Error(), "itemNotFound") { if !strings.Contains(err.Error(), "itemNotFound") {
return err return err
} }
_ = onedrive.createFolder(path.Dir(parent)) _ = o.createFolder(path.Dir(parent))
} }
item2, err := onedrive.loadIDByPath(path.Dir(parent)) item2, err := o.loadIDByPath(path.Dir(parent))
if err != nil { if err != nil {
return err return err
} }
if _, err := onedrive.client.DriveItems.CreateNewFolder(context.Background(), "", item2, path.Base(parent)); err != nil { if _, err := o.client.DriveItems.CreateNewFolder(context.Background(), "", item2, path.Base(parent)); err != nil {
return err return err
} }
return nil return nil
@ -307,8 +325,8 @@ type DriveItem struct {
WebURL string `json:"webUrl"` WebURL string `json:"webUrl"`
} }
func (onedrive *oneDriveClient) NewSessionFileUploadRequest(absoluteUrl string, grandOffset, grandTotalSize int64, byteReader *bytes.Reader) (*http.Request, error) { func (o *oneDriveClient) NewSessionFileUploadRequest(absoluteUrl string, grandOffset, grandTotalSize int64, byteReader *bytes.Reader) (*http.Request, error) {
apiUrl, err := onedrive.client.BaseURL.Parse(absoluteUrl) apiUrl, err := o.client.BaseURL.Parse(absoluteUrl)
if err != nil { if err != nil {
return nil, err return nil, err
} }

113
backend/utils/cloud_storage/client/oss.go
View File

@ -1,54 +1,36 @@
package client package client
import ( import (
"github.com/1Panel-dev/1Panel/backend/constant" "fmt"
osssdk "github.com/aliyun/aliyun-oss-go-sdk/oss" osssdk "github.com/aliyun/aliyun-oss-go-sdk/oss"
) )
type ossClient struct { type ossClient struct {
scType string scType string
Vars map[string]interface{} bucketStr string
client osssdk.Client client osssdk.Client
} }
func NewOssClient(vars map[string]interface{}) (*ossClient, error) { func NewOssClient(vars map[string]interface{}) (*ossClient, error) {
var endpoint string endpoint := loadParamFromVars("endpoint", true, vars)
var accessKey string accessKey := loadParamFromVars("accessKey", true, vars)
var secretKey string secretKey := loadParamFromVars("secretKey", true, vars)
var scType string bucketStr := loadParamFromVars("bucket", true, vars)
if _, ok := vars["endpoint"]; ok { scType := loadParamFromVars("scType", true, vars)
endpoint = vars["endpoint"].(string) if len(scType) == 0 {
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["accessKey"]; ok {
accessKey = vars["accessKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["scType"]; ok {
scType = vars["scType"].(string)
} else {
scType = "Standard" scType = "Standard"
} }
if _, ok := vars["secretKey"]; ok {
secretKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
client, err := osssdk.New(endpoint, accessKey, secretKey) client, err := osssdk.New(endpoint, accessKey, secretKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &ossClient{
scType: scType, return &ossClient{scType: scType, bucketStr: bucketStr, client: *client}, nil
Vars: vars,
client: *client,
}, nil
} }
func (oss ossClient) ListBuckets() ([]interface{}, error) { func (o ossClient) ListBuckets() ([]interface{}, error) {
response, err := oss.client.ListBuckets() response, err := o.client.ListBuckets()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -59,69 +41,72 @@ func (oss ossClient) ListBuckets() ([]interface{}, error) {
return result, err return result, err
} }
func (oss ossClient) Exist(path string) (bool, error) { func (o ossClient) Exist(path string) (bool, error) {
bucket, err := oss.GetBucket() bucket, err := o.client.Bucket(o.bucketStr)
if err != nil { if err != nil {
return false, err return false, err
} }
return bucket.IsObjectExist(path) return bucket.IsObjectExist(path)
} }
func (oss ossClient) Delete(path string) (bool, error) { func (o ossClient) Size(path string) (int64, error) {
bucket, err := oss.GetBucket() bucket, err := o.client.Bucket(o.bucketStr)
if err != nil {
return 0, err
}
lor, err := bucket.ListObjectsV2(osssdk.Prefix(path))
if err != nil {
return 0, err
}
if len(lor.Objects) == 0 {
return 0, fmt.Errorf("no such file %s", path)
}
return lor.Objects[0].Size, nil
}
func (o ossClient) Delete(path string) (bool, error) {
bucket, err := o.client.Bucket(o.bucketStr)
if err != nil { if err != nil {
return false, err return false, err
} }
err = bucket.DeleteObject(path) if err := bucket.DeleteObject(path); err != nil {
if err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (oss ossClient) Upload(src, target string) (bool, error) { func (o ossClient) Upload(src, target string) (bool, error) {
bucket, err := oss.GetBucket() bucket, err := o.client.Bucket(o.bucketStr)
if err != nil { if err != nil {
return false, err return false, err
} }
err = bucket.UploadFile(target, src, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, ""), osssdk.ObjectStorageClass(osssdk.StorageClassType(oss.scType))) if err := bucket.UploadFile(target, src,
if err != nil { 200*1024*1024,
osssdk.Routines(5),
osssdk.Checkpoint(true, ""),
osssdk.ObjectStorageClass(osssdk.StorageClassType(o.scType))); err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (oss ossClient) Download(src, target string) (bool, error) { func (o ossClient) Download(src, target string) (bool, error) {
bucket, err := oss.GetBucket() bucket, err := o.client.Bucket(o.bucketStr)
if err != nil { if err != nil {
return false, err return false, err
} }
err = bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, "")) if err := bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, "")); err != nil {
if err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (oss *ossClient) GetBucket() (*osssdk.Bucket, error) { func (o *ossClient) ListObjects(prefix string) ([]string, error) {
if _, ok := oss.Vars["bucket"]; ok { bucket, err := o.client.Bucket(o.bucketStr)
bucket, err := oss.client.Bucket(oss.Vars["bucket"].(string))
if err != nil {
return nil, err
}
return bucket, nil
} else {
return nil, constant.ErrInvalidParams
}
}
func (oss *ossClient) ListObjects(prefix string) ([]string, error) {
bucket, err := oss.GetBucket()
if err != nil { if err != nil {
return nil, constant.ErrInvalidParams return nil, err
} }
lor, err := bucket.ListObjects(osssdk.Prefix(prefix)) lor, err := bucket.ListObjectsV2(osssdk.Prefix(prefix))
if err != nil { if err != nil {
return nil, err return nil, err
} }

166
backend/utils/cloud_storage/client/s3.go
View File

@ -3,7 +3,6 @@ package client
import ( import (
"os" "os"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials"
@ -14,41 +13,20 @@ import (
type s3Client struct { type s3Client struct {
scType string scType string
Vars map[string]interface{} bucket string
Sess session.Session Sess session.Session
} }
func NewS3Client(vars map[string]interface{}) (*s3Client, error) { func NewS3Client(vars map[string]interface{}) (*s3Client, error) {
var accessKey string accessKey := loadParamFromVars("accessKey", true, vars)
var secretKey string secretKey := loadParamFromVars("secretKey", true, vars)
var endpoint string endpoint := loadParamFromVars("endpoint", true, vars)
var scType string region := loadParamFromVars("region", true, vars)
var region string bucket := loadParamFromVars("bucket", true, vars)
if _, ok := vars["accessKey"]; ok { scType := loadParamFromVars("scType", true, vars)
accessKey = vars["accessKey"].(string) if len(scType) == 0 {
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["secretKey"]; ok {
secretKey = vars["secretKey"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["scType"]; ok {
scType = vars["scType"].(string)
} else {
scType = "Standard" scType = "Standard"
} }
if _, ok := vars["endpoint"]; ok {
endpoint = vars["endpoint"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["region"]; ok {
region = vars["region"].(string)
} else {
return nil, constant.ErrInvalidParams
}
sess, err := session.NewSession(&aws.Config{ sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
Endpoint: aws.String(endpoint), Endpoint: aws.String(endpoint),
@ -59,16 +37,12 @@ func NewS3Client(vars map[string]interface{}) (*s3Client, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &s3Client{ return &s3Client{scType: scType, bucket: bucket, Sess: *sess}, nil
scType: scType,
Vars: vars,
Sess: *sess,
}, nil
} }
func (s3C s3Client) ListBuckets() ([]interface{}, error) { func (s s3Client) ListBuckets() ([]interface{}, error) {
var result []interface{} var result []interface{}
svc := s3.New(&s3C.Sess) svc := s3.New(&s.Sess)
res, err := svc.ListBuckets(nil) res, err := svc.ListBuckets(nil)
if err != nil { if err != nil {
return nil, err return nil, err
@ -79,17 +53,12 @@ func (s3C s3Client) ListBuckets() ([]interface{}, error) {
return result, nil return result, nil
} }
func (s3C s3Client) Exist(path string) (bool, error) { func (s s3Client) Exist(path string) (bool, error) {
bucket, err := s3C.getBucket() svc := s3.New(&s.Sess)
if err != nil { if _, err := svc.HeadObject(&s3.HeadObjectInput{
return false, err Bucket: &s.bucket,
}
svc := s3.New(&s3C.Sess)
_, err = svc.HeadObject(&s3.HeadObjectInput{
Bucket: &bucket,
Key: &path, Key: &path,
}) }); err != nil {
if err != nil {
if aerr, ok := err.(awserr.RequestFailure); ok { if aerr, ok := err.(awserr.RequestFailure); ok {
if aerr.StatusCode() == 404 { if aerr.StatusCode() == 404 {
return false, nil return false, nil
@ -101,57 +70,53 @@ func (s3C s3Client) Exist(path string) (bool, error) {
return true, nil return true, nil
} }
func (s3C s3Client) Delete(path string) (bool, error) { func (s *s3Client) Size(path string) (int64, error) {
bucket, err := s3C.getBucket() svc := s3.New(&s.Sess)
if err != nil { file, err := svc.GetObject(&s3.GetObjectInput{
return false, err Bucket: &s.bucket,
} Key: &path,
svc := s3.New(&s3C.Sess)
_, err = svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(path)})
if err != nil {
return false, err
}
err = svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(path),
}) })
if err != nil { if err != nil {
return 0, err
}
return *file.ContentLength, nil
}
func (s s3Client) Delete(path string) (bool, error) {
svc := s3.New(&s.Sess)
if _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(path)}); err != nil {
return false, err
}
if err := svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
}); err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (s3C s3Client) Upload(src, target string) (bool, error) { func (s s3Client) Upload(src, target string) (bool, error) {
bucket, err := s3C.getBucket()
if err != nil {
return false, err
}
file, err := os.Open(src) file, err := os.Open(src)
if err != nil { if err != nil {
return false, err return false, err
} }
defer file.Close() defer file.Close()
uploader := s3manager.NewUploader(&s3C.Sess) uploader := s3manager.NewUploader(&s.Sess)
_, err = uploader.Upload(&s3manager.UploadInput{ if _, err := uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(bucket), Bucket: aws.String(s.bucket),
Key: aws.String(target), Key: aws.String(target),
Body: file, Body: file,
StorageClass: &s3C.scType, StorageClass: &s.scType,
}) }); err != nil {
if err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func (s3C s3Client) Download(src, target string) (bool, error) { func (s s3Client) Download(src, target string) (bool, error) {
bucket, err := s3C.getBucket() if _, err := os.Stat(target); err != nil {
if err != nil {
return false, err
}
_, err = os.Stat(target)
if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
os.Remove(target) os.Remove(target)
} else { } else {
@ -163,44 +128,29 @@ func (s3C s3Client) Download(src, target string) (bool, error) {
return false, err return false, err
} }
defer file.Close() defer file.Close()
downloader := s3manager.NewDownloader(&s3C.Sess) downloader := s3manager.NewDownloader(&s.Sess)
_, err = downloader.Download(file, if _, err = downloader.Download(file, &s3.GetObjectInput{
&s3.GetObjectInput{ Bucket: aws.String(s.bucket),
Bucket: aws.String(bucket), Key: aws.String(src),
Key: aws.String(src), }); err != nil {
})
if err != nil {
os.Remove(target) os.Remove(target)
return false, err return false, err
} }
return true, nil return true, nil
} }
func (s3C *s3Client) getBucket() (string, error) { func (s *s3Client) ListObjects(prefix string) ([]string, error) {
if _, ok := s3C.Vars["bucket"]; ok { svc := s3.New(&s.Sess)
return s3C.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (s3C *s3Client) ListObjects(prefix string) ([]string, error) {
bucket, err := s3C.getBucket()
if err != nil {
return nil, constant.ErrInvalidParams
}
svc := s3.New(&s3C.Sess)
var result []string var result []string
if err := svc.ListObjectsPages(&s3.ListObjectsInput{ outputs, err := svc.ListObjects(&s3.ListObjectsInput{
Bucket: &bucket, Bucket: &s.bucket,
Prefix: &prefix, Prefix: &prefix,
}, func(p *s3.ListObjectsOutput, last bool) (shouldContinue bool) { })
for _, obj := range p.Contents { if err != nil {
result = append(result, *obj.Key) return result, err
} }
return true for _, item := range outputs.Contents {
}); err != nil { result = append(result, *item.Key)
return nil, err
} }
return result, nil return result, nil
} }

174
backend/utils/cloud_storage/client/sftp.go
View File

@ -6,53 +6,49 @@ import (
"net" "net"
"os" "os"
"path" "path"
"strconv"
"time" "time"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/pkg/sftp" "github.com/pkg/sftp"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
type sftpClient struct { type sftpClient struct {
Bucket string bucket string
Vars map[string]interface{} connInfo string
client *sftp.Client config *ssh.ClientConfig
} }
func NewSftpClient(vars map[string]interface{}) (*sftpClient, error) { func NewSftpClient(vars map[string]interface{}) (*sftpClient, error) {
if _, ok := vars["address"]; !ok { address := loadParamFromVars("address", true, vars)
return nil, constant.ErrInvalidParams port := loadParamFromVars("port", false, vars)
} password := loadParamFromVars("password", true, vars)
if _, ok := vars["port"].(float64); !ok { username := loadParamFromVars("username", true, vars)
return nil, constant.ErrInvalidParams bucket := loadParamFromVars("bucket", true, vars)
}
if _, ok := vars["password"]; !ok { auth := []ssh.AuthMethod{ssh.Password(password)}
return nil, constant.ErrInvalidParams clientConfig := &ssh.ClientConfig{
} User: username,
if _, ok := vars["username"]; !ok { Auth: auth,
return nil, constant.ErrInvalidParams Timeout: 30 * time.Second,
} HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
var bucket string return nil
if _, ok := vars["bucket"]; ok { },
bucket = vars["bucket"].(string)
} else {
return nil, constant.ErrInvalidParams
} }
port, err := strconv.Atoi(strconv.FormatFloat(vars["port"].(float64), 'G', -1, 64)) return &sftpClient{bucket: bucket, connInfo: fmt.Sprintf("%s:%s", address, port), config: clientConfig}, nil
if err != nil {
return nil, err
}
sftpC, err := connect(vars["username"].(string), vars["password"].(string), vars["address"].(string), port)
if err != nil {
return nil, err
}
return &sftpClient{Bucket: bucket, client: sftpC, Vars: vars}, nil
} }
func (s sftpClient) Upload(src, target string) (bool, error) { func (s sftpClient) Upload(src, target string) (bool, error) {
defer s.client.Close() sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return false, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return false, err
}
defer client.Close()
defer sshClient.Close()
srcFile, err := os.Open(src) srcFile, err := os.Open(src)
if err != nil { if err != nil {
@ -60,18 +56,18 @@ func (s sftpClient) Upload(src, target string) (bool, error) {
} }
defer srcFile.Close() defer srcFile.Close()
targetFilePath := s.Bucket + "/" + target targetFilePath := s.bucket + "/" + target
targetDir, _ := path.Split(targetFilePath) targetDir, _ := path.Split(targetFilePath)
if _, err = s.client.Stat(targetDir); err != nil { if _, err = client.Stat(targetDir); err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
if err = s.client.MkdirAll(targetDir); err != nil { if err = client.MkdirAll(targetDir); err != nil {
return false, err return false, err
} }
} else { } else {
return false, err return false, err
} }
} }
dstFile, err := s.client.Create(targetFilePath) dstFile, err := client.Create(targetFilePath)
if err != nil { if err != nil {
return false, err return false, err
} }
@ -97,8 +93,17 @@ func (s sftpClient) ListBuckets() ([]interface{}, error) {
} }
func (s sftpClient) Download(src, target string) (bool, error) { func (s sftpClient) Download(src, target string) (bool, error) {
defer s.client.Close() sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
srcFile, err := s.client.Open(s.Bucket + "/" + src) if err != nil {
return false, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return false, err
}
defer client.Close()
defer sshClient.Close()
srcFile, err := client.Open(s.bucket + "/" + src)
if err != nil { if err != nil {
return false, err return false, err
} }
@ -117,8 +122,18 @@ func (s sftpClient) Download(src, target string) (bool, error) {
} }
func (s sftpClient) Exist(path string) (bool, error) { func (s sftpClient) Exist(path string) (bool, error) {
defer s.client.Close() sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
srcFile, err := s.client.Open(s.Bucket + "/" + path) if err != nil {
return false, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return false, err
}
defer client.Close()
defer sshClient.Close()
srcFile, err := client.Open(s.bucket + "/" + path)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return false, nil return false, nil
@ -130,48 +145,57 @@ func (s sftpClient) Exist(path string) (bool, error) {
return true, err return true, err
} }
func (s sftpClient) Size(path string) (int64, error) {
sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
if err != nil {
return 0, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return 0, err
}
defer client.Close()
defer sshClient.Close()
files, err := client.Stat(s.bucket + "/" + path)
if err != nil {
return 0, err
}
return files.Size(), nil
}
func (s sftpClient) Delete(filePath string) (bool, error) { func (s sftpClient) Delete(filePath string) (bool, error) {
defer s.client.Close() sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
targetFilePath := s.Bucket + "/" + filePath if err != nil {
if err := s.client.Remove(targetFilePath); err != nil { return false, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return false, err
}
defer client.Close()
defer sshClient.Close()
targetFilePath := s.bucket + "/" + filePath
if err := client.Remove(targetFilePath); err != nil {
return false, err return false, err
} }
return true, nil return true, nil
} }
func connect(user, password, host string, port int) (*sftp.Client, error) {
var (
auth []ssh.AuthMethod
addr string
clientConfig *ssh.ClientConfig
sshClient *ssh.Client
sftpClient *sftp.Client
err error
)
auth = make([]ssh.AuthMethod, 0)
auth = append(auth, ssh.Password(password))
clientConfig = &ssh.ClientConfig{
User: user,
Auth: auth,
Timeout: 30 * time.Second,
HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
return nil
},
}
addr = fmt.Sprintf("%s:%d", host, port)
if sshClient, err = ssh.Dial("tcp", addr, clientConfig); err != nil {
return nil, err
}
if sftpClient, err = sftp.NewClient(sshClient); err != nil {
return nil, err
}
return sftpClient, nil
}
func (s sftpClient) ListObjects(prefix string) ([]string, error) { func (s sftpClient) ListObjects(prefix string) ([]string, error) {
defer s.client.Close() sshClient, err := ssh.Dial("tcp", s.connInfo, s.config)
files, err := s.client.ReadDir(s.Bucket + "/" + prefix) if err != nil {
return nil, err
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return nil, err
}
defer client.Close()
defer sshClient.Close()
files, err := client.ReadDir(s.bucket + "/" + prefix)
if err != nil { if err != nil {
return nil, err return nil, err
} }

48
backend/utils/cloud_storage/client/webdav.go
View File

@ -5,52 +5,26 @@ import (
"io" "io"
"os" "os"
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/studio-b12/gowebdav" "github.com/studio-b12/gowebdav"
) )
type webDAVClient struct { type webDAVClient struct {
Bucket string Bucket string
client *gowebdav.Client client *gowebdav.Client
Vars map[string]interface{}
} }
func NewWebDAVClient(vars map[string]interface{}) (*webDAVClient, error) { func NewWebDAVClient(vars map[string]interface{}) (*webDAVClient, error) {
var ( address := loadParamFromVars("address", true, vars)
address string port := loadParamFromVars("port", false, vars)
username string password := loadParamFromVars("password", true, vars)
password string username := loadParamFromVars("username", true, vars)
bucket string bucket := loadParamFromVars("bucket", true, vars)
)
if _, ok := vars["address"]; ok {
address = vars["address"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["port"].(float64); !ok {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["username"]; ok {
username = vars["username"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["password"]; ok {
password = vars["password"].(string)
} else {
return nil, constant.ErrInvalidParams
}
if _, ok := vars["bucket"]; ok {
bucket = vars["bucket"].(string)
} else {
return nil, constant.ErrInvalidParams
}
client := gowebdav.NewClient(fmt.Sprintf("%s:%v", address, vars["port"]), username, password) client := gowebdav.NewClient(fmt.Sprintf("%s:%s", address, port), username, password)
if err := client.Connect(); err != nil { if err := client.Connect(); err != nil {
return nil, err return nil, err
} }
return &webDAVClient{Vars: vars, Bucket: bucket, client: client}, nil return &webDAVClient{Bucket: bucket, client: client}, nil
} }
func (s webDAVClient) Upload(src, target string) (bool, error) { func (s webDAVClient) Upload(src, target string) (bool, error) {
@ -115,6 +89,14 @@ func (s webDAVClient) Exist(path string) (bool, error) {
return true, nil return true, nil
} }
func (s webDAVClient) Size(path string) (int64, error) {
file, err := s.client.Stat(s.Bucket + "/" + path)
if err != nil {
return 0, err
}
return file.Size(), nil
}
func (s webDAVClient) Delete(filePath string) (bool, error) { func (s webDAVClient) Delete(filePath string) (bool, error) {
if err := s.client.Remove(s.Bucket + "/" + filePath); err != nil { if err := s.client.Remove(s.Bucket + "/" + filePath); err != nil {
return false, err return false, err

2
backend/utils/cloud_storage/cloud_storage_client.go
View File

@ -12,6 +12,8 @@ type CloudStorageClient interface {
Delete(path string) (bool, error) Delete(path string) (bool, error)
Upload(src, target string) (bool, error) Upload(src, target string) (bool, error)
Download(src, target string) (bool, error) Download(src, target string) (bool, error)
Size(path string) (int64, error)
} }
func NewCloudStorageClient(backupType string, vars map[string]interface{}) (CloudStorageClient, error) { func NewCloudStorageClient(backupType string, vars map[string]interface{}) (CloudStorageClient, error) {

23
frontend/src/components/backup/index.vue
View File

@ -27,6 +27,14 @@
</template> </template>
<el-table-column type="selection" fix /> <el-table-column type="selection" fix />
<el-table-column :label="$t('commons.table.name')" prop="fileName" show-overflow-tooltip /> <el-table-column :label="$t('commons.table.name')" prop="fileName" show-overflow-tooltip />
<el-table-column :label="$t('file.size')" prop="size" show-overflow-tooltip>
<template #default="{ row }">
<span v-if="row.size">
{{ computeSize(row.size) }}
</span>
<span v-else>-</span>
</template>
</el-table-column>
<el-table-column :label="$t('database.source')" prop="backupType"> <el-table-column :label="$t('database.source')" prop="backupType">
<template #default="{ row }"> <template #default="{ row }">
<span v-if="row.source"> <span v-if="row.source">
@ -52,7 +60,7 @@
<script lang="ts" setup> <script lang="ts" setup>
import { reactive, ref } from 'vue'; import { reactive, ref } from 'vue';
import OpDialog from '@/components/del-dialog/index.vue'; import OpDialog from '@/components/del-dialog/index.vue';
import { dateFormat, downloadFile } from '@/utils/util'; import { computeSize, dateFormat, downloadFile } from '@/utils/util';
import { handleBackup, handleRecover } from '@/api/modules/setting'; import { handleBackup, handleRecover } from '@/api/modules/setting';
import i18n from '@/lang'; import i18n from '@/lang';
import DrawerHeader from '@/components/drawer-header/index.vue'; import DrawerHeader from '@/components/drawer-header/index.vue';
@ -101,9 +109,16 @@ const search = async () => {
name: name.value, name: name.value,
detailName: detailName.value, detailName: detailName.value,
}; };
const res = await searchBackupRecords(params); loading.value = true;
data.value = res.data.items || []; await searchBackupRecords(params)
paginationConfig.total = res.data.total; .then((res) => {
loading.value = false;
data.value = res.data.items || [];
paginationConfig.total = res.data.total;
})
.catch(() => {
loading.value = false;
});
}; };
const onBackup = async () => { const onBackup = async () => {

2
frontend/src/views/setting/backup-account/operate/index.vue
View File

@ -211,7 +211,7 @@
:rules="Rules.requiredInput" :rules="Rules.requiredInput"
> >
<el-input v-model.trim="dialogData.rowData!.varsJson['address']" /> <el-input v-model.trim="dialogData.rowData!.varsJson['address']" />
<span class="input-help">https://172.16.10.111</span> <span class="input-help">http://172.16.10.111</span>
</el-form-item> </el-form-item>
<el-form-item :label="$t('commons.table.port')" prop="varsJson.port" :rules="[Rules.port]"> <el-form-item :label="$t('commons.table.port')" prop="varsJson.port" :rules="[Rules.port]">
<el-input-number <el-input-number