gh_mirrors/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-03-13 17:24:44 +08:00
Code

feat: oss 备份保存个数限制

Browse Source
This commit is contained in:
ssongliu 2022-09-28 00:08:21 +08:00 committed by ssongliu
parent 7d14d37057
commit a0c73e8eab
13 changed files with 198 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
backend/app/api/v1/cronjob.go
View File

@ -152,3 +152,22 @@ func (b *BaseApi) LoadRecordDetail(c *gin.Context) {
}
helper.SuccessWithData(c, string(buf))
}
func (b *BaseApi) TargetDownload(c *gin.Context) {
var req dto.CronjobDownload
if err := c.ShouldBindJSON(&req); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
if err := global.VALID.Struct(req); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
dir, err := cronjobService.Download(req)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithData(c, dir)
}

5
backend/app/dto/cronjob.go
View File

@ -44,6 +44,11 @@ type CronjobUpdateStatus struct {
Status string `json:"status" validate:"required"`
}
type CronjobDownload struct {
RecordID uint `json:"recordID" validate:"required"`
BackupAccountID uint `json:"backupAccountID" validate:"required"`
}
type DetailFile struct {
Path string `json:"path" validate:"required"`
}

11
backend/app/repo/cronjob.go
View File

@ -13,6 +13,7 @@ type CronjobRepo struct{}
type ICronjobRepo interface {
Get(opts ...DBOption) (model.Cronjob, error)
GetRecord(opts ...DBOption) (model.JobRecords, error)
List(opts ...DBOption) ([]model.Cronjob, error)
Page(limit, offset int, opts ...DBOption) (int64, []model.Cronjob, error)
Create(cronjob *model.Cronjob) error
@ -40,6 +41,16 @@ func (u *CronjobRepo) Get(opts ...DBOption) (model.Cronjob, error) {
return cronjob, err
}
func (u *CronjobRepo) GetRecord(opts ...DBOption) (model.JobRecords, error) {
var record model.JobRecords
db := global.DB
for _, opt := range opts {
db = opt(db)
}
err := db.First(&record).Error
return record, err
}
func (u *CronjobRepo) List(opts ...DBOption) ([]model.Cronjob, error) {
var cronjobs []model.Cronjob
db := global.DB.Model(&model.Cronjob{})

55
backend/app/service/cornjob.go
View File

@ -82,6 +82,53 @@ func (u *CronjobService) SearchRecords(search dto.SearchRecord) (int64, interfac
return total, dtoCronjobs, err
}
func (u *CronjobService) Download(down dto.CronjobDownload) (string, error) {
record, _ := cronjobRepo.GetRecord(commonRepo.WithByID(down.RecordID))
if record.ID != 0 {
return "", constant.ErrRecordExist
}
cronjob, _ := cronjobRepo.Get(commonRepo.WithByID(record.CronjobID))
if cronjob.ID != 0 {
return "", constant.ErrRecordExist
}
backup, _ := backupRepo.Get(commonRepo.WithByID(down.BackupAccountID))
if cronjob.ID != 0 {
return "", constant.ErrRecordExist
}
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
return "", err
}
varMap["type"] = backup.Type
if backup.Type != "LOCAL" {
varMap["bucket"] = backup.Bucket
switch backup.Type {
case constant.Sftp:
varMap["password"] = backup.Credential
case constant.OSS, constant.S3, constant.MinIo:
varMap["secretKey"] = backup.Credential
}
backClient, err := cloud_storage.NewCloudStorageClient(varMap)
if err != nil {
return "", fmt.Errorf("new cloud storage client failed, err: %v", err)
}
name := fmt.Sprintf("%s/%s/%s.tar.gz", cronjob.Type, cronjob.Name, record.StartTime.Format("20060102150405"))
if cronjob.Type == "database" {
name = fmt.Sprintf("%s/%s/%s.gz", cronjob.Type, cronjob.Name, record.StartTime.Format("20060102150405"))
}
isOK, err := backClient.Download(name, constant.DownloadDir)
if !isOK {
return "", fmt.Errorf("cloud storage download failed, err: %v", err)
}
return constant.DownloadDir, nil
}
if _, ok := varMap["dir"]; !ok {
return "", errors.New("load local backup dir failed")
}
return fmt.Sprintf("%v/%s/%s", varMap["dir"], cronjob.Type, cronjob.Name), nil
}
func (u *CronjobService) Create(cronjobDto dto.CronjobCreate) error {
cronjob, _ := cronjobRepo.Get(commonRepo.WithByName(cronjobDto.Name))
if cronjob.ID != 0 {
@ -387,6 +434,14 @@ func tarWithExclude(cronjob *model.Cronjob, startTime time.Time) ([]byte, error)
if !isOK {
return nil, fmt.Errorf("cloud storage upload failed, err: %v", err)
}
currentObjs, _ := backClient.ListObjects(fmt.Sprintf("%s/%s/", cronjob.Type, cronjob.Name))
if len(currentObjs) > int(cronjob.RetainCopies) {
for i := 0; i < len(currentObjs)-int(cronjob.RetainCopies); i++ {
if path, ok := currentObjs[i].(string); ok {
_, _ = backClient.Delete(path)
}
}
}
if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TmpDir, cronjob.Type, cronjob.Name, cronjob.ID)); err != nil {
global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID, err)
}

5
backend/constant/cronjob.go
View File

@ -1,6 +1,7 @@
package constant
const (
TmpDir = "/opt/1Panel/task/tmp/"
TaskDir = "/opt/1Panel/task/"
TmpDir = "/opt/1Panel/task/tmp/"
TaskDir = "/opt/1Panel/task/"
DownloadDir = "/opt/1Panel/download"
)

1
backend/router/ro_cronjob.go
View File

@ -18,6 +18,7 @@ func (s *CronjobRouter) InitCronjobRouter(Router *gin.RouterGroup) {
withRecordRouter.POST("/del", baseApi.DeleteCronjob)
withRecordRouter.PUT(":id", baseApi.UpdateCronjob)
withRecordRouter.POST("/status", baseApi.UpdateCronjobStatus)
withRecordRouter.POST("/download", baseApi.TargetDownload)
cmdRouter.POST("/search", baseApi.SearchCronjob)
cmdRouter.POST("/search/records", baseApi.SearchJobRecords)
cmdRouter.POST("/search/detail", baseApi.LoadRecordDetail)

4
backend/utils/cloud_storage/client/minio.go
View File

@ -156,3 +156,7 @@ func (minIo minIoClient) Download(src, target string) (bool, error) {
return false, constant.ErrInvalidParams
}
}
func (minIo minIoClient) ListObjects(prefix string) ([]interface{}, error) {
return nil, nil
}

28
backend/utils/cloud_storage/client/oss.go
View File

@ -2,12 +2,12 @@ package client
import (
"github.com/1Panel-dev/1Panel/constant"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
osssdk "github.com/aliyun/aliyun-oss-go-sdk/oss"
)
type ossClient struct {
Vars map[string]interface{}
client oss.Client
client osssdk.Client
}
func NewOssClient(vars map[string]interface{}) (*ossClient, error) {
@ -29,7 +29,7 @@ func NewOssClient(vars map[string]interface{}) (*ossClient, error) {
} else {
return nil, constant.ErrInvalidParams
}
client, err := oss.New(endpoint, accessKey, secretKey)
client, err := osssdk.New(endpoint, accessKey, secretKey)
if err != nil {
return nil, err
}
@ -96,7 +96,7 @@ func (oss ossClient) Download(src, target string) (bool, error) {
return true, nil
}
func (oss *ossClient) GetBucket() (*oss.Bucket, error) {
func (oss *ossClient) GetBucket() (*osssdk.Bucket, error) {
if _, ok := oss.Vars["bucket"]; ok {
bucket, err := oss.client.Bucket(oss.Vars["bucket"].(string))
if err != nil {
@ -107,3 +107,23 @@ func (oss *ossClient) GetBucket() (*oss.Bucket, error) {
return nil, constant.ErrInvalidParams
}
}
func (oss *ossClient) ListObjects(prefix string) ([]interface{}, error) {
if _, ok := oss.Vars["bucket"]; ok {
bucket, err := oss.client.Bucket(oss.Vars["bucket"].(string))
if err != nil {
return nil, err
}
lor, err := bucket.ListObjects(osssdk.Prefix(prefix))
if err != nil {
return nil, err
}
var result []interface{}
for _, obj := range lor.Objects {
result = append(result, obj.Key)
}
return result, nil
} else {
return nil, constant.ErrInvalidParams
}
}

63
backend/utils/cloud_storage/client/oss_test.go Normal file
View File

@ -0,0 +1,63 @@
package service
import (
"encoding/json"
"fmt"
"testing"
"github.com/1Panel-dev/1Panel/app/model"
"github.com/1Panel-dev/1Panel/constant"
"github.com/1Panel-dev/1Panel/global"
"github.com/1Panel-dev/1Panel/init/db"
"github.com/1Panel-dev/1Panel/init/log"
"github.com/1Panel-dev/1Panel/init/viper"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
)
func TestCron(t *testing.T) {
viper.Init()
log.Init()
db.Init()
var backup model.BackupAccount
if err := global.DB.Where("id = ?", 2).First(&backup).Error; err != nil {
fmt.Println(err)
}
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
fmt.Println(err)
}
varMap["type"] = backup.Type
varMap["bucket"] = backup.Bucket
switch backup.Type {
case constant.Sftp:
varMap["password"] = backup.Credential
case constant.OSS, constant.S3, constant.MinIo:
varMap["secretKey"] = backup.Credential
}
endpoint := varMap["endpoint"].(string)
accessKey := varMap["accessKey"].(string)
secretKey := varMap["secretKey"].(string)
client, err := oss.New(endpoint, accessKey, secretKey)
if err != nil {
fmt.Println(err)
}
bucket, err := client.Bucket(backup.Bucket)
if err != nil {
fmt.Println(err)
}
lor, err := bucket.ListObjects(oss.Prefix("directory/directory-test1/"))
if err != nil {
fmt.Println(err)
}
fmt.Println("my objects:", getObjectsFormResponse(lor))
}
func getObjectsFormResponse(lor oss.ListObjectsResult) string {
var output string
for _, object := range lor.Objects {
output += object.Key + " "
}
return output
}

4
backend/utils/cloud_storage/client/s3.go
View File

@ -175,3 +175,7 @@ func (s3C *s3Client) getBucket() (string, error) {
return "", constant.ErrInvalidParams
}
}
func (s3C *s3Client) ListObjects(prefix string) ([]interface{}, error) {
return nil, nil
}

4
backend/utils/cloud_storage/client/sftp.go
View File

@ -210,3 +210,7 @@ func (s sftpClient) getBucket() (string, error) {
return "", constant.ErrInvalidParams
}
}
func (s sftpClient) ListObjects(prefix string) ([]interface{}, error) {
return nil, nil
}

1
backend/utils/cloud_storage/cloud_storage_client.go
View File

@ -7,6 +7,7 @@ import (
type CloudStorageClient interface {
ListBuckets() ([]interface{}, error)
ListObjects(prefix string) ([]interface{}, error)
Exist(path string) (bool, error)
Delete(path string) (bool, error)
Upload(src, target string) (bool, error)

4
frontend/src/api/interface/cronjob.ts
View File

@ -61,6 +61,10 @@ export namespace Cronjob {
id: number;
status: string;
}
export interface UpdateStatus {
recordID: number;
backupAccountID: number;
}
export interface SearchRecord extends ReqPage {
cronjobID: number;
startTime: Date;