From c5e9a2e085456e75d4669c89af86793112c10f8f Mon Sep 17 00:00:00 2001 From: ssongliu Date: Wed, 28 Sep 2022 18:11:36 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20sftp=E3=80=81s3=E3=80=81minio=20?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E5=A4=87=E4=BB=BD=E5=8A=9F=E8=83=BD=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/v1/cronjob.go | 4 +- backend/app/dto/cronjob.go | 6 +- backend/app/model/cronjob.go | 15 ++- backend/app/repo/cronjob.go | 32 +++-- backend/app/service/cornjob.go | 117 +++++++++++++----- backend/constant/cronjob.go | 2 +- backend/cron/cron.go | 9 ++ backend/utils/cloud_storage/client/minio.go | 31 ++++- backend/utils/cloud_storage/client/oss.go | 26 ++-- .../utils/cloud_storage/client/oss_test.go | 26 +++- backend/utils/cloud_storage/client/s3.go | 20 ++- backend/utils/cloud_storage/client/sftp.go | 23 +++- frontend/src/api/interface/cronjob.ts | 8 +- frontend/src/api/modules/cronjob.ts | 4 + frontend/src/lang/modules/en.ts | 58 ++++++++- frontend/src/lang/modules/zh.ts | 6 +- frontend/src/routers/modules/cronjob.ts | 2 +- frontend/src/utils/util.ts | 16 +++ frontend/src/views/cronjob/index.vue | 12 +- frontend/src/views/cronjob/operate/index.vue | 8 +- frontend/src/views/cronjob/options.ts | 2 +- frontend/src/views/cronjob/record/index.vue | 54 ++++++-- frontend/src/views/setting/tabs/backup.vue | 12 +- 23 files changed, 387 insertions(+), 106 deletions(-) diff --git a/backend/app/api/v1/cronjob.go b/backend/app/api/v1/cronjob.go index c216eaa4f..62b0c5ca2 100644 --- a/backend/app/api/v1/cronjob.go +++ b/backend/app/api/v1/cronjob.go @@ -164,10 +164,10 @@ func (b *BaseApi) TargetDownload(c *gin.Context) { return } - dir, err := cronjobService.Download(req) + filePath, err := cronjobService.Download(req) if err != nil { helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) return } - helper.SuccessWithData(c, dir) + c.File(filePath) } diff --git a/backend/app/dto/cronjob.go b/backend/app/dto/cronjob.go index 33af95ebb..08cc03e4c 100644 --- a/backend/app/dto/cronjob.go +++ b/backend/app/dto/cronjob.go @@ -18,7 +18,7 @@ type CronjobCreate struct { URL string `json:"url"` SourceDir string `json:"sourceDir"` TargetDirID int `json:"targetDirID"` - RetainCopies int `json:"retainCopies" validate:"number,min=1"` + RetainDays int `json:"retainDays" validate:"number,min=1"` } type CronjobUpdate struct { @@ -36,7 +36,7 @@ type CronjobUpdate struct { URL string `json:"url"` SourceDir string `json:"sourceDir"` TargetDirID int `json:"targetDirID"` - RetainCopies int `json:"retainCopies" validate:"number,min=1"` + RetainDays int `json:"retainDays" validate:"number,min=1"` } type CronjobUpdateStatus struct { @@ -71,7 +71,7 @@ type CronjobInfo struct { SourceDir string `json:"sourceDir"` TargetDir string `json:"targetDir"` TargetDirID int `json:"targetDirID"` - RetainCopies int `json:"retainCopies"` + RetainDays int `json:"retainDays"` Status string `json:"status"` } diff --git a/backend/app/model/cronjob.go b/backend/app/model/cronjob.go index 35d40a527..18fa3f586 100644 --- a/backend/app/model/cronjob.go +++ b/backend/app/model/cronjob.go @@ -21,7 +21,7 @@ type Cronjob struct { SourceDir string `gorm:"type:varchar(256)" json:"sourceDir"` TargetDirID uint64 `gorm:"type:decimal" json:"targetDirID"` ExclusionRules string `gorm:"longtext" json:"exclusionRules"` - RetainCopies uint64 `gorm:"type:decimal" json:"retainCopies"` + RetainDays uint64 `gorm:"type:decimal" json:"retainDays"` Status string `gorm:"type:varchar(64)" json:"status"` EntryID uint64 `gorm:"type:decimal" json:"entryID"` @@ -31,11 +31,10 @@ type Cronjob struct { type JobRecords struct { BaseModel - CronjobID uint `gorm:"type:varchar(64);not null" json:"cronjobID"` - StartTime time.Time `gorm:"type:datetime" json:"startTime"` - Interval float64 `gorm:"type:float" json:"interval"` - Records string `gorm:"longtext" json:"records"` - Status string `gorm:"type:varchar(64)" json:"status"` - Message string `gorm:"longtext" json:"message"` - TargetPath string `gorm:"type:varchar(256)" json:"targetPath"` + CronjobID uint `gorm:"type:varchar(64);not null" json:"cronjobID"` + StartTime time.Time `gorm:"type:datetime" json:"startTime"` + Interval float64 `gorm:"type:float" json:"interval"` + Records string `gorm:"longtext" json:"records"` + Status string `gorm:"type:varchar(64)" json:"status"` + Message string `gorm:"longtext" json:"message"` } diff --git a/backend/app/repo/cronjob.go b/backend/app/repo/cronjob.go index 66a38bc52..dfa8e60c8 100644 --- a/backend/app/repo/cronjob.go +++ b/backend/app/repo/cronjob.go @@ -14,6 +14,7 @@ type CronjobRepo struct{} type ICronjobRepo interface { Get(opts ...DBOption) (model.Cronjob, error) GetRecord(opts ...DBOption) (model.JobRecords, error) + ListRecord(opts ...DBOption) ([]model.JobRecords, error) List(opts ...DBOption) ([]model.Cronjob, error) Page(limit, offset int, opts ...DBOption) (int64, []model.Cronjob, error) Create(cronjob *model.Cronjob) error @@ -22,7 +23,7 @@ type ICronjobRepo interface { Save(id uint, cronjob model.Cronjob) error Update(id uint, vars map[string]interface{}) error Delete(opts ...DBOption) error - DeleteRecord(jobID uint) error + DeleteRecord(opts ...DBOption) error StartRecords(cronjobID uint, targetPath string) model.JobRecords EndRecords(record model.JobRecords, status, message, records string) } @@ -57,8 +58,16 @@ func (u *CronjobRepo) List(opts ...DBOption) ([]model.Cronjob, error) { for _, opt := range opts { db = opt(db) } - count := int64(0) - db = db.Count(&count) + err := db.Find(&cronjobs).Error + return cronjobs, err +} + +func (u *CronjobRepo) ListRecord(opts ...DBOption) ([]model.JobRecords, error) { + var cronjobs []model.JobRecords + db := global.DB.Model(&model.JobRecords{}) + for _, opt := range opts { + db = opt(db) + } err := db.Find(&cronjobs).Error return cronjobs, err } @@ -71,7 +80,7 @@ func (u *CronjobRepo) Page(page, size int, opts ...DBOption) (int64, []model.Cro } count := int64(0) db = db.Count(&count) - err := db.Order("created_at").Limit(size).Offset(size * (page - 1)).Find(&cronjobs).Error + err := db.Order("created_at desc").Limit(size).Offset(size * (page - 1)).Find(&cronjobs).Error return count, cronjobs, err } @@ -83,7 +92,7 @@ func (u *CronjobRepo) PageRecords(page, size int, opts ...DBOption) (int64, []mo } count := int64(0) db = db.Count(&count) - err := db.Order("created_at").Limit(size).Offset(size * (page - 1)).Find(&cronjobs).Error + err := db.Order("created_at desc").Limit(size).Offset(size * (page - 1)).Find(&cronjobs).Error return count, cronjobs, err } @@ -96,6 +105,11 @@ func (c *CronjobRepo) WithByDate(startTime, endTime time.Time) DBOption { return g.Where("start_time > ? AND start_time < ?", startTime, endTime) } } +func (c *CronjobRepo) WithByStartDate(startTime time.Time) DBOption { + return func(g *gorm.DB) *gorm.DB { + return g.Where("start_time < ?", startTime) + } +} func (c *CronjobRepo) WithByJobID(id int) DBOption { return func(g *gorm.DB) *gorm.DB { return g.Where("cronjob_id = ?", id) @@ -137,6 +151,10 @@ func (u *CronjobRepo) Delete(opts ...DBOption) error { } return db.Delete(&model.Cronjob{}).Error } -func (u *CronjobRepo) DeleteRecord(jobID uint) error { - return global.DB.Where("cronjob_id = ?", jobID).Delete(&model.JobRecords{}).Error +func (u *CronjobRepo) DeleteRecord(opts ...DBOption) error { + db := global.DB + for _, opt := range opts { + db = opt(db) + } + return db.Delete(&model.JobRecords{}).Error } diff --git a/backend/app/service/cornjob.go b/backend/app/service/cornjob.go index 72a7dc365..b2cb73b15 100644 --- a/backend/app/service/cornjob.go +++ b/backend/app/service/cornjob.go @@ -84,16 +84,16 @@ func (u *CronjobService) SearchRecords(search dto.SearchRecord) (int64, interfac func (u *CronjobService) Download(down dto.CronjobDownload) (string, error) { record, _ := cronjobRepo.GetRecord(commonRepo.WithByID(down.RecordID)) - if record.ID != 0 { - return "", constant.ErrRecordExist + if record.ID == 0 { + return "", constant.ErrRecordNotFound } cronjob, _ := cronjobRepo.Get(commonRepo.WithByID(record.CronjobID)) - if cronjob.ID != 0 { - return "", constant.ErrRecordExist + if cronjob.ID == 0 { + return "", constant.ErrRecordNotFound } backup, _ := backupRepo.Get(commonRepo.WithByID(down.BackupAccountID)) - if cronjob.ID != 0 { - return "", constant.ErrRecordExist + if cronjob.ID == 0 { + return "", constant.ErrRecordNotFound } varMap := make(map[string]interface{}) if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil { @@ -112,21 +112,35 @@ func (u *CronjobService) Download(down dto.CronjobDownload) (string, error) { if err != nil { return "", fmt.Errorf("new cloud storage client failed, err: %v", err) } - name := fmt.Sprintf("%s/%s/%s.tar.gz", cronjob.Type, cronjob.Name, record.StartTime.Format("20060102150405")) + commonDir := fmt.Sprintf("%s/%s/", cronjob.Type, cronjob.Name) + name := fmt.Sprintf("%s%s.tar.gz", commonDir, record.StartTime.Format("20060102150405")) if cronjob.Type == "database" { - name = fmt.Sprintf("%s/%s/%s.gz", cronjob.Type, cronjob.Name, record.StartTime.Format("20060102150405")) + name = fmt.Sprintf("%s%s.gz", commonDir, record.StartTime.Format("20060102150405")) } - isOK, err := backClient.Download(name, constant.DownloadDir) - if !isOK { - return "", fmt.Errorf("cloud storage download failed, err: %v", err) + tempPath := fmt.Sprintf("%s%s", constant.DownloadDir, commonDir) + if _, err := os.Stat(tempPath); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(tempPath, os.ModePerm); err != nil { + fmt.Println(err) + } } - return constant.DownloadDir, nil + targetPath := tempPath + strings.ReplaceAll(name, commonDir, "") + if _, err = os.Stat(targetPath); err != nil && os.IsNotExist(err) { + isOK, err := backClient.Download(name, targetPath) + if !isOK { + return "", fmt.Errorf("cloud storage download failed, err: %v", err) + } + } + return targetPath, nil } if _, ok := varMap["dir"]; !ok { return "", errors.New("load local backup dir failed") } - return fmt.Sprintf("%v/%s/%s", varMap["dir"], cronjob.Type, cronjob.Name), nil - + dir := fmt.Sprintf("%v/%s/%s/", varMap["dir"], cronjob.Type, cronjob.Name) + name := fmt.Sprintf("%s%s.tar.gz", dir, record.StartTime.Format("20060102150405")) + if cronjob.Type == "database" { + name = fmt.Sprintf("%s%s.gz", dir, record.StartTime.Format("20060102150405")) + } + return name, nil } func (u *CronjobService) Create(cronjobDto dto.CronjobCreate) error { @@ -183,7 +197,7 @@ func (u *CronjobService) Delete(ids []uint) error { return constant.ErrRecordNotFound } global.Cron.Remove(cron.EntryID(cronjob.EntryID)) - _ = cronjobRepo.DeleteRecord(ids[0]) + _ = cronjobRepo.DeleteRecord(cronjobRepo.WithByJobID(int(ids[0]))) if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID)); err != nil { global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID, err) @@ -196,7 +210,7 @@ func (u *CronjobService) Delete(ids []uint) error { } for i := range cronjobs { global.Cron.Remove(cron.EntryID(cronjobs[i].EntryID)) - _ = cronjobRepo.DeleteRecord(cronjobs[i].ID) + _ = cronjobRepo.DeleteRecord(cronjobRepo.WithByJobID(int(cronjobs[i].ID))) if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TaskDir, cronjobs[i].Type, cronjobs[i].Name, cronjobs[i].ID)); err != nil { global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjobs[i].Type, cronjobs[i].Name, cronjobs[i].ID, err) } @@ -425,8 +439,9 @@ func tarWithExclude(cronjob *model.Cronjob, startTime time.Time) ([]byte, error) return nil, fmt.Errorf("tar zcPf failed, err: %v", err) } + var backClient cloud_storage.CloudStorageClient if varMaps["type"] != "LOCAL" { - backClient, err := cloud_storage.NewCloudStorageClient(varMaps) + backClient, err = cloud_storage.NewCloudStorageClient(varMaps) if err != nil { return stdout, fmt.Errorf("new cloud storage client failed, err: %v", err) } @@ -434,17 +449,9 @@ func tarWithExclude(cronjob *model.Cronjob, startTime time.Time) ([]byte, error) if !isOK { return nil, fmt.Errorf("cloud storage upload failed, err: %v", err) } - currentObjs, _ := backClient.ListObjects(fmt.Sprintf("%s/%s/", cronjob.Type, cronjob.Name)) - if len(currentObjs) > int(cronjob.RetainCopies) { - for i := 0; i < len(currentObjs)-int(cronjob.RetainCopies); i++ { - if path, ok := currentObjs[i].(string); ok { - _, _ = backClient.Delete(path) - } - } - } - if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TmpDir, cronjob.Type, cronjob.Name, cronjob.ID)); err != nil { - global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID, err) - } + } + if backType, ok := varMaps["type"].(string); ok { + rmOverdueCloud(backType, targetdir, cronjob, backClient) } return stdout, nil } @@ -486,6 +493,60 @@ func loadTargetInfo(cronjob *model.Cronjob) (map[string]interface{}, string, err return varMap, dir, nil } +func rmOverdueCloud(backType, path string, cronjob *model.Cronjob, backClient cloud_storage.CloudStorageClient) { + timeNow := time.Now() + timeZero := time.Date(timeNow.Year(), timeNow.Month(), timeNow.Day(), 0, 0, 0, 0, timeNow.Location()) + timeStart := timeZero.AddDate(0, 0, -int(cronjob.RetainDays)+1) + var timePrefixs []string + for i := 0; i < int(cronjob.RetainDays); i++ { + timePrefixs = append(timePrefixs, timeZero.AddDate(0, 0, i).Format("20060102")) + } + if backType != "LOCAL" { + dir := fmt.Sprintf("%s/%s/", cronjob.Type, cronjob.Name) + currentObjs, err := backClient.ListObjects(dir) + if err != nil { + global.LOG.Errorf("list bucket object %s failed, err: %v", dir, err) + return + } + for _, obj := range currentObjs { + objKey, ok := obj.(string) + if !ok { + continue + } + objKey = strings.ReplaceAll(objKey, dir, "") + isOk := false + for _, pre := range timePrefixs { + if strings.HasPrefix(objKey, pre) { + isOk = true + break + } + } + if !isOk { + _, _ = backClient.Delete(objKey) + } + } + return + } + files, err := ioutil.ReadDir(path) + if err != nil { + global.LOG.Errorf("read dir %s failed, err: %v", path, err) + return + } + for _, file := range files { + isOk := false + for _, pre := range timePrefixs { + if strings.HasPrefix(file.Name(), pre) { + isOk = true + break + } + } + if !isOk { + _ = os.Remove(path + "/" + file.Name()) + } + } + _ = cronjobRepo.DeleteRecord(cronjobRepo.WithByStartDate(timeStart)) +} + func loadSpec(cronjob model.Cronjob) string { switch cronjob.SpecType { case "perMonth": diff --git a/backend/constant/cronjob.go b/backend/constant/cronjob.go index ab6ef7b45..226bb8180 100644 --- a/backend/constant/cronjob.go +++ b/backend/constant/cronjob.go @@ -3,5 +3,5 @@ package constant const ( TmpDir = "/opt/1Panel/task/tmp/" TaskDir = "/opt/1Panel/task/" - DownloadDir = "/opt/1Panel/download" + DownloadDir = "/opt/1Panel/download/" ) diff --git a/backend/cron/cron.go b/backend/cron/cron.go index 057331f26..4d95a6867 100644 --- a/backend/cron/cron.go +++ b/backend/cron/cron.go @@ -26,6 +26,15 @@ func Run() { if err := global.DB.Where("status = ?", constant.StatusEnable).Find(&Cronjobs).Error; err != nil { global.LOG.Errorf("start my cronjob failed, err: %v", err) } + if err := global.DB.Model(&model.JobRecords{}). + Where("status = ?", constant.StatusRunning). + Updates(map[string]interface{}{ + "status": constant.StatusFailed, + "message": "Task Cancel", + "records": "errHandle", + }).Error; err != nil { + global.LOG.Errorf("start my cronjob failed, err: %v", err) + } for _, cronjob := range Cronjobs { if err := service.ServiceGroupApp.StartJob(&cronjob); err != nil { global.LOG.Errorf("start %s job %s failed, err: %v", cronjob.Type, cronjob.Name, err) diff --git a/backend/utils/cloud_storage/client/minio.go b/backend/utils/cloud_storage/client/minio.go index 11b8db7c2..813c10881 100644 --- a/backend/utils/cloud_storage/client/minio.go +++ b/backend/utils/cloud_storage/client/minio.go @@ -51,7 +51,7 @@ func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) { var transport http.RoundTripper = &http.Transport{ TLSClientConfig: tlsConfig, } - client, err := minio.New(endpoint, &minio.Options{ + client, err := minio.New(strings.ReplaceAll(endpoint, ssl+"://", ""), &minio.Options{ Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Secure: secure, Transport: transport, @@ -113,7 +113,6 @@ func (minIo minIoClient) Delete(path string) (bool, error) { } func (minIo minIoClient) Upload(src, target string) (bool, error) { - var bucket string if _, ok := minIo.Vars["bucket"]; ok { bucket = minIo.Vars["bucket"].(string) @@ -157,6 +156,30 @@ func (minIo minIoClient) Download(src, target string) (bool, error) { } } -func (minIo minIoClient) ListObjects(prefix string) ([]interface{}, error) { - return nil, nil +func (minIo *minIoClient) GetBucket() (string, error) { + if _, ok := minIo.Vars["bucket"]; ok { + return minIo.Vars["bucket"].(string), nil + } else { + return "", constant.ErrInvalidParams + } +} + +func (minIo minIoClient) ListObjects(prefix string) ([]interface{}, error) { + bucket, err := minIo.GetBucket() + if err != nil { + return nil, constant.ErrInvalidParams + } + opts := minio.ListObjectsOptions{ + Recursive: true, + Prefix: prefix, + } + + var result []interface{} + for object := range minIo.client.ListObjects(context.Background(), bucket, opts) { + if object.Err != nil { + continue + } + result = append(result, object.Key) + } + return result, nil } diff --git a/backend/utils/cloud_storage/client/oss.go b/backend/utils/cloud_storage/client/oss.go index 2908dd209..162c9fcb6 100644 --- a/backend/utils/cloud_storage/client/oss.go +++ b/backend/utils/cloud_storage/client/oss.go @@ -109,21 +109,17 @@ func (oss *ossClient) GetBucket() (*osssdk.Bucket, error) { } func (oss *ossClient) ListObjects(prefix string) ([]interface{}, error) { - if _, ok := oss.Vars["bucket"]; ok { - bucket, err := oss.client.Bucket(oss.Vars["bucket"].(string)) - if err != nil { - return nil, err - } - lor, err := bucket.ListObjects(osssdk.Prefix(prefix)) - if err != nil { - return nil, err - } - var result []interface{} - for _, obj := range lor.Objects { - result = append(result, obj.Key) - } - return result, nil - } else { + bucket, err := oss.GetBucket() + if err != nil { return nil, constant.ErrInvalidParams } + lor, err := bucket.ListObjects(osssdk.Prefix(prefix)) + if err != nil { + return nil, err + } + var result []interface{} + for _, obj := range lor.Objects { + result = append(result, obj.Key) + } + return result, nil } diff --git a/backend/utils/cloud_storage/client/oss_test.go b/backend/utils/cloud_storage/client/oss_test.go index b44293cc0..675a8f71e 100644 --- a/backend/utils/cloud_storage/client/oss_test.go +++ b/backend/utils/cloud_storage/client/oss_test.go @@ -1,8 +1,10 @@ -package service +package client import ( "encoding/json" "fmt" + "io/ioutil" + "os" "testing" "github.com/1Panel-dev/1Panel/app/model" @@ -52,6 +54,28 @@ func TestCron(t *testing.T) { fmt.Println(err) } fmt.Println("my objects:", getObjectsFormResponse(lor)) + + name := "directory/directory-test1/20220928104331.tar.gz" + targetPath := constant.DownloadDir + "directory/directory-test1/" + if _, err := os.Stat(targetPath); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(targetPath, os.ModePerm); err != nil { + fmt.Println(err) + } + } + if err := bucket.GetObjectToFile(name, targetPath+"20220928104231.tar.gz"); err != nil { + fmt.Println(err) + } +} + +func TestDir(t *testing.T) { + files, err := ioutil.ReadDir("/opt/1Panel/task/directory/directory-test1-3") + if len(files) <= 10 { + return + } + for i := 0; i < len(files)-10; i++ { + os.Remove("/opt/1Panel/task/directory/directory-test1-3/" + files[i].Name()) + } + fmt.Println(files, err) } func getObjectsFormResponse(lor oss.ListObjectsResult) string { diff --git a/backend/utils/cloud_storage/client/s3.go b/backend/utils/cloud_storage/client/s3.go index 1348f58b5..0929ff54c 100644 --- a/backend/utils/cloud_storage/client/s3.go +++ b/backend/utils/cloud_storage/client/s3.go @@ -18,7 +18,6 @@ type s3Client struct { } func NewS3Client(vars map[string]interface{}) (*s3Client, error) { - var accessKey string var secretKey string var endpoint string @@ -177,5 +176,22 @@ func (s3C *s3Client) getBucket() (string, error) { } func (s3C *s3Client) ListObjects(prefix string) ([]interface{}, error) { - return nil, nil + bucket, err := s3C.getBucket() + if err != nil { + return nil, constant.ErrInvalidParams + } + svc := s3.New(&s3C.Sess) + var result []interface{} + if err := svc.ListObjectsPages(&s3.ListObjectsInput{ + Bucket: &bucket, + Prefix: &prefix, + }, func(p *s3.ListObjectsOutput, last bool) (shouldContinue bool) { + for _, obj := range p.Contents { + result = append(result, obj) + } + return true + }); err != nil { + return nil, err + } + return result, nil } diff --git a/backend/utils/cloud_storage/client/sftp.go b/backend/utils/cloud_storage/client/sftp.go index 52d470f41..098461179 100644 --- a/backend/utils/cloud_storage/client/sftp.go +++ b/backend/utils/cloud_storage/client/sftp.go @@ -212,5 +212,26 @@ func (s sftpClient) getBucket() (string, error) { } func (s sftpClient) ListObjects(prefix string) ([]interface{}, error) { - return nil, nil + bucket, err := s.getBucket() + if err != nil { + return nil, err + } + port, err := strconv.Atoi(strconv.FormatFloat(s.Vars["port"].(float64), 'G', -1, 64)) + if err != nil { + return nil, err + } + sftpC, err := connect(s.Vars["username"].(string), s.Vars["password"].(string), s.Vars["address"].(string), port) + if err != nil { + return nil, err + } + defer sftpC.Close() + files, err := sftpC.ReadDir(bucket + "/" + prefix) + if err != nil { + return nil, err + } + var result []interface{} + for _, file := range files { + result = append(result, file.Name()) + } + return result, nil } diff --git a/frontend/src/api/interface/cronjob.ts b/frontend/src/api/interface/cronjob.ts index ad242da27..de7fc64cf 100644 --- a/frontend/src/api/interface/cronjob.ts +++ b/frontend/src/api/interface/cronjob.ts @@ -19,7 +19,7 @@ export namespace Cronjob { sourceDir: string; targetDirID: number; targetDir: string; - retainCopies: number; + retainDays: number; status: string; } export interface CronjobCreate { @@ -38,7 +38,7 @@ export namespace Cronjob { url: string; sourceDir: string; targetDirID: number; - retainCopies: number; + retainDays: number; } export interface CronjobUpdate { id: number; @@ -55,13 +55,13 @@ export namespace Cronjob { url: string; sourceDir: string; targetDirID: number; - retainCopies: number; + retainDays: number; } export interface UpdateStatus { id: number; status: string; } - export interface UpdateStatus { + export interface Download { recordID: number; backupAccountID: number; } diff --git a/frontend/src/api/modules/cronjob.ts b/frontend/src/api/modules/cronjob.ts index 462afce06..292f4ee44 100644 --- a/frontend/src/api/modules/cronjob.ts +++ b/frontend/src/api/modules/cronjob.ts @@ -29,3 +29,7 @@ export const getRecordDetail = (params: string) => { export const updateStatus = (params: Cronjob.UpdateStatus) => { return http.post(`cronjobs/status`, params); }; + +export const download = (params: Cronjob.Download) => { + return http.download(`cronjobs/download`, params, { responseType: 'blob' }); +}; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index d0b003031..f6bb51d6a 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -17,6 +17,9 @@ export default { clean: 'Clean', login: 'Login', close: 'Close', + view: 'View', + expand: 'Expand', + log: 'Log', saveAndEnable: 'Save and enable', }, search: { @@ -27,9 +30,13 @@ export default { dateEnd: 'Date end', }, table: { + total: 'Total {0}', name: 'Name', type: 'Type', status: 'Status', + statusSuccess: 'Success', + statusFailed: 'Failed', + records: 'Records', group: 'Group', createdAt: 'Creation Time', date: 'Date', @@ -37,16 +44,18 @@ export default { operate: 'Operations', message: 'Message', description: 'Description', + interval: 'Interval', }, msg: { delete: 'This operation cannot be rolled back. Do you want to continue', deleteTitle: 'Delete', deleteSuccess: 'Delete Success', loginSuccess: 'Login Success', - requestTimeout: 'The request timed out, please try again later', operationSuccess: 'Successful operation', notSupportOperation: 'This operation is not supported', + requestTimeout: 'The request timed out, please try again later', infoTitle: 'Hint', + notRecords: 'No execution record is generated for the current task', sureLogOut: 'Are you sure you want to log out?', createSuccess: 'Create Success', updateSuccess: 'Update Success', @@ -136,6 +145,51 @@ export default { header: { logout: 'Logout', }, + cronjob: { + cronTask: 'Task', + taskType: 'Task type', + shell: 'shell', + website: 'website', + failedFilter: 'Failed Task Filtering', + all: 'all', + database: 'database', + missBackupAccount: 'The backup account could not be found', + syncDate: 'Synchronization time ', + releaseMemory: 'Free memory', + curl: 'Crul', + taskName: 'Task name', + cronSpec: 'Lifecycle', + directory: 'Backup directory', + sourceDir: 'Backup directory', + exclusionRules: 'Exclusive rule', + url: 'URL Address', + target: 'Target', + retainDays: 'Retain days', + cronSpecRule: 'Please enter a correct lifecycle', + perMonth: 'Per monthly', + perWeek: 'Per week', + perHour: 'Per hour', + perNDay: 'Every N days', + perNHour: 'Every N hours', + perNMinute: 'Every N minutes', + per: 'Every ', + handle: 'Handle', + day: 'Day', + day1: 'Day', + hour: ' Hour', + minute: ' Minute', + monday: 'Monday', + tuesday: 'Tuesday', + wednesday: 'Wednesday', + thursday: 'Thursday', + friday: 'Friday', + saturday: 'Saturday', + sunday: 'Sunday', + shellContent: 'Script content', + errRecord: 'Incorrect logging', + errHandle: 'Task execution failure', + noRecord: 'The execution did not generate any logs', + }, monitor: { avgLoad: 'Average load', loadDetail: 'Load detail', @@ -207,7 +261,7 @@ export default { file: { dir: 'folder', upload: 'Upload', - download: 'download', + download: 'Download', fileName: 'file name', search: 'find', mode: 'permission', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 38a2ae5ed..08b7494e0 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -30,6 +30,7 @@ export default { dateEnd: '结束日期', }, table: { + total: '共 {0} 条', name: '名称', type: '类型', status: '状态', @@ -119,7 +120,7 @@ export default { firewall: '防火墙', database: '数据库', container: '容器', - cron: '计划任务', + cronjob: '计划任务', host: '主机', security: '安全', files: '文件管理', @@ -151,7 +152,6 @@ export default { database: '备份数据库', missBackupAccount: '未能找到备份账号', syncDate: '同步时间 ', - syncDateName: '定期同步服务器时间 ', releaseMemory: '释放内存', curl: '访问', taskName: '任务名称', @@ -161,7 +161,7 @@ export default { exclusionRules: '排除规则', url: 'URL 地址', target: '备份到', - retainCopies: '保留份数', + retainDays: '保留天数', cronSpecRule: '请输入正确的执行周期', perMonth: '每月', perWeek: '每周', diff --git a/frontend/src/routers/modules/cronjob.ts b/frontend/src/routers/modules/cronjob.ts index 6353e89c5..7d7e49e05 100644 --- a/frontend/src/routers/modules/cronjob.ts +++ b/frontend/src/routers/modules/cronjob.ts @@ -7,7 +7,7 @@ const cronRouter = { redirect: '/cronjobs', meta: { icon: 'p-plan', - title: 'menu.cron', + title: 'menu.cronjob', }, children: [ { diff --git a/frontend/src/utils/util.ts b/frontend/src/utils/util.ts index 1af0661f4..66afa4d01 100644 --- a/frontend/src/utils/util.ts +++ b/frontend/src/utils/util.ts @@ -49,6 +49,22 @@ export function dateFromat(row: number, col: number, dataStr: any) { return `${String(y)}-${String(m)}-${String(d)} ${String(h)}:${String(minute)}:${String(second)}`; } +export function dateFromatForName(dataStr: any) { + const date = new Date(dataStr); + const y = date.getFullYear(); + let m: string | number = date.getMonth() + 1; + m = m < 10 ? `0${String(m)}` : m; + let d: string | number = date.getDate(); + d = d < 10 ? `0${String(d)}` : d; + let h: string | number = date.getHours(); + h = h < 10 ? `0${String(h)}` : h; + let minute: string | number = date.getMinutes(); + minute = minute < 10 ? `0${String(minute)}` : minute; + let second: string | number = date.getSeconds(); + second = second < 10 ? `0${String(second)}` : second; + return `${String(y)}${String(m)}${String(d)}${String(h)}${String(minute)}${String(second)}`; +} + export function dateFromatWithoutYear(dataStr: any) { const date = new Date(dataStr); let m: string | number = date.getMonth() + 1; diff --git a/frontend/src/views/cronjob/index.vue b/frontend/src/views/cronjob/index.vue index 32d96ab1d..e59ce907d 100644 --- a/frontend/src/views/cronjob/index.vue +++ b/frontend/src/views/cronjob/index.vue @@ -55,8 +55,12 @@ {{ $t('cronjob.handle') }} - - + + + + @@ -71,8 +75,8 @@ import OperatrDialog from '@/views/cronjob/operate/index.vue'; import RecordDialog from '@/views/cronjob/record/index.vue'; import { loadZero } from '@/views/cronjob/options'; import { onMounted, reactive, ref } from 'vue'; -import { loadBackupName } from '@/views/setting/helper'; import { deleteCronjob, getCronjobPage, updateStatus } from '@/api/modules/cronjob'; +import { loadBackupName } from '@/views/setting/helper'; import { loadWeek } from './options'; import i18n from '@/lang'; import { Cronjob } from '@/api/interface/cronjob'; @@ -121,7 +125,7 @@ const onOpenDialog = async ( day: 1, hour: 2, minute: 3, - retainCopies: 3, + retainDays: 7, }, ) => { let params = { diff --git a/frontend/src/views/cronjob/operate/index.vue b/frontend/src/views/cronjob/operate/index.vue index 43e956766..4a274bb1a 100644 --- a/frontend/src/views/cronjob/operate/index.vue +++ b/frontend/src/views/cronjob/operate/index.vue @@ -21,7 +21,7 @@ - + - - + + @@ -241,7 +241,7 @@ const rules = reactive({ url: [Rules.requiredInput], sourceDir: [Rules.requiredSelect], targetDirID: [Rules.requiredSelect, Rules.number], - retainCopies: [Rules.number], + retainDays: [Rules.number], }); type FormInstance = InstanceType; diff --git a/frontend/src/views/cronjob/options.ts b/frontend/src/views/cronjob/options.ts index 448d052cf..182e4379a 100644 --- a/frontend/src/views/cronjob/options.ts +++ b/frontend/src/views/cronjob/options.ts @@ -6,7 +6,7 @@ export const typeOptions = [ { label: i18n.global.t('cronjob.database'), value: 'database' }, { label: i18n.global.t('cronjob.directory'), value: 'directory' }, { label: i18n.global.t('cronjob.syncDate'), value: 'sync' }, - { label: i18n.global.t('cronjob.releaseMemory'), value: 'release' }, + // { label: i18n.global.t('cronjob.releaseMemory'), value: 'release' }, { label: i18n.global.t('cronjob.curl') + ' URL', value: 'curl' }, ]; diff --git a/frontend/src/views/cronjob/record/index.vue b/frontend/src/views/cronjob/record/index.vue index cca69909b..02cb1a2b7 100644 --- a/frontend/src/views/cronjob/record/index.vue +++ b/frontend/src/views/cronjob/record/index.vue @@ -32,10 +32,13 @@ {{ dateFromat(0, 0, item.startTime) }} +
+ {{ $t('commons.table.total', [searchInfo.recordTotal]) }} +
- + @@ -122,11 +125,21 @@ {{ loadBackupName(dialogData.rowData!.targetDir) }} + + {{ $t('file.download') }} + - - {{ dialogData.rowData!.retainCopies }} + + {{ dialogData.rowData!.retainDays }} @@ -179,11 +192,13 @@ - {{ $t('cronjob.' + currentRecord?.records!) }} + {{ currentRecord?.message }} { + if (timeRangeLoad.value && timeRangeLoad.value.length === 2) { + searchInfo.startTime = timeRangeLoad.value[0]; + searchInfo.endTime = timeRangeLoad.value[1]; + } else { + searchInfo.startTime = new Date(new Date().setHours(0, 0, 0, 0)); + searchInfo.endTime = new Date(); + } let params = { page: searchInfo.page, pageSize: searchInfo.pageSize, @@ -323,6 +345,24 @@ const search = async () => { records.value = res.data.items || []; searchInfo.recordTotal = res.data.total; }; +const onDownload = async (recordID: number, backupID: number) => { + let params = { + recordID: recordID, + backupAccountID: backupID, + }; + const res = await download(params); + const downloadUrl = window.URL.createObjectURL(new Blob([res])); + const a = document.createElement('a'); + a.style.display = 'none'; + a.href = downloadUrl; + if (dialogData.value.rowData!.type === 'database') { + a.download = dateFromatForName(currentRecord.value?.startTime) + '.sql.gz'; + } else { + a.download = dateFromatForName(currentRecord.value?.startTime) + '.tar.gz'; + } + const event = new MouseEvent('click'); + a.dispatchEvent(event); +}; const nextPage = async () => { if (searchInfo.pageSize >= searchInfo.recordTotal) { diff --git a/frontend/src/views/setting/tabs/backup.vue b/frontend/src/views/setting/tabs/backup.vue index 9678add79..e9951a161 100644 --- a/frontend/src/views/setting/tabs/backup.vue +++ b/frontend/src/views/setting/tabs/backup.vue @@ -8,8 +8,8 @@ {{ $t('commons.button.create') }} - - + +