1
0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-01-31 22:18:07 +08:00

feat: sftp、s3、minio 定时备份功能实现

This commit is contained in:
ssongliu 2022-09-28 18:11:36 +08:00 committed by ssongliu
parent a0c73e8eab
commit c5e9a2e085
23 changed files with 387 additions and 106 deletions

View File

@ -164,10 +164,10 @@ func (b *BaseApi) TargetDownload(c *gin.Context) {
return return
} }
dir, err := cronjobService.Download(req) filePath, err := cronjobService.Download(req)
if err != nil { if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return return
} }
helper.SuccessWithData(c, dir) c.File(filePath)
} }

View File

@ -18,7 +18,7 @@ type CronjobCreate struct {
URL string `json:"url"` URL string `json:"url"`
SourceDir string `json:"sourceDir"` SourceDir string `json:"sourceDir"`
TargetDirID int `json:"targetDirID"` TargetDirID int `json:"targetDirID"`
RetainCopies int `json:"retainCopies" validate:"number,min=1"` RetainDays int `json:"retainDays" validate:"number,min=1"`
} }
type CronjobUpdate struct { type CronjobUpdate struct {
@ -36,7 +36,7 @@ type CronjobUpdate struct {
URL string `json:"url"` URL string `json:"url"`
SourceDir string `json:"sourceDir"` SourceDir string `json:"sourceDir"`
TargetDirID int `json:"targetDirID"` TargetDirID int `json:"targetDirID"`
RetainCopies int `json:"retainCopies" validate:"number,min=1"` RetainDays int `json:"retainDays" validate:"number,min=1"`
} }
type CronjobUpdateStatus struct { type CronjobUpdateStatus struct {
@ -71,7 +71,7 @@ type CronjobInfo struct {
SourceDir string `json:"sourceDir"` SourceDir string `json:"sourceDir"`
TargetDir string `json:"targetDir"` TargetDir string `json:"targetDir"`
TargetDirID int `json:"targetDirID"` TargetDirID int `json:"targetDirID"`
RetainCopies int `json:"retainCopies"` RetainDays int `json:"retainDays"`
Status string `json:"status"` Status string `json:"status"`
} }

View File

@ -21,7 +21,7 @@ type Cronjob struct {
SourceDir string `gorm:"type:varchar(256)" json:"sourceDir"` SourceDir string `gorm:"type:varchar(256)" json:"sourceDir"`
TargetDirID uint64 `gorm:"type:decimal" json:"targetDirID"` TargetDirID uint64 `gorm:"type:decimal" json:"targetDirID"`
ExclusionRules string `gorm:"longtext" json:"exclusionRules"` ExclusionRules string `gorm:"longtext" json:"exclusionRules"`
RetainCopies uint64 `gorm:"type:decimal" json:"retainCopies"` RetainDays uint64 `gorm:"type:decimal" json:"retainDays"`
Status string `gorm:"type:varchar(64)" json:"status"` Status string `gorm:"type:varchar(64)" json:"status"`
EntryID uint64 `gorm:"type:decimal" json:"entryID"` EntryID uint64 `gorm:"type:decimal" json:"entryID"`
@ -37,5 +37,4 @@ type JobRecords struct {
Records string `gorm:"longtext" json:"records"` Records string `gorm:"longtext" json:"records"`
Status string `gorm:"type:varchar(64)" json:"status"` Status string `gorm:"type:varchar(64)" json:"status"`
Message string `gorm:"longtext" json:"message"` Message string `gorm:"longtext" json:"message"`
TargetPath string `gorm:"type:varchar(256)" json:"targetPath"`
} }

View File

@ -14,6 +14,7 @@ type CronjobRepo struct{}
type ICronjobRepo interface { type ICronjobRepo interface {
Get(opts ...DBOption) (model.Cronjob, error) Get(opts ...DBOption) (model.Cronjob, error)
GetRecord(opts ...DBOption) (model.JobRecords, error) GetRecord(opts ...DBOption) (model.JobRecords, error)
ListRecord(opts ...DBOption) ([]model.JobRecords, error)
List(opts ...DBOption) ([]model.Cronjob, error) List(opts ...DBOption) ([]model.Cronjob, error)
Page(limit, offset int, opts ...DBOption) (int64, []model.Cronjob, error) Page(limit, offset int, opts ...DBOption) (int64, []model.Cronjob, error)
Create(cronjob *model.Cronjob) error Create(cronjob *model.Cronjob) error
@ -22,7 +23,7 @@ type ICronjobRepo interface {
Save(id uint, cronjob model.Cronjob) error Save(id uint, cronjob model.Cronjob) error
Update(id uint, vars map[string]interface{}) error Update(id uint, vars map[string]interface{}) error
Delete(opts ...DBOption) error Delete(opts ...DBOption) error
DeleteRecord(jobID uint) error DeleteRecord(opts ...DBOption) error
StartRecords(cronjobID uint, targetPath string) model.JobRecords StartRecords(cronjobID uint, targetPath string) model.JobRecords
EndRecords(record model.JobRecords, status, message, records string) EndRecords(record model.JobRecords, status, message, records string)
} }
@ -57,8 +58,16 @@ func (u *CronjobRepo) List(opts ...DBOption) ([]model.Cronjob, error) {
for _, opt := range opts { for _, opt := range opts {
db = opt(db) db = opt(db)
} }
count := int64(0) err := db.Find(&cronjobs).Error
db = db.Count(&count) return cronjobs, err
}
func (u *CronjobRepo) ListRecord(opts ...DBOption) ([]model.JobRecords, error) {
var cronjobs []model.JobRecords
db := global.DB.Model(&model.JobRecords{})
for _, opt := range opts {
db = opt(db)
}
err := db.Find(&cronjobs).Error err := db.Find(&cronjobs).Error
return cronjobs, err return cronjobs, err
} }
@ -71,7 +80,7 @@ func (u *CronjobRepo) Page(page, size int, opts ...DBOption) (int64, []model.Cro
} }
count := int64(0) count := int64(0)
db = db.Count(&count) db = db.Count(&count)
err := db.Order("created_at").Limit(size).Offset(size * (page - 1)).Find(&cronjobs).Error err := db.Order("created_at desc").Limit(size).Offset(size * (page - 1)).Find(&cronjobs).Error
return count, cronjobs, err return count, cronjobs, err
} }
@ -83,7 +92,7 @@ func (u *CronjobRepo) PageRecords(page, size int, opts ...DBOption) (int64, []mo
} }
count := int64(0) count := int64(0)
db = db.Count(&count) db = db.Count(&count)
err := db.Order("created_at").Limit(size).Offset(size * (page - 1)).Find(&cronjobs).Error err := db.Order("created_at desc").Limit(size).Offset(size * (page - 1)).Find(&cronjobs).Error
return count, cronjobs, err return count, cronjobs, err
} }
@ -96,6 +105,11 @@ func (c *CronjobRepo) WithByDate(startTime, endTime time.Time) DBOption {
return g.Where("start_time > ? AND start_time < ?", startTime, endTime) return g.Where("start_time > ? AND start_time < ?", startTime, endTime)
} }
} }
func (c *CronjobRepo) WithByStartDate(startTime time.Time) DBOption {
return func(g *gorm.DB) *gorm.DB {
return g.Where("start_time < ?", startTime)
}
}
func (c *CronjobRepo) WithByJobID(id int) DBOption { func (c *CronjobRepo) WithByJobID(id int) DBOption {
return func(g *gorm.DB) *gorm.DB { return func(g *gorm.DB) *gorm.DB {
return g.Where("cronjob_id = ?", id) return g.Where("cronjob_id = ?", id)
@ -137,6 +151,10 @@ func (u *CronjobRepo) Delete(opts ...DBOption) error {
} }
return db.Delete(&model.Cronjob{}).Error return db.Delete(&model.Cronjob{}).Error
} }
func (u *CronjobRepo) DeleteRecord(jobID uint) error { func (u *CronjobRepo) DeleteRecord(opts ...DBOption) error {
return global.DB.Where("cronjob_id = ?", jobID).Delete(&model.JobRecords{}).Error db := global.DB
for _, opt := range opts {
db = opt(db)
}
return db.Delete(&model.JobRecords{}).Error
} }

View File

@ -84,16 +84,16 @@ func (u *CronjobService) SearchRecords(search dto.SearchRecord) (int64, interfac
func (u *CronjobService) Download(down dto.CronjobDownload) (string, error) { func (u *CronjobService) Download(down dto.CronjobDownload) (string, error) {
record, _ := cronjobRepo.GetRecord(commonRepo.WithByID(down.RecordID)) record, _ := cronjobRepo.GetRecord(commonRepo.WithByID(down.RecordID))
if record.ID != 0 { if record.ID == 0 {
return "", constant.ErrRecordExist return "", constant.ErrRecordNotFound
} }
cronjob, _ := cronjobRepo.Get(commonRepo.WithByID(record.CronjobID)) cronjob, _ := cronjobRepo.Get(commonRepo.WithByID(record.CronjobID))
if cronjob.ID != 0 { if cronjob.ID == 0 {
return "", constant.ErrRecordExist return "", constant.ErrRecordNotFound
} }
backup, _ := backupRepo.Get(commonRepo.WithByID(down.BackupAccountID)) backup, _ := backupRepo.Get(commonRepo.WithByID(down.BackupAccountID))
if cronjob.ID != 0 { if cronjob.ID == 0 {
return "", constant.ErrRecordExist return "", constant.ErrRecordNotFound
} }
varMap := make(map[string]interface{}) varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil { if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
@ -112,21 +112,35 @@ func (u *CronjobService) Download(down dto.CronjobDownload) (string, error) {
if err != nil { if err != nil {
return "", fmt.Errorf("new cloud storage client failed, err: %v", err) return "", fmt.Errorf("new cloud storage client failed, err: %v", err)
} }
name := fmt.Sprintf("%s/%s/%s.tar.gz", cronjob.Type, cronjob.Name, record.StartTime.Format("20060102150405")) commonDir := fmt.Sprintf("%s/%s/", cronjob.Type, cronjob.Name)
name := fmt.Sprintf("%s%s.tar.gz", commonDir, record.StartTime.Format("20060102150405"))
if cronjob.Type == "database" { if cronjob.Type == "database" {
name = fmt.Sprintf("%s/%s/%s.gz", cronjob.Type, cronjob.Name, record.StartTime.Format("20060102150405")) name = fmt.Sprintf("%s%s.gz", commonDir, record.StartTime.Format("20060102150405"))
} }
isOK, err := backClient.Download(name, constant.DownloadDir) tempPath := fmt.Sprintf("%s%s", constant.DownloadDir, commonDir)
if _, err := os.Stat(tempPath); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(tempPath, os.ModePerm); err != nil {
fmt.Println(err)
}
}
targetPath := tempPath + strings.ReplaceAll(name, commonDir, "")
if _, err = os.Stat(targetPath); err != nil && os.IsNotExist(err) {
isOK, err := backClient.Download(name, targetPath)
if !isOK { if !isOK {
return "", fmt.Errorf("cloud storage download failed, err: %v", err) return "", fmt.Errorf("cloud storage download failed, err: %v", err)
} }
return constant.DownloadDir, nil }
return targetPath, nil
} }
if _, ok := varMap["dir"]; !ok { if _, ok := varMap["dir"]; !ok {
return "", errors.New("load local backup dir failed") return "", errors.New("load local backup dir failed")
} }
return fmt.Sprintf("%v/%s/%s", varMap["dir"], cronjob.Type, cronjob.Name), nil dir := fmt.Sprintf("%v/%s/%s/", varMap["dir"], cronjob.Type, cronjob.Name)
name := fmt.Sprintf("%s%s.tar.gz", dir, record.StartTime.Format("20060102150405"))
if cronjob.Type == "database" {
name = fmt.Sprintf("%s%s.gz", dir, record.StartTime.Format("20060102150405"))
}
return name, nil
} }
func (u *CronjobService) Create(cronjobDto dto.CronjobCreate) error { func (u *CronjobService) Create(cronjobDto dto.CronjobCreate) error {
@ -183,7 +197,7 @@ func (u *CronjobService) Delete(ids []uint) error {
return constant.ErrRecordNotFound return constant.ErrRecordNotFound
} }
global.Cron.Remove(cron.EntryID(cronjob.EntryID)) global.Cron.Remove(cron.EntryID(cronjob.EntryID))
_ = cronjobRepo.DeleteRecord(ids[0]) _ = cronjobRepo.DeleteRecord(cronjobRepo.WithByJobID(int(ids[0])))
if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID)); err != nil { if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID)); err != nil {
global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID, err) global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID, err)
@ -196,7 +210,7 @@ func (u *CronjobService) Delete(ids []uint) error {
} }
for i := range cronjobs { for i := range cronjobs {
global.Cron.Remove(cron.EntryID(cronjobs[i].EntryID)) global.Cron.Remove(cron.EntryID(cronjobs[i].EntryID))
_ = cronjobRepo.DeleteRecord(cronjobs[i].ID) _ = cronjobRepo.DeleteRecord(cronjobRepo.WithByJobID(int(cronjobs[i].ID)))
if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TaskDir, cronjobs[i].Type, cronjobs[i].Name, cronjobs[i].ID)); err != nil { if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TaskDir, cronjobs[i].Type, cronjobs[i].Name, cronjobs[i].ID)); err != nil {
global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjobs[i].Type, cronjobs[i].Name, cronjobs[i].ID, err) global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjobs[i].Type, cronjobs[i].Name, cronjobs[i].ID, err)
} }
@ -425,8 +439,9 @@ func tarWithExclude(cronjob *model.Cronjob, startTime time.Time) ([]byte, error)
return nil, fmt.Errorf("tar zcPf failed, err: %v", err) return nil, fmt.Errorf("tar zcPf failed, err: %v", err)
} }
var backClient cloud_storage.CloudStorageClient
if varMaps["type"] != "LOCAL" { if varMaps["type"] != "LOCAL" {
backClient, err := cloud_storage.NewCloudStorageClient(varMaps) backClient, err = cloud_storage.NewCloudStorageClient(varMaps)
if err != nil { if err != nil {
return stdout, fmt.Errorf("new cloud storage client failed, err: %v", err) return stdout, fmt.Errorf("new cloud storage client failed, err: %v", err)
} }
@ -434,17 +449,9 @@ func tarWithExclude(cronjob *model.Cronjob, startTime time.Time) ([]byte, error)
if !isOK { if !isOK {
return nil, fmt.Errorf("cloud storage upload failed, err: %v", err) return nil, fmt.Errorf("cloud storage upload failed, err: %v", err)
} }
currentObjs, _ := backClient.ListObjects(fmt.Sprintf("%s/%s/", cronjob.Type, cronjob.Name))
if len(currentObjs) > int(cronjob.RetainCopies) {
for i := 0; i < len(currentObjs)-int(cronjob.RetainCopies); i++ {
if path, ok := currentObjs[i].(string); ok {
_, _ = backClient.Delete(path)
}
}
}
if err := os.RemoveAll(fmt.Sprintf("%s/%s/%s-%v", constant.TmpDir, cronjob.Type, cronjob.Name, cronjob.ID)); err != nil {
global.LOG.Errorf("rm file %s/%s/%s-%v failed, err: %v", constant.TaskDir, cronjob.Type, cronjob.Name, cronjob.ID, err)
} }
if backType, ok := varMaps["type"].(string); ok {
rmOverdueCloud(backType, targetdir, cronjob, backClient)
} }
return stdout, nil return stdout, nil
} }
@ -486,6 +493,60 @@ func loadTargetInfo(cronjob *model.Cronjob) (map[string]interface{}, string, err
return varMap, dir, nil return varMap, dir, nil
} }
func rmOverdueCloud(backType, path string, cronjob *model.Cronjob, backClient cloud_storage.CloudStorageClient) {
timeNow := time.Now()
timeZero := time.Date(timeNow.Year(), timeNow.Month(), timeNow.Day(), 0, 0, 0, 0, timeNow.Location())
timeStart := timeZero.AddDate(0, 0, -int(cronjob.RetainDays)+1)
var timePrefixs []string
for i := 0; i < int(cronjob.RetainDays); i++ {
timePrefixs = append(timePrefixs, timeZero.AddDate(0, 0, i).Format("20060102"))
}
if backType != "LOCAL" {
dir := fmt.Sprintf("%s/%s/", cronjob.Type, cronjob.Name)
currentObjs, err := backClient.ListObjects(dir)
if err != nil {
global.LOG.Errorf("list bucket object %s failed, err: %v", dir, err)
return
}
for _, obj := range currentObjs {
objKey, ok := obj.(string)
if !ok {
continue
}
objKey = strings.ReplaceAll(objKey, dir, "")
isOk := false
for _, pre := range timePrefixs {
if strings.HasPrefix(objKey, pre) {
isOk = true
break
}
}
if !isOk {
_, _ = backClient.Delete(objKey)
}
}
return
}
files, err := ioutil.ReadDir(path)
if err != nil {
global.LOG.Errorf("read dir %s failed, err: %v", path, err)
return
}
for _, file := range files {
isOk := false
for _, pre := range timePrefixs {
if strings.HasPrefix(file.Name(), pre) {
isOk = true
break
}
}
if !isOk {
_ = os.Remove(path + "/" + file.Name())
}
}
_ = cronjobRepo.DeleteRecord(cronjobRepo.WithByStartDate(timeStart))
}
func loadSpec(cronjob model.Cronjob) string { func loadSpec(cronjob model.Cronjob) string {
switch cronjob.SpecType { switch cronjob.SpecType {
case "perMonth": case "perMonth":

View File

@ -3,5 +3,5 @@ package constant
const ( const (
TmpDir = "/opt/1Panel/task/tmp/" TmpDir = "/opt/1Panel/task/tmp/"
TaskDir = "/opt/1Panel/task/" TaskDir = "/opt/1Panel/task/"
DownloadDir = "/opt/1Panel/download" DownloadDir = "/opt/1Panel/download/"
) )

View File

@ -26,6 +26,15 @@ func Run() {
if err := global.DB.Where("status = ?", constant.StatusEnable).Find(&Cronjobs).Error; err != nil { if err := global.DB.Where("status = ?", constant.StatusEnable).Find(&Cronjobs).Error; err != nil {
global.LOG.Errorf("start my cronjob failed, err: %v", err) global.LOG.Errorf("start my cronjob failed, err: %v", err)
} }
if err := global.DB.Model(&model.JobRecords{}).
Where("status = ?", constant.StatusRunning).
Updates(map[string]interface{}{
"status": constant.StatusFailed,
"message": "Task Cancel",
"records": "errHandle",
}).Error; err != nil {
global.LOG.Errorf("start my cronjob failed, err: %v", err)
}
for _, cronjob := range Cronjobs { for _, cronjob := range Cronjobs {
if err := service.ServiceGroupApp.StartJob(&cronjob); err != nil { if err := service.ServiceGroupApp.StartJob(&cronjob); err != nil {
global.LOG.Errorf("start %s job %s failed, err: %v", cronjob.Type, cronjob.Name, err) global.LOG.Errorf("start %s job %s failed, err: %v", cronjob.Type, cronjob.Name, err)

View File

@ -51,7 +51,7 @@ func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) {
var transport http.RoundTripper = &http.Transport{ var transport http.RoundTripper = &http.Transport{
TLSClientConfig: tlsConfig, TLSClientConfig: tlsConfig,
} }
client, err := minio.New(endpoint, &minio.Options{ client, err := minio.New(strings.ReplaceAll(endpoint, ssl+"://", ""), &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: secure, Secure: secure,
Transport: transport, Transport: transport,
@ -113,7 +113,6 @@ func (minIo minIoClient) Delete(path string) (bool, error) {
} }
func (minIo minIoClient) Upload(src, target string) (bool, error) { func (minIo minIoClient) Upload(src, target string) (bool, error) {
var bucket string var bucket string
if _, ok := minIo.Vars["bucket"]; ok { if _, ok := minIo.Vars["bucket"]; ok {
bucket = minIo.Vars["bucket"].(string) bucket = minIo.Vars["bucket"].(string)
@ -157,6 +156,30 @@ func (minIo minIoClient) Download(src, target string) (bool, error) {
} }
} }
func (minIo minIoClient) ListObjects(prefix string) ([]interface{}, error) { func (minIo *minIoClient) GetBucket() (string, error) {
return nil, nil if _, ok := minIo.Vars["bucket"]; ok {
return minIo.Vars["bucket"].(string), nil
} else {
return "", constant.ErrInvalidParams
}
}
func (minIo minIoClient) ListObjects(prefix string) ([]interface{}, error) {
bucket, err := minIo.GetBucket()
if err != nil {
return nil, constant.ErrInvalidParams
}
opts := minio.ListObjectsOptions{
Recursive: true,
Prefix: prefix,
}
var result []interface{}
for object := range minIo.client.ListObjects(context.Background(), bucket, opts) {
if object.Err != nil {
continue
}
result = append(result, object.Key)
}
return result, nil
} }

View File

@ -109,10 +109,9 @@ func (oss *ossClient) GetBucket() (*osssdk.Bucket, error) {
} }
func (oss *ossClient) ListObjects(prefix string) ([]interface{}, error) { func (oss *ossClient) ListObjects(prefix string) ([]interface{}, error) {
if _, ok := oss.Vars["bucket"]; ok { bucket, err := oss.GetBucket()
bucket, err := oss.client.Bucket(oss.Vars["bucket"].(string))
if err != nil { if err != nil {
return nil, err return nil, constant.ErrInvalidParams
} }
lor, err := bucket.ListObjects(osssdk.Prefix(prefix)) lor, err := bucket.ListObjects(osssdk.Prefix(prefix))
if err != nil { if err != nil {
@ -123,7 +122,4 @@ func (oss *ossClient) ListObjects(prefix string) ([]interface{}, error) {
result = append(result, obj.Key) result = append(result, obj.Key)
} }
return result, nil return result, nil
} else {
return nil, constant.ErrInvalidParams
}
} }

View File

@ -1,8 +1,10 @@
package service package client
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil"
"os"
"testing" "testing"
"github.com/1Panel-dev/1Panel/app/model" "github.com/1Panel-dev/1Panel/app/model"
@ -52,6 +54,28 @@ func TestCron(t *testing.T) {
fmt.Println(err) fmt.Println(err)
} }
fmt.Println("my objects:", getObjectsFormResponse(lor)) fmt.Println("my objects:", getObjectsFormResponse(lor))
name := "directory/directory-test1/20220928104331.tar.gz"
targetPath := constant.DownloadDir + "directory/directory-test1/"
if _, err := os.Stat(targetPath); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(targetPath, os.ModePerm); err != nil {
fmt.Println(err)
}
}
if err := bucket.GetObjectToFile(name, targetPath+"20220928104231.tar.gz"); err != nil {
fmt.Println(err)
}
}
func TestDir(t *testing.T) {
files, err := ioutil.ReadDir("/opt/1Panel/task/directory/directory-test1-3")
if len(files) <= 10 {
return
}
for i := 0; i < len(files)-10; i++ {
os.Remove("/opt/1Panel/task/directory/directory-test1-3/" + files[i].Name())
}
fmt.Println(files, err)
} }
func getObjectsFormResponse(lor oss.ListObjectsResult) string { func getObjectsFormResponse(lor oss.ListObjectsResult) string {

View File

@ -18,7 +18,6 @@ type s3Client struct {
} }
func NewS3Client(vars map[string]interface{}) (*s3Client, error) { func NewS3Client(vars map[string]interface{}) (*s3Client, error) {
var accessKey string var accessKey string
var secretKey string var secretKey string
var endpoint string var endpoint string
@ -177,5 +176,22 @@ func (s3C *s3Client) getBucket() (string, error) {
} }
func (s3C *s3Client) ListObjects(prefix string) ([]interface{}, error) { func (s3C *s3Client) ListObjects(prefix string) ([]interface{}, error) {
return nil, nil bucket, err := s3C.getBucket()
if err != nil {
return nil, constant.ErrInvalidParams
}
svc := s3.New(&s3C.Sess)
var result []interface{}
if err := svc.ListObjectsPages(&s3.ListObjectsInput{
Bucket: &bucket,
Prefix: &prefix,
}, func(p *s3.ListObjectsOutput, last bool) (shouldContinue bool) {
for _, obj := range p.Contents {
result = append(result, obj)
}
return true
}); err != nil {
return nil, err
}
return result, nil
} }

View File

@ -212,5 +212,26 @@ func (s sftpClient) getBucket() (string, error) {
} }
func (s sftpClient) ListObjects(prefix string) ([]interface{}, error) { func (s sftpClient) ListObjects(prefix string) ([]interface{}, error) {
return nil, nil bucket, err := s.getBucket()
if err != nil {
return nil, err
}
port, err := strconv.Atoi(strconv.FormatFloat(s.Vars["port"].(float64), 'G', -1, 64))
if err != nil {
return nil, err
}
sftpC, err := connect(s.Vars["username"].(string), s.Vars["password"].(string), s.Vars["address"].(string), port)
if err != nil {
return nil, err
}
defer sftpC.Close()
files, err := sftpC.ReadDir(bucket + "/" + prefix)
if err != nil {
return nil, err
}
var result []interface{}
for _, file := range files {
result = append(result, file.Name())
}
return result, nil
} }

View File

@ -19,7 +19,7 @@ export namespace Cronjob {
sourceDir: string; sourceDir: string;
targetDirID: number; targetDirID: number;
targetDir: string; targetDir: string;
retainCopies: number; retainDays: number;
status: string; status: string;
} }
export interface CronjobCreate { export interface CronjobCreate {
@ -38,7 +38,7 @@ export namespace Cronjob {
url: string; url: string;
sourceDir: string; sourceDir: string;
targetDirID: number; targetDirID: number;
retainCopies: number; retainDays: number;
} }
export interface CronjobUpdate { export interface CronjobUpdate {
id: number; id: number;
@ -55,13 +55,13 @@ export namespace Cronjob {
url: string; url: string;
sourceDir: string; sourceDir: string;
targetDirID: number; targetDirID: number;
retainCopies: number; retainDays: number;
} }
export interface UpdateStatus { export interface UpdateStatus {
id: number; id: number;
status: string; status: string;
} }
export interface UpdateStatus { export interface Download {
recordID: number; recordID: number;
backupAccountID: number; backupAccountID: number;
} }

View File

@ -29,3 +29,7 @@ export const getRecordDetail = (params: string) => {
export const updateStatus = (params: Cronjob.UpdateStatus) => { export const updateStatus = (params: Cronjob.UpdateStatus) => {
return http.post(`cronjobs/status`, params); return http.post(`cronjobs/status`, params);
}; };
export const download = (params: Cronjob.Download) => {
return http.download<BlobPart>(`cronjobs/download`, params, { responseType: 'blob' });
};

View File

@ -17,6 +17,9 @@ export default {
clean: 'Clean', clean: 'Clean',
login: 'Login', login: 'Login',
close: 'Close', close: 'Close',
view: 'View',
expand: 'Expand',
log: 'Log',
saveAndEnable: 'Save and enable', saveAndEnable: 'Save and enable',
}, },
search: { search: {
@ -27,9 +30,13 @@ export default {
dateEnd: 'Date end', dateEnd: 'Date end',
}, },
table: { table: {
total: 'Total {0}',
name: 'Name', name: 'Name',
type: 'Type', type: 'Type',
status: 'Status', status: 'Status',
statusSuccess: 'Success',
statusFailed: 'Failed',
records: 'Records',
group: 'Group', group: 'Group',
createdAt: 'Creation Time', createdAt: 'Creation Time',
date: 'Date', date: 'Date',
@ -37,16 +44,18 @@ export default {
operate: 'Operations', operate: 'Operations',
message: 'Message', message: 'Message',
description: 'Description', description: 'Description',
interval: 'Interval',
}, },
msg: { msg: {
delete: 'This operation cannot be rolled back. Do you want to continue', delete: 'This operation cannot be rolled back. Do you want to continue',
deleteTitle: 'Delete', deleteTitle: 'Delete',
deleteSuccess: 'Delete Success', deleteSuccess: 'Delete Success',
loginSuccess: 'Login Success', loginSuccess: 'Login Success',
requestTimeout: 'The request timed out, please try again later',
operationSuccess: 'Successful operation', operationSuccess: 'Successful operation',
notSupportOperation: 'This operation is not supported', notSupportOperation: 'This operation is not supported',
requestTimeout: 'The request timed out, please try again later',
infoTitle: 'Hint', infoTitle: 'Hint',
notRecords: 'No execution record is generated for the current task',
sureLogOut: 'Are you sure you want to log out?', sureLogOut: 'Are you sure you want to log out?',
createSuccess: 'Create Success', createSuccess: 'Create Success',
updateSuccess: 'Update Success', updateSuccess: 'Update Success',
@ -136,6 +145,51 @@ export default {
header: { header: {
logout: 'Logout', logout: 'Logout',
}, },
cronjob: {
cronTask: 'Task',
taskType: 'Task type',
shell: 'shell',
website: 'website',
failedFilter: 'Failed Task Filtering',
all: 'all',
database: 'database',
missBackupAccount: 'The backup account could not be found',
syncDate: 'Synchronization time ',
releaseMemory: 'Free memory',
curl: 'Crul',
taskName: 'Task name',
cronSpec: 'Lifecycle',
directory: 'Backup directory',
sourceDir: 'Backup directory',
exclusionRules: 'Exclusive rule',
url: 'URL Address',
target: 'Target',
retainDays: 'Retain days',
cronSpecRule: 'Please enter a correct lifecycle',
perMonth: 'Per monthly',
perWeek: 'Per week',
perHour: 'Per hour',
perNDay: 'Every N days',
perNHour: 'Every N hours',
perNMinute: 'Every N minutes',
per: 'Every ',
handle: 'Handle',
day: 'Day',
day1: 'Day',
hour: ' Hour',
minute: ' Minute',
monday: 'Monday',
tuesday: 'Tuesday',
wednesday: 'Wednesday',
thursday: 'Thursday',
friday: 'Friday',
saturday: 'Saturday',
sunday: 'Sunday',
shellContent: 'Script content',
errRecord: 'Incorrect logging',
errHandle: 'Task execution failure',
noRecord: 'The execution did not generate any logs',
},
monitor: { monitor: {
avgLoad: 'Average load', avgLoad: 'Average load',
loadDetail: 'Load detail', loadDetail: 'Load detail',
@ -207,7 +261,7 @@ export default {
file: { file: {
dir: 'folder', dir: 'folder',
upload: 'Upload', upload: 'Upload',
download: 'download', download: 'Download',
fileName: 'file name', fileName: 'file name',
search: 'find', search: 'find',
mode: 'permission', mode: 'permission',

View File

@ -30,6 +30,7 @@ export default {
dateEnd: '结束日期', dateEnd: '结束日期',
}, },
table: { table: {
total: ' {0} ',
name: '名称', name: '名称',
type: '类型', type: '类型',
status: '状态', status: '状态',
@ -119,7 +120,7 @@ export default {
firewall: '防火墙', firewall: '防火墙',
database: '数据库', database: '数据库',
container: '容器', container: '容器',
cron: '计划任务', cronjob: '计划任务',
host: '主机', host: '主机',
security: '安全', security: '安全',
files: '文件管理', files: '文件管理',
@ -151,7 +152,6 @@ export default {
database: '备份数据库', database: '备份数据库',
missBackupAccount: '未能找到备份账号', missBackupAccount: '未能找到备份账号',
syncDate: '同步时间 ', syncDate: '同步时间 ',
syncDateName: '定期同步服务器时间 ',
releaseMemory: '释放内存', releaseMemory: '释放内存',
curl: '访问', curl: '访问',
taskName: '任务名称', taskName: '任务名称',
@ -161,7 +161,7 @@ export default {
exclusionRules: '排除规则', exclusionRules: '排除规则',
url: 'URL 地址', url: 'URL 地址',
target: '备份到', target: '备份到',
retainCopies: '保留份', retainDays: '保留天',
cronSpecRule: '请输入正确的执行周期', cronSpecRule: '请输入正确的执行周期',
perMonth: '每月', perMonth: '每月',
perWeek: '每周', perWeek: '每周',

View File

@ -7,7 +7,7 @@ const cronRouter = {
redirect: '/cronjobs', redirect: '/cronjobs',
meta: { meta: {
icon: 'p-plan', icon: 'p-plan',
title: 'menu.cron', title: 'menu.cronjob',
}, },
children: [ children: [
{ {

View File

@ -49,6 +49,22 @@ export function dateFromat(row: number, col: number, dataStr: any) {
return `${String(y)}-${String(m)}-${String(d)} ${String(h)}:${String(minute)}:${String(second)}`; return `${String(y)}-${String(m)}-${String(d)} ${String(h)}:${String(minute)}:${String(second)}`;
} }
export function dateFromatForName(dataStr: any) {
const date = new Date(dataStr);
const y = date.getFullYear();
let m: string | number = date.getMonth() + 1;
m = m < 10 ? `0${String(m)}` : m;
let d: string | number = date.getDate();
d = d < 10 ? `0${String(d)}` : d;
let h: string | number = date.getHours();
h = h < 10 ? `0${String(h)}` : h;
let minute: string | number = date.getMinutes();
minute = minute < 10 ? `0${String(minute)}` : minute;
let second: string | number = date.getSeconds();
second = second < 10 ? `0${String(second)}` : second;
return `${String(y)}${String(m)}${String(d)}${String(h)}${String(minute)}${String(second)}`;
}
export function dateFromatWithoutYear(dataStr: any) { export function dateFromatWithoutYear(dataStr: any) {
const date = new Date(dataStr); const date = new Date(dataStr);
let m: string | number = date.getMonth() + 1; let m: string | number = date.getMonth() + 1;

View File

@ -55,8 +55,12 @@
{{ $t('cronjob.handle') }} {{ $t('cronjob.handle') }}
</template> </template>
</el-table-column> </el-table-column>
<el-table-column :label="$t('cronjob.retainCopies')" prop="retainCopies" /> <el-table-column :label="$t('cronjob.retainDays')" prop="retainDays" />
<el-table-column :label="$t('cronjob.target')" prop="targetDir" /> <el-table-column :label="$t('cronjob.target')" prop="targetDir">
<template #default="{ row }">
{{ loadBackupName(row.targetDir) }}
</template>
</el-table-column>
<fu-table-operations type="icon" :buttons="buttons" :label="$t('commons.table.operate')" fix /> <fu-table-operations type="icon" :buttons="buttons" :label="$t('commons.table.operate')" fix />
</ComplexTable> </ComplexTable>
@ -71,8 +75,8 @@ import OperatrDialog from '@/views/cronjob/operate/index.vue';
import RecordDialog from '@/views/cronjob/record/index.vue'; import RecordDialog from '@/views/cronjob/record/index.vue';
import { loadZero } from '@/views/cronjob/options'; import { loadZero } from '@/views/cronjob/options';
import { onMounted, reactive, ref } from 'vue'; import { onMounted, reactive, ref } from 'vue';
import { loadBackupName } from '@/views/setting/helper';
import { deleteCronjob, getCronjobPage, updateStatus } from '@/api/modules/cronjob'; import { deleteCronjob, getCronjobPage, updateStatus } from '@/api/modules/cronjob';
import { loadBackupName } from '@/views/setting/helper';
import { loadWeek } from './options'; import { loadWeek } from './options';
import i18n from '@/lang'; import i18n from '@/lang';
import { Cronjob } from '@/api/interface/cronjob'; import { Cronjob } from '@/api/interface/cronjob';
@ -121,7 +125,7 @@ const onOpenDialog = async (
day: 1, day: 1,
hour: 2, hour: 2,
minute: 3, minute: 3,
retainCopies: 3, retainDays: 7,
}, },
) => { ) => {
let params = { let params = {

View File

@ -21,7 +21,7 @@
</el-form-item> </el-form-item>
<el-form-item :label="$t('cronjob.cronSpec')" prop="spec"> <el-form-item :label="$t('cronjob.cronSpec')" prop="spec">
<el-select style="width: 15%" v-model="dialogData.rowData!.specType"> <el-select style="width: 20%" v-model="dialogData.rowData!.specType">
<el-option v-for="item in specOptions" :key="item.label" :value="item.value" :label="item.label" /> <el-option v-for="item in specOptions" :key="item.label" :value="item.value" :label="item.label" />
</el-select> </el-select>
<el-select <el-select
@ -108,8 +108,8 @@
/> />
</el-select> </el-select>
</el-form-item> </el-form-item>
<el-form-item v-if="isBackup()" :label="$t('cronjob.retainCopies')" prop="retainCopies"> <el-form-item v-if="isBackup()" :label="$t('cronjob.retainDays')" prop="retainDays">
<el-input-number :min="1" v-model.number="dialogData.rowData!.retainCopies"></el-input-number> <el-input-number :min="1" :max="30" v-model.number="dialogData.rowData!.retainDays"></el-input-number>
</el-form-item> </el-form-item>
<el-form-item v-if="dialogData.rowData!.type === 'curl'" :label="$t('cronjob.url') + 'URL'" prop="url"> <el-form-item v-if="dialogData.rowData!.type === 'curl'" :label="$t('cronjob.url') + 'URL'" prop="url">
@ -241,7 +241,7 @@ const rules = reactive({
url: [Rules.requiredInput], url: [Rules.requiredInput],
sourceDir: [Rules.requiredSelect], sourceDir: [Rules.requiredSelect],
targetDirID: [Rules.requiredSelect, Rules.number], targetDirID: [Rules.requiredSelect, Rules.number],
retainCopies: [Rules.number], retainDays: [Rules.number],
}); });
type FormInstance = InstanceType<typeof ElForm>; type FormInstance = InstanceType<typeof ElForm>;

View File

@ -6,7 +6,7 @@ export const typeOptions = [
{ label: i18n.global.t('cronjob.database'), value: 'database' }, { label: i18n.global.t('cronjob.database'), value: 'database' },
{ label: i18n.global.t('cronjob.directory'), value: 'directory' }, { label: i18n.global.t('cronjob.directory'), value: 'directory' },
{ label: i18n.global.t('cronjob.syncDate'), value: 'sync' }, { label: i18n.global.t('cronjob.syncDate'), value: 'sync' },
{ label: i18n.global.t('cronjob.releaseMemory'), value: 'release' }, // { label: i18n.global.t('cronjob.releaseMemory'), value: 'release' },
{ label: i18n.global.t('cronjob.curl') + ' URL', value: 'curl' }, { label: i18n.global.t('cronjob.curl') + ' URL', value: 'curl' },
]; ];

View File

@ -32,10 +32,13 @@
{{ dateFromat(0, 0, item.startTime) }} {{ dateFromat(0, 0, item.startTime) }}
</li> </li>
</ul> </ul>
<div style="margin-top: 10px; margin-bottom: 5px; font-size: 12px; float: right">
<span>{{ $t('commons.table.total', [searchInfo.recordTotal]) }}</span>
</div>
</el-card> </el-card>
</el-col> </el-col>
<el-col :span="18"> <el-col :span="18">
<el-card style="height: 340px"> <el-card style="height: 352px">
<el-form> <el-form>
<el-row> <el-row>
<el-col :span="8"> <el-col :span="8">
@ -122,11 +125,21 @@
<el-col :span="8" v-if="isBackup()"> <el-col :span="8" v-if="isBackup()">
<el-form-item :label="$t('cronjob.target')"> <el-form-item :label="$t('cronjob.target')">
{{ loadBackupName(dialogData.rowData!.targetDir) }} {{ loadBackupName(dialogData.rowData!.targetDir) }}
<el-button
v-if="currentRecord?.records! !== 'errHandle'"
type="primary"
style="margin-left: 10px"
link
icon="Download"
@click="onDownload(currentRecord!.id, dialogData.rowData!.targetDirID)"
>
{{ $t('file.download') }}
</el-button>
</el-form-item> </el-form-item>
</el-col> </el-col>
<el-col :span="8" v-if="isBackup()"> <el-col :span="8" v-if="isBackup()">
<el-form-item :label="$t('cronjob.retainCopies')"> <el-form-item :label="$t('cronjob.retainDays')">
{{ dialogData.rowData!.retainCopies }} {{ dialogData.rowData!.retainDays }}
</el-form-item> </el-form-item>
</el-col> </el-col>
<el-col :span="8" v-if="dialogData.rowData!.type === 'curl'"> <el-col :span="8" v-if="dialogData.rowData!.type === 'curl'">
@ -179,11 +192,13 @@
<el-col :span="24"> <el-col :span="24">
<el-form-item :label="$t('commons.table.records')"> <el-form-item :label="$t('commons.table.records')">
<span <span
style="color: red"
v-if="currentRecord?.records! === 'errRecord' || currentRecord?.records! === 'errHandle'|| currentRecord?.records! === 'noRecord'" v-if="currentRecord?.records! === 'errRecord' || currentRecord?.records! === 'errHandle'|| currentRecord?.records! === 'noRecord'"
> >
{{ $t('cronjob.' + currentRecord?.records!) }} {{ currentRecord?.message }}
</span> </span>
<el-popover <el-popover
v-else
placement="right" placement="right"
:width="600" :width="600"
trigger="click" trigger="click"
@ -220,8 +235,8 @@ import { reactive, ref } from 'vue';
import { Cronjob } from '@/api/interface/cronjob'; import { Cronjob } from '@/api/interface/cronjob';
import { loadZero, loadWeek } from '@/views/cronjob/options'; import { loadZero, loadWeek } from '@/views/cronjob/options';
import { loadBackupName } from '@/views/setting/helper'; import { loadBackupName } from '@/views/setting/helper';
import { searchRecords, getRecordDetail } from '@/api/modules/cronjob'; import { searchRecords, getRecordDetail, download } from '@/api/modules/cronjob';
import { dateFromat } from '@/utils/util'; import { dateFromat, dateFromatForName } from '@/utils/util';
import i18n from '@/lang'; import i18n from '@/lang';
import { ElMessage } from 'element-plus'; import { ElMessage } from 'element-plus';
@ -305,12 +320,19 @@ const searchInfo = reactive({
pageSize: 5, pageSize: 5,
recordTotal: 0, recordTotal: 0,
cronjobID: 0, cronjobID: 0,
startTime: new Date(new Date().getTime() - 3600 * 1000 * 24 * 30), startTime: new Date(new Date().setHours(0, 0, 0, 0)),
endTime: new Date(), endTime: new Date(),
status: false, status: false,
}); });
const search = async () => { const search = async () => {
if (timeRangeLoad.value && timeRangeLoad.value.length === 2) {
searchInfo.startTime = timeRangeLoad.value[0];
searchInfo.endTime = timeRangeLoad.value[1];
} else {
searchInfo.startTime = new Date(new Date().setHours(0, 0, 0, 0));
searchInfo.endTime = new Date();
}
let params = { let params = {
page: searchInfo.page, page: searchInfo.page,
pageSize: searchInfo.pageSize, pageSize: searchInfo.pageSize,
@ -323,6 +345,24 @@ const search = async () => {
records.value = res.data.items || []; records.value = res.data.items || [];
searchInfo.recordTotal = res.data.total; searchInfo.recordTotal = res.data.total;
}; };
const onDownload = async (recordID: number, backupID: number) => {
let params = {
recordID: recordID,
backupAccountID: backupID,
};
const res = await download(params);
const downloadUrl = window.URL.createObjectURL(new Blob([res]));
const a = document.createElement('a');
a.style.display = 'none';
a.href = downloadUrl;
if (dialogData.value.rowData!.type === 'database') {
a.download = dateFromatForName(currentRecord.value?.startTime) + '.sql.gz';
} else {
a.download = dateFromatForName(currentRecord.value?.startTime) + '.tar.gz';
}
const event = new MouseEvent('click');
a.dispatchEvent(event);
};
const nextPage = async () => { const nextPage = async () => {
if (searchInfo.pageSize >= searchInfo.recordTotal) { if (searchInfo.pageSize >= searchInfo.recordTotal) {

View File

@ -8,8 +8,8 @@
<el-button type="primary" @click="onCreate"> <el-button type="primary" @click="onCreate">
{{ $t('commons.button.create') }} {{ $t('commons.button.create') }}
</el-button> </el-button>
<el-row :gutter="20" class="row-box" style="margin-top: 10px; margin-bottom: 20px"> <el-row :gutter="20" class="row-box">
<el-col v-for="item in data" :key="item.id" :span="8"> <el-col v-for="item in data" :key="item.id" :span="8" style="margin-top: 20px">
<el-card class="el-card"> <el-card class="el-card">
<template #header> <template #header>
<div class="card-header"> <div class="card-header">
@ -131,12 +131,8 @@
<el-form-item :label="$t('setting.address')" prop="varsJson.address" :rules="Rules.requiredInput"> <el-form-item :label="$t('setting.address')" prop="varsJson.address" :rules="Rules.requiredInput">
<el-input v-model="form.varsJson['address']" /> <el-input v-model="form.varsJson['address']" />
</el-form-item> </el-form-item>
<el-form-item <el-form-item :label="$t('setting.port')" prop="varsJson.port" :rules="[Rules.number]">
:label="$t('setting.port')" <el-input-number :min="0" :max="65535" v-model.number="form.varsJson['port']" />
prop="varsJson.port"
:rules="[Rules.number, { max: 65535 }]"
>
<el-input v-model.number="form.varsJson['port']" />
</el-form-item> </el-form-item>
<el-form-item <el-form-item
:label="$t('setting.username')" :label="$t('setting.username')"