1
0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-03-13 17:24:44 +08:00

feat: The result of the scheduled task execution is based on the task (#7586)

This commit is contained in:
ssongliu 2024-12-28 21:09:26 +08:00 committed by GitHub
parent 72c86c3525
commit ba1d65f35f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
23 changed files with 438 additions and 266 deletions

View File

@ -36,7 +36,7 @@ func (b *BaseApi) CreateSnapshot(c *gin.Context) {
return
}
if err := snapshotService.SnapshotCreate(req); err != nil {
if err := snapshotService.SnapshotCreate(req, false); err != nil {
helper.InternalServer(c, err)
return
}

View File

@ -134,6 +134,7 @@ type SearchRecord struct {
type Record struct {
ID uint `json:"id"`
TaskID string `json:"taskID"`
StartTime string `json:"startTime"`
Records string `json:"records"`
Status string `json:"status"`

View File

@ -6,6 +6,7 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/google/uuid"
"gorm.io/gorm"
)
@ -146,6 +147,7 @@ func (u *CronjobRepo) StartRecords(cronjobID uint, targetPath string) model.JobR
var record model.JobRecords
record.StartTime = time.Now()
record.CronjobID = cronjobID
record.TaskID = uuid.New().String()
record.Status = constant.StatusWaiting
if err := global.DB.Create(&record).Error; err != nil {
global.LOG.Errorf("create record status failed, err: %v", err)

View File

@ -17,7 +17,7 @@ import (
"github.com/pkg/errors"
)
func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time, taskID string) error {
var apps []model.AppInstall
if cronjob.AppID == "all" {
apps, _ = appInstallRepo.ListBy()
@ -46,7 +46,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time) e
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("app/%s/%s", app.App.Key, app.Name))
record.FileName = fmt.Sprintf("app_%s_%s.tar.gz", app.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if err := handleAppBackup(&app, nil, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, ""); err != nil {
if err := handleAppBackup(&app, nil, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, taskID); err != nil {
return err
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName))
@ -63,7 +63,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time) e
return nil
}
func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Time, taskID string) error {
webs := loadWebsForJob(cronjob)
if len(webs) == 0 {
return errors.New("no such website in database!")
@ -82,7 +82,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("website/%s", web.PrimaryDomain))
record.FileName = fmt.Sprintf("website_%s_%s.tar.gz", web.PrimaryDomain, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if err := handleWebsiteBackup(&web, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, ""); err != nil {
if err := handleWebsiteBackup(&web, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret, taskID); err != nil {
return err
}
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName))
@ -99,7 +99,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
return nil
}
func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Time, taskID string) error {
dbs := loadDbsForJob(cronjob)
if len(dbs) == 0 {
return errors.New("no such db in database!")
@ -120,11 +120,11 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name))
record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
if err := handleMysqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil {
if err := handleMysqlBackup(dbInfo, nil, backupDir, record.FileName, taskID); err != nil {
return err
}
} else {
if err := handlePostgresqlBackup(dbInfo, nil, backupDir, record.FileName, ""); err != nil {
if err := handlePostgresqlBackup(dbInfo, nil, backupDir, record.FileName, taskID); err != nil {
return err
}
}
@ -212,11 +212,15 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T
return nil
}
func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Time) error {
func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Time, taskID string) error {
accountMap, err := NewBackupClientMap(strings.Split(cronjob.SourceAccountIDs, ","))
if err != nil {
return err
}
itemData, err := NewISnapshotService().LoadSnapshotData()
if err != nil {
return err
}
var record model.BackupRecord
record.From = "cronjob"
@ -227,14 +231,28 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti
record.FileDir = "system_snapshot"
versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion"))
scope := "core"
if !global.IsMaster {
scope = "agent"
}
req := dto.SnapshotCreate{
Name: fmt.Sprintf("snapshot-1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)),
Name: fmt.Sprintf("snapshot-1panel-%s-%s-linux-%s-%s", scope, versionItem.Value, loadOs(), startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5)),
Secret: cronjob.Secret,
TaskID: taskID,
SourceAccountIDs: record.SourceAccountIDs,
DownloadAccountID: cronjob.DownloadAccountID,
AppData: itemData.AppData,
PanelData: itemData.PanelData,
BackupData: itemData.BackupData,
WithMonitorData: true,
WithLoginLog: true,
WithOperationLog: true,
WithSystemLog: true,
WithTaskLog: true,
}
if err := NewISnapshotService().HandleSnapshot(req); err != nil {
if err := NewISnapshotService().SnapshotCreate(req, true); err != nil {
return err
}
record.FileName = req.Name + ".tar.gz"

View File

@ -10,6 +10,7 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
@ -31,36 +32,26 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
if len(cronjob.Script) == 0 {
return
}
record.Records = u.generateLogsPath(*cronjob, record.StartTime)
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = u.handleShell(*cronjob, record.Records)
u.removeExpiredLog(*cronjob)
err = u.handleShell(*cronjob, record.TaskID)
case "curl":
if len(cronjob.URL) == 0 {
return
}
record.Records = u.generateLogsPath(*cronjob, record.StartTime)
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = cmd.ExecShell(record.Records, 24*time.Hour, "bash", "-c", "curl", cronjob.URL)
u.removeExpiredLog(*cronjob)
err = u.handleCurl(*cronjob, record.TaskID)
case "ntp":
err = u.handleNtpSync()
u.removeExpiredLog(*cronjob)
err = u.handleNtpSync(*cronjob, record.TaskID)
case "cutWebsiteLog":
var messageItem []string
messageItem, record.File, err = u.handleCutWebsiteLog(cronjob, record.StartTime)
message = []byte(strings.Join(messageItem, "\n"))
case "clean":
messageItem := ""
messageItem, err = u.handleSystemClean()
message = []byte(messageItem)
u.removeExpiredLog(*cronjob)
err = u.handleSystemClean(*cronjob, record.TaskID)
case "website":
err = u.handleWebsite(*cronjob, record.StartTime)
err = u.handleWebsite(*cronjob, record.StartTime, record.TaskID)
case "app":
err = u.handleApp(*cronjob, record.StartTime)
err = u.handleApp(*cronjob, record.StartTime, record.TaskID)
case "database":
err = u.handleDatabase(*cronjob, record.StartTime)
err = u.handleDatabase(*cronjob, record.StartTime, record.TaskID)
case "directory":
if len(cronjob.SourceDir) == 0 {
return
@ -70,7 +61,7 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
err = u.handleSystemLog(*cronjob, record.StartTime)
case "snapshot":
_ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records})
err = u.handleSnapshot(*cronjob, record.StartTime)
err = u.handleSnapshot(*cronjob, record.StartTime, record.TaskID)
}
if err != nil {
@ -90,53 +81,95 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
}()
}
func (u *CronjobService) handleShell(cronjob model.Cronjob, logPath string) error {
if len(cronjob.ContainerName) != 0 {
command := "sh"
if len(cronjob.Command) != 0 {
command = cronjob.Command
func (u *CronjobService) handleShell(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
taskItem.AddSubTask(i18n.GetWithName("HandleShell", cronjob.Name), func(t *task.Task) error {
if len(cronjob.ContainerName) != 0 {
command := "sh"
if len(cronjob.Command) != 0 {
command = cronjob.Command
}
scriptFile, _ := os.ReadFile(cronjob.Script)
return cmd.ExecShellWithTask(taskItem, 24*time.Hour, "docker", "exec", cronjob.ContainerName, command, "-c", strings.ReplaceAll(string(scriptFile), "\"", "\\\""))
}
scriptFile, _ := os.ReadFile(cronjob.Script)
return cmd.ExecShell(logPath, 24*time.Hour, "docker", "exec", cronjob.ContainerName, command, "-c", strings.ReplaceAll(string(scriptFile), "\"", "\\\""))
if len(cronjob.Executor) == 0 {
cronjob.Executor = "bash"
}
if cronjob.ScriptMode == "input" {
fileItem := pathUtils.Join(global.CONF.System.BaseDir, "1panel", "task", "shell", cronjob.Name, cronjob.Name+".sh")
_ = os.MkdirAll(pathUtils.Dir(fileItem), os.ModePerm)
shellFile, err := os.OpenFile(fileItem, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, constant.FilePerm)
if err != nil {
return err
}
defer shellFile.Close()
if _, err := shellFile.WriteString(cronjob.Script); err != nil {
return err
}
if len(cronjob.User) == 0 {
return cmd.ExecShellWithTask(taskItem, 24*time.Hour, cronjob.Executor, fileItem)
}
return cmd.ExecShellWithTask(taskItem, 24*time.Hour, "sudo", "-u", cronjob.User, cronjob.Executor, fileItem)
}
if len(cronjob.User) == 0 {
return cmd.ExecShellWithTask(taskItem, 24*time.Hour, cronjob.Executor, cronjob.Script)
}
if err := cmd.ExecShellWithTask(taskItem, 24*time.Hour, "sudo", "-u", cronjob.User, cronjob.Executor, cronjob.Script); err != nil {
return err
}
return nil
},
nil,
)
return taskItem.Execute()
}
func (u *CronjobService) handleCurl(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
if len(cronjob.Executor) == 0 {
cronjob.Executor = "bash"
taskItem.AddSubTask(i18n.GetWithName("HandleShell", cronjob.Name), func(t *task.Task) error {
if err := cmd.ExecShellWithTask(taskItem, 24*time.Hour, "bash", "-c", "curl", cronjob.URL); err != nil {
return err
}
return nil
},
nil,
)
return taskItem.Execute()
}
func (u *CronjobService) handleNtpSync(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for exec shell failed, err: %v", err)
return err
}
if cronjob.ScriptMode == "input" {
fileItem := pathUtils.Join(global.CONF.System.BaseDir, "1panel", "task", "shell", cronjob.Name, cronjob.Name+".sh")
_ = os.MkdirAll(pathUtils.Dir(fileItem), os.ModePerm)
shellFile, err := os.OpenFile(fileItem, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, constant.FilePerm)
taskItem.AddSubTask(i18n.GetMsgByKey("HandleNtpSync"), func(t *task.Task) error {
ntpServer, err := settingRepo.Get(settingRepo.WithByKey("NtpSite"))
if err != nil {
return err
}
defer shellFile.Close()
if _, err := shellFile.WriteString(cronjob.Script); err != nil {
taskItem.Logf("ntp server: %s", ntpServer.Value)
ntime, err := ntp.GetRemoteTime(ntpServer.Value)
if err != nil {
return err
}
if len(cronjob.User) == 0 {
return cmd.ExecShell(logPath, 24*time.Hour, cronjob.Executor, fileItem)
if err := ntp.UpdateSystemTime(ntime.Format(constant.DateTimeLayout)); err != nil {
return err
}
return cmd.ExecShell(logPath, 24*time.Hour, "sudo", "-u", cronjob.User, cronjob.Executor, fileItem)
}
if len(cronjob.User) == 0 {
return cmd.ExecShell(logPath, 24*time.Hour, cronjob.Executor, cronjob.Script)
}
return cmd.ExecShell(logPath, 24*time.Hour, "sudo", "-u", cronjob.User, cronjob.Executor, cronjob.Script)
}
func (u *CronjobService) handleNtpSync() error {
ntpServer, err := settingRepo.Get(settingRepo.WithByKey("NtpSite"))
if err != nil {
return err
}
ntime, err := ntp.GetRemoteTime(ntpServer.Value)
if err != nil {
return err
}
if err := ntp.UpdateSystemTime(ntime.Format(constant.DateTimeLayout)); err != nil {
return err
}
return nil
return nil
}, nil)
return taskItem.Execute()
}
func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time) ([]string, string, error) {
@ -201,8 +234,13 @@ func backupLogFile(dstFilePath, websiteLogDir string, fileOp files.FileOp) error
return nil
}
func (u *CronjobService) handleSystemClean() (string, error) {
return NewIDeviceService().CleanForCronjob()
func (u *CronjobService) handleSystemClean(cronjob model.Cronjob, taskID string) error {
taskItem, err := task.NewTaskWithOps(fmt.Sprintf("cronjob-%s", cronjob.Name), task.TaskHandle, task.TaskScopeCronjob, taskID, cronjob.ID)
if err != nil {
global.LOG.Errorf("new task for system clean failed, err: %v", err)
return err
}
return systemClean(taskItem)
}
func (u *CronjobService) uploadCronjobBackFile(cronjob model.Cronjob, accountMap map[string]backupClientHelper, file string) (string, error) {
@ -274,16 +312,6 @@ func (u *CronjobService) removeExpiredLog(cronjob model.Cronjob) {
}
}
func (u *CronjobService) generateLogsPath(cronjob model.Cronjob, startTime time.Time) string {
dir := fmt.Sprintf("%s/task/%s/%s", constant.DataDir, cronjob.Type, cronjob.Name)
if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
_ = os.MkdirAll(dir, os.ModePerm)
}
path := fmt.Sprintf("%s/%s.log", dir, startTime.Format(constant.DateTimeSlimLayout))
return path
}
func hasBackup(cronjobType string) bool {
return cronjobType == "app" || cronjobType == "database" || cronjobType == "website" || cronjobType == "directory" || cronjobType == "snapshot" || cronjobType == "log"
}

View File

@ -41,7 +41,6 @@ type IDeviceService interface {
Scan() dto.CleanData
Clean(req []dto.Clean)
CleanForCronjob() (string, error)
}
func NewIDeviceService() IDeviceService {

View File

@ -10,11 +10,13 @@ import (
"time"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/docker"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
"github.com/1Panel-dev/1Panel/agent/utils/common"
@ -290,64 +292,71 @@ func (u *DeviceService) Clean(req []dto.Clean) {
}
}
func (u *DeviceService) CleanForCronjob() (string, error) {
logs := ""
size := int64(0)
fileCount := 0
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, "1panel_original"), &logs, &size, &fileCount)
func systemClean(taskItem *task.Task) error {
taskItem.AddSubTask(i18n.GetMsgByKey("HandleSystemClean"), func(t *task.Task) error {
size := int64(0)
fileCount := 0
dropWithTask(path.Join(global.CONF.System.BaseDir, "1panel_original"), taskItem, &size, &fileCount)
upgradePath := path.Join(global.CONF.System.BaseDir, upgradePath)
upgradeFiles, _ := os.ReadDir(upgradePath)
if len(upgradeFiles) != 0 {
sort.Slice(upgradeFiles, func(i, j int) bool {
return upgradeFiles[i].Name() > upgradeFiles[j].Name()
})
for i := 0; i < len(upgradeFiles); i++ {
if i != 0 {
dropFileOrDirWithLog(path.Join(upgradePath, upgradeFiles[i].Name()), &logs, &size, &fileCount)
upgradePath := path.Join(global.CONF.System.BaseDir, upgradePath)
upgradeFiles, _ := os.ReadDir(upgradePath)
if len(upgradeFiles) != 0 {
sort.Slice(upgradeFiles, func(i, j int) bool {
return upgradeFiles[i].Name() > upgradeFiles[j].Name()
})
for i := 0; i < len(upgradeFiles); i++ {
if i != 0 {
dropWithTask(path.Join(upgradePath, upgradeFiles[i].Name()), taskItem, &size, &fileCount)
}
}
}
}
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, snapshotTmpPath), &logs, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.Backup, "system"), &logs, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, snapshotTmpPath), taskItem, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.Backup, "system"), taskItem, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, rollbackPath, "app"), &logs, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, rollbackPath, "website"), &logs, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, rollbackPath, "database"), &logs, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, rollbackPath, "app"), taskItem, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, rollbackPath, "website"), taskItem, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, rollbackPath, "database"), taskItem, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, oldOriginalPath), &logs, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, oldAppBackupPath), &logs, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, oldDownloadPath), &logs, &size, &fileCount)
oldUpgradePath := path.Join(global.CONF.System.BaseDir, oldUpgradePath)
oldUpgradeFiles, _ := os.ReadDir(oldUpgradePath)
if len(oldUpgradeFiles) != 0 {
for i := 0; i < len(oldUpgradeFiles); i++ {
dropFileOrDirWithLog(path.Join(oldUpgradePath, oldUpgradeFiles[i].Name()), &logs, &size, &fileCount)
}
}
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, tmpUploadPath), &logs, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, uploadPath), &logs, &size, &fileCount)
dropFileOrDirWithLog(path.Join(global.CONF.System.BaseDir, downloadPath), &logs, &size, &fileCount)
logPath := path.Join(global.CONF.System.BaseDir, logPath)
logFiles, _ := os.ReadDir(logPath)
if len(logFiles) != 0 {
for i := 0; i < len(logFiles); i++ {
if logFiles[i].Name() != "1Panel.log" {
dropFileOrDirWithLog(path.Join(logPath, logFiles[i].Name()), &logs, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, oldOriginalPath), taskItem, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, oldAppBackupPath), taskItem, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, oldDownloadPath), taskItem, &size, &fileCount)
oldUpgradePath := path.Join(global.CONF.System.BaseDir, oldUpgradePath)
oldUpgradeFiles, _ := os.ReadDir(oldUpgradePath)
if len(oldUpgradeFiles) != 0 {
for i := 0; i < len(oldUpgradeFiles); i++ {
dropWithTask(path.Join(oldUpgradePath, oldUpgradeFiles[i].Name()), taskItem, &size, &fileCount)
}
}
}
timeNow := time.Now().Format(constant.DateTimeLayout)
logs += fmt.Sprintf("\n%s: total clean: %s, total count: %d", timeNow, common.LoadSizeUnit2F(float64(size)), fileCount)
_ = settingRepo.Update("LastCleanTime", timeNow)
_ = settingRepo.Update("LastCleanSize", fmt.Sprintf("%v", size))
_ = settingRepo.Update("LastCleanData", fmt.Sprintf("%v", fileCount))
dropWithTask(path.Join(global.CONF.System.BaseDir, tmpUploadPath), taskItem, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, uploadPath), taskItem, &size, &fileCount)
dropWithTask(path.Join(global.CONF.System.BaseDir, downloadPath), taskItem, &size, &fileCount)
return logs, nil
logPath := path.Join(global.CONF.System.BaseDir, logPath)
logFiles, _ := os.ReadDir(logPath)
if len(logFiles) != 0 {
for i := 0; i < len(logFiles); i++ {
if logFiles[i].IsDir() {
continue
}
if logFiles[i].Name() != "1Panel.log" {
dropWithTask(path.Join(logPath, logFiles[i].Name()), taskItem, &size, &fileCount)
}
}
}
timeNow := time.Now().Format(constant.DateTimeLayout)
if fileCount != 0 {
taskItem.LogSuccessF("%s: total clean: %s, total count: %d", timeNow, common.LoadSizeUnit2F(float64(size)), fileCount)
}
_ = settingRepo.Update("LastCleanTime", timeNow)
_ = settingRepo.Update("LastCleanSize", fmt.Sprintf("%v", size))
_ = settingRepo.Update("LastCleanData", fmt.Sprintf("%v", fileCount))
return nil
}, nil)
return taskItem.Execute()
}
func loadSnapshotTree(fileOp fileUtils.FileOp) []dto.CleanTree {
@ -695,18 +704,19 @@ func dropVolumes() {
}
}
func dropFileOrDirWithLog(itemPath string, log *string, size *int64, count *int) {
func dropWithTask(itemPath string, taskItem *task.Task, size *int64, count *int) {
itemSize := int64(0)
itemCount := 0
scanFile(itemPath, &itemSize, &itemCount)
*size += itemSize
*count += itemCount
if err := os.RemoveAll(itemPath); err != nil {
global.LOG.Errorf("drop file %s failed, err %v", itemPath, err)
*log += fmt.Sprintf("- drop file %s failed, err: %v \n\n", itemPath, err)
taskItem.LogFailed(fmt.Sprintf("drop file %s, err %v", itemPath, err))
return
}
*log += fmt.Sprintf("+ drop file %s successful!, size: %s, count: %d \n\n", itemPath, common.LoadSizeUnit2F(float64(itemSize)), itemCount)
if itemCount != 0 {
taskItem.LogSuccessF("drop file %s, size: %s, count: %d", itemPath, common.LoadSizeUnit2F(float64(itemSize)), itemCount)
}
}
func scanFile(pathItem string, size *int64, count *int) {

View File

@ -30,7 +30,7 @@ type ISnapshotService interface {
SearchWithPage(req dto.PageSnapshot) (int64, interface{}, error)
LoadSize(req dto.SearchWithPage) ([]dto.SnapshotFile, error)
LoadSnapshotData() (dto.SnapshotData, error)
SnapshotCreate(req dto.SnapshotCreate) error
SnapshotCreate(req dto.SnapshotCreate, isCron bool) error
SnapshotReCreate(id uint) error
SnapshotRecover(req dto.SnapshotRecover) error
SnapshotRollback(req dto.SnapshotRecover) error
@ -38,8 +38,6 @@ type ISnapshotService interface {
Delete(req dto.SnapshotBatchDelete) error
UpdateDescription(req dto.UpdateDescription) error
HandleSnapshot(req dto.SnapshotCreate) error
}
func NewISnapshotService() ISnapshotService {
@ -87,6 +85,8 @@ func (u *SnapshotService) LoadSize(req dto.SearchWithPage) ([]dto.SnapshotFile,
backupName := fmt.Sprintf("%s - %s", backup.Type, backup.Name)
clientMap[uint(itemVal)] = loadSizeHelper{backupPath: strings.TrimLeft(backup.BackupPath, "/"), client: client, isOk: true, backupName: backupName}
accountNames = append(accountNames, backupName)
} else {
accountNames = append(accountNames, clientMap[uint(itemVal)].backupName)
}
}
data.DefaultDownload = clientMap[records[i].DownloadAccountID].backupName

View File

@ -24,10 +24,16 @@ import (
"gorm.io/gorm"
)
func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error {
func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate, isCron bool) error {
versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion"))
req.Name = fmt.Sprintf("1panel-%s-linux-%s-%s", versionItem.Value, loadOs(), time.Now().Format(constant.DateTimeSlimLayout))
scope := "core"
if !global.IsMaster {
scope = "agent"
}
if !isCron {
req.Name = fmt.Sprintf("1panel-%s-%s-linux-%s-%s", versionItem.Value, scope, loadOs(), time.Now().Format(constant.DateTimeSlimLayout))
}
appItem, _ := json.Marshal(req.AppData)
panelItem, _ := json.Marshal(req.PanelData)
backupItem, _ := json.Marshal(req.BackupData)
@ -57,9 +63,16 @@ func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error {
}
req.ID = snap.ID
if err := u.HandleSnapshot(req); err != nil {
taskItem, err := task.NewTaskWithOps(req.Name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, req.ID)
if err != nil {
global.LOG.Errorf("new task for create snapshot failed, err: %v", err)
return err
}
if !isCron {
go handleSnapshot(req, taskItem)
return nil
}
handleSnapshot(req, taskItem)
return nil
}
@ -85,101 +98,95 @@ func (u *SnapshotService) SnapshotReCreate(id uint) error {
return err
}
req.TaskID = taskModel.ID
if err := u.HandleSnapshot(req); err != nil {
return err
}
return nil
}
func (u *SnapshotService) HandleSnapshot(req dto.SnapshotCreate) error {
taskItem, err := task.NewTaskWithOps(req.Name, task.TaskCreate, task.TaskScopeSnapshot, req.TaskID, req.ID)
if err != nil {
global.LOG.Errorf("new task for create snapshot failed, err: %v", err)
return err
}
go handleSnapshot(req, taskItem)
return nil
}
func handleSnapshot(req dto.SnapshotCreate, taskItem *task.Task) {
rootDir := path.Join(global.CONF.System.BaseDir, "1panel/tmp/system", req.Name)
itemHelper := snapHelper{SnapID: req.ID, Task: *taskItem, FileOp: files.NewFileOp(), Ctx: context.Background()}
baseDir := path.Join(rootDir, "base")
_ = os.MkdirAll(baseDir, os.ModePerm)
go func() {
taskItem.AddSubTaskWithAlias(
"SnapDBInfo",
func(t *task.Task) error { return loadDbConn(&itemHelper, rootDir, req) },
nil,
)
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapBaseInfo" {
taskItem.AddSubTaskWithAlias(
"SnapDBInfo",
func(t *task.Task) error { return loadDbConn(&itemHelper, rootDir, req) },
"SnapBaseInfo",
func(t *task.Task) error { return snapBaseData(itemHelper, baseDir) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapInstallApp" {
taskItem.AddSubTaskWithAlias(
"SnapInstallApp",
func(t *task.Task) error { return snapAppImage(itemHelper, req, rootDir) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapLocalBackup" {
taskItem.AddSubTaskWithAlias(
"SnapLocalBackup",
func(t *task.Task) error { return snapBackupData(itemHelper, req, rootDir) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapPanelData" {
taskItem.AddSubTaskWithAlias(
"SnapPanelData",
func(t *task.Task) error { return snapPanelData(itemHelper, req, rootDir) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapBaseInfo" {
taskItem.AddSubTaskWithAlias(
"SnapBaseInfo",
func(t *task.Task) error { return snapBaseData(itemHelper, baseDir) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapInstallApp" {
taskItem.AddSubTaskWithAlias(
"SnapInstallApp",
func(t *task.Task) error { return snapAppImage(itemHelper, req, rootDir) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapLocalBackup" {
taskItem.AddSubTaskWithAlias(
"SnapLocalBackup",
func(t *task.Task) error { return snapBackupData(itemHelper, req, rootDir) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapPanelData" {
taskItem.AddSubTaskWithAlias(
"SnapPanelData",
func(t *task.Task) error { return snapPanelData(itemHelper, req, rootDir) },
nil,
)
req.InterruptStep = ""
}
taskItem.AddSubTask(
"SnapCloseDBConn",
taskItem.AddSubTask(
"SnapCloseDBConn",
func(t *task.Task) error {
taskItem.Log("---------------------- 6 / 8 ----------------------")
common.CloseDB(itemHelper.snapAgentDB)
common.CloseDB(itemHelper.snapCoreDB)
return nil
},
nil,
)
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapCompress" {
taskItem.AddSubTaskWithAlias(
"SnapCompress",
func(t *task.Task) error { return snapCompress(itemHelper, rootDir, req.Secret) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapUpload" {
taskItem.AddSubTaskWithAlias(
"SnapUpload",
func(t *task.Task) error {
taskItem.Log("---------------------- 6 / 8 ----------------------")
common.CloseDB(itemHelper.snapAgentDB)
common.CloseDB(itemHelper.snapCoreDB)
return nil
return snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir))
},
nil,
)
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapCompress" {
taskItem.AddSubTaskWithAlias(
"SnapCompress",
func(t *task.Task) error { return snapCompress(itemHelper, rootDir, req.Secret) },
nil,
)
req.InterruptStep = ""
}
if len(req.InterruptStep) == 0 || req.InterruptStep == "SnapUpload" {
taskItem.AddSubTaskWithAlias(
"SnapUpload",
func(t *task.Task) error {
return snapUpload(itemHelper, req.SourceAccountIDs, fmt.Sprintf("%s.tar.gz", rootDir))
},
nil,
)
req.InterruptStep = ""
}
if err := taskItem.Execute(); err != nil {
_ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep})
return
}
_ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusSuccess, "interrupt_step": ""})
_ = os.RemoveAll(rootDir)
}()
return nil
req.InterruptStep = ""
}
if err := taskItem.Execute(); err != nil {
_ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep})
return
}
_ = snapshotRepo.Update(req.ID, map[string]interface{}{"status": constant.StatusSuccess, "interrupt_step": ""})
_ = os.RemoveAll(rootDir)
}
type snapHelper struct {

View File

@ -3,13 +3,14 @@ package task
import (
"context"
"fmt"
"github.com/1Panel-dev/1Panel/agent/buserr"
"log"
"os"
"path"
"strconv"
"time"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/constant"
@ -68,6 +69,7 @@ const (
TaskScopeRuntime = "Runtime"
TaskScopeDatabase = "Database"
TaskScopeCronjob = "Cronjob"
TaskScopeSystem = "System"
TaskScopeAppStore = "AppStore"
TaskScopeSnapshot = "Snapshot"
TaskScopeContainer = "Container"
@ -234,14 +236,14 @@ func (t *Task) DeleteLogFile() {
func (t *Task) LogWithStatus(msg string, err error) {
if err != nil {
t.Logger.Printf(i18n.GetWithNameAndErr("FailedStatus", msg, err))
t.Logger.Print(i18n.GetWithNameAndErr("FailedStatus", msg, err))
} else {
t.Logger.Printf(i18n.GetWithName("SuccessStatus", msg))
t.Logger.Print(i18n.GetWithName("SuccessStatus", msg))
}
}
func (t *Task) Log(msg string) {
t.Logger.Printf(msg)
t.Logger.Print(msg)
}
func (t *Task) Logf(format string, v ...any) {

View File

@ -187,6 +187,9 @@ ErrFirewallBoth: "Both firewalld and ufw services are detected on the system. To
#cronjob
ErrCutWebsiteLog: "{{ .name }} website log cutting failed, error {{ .err }}"
CutWebsiteLogSuccess: "{{ .name }} website log cut successfully, backup path {{ .path }}"
HandleShell: "Execute script {{ .name }}"
HandleNtpSync: "System time synchronization"
HandleSystemClean: "System cache cleanup"
#toolbox
ErrNotExistUser: "The current user does not exist. Please modify and retry!"

View File

@ -188,6 +188,9 @@ ErrFirewallBoth: "檢測到系統同時存在 firewalld 或 ufw 服務,為避
#cronjob
ErrCutWebsiteLog: "{{ .name }} 網站日誌切割失敗,錯誤 {{ .err }}"
CutWebsiteLogSuccess: "{{ .name }} 網站日誌切割成功,備份路徑 {{ .path }}"
HandleShell: "執行腳本 {{ .name }}"
HandleNtpSync: "系統時間同步"
HandleSystemClean: "系統快取清理"
#toolbox
ErrNotExistUser: "當前使用者不存在,請修改後重試!"

View File

@ -186,6 +186,9 @@ ErrFirewallBoth: "检测到系统同时存在 firewalld 或 ufw 服务,为避
#cronjob
ErrCutWebsiteLog: "{{ .name }} 网站日志切割失败,错误 {{ .err }}"
CutWebsiteLogSuccess: "{{ .name }} 网站日志切割成功,备份路径 {{ .path }}"
HandleShell: "执行脚本 {{ .name }}"
HandleNtpSync: "系统时间同步"
HandleSystemClean: "系统缓存清理"
#toolbox
ErrNotExistUser: "当前用户不存在,请修改后重试!"

View File

@ -11,6 +11,7 @@ import (
"strings"
"time"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
)
@ -137,6 +138,39 @@ func ExecShell(outPath string, timeout time.Duration, name string, arg ...string
return nil
}
type CustomWriter struct {
taskItem *task.Task
}
func (cw *CustomWriter) Write(p []byte) (n int, err error) {
cw.taskItem.Log(string(p))
return len(p), nil
}
func ExecShellWithTask(taskItem *task.Task, timeout time.Duration, name string, arg ...string) error {
customWriter := &CustomWriter{taskItem: taskItem}
cmd := exec.Command(name, arg...)
cmd.Stdout = customWriter
cmd.Stderr = customWriter
if err := cmd.Start(); err != nil {
return err
}
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()
after := time.After(timeout)
select {
case <-after:
_ = cmd.Process.Kill()
return buserr.New(constant.ErrCmdTimeout)
case err := <-done:
if err != nil {
return err
}
}
return nil
}
func Execf(cmdStr string, a ...interface{}) (string, error) {
cmd := exec.Command("bash", "-c", fmt.Sprintf(cmdStr, a...))
var stdout, stderr bytes.Buffer

View File

@ -105,6 +105,7 @@ export namespace Cronjob {
}
export interface Record {
id: number;
taskID: string;
file: string;
startTime: string;
records: string;

View File

@ -4,7 +4,13 @@
<el-checkbox border v-model="tailLog" class="float-left" @change="changeTail(false)" v-if="showTail">
{{ $t('commons.button.watch') }}
</el-checkbox>
<el-button class="ml-2.5" @click="onDownload" icon="Download" :disabled="logs.length === 0">
<el-button
class="ml-2.5"
v-if="showDownload"
@click="onDownload"
icon="Download"
:disabled="logs.length === 0"
>
{{ $t('file.download') }}
</el-button>
<span v-if="$slots.button" class="ml-2.5">
@ -67,6 +73,10 @@ const props = defineProps({
type: Boolean,
default: true,
},
showDownload: {
type: Boolean,
default: true,
},
});
const stopSignals = [
'docker-compose up failed!',

View File

@ -0,0 +1,22 @@
<template>
<div>
<LogFile :config="config" :showTail="false" :showDownload="false" :heightDiff="heightDiff"></LogFile>
</div>
</template>
<script lang="ts" setup>
import { reactive } from 'vue';
const props = defineProps({
taskID: String,
heightDiff: Number,
});
const config = reactive({
taskID: props.taskID,
type: 'task',
taskOperate: '',
resourceID: 0,
taskType: '',
tail: false,
});
</script>

View File

@ -1658,6 +1658,8 @@ const message = {
'Backup files not in the current backup list, please try downloading from the file directory and importing for backup.',
snapshot: 'Snapshot',
noAppData: 'No system applications available for selection',
noBackupData: 'No backup data available for selection',
stepBaseData: 'Base Data',
stepAppData: 'System Application',
stepPanelData: 'System Data',

View File

@ -1469,6 +1469,8 @@ const message = {
backupJump: '未在當前備份列表中的備份檔案請嘗試從檔案目錄中下載後導入備份',
snapshot: '快照',
noAppData: '暫無可選擇系統應用',
noBackupData: '暫無可選擇備份數據',
stepBaseData: '基礎數據',
stepAppData: '系統應用',
stepPanelData: '系統數據',

View File

@ -1470,6 +1470,8 @@ const message = {
backupJump: '未在当前备份列表中的备份文件请尝试从文件目录中下载后导入备份',
snapshot: '快照',
noAppData: '暂无可选择系统应用',
noBackupData: '暂无可选择备份数据',
stepBaseData: '基础数据',
stepAppData: '系统应用',
stepPanelData: '系统数据',

View File

@ -368,7 +368,7 @@
</el-form-item>
</div>
<el-form-item :label="$t('cronjob.backupContent')">
<el-form-item v-if="dialogData.rowData!.type === 'directory'" :label="$t('cronjob.backupContent')">
<el-radio-group v-model="dialogData.rowData!.isDir">
<el-radio :value="true">{{ $t('file.dir') }}</el-radio>
<el-radio :value="false">{{ $t('file.file') }}</el-radio>

View File

@ -172,6 +172,14 @@
></highlightjs>
</div>
</el-row>
<el-row v-if="currentRecord?.taskID">
<TaskLog
class="w-full"
:taskID="currentRecord?.taskID"
:key="currentRecord?.taskID"
:heightDiff="200"
/>
</el-row>
</el-form>
</el-col>
</el-row>
@ -221,6 +229,7 @@ import { MsgSuccess } from '@/utils/message';
import { listDbItems } from '@/api/modules/database';
import { ListAppInstalled } from '@/api/modules/app';
import { shortcuts } from '@/utils/shortcuts';
import TaskLog from '@/components/task-log/log-without-dialog.vue';
const loading = ref();
const refresh = ref(false);

View File

@ -53,31 +53,36 @@
</el-form>
</fu-step>
<fu-step id="appData" :title="$t('setting.stepAppData')">
<el-checkbox
class="ml-6"
v-model="form.backupAllImage"
@change="selectAllImage"
:label="$t('setting.selectAllImage')"
size="large"
/>
<el-tree
style="max-width: 600px"
ref="appRef"
node-key="id"
:data="form.appData"
:props="defaultProps"
@check-change="onChangeAppData"
show-checkbox
>
<template #default="{ data }">
<div class="float-left">
<span>{{ loadApp18n(data.label) }}</span>
</div>
<div class="ml-4 float-left">
<span v-if="data.size">{{ computeSize(data.size) }}</span>
</div>
</template>
</el-tree>
<div class="mt-5 mb-5" v-if="!form.appData || form.appData.length === 0">
<span class="input-help">{{ $t('setting.noAppData') }}</span>
</div>
<div v-else>
<el-checkbox
class="ml-6"
v-model="form.backupAllImage"
@change="selectAllImage"
:label="$t('setting.selectAllImage')"
size="large"
/>
<el-tree
style="max-width: 600px"
ref="appRef"
node-key="id"
:data="form.appData"
:props="defaultProps"
@check-change="onChangeAppData"
show-checkbox
>
<template #default="{ data }">
<div class="float-left">
<span>{{ loadApp18n(data.label) }}</span>
</div>
<div class="ml-4 float-left">
<span v-if="data.size">{{ computeSize(data.size) }}</span>
</div>
</template>
</el-tree>
</div>
</fu-step>
<fu-step id="panelData" :title="$t('setting.stepPanelData')">
<el-tree
@ -99,23 +104,28 @@
</el-tree>
</fu-step>
<fu-step id="backupData" :title="$t('setting.stepBackupData')">
<el-tree
style="max-width: 600px"
ref="backupRef"
node-key="id"
:data="form.backupData"
:props="defaultProps"
show-checkbox
>
<template #default="{ node, data }">
<div class="float-left">
<span>{{ load18n(node, data.label) }}</span>
</div>
<div class="ml-4 float-left">
<span v-if="data.size">{{ computeSize(data.size) }}</span>
</div>
</template>
</el-tree>
<div class="mt-5 mb-5" v-if="!form.appData || form.appData.length === 0">
<span class="input-help">{{ $t('setting.noBackupData') }}</span>
</div>
<div v-else>
<el-tree
style="max-width: 600px"
ref="backupRef"
node-key="id"
:data="form.backupData"
:props="defaultProps"
show-checkbox
>
<template #default="{ node, data }">
<div class="float-left">
<span>{{ load18n(node, data.label) }}</span>
</div>
<div class="ml-4 float-left">
<span v-if="data.size">{{ computeSize(data.size) }}</span>
</div>
</template>
</el-tree>
</div>
</fu-step>
<fu-step id="otherData" :title="$t('setting.stepOtherData')">
<div class="ml-5">
@ -239,16 +249,20 @@ const beforeLeave = async (stepItem: any) => {
return false;
}
case 'appData':
let appChecks = appRef.value.getCheckedNodes();
loadCheckForSubmit(appChecks, form.appData);
if (form.appData && form.appData.length !== 0) {
let appChecks = appRef.value.getCheckedNodes();
loadCheckForSubmit(appChecks, form.appData);
}
return true;
case 'panelData':
let panelChecks = panelRef.value.getCheckedNodes();
loadCheckForSubmit(panelChecks, form.panelData);
return true;
case 'backupData':
let backupChecks = backupRef.value.getCheckedNodes();
loadCheckForSubmit(backupChecks, form.backupData);
if (form.backupData && form.backupData.length !== 0) {
let backupChecks = backupRef.value.getCheckedNodes();
loadCheckForSubmit(backupChecks, form.backupData);
}
return true;
}
};