diff --git a/backend/app/service/snapshot.go b/backend/app/service/snapshot.go index 66c0e404f..c81bbd3e0 100644 --- a/backend/app/service/snapshot.go +++ b/backend/app/service/snapshot.go @@ -6,8 +6,6 @@ import ( "fmt" "os" "path" - "regexp" - "strconv" "strings" "sync" "time" @@ -18,7 +16,6 @@ import ( "github.com/1Panel-dev/1Panel/backend/global" "github.com/1Panel-dev/1Panel/backend/utils/cmd" "github.com/1Panel-dev/1Panel/backend/utils/compose" - "github.com/1Panel-dev/1Panel/backend/utils/docker" "github.com/1Panel-dev/1Panel/backend/utils/files" "github.com/jinzhu/copier" "github.com/pkg/errors" @@ -140,23 +137,7 @@ func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { if !req.IsNew && len(snap.InterruptStep) != 0 && len(snap.RollbackStatus) != 0 { return fmt.Errorf("the snapshot has been rolled back and cannot be restored again") } - isNewSnapshot := isNewSnapVersion(snap.Version) - isReTry := false - if len(snap.InterruptStep) != 0 && !req.IsNew { - isReTry = true - } - backup, err := backupRepo.Get(commonRepo.WithByType(snap.DefaultDownload)) - if err != nil { - return err - } - client, err := NewIBackupService().NewClient(&backup) - if err != nil { - return err - } - localDir, err := loadLocalDir() - if err != nil { - return err - } + baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { _ = os.MkdirAll(baseDir, os.ModePerm) @@ -164,152 +145,7 @@ func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting}) _ = settingRepo.Update("SystemStatus", "Recovering") - go func() { - _ = global.Cron.Stop() - defer func() { - global.Cron.Start() - }() - operation := "recover" - if isReTry { - operation = "re-recover" - } - if !isReTry || snap.InterruptStep == "Download" || (isReTry && req.ReDownload) { - itemBackupPath := strings.TrimPrefix(backup.BackupPath, "/") - itemBackupPath = strings.TrimSuffix(itemBackupPath, "/") - ok, err := client.Download(fmt.Sprintf("%s/system_snapshot/%s.tar.gz", itemBackupPath, snap.Name), fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name)) - if err != nil || !ok { - if req.ReDownload { - updateRecoverStatus(snap.ID, snap.InterruptStep, constant.StatusFailed, fmt.Sprintf("download file %s from %s failed, err: %v", snap.Name, backup.Type, err)) - return - } - updateRecoverStatus(snap.ID, "Download", constant.StatusFailed, fmt.Sprintf("download file %s from %s failed, err: %v", snap.Name, backup.Type, err)) - return - } - isReTry = false - } - fileOp := files.NewFileOp() - if !isReTry || snap.InterruptStep == "Decompress" || (isReTry && req.ReDownload) { - if err := handleUnTar(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir); err != nil { - if req.ReDownload { - updateRecoverStatus(snap.ID, snap.InterruptStep, constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return - } - updateRecoverStatus(snap.ID, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return - } - isReTry = false - } - rootDir := fmt.Sprintf("%s/%s", baseDir, snap.Name) - - snapJson, err := u.readFromJson(fmt.Sprintf("%s/snapshot.json", rootDir)) - if err != nil { - updateRecoverStatus(snap.ID, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) - return - } - if snap.InterruptStep == "Readjson" { - isReTry = false - } - u.OriginalPath = fmt.Sprintf("%s/1panel_original/original_%s", snapJson.BaseDir, snap.Name) - _ = os.MkdirAll(u.OriginalPath, os.ModePerm) - - snapJson.OldBaseDir = global.CONF.System.BaseDir - snapJson.OldPanelDataDir = path.Join(global.CONF.System.BaseDir, "1panel") - snapJson.OldBackupDataDir = localDir - recoverPanelDir := fmt.Sprintf("%s/%s/1panel", baseDir, snap.Name) - liveRestore := false - - if !isNewSnapshot { - if !isReTry || snap.InterruptStep == "LoadDockerJson" { - snapJson.OldDockerDataDir, liveRestore, err = u.loadDockerDataDir() - if err != nil { - updateRecoverStatus(snap.ID, "LoadDockerJson", constant.StatusFailed, fmt.Sprintf("load docker data dir failed, err: %v", err)) - return - } - isReTry = false - } - if liveRestore { - if err := u.updateLiveRestore(false); err != nil { - updateRecoverStatus(snap.ID, "UpdateLiveRestore", constant.StatusFailed, fmt.Sprintf("update docker daemon.json live-restore conf failed, err: %v", err)) - return - } - isReTry = false - } - _ = u.saveJson(snapJson, u.OriginalPath) - - _, _ = cmd.Exec("systemctl stop docker") - if !isReTry || snap.InterruptStep == "DockerDir" { - if err := u.handleDockerDatas(fileOp, operation, rootDir, snapJson.DockerDataDir); err != nil { - updateRecoverStatus(snap.ID, "DockerDir", constant.StatusFailed, err.Error()) - return - } - isReTry = false - } - } else { - _ = u.saveJson(snapJson, u.OriginalPath) - if !isReTry || snap.InterruptStep == "DockerDir" { - if err := u.handleDockerDatasWithSave(fileOp, operation, rootDir, ""); err != nil { - updateRecoverStatus(snap.ID, "DockerDir", constant.StatusFailed, err.Error()) - return - } - isReTry = false - } - } - - if !isReTry || snap.InterruptStep == "DaemonJson" { - if err := u.handleDaemonJson(fileOp, operation, rootDir+"/docker/daemon.json", u.OriginalPath); err != nil { - updateRecoverStatus(snap.ID, "DaemonJson", constant.StatusFailed, err.Error()) - return - } - isReTry = false - } - _, _ = cmd.Exec("systemctl restart docker") - - if !isReTry || snap.InterruptStep == "1PanelBinary" { - if err := u.handlePanelBinary(fileOp, operation, recoverPanelDir+"/1panel", u.OriginalPath+"/1panel"); err != nil { - updateRecoverStatus(snap.ID, "1PanelBinary", constant.StatusFailed, err.Error()) - return - } - isReTry = false - } - if !isReTry || snap.InterruptStep == "1PctlBinary" { - if err := u.handlePanelctlBinary(fileOp, operation, recoverPanelDir+"/1pctl", u.OriginalPath+"/1pctl"); err != nil { - updateRecoverStatus(snap.ID, "1PctlBinary", constant.StatusFailed, err.Error()) - return - } - isReTry = false - } - if !isReTry || snap.InterruptStep == "1PanelService" { - if err := u.handlePanelService(fileOp, operation, recoverPanelDir+"/1panel.service", u.OriginalPath+"/1panel.service"); err != nil { - updateRecoverStatus(snap.ID, "1PanelService", constant.StatusFailed, err.Error()) - return - } - isReTry = false - } - - if !isReTry || snap.InterruptStep == "1PanelBackups" { - if err := u.handleBackupDatas(fileOp, operation, rootDir, snapJson.BackupDataDir); err != nil { - updateRecoverStatus(snap.ID, "1PanelBackups", constant.StatusFailed, err.Error()) - return - } - isReTry = false - } - - if !isReTry || snap.InterruptStep == "1PanelData" { - if err := u.handlePanelDatas(snap.ID, fileOp, operation, rootDir, snapJson.PanelDataDir, localDir, snapJson.OldDockerDataDir); err != nil { - updateRecoverStatus(snap.ID, "1PanelData", constant.StatusFailed, err.Error()) - return - } - isReTry = false - } - - if isNewSnapshot { - _ = rebuildAllAppInstall() - } - - _ = os.RemoveAll(rootDir) - global.LOG.Info("recover successful") - _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") - }() + go u.HandleSnapshotRecover(snap, true, req) return nil } @@ -319,133 +155,9 @@ func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error { if err != nil { return err } - if snap.InterruptStep == "Download" || snap.InterruptStep == "Decompress" || snap.InterruptStep == "Readjson" { - return nil - } - localDir, err := loadLocalDir() - if err != nil { - return err - } - fileOp := files.NewFileOp() - isNewSnapshot := isNewSnapVersion(snap.Version) - - rootDir := path.Join(localDir, fmt.Sprintf("system/%s/%s", snap.Name, snap.Name)) - - _ = settingRepo.Update("SystemStatus", "Rollbacking") - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"rollback_status": constant.StatusWaiting}) - go func() { - _ = global.Cron.Stop() - defer func() { - global.Cron.Start() - }() - - u.OriginalPath = fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) - if _, err := os.Stat(u.OriginalPath); err != nil && os.IsNotExist(err) { - return - } - snapJson, err := u.readFromJson(fmt.Sprintf("%s/snapshot.json", u.OriginalPath)) - if err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, fmt.Sprintf("read snapjson file failed, err: %v", err)) - return - } - - if !isNewSnapshot { - _, _ = cmd.Exec("systemctl stop docker") - if err := u.handleDockerDatas(fileOp, "rollback", u.OriginalPath, snapJson.OldDockerDataDir); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - defer func() { - _, _ = cmd.Exec("systemctl restart docker") - }() - if snap.InterruptStep == "DockerDir" { - return - } - if snapJson.LiveRestoreEnabled { - if err := u.updateLiveRestore(true); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - } - if snap.InterruptStep == "UpdateLiveRestore" { - return - } - } else { - if err := u.handleDockerDatasWithSave(fileOp, "rollback", u.OriginalPath, ""); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - defer func() { - _ = rebuildAllAppInstall() - }() - if snap.InterruptStep == "DockerDir" { - return - } - } - - if err := u.handleDaemonJson(fileOp, "rollback", u.OriginalPath+"/daemon.json", ""); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - if snap.InterruptStep == "DaemonJson" { - return - } - - if err := u.handlePanelBinary(fileOp, "rollback", u.OriginalPath+"/1panel", ""); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - if snap.InterruptStep == "1PanelBinary" { - return - } - - if err := u.handlePanelctlBinary(fileOp, "rollback", u.OriginalPath+"/1pctl", ""); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - if snap.InterruptStep == "1PctlBinary" { - return - } - - if err := u.handlePanelService(fileOp, "rollback", u.OriginalPath+"/1panel.service", ""); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - if snap.InterruptStep == "1PanelService" { - _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") - return - } - - if err := u.handleBackupDatas(fileOp, "rollback", u.OriginalPath, snapJson.OldBackupDataDir); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - if snap.InterruptStep == "1PanelBackups" { - _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") - return - } - - if err := u.handlePanelDatas(snap.ID, fileOp, "rollback", u.OriginalPath, snapJson.OldPanelDataDir, "", ""); err != nil { - updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - if snap.InterruptStep == "1PanelData" { - _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") - return - } - - _ = os.RemoveAll(rootDir) - global.LOG.Info("rollback successful") - _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") - }() - return nil -} - -func (u *SnapshotService) saveJson(snapJson SnapshotJson, path string) error { - remarkInfo, _ := json.MarshalIndent(snapJson, "", "\t") - if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", path), remarkInfo, 0640); err != nil { - return err - } + req.IsNew = false + snap.InterruptStep = "Readjson" + go u.HandleSnapshotRecover(snap, false, req) return nil } @@ -599,263 +311,6 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto return snap.Name, nil } -func (u *SnapshotService) handleDockerDatas(fileOp files.FileOp, operation string, source, target string) error { - switch operation { - case "snapshot": - if err := handleSnapTar(source, target, "docker_data.tar.gz", ""); err != nil { - return fmt.Errorf("backup docker data failed, err: %v", err) - } - case "recover": - if err := handleSnapTar(target, u.OriginalPath, "docker_data.tar.gz", ""); err != nil { - return fmt.Errorf("backup docker data failed, err: %v", err) - } - if err := u.handleUnTar(source+"/docker/docker_data.tar.gz", target); err != nil { - return fmt.Errorf("recover docker data failed, err: %v", err) - } - case "re-recover": - if err := u.handleUnTar(source+"/docker/docker_data.tar.gz", target); err != nil { - return fmt.Errorf("re-recover docker data failed, err: %v", err) - } - case "rollback": - if err := u.handleUnTar(source+"/docker_data.tar.gz", target); err != nil { - return fmt.Errorf("rollback docker data failed, err: %v", err) - } - } - global.LOG.Info("handle docker data dir successful!") - return nil -} - -func (u *SnapshotService) handleDockerDatasWithSave(fileOp files.FileOp, operation, source, target string) error { - switch operation { - case "snapshot": - var wg sync.WaitGroup - wg.Add(1) - var status model.SnapshotStatus - go snapAppData(snapHelper{Wg: &wg, Status: &status}, u.OriginalPath) - wg.Wait() - if status.AppData != constant.StatusDone { - return errors.New(status.AppData) - } - case "recover": - if err := u.handleDockerDatasWithSave(fileOp, "snapshot", "", u.OriginalPath); err != nil { - return fmt.Errorf("backup docker data failed, err: %v", err) - } - if _, err := os.Stat(path.Join(source, "docker/docker_image.tar")); err != nil { - global.LOG.Debug("no such docker images in snapshot") - return nil - } - std, err := cmd.Execf("docker load < %s", path.Join(source, "docker/docker_image.tar")) - if err != nil { - return errors.New(std) - } - case "re-recover": - if _, err := os.Stat(path.Join(source, "docker/docker_image.tar")); err != nil { - global.LOG.Debug("no such docker images in snapshot") - return nil - } - std, err := cmd.Execf("docker load < %s", path.Join(source, "docker/docker_image.tar")) - if err != nil { - return errors.New(std) - } - case "rollback": - if _, err := os.Stat(path.Join(source, "docker/docker_image.tar")); err != nil { - global.LOG.Debug("no such docker images in snapshot") - return nil - } - std, err := cmd.Execf("docker load < %s", path.Join(source, "docker_image.tar")) - if err != nil { - return errors.New(std) - } - } - global.LOG.Info("handle docker data successful!") - return nil -} - -func (u *SnapshotService) handleDaemonJson(fileOp files.FileOp, operation string, source, target string) error { - daemonJsonPath := "/etc/docker/daemon.json" - if operation == "snapshot" || operation == "recover" { - _, err := os.Stat(daemonJsonPath) - if os.IsNotExist(err) { - global.LOG.Info("no daemon.json in snapshot and system now, nothing happened") - } - if err == nil { - if err := fileOp.CopyFile(daemonJsonPath, target); err != nil { - return fmt.Errorf("backup docker daemon.json failed, err: %v", err) - } - } - } - if operation == "recover" || operation == "rollback" || operation == "re-recover" { - _, sourceErr := os.Stat(source) - if os.IsNotExist(sourceErr) { - _ = os.Remove(daemonJsonPath) - } - if sourceErr == nil { - if err := fileOp.CopyFile(source, "/etc/docker"); err != nil { - return fmt.Errorf("recover docker daemon.json failed, err: %v", err) - } - } - } - global.LOG.Info("handle docker daemon.json successful!") - return nil -} - -func (u *SnapshotService) handlePanelBinary(fileOp files.FileOp, operation string, source, target string) error { - panelPath := "/usr/local/bin/1panel" - if operation == "snapshot" || operation == "recover" { - if _, err := os.Stat(panelPath); err != nil { - return fmt.Errorf("1panel binary is not found in %s, err: %v", panelPath, err) - } else { - if err := cpBinary([]string{panelPath}, target); err != nil { - return fmt.Errorf("backup 1panel binary failed, err: %v", err) - } - } - } - if operation == "recover" || operation == "rollback" || operation == "re-recover" { - if _, err := os.Stat(source); err != nil { - return fmt.Errorf("1panel binary is not found in snapshot, err: %v", err) - } else { - if err := cpBinary([]string{source}, "/usr/local/bin/1panel"); err != nil { - return fmt.Errorf("recover 1panel binary failed, err: %v", err) - } - } - } - global.LOG.Info("handle binary panel successful!") - return nil -} - -func (u *SnapshotService) handlePanelctlBinary(fileOp files.FileOp, operation string, source, target string) error { - panelctlPath := "/usr/local/bin/1pctl" - if operation == "snapshot" || operation == "recover" { - if _, err := os.Stat(panelctlPath); err != nil { - return fmt.Errorf("1pctl binary is not found in %s, err: %v", panelctlPath, err) - } else { - if err := cpBinary([]string{panelctlPath}, target); err != nil { - return fmt.Errorf("backup 1pctl binary failed, err: %v", err) - } - } - } - if operation == "recover" || operation == "rollback" || operation == "re-recover" { - if _, err := os.Stat(source); err != nil { - return fmt.Errorf("1pctl binary is not found in snapshot, err: %v", err) - } else { - if err := cpBinary([]string{source}, "/usr/local/bin/1pctl"); err != nil { - return fmt.Errorf("recover 1pctl binary failed, err: %v", err) - } - } - } - global.LOG.Info("handle binary 1pactl successful!") - return nil -} - -func (u *SnapshotService) handlePanelService(fileOp files.FileOp, operation string, source, target string) error { - panelServicePath := "/etc/systemd/system/1panel.service" - if operation == "snapshot" || operation == "recover" { - if _, err := os.Stat(panelServicePath); err != nil { - return fmt.Errorf("1panel service is not found in %s, err: %v", panelServicePath, err) - } else { - if err := cpBinary([]string{panelServicePath}, target); err != nil { - return fmt.Errorf("backup 1panel service failed, err: %v", err) - } - } - } - if operation == "recover" || operation == "rollback" || operation == "re-recover" { - if _, err := os.Stat(source); err != nil { - return fmt.Errorf("1panel service is not found in snapshot, err: %v", err) - } else { - if err := cpBinary([]string{source}, "/etc/systemd/system/1panel.service"); err != nil { - return fmt.Errorf("recover 1panel service failed, err: %v", err) - } - } - } - global.LOG.Info("handle panel service successful!") - return nil -} - -func (u *SnapshotService) handleBackupDatas(fileOp files.FileOp, operation string, source, target string) error { - switch operation { - case "recover": - if err := handleSnapTar(target, u.OriginalPath, "1panel_backup.tar.gz", "./system;"); err != nil { - return fmt.Errorf("restore original local backup dir data failed, err: %v", err) - } - if err := u.handleUnTar(source+"/1panel/1panel_backup.tar.gz", target); err != nil { - return fmt.Errorf("recover local backup dir data failed, err: %v", err) - } - case "re-recover": - if err := u.handleUnTar(source+"/1panel/1panel_backup.tar.gz", target); err != nil { - return fmt.Errorf("retry recover local backup dir data failed, err: %v", err) - } - case "rollback": - if err := u.handleUnTar(source+"/1panel_backup.tar.gz", target); err != nil { - return fmt.Errorf("rollback local backup dir data failed, err: %v", err) - } - } - global.LOG.Info("handle backup data successful!") - return nil -} - -func (u *SnapshotService) handlePanelDatas(snapID uint, fileOp files.FileOp, operation string, source, target, backupDir, dockerDir string) error { - switch operation { - case "snapshot": - exclusionRules := "./tmp;./log;./cache;" - if strings.Contains(backupDir, source) { - exclusionRules += ("." + strings.ReplaceAll(backupDir, source, "") + ";") - } - if strings.Contains(dockerDir, source) { - exclusionRules += ("." + strings.ReplaceAll(dockerDir, source, "") + ";") - } - - if err := handleSnapTar(source, target, "1panel_data.tar.gz", exclusionRules); err != nil { - return fmt.Errorf("backup panel data failed, err: %v", err) - } - case "recover": - exclusionRules := "./tmp;./log;./cache;./db/1Panel.db-*;" - if strings.Contains(backupDir, target) { - exclusionRules += ("." + strings.ReplaceAll(backupDir, target, "") + ";") - } - if strings.Contains(dockerDir, target) { - exclusionRules += ("." + strings.ReplaceAll(dockerDir, target, "") + ";") - } - - _ = snapshotRepo.Update(snapID, map[string]interface{}{"recover_status": ""}) - checkPointOfWal() - if err := handleSnapTar(target, u.OriginalPath, "1panel_data.tar.gz", exclusionRules); err != nil { - return fmt.Errorf("restore original panel data failed, err: %v", err) - } - _ = snapshotRepo.Update(snapID, map[string]interface{}{"recover_status": constant.StatusWaiting}) - - _ = fileOp.Fs.RemoveAll(path.Join(target, "apps")) - checkPointOfWal() - if err := u.handleUnTar(source+"/1panel/1panel_data.tar.gz", target); err != nil { - return fmt.Errorf("recover panel data failed, err: %v", err) - } - case "re-recover": - _ = fileOp.Fs.RemoveAll(path.Join(target, "apps")) - checkPointOfWal() - if err := u.handleUnTar(source+"/1panel/1panel_data.tar.gz", target); err != nil { - return fmt.Errorf("retry recover panel data failed, err: %v", err) - } - case "rollback": - _ = fileOp.Fs.RemoveAll(path.Join(target, "apps")) - checkPointOfWal() - if err := u.handleUnTar(source+"/1panel_data.tar.gz", target); err != nil { - return fmt.Errorf("rollback panel data failed, err: %v", err) - } - } - return nil -} - -func (u *SnapshotService) loadDockerDataDir() (string, bool, error) { - client, err := docker.NewDockerClient() - if err != nil { - return "", false, fmt.Errorf("new docker client failed, err: %v", err) - } - info, err := client.Info(context.Background()) - if err != nil { - return "", false, fmt.Errorf("load docker info failed, err: %v", err) - } - return info.DockerRootDir, info.LiveRestoreEnabled, nil -} - func (u *SnapshotService) Delete(req dto.BatchDeleteReq) error { backups, _ := snapshotRepo.GetList(commonRepo.WithIdsIn(req.Ids)) localDir, err := loadLocalDir() @@ -878,21 +333,22 @@ func (u *SnapshotService) Delete(req dto.BatchDeleteReq) error { return nil } -func updateRecoverStatus(id uint, interruptStep, status string, message string) { - if status != constant.StatusSuccess { - global.LOG.Errorf("recover failed, err: %s", message) +func updateRecoverStatus(id uint, isRecover bool, interruptStep, status, message string) { + if isRecover { + if status != constant.StatusSuccess { + global.LOG.Errorf("recover failed, err: %s", message) + } + if err := snapshotRepo.Update(id, map[string]interface{}{ + "interrupt_step": interruptStep, + "recover_status": status, + "recover_message": message, + "last_recovered_at": time.Now().Format("2006-01-02 15:04:05"), + }); err != nil { + global.LOG.Errorf("update snap recover status failed, err: %v", err) + } + _ = settingRepo.Update("SystemStatus", "Free") + return } - if err := snapshotRepo.Update(id, map[string]interface{}{ - "interrupt_step": interruptStep, - "recover_status": status, - "recover_message": message, - "last_recovered_at": time.Now().Format("2006-01-02 15:04:05"), - }); err != nil { - global.LOG.Errorf("update snap recover status failed, err: %v", err) - } - _ = settingRepo.Update("SystemStatus", "Free") -} -func updateRollbackStatus(id uint, status string, message string) { _ = settingRepo.Update("SystemStatus", "Free") if status == constant.StatusSuccess { if err := snapshotRepo.Update(id, map[string]interface{}{ @@ -926,53 +382,6 @@ func cpBinary(src []string, dst string) error { return nil } -func (u *SnapshotService) updateLiveRestore(enabled bool) error { - if _, err := os.Stat(constant.DaemonJsonPath); err != nil { - return fmt.Errorf("load docker daemon.json conf failed, err: %v", err) - } - file, err := os.ReadFile(constant.DaemonJsonPath) - if err != nil { - return err - } - daemonMap := make(map[string]interface{}) - _ = json.Unmarshal(file, &daemonMap) - - if !enabled { - delete(daemonMap, "live-restore") - } else { - daemonMap["live-restore"] = enabled - } - newJson, err := json.MarshalIndent(daemonMap, "", "\t") - if err != nil { - return err - } - if err := os.WriteFile(constant.DaemonJsonPath, newJson, 0640); err != nil { - return err - } - - stdout, err := cmd.Exec("systemctl restart docker") - if err != nil { - return errors.New(stdout) - } - - ticker := time.NewTicker(3 * time.Second) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - for range ticker.C { - select { - case <-ctx.Done(): - return errors.New("the docker service cannot be restarted") - default: - stdout, err := cmd.Exec("systemctl is-active docker") - if string(stdout) == "active\n" && err == nil { - global.LOG.Info("docker restart with new live-restore successful!") - return nil - } - } - } - return nil -} - func (u *SnapshotService) handleUnTar(sourceDir, targetDir string) error { if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { @@ -1025,64 +434,32 @@ func rebuildAllAppInstall() error { return nil } -func isNewSnapVersion(version string) bool { - versionItem := "v1.5.0" - if version == versionItem { - return true - } - version1s := strings.Split(version, ".") - version2s := strings.Split(versionItem, ".") - - n := min(len(version1s), len(version2s)) - re := regexp.MustCompile("[0-9]+") - for i := 0; i < n; i++ { - sVersion1s := re.FindAllString(version1s[i], -1) - sVersion2s := re.FindAllString(version2s[i], -1) - if len(sVersion1s) == 0 { - return false - } - if len(sVersion2s) == 0 { - return false - } - v1num, _ := strconv.Atoi(sVersion1s[0]) - v2num, _ := strconv.Atoi(sVersion2s[0]) - if v1num == v2num { - continue - } else { - return v1num > v2num - } - } - return true -} - -func min(a, b int) int { - if a < b { - return a - } - return b -} - func checkIsAllDone(snapID uint) bool { status, err := snapshotRepo.GetStatus(snapID) if err != nil { return false } + isOK, _ := checkAllDone(status) + return isOK +} + +func checkAllDone(status model.SnapshotStatus) (bool, string) { if status.Panel != constant.StatusDone { - return false + return false, status.Panel } if status.PanelInfo != constant.StatusDone { - return false + return false, status.PanelInfo } if status.DaemonJson != constant.StatusDone { - return false + return false, status.DaemonJson } if status.AppData != constant.StatusDone { - return false + return false, status.AppData } if status.BackupData != constant.StatusDone { - return false + return false, status.BackupData } - return true + return true, "" } func loadLogByStatus(status model.SnapshotStatus, logPath string) { @@ -1099,7 +476,6 @@ func loadLogByStatus(status model.SnapshotStatus, logPath string) { file, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { - global.LOG.Errorf("write snapshot logs failed, err: %v", err) return } defer file.Close() diff --git a/backend/app/service/snapshot_recover.go b/backend/app/service/snapshot_recover.go new file mode 100644 index 000000000..f38a6be33 --- /dev/null +++ b/backend/app/service/snapshot_recover.go @@ -0,0 +1,263 @@ +package service + +import ( + "context" + "fmt" + "os" + "path" + "strings" + "sync" + + "github.com/1Panel-dev/1Panel/backend/app/dto" + "github.com/1Panel-dev/1Panel/backend/app/model" + "github.com/1Panel-dev/1Panel/backend/constant" + "github.com/1Panel-dev/1Panel/backend/global" + "github.com/1Panel-dev/1Panel/backend/utils/cmd" + "github.com/1Panel-dev/1Panel/backend/utils/files" + "github.com/pkg/errors" +) + +func (u *SnapshotService) HandleSnapshotRecover(snap model.Snapshot, isRecover bool, req dto.SnapshotRecover) { + _ = global.Cron.Stop() + defer func() { + global.Cron.Start() + }() + + snapFileDir := "" + if isRecover { + baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name)) + if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { + _ = os.MkdirAll(baseDir, os.ModePerm) + } + if req.IsNew || snap.InterruptStep == "Backup" { + if err := backupBeforeRecover(snap); err != nil { + updateRecoverStatus(snap.ID, isRecover, "Backup", constant.StatusFailed, fmt.Sprintf("handle backup before recover failed, err: %v", err)) + return + } + global.LOG.Debug("handle backup before recover successful!") + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "Download" || (!req.IsNew || req.ReDownload) { + if err := handleDownloadSnapshot(snap, baseDir); err != nil { + updateRecoverStatus(snap.ID, isRecover, "Backup", constant.StatusFailed, err.Error()) + return + } + global.LOG.Debugf("download snapshot file to %s successful!", baseDir) + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "Decompress" { + if err := handleUnTar(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir); err != nil { + updateRecoverStatus(snap.ID, isRecover, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) + return + } + global.LOG.Debug("decompress snapshot file successful!", baseDir) + req.IsNew = true + } + snapFileDir = fmt.Sprintf("%s/%s", baseDir, snap.Name) + } else { + snapFileDir = fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) + if _, err := os.Stat(snapFileDir); err != nil { + updateRecoverStatus(snap.ID, isRecover, "", constant.StatusFailed, fmt.Sprintf("cannot find the backup file %s, please try to recover again.", snapFileDir)) + return + } + } + snapJson, err := u.readFromJson(fmt.Sprintf("%s/snapshot.json", snapFileDir)) + if err != nil { + updateRecoverStatus(snap.ID, isRecover, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err)) + return + } + if snap.InterruptStep == "Readjson" { + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "AppData" { + if err := recoverAppData(snapFileDir); err != nil { + updateRecoverStatus(snap.ID, isRecover, "DockerDir", constant.StatusFailed, fmt.Sprintf("handle recover app data failed, err: %v", err)) + return + } + global.LOG.Debug("recover app data from snapshot file successful!") + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "DaemonJson" { + fileOp := files.NewFileOp() + if err := recoverDaemonJson(snapFileDir, fileOp); err != nil { + updateRecoverStatus(snap.ID, isRecover, "DaemonJson", constant.StatusFailed, err.Error()) + return + } + global.LOG.Debug("recover daemon.json from snapshot file successful!") + req.IsNew = true + } + + if req.IsNew || snap.InterruptStep == "1PanelBinary" { + if err := recoverPanel(path.Join(snapFileDir, "1panel/1panel"), "/usr/local/bin/1panel"); err != nil { + updateRecoverStatus(snap.ID, isRecover, "1PanelBinary", constant.StatusFailed, err.Error()) + return + } + global.LOG.Debug("recover 1panel binary from snapshot file successful!") + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "1PctlBinary" { + if err := recoverPanel(path.Join(snapFileDir, "1panel/1pctl"), "/usr/local/bin/1pctl"); err != nil { + updateRecoverStatus(snap.ID, isRecover, "1PctlBinary", constant.StatusFailed, err.Error()) + return + } + global.LOG.Debug("recover 1pctl from snapshot file successful!") + req.IsNew = true + } + if req.IsNew || snap.InterruptStep == "1PanelService" { + if err := recoverPanel(path.Join(snapFileDir, "1panel/1panel.service"), "/etc/systemd/system/1panel.service"); err != nil { + updateRecoverStatus(snap.ID, isRecover, "1PanelService", constant.StatusFailed, err.Error()) + return + } + global.LOG.Debug("recover 1panel service from snapshot file successful!") + req.IsNew = true + } + + if req.IsNew || snap.InterruptStep == "1PanelBackups" { + if err := u.handleUnTar(path.Join(snapFileDir, "/1panel/1panel_backup.tar.gz"), snapJson.BackupDataDir); err != nil { + updateRecoverStatus(snap.ID, isRecover, "1PanelBackups", constant.StatusFailed, err.Error()) + return + } + global.LOG.Debug("recover 1panel backups from snapshot file successful!") + req.IsNew = true + } + + if req.IsNew || snap.InterruptStep == "1PanelData" { + checkPointOfWal() + if err := u.handleUnTar(path.Join(snapFileDir, "/1panel/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel")); err != nil { + updateRecoverStatus(snap.ID, isRecover, "1PanelData", constant.StatusFailed, err.Error()) + return + } + global.LOG.Debug("recover 1panel data from snapshot file successful!") + req.IsNew = true + } + _ = rebuildAllAppInstall() + restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose")) + + global.LOG.Info("recover successful") + if !isRecover { + oriPath := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) + global.LOG.Debugf("remove the file %s after the operation is successful", oriPath) + _ = os.RemoveAll(oriPath) + } else { + global.LOG.Debugf("remove the file %s after the operation is successful", path.Dir(snapFileDir)) + _ = os.RemoveAll(path.Dir(snapFileDir)) + } + _, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service") +} + +func backupBeforeRecover(snap model.Snapshot) error { + baseDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, snap.Name) + var wg sync.WaitGroup + var status model.SnapshotStatus + itemHelper := snapHelper{SnapID: 0, Status: &status, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} + + jsonItem := SnapshotJson{ + BaseDir: global.CONF.System.BaseDir, + BackupDataDir: global.CONF.System.Backup, + PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), + } + _ = os.MkdirAll(path.Join(baseDir, "1panel"), os.ModePerm) + _ = os.MkdirAll(path.Join(baseDir, "docker"), os.ModePerm) + + wg.Add(5) + itemHelper.Wg = &wg + go snapJson(itemHelper, jsonItem, baseDir) + go snapPanel(itemHelper, path.Join(baseDir, "1panel")) + go snapDaemonJson(itemHelper, path.Join(baseDir, "docker")) + go snapAppData(itemHelper, path.Join(baseDir, "docker")) + go snapBackup(itemHelper, global.CONF.System.Backup, path.Join(baseDir, "1panel")) + wg.Wait() + + allDone, msg := checkAllDone(status) + if !allDone { + return errors.New(msg) + } + snapPanelData(itemHelper, global.CONF.System.BaseDir, path.Join(baseDir, "1panel")) + if status.PanelData != constant.StatusDone { + return errors.New(status.PanelData) + } + return nil +} + +func handleDownloadSnapshot(snap model.Snapshot, targetDir string) error { + backup, err := backupRepo.Get(commonRepo.WithByType(snap.DefaultDownload)) + if err != nil { + return err + } + client, err := NewIBackupService().NewClient(&backup) + if err != nil { + return err + } + pathItem := backup.BackupPath + if backup.BackupPath != "/" { + pathItem = strings.TrimPrefix(backup.BackupPath, "/") + } + filePath := fmt.Sprintf("%s/%s.tar.gz", targetDir, snap.Name) + _ = os.RemoveAll(filePath) + ok, err := client.Download(path.Join(pathItem, fmt.Sprintf("system_snapshot/%s.tar.gz", snap.Name)), filePath) + if err != nil || !ok { + return fmt.Errorf("download file %s from %s failed, err: %v", snap.Name, backup.Type, err) + } + return nil +} + +func recoverAppData(src string) error { + if _, err := os.Stat(path.Join(src, "docker/docker_image.tar")); err != nil { + global.LOG.Debug("no such docker images in snapshot") + return nil + } + std, err := cmd.Execf("docker load < %s", path.Join(src, "docker/docker_image.tar")) + if err != nil { + return errors.New(std) + } + return err +} + +func recoverDaemonJson(src string, fileOp files.FileOp) error { + daemonJsonPath := "/etc/docker/daemon.json" + _, errSrc := os.Stat(src) + _, errPath := os.Stat(daemonJsonPath) + if os.IsNotExist(errSrc) && os.IsNotExist(errPath) { + global.LOG.Debug("the daemon.json file does not exist, nothing happens.") + return nil + } + if errSrc == nil { + if err := fileOp.CopyFile(path.Join(src, "docker/daemon.json"), "/etc/docker"); err != nil { + return fmt.Errorf("recover docker daemon.json failed, err: %v", err) + } + } + + _, _ = cmd.Exec("systemctl restart docker") + return nil +} + +func recoverPanel(src string, dst string) error { + if _, err := os.Stat(src); err != nil { + return fmt.Errorf("file is not found in %s, err: %v", src, err) + } + global.LOG.Debugf(fmt.Sprintf("\\cp -f %s %s", src, dst)) + stdout, err := cmd.Exec(fmt.Sprintf("\\cp -f %s %s", src, dst)) + if err != nil { + return fmt.Errorf("cp file failed, stdout: %v, err: %v", stdout, err) + } + return nil +} + +func restartCompose(composePath string) { + composes, err := composeRepo.ListRecord() + if err != nil { + return + } + for _, compose := range composes { + pathItem := path.Join(composePath, compose.Name, "docker-compose.yml") + if _, err := os.Stat(pathItem); err != nil { + continue + } + upCmd := fmt.Sprintf("docker-compose -f %s up -d", pathItem) + stdout, err := cmd.Exec(upCmd) + if err != nil { + global.LOG.Debugf("%s failed, err: %v", upCmd, stdout) + } + } + global.LOG.Debug("restart all compose successful!") +} diff --git a/frontend/src/views/database/mysql/remote/delete/index.vue b/frontend/src/views/database/mysql/remote/delete/index.vue index 10904f6a9..04f4b4a63 100644 --- a/frontend/src/views/database/mysql/remote/delete/index.vue +++ b/frontend/src/views/database/mysql/remote/delete/index.vue @@ -1,19 +1,19 @@