1Panel/agent/app/service/backup.go

package service

import (
	"bufio"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/1Panel-dev/1Panel/agent/app/repo"

	"github.com/1Panel-dev/1Panel/agent/app/dto"
	"github.com/1Panel-dev/1Panel/agent/app/model"
	"github.com/1Panel-dev/1Panel/agent/buserr"
	"github.com/1Panel-dev/1Panel/agent/constant"
	"github.com/1Panel-dev/1Panel/agent/global"
	"github.com/1Panel-dev/1Panel/agent/utils/cloud_storage"
	"github.com/1Panel-dev/1Panel/agent/utils/cloud_storage/client"
	"github.com/1Panel-dev/1Panel/agent/utils/encrypt"
	"github.com/1Panel-dev/1Panel/agent/utils/files"
	"github.com/jinzhu/copier"
	"github.com/pkg/errors"
)

type BackupService struct{}

type IBackupService interface {
	CheckUsed(id uint) error
	Sync(req dto.SyncFromMaster) error

	LoadBackupOptions() ([]dto.BackupOption, error)
	SearchWithPage(search dto.SearchPageWithType) (int64, interface{}, error)
	Create(backupDto dto.BackupOperate) error
	GetBuckets(backupDto dto.ForBuckets) ([]interface{}, error)
	Update(req dto.BackupOperate) error
	Delete(id uint) error
	RefreshToken(req dto.OperateByID) error
	GetLocalDir() (string, error)

	MysqlBackup(db dto.CommonBackup) error
	PostgresqlBackup(db dto.CommonBackup) error
	MysqlRecover(db dto.CommonRecover) error
	PostgresqlRecover(db dto.CommonRecover) error
	MysqlRecoverByUpload(req dto.CommonRecover) error
	PostgresqlRecoverByUpload(req dto.CommonRecover) error

	RedisBackup(db dto.CommonBackup) error
	RedisRecover(db dto.CommonRecover) error

	WebsiteBackup(db dto.CommonBackup) error
	WebsiteRecover(req dto.CommonRecover) error

	AppBackup(db dto.CommonBackup) (*model.BackupRecord, error)
	AppRecover(req dto.CommonRecover) error
}

func NewIBackupService() IBackupService {
	return &BackupService{}
}

func (u *BackupService) GetLocalDir() (string, error) {
	account, err := backupRepo.Get(repo.WithByType(constant.Local))
	if err != nil {
		return "", err
	}
	return account.BackupPath, nil
}

func (u *BackupService) SearchWithPage(req dto.SearchPageWithType) (int64, interface{}, error) {
	options := []repo.DBOption{repo.WithOrderBy("created_at desc")}
	if len(req.Type) != 0 {
		options = append(options, repo.WithByType(req.Type))
	}
	if len(req.Info) != 0 {
		options = append(options, repo.WithByType(req.Info))
	}
	count, accounts, err := backupRepo.Page(req.Page, req.PageSize, options...)
	if err != nil {
		return 0, nil, err
	}
	var data []dto.BackupInfo
	for _, account := range accounts {
		var item dto.BackupInfo
		if err := copier.Copy(&item, &account); err != nil {
			global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err)
		}
		if item.Type != constant.Sftp && item.Type != constant.Local {
			item.BackupPath = path.Join("/", strings.TrimPrefix(item.BackupPath, "/"))
		}
		if !item.RememberAuth {
			item.AccessKey = ""
			item.Credential = ""
			if account.Type == constant.Sftp {
				varMap := make(map[string]interface{})
				if err := json.Unmarshal([]byte(item.Vars), &varMap); err != nil {
					continue
				}
				delete(varMap, "passPhrase")
				itemVars, _ := json.Marshal(varMap)
				item.Vars = string(itemVars)
			}
		} else {
			item.AccessKey = base64.StdEncoding.EncodeToString([]byte(item.AccessKey))
			item.Credential = base64.StdEncoding.EncodeToString([]byte(item.Credential))
		}

		if account.Type == constant.OneDrive || account.Type == constant.ALIYUN || account.Type == constant.GoogleDrive {
			varMap := make(map[string]interface{})
			if err := json.Unmarshal([]byte(item.Vars), &varMap); err != nil {
				continue
			}
			delete(varMap, "refresh_token")
			delete(varMap, "drive_id")
			itemVars, _ := json.Marshal(varMap)
			item.Vars = string(itemVars)
		}
		data = append(data, item)
	}
	return count, data, nil
}

func (u *BackupService) Create(req dto.BackupOperate) error {
	if req.Type == constant.Local {
		return buserr.New(constant.ErrBackupLocalCreate)
	}
	if req.Type != constant.Sftp && req.BackupPath != "/" {
		req.BackupPath = strings.TrimPrefix(req.BackupPath, "/")
	}
	backup, _ := backupRepo.Get(repo.WithByName(req.Name))
	if backup.ID != 0 {
		return constant.ErrRecordExist
	}
	if err := copier.Copy(&backup, &req); err != nil {
		return errors.WithMessage(constant.ErrStructTransform, err.Error())
	}
	itemAccessKey, err := base64.StdEncoding.DecodeString(backup.AccessKey)
	if err != nil {
		return err
	}
	backup.AccessKey = string(itemAccessKey)
	itemCredential, err := base64.StdEncoding.DecodeString(backup.Credential)
	if err != nil {
		return err
	}
	backup.Credential = string(itemCredential)

	if req.Type == constant.OneDrive || req.Type == constant.GoogleDrive {
		if err := loadRefreshTokenByCode(&backup); err != nil {
			return err
		}
	}
	if req.Type != "LOCAL" {
		isOk, err := u.checkBackupConn(&backup)
		if err != nil || !isOk {
			return buserr.WithMap(constant.ErrBackupCheck, map[string]interface{}{"err": err.Error()}, err)
		}
	}

	backup.AccessKey, err = encrypt.StringEncrypt(backup.AccessKey)
	if err != nil {
		return err
	}
	backup.Credential, err = encrypt.StringEncrypt(backup.Credential)
	if err != nil {
		return err
	}
	if err := backupRepo.Create(&backup); err != nil {
		return err
	}
	return nil
}

func (u *BackupService) GetBuckets(req dto.ForBuckets) ([]interface{}, error) {
	itemAccessKey, err := base64.StdEncoding.DecodeString(req.AccessKey)
	if err != nil {
		return nil, err
	}
	req.AccessKey = string(itemAccessKey)
	itemCredential, err := base64.StdEncoding.DecodeString(req.Credential)
	if err != nil {
		return nil, err
	}
	req.Credential = string(itemCredential)

	varMap := make(map[string]interface{})
	if err := json.Unmarshal([]byte(req.Vars), &varMap); err != nil {
		return nil, err
	}
	switch req.Type {
	case constant.Sftp, constant.WebDAV:
		varMap["username"] = req.AccessKey
		varMap["password"] = req.Credential
	case constant.OSS, constant.S3, constant.MinIo, constant.Cos, constant.Kodo:
		varMap["accessKey"] = req.AccessKey
		varMap["secretKey"] = req.Credential
	}
	client, err := cloud_storage.NewCloudStorageClient(req.Type, varMap)
	if err != nil {
		return nil, err
	}
	return client.ListBuckets()
}

func (u *BackupService) Delete(id uint) error {
	backup, _ := backupRepo.Get(repo.WithByID(id))
	if backup.ID == 0 {
		return constant.ErrRecordNotFound
	}
	if backup.Type == constant.Local {
		return buserr.New(constant.ErrBackupLocalDelete)
	}
	return backupRepo.Delete(repo.WithByID(id))
}

func (u *BackupService) Update(req dto.BackupOperate) error {
	backup, _ := backupRepo.Get(repo.WithByID(req.ID))
	if backup.ID == 0 {
		return constant.ErrRecordNotFound
	}
	if req.Type != constant.Sftp && req.Type != constant.Local && req.BackupPath != "/" {
		req.BackupPath = strings.TrimPrefix(req.BackupPath, "/")
	}
	var newBackup model.BackupAccount
	if err := copier.Copy(&newBackup, &req); err != nil {
		return errors.WithMessage(constant.ErrStructTransform, err.Error())
	}
	itemAccessKey, err := base64.StdEncoding.DecodeString(newBackup.AccessKey)
	if err != nil {
		return err
	}
	newBackup.AccessKey = string(itemAccessKey)
	itemCredential, err := base64.StdEncoding.DecodeString(newBackup.Credential)
	if err != nil {
		return err
	}
	newBackup.Credential = string(itemCredential)
	if backup.Type == constant.Local {
		if newBackup.Vars != backup.Vars {
			oldPath := backup.BackupPath
			newPath := newBackup.BackupPath
			if strings.HasSuffix(newPath, "/") && newPath != "/" {
				newPath = newPath[:strings.LastIndex(newPath, "/")]
			}
			if err := files.NewFileOp().CopyDir(oldPath, newPath); err != nil {
				return err
			}
		}
	}

	if newBackup.Type == constant.OneDrive || newBackup.Type == constant.GoogleDrive {
		if err := loadRefreshTokenByCode(&backup); err != nil {
			return err
		}
	}
	if backup.Type != "LOCAL" {
		isOk, err := u.checkBackupConn(&newBackup)
		if err != nil || !isOk {
			return buserr.WithMap("ErrBackupCheck", map[string]interface{}{"err": err.Error()}, err)
		}
	}

	newBackup.AccessKey, err = encrypt.StringEncrypt(newBackup.AccessKey)
	if err != nil {
		return err
	}
	newBackup.Credential, err = encrypt.StringEncrypt(newBackup.Credential)
	if err != nil {
		return err
	}
	newBackup.ID = backup.ID
	if err := backupRepo.Save(&newBackup); err != nil {
		return err
	}
	return nil
}

func (u *BackupService) RefreshToken(req dto.OperateByID) error {
	backup, _ := backupRepo.Get(repo.WithByID(req.ID))
	if backup.ID == 0 {
		return constant.ErrRecordNotFound
	}
	varMap := make(map[string]interface{})
	if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
		return fmt.Errorf("Failed to refresh %s - %s token, please retry, err: %v", backup.Type, backup.Name, err)
	}
	var (
		refreshToken string
		err          error
	)
	switch backup.Type {
	case constant.OneDrive:
		refreshToken, err = client.RefreshToken("refresh_token", "refreshToken", varMap)
	case constant.GoogleDrive:
		refreshToken, err = client.RefreshGoogleToken("refresh_token", "refreshToken", varMap)
	case constant.ALIYUN:
		refreshToken, err = client.RefreshALIToken(varMap)
	}
	if err != nil {
		varMap["refresh_status"] = constant.StatusFailed
		varMap["refresh_msg"] = err.Error()
		return fmt.Errorf("Failed to refresh %s-%s token, please retry, err: %v", backup.Type, backup.Name, err)
	}
	varMap["refresh_status"] = constant.StatusSuccess
	varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
	varMap["refresh_token"] = refreshToken

	varsItem, _ := json.Marshal(varMap)
	backup.Vars = string(varsItem)
	return backupRepo.Save(&backup)
}

func (u *BackupService) checkBackupConn(backup *model.BackupAccount) (bool, error) {
	client, err := newClient(backup)
	if err != nil {
		return false, err
	}
	fileItem := path.Join(global.CONF.System.BaseDir, "1panel/tmp/test/1panel")
	if _, err := os.Stat(path.Dir(fileItem)); err != nil && os.IsNotExist(err) {
		if err = os.MkdirAll(path.Dir(fileItem), os.ModePerm); err != nil {
			return false, err
		}
	}
	file, err := os.OpenFile(fileItem, os.O_WRONLY|os.O_CREATE, constant.FilePerm)
	if err != nil {
		return false, err
	}
	defer file.Close()
	write := bufio.NewWriter(file)
	_, _ = write.WriteString("1Panel 备份账号测试文件。\n")
	_, _ = write.WriteString("1Panel 備份賬號測試文件。\n")
	_, _ = write.WriteString("1Panel Backs up account test files.\n")
	_, _ = write.WriteString("1Panelアカウントのテストファイルをバックアップします。\n")
	write.Flush()

	targetPath := path.Join(backup.BackupPath, "test/1panel")
	if backup.Type != constant.Sftp && backup.Type != constant.Local && targetPath != "/" {
		targetPath = strings.TrimPrefix(targetPath, "/")
	}
	return client.Upload(fileItem, targetPath)
}

func (u *BackupService) Sync(req dto.SyncFromMaster) error {
	var accountItem model.BackupAccount
	if err := json.Unmarshal([]byte(req.Data), &accountItem); err != nil {
		return err
	}
	accountItem.AccessKey, _ = encrypt.StringEncryptWithBase64(accountItem.AccessKey)
	accountItem.Credential, _ = encrypt.StringEncryptWithBase64(accountItem.Credential)
	account, _ := backupRepo.Get(repo.WithByName(req.Name))
	switch req.Operation {
	case "create":
		if account.ID != 0 {
			accountItem.ID = account.ID
			return backupRepo.Save(&accountItem)
		}
		return backupRepo.Create(&accountItem)
	case "delete":
		if account.ID == 0 {
			return constant.ErrRecordNotFound
		}
		return backupRepo.Delete(repo.WithByID(account.ID))
	case "update":
		if account.ID == 0 {
			return constant.ErrRecordNotFound
		}
		accountItem.ID = account.ID
		return backupRepo.Save(&accountItem)
	default:
		return fmt.Errorf("not support such operation %s", req.Operation)
	}
}

func (u *BackupService) LoadBackupOptions() ([]dto.BackupOption, error) {
	accounts, err := backupRepo.List(repo.WithOrderBy("created_at desc"))
	if err != nil {
		return nil, err
	}
	var data []dto.BackupOption
	for _, account := range accounts {
		var item dto.BackupOption
		if err := copier.Copy(&item, &account); err != nil {
			global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err)
		}
		data = append(data, item)
	}
	return data, nil
}

func (u *BackupService) CheckUsed(id uint) error {
	cronjobs, _ := cronjobRepo.List()
	for _, job := range cronjobs {
		if job.DownloadAccountID == id {
			return buserr.New(constant.ErrBackupInUsed)
		}
		ids := strings.Split(job.SourceAccountIDs, ",")
		for _, idItem := range ids {
			if idItem == fmt.Sprintf("%v", id) {
				return buserr.New(constant.ErrBackupInUsed)
			}
		}
	}
	return nil
}

func NewBackupClientWithID(id uint) (*model.BackupAccount, cloud_storage.CloudStorageClient, error) {
	account, _ := backupRepo.Get(repo.WithByID(id))
	backClient, err := newClient(&account)
	if err != nil {
		return nil, nil, err
	}
	return &account, backClient, nil
}

type backupClientHelper struct {
	id          uint
	accountType string
	name        string
	backupPath  string
	client      cloud_storage.CloudStorageClient
}

func NewBackupClientMap(ids []string) (map[string]backupClientHelper, error) {
	var accounts []model.BackupAccount
	var idItems []uint
	for i := 0; i < len(ids); i++ {
		item, _ := strconv.Atoi(ids[i])
		idItems = append(idItems, uint(item))
	}
	accounts, _ = backupRepo.List(repo.WithByIDs(idItems))
	clientMap := make(map[string]backupClientHelper)
	for _, item := range accounts {
		backClient, err := newClient(&item)
		if err != nil {
			return nil, err
		}
		clientMap[fmt.Sprintf("%v", item.ID)] = backupClientHelper{
			client:      backClient,
			name:        item.Name,
			backupPath:  item.BackupPath,
			accountType: item.Type,
			id:          item.ID,
		}
	}
	return clientMap, nil
}

func newClient(account *model.BackupAccount) (cloud_storage.CloudStorageClient, error) {
	varMap := make(map[string]interface{})
	if len(account.Vars) != 0 {
		if err := json.Unmarshal([]byte(account.Vars), &varMap); err != nil {
			return nil, err
		}
	}
	varMap["bucket"] = account.Bucket
	varMap["backupPath"] = account.BackupPath
	account.AccessKey, _ = encrypt.StringDecrypt(account.AccessKey)
	account.Credential, _ = encrypt.StringDecrypt(account.Credential)
	switch account.Type {
	case constant.Sftp, constant.WebDAV:
		varMap["username"] = account.AccessKey
		varMap["password"] = account.Credential
	case constant.OSS, constant.S3, constant.MinIo, constant.Cos, constant.Kodo:
		varMap["accessKey"] = account.AccessKey
		varMap["secretKey"] = account.Credential
	case constant.UPYUN:
		varMap["operator"] = account.AccessKey
		varMap["password"] = account.Credential
	}

	client, err := cloud_storage.NewCloudStorageClient(account.Type, varMap)
	if err != nil {
		return nil, err
	}
	return client, nil
}

func loadRefreshTokenByCode(backup *model.BackupAccount) error {
	varMap := make(map[string]interface{})
	if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
		return fmt.Errorf("unmarshal backup vars failed, err: %v", err)
	}
	refreshToken := ""
	var err error
	if backup.Type == constant.GoogleDrive {
		refreshToken, err = client.RefreshGoogleToken("authorization_code", "refreshToken", varMap)
		if err != nil {
			return err
		}
	} else {
		refreshToken, err = client.RefreshToken("authorization_code", "refreshToken", varMap)
		if err != nil {
			return err
		}
	}
	delete(varMap, "code")
	varMap["refresh_status"] = constant.StatusSuccess
	varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
	varMap["refresh_token"] = refreshToken
	itemVars, err := json.Marshal(varMap)
	if err != nil {
		return fmt.Errorf("json marshal var map failed, err: %v", err)
	}
	backup.Vars = string(itemVars)
	return nil
}

func loadBackupNamesByID(accountIDs string, downloadID uint) ([]string, string, error) {
	accountIDList := strings.Split(accountIDs, ",")
	var ids []uint
	for _, item := range accountIDList {
		if len(item) != 0 {
			itemID, _ := strconv.Atoi(item)
			ids = append(ids, uint(itemID))
		}
	}
	list, err := backupRepo.List(repo.WithByIDs(ids))
	if err != nil {
		return nil, "", err
	}
	var accounts []string
	var downloadAccount string
	for _, item := range list {
		itemName := fmt.Sprintf("%s - %s", item.Type, item.Name)
		accounts = append(accounts, itemName)
		if item.ID == downloadID {
			downloadAccount = itemName
		}
	}
	return accounts, downloadAccount, nil
}