pref: 优化 MySQL 备份恢复方式 (#4059)

Refs #3955

backend/app/service/backup_app.go

@@ -115,7 +115,7 @@ func handleAppBackup(install *model.AppInstall, backupDir, fileName string) erro
 			if err != nil {
 				return err
 			}
-			if err := handleMysqlBackup(db.MysqlName, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", install.Name)); err != nil {
+			if err := handleMysqlBackup(db.MysqlName, resource.Key, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", install.Name)); err != nil {
 				return err
 			}
 		case constant.AppPostgresql:

backend/app/service/backup_mysql.go

@@ -29,7 +29,7 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error {
 	targetDir := path.Join(localDir, itemDir)
 	fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5))
 
-	if err := handleMysqlBackup(req.Name, req.DetailName, targetDir, fileName); err != nil {
+	if err := handleMysqlBackup(req.Name, req.Type, req.DetailName, targetDir, fileName); err != nil {
 		return err
 	}
 
@@ -100,18 +100,20 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error {
 	return nil
 }
 
-func handleMysqlBackup(database, dbName, targetDir, fileName string) error {
+func handleMysqlBackup(database, dbType, dbName, targetDir, fileName string) error {
 	dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(dbName), mysqlRepo.WithByMysqlName(database))
 	if err != nil {
 		return err
 	}
-	cli, _, err := LoadMysqlClientByFrom(database)
+	cli, version, err := LoadMysqlClientByFrom(database)
 	if err != nil {
 		return err
 	}
 
 	backupInfo := client.BackupInfo{
 		Name:      dbName,
+		Type:      dbType,
+		Version:   version,
 		Format:    dbInfo.Format,
 		TargetDir: targetDir,
 		FileName:  fileName,
@@ -134,7 +136,7 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
 	if err != nil {
 		return err
 	}
-	cli, _, err := LoadMysqlClientByFrom(req.Name)
+	cli, version, err := LoadMysqlClientByFrom(req.Name)
 	if err != nil {
 		return err
 	}
@@ -143,6 +145,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
 		rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format("20060102150405")))
 		if err := cli.Backup(client.BackupInfo{
 			Name:      req.DetailName,
+			Type:      req.Type,
+			Version:   version,
 			Format:    dbInfo.Format,
 			TargetDir: path.Dir(rollbackFile),
 			FileName:  path.Base(rollbackFile),
@@ -156,6 +160,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
 				global.LOG.Info("recover failed, start to rollback now")
 				if err := cli.Recover(client.RecoverInfo{
 					Name:       req.DetailName,
+					Type:       req.Type,
+					Version:    version,
 					Format:     dbInfo.Format,
 					SourceFile: rollbackFile,
 
@@ -172,6 +178,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
 	}
 	if err := cli.Recover(client.RecoverInfo{
 		Name:       req.DetailName,
+		Type:       req.Type,
+		Version:    version,
 		Format:     dbInfo.Format,
 		SourceFile: req.File,
 

backend/app/service/cronjob_backup.go

@@ -109,7 +109,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
 		backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name))
 		record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format("20060102150405")+common.RandStrAndNum(5))
 		if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
-			if err := handleMysqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil {
+			if err := handleMysqlBackup(dbInfo.Database, dbInfo.DBType, dbInfo.Name, backupDir, record.FileName); err != nil {
 				return err
 			}
 		} else {

backend/utils/mysql/client/info.go

@@ -70,6 +70,8 @@ type AccessChangeInfo struct {
 
 type BackupInfo struct {
 	Name      string `json:"name"`
+	Type      string `json:"type"`
+	Version   string `json:"version"`
 	Format    string `json:"format"`
 	TargetDir string `json:"targetDir"`
 	FileName  string `json:"fileName"`
@@ -79,6 +81,8 @@ type BackupInfo struct {
 
 type RecoverInfo struct {
 	Name       string `json:"name"`
+	Type       string `json:"type"`
+	Version    string `json:"version"`
 	Format     string `json:"format"`
 	SourceFile string `json:"sourceFile"`
 

backend/utils/mysql/client/remote.go

@@ -1,11 +1,14 @@
 package client
 
 import (
+	"compress/gzip"
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
 	"strings"
 	"time"
 
@@ -13,7 +16,8 @@ import (
 	"github.com/1Panel-dev/1Panel/backend/constant"
 	"github.com/1Panel-dev/1Panel/backend/global"
 	"github.com/1Panel-dev/1Panel/backend/utils/files"
-	"github.com/1Panel-dev/1Panel/backend/utils/mysql/helper"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
 )
 
 type Remote struct {
@@ -229,56 +233,63 @@ func (r *Remote) Backup(info BackupInfo) error {
 			return fmt.Errorf("mkdir %s failed, err: %v", info.TargetDir, err)
 		}
 	}
-	fileNameItem := info.TargetDir + "/" + strings.TrimSuffix(info.FileName, ".gz")
+	outfile, _ := os.OpenFile(path.Join(info.TargetDir, info.FileName), os.O_RDWR|os.O_CREATE, 0755)
-
+	global.LOG.Infof("start to mysqldump | gzip > %s.gzip", info.TargetDir+"/"+info.FileName)
-	tlsItem, err := ConnWithSSL(r.SSL, r.SkipVerify, r.ClientKey, r.ClientCert, r.RootCert)
+	image, err := loadImage(info.Type, info.Version)
 	if err != nil {
 		return err
 	}
-	dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai%s", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F", tlsItem)
+	backupCmd := fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'mysqldump -h %s -P %d -u%s -p%s %s --default-character-set=%s %s'",
+		image, r.Address, r.Port, r.User, r.Password, sslSkip(info.Version), info.Format, info.Name)
 
-	f, _ := os.OpenFile(fileNameItem, os.O_RDWR|os.O_CREATE, 0755)
+	global.LOG.Debug(backupCmd)
-	defer f.Close()
+	cmd := exec.Command("bash", "-c", backupCmd)
-	if err := helper.Dump(dns, helper.WithData(), helper.WithDropTable(), helper.WithWriter(f)); err != nil {
-		return err
-	}
 
-	gzipCmd := exec.Command("gzip", fileNameItem)
+	gzipCmd := exec.Command("gzip", "-cf")
-	stdout, err := gzipCmd.CombinedOutput()
+	gzipCmd.Stdin, _ = cmd.StdoutPipe()
-	if err != nil {
+	gzipCmd.Stdout = outfile
-		return fmt.Errorf("gzip file %s failed, stdout: %v, err: %v", strings.TrimSuffix(info.FileName, ".gz"), string(stdout), err)
+	_ = gzipCmd.Start()
-	}
+	_ = cmd.Run()
+	_ = gzipCmd.Wait()
 	return nil
 }
 
 func (r *Remote) Recover(info RecoverInfo) error {
-	fileName := info.SourceFile
+	fi, _ := os.Open(info.SourceFile)
-	if strings.HasSuffix(info.SourceFile, ".sql.gz") {
+	defer fi.Close()
-		fileName = strings.TrimSuffix(info.SourceFile, ".gz")
-		gzipCmd := exec.Command("gunzip", info.SourceFile)
-		stdout, err := gzipCmd.CombinedOutput()
-		if err != nil {
-			return fmt.Errorf("gunzip file %s failed, stdout: %v, err: %v", info.SourceFile, string(stdout), err)
-		}
-		defer func() {
-			gzipCmd := exec.Command("gzip", fileName)
-			_, _ = gzipCmd.CombinedOutput()
-		}()
-	}
-	tlsItem, err := ConnWithSSL(r.SSL, r.SkipVerify, r.ClientKey, r.ClientCert, r.RootCert)
-	if err != nil {
-		return err
-	}
-	dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai%s", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F", tlsItem)
 
-	f, err := os.Open(fileName)
+	image, err := loadImage(info.Type, info.Version)
 	if err != nil {
 		return err
 	}
-	defer f.Close()
+
-	if err := helper.Source(dns, f, helper.WithMergeInsert(1000)); err != nil {
+	recoverCmd := fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'mysql -h %s -P %d -u%s -p%s %s --default-character-set=%s %s'",
-		return err
+		image, r.Address, r.Port, r.User, r.Password, sslSkip(info.Version), info.Format, info.Name)
+
+	global.LOG.Debug(recoverCmd)
+	cmd := exec.Command("bash", "-c", recoverCmd)
+
+	if strings.HasSuffix(info.SourceFile, ".gz") {
+		gzipFile, err := os.Open(info.SourceFile)
+		if err != nil {
+			return err
+		}
+		defer gzipFile.Close()
+		gzipReader, err := gzip.NewReader(gzipFile)
+		if err != nil {
+			return err
+		}
+		defer gzipReader.Close()
+		cmd.Stdin = gzipReader
+	} else {
+		cmd.Stdin = fi
 	}
+	stdout, err := cmd.CombinedOutput()
+	stdStr := strings.ReplaceAll(string(stdout), "mysql: [Warning] Using a password on the command line interface can be insecure.\n", "")
+	if err != nil || strings.HasPrefix(string(stdStr), "ERROR ") {
+		return errors.New(stdStr)
+	}
+
 	return nil
 }
 
@@ -389,3 +400,45 @@ func (r *Remote) ExecSQLForHosts(timeout uint) ([]string, error) {
 	}
 	return rows, nil
 }
+
+func loadImage(dbType, version string) (string, error) {
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		fmt.Println(err)
+	}
+	images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	for _, image := range images {
+		for _, tag := range image.RepoTags {
+			if !strings.HasPrefix(tag, dbType+":") {
+				continue
+			}
+			if dbType == "mariadb" && strings.HasPrefix(tag, "mariadb:") {
+				return tag, nil
+			}
+			if strings.HasPrefix(version, "5.6") && strings.HasPrefix(tag, "mysql:5.6") {
+				return tag, nil
+			}
+			if strings.HasPrefix(version, "5.7") && strings.HasPrefix(tag, "mysql:5.7") {
+				return tag, nil
+			}
+			if strings.HasPrefix(version, "8.") && strings.HasPrefix(tag, "mysql:8.") {
+				return tag, nil
+			}
+		}
+	}
+	if dbType == "mariadb" || version == "8.x" {
+		return "mysql:8.2.0", nil
+	}
+	return "mysql:" + version, nil
+}
+
+func sslSkip(version string) string {
+	if strings.HasPrefix(version, "5.6") || strings.HasPrefix(version, "5.7") {
+		return "--skip-ssl"
+	}
+	return "--ssl-mode=DISABLED"
+}