gh_mirrors/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-01-19 08:19:15 +08:00
Code

pref: 优化 MySQL 备份恢复方式 (#4059)

Browse Source 
Refs #3955
This commit is contained in:
ssongliu 2024-03-04 16:35:05 +08:00 committed by GitHub
parent dca29174ea
commit e7f172520a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 108 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/app/service/backup_app.go
View File

@ -115,7 +115,7 @@ func handleAppBackup(install *model.AppInstall, backupDir, fileName string) erro
if err != nil {
return err
}
if err := handleMysqlBackup(db.MysqlName, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", install.Name)); err != nil {
if err := handleMysqlBackup(db.MysqlName, resource.Key, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", install.Name)); err != nil {
return err
}
case constant.AppPostgresql:

16
backend/app/service/backup_mysql.go
View File

@ -29,7 +29,7 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error {
targetDir := path.Join(localDir, itemDir)
fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5))
if err := handleMysqlBackup(req.Name, req.DetailName, targetDir, fileName); err != nil {
if err := handleMysqlBackup(req.Name, req.Type, req.DetailName, targetDir, fileName); err != nil {
return err
}
@ -100,18 +100,20 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error {
return nil
}
func handleMysqlBackup(database, dbName, targetDir, fileName string) error {
func handleMysqlBackup(database, dbType, dbName, targetDir, fileName string) error {
dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(dbName), mysqlRepo.WithByMysqlName(database))
if err != nil {
return err
}
cli, _, err := LoadMysqlClientByFrom(database)
cli, version, err := LoadMysqlClientByFrom(database)
if err != nil {
return err
}
backupInfo := client.BackupInfo{
Name: dbName,
Type: dbType,
Version: version,
Format: dbInfo.Format,
TargetDir: targetDir,
FileName: fileName,
@ -134,7 +136,7 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
if err != nil {
return err
}
cli, _, err := LoadMysqlClientByFrom(req.Name)
cli, version, err := LoadMysqlClientByFrom(req.Name)
if err != nil {
return err
}
@ -143,6 +145,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format("20060102150405")))
if err := cli.Backup(client.BackupInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
TargetDir: path.Dir(rollbackFile),
FileName: path.Base(rollbackFile),
@ -156,6 +160,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
global.LOG.Info("recover failed, start to rollback now")
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
SourceFile: rollbackFile,
@ -172,6 +178,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error {
}
if err := cli.Recover(client.RecoverInfo{
Name: req.DetailName,
Type: req.Type,
Version: version,
Format: dbInfo.Format,
SourceFile: req.File,

2
backend/app/service/cronjob_backup.go
View File

@ -109,7 +109,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name))
record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format("20060102150405")+common.RandStrAndNum(5))
if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" {
if err := handleMysqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil {
if err := handleMysqlBackup(dbInfo.Database, dbInfo.DBType, dbInfo.Name, backupDir, record.FileName); err != nil {
return err
}
} else {

4
backend/utils/mysql/client/info.go
View File

@ -70,6 +70,8 @@ type AccessChangeInfo struct {
type BackupInfo struct {
Name string `json:"name"`
Type string `json:"type"`
Version string `json:"version"`
Format string `json:"format"`
TargetDir string `json:"targetDir"`
FileName string `json:"fileName"`
@ -79,6 +81,8 @@ type BackupInfo struct {
type RecoverInfo struct {
Name string `json:"name"`
Type string `json:"type"`
Version string `json:"version"`
Format string `json:"format"`
SourceFile string `json:"sourceFile"`

127
backend/utils/mysql/client/remote.go
View File

@ -1,11 +1,14 @@
package client
import (
"compress/gzip"
"context"
"database/sql"
"errors"
"fmt"
"os"
"os/exec"
"path"
"strings"
"time"
@ -13,7 +16,8 @@ import (
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/1Panel-dev/1Panel/backend/global"
"github.com/1Panel-dev/1Panel/backend/utils/files"
"github.com/1Panel-dev/1Panel/backend/utils/mysql/helper"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
)
type Remote struct {
@ -229,56 +233,63 @@ func (r *Remote) Backup(info BackupInfo) error {
return fmt.Errorf("mkdir %s failed, err: %v", info.TargetDir, err)
}
}
fileNameItem := info.TargetDir + "/" + strings.TrimSuffix(info.FileName, ".gz")
tlsItem, err := ConnWithSSL(r.SSL, r.SkipVerify, r.ClientKey, r.ClientCert, r.RootCert)
outfile, _ := os.OpenFile(path.Join(info.TargetDir, info.FileName), os.O_RDWR|os.O_CREATE, 0755)
global.LOG.Infof("start to mysqldump | gzip > %s.gzip", info.TargetDir+"/"+info.FileName)
image, err := loadImage(info.Type, info.Version)
if err != nil {
return err
}
dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai%s", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F", tlsItem)
backupCmd := fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'mysqldump -h %s -P %d -u%s -p%s %s --default-character-set=%s %s'",
image, r.Address, r.Port, r.User, r.Password, sslSkip(info.Version), info.Format, info.Name)
f, _ := os.OpenFile(fileNameItem, os.O_RDWR|os.O_CREATE, 0755)
defer f.Close()
if err := helper.Dump(dns, helper.WithData(), helper.WithDropTable(), helper.WithWriter(f)); err != nil {
return err
}
global.LOG.Debug(backupCmd)
cmd := exec.Command("bash", "-c", backupCmd)
gzipCmd := exec.Command("gzip", fileNameItem)
stdout, err := gzipCmd.CombinedOutput()
if err != nil {
return fmt.Errorf("gzip file %s failed, stdout: %v, err: %v", strings.TrimSuffix(info.FileName, ".gz"), string(stdout), err)
}
gzipCmd := exec.Command("gzip", "-cf")
gzipCmd.Stdin, _ = cmd.StdoutPipe()
gzipCmd.Stdout = outfile
_ = gzipCmd.Start()
_ = cmd.Run()
_ = gzipCmd.Wait()
return nil
}
func (r *Remote) Recover(info RecoverInfo) error {
fileName := info.SourceFile
if strings.HasSuffix(info.SourceFile, ".sql.gz") {
fileName = strings.TrimSuffix(info.SourceFile, ".gz")
gzipCmd := exec.Command("gunzip", info.SourceFile)
stdout, err := gzipCmd.CombinedOutput()
if err != nil {
return fmt.Errorf("gunzip file %s failed, stdout: %v, err: %v", info.SourceFile, string(stdout), err)
}
defer func() {
gzipCmd := exec.Command("gzip", fileName)
_, _ = gzipCmd.CombinedOutput()
}()
}
tlsItem, err := ConnWithSSL(r.SSL, r.SkipVerify, r.ClientKey, r.ClientCert, r.RootCert)
if err != nil {
return err
}
dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai%s", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F", tlsItem)
fi, _ := os.Open(info.SourceFile)
defer fi.Close()
f, err := os.Open(fileName)
image, err := loadImage(info.Type, info.Version)
if err != nil {
return err
}
defer f.Close()
if err := helper.Source(dns, f, helper.WithMergeInsert(1000)); err != nil {
return err
recoverCmd := fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'mysql -h %s -P %d -u%s -p%s %s --default-character-set=%s %s'",
image, r.Address, r.Port, r.User, r.Password, sslSkip(info.Version), info.Format, info.Name)
global.LOG.Debug(recoverCmd)
cmd := exec.Command("bash", "-c", recoverCmd)
if strings.HasSuffix(info.SourceFile, ".gz") {
gzipFile, err := os.Open(info.SourceFile)
if err != nil {
return err
}
defer gzipFile.Close()
gzipReader, err := gzip.NewReader(gzipFile)
if err != nil {
return err
}
defer gzipReader.Close()
cmd.Stdin = gzipReader
} else {
cmd.Stdin = fi
}
stdout, err := cmd.CombinedOutput()
stdStr := strings.ReplaceAll(string(stdout), "mysql: [Warning] Using a password on the command line interface can be insecure.\n", "")
if err != nil || strings.HasPrefix(string(stdStr), "ERROR ") {
return errors.New(stdStr)
}
return nil
}
@ -389,3 +400,45 @@ func (r *Remote) ExecSQLForHosts(timeout uint) ([]string, error) {
}
return rows, nil
}
func loadImage(dbType, version string) (string, error) {
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
fmt.Println(err)
}
images, err := cli.ImageList(context.Background(), types.ImageListOptions{})
if err != nil {
fmt.Println(err)
}
for _, image := range images {
for _, tag := range image.RepoTags {
if !strings.HasPrefix(tag, dbType+":") {
continue
}
if dbType == "mariadb" && strings.HasPrefix(tag, "mariadb:") {
return tag, nil
}
if strings.HasPrefix(version, "5.6") && strings.HasPrefix(tag, "mysql:5.6") {
return tag, nil
}
if strings.HasPrefix(version, "5.7") && strings.HasPrefix(tag, "mysql:5.7") {
return tag, nil
}
if strings.HasPrefix(version, "8.") && strings.HasPrefix(tag, "mysql:8.") {
return tag, nil
}
}
}
if dbType == "mariadb" || version == "8.x" {
return "mysql:8.2.0", nil
}
return "mysql:" + version, nil
}
func sslSkip(version string) string {
if strings.HasPrefix(version, "5.6") || strings.HasPrefix(version, "5.7") {
return "--skip-ssl"
}
return "--ssl-mode=DISABLED"
}