From e7f172520af1da623755eaed0ad105d9d3797cdc Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:35:05 +0800 Subject: [PATCH] =?UTF-8?q?pref:=20=E4=BC=98=E5=8C=96=20MySQL=20=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E6=81=A2=E5=A4=8D=E6=96=B9=E5=BC=8F=20(#4059)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #3955 --- backend/app/service/backup_app.go | 2 +- backend/app/service/backup_mysql.go | 16 +++- backend/app/service/cronjob_backup.go | 2 +- backend/utils/mysql/client/info.go | 4 + backend/utils/mysql/client/remote.go | 127 ++++++++++++++++++-------- 5 files changed, 108 insertions(+), 43 deletions(-) diff --git a/backend/app/service/backup_app.go b/backend/app/service/backup_app.go index 8718d9c28..1b7941593 100644 --- a/backend/app/service/backup_app.go +++ b/backend/app/service/backup_app.go @@ -115,7 +115,7 @@ func handleAppBackup(install *model.AppInstall, backupDir, fileName string) erro if err != nil { return err } - if err := handleMysqlBackup(db.MysqlName, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", install.Name)); err != nil { + if err := handleMysqlBackup(db.MysqlName, resource.Key, db.Name, tmpDir, fmt.Sprintf("%s.sql.gz", install.Name)); err != nil { return err } case constant.AppPostgresql: diff --git a/backend/app/service/backup_mysql.go b/backend/app/service/backup_mysql.go index 8fb57aaad..5d746701f 100644 --- a/backend/app/service/backup_mysql.go +++ b/backend/app/service/backup_mysql.go @@ -29,7 +29,7 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error { targetDir := path.Join(localDir, itemDir) fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow+common.RandStrAndNum(5)) - if err := handleMysqlBackup(req.Name, req.DetailName, targetDir, fileName); err != nil { + if err := handleMysqlBackup(req.Name, req.Type, req.DetailName, targetDir, fileName); err != nil { return err } @@ -100,18 +100,20 @@ func (u *BackupService) MysqlRecoverByUpload(req dto.CommonRecover) error { return nil } -func handleMysqlBackup(database, dbName, targetDir, fileName string) error { +func handleMysqlBackup(database, dbType, dbName, targetDir, fileName string) error { dbInfo, err := mysqlRepo.Get(commonRepo.WithByName(dbName), mysqlRepo.WithByMysqlName(database)) if err != nil { return err } - cli, _, err := LoadMysqlClientByFrom(database) + cli, version, err := LoadMysqlClientByFrom(database) if err != nil { return err } backupInfo := client.BackupInfo{ Name: dbName, + Type: dbType, + Version: version, Format: dbInfo.Format, TargetDir: targetDir, FileName: fileName, @@ -134,7 +136,7 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error { if err != nil { return err } - cli, _, err := LoadMysqlClientByFrom(req.Name) + cli, version, err := LoadMysqlClientByFrom(req.Name) if err != nil { return err } @@ -143,6 +145,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error { rollbackFile := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s_%s.sql.gz", req.Type, req.DetailName, time.Now().Format("20060102150405"))) if err := cli.Backup(client.BackupInfo{ Name: req.DetailName, + Type: req.Type, + Version: version, Format: dbInfo.Format, TargetDir: path.Dir(rollbackFile), FileName: path.Base(rollbackFile), @@ -156,6 +160,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error { global.LOG.Info("recover failed, start to rollback now") if err := cli.Recover(client.RecoverInfo{ Name: req.DetailName, + Type: req.Type, + Version: version, Format: dbInfo.Format, SourceFile: rollbackFile, @@ -172,6 +178,8 @@ func handleMysqlRecover(req dto.CommonRecover, isRollback bool) error { } if err := cli.Recover(client.RecoverInfo{ Name: req.DetailName, + Type: req.Type, + Version: version, Format: dbInfo.Format, SourceFile: req.File, diff --git a/backend/app/service/cronjob_backup.go b/backend/app/service/cronjob_backup.go index 95e845773..fce556dcb 100644 --- a/backend/app/service/cronjob_backup.go +++ b/backend/app/service/cronjob_backup.go @@ -109,7 +109,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name)) record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format("20060102150405")+common.RandStrAndNum(5)) if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { - if err := handleMysqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil { + if err := handleMysqlBackup(dbInfo.Database, dbInfo.DBType, dbInfo.Name, backupDir, record.FileName); err != nil { return err } } else { diff --git a/backend/utils/mysql/client/info.go b/backend/utils/mysql/client/info.go index 8b9320a49..0dbc20315 100644 --- a/backend/utils/mysql/client/info.go +++ b/backend/utils/mysql/client/info.go @@ -70,6 +70,8 @@ type AccessChangeInfo struct { type BackupInfo struct { Name string `json:"name"` + Type string `json:"type"` + Version string `json:"version"` Format string `json:"format"` TargetDir string `json:"targetDir"` FileName string `json:"fileName"` @@ -79,6 +81,8 @@ type BackupInfo struct { type RecoverInfo struct { Name string `json:"name"` + Type string `json:"type"` + Version string `json:"version"` Format string `json:"format"` SourceFile string `json:"sourceFile"` diff --git a/backend/utils/mysql/client/remote.go b/backend/utils/mysql/client/remote.go index d385b2b47..336d778e8 100644 --- a/backend/utils/mysql/client/remote.go +++ b/backend/utils/mysql/client/remote.go @@ -1,11 +1,14 @@ package client import ( + "compress/gzip" "context" "database/sql" + "errors" "fmt" "os" "os/exec" + "path" "strings" "time" @@ -13,7 +16,8 @@ import ( "github.com/1Panel-dev/1Panel/backend/constant" "github.com/1Panel-dev/1Panel/backend/global" "github.com/1Panel-dev/1Panel/backend/utils/files" - "github.com/1Panel-dev/1Panel/backend/utils/mysql/helper" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" ) type Remote struct { @@ -229,56 +233,63 @@ func (r *Remote) Backup(info BackupInfo) error { return fmt.Errorf("mkdir %s failed, err: %v", info.TargetDir, err) } } - fileNameItem := info.TargetDir + "/" + strings.TrimSuffix(info.FileName, ".gz") - - tlsItem, err := ConnWithSSL(r.SSL, r.SkipVerify, r.ClientKey, r.ClientCert, r.RootCert) + outfile, _ := os.OpenFile(path.Join(info.TargetDir, info.FileName), os.O_RDWR|os.O_CREATE, 0755) + global.LOG.Infof("start to mysqldump | gzip > %s.gzip", info.TargetDir+"/"+info.FileName) + image, err := loadImage(info.Type, info.Version) if err != nil { return err } - dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai%s", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F", tlsItem) + backupCmd := fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'mysqldump -h %s -P %d -u%s -p%s %s --default-character-set=%s %s'", + image, r.Address, r.Port, r.User, r.Password, sslSkip(info.Version), info.Format, info.Name) - f, _ := os.OpenFile(fileNameItem, os.O_RDWR|os.O_CREATE, 0755) - defer f.Close() - if err := helper.Dump(dns, helper.WithData(), helper.WithDropTable(), helper.WithWriter(f)); err != nil { - return err - } + global.LOG.Debug(backupCmd) + cmd := exec.Command("bash", "-c", backupCmd) - gzipCmd := exec.Command("gzip", fileNameItem) - stdout, err := gzipCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("gzip file %s failed, stdout: %v, err: %v", strings.TrimSuffix(info.FileName, ".gz"), string(stdout), err) - } + gzipCmd := exec.Command("gzip", "-cf") + gzipCmd.Stdin, _ = cmd.StdoutPipe() + gzipCmd.Stdout = outfile + _ = gzipCmd.Start() + _ = cmd.Run() + _ = gzipCmd.Wait() return nil } func (r *Remote) Recover(info RecoverInfo) error { - fileName := info.SourceFile - if strings.HasSuffix(info.SourceFile, ".sql.gz") { - fileName = strings.TrimSuffix(info.SourceFile, ".gz") - gzipCmd := exec.Command("gunzip", info.SourceFile) - stdout, err := gzipCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("gunzip file %s failed, stdout: %v, err: %v", info.SourceFile, string(stdout), err) - } - defer func() { - gzipCmd := exec.Command("gzip", fileName) - _, _ = gzipCmd.CombinedOutput() - }() - } - tlsItem, err := ConnWithSSL(r.SSL, r.SkipVerify, r.ClientKey, r.ClientCert, r.RootCert) - if err != nil { - return err - } - dns := fmt.Sprintf("%s:%s@tcp(%s:%v)/%s?charset=%s&parseTime=true&loc=Asia%sShanghai%s", r.User, r.Password, r.Address, r.Port, info.Name, info.Format, "%2F", tlsItem) + fi, _ := os.Open(info.SourceFile) + defer fi.Close() - f, err := os.Open(fileName) + image, err := loadImage(info.Type, info.Version) if err != nil { return err } - defer f.Close() - if err := helper.Source(dns, f, helper.WithMergeInsert(1000)); err != nil { - return err + + recoverCmd := fmt.Sprintf("docker run --rm --net=host -i %s /bin/bash -c 'mysql -h %s -P %d -u%s -p%s %s --default-character-set=%s %s'", + image, r.Address, r.Port, r.User, r.Password, sslSkip(info.Version), info.Format, info.Name) + + global.LOG.Debug(recoverCmd) + cmd := exec.Command("bash", "-c", recoverCmd) + + if strings.HasSuffix(info.SourceFile, ".gz") { + gzipFile, err := os.Open(info.SourceFile) + if err != nil { + return err + } + defer gzipFile.Close() + gzipReader, err := gzip.NewReader(gzipFile) + if err != nil { + return err + } + defer gzipReader.Close() + cmd.Stdin = gzipReader + } else { + cmd.Stdin = fi } + stdout, err := cmd.CombinedOutput() + stdStr := strings.ReplaceAll(string(stdout), "mysql: [Warning] Using a password on the command line interface can be insecure.\n", "") + if err != nil || strings.HasPrefix(string(stdStr), "ERROR ") { + return errors.New(stdStr) + } + return nil } @@ -389,3 +400,45 @@ func (r *Remote) ExecSQLForHosts(timeout uint) ([]string, error) { } return rows, nil } + +func loadImage(dbType, version string) (string, error) { + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + fmt.Println(err) + } + images, err := cli.ImageList(context.Background(), types.ImageListOptions{}) + if err != nil { + fmt.Println(err) + } + + for _, image := range images { + for _, tag := range image.RepoTags { + if !strings.HasPrefix(tag, dbType+":") { + continue + } + if dbType == "mariadb" && strings.HasPrefix(tag, "mariadb:") { + return tag, nil + } + if strings.HasPrefix(version, "5.6") && strings.HasPrefix(tag, "mysql:5.6") { + return tag, nil + } + if strings.HasPrefix(version, "5.7") && strings.HasPrefix(tag, "mysql:5.7") { + return tag, nil + } + if strings.HasPrefix(version, "8.") && strings.HasPrefix(tag, "mysql:8.") { + return tag, nil + } + } + } + if dbType == "mariadb" || version == "8.x" { + return "mysql:8.2.0", nil + } + return "mysql:" + version, nil +} + +func sslSkip(version string) string { + if strings.HasPrefix(version, "5.6") || strings.HasPrefix(version, "5.7") { + return "--skip-ssl" + } + return "--ssl-mode=DISABLED" +}