1
0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-01-31 14:08:06 +08:00

feat: 备份账号增删改查操作同步到节点 (#6081)

This commit is contained in:
ssongliu 2024-08-09 18:30:39 +08:00 committed by GitHub
parent 9114df9c2a
commit fecba858a5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 24266 additions and 1024 deletions

View File

@ -1,7 +1,6 @@
package v2
import (
"encoding/base64"
"fmt"
"path"
@ -11,125 +10,14 @@ import (
"github.com/gin-gonic/gin"
)
// @Tags Backup Account
// @Summary Create backup account
// @Description 创建备份账号
// @Accept json
// @Param request body dto.BackupOperate true "request"
// @Success 200
// @Security ApiKeyAuth
// @Router /settings/backup [post]
// @x-panel-log {"bodyKeys":["type"],"paramKeys":[],"BeforeFunctions":[],"formatZH":"创建备份账号 [type]","formatEN":"create backup account [type]"}
func (b *BaseApi) CreateBackup(c *gin.Context) {
func (b *BaseApi) OperateBackup(c *gin.Context) {
var req dto.BackupOperate
if err := helper.CheckBindAndValidate(&req, c); err != nil {
return
}
if len(req.Credential) != 0 {
credential, err := base64.StdEncoding.DecodeString(req.Credential)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.Credential = string(credential)
}
if len(req.AccessKey) != 0 {
accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.AccessKey = string(accessKey)
}
if err := backupService.Create(req); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithData(c, nil)
}
// @Tags Backup Account
// @Summary Refresh OneDrive token
// @Description 刷新 OneDrive token
// @Success 200
// @Security ApiKeyAuth
// @Router /settings/backup/refresh/onedrive [post]
func (b *BaseApi) RefreshOneDriveToken(c *gin.Context) {
backupService.Run()
helper.SuccessWithData(c, nil)
}
// @Tags Backup Account
// @Summary List buckets
// @Description 获取 bucket 列表
// @Accept json
// @Param request body dto.ForBuckets true "request"
// @Success 200 {array} string
// @Security ApiKeyAuth
// @Router /settings/backup/search [post]
func (b *BaseApi) ListBuckets(c *gin.Context) {
var req dto.ForBuckets
if err := helper.CheckBindAndValidate(&req, c); err != nil {
return
}
if len(req.Credential) != 0 {
credential, err := base64.StdEncoding.DecodeString(req.Credential)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.Credential = string(credential)
}
if len(req.AccessKey) != 0 {
accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.AccessKey = string(accessKey)
}
buckets, err := backupService.GetBuckets(req)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithData(c, buckets)
}
// @Tags Backup Account
// @Summary Load OneDrive info
// @Description 获取 OneDrive 信息
// @Accept json
// @Success 200 {object} dto.OneDriveInfo
// @Security ApiKeyAuth
// @Router /settings/backup/onedrive [get]
func (b *BaseApi) LoadOneDriveInfo(c *gin.Context) {
data, err := backupService.LoadOneDriveInfo()
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithData(c, data)
}
// @Tags Backup Account
// @Summary Delete backup account
// @Description 删除备份账号
// @Accept json
// @Param request body dto.OperateByID true "request"
// @Success 200
// @Security ApiKeyAuth
// @Router /settings/backup/del [post]
// @x-panel-log {"bodyKeys":["id"],"paramKeys":[],"BeforeFunctions":[{"input_column":"id","input_value":"id","isList":false,"db":"backup_accounts","output_column":"type","output_value":"types"}],"formatZH":"删除备份账号 [types]","formatEN":"delete backup account [types]"}
func (b *BaseApi) DeleteBackup(c *gin.Context) {
var req dto.OperateByID
if err := helper.CheckBindAndValidate(&req, c); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInvalidParams, err)
return
}
if err := backupService.Delete(req.ID); err != nil {
if err := backupService.Operate(req); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
@ -233,61 +121,6 @@ func (b *BaseApi) DeleteBackupRecord(c *gin.Context) {
helper.SuccessWithData(c, nil)
}
// @Tags Backup Account
// @Summary Update backup account
// @Description 更新备份账号信息
// @Accept json
// @Param request body dto.BackupOperate true "request"
// @Success 200
// @Security ApiKeyAuth
// @Router /settings/backup/update [post]
// @x-panel-log {"bodyKeys":["type"],"paramKeys":[],"BeforeFunctions":[],"formatZH":"更新备份账号 [types]","formatEN":"update backup account [types]"}
func (b *BaseApi) UpdateBackup(c *gin.Context) {
var req dto.BackupOperate
if err := helper.CheckBindAndValidate(&req, c); err != nil {
return
}
if len(req.Credential) != 0 {
credential, err := base64.StdEncoding.DecodeString(req.Credential)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.Credential = string(credential)
}
if len(req.AccessKey) != 0 {
accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.AccessKey = string(accessKey)
}
if err := backupService.Update(req); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithData(c, nil)
}
// @Tags Backup Account
// @Summary List backup accounts
// @Description 获取备份账号列表
// @Success 200 {array} dto.BackupInfo
// @Security ApiKeyAuth
// @Router /settings/backup/search [get]
func (b *BaseApi) ListBackup(c *gin.Context) {
data, err := backupService.List()
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithData(c, data)
}
// @Tags Backup Account
// @Summary List files from backup accounts
// @Description 获取备份账号内文件列表

View File

@ -1,15 +1,14 @@
package dto
import "time"
import (
"time"
"github.com/1Panel-dev/1Panel/agent/app/model"
)
type BackupOperate struct {
ID uint `json:"id"`
Type string `json:"type" validate:"required"`
Bucket string `json:"bucket"`
AccessKey string `json:"accessKey"`
Credential string `json:"credential"`
BackupPath string `json:"backupPath"`
Vars string `json:"vars" validate:"required"`
Operate string `json:"operate" validate:"required,oneof=add remove update"`
Data []model.BackupAccount `json:"data" validate:"required"`
}
type BackupInfo struct {
@ -21,12 +20,6 @@ type BackupInfo struct {
Vars string `json:"vars"`
}
type OneDriveInfo struct {
ClientID string `json:"client_id"`
ClientSecret string `json:"client_secret"`
RedirectUri string `json:"redirect_uri"`
}
type BackupSearchFile struct {
Type string `json:"type" validate:"required"`
}
@ -73,10 +66,3 @@ type DownloadRecord struct {
FileDir string `json:"fileDir" validate:"required"`
FileName string `json:"fileName" validate:"required"`
}
type ForBuckets struct {
Type string `json:"type" validate:"required"`
AccessKey string `json:"accessKey"`
Credential string `json:"credential" validate:"required"`
Vars string `json:"vars" validate:"required"`
}

View File

@ -12,13 +12,14 @@ type BackupRepo struct{}
type IBackupRepo interface {
Get(opts ...DBOption) (model.BackupAccount, error)
List(opts ...DBOption) ([]model.BackupAccount, error)
Create(backup []model.BackupAccount) error
Save(backup *model.BackupAccount) error
Delete(opts ...DBOption) error
ListRecord(opts ...DBOption) ([]model.BackupRecord, error)
PageRecord(page, size int, opts ...DBOption) (int64, []model.BackupRecord, error)
List(opts ...DBOption) ([]model.BackupAccount, error)
Create(backup *model.BackupAccount) error
CreateRecord(record *model.BackupRecord) error
Update(id uint, vars map[string]interface{}) error
Delete(opts ...DBOption) error
DeleteRecord(ctx context.Context, opts ...DBOption) error
UpdateRecord(record *model.BackupRecord) error
WithByDetailName(detailName string) DBOption
@ -41,6 +42,10 @@ func (u *BackupRepo) Get(opts ...DBOption) (model.BackupAccount, error) {
return backup, err
}
func (u *BackupRepo) Save(backup *model.BackupAccount) error {
return global.DB.Save(backup).Error
}
func (u *BackupRepo) ListRecord(opts ...DBOption) ([]model.BackupRecord, error) {
var users []model.BackupRecord
db := global.DB.Model(&model.BackupRecord{})
@ -100,7 +105,7 @@ func (u *BackupRepo) List(opts ...DBOption) ([]model.BackupAccount, error) {
return ops, err
}
func (u *BackupRepo) Create(backup *model.BackupAccount) error {
func (u *BackupRepo) Create(backup []model.BackupAccount) error {
return global.DB.Create(backup).Error
}
@ -112,10 +117,6 @@ func (u *BackupRepo) UpdateRecord(record *model.BackupRecord) error {
return global.DB.Save(record).Error
}
func (u *BackupRepo) Update(id uint, vars map[string]interface{}) error {
return global.DB.Model(&model.BackupAccount{}).Where("id = ?", id).Updates(vars).Error
}
func (u *BackupRepo) Delete(opts ...DBOption) error {
db := global.DB
for _, opt := range opts {

View File

@ -21,6 +21,7 @@ type ICommonRepo interface {
WithByGroupID(groupID uint) DBOption
WithLikeName(name string) DBOption
WithIdsIn(ids []uint) DBOption
WithNamesIn(names []string) DBOption
WithByDate(startTime, endTime time.Time) DBOption
WithByCreatedAt(startTime, endTime time.Time) DBOption
WithByStartDate(startTime time.Time) DBOption
@ -124,6 +125,12 @@ func (c *CommonRepo) WithOrderRuleBy(orderBy, order string) DBOption {
}
}
func (c *CommonRepo) WithNamesIn(names []string) DBOption {
return func(g *gorm.DB) *gorm.DB {
return g.Where("name in (?)", names)
}
}
func (c *CommonRepo) WithIdsIn(ids []uint) DBOption {
return func(g *gorm.DB) *gorm.DB {
return g.Where("id in (?)", ids)

View File

@ -1,9 +1,7 @@
package service
import (
"bufio"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"os"
@ -11,15 +9,13 @@ import (
"sort"
"strings"
"sync"
"time"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/cloud_storage"
"github.com/1Panel-dev/1Panel/agent/utils/cloud_storage/client"
"github.com/1Panel-dev/1Panel/agent/utils/encrypt"
fileUtils "github.com/1Panel-dev/1Panel/agent/utils/files"
"github.com/jinzhu/copier"
"github.com/pkg/errors"
@ -28,15 +24,11 @@ import (
type BackupService struct{}
type IBackupService interface {
List() ([]dto.BackupInfo, error)
Operate(req dto.BackupOperate) error
SearchRecordsWithPage(search dto.RecordSearch) (int64, []dto.BackupRecords, error)
SearchRecordsByCronjobWithPage(search dto.RecordSearchByCronjob) (int64, []dto.BackupRecords, error)
LoadOneDriveInfo() (dto.OneDriveInfo, error)
DownloadRecord(info dto.DownloadRecord) (string, error)
Create(backupDto dto.BackupOperate) error
GetBuckets(backupDto dto.ForBuckets) ([]interface{}, error)
Update(ireq dto.BackupOperate) error
Delete(id uint) error
DeleteRecordByName(backupType, name, detailName string, withDeleteFile bool) error
BatchDeleteRecord(ids []uint) error
NewClient(backup *model.BackupAccount) (cloud_storage.CloudStorageClient, error)
@ -58,29 +50,12 @@ type IBackupService interface {
AppBackup(db dto.CommonBackup) (*model.BackupRecord, error)
AppRecover(req dto.CommonRecover) error
Run()
}
func NewIBackupService() IBackupService {
return &BackupService{}
}
func (u *BackupService) List() ([]dto.BackupInfo, error) {
ops, err := backupRepo.List(commonRepo.WithOrderBy("created_at desc"))
var dtobas []dto.BackupInfo
dtobas = append(dtobas, u.loadByType("LOCAL", ops))
dtobas = append(dtobas, u.loadByType("OSS", ops))
dtobas = append(dtobas, u.loadByType("S3", ops))
dtobas = append(dtobas, u.loadByType("SFTP", ops))
dtobas = append(dtobas, u.loadByType("MINIO", ops))
dtobas = append(dtobas, u.loadByType("COS", ops))
dtobas = append(dtobas, u.loadByType("KODO", ops))
dtobas = append(dtobas, u.loadByType("OneDrive", ops))
dtobas = append(dtobas, u.loadByType("WebDAV", ops))
return dtobas, err
}
func (u *BackupService) SearchRecordsWithPage(search dto.RecordSearch) (int64, []dto.BackupRecords, error) {
total, records, err := backupRepo.PageRecord(
search.Page, search.PageSize,
@ -123,32 +98,11 @@ type loadSizeHelper struct {
client cloud_storage.CloudStorageClient
}
func (u *BackupService) LoadOneDriveInfo() (dto.OneDriveInfo, error) {
var data dto.OneDriveInfo
data.RedirectUri = constant.OneDriveRedirectURI
clientID, err := settingRepo.Get(settingRepo.WithByKey("OneDriveID"))
if err != nil {
return data, err
}
idItem, err := base64.StdEncoding.DecodeString(clientID.Value)
if err != nil {
return data, err
}
data.ClientID = string(idItem)
clientSecret, err := settingRepo.Get(settingRepo.WithByKey("OneDriveSc"))
if err != nil {
return data, err
}
secretItem, err := base64.StdEncoding.DecodeString(clientSecret.Value)
if err != nil {
return data, err
}
data.ClientSecret = string(secretItem)
return data, err
}
func (u *BackupService) DownloadRecord(info dto.DownloadRecord) (string, error) {
backup, _ := backupRepo.Get(commonRepo.WithByType(info.Source))
if backup.ID == 0 {
return "", constant.ErrRecordNotFound
}
if info.Source == "LOCAL" {
localDir, err := loadLocalDir()
if err != nil {
@ -156,10 +110,6 @@ func (u *BackupService) DownloadRecord(info dto.DownloadRecord) (string, error)
}
return path.Join(localDir, info.FileDir, info.FileName), nil
}
backup, _ := backupRepo.Get(commonRepo.WithByType(info.Source))
if backup.ID == 0 {
return "", constant.ErrRecordNotFound
}
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
return "", err
@ -198,69 +148,67 @@ func (u *BackupService) DownloadRecord(info dto.DownloadRecord) (string, error)
return targetPath, nil
}
func (u *BackupService) Create(req dto.BackupOperate) error {
backup, _ := backupRepo.Get(commonRepo.WithByType(req.Type))
if backup.ID != 0 {
return constant.ErrRecordExist
func (u *BackupService) Operate(req dto.BackupOperate) error {
for i := 0; i < len(req.Data); i++ {
encryptKeyItem, err := encrypt.StringEncryptWithBase64(req.Data[i].AccessKey)
if err != nil {
return err
}
req.Data[i].AccessKey = encryptKeyItem
encryptCredentialItem, err := encrypt.StringEncryptWithBase64(req.Data[i].Credential)
if err != nil {
return err
}
req.Data[i].Credential = encryptCredentialItem
}
if err := copier.Copy(&backup, &req); err != nil {
return errors.WithMessage(constant.ErrStructTransform, err.Error())
if req.Operate == "add" {
return backupRepo.Create(req.Data)
}
if req.Operate == "remove" {
var names []string
for _, item := range req.Data {
names = append(names, item.Name)
}
return backupRepo.Delete(commonRepo.WithNamesIn(names))
}
global.LOG.Debug("走到了这里")
for _, item := range req.Data {
local, _ := backupRepo.Get(commonRepo.WithByName(item.Name))
if local.ID == 0 {
if err := backupRepo.Create([]model.BackupAccount{item}); err != nil {
return err
}
continue
}
if item.Type == constant.Local {
if local.ID != 0 && item.Vars != local.Vars {
oldPath, err := loadLocalDirByStr(local.Vars)
if err != nil {
return err
}
newPath, err := loadLocalDirByStr(item.Vars)
if err != nil {
return err
}
if strings.HasSuffix(newPath, "/") && newPath != "/" {
newPath = newPath[:strings.LastIndex(newPath, "/")]
}
if err := copyDir(oldPath, newPath); err != nil {
return err
}
global.CONF.System.Backup = newPath
}
}
item.ID = local.ID
if req.Type == constant.OneDrive {
if err := u.loadAccessToken(&backup); err != nil {
global.LOG.Debug("走到了这里111")
if err := backupRepo.Save(&item); err != nil {
return err
}
}
if req.Type != "LOCAL" {
if _, err := u.checkBackupConn(&backup); err != nil {
return buserr.WithMap("ErrBackupCheck", map[string]interface{}{"err": err.Error()}, err)
}
}
if backup.Type == constant.OneDrive {
StartRefreshOneDriveToken()
}
if err := backupRepo.Create(&backup); err != nil {
return err
}
return nil
}
func (u *BackupService) GetBuckets(backupDto dto.ForBuckets) ([]interface{}, error) {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backupDto.Vars), &varMap); err != nil {
return nil, err
}
switch backupDto.Type {
case constant.Sftp, constant.WebDAV:
varMap["username"] = backupDto.AccessKey
varMap["password"] = backupDto.Credential
case constant.OSS, constant.S3, constant.MinIo, constant.Cos, constant.Kodo:
varMap["accessKey"] = backupDto.AccessKey
varMap["secretKey"] = backupDto.Credential
}
client, err := cloud_storage.NewCloudStorageClient(backupDto.Type, varMap)
if err != nil {
return nil, err
}
return client.ListBuckets()
}
func (u *BackupService) Delete(id uint) error {
backup, _ := backupRepo.Get(commonRepo.WithByID(id))
if backup.ID == 0 {
return constant.ErrRecordNotFound
}
if backup.Type == constant.OneDrive {
global.Cron.Remove(global.OneDriveCronID)
}
cronjobs, _ := cronjobRepo.List(cronjobRepo.WithByDefaultDownload(backup.Type))
if len(cronjobs) != 0 {
return buserr.New(constant.ErrBackupInUsed)
}
return backupRepo.Delete(commonRepo.WithByID(id))
}
func (u *BackupService) DeleteRecordByName(backupType, name, detailName string, withDeleteFile bool) error {
if !withDeleteFile {
return backupRepo.DeleteRecord(context.Background(), commonRepo.WithByType(backupType), commonRepo.WithByName(name), backupRepo.WithByDetailName(detailName))
@ -313,67 +261,6 @@ func (u *BackupService) BatchDeleteRecord(ids []uint) error {
return backupRepo.DeleteRecord(context.Background(), commonRepo.WithIdsIn(ids))
}
func (u *BackupService) Update(req dto.BackupOperate) error {
backup, err := backupRepo.Get(commonRepo.WithByID(req.ID))
if err != nil {
return constant.ErrRecordNotFound
}
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(req.Vars), &varMap); err != nil {
return err
}
oldVars := backup.Vars
oldDir, err := loadLocalDir()
if err != nil {
return err
}
upMap := make(map[string]interface{})
upMap["bucket"] = req.Bucket
upMap["access_key"] = req.AccessKey
upMap["credential"] = req.Credential
upMap["backup_path"] = req.BackupPath
upMap["vars"] = req.Vars
backup.Bucket = req.Bucket
backup.Vars = req.Vars
backup.Credential = req.Credential
backup.AccessKey = req.AccessKey
backup.BackupPath = req.BackupPath
if req.Type == constant.OneDrive {
if err := u.loadAccessToken(&backup); err != nil {
return err
}
upMap["credential"] = backup.Credential
upMap["vars"] = backup.Vars
}
if backup.Type != "LOCAL" {
isOk, err := u.checkBackupConn(&backup)
if err != nil || !isOk {
return buserr.WithMap("ErrBackupCheck", map[string]interface{}{"err": err.Error()}, err)
}
}
if err := backupRepo.Update(req.ID, upMap); err != nil {
return err
}
if backup.Type == "LOCAL" {
if dir, ok := varMap["dir"]; ok {
if dirStr, isStr := dir.(string); isStr {
if strings.HasSuffix(dirStr, "/") && dirStr != "/" {
dirStr = dirStr[:strings.LastIndex(dirStr, "/")]
}
if err := copyDir(oldDir, dirStr); err != nil {
_ = backupRepo.Update(req.ID, map[string]interface{}{"vars": oldVars})
return err
}
global.CONF.System.Backup = dirStr
}
}
}
return nil
}
func (u *BackupService) ListFiles(req dto.BackupSearchFile) []string {
var datas []string
backup, err := backupRepo.Get(backupRepo.WithByType(req.Type))
@ -424,49 +311,6 @@ func (u *BackupService) NewClient(backup *model.BackupAccount) (cloud_storage.Cl
return backClient, nil
}
func (u *BackupService) loadByType(accountType string, accounts []model.BackupAccount) dto.BackupInfo {
for _, account := range accounts {
if account.Type == accountType {
var item dto.BackupInfo
if err := copier.Copy(&item, &account); err != nil {
global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err)
}
if account.Type == constant.OneDrive {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(item.Vars), &varMap); err != nil {
return dto.BackupInfo{Type: accountType}
}
delete(varMap, "refresh_token")
itemVars, _ := json.Marshal(varMap)
item.Vars = string(itemVars)
}
return item
}
}
return dto.BackupInfo{Type: accountType}
}
func (u *BackupService) loadAccessToken(backup *model.BackupAccount) error {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
return fmt.Errorf("unmarshal backup vars failed, err: %v", err)
}
refreshToken, err := client.RefreshToken("authorization_code", "refreshToken", varMap)
if err != nil {
return err
}
delete(varMap, "code")
varMap["refresh_status"] = constant.StatusSuccess
varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
varMap["refresh_token"] = refreshToken
itemVars, err := json.Marshal(varMap)
if err != nil {
return fmt.Errorf("json marshal var map failed, err: %v", err)
}
backup.Vars = string(itemVars)
return nil
}
func (u *BackupService) loadRecordSize(records []model.BackupRecord) ([]dto.BackupRecords, error) {
var datas []dto.BackupRecords
clientMap := make(map[string]loadSizeHelper)
@ -517,8 +361,12 @@ func loadLocalDir() (string, error) {
if err != nil {
return "", err
}
return loadLocalDirByStr(backup.Vars)
}
func loadLocalDirByStr(vars string) (string, error) {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
if err := json.Unmarshal([]byte(vars), &varMap); err != nil {
return "", err
}
if _, ok := varMap["dir"]; !ok {
@ -531,7 +379,6 @@ func loadLocalDir() (string, error) {
return "", fmt.Errorf("mkdir %s failed, err: %v", baseDir, err)
}
}
return baseDir, nil
}
return "", fmt.Errorf("error type dir: %T", varMap["dir"])
}
@ -566,72 +413,3 @@ func copyDir(src, dst string) error {
return nil
}
func (u *BackupService) checkBackupConn(backup *model.BackupAccount) (bool, error) {
client, err := u.NewClient(backup)
if err != nil {
return false, err
}
fileItem := path.Join(global.CONF.System.TmpDir, "test", "1panel")
if _, err := os.Stat(path.Dir(fileItem)); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(path.Dir(fileItem), os.ModePerm); err != nil {
return false, err
}
}
file, err := os.OpenFile(fileItem, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
return false, err
}
defer file.Close()
write := bufio.NewWriter(file)
_, _ = write.WriteString("1Panel 备份账号测试文件。\n")
_, _ = write.WriteString("1Panel 備份賬號測試文件。\n")
_, _ = write.WriteString("1Panel Backs up account test files.\n")
_, _ = write.WriteString("1Panelアカウントのテストファイルをバックアップします。\n")
write.Flush()
targetPath := strings.TrimPrefix(path.Join(backup.BackupPath, "test/1panel"), "/")
return client.Upload(fileItem, targetPath)
}
func StartRefreshOneDriveToken() {
service := NewIBackupService()
oneDriveCronID, err := global.Cron.AddJob("0 3 */31 * *", service)
if err != nil {
global.LOG.Errorf("can not add OneDrive corn job: %s", err.Error())
return
}
global.OneDriveCronID = oneDriveCronID
}
func (u *BackupService) Run() {
var backupItem model.BackupAccount
_ = global.DB.Where("`type` = ?", "OneDrive").First(&backupItem)
if backupItem.ID == 0 {
return
}
global.LOG.Info("start to refresh token of OneDrive ...")
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil {
global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err)
return
}
refreshToken, err := client.RefreshToken("refresh_token", "refreshToken", varMap)
varMap["refresh_status"] = constant.StatusSuccess
varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout)
if err != nil {
varMap["refresh_status"] = constant.StatusFailed
varMap["refresh_msg"] = err.Error()
global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err)
return
}
varMap["refresh_token"] = refreshToken
varsItem, _ := json.Marshal(varMap)
_ = global.DB.Model(&model.BackupAccount{}).
Where("id = ?", backupItem.ID).
Updates(map[string]interface{}{
"vars": varsItem,
}).Error
global.LOG.Info("Successfully refreshed OneDrive token.")
}

View File

@ -15,6 +15,4 @@ type System struct {
Version string `mapstructure:"version"`
IsDemo bool `mapstructure:"is_demo"`
AppRepo string `mapstructure:"app_repo"`
OneDriveID string `mapstructure:"one_drive_id"`
OneDriveSc string `mapstructure:"one_drive_sc"`
}

View File

@ -50,13 +50,6 @@ func Run() {
global.LOG.Errorf("can not add cache corn job: %s", err.Error())
}
var backup model.BackupAccount
_ = global.DB.Where("type = ?", "OneDrive").Find(&backup).Error
if backup.ID != 0 {
service.StartRefreshOneDriveToken()
}
global.Cron.Start()
var cronJobs []model.Cronjob
if err := global.DB.Where("status = ?", constant.StatusEnable).Find(&cronJobs).Error; err != nil {
global.LOG.Errorf("start my cronjob failed, err: %v", err)

View File

@ -1,7 +1,6 @@
package hook
import (
"encoding/base64"
"encoding/json"
"os"
"path"
@ -15,25 +14,17 @@ import (
func Init() {
settingRepo := repo.NewISettingRepo()
OneDriveID, err := settingRepo.Get(settingRepo.WithByKey("OneDriveID"))
if err != nil {
global.LOG.Errorf("load onedrive info from setting failed, err: %v", err)
}
idItem, _ := base64.StdEncoding.DecodeString(OneDriveID.Value)
global.CONF.System.OneDriveID = string(idItem)
OneDriveSc, err := settingRepo.Get(settingRepo.WithByKey("OneDriveSc"))
if err != nil {
global.LOG.Errorf("load onedrive info from setting failed, err: %v", err)
}
scItem, _ := base64.StdEncoding.DecodeString(OneDriveSc.Value)
global.CONF.System.OneDriveSc = string(scItem)
if _, err := settingRepo.Get(settingRepo.WithByKey("SystemStatus")); err != nil {
_ = settingRepo.Create("SystemStatus", "Free")
}
if err := settingRepo.Update("SystemStatus", "Free"); err != nil {
global.LOG.Fatalf("init service before start failed, err: %v", err)
}
node, err := settingRepo.Get(settingRepo.WithByKey("CurrentNode"))
if err != nil {
global.LOG.Fatalf("load current node before start failed, err: %v", err)
}
global.CurrentNode = node.Value
handleCronjobStatus()
handleSnapStatus()

View File

@ -24,17 +24,11 @@ func (s *SettingRouter) InitRouter(Router *gin.RouterGroup) {
settingRouter.POST("/snapshot/rollback", baseApi.RollbackSnapshot)
settingRouter.POST("/snapshot/description/update", baseApi.UpdateSnapDescription)
settingRouter.GET("/backup/search", baseApi.ListBackup)
settingRouter.GET("/backup/onedrive", baseApi.LoadOneDriveInfo)
settingRouter.POST("/backup/operate", baseApi.OperateBackup)
settingRouter.POST("/backup/backup", baseApi.Backup)
settingRouter.POST("/backup/refresh/onedrive", baseApi.RefreshOneDriveToken)
settingRouter.POST("/backup/recover", baseApi.Recover)
settingRouter.POST("/backup/recover/byupload", baseApi.RecoverByUpload)
settingRouter.POST("/backup/search/files", baseApi.LoadFilesFromBackup)
settingRouter.POST("/backup/buckets", baseApi.ListBuckets)
settingRouter.POST("/backup", baseApi.CreateBackup)
settingRouter.POST("/backup/del", baseApi.DeleteBackup)
settingRouter.POST("/backup/update", baseApi.UpdateBackup)
settingRouter.POST("/backup/record/search", baseApi.SearchBackupRecords)
settingRouter.POST("/backup/record/search/bycronjob", baseApi.SearchBackupRecordsByCronjob)
settingRouter.POST("/backup/record/download", baseApi.DownloadRecord)

View File

@ -13,6 +13,18 @@ import (
"github.com/1Panel-dev/1Panel/agent/global"
)
func StringEncryptWithBase64(text string) (string, error) {
accessKeyItem, err := base64.StdEncoding.DecodeString(text)
if err != nil {
return "", err
}
encryptKeyItem, err := StringEncrypt(string(accessKeyItem))
if err != nil {
return "", err
}
return encryptKeyItem, nil
}
func StringEncrypt(text string) (string, error) {
if len(text) == 0 {
return "", nil

View File

@ -45,21 +45,26 @@ var restoreCmd = &cobra.Command{
tmpPath = path.Join(upgradeDir, tmpPath, "original")
fmt.Printf("(0/4) 开始从 %s 目录回滚 1Panel 服务及数据... \n", tmpPath)
if err := files.CopyFile(path.Join(tmpPath, "1panel"), "/usr/local/bin"); err != nil {
if err := files.CopyFile(path.Join(tmpPath, "1panel"), "/usr/local/bin", false); err != nil {
return err
}
fmt.Println("(1/4) 1panel 二进制回滚成功")
if err := files.CopyFile(path.Join(tmpPath, "1pctl"), "/usr/local/bin"); err != nil {
if err := files.CopyFile(path.Join(tmpPath, "1pctl"), "/usr/local/bin", false); err != nil {
return err
}
fmt.Println("(2/4) 1panel 脚本回滚成功")
if err := files.CopyFile(path.Join(tmpPath, "1panel.service"), "/etc/systemd/system"); err != nil {
if err := files.CopyFile(path.Join(tmpPath, "1panel.service"), "/etc/systemd/system", false); err != nil {
return err
}
fmt.Println("(3/4) 1panel 服务回滚成功")
checkPointOfWal()
if _, err := os.Stat(path.Join(tmpPath, "1Panel.db")); err == nil {
if err := files.CopyFile(path.Join(tmpPath, "1Panel.db"), path.Join(baseDir, "1panel/db")); err != nil {
if _, err := os.Stat(path.Join(tmpPath, "core.db")); err == nil {
if err := files.CopyFile(path.Join(tmpPath, "core.db"), path.Join(baseDir, "core/db"), false); err != nil {
return err
}
}
if _, err := os.Stat(path.Join(tmpPath, "agent.db")); err == nil {
if err := files.CopyFile(path.Join(tmpPath, "agent.db"), path.Join(baseDir, "1panel/db"), false); err != nil {
return err
}
}

View File

@ -16,7 +16,7 @@ const docTemplate = `{
"version": "1.0"
},
"host": "localhost",
"basePath": "/api/v1",
"basePath": "/api/v2",
"paths": {
"/apps/:key": {
"get": {
@ -3365,6 +3365,255 @@ const docTemplate = `{
]
}
},
"/core/backup": {
"post": {
"consumes": [
"application/json"
],
"description": "创建备份账号",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.BackupOperate"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Create backup account",
"tags": [
"Backup Account"
],
"x-panel-log": {
"BeforeFunctions": [],
"bodyKeys": [
"type"
],
"formatEN": "create backup account [type]",
"formatZH": "创建备份账号 [type]",
"paramKeys": []
}
}
},
"/core/backup/del": {
"post": {
"consumes": [
"application/json"
],
"description": "删除备份账号",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.OperateByID"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Delete backup account",
"tags": [
"Backup Account"
],
"x-panel-log": {
"BeforeFunctions": [
{
"db": "backup_accounts",
"input_column": "id",
"input_value": "id",
"isList": false,
"output_column": "type",
"output_value": "types"
}
],
"bodyKeys": [
"id"
],
"formatEN": "delete backup account [types]",
"formatZH": "删除备份账号 [types]",
"paramKeys": []
}
}
},
"/core/backup/onedrive": {
"get": {
"consumes": [
"application/json"
],
"description": "获取 OneDrive 信息",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/dto.OneDriveInfo"
}
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Load OneDrive info",
"tags": [
"Backup Account"
]
}
},
"/core/backup/refresh/onedrive": {
"post": {
"description": "刷新 OneDrive token",
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Refresh OneDrive token",
"tags": [
"Backup Account"
]
}
},
"/core/backup/search": {
"get": {
"consumes": [
"application/json"
],
"description": "获取备份账号列表",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.SearchPageWithType"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Search backup accounts with page",
"tags": [
"Backup Account"
]
},
"post": {
"consumes": [
"application/json"
],
"description": "获取 bucket 列表",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.ForBuckets"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"items": {
"type": "string"
},
"type": "array"
}
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "List buckets",
"tags": [
"Backup Account"
]
}
},
"/core/backup/update": {
"post": {
"consumes": [
"application/json"
],
"description": "更新备份账号信息",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.BackupOperate"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Update backup account",
"tags": [
"Backup Account"
],
"x-panel-log": {
"BeforeFunctions": [],
"bodyKeys": [
"type"
],
"formatEN": "update backup account [types]",
"formatZH": "更新备份账号 [types]",
"paramKeys": []
}
}
},
"/core/logs/clean": {
"post": {
"consumes": [
@ -10207,48 +10456,6 @@ const docTemplate = `{
}
}
},
"/settings/backup": {
"post": {
"consumes": [
"application/json"
],
"description": "创建备份账号",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.BackupOperate"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Create backup account",
"tags": [
"Backup Account"
],
"x-panel-log": {
"BeforeFunctions": [],
"bodyKeys": [
"type"
],
"formatEN": "create backup account [type]",
"formatZH": "创建备份账号 [type]",
"paramKeys": []
}
}
},
"/settings/backup/backup": {
"post": {
"consumes": [
@ -10293,82 +10500,6 @@ const docTemplate = `{
}
}
},
"/settings/backup/del": {
"post": {
"consumes": [
"application/json"
],
"description": "删除备份账号",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.OperateByID"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Delete backup account",
"tags": [
"Backup Account"
],
"x-panel-log": {
"BeforeFunctions": [
{
"db": "backup_accounts",
"input_column": "id",
"input_value": "id",
"isList": false,
"output_column": "type",
"output_value": "types"
}
],
"bodyKeys": [
"id"
],
"formatEN": "delete backup account [types]",
"formatZH": "删除备份账号 [types]",
"paramKeys": []
}
}
},
"/settings/backup/onedrive": {
"get": {
"consumes": [
"application/json"
],
"description": "获取 OneDrive 信息",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/dto.OneDriveInfo"
}
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Load OneDrive info",
"tags": [
"Backup Account"
]
}
},
"/settings/backup/record/del": {
"post": {
"consumes": [
@ -10619,87 +10750,6 @@ const docTemplate = `{
}
}
},
"/settings/backup/refresh/onedrive": {
"post": {
"description": "刷新 OneDrive token",
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Refresh OneDrive token",
"tags": [
"Backup Account"
]
}
},
"/settings/backup/search": {
"get": {
"description": "获取备份账号列表",
"responses": {
"200": {
"description": "OK",
"schema": {
"items": {
"$ref": "#/definitions/dto.BackupInfo"
},
"type": "array"
}
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "List backup accounts",
"tags": [
"Backup Account"
]
},
"post": {
"consumes": [
"application/json"
],
"description": "获取 bucket 列表",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.ForBuckets"
}
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"items": {
"type": "string"
},
"type": "array"
}
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "List buckets",
"tags": [
"Backup Account"
]
}
},
"/settings/backup/search/files": {
"post": {
"consumes": [
@ -10739,48 +10789,6 @@ const docTemplate = `{
]
}
},
"/settings/backup/update": {
"post": {
"consumes": [
"application/json"
],
"description": "更新备份账号信息",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/dto.BackupOperate"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Update backup account",
"tags": [
"Backup Account"
],
"x-panel-log": {
"BeforeFunctions": [],
"bodyKeys": [
"type"
],
"formatEN": "update backup account [types]",
"formatZH": "更新备份账号 [types]",
"paramKeys": []
}
}
},
"/settings/basedir": {
"get": {
"description": "获取安装根目录",
@ -13027,6 +13035,72 @@ const docTemplate = `{
]
}
},
"/websites/auths/path": {
"post": {
"consumes": [
"application/json"
],
"description": "获取路由密码访问配置",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/request.NginxAuthReq"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Get AuthBasic conf",
"tags": [
"Website"
]
}
},
"/websites/auths/path/update": {
"post": {
"consumes": [
"application/json"
],
"description": "更新路由密码访问配置",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/request.NginxPathAuthUpdate"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Get AuthBasic conf",
"tags": [
"Website"
]
}
},
"/websites/auths/update": {
"post": {
"consumes": [
@ -14114,6 +14188,57 @@ const docTemplate = `{
}
}
},
"/websites/domains/update": {
"post": {
"consumes": [
"application/json"
],
"description": "更新网站域名",
"parameters": [
{
"description": "request",
"in": "body",
"name": "request",
"required": true,
"schema": {
"$ref": "#/definitions/request.WebsiteDomainUpdate"
}
}
],
"responses": {
"200": {
"description": "OK"
}
},
"security": [
{
"ApiKeyAuth": []
}
],
"summary": "Update website domain",
"tags": [
"Website Domain"
],
"x-panel-log": {
"BeforeFunctions": [
{
"db": "website_domains",
"input_column": "id",
"input_value": "id",
"isList": false,
"output_column": "domain",
"output_value": "domain"
}
],
"bodyKeys": [
"id"
],
"formatEN": "Update domain [domain]",
"formatZH": "更新域名 [domain]",
"paramKeys": []
}
}
},
"/websites/leech": {
"post": {
"consumes": [
@ -15514,59 +15639,6 @@ const docTemplate = `{
},
"type": "object"
},
"dto.BackupInfo": {
"properties": {
"backupPath": {
"type": "string"
},
"bucket": {
"type": "string"
},
"createdAt": {
"type": "string"
},
"id": {
"type": "integer"
},
"type": {
"type": "string"
},
"vars": {
"type": "string"
}
},
"type": "object"
},
"dto.BackupOperate": {
"properties": {
"accessKey": {
"type": "string"
},
"backupPath": {
"type": "string"
},
"bucket": {
"type": "string"
},
"credential": {
"type": "string"
},
"id": {
"type": "integer"
},
"type": {
"type": "string"
},
"vars": {
"type": "string"
}
},
"required": [
"type",
"vars"
],
"type": "object"
},
"dto.BackupSearchFile": {
"properties": {
"type": {
@ -17355,28 +17427,6 @@ const docTemplate = `{
],
"type": "object"
},
"dto.ForBuckets": {
"properties": {
"accessKey": {
"type": "string"
},
"credential": {
"type": "string"
},
"type": {
"type": "string"
},
"vars": {
"type": "string"
}
},
"required": [
"credential",
"type",
"vars"
],
"type": "object"
},
"dto.ForwardRuleOperate": {
"properties": {
"rules": {
@ -18486,20 +18536,6 @@ const docTemplate = `{
"ProxyCache"
]
},
"dto.OneDriveInfo": {
"properties": {
"client_id": {
"type": "string"
},
"client_secret": {
"type": "string"
},
"redirect_uri": {
"type": "string"
}
},
"type": "object"
},
"dto.Operate": {
"properties": {
"operation": {
@ -20142,6 +20178,9 @@ const docTemplate = `{
"port": {
"type": "integer"
},
"ssl": {
"type": "boolean"
},
"updatedAt": {
"type": "string"
},
@ -21180,6 +21219,36 @@ const docTemplate = `{
],
"type": "object"
},
"request.NginxPathAuthUpdate": {
"properties": {
"name": {
"type": "string"
},
"operate": {
"type": "string"
},
"password": {
"type": "string"
},
"path": {
"type": "string"
},
"remark": {
"type": "string"
},
"username": {
"type": "string"
},
"websiteID": {
"type": "integer"
}
},
"required": [
"operate",
"websiteID"
],
"type": "object"
},
"request.NginxProxyUpdate": {
"properties": {
"content": {
@ -21872,21 +21941,24 @@ const docTemplate = `{
"dbUser": {
"type": "string"
},
"domains": {
"items": {
"$ref": "#/definitions/request.WebsiteDomain"
},
"type": "array"
},
"enableSSL": {
"type": "boolean"
},
"ftpPassword": {
"type": "string"
},
"ftpUser": {
"type": "string"
},
"otherDomains": {
"type": "string"
},
"port": {
"type": "integer"
},
"primaryDomain": {
"type": "string"
},
"proxy": {
"type": "string"
},
@ -21907,11 +21979,13 @@ const docTemplate = `{
},
"webSiteGroupID": {
"type": "integer"
},
"websiteSSLID": {
"type": "integer"
}
},
"required": [
"alias",
"primaryDomain",
"type",
"webSiteGroupID"
],
@ -22011,10 +22085,30 @@ const docTemplate = `{
],
"type": "object"
},
"request.WebsiteDomain": {
"properties": {
"domain": {
"type": "string"
},
"port": {
"type": "integer"
},
"ssl": {
"type": "boolean"
}
},
"required": [
"domain"
],
"type": "object"
},
"request.WebsiteDomainCreate": {
"properties": {
"domains": {
"type": "string"
"items": {
"$ref": "#/definitions/request.WebsiteDomain"
},
"type": "array"
},
"websiteID": {
"type": "integer"
@ -22037,6 +22131,20 @@ const docTemplate = `{
],
"type": "object"
},
"request.WebsiteDomainUpdate": {
"properties": {
"id": {
"type": "integer"
},
"ssl": {
"type": "boolean"
}
},
"required": [
"id"
],
"type": "object"
},
"request.WebsiteHTTPSOp": {
"properties": {
"SSLProtocol": {
@ -22068,6 +22176,12 @@ const docTemplate = `{
],
"type": "string"
},
"httpsPorts": {
"items": {
"type": "integer"
},
"type": "array"
},
"importType": {
"type": "string"
},
@ -23277,6 +23391,15 @@ const docTemplate = `{
},
"httpConfig": {
"type": "string"
},
"httpsPort": {
"type": "string"
},
"httpsPorts": {
"items": {
"type": "integer"
},
"type": "array"
}
},
"type": "object"
@ -23347,4 +23470,4 @@ var SwaggerInfo = &swag.Spec{
func init() {
swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
}
}

23452
cmd/server/docs/swagger.json Normal file

File diff suppressed because it is too large Load Diff

View File

@ -469,6 +469,42 @@
"formatZH": "删除容器存储卷 [names]",
"paramKeys": []
},
"/core/backup": {
"BeforeFunctions": [],
"bodyKeys": [
"type"
],
"formatEN": "create backup account [type]",
"formatZH": "创建备份账号 [type]",
"paramKeys": []
},
"/core/backup/del": {
"BeforeFunctions": [
{
"db": "backup_accounts",
"input_column": "id",
"input_value": "id",
"isList": false,
"output_column": "type",
"output_value": "types"
}
],
"bodyKeys": [
"id"
],
"formatEN": "delete backup account [types]",
"formatZH": "删除备份账号 [types]",
"paramKeys": []
},
"/core/backup/update": {
"BeforeFunctions": [],
"bodyKeys": [
"type"
],
"formatEN": "update backup account [types]",
"formatZH": "更新备份账号 [types]",
"paramKeys": []
},
"/core/logs/clean": {
"BeforeFunctions": [],
"bodyKeys": [
@ -1519,15 +1555,6 @@
"formatZH": "更新运行环境 [name]",
"paramKeys": []
},
"/settings/backup": {
"BeforeFunctions": [],
"bodyKeys": [
"type"
],
"formatEN": "create backup account [type]",
"formatZH": "创建备份账号 [type]",
"paramKeys": []
},
"/settings/backup/backup": {
"BeforeFunctions": [],
"bodyKeys": [
@ -1539,24 +1566,6 @@
"formatZH": "备份 [type] 数据 [name][detailName]",
"paramKeys": []
},
"/settings/backup/del": {
"BeforeFunctions": [
{
"db": "backup_accounts",
"input_column": "id",
"input_value": "id",
"isList": false,
"output_column": "type",
"output_value": "types"
}
],
"bodyKeys": [
"id"
],
"formatEN": "delete backup account [types]",
"formatZH": "删除备份账号 [types]",
"paramKeys": []
},
"/settings/backup/record/del": {
"BeforeFunctions": [
{
@ -1609,15 +1618,6 @@
"formatZH": "从 [file] 恢复 [type] 数据 [name][detailName]",
"paramKeys": []
},
"/settings/backup/update": {
"BeforeFunctions": [],
"bodyKeys": [
"type"
],
"formatEN": "update backup account [types]",
"formatZH": "更新备份账号 [types]",
"paramKeys": []
},
"/settings/snapshot": {
"BeforeFunctions": [],
"bodyKeys": [
@ -2238,6 +2238,24 @@
"formatZH": "删除域名 [domain]",
"paramKeys": []
},
"/websites/domains/update": {
"BeforeFunctions": [
{
"db": "website_domains",
"input_column": "id",
"input_value": "id",
"isList": false,
"output_column": "domain",
"output_value": "domain"
}
],
"bodyKeys": [
"id"
],
"formatEN": "Update domain [domain]",
"formatZH": "更新域名 [domain]",
"paramKeys": []
},
"/websites/log": {
"BeforeFunctions": [
{

View File

@ -1,8 +1,6 @@
package v2
import (
"encoding/base64"
"github.com/1Panel-dev/1Panel/core/app/api/v2/helper"
"github.com/1Panel-dev/1Panel/core/app/dto"
"github.com/1Panel-dev/1Panel/core/constant"
@ -23,22 +21,6 @@ func (b *BaseApi) CreateBackup(c *gin.Context) {
if err := helper.CheckBindAndValidate(&req, c); err != nil {
return
}
if len(req.Credential) != 0 {
credential, err := base64.StdEncoding.DecodeString(req.Credential)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.Credential = string(credential)
}
if len(req.AccessKey) != 0 {
accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.AccessKey = string(accessKey)
}
if err := backupService.Create(req); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
@ -71,22 +53,6 @@ func (b *BaseApi) ListBuckets(c *gin.Context) {
if err := helper.CheckBindAndValidate(&req, c); err != nil {
return
}
if len(req.Credential) != 0 {
credential, err := base64.StdEncoding.DecodeString(req.Credential)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.Credential = string(credential)
}
if len(req.AccessKey) != 0 {
accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.AccessKey = string(accessKey)
}
buckets, err := backupService.GetBuckets(req)
if err != nil {
@ -149,23 +115,6 @@ func (b *BaseApi) UpdateBackup(c *gin.Context) {
return
}
if len(req.Credential) != 0 {
credential, err := base64.StdEncoding.DecodeString(req.Credential)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.Credential = string(credential)
}
if len(req.AccessKey) != 0 {
accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err)
return
}
req.AccessKey = string(accessKey)
}
if err := backupService.Update(req); err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
@ -178,7 +127,7 @@ func (b *BaseApi) UpdateBackup(c *gin.Context) {
// @Description 获取备份账号列表
// @Accept json
// @Param request body dto.SearchPageWithType true "request"
// @Success 200 {array} dto.BackupList
// @Success 200
// @Security ApiKeyAuth
// @Router /core/backup/search [get]
func (b *BaseApi) SearchBackup(c *gin.Context) {

View File

@ -12,7 +12,7 @@ type IBackupRepo interface {
List(opts ...DBOption) ([]model.BackupAccount, error)
Page(limit, offset int, opts ...DBOption) (int64, []model.BackupAccount, error)
Create(backup *model.BackupAccount) error
Update(id uint, vars map[string]interface{}) error
Save(backup *model.BackupAccount) error
Delete(opts ...DBOption) error
}
@ -56,8 +56,8 @@ func (u *BackupRepo) Create(backup *model.BackupAccount) error {
return global.DB.Create(backup).Error
}
func (u *BackupRepo) Update(id uint, vars map[string]interface{}) error {
return global.DB.Model(&model.BackupAccount{}).Where("id = ?", id).Updates(vars).Error
func (u *BackupRepo) Save(backup *model.BackupAccount) error {
return global.DB.Save(backup).Error
}
func (u *BackupRepo) Delete(opts ...DBOption) error {

View File

@ -17,7 +17,9 @@ import (
"github.com/1Panel-dev/1Panel/core/global"
"github.com/1Panel-dev/1Panel/core/utils/cloud_storage"
"github.com/1Panel-dev/1Panel/core/utils/cloud_storage/client"
"github.com/1Panel-dev/1Panel/core/utils/encrypt"
fileUtils "github.com/1Panel-dev/1Panel/core/utils/files"
"github.com/1Panel-dev/1Panel/core/utils/xpack"
"github.com/jinzhu/copier"
"github.com/pkg/errors"
"github.com/robfig/cron/v3"
@ -113,6 +115,16 @@ func (u *BackupService) Create(req dto.BackupOperate) error {
if err := copier.Copy(&backup, &req); err != nil {
return errors.WithMessage(constant.ErrStructTransform, err.Error())
}
itemAccessKey, err := base64.StdEncoding.DecodeString(backup.AccessKey)
if err != nil {
return err
}
backup.AccessKey = string(itemAccessKey)
itemCredential, err := base64.StdEncoding.DecodeString(backup.Credential)
if err != nil {
return err
}
backup.Credential = string(itemCredential)
if req.Type == constant.OneDrive {
if err := u.loadAccessToken(&backup); err != nil {
@ -129,26 +141,49 @@ func (u *BackupService) Create(req dto.BackupOperate) error {
return err
}
}
if err := xpack.SyncBackupOperation("add", []model.BackupAccount{backup}); err != nil {
return err
}
backup.AccessKey, err = encrypt.StringEncrypt(backup.AccessKey)
if err != nil {
return err
}
backup.Credential, err = encrypt.StringEncrypt(backup.Credential)
if err != nil {
return err
}
if err := backupRepo.Create(&backup); err != nil {
return err
}
return nil
}
func (u *BackupService) GetBuckets(backupDto dto.ForBuckets) ([]interface{}, error) {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backupDto.Vars), &varMap); err != nil {
func (u *BackupService) GetBuckets(req dto.ForBuckets) ([]interface{}, error) {
itemAccessKey, err := base64.StdEncoding.DecodeString(req.AccessKey)
if err != nil {
return nil, err
}
switch backupDto.Type {
case constant.Sftp, constant.WebDAV:
varMap["username"] = backupDto.AccessKey
varMap["password"] = backupDto.Credential
case constant.OSS, constant.S3, constant.MinIo, constant.Cos, constant.Kodo:
varMap["accessKey"] = backupDto.AccessKey
varMap["secretKey"] = backupDto.Credential
req.AccessKey = string(itemAccessKey)
itemCredential, err := base64.StdEncoding.DecodeString(req.Credential)
if err != nil {
return nil, err
}
client, err := cloud_storage.NewCloudStorageClient(backupDto.Type, varMap)
req.Credential = string(itemCredential)
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(req.Vars), &varMap); err != nil {
return nil, err
}
switch req.Type {
case constant.Sftp, constant.WebDAV:
varMap["username"] = req.AccessKey
varMap["password"] = req.Credential
case constant.OSS, constant.S3, constant.MinIo, constant.Cos, constant.Kodo:
varMap["accessKey"] = req.AccessKey
varMap["secretKey"] = req.Credential
}
client, err := cloud_storage.NewCloudStorageClient(req.Type, varMap)
if err != nil {
return nil, err
}
@ -169,70 +204,80 @@ func (u *BackupService) Delete(id uint) error {
if backup.Type == constant.OneDrive {
global.Cron.Remove(cron.EntryID(backup.EntryID))
}
if err := xpack.SyncBackupOperation("remove", []model.BackupAccount{backup}); err != nil {
return err
}
return backupRepo.Delete(commonRepo.WithByID(id))
}
func (u *BackupService) Update(req dto.BackupOperate) error {
backup, err := backupRepo.Get(commonRepo.WithByID(req.ID))
if err != nil {
backup, _ := backupRepo.Get(commonRepo.WithByID(req.ID))
if backup.ID == 0 {
return constant.ErrRecordNotFound
}
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(req.Vars), &varMap); err != nil {
return err
var newBackup model.BackupAccount
if err := copier.Copy(&newBackup, &req); err != nil {
return errors.WithMessage(constant.ErrStructTransform, err.Error())
}
oldVars := backup.Vars
oldDir, err := loadLocalDir()
itemAccessKey, err := base64.StdEncoding.DecodeString(newBackup.AccessKey)
if err != nil {
return err
}
upMap := make(map[string]interface{})
upMap["bucket"] = req.Bucket
upMap["access_key"] = req.AccessKey
upMap["credential"] = req.Credential
upMap["backup_path"] = req.BackupPath
upMap["vars"] = req.Vars
backup.Bucket = req.Bucket
backup.Vars = req.Vars
backup.Credential = req.Credential
backup.AccessKey = req.AccessKey
backup.BackupPath = req.BackupPath
newBackup.AccessKey = string(itemAccessKey)
itemCredential, err := base64.StdEncoding.DecodeString(newBackup.Credential)
if err != nil {
return err
}
newBackup.Credential = string(itemCredential)
if backup.Type == constant.Local {
if newBackup.Vars != backup.Vars {
oldPath, err := loadLocalDirByStr(backup.Vars)
if err != nil {
return err
}
newPath, err := loadLocalDirByStr(newBackup.Vars)
if err != nil {
return err
}
if strings.HasSuffix(newPath, "/") && newPath != "/" {
newPath = newPath[:strings.LastIndex(newPath, "/")]
}
if err := copyDir(oldPath, newPath); err != nil {
return err
}
global.CONF.System.BackupDir = newPath
}
}
if req.Type == constant.OneDrive {
if newBackup.Type == constant.OneDrive {
global.Cron.Remove(cron.EntryID(backup.EntryID))
if err := u.loadAccessToken(&backup); err != nil {
return err
}
upMap["credential"] = backup.Credential
upMap["vars"] = backup.Vars
if err := StartRefreshOneDriveToken(&backup); err != nil {
return err
}
upMap["entry_id"] = backup.EntryID
}
if backup.Type != "LOCAL" {
isOk, err := u.checkBackupConn(&backup)
isOk, err := u.checkBackupConn(&newBackup)
if err != nil || !isOk {
return buserr.WithMap("ErrBackupCheck", map[string]interface{}{"err": err.Error()}, err)
}
}
if err := backupRepo.Update(req.ID, upMap); err != nil {
if err := xpack.SyncBackupOperation("update", []model.BackupAccount{newBackup}); err != nil {
return err
}
if backup.Type == "LOCAL" {
if dir, ok := varMap["dir"]; ok {
if dirStr, isStr := dir.(string); isStr {
if strings.HasSuffix(dirStr, "/") && dirStr != "/" {
dirStr = dirStr[:strings.LastIndex(dirStr, "/")]
}
if err := copyDir(oldDir, dirStr); err != nil {
_ = backupRepo.Update(req.ID, map[string]interface{}{"vars": oldVars})
return err
}
}
}
newBackup.AccessKey, err = encrypt.StringEncrypt(newBackup.AccessKey)
if err != nil {
return err
}
newBackup.Credential, err = encrypt.StringEncrypt(newBackup.Credential)
if err != nil {
return err
}
newBackup.ID = backup.ID
if err := backupRepo.Save(&newBackup); err != nil {
return err
}
return nil
}
@ -281,13 +326,9 @@ func (u *BackupService) loadAccessToken(backup *model.BackupAccount) error {
return nil
}
func loadLocalDir() (string, error) {
backup, err := backupRepo.Get(commonRepo.WithByType("LOCAL"))
if err != nil {
return "", err
}
func loadLocalDirByStr(vars string) (string, error) {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
if err := json.Unmarshal([]byte(vars), &varMap); err != nil {
return "", err
}
if _, ok := varMap["dir"]; !ok {
@ -300,7 +341,6 @@ func loadLocalDir() (string, error) {
return "", fmt.Errorf("mkdir %s failed, err: %v", baseDir, err)
}
}
return baseDir, nil
}
return "", fmt.Errorf("error type dir: %T", varMap["dir"])
}
@ -325,7 +365,7 @@ func copyDir(src, dst string) error {
global.LOG.Errorf("copy dir %s to %s failed, err: %v", srcPath, dstPath, err)
}
} else {
if err := fileUtils.CopyFile(srcPath, dst); err != nil {
if err := fileUtils.CopyFile(srcPath, dst, false); err != nil {
global.LOG.Errorf("copy file %s to %s failed, err: %v", srcPath, dstPath, err)
}
}

View File

@ -127,13 +127,13 @@ func (u *UpgradeService) Upgrade(req dto.Upgrade) error {
}
global.LOG.Info("backup original data successful, now start to upgrade!")
if err := files.CopyFile(path.Join(tmpDir, "1panel"), "/usr/local/bin"); err != nil {
if err := files.CopyFile(path.Join(tmpDir, "1panel"), "/usr/local/bin", false); err != nil {
global.LOG.Errorf("upgrade 1panel failed, err: %v", err)
u.handleRollback(originalDir, 1)
return
}
if err := files.CopyFile(path.Join(tmpDir, "1pctl"), "/usr/local/bin"); err != nil {
if err := files.CopyFile(path.Join(tmpDir, "1pctl"), "/usr/local/bin", false); err != nil {
global.LOG.Errorf("upgrade 1pctl failed, err: %v", err)
u.handleRollback(originalDir, 2)
return
@ -144,7 +144,7 @@ func (u *UpgradeService) Upgrade(req dto.Upgrade) error {
return
}
if err := files.CopyFile(path.Join(tmpDir, "1panel.service"), "/etc/systemd/system"); err != nil {
if err := files.CopyFile(path.Join(tmpDir, "1panel.service"), "/etc/systemd/system", false); err != nil {
global.LOG.Errorf("upgrade 1panel.service failed, err: %v", err)
u.handleRollback(originalDir, 3)
return
@ -161,13 +161,13 @@ func (u *UpgradeService) Upgrade(req dto.Upgrade) error {
}
func (u *UpgradeService) handleBackup(originalDir string) error {
if err := files.CopyFile("/usr/local/bin/1panel", originalDir); err != nil {
if err := files.CopyFile("/usr/local/bin/1panel", originalDir, false); err != nil {
return err
}
if err := files.CopyFile("/usr/local/bin/1pctl", originalDir); err != nil {
if err := files.CopyFile("/usr/local/bin/1pctl", originalDir, false); err != nil {
return err
}
if err := files.CopyFile("/etc/systemd/system/1panel.service", originalDir); err != nil {
if err := files.CopyFile("/etc/systemd/system/1panel.service", originalDir, false); err != nil {
return err
}
checkPointOfWal()
@ -183,7 +183,7 @@ func (u *UpgradeService) handleRollback(originalDir string, errStep int) {
checkPointOfWal()
dbPath := path.Join(global.CONF.System.BaseDir, "1panel/db")
if _, err := os.Stat(path.Join(originalDir, "1Panel.db")); err == nil {
if err := files.CopyFile(path.Join(originalDir, "1Panel.db"), dbPath); err != nil {
if err := files.CopyFile(path.Join(originalDir, "1Panel.db"), dbPath, false); err != nil {
global.LOG.Errorf("rollback 1panel db failed, err: %v", err)
}
}
@ -192,19 +192,19 @@ func (u *UpgradeService) handleRollback(originalDir string, errStep int) {
global.LOG.Errorf("rollback 1panel db failed, err: %v", err)
}
}
if err := files.CopyFile(path.Join(originalDir, "1panel"), "/usr/local/bin"); err != nil {
if err := files.CopyFile(path.Join(originalDir, "1panel"), "/usr/local/bin", false); err != nil {
global.LOG.Errorf("rollback 1pctl failed, err: %v", err)
}
if errStep == 1 {
return
}
if err := files.CopyFile(path.Join(originalDir, "1pctl"), "/usr/local/bin"); err != nil {
if err := files.CopyFile(path.Join(originalDir, "1pctl"), "/usr/local/bin", false); err != nil {
global.LOG.Errorf("rollback 1panel failed, err: %v", err)
}
if errStep == 2 {
return
}
if err := files.CopyFile(path.Join(originalDir, "1panel.service"), "/etc/systemd/system"); err != nil {
if err := files.CopyFile(path.Join(originalDir, "1panel.service"), "/etc/systemd/system", false); err != nil {
global.LOG.Errorf("rollback 1panel failed, err: %v", err)
}
}

View File

@ -8,6 +8,7 @@ type System struct {
DbCoreFile string `mapstructure:"db_core_file"`
EncryptKey string `mapstructure:"encrypt_key"`
BaseDir string `mapstructure:"base_dir"`
BackupDir string `mapstructure:"backup_dir"`
Mode string `mapstructure:"mode"`
RepoUrl string `mapstructure:"repo_url"`
Version string `mapstructure:"version"`

View File

@ -1,8 +1,11 @@
package hook
import (
"encoding/json"
"os"
"strings"
"github.com/1Panel-dev/1Panel/core/app/model"
"github.com/1Panel-dev/1Panel/core/app/repo"
"github.com/1Panel-dev/1Panel/core/global"
"github.com/1Panel-dev/1Panel/core/utils/cmd"
@ -41,6 +44,7 @@ func Init() {
}
handleUserInfo(global.CONF.System.ChangeUserInfo, settingRepo)
loadLocalDir()
}
func handleUserInfo(tags string, settingRepo repo.ISettingRepo) {
@ -80,3 +84,32 @@ func handleUserInfo(tags string, settingRepo repo.ISettingRepo) {
sudo := cmd.SudoHandleCmd()
_, _ = cmd.Execf("%s sed -i '/CHANGE_USER_INFO=%v/d' /usr/local/bin/1pctl", sudo, global.CONF.System.ChangeUserInfo)
}
func loadLocalDir() {
var backup model.BackupAccount
_ = global.DB.Where("type = ?", "LOCAL").First(&backup).Error
if backup.ID == 0 {
global.LOG.Errorf("no such backup account `%s` in db", "LOCAL")
return
}
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
global.LOG.Errorf("json unmarshal backup.Vars: %v failed, err: %v", backup.Vars, err)
return
}
if _, ok := varMap["dir"]; !ok {
global.LOG.Error("load local backup dir failed")
return
}
baseDir, ok := varMap["dir"].(string)
if ok {
if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(baseDir, os.ModePerm); err != nil {
global.LOG.Errorf("mkdir %s failed, err: %v", baseDir, err)
return
}
}
}
global.CONF.System.BackupDir = baseDir
global.LOG.Errorf("error type dir: %T", varMap["dir"])
}

View File

@ -33,7 +33,7 @@ func (c localClient) Upload(src, target string) (bool, error) {
}
}
if err := files.CopyFile(src, targetFilePath); err != nil {
if err := files.CopyFile(src, targetFilePath, false); err != nil {
return false, fmt.Errorf("cp file failed, err: %v", err)
}
return true, nil

View File

@ -5,6 +5,7 @@ import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base32"
"encoding/base64"
"fmt"
"io"
@ -13,6 +14,26 @@ import (
"github.com/1Panel-dev/1Panel/core/global"
)
func StringEncryptWithBase64(text string) (string, error) {
base64Item, err := base64.StdEncoding.DecodeString(text)
if err != nil {
return "", err
}
encryptItem, err := StringEncrypt(string(base64Item))
if err != nil {
return "", err
}
return encryptItem, nil
}
func StringDecryptWithBase64(text string) (string, error) {
decryptItem, err := StringDecrypt(text)
if err != nil {
return "", err
}
return base32.StdEncoding.EncodeToString([]byte(decryptItem)), nil
}
func StringEncrypt(text string) (string, error) {
if len(text) == 0 {
return "", nil

View File

@ -17,14 +17,14 @@ import (
httpUtil "github.com/1Panel-dev/1Panel/core/utils/http"
)
func CopyFile(src, dst string) error {
func CopyFile(src, dst string, withName bool) error {
source, err := os.Open(src)
if err != nil {
return err
}
defer source.Close()
if path.Base(src) != path.Base(dst) {
if path.Base(src) != path.Base(dst) && !withName {
dst = path.Join(dst, path.Base(src))
}
if _, err := os.Stat(path.Dir(dst)); err != nil {

View File

@ -3,9 +3,16 @@
package xpack
import (
"github.com/1Panel-dev/1Panel/core/app/model"
"github.com/gin-gonic/gin"
)
func Proxy(c *gin.Context, currentNode string) error {
return nil
}
type Node struct{}
func SyncBackupOperation(operate string, accounts []model.BackupAccount) error {
return nil
}