diff --git a/agent/app/model/backup.go b/agent/app/model/backup.go index 50690dcaa..61c23af43 100644 --- a/agent/app/model/backup.go +++ b/agent/app/model/backup.go @@ -2,6 +2,7 @@ package model type BackupAccount struct { BaseModel + Name string `gorm:"type:varchar(64);unique;not null" json:"name"` Type string `gorm:"type:varchar(64);unique;not null" json:"type"` Bucket string `gorm:"type:varchar(256)" json:"bucket"` AccessKey string `gorm:"type:varchar(256)" json:"accessKey"` diff --git a/agent/cmd/server/main.go b/agent/cmd/server/main.go index cb3524b64..f8f079f67 100644 --- a/agent/cmd/server/main.go +++ b/agent/cmd/server/main.go @@ -13,7 +13,7 @@ import ( // @license.name Apache 2.0 // @license.url http://www.apache.org/licenses/LICENSE-2.0.html // @host localhost -// @BasePath /api/v1 +// @BasePath /api/v2 func main() { server.Start() } diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index bacc40c47..a029c0e2e 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -178,13 +178,6 @@ var InitSetting = &gormigrate.Migration{ return err } - if err := tx.Create(&model.Setting{Key: "OneDriveID", Value: "MDEwOTM1YTktMWFhOS00ODU0LWExZGMtNmU0NWZlNjI4YzZi"}).Error; err != nil { - return err - } - if err := tx.Create(&model.Setting{Key: "OneDriveSc", Value: "akpuOFF+YkNXOU1OLWRzS1ZSRDdOcG1LT2ZRM0RLNmdvS1RkVWNGRA=="}).Error; err != nil { - return err - } - if err := tx.Create(&model.Setting{Key: "FileRecycleBin", Value: "enable"}).Error; err != nil { return err } diff --git a/agent/init/router/router.go b/agent/init/router/router.go index e131e30f1..58bfea868 100644 --- a/agent/init/router/router.go +++ b/agent/init/router/router.go @@ -23,7 +23,7 @@ func Routers() *gin.Engine { c.JSON(200, "ok") }) PublicGroup.Use(gzip.Gzip(gzip.DefaultCompression)) - PublicGroup.Static("/api/v1/images", "./uploads") + PublicGroup.Static("/api/v2/images", "./uploads") } PrivateGroup := Router.Group("/api/v2") if global.CurrentNode != "127.0.0.1" { diff --git a/agent/middleware/demo_handle.go b/agent/middleware/demo_handle.go index 1843406ba..401b87e74 100644 --- a/agent/middleware/demo_handle.go +++ b/agent/middleware/demo_handle.go @@ -11,30 +11,30 @@ import ( ) var whiteUrlList = map[string]struct{}{ - "/api/v1/auth/login": {}, - "/api/v1/websites/config": {}, - "/api/v1/websites/waf/config": {}, - "/api/v1/files/loadfile": {}, - "/api/v1/files/size": {}, - "/api/v1/logs/operation": {}, - "/api/v1/logs/login": {}, - "/api/v1/auth/logout": {}, + "/api/v2/auth/login": {}, + "/api/v2/websites/config": {}, + "/api/v2/websites/waf/config": {}, + "/api/v2/files/loadfile": {}, + "/api/v2/files/size": {}, + "/api/v2/logs/operation": {}, + "/api/v2/logs/login": {}, + "/api/v2/auth/logout": {}, - "/api/v1/apps/installed/loadport": {}, - "/api/v1/apps/installed/check": {}, - "/api/v1/apps/installed/conninfo": {}, - "/api/v1/databases/load/file": {}, - "/api/v1/databases/variables": {}, - "/api/v1/databases/status": {}, - "/api/v1/databases/baseinfo": {}, + "/api/v2/apps/installed/loadport": {}, + "/api/v2/apps/installed/check": {}, + "/api/v2/apps/installed/conninfo": {}, + "/api/v2/databases/load/file": {}, + "/api/v2/databases/variables": {}, + "/api/v2/databases/status": {}, + "/api/v2/databases/baseinfo": {}, - "/api/v1/waf/attack/stat": {}, - "/api/v1/waf/config/website": {}, + "/api/v2/waf/attack/stat": {}, + "/api/v2/waf/config/website": {}, - "/api/v1/monitor/stat": {}, - "/api/v1/monitor/visitors": {}, - "/api/v1/monitor/visitors/loc": {}, - "/api/v1/monitor/qps": {}, + "/api/v2/monitor/stat": {}, + "/api/v2/monitor/visitors": {}, + "/api/v2/monitor/visitors/loc": {}, + "/api/v2/monitor/qps": {}, } func DemoHandle() gin.HandlerFunc { diff --git a/core/app/api/v2/backup.go b/core/app/api/v2/backup.go new file mode 100644 index 000000000..94778f008 --- /dev/null +++ b/core/app/api/v2/backup.go @@ -0,0 +1,200 @@ +package v2 + +import ( + "encoding/base64" + + "github.com/1Panel-dev/1Panel/core/app/api/v2/helper" + "github.com/1Panel-dev/1Panel/core/app/dto" + "github.com/1Panel-dev/1Panel/core/constant" + "github.com/gin-gonic/gin" +) + +// @Tags Backup Account +// @Summary Create backup account +// @Description 创建备份账号 +// @Accept json +// @Param request body dto.BackupOperate true "request" +// @Success 200 +// @Security ApiKeyAuth +// @Router /core/backup [post] +// @x-panel-log {"bodyKeys":["type"],"paramKeys":[],"BeforeFunctions":[],"formatZH":"创建备份账号 [type]","formatEN":"create backup account [type]"} +func (b *BaseApi) CreateBackup(c *gin.Context) { + var req dto.BackupOperate + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + if len(req.Credential) != 0 { + credential, err := base64.StdEncoding.DecodeString(req.Credential) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) + return + } + req.Credential = string(credential) + } + if len(req.AccessKey) != 0 { + accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) + return + } + req.AccessKey = string(accessKey) + } + + if err := backupService.Create(req); err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, nil) +} + +// @Tags Backup Account +// @Summary Refresh OneDrive token +// @Description 刷新 OneDrive token +// @Success 200 +// @Security ApiKeyAuth +// @Router /core/backup/refresh/onedrive [post] +func (b *BaseApi) RefreshOneDriveToken(c *gin.Context) { + backupService.Run() + helper.SuccessWithData(c, nil) +} + +// @Tags Backup Account +// @Summary List buckets +// @Description 获取 bucket 列表 +// @Accept json +// @Param request body dto.ForBuckets true "request" +// @Success 200 {array} string +// @Security ApiKeyAuth +// @Router /core/backup/search [post] +func (b *BaseApi) ListBuckets(c *gin.Context) { + var req dto.ForBuckets + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + if len(req.Credential) != 0 { + credential, err := base64.StdEncoding.DecodeString(req.Credential) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) + return + } + req.Credential = string(credential) + } + if len(req.AccessKey) != 0 { + accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) + return + } + req.AccessKey = string(accessKey) + } + + buckets, err := backupService.GetBuckets(req) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, buckets) +} + +// @Tags Backup Account +// @Summary Load OneDrive info +// @Description 获取 OneDrive 信息 +// @Accept json +// @Success 200 {object} dto.OneDriveInfo +// @Security ApiKeyAuth +// @Router /core/backup/onedrive [get] +func (b *BaseApi) LoadOneDriveInfo(c *gin.Context) { + data, err := backupService.LoadOneDriveInfo() + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, data) +} + +// @Tags Backup Account +// @Summary Delete backup account +// @Description 删除备份账号 +// @Accept json +// @Param request body dto.OperateByID true "request" +// @Success 200 +// @Security ApiKeyAuth +// @Router /core/backup/del [post] +// @x-panel-log {"bodyKeys":["id"],"paramKeys":[],"BeforeFunctions":[{"input_column":"id","input_value":"id","isList":false,"db":"backup_accounts","output_column":"type","output_value":"types"}],"formatZH":"删除备份账号 [types]","formatEN":"delete backup account [types]"} +func (b *BaseApi) DeleteBackup(c *gin.Context) { + var req dto.OperateByID + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + + if err := backupService.Delete(req.ID); err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, nil) +} + +// @Tags Backup Account +// @Summary Update backup account +// @Description 更新备份账号信息 +// @Accept json +// @Param request body dto.BackupOperate true "request" +// @Success 200 +// @Security ApiKeyAuth +// @Router /core/backup/update [post] +// @x-panel-log {"bodyKeys":["type"],"paramKeys":[],"BeforeFunctions":[],"formatZH":"更新备份账号 [types]","formatEN":"update backup account [types]"} +func (b *BaseApi) UpdateBackup(c *gin.Context) { + var req dto.BackupOperate + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + + if len(req.Credential) != 0 { + credential, err := base64.StdEncoding.DecodeString(req.Credential) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) + return + } + req.Credential = string(credential) + } + if len(req.AccessKey) != 0 { + accessKey, err := base64.StdEncoding.DecodeString(req.AccessKey) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) + return + } + req.AccessKey = string(accessKey) + } + + if err := backupService.Update(req); err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, nil) +} + +// @Tags Backup Account +// @Summary Search backup accounts with page +// @Description 获取备份账号列表 +// @Accept json +// @Param request body dto.SearchPageWithType true "request" +// @Success 200 {array} dto.BackupList +// @Security ApiKeyAuth +// @Router /core/backup/search [get] +func (b *BaseApi) SearchBackup(c *gin.Context) { + var req dto.SearchPageWithType + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + + total, list, err := backupService.SearchWithPage(req) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + + helper.SuccessWithData(c, dto.PageResult{ + Items: list, + Total: total, + }) +} diff --git a/core/app/api/v2/entry.go b/core/app/api/v2/entry.go index 3cd4f0dcc..b83ba1774 100644 --- a/core/app/api/v2/entry.go +++ b/core/app/api/v2/entry.go @@ -10,6 +10,7 @@ var ApiGroupApp = new(ApiGroup) var ( authService = service.NewIAuthService() + backupService = service.NewIBackupService() settingService = service.NewISettingService() logService = service.NewILogService() upgradeService = service.NewIUpgradeService() diff --git a/core/app/dto/backup.go b/core/app/dto/backup.go new file mode 100644 index 000000000..5be3f3b1d --- /dev/null +++ b/core/app/dto/backup.go @@ -0,0 +1,43 @@ +package dto + +import "time" + +type BackupOperate struct { + ID uint `json:"id"` + Name string `json:"name"` + Type string `json:"type" validate:"required"` + Bucket string `json:"bucket"` + AccessKey string `json:"accessKey"` + Credential string `json:"credential"` + BackupPath string `json:"backupPath"` + Vars string `json:"vars" validate:"required"` + + RememberAuth bool `json:"rememberAuth"` +} + +type BackupInfo struct { + ID uint `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + Bucket string `json:"bucket"` + AccessKey string `json:"accessKey"` + Credential string `json:"credential"` + BackupPath string `json:"backupPath"` + Vars string `json:"vars"` + CreatedAt time.Time `json:"createdAt"` + + RememberAuth bool `json:"rememberAuth"` +} + +type OneDriveInfo struct { + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + RedirectUri string `json:"redirect_uri"` +} + +type ForBuckets struct { + Type string `json:"type" validate:"required"` + AccessKey string `json:"accessKey"` + Credential string `json:"credential" validate:"required"` + Vars string `json:"vars" validate:"required"` +} diff --git a/core/app/dto/common.go b/core/app/dto/common.go index cca92a13d..789a01252 100644 --- a/core/app/dto/common.go +++ b/core/app/dto/common.go @@ -5,6 +5,12 @@ type SearchWithPage struct { Info string `json:"info"` } +type SearchPageWithType struct { + PageInfo + Type string `json:"type"` + Info string `json:"info"` +} + type PageInfo struct { Page int `json:"page" validate:"required,number"` PageSize int `json:"pageSize" validate:"required,number"` @@ -24,3 +30,7 @@ type Response struct { type Options struct { Option string `json:"option"` } + +type OperateByID struct { + ID uint `json:"id"` +} diff --git a/core/app/model/backup.go b/core/app/model/backup.go new file mode 100644 index 000000000..2eb37a0aa --- /dev/null +++ b/core/app/model/backup.go @@ -0,0 +1,16 @@ +package model + +type BackupAccount struct { + BaseModel + Name string `gorm:"type:varchar(64);unique;not null" json:"name"` + Type string `gorm:"type:varchar(64);unique;not null" json:"type"` + Bucket string `gorm:"type:varchar(256)" json:"bucket"` + AccessKey string `gorm:"type:varchar(256)" json:"accessKey"` + Credential string `gorm:"type:varchar(256)" json:"credential"` + BackupPath string `gorm:"type:varchar(256)" json:"backupPath"` + Vars string `gorm:"type:longText" json:"vars"` + + RememberAuth bool `gorm:"type:varchar(64)" json:"rememberAuth"` + InUsed bool `gorm:"type:varchar(64)" json:"inUsed"` + EntryID uint `gorm:"type:varchar(64)" json:"entryID"` +} diff --git a/core/app/repo/backup.go b/core/app/repo/backup.go new file mode 100644 index 000000000..a336dcdd0 --- /dev/null +++ b/core/app/repo/backup.go @@ -0,0 +1,69 @@ +package repo + +import ( + "github.com/1Panel-dev/1Panel/core/app/model" + "github.com/1Panel-dev/1Panel/core/global" +) + +type BackupRepo struct{} + +type IBackupRepo interface { + Get(opts ...DBOption) (model.BackupAccount, error) + List(opts ...DBOption) ([]model.BackupAccount, error) + Page(limit, offset int, opts ...DBOption) (int64, []model.BackupAccount, error) + Create(backup *model.BackupAccount) error + Update(id uint, vars map[string]interface{}) error + Delete(opts ...DBOption) error +} + +func NewIBackupRepo() IBackupRepo { + return &BackupRepo{} +} + +func (u *BackupRepo) Get(opts ...DBOption) (model.BackupAccount, error) { + var backup model.BackupAccount + db := global.DB + for _, opt := range opts { + db = opt(db) + } + err := db.First(&backup).Error + return backup, err +} + +func (u *BackupRepo) Page(page, size int, opts ...DBOption) (int64, []model.BackupAccount, error) { + var ops []model.BackupAccount + db := global.DB.Model(&model.BackupAccount{}) + for _, opt := range opts { + db = opt(db) + } + count := int64(0) + db = db.Count(&count) + err := db.Limit(size).Offset(size * (page - 1)).Find(&ops).Error + return count, ops, err +} + +func (u *BackupRepo) List(opts ...DBOption) ([]model.BackupAccount, error) { + var ops []model.BackupAccount + db := global.DB.Model(&model.BackupAccount{}) + for _, opt := range opts { + db = opt(db) + } + err := db.Find(&ops).Error + return ops, err +} + +func (u *BackupRepo) Create(backup *model.BackupAccount) error { + return global.DB.Create(backup).Error +} + +func (u *BackupRepo) Update(id uint, vars map[string]interface{}) error { + return global.DB.Model(&model.BackupAccount{}).Where("id = ?", id).Updates(vars).Error +} + +func (u *BackupRepo) Delete(opts ...DBOption) error { + db := global.DB + for _, opt := range opts { + db = opt(db) + } + return db.Delete(&model.BackupAccount{}).Error +} diff --git a/core/app/repo/common.go b/core/app/repo/common.go index 1d054b053..6c5331a1d 100644 --- a/core/app/repo/common.go +++ b/core/app/repo/common.go @@ -8,6 +8,8 @@ type DBOption func(*gorm.DB) *gorm.DB type ICommonRepo interface { WithByID(id uint) DBOption + WithByName(name string) DBOption + WithByType(ty string) DBOption WithOrderBy(orderStr string) DBOption } @@ -22,6 +24,22 @@ func (c *CommonRepo) WithByID(id uint) DBOption { return g.Where("id = ?", id) } } +func (c *CommonRepo) WithByName(name string) DBOption { + return func(g *gorm.DB) *gorm.DB { + if len(name) == 0 { + return g + } + return g.Where("`name` = ?", name) + } +} +func (c *CommonRepo) WithByType(ty string) DBOption { + return func(g *gorm.DB) *gorm.DB { + if len(ty) == 0 { + return g + } + return g.Where("`type` = ?", ty) + } +} func (c *CommonRepo) WithOrderBy(orderStr string) DBOption { return func(g *gorm.DB) *gorm.DB { return g.Order(orderStr) diff --git a/core/app/service/backup.go b/core/app/service/backup.go new file mode 100644 index 000000000..2c8ae2fce --- /dev/null +++ b/core/app/service/backup.go @@ -0,0 +1,405 @@ +package service + +import ( + "bufio" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path" + "strings" + "time" + + "github.com/1Panel-dev/1Panel/core/app/dto" + "github.com/1Panel-dev/1Panel/core/app/model" + "github.com/1Panel-dev/1Panel/core/buserr" + "github.com/1Panel-dev/1Panel/core/constant" + "github.com/1Panel-dev/1Panel/core/global" + "github.com/1Panel-dev/1Panel/core/utils/cloud_storage" + "github.com/1Panel-dev/1Panel/core/utils/cloud_storage/client" + fileUtils "github.com/1Panel-dev/1Panel/core/utils/files" + "github.com/jinzhu/copier" + "github.com/pkg/errors" + "github.com/robfig/cron/v3" +) + +type BackupService struct{} + +type IBackupService interface { + SearchWithPage(search dto.SearchPageWithType) (int64, interface{}, error) + LoadOneDriveInfo() (dto.OneDriveInfo, error) + Create(backupDto dto.BackupOperate) error + GetBuckets(backupDto dto.ForBuckets) ([]interface{}, error) + Update(req dto.BackupOperate) error + Delete(id uint) error + NewClient(backup *model.BackupAccount) (cloud_storage.CloudStorageClient, error) + + Run() +} + +func NewIBackupService() IBackupService { + return &BackupService{} +} + +func (u *BackupService) SearchWithPage(req dto.SearchPageWithType) (int64, interface{}, error) { + count, accounts, err := backupRepo.Page( + req.Page, + req.PageSize, + commonRepo.WithByType(req.Type), + commonRepo.WithByName(req.Info), + commonRepo.WithOrderBy("created_at desc"), + ) + if err != nil { + return 0, nil, err + } + var data []dto.BackupInfo + for _, account := range accounts { + var item dto.BackupInfo + if err := copier.Copy(&item, &account); err != nil { + global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err) + } + if !item.RememberAuth { + item.AccessKey = "" + item.Credential = "" + } else { + item.AccessKey = base64.StdEncoding.EncodeToString([]byte(item.AccessKey)) + item.Credential = base64.StdEncoding.EncodeToString([]byte(item.Credential)) + } + + if account.Type == constant.OneDrive { + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(item.Vars), &varMap); err != nil { + continue + } + delete(varMap, "refresh_token") + itemVars, _ := json.Marshal(varMap) + item.Vars = string(itemVars) + } + data = append(data, item) + } + return count, data, nil +} + +func (u *BackupService) LoadOneDriveInfo() (dto.OneDriveInfo, error) { + var data dto.OneDriveInfo + data.RedirectUri = constant.OneDriveRedirectURI + clientID, err := settingRepo.Get(settingRepo.WithByKey("OneDriveID")) + if err != nil { + return data, err + } + idItem, err := base64.StdEncoding.DecodeString(clientID.Value) + if err != nil { + return data, err + } + data.ClientID = string(idItem) + clientSecret, err := settingRepo.Get(settingRepo.WithByKey("OneDriveSc")) + if err != nil { + return data, err + } + secretItem, err := base64.StdEncoding.DecodeString(clientSecret.Value) + if err != nil { + return data, err + } + data.ClientSecret = string(secretItem) + + return data, err +} + +func (u *BackupService) Create(req dto.BackupOperate) error { + backup, _ := backupRepo.Get(commonRepo.WithByName(req.Name)) + if backup.ID != 0 { + return constant.ErrRecordExist + } + if err := copier.Copy(&backup, &req); err != nil { + return errors.WithMessage(constant.ErrStructTransform, err.Error()) + } + + if req.Type == constant.OneDrive { + if err := u.loadAccessToken(&backup); err != nil { + return err + } + } + if req.Type != "LOCAL" { + if _, err := u.checkBackupConn(&backup); err != nil { + return buserr.WithMap("ErrBackupCheck", map[string]interface{}{"err": err.Error()}, err) + } + } + if backup.Type == constant.OneDrive { + if err := StartRefreshOneDriveToken(&backup); err != nil { + return err + } + } + if err := backupRepo.Create(&backup); err != nil { + return err + } + return nil +} + +func (u *BackupService) GetBuckets(backupDto dto.ForBuckets) ([]interface{}, error) { + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(backupDto.Vars), &varMap); err != nil { + return nil, err + } + switch backupDto.Type { + case constant.Sftp, constant.WebDAV: + varMap["username"] = backupDto.AccessKey + varMap["password"] = backupDto.Credential + case constant.OSS, constant.S3, constant.MinIo, constant.Cos, constant.Kodo: + varMap["accessKey"] = backupDto.AccessKey + varMap["secretKey"] = backupDto.Credential + } + client, err := cloud_storage.NewCloudStorageClient(backupDto.Type, varMap) + if err != nil { + return nil, err + } + return client.ListBuckets() +} + +func (u *BackupService) Delete(id uint) error { + backup, _ := backupRepo.Get(commonRepo.WithByID(id)) + if backup.ID == 0 { + return constant.ErrRecordNotFound + } + if backup.Type == constant.Local { + return buserr.New(constant.ErrBackupLocalDelete) + } + if backup.InUsed { + return buserr.New(constant.ErrBackupInUsed) + } + if backup.Type == constant.OneDrive { + global.Cron.Remove(cron.EntryID(backup.EntryID)) + } + return backupRepo.Delete(commonRepo.WithByID(id)) +} + +func (u *BackupService) Update(req dto.BackupOperate) error { + backup, err := backupRepo.Get(commonRepo.WithByID(req.ID)) + if err != nil { + return constant.ErrRecordNotFound + } + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(req.Vars), &varMap); err != nil { + return err + } + + oldVars := backup.Vars + oldDir, err := loadLocalDir() + if err != nil { + return err + } + upMap := make(map[string]interface{}) + upMap["bucket"] = req.Bucket + upMap["access_key"] = req.AccessKey + upMap["credential"] = req.Credential + upMap["backup_path"] = req.BackupPath + upMap["vars"] = req.Vars + backup.Bucket = req.Bucket + backup.Vars = req.Vars + backup.Credential = req.Credential + backup.AccessKey = req.AccessKey + backup.BackupPath = req.BackupPath + + if req.Type == constant.OneDrive { + global.Cron.Remove(cron.EntryID(backup.EntryID)) + if err := u.loadAccessToken(&backup); err != nil { + return err + } + upMap["credential"] = backup.Credential + upMap["vars"] = backup.Vars + if err := StartRefreshOneDriveToken(&backup); err != nil { + return err + } + upMap["entry_id"] = backup.EntryID + } + if backup.Type != "LOCAL" { + isOk, err := u.checkBackupConn(&backup) + if err != nil || !isOk { + return buserr.WithMap("ErrBackupCheck", map[string]interface{}{"err": err.Error()}, err) + } + } + + if err := backupRepo.Update(req.ID, upMap); err != nil { + return err + } + if backup.Type == "LOCAL" { + if dir, ok := varMap["dir"]; ok { + if dirStr, isStr := dir.(string); isStr { + if strings.HasSuffix(dirStr, "/") && dirStr != "/" { + dirStr = dirStr[:strings.LastIndex(dirStr, "/")] + } + if err := copyDir(oldDir, dirStr); err != nil { + _ = backupRepo.Update(req.ID, map[string]interface{}{"vars": oldVars}) + return err + } + } + } + } + return nil +} + +func (u *BackupService) NewClient(backup *model.BackupAccount) (cloud_storage.CloudStorageClient, error) { + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil { + return nil, err + } + varMap["bucket"] = backup.Bucket + switch backup.Type { + case constant.Sftp, constant.WebDAV: + varMap["username"] = backup.AccessKey + varMap["password"] = backup.Credential + case constant.OSS, constant.S3, constant.MinIo, constant.Cos, constant.Kodo: + varMap["accessKey"] = backup.AccessKey + varMap["secretKey"] = backup.Credential + } + + backClient, err := cloud_storage.NewCloudStorageClient(backup.Type, varMap) + if err != nil { + return nil, err + } + + return backClient, nil +} + +func (u *BackupService) loadAccessToken(backup *model.BackupAccount) error { + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil { + return fmt.Errorf("unmarshal backup vars failed, err: %v", err) + } + refreshToken, err := client.RefreshToken("authorization_code", "refreshToken", varMap) + if err != nil { + return err + } + delete(varMap, "code") + varMap["refresh_status"] = constant.StatusSuccess + varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout) + varMap["refresh_token"] = refreshToken + itemVars, err := json.Marshal(varMap) + if err != nil { + return fmt.Errorf("json marshal var map failed, err: %v", err) + } + backup.Vars = string(itemVars) + return nil +} + +func loadLocalDir() (string, error) { + backup, err := backupRepo.Get(commonRepo.WithByType("LOCAL")) + if err != nil { + return "", err + } + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil { + return "", err + } + if _, ok := varMap["dir"]; !ok { + return "", errors.New("load local backup dir failed") + } + baseDir, ok := varMap["dir"].(string) + if ok { + if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(baseDir, os.ModePerm); err != nil { + return "", fmt.Errorf("mkdir %s failed, err: %v", baseDir, err) + } + } + return baseDir, nil + } + return "", fmt.Errorf("error type dir: %T", varMap["dir"]) +} + +func copyDir(src, dst string) error { + srcInfo, err := os.Stat(src) + if err != nil { + return err + } + if err = os.MkdirAll(dst, srcInfo.Mode()); err != nil { + return err + } + files, err := os.ReadDir(src) + if err != nil { + return err + } + for _, file := range files { + srcPath := fmt.Sprintf("%s/%s", src, file.Name()) + dstPath := fmt.Sprintf("%s/%s", dst, file.Name()) + if file.IsDir() { + if err = copyDir(srcPath, dstPath); err != nil { + global.LOG.Errorf("copy dir %s to %s failed, err: %v", srcPath, dstPath, err) + } + } else { + if err := fileUtils.CopyFile(srcPath, dst); err != nil { + global.LOG.Errorf("copy file %s to %s failed, err: %v", srcPath, dstPath, err) + } + } + } + + return nil +} + +func (u *BackupService) checkBackupConn(backup *model.BackupAccount) (bool, error) { + client, err := u.NewClient(backup) + if err != nil { + return false, err + } + fileItem := path.Join(global.CONF.System.BaseDir, "1panel/tmp/test/1panel") + if _, err := os.Stat(path.Dir(fileItem)); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(fileItem), os.ModePerm); err != nil { + return false, err + } + } + file, err := os.OpenFile(fileItem, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return false, err + } + defer file.Close() + write := bufio.NewWriter(file) + _, _ = write.WriteString("1Panel 备份账号测试文件。\n") + _, _ = write.WriteString("1Panel 備份賬號測試文件。\n") + _, _ = write.WriteString("1Panel Backs up account test files.\n") + _, _ = write.WriteString("1Panelアカウントのテストファイルをバックアップします。\n") + write.Flush() + + targetPath := strings.TrimPrefix(path.Join(backup.BackupPath, "test/1panel"), "/") + return client.Upload(fileItem, targetPath) +} + +func StartRefreshOneDriveToken(backup *model.BackupAccount) error { + service := NewIBackupService() + oneDriveCronID, err := global.Cron.AddJob("0 3 */31 * *", service) + if err != nil { + global.LOG.Errorf("can not add OneDrive corn job: %s", err.Error()) + return err + } + backup.EntryID = uint(oneDriveCronID) + return nil +} + +func (u *BackupService) Run() { + var backupItem model.BackupAccount + _ = global.DB.Where("`type` = ?", "OneDrive").First(&backupItem) + if backupItem.ID == 0 { + return + } + global.LOG.Info("start to refresh token of OneDrive ...") + varMap := make(map[string]interface{}) + if err := json.Unmarshal([]byte(backupItem.Vars), &varMap); err != nil { + global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err) + return + } + refreshToken, err := client.RefreshToken("refresh_token", "refreshToken", varMap) + varMap["refresh_status"] = constant.StatusSuccess + varMap["refresh_time"] = time.Now().Format(constant.DateTimeLayout) + if err != nil { + varMap["refresh_status"] = constant.StatusFailed + varMap["refresh_msg"] = err.Error() + global.LOG.Errorf("Failed to refresh OneDrive token, please retry, err: %v", err) + return + } + varMap["refresh_token"] = refreshToken + + varsItem, _ := json.Marshal(varMap) + _ = global.DB.Model(&model.BackupAccount{}). + Where("id = ?", backupItem.ID). + Updates(map[string]interface{}{ + "vars": varsItem, + }).Error + global.LOG.Info("Successfully refreshed OneDrive token.") +} diff --git a/core/app/service/entry.go b/core/app/service/entry.go index 834443f26..f6b25a2c6 100644 --- a/core/app/service/entry.go +++ b/core/app/service/entry.go @@ -5,5 +5,6 @@ import "github.com/1Panel-dev/1Panel/core/app/repo" var ( commonRepo = repo.NewICommonRepo() settingRepo = repo.NewISettingRepo() + backupRepo = repo.NewIBackupRepo() logRepo = repo.NewILogRepo() ) diff --git a/core/app/service/upgrade.go b/core/app/service/upgrade.go index 3e9e5b8c2..53938d1a2 100644 --- a/core/app/service/upgrade.go +++ b/core/app/service/upgrade.go @@ -83,8 +83,8 @@ func (u *UpgradeService) LoadNotes(req dto.Upgrade) (string, error) { func (u *UpgradeService) Upgrade(req dto.Upgrade) error { global.LOG.Info("start to upgrade now...") timeStr := time.Now().Format(constant.DateTimeSlimLayout) - rootDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("upgrade/upgrade_%s/downloads", timeStr)) - originalDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("upgrade/upgrade_%s/original", timeStr)) + rootDir := path.Join(global.CONF.System.BaseDir, fmt.Sprintf("1panel/tmp/upgrade/upgrade_%s/downloads", timeStr)) + originalDir := path.Join(global.CONF.System.BaseDir, fmt.Sprintf("1panel/tmp/upgrade/upgrade_%s/original", timeStr)) if err := os.MkdirAll(rootDir, os.ModePerm); err != nil { return err } @@ -181,13 +181,14 @@ func (u *UpgradeService) handleRollback(originalDir string, errStep int) { _ = settingRepo.Update("SystemStatus", "Free") checkPointOfWal() + dbPath := path.Join(global.CONF.System.BaseDir, "1panel/db") if _, err := os.Stat(path.Join(originalDir, "1Panel.db")); err == nil { - if err := files.CopyFile(path.Join(originalDir, "1Panel.db"), global.CONF.System.DbPath); err != nil { + if err := files.CopyFile(path.Join(originalDir, "1Panel.db"), dbPath); err != nil { global.LOG.Errorf("rollback 1panel db failed, err: %v", err) } } if _, err := os.Stat(path.Join(originalDir, "db.tar.gz")); err == nil { - if err := files.HandleUnTar(path.Join(originalDir, "db.tar.gz"), global.CONF.System.DbPath, ""); err != nil { + if err := files.HandleUnTar(path.Join(originalDir, "db.tar.gz"), dbPath, ""); err != nil { global.LOG.Errorf("rollback 1panel db failed, err: %v", err) } } diff --git a/core/configs/system.go b/core/configs/system.go index 1f165c489..c78995c23 100644 --- a/core/configs/system.go +++ b/core/configs/system.go @@ -6,12 +6,6 @@ type System struct { BindAddress string `mapstructure:"bindAddress"` SSL string `mapstructure:"ssl"` DbCoreFile string `mapstructure:"db_core_file"` - DbPath string `mapstructure:"db_path"` - LogPath string `mapstructure:"log_path"` - DataDir string `mapstructure:"data_dir"` - TmpDir string `mapstructure:"tmp_dir"` - Cache string `mapstructure:"cache"` - Backup string `mapstructure:"backup"` EncryptKey string `mapstructure:"encrypt_key"` BaseDir string `mapstructure:"base_dir"` Mode string `mapstructure:"mode"` diff --git a/core/constant/common.go b/core/constant/common.go index eca15f1d0..fdb09a806 100644 --- a/core/constant/common.go +++ b/core/constant/common.go @@ -17,4 +17,16 @@ const ( StatusEnable = "Enable" StatusDisable = "Disable" + + // backup + S3 = "S3" + OSS = "OSS" + Sftp = "SFTP" + OneDrive = "OneDrive" + MinIo = "MINIO" + Cos = "COS" + Kodo = "KODO" + WebDAV = "WebDAV" + Local = "LOCAL" + OneDriveRedirectURI = "http://localhost/login/authorized" ) diff --git a/core/constant/dir.go b/core/constant/dir.go deleted file mode 100644 index cdb8de0dd..000000000 --- a/core/constant/dir.go +++ /dev/null @@ -1,9 +0,0 @@ -package constant - -import ( - "github.com/1Panel-dev/1Panel/core/global" -) - -var ( - DataDir = global.CONF.System.DataDir -) diff --git a/core/constant/errs.go b/core/constant/errs.go index 2fb82cd63..e61746ede 100644 --- a/core/constant/errs.go +++ b/core/constant/errs.go @@ -30,6 +30,7 @@ var ( ErrTransform = errors.New("ErrTransform") ErrInitialPassword = errors.New("ErrInitialPassword") ErrInvalidParams = errors.New("ErrInvalidParams") + ErrNotSupportType = errors.New("ErrNotSupportType") ErrTokenParse = errors.New("ErrTokenParse") ErrStructTransform = errors.New("ErrStructTransform") @@ -49,3 +50,9 @@ var ( ErrProxy = "ErrProxy" ErrLocalDelete = "ErrLocalDelete" ) + +// backup +var ( + ErrBackupInUsed = "ErrBackupInUsed" + ErrBackupLocalDelete = "ErrBackupLocalDelete" +) diff --git a/core/global/global.go b/core/global/global.go index 5d66ba05f..e27b7946f 100644 --- a/core/global/global.go +++ b/core/global/global.go @@ -7,6 +7,7 @@ import ( "github.com/dgraph-io/badger/v4" "github.com/go-playground/validator/v10" "github.com/nicksnyder/go-i18n/v2/i18n" + "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "github.com/spf13/viper" "gorm.io/gorm" @@ -23,4 +24,6 @@ var ( Viper *viper.Viper I18n *i18n.Localizer + + Cron *cron.Cron ) diff --git a/core/i18n/lang/en.yaml b/core/i18n/lang/en.yaml index d602837ef..7b7ed8d25 100644 --- a/core/i18n/lang/en.yaml +++ b/core/i18n/lang/en.yaml @@ -23,4 +23,9 @@ ErrCreateHttpClient: "Failed to create HTTP request {{.err}}" ErrHttpReqTimeOut: "Request timed out {{.err}}" ErrHttpReqFailed: "Request failed {{.err}}" ErrHttpReqNotFound: "The file does not exist" -ErrNoSuchHost: "Network connection failed" \ No newline at end of file +ErrNoSuchHost: "Network connection failed" + +#backup +ErrBackupInUsed: "The backup account is currently in use in a scheduled task and cannot be deleted." +ErrBackupCheck: "Backup account test connection failed {{.err}}" +ErrBackupLocalDelete: "Deleting local server backup accounts is not currently supported." diff --git a/core/i18n/lang/zh-Hant.yaml b/core/i18n/lang/zh-Hant.yaml index 1a11d7194..e1c529cd5 100644 --- a/core/i18n/lang/zh-Hant.yaml +++ b/core/i18n/lang/zh-Hant.yaml @@ -23,4 +23,9 @@ ErrCreateHttpClient: "創建HTTP請求失敗 {{.err}}" ErrHttpReqTimeOut: "請求超時 {{.err}}" ErrHttpReqFailed: "請求失敗 {{.err}}" ErrHttpReqNotFound: "文件不存在" -ErrNoSuchHost: "網路連接失敗" \ No newline at end of file +ErrNoSuchHost: "網路連接失敗" + +#backup +ErrBackupInUsed: "該備份帳號已在計劃任務中使用，無法刪除" +ErrBackupCheck: "備份帳號測試連接失敗 {{.err}}" +ErrBackupLocalDelete: "暫不支持刪除本地伺服器備份帳號" \ No newline at end of file diff --git a/core/i18n/lang/zh.yaml b/core/i18n/lang/zh.yaml index 975d6b449..de576e54e 100644 --- a/core/i18n/lang/zh.yaml +++ b/core/i18n/lang/zh.yaml @@ -23,4 +23,9 @@ ErrCreateHttpClient: "创建HTTP请求失败 {{.err}}" ErrHttpReqTimeOut: "请求超时 {{.err}}" ErrHttpReqFailed: "请求失败 {{.err}}" ErrHttpReqNotFound: "文件不存在" -ErrNoSuchHost: "网络连接失败" \ No newline at end of file +ErrNoSuchHost: "网络连接失败" + +#backup +ErrBackupInUsed: "该备份账号已在计划任务中使用，无法删除" +ErrBackupCheck: "备份账号测试连接失败 {{ .err}}" +ErrBackupLocalDelete: "暂不支持删除本地服务器备份账号" \ No newline at end of file diff --git a/core/init/cache/cache.go b/core/init/cache/cache.go index 2d9f1ade2..8f3b8905d 100644 --- a/core/init/cache/cache.go +++ b/core/init/cache/cache.go @@ -1,6 +1,7 @@ package cache import ( + "path" "time" "github.com/1Panel-dev/1Panel/core/global" @@ -9,7 +10,7 @@ import ( ) func Init() { - c := global.CONF.System.Cache + c := path.Join(global.CONF.System.BaseDir, "1panel/cache") options := badger.Options{ Dir: c, diff --git a/core/init/cron/cron.go b/core/init/cron/cron.go new file mode 100644 index 000000000..db215b3c3 --- /dev/null +++ b/core/init/cron/cron.go @@ -0,0 +1,23 @@ +package cron + +import ( + "time" + + "github.com/1Panel-dev/1Panel/core/app/model" + "github.com/1Panel-dev/1Panel/core/app/service" + "github.com/1Panel-dev/1Panel/core/global" + "github.com/1Panel-dev/1Panel/core/utils/common" + "github.com/robfig/cron/v3" +) + +func Init() { + nyc, _ := time.LoadLocation(common.LoadTimeZone()) + global.Cron = cron.New(cron.WithLocation(nyc), cron.WithChain(cron.Recover(cron.DefaultLogger)), cron.WithChain(cron.DelayIfStillRunning(cron.DefaultLogger))) + + var accounts []model.BackupAccount + _ = global.DB.Where("type = ?", "OneDrive").Find(&accounts).Error + for i := 0; i < len(accounts); i++ { + _ = service.StartRefreshOneDriveToken(&accounts[i]) + } + global.Cron.Start() +} diff --git a/core/init/db/db.go b/core/init/db/db.go index 63051b6b5..f69d2ad83 100644 --- a/core/init/db/db.go +++ b/core/init/db/db.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "os" + "path" "time" "github.com/1Panel-dev/1Panel/core/global" @@ -13,12 +14,13 @@ import ( ) func Init() { - if _, err := os.Stat(global.CONF.System.DbPath); err != nil { - if err := os.MkdirAll(global.CONF.System.DbPath, os.ModePerm); err != nil { + dbPath := path.Join(global.CONF.System.BaseDir, "1panel/db") + if _, err := os.Stat(dbPath); err != nil { + if err := os.MkdirAll(dbPath, os.ModePerm); err != nil { panic(fmt.Errorf("init db dir failed, err: %v", err)) } } - fullPath := global.CONF.System.DbPath + "/" + global.CONF.System.DbCoreFile + fullPath := path.Join(dbPath, global.CONF.System.DbCoreFile) if _, err := os.Stat(fullPath); err != nil { f, err := os.Create(fullPath) if err != nil { diff --git a/core/init/log/log.go b/core/init/log/log.go index 52dc8cdd6..a22ece668 100644 --- a/core/init/log/log.go +++ b/core/init/log/log.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "path" "strings" "time" @@ -29,7 +30,7 @@ func Init() { func setOutput(logger *logrus.Logger, config configs.LogConfig) { writer, err := log.NewWriterFromConfig(&log.Config{ - LogPath: global.CONF.System.LogPath, + LogPath: path.Join(global.CONF.System.BaseDir, "1panel/log"), FileName: config.LogName, TimeTagFormat: FileTImeFormat, MaxRemain: config.MaxBackup, diff --git a/core/init/migration/migrate.go b/core/init/migration/migrate.go index 1eb29b80f..99e2a0f44 100644 --- a/core/init/migration/migrate.go +++ b/core/init/migration/migrate.go @@ -11,6 +11,7 @@ func Init() { m := gormigrate.New(global.DB, gormigrate.DefaultOptions, []*gormigrate.Migration{ migrations.AddTable, migrations.InitSetting, + migrations.InitOneDrive, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/core/init/migration/migrations/init.go b/core/init/migration/migrations/init.go index 67ea03ad0..35809685b 100644 --- a/core/init/migration/migrations/init.go +++ b/core/init/migration/migrations/init.go @@ -1,6 +1,8 @@ package migrations import ( + "fmt" + "path" "time" "github.com/1Panel-dev/1Panel/core/app/model" @@ -13,12 +15,13 @@ import ( ) var AddTable = &gormigrate.Migration{ - ID: "20240722-add-table", + ID: "20240808-add-table", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate( &model.OperationLog{}, &model.LoginLog{}, &model.Setting{}, + &model.BackupAccount{}, ) }, } @@ -137,3 +140,23 @@ var InitSetting = &gormigrate.Migration{ return nil }, } + +var InitOneDrive = &gormigrate.Migration{ + ID: "20240808-init-one-drive", + Migrate: func(tx *gorm.DB) error { + if err := tx.Create(&model.Setting{Key: "OneDriveID", Value: "MDEwOTM1YTktMWFhOS00ODU0LWExZGMtNmU0NWZlNjI4YzZi"}).Error; err != nil { + return err + } + if err := tx.Create(&model.Setting{Key: "OneDriveSc", Value: "akpuOFF+YkNXOU1OLWRzS1ZSRDdOcG1LT2ZRM0RLNmdvS1RkVWNGRA=="}).Error; err != nil { + return err + } + if err := tx.Create(&model.BackupAccount{ + Name: "localhost", + Type: "LOCAL", + Vars: fmt.Sprintf("{\"dir\":\"%s\"}", path.Join(global.CONF.System.BaseDir, "1panel/backup")), + }).Error; err != nil { + return err + } + return nil + }, +} diff --git a/core/init/router/router.go b/core/init/router/router.go index 2f9ed4c19..70e4c9a7a 100644 --- a/core/init/router/router.go +++ b/core/init/router/router.go @@ -22,7 +22,7 @@ var ( func setWebStatic(rootRouter *gin.RouterGroup) { rootRouter.StaticFS("/public", http.FS(web.Favicon)) - rootRouter.Static("/api/v1/images", "./uploads") + rootRouter.Static("/api/v2/images", "./uploads") rootRouter.Use(func(c *gin.Context) { c.Next() }) diff --git a/core/init/viper/viper.go b/core/init/viper/viper.go index bcce2018d..771dbb0bc 100644 --- a/core/init/viper/viper.go +++ b/core/init/viper/viper.go @@ -86,12 +86,6 @@ func Init() { global.CONF = serverConfig global.CONF.System.BaseDir = baseDir global.CONF.System.IsDemo = v.GetBool("system.is_demo") - global.CONF.System.DataDir = path.Join(global.CONF.System.BaseDir, "1panel") - global.CONF.System.Cache = path.Join(global.CONF.System.DataDir, "cache") - global.CONF.System.Backup = path.Join(global.CONF.System.DataDir, "backup") - global.CONF.System.DbPath = path.Join(global.CONF.System.DataDir, "db") - global.CONF.System.LogPath = path.Join(global.CONF.System.DataDir, "log") - global.CONF.System.TmpDir = path.Join(global.CONF.System.DataDir, "tmp") global.CONF.System.Port = port global.CONF.System.Version = version global.CONF.System.Username = username diff --git a/core/router/common.go b/core/router/common.go index fada5160c..78748f230 100644 --- a/core/router/common.go +++ b/core/router/common.go @@ -3,6 +3,7 @@ package router func commonGroups() []CommonRouter { return []CommonRouter{ &BaseRouter{}, + &BackupRouter{}, &LogRouter{}, &SettingRouter{}, } diff --git a/core/router/ro_backup.go b/core/router/ro_backup.go new file mode 100644 index 000000000..bb74e2b72 --- /dev/null +++ b/core/router/ro_backup.go @@ -0,0 +1,22 @@ +package router + +import ( + v2 "github.com/1Panel-dev/1Panel/core/app/api/v2" + "github.com/gin-gonic/gin" +) + +type BackupRouter struct{} + +func (s *BackupRouter) InitRouter(Router *gin.RouterGroup) { + backupRouter := Router.Group("backup") + baseApi := v2.ApiGroupApp.BaseApi + { + backupRouter.GET("/onedrive", baseApi.LoadOneDriveInfo) + backupRouter.POST("/search", baseApi.SearchBackup) + backupRouter.POST("/refresh/onedrive", baseApi.RefreshOneDriveToken) + backupRouter.POST("/buckets", baseApi.ListBuckets) + backupRouter.POST("", baseApi.CreateBackup) + backupRouter.POST("/del", baseApi.DeleteBackup) + backupRouter.POST("/update", baseApi.UpdateBackup) + } +} diff --git a/core/server/server.go b/core/server/server.go index b58ce5aa0..dc6cddc18 100644 --- a/core/server/server.go +++ b/core/server/server.go @@ -12,6 +12,7 @@ import ( "github.com/1Panel-dev/1Panel/core/global" "github.com/1Panel-dev/1Panel/core/i18n" "github.com/1Panel-dev/1Panel/core/init/cache" + "github.com/1Panel-dev/1Panel/core/init/cron" "github.com/1Panel-dev/1Panel/core/init/db" "github.com/1Panel-dev/1Panel/core/init/hook" "github.com/1Panel-dev/1Panel/core/init/log" @@ -34,6 +35,7 @@ func Start() { validator.Init() gob.Register(psession.SessionUser{}) cache.Init() + cron.Init() session.Init() gin.SetMode("debug") InitOthers() diff --git a/core/utils/cloud_storage/client/cos.go b/core/utils/cloud_storage/client/cos.go new file mode 100644 index 000000000..d000dc232 --- /dev/null +++ b/core/utils/cloud_storage/client/cos.go @@ -0,0 +1,96 @@ +package client + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + + cosSDK "github.com/tencentyun/cos-go-sdk-v5" +) + +type cosClient struct { + scType string + client *cosSDK.Client + clientWithBucket *cosSDK.Client +} + +func NewCosClient(vars map[string]interface{}) (*cosClient, error) { + region := loadParamFromVars("region", vars) + accessKey := loadParamFromVars("accessKey", vars) + secretKey := loadParamFromVars("secretKey", vars) + bucket := loadParamFromVars("bucket", vars) + scType := loadParamFromVars("scType", vars) + if len(scType) == 0 { + scType = "Standard" + } + + u, _ := url.Parse(fmt.Sprintf("https://cos.%s.myqcloud.com", region)) + b := &cosSDK.BaseURL{BucketURL: u} + client := cosSDK.NewClient(b, &http.Client{ + Transport: &cosSDK.AuthorizationTransport{ + SecretID: accessKey, + SecretKey: secretKey, + }, + }) + + if len(bucket) != 0 { + u2, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", bucket, region)) + b2 := &cosSDK.BaseURL{BucketURL: u2} + clientWithBucket := cosSDK.NewClient(b2, &http.Client{ + Transport: &cosSDK.AuthorizationTransport{ + SecretID: accessKey, + SecretKey: secretKey, + }, + }) + return &cosClient{client: client, clientWithBucket: clientWithBucket, scType: scType}, nil + } + + return &cosClient{client: client, clientWithBucket: nil, scType: scType}, nil +} + +func (c cosClient) ListBuckets() ([]interface{}, error) { + buckets, _, err := c.client.Service.Get(context.Background()) + if err != nil { + return nil, err + } + var datas []interface{} + for _, bucket := range buckets.Buckets { + datas = append(datas, bucket.Name) + } + return datas, nil +} + +func (c cosClient) Upload(src, target string) (bool, error) { + fileInfo, err := os.Stat(src) + if err != nil { + return false, err + } + if fileInfo.Size() > 5368709120 { + opt := &cosSDK.MultiUploadOptions{ + OptIni: &cosSDK.InitiateMultipartUploadOptions{ + ACLHeaderOptions: nil, + ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{ + XCosStorageClass: c.scType, + }, + }, + PartSize: 200, + } + if _, _, err := c.clientWithBucket.Object.MultiUpload( + context.Background(), target, src, opt, + ); err != nil { + return false, err + } + return true, nil + } + if _, err := c.clientWithBucket.Object.PutFromFile(context.Background(), target, src, &cosSDK.ObjectPutOptions{ + ACLHeaderOptions: nil, + ObjectPutHeaderOptions: &cosSDK.ObjectPutHeaderOptions{ + XCosStorageClass: c.scType, + }, + }); err != nil { + return false, err + } + return true, nil +} diff --git a/core/utils/cloud_storage/client/helper.go b/core/utils/cloud_storage/client/helper.go new file mode 100644 index 000000000..08134748c --- /dev/null +++ b/core/utils/cloud_storage/client/helper.go @@ -0,0 +1,18 @@ +package client + +import ( + "fmt" + + "github.com/1Panel-dev/1Panel/core/global" +) + +func loadParamFromVars(key string, vars map[string]interface{}) string { + if _, ok := vars[key]; !ok { + if key != "bucket" && key != "port" { + global.LOG.Errorf("load param %s from vars failed, err: not exist!", key) + } + return "" + } + + return fmt.Sprintf("%v", vars[key]) +} diff --git a/core/utils/cloud_storage/client/kodo.go b/core/utils/cloud_storage/client/kodo.go new file mode 100644 index 000000000..dd21e8614 --- /dev/null +++ b/core/utils/cloud_storage/client/kodo.go @@ -0,0 +1,66 @@ +package client + +import ( + "context" + "strconv" + + "github.com/qiniu/go-sdk/v7/auth" + "github.com/qiniu/go-sdk/v7/storage" +) + +type kodoClient struct { + bucket string + domain string + timeout string + auth *auth.Credentials + client *storage.BucketManager +} + +func NewKodoClient(vars map[string]interface{}) (*kodoClient, error) { + accessKey := loadParamFromVars("accessKey", vars) + secretKey := loadParamFromVars("secretKey", vars) + bucket := loadParamFromVars("bucket", vars) + domain := loadParamFromVars("domain", vars) + timeout := loadParamFromVars("timeout", vars) + if timeout == "" { + timeout = "1" + } + conn := auth.New(accessKey, secretKey) + cfg := storage.Config{ + UseHTTPS: false, + } + bucketManager := storage.NewBucketManager(conn, &cfg) + + return &kodoClient{client: bucketManager, auth: conn, bucket: bucket, domain: domain, timeout: timeout}, nil +} + +func (k kodoClient) ListBuckets() ([]interface{}, error) { + buckets, err := k.client.Buckets(true) + if err != nil { + return nil, err + } + var datas []interface{} + for _, bucket := range buckets { + datas = append(datas, bucket) + } + return datas, nil +} + +func (k kodoClient) Upload(src, target string) (bool, error) { + int64Value, _ := strconv.ParseInt(k.timeout, 10, 64) + unixTimestamp := int64Value * 3600 + + putPolicy := storage.PutPolicy{ + Scope: k.bucket, + Expires: uint64(unixTimestamp), + } + upToken := putPolicy.UploadToken(k.auth) + cfg := storage.Config{UseHTTPS: true, UseCdnDomains: false} + resumeUploader := storage.NewResumeUploaderV2(&cfg) + ret := storage.PutRet{} + putExtra := storage.RputV2Extra{} + if err := resumeUploader.PutFile(context.Background(), &ret, upToken, target, src, &putExtra); err != nil { + return false, err + } + return true, nil +} diff --git a/core/utils/cloud_storage/client/local.go b/core/utils/cloud_storage/client/local.go new file mode 100644 index 000000000..5904c91ee --- /dev/null +++ b/core/utils/cloud_storage/client/local.go @@ -0,0 +1,40 @@ +package client + +import ( + "fmt" + "os" + "path" + + "github.com/1Panel-dev/1Panel/core/utils/files" +) + +type localClient struct { + dir string +} + +func NewLocalClient(vars map[string]interface{}) (*localClient, error) { + dir := loadParamFromVars("dir", vars) + return &localClient{dir: dir}, nil +} + +func (c localClient) ListBuckets() ([]interface{}, error) { + return nil, nil +} + +func (c localClient) Upload(src, target string) (bool, error) { + targetFilePath := path.Join(c.dir, target) + if _, err := os.Stat(path.Dir(targetFilePath)); err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(targetFilePath), os.ModePerm); err != nil { + return false, err + } + } else { + return false, err + } + } + + if err := files.CopyFile(src, targetFilePath); err != nil { + return false, fmt.Errorf("cp file failed, err: %v", err) + } + return true, nil +} diff --git a/core/utils/cloud_storage/client/minio.go b/core/utils/cloud_storage/client/minio.go new file mode 100644 index 000000000..2c9db7a86 --- /dev/null +++ b/core/utils/cloud_storage/client/minio.go @@ -0,0 +1,78 @@ +package client + +import ( + "context" + "crypto/tls" + "net/http" + "os" + "strings" + + "github.com/1Panel-dev/1Panel/core/constant" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type minIoClient struct { + bucket string + client *minio.Client +} + +func NewMinIoClient(vars map[string]interface{}) (*minIoClient, error) { + endpoint := loadParamFromVars("endpoint", vars) + accessKeyID := loadParamFromVars("accessKey", vars) + secretAccessKey := loadParamFromVars("secretKey", vars) + bucket := loadParamFromVars("bucket", vars) + ssl := strings.Split(endpoint, ":")[0] + if len(ssl) == 0 || (ssl != "https" && ssl != "http") { + return nil, constant.ErrInvalidParams + } + + secure := false + tlsConfig := &tls.Config{} + if ssl == "https" { + secure = true + tlsConfig.InsecureSkipVerify = true + } + var transport http.RoundTripper = &http.Transport{ + TLSClientConfig: tlsConfig, + } + client, err := minio.New(strings.ReplaceAll(endpoint, ssl+"://", ""), &minio.Options{ + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Secure: secure, + Transport: transport, + }) + if err != nil { + return nil, err + } + return &minIoClient{bucket: bucket, client: client}, nil +} + +func (m minIoClient) ListBuckets() ([]interface{}, error) { + buckets, err := m.client.ListBuckets(context.Background()) + if err != nil { + return nil, err + } + var result []interface{} + for _, bucket := range buckets { + result = append(result, bucket.Name) + } + return result, err +} + +func (m minIoClient) Upload(src, target string) (bool, error) { + file, err := os.Open(src) + if err != nil { + return false, err + } + defer file.Close() + + fileStat, err := file.Stat() + if err != nil { + return false, err + } + _, err = m.client.PutObject(context.Background(), m.bucket, target, file, fileStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) + if err != nil { + return false, err + } + return true, nil +} diff --git a/core/utils/cloud_storage/client/onedrive.go b/core/utils/cloud_storage/client/onedrive.go new file mode 100644 index 000000000..47349fd07 --- /dev/null +++ b/core/utils/cloud_storage/client/onedrive.go @@ -0,0 +1,350 @@ +package client + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "strconv" + "strings" + "time" + + odsdk "github.com/goh-chunlin/go-onedrive/onedrive" + "golang.org/x/oauth2" +) + +type oneDriveClient struct { + client odsdk.Client +} + +func NewOneDriveClient(vars map[string]interface{}) (*oneDriveClient, error) { + token, err := RefreshToken("refresh_token", "accessToken", vars) + if err != nil { + return nil, err + } + isCN := loadParamFromVars("isCN", vars) + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + + client := odsdk.NewClient(tc) + if isCN == "true" { + client.BaseURL, _ = url.Parse("https://microsoftgraph.chinacloudapi.cn/v1.0/") + } + return &oneDriveClient{client: *client}, nil +} + +func (o oneDriveClient) ListBuckets() ([]interface{}, error) { + return nil, nil +} + +func (o oneDriveClient) Upload(src, target string) (bool, error) { + target = "/" + strings.TrimPrefix(target, "/") + if _, err := o.loadIDByPath(path.Dir(target)); err != nil { + if !strings.Contains(err.Error(), "itemNotFound") { + return false, err + } + if err := o.createFolder(path.Dir(target)); err != nil { + return false, fmt.Errorf("create dir before upload failed, err: %v", err) + } + } + + ctx := context.Background() + folderID, err := o.loadIDByPath(path.Dir(target)) + if err != nil { + return false, err + } + fileInfo, err := os.Stat(src) + if err != nil { + return false, err + } + if fileInfo.IsDir() { + return false, errors.New("only file is allowed to be uploaded here") + } + var isOk bool + if fileInfo.Size() < 4*1024*1024 { + isOk, err = o.upSmall(src, folderID, fileInfo.Size()) + } else { + isOk, err = o.upBig(ctx, src, folderID, fileInfo.Size()) + } + return isOk, err +} + +func (o oneDriveClient) Download(src, target string) (bool, error) { + src = "/" + strings.TrimPrefix(src, "/") + req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/root:%s", src), nil) + if err != nil { + return false, fmt.Errorf("new request for file id failed, err: %v", err) + } + var driveItem *odsdk.DriveItem + if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { + return false, fmt.Errorf("do request for file id failed, err: %v", err) + } + + resp, err := http.Get(driveItem.DownloadURL) + if err != nil { + return false, err + } + defer resp.Body.Close() + + out, err := os.Create(target) + if err != nil { + return false, err + } + defer out.Close() + buffer := make([]byte, 2*1024*1024) + + _, err = io.CopyBuffer(out, resp.Body, buffer) + if err != nil { + return false, err + } + + return true, nil +} + +func (o *oneDriveClient) loadIDByPath(path string) (string, error) { + pathItem := "root:" + path + if path == "/" { + pathItem = "root" + } + req, err := o.client.NewRequest("GET", fmt.Sprintf("me/drive/%s", pathItem), nil) + if err != nil { + return "", fmt.Errorf("new request for file id failed, err: %v", err) + } + var driveItem *odsdk.DriveItem + if err := o.client.Do(context.Background(), req, false, &driveItem); err != nil { + return "", fmt.Errorf("do request for file id failed, err: %v", err) + } + return driveItem.Id, nil +} + +func RefreshToken(grantType string, tokenType string, varMap map[string]interface{}) (string, error) { + data := url.Values{} + isCN := loadParamFromVars("isCN", varMap) + data.Set("client_id", loadParamFromVars("client_id", varMap)) + data.Set("client_secret", loadParamFromVars("client_secret", varMap)) + if grantType == "refresh_token" { + data.Set("grant_type", "refresh_token") + data.Set("refresh_token", loadParamFromVars("refresh_token", varMap)) + } else { + data.Set("grant_type", "authorization_code") + data.Set("code", loadParamFromVars("code", varMap)) + } + data.Set("redirect_uri", loadParamFromVars("redirect_uri", varMap)) + client := &http.Client{} + url := "https://login.microsoftonline.com/common/oauth2/v2.0/token" + if isCN == "true" { + url = "https://login.chinacloudapi.cn/common/oauth2/v2.0/token" + } + req, err := http.NewRequest("POST", url, strings.NewReader(data.Encode())) + if err != nil { + return "", fmt.Errorf("new http post client for access token failed, err: %v", err) + } + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("request for access token failed, err: %v", err) + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read data from response body failed, err: %v", err) + } + + tokenMap := map[string]interface{}{} + if err := json.Unmarshal(respBody, &tokenMap); err != nil { + return "", fmt.Errorf("unmarshal data from response body failed, err: %v", err) + } + if tokenType == "accessToken" { + accessToken, ok := tokenMap["access_token"].(string) + if !ok { + return "", errors.New("no such access token in response") + } + tokenMap = nil + return accessToken, nil + } + refreshToken, ok := tokenMap["refresh_token"].(string) + if !ok { + return "", errors.New("no such access token in response") + } + tokenMap = nil + return refreshToken, nil +} + +func (o *oneDriveClient) createFolder(parent string) error { + if _, err := o.loadIDByPath(path.Dir(parent)); err != nil { + if !strings.Contains(err.Error(), "itemNotFound") { + return err + } + _ = o.createFolder(path.Dir(parent)) + } + item2, err := o.loadIDByPath(path.Dir(parent)) + if err != nil { + return err + } + if _, err := o.client.DriveItems.CreateNewFolder(context.Background(), "", item2, path.Base(parent)); err != nil { + return err + } + return nil +} + +type NewUploadSessionCreationRequest struct { + ConflictBehavior string `json:"@microsoft.graph.conflictBehavior,omitempty"` +} +type NewUploadSessionCreationResponse struct { + UploadURL string `json:"uploadUrl"` + ExpirationDateTime string `json:"expirationDateTime"` +} +type UploadSessionUploadResponse struct { + ExpirationDateTime string `json:"expirationDateTime"` + NextExpectedRanges []string `json:"nextExpectedRanges"` + DriveItem +} +type DriveItem struct { + Name string `json:"name"` + Id string `json:"id"` + DownloadURL string `json:"@microsoft.graph.downloadUrl"` + Description string `json:"description"` + Size int64 `json:"size"` + WebURL string `json:"webUrl"` +} + +func (o *oneDriveClient) NewSessionFileUploadRequest(absoluteUrl string, grandOffset, grandTotalSize int64, byteReader *bytes.Reader) (*http.Request, error) { + apiUrl, err := o.client.BaseURL.Parse(absoluteUrl) + if err != nil { + return nil, err + } + absoluteUrl = apiUrl.String() + contentLength := byteReader.Size() + req, err := http.NewRequest("PUT", absoluteUrl, byteReader) + req.Header.Set("Content-Length", strconv.FormatInt(contentLength, 10)) + preliminaryLength := grandOffset + preliminaryRange := grandOffset + contentLength - 1 + if preliminaryRange >= grandTotalSize { + preliminaryRange = grandTotalSize - 1 + preliminaryLength = preliminaryRange - grandOffset + 1 + } + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", preliminaryLength, preliminaryRange, grandTotalSize)) + + return req, err +} + +func (o *oneDriveClient) upSmall(srcPath, folderID string, fileSize int64) (bool, error) { + file, err := os.Open(srcPath) + if err != nil { + return false, err + } + defer file.Close() + + buffer := make([]byte, fileSize) + _, _ = file.Read(buffer) + fileReader := bytes.NewReader(buffer) + apiURL := fmt.Sprintf("me/drive/items/%s:/%s:/content?@microsoft.graph.conflictBehavior=rename", url.PathEscape(folderID), path.Base(srcPath)) + + mimeType := getMimeType(srcPath) + req, err := o.client.NewFileUploadRequest(apiURL, mimeType, fileReader) + if err != nil { + return false, err + } + var response *DriveItem + if err := o.client.Do(context.Background(), req, false, &response); err != nil { + return false, fmt.Errorf("do request for list failed, err: %v", err) + } + return true, nil +} + +func (o *oneDriveClient) upBig(ctx context.Context, srcPath, folderID string, fileSize int64) (bool, error) { + file, err := os.Open(srcPath) + if err != nil { + return false, err + } + defer file.Close() + + apiURL := fmt.Sprintf("me/drive/items/%s:/%s:/createUploadSession", url.PathEscape(folderID), path.Base(srcPath)) + sessionCreationRequestInside := NewUploadSessionCreationRequest{ + ConflictBehavior: "rename", + } + + sessionCreationRequest := struct { + Item NewUploadSessionCreationRequest `json:"item"` + DeferCommit bool `json:"deferCommit"` + }{sessionCreationRequestInside, false} + + sessionCreationReq, err := o.client.NewRequest("POST", apiURL, sessionCreationRequest) + if err != nil { + return false, err + } + + var sessionCreationResp *NewUploadSessionCreationResponse + err = o.client.Do(ctx, sessionCreationReq, false, &sessionCreationResp) + if err != nil { + return false, fmt.Errorf("session creation failed %w", err) + } + + fileSessionUploadUrl := sessionCreationResp.UploadURL + + sizePerSplit := int64(5 * 1024 * 1024) + buffer := make([]byte, 5*1024*1024) + splitCount := fileSize / sizePerSplit + if fileSize%sizePerSplit != 0 { + splitCount += 1 + } + bfReader := bufio.NewReader(file) + httpClient := http.Client{ + Timeout: time.Minute * 10, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + for splitNow := int64(0); splitNow < splitCount; splitNow++ { + length, err := bfReader.Read(buffer) + if err != nil { + return false, err + } + if int64(length) < sizePerSplit { + bufferLast := buffer[:length] + buffer = bufferLast + } + sessionFileUploadReq, err := o.NewSessionFileUploadRequest(fileSessionUploadUrl, splitNow*sizePerSplit, fileSize, bytes.NewReader(buffer)) + if err != nil { + return false, err + } + res, err := httpClient.Do(sessionFileUploadReq) + if err != nil { + return false, err + } + res.Body.Close() + if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { + data, _ := io.ReadAll(res.Body) + return false, errors.New(string(data)) + } + } + return true, nil +} + +func getMimeType(path string) string { + file, err := os.Open(path) + if err != nil { + return "" + } + defer file.Close() + + buffer := make([]byte, 512) + _, err = file.Read(buffer) + if err != nil { + return "" + } + mimeType := http.DetectContentType(buffer) + return mimeType +} diff --git a/core/utils/cloud_storage/client/oss.go b/core/utils/cloud_storage/client/oss.go new file mode 100644 index 000000000..e4fed15d7 --- /dev/null +++ b/core/utils/cloud_storage/client/oss.go @@ -0,0 +1,118 @@ +package client + +import ( + "fmt" + + osssdk "github.com/aliyun/aliyun-oss-go-sdk/oss" +) + +type ossClient struct { + scType string + bucketStr string + client osssdk.Client +} + +func NewOssClient(vars map[string]interface{}) (*ossClient, error) { + endpoint := loadParamFromVars("endpoint", vars) + accessKey := loadParamFromVars("accessKey", vars) + secretKey := loadParamFromVars("secretKey", vars) + bucketStr := loadParamFromVars("bucket", vars) + scType := loadParamFromVars("scType", vars) + if len(scType) == 0 { + scType = "Standard" + } + client, err := osssdk.New(endpoint, accessKey, secretKey) + if err != nil { + return nil, err + } + + return &ossClient{scType: scType, bucketStr: bucketStr, client: *client}, nil +} + +func (o ossClient) ListBuckets() ([]interface{}, error) { + response, err := o.client.ListBuckets() + if err != nil { + return nil, err + } + var result []interface{} + for _, bucket := range response.Buckets { + result = append(result, bucket.Name) + } + return result, err +} + +func (o ossClient) Exist(path string) (bool, error) { + bucket, err := o.client.Bucket(o.bucketStr) + if err != nil { + return false, err + } + return bucket.IsObjectExist(path) +} + +func (o ossClient) Size(path string) (int64, error) { + bucket, err := o.client.Bucket(o.bucketStr) + if err != nil { + return 0, err + } + lor, err := bucket.ListObjectsV2(osssdk.Prefix(path)) + if err != nil { + return 0, err + } + if len(lor.Objects) == 0 { + return 0, fmt.Errorf("no such file %s", path) + } + return lor.Objects[0].Size, nil +} + +func (o ossClient) Delete(path string) (bool, error) { + bucket, err := o.client.Bucket(o.bucketStr) + if err != nil { + return false, err + } + if err := bucket.DeleteObject(path); err != nil { + return false, err + } + return true, nil +} + +func (o ossClient) Upload(src, target string) (bool, error) { + bucket, err := o.client.Bucket(o.bucketStr) + if err != nil { + return false, err + } + if err := bucket.UploadFile(target, src, + 200*1024*1024, + osssdk.Routines(5), + osssdk.Checkpoint(true, ""), + osssdk.ObjectStorageClass(osssdk.StorageClassType(o.scType))); err != nil { + return false, err + } + return true, nil +} + +func (o ossClient) Download(src, target string) (bool, error) { + bucket, err := o.client.Bucket(o.bucketStr) + if err != nil { + return false, err + } + if err := bucket.DownloadFile(src, target, 200*1024*1024, osssdk.Routines(5), osssdk.Checkpoint(true, "")); err != nil { + return false, err + } + return true, nil +} + +func (o *ossClient) ListObjects(prefix string) ([]string, error) { + bucket, err := o.client.Bucket(o.bucketStr) + if err != nil { + return nil, err + } + lor, err := bucket.ListObjectsV2(osssdk.Prefix(prefix)) + if err != nil { + return nil, err + } + var result []string + for _, obj := range lor.Objects { + result = append(result, obj.Key) + } + return result, nil +} diff --git a/core/utils/cloud_storage/client/s3.go b/core/utils/cloud_storage/client/s3.go new file mode 100644 index 000000000..9fc7ebadd --- /dev/null +++ b/core/utils/cloud_storage/client/s3.go @@ -0,0 +1,163 @@ +package client + +import ( + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +type s3Client struct { + scType string + bucket string + Sess session.Session +} + +func NewS3Client(vars map[string]interface{}) (*s3Client, error) { + accessKey := loadParamFromVars("accessKey", vars) + secretKey := loadParamFromVars("secretKey", vars) + endpoint := loadParamFromVars("endpoint", vars) + region := loadParamFromVars("region", vars) + bucket := loadParamFromVars("bucket", vars) + scType := loadParamFromVars("scType", vars) + if len(scType) == 0 { + scType = "Standard" + } + sess, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""), + Endpoint: aws.String(endpoint), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(false), + }) + if err != nil { + return nil, err + } + return &s3Client{scType: scType, bucket: bucket, Sess: *sess}, nil +} + +func (s s3Client) ListBuckets() ([]interface{}, error) { + var result []interface{} + svc := s3.New(&s.Sess) + res, err := svc.ListBuckets(nil) + if err != nil { + return nil, err + } + for _, b := range res.Buckets { + result = append(result, b.Name) + } + return result, nil +} + +func (s s3Client) Exist(path string) (bool, error) { + svc := s3.New(&s.Sess) + if _, err := svc.HeadObject(&s3.HeadObjectInput{ + Bucket: &s.bucket, + Key: &path, + }); err != nil { + if aerr, ok := err.(awserr.RequestFailure); ok { + if aerr.StatusCode() == 404 { + return false, nil + } + } else { + return false, aerr + } + } + return true, nil +} + +func (s *s3Client) Size(path string) (int64, error) { + svc := s3.New(&s.Sess) + file, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: &s.bucket, + Key: &path, + }) + if err != nil { + return 0, err + } + return *file.ContentLength, nil +} + +func (s s3Client) Delete(path string) (bool, error) { + svc := s3.New(&s.Sess) + if _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(path)}); err != nil { + return false, err + } + if err := svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(path), + }); err != nil { + return false, err + } + return true, nil +} + +func (s s3Client) Upload(src, target string) (bool, error) { + fileInfo, err := os.Stat(src) + if err != nil { + return false, err + } + file, err := os.Open(src) + if err != nil { + return false, err + } + defer file.Close() + + uploader := s3manager.NewUploader(&s.Sess) + if fileInfo.Size() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = fileInfo.Size() / (s3manager.MaxUploadParts - 1) + } + if _, err := uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(target), + Body: file, + StorageClass: &s.scType, + }); err != nil { + return false, err + } + return true, nil +} + +func (s s3Client) Download(src, target string) (bool, error) { + if _, err := os.Stat(target); err != nil { + if os.IsNotExist(err) { + os.Remove(target) + } else { + return false, err + } + } + file, err := os.Create(target) + if err != nil { + return false, err + } + defer file.Close() + downloader := s3manager.NewDownloader(&s.Sess) + if _, err = downloader.Download(file, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(src), + }); err != nil { + os.Remove(target) + return false, err + } + return true, nil +} + +func (s *s3Client) ListObjects(prefix string) ([]string, error) { + svc := s3.New(&s.Sess) + var result []string + outputs, err := svc.ListObjects(&s3.ListObjectsInput{ + Bucket: &s.bucket, + Prefix: &prefix, + }) + if err != nil { + return result, err + } + for _, item := range outputs.Contents { + result = append(result, *item.Key) + } + return result, nil +} diff --git a/core/utils/cloud_storage/client/sftp.go b/core/utils/cloud_storage/client/sftp.go new file mode 100644 index 000000000..a9f552963 --- /dev/null +++ b/core/utils/cloud_storage/client/sftp.go @@ -0,0 +1,122 @@ +package client + +import ( + "fmt" + "io" + "net" + "os" + "path" + "time" + + "github.com/1Panel-dev/1Panel/core/global" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +type sftpClient struct { + bucket string + connInfo string + config *ssh.ClientConfig +} + +func NewSftpClient(vars map[string]interface{}) (*sftpClient, error) { + address := loadParamFromVars("address", vars) + port := loadParamFromVars("port", vars) + if len(port) == 0 { + global.LOG.Errorf("load param port from vars failed, err: not exist!") + } + authMode := loadParamFromVars("authMode", vars) + privateKey := loadParamFromVars("privateKey", vars) + password := loadParamFromVars("password", vars) + bucket := loadParamFromVars("bucket", vars) + + var auth []ssh.AuthMethod + if authMode == "key" { + itemPrivateKey, err := ssh.ParsePrivateKey([]byte(privateKey)) + if err != nil { + return nil, err + } + auth = []ssh.AuthMethod{ssh.PublicKeys(itemPrivateKey)} + } else { + auth = []ssh.AuthMethod{ssh.Password(password)} + } + username := loadParamFromVars("username", vars) + + clientConfig := &ssh.ClientConfig{ + User: username, + Auth: auth, + Timeout: 30 * time.Second, + HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { + return nil + }, + } + if _, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", address, port), clientConfig); err != nil { + return nil, err + } + + return &sftpClient{connInfo: fmt.Sprintf("%s:%s", address, port), config: clientConfig, bucket: bucket}, nil +} + +func (s sftpClient) Upload(src, target string) (bool, error) { + sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) + if err != nil { + return false, err + } + defer sshClient.Close() + client, err := sftp.NewClient(sshClient) + if err != nil { + return false, err + } + defer client.Close() + + srcFile, err := os.Open(src) + if err != nil { + return false, err + } + defer srcFile.Close() + + targetFilePath := path.Join(s.bucket, target) + targetDir, _ := path.Split(targetFilePath) + if _, err = client.Stat(targetDir); err != nil { + if os.IsNotExist(err) { + if err = client.MkdirAll(targetDir); err != nil { + return false, err + } + } else { + return false, err + } + } + dstFile, err := client.Create(path.Join(s.bucket, target)) + if err != nil { + return false, err + } + defer dstFile.Close() + + if _, err := io.Copy(dstFile, srcFile); err != nil { + return false, err + } + return true, nil +} + +func (s sftpClient) ListBuckets() ([]interface{}, error) { + var result []interface{} + return result, nil +} + +func (s sftpClient) Delete(filePath string) error { + sshClient, err := ssh.Dial("tcp", s.connInfo, s.config) + if err != nil { + return err + } + client, err := sftp.NewClient(sshClient) + if err != nil { + return err + } + defer client.Close() + defer sshClient.Close() + + if err := client.Remove(filePath); err != nil { + return err + } + return nil +} diff --git a/core/utils/cloud_storage/client/webdav.go b/core/utils/cloud_storage/client/webdav.go new file mode 100644 index 000000000..c1013357c --- /dev/null +++ b/core/utils/cloud_storage/client/webdav.go @@ -0,0 +1,121 @@ +package client + +import ( + "crypto/tls" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + + "github.com/studio-b12/gowebdav" +) + +type webDAVClient struct { + Bucket string + client *gowebdav.Client +} + +func NewWebDAVClient(vars map[string]interface{}) (*webDAVClient, error) { + address := loadParamFromVars("address", vars) + port := loadParamFromVars("port", vars) + password := loadParamFromVars("password", vars) + username := loadParamFromVars("username", vars) + bucket := loadParamFromVars("bucket", vars) + + url := fmt.Sprintf("%s:%s", address, port) + if len(port) == 0 { + url = address + } + client := gowebdav.NewClient(url, username, password) + tlsConfig := &tls.Config{} + if strings.HasPrefix(address, "https") { + tlsConfig.InsecureSkipVerify = true + } + var transport http.RoundTripper = &http.Transport{ + TLSClientConfig: tlsConfig, + } + client.SetTransport(transport) + if err := client.Connect(); err != nil { + return nil, err + } + return &webDAVClient{Bucket: bucket, client: client}, nil +} + +func (s webDAVClient) Upload(src, target string) (bool, error) { + targetFilePath := path.Join(s.Bucket, target) + srcFile, err := os.Open(src) + if err != nil { + return false, err + } + defer srcFile.Close() + + if err := s.client.WriteStream(targetFilePath, srcFile, 0644); err != nil { + return false, err + } + return true, nil +} + +func (s webDAVClient) ListBuckets() ([]interface{}, error) { + var result []interface{} + return result, nil +} + +func (s webDAVClient) Download(src, target string) (bool, error) { + srcPath := path.Join(s.Bucket, src) + info, err := s.client.Stat(srcPath) + if err != nil { + return false, err + } + targetStat, err := os.Stat(target) + if err == nil { + if info.Size() == targetStat.Size() { + return true, nil + } + } + file, err := os.Create(target) + if err != nil { + return false, err + } + defer file.Close() + reader, _ := s.client.ReadStream(srcPath) + if _, err := io.Copy(file, reader); err != nil { + return false, err + } + return true, err +} + +func (s webDAVClient) Exist(pathItem string) (bool, error) { + if _, err := s.client.Stat(path.Join(s.Bucket, pathItem)); err != nil { + return false, err + } + return true, nil +} + +func (s webDAVClient) Size(pathItem string) (int64, error) { + file, err := s.client.Stat(path.Join(s.Bucket, pathItem)) + if err != nil { + return 0, err + } + return file.Size(), nil +} + +func (s webDAVClient) Delete(pathItem string) (bool, error) { + if err := s.client.Remove(path.Join(s.Bucket, pathItem)); err != nil { + return false, err + } + return true, nil +} + +func (s webDAVClient) ListObjects(prefix string) ([]string, error) { + files, err := s.client.ReadDir(path.Join(s.Bucket, prefix)) + if err != nil { + return nil, err + } + var result []string + for _, file := range files { + result = append(result, file.Name()) + } + return result, nil +} diff --git a/core/utils/cloud_storage/cloud_storage_client.go b/core/utils/cloud_storage/cloud_storage_client.go new file mode 100644 index 000000000..6c4cb4f26 --- /dev/null +++ b/core/utils/cloud_storage/cloud_storage_client.go @@ -0,0 +1,36 @@ +package cloud_storage + +import ( + "github.com/1Panel-dev/1Panel/core/constant" + "github.com/1Panel-dev/1Panel/core/utils/cloud_storage/client" +) + +type CloudStorageClient interface { + ListBuckets() ([]interface{}, error) + Upload(src, target string) (bool, error) +} + +func NewCloudStorageClient(backupType string, vars map[string]interface{}) (CloudStorageClient, error) { + switch backupType { + case constant.Local: + return client.NewLocalClient(vars) + case constant.S3: + return client.NewS3Client(vars) + case constant.OSS: + return client.NewOssClient(vars) + case constant.Sftp: + return client.NewSftpClient(vars) + case constant.WebDAV: + return client.NewWebDAVClient(vars) + case constant.MinIo: + return client.NewMinIoClient(vars) + case constant.Cos: + return client.NewCosClient(vars) + case constant.Kodo: + return client.NewKodoClient(vars) + case constant.OneDrive: + return client.NewOneDriveClient(vars) + default: + return nil, constant.ErrNotSupportType + } +} diff --git a/frontend/src/api/interface/backup.ts b/frontend/src/api/interface/backup.ts index d87675aee..aed28c9d8 100644 --- a/frontend/src/api/interface/backup.ts +++ b/frontend/src/api/interface/backup.ts @@ -1,12 +1,18 @@ import { ReqPage } from '.'; export namespace Backup { + export interface SearchWithType extends ReqPage { + type: string; + name: string; + } export interface BackupInfo { id: number; + name: string; type: string; accessKey: string; bucket: string; credential: string; + rememberAuth: boolean; backupPath: string; vars: string; varsJson: object; diff --git a/frontend/src/api/modules/setting.ts b/frontend/src/api/modules/setting.ts index 17bfee87e..c957fb150 100644 --- a/frontend/src/api/modules/setting.ts +++ b/frontend/src/api/modules/setting.ts @@ -27,6 +27,9 @@ export const unbindLicense = () => { export const loadBaseDir = () => { return http.get(`/settings/basedir`); }; +export const loadDaemonJsonPath = () => { + return http.get(`/settings/daemonjson`, {}); +}; // core export const getSettingInfo = () => { @@ -35,15 +38,12 @@ export const getSettingInfo = () => { export const getSystemAvailable = () => { return http.get(`/settings/search/available`); }; - export const updateSetting = (param: Setting.SettingUpdate) => { return http.post(`/core/settings/update`, param); }; - export const updateMenu = (param: Setting.SettingUpdate) => { return http.post(`/core/settings/menu/update`, param); }; - export const updateProxy = (params: Setting.ProxyUpdate) => { let request = deepCopy(params) as Setting.ProxyUpdate; if (request.proxyPasswd) { @@ -52,23 +52,18 @@ export const updateProxy = (params: Setting.ProxyUpdate) => { request.proxyType = request.proxyType === 'close' ? '' : request.proxyType; return http.post(`/core/settings/proxy/update`, request); }; - export const updatePassword = (param: Setting.PasswordUpdate) => { return http.post(`/core/settings/password/update`, param); }; - export const loadInterfaceAddr = () => { return http.get(`/core/settings/interface`); }; - export const updateBindInfo = (ipv6: string, bindAddress: string) => { return http.post(`/core/settings/bind/update`, { ipv6: ipv6, bindAddress: bindAddress }); }; - export const updatePort = (param: Setting.PortUpdate) => { return http.post(`/core/settings/port/update`, param); }; - export const updateSSL = (param: Setting.SSLUpdate) => { return http.post(`/core/settings/ssl/update`, param); }; @@ -78,24 +73,17 @@ export const loadSSLInfo = () => { export const downloadSSL = () => { return http.download(`/core/settings/ssl/download`); }; - export const handleExpired = (param: Setting.PasswordUpdate) => { return http.post(`/core/settings/expired/handle`, param); }; - export const loadMFA = (param: Setting.MFARequest) => { return http.post(`/core/settings/mfa`, param); }; - -export const loadDaemonJsonPath = () => { - return http.get(`/core/settings/daemonjson`, {}); -}; - export const bindMFA = (param: Setting.MFABind) => { return http.post(`/core/settings/mfa/bind`, param); }; -// backup +// backup-agent export const handleBackup = (params: Backup.Backup) => { return http.post(`/settings/backup/backup`, params, TimeoutEnum.T_1H); }; @@ -105,9 +93,6 @@ export const handleRecover = (params: Backup.Recover) => { export const handleRecoverByUpload = (params: Backup.Recover) => { return http.post(`/settings/backup/recover/byupload`, params, TimeoutEnum.T_1D); }; -export const refreshOneDrive = () => { - return http.post(`/settings/backup/refresh/onedrive`, {}); -}; export const downloadBackupRecord = (params: Backup.RecordDownload) => { return http.post(`/settings/backup/record/download`, params, TimeoutEnum.T_10M); }; @@ -120,16 +105,23 @@ export const searchBackupRecords = (params: Backup.SearchBackupRecord) => { export const searchBackupRecordsByCronjob = (params: Backup.SearchBackupRecordByCronjob) => { return http.post>(`/settings/backup/record/search/bycronjob`, params, TimeoutEnum.T_5M); }; - -export const getBackupList = () => { - return http.get>(`/settings/backup/search`); -}; -export const getOneDriveInfo = () => { - return http.get(`/settings/backup/onedrive`); -}; export const getFilesFromBackup = (type: string) => { return http.post>(`/settings/backup/search/files`, { type: type }); }; + +// backup-core +export const refreshOneDrive = () => { + return http.post(`/core/backup/refresh/onedrive`, {}); +}; +export const getBackupList = () => { + return http.post>(`/core/backup/list`); +}; +export const searchBackup = (params: Backup.SearchWithType) => { + return http.post>(`/core/backup/search`, params); +}; +export const getOneDriveInfo = () => { + return http.get(`/core/backup/onedrive`); +}; export const addBackup = (params: Backup.BackupOperate) => { let request = deepCopy(params) as Backup.BackupOperate; if (request.accessKey) { @@ -138,7 +130,7 @@ export const addBackup = (params: Backup.BackupOperate) => { if (request.credential) { request.credential = Base64.encode(request.credential); } - return http.post(`/settings/backup`, request); + return http.post(`/core/backup`, request, TimeoutEnum.T_60S); }; export const editBackup = (params: Backup.BackupOperate) => { let request = deepCopy(params) as Backup.BackupOperate; @@ -148,10 +140,10 @@ export const editBackup = (params: Backup.BackupOperate) => { if (request.credential) { request.credential = Base64.encode(request.credential); } - return http.post(`/settings/backup/update`, request); + return http.post(`/core/backup/update`, request); }; export const deleteBackup = (params: { id: number }) => { - return http.post(`/settings/backup/del`, params); + return http.post(`/core/backup/del`, params); }; export const listBucket = (params: Backup.ForBucket) => { let request = deepCopy(params) as Backup.BackupOperate; @@ -161,7 +153,7 @@ export const listBucket = (params: Backup.ForBucket) => { if (request.credential) { request.credential = Base64.encode(request.credential); } - return http.post(`/settings/backup/buckets`, request); + return http.post(`/core/backup/buckets`, request); }; // snapshot diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 42cfa2457..7f8b81cf0 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1423,6 +1423,7 @@ const message = { client_secret: 'Client Secret', redirect_uri: 'Redirect URL', onedrive_helper: 'Custom configuration can be referred to in the official documentation', + clickToRefresh: 'Click to refresh', refreshTime: 'Token Refresh Time', refreshStatus: 'Token Refresh Status', backupDir: 'Backup Dir', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index 11954a080..fad659d0b 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -1336,6 +1336,7 @@ const message = { client_secret: '客戶端密鑰', redirect_uri: '重定向 URL', onedrive_helper: '自訂配置可參考官方文件', + clickToRefresh: '單擊可手動刷新', refreshTime: '令牌刷新時間', refreshStatus: '令牌刷新狀態', codeWarning: '當前授權碼格式錯誤，請重新確認！', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 022d3391a..2bad460fc 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1338,6 +1338,7 @@ const message = { client_secret: '客户端密钥', redirect_uri: '重定向 Url', onedrive_helper: '自定义配置可参考官方文档', + clickToRefresh: '单击可手动刷新', refreshTime: '令牌刷新时间', refreshStatus: '令牌刷新状态', codeWarning: '当前授权码格式错误，请重新确认！', diff --git a/frontend/src/layout/components/Sidebar/components/Logo.vue b/frontend/src/layout/components/Sidebar/components/Logo.vue index 34efac09b..8ea024b3c 100644 --- a/frontend/src/layout/components/Sidebar/components/Logo.vue +++ b/frontend/src/layout/components/Sidebar/components/Logo.vue @@ -1,13 +1,13 @@