From 79fea272aa281d98c35a7fe32267c99977fd36a1 Mon Sep 17 00:00:00 2001 From: zhengkunwang <31820853+zhengkunwang223@users.noreply.github.com> Date: Mon, 30 Dec 2024 13:37:49 +0800 Subject: [PATCH] feat(task): Replace QQWry with GeoIP.mmdb (#7599) --- agent/app/api/v2/ssh.go | 2 +- agent/app/model/backup.go | 8 ++-- agent/app/service/ssh.go | 29 ++++++------ agent/go.mod | 3 +- agent/go.sum | 2 + agent/init/migration/migrations/init.go | 3 +- agent/utils/common/common.go | 9 ++++ agent/utils/geo/geo.go | 47 ++++++++++++++++++++ frontend/src/components/app-status/index.vue | 14 +++--- 9 files changed, 90 insertions(+), 27 deletions(-) create mode 100644 agent/utils/geo/geo.go diff --git a/agent/app/api/v2/ssh.go b/agent/app/api/v2/ssh.go index f9faf6d4c..4772818ac 100644 --- a/agent/app/api/v2/ssh.go +++ b/agent/app/api/v2/ssh.go @@ -144,7 +144,7 @@ func (b *BaseApi) LoadSSHLogs(c *gin.Context) { return } - data, err := sshService.LoadLog(req) + data, err := sshService.LoadLog(c, req) if err != nil { helper.InternalServer(c, err) return diff --git a/agent/app/model/backup.go b/agent/app/model/backup.go index 79680f46b..a9a301534 100644 --- a/agent/app/model/backup.go +++ b/agent/app/model/backup.go @@ -2,8 +2,8 @@ package model type BackupAccount struct { BaseModel - Name string `gorm:"not null" json:"name"` - Type string `gorm:"not null" json:"type"` + Name string `gorm:"not null;default:''" json:"name"` + Type string `gorm:"not null;default:''" json:"type"` Bucket string `json:"bucket"` AccessKey string `json:"accessKey"` Credential string `json:"credential"` @@ -22,8 +22,8 @@ type BackupRecord struct { SourceAccountIDs string `json:"sourceAccountIDs"` DownloadAccountID uint `json:"downloadAccountID"` - Type string `gorm:"not null" json:"type"` - Name string `gorm:"not null" json:"name"` + Type string `gorm:"not null;default:''" json:"type"` + Name string `gorm:"not null;default:''" json:"name"` DetailName string `json:"detailName"` FileDir string `json:"fileDir"` FileName string `json:"fileName"` diff --git a/agent/app/service/ssh.go b/agent/app/service/ssh.go index b164d47a1..75b94c879 100644 --- a/agent/app/service/ssh.go +++ b/agent/app/service/ssh.go @@ -2,6 +2,8 @@ package service import ( "fmt" + "github.com/1Panel-dev/1Panel/agent/utils/geo" + "github.com/gin-gonic/gin" "os" "os/user" "path" @@ -17,7 +19,6 @@ import ( "github.com/1Panel-dev/1Panel/agent/utils/cmd" "github.com/1Panel-dev/1Panel/agent/utils/common" "github.com/1Panel-dev/1Panel/agent/utils/files" - "github.com/1Panel-dev/1Panel/agent/utils/qqwry" "github.com/1Panel-dev/1Panel/agent/utils/systemctl" "github.com/pkg/errors" ) @@ -33,7 +34,7 @@ type ISSHService interface { Update(req dto.SSHUpdate) error GenerateSSH(req dto.GenerateSSH) error LoadSSHSecret(mode string) (string, error) - LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) + LoadLog(ctx *gin.Context, req dto.SearchSSHLog) (*dto.SSHLog, error) LoadSSHConf() (string, error) } @@ -282,7 +283,7 @@ type sshFileItem struct { Year int } -func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) { +func (u *SSHService) LoadLog(ctx *gin.Context, req dto.SearchSSHLog) (*dto.SSHLog, error) { var fileList []sshFileItem var data dto.SSHLog baseDir := "/var/log" @@ -316,10 +317,6 @@ func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) { showCountFrom := (req.Page - 1) * req.PageSize showCountTo := req.Page * req.PageSize nyc, _ := time.LoadLocation(common.LoadTimeZoneByCmd()) - qqWry, err := qqwry.NewQQwry() - if err != nil { - global.LOG.Errorf("load qqwry datas failed: %s", err) - } for _, file := range fileList { commandItem := "" if strings.HasPrefix(path.Base(file.Name), "secure") { @@ -342,7 +339,7 @@ func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) { commandItem = fmt.Sprintf("cat %s | grep -aE \"(Failed password for|Connection closed by authenticating user|Accepted)\" %s", file.Name, command) } } - dataItem, successCount, failedCount := loadSSHData(commandItem, showCountFrom, showCountTo, file.Year, qqWry, nyc) + dataItem, successCount, failedCount := loadSSHData(ctx, commandItem, showCountFrom, showCountTo, file.Year, nyc) data.FailedCount += failedCount data.TotalCount += successCount + failedCount showCountFrom = showCountFrom - (successCount + failedCount) @@ -415,17 +412,21 @@ func updateSSHConf(oldFiles []string, param string, value string) []string { return newFiles } -func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qqWry *qqwry.QQwry, nyc *time.Location) ([]dto.SSHHistory, int, int) { +func loadSSHData(ctx *gin.Context, command string, showCountFrom, showCountTo, currentYear int, nyc *time.Location) ([]dto.SSHHistory, int, int) { var ( datas []dto.SSHHistory successCount int failedCount int ) - stdout2, err := cmd.Exec(command) + getLoc, err := geo.NewGeo() if err != nil { return datas, 0, 0 } - lines := strings.Split(string(stdout2), "\n") + stdout, err := cmd.Exec(command) + if err != nil { + return datas, 0, 0 + } + lines := strings.Split(stdout, "\n") for i := len(lines) - 1; i >= 0; i-- { var itemData dto.SSHHistory switch { @@ -433,7 +434,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq itemData = loadFailedSecureDatas(lines[i]) if len(itemData.Address) != 0 { if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo { - itemData.Area = qqWry.Find(itemData.Address).Area + itemData.Area, _ = geo.GetIPLocation(getLoc, itemData.Address, common.GetLang(ctx)) itemData.Date = loadDate(currentYear, itemData.DateStr, nyc) datas = append(datas, itemData) } @@ -443,7 +444,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq itemData = loadFailedAuthDatas(lines[i]) if len(itemData.Address) != 0 { if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo { - itemData.Area = qqWry.Find(itemData.Address).Area + itemData.Area, _ = geo.GetIPLocation(getLoc, itemData.Address, common.GetLang(ctx)) itemData.Date = loadDate(currentYear, itemData.DateStr, nyc) datas = append(datas, itemData) } @@ -453,7 +454,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq itemData = loadSuccessDatas(lines[i]) if len(itemData.Address) != 0 { if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo { - itemData.Area = qqWry.Find(itemData.Address).Area + itemData.Area, _ = geo.GetIPLocation(getLoc, itemData.Address, common.GetLang(ctx)) itemData.Date = loadDate(currentYear, itemData.DateStr, nyc) datas = append(datas, itemData) } diff --git a/agent/go.mod b/agent/go.mod index bd38f9122..8ff355476 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -1,6 +1,6 @@ module github.com/1Panel-dev/1Panel/agent -go 1.22.4 +go 1.23 require ( github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible @@ -171,6 +171,7 @@ require ( github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect + github.com/oschwald/maxminddb-golang v1.13.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect diff --git a/agent/go.sum b/agent/go.sum index d7c18deaa..6f841f001 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -586,6 +586,8 @@ github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M5 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= +github.com/oschwald/maxminddb-golang v1.13.1 h1:G3wwjdN9JmIK2o/ermkHM+98oX5fS+k5MbwsmL4MRQE= +github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk4IsCNThNdTmnaBZ/F8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index 986d894c3..c16448112 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -17,7 +17,7 @@ import ( ) var AddTable = &gormigrate.Migration{ - ID: "20241226-add-table", + ID: "20241231-add-table", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate( &model.AppDetail{}, @@ -56,7 +56,6 @@ var AddTable = &gormigrate.Migration{ &model.WebsiteDnsAccount{}, &model.WebsiteDomain{}, &model.WebsiteSSL{}, - &model.Task{}, ) }, } diff --git a/agent/utils/common/common.go b/agent/utils/common/common.go index b71ba0488..2b4d3006f 100644 --- a/agent/utils/common/common.go +++ b/agent/utils/common/common.go @@ -3,6 +3,7 @@ package common import ( "crypto/rand" "fmt" + "github.com/gin-gonic/gin" "io" mathRand "math/rand" "net" @@ -323,3 +324,11 @@ func SplitStr(str string, spi ...string) []string { func IsValidIP(ip string) bool { return net.ParseIP(ip) != nil } + +func GetLang(context *gin.Context) string { + lang := context.GetHeader("Accept-Language") + if strings.Contains(lang, "zh") { + return "zh" + } + return "en" +} diff --git a/agent/utils/geo/geo.go b/agent/utils/geo/geo.go new file mode 100644 index 000000000..df469c91b --- /dev/null +++ b/agent/utils/geo/geo.go @@ -0,0 +1,47 @@ +package geo + +import ( + "github.com/1Panel-dev/1Panel/agent/global" + "github.com/oschwald/maxminddb-golang" + "net" + "path" +) + +type Location struct { + En string `maxminddb:"en"` + Zh string `maxminddb:"zh"` +} + +type LocationRes struct { + Iso string `maxminddb:"iso"` + Country Location `maxminddb:"country"` + Latitude float64 `maxminddb:"latitude"` + Longitude float64 `maxminddb:"longitude"` + Province Location `maxminddb:"province"` +} + +func NewGeo() (*maxminddb.Reader, error) { + geoPath := path.Join(global.CONF.System.BaseDir, "1panel", "geo", "GeoIP.mmdb") + return maxminddb.Open(geoPath) +} + +func GetIPLocation(reader *maxminddb.Reader, ip, lang string) (string, error) { + var err error + var geoLocation LocationRes + if reader == nil { + geoPath := path.Join(global.CONF.System.BaseDir, "1panel", "geo", "GeoIP.mmdb") + reader, err = maxminddb.Open(geoPath) + if err != nil { + return "", err + } + } + ipNet := net.ParseIP(ip) + err = reader.Lookup(ipNet, &geoLocation) + if err != nil { + return "", err + } + if lang == "zh" { + return geoLocation.Country.Zh + geoLocation.Province.Zh, nil + } + return geoLocation.Country.En + geoLocation.Province.En, nil +} diff --git a/frontend/src/components/app-status/index.vue b/frontend/src/components/app-status/index.vue index 17fe2cfee..b79eec2b3 100644 --- a/frontend/src/components/app-status/index.vue +++ b/frontend/src/components/app-status/index.vue @@ -136,11 +136,15 @@ const onCheck = async (key: any, name: any) => { const onOperate = async (operation: string) => { em('update:maskShow', false); operateReq.operate = operation; - ElMessageBox.confirm(i18n.global.t(`app.${operation}OperatorHelper`), i18n.global.t('app.' + operation), { - confirmButtonText: i18n.global.t('commons.button.confirm'), - cancelButtonText: i18n.global.t('commons.button.cancel'), - type: 'info', - }) + ElMessageBox.confirm( + i18n.global.t('app.operatorHelper', [i18n.global.t('app.' + operation)]), + i18n.global.t('app.' + operation), + { + confirmButtonText: i18n.global.t('commons.button.confirm'), + cancelButtonText: i18n.global.t('commons.button.cancel'), + type: 'info', + }, + ) .then(() => { em('update:maskShow', true); em('update:loading', true);