gh_mirrors/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2025-01-19 00:09:16 +08:00
Code

feat(logs): Change IP Database to Support Internationalization (#7212)

Browse Source 
change ip database from qqwry to geoip
This commit is contained in:
zhengkunwang 2024-11-29 11:46:44 +08:00 committed by GitHub
parent 5e9a69d8f3
commit 23087bdb6f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 66 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/app/api/v1/auth.go
View File

@ -8,7 +8,8 @@ import (
"github.com/1Panel-dev/1Panel/backend/constant"
"github.com/1Panel-dev/1Panel/backend/global"
"github.com/1Panel-dev/1Panel/backend/utils/captcha"
"github.com/1Panel-dev/1Panel/backend/utils/qqwry"
"github.com/1Panel-dev/1Panel/backend/utils/common"
"github.com/1Panel-dev/1Panel/backend/utils/geo"
"github.com/gin-gonic/gin"
)
@ -162,12 +163,11 @@ func saveLoginLogs(c *gin.Context, err error) {
logs.Status = constant.StatusSuccess
}
logs.IP = c.ClientIP()
qqWry, err := qqwry.NewQQwry()
address, err := geo.GetIPLocation(logs.IP, common.GetLang(c))
if err != nil {
global.LOG.Errorf("load qqwry datas failed: %s", err)
global.LOG.Errorf("get ip location failed: %s", err)
}
res := qqWry.Find(logs.IP)
logs.Agent = c.GetHeader("User-Agent")
logs.Address = res.Area
logs.Address = address
_ = logService.CreateLoginLog(logs)
}

2
backend/app/api/v1/ssh.go
View File

@ -145,7 +145,7 @@ func (b *BaseApi) LoadSSHLogs(c *gin.Context) {
return
}
data, err := sshService.LoadLog(req)
data, err := sshService.LoadLog(c, req)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return

21
backend/app/service/ssh.go
View File

@ -2,6 +2,8 @@ package service
import (
"fmt"
"github.com/1Panel-dev/1Panel/backend/utils/geo"
"github.com/gin-gonic/gin"
"os"
"os/user"
"path"
@ -17,7 +19,6 @@ import (
"github.com/1Panel-dev/1Panel/backend/utils/cmd"
"github.com/1Panel-dev/1Panel/backend/utils/common"
"github.com/1Panel-dev/1Panel/backend/utils/files"
"github.com/1Panel-dev/1Panel/backend/utils/qqwry"
"github.com/1Panel-dev/1Panel/backend/utils/systemctl"
"github.com/pkg/errors"
)
@ -33,7 +34,7 @@ type ISSHService interface {
Update(req dto.SSHUpdate) error
GenerateSSH(req dto.GenerateSSH) error
LoadSSHSecret(mode string) (string, error)
LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error)
LoadLog(c *gin.Context, req dto.SearchSSHLog) (*dto.SSHLog, error)
LoadSSHConf() (string, error)
}
@ -289,7 +290,7 @@ type sshFileItem struct {
Year int
}
func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) {
func (u *SSHService) LoadLog(c *gin.Context, req dto.SearchSSHLog) (*dto.SSHLog, error) {
var fileList []sshFileItem
var data dto.SSHLog
baseDir := "/var/log"
@ -323,10 +324,6 @@ func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) {
showCountFrom := (req.Page - 1) * req.PageSize
showCountTo := req.Page * req.PageSize
nyc, _ := time.LoadLocation(common.LoadTimeZoneByCmd())
qqWry, err := qqwry.NewQQwry()
if err != nil {
global.LOG.Errorf("load qqwry datas failed: %s", err)
}
for _, file := range fileList {
commandItem := ""
if strings.HasPrefix(path.Base(file.Name), "secure") {
@ -349,7 +346,7 @@ func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) {
commandItem = fmt.Sprintf("cat %s | grep -aE \"(Failed password for|Connection closed by authenticating user|Accepted)\" %s", file.Name, command)
}
}
dataItem, successCount, failedCount := loadSSHData(commandItem, showCountFrom, showCountTo, file.Year, qqWry, nyc)
dataItem, successCount, failedCount := loadSSHData(c, commandItem, showCountFrom, showCountTo, file.Year, nyc)
data.FailedCount += failedCount
data.TotalCount += successCount + failedCount
showCountFrom = showCountFrom - (successCount + failedCount)
@ -422,7 +419,7 @@ func updateSSHConf(oldFiles []string, param string, value string) []string {
return newFiles
}
func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qqWry *qqwry.QQwry, nyc *time.Location) ([]dto.SSHHistory, int, int) {
func loadSSHData(c *gin.Context, command string, showCountFrom, showCountTo, currentYear int, nyc *time.Location) ([]dto.SSHHistory, int, int) {
var (
datas []dto.SSHHistory
successCount int
@ -440,7 +437,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq
itemData = loadFailedSecureDatas(lines[i])
if len(itemData.Address) != 0 {
if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo {
itemData.Area = qqWry.Find(itemData.Address).Area
itemData.Area, _ = geo.GetIPLocation(itemData.Address, common.GetLang(c))
itemData.Date = loadDate(currentYear, itemData.DateStr, nyc)
datas = append(datas, itemData)
}
@ -450,7 +447,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq
itemData = loadFailedAuthDatas(lines[i])
if len(itemData.Address) != 0 {
if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo {
itemData.Area = qqWry.Find(itemData.Address).Area
itemData.Area, _ = geo.GetIPLocation(itemData.Address, common.GetLang(c))
itemData.Date = loadDate(currentYear, itemData.DateStr, nyc)
datas = append(datas, itemData)
}
@ -460,7 +457,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq
itemData = loadSuccessDatas(lines[i])
if len(itemData.Address) != 0 {
if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo {
itemData.Area = qqWry.Find(itemData.Address).Area
itemData.Area, _ = geo.GetIPLocation(itemData.Address, common.GetLang(c))
itemData.Date = loadDate(currentYear, itemData.DateStr, nyc)
datas = append(datas, itemData)
}

9
backend/utils/common/common.go
View File

@ -3,6 +3,7 @@ package common
import (
"crypto/rand"
"fmt"
"github.com/gin-gonic/gin"
"io"
mathRand "math/rand"
"net"
@ -342,3 +343,11 @@ func FormatBytes(bytes uint64) string {
func FormatPercent(percent float64) string {
return fmt.Sprintf("%.2f%%", percent)
}
func GetLang(c *gin.Context) string {
lang := c.GetHeader("Accept-Language")
if lang == "" {
lang = "en"
}
return lang
}

39
backend/utils/geo/geo.go Normal file
View File

@ -0,0 +1,39 @@
package geo
import (
"github.com/1Panel-dev/1Panel/backend/global"
"github.com/oschwald/maxminddb-golang"
"net"
"path"
)
type Location struct {
En string `maxminddb:"en"`
Zh string `maxminddb:"zh"`
}
type LocationRes struct {
Iso string `maxminddb:"iso"`
Country Location `maxminddb:"country"`
Latitude float64 `maxminddb:"latitude"`
Longitude float64 `maxminddb:"longitude"`
Province Location `maxminddb:"province"`
}
func GetIPLocation(ip, lang string) (string, error) {
geoPath := path.Join(global.CONF.System.BaseDir, "1panel", "geo", "GeoIP.mmdb")
reader, err := maxminddb.Open(geoPath)
if err != nil {
return "", err
}
var geoLocation LocationRes
ipNet := net.ParseIP(ip)
err = reader.Lookup(ipNet, &geoLocation)
if err != nil {
return "", err
}
if lang == "en" {
return geoLocation.Country.En + geoLocation.Province.En, nil
}
return geoLocation.Country.Zh + geoLocation.Province.Zh, nil
}

165
backend/utils/qqwry/qqwry.go
View File

@ -1,165 +0,0 @@
package qqwry
import (
"encoding/binary"
"net"
"strings"
"github.com/1Panel-dev/1Panel/cmd/server/qqwry"
"golang.org/x/text/encoding/simplifiedchinese"
)
const (
indexLen = 7
redirectMode1 = 0x01
redirectMode2 = 0x02
)
var IpCommonDictionary []byte
type QQwry struct {
Data []byte
Offset int64
}
func NewQQwry() (*QQwry, error) {
IpCommonDictionary := qqwry.QQwryByte
return &QQwry{Data: IpCommonDictionary}, nil
}
// readData 从文件中读取数据
func (q *QQwry) readData(num int, offset ...int64) (rs []byte) {
if len(offset) > 0 {
q.setOffset(offset[0])
}
nums := int64(num)
end := q.Offset + nums
dataNum := int64(len(q.Data))
if q.Offset > dataNum {
return nil
}
if end > dataNum {
end = dataNum
}
rs = q.Data[q.Offset:end]
q.Offset = end
return
}
// setOffset 设置偏移量
func (q *QQwry) setOffset(offset int64) {
q.Offset = offset
}
// Find ip地址查询对应归属地信息
func (q *QQwry) Find(ip string) (res ResultQQwry) {
res = ResultQQwry{}
res.IP = ip
if strings.Count(ip, ".") != 3 {
return res
}
offset := q.searchIndex(binary.BigEndian.Uint32(net.ParseIP(ip).To4()))
if offset <= 0 {
return
}
var area []byte
mode := q.readMode(offset + 4)
if mode == redirectMode1 {
countryOffset := q.readUInt24()
mode = q.readMode(countryOffset)
if mode == redirectMode2 {
c := q.readUInt24()
area = q.readString(c)
} else {
area = q.readString(countryOffset)
}
} else if mode == redirectMode2 {
countryOffset := q.readUInt24()
area = q.readString(countryOffset)
} else {
area = q.readString(offset + 4)
}
enc := simplifiedchinese.GBK.NewDecoder()
res.Area, _ = enc.String(string(area))
return
}
type ResultQQwry struct {
IP string `json:"ip"`
Area string `json:"area"`
}
// readMode 获取偏移值类型
func (q *QQwry) readMode(offset uint32) byte {
mode := q.readData(1, int64(offset))
return mode[0]
}
// readString 获取字符串
func (q *QQwry) readString(offset uint32) []byte {
q.setOffset(int64(offset))
data := make([]byte, 0, 30)
for {
buf := q.readData(1)
if buf[0] == 0 {
break
}
data = append(data, buf[0])
}
return data
}
// searchIndex 查找索引位置
func (q *QQwry) searchIndex(ip uint32) uint32 {
header := q.readData(8, 0)
start := binary.LittleEndian.Uint32(header[:4])
end := binary.LittleEndian.Uint32(header[4:])
for {
mid := q.getMiddleOffset(start, end)
buf := q.readData(indexLen, int64(mid))
_ip := binary.LittleEndian.Uint32(buf[:4])
if end-start == indexLen {
offset := byteToUInt32(buf[4:])
buf = q.readData(indexLen)
if ip < binary.LittleEndian.Uint32(buf[:4]) {
return offset
}
return 0
}
if _ip > ip {
end = mid
} else if _ip < ip {
start = mid
} else if _ip == ip {
return byteToUInt32(buf[4:])
}
}
}
// readUInt24
func (q *QQwry) readUInt24() uint32 {
buf := q.readData(3)
return byteToUInt32(buf)
}
// getMiddleOffset
func (q *QQwry) getMiddleOffset(start uint32, end uint32) uint32 {
records := ((end - start) / indexLen) >> 1
return start + records*indexLen
}
// byteToUInt32 将 byte 转换为uint32
func byteToUInt32(data []byte) uint32 {
i := uint32(data[0]) & 0xff
i |= (uint32(data[1]) << 8) & 0xff00
i |= (uint32(data[2]) << 16) & 0xff0000
return i
}

6
cmd/server/qqwry/qqwey.go
View File

@ -1,6 +0,0 @@
package qqwry
import _ "embed"
//go:embed qqwry.dat
var QQwryByte []byte

BIN
cmd/server/qqwry/qqwry.dat
View File

Binary file not shown.

1
go.mod
View File

@ -196,6 +196,7 @@ require (
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
github.com/oschwald/maxminddb-golang v1.13.1 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect

2
go.sum
View File

@ -792,6 +792,8 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A=
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU=
github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE=
github.com/oschwald/maxminddb-golang v1.13.1 h1:G3wwjdN9JmIK2o/ermkHM+98oX5fS+k5MbwsmL4MRQE=
github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk4IsCNThNdTmnaBZ/F8=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=