feat(logs): Change IP Database to Support Internationalization (#7212)

change ip database from qqwry to geoip

backend/app/api/v1/auth.go

@@ -8,7 +8,8 @@ import (
 	"github.com/1Panel-dev/1Panel/backend/constant"
 	"github.com/1Panel-dev/1Panel/backend/global"
 	"github.com/1Panel-dev/1Panel/backend/utils/captcha"
-	"github.com/1Panel-dev/1Panel/backend/utils/qqwry"
+	"github.com/1Panel-dev/1Panel/backend/utils/common"
+	"github.com/1Panel-dev/1Panel/backend/utils/geo"
 	"github.com/gin-gonic/gin"
 )
 
@@ -162,12 +163,11 @@ func saveLoginLogs(c *gin.Context, err error) {
 		logs.Status = constant.StatusSuccess
 	}
 	logs.IP = c.ClientIP()
-	qqWry, err := qqwry.NewQQwry()
+	address, err := geo.GetIPLocation(logs.IP, common.GetLang(c))
 	if err != nil {
-		global.LOG.Errorf("load qqwry datas failed: %s", err)
+		global.LOG.Errorf("get ip location failed: %s", err)
 	}
-	res := qqWry.Find(logs.IP)
 	logs.Agent = c.GetHeader("User-Agent")
-	logs.Address = res.Area
+	logs.Address = address
 	_ = logService.CreateLoginLog(logs)
 }

backend/app/api/v1/ssh.go

@@ -145,7 +145,7 @@ func (b *BaseApi) LoadSSHLogs(c *gin.Context) {
 		return
 	}
 
-	data, err := sshService.LoadLog(req)
+	data, err := sshService.LoadLog(c, req)
 	if err != nil {
 		helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
 		return

backend/app/service/ssh.go

@@ -2,6 +2,8 @@ package service
 
 import (
 	"fmt"
+	"github.com/1Panel-dev/1Panel/backend/utils/geo"
+	"github.com/gin-gonic/gin"
 	"os"
 	"os/user"
 	"path"
@@ -17,7 +19,6 @@ import (
 	"github.com/1Panel-dev/1Panel/backend/utils/cmd"
 	"github.com/1Panel-dev/1Panel/backend/utils/common"
 	"github.com/1Panel-dev/1Panel/backend/utils/files"
-	"github.com/1Panel-dev/1Panel/backend/utils/qqwry"
 	"github.com/1Panel-dev/1Panel/backend/utils/systemctl"
 	"github.com/pkg/errors"
 )
@@ -33,7 +34,7 @@ type ISSHService interface {
 	Update(req dto.SSHUpdate) error
 	GenerateSSH(req dto.GenerateSSH) error
 	LoadSSHSecret(mode string) (string, error)
-	LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error)
+	LoadLog(c *gin.Context, req dto.SearchSSHLog) (*dto.SSHLog, error)
 
 	LoadSSHConf() (string, error)
 }
@@ -289,7 +290,7 @@ type sshFileItem struct {
 	Year int
 }
 
-func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) {
+func (u *SSHService) LoadLog(c *gin.Context, req dto.SearchSSHLog) (*dto.SSHLog, error) {
 	var fileList []sshFileItem
 	var data dto.SSHLog
 	baseDir := "/var/log"
@@ -323,10 +324,6 @@ func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) {
 	showCountFrom := (req.Page - 1) * req.PageSize
 	showCountTo := req.Page * req.PageSize
 	nyc, _ := time.LoadLocation(common.LoadTimeZoneByCmd())
-	qqWry, err := qqwry.NewQQwry()
-	if err != nil {
-		global.LOG.Errorf("load qqwry datas failed: %s", err)
-	}
 	for _, file := range fileList {
 		commandItem := ""
 		if strings.HasPrefix(path.Base(file.Name), "secure") {
@@ -349,7 +346,7 @@ func (u *SSHService) LoadLog(req dto.SearchSSHLog) (*dto.SSHLog, error) {
 				commandItem = fmt.Sprintf("cat %s | grep -aE \"(Failed password for|Connection closed by authenticating user|Accepted)\" %s", file.Name, command)
 			}
 		}
-		dataItem, successCount, failedCount := loadSSHData(commandItem, showCountFrom, showCountTo, file.Year, qqWry, nyc)
+		dataItem, successCount, failedCount := loadSSHData(c, commandItem, showCountFrom, showCountTo, file.Year, nyc)
 		data.FailedCount += failedCount
 		data.TotalCount += successCount + failedCount
 		showCountFrom = showCountFrom - (successCount + failedCount)
@@ -422,7 +419,7 @@ func updateSSHConf(oldFiles []string, param string, value string) []string {
 	return newFiles
 }
 
-func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qqWry *qqwry.QQwry, nyc *time.Location) ([]dto.SSHHistory, int, int) {
+func loadSSHData(c *gin.Context, command string, showCountFrom, showCountTo, currentYear int, nyc *time.Location) ([]dto.SSHHistory, int, int) {
 	var (
 		datas        []dto.SSHHistory
 		successCount int
@@ -440,7 +437,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq
 			itemData = loadFailedSecureDatas(lines[i])
 			if len(itemData.Address) != 0 {
 				if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo {
-					itemData.Area = qqWry.Find(itemData.Address).Area
+					itemData.Area, _ = geo.GetIPLocation(itemData.Address, common.GetLang(c))
 					itemData.Date = loadDate(currentYear, itemData.DateStr, nyc)
 					datas = append(datas, itemData)
 				}
@@ -450,7 +447,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq
 			itemData = loadFailedAuthDatas(lines[i])
 			if len(itemData.Address) != 0 {
 				if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo {
-					itemData.Area = qqWry.Find(itemData.Address).Area
+					itemData.Area, _ = geo.GetIPLocation(itemData.Address, common.GetLang(c))
 					itemData.Date = loadDate(currentYear, itemData.DateStr, nyc)
 					datas = append(datas, itemData)
 				}
@@ -460,7 +457,7 @@ func loadSSHData(command string, showCountFrom, showCountTo, currentYear int, qq
 			itemData = loadSuccessDatas(lines[i])
 			if len(itemData.Address) != 0 {
 				if successCount+failedCount >= showCountFrom && successCount+failedCount < showCountTo {
-					itemData.Area = qqWry.Find(itemData.Address).Area
+					itemData.Area, _ = geo.GetIPLocation(itemData.Address, common.GetLang(c))
 					itemData.Date = loadDate(currentYear, itemData.DateStr, nyc)
 					datas = append(datas, itemData)
 				}

backend/utils/common/common.go

@@ -3,6 +3,7 @@ package common
 import (
 	"crypto/rand"
 	"fmt"
+	"github.com/gin-gonic/gin"
 	"io"
 	mathRand "math/rand"
 	"net"
@@ -342,3 +343,11 @@ func FormatBytes(bytes uint64) string {
 func FormatPercent(percent float64) string {
 	return fmt.Sprintf("%.2f%%", percent)
 }
+
+func GetLang(c *gin.Context) string {
+	lang := c.GetHeader("Accept-Language")
+	if lang == "" {
+		lang = "en"
+	}
+	return lang
+}

backend/utils/geo/geo.go

@@ -0,0 +1,39 @@
+package geo
+
+import (
+	"github.com/1Panel-dev/1Panel/backend/global"
+	"github.com/oschwald/maxminddb-golang"
+	"net"
+	"path"
+)
+
+type Location struct {
+	En string `maxminddb:"en"`
+	Zh string `maxminddb:"zh"`
+}
+
+type LocationRes struct {
+	Iso       string   `maxminddb:"iso"`
+	Country   Location `maxminddb:"country"`
+	Latitude  float64  `maxminddb:"latitude"`
+	Longitude float64  `maxminddb:"longitude"`
+	Province  Location `maxminddb:"province"`
+}
+
+func GetIPLocation(ip, lang string) (string, error) {
+	geoPath := path.Join(global.CONF.System.BaseDir, "1panel", "geo", "GeoIP.mmdb")
+	reader, err := maxminddb.Open(geoPath)
+	if err != nil {
+		return "", err
+	}
+	var geoLocation LocationRes
+	ipNet := net.ParseIP(ip)
+	err = reader.Lookup(ipNet, &geoLocation)
+	if err != nil {
+		return "", err
+	}
+	if lang == "en" {
+		return geoLocation.Country.En + geoLocation.Province.En, nil
+	}
+	return geoLocation.Country.Zh + geoLocation.Province.Zh, nil
+}

backend/utils/qqwry/qqwry.go

@@ -1,165 +0,0 @@
-package qqwry
-
-import (
-	"encoding/binary"
-	"net"
-	"strings"
-
-	"github.com/1Panel-dev/1Panel/cmd/server/qqwry"
-	"golang.org/x/text/encoding/simplifiedchinese"
-)
-
-const (
-	indexLen      = 7
-	redirectMode1 = 0x01
-	redirectMode2 = 0x02
-)
-
-var IpCommonDictionary []byte
-
-type QQwry struct {
-	Data   []byte
-	Offset int64
-}
-
-func NewQQwry() (*QQwry, error) {
-	IpCommonDictionary := qqwry.QQwryByte
-	return &QQwry{Data: IpCommonDictionary}, nil
-}
-
-// readData 从文件中读取数据
-func (q *QQwry) readData(num int, offset ...int64) (rs []byte) {
-	if len(offset) > 0 {
-		q.setOffset(offset[0])
-	}
-	nums := int64(num)
-	end := q.Offset + nums
-	dataNum := int64(len(q.Data))
-	if q.Offset > dataNum {
-		return nil
-	}
-
-	if end > dataNum {
-		end = dataNum
-	}
-	rs = q.Data[q.Offset:end]
-	q.Offset = end
-	return
-}
-
-// setOffset 设置偏移量
-func (q *QQwry) setOffset(offset int64) {
-	q.Offset = offset
-}
-
-// Find ip地址查询对应归属地信息
-func (q *QQwry) Find(ip string) (res ResultQQwry) {
-	res = ResultQQwry{}
-	res.IP = ip
-	if strings.Count(ip, ".") != 3 {
-		return res
-	}
-	offset := q.searchIndex(binary.BigEndian.Uint32(net.ParseIP(ip).To4()))
-	if offset <= 0 {
-		return
-	}
-
-	var area []byte
-	mode := q.readMode(offset + 4)
-	if mode == redirectMode1 {
-		countryOffset := q.readUInt24()
-		mode = q.readMode(countryOffset)
-		if mode == redirectMode2 {
-			c := q.readUInt24()
-			area = q.readString(c)
-		} else {
-			area = q.readString(countryOffset)
-		}
-	} else if mode == redirectMode2 {
-		countryOffset := q.readUInt24()
-		area = q.readString(countryOffset)
-	} else {
-		area = q.readString(offset + 4)
-	}
-
-	enc := simplifiedchinese.GBK.NewDecoder()
-	res.Area, _ = enc.String(string(area))
-
-	return
-}
-
-type ResultQQwry struct {
-	IP   string `json:"ip"`
-	Area string `json:"area"`
-}
-
-// readMode 获取偏移值类型
-func (q *QQwry) readMode(offset uint32) byte {
-	mode := q.readData(1, int64(offset))
-	return mode[0]
-}
-
-// readString 获取字符串
-func (q *QQwry) readString(offset uint32) []byte {
-	q.setOffset(int64(offset))
-	data := make([]byte, 0, 30)
-	for {
-		buf := q.readData(1)
-		if buf[0] == 0 {
-			break
-		}
-		data = append(data, buf[0])
-	}
-	return data
-}
-
-// searchIndex 查找索引位置
-func (q *QQwry) searchIndex(ip uint32) uint32 {
-	header := q.readData(8, 0)
-
-	start := binary.LittleEndian.Uint32(header[:4])
-	end := binary.LittleEndian.Uint32(header[4:])
-
-	for {
-		mid := q.getMiddleOffset(start, end)
-		buf := q.readData(indexLen, int64(mid))
-		_ip := binary.LittleEndian.Uint32(buf[:4])
-
-		if end-start == indexLen {
-			offset := byteToUInt32(buf[4:])
-			buf = q.readData(indexLen)
-			if ip < binary.LittleEndian.Uint32(buf[:4]) {
-				return offset
-			}
-			return 0
-		}
-
-		if _ip > ip {
-			end = mid
-		} else if _ip < ip {
-			start = mid
-		} else if _ip == ip {
-			return byteToUInt32(buf[4:])
-		}
-	}
-}
-
-// readUInt24
-func (q *QQwry) readUInt24() uint32 {
-	buf := q.readData(3)
-	return byteToUInt32(buf)
-}
-
-// getMiddleOffset
-func (q *QQwry) getMiddleOffset(start uint32, end uint32) uint32 {
-	records := ((end - start) / indexLen) >> 1
-	return start + records*indexLen
-}
-
-// byteToUInt32 将 byte 转换为uint32
-func byteToUInt32(data []byte) uint32 {
-	i := uint32(data[0]) & 0xff
-	i |= (uint32(data[1]) << 8) & 0xff00
-	i |= (uint32(data[2]) << 16) & 0xff0000
-	return i
-}

cmd/server/qqwry/qqwey.go

@@ -1,6 +0,0 @@
-package qqwry
-
-import _ "embed"
-
-//go:embed qqwry.dat
-var QQwryByte []byte

go.mod

@@ -196,6 +196,7 @@ require (
 	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
+	github.com/oschwald/maxminddb-golang v1.13.1 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.21 // indirect

go.sum

@@ -792,6 +792,8 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr
 github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A=
 github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU=
 github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE=
+github.com/oschwald/maxminddb-golang v1.13.1 h1:G3wwjdN9JmIK2o/ermkHM+98oX5fS+k5MbwsmL4MRQE=
+github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk4IsCNThNdTmnaBZ/F8=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=