xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #462 from lionsoul2014/opt_golang_ip_util 

optimize the golang ip utils
This commit is contained in:
Leon / 狮子的魂 2026-04-02 11:28:09 +08:00 committed by GitHub
parent 8a7b088216 62aa42e05e
commit 1209b72452
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 164 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.gitignore vendored
View File

@ -29,10 +29,6 @@ META-INF/
.vscode
# binding
/v1.0//binding/java/classes/
/v1.0//binding/java/doc/
/v1.0//binding/java/target/
/v1.0//binding/java/*.jar
/binding/java/classes/
/binding/java/doc/
/binding/java/target/
@ -61,18 +57,12 @@ target
# VS ignore cases
/**/*.sln
/v1.0/binding/c#/**/.vs/
/v1.0/binding/c#/**/packages
/v1.0/binding/c#/**/bin
/v1.0/binding/c#/**/obj
/binding/c#/**/.vs/
/binding/c#/**/packages
/binding/c#/**/bin
/binding/c#/**/obj
# Nodejs
/v1.0/binding/nodejs/tests/unitTests/__snapshots__
/v1.0/binding/nodejs/coverage
/binding/nodejs/tests/unitTests/__snapshots__
/binding/nodejs/coverage
/binding/nodejs/node_modules

2
maker/golang/cmd/bench.go
View File

@ -90,7 +90,7 @@ func Bench() {
var count, errCount, tStart = 0, 0, time.Now()
slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile)
var iErr = xdb.IterateSegments(handle, nil, nil, func(seg *xdb.Segment) error {
_, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error {
var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
slog.Debug("try to bench", "segment", l)
// mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)

4
maker/golang/xdb/editor.go
View File

@ -60,7 +60,7 @@ func (e *Editor) loadSegments() error {
var segments []*Segment
var sorting = false
var iErr = IterateSegments(e.srcHandle, func(l string) {
_, _, iErr := IterateSegments(e.srcHandle, true, func(l string) {
// do nothing here
}, nil, func(seg *Segment) error {
// version check
@ -302,7 +302,7 @@ func (e *Editor) PutFile(src string) (int, int, error) {
}
var oldRows, newRows = 0, 0
iErr := IterateSegments(handle, func(l string) {
_, _, iErr := IterateSegments(handle, true, func(l string) {
// do nothing here
}, nil, func(seg *Segment) error {
o, n, err := e.PutSegment(seg)

4
maker/golang/xdb/maker.go
View File

@ -168,7 +168,7 @@ func (m *Maker) loadSegments() error {
var tStart = time.Now()
var sorting = false
var iErr = IterateSegments(m.srcHandle, func(l string) {
_, mergeCount, iErr := IterateSegments(m.srcHandle, true, func(l string) {
slog.Debug("loaded", "segment", l)
}, func(region string) (string, error) {
// apply the field filter
@ -217,7 +217,7 @@ func (m *Maker) loadSegments() error {
}
}
slog.Info("all segments loaded", "length", len(m.segments), "sorting", sorting, "elapsed", time.Since(tStart))
slog.Info("all segments loaded", "length", len(m.segments), "merged", mergeCount, "sorting", sorting, "elapsed", time.Since(tStart))
return nil
}

4
maker/golang/xdb/processor.go
View File

@ -62,7 +62,7 @@ func (p *Processor) loadSegments() error {
slog.Info("try to load the segments ... ")
var tStart = time.Now()
var iErr = IterateSegments(p.srcHandle, func(l string) {
_, mergeCount, iErr := IterateSegments(p.srcHandle, true, func(l string) {
slog.Debug("loaded", "segment", l)
}, func(region string) (string, error) {
if p.clearBasedIndex > -1 {
@ -108,7 +108,7 @@ func (p *Processor) loadSegments() error {
return fmt.Errorf("failed to load segments: %s", iErr)
}
slog.Info("all segments loaded", "length", len(p.segments), "elapsed", time.Since(tStart))
slog.Info("all segments loaded", "length", len(p.segments), "merged", mergeCount, "elapsed", time.Since(tStart))
return nil
}

76
maker/golang/xdb/util.go
View File

@ -87,22 +87,62 @@ func IPSubOne(ip []byte) []byte {
return r
}
func IPMiddle(sip, eip []byte) []byte {
var result = make([]byte, len(sip))
var carry uint16 = 0
// IPSub Sub the spcecified two byte ip
func IPSub(sip, eip []byte) ([]byte, error) {
if len(sip) != len(eip) {
return []byte{}, fmt.Errorf("length of the two ips are not the same")
}
var carry uint16 = 0
var result = make([]byte, len(sip)+1)
// Add the two addresses with carry
for i := len(sip) - 1; i >= 0; i-- {
sum := uint16(sip[i]) + uint16(eip[i]) + carry
result[i] = byte(sum >> 0x01) // Divide by 2
carry = (sum & 0x01) << 7 // Carry for next byte (shift to MSB)
result[i+1] = byte(sum) // Store standard 8-bit result
carry = sum >> 8 // Extract the 1-bit carry for the next byte
}
// check and append the carry
if carry > 0 {
result[0] = byte(carry)
return result, nil
} else {
return result[1:], nil
}
}
// IPHalf get the half value of an input byte ip
func IPHalf(ip []byte) []byte {
var length = len(ip)
var result = make([]byte, length)
// Tracks the bit falling off from the previous byte
var carry byte = 0
for i := 0; i < length; i++ {
// 1. Shift current byte right by 1
// 2. Or (|) with the carry from the previous byte (shifted to the MSB position)
result[i] = (ip[i] >> 1) | (carry << 7)
// 3. Capture the Least Significant Bit (LSB) to use as carry for the next byte
carry = ip[i] & 1
}
return result
}
func IterateSegments(handle *os.File, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) error {
// IPMiddle get the middle value of two input ip address
func IPMiddle(sip, eip []byte) ([]byte, error) {
buf, err := IPSub(sip, eip)
if err != nil {
return []byte{}, fmt.Errorf("IPSub(%s, %s): %w", IP2String(sip), IP2String(eip), err)
}
return IPHalf(buf), nil
}
func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) {
var last *Segment = nil
var totalCount, mergeCount = 0, 0
var scanner = bufio.NewScanner(handle)
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
@ -115,31 +155,32 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
continue
}
totalCount++
if before != nil {
before(l)
}
var ps = strings.SplitN(l, "|", 3)
if len(ps) != 3 {
return fmt.Errorf("invalid ip segment line `%s`", l)
return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l)
}
sip, err := ParseIP(ps[0])
if err != nil {
return fmt.Errorf("check start ip `%s`: %s", ps[0], err)
return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err)
}
eip, err := ParseIP(ps[1])
if err != nil {
return fmt.Errorf("check end ip `%s`: %s", ps[1], err)
return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err)
}
if len(sip) != len(eip) {
return fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l)
return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l)
}
if IPCompare(sip, eip) > 0 {
return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
}
// Allow empty region info since 2024/09/24
@ -152,7 +193,7 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
if filter != nil {
region, err = filter(ps[2])
if err != nil {
return fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
}
}
@ -168,15 +209,16 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
if last == nil {
last = seg
continue
} else if last.Region == seg.Region {
} else if autoMerge && last.Region == seg.Region {
if err = seg.RightBehind(last); err == nil {
mergeCount++
last.EndIP = seg.EndIP
continue
}
}
if err = done(last); err != nil {
return err
return totalCount, mergeCount, err
}
// reset the last
@ -185,10 +227,10 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
// process the last segment
if last != nil {
return done(last)
return totalCount, mergeCount, done(last)
}
return nil
return totalCount, mergeCount, nil
}
func CheckSegments(segList []*Segment) error {

99
maker/golang/xdb/util_test.go
View File

@ -5,6 +5,7 @@
package xdb
import (
"encoding/binary"
"fmt"
"os"
"testing"
@ -99,6 +100,102 @@ func TestIPSubOne2(t *testing.T) {
fmt.Printf("nip: %+v, ip:%+v", ip, nip)
}
func TestIPSub(t *testing.T) {
var strToSub = "1.2.3.4"
bytesToSub, err := ParseIP(strToSub)
if err != nil {
t.Fatalf("failed to parse ip %s", strToSub)
}
var intToSub = int(binary.BigEndian.Uint32(bytesToSub))
t.Logf("to sub ip: %d -> %s", intToSub, strToSub)
counter := 0
buf := make([]byte, 4)
for i := 0; i < 0x2FFFFFFF; i++ {
binary.BigEndian.PutUint32(buf, uint32(i))
subVal, err := IPSub(buf, bytesToSub)
if err != nil {
t.Fatalf("failed to IPSub(%s,%s): %s", IP2String(buf), strToSub, err)
}
// do it as two integers
byteSub := int(binary.BigEndian.Uint32(subVal))
intSub := i + intToSub
if byteSub != intSub {
t.Fatal("byte and int sub value are not the same")
}
counter++
}
t.Logf("test done with %d ips", counter)
}
func TestIPHalf(t *testing.T) {
var buf = make([]byte, 4)
for i := 0; i < 0xFFFFFFFF; i++ {
binary.BigEndian.PutUint32(buf, uint32(i))
half := IPHalf(buf)
// do it as two integers
byteMiddle := binary.BigEndian.Uint32(half)
intMidle := i >> 1
if byteMiddle != uint32(intMidle) {
t.Fatal("byte middle and int middle are not the same")
}
}
}
func TestSubOverflow(t *testing.T) {
var ip1Str = "255.255.255.250"
ip1Bytes, err := ParseIP(ip1Str)
if err != nil {
t.Fatalf("failed to ParseIP(%s): %s", ip1Str, err)
}
var buff = make([]byte, 4)
for i := 0; i < 10; i++ {
binary.BigEndian.PutUint32(buff, uint32(i))
ipSub, err := IPSub(ip1Bytes, buff)
if err != nil {
t.Fatalf("failed to IPSub(%s, %s): %s", ip1Str, IP2String(buff), err)
}
t.Logf("IPSub(%s, %s) = %+v", ip1Str, IP2String(buff), ipSub)
}
}
func TestIPMiddle(t *testing.T) {
var sIPStr = "0.0.0.0"
sBytes, err := ParseIP(sIPStr)
if err != nil {
t.Fatalf("failed to parse ip %s", sIPStr)
}
var sInt = int(binary.BigEndian.Uint32(sBytes))
t.Logf("start ip: %d -> %s", sInt, sIPStr)
counter := 0
buf := make([]byte, 4)
for i := 0; i < 0x0FFFFFFF; i++ {
binary.BigEndian.PutUint32(buf, uint32(i))
midVal, err := IPMiddle(sBytes, buf)
if err != nil {
t.Fatalf("failed to IPMiddle(%s,%s): %s", sIPStr, IP2String(buf), err)
}
// do it as two integers
byteMid := int(binary.BigEndian.Uint32(midVal))
intMid := (sInt + i) >> 1
if byteMid != intMid {
t.Fatal("byte and int middle value are not the same")
}
counter++
}
t.Logf("test done with %d ips", counter)
}
func TestSplitSegmentV4(t *testing.T) {
// var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信"
// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
@ -161,7 +258,7 @@ func TestIterateSegments(t *testing.T) {
t.Fatalf("failed to open tests file: %s", err)
}
_ = IterateSegments(handle, func(l string) {
_, _, _ = IterateSegments(handle, true, func(l string) {
// fmt.Printf("load segment: `%s`\n", l)
}, nil, func(seg *Segment) error {
fmt.Printf("get segment: `%s`\n", seg)