Merge pull request #462 from lionsoul2014/opt_golang_ip_util

optimize the golang ip utils

.gitignore

@@ -29,10 +29,6 @@ META-INF/
 .vscode
 
 # binding
-/v1.0//binding/java/classes/
-/v1.0//binding/java/doc/
-/v1.0//binding/java/target/
-/v1.0//binding/java/*.jar
 /binding/java/classes/
 /binding/java/doc/
 /binding/java/target/
@@ -61,18 +57,12 @@ target
 
 # VS ignore cases
 /**/*.sln
-/v1.0/binding/c#/**/.vs/
-/v1.0/binding/c#/**/packages
-/v1.0/binding/c#/**/bin
-/v1.0/binding/c#/**/obj
 /binding/c#/**/.vs/
 /binding/c#/**/packages
 /binding/c#/**/bin
 /binding/c#/**/obj
 
 # Nodejs
-/v1.0/binding/nodejs/tests/unitTests/__snapshots__
-/v1.0/binding/nodejs/coverage
 /binding/nodejs/tests/unitTests/__snapshots__
 /binding/nodejs/coverage
 /binding/nodejs/node_modules

maker/golang/cmd/bench.go

@@ -90,7 +90,7 @@ func Bench() {
 
 	var count, errCount, tStart = 0, 0, time.Now()
 	slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile)
-	var iErr = xdb.IterateSegments(handle, nil, nil, func(seg *xdb.Segment) error {
+	_, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error {
 		var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
 		slog.Debug("try to bench", "segment", l)
 		// mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)

maker/golang/xdb/editor.go

@@ -60,7 +60,7 @@ func (e *Editor) loadSegments() error {
 	var segments []*Segment
 	var sorting = false
 
-	var iErr = IterateSegments(e.srcHandle, func(l string) {
+	_, _, iErr := IterateSegments(e.srcHandle, true, func(l string) {
 		// do nothing here
 	}, nil, func(seg *Segment) error {
 		// version check
@@ -302,7 +302,7 @@ func (e *Editor) PutFile(src string) (int, int, error) {
 	}
 
 	var oldRows, newRows = 0, 0
-	iErr := IterateSegments(handle, func(l string) {
+	_, _, iErr := IterateSegments(handle, true, func(l string) {
 		// do nothing here
 	}, nil, func(seg *Segment) error {
 		o, n, err := e.PutSegment(seg)

maker/golang/xdb/maker.go

@@ -168,7 +168,7 @@ func (m *Maker) loadSegments() error {
 	var tStart = time.Now()
 	var sorting = false
 
-	var iErr = IterateSegments(m.srcHandle, func(l string) {
+	_, mergeCount, iErr := IterateSegments(m.srcHandle, true, func(l string) {
 		slog.Debug("loaded", "segment", l)
 	}, func(region string) (string, error) {
 		// apply the field filter
@@ -217,7 +217,7 @@ func (m *Maker) loadSegments() error {
 		}
 	}
 
-	slog.Info("all segments loaded", "length", len(m.segments), "sorting", sorting, "elapsed", time.Since(tStart))
+	slog.Info("all segments loaded", "length", len(m.segments), "merged", mergeCount, "sorting", sorting, "elapsed", time.Since(tStart))
 	return nil
 }
 

maker/golang/xdb/processor.go

@@ -62,7 +62,7 @@ func (p *Processor) loadSegments() error {
 	slog.Info("try to load the segments ... ")
 	var tStart = time.Now()
 
-	var iErr = IterateSegments(p.srcHandle, func(l string) {
+	_, mergeCount, iErr := IterateSegments(p.srcHandle, true, func(l string) {
 		slog.Debug("loaded", "segment", l)
 	}, func(region string) (string, error) {
 		if p.clearBasedIndex > -1 {
@@ -108,7 +108,7 @@ func (p *Processor) loadSegments() error {
 		return fmt.Errorf("failed to load segments: %s", iErr)
 	}
 
-	slog.Info("all segments loaded", "length", len(p.segments), "elapsed", time.Since(tStart))
+	slog.Info("all segments loaded", "length", len(p.segments), "merged", mergeCount, "elapsed", time.Since(tStart))
 	return nil
 }
 

maker/golang/xdb/util.go

@@ -87,22 +87,62 @@ func IPSubOne(ip []byte) []byte {
 	return r
 }
 
-func IPMiddle(sip, eip []byte) []byte {
+// IPSub Sub the spcecified two byte ip
-	var result = make([]byte, len(sip))
+func IPSub(sip, eip []byte) ([]byte, error) {
-	var carry uint16 = 0
+	if len(sip) != len(eip) {
+		return []byte{}, fmt.Errorf("length of the two ips are not the same")
+	}
+
+	var carry uint16 = 0
+	var result = make([]byte, len(sip)+1)
 
-	// Add the two addresses with carry
 	for i := len(sip) - 1; i >= 0; i-- {
 		sum := uint16(sip[i]) + uint16(eip[i]) + carry
-		result[i] = byte(sum >> 0x01) // Divide by 2
+		result[i+1] = byte(sum) // Store standard 8-bit result
-		carry = (sum & 0x01) << 7     // Carry for next byte (shift to MSB)
+		carry = sum >> 8        // Extract the 1-bit carry for the next byte
+	}
+
+	// check and append the carry
+	if carry > 0 {
+		result[0] = byte(carry)
+		return result, nil
+	} else {
+		return result[1:], nil
+	}
+}
+
+// IPHalf get the half value of an input byte ip
+func IPHalf(ip []byte) []byte {
+	var length = len(ip)
+	var result = make([]byte, length)
+	// Tracks the bit falling off from the previous byte
+	var carry byte = 0
+
+	for i := 0; i < length; i++ {
+		// 1. Shift current byte right by 1
+		// 2. Or (|) with the carry from the previous byte (shifted to the MSB position)
+		result[i] = (ip[i] >> 1) | (carry << 7)
+
+		// 3. Capture the Least Significant Bit (LSB) to use as carry for the next byte
+		carry = ip[i] & 1
 	}
 
 	return result
 }
 
-func IterateSegments(handle *os.File, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) error {
+// IPMiddle get the middle value of two input ip address
+func IPMiddle(sip, eip []byte) ([]byte, error) {
+	buf, err := IPSub(sip, eip)
+	if err != nil {
+		return []byte{}, fmt.Errorf("IPSub(%s, %s): %w", IP2String(sip), IP2String(eip), err)
+	}
+
+	return IPHalf(buf), nil
+}
+
+func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) {
 	var last *Segment = nil
+	var totalCount, mergeCount = 0, 0
 	var scanner = bufio.NewScanner(handle)
 	scanner.Split(bufio.ScanLines)
 	for scanner.Scan() {
@@ -115,31 +155,32 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
 			continue
 		}
 
+		totalCount++
 		if before != nil {
 			before(l)
 		}
 
 		var ps = strings.SplitN(l, "|", 3)
 		if len(ps) != 3 {
-			return fmt.Errorf("invalid ip segment line `%s`", l)
+			return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l)
 		}
 
 		sip, err := ParseIP(ps[0])
 		if err != nil {
-			return fmt.Errorf("check start ip `%s`: %s", ps[0], err)
+			return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err)
 		}
 
 		eip, err := ParseIP(ps[1])
 		if err != nil {
-			return fmt.Errorf("check end ip `%s`: %s", ps[1], err)
+			return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err)
 		}
 
 		if len(sip) != len(eip) {
-			return fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l)
+			return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l)
 		}
 
 		if IPCompare(sip, eip) > 0 {
-			return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
+			return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
 		}
 
 		// Allow empty region info since 2024/09/24
@@ -152,7 +193,7 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
 		if filter != nil {
 			region, err = filter(ps[2])
 			if err != nil {
-				return fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
+				return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
 			}
 		}
 
@@ -168,15 +209,16 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
 		if last == nil {
 			last = seg
 			continue
-		} else if last.Region == seg.Region {
+		} else if autoMerge && last.Region == seg.Region {
 			if err = seg.RightBehind(last); err == nil {
+				mergeCount++
 				last.EndIP = seg.EndIP
 				continue
 			}
 		}
 
 		if err = done(last); err != nil {
-			return err
+			return totalCount, mergeCount, err
 		}
 
 		// reset the last
@@ -185,10 +227,10 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
 
 	// process the last segment
 	if last != nil {
-		return done(last)
+		return totalCount, mergeCount, done(last)
 	}
 
-	return nil
+	return totalCount, mergeCount, nil
 }
 
 func CheckSegments(segList []*Segment) error {

maker/golang/xdb/util_test.go

@@ -5,6 +5,7 @@
 package xdb
 
 import (
+	"encoding/binary"
 	"fmt"
 	"os"
 	"testing"
@@ -99,6 +100,102 @@ func TestIPSubOne2(t *testing.T) {
 	fmt.Printf("nip: %+v, ip:%+v", ip, nip)
 }
 
+func TestIPSub(t *testing.T) {
+	var strToSub = "1.2.3.4"
+	bytesToSub, err := ParseIP(strToSub)
+	if err != nil {
+		t.Fatalf("failed to parse ip %s", strToSub)
+	}
+	var intToSub = int(binary.BigEndian.Uint32(bytesToSub))
+	t.Logf("to sub ip: %d -> %s", intToSub, strToSub)
+
+	counter := 0
+	buf := make([]byte, 4)
+	for i := 0; i < 0x2FFFFFFF; i++ {
+		binary.BigEndian.PutUint32(buf, uint32(i))
+		subVal, err := IPSub(buf, bytesToSub)
+		if err != nil {
+			t.Fatalf("failed to IPSub(%s,%s): %s", IP2String(buf), strToSub, err)
+		}
+
+		// do it as two integers
+		byteSub := int(binary.BigEndian.Uint32(subVal))
+		intSub := i + intToSub
+		if byteSub != intSub {
+			t.Fatal("byte and int sub value are not the same")
+		}
+
+		counter++
+	}
+
+	t.Logf("test done with %d ips", counter)
+}
+
+func TestIPHalf(t *testing.T) {
+	var buf = make([]byte, 4)
+	for i := 0; i < 0xFFFFFFFF; i++ {
+		binary.BigEndian.PutUint32(buf, uint32(i))
+		half := IPHalf(buf)
+
+		// do it as two integers
+		byteMiddle := binary.BigEndian.Uint32(half)
+		intMidle := i >> 1
+		if byteMiddle != uint32(intMidle) {
+			t.Fatal("byte middle and int middle are not the same")
+		}
+	}
+}
+
+func TestSubOverflow(t *testing.T) {
+	var ip1Str = "255.255.255.250"
+	ip1Bytes, err := ParseIP(ip1Str)
+	if err != nil {
+		t.Fatalf("failed to ParseIP(%s): %s", ip1Str, err)
+	}
+
+	var buff = make([]byte, 4)
+	for i := 0; i < 10; i++ {
+		binary.BigEndian.PutUint32(buff, uint32(i))
+		ipSub, err := IPSub(ip1Bytes, buff)
+		if err != nil {
+			t.Fatalf("failed to IPSub(%s, %s): %s", ip1Str, IP2String(buff), err)
+		}
+
+		t.Logf("IPSub(%s, %s) = %+v", ip1Str, IP2String(buff), ipSub)
+	}
+}
+
+func TestIPMiddle(t *testing.T) {
+	var sIPStr = "0.0.0.0"
+	sBytes, err := ParseIP(sIPStr)
+	if err != nil {
+		t.Fatalf("failed to parse ip %s", sIPStr)
+	}
+	var sInt = int(binary.BigEndian.Uint32(sBytes))
+	t.Logf("start ip: %d -> %s", sInt, sIPStr)
+
+	counter := 0
+	buf := make([]byte, 4)
+	for i := 0; i < 0x0FFFFFFF; i++ {
+		binary.BigEndian.PutUint32(buf, uint32(i))
+		midVal, err := IPMiddle(sBytes, buf)
+		if err != nil {
+			t.Fatalf("failed to IPMiddle(%s,%s): %s", sIPStr, IP2String(buf), err)
+		}
+
+		// do it as two integers
+		byteMid := int(binary.BigEndian.Uint32(midVal))
+		intMid := (sInt + i) >> 1
+		if byteMid != intMid {
+			t.Fatal("byte and int middle value are not the same")
+		}
+
+		counter++
+	}
+
+	t.Logf("test done with %d ips", counter)
+}
+
 func TestSplitSegmentV4(t *testing.T) {
 	// var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信"
 	// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
@@ -161,7 +258,7 @@ func TestIterateSegments(t *testing.T) {
 		t.Fatalf("failed to open tests file: %s", err)
 	}
 
-	_ = IterateSegments(handle, func(l string) {
+	_, _, _ = IterateSegments(handle, true, func(l string) {
 		// fmt.Printf("load segment: `%s`\n", l)
 	}, nil, func(seg *Segment) error {
 		fmt.Printf("get segment: `%s`\n", seg)