IPv6 RC verison with IPv4 compatible with the old implementation

binding/golang/main.go

@@ -86,7 +86,7 @@ func testSearch() {
 	fmt.Printf(`ip2region xdb searcher test program
 source xdb: %s (%s, %s)
 type 'quit' to exit
-`, dbPath, searcher.GetIPVersion().Name, cachePolicy)
+`, dbPath, searcher.IPVersion().Name, cachePolicy)
 	reader := bufio.NewReader(os.Stdin)
 	for {
 		fmt.Print("ip2region>> ")
@@ -257,7 +257,7 @@ func createSearcher(dbPath string, cachePolicy string) (*xdb.Searcher, error) {
 			return nil, fmt.Errorf("failed to load content from '%s': %w", dbPath, err)
 		}
 
-		return xdb.NewWithBuffer(cBuff)
+		return xdb.NewWithBuffer(version, cBuff)
 	default:
 		return nil, fmt.Errorf("invalid cache policy `%s`, options: file/vectorIndex/content", cachePolicy)
 	}

binding/golang/xdb/searcher.go

@@ -67,7 +67,7 @@ func NewHeader(input []byte) (*Header, error) {
 	}
 
 	return &Header{
-		Version:       binary.LittleEndian.Uint16(input),
+		Version:       binary.LittleEndian.Uint16(input[0:]),
 		IndexPolicy:   IndexPolicy(binary.LittleEndian.Uint16(input[2:])),
 		CreatedAt:     binary.LittleEndian.Uint32(input[4:]),
 		StartIndexPtr: binary.LittleEndian.Uint32(input[8:]),
@@ -104,6 +104,7 @@ func baseNew(version *Version, dbFile string, vIndex []byte, cBuff []byte) (*Sea
 	// content buff first
 	if cBuff != nil {
 		return &Searcher{
+			version:     version,
 			vectorIndex: nil,
 			contentBuff: cBuff,
 		}, nil
@@ -130,15 +131,8 @@ func NewWithVectorIndex(version *Version, dbFile string, vIndex []byte) (*Search
 	return baseNew(version, dbFile, vIndex, nil)
 }
 
-func NewWithBuffer(cBuff []byte) (*Searcher, error) {
+func NewWithBuffer(version *Version, cBuff []byte) (*Searcher, error) {
-	versionNo := binary.LittleEndian.Uint16(cBuff[16:])
+	return baseNew(version, "", nil, cBuff)
-	if versionNo == IPv4VersionNo {
-		return baseNew(IPv4, "", nil, cBuff)
-	} else if versionNo == IPv6VersionNo {
-		return baseNew(IPv6, "", nil, cBuff)
-	} else {
-		return nil, fmt.Errorf("invalid version number `%d`", versionNo)
-	}
 }
 
 func (s *Searcher) Close() {
@@ -150,8 +144,8 @@ func (s *Searcher) Close() {
 	}
 }
 
-// GetIPVersion return the ip version
+// IPVersion return the ip version
-func (s *Searcher) GetIPVersion() *Version {
+func (s *Searcher) IPVersion() *Version {
 	return s.version
 }
 
@@ -219,9 +213,9 @@ func (s *Searcher) Search(ip []byte) (string, error) {
 		}
 
 		// decode the data step by step to reduce the unnecessary operations
-		if IPCompare(ip, buff[0:bytes]) < 0 {
+		if s.version.IPCompare(ip, buff[0:bytes]) < 0 {
 			h = m - 1
-		} else if IPCompare(ip, buff[bytes:dBytes]) > 0 {
+		} else if s.version.IPCompare(ip, buff[bytes:dBytes]) > 0 {
 			l = m + 1
 		} else {
 			dataLen = int(binary.LittleEndian.Uint16(buff[dBytes:]))

binding/golang/xdb/util.go

@@ -9,10 +9,10 @@
 package xdb
 
 import (
+	"bytes"
 	"embed"
 	"fmt"
 	"io"
-	"math/big"
 	"net"
 	"os"
 )
@@ -40,51 +40,19 @@ func IP2String(ip []byte) string {
 	return net.IP(ip[:]).String()
 }
 
-func IP2Long(ip []byte) *big.Int {
-	return big.NewInt(0).SetBytes(ip)
-}
-
 // IPCompare compares two IP addresses
 // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2
 func IPCompare(ip1, ip2 []byte) int {
-	for i := 0; i < len(ip1); i++ {
+	// for i := 0; i < len(ip1); i++ {
-		if ip1[i] < ip2[i] {
+	// 	if ip1[i] < ip2[i] {
-			return -1
+	// 		return -1
-		}
+	// 	}
-
+	// 	if ip1[i] > ip2[i] {
-		if ip1[i] > ip2[i] {
+	// 		return 1
-			return 1
+	// 	}
-		}
+	// }
-	}
+	// return 0
-
+	return bytes.Compare(ip1, ip2)
-	return 0
-}
-
-func IPAddOne(ip []byte) []byte {
-	var r = make([]byte, len(ip))
-	copy(r, ip)
-	for i := len(ip) - 1; i >= 0; i-- {
-		r[i]++
-		if r[i] != 0 { // No overflow
-			break
-		}
-	}
-
-	return r
-}
-
-func IPSubOne(ip []byte) []byte {
-	var r = make([]byte, len(ip))
-	copy(r, ip)
-	for i := len(ip) - 1; i >= 0; i-- {
-		if r[i] != 0 { // No borrow needed
-			r[i]--
-			break
-		}
-		r[i] = 0xFF // borrow from the next byte
-	}
-
-	return r
 }
 
 // LoadHeader load the header info from the specified handle

binding/golang/xdb/util_test.go

@@ -41,70 +41,8 @@ func TestIPCompare(t *testing.T) {
 	}
 }
 
-func TestIPAddOne(t *testing.T) {
-	var ipPairs = [][]string{
-		{"1.2.3.4", "1.2.3.5"},
-		{"2.3.4.5", "2.3.4.6"},
-		{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
-		{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"},
-		{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"},
-	}
-
-	for _, pairs := range ipPairs {
-		sip, err := ParseIP(pairs[0])
-		if err != nil {
-			t.Errorf("parse ip `%s`: %s\n", pairs[0], err)
-		}
-
-		eip, err := ParseIP(pairs[1])
-		if err != nil {
-			t.Errorf("parse ip `%s`: %s\n", pairs[1], err)
-		}
-
-		fmt.Printf("IPAddOne(%s) = %s ? %d\n",
-			pairs[0], pairs[1], IPCompare(IPAddOne(sip), eip))
-	}
-}
-
-func TestIPAddOne2(t *testing.T) {
-	var ip = []byte{0, 1, 2, 3}
-	nip := IPAddOne(ip)
-	fmt.Printf("nip: %+v, ip:%+v", ip, nip)
-}
-
-func TestIPSubOne(t *testing.T) {
-	var ipPairs = [][]string{
-		{"1.2.3.4", "1.2.3.5"},
-		{"2.3.4.5", "2.3.4.6"},
-		{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
-		{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"},
-		{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"},
-	}
-
-	for _, pairs := range ipPairs {
-		sip, err := ParseIP(pairs[0])
-		if err != nil {
-			t.Errorf("parse ip `%s`: %s\n", pairs[0], err)
-		}
-
-		eip, err := ParseIP(pairs[1])
-		if err != nil {
-			t.Errorf("parse ip `%s`: %s\n", pairs[1], err)
-		}
-
-		fmt.Printf("IPSubOne(%s) = %s ? %d\n",
-			pairs[1], pairs[0], IPCompare(IPSubOne(eip), sip))
-	}
-}
-
-func TestIPSubOne2(t *testing.T) {
-	var ip = []byte{0, 1, 2, 3}
-	nip := IPSubOne(ip)
-	fmt.Printf("nip: %+v, ip:%+v", ip, nip)
-}
-
 func TestLoadVectorIndex(t *testing.T) {
-	vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region.xdb")
+	vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region_v4.xdb")
 	if err != nil {
 		fmt.Printf("failed to load vector index: %s\n", err)
 		return
@@ -114,7 +52,7 @@ func TestLoadVectorIndex(t *testing.T) {
 }
 
 func TestLoadContent(t *testing.T) {
-	buff, err := LoadContentFromFile("../../../data/ip2region.xdb")
+	buff, err := LoadContentFromFile("../../../data/ip2region_v4.xdb")
 	if err != nil {
 		fmt.Printf("failed to load xdb content: %s\n", err)
 		return
@@ -124,15 +62,17 @@ func TestLoadContent(t *testing.T) {
 }
 
 func TestLoadHeader(t *testing.T) {
-	header, err := LoadHeaderFromFile("../../../data/ip2region.xdb")
+	header, err := LoadHeaderFromFile("../../../data/ip2region_v4.xdb")
 	if err != nil {
 		fmt.Printf("failed to load xdb header info: %s\n", err)
 		return
 	}
 
-	fmt.Printf("Version        : %d\n", header.Version)
+	fmt.Printf("Version         : %d\n", header.Version)
-	fmt.Printf("IndexPolicy    : %s\n", header.IndexPolicy.String())
+	fmt.Printf("IndexPolicy     : %s\n", header.IndexPolicy.String())
-	fmt.Printf("CreatedAt      : %d(%s)\n", header.CreatedAt, time.Unix(int64(header.CreatedAt), 0).Format(time.RFC3339))
+	fmt.Printf("CreatedAt       : %d(%s)\n", header.CreatedAt, time.Unix(int64(header.CreatedAt), 0).Format(time.RFC3339))
-	fmt.Printf("StartIndexPtr  : %d\n", header.StartIndexPtr)
+	fmt.Printf("StartIndexPtr   : %d\n", header.StartIndexPtr)
-	fmt.Printf("EndIndexPtr    : %d\n", header.EndIndexPtr)
+	fmt.Printf("EndIndexPtr     : %d\n", header.EndIndexPtr)
+	fmt.Printf("IPVersion       : %d\n", header.IPVersion)
+	fmt.Printf("RuntimePtrBytes : %d\n", header.RuntimePtrBytes)
 }

binding/golang/xdb/version.go

@@ -5,6 +5,7 @@
 package xdb
 
 import (
+	"bytes"
 	"fmt"
 	"strings"
 )
@@ -14,6 +15,9 @@ type Version struct {
 	Name             string
 	Bytes            int
 	SegmentIndexSize int
+
+	// function to compare two ips
+	IPCompare func([]byte, []byte) int
 }
 
 const (
@@ -27,23 +31,33 @@ var (
 		Id:               IPv4VersionNo,
 		Name:             "IPv4",
 		Bytes:            4,
-		SegmentIndexSize: 14, // 4 + 4 + 2 + 4
+		SegmentIndexSize: 14, // 4 + 4 + 2 + 4,
+		IPCompare: func(ip1, ip2 []byte) int {
+			// ip1 - with Bit endian parsed from an input
+			// ip2 - with Little endian read from the xdb index
+			ip2[0], ip2[3] = ip2[3], ip2[0]
+			ip2[1], ip2[2] = ip2[2], ip2[1]
+			return bytes.Compare(ip1, ip2)
+		},
 	}
 	IPv6 = &Version{
 		Id:               IPv6VersionNo,
 		Name:             "IPv6",
 		Bytes:            16,
-		SegmentIndexSize: 38, // 16 + 16 + 2 + 4
+		SegmentIndexSize: 38, // 16 + 16 + 2 + 4,
+		IPCompare: func(ip1, ip2 []byte) int {
+			return bytes.Compare(ip1, ip2)
+		},
 	}
 )
 
 func VersionFromIP(ip string) (*Version, error) {
-	bytes, err := ParseIP(ip)
+	r, err := ParseIP(ip)
 	if err != nil {
 		return IPvx, fmt.Errorf("parse ip fail: %w", err)
 	}
 
-	if len(bytes) == 4 {
+	if len(r) == 4 {
 		return IPv4, nil
 	}
 

maker/golang/xdb/maker.go

@@ -302,9 +302,15 @@ func (m *Maker) Start() error {
 				return fmt.Errorf("segment index ptr exceed the max length of %d", math.MaxUint32)
 			}
 
-			// encode the segment index
+			// encode the segment index.
-			copy(indexBuff[0:], s.StartIP)
+			// @Note by Leon at 2025/09/05:
-			copy(indexBuff[len(s.StartIP):], s.EndIP)
+			// This is a tough decision since the directly copy of the bytes will make everything simpler.
+			// But in order to compatible with the old searcher implementation we had to keep encoding the IPv4 bytes with little endian.
+			// @TODO: we may choose to use the big-endian byte order in the future.
+			// But now compatibility is the most important !!!
+
+			m.version.PutBytes(indexBuff[0:], s.StartIP)
+			m.version.PutBytes(indexBuff[len(s.StartIP):], s.EndIP)
 			_offset = len(s.StartIP) + len(s.EndIP)
 			binary.LittleEndian.PutUint16(indexBuff[_offset:], uint16(dataLen))
 			binary.LittleEndian.PutUint32(indexBuff[_offset+2:], dataPtr)

maker/golang/xdb/searcher.go

@@ -151,9 +151,9 @@ func (s *Searcher) Search(ip []byte) (string, int, error) {
 		}
 
 		// decode the data step by step to reduce the unnecessary calculations
-		if IPCompare(ip, buff[0:bytes]) < 0 {
+		if s.version.IPCompare(ip, buff[0:bytes]) < 0 {
 			h = m - 1
-		} else if IPCompare(ip, buff[bytes:tBytes]) > 0 {
+		} else if s.version.IPCompare(ip, buff[bytes:tBytes]) > 0 {
 			l = m + 1
 		} else {
 			dataLen = int(binary.LittleEndian.Uint16(buff[tBytes:]))

maker/golang/xdb/util.go

@@ -6,6 +6,7 @@ package xdb
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"math/big"
 	"net"
@@ -45,17 +46,18 @@ func IP2Long(ip []byte) *big.Int {
 // IPCompare compares two IP addresses
 // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2
 func IPCompare(ip1, ip2 []byte) int {
-	for i := 0; i < len(ip1); i++ {
+	// for i := 0; i < len(ip1); i++ {
-		if ip1[i] < ip2[i] {
+	// 	if ip1[i] < ip2[i] {
-			return -1
+	// 		return -1
-		}
+	// 	}
 
-		if ip1[i] > ip2[i] {
+	// 	if ip1[i] > ip2[i] {
-			return 1
+	// 		return 1
-		}
+	// 	}
-	}
+	// }
 
-	return 0
+	// return 0
+	return bytes.Compare(ip1, ip2)
 }
 
 func IPAddOne(ip []byte) []byte {

maker/golang/xdb/version.go

@@ -5,6 +5,7 @@
 package xdb
 
 import (
+	"bytes"
 	"fmt"
 	"strings"
 )
@@ -19,6 +20,16 @@ type Version struct {
 	Name             string
 	Bytes            int
 	SegmentIndexSize int
+
+	// bytes encode
+	PutBytes func([]byte, []byte) int
+
+	// ip compares
+	IPCompare func([]byte, []byte) int
+}
+
+func (v *Version) String() string {
+	return fmt.Sprintf("{Id:%d, Name:%d, Bytes:%d, IndexSize: %d}", v.Id, v.Name, v.Bytes, v.SegmentIndexSize)
 }
 
 var (
@@ -27,23 +38,46 @@ var (
 		Id:               4,
 		Name:             "IPv4",
 		Bytes:            4,
-		SegmentIndexSize: 14, // 4 + 4 + 2 + 4
+		SegmentIndexSize: 14, // 4 + 4 + 2 + 4,
+		PutBytes: func(buff []byte, ip []byte) int {
+			// binary.LittleEndian.PutUint32(buff, binary.BigEndian.Uint32(ip))
+			// Little Endian byte order for compatible with the old searcher implementation
+			buff[0] = ip[3]
+			buff[1] = ip[2]
+			buff[2] = ip[1]
+			buff[3] = ip[0]
+			return len(ip)
+		},
+		IPCompare: func(ip1 []byte, ip2 []byte) int {
+			// ip1 - with Bit endian parsed from an input
+			// ip2 - with Little endian read from the xdb index
+			ip2[0], ip2[3] = ip2[3], ip2[0]
+			ip2[1], ip2[2] = ip2[2], ip2[1]
+			return bytes.Compare(ip1, ip2)
+		},
 	}
 	IPv6 = &Version{
 		Id:               6,
 		Name:             "IPv6",
 		Bytes:            16,
-		SegmentIndexSize: 38, // 16 + 16 + 2 + 4
+		SegmentIndexSize: 38, // 16 + 16 + 2 + 4,
+		// Big Endian byte order to follow the network byte order
+		PutBytes: func(buff []byte, ip []byte) int {
+			return copy(buff, ip)
+		},
+		IPCompare: func(ip1, ip2 []byte) int {
+			return bytes.Compare(ip1, ip2)
+		},
 	}
 )
 
 func VersionFromIP(ip string) (*Version, error) {
-	bytes, err := ParseIP(ip)
+	r, err := ParseIP(ip)
 	if err != nil {
 		return IPvx, fmt.Errorf("parse ip fail: %w", err)
 	}
 
-	if len(bytes) == 4 {
+	if len(r) == 4 {
 		return IPv4, nil
 	}