diff --git a/binding/golang/main.go b/binding/golang/main.go index 68ad82b..444e5c0 100644 --- a/binding/golang/main.go +++ b/binding/golang/main.go @@ -86,7 +86,7 @@ func testSearch() { fmt.Printf(`ip2region xdb searcher test program source xdb: %s (%s, %s) type 'quit' to exit -`, dbPath, searcher.GetIPVersion().Name, cachePolicy) +`, dbPath, searcher.IPVersion().Name, cachePolicy) reader := bufio.NewReader(os.Stdin) for { fmt.Print("ip2region>> ") @@ -257,7 +257,7 @@ func createSearcher(dbPath string, cachePolicy string) (*xdb.Searcher, error) { return nil, fmt.Errorf("failed to load content from '%s': %w", dbPath, err) } - return xdb.NewWithBuffer(cBuff) + return xdb.NewWithBuffer(version, cBuff) default: return nil, fmt.Errorf("invalid cache policy `%s`, options: file/vectorIndex/content", cachePolicy) } diff --git a/binding/golang/xdb/searcher.go b/binding/golang/xdb/searcher.go index 3cd8e18..b1044b8 100644 --- a/binding/golang/xdb/searcher.go +++ b/binding/golang/xdb/searcher.go @@ -67,7 +67,7 @@ func NewHeader(input []byte) (*Header, error) { } return &Header{ - Version: binary.LittleEndian.Uint16(input), + Version: binary.LittleEndian.Uint16(input[0:]), IndexPolicy: IndexPolicy(binary.LittleEndian.Uint16(input[2:])), CreatedAt: binary.LittleEndian.Uint32(input[4:]), StartIndexPtr: binary.LittleEndian.Uint32(input[8:]), @@ -104,6 +104,7 @@ func baseNew(version *Version, dbFile string, vIndex []byte, cBuff []byte) (*Sea // content buff first if cBuff != nil { return &Searcher{ + version: version, vectorIndex: nil, contentBuff: cBuff, }, nil @@ -130,15 +131,8 @@ func NewWithVectorIndex(version *Version, dbFile string, vIndex []byte) (*Search return baseNew(version, dbFile, vIndex, nil) } -func NewWithBuffer(cBuff []byte) (*Searcher, error) { - versionNo := binary.LittleEndian.Uint16(cBuff[16:]) - if versionNo == IPv4VersionNo { - return baseNew(IPv4, "", nil, cBuff) - } else if versionNo == IPv6VersionNo { - return baseNew(IPv6, "", nil, cBuff) - } else { - return nil, fmt.Errorf("invalid version number `%d`", versionNo) - } +func NewWithBuffer(version *Version, cBuff []byte) (*Searcher, error) { + return baseNew(version, "", nil, cBuff) } func (s *Searcher) Close() { @@ -150,8 +144,8 @@ func (s *Searcher) Close() { } } -// GetIPVersion return the ip version -func (s *Searcher) GetIPVersion() *Version { +// IPVersion return the ip version +func (s *Searcher) IPVersion() *Version { return s.version } @@ -219,9 +213,9 @@ func (s *Searcher) Search(ip []byte) (string, error) { } // decode the data step by step to reduce the unnecessary operations - if IPCompare(ip, buff[0:bytes]) < 0 { + if s.version.IPCompare(ip, buff[0:bytes]) < 0 { h = m - 1 - } else if IPCompare(ip, buff[bytes:dBytes]) > 0 { + } else if s.version.IPCompare(ip, buff[bytes:dBytes]) > 0 { l = m + 1 } else { dataLen = int(binary.LittleEndian.Uint16(buff[dBytes:])) diff --git a/binding/golang/xdb/util.go b/binding/golang/xdb/util.go index 8fe7c95..c016b91 100644 --- a/binding/golang/xdb/util.go +++ b/binding/golang/xdb/util.go @@ -9,10 +9,10 @@ package xdb import ( + "bytes" "embed" "fmt" "io" - "math/big" "net" "os" ) @@ -40,51 +40,19 @@ func IP2String(ip []byte) string { return net.IP(ip[:]).String() } -func IP2Long(ip []byte) *big.Int { - return big.NewInt(0).SetBytes(ip) -} - // IPCompare compares two IP addresses // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 func IPCompare(ip1, ip2 []byte) int { - for i := 0; i < len(ip1); i++ { - if ip1[i] < ip2[i] { - return -1 - } - - if ip1[i] > ip2[i] { - return 1 - } - } - - return 0 -} - -func IPAddOne(ip []byte) []byte { - var r = make([]byte, len(ip)) - copy(r, ip) - for i := len(ip) - 1; i >= 0; i-- { - r[i]++ - if r[i] != 0 { // No overflow - break - } - } - - return r -} - -func IPSubOne(ip []byte) []byte { - var r = make([]byte, len(ip)) - copy(r, ip) - for i := len(ip) - 1; i >= 0; i-- { - if r[i] != 0 { // No borrow needed - r[i]-- - break - } - r[i] = 0xFF // borrow from the next byte - } - - return r + // for i := 0; i < len(ip1); i++ { + // if ip1[i] < ip2[i] { + // return -1 + // } + // if ip1[i] > ip2[i] { + // return 1 + // } + // } + // return 0 + return bytes.Compare(ip1, ip2) } // LoadHeader load the header info from the specified handle diff --git a/binding/golang/xdb/util_test.go b/binding/golang/xdb/util_test.go index 96a94c0..75696a7 100644 --- a/binding/golang/xdb/util_test.go +++ b/binding/golang/xdb/util_test.go @@ -41,70 +41,8 @@ func TestIPCompare(t *testing.T) { } } -func TestIPAddOne(t *testing.T) { - var ipPairs = [][]string{ - {"1.2.3.4", "1.2.3.5"}, - {"2.3.4.5", "2.3.4.6"}, - {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, - {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"}, - {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"}, - } - - for _, pairs := range ipPairs { - sip, err := ParseIP(pairs[0]) - if err != nil { - t.Errorf("parse ip `%s`: %s\n", pairs[0], err) - } - - eip, err := ParseIP(pairs[1]) - if err != nil { - t.Errorf("parse ip `%s`: %s\n", pairs[1], err) - } - - fmt.Printf("IPAddOne(%s) = %s ? %d\n", - pairs[0], pairs[1], IPCompare(IPAddOne(sip), eip)) - } -} - -func TestIPAddOne2(t *testing.T) { - var ip = []byte{0, 1, 2, 3} - nip := IPAddOne(ip) - fmt.Printf("nip: %+v, ip:%+v", ip, nip) -} - -func TestIPSubOne(t *testing.T) { - var ipPairs = [][]string{ - {"1.2.3.4", "1.2.3.5"}, - {"2.3.4.5", "2.3.4.6"}, - {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, - {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"}, - {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"}, - } - - for _, pairs := range ipPairs { - sip, err := ParseIP(pairs[0]) - if err != nil { - t.Errorf("parse ip `%s`: %s\n", pairs[0], err) - } - - eip, err := ParseIP(pairs[1]) - if err != nil { - t.Errorf("parse ip `%s`: %s\n", pairs[1], err) - } - - fmt.Printf("IPSubOne(%s) = %s ? %d\n", - pairs[1], pairs[0], IPCompare(IPSubOne(eip), sip)) - } -} - -func TestIPSubOne2(t *testing.T) { - var ip = []byte{0, 1, 2, 3} - nip := IPSubOne(ip) - fmt.Printf("nip: %+v, ip:%+v", ip, nip) -} - func TestLoadVectorIndex(t *testing.T) { - vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region.xdb") + vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region_v4.xdb") if err != nil { fmt.Printf("failed to load vector index: %s\n", err) return @@ -114,7 +52,7 @@ func TestLoadVectorIndex(t *testing.T) { } func TestLoadContent(t *testing.T) { - buff, err := LoadContentFromFile("../../../data/ip2region.xdb") + buff, err := LoadContentFromFile("../../../data/ip2region_v4.xdb") if err != nil { fmt.Printf("failed to load xdb content: %s\n", err) return @@ -124,15 +62,17 @@ func TestLoadContent(t *testing.T) { } func TestLoadHeader(t *testing.T) { - header, err := LoadHeaderFromFile("../../../data/ip2region.xdb") + header, err := LoadHeaderFromFile("../../../data/ip2region_v4.xdb") if err != nil { fmt.Printf("failed to load xdb header info: %s\n", err) return } - fmt.Printf("Version : %d\n", header.Version) - fmt.Printf("IndexPolicy : %s\n", header.IndexPolicy.String()) - fmt.Printf("CreatedAt : %d(%s)\n", header.CreatedAt, time.Unix(int64(header.CreatedAt), 0).Format(time.RFC3339)) - fmt.Printf("StartIndexPtr : %d\n", header.StartIndexPtr) - fmt.Printf("EndIndexPtr : %d\n", header.EndIndexPtr) + fmt.Printf("Version : %d\n", header.Version) + fmt.Printf("IndexPolicy : %s\n", header.IndexPolicy.String()) + fmt.Printf("CreatedAt : %d(%s)\n", header.CreatedAt, time.Unix(int64(header.CreatedAt), 0).Format(time.RFC3339)) + fmt.Printf("StartIndexPtr : %d\n", header.StartIndexPtr) + fmt.Printf("EndIndexPtr : %d\n", header.EndIndexPtr) + fmt.Printf("IPVersion : %d\n", header.IPVersion) + fmt.Printf("RuntimePtrBytes : %d\n", header.RuntimePtrBytes) } diff --git a/binding/golang/xdb/version.go b/binding/golang/xdb/version.go index ed7eba5..7a5358c 100644 --- a/binding/golang/xdb/version.go +++ b/binding/golang/xdb/version.go @@ -5,6 +5,7 @@ package xdb import ( + "bytes" "fmt" "strings" ) @@ -14,6 +15,9 @@ type Version struct { Name string Bytes int SegmentIndexSize int + + // function to compare two ips + IPCompare func([]byte, []byte) int } const ( @@ -27,23 +31,33 @@ var ( Id: IPv4VersionNo, Name: "IPv4", Bytes: 4, - SegmentIndexSize: 14, // 4 + 4 + 2 + 4 + SegmentIndexSize: 14, // 4 + 4 + 2 + 4, + IPCompare: func(ip1, ip2 []byte) int { + // ip1 - with Bit endian parsed from an input + // ip2 - with Little endian read from the xdb index + ip2[0], ip2[3] = ip2[3], ip2[0] + ip2[1], ip2[2] = ip2[2], ip2[1] + return bytes.Compare(ip1, ip2) + }, } IPv6 = &Version{ Id: IPv6VersionNo, Name: "IPv6", Bytes: 16, - SegmentIndexSize: 38, // 16 + 16 + 2 + 4 + SegmentIndexSize: 38, // 16 + 16 + 2 + 4, + IPCompare: func(ip1, ip2 []byte) int { + return bytes.Compare(ip1, ip2) + }, } ) func VersionFromIP(ip string) (*Version, error) { - bytes, err := ParseIP(ip) + r, err := ParseIP(ip) if err != nil { return IPvx, fmt.Errorf("parse ip fail: %w", err) } - if len(bytes) == 4 { + if len(r) == 4 { return IPv4, nil } diff --git a/data/ip2region.xdb b/data/ip2region.xdb deleted file mode 100644 index 7052c05..0000000 Binary files a/data/ip2region.xdb and /dev/null differ diff --git a/data/ip2region_v4.xdb b/data/ip2region_v4.xdb index fda539b..7052c05 100644 Binary files a/data/ip2region_v4.xdb and b/data/ip2region_v4.xdb differ diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index 45b7bd1..8f5986b 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -302,9 +302,15 @@ func (m *Maker) Start() error { return fmt.Errorf("segment index ptr exceed the max length of %d", math.MaxUint32) } - // encode the segment index - copy(indexBuff[0:], s.StartIP) - copy(indexBuff[len(s.StartIP):], s.EndIP) + // encode the segment index. + // @Note by Leon at 2025/09/05: + // This is a tough decision since the directly copy of the bytes will make everything simpler. + // But in order to compatible with the old searcher implementation we had to keep encoding the IPv4 bytes with little endian. + // @TODO: we may choose to use the big-endian byte order in the future. + // But now compatibility is the most important !!! + + m.version.PutBytes(indexBuff[0:], s.StartIP) + m.version.PutBytes(indexBuff[len(s.StartIP):], s.EndIP) _offset = len(s.StartIP) + len(s.EndIP) binary.LittleEndian.PutUint16(indexBuff[_offset:], uint16(dataLen)) binary.LittleEndian.PutUint32(indexBuff[_offset+2:], dataPtr) diff --git a/maker/golang/xdb/searcher.go b/maker/golang/xdb/searcher.go index 986e5a5..f34aa10 100644 --- a/maker/golang/xdb/searcher.go +++ b/maker/golang/xdb/searcher.go @@ -151,9 +151,9 @@ func (s *Searcher) Search(ip []byte) (string, int, error) { } // decode the data step by step to reduce the unnecessary calculations - if IPCompare(ip, buff[0:bytes]) < 0 { + if s.version.IPCompare(ip, buff[0:bytes]) < 0 { h = m - 1 - } else if IPCompare(ip, buff[bytes:tBytes]) > 0 { + } else if s.version.IPCompare(ip, buff[bytes:tBytes]) > 0 { l = m + 1 } else { dataLen = int(binary.LittleEndian.Uint16(buff[tBytes:])) diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index a57f9a9..e10d093 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -6,6 +6,7 @@ package xdb import ( "bufio" + "bytes" "fmt" "math/big" "net" @@ -45,17 +46,18 @@ func IP2Long(ip []byte) *big.Int { // IPCompare compares two IP addresses // Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 func IPCompare(ip1, ip2 []byte) int { - for i := 0; i < len(ip1); i++ { - if ip1[i] < ip2[i] { - return -1 - } + // for i := 0; i < len(ip1); i++ { + // if ip1[i] < ip2[i] { + // return -1 + // } - if ip1[i] > ip2[i] { - return 1 - } - } + // if ip1[i] > ip2[i] { + // return 1 + // } + // } - return 0 + // return 0 + return bytes.Compare(ip1, ip2) } func IPAddOne(ip []byte) []byte { diff --git a/maker/golang/xdb/version.go b/maker/golang/xdb/version.go index bf9c672..04c582c 100644 --- a/maker/golang/xdb/version.go +++ b/maker/golang/xdb/version.go @@ -5,6 +5,7 @@ package xdb import ( + "bytes" "fmt" "strings" ) @@ -19,6 +20,16 @@ type Version struct { Name string Bytes int SegmentIndexSize int + + // bytes encode + PutBytes func([]byte, []byte) int + + // ip compares + IPCompare func([]byte, []byte) int +} + +func (v *Version) String() string { + return fmt.Sprintf("{Id:%d, Name:%d, Bytes:%d, IndexSize: %d}", v.Id, v.Name, v.Bytes, v.SegmentIndexSize) } var ( @@ -27,23 +38,46 @@ var ( Id: 4, Name: "IPv4", Bytes: 4, - SegmentIndexSize: 14, // 4 + 4 + 2 + 4 + SegmentIndexSize: 14, // 4 + 4 + 2 + 4, + PutBytes: func(buff []byte, ip []byte) int { + // binary.LittleEndian.PutUint32(buff, binary.BigEndian.Uint32(ip)) + // Little Endian byte order for compatible with the old searcher implementation + buff[0] = ip[3] + buff[1] = ip[2] + buff[2] = ip[1] + buff[3] = ip[0] + return len(ip) + }, + IPCompare: func(ip1 []byte, ip2 []byte) int { + // ip1 - with Bit endian parsed from an input + // ip2 - with Little endian read from the xdb index + ip2[0], ip2[3] = ip2[3], ip2[0] + ip2[1], ip2[2] = ip2[2], ip2[1] + return bytes.Compare(ip1, ip2) + }, } IPv6 = &Version{ Id: 6, Name: "IPv6", Bytes: 16, - SegmentIndexSize: 38, // 16 + 16 + 2 + 4 + SegmentIndexSize: 38, // 16 + 16 + 2 + 4, + // Big Endian byte order to follow the network byte order + PutBytes: func(buff []byte, ip []byte) int { + return copy(buff, ip) + }, + IPCompare: func(ip1, ip2 []byte) int { + return bytes.Compare(ip1, ip2) + }, } ) func VersionFromIP(ip string) (*Version, error) { - bytes, err := ParseIP(ip) + r, err := ParseIP(ip) if err != nil { return IPvx, fmt.Errorf("parse ip fail: %w", err) } - if len(bytes) == 4 { + if len(r) == 4 { return IPv4, nil }