xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

IPv6 RC verison with IPv4 compatible with the old implementation

This commit is contained in:
lion 2025-09-06 00:36:10 +08:00
parent ae69123857
commit 0307a3df14
11 changed files with 109 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
binding/golang/main.go
View File

@ -86,7 +86,7 @@ func testSearch() {
fmt.Printf(`ip2region xdb searcher test program
source xdb: %s (%s, %s)
type 'quit' to exit
`, dbPath, searcher.GetIPVersion().Name, cachePolicy)
`, dbPath, searcher.IPVersion().Name, cachePolicy)
reader := bufio.NewReader(os.Stdin)
for {
fmt.Print("ip2region>> ")
@ -257,7 +257,7 @@ func createSearcher(dbPath string, cachePolicy string) (*xdb.Searcher, error) {
return nil, fmt.Errorf("failed to load content from '%s': %w", dbPath, err)
}
return xdb.NewWithBuffer(cBuff)
return xdb.NewWithBuffer(version, cBuff)
default:
return nil, fmt.Errorf("invalid cache policy `%s`, options: file/vectorIndex/content", cachePolicy)
}

22
binding/golang/xdb/searcher.go
View File

@ -67,7 +67,7 @@ func NewHeader(input []byte) (*Header, error) {
}
return &Header{
Version: binary.LittleEndian.Uint16(input),
Version: binary.LittleEndian.Uint16(input[0:]),
IndexPolicy: IndexPolicy(binary.LittleEndian.Uint16(input[2:])),
CreatedAt: binary.LittleEndian.Uint32(input[4:]),
StartIndexPtr: binary.LittleEndian.Uint32(input[8:]),
@ -104,6 +104,7 @@ func baseNew(version *Version, dbFile string, vIndex []byte, cBuff []byte) (*Sea
// content buff first
if cBuff != nil {
return &Searcher{
version: version,
vectorIndex: nil,
contentBuff: cBuff,
}, nil
@ -130,15 +131,8 @@ func NewWithVectorIndex(version *Version, dbFile string, vIndex []byte) (*Search
return baseNew(version, dbFile, vIndex, nil)
}
func NewWithBuffer(cBuff []byte) (*Searcher, error) {
versionNo := binary.LittleEndian.Uint16(cBuff[16:])
if versionNo == IPv4VersionNo {
return baseNew(IPv4, "", nil, cBuff)
} else if versionNo == IPv6VersionNo {
return baseNew(IPv6, "", nil, cBuff)
} else {
return nil, fmt.Errorf("invalid version number `%d`", versionNo)
}
func NewWithBuffer(version *Version, cBuff []byte) (*Searcher, error) {
return baseNew(version, "", nil, cBuff)
}
func (s *Searcher) Close() {
@ -150,8 +144,8 @@ func (s *Searcher) Close() {
}
}
// GetIPVersion return the ip version
func (s *Searcher) GetIPVersion() *Version {
// IPVersion return the ip version
func (s *Searcher) IPVersion() *Version {
return s.version
}
@ -219,9 +213,9 @@ func (s *Searcher) Search(ip []byte) (string, error) {
}
// decode the data step by step to reduce the unnecessary operations
if IPCompare(ip, buff[0:bytes]) < 0 {
if s.version.IPCompare(ip, buff[0:bytes]) < 0 {
h = m - 1
} else if IPCompare(ip, buff[bytes:dBytes]) > 0 {
} else if s.version.IPCompare(ip, buff[bytes:dBytes]) > 0 {
l = m + 1
} else {
dataLen = int(binary.LittleEndian.Uint16(buff[dBytes:]))

54
binding/golang/xdb/util.go
View File

@ -9,10 +9,10 @@
package xdb
import (
"bytes"
"embed"
"fmt"
"io"
"math/big"
"net"
"os"
)
@ -40,51 +40,19 @@ func IP2String(ip []byte) string {
return net.IP(ip[:]).String()
}
func IP2Long(ip []byte) *big.Int {
return big.NewInt(0).SetBytes(ip)
}
// IPCompare compares two IP addresses
// Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2
func IPCompare(ip1, ip2 []byte) int {
for i := 0; i < len(ip1); i++ {
if ip1[i] < ip2[i] {
return -1
}
if ip1[i] > ip2[i] {
return 1
}
}
return 0
}
func IPAddOne(ip []byte) []byte {
var r = make([]byte, len(ip))
copy(r, ip)
for i := len(ip) - 1; i >= 0; i-- {
r[i]++
if r[i] != 0 { // No overflow
break
}
}
return r
}
func IPSubOne(ip []byte) []byte {
var r = make([]byte, len(ip))
copy(r, ip)
for i := len(ip) - 1; i >= 0; i-- {
if r[i] != 0 { // No borrow needed
r[i]--
break
}
r[i] = 0xFF // borrow from the next byte
}
return r
// for i := 0; i < len(ip1); i++ {
// if ip1[i] < ip2[i] {
// return -1
// }
// if ip1[i] > ip2[i] {
// return 1
// }
// }
// return 0
return bytes.Compare(ip1, ip2)
}
// LoadHeader load the header info from the specified handle

70
binding/golang/xdb/util_test.go
View File

@ -41,70 +41,8 @@ func TestIPCompare(t *testing.T) {
}
}
func TestIPAddOne(t *testing.T) {
var ipPairs = [][]string{
{"1.2.3.4", "1.2.3.5"},
{"2.3.4.5", "2.3.4.6"},
{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"},
{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"},
}
for _, pairs := range ipPairs {
sip, err := ParseIP(pairs[0])
if err != nil {
t.Errorf("parse ip `%s`: %s\n", pairs[0], err)
}
eip, err := ParseIP(pairs[1])
if err != nil {
t.Errorf("parse ip `%s`: %s\n", pairs[1], err)
}
fmt.Printf("IPAddOne(%s) = %s ? %d\n",
pairs[0], pairs[1], IPCompare(IPAddOne(sip), eip))
}
}
func TestIPAddOne2(t *testing.T) {
var ip = []byte{0, 1, 2, 3}
nip := IPAddOne(ip)
fmt.Printf("nip: %+v, ip:%+v", ip, nip)
}
func TestIPSubOne(t *testing.T) {
var ipPairs = [][]string{
{"1.2.3.4", "1.2.3.5"},
{"2.3.4.5", "2.3.4.6"},
{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"},
{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"},
}
for _, pairs := range ipPairs {
sip, err := ParseIP(pairs[0])
if err != nil {
t.Errorf("parse ip `%s`: %s\n", pairs[0], err)
}
eip, err := ParseIP(pairs[1])
if err != nil {
t.Errorf("parse ip `%s`: %s\n", pairs[1], err)
}
fmt.Printf("IPSubOne(%s) = %s ? %d\n",
pairs[1], pairs[0], IPCompare(IPSubOne(eip), sip))
}
}
func TestIPSubOne2(t *testing.T) {
var ip = []byte{0, 1, 2, 3}
nip := IPSubOne(ip)
fmt.Printf("nip: %+v, ip:%+v", ip, nip)
}
func TestLoadVectorIndex(t *testing.T) {
vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region.xdb")
vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region_v4.xdb")
if err != nil {
fmt.Printf("failed to load vector index: %s\n", err)
return
@ -114,7 +52,7 @@ func TestLoadVectorIndex(t *testing.T) {
}
func TestLoadContent(t *testing.T) {
buff, err := LoadContentFromFile("../../../data/ip2region.xdb")
buff, err := LoadContentFromFile("../../../data/ip2region_v4.xdb")
if err != nil {
fmt.Printf("failed to load xdb content: %s\n", err)
return
@ -124,7 +62,7 @@ func TestLoadContent(t *testing.T) {
}
func TestLoadHeader(t *testing.T) {
header, err := LoadHeaderFromFile("../../../data/ip2region.xdb")
header, err := LoadHeaderFromFile("../../../data/ip2region_v4.xdb")
if err != nil {
fmt.Printf("failed to load xdb header info: %s\n", err)
return
@ -135,4 +73,6 @@ func TestLoadHeader(t *testing.T) {
fmt.Printf("CreatedAt : %d(%s)\n", header.CreatedAt, time.Unix(int64(header.CreatedAt), 0).Format(time.RFC3339))
fmt.Printf("StartIndexPtr : %d\n", header.StartIndexPtr)
fmt.Printf("EndIndexPtr : %d\n", header.EndIndexPtr)
fmt.Printf("IPVersion : %d\n", header.IPVersion)
fmt.Printf("RuntimePtrBytes : %d\n", header.RuntimePtrBytes)
}

22
binding/golang/xdb/version.go
View File

@ -5,6 +5,7 @@
package xdb
import (
"bytes"
"fmt"
"strings"
)
@ -14,6 +15,9 @@ type Version struct {
Name string
Bytes int
SegmentIndexSize int
// function to compare two ips
IPCompare func([]byte, []byte) int
}
const (
@ -27,23 +31,33 @@ var (
Id: IPv4VersionNo,
Name: "IPv4",
Bytes: 4,
SegmentIndexSize: 14, // 4 + 4 + 2 + 4
SegmentIndexSize: 14, // 4 + 4 + 2 + 4,
IPCompare: func(ip1, ip2 []byte) int {
// ip1 - with Bit endian parsed from an input
// ip2 - with Little endian read from the xdb index
ip2[0], ip2[3] = ip2[3], ip2[0]
ip2[1], ip2[2] = ip2[2], ip2[1]
return bytes.Compare(ip1, ip2)
},
}
IPv6 = &Version{
Id: IPv6VersionNo,
Name: "IPv6",
Bytes: 16,
SegmentIndexSize: 38, // 16 + 16 + 2 + 4
SegmentIndexSize: 38, // 16 + 16 + 2 + 4,
IPCompare: func(ip1, ip2 []byte) int {
return bytes.Compare(ip1, ip2)
},
}
)
func VersionFromIP(ip string) (*Version, error) {
bytes, err := ParseIP(ip)
r, err := ParseIP(ip)
if err != nil {
return IPvx, fmt.Errorf("parse ip fail: %w", err)
}
if len(bytes) == 4 {
if len(r) == 4 {
return IPv4, nil
}

BIN
data/ip2region.xdb
View File

Binary file not shown.

BIN
data/ip2region_v4.xdb
View File

Binary file not shown.

12
maker/golang/xdb/maker.go
View File

@ -302,9 +302,15 @@ func (m *Maker) Start() error {
return fmt.Errorf("segment index ptr exceed the max length of %d", math.MaxUint32)
}
// encode the segment index
copy(indexBuff[0:], s.StartIP)
copy(indexBuff[len(s.StartIP):], s.EndIP)
// encode the segment index.
// @Note by Leon at 2025/09/05:
// This is a tough decision since the directly copy of the bytes will make everything simpler.
// But in order to compatible with the old searcher implementation we had to keep encoding the IPv4 bytes with little endian.
// @TODO: we may choose to use the big-endian byte order in the future.
// But now compatibility is the most important !!!
m.version.PutBytes(indexBuff[0:], s.StartIP)
m.version.PutBytes(indexBuff[len(s.StartIP):], s.EndIP)
_offset = len(s.StartIP) + len(s.EndIP)
binary.LittleEndian.PutUint16(indexBuff[_offset:], uint16(dataLen))
binary.LittleEndian.PutUint32(indexBuff[_offset+2:], dataPtr)

4
maker/golang/xdb/searcher.go
View File

@ -151,9 +151,9 @@ func (s *Searcher) Search(ip []byte) (string, int, error) {
}
// decode the data step by step to reduce the unnecessary calculations
if IPCompare(ip, buff[0:bytes]) < 0 {
if s.version.IPCompare(ip, buff[0:bytes]) < 0 {
h = m - 1
} else if IPCompare(ip, buff[bytes:tBytes]) > 0 {
} else if s.version.IPCompare(ip, buff[bytes:tBytes]) > 0 {
l = m + 1
} else {
dataLen = int(binary.LittleEndian.Uint16(buff[tBytes:]))

20
maker/golang/xdb/util.go
View File

@ -6,6 +6,7 @@ package xdb
import (
"bufio"
"bytes"
"fmt"
"math/big"
"net"
@ -45,17 +46,18 @@ func IP2Long(ip []byte) *big.Int {
// IPCompare compares two IP addresses
// Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2
func IPCompare(ip1, ip2 []byte) int {
for i := 0; i < len(ip1); i++ {
if ip1[i] < ip2[i] {
return -1
}
// for i := 0; i < len(ip1); i++ {
// if ip1[i] < ip2[i] {
// return -1
// }
if ip1[i] > ip2[i] {
return 1
}
}
// if ip1[i] > ip2[i] {
// return 1
// }
// }
return 0
// return 0
return bytes.Compare(ip1, ip2)
}
func IPAddOne(ip []byte) []byte {

42
maker/golang/xdb/version.go
View File

@ -5,6 +5,7 @@
package xdb
import (
"bytes"
"fmt"
"strings"
)
@ -19,6 +20,16 @@ type Version struct {
Name string
Bytes int
SegmentIndexSize int
// bytes encode
PutBytes func([]byte, []byte) int
// ip compares
IPCompare func([]byte, []byte) int
}
func (v *Version) String() string {
return fmt.Sprintf("{Id:%d, Name:%d, Bytes:%d, IndexSize: %d}", v.Id, v.Name, v.Bytes, v.SegmentIndexSize)
}
var (
@ -27,23 +38,46 @@ var (
Id: 4,
Name: "IPv4",
Bytes: 4,
SegmentIndexSize: 14, // 4 + 4 + 2 + 4
SegmentIndexSize: 14, // 4 + 4 + 2 + 4,
PutBytes: func(buff []byte, ip []byte) int {
// binary.LittleEndian.PutUint32(buff, binary.BigEndian.Uint32(ip))
// Little Endian byte order for compatible with the old searcher implementation
buff[0] = ip[3]
buff[1] = ip[2]
buff[2] = ip[1]
buff[3] = ip[0]
return len(ip)
},
IPCompare: func(ip1 []byte, ip2 []byte) int {
// ip1 - with Bit endian parsed from an input
// ip2 - with Little endian read from the xdb index
ip2[0], ip2[3] = ip2[3], ip2[0]
ip2[1], ip2[2] = ip2[2], ip2[1]
return bytes.Compare(ip1, ip2)
},
}
IPv6 = &Version{
Id: 6,
Name: "IPv6",
Bytes: 16,
SegmentIndexSize: 38, // 16 + 16 + 2 + 4
SegmentIndexSize: 38, // 16 + 16 + 2 + 4,
// Big Endian byte order to follow the network byte order
PutBytes: func(buff []byte, ip []byte) int {
return copy(buff, ip)
},
IPCompare: func(ip1, ip2 []byte) int {
return bytes.Compare(ip1, ip2)
},
}
)
func VersionFromIP(ip string) (*Version, error) {
bytes, err := ParseIP(ip)
r, err := ParseIP(ip)
if err != nil {
return IPvx, fmt.Errorf("parse ip fail: %w", err)
}
if len(bytes) == 4 {
if len(r) == 4 {
return IPv4, nil
}