From ae6912385752a2ae5ccdc2600a609a6eabd76ea0 Mon Sep 17 00:00:00 2001 From: lion Date: Fri, 5 Sep 2025 17:15:01 +0800 Subject: [PATCH] add IPv6 supports for golang binding --- binding/golang/main.go | 38 ++++++++---- binding/golang/xdb/searcher.go | 88 +++++++++++++++++---------- binding/golang/xdb/util.go | 88 +++++++++++++++++++-------- binding/golang/xdb/util_test.go | 103 +++++++++++++++++++++++++------- binding/golang/xdb/version.go | 82 +++++++++++++++++++++++++ 5 files changed, 306 insertions(+), 93 deletions(-) create mode 100644 binding/golang/xdb/version.go diff --git a/binding/golang/main.go b/binding/golang/main.go index ad58abb..68ad82b 100644 --- a/binding/golang/main.go +++ b/binding/golang/main.go @@ -11,12 +11,13 @@ package main import ( "bufio" "fmt" - "github.com/lionsoul2014/ip2region/binding/golang/xdb" - "github.com/mitchellh/go-homedir" "log" "os" "strings" "time" + + "github.com/lionsoul2014/ip2region/binding/golang/xdb" + "github.com/mitchellh/go-homedir" ) func printHelp() { @@ -82,9 +83,10 @@ func testSearch() { fmt.Printf("searcher test program exited, thanks for trying\n") }() - fmt.Printf(`ip2region xdb searcher test program, cachePolicy: %s + fmt.Printf(`ip2region xdb searcher test program +source xdb: %s (%s, %s) type 'quit' to exit -`, cachePolicy) +`, dbPath, searcher.GetIPVersion().Name, cachePolicy) reader := bufio.NewReader(os.Stdin) for { fmt.Print("ip2region>> ") @@ -185,29 +187,28 @@ func testBench() { return } - sip, err := xdb.CheckIP(ps[0]) + sip, err := xdb.ParseIP(ps[0]) if err != nil { fmt.Printf("check start ip `%s`: %s\n", ps[0], err) return } - eip, err := xdb.CheckIP(ps[1]) + eip, err := xdb.ParseIP(ps[1]) if err != nil { fmt.Printf("check end ip `%s`: %s\n", ps[1], err) return } - if sip > eip { + if xdb.IPCompare(sip, eip) > 0 { fmt.Printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]) return } - mip := xdb.MidIP(sip, eip) - for _, ip := range []uint32{sip, xdb.MidIP(sip, mip), mip, xdb.MidIP(mip, eip), eip} { + for _, ip := range [][]byte{sip, eip} { sTime := time.Now() region, err := searcher.Search(ip) if err != nil { - fmt.Printf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) + fmt.Printf("failed to search ip '%s': %s\n", xdb.IP2String(ip), err) return } @@ -215,7 +216,7 @@ func testBench() { // check the region info if region != ps[2] { - fmt.Printf("failed Search(%s) with (%s != %s)\n", xdb.Long2IP(ip), region, ps[2]) + fmt.Printf("failed Search(%s) with (%s != %s)\n", xdb.IP2String(ip), region, ps[2]) return } @@ -229,16 +230,27 @@ func testBench() { } func createSearcher(dbPath string, cachePolicy string) (*xdb.Searcher, error) { + // auto-detect the ip version from the xdb header + header, err := xdb.LoadHeaderFromFile(dbPath) + if err != nil { + return nil, fmt.Errorf("failed to load header from `%s`: %s", dbPath, err) + } + + version, err := xdb.VersionFromHeader(header) + if err != nil { + return nil, fmt.Errorf("failed to detect IP version from `%s`: %s", dbPath, err) + } + switch cachePolicy { case "nil", "file": - return xdb.NewWithFileOnly(dbPath) + return xdb.NewWithFileOnly(version, dbPath) case "vectorIndex": vIndex, err := xdb.LoadVectorIndexFromFile(dbPath) if err != nil { return nil, fmt.Errorf("failed to load vector index from `%s`: %w", dbPath, err) } - return xdb.NewWithVectorIndex(dbPath, vIndex) + return xdb.NewWithVectorIndex(version, dbPath, vIndex) case "content": cBuff, err := xdb.LoadContentFromFile(dbPath) if err != nil { diff --git a/binding/golang/xdb/searcher.go b/binding/golang/xdb/searcher.go index b5788e7..3cd8e18 100644 --- a/binding/golang/xdb/searcher.go +++ b/binding/golang/xdb/searcher.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // --- -// ip2region database v2.0 searcher. +// Ip2Region database v2.0 searcher. // @Note this is a Not thread safe implementation. // // @Author Lion @@ -18,11 +18,12 @@ import ( ) const ( - HeaderInfoLength = 256 - VectorIndexRows = 256 - VectorIndexCols = 256 - VectorIndexSize = 8 - SegmentIndexBlockSize = 14 + Structure20 = 2 + Structure30 = 3 + HeaderInfoLength = 256 + VectorIndexRows = 256 + VectorIndexCols = 256 + VectorIndexSize = 8 ) // --- Index policy define @@ -54,6 +55,10 @@ type Header struct { CreatedAt uint32 StartIndexPtr uint32 EndIndexPtr uint32 + + // since IPv6 supporting + IPVersion int + RuntimePtrBytes int } func NewHeader(input []byte) (*Header, error) { @@ -67,13 +72,17 @@ func NewHeader(input []byte) (*Header, error) { CreatedAt: binary.LittleEndian.Uint32(input[4:]), StartIndexPtr: binary.LittleEndian.Uint32(input[8:]), EndIndexPtr: binary.LittleEndian.Uint32(input[12:]), + + IPVersion: int(binary.LittleEndian.Uint16(input[16:])), + RuntimePtrBytes: int(binary.LittleEndian.Uint16(input[18:])), }, nil } // --- searcher implementation type Searcher struct { - handle *os.File + version *Version + handle *os.File // header info header *Header @@ -89,7 +98,7 @@ type Searcher struct { contentBuff []byte } -func baseNew(dbFile string, vIndex []byte, cBuff []byte) (*Searcher, error) { +func baseNew(version *Version, dbFile string, vIndex []byte, cBuff []byte) (*Searcher, error) { var err error // content buff first @@ -107,21 +116,29 @@ func baseNew(dbFile string, vIndex []byte, cBuff []byte) (*Searcher, error) { } return &Searcher{ + version: version, handle: handle, vectorIndex: vIndex, }, nil } -func NewWithFileOnly(dbFile string) (*Searcher, error) { - return baseNew(dbFile, nil, nil) +func NewWithFileOnly(version *Version, dbFile string) (*Searcher, error) { + return baseNew(version, dbFile, nil, nil) } -func NewWithVectorIndex(dbFile string, vIndex []byte) (*Searcher, error) { - return baseNew(dbFile, vIndex, nil) +func NewWithVectorIndex(version *Version, dbFile string, vIndex []byte) (*Searcher, error) { + return baseNew(version, dbFile, vIndex, nil) } func NewWithBuffer(cBuff []byte) (*Searcher, error) { - return baseNew("", nil, cBuff) + versionNo := binary.LittleEndian.Uint16(cBuff[16:]) + if versionNo == IPv4VersionNo { + return baseNew(IPv4, "", nil, cBuff) + } else if versionNo == IPv6VersionNo { + return baseNew(IPv6, "", nil, cBuff) + } else { + return nil, fmt.Errorf("invalid version number `%d`", versionNo) + } } func (s *Searcher) Close() { @@ -133,6 +150,11 @@ func (s *Searcher) Close() { } } +// GetIPVersion return the ip version +func (s *Searcher) GetIPVersion() *Version { + return s.version +} + // GetIOCount return the global io count for the last search func (s *Searcher) GetIOCount() int { return s.ioCount @@ -140,7 +162,7 @@ func (s *Searcher) GetIOCount() int { // SearchByStr find the region for the specified ip string func (s *Searcher) SearchByStr(str string) (string, error) { - ip, err := CheckIP(str) + ip, err := ParseIP(str) if err != nil { return "", err } @@ -149,13 +171,17 @@ func (s *Searcher) SearchByStr(str string) (string, error) { } // Search find the region for the specified long ip -func (s *Searcher) Search(ip uint32) (string, error) { +func (s *Searcher) Search(ip []byte) (string, error) { + // ip version check + if len(ip) != s.version.Bytes { + return "", fmt.Errorf("invalid ip address(%s expected)", s.version.Name) + } + // reset the global ioCount s.ioCount = 0 // locate the segment index block based on the vector index - var il0 = (ip >> 24) & 0xFF - var il1 = (ip >> 16) & 0xFF + var il0, il1 = int(ip[0]), int(ip[1]) var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize var sPtr, ePtr = uint32(0), uint32(0) if s.vectorIndex != nil { @@ -176,37 +202,35 @@ func (s *Searcher) Search(ip uint32) (string, error) { ePtr = binary.LittleEndian.Uint32(buff[4:]) } - // fmt.Printf("sPtr=%d, ePtr=%d", sPtr, ePtr) + // fmt.Printf("sPtr=%d, ePtr=%d\n", sPtr, ePtr) // binary search the segment index to get the region + var bytes, dBytes = len(ip), len(ip) << 1 + var segIndexSize = uint32(s.version.SegmentIndexSize) var dataLen, dataPtr = 0, uint32(0) - var buff = make([]byte, SegmentIndexBlockSize) - var l, h = 0, int((ePtr - sPtr) / SegmentIndexBlockSize) + var buff = make([]byte, segIndexSize) + var l, h = 0, int((ePtr - sPtr) / segIndexSize) for l <= h { m := (l + h) >> 1 - p := sPtr + uint32(m*SegmentIndexBlockSize) + p := sPtr + uint32(m)*segIndexSize err := s.read(int64(p), buff) if err != nil { return "", fmt.Errorf("read segment index at %d: %w", p, err) } // decode the data step by step to reduce the unnecessary operations - sip := binary.LittleEndian.Uint32(buff) - if ip < sip { + if IPCompare(ip, buff[0:bytes]) < 0 { h = m - 1 + } else if IPCompare(ip, buff[bytes:dBytes]) > 0 { + l = m + 1 } else { - eip := binary.LittleEndian.Uint32(buff[4:]) - if ip > eip { - l = m + 1 - } else { - dataLen = int(binary.LittleEndian.Uint16(buff[8:])) - dataPtr = binary.LittleEndian.Uint32(buff[10:]) - break - } + dataLen = int(binary.LittleEndian.Uint16(buff[dBytes:])) + dataPtr = binary.LittleEndian.Uint32(buff[dBytes+2:]) + break } } - //fmt.Printf("dataLen: %d, dataPtr: %d", dataLen, dataPtr) + // fmt.Printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr) if dataLen == 0 { return "", nil } diff --git a/binding/golang/xdb/util.go b/binding/golang/xdb/util.go index b35eb56..8fe7c95 100644 --- a/binding/golang/xdb/util.go +++ b/binding/golang/xdb/util.go @@ -12,43 +12,79 @@ import ( "embed" "fmt" "io" + "math/big" + "net" "os" - "strconv" - "strings" ) -var shiftIndex = []int{24, 16, 8, 0} - -func CheckIP(ip string) (uint32, error) { - var ps = strings.Split(strings.TrimSpace(ip), ".") - if len(ps) != 4 { - return 0, fmt.Errorf("invalid ip address `%s`", ip) +func ParseIP(ip string) ([]byte, error) { + parsedIP := net.ParseIP(ip) + if parsedIP == nil { + return nil, fmt.Errorf("invalid ip address: %s", ip) } - var val = uint32(0) - for i, s := range ps { - d, err := strconv.Atoi(s) - if err != nil { - return 0, fmt.Errorf("the %dth part `%s` is not an integer", i, s) - } - - if d < 0 || d > 255 { - return 0, fmt.Errorf("the %dth part `%s` should be an integer bettween 0 and 255", i, s) - } - - val |= uint32(d) << shiftIndex[i] + v4 := parsedIP.To4() + if v4 != nil { + return v4, nil } - // convert the ip to integer - return val, nil + v6 := parsedIP.To16() + if v6 != nil { + return v6, nil + } + + return nil, fmt.Errorf("invalid ip address: %s", ip) } -func Long2IP(ip uint32) string { - return fmt.Sprintf("%d.%d.%d.%d", (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, ip&0xFF) +func IP2String(ip []byte) string { + return net.IP(ip[:]).String() } -func MidIP(sip uint32, eip uint32) uint32 { - return uint32((uint64(sip) + uint64(eip)) >> 1) +func IP2Long(ip []byte) *big.Int { + return big.NewInt(0).SetBytes(ip) +} + +// IPCompare compares two IP addresses +// Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 +func IPCompare(ip1, ip2 []byte) int { + for i := 0; i < len(ip1); i++ { + if ip1[i] < ip2[i] { + return -1 + } + + if ip1[i] > ip2[i] { + return 1 + } + } + + return 0 +} + +func IPAddOne(ip []byte) []byte { + var r = make([]byte, len(ip)) + copy(r, ip) + for i := len(ip) - 1; i >= 0; i-- { + r[i]++ + if r[i] != 0 { // No overflow + break + } + } + + return r +} + +func IPSubOne(ip []byte) []byte { + var r = make([]byte, len(ip)) + copy(r, ip) + for i := len(ip) - 1; i >= 0; i-- { + if r[i] != 0 { // No borrow needed + r[i]-- + break + } + r[i] = 0xFF // borrow from the next byte + } + + return r } // LoadHeader load the header info from the specified handle diff --git a/binding/golang/xdb/util_test.go b/binding/golang/xdb/util_test.go index bea7711..96a94c0 100644 --- a/binding/golang/xdb/util_test.go +++ b/binding/golang/xdb/util_test.go @@ -9,39 +9,98 @@ package xdb import ( - "encoding/binary" "fmt" - "net" "testing" "time" ) -func TestCheckIP(t *testing.T) { - var str = "29.34.191.255" - ip, err := CheckIP(str) - if err != nil { - t.Errorf("check ip `%s`: %s\n", str, err) - } +func TestParseIP(t *testing.T) { + var ips = []string{"29.34.191.255", "2c0f:fff0::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} + for _, ip := range ips { + bytes, err := ParseIP(ip) + if err != nil { + t.Errorf("check ip `%s`: %s\n", IP2String(bytes), err) + } - netIP := net.ParseIP(str).To4() - if netIP == nil { - t.Fatalf("parse ip `%s` failed", str) + nip := IP2String(bytes) + fmt.Printf("checkip: (%s / %s), isEqual: %v\n", ip, nip, ip == nip) } - - u32 := binary.BigEndian.Uint32(netIP) - fmt.Printf("checkip: %d, parseip: %d, isEqual: %v\n", ip, u32, ip == u32) } -func TestLong2IP(t *testing.T) { - var str = "29.34.191.255" - netIP := net.ParseIP(str).To4() - if netIP == nil { - t.Fatalf("parse ip `%s` failed", str) +func TestIPCompare(t *testing.T) { + var ipPairs = [][]string{ + {"1.2.3.4", "1.2.3.5"}, + {"58.250.36.41", "58.250.30.41"}, + {"2c10::", "2e00::"}, + {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, + {"fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, } - u32 := binary.BigEndian.Uint32(netIP) - ipStr := Long2IP(u32) - fmt.Printf("originIP: %s, Long2IP: %s, isEqual: %v\n", str, ipStr, ipStr == str) + for _, pairs := range ipPairs { + fmt.Printf("IPCompare(%s, %s): %d\n", pairs[0], pairs[1], IPCompare([]byte(pairs[0]), []byte(pairs[1]))) + } +} + +func TestIPAddOne(t *testing.T) { + var ipPairs = [][]string{ + {"1.2.3.4", "1.2.3.5"}, + {"2.3.4.5", "2.3.4.6"}, + {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, + {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"}, + {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"}, + } + + for _, pairs := range ipPairs { + sip, err := ParseIP(pairs[0]) + if err != nil { + t.Errorf("parse ip `%s`: %s\n", pairs[0], err) + } + + eip, err := ParseIP(pairs[1]) + if err != nil { + t.Errorf("parse ip `%s`: %s\n", pairs[1], err) + } + + fmt.Printf("IPAddOne(%s) = %s ? %d\n", + pairs[0], pairs[1], IPCompare(IPAddOne(sip), eip)) + } +} + +func TestIPAddOne2(t *testing.T) { + var ip = []byte{0, 1, 2, 3} + nip := IPAddOne(ip) + fmt.Printf("nip: %+v, ip:%+v", ip, nip) +} + +func TestIPSubOne(t *testing.T) { + var ipPairs = [][]string{ + {"1.2.3.4", "1.2.3.5"}, + {"2.3.4.5", "2.3.4.6"}, + {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, + {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"}, + {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"}, + } + + for _, pairs := range ipPairs { + sip, err := ParseIP(pairs[0]) + if err != nil { + t.Errorf("parse ip `%s`: %s\n", pairs[0], err) + } + + eip, err := ParseIP(pairs[1]) + if err != nil { + t.Errorf("parse ip `%s`: %s\n", pairs[1], err) + } + + fmt.Printf("IPSubOne(%s) = %s ? %d\n", + pairs[1], pairs[0], IPCompare(IPSubOne(eip), sip)) + } +} + +func TestIPSubOne2(t *testing.T) { + var ip = []byte{0, 1, 2, 3} + nip := IPSubOne(ip) + fmt.Printf("nip: %+v, ip:%+v", ip, nip) } func TestLoadVectorIndex(t *testing.T) { diff --git a/binding/golang/xdb/version.go b/binding/golang/xdb/version.go new file mode 100644 index 0000000..ed7eba5 --- /dev/null +++ b/binding/golang/xdb/version.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package xdb + +import ( + "fmt" + "strings" +) + +type Version struct { + Id int + Name string + Bytes int + SegmentIndexSize int +} + +const ( + IPv4VersionNo = 4 + IPv6VersionNo = 6 +) + +var ( + IPvx = &Version{} + IPv4 = &Version{ + Id: IPv4VersionNo, + Name: "IPv4", + Bytes: 4, + SegmentIndexSize: 14, // 4 + 4 + 2 + 4 + } + IPv6 = &Version{ + Id: IPv6VersionNo, + Name: "IPv6", + Bytes: 16, + SegmentIndexSize: 38, // 16 + 16 + 2 + 4 + } +) + +func VersionFromIP(ip string) (*Version, error) { + bytes, err := ParseIP(ip) + if err != nil { + return IPvx, fmt.Errorf("parse ip fail: %w", err) + } + + if len(bytes) == 4 { + return IPv4, nil + } + + return IPv6, nil +} + +func VersionFromName(name string) (*Version, error) { + switch strings.ToUpper(name) { + case "V4", "IPV4": + return IPv4, nil + case "V6", "IPV6": + return IPv6, nil + default: + return IPvx, fmt.Errorf("invalid version name `%s`", name) + } +} + +func VersionFromHeader(header *Header) (*Version, error) { + // old structure with IPv4 supports ONLY + if header.Version == Structure20 { + return IPv4, nil + } + + // structure 3.0 after IPv6 supporting + if header.Version == Structure30 { + if header.IPVersion == IPv4VersionNo { + return IPv4, nil + } else if header.IPVersion == IPv6VersionNo { + return IPv6, nil + } else { + return IPvx, fmt.Errorf("invalid version `%d`", header.IPVersion) + } + } + + return IPvx, fmt.Errorf("invalid version `%d`", header.Version) +}