xdb ipv6 supporting (bytes operation is working)

data/segments.tests.mixed

@@ -0,0 +1,10 @@
+192.168.2.1|192.168.2.20|0|0|0|内网IP|办公室A
+192.168.2.21|192.168.2.30|0|0|0|内网IP|办公室A
+192.168.2.31|192.168.2.60|0|0|0|内网IP|办公室B
+192.168.2.61|192.168.2.91|0|0|0|内网IP|办公室B
+223.255.236.0|223.255.239.255|中国|0|上海|上海市|电信
+2c0f:fff1::|2c0f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|毛里求斯|威廉平原区||卡特勒博尔纳||专线用户
+2e00::|2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|德国|黑森||美因河畔法兰克福||专线用户
+3000::|fbff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA
+fe00::|fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA
+fe80::|febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA

maker/golang/main.go

@@ -7,7 +7,6 @@ package main
 import (
 	"bufio"
 	"fmt"
-	"github.com/lionsoul2014/ip2region/maker/golang/xdb"
 	"log"
 	"log/slog"
 	"os"
@@ -16,6 +15,8 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/lionsoul2014/ip2region/maker/golang/xdb"
 )
 
 func printHelp() {
@@ -151,7 +152,7 @@ func getFilterFields(fieldList string) ([]int, error) {
 func genDb() {
 	var err error
 	var srcFile, dstFile = "", ""
-	var fieldList, logLevel = "", "info"
+	var ipVersion, fieldList, logLevel = "auto", "", "info"
 	var indexPolicy = xdb.VectorIndexPolicy
 	var fErr = iterateFlags(func(key string, val string) error {
 		switch key {
@@ -159,6 +160,8 @@ func genDb() {
 			srcFile = val
 		case "dst":
 			dstFile = val
+		case "version":
+			ipVersion = val
 		case "log-level":
 			logLevel = val
 		case "field-list":
@@ -184,6 +187,7 @@ func genDb() {
 		fmt.Printf("options:\n")
 		fmt.Printf(" --src string           source ip text file path\n")
 		fmt.Printf(" --dst string           destination binary xdb file path\n")
+		fmt.Printf(" --version string       IP version, options: ipv4/ipv6, default auto (detect from the source file)\n")
 		fmt.Printf(" --field-list string    field index list imploded with ',' eg: 0,1,2,3-6,7\n")
 		fmt.Printf(" --log-level string     set the log level, options: debug/info/warn/error\n")
 		return
@@ -202,9 +206,19 @@ func genDb() {
 		return
 	}
 
+	// check and define the IP version
+	var version *xdb.Version = nil
+	if ipVersion == "auto" {
+		version = xdb.V4
+	} else if v, err := xdb.VersionFromName(ipVersion); err != nil {
+		slog.Error("failed to parse version name", "error", err)
+	} else {
+		version = v
+	}
+
 	// make the binary file
 	tStart := time.Now()
-	maker, err := xdb.NewMaker(indexPolicy, srcFile, dstFile, fields)
+	maker, err := xdb.NewMaker(version, indexPolicy, srcFile, dstFile, fields)
 	if err != nil {
 		fmt.Printf("failed to create %s\n", err)
 		return
@@ -254,7 +268,7 @@ func testSearch() {
 		return
 	}
 
-	searcher, err := xdb.NewSearcher(dbFile)
+	searcher, err := xdb.NewSearcher(xdb.V4, dbFile)
 	if err != nil {
 		fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err.Error())
 		return
@@ -299,7 +313,7 @@ quit      : exit the test program`)
 			break
 		}
 
-		ip, err := xdb.CheckIP(line)
+		ip, err := xdb.ParseIP(line)
 		if err != nil {
 			fmt.Printf("invalid ip address `%s`\n", line)
 			continue
@@ -362,7 +376,7 @@ func testBench() {
 		return
 	}
 
-	searcher, err := xdb.NewSearcher(dbFile)
+	searcher, err := xdb.NewSearcher(xdb.V4, dbFile)
 	if err != nil {
 		fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err)
 		return
@@ -382,12 +396,13 @@ func testBench() {
 	var iErr = xdb.IterateSegments(handle, nil, func(seg *xdb.Segment) error {
 		var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
 		slog.Debug("try to bench", "segment", l)
-		mip := xdb.MidIP(seg.StartIP, seg.EndIP)
+		mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)
-		for _, ip := range []uint32{seg.StartIP, xdb.MidIP(seg.EndIP, mip), mip, xdb.MidIP(mip, seg.EndIP), seg.EndIP} {
+		for _, ip := range [][]byte{
-			slog.Debug("|-try to bench", "ip", xdb.Long2IP(ip))
+			seg.StartIP, xdb.IPMiddle(seg.EndIP, mip), mip, xdb.IPMiddle(mip, seg.EndIP), seg.EndIP} {
+			slog.Debug("|-try to bench", "ip", xdb.IP2String(ip))
 			r, _, err := searcher.Search(ip)
 			if err != nil {
-				return fmt.Errorf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err)
+				return fmt.Errorf("failed to search ip '%s': %s\n", xdb.IP2Long(ip), err)
 			}
 
 			// check the region info

maker/golang/xdb/editor.go

@@ -120,11 +120,13 @@ func (e *Editor) Put(ip string) (int, int, error) {
 // the following position relationships.
 // 1, A - fully contained like:
 // StartIP------seg.StartIP--------seg.EndIP----EndIP
-//                 |------------------|
+//
+//	|------------------|
+//
 // 2, B - intersect like:
 // StartIP------seg.StartIP------EndIP------|
-//                 |---------------------seg.EndIP
 //
+//	|---------------------seg.EndIP
 func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
 	var next *list.Element
 	var eList []*list.Element
@@ -138,7 +140,7 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
 		}
 
 		// found the related segment
-		if seg.StartIP <= s.EndIP && seg.StartIP >= s.StartIP {
+		if IPCompare(seg.StartIP, s.EndIP) <= 0 && IPCompare(seg.StartIP, s.StartIP) >= 0 {
 			found = true
 		}
 
@@ -147,7 +149,7 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
 		}
 
 		eList = append(eList, ele)
-		if seg.EndIP <= s.EndIP {
+		if IPCompare(seg.EndIP, s.EndIP) <= 0 {
 			break
 		}
 	}
@@ -167,10 +169,10 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
 	// segment split
 	var sList []*Segment
 	var head = eList[0].Value.(*Segment)
-	if seg.StartIP > head.StartIP {
+	if IPCompare(seg.StartIP, head.StartIP) > 0 {
 		sList = append(sList, &Segment{
 			StartIP: head.StartIP,
-			EndIP:   seg.StartIP - 1,
+			EndIP:   IPSubOne(seg.StartIP),
 			Region:  head.Region,
 		})
 	}
@@ -182,9 +184,9 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
 	if len(sList) > 0 {
 		// check and append the tailing
 		var tail = eList[len(eList)-1].Value.(*Segment)
-		if seg.EndIP < tail.EndIP {
+		if IPCompare(seg.EndIP, tail.EndIP) < 0 {
 			sList = append(sList, &Segment{
-				StartIP: seg.EndIP + 1,
+				StartIP: IPAddOne(seg.EndIP),
 				EndIP:   tail.EndIP,
 				Region:  tail.Region,
 			})

maker/golang/xdb/maker.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // ----
-// ip2region database v2.0 structure
+// Ip2Region database v2.0 structure
 //
 // +----------------+-------------------+---------------+--------------+
 // | header space   | speed up index    |  data payload | block index  |
@@ -53,20 +53,22 @@ import (
 	"encoding/binary"
 	"fmt"
 	"log/slog"
+	"math"
 	"os"
 	"strings"
 	"time"
 )
 
-const VersionNo = 2
+const VersionNo = 3
 const HeaderInfoLength = 256
 const VectorIndexRows = 256
 const VectorIndexCols = 256
 const VectorIndexSize = 8
-const SegmentIndexSize = 14
 const VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize
 
 type Maker struct {
+	version *Version
+
 	srcHandle *os.File
 	dstHandle *os.File
 
@@ -79,7 +81,7 @@ type Maker struct {
 	vectorIndex []byte
 }
 
-func NewMaker(policy IndexPolicy, srcFile string, dstFile string, fields []int) (*Maker, error) {
+func NewMaker(version *Version, policy IndexPolicy, srcFile string, dstFile string, fields []int) (*Maker, error) {
 	// open the source file with READONLY mode
 	srcHandle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600)
 	if err != nil {
@@ -93,6 +95,8 @@ func NewMaker(policy IndexPolicy, srcFile string, dstFile string, fields []int)
 	}
 
 	return &Maker{
+		version: version,
+
 		srcHandle: srcHandle,
 		dstHandle: dstHandle,
 
@@ -117,7 +121,7 @@ func (m *Maker) initDbHeader() error {
 	// make and write the header space
 	var header = make([]byte, 256)
 
-	// 1, version number
+	// 1, data version number
 	binary.LittleEndian.PutUint16(header, uint16(VersionNo))
 
 	// 2, index policy code
@@ -132,6 +136,9 @@ func (m *Maker) initDbHeader() error {
 	// 5, index block end ptr
 	binary.LittleEndian.PutUint32(header[12:], uint32(0))
 
+	// 6, ip version
+	binary.LittleEndian.PutUint16(header[16:], uint16(m.version.Id))
+
 	_, err = m.dstHandle.Write(header)
 	if err != nil {
 		return err
@@ -213,16 +220,16 @@ func (m *Maker) Init() error {
 }
 
 // refresh the vector index of the specified ip
-func (m *Maker) setVectorIndex(ip uint32, ptr uint32) {
+func (m *Maker) setVectorIndex(ip []byte, ptr uint32) {
-	var il0 = (ip >> 24) & 0xFF
+	var segIdxSize = uint32(m.version.SegmentIndexSize)
-	var il1 = (ip >> 16) & 0xFF
+	var il0, il1 = int(ip[0]), int(ip[1])
 	var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize
 	var sPtr = binary.LittleEndian.Uint32(m.vectorIndex[idx:])
 	if sPtr == 0 {
 		binary.LittleEndian.PutUint32(m.vectorIndex[idx:], ptr)
-		binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+SegmentIndexSize)
+		binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+segIdxSize)
 	} else {
-		binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+SegmentIndexSize)
+		binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+segIdxSize)
 	}
 }
 
@@ -258,6 +265,11 @@ func (m *Maker) Start() error {
 			return fmt.Errorf("seek to current ptr: %w", err)
 		}
 
+		// @TODO: remove this if the long ptr operation were supported
+		if pos >= math.MaxUint32 {
+			return fmt.Errorf("region ptr exceed the max length of %d", math.MaxUint32)
+		}
+
 		_, err = m.dstHandle.Write(region)
 		if err != nil {
 			return fmt.Errorf("write region '%s': %w", seg.Region, err)
@@ -269,7 +281,7 @@ func (m *Maker) Start() error {
 
 	// 2, write the index block and cache the super index block
 	slog.Info("try to write the segment index block ... ")
-	var indexBuff = make([]byte, SegmentIndexSize)
+	var indexBuff = make([]byte, m.version.SegmentIndexSize)
 	var counter, startIndexPtr, endIndexPtr = 0, int64(-1), int64(-1)
 	for _, seg := range m.segments {
 		dataPtr, has := m.regionPool[seg.Region]
@@ -285,6 +297,7 @@ func (m *Maker) Start() error {
 			return fmt.Errorf("empty region info for segment '%s'", seg)
 		}
 
+		var _offset = 0
 		var segList = seg.Split()
 		slog.Debug("try to index segment", "length", len(segList), "splits", seg.String())
 		for _, s := range segList {
@@ -293,11 +306,17 @@ func (m *Maker) Start() error {
 				return fmt.Errorf("seek to segment index block: %w", err)
 			}
 
+			// @TODO: remove this if the long ptr operation were supported
+			if pos >= math.MaxUint32 {
+				return fmt.Errorf("segment index ptr exceed the max length of %d", math.MaxUint32)
+			}
+
 			// encode the segment index
-			binary.LittleEndian.PutUint32(indexBuff, s.StartIP)
+			copy(indexBuff[0:], s.StartIP)
-			binary.LittleEndian.PutUint32(indexBuff[4:], s.EndIP)
+			copy(indexBuff[len(s.StartIP):], s.EndIP)
-			binary.LittleEndian.PutUint16(indexBuff[8:], uint16(dataLen))
+			_offset = len(s.StartIP) + len(s.EndIP)
-			binary.LittleEndian.PutUint32(indexBuff[10:], dataPtr)
+			binary.LittleEndian.PutUint16(indexBuff[_offset:], uint16(dataLen))
+			binary.LittleEndian.PutUint32(indexBuff[_offset+2:], dataPtr)
 			_, err = m.dstHandle.Write(indexBuff)
 			if err != nil {
 				return fmt.Errorf("write segment index for '%s': %w", s.String(), err)

maker/golang/xdb/searcher.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // ---
-// ip2region database v2.0 searcher.
+// Ip2Region database v2.0 searcher.
 // this is part of the maker for testing and validate.
 // please use the searcher in binding/golang for production use.
 // And this is a Not thread safe implementation.
@@ -17,6 +17,8 @@ import (
 )
 
 type Searcher struct {
+	version *Version
+
 	handle *os.File
 
 	// header info
@@ -28,13 +30,15 @@ type Searcher struct {
 	vectorIndex []byte
 }
 
-func NewSearcher(dbFile string) (*Searcher, error) {
+func NewSearcher(version *Version, dbFile string) (*Searcher, error) {
 	handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Searcher{
+		version: version,
+
 		handle: handle,
 		header: nil,
 
@@ -85,11 +89,10 @@ func (s *Searcher) ClearVectorIndex() {
 }
 
 // Search find the region for the specified ip address
-func (s *Searcher) Search(ip uint32) (string, int, error) {
+func (s *Searcher) Search(ip []byte) (string, int, error) {
 	// locate the segment index block based on the vector index
 	var ioCount = 0
-	var il0 = (ip >> 24) & 0xFF
+	var il0, il1, bytes, tBytes = int(ip[0]), int(ip[1]), len(ip), len(ip) << 1
-	var il1 = (ip >> 16) & 0xFF
 	var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize
 	var sPtr, ePtr = uint32(0), uint32(0)
 	if s.vectorIndex != nil {
@@ -118,13 +121,14 @@ func (s *Searcher) Search(ip uint32) (string, int, error) {
 
 	//log.Printf("vIndex=%s", vIndex)
 	// binary search the segment index to get the region
+	var segIndexSize = uint32(s.version.SegmentIndexSize)
 	var dataLen, dataPtr = 0, uint32(0)
-	var buff = make([]byte, SegmentIndexSize)
+	var buff = make([]byte, segIndexSize)
-	var l, h = 0, int((ePtr - sPtr) / SegmentIndexSize)
+	var l, h = 0, int((ePtr - sPtr) / segIndexSize)
 	for l <= h {
 		// log.Printf("l=%d, h=%d", l, h)
 		m := (l + h) >> 1
-		p := sPtr + uint32(m*SegmentIndexSize)
+		p := sPtr + uint32(m)*segIndexSize
 		// log.Printf("m=%d, p=%d", m, p)
 		_, err := s.handle.Seek(int64(p), 0)
 		if err != nil {
@@ -142,18 +146,14 @@ func (s *Searcher) Search(ip uint32) (string, int, error) {
 		}
 
 		// decode the data step by step to reduce the unnecessary calculations
-		sip := binary.LittleEndian.Uint32(buff)
+		if IPCompare(ip, buff[0:bytes]) < 0 {
-		if ip < sip {
 			h = m - 1
+		} else if IPCompare(ip, buff[bytes:tBytes]) > 0 {
+			l = m + 1
 		} else {
-			eip := binary.LittleEndian.Uint32(buff[4:])
+			dataLen = int(binary.LittleEndian.Uint16(buff[tBytes:]))
-			if ip > eip {
+			dataPtr = binary.LittleEndian.Uint32(buff[tBytes+2:])
-				l = m + 1
+			break
-			} else {
-				dataLen = int(binary.LittleEndian.Uint16(buff[8:]))
-				dataPtr = binary.LittleEndian.Uint32(buff[10:])
-				break
-			}
 		}
 	}
 

maker/golang/xdb/segment.go

@@ -10,8 +10,8 @@ import (
 )
 
 type Segment struct {
-	StartIP uint32
+	StartIP []byte
-	EndIP   uint32
+	EndIP   []byte
 	Region  string
 }
 
@@ -21,17 +21,17 @@ func SegmentFrom(seg string) (*Segment, error) {
 		return nil, fmt.Errorf("invalid ip segment `%s`", seg)
 	}
 
-	sip, err := CheckIP(ps[0])
+	sip, err := ParseIP(ps[0])
 	if err != nil {
 		return nil, fmt.Errorf("check start ip `%s`: %s", ps[0], err)
 	}
 
-	eip, err := CheckIP(ps[1])
+	eip, err := ParseIP(ps[1])
 	if err != nil {
 		return nil, fmt.Errorf("check end ip `%s`: %s", ps[1], err)
 	}
 
-	if sip > eip {
+	if IPCompare(sip, eip) > 0 {
 		return nil, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
 	}
 
@@ -45,10 +45,10 @@ func SegmentFrom(seg string) (*Segment, error) {
 // AfterCheck check the current segment is the one just after the specified one
 func (s *Segment) AfterCheck(last *Segment) error {
 	if last != nil {
-		if last.EndIP+1 != s.StartIP {
+		if IPCompare(IPAddOne(last.EndIP), s.StartIP) != 0 {
 			return fmt.Errorf(
-				"discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)",
+				"discontinuous data segment: last.eip(%s)+1 != seg.sip(%s, %s)",
-				last.EndIP+1, s.StartIP, s.Region,
+				IP2String(last.EndIP), IP2String(s.StartIP), s.Region,
 			)
 		}
 	}
@@ -60,17 +60,39 @@ func (s *Segment) AfterCheck(last *Segment) error {
 func (s *Segment) Split() []*Segment {
 	// 1, split the segment with the first byte
 	var tList []*Segment
-	var sByte1, eByte1 = (s.StartIP >> 24) & 0xFF, (s.EndIP >> 24) & 0xFF
+	var sByte1, eByte1 = int(s.StartIP[0]), int(s.EndIP[0])
-	var nSip = s.StartIP
+	// var nSip = s.StartIP
 	for i := sByte1; i <= eByte1; i++ {
-		sip := (i << 24) | (nSip & 0xFFFFFF)
+		// Make and init the new start & end IP
-		eip := (i << 24) | 0xFFFFFF
+		sip := make([]byte, len(s.StartIP))
-		if eip < s.EndIP {
+		eip := make([]byte, len(s.StartIP))
-			nSip = (i + 1) << 24
+
+		if i == sByte1 {
+			sip = s.StartIP
 		} else {
-			eip = s.EndIP
+			sip[0] = byte(i)
 		}
 
+		if i == eByte1 {
+			eip = s.EndIP
+		} else {
+			// set the first byte
+			eip[0] = byte(i)
+			// fill the buffer with 0xFF
+			for j := 1; j < len(eip); j++ {
+				eip[j] = 0xFF
+			}
+		}
+
+		// sip := (i << 24) | (nSip & 0xFFFFFF)
+		// eip := (i << 24) | 0xFFFFFF
+		// if eip < s.EndIP {
+		// 	nSip = (i + 1) << 24
+		// } else {
+		// 	eip = s.EndIP
+		// }
+
+		// fmt.Printf("sip:%+v, eip: %+v\n", sip, eip)
 		// append the new segment (maybe)
 		tList = append(tList, &Segment{
 			StartIP: sip,
@@ -83,18 +105,41 @@ func (s *Segment) Split() []*Segment {
 	// 2, split the segments with the second byte
 	var segList []*Segment
 	for _, seg := range tList {
-		base := seg.StartIP & 0xFF000000
+		// base := seg.StartIP & 0xFF000000
-		nSip := seg.StartIP
+		// nSip := seg.StartIP
-		sb2, eb2 := (seg.StartIP>>16)&0xFF, (seg.EndIP>>16)&0xFF
+		// sb2, eb2 := (seg.StartIP>>16)&0xFF, (seg.EndIP>>16)&0xFF
+		sb2, eb2 := int(seg.StartIP[1]), int(seg.EndIP[1])
+		// fmt.Printf("seg: %s, sb2: %d, eb2: %d\n", seg.String(), sb2, eb2)
 		for i := sb2; i <= eb2; i++ {
-			sip := base | (i << 16) | (nSip & 0xFFFF)
+			// sip := base | (i << 16) | (nSip & 0xFFFF)
-			eip := base | (i << 16) | 0xFFFF
+			// eip := base | (i << 16) | 0xFFFF
-			if eip < seg.EndIP {
+			// if eip < seg.EndIP {
-				nSip = 0
+			// 	nSip = 0
+			// } else {
+			// 	eip = seg.EndIP
+			// }
+
+			sip := make([]byte, len(s.StartIP))
+			eip := make([]byte, len(s.StartIP))
+			sip[0] = seg.StartIP[0]
+			eip[0] = seg.StartIP[0]
+
+			if i == sb2 {
+				sip = seg.StartIP
 			} else {
-				eip = seg.EndIP
+				sip[1] = byte(i)
 			}
 
+			if i == eb2 {
+				eip = seg.EndIP
+			} else {
+				eip[1] = byte(i)
+				for j := 2; j < len(eip); j++ {
+					eip[j] = 0xFF
+				}
+			}
+
+			// fmt.Printf("i=%d, sip:%+v, eip: %+v\n", i, sip, eip)
 			segList = append(segList, &Segment{
 				StartIP: sip,
 				EndIP:   eip,
@@ -107,5 +152,10 @@ func (s *Segment) Split() []*Segment {
 }
 
 func (s *Segment) String() string {
-	return fmt.Sprintf("%s|%s|%s", Long2IP(s.StartIP), Long2IP(s.EndIP), s.Region)
+	return fmt.Sprintf("%s|%s|%s", IP2String(s.StartIP), IP2String(s.EndIP), s.Region)
+}
+
+// Contains checks if an IP address is within this segment
+func (s *Segment) Contains(ip []byte) bool {
+	return IPCompare(s.StartIP, ip) <= 0 && IPCompare(ip, s.EndIP) <= 0
 }

maker/golang/xdb/util.go

@@ -7,44 +7,96 @@ package xdb
 import (
 	"bufio"
 	"fmt"
+	"math/big"
+	"net"
 	"os"
-	"strconv"
 	"strings"
 )
 
 // Util function
 
-var shiftIndex = []int{24, 16, 8, 0}
+func ParseIP(ip string) ([]byte, error) {
-
+	parsedIP := net.ParseIP(ip)
-func CheckIP(ip string) (uint32, error) {
+	if parsedIP == nil {
-	var ps = strings.Split(ip, ".")
+		return nil, fmt.Errorf("invalid ip address: %s", ip)
-	if len(ps) != 4 {
-		return 0, fmt.Errorf("invalid ip address `%s`", ip)
 	}
 
-	var val uint32
+	v4 := parsedIP.To4()
-	for i, s := range ps {
+	if v4 != nil {
-		d, err := strconv.Atoi(s)
+		return v4, nil
-		if err != nil {
-			return 0, fmt.Errorf("the %dth part `%s` is not an integer", i, s)
-		}
-
-		if d < 0 || d > 255 {
-			return 0, fmt.Errorf("the %dth part `%s` should be an integer bettween 0 and 255", i, s)
-		}
-
-		val |= uint32(d) << shiftIndex[i]
 	}
 
-	return val, nil
+	v6 := parsedIP.To16()
+	if v6 != nil {
+		return v6, nil
+	}
+
+	return nil, fmt.Errorf("invalid ip address: %s", ip)
 }
 
-func Long2IP(ip uint32) string {
+func IP2String(ip []byte) string {
-	return fmt.Sprintf("%d.%d.%d.%d", (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, (ip>>0)&0xFF)
+	return net.IP(ip[:]).String()
 }
 
-func MidIP(sip uint32, eip uint32) uint32 {
+func IP2Long(ip []byte) *big.Int {
-	return uint32((uint64(sip) + uint64(eip)) >> 1)
+	return big.NewInt(0).SetBytes(ip)
+}
+
+// IPCompare compares two IP addresses
+// Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2
+func IPCompare(ip1, ip2 []byte) int {
+	for i := 0; i < len(ip1); i++ {
+		if ip1[i] < ip2[i] {
+			return -1
+		}
+
+		if ip1[i] > ip2[i] {
+			return 1
+		}
+	}
+
+	return 0
+}
+
+func IPAddOne(ip []byte) []byte {
+	var r = make([]byte, len(ip))
+	copy(r, ip)
+	for i := len(ip) - 1; i >= 0; i-- {
+		r[i]++
+		if r[i] != 0 { // No overflow
+			break
+		}
+	}
+
+	return r
+}
+
+func IPSubOne(ip []byte) []byte {
+	var r = make([]byte, len(ip))
+	copy(r, ip)
+	for i := len(ip) - 1; i >= 0; i-- {
+		if r[i] != 0 { // No borrow needed
+			r[i]--
+			break
+		}
+		r[i] = 0xFF // borrow from the next byte
+	}
+
+	return r
+}
+
+func IPMiddle(sip, eip []byte) []byte {
+	var result = make([]byte, len(sip))
+	var carry uint16 = 0
+
+	// Add the two addresses with carry
+	for i := len(sip) - 1; i >= 0; i-- {
+		sum := uint16(sip[i]) + uint16(eip[i]) + carry
+		result[i] = byte(sum >> 0x01) // Divide by 2
+		carry = (sum & 0x01) << 7     // Carry for next byte (shift to MSB)
+	}
+
+	return result
 }
 
 func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segment) error) error {
@@ -70,17 +122,17 @@ func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segmen
 			return fmt.Errorf("invalid ip segment line `%s`", l)
 		}
 
-		sip, err := CheckIP(ps[0])
+		sip, err := ParseIP(ps[0])
 		if err != nil {
 			return fmt.Errorf("check start ip `%s`: %s", ps[0], err)
 		}
 
-		eip, err := CheckIP(ps[1])
+		eip, err := ParseIP(ps[1])
 		if err != nil {
 			return fmt.Errorf("check end ip `%s`: %s", ps[1], err)
 		}
 
-		if sip > eip {
+		if IPCompare(sip, eip) > 0 {
 			return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
 		}
 
@@ -127,13 +179,13 @@ func CheckSegments(segList []*Segment) error {
 	var last *Segment
 	for _, seg := range segList {
 		// sip must <= eip
-		if seg.StartIP > seg.EndIP {
+		if IPCompare(seg.StartIP, seg.EndIP) > 0 {
 			return fmt.Errorf("segment `%s`: start ip should not be greater than end ip", seg.String())
 		}
 
 		// check the continuity of the data segment
 		if last != nil {
-			if last.EndIP+1 != seg.StartIP {
+			if IPCompare(IPAddOne(last.EndIP), seg.StartIP) != 0 {
 				return fmt.Errorf("discontinuous segment `%s`: last.eip+1 != cur.sip", seg.String())
 			}
 		}

maker/golang/xdb/util_test.go

@@ -5,44 +5,104 @@
 package xdb
 
 import (
-	"encoding/binary"
 	"fmt"
-	"net"
 	"os"
 	"testing"
 )
 
-func TestCheckIP(t *testing.T) {
+func TestParseIP(t *testing.T) {
-	var str = "29.34.191.255"
+	var ips = []string{"29.34.191.255", "2c0f:fff0::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}
-	ip, err := CheckIP(str)
+	for _, ip := range ips {
-	if err != nil {
+		bytes, err := ParseIP(ip)
-		t.Errorf("check ip `%s`: %s\n", str, err)
+		if err != nil {
-	}
+			t.Errorf("check ip `%s`: %s\n", IP2String(bytes), err)
+		}
 
-	netIP := net.ParseIP(str).To4()
+		nip := IP2String(bytes)
-	if netIP == nil {
+		fmt.Printf("checkip: (%s / %s), isEqual: %v\n", ip, nip, ip == nip)
-		t.Fatalf("parse ip `%s` failed", str)
 	}
-
-	u32 := binary.BigEndian.Uint32(netIP)
-	fmt.Printf("checkip: %d, parseip: %d, isEqual: %v\n", ip, u32, ip == u32)
 }
 
-func TestLong2IP(t *testing.T) {
+func TestIPCompare(t *testing.T) {
-	var str = "29.34.191.255"
+	var ipPairs = [][]string{
-	netIP := net.ParseIP(str).To4()
+		{"1.2.3.4", "1.2.3.5"},
-	if netIP == nil {
+		{"58.250.36.41", "58.250.30.41"},
-		t.Fatalf("parse ip `%s` failed", str)
+		{"2c10::", "2e00::"},
+		{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"},
+		{"fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
 	}
 
-	u32 := binary.BigEndian.Uint32(netIP)
+	for _, pairs := range ipPairs {
-	ipStr := Long2IP(u32)
+		fmt.Printf("IPCompare(%s, %s): %d\n", pairs[0], pairs[1], IPCompare([]byte(pairs[0]), []byte(pairs[1])))
-	fmt.Printf("originIP: %s, Long2IP: %s, isEqual: %v\n", str, ipStr, ipStr == str)
+	}
 }
 
-func TestSplitSegment(t *testing.T) {
+func TestIPAddOne(t *testing.T) {
+	var ipPairs = [][]string{
+		{"1.2.3.4", "1.2.3.5"},
+		{"2.3.4.5", "2.3.4.6"},
+		{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
+		{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"},
+		{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"},
+	}
+
+	for _, pairs := range ipPairs {
+		sip, err := ParseIP(pairs[0])
+		if err != nil {
+			t.Errorf("parse ip `%s`: %s\n", pairs[0], err)
+		}
+
+		eip, err := ParseIP(pairs[1])
+		if err != nil {
+			t.Errorf("parse ip `%s`: %s\n", pairs[1], err)
+		}
+
+		fmt.Printf("IPAddOne(%s) = %s ? %d\n",
+			pairs[0], pairs[1], IPCompare(IPAddOne(sip), eip))
+	}
+}
+
+func TestIPAddOne2(t *testing.T) {
+	var ip = []byte{0, 1, 2, 3}
+	nip := IPAddOne(ip)
+	fmt.Printf("nip: %+v, ip:%+v", ip, nip)
+}
+
+func TestIPSubOne(t *testing.T) {
+	var ipPairs = [][]string{
+		{"1.2.3.4", "1.2.3.5"},
+		{"2.3.4.5", "2.3.4.6"},
+		{"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"},
+		{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"},
+		{"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"},
+	}
+
+	for _, pairs := range ipPairs {
+		sip, err := ParseIP(pairs[0])
+		if err != nil {
+			t.Errorf("parse ip `%s`: %s\n", pairs[0], err)
+		}
+
+		eip, err := ParseIP(pairs[1])
+		if err != nil {
+			t.Errorf("parse ip `%s`: %s\n", pairs[1], err)
+		}
+
+		fmt.Printf("IPSubOne(%s) = %s ? %d\n",
+			pairs[1], pairs[0], IPCompare(IPSubOne(eip), sip))
+	}
+}
+
+func TestIPSubOne2(t *testing.T) {
+	var ip = []byte{0, 1, 2, 3}
+	nip := IPSubOne(ip)
+	fmt.Printf("nip: %+v, ip:%+v", ip, nip)
+}
+
+func TestSplitSegmentV4(t *testing.T) {
 	// var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信"
 	// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
+	// var str = "29.0.0.0|29.34.191.255|美国|0|0|0|0"
 	var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0"
 	seg, err := SegmentFrom(str)
 	if err != nil {
@@ -61,8 +121,27 @@ func TestSplitSegment(t *testing.T) {
 	}
 }
 
+func TestSplitSegmentV6(t *testing.T) {
+	var str = "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff||瑞士|弗里堡州||||专线用户|IANA"
+	seg, err := SegmentFrom(str)
+	if err != nil {
+		t.Fatalf("failed to parser segment '%s': %s", str, err)
+	}
+
+	fmt.Printf("idx: src, seg: %s\n", seg.String())
+	var segList = seg.Split()
+	err = CheckSegments(segList)
+	if err != nil {
+		t.Fatalf("check segments: %s", err.Error())
+	}
+
+	for i, s := range segList {
+		fmt.Printf("idx: %3d, seg: %s\n", i, s.String())
+	}
+}
+
 func TestIterateSegments(t *testing.T) {
-	handle, err := os.OpenFile("../../../data/segments.tests", os.O_RDONLY, 0600)
+	handle, err := os.OpenFile("../../../data/segments.tests.mixed", os.O_RDONLY, 0600)
 	if err != nil {
 		t.Fatalf("failed to open tests file: %s", err)
 	}

maker/golang/xdb/version.go

@@ -0,0 +1,51 @@
+// Copyright 2022 The Ip2Region Authors. All rights reserved.
+// Use of this source code is governed by a Apache2.0-style
+// license that can be found in the LICENSE file.
+
+package xdb
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Version struct {
+	Id               int
+	SegmentIndexSize int
+}
+
+var (
+	VX = &Version{}
+	V4 = &Version{
+		Id:               4,
+		SegmentIndexSize: 14, // 4 + 4 + 2 + 4
+	}
+	V6 = &Version{
+		Id:               6,
+		SegmentIndexSize: 38, // 16 + 16 + 2 + 4
+	}
+)
+
+func VersionFromData(ip string) (*Version, error) {
+	bytes, err := ParseIP(ip)
+	if err != nil {
+		return VX, fmt.Errorf("parse ip fail: %w", err)
+	}
+
+	if len(bytes) == 4 {
+		return V4, nil
+	}
+
+	return V6, nil
+}
+
+func VersionFromName(name string) (*Version, error) {
+	switch strings.ToUpper(name) {
+	case "V4", "IPV4":
+		return V4, nil
+	case "V6", "IPV6":
+		return V6, nil
+	default:
+		return VX, fmt.Errorf("invalid version name `%s`", name)
+	}
+}