diff --git a/data/segments.tests.mixed b/data/segments.tests.mixed new file mode 100644 index 0000000..a301731 --- /dev/null +++ b/data/segments.tests.mixed @@ -0,0 +1,10 @@ +192.168.2.1|192.168.2.20|0|0|0|内网IP|办公室A +192.168.2.21|192.168.2.30|0|0|0|内网IP|办公室A +192.168.2.31|192.168.2.60|0|0|0|内网IP|办公室B +192.168.2.61|192.168.2.91|0|0|0|内网IP|办公室B +223.255.236.0|223.255.239.255|中国|0|上海|上海市|电信 +2c0f:fff1::|2c0f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|毛里求斯|威廉平原区||卡特勒博尔纳||专线用户 +2e00::|2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|德国|黑森||美因河畔法兰克福||专线用户 +3000::|fbff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA +fe00::|fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA +fe80::|febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||||专线用户|IANA diff --git a/maker/golang/main.go b/maker/golang/main.go index c7f0f27..c90e257 100644 --- a/maker/golang/main.go +++ b/maker/golang/main.go @@ -7,7 +7,6 @@ package main import ( "bufio" "fmt" - "github.com/lionsoul2014/ip2region/maker/golang/xdb" "log" "log/slog" "os" @@ -16,6 +15,8 @@ import ( "strconv" "strings" "time" + + "github.com/lionsoul2014/ip2region/maker/golang/xdb" ) func printHelp() { @@ -151,7 +152,7 @@ func getFilterFields(fieldList string) ([]int, error) { func genDb() { var err error var srcFile, dstFile = "", "" - var fieldList, logLevel = "", "info" + var ipVersion, fieldList, logLevel = "auto", "", "info" var indexPolicy = xdb.VectorIndexPolicy var fErr = iterateFlags(func(key string, val string) error { switch key { @@ -159,6 +160,8 @@ func genDb() { srcFile = val case "dst": dstFile = val + case "version": + ipVersion = val case "log-level": logLevel = val case "field-list": @@ -184,6 +187,7 @@ func genDb() { fmt.Printf("options:\n") fmt.Printf(" --src string source ip text file path\n") fmt.Printf(" --dst string destination binary xdb file path\n") + fmt.Printf(" --version string IP version, options: ipv4/ipv6, default auto (detect from the source file)\n") fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") return @@ -202,9 +206,19 @@ func genDb() { return } + // check and define the IP version + var version *xdb.Version = nil + if ipVersion == "auto" { + version = xdb.V4 + } else if v, err := xdb.VersionFromName(ipVersion); err != nil { + slog.Error("failed to parse version name", "error", err) + } else { + version = v + } + // make the binary file tStart := time.Now() - maker, err := xdb.NewMaker(indexPolicy, srcFile, dstFile, fields) + maker, err := xdb.NewMaker(version, indexPolicy, srcFile, dstFile, fields) if err != nil { fmt.Printf("failed to create %s\n", err) return @@ -254,7 +268,7 @@ func testSearch() { return } - searcher, err := xdb.NewSearcher(dbFile) + searcher, err := xdb.NewSearcher(xdb.V4, dbFile) if err != nil { fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err.Error()) return @@ -299,7 +313,7 @@ quit : exit the test program`) break } - ip, err := xdb.CheckIP(line) + ip, err := xdb.ParseIP(line) if err != nil { fmt.Printf("invalid ip address `%s`\n", line) continue @@ -362,7 +376,7 @@ func testBench() { return } - searcher, err := xdb.NewSearcher(dbFile) + searcher, err := xdb.NewSearcher(xdb.V4, dbFile) if err != nil { fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err) return @@ -382,12 +396,13 @@ func testBench() { var iErr = xdb.IterateSegments(handle, nil, func(seg *xdb.Segment) error { var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) slog.Debug("try to bench", "segment", l) - mip := xdb.MidIP(seg.StartIP, seg.EndIP) - for _, ip := range []uint32{seg.StartIP, xdb.MidIP(seg.EndIP, mip), mip, xdb.MidIP(mip, seg.EndIP), seg.EndIP} { - slog.Debug("|-try to bench", "ip", xdb.Long2IP(ip)) + mip := xdb.IPMiddle(seg.StartIP, seg.EndIP) + for _, ip := range [][]byte{ + seg.StartIP, xdb.IPMiddle(seg.EndIP, mip), mip, xdb.IPMiddle(mip, seg.EndIP), seg.EndIP} { + slog.Debug("|-try to bench", "ip", xdb.IP2String(ip)) r, _, err := searcher.Search(ip) if err != nil { - return fmt.Errorf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) + return fmt.Errorf("failed to search ip '%s': %s\n", xdb.IP2Long(ip), err) } // check the region info diff --git a/maker/golang/xdb/editor.go b/maker/golang/xdb/editor.go index 42232c9..7258fc5 100644 --- a/maker/golang/xdb/editor.go +++ b/maker/golang/xdb/editor.go @@ -120,11 +120,13 @@ func (e *Editor) Put(ip string) (int, int, error) { // the following position relationships. // 1, A - fully contained like: // StartIP------seg.StartIP--------seg.EndIP----EndIP -// |------------------| +// +// |------------------| +// // 2, B - intersect like: // StartIP------seg.StartIP------EndIP------| -// |---------------------seg.EndIP // +// |---------------------seg.EndIP func (e *Editor) PutSegment(seg *Segment) (int, int, error) { var next *list.Element var eList []*list.Element @@ -138,7 +140,7 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) { } // found the related segment - if seg.StartIP <= s.EndIP && seg.StartIP >= s.StartIP { + if IPCompare(seg.StartIP, s.EndIP) <= 0 && IPCompare(seg.StartIP, s.StartIP) >= 0 { found = true } @@ -147,7 +149,7 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) { } eList = append(eList, ele) - if seg.EndIP <= s.EndIP { + if IPCompare(seg.EndIP, s.EndIP) <= 0 { break } } @@ -167,10 +169,10 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) { // segment split var sList []*Segment var head = eList[0].Value.(*Segment) - if seg.StartIP > head.StartIP { + if IPCompare(seg.StartIP, head.StartIP) > 0 { sList = append(sList, &Segment{ StartIP: head.StartIP, - EndIP: seg.StartIP - 1, + EndIP: IPSubOne(seg.StartIP), Region: head.Region, }) } @@ -182,9 +184,9 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) { if len(sList) > 0 { // check and append the tailing var tail = eList[len(eList)-1].Value.(*Segment) - if seg.EndIP < tail.EndIP { + if IPCompare(seg.EndIP, tail.EndIP) < 0 { sList = append(sList, &Segment{ - StartIP: seg.EndIP + 1, + StartIP: IPAddOne(seg.EndIP), EndIP: tail.EndIP, Region: tail.Region, }) diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index b4b14a0..cdf58e7 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // ---- -// ip2region database v2.0 structure +// Ip2Region database v2.0 structure // // +----------------+-------------------+---------------+--------------+ // | header space | speed up index | data payload | block index | @@ -53,20 +53,22 @@ import ( "encoding/binary" "fmt" "log/slog" + "math" "os" "strings" "time" ) -const VersionNo = 2 +const VersionNo = 3 const HeaderInfoLength = 256 const VectorIndexRows = 256 const VectorIndexCols = 256 const VectorIndexSize = 8 -const SegmentIndexSize = 14 const VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize type Maker struct { + version *Version + srcHandle *os.File dstHandle *os.File @@ -79,7 +81,7 @@ type Maker struct { vectorIndex []byte } -func NewMaker(policy IndexPolicy, srcFile string, dstFile string, fields []int) (*Maker, error) { +func NewMaker(version *Version, policy IndexPolicy, srcFile string, dstFile string, fields []int) (*Maker, error) { // open the source file with READONLY mode srcHandle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) if err != nil { @@ -93,6 +95,8 @@ func NewMaker(policy IndexPolicy, srcFile string, dstFile string, fields []int) } return &Maker{ + version: version, + srcHandle: srcHandle, dstHandle: dstHandle, @@ -117,7 +121,7 @@ func (m *Maker) initDbHeader() error { // make and write the header space var header = make([]byte, 256) - // 1, version number + // 1, data version number binary.LittleEndian.PutUint16(header, uint16(VersionNo)) // 2, index policy code @@ -132,6 +136,9 @@ func (m *Maker) initDbHeader() error { // 5, index block end ptr binary.LittleEndian.PutUint32(header[12:], uint32(0)) + // 6, ip version + binary.LittleEndian.PutUint16(header[16:], uint16(m.version.Id)) + _, err = m.dstHandle.Write(header) if err != nil { return err @@ -213,16 +220,16 @@ func (m *Maker) Init() error { } // refresh the vector index of the specified ip -func (m *Maker) setVectorIndex(ip uint32, ptr uint32) { - var il0 = (ip >> 24) & 0xFF - var il1 = (ip >> 16) & 0xFF +func (m *Maker) setVectorIndex(ip []byte, ptr uint32) { + var segIdxSize = uint32(m.version.SegmentIndexSize) + var il0, il1 = int(ip[0]), int(ip[1]) var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize var sPtr = binary.LittleEndian.Uint32(m.vectorIndex[idx:]) if sPtr == 0 { binary.LittleEndian.PutUint32(m.vectorIndex[idx:], ptr) - binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+SegmentIndexSize) + binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+segIdxSize) } else { - binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+SegmentIndexSize) + binary.LittleEndian.PutUint32(m.vectorIndex[idx+4:], ptr+segIdxSize) } } @@ -258,6 +265,11 @@ func (m *Maker) Start() error { return fmt.Errorf("seek to current ptr: %w", err) } + // @TODO: remove this if the long ptr operation were supported + if pos >= math.MaxUint32 { + return fmt.Errorf("region ptr exceed the max length of %d", math.MaxUint32) + } + _, err = m.dstHandle.Write(region) if err != nil { return fmt.Errorf("write region '%s': %w", seg.Region, err) @@ -269,7 +281,7 @@ func (m *Maker) Start() error { // 2, write the index block and cache the super index block slog.Info("try to write the segment index block ... ") - var indexBuff = make([]byte, SegmentIndexSize) + var indexBuff = make([]byte, m.version.SegmentIndexSize) var counter, startIndexPtr, endIndexPtr = 0, int64(-1), int64(-1) for _, seg := range m.segments { dataPtr, has := m.regionPool[seg.Region] @@ -285,6 +297,7 @@ func (m *Maker) Start() error { return fmt.Errorf("empty region info for segment '%s'", seg) } + var _offset = 0 var segList = seg.Split() slog.Debug("try to index segment", "length", len(segList), "splits", seg.String()) for _, s := range segList { @@ -293,11 +306,17 @@ func (m *Maker) Start() error { return fmt.Errorf("seek to segment index block: %w", err) } + // @TODO: remove this if the long ptr operation were supported + if pos >= math.MaxUint32 { + return fmt.Errorf("segment index ptr exceed the max length of %d", math.MaxUint32) + } + // encode the segment index - binary.LittleEndian.PutUint32(indexBuff, s.StartIP) - binary.LittleEndian.PutUint32(indexBuff[4:], s.EndIP) - binary.LittleEndian.PutUint16(indexBuff[8:], uint16(dataLen)) - binary.LittleEndian.PutUint32(indexBuff[10:], dataPtr) + copy(indexBuff[0:], s.StartIP) + copy(indexBuff[len(s.StartIP):], s.EndIP) + _offset = len(s.StartIP) + len(s.EndIP) + binary.LittleEndian.PutUint16(indexBuff[_offset:], uint16(dataLen)) + binary.LittleEndian.PutUint32(indexBuff[_offset+2:], dataPtr) _, err = m.dstHandle.Write(indexBuff) if err != nil { return fmt.Errorf("write segment index for '%s': %w", s.String(), err) diff --git a/maker/golang/xdb/searcher.go b/maker/golang/xdb/searcher.go index 62151e4..9fcc743 100644 --- a/maker/golang/xdb/searcher.go +++ b/maker/golang/xdb/searcher.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // --- -// ip2region database v2.0 searcher. +// Ip2Region database v2.0 searcher. // this is part of the maker for testing and validate. // please use the searcher in binding/golang for production use. // And this is a Not thread safe implementation. @@ -17,6 +17,8 @@ import ( ) type Searcher struct { + version *Version + handle *os.File // header info @@ -28,13 +30,15 @@ type Searcher struct { vectorIndex []byte } -func NewSearcher(dbFile string) (*Searcher, error) { +func NewSearcher(version *Version, dbFile string) (*Searcher, error) { handle, err := os.OpenFile(dbFile, os.O_RDONLY, 0600) if err != nil { return nil, err } return &Searcher{ + version: version, + handle: handle, header: nil, @@ -85,11 +89,10 @@ func (s *Searcher) ClearVectorIndex() { } // Search find the region for the specified ip address -func (s *Searcher) Search(ip uint32) (string, int, error) { +func (s *Searcher) Search(ip []byte) (string, int, error) { // locate the segment index block based on the vector index var ioCount = 0 - var il0 = (ip >> 24) & 0xFF - var il1 = (ip >> 16) & 0xFF + var il0, il1, bytes, tBytes = int(ip[0]), int(ip[1]), len(ip), len(ip) << 1 var idx = il0*VectorIndexCols*VectorIndexSize + il1*VectorIndexSize var sPtr, ePtr = uint32(0), uint32(0) if s.vectorIndex != nil { @@ -118,13 +121,14 @@ func (s *Searcher) Search(ip uint32) (string, int, error) { //log.Printf("vIndex=%s", vIndex) // binary search the segment index to get the region + var segIndexSize = uint32(s.version.SegmentIndexSize) var dataLen, dataPtr = 0, uint32(0) - var buff = make([]byte, SegmentIndexSize) - var l, h = 0, int((ePtr - sPtr) / SegmentIndexSize) + var buff = make([]byte, segIndexSize) + var l, h = 0, int((ePtr - sPtr) / segIndexSize) for l <= h { // log.Printf("l=%d, h=%d", l, h) m := (l + h) >> 1 - p := sPtr + uint32(m*SegmentIndexSize) + p := sPtr + uint32(m)*segIndexSize // log.Printf("m=%d, p=%d", m, p) _, err := s.handle.Seek(int64(p), 0) if err != nil { @@ -142,18 +146,14 @@ func (s *Searcher) Search(ip uint32) (string, int, error) { } // decode the data step by step to reduce the unnecessary calculations - sip := binary.LittleEndian.Uint32(buff) - if ip < sip { + if IPCompare(ip, buff[0:bytes]) < 0 { h = m - 1 + } else if IPCompare(ip, buff[bytes:tBytes]) > 0 { + l = m + 1 } else { - eip := binary.LittleEndian.Uint32(buff[4:]) - if ip > eip { - l = m + 1 - } else { - dataLen = int(binary.LittleEndian.Uint16(buff[8:])) - dataPtr = binary.LittleEndian.Uint32(buff[10:]) - break - } + dataLen = int(binary.LittleEndian.Uint16(buff[tBytes:])) + dataPtr = binary.LittleEndian.Uint32(buff[tBytes+2:]) + break } } diff --git a/maker/golang/xdb/segment.go b/maker/golang/xdb/segment.go index ac5e253..cabc4b7 100644 --- a/maker/golang/xdb/segment.go +++ b/maker/golang/xdb/segment.go @@ -10,8 +10,8 @@ import ( ) type Segment struct { - StartIP uint32 - EndIP uint32 + StartIP []byte + EndIP []byte Region string } @@ -21,17 +21,17 @@ func SegmentFrom(seg string) (*Segment, error) { return nil, fmt.Errorf("invalid ip segment `%s`", seg) } - sip, err := CheckIP(ps[0]) + sip, err := ParseIP(ps[0]) if err != nil { return nil, fmt.Errorf("check start ip `%s`: %s", ps[0], err) } - eip, err := CheckIP(ps[1]) + eip, err := ParseIP(ps[1]) if err != nil { return nil, fmt.Errorf("check end ip `%s`: %s", ps[1], err) } - if sip > eip { + if IPCompare(sip, eip) > 0 { return nil, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) } @@ -45,10 +45,10 @@ func SegmentFrom(seg string) (*Segment, error) { // AfterCheck check the current segment is the one just after the specified one func (s *Segment) AfterCheck(last *Segment) error { if last != nil { - if last.EndIP+1 != s.StartIP { + if IPCompare(IPAddOne(last.EndIP), s.StartIP) != 0 { return fmt.Errorf( - "discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", - last.EndIP+1, s.StartIP, s.Region, + "discontinuous data segment: last.eip(%s)+1 != seg.sip(%s, %s)", + IP2String(last.EndIP), IP2String(s.StartIP), s.Region, ) } } @@ -60,17 +60,39 @@ func (s *Segment) AfterCheck(last *Segment) error { func (s *Segment) Split() []*Segment { // 1, split the segment with the first byte var tList []*Segment - var sByte1, eByte1 = (s.StartIP >> 24) & 0xFF, (s.EndIP >> 24) & 0xFF - var nSip = s.StartIP + var sByte1, eByte1 = int(s.StartIP[0]), int(s.EndIP[0]) + // var nSip = s.StartIP for i := sByte1; i <= eByte1; i++ { - sip := (i << 24) | (nSip & 0xFFFFFF) - eip := (i << 24) | 0xFFFFFF - if eip < s.EndIP { - nSip = (i + 1) << 24 + // Make and init the new start & end IP + sip := make([]byte, len(s.StartIP)) + eip := make([]byte, len(s.StartIP)) + + if i == sByte1 { + sip = s.StartIP } else { - eip = s.EndIP + sip[0] = byte(i) } + if i == eByte1 { + eip = s.EndIP + } else { + // set the first byte + eip[0] = byte(i) + // fill the buffer with 0xFF + for j := 1; j < len(eip); j++ { + eip[j] = 0xFF + } + } + + // sip := (i << 24) | (nSip & 0xFFFFFF) + // eip := (i << 24) | 0xFFFFFF + // if eip < s.EndIP { + // nSip = (i + 1) << 24 + // } else { + // eip = s.EndIP + // } + + // fmt.Printf("sip:%+v, eip: %+v\n", sip, eip) // append the new segment (maybe) tList = append(tList, &Segment{ StartIP: sip, @@ -83,18 +105,41 @@ func (s *Segment) Split() []*Segment { // 2, split the segments with the second byte var segList []*Segment for _, seg := range tList { - base := seg.StartIP & 0xFF000000 - nSip := seg.StartIP - sb2, eb2 := (seg.StartIP>>16)&0xFF, (seg.EndIP>>16)&0xFF + // base := seg.StartIP & 0xFF000000 + // nSip := seg.StartIP + // sb2, eb2 := (seg.StartIP>>16)&0xFF, (seg.EndIP>>16)&0xFF + sb2, eb2 := int(seg.StartIP[1]), int(seg.EndIP[1]) + // fmt.Printf("seg: %s, sb2: %d, eb2: %d\n", seg.String(), sb2, eb2) for i := sb2; i <= eb2; i++ { - sip := base | (i << 16) | (nSip & 0xFFFF) - eip := base | (i << 16) | 0xFFFF - if eip < seg.EndIP { - nSip = 0 + // sip := base | (i << 16) | (nSip & 0xFFFF) + // eip := base | (i << 16) | 0xFFFF + // if eip < seg.EndIP { + // nSip = 0 + // } else { + // eip = seg.EndIP + // } + + sip := make([]byte, len(s.StartIP)) + eip := make([]byte, len(s.StartIP)) + sip[0] = seg.StartIP[0] + eip[0] = seg.StartIP[0] + + if i == sb2 { + sip = seg.StartIP } else { - eip = seg.EndIP + sip[1] = byte(i) } + if i == eb2 { + eip = seg.EndIP + } else { + eip[1] = byte(i) + for j := 2; j < len(eip); j++ { + eip[j] = 0xFF + } + } + + // fmt.Printf("i=%d, sip:%+v, eip: %+v\n", i, sip, eip) segList = append(segList, &Segment{ StartIP: sip, EndIP: eip, @@ -107,5 +152,10 @@ func (s *Segment) Split() []*Segment { } func (s *Segment) String() string { - return fmt.Sprintf("%s|%s|%s", Long2IP(s.StartIP), Long2IP(s.EndIP), s.Region) + return fmt.Sprintf("%s|%s|%s", IP2String(s.StartIP), IP2String(s.EndIP), s.Region) +} + +// Contains checks if an IP address is within this segment +func (s *Segment) Contains(ip []byte) bool { + return IPCompare(s.StartIP, ip) <= 0 && IPCompare(ip, s.EndIP) <= 0 } diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index 0c5ac21..dcd344e 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -7,44 +7,96 @@ package xdb import ( "bufio" "fmt" + "math/big" + "net" "os" - "strconv" "strings" ) // Util function -var shiftIndex = []int{24, 16, 8, 0} - -func CheckIP(ip string) (uint32, error) { - var ps = strings.Split(ip, ".") - if len(ps) != 4 { - return 0, fmt.Errorf("invalid ip address `%s`", ip) +func ParseIP(ip string) ([]byte, error) { + parsedIP := net.ParseIP(ip) + if parsedIP == nil { + return nil, fmt.Errorf("invalid ip address: %s", ip) } - var val uint32 - for i, s := range ps { - d, err := strconv.Atoi(s) - if err != nil { - return 0, fmt.Errorf("the %dth part `%s` is not an integer", i, s) - } - - if d < 0 || d > 255 { - return 0, fmt.Errorf("the %dth part `%s` should be an integer bettween 0 and 255", i, s) - } - - val |= uint32(d) << shiftIndex[i] + v4 := parsedIP.To4() + if v4 != nil { + return v4, nil } - return val, nil + v6 := parsedIP.To16() + if v6 != nil { + return v6, nil + } + + return nil, fmt.Errorf("invalid ip address: %s", ip) } -func Long2IP(ip uint32) string { - return fmt.Sprintf("%d.%d.%d.%d", (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, (ip>>0)&0xFF) +func IP2String(ip []byte) string { + return net.IP(ip[:]).String() } -func MidIP(sip uint32, eip uint32) uint32 { - return uint32((uint64(sip) + uint64(eip)) >> 1) +func IP2Long(ip []byte) *big.Int { + return big.NewInt(0).SetBytes(ip) +} + +// IPCompare compares two IP addresses +// Returns: -1 if ip1 < ip2, 0 if ip1 == ip2, 1 if ip1 > ip2 +func IPCompare(ip1, ip2 []byte) int { + for i := 0; i < len(ip1); i++ { + if ip1[i] < ip2[i] { + return -1 + } + + if ip1[i] > ip2[i] { + return 1 + } + } + + return 0 +} + +func IPAddOne(ip []byte) []byte { + var r = make([]byte, len(ip)) + copy(r, ip) + for i := len(ip) - 1; i >= 0; i-- { + r[i]++ + if r[i] != 0 { // No overflow + break + } + } + + return r +} + +func IPSubOne(ip []byte) []byte { + var r = make([]byte, len(ip)) + copy(r, ip) + for i := len(ip) - 1; i >= 0; i-- { + if r[i] != 0 { // No borrow needed + r[i]-- + break + } + r[i] = 0xFF // borrow from the next byte + } + + return r +} + +func IPMiddle(sip, eip []byte) []byte { + var result = make([]byte, len(sip)) + var carry uint16 = 0 + + // Add the two addresses with carry + for i := len(sip) - 1; i >= 0; i-- { + sum := uint16(sip[i]) + uint16(eip[i]) + carry + result[i] = byte(sum >> 0x01) // Divide by 2 + carry = (sum & 0x01) << 7 // Carry for next byte (shift to MSB) + } + + return result } func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segment) error) error { @@ -70,17 +122,17 @@ func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segmen return fmt.Errorf("invalid ip segment line `%s`", l) } - sip, err := CheckIP(ps[0]) + sip, err := ParseIP(ps[0]) if err != nil { return fmt.Errorf("check start ip `%s`: %s", ps[0], err) } - eip, err := CheckIP(ps[1]) + eip, err := ParseIP(ps[1]) if err != nil { return fmt.Errorf("check end ip `%s`: %s", ps[1], err) } - if sip > eip { + if IPCompare(sip, eip) > 0 { return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) } @@ -127,13 +179,13 @@ func CheckSegments(segList []*Segment) error { var last *Segment for _, seg := range segList { // sip must <= eip - if seg.StartIP > seg.EndIP { + if IPCompare(seg.StartIP, seg.EndIP) > 0 { return fmt.Errorf("segment `%s`: start ip should not be greater than end ip", seg.String()) } // check the continuity of the data segment if last != nil { - if last.EndIP+1 != seg.StartIP { + if IPCompare(IPAddOne(last.EndIP), seg.StartIP) != 0 { return fmt.Errorf("discontinuous segment `%s`: last.eip+1 != cur.sip", seg.String()) } } diff --git a/maker/golang/xdb/util_test.go b/maker/golang/xdb/util_test.go index 84f1aea..b4478d6 100644 --- a/maker/golang/xdb/util_test.go +++ b/maker/golang/xdb/util_test.go @@ -5,44 +5,104 @@ package xdb import ( - "encoding/binary" "fmt" - "net" "os" "testing" ) -func TestCheckIP(t *testing.T) { - var str = "29.34.191.255" - ip, err := CheckIP(str) - if err != nil { - t.Errorf("check ip `%s`: %s\n", str, err) - } +func TestParseIP(t *testing.T) { + var ips = []string{"29.34.191.255", "2c0f:fff0::", "2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"} + for _, ip := range ips { + bytes, err := ParseIP(ip) + if err != nil { + t.Errorf("check ip `%s`: %s\n", IP2String(bytes), err) + } - netIP := net.ParseIP(str).To4() - if netIP == nil { - t.Fatalf("parse ip `%s` failed", str) + nip := IP2String(bytes) + fmt.Printf("checkip: (%s / %s), isEqual: %v\n", ip, nip, ip == nip) } - - u32 := binary.BigEndian.Uint32(netIP) - fmt.Printf("checkip: %d, parseip: %d, isEqual: %v\n", ip, u32, ip == u32) } -func TestLong2IP(t *testing.T) { - var str = "29.34.191.255" - netIP := net.ParseIP(str).To4() - if netIP == nil { - t.Fatalf("parse ip `%s` failed", str) +func TestIPCompare(t *testing.T) { + var ipPairs = [][]string{ + {"1.2.3.4", "1.2.3.5"}, + {"58.250.36.41", "58.250.30.41"}, + {"2c10::", "2e00::"}, + {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff"}, + {"fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, } - u32 := binary.BigEndian.Uint32(netIP) - ipStr := Long2IP(u32) - fmt.Printf("originIP: %s, Long2IP: %s, isEqual: %v\n", str, ipStr, ipStr == str) + for _, pairs := range ipPairs { + fmt.Printf("IPCompare(%s, %s): %d\n", pairs[0], pairs[1], IPCompare([]byte(pairs[0]), []byte(pairs[1]))) + } } -func TestSplitSegment(t *testing.T) { +func TestIPAddOne(t *testing.T) { + var ipPairs = [][]string{ + {"1.2.3.4", "1.2.3.5"}, + {"2.3.4.5", "2.3.4.6"}, + {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, + {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"}, + {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"}, + } + + for _, pairs := range ipPairs { + sip, err := ParseIP(pairs[0]) + if err != nil { + t.Errorf("parse ip `%s`: %s\n", pairs[0], err) + } + + eip, err := ParseIP(pairs[1]) + if err != nil { + t.Errorf("parse ip `%s`: %s\n", pairs[1], err) + } + + fmt.Printf("IPAddOne(%s) = %s ? %d\n", + pairs[0], pairs[1], IPCompare(IPAddOne(sip), eip)) + } +} + +func TestIPAddOne2(t *testing.T) { + var ip = []byte{0, 1, 2, 3} + nip := IPAddOne(ip) + fmt.Printf("nip: %+v, ip:%+v", ip, nip) +} + +func TestIPSubOne(t *testing.T) { + var ipPairs = [][]string{ + {"1.2.3.4", "1.2.3.5"}, + {"2.3.4.5", "2.3.4.6"}, + {"fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "fe00::"}, + {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::"}, + {"2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", "3000::1"}, + } + + for _, pairs := range ipPairs { + sip, err := ParseIP(pairs[0]) + if err != nil { + t.Errorf("parse ip `%s`: %s\n", pairs[0], err) + } + + eip, err := ParseIP(pairs[1]) + if err != nil { + t.Errorf("parse ip `%s`: %s\n", pairs[1], err) + } + + fmt.Printf("IPSubOne(%s) = %s ? %d\n", + pairs[1], pairs[0], IPCompare(IPSubOne(eip), sip)) + } +} + +func TestIPSubOne2(t *testing.T) { + var ip = []byte{0, 1, 2, 3} + nip := IPSubOne(ip) + fmt.Printf("nip: %+v, ip:%+v", ip, nip) +} + +func TestSplitSegmentV4(t *testing.T) { // var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信" // var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP" + // var str = "29.0.0.0|29.34.191.255|美国|0|0|0|0" var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0" seg, err := SegmentFrom(str) if err != nil { @@ -61,8 +121,27 @@ func TestSplitSegment(t *testing.T) { } } +func TestSplitSegmentV6(t *testing.T) { + var str = "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff||瑞士|弗里堡州||||专线用户|IANA" + seg, err := SegmentFrom(str) + if err != nil { + t.Fatalf("failed to parser segment '%s': %s", str, err) + } + + fmt.Printf("idx: src, seg: %s\n", seg.String()) + var segList = seg.Split() + err = CheckSegments(segList) + if err != nil { + t.Fatalf("check segments: %s", err.Error()) + } + + for i, s := range segList { + fmt.Printf("idx: %3d, seg: %s\n", i, s.String()) + } +} + func TestIterateSegments(t *testing.T) { - handle, err := os.OpenFile("../../../data/segments.tests", os.O_RDONLY, 0600) + handle, err := os.OpenFile("../../../data/segments.tests.mixed", os.O_RDONLY, 0600) if err != nil { t.Fatalf("failed to open tests file: %s", err) } diff --git a/maker/golang/xdb/version.go b/maker/golang/xdb/version.go new file mode 100644 index 0000000..51e89c9 --- /dev/null +++ b/maker/golang/xdb/version.go @@ -0,0 +1,51 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package xdb + +import ( + "fmt" + "strings" +) + +type Version struct { + Id int + SegmentIndexSize int +} + +var ( + VX = &Version{} + V4 = &Version{ + Id: 4, + SegmentIndexSize: 14, // 4 + 4 + 2 + 4 + } + V6 = &Version{ + Id: 6, + SegmentIndexSize: 38, // 16 + 16 + 2 + 4 + } +) + +func VersionFromData(ip string) (*Version, error) { + bytes, err := ParseIP(ip) + if err != nil { + return VX, fmt.Errorf("parse ip fail: %w", err) + } + + if len(bytes) == 4 { + return V4, nil + } + + return V6, nil +} + +func VersionFromName(name string) (*Version, error) { + switch strings.ToUpper(name) { + case "V4", "IPV4": + return V4, nil + case "V6", "IPV6": + return V6, nil + default: + return VX, fmt.Errorf("invalid version name `%s`", name) + } +}