Merge pull request #456 from lionsoul2014/fr_go_non_contiguous_range

non-contiguous ip ranges supports

maker/golang/xdb/editor.go

@@ -66,7 +66,7 @@ func (e *Editor) loadSegments() error {
 		}
 
 		// check the continuity of the data segment
-		if err := seg.AfterCheck(last); err != nil {
+		if err := seg.RightBehind(last); err != nil {
 			return err
 		}
 
@@ -189,16 +189,13 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) {
 	sList = append(sList, seg)
 
 	// check and do the tailing segment append
-	if len(sList) > 0 {
+	var tail = eList[len(eList)-1].Value.(*Segment)
-		// check and append the tailing
+	if IPCompare(seg.EndIP, tail.EndIP) < 0 {
-		var tail = eList[len(eList)-1].Value.(*Segment)
+		sList = append(sList, &Segment{
-		if IPCompare(seg.EndIP, tail.EndIP) < 0 {
+			StartIP: IPAddOne(seg.EndIP),
-			sList = append(sList, &Segment{
+			EndIP:   tail.EndIP,
-				StartIP: IPAddOne(seg.EndIP),
+			Region:  tail.Region,
-				EndIP:   tail.EndIP,
+		})
-				Region:  tail.Region,
-			})
-		}
 	}
 
 	// print for debug

maker/golang/xdb/maker.go

@@ -58,6 +58,7 @@ import (
 	"log/slog"
 	"math"
 	"os"
+	"sort"
 	"time"
 )
 
@@ -159,6 +160,7 @@ func (m *Maker) loadSegments() error {
 	slog.Info("try to load the segments ... ")
 	var last *Segment = nil
 	var tStart = time.Now()
+	var sorting = false
 
 	var iErr = IterateSegments(m.srcHandle, func(l string) {
 		slog.Debug("loaded", "segment", l)
@@ -171,9 +173,13 @@ func (m *Maker) loadSegments() error {
 			return fmt.Errorf("invalid ip segment(%s expected)", m.version.Name)
 		}
 
-		// check the continuity of the data segment
+		// check the order of the data segment
-		if err := seg.AfterCheck(last); err != nil {
+		// if err := seg.RightBehind(last); err != nil {
-			return err
+		if err := seg.After(last); err != nil {
+			// return err
+			// @Note: If the continuity is disrupted,
+			// we will sort all these segments later.
+			sorting = true
 		}
 
 		m.segments = append(m.segments, seg)
@@ -184,7 +190,27 @@ func (m *Maker) loadSegments() error {
 		return fmt.Errorf("failed to load segments: %s", iErr)
 	}
 
-	slog.Info("all segments loaded", "length", len(m.segments), "elapsed", time.Since(tStart))
+	// check and do the sorting
+	if sorting {
+		slog.Info("try to sort all the segments based on its start ip ...")
+		sort.Slice(m.segments, func(i, j int) bool {
+			return IPCompare(m.segments[i].StartIP, m.segments[j].StartIP) < 0
+		})
+
+		slog.Info("try to check if there is overlap in the segments  ...")
+		last = nil
+		for _, seg := range m.segments {
+			// check the order of the data segment
+			if err := seg.After(last); err != nil {
+				return fmt.Errorf("overlap checking: %w", err)
+			}
+
+			// reset the last
+			last = seg
+		}
+	}
+
+	slog.Info("all segments loaded", "length", len(m.segments), "sorting", sorting, "elapsed", time.Since(tStart))
 	return nil
 }
 

maker/golang/xdb/searcher.go

@@ -125,6 +125,10 @@ func (s *Searcher) Search(ip []byte) (string, int, error) {
 	}
 
 	//log.Printf("vIndex=%s", vIndex)
+	if sPtr == 0 || ePtr == 0 {
+		return "", ioCount, nil
+	}
+
 	// binary search the segment index to get the region
 	var segIndexSize = uint32(s.version.SegmentIndexSize)
 	var dataLen, dataPtr = 0, uint32(0)

maker/golang/xdb/segment.go

@@ -42,8 +42,9 @@ func SegmentFrom(seg string) (*Segment, error) {
 	}, nil
 }
 
-// AfterCheck check the current segment is the one just after the specified one
+// RightBehind check the current segment is just right behind the specified one
-func (s *Segment) AfterCheck(last *Segment) error {
+// which mean last.EndIP + 1 = s.startIP
+func (s *Segment) RightBehind(last *Segment) error {
 	if last != nil {
 		if IPCompare(IPAddOne(last.EndIP), s.StartIP) != 0 {
 			return fmt.Errorf(
@@ -56,6 +57,21 @@ func (s *Segment) AfterCheck(last *Segment) error {
 	return nil
 }
 
+// After check the current segment is after the specified one
+// which means last.EndIP < s.startIP
+func (s *Segment) After(last *Segment) error {
+	if last != nil {
+		if IPCompare(last.EndIP, s.StartIP) >= 0 {
+			return fmt.Errorf(
+				"disorder data segment: last.eip(%s) >= seg.sip(%s, %s)",
+				IP2String(last.EndIP), IP2String(s.StartIP), s.Region,
+			)
+		}
+	}
+
+	return nil
+}
+
 // Split the segment based on the pre-two bytes
 func (s *Segment) Split() []*Segment {
 	// 1, split the segment with the first byte

maker/golang/xdb/util.go

@@ -169,7 +169,7 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
 			last = seg
 			continue
 		} else if last.Region == seg.Region {
-			if err = seg.AfterCheck(last); err == nil {
+			if err = seg.RightBehind(last); err == nil {
 				last.EndIP = seg.EndIP
 				continue
 			}