diff --git a/maker/golang/xdb/editor.go b/maker/golang/xdb/editor.go index 3a52804..6f26d20 100644 --- a/maker/golang/xdb/editor.go +++ b/maker/golang/xdb/editor.go @@ -66,7 +66,7 @@ func (e *Editor) loadSegments() error { } // check the continuity of the data segment - if err := seg.AfterCheck(last); err != nil { + if err := seg.RightBehind(last); err != nil { return err } @@ -189,16 +189,13 @@ func (e *Editor) PutSegment(seg *Segment) (int, int, error) { sList = append(sList, seg) // check and do the tailing segment append - if len(sList) > 0 { - // check and append the tailing - var tail = eList[len(eList)-1].Value.(*Segment) - if IPCompare(seg.EndIP, tail.EndIP) < 0 { - sList = append(sList, &Segment{ - StartIP: IPAddOne(seg.EndIP), - EndIP: tail.EndIP, - Region: tail.Region, - }) - } + var tail = eList[len(eList)-1].Value.(*Segment) + if IPCompare(seg.EndIP, tail.EndIP) < 0 { + sList = append(sList, &Segment{ + StartIP: IPAddOne(seg.EndIP), + EndIP: tail.EndIP, + Region: tail.Region, + }) } // print for debug diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index ef94f12..570edc2 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -58,6 +58,7 @@ import ( "log/slog" "math" "os" + "sort" "time" ) @@ -159,6 +160,7 @@ func (m *Maker) loadSegments() error { slog.Info("try to load the segments ... ") var last *Segment = nil var tStart = time.Now() + var sorting = false var iErr = IterateSegments(m.srcHandle, func(l string) { slog.Debug("loaded", "segment", l) @@ -171,9 +173,13 @@ func (m *Maker) loadSegments() error { return fmt.Errorf("invalid ip segment(%s expected)", m.version.Name) } - // check the continuity of the data segment - if err := seg.AfterCheck(last); err != nil { - return err + // check the order of the data segment + // if err := seg.RightBehind(last); err != nil { + if err := seg.After(last); err != nil { + // return err + // @Note: If the continuity is disrupted, + // we will sort all these segments later. + sorting = true } m.segments = append(m.segments, seg) @@ -184,7 +190,27 @@ func (m *Maker) loadSegments() error { return fmt.Errorf("failed to load segments: %s", iErr) } - slog.Info("all segments loaded", "length", len(m.segments), "elapsed", time.Since(tStart)) + // check and do the sorting + if sorting { + slog.Info("try to sort all the segments based on its start ip ...") + sort.Slice(m.segments, func(i, j int) bool { + return IPCompare(m.segments[i].StartIP, m.segments[j].StartIP) < 0 + }) + + slog.Info("try to check if there is overlap in the segments ...") + last = nil + for _, seg := range m.segments { + // check the order of the data segment + if err := seg.After(last); err != nil { + return fmt.Errorf("overlap checking: %w", err) + } + + // reset the last + last = seg + } + } + + slog.Info("all segments loaded", "length", len(m.segments), "sorting", sorting, "elapsed", time.Since(tStart)) return nil } diff --git a/maker/golang/xdb/searcher.go b/maker/golang/xdb/searcher.go index f34aa10..c0ccd0c 100644 --- a/maker/golang/xdb/searcher.go +++ b/maker/golang/xdb/searcher.go @@ -125,6 +125,10 @@ func (s *Searcher) Search(ip []byte) (string, int, error) { } //log.Printf("vIndex=%s", vIndex) + if sPtr == 0 || ePtr == 0 { + return "", ioCount, nil + } + // binary search the segment index to get the region var segIndexSize = uint32(s.version.SegmentIndexSize) var dataLen, dataPtr = 0, uint32(0) diff --git a/maker/golang/xdb/segment.go b/maker/golang/xdb/segment.go index cabc4b7..6a5a995 100644 --- a/maker/golang/xdb/segment.go +++ b/maker/golang/xdb/segment.go @@ -42,8 +42,9 @@ func SegmentFrom(seg string) (*Segment, error) { }, nil } -// AfterCheck check the current segment is the one just after the specified one -func (s *Segment) AfterCheck(last *Segment) error { +// RightBehind check the current segment is just right behind the specified one +// which mean last.EndIP + 1 = s.startIP +func (s *Segment) RightBehind(last *Segment) error { if last != nil { if IPCompare(IPAddOne(last.EndIP), s.StartIP) != 0 { return fmt.Errorf( @@ -56,6 +57,21 @@ func (s *Segment) AfterCheck(last *Segment) error { return nil } +// After check the current segment is after the specified one +// which means last.EndIP < s.startIP +func (s *Segment) After(last *Segment) error { + if last != nil { + if IPCompare(last.EndIP, s.StartIP) >= 0 { + return fmt.Errorf( + "disorder data segment: last.eip(%s) >= seg.sip(%s, %s)", + IP2String(last.EndIP), IP2String(s.StartIP), s.Region, + ) + } + } + + return nil +} + // Split the segment based on the pre-two bytes func (s *Segment) Split() []*Segment { // 1, split the segment with the first byte diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index 651a577..2add0df 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -169,7 +169,7 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region last = seg continue } else if last.Region == seg.Region { - if err = seg.AfterCheck(last); err == nil { + if err = seg.RightBehind(last); err == nil { last.EndIP = seg.EndIP continue }