diff --git a/maker/golang/cmd/bench.go b/maker/golang/cmd/bench.go index 1c17350..7f1fa51 100644 --- a/maker/golang/cmd/bench.go +++ b/maker/golang/cmd/bench.go @@ -90,7 +90,7 @@ func Bench() { var count, errCount, tStart = 0, 0, time.Now() slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile) - var iErr = xdb.IterateSegments(handle, nil, nil, func(seg *xdb.Segment) error { + _, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error { var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) slog.Debug("try to bench", "segment", l) // mip := xdb.IPMiddle(seg.StartIP, seg.EndIP) diff --git a/maker/golang/xdb/editor.go b/maker/golang/xdb/editor.go index 3f4ad13..2c4f259 100644 --- a/maker/golang/xdb/editor.go +++ b/maker/golang/xdb/editor.go @@ -60,7 +60,7 @@ func (e *Editor) loadSegments() error { var segments []*Segment var sorting = false - var iErr = IterateSegments(e.srcHandle, func(l string) { + _, _, iErr := IterateSegments(e.srcHandle, true, func(l string) { // do nothing here }, nil, func(seg *Segment) error { // version check @@ -302,7 +302,7 @@ func (e *Editor) PutFile(src string) (int, int, error) { } var oldRows, newRows = 0, 0 - iErr := IterateSegments(handle, func(l string) { + _, _, iErr := IterateSegments(handle, true, func(l string) { // do nothing here }, nil, func(seg *Segment) error { o, n, err := e.PutSegment(seg) diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index be48407..bd4f56c 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -168,7 +168,7 @@ func (m *Maker) loadSegments() error { var tStart = time.Now() var sorting = false - var iErr = IterateSegments(m.srcHandle, func(l string) { + _, mergeCount, iErr := IterateSegments(m.srcHandle, true, func(l string) { slog.Debug("loaded", "segment", l) }, func(region string) (string, error) { // apply the field filter @@ -217,7 +217,7 @@ func (m *Maker) loadSegments() error { } } - slog.Info("all segments loaded", "length", len(m.segments), "sorting", sorting, "elapsed", time.Since(tStart)) + slog.Info("all segments loaded", "length", len(m.segments), "merged", mergeCount, "sorting", sorting, "elapsed", time.Since(tStart)) return nil } diff --git a/maker/golang/xdb/processor.go b/maker/golang/xdb/processor.go index 43cfc09..e791d0f 100644 --- a/maker/golang/xdb/processor.go +++ b/maker/golang/xdb/processor.go @@ -62,7 +62,7 @@ func (p *Processor) loadSegments() error { slog.Info("try to load the segments ... ") var tStart = time.Now() - var iErr = IterateSegments(p.srcHandle, func(l string) { + _, mergeCount, iErr := IterateSegments(p.srcHandle, true, func(l string) { slog.Debug("loaded", "segment", l) }, func(region string) (string, error) { if p.clearBasedIndex > -1 { @@ -108,7 +108,7 @@ func (p *Processor) loadSegments() error { return fmt.Errorf("failed to load segments: %s", iErr) } - slog.Info("all segments loaded", "length", len(p.segments), "elapsed", time.Since(tStart)) + slog.Info("all segments loaded", "length", len(p.segments), "merged", mergeCount, "elapsed", time.Since(tStart)) return nil } diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index 2c5f907..1235ebd 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -140,8 +140,9 @@ func IPMiddle(sip, eip []byte) ([]byte, error) { return IPHalf(buf), nil } -func IterateSegments(handle *os.File, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) error { +func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) { var last *Segment = nil + var totalCount, mergeCount = 0, 0 var scanner = bufio.NewScanner(handle) scanner.Split(bufio.ScanLines) for scanner.Scan() { @@ -154,31 +155,32 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region continue } + totalCount++ if before != nil { before(l) } var ps = strings.SplitN(l, "|", 3) if len(ps) != 3 { - return fmt.Errorf("invalid ip segment line `%s`", l) + return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l) } sip, err := ParseIP(ps[0]) if err != nil { - return fmt.Errorf("check start ip `%s`: %s", ps[0], err) + return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err) } eip, err := ParseIP(ps[1]) if err != nil { - return fmt.Errorf("check end ip `%s`: %s", ps[1], err) + return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err) } if len(sip) != len(eip) { - return fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l) + return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l) } if IPCompare(sip, eip) > 0 { - return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) + return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) } // Allow empty region info since 2024/09/24 @@ -191,7 +193,7 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region if filter != nil { region, err = filter(ps[2]) if err != nil { - return fmt.Errorf("failed to filter region `%s`: %s", ps[2], err) + return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err) } } @@ -207,15 +209,16 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region if last == nil { last = seg continue - } else if last.Region == seg.Region { + } else if autoMerge && last.Region == seg.Region { if err = seg.RightBehind(last); err == nil { + mergeCount++ last.EndIP = seg.EndIP continue } } if err = done(last); err != nil { - return err + return totalCount, mergeCount, err } // reset the last @@ -224,10 +227,10 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region // process the last segment if last != nil { - return done(last) + return totalCount, mergeCount, done(last) } - return nil + return totalCount, mergeCount, nil } func CheckSegments(segList []*Segment) error { diff --git a/maker/golang/xdb/util_test.go b/maker/golang/xdb/util_test.go index 249a8b6..f428974 100644 --- a/maker/golang/xdb/util_test.go +++ b/maker/golang/xdb/util_test.go @@ -258,7 +258,7 @@ func TestIterateSegments(t *testing.T) { t.Fatalf("failed to open tests file: %s", err) } - _ = IterateSegments(handle, func(l string) { + _, _, _ = IterateSegments(handle, true, func(l string) { // fmt.Printf("load segment: `%s`\n", l) }, nil, func(seg *Segment) error { fmt.Printf("get segment: `%s`\n", seg)