xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add autoMege flag and return total/merged count for segments iterate

This commit is contained in:
lionsoul2014 2026-04-02 11:24:56 +08:00
parent edb29aea3b
commit e76f995ceb
6 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
maker/golang/cmd/bench.go
View File

@ -90,7 +90,7 @@ func Bench() {
var count, errCount, tStart = 0, 0, time.Now()
slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile)
var iErr = xdb.IterateSegments(handle, nil, nil, func(seg *xdb.Segment) error {
_, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error {
var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
slog.Debug("try to bench", "segment", l)
// mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)

4
maker/golang/xdb/editor.go
View File

@ -60,7 +60,7 @@ func (e *Editor) loadSegments() error {
var segments []*Segment
var sorting = false
var iErr = IterateSegments(e.srcHandle, func(l string) {
_, _, iErr := IterateSegments(e.srcHandle, true, func(l string) {
// do nothing here
}, nil, func(seg *Segment) error {
// version check
@ -302,7 +302,7 @@ func (e *Editor) PutFile(src string) (int, int, error) {
}
var oldRows, newRows = 0, 0
iErr := IterateSegments(handle, func(l string) {
_, _, iErr := IterateSegments(handle, true, func(l string) {
// do nothing here
}, nil, func(seg *Segment) error {
o, n, err := e.PutSegment(seg)

4
maker/golang/xdb/maker.go
View File

@ -168,7 +168,7 @@ func (m *Maker) loadSegments() error {
var tStart = time.Now()
var sorting = false
var iErr = IterateSegments(m.srcHandle, func(l string) {
_, mergeCount, iErr := IterateSegments(m.srcHandle, true, func(l string) {
slog.Debug("loaded", "segment", l)
}, func(region string) (string, error) {
// apply the field filter
@ -217,7 +217,7 @@ func (m *Maker) loadSegments() error {
}
}
slog.Info("all segments loaded", "length", len(m.segments), "sorting", sorting, "elapsed", time.Since(tStart))
slog.Info("all segments loaded", "length", len(m.segments), "merged", mergeCount, "sorting", sorting, "elapsed", time.Since(tStart))
return nil
}

4
maker/golang/xdb/processor.go
View File

@ -62,7 +62,7 @@ func (p *Processor) loadSegments() error {
slog.Info("try to load the segments ... ")
var tStart = time.Now()
var iErr = IterateSegments(p.srcHandle, func(l string) {
_, mergeCount, iErr := IterateSegments(p.srcHandle, true, func(l string) {
slog.Debug("loaded", "segment", l)
}, func(region string) (string, error) {
if p.clearBasedIndex > -1 {
@ -108,7 +108,7 @@ func (p *Processor) loadSegments() error {
return fmt.Errorf("failed to load segments: %s", iErr)
}
slog.Info("all segments loaded", "length", len(p.segments), "elapsed", time.Since(tStart))
slog.Info("all segments loaded", "length", len(p.segments), "merged", mergeCount, "elapsed", time.Since(tStart))
return nil
}

25
maker/golang/xdb/util.go
View File

@ -140,8 +140,9 @@ func IPMiddle(sip, eip []byte) ([]byte, error) {
return IPHalf(buf), nil
}
func IterateSegments(handle *os.File, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) error {
func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) {
var last *Segment = nil
var totalCount, mergeCount = 0, 0
var scanner = bufio.NewScanner(handle)
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
@ -154,31 +155,32 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
continue
}
totalCount++
if before != nil {
before(l)
}
var ps = strings.SplitN(l, "|", 3)
if len(ps) != 3 {
return fmt.Errorf("invalid ip segment line `%s`", l)
return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l)
}
sip, err := ParseIP(ps[0])
if err != nil {
return fmt.Errorf("check start ip `%s`: %s", ps[0], err)
return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err)
}
eip, err := ParseIP(ps[1])
if err != nil {
return fmt.Errorf("check end ip `%s`: %s", ps[1], err)
return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err)
}
if len(sip) != len(eip) {
return fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l)
return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l)
}
if IPCompare(sip, eip) > 0 {
return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
}
// Allow empty region info since 2024/09/24
@ -191,7 +193,7 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
if filter != nil {
region, err = filter(ps[2])
if err != nil {
return fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
}
}
@ -207,15 +209,16 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
if last == nil {
last = seg
continue
} else if last.Region == seg.Region {
} else if autoMerge && last.Region == seg.Region {
if err = seg.RightBehind(last); err == nil {
mergeCount++
last.EndIP = seg.EndIP
continue
}
}
if err = done(last); err != nil {
return err
return totalCount, mergeCount, err
}
// reset the last
@ -224,10 +227,10 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
// process the last segment
if last != nil {
return done(last)
return totalCount, mergeCount, done(last)
}
return nil
return totalCount, mergeCount, nil
}
func CheckSegments(segList []*Segment) error {

2
maker/golang/xdb/util_test.go
View File

@ -258,7 +258,7 @@ func TestIterateSegments(t *testing.T) {
t.Fatalf("failed to open tests file: %s", err)
}
_ = IterateSegments(handle, func(l string) {
_, _, _ = IterateSegments(handle, true, func(l string) {
// fmt.Printf("load segment: `%s`\n", l)
}, nil, func(seg *Segment) error {
fmt.Printf("get segment: `%s`\n", seg)