xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add autoMege flag and return total/merged count for segments iterate

This commit is contained in:
lionsoul2014 2026-04-02 11:24:56 +08:00
parent edb29aea3b
commit e76f995ceb
6 changed files with 22 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
maker/golang/cmd/bench.go
View File

@ -90,7 +90,7 @@ func Bench() {
var count, errCount, tStart = 0, 0, time.Now() var count, errCount, tStart = 0, 0, time.Now()
slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile) slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile)
var iErr = xdb.IterateSegments(handle, nil, nil, func(seg *xdb.Segment) error { _, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error {
var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
slog.Debug("try to bench", "segment", l) slog.Debug("try to bench", "segment", l)
// mip := xdb.IPMiddle(seg.StartIP, seg.EndIP) // mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)

4
maker/golang/xdb/editor.go
View File

@ -60,7 +60,7 @@ func (e *Editor) loadSegments() error {
var segments []*Segment var segments []*Segment
var sorting = false var sorting = false
var iErr = IterateSegments(e.srcHandle, func(l string) { _, _, iErr := IterateSegments(e.srcHandle, true, func(l string) {
// do nothing here // do nothing here
}, nil, func(seg *Segment) error { }, nil, func(seg *Segment) error {
// version check // version check
@ -302,7 +302,7 @@ func (e *Editor) PutFile(src string) (int, int, error) {
} }
var oldRows, newRows = 0, 0 var oldRows, newRows = 0, 0
iErr := IterateSegments(handle, func(l string) { _, _, iErr := IterateSegments(handle, true, func(l string) {
// do nothing here // do nothing here
}, nil, func(seg *Segment) error { }, nil, func(seg *Segment) error {
o, n, err := e.PutSegment(seg) o, n, err := e.PutSegment(seg)

4
maker/golang/xdb/maker.go
View File

@ -168,7 +168,7 @@ func (m *Maker) loadSegments() error {
var tStart = time.Now() var tStart = time.Now()
var sorting = false var sorting = false
var iErr = IterateSegments(m.srcHandle, func(l string) { _, mergeCount, iErr := IterateSegments(m.srcHandle, true, func(l string) {
slog.Debug("loaded", "segment", l) slog.Debug("loaded", "segment", l)
}, func(region string) (string, error) { }, func(region string) (string, error) {
// apply the field filter // apply the field filter
@ -217,7 +217,7 @@ func (m *Maker) loadSegments() error {
} }
} }
slog.Info("all segments loaded", "length", len(m.segments), "sorting", sorting, "elapsed", time.Since(tStart)) slog.Info("all segments loaded", "length", len(m.segments), "merged", mergeCount, "sorting", sorting, "elapsed", time.Since(tStart))
return nil return nil
} }

4
maker/golang/xdb/processor.go
View File

@ -62,7 +62,7 @@ func (p *Processor) loadSegments() error {
slog.Info("try to load the segments ... ") slog.Info("try to load the segments ... ")
var tStart = time.Now() var tStart = time.Now()
var iErr = IterateSegments(p.srcHandle, func(l string) { _, mergeCount, iErr := IterateSegments(p.srcHandle, true, func(l string) {
slog.Debug("loaded", "segment", l) slog.Debug("loaded", "segment", l)
}, func(region string) (string, error) { }, func(region string) (string, error) {
if p.clearBasedIndex > -1 { if p.clearBasedIndex > -1 {
@ -108,7 +108,7 @@ func (p *Processor) loadSegments() error {
return fmt.Errorf("failed to load segments: %s", iErr) return fmt.Errorf("failed to load segments: %s", iErr)
} }
slog.Info("all segments loaded", "length", len(p.segments), "elapsed", time.Since(tStart)) slog.Info("all segments loaded", "length", len(p.segments), "merged", mergeCount, "elapsed", time.Since(tStart))
return nil return nil
} }

25
maker/golang/xdb/util.go
View File

@ -140,8 +140,9 @@ func IPMiddle(sip, eip []byte) ([]byte, error) {
return IPHalf(buf), nil return IPHalf(buf), nil
} }
func IterateSegments(handle *os.File, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) error { func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) {
var last *Segment = nil var last *Segment = nil
var totalCount, mergeCount = 0, 0
var scanner = bufio.NewScanner(handle) var scanner = bufio.NewScanner(handle)
scanner.Split(bufio.ScanLines) scanner.Split(bufio.ScanLines)
for scanner.Scan() { for scanner.Scan() {
@ -154,31 +155,32 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
continue continue
} }
totalCount++
if before != nil { if before != nil {
before(l) before(l)
} }
var ps = strings.SplitN(l, "|", 3) var ps = strings.SplitN(l, "|", 3)
if len(ps) != 3 { if len(ps) != 3 {
return fmt.Errorf("invalid ip segment line `%s`", l) return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l)
} }
sip, err := ParseIP(ps[0]) sip, err := ParseIP(ps[0])
if err != nil { if err != nil {
return fmt.Errorf("check start ip `%s`: %s", ps[0], err) return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err)
} }
eip, err := ParseIP(ps[1]) eip, err := ParseIP(ps[1])
if err != nil { if err != nil {
return fmt.Errorf("check end ip `%s`: %s", ps[1], err) return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err)
} }
if len(sip) != len(eip) { if len(sip) != len(eip) {
return fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l) return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l)
} }
if IPCompare(sip, eip) > 0 { if IPCompare(sip, eip) > 0 {
return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
} }
// Allow empty region info since 2024/09/24 // Allow empty region info since 2024/09/24
@ -191,7 +193,7 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
if filter != nil { if filter != nil {
region, err = filter(ps[2]) region, err = filter(ps[2])
if err != nil { if err != nil {
return fmt.Errorf("failed to filter region `%s`: %s", ps[2], err) return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
} }
} }
@ -207,15 +209,16 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
if last == nil { if last == nil {
last = seg last = seg
continue continue
} else if last.Region == seg.Region { } else if autoMerge && last.Region == seg.Region {
if err = seg.RightBehind(last); err == nil { if err = seg.RightBehind(last); err == nil {
mergeCount++
last.EndIP = seg.EndIP last.EndIP = seg.EndIP
continue continue
} }
} }
if err = done(last); err != nil { if err = done(last); err != nil {
return err return totalCount, mergeCount, err
} }
// reset the last // reset the last
@ -224,10 +227,10 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region
// process the last segment // process the last segment
if last != nil { if last != nil {
return done(last) return totalCount, mergeCount, done(last)
} }
return nil return totalCount, mergeCount, nil
} }
func CheckSegments(segList []*Segment) error { func CheckSegments(segList []*Segment) error {

2
maker/golang/xdb/util_test.go
View File

@ -258,7 +258,7 @@ func TestIterateSegments(t *testing.T) {
t.Fatalf("failed to open tests file: %s", err) t.Fatalf("failed to open tests file: %s", err)
} }
_ = IterateSegments(handle, func(l string) { _, _, _ = IterateSegments(handle, true, func(l string) {
// fmt.Printf("load segment: `%s`\n", l) // fmt.Printf("load segment: `%s`\n", l)
}, nil, func(seg *Segment) error { }, nil, func(seg *Segment) error {
fmt.Printf("get segment: `%s`\n", seg) fmt.Printf("get segment: `%s`\n", seg)