diff --git a/maker/golang/main.go b/maker/golang/main.go index f8549cd..c4f2651 100644 --- a/maker/golang/main.go +++ b/maker/golang/main.go @@ -246,12 +246,11 @@ func testBench() { } var count, errCount, tStart = 0, 0, time.Now() - var iErr = xdb.IterateSegments(handle, nil, func(sip uint32, eip uint32, region *string) error { - var s = *region - var l = fmt.Sprintf("%d|%d|%s", sip, eip, s) + var iErr = xdb.IterateSegments(handle, nil, func(seg *xdb.Segment) error { + var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) fmt.Printf("try to bench segment: `%s`\n", l) - mip := xdb.MidIP(sip, eip) - for _, ip := range []uint32{sip, xdb.MidIP(sip, mip), mip, xdb.MidIP(mip, eip), eip} { + mip := xdb.MidIP(seg.StartIP, seg.EndIP) + for _, ip := range []uint32{seg.StartIP, xdb.MidIP(seg.EndIP, mip), mip, xdb.MidIP(mip, seg.EndIP), seg.EndIP} { fmt.Printf("|-try to bench ip '%s' ... ", xdb.Long2IP(ip)) r, _, err := searcher.Search(ip) if err != nil { @@ -260,9 +259,9 @@ func testBench() { // check the region info count++ - if r != s { + if r != seg.Region { errCount++ - fmt.Printf(" --[Failed] (%s != %s)\n", r, s) + fmt.Printf(" --[Failed] (%s != %s)\n", r, seg.Region) if ignoreError == false { return fmt.Errorf("") } diff --git a/maker/golang/segments.tests b/maker/golang/segments.tests new file mode 100644 index 0000000..0d46f0f --- /dev/null +++ b/maker/golang/segments.tests @@ -0,0 +1,5 @@ +192.168.2.1|192.168.2.20|办公室A内网IP +192.168.2.21|192.168.2.30|办公室A内网IP +192.168.2.31|192.168.2.60|办公室B内网IP +192.168.2.61|192.168.2.91|办公室B内网IP +223.255.236.0|223.255.239.255|中国|0|上海|上海市|电信 \ No newline at end of file diff --git a/maker/golang/xdb/editor.go b/maker/golang/xdb/editor.go index ea9417e..733b4f4 100644 --- a/maker/golang/xdb/editor.go +++ b/maker/golang/xdb/editor.go @@ -76,19 +76,10 @@ func (e *Editor) loadSegments() error { var last *Segment = nil var tStart = time.Now() - var err = IterateSegments(e.srcHandle, nil, func(sip uint32, eip uint32, region *string) error { - var str = *region - var seg = &Segment{ - StartIP: sip, - EndIP: eip, - Region: str, - } - + var err = IterateSegments(e.srcHandle, nil, func(seg *Segment) error { // check the continuity of the data segment - if last != nil { - if last.EndIP+1 != seg.StartIP { - return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, str) - } + if err := seg.AfterCheck(last); err != nil { + return err } last = seg diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index 1ddd38e..b2d1cfa 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -138,29 +138,20 @@ func (m *Maker) loadSegments() error { var last *Segment = nil var tStart = time.Now() - var err = IterateSegments(m.srcHandle, func(l string) { + var iErr = IterateSegments(m.srcHandle, func(l string) { log.Printf("load segment: `%s`", l) - }, func(sip uint32, eip uint32, region *string) error { - var str = *region - var seg = &Segment{ - StartIP: sip, - EndIP: eip, - Region: str, - } - + }, func(seg *Segment) error { // check the continuity of the data segment - if last != nil { - if last.EndIP+1 != seg.StartIP { - return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, str) - } + if err := seg.AfterCheck(last); err != nil { + return err } m.segments = append(m.segments, seg) last = seg return nil }) - if err != nil { - return fmt.Errorf("failed to load segments: %s", err) + if iErr != nil { + return fmt.Errorf("failed to load segments: %s", iErr) } log.Printf("all segments loaded, length: %d, elapsed: %s", len(m.segments), time.Since(tStart)) diff --git a/maker/golang/xdb/segment.go b/maker/golang/xdb/segment.go index cdc48ac..0ad95f4 100644 --- a/maker/golang/xdb/segment.go +++ b/maker/golang/xdb/segment.go @@ -42,6 +42,20 @@ func SegmentFrom(seg string) (*Segment, error) { }, nil } +// AfterCheck check the current segment is the one just after the specified one +func (s *Segment) AfterCheck(last *Segment) error { + if last != nil { + if last.EndIP+1 != s.StartIP { + return fmt.Errorf( + "discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", + last.EndIP+1, s.StartIP, s.Region, + ) + } + } + + return nil +} + // Split the segment based on the pre-two bytes func (s *Segment) Split() []*Segment { // 1, split the segment with the first byte diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index 7bc17e6..870c232 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -47,7 +47,8 @@ func MidIP(sip uint32, eip uint32) uint32 { return uint32((uint64(sip) + uint64(eip)) >> 1) } -func IterateSegments(handle *os.File, before func(l string), cb func(sip uint32, eip uint32, region *string) error) error { +func IterateSegments(handle *os.File, before func(l string), cb func(seg *Segment) error) error { + var last *Segment = nil var scanner = bufio.NewScanner(handle) scanner.Split(bufio.ScanLines) for scanner.Scan() { @@ -79,9 +80,36 @@ func IterateSegments(handle *os.File, before func(l string), cb func(sip uint32, return fmt.Errorf("empty region info in segment line `%s`", l) } - if err = cb(sip, eip, &ps[2]); err != nil { + var seg = &Segment{ + StartIP: sip, + EndIP: eip, + Region: ps[2], + } + + // check and automatic merging the Consecutive Segments which means: + // 1, region info is the same + // 2, last.eip+1 = cur.sip + if last == nil { + last = seg + continue + } else if last.Region == seg.Region { + if err = seg.AfterCheck(last); err == nil { + last.EndIP = seg.EndIP + continue + } + } + + if err = cb(last); err != nil { return err } + + // reset the last + last = seg + } + + // process the last segment + if last != nil { + return cb(last) } return nil diff --git a/maker/golang/xdb/util_test.go b/maker/golang/xdb/util_test.go index 7e918c1..ce8328e 100644 --- a/maker/golang/xdb/util_test.go +++ b/maker/golang/xdb/util_test.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "fmt" "net" + "os" "testing" ) @@ -59,3 +60,17 @@ func TestSplitSegment(t *testing.T) { fmt.Printf("idx: %3d, seg: %s\n", i, s.String()) } } + +func TestIterateSegments(t *testing.T) { + handle, err := os.OpenFile("../segments.tests", os.O_RDONLY, 0600) + if err != nil { + t.Fatalf("failed to open tests file: %s", err) + } + + err = IterateSegments(handle, func(l string) { + // fmt.Printf("load segment: `%s`\n", l) + }, func(seg *Segment) error { + fmt.Printf("get segment: `%s`\n", seg) + return nil + }) +}