From 787bd7ae070e9759b9b26ee35d2939fffcec4d08 Mon Sep 17 00:00:00 2001 From: lionsoul2014 Date: Wed, 29 Apr 2026 15:52:40 +0800 Subject: [PATCH] use the Region cache --- maker/golang/cmd/bench.go | 6 ++- maker/golang/xdb/editor.go | 13 +++--- maker/golang/xdb/maker.go | 4 +- maker/golang/xdb/processor.go | 6 ++- maker/golang/xdb/region.go | 86 +++++++++++++++++++++-------------- maker/golang/xdb/segment.go | 4 +- maker/golang/xdb/util.go | 6 +-- maker/golang/xdb/util_test.go | 8 ++-- 8 files changed, 81 insertions(+), 52 deletions(-) diff --git a/maker/golang/cmd/bench.go b/maker/golang/cmd/bench.go index 8500a9f..a008a72 100644 --- a/maker/golang/cmd/bench.go +++ b/maker/golang/cmd/bench.go @@ -90,12 +90,14 @@ func Bench(sCmd string) { fmt.Printf("failed to open source text file: %s\n", err) return } - defer handle.Close() + var rgCache = xdb.NewRegionCache() var count, errCount, tStart = 0, 0, time.Now() slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile) - _, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error { + _, _, iErr := xdb.IterateSegments(handle, false, func(l string) { + // do thing here + }, nil, rgCache.Region, func(seg *xdb.Segment) error { var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) slog.Debug("try to bench", "segment", l) // mip := xdb.IPMiddle(seg.StartIP, seg.EndIP) diff --git a/maker/golang/xdb/editor.go b/maker/golang/xdb/editor.go index 4ffbbb2..e1bc7d7 100644 --- a/maker/golang/xdb/editor.go +++ b/maker/golang/xdb/editor.go @@ -24,6 +24,9 @@ type Editor struct { // segments list segments *list.List + + // region cache + rgCache *RegionCache } func NewEditor(version *Version, srcFile string) (*Editor, error) { @@ -44,6 +47,7 @@ func NewEditor(version *Version, srcFile string) (*Editor, error) { srcHandle: srcHandle, toSave: false, segments: list.New(), + rgCache: NewRegionCache(), } // load the segments @@ -62,7 +66,7 @@ func (e *Editor) loadSegments() error { _, _, iErr := IterateSegments(e.srcHandle, true, func(l string) { // do nothing here - }, nil, func(seg *Segment) error { + }, nil, e.rgCache.Region, func(seg *Segment) error { // version check if len(seg.StartIP) != e.verison.Bytes { return fmt.Errorf("invalid ip segment(%s expected)", e.verison.Name) @@ -100,9 +104,6 @@ func (e *Editor) loadSegments() error { // to Keep the entire data continuous. last = nil for _, seg := range segments { - if err := seg.After(last); err != nil { - } - if last == nil { if IPCompare(seg.StartIP, e.verison.Min) > 0 { e.segments.PushBack(&Segment{ @@ -182,7 +183,7 @@ func (e *Editor) Slice(offset int, size int) []*Segment { } func (e *Editor) Put(ip string, cb func(newSeg *Segment, oldList []*Segment) []*Segment) (int, int, error) { - seg, err := SegmentFrom(ip) + seg, err := SegmentFrom(ip, e.rgCache.Region) if err != nil { return 0, 0, err } @@ -350,7 +351,7 @@ func (e *Editor) PutFile(src string, cb func(newSeg *Segment, oldList []*Segment var oldRows, newRows = 0, 0 _, _, iErr := IterateSegments(handle, true, func(l string) { // do nothing here - }, nil, func(seg *Segment) error { + }, nil, NewRegion, func(seg *Segment) error { o, n, err := e.PutSegment(seg, cb) if err == nil { oldRows += o diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index 3aee6a3..a9aea1c 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -84,6 +84,7 @@ type Maker struct { indexPolicy IndexPolicy segments []*Segment regionPool map[string]uint32 + regionCache *RegionCache vectorIndex []byte } @@ -118,6 +119,7 @@ func NewMaker(version *Version, policy IndexPolicy, srcFile string, dstFile stri indexPolicy: policy, segments: []*Segment{}, regionPool: map[string]uint32{}, + regionCache: NewRegionCache(), vectorIndex: make([]byte, VectorIndexLength), }, nil } @@ -173,7 +175,7 @@ func (m *Maker) loadSegments() error { }, func(region string) (string, error) { // apply the field filter return RegionFiltering(region, m.fields) - }, func(seg *Segment) error { + }, m.regionCache.Region, func(seg *Segment) error { // ip version check if len(seg.StartIP) != m.version.Bytes { return fmt.Errorf("invalid ip segment(%s expected)", m.version.Name) diff --git a/maker/golang/xdb/processor.go b/maker/golang/xdb/processor.go index e791d0f..af2f20e 100644 --- a/maker/golang/xdb/processor.go +++ b/maker/golang/xdb/processor.go @@ -26,6 +26,9 @@ type Processor struct { fields []int segments []*Segment + + // region cache + rgCache *RegionCache } func NewProcessor(srcFile string, dstFile string, fields []int, @@ -55,6 +58,7 @@ func NewProcessor(srcFile string, dstFile string, fields []int, fields: fields, segments: []*Segment{}, + rgCache: NewRegionCache(), }, nil } @@ -94,7 +98,7 @@ func (p *Processor) loadSegments() error { } return RegionFiltering(region, p.fields) - }, func(seg *Segment) error { + }, p.rgCache.Region, func(seg *Segment) error { // check the continuity of the data segment // if err := seg.AfterCheck(last); err != nil { // return err diff --git a/maker/golang/xdb/region.go b/maker/golang/xdb/region.go index e011ea4..88e6579 100644 --- a/maker/golang/xdb/region.go +++ b/maker/golang/xdb/region.go @@ -10,39 +10,14 @@ import ( // 1, content cache. // 2, util functions -// global cache map -var rcLock sync.Mutex -var regionCache = map[string]*Region{} +// --- region type Region struct { Str string // region string fields []string // region fields } -var EmptyRegion = CacheRegion("") - -// Create or get the region from the global cache. -// And it is a thread-safe implementation. -func CacheRegion(str string) *Region { - // check the cache and return it directly - // if there is a cache available - rcLock.Lock() - defer rcLock.Unlock() - - region, ok := regionCache[str] - if ok { - return region - } - - // cache the new region - region = &Region{ - Str: str, - fields: nil, - } - - regionCache[str] = region - return region -} +var EmptyRegion = NewRegion("") // Create a new region without checking cache info func NewRegion(str string) *Region { @@ -87,12 +62,10 @@ func (r *Region) Filtering(fields []int) (*Region, error) { sb = append(sb, fs[idx]) } - new := CacheRegion(strings.Join(sb, "|")) - if new.fields == nil { - new.fields = sb - } - - return new, nil + return &Region{ + Str: strings.Join(sb, "|"), + fields: sb, + }, nil } // Equal check ptr (share the same region cache) or the Str is the same. @@ -107,3 +80,50 @@ func (r *Region) IsEmpty() bool { func (r *Region) String() string { return r.Str } + +// --- +// --- region cache + +type RegionCache struct { + lock sync.Mutex + cache map[string]*Region +} + +func NewRegionCache() *RegionCache { + return &RegionCache{ + lock: sync.Mutex{}, + cache: make(map[string]*Region), + } +} + +func (rc *RegionCache) Region(str string) *Region { + rc.lock.Lock() + defer rc.lock.Unlock() + + region, ok := rc.cache[str] + if ok { + return region + } + + // cache the new region + region = &Region{ + Str: str, + fields: nil, + } + + rc.cache[str] = region + return region +} + +func (rc *RegionCache) Swap(r *Region) *Region { + return rc.Region(r.Str) +} + +func (rc *RegionCache) Clean() { + rc.lock.Lock() + defer rc.lock.Unlock() + + for k := range rc.cache { + delete(rc.cache, k) + } +} diff --git a/maker/golang/xdb/segment.go b/maker/golang/xdb/segment.go index 179cc88..831bdf9 100644 --- a/maker/golang/xdb/segment.go +++ b/maker/golang/xdb/segment.go @@ -15,7 +15,7 @@ type Segment struct { Region *Region } -func SegmentFrom(seg string) (*Segment, error) { +func SegmentFrom(seg string, cRegion func(string) *Region) (*Segment, error) { var ps = strings.SplitN(strings.TrimSpace(seg), "|", 3) if len(ps) != 3 { return nil, fmt.Errorf("invalid ip segment `%s`", seg) @@ -38,7 +38,7 @@ func SegmentFrom(seg string) (*Segment, error) { return &Segment{ StartIP: sip, EndIP: eip, - Region: CacheRegion(ps[2]), + Region: cRegion(ps[2]), }, nil } diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index 6e8fb58..4d394ea 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -140,7 +140,7 @@ func IPMiddle(sip, eip []byte) ([]byte, error) { return IPHalf(buf), nil } -func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) { +func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), cRegion func(string) *Region, done func(seg *Segment) error) (int, int, error) { var last *Segment = nil var totalCount, mergeCount = 0, 0 var scanner = bufio.NewScanner(handle) @@ -193,14 +193,14 @@ func IterateSegments(handle *os.File, autoMerge bool, before func(l string), fil if filter != nil { region, err = filter(ps[2]) if err != nil { - return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err) + return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", region, err) } } var seg = &Segment{ StartIP: sip, EndIP: eip, - Region: CacheRegion(region), + Region: cRegion(region), } // check and automatic merging the Consecutive Segments, which means: diff --git a/maker/golang/xdb/util_test.go b/maker/golang/xdb/util_test.go index e7a151a..7f5b8cb 100644 --- a/maker/golang/xdb/util_test.go +++ b/maker/golang/xdb/util_test.go @@ -201,7 +201,7 @@ func TestSplitSegmentV4(t *testing.T) { // var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP" // var str = "29.0.0.0|29.34.191.255|美国|0|0|0|0" var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0" - seg, err := SegmentFrom(str) + seg, err := SegmentFrom(str, NewRegion) if err != nil { t.Fatalf("failed to parser segment '%s': %s", str, err) } @@ -220,7 +220,7 @@ func TestSplitSegmentV4(t *testing.T) { func TestRegionFiltering(t *testing.T) { var line = "2001:1203:31:8000::|2001:1203:31:bfff:ffff:ffff:ffff:ffff||墨西哥|瓜纳华托州||||专线用户|" - seg, err := SegmentFrom(line) + seg, err := SegmentFrom(line, NewRegion) if err != nil { t.Fatalf("failed to parse segment '%s': %s", line, err) } @@ -235,7 +235,7 @@ func TestRegionFiltering(t *testing.T) { func TestSplitSegmentV6(t *testing.T) { var str = "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff||瑞士|弗里堡州||||专线用户|IANA" - seg, err := SegmentFrom(str) + seg, err := SegmentFrom(str, NewRegion) if err != nil { t.Fatalf("failed to parser segment '%s': %s", str, err) } @@ -260,7 +260,7 @@ func TestIterateSegments(t *testing.T) { _, _, _ = IterateSegments(handle, true, func(l string) { // fmt.Printf("load segment: `%s`\n", l) - }, nil, func(seg *Segment) error { + }, nil, NewRegion, func(seg *Segment) error { fmt.Printf("get segment: `%s`\n", seg) return nil })