use the Region cache

maker/golang/cmd/bench.go

@@ -90,12 +90,14 @@ func Bench(sCmd string) {
 		fmt.Printf("failed to open source text file: %s\n", err)
 		return
 	}
-
 	defer handle.Close()
 
+	var rgCache = xdb.NewRegionCache()
 	var count, errCount, tStart = 0, 0, time.Now()
 	slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile)
-	_, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error {
+	_, _, iErr := xdb.IterateSegments(handle, false, func(l string) {
+		// do thing here
+	}, nil, rgCache.Region, func(seg *xdb.Segment) error {
 		var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
 		slog.Debug("try to bench", "segment", l)
 		// mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)

maker/golang/xdb/editor.go

@@ -24,6 +24,9 @@ type Editor struct {
 
 	// segments list
 	segments *list.List
+
+	// region cache
+	rgCache *RegionCache
 }
 
 func NewEditor(version *Version, srcFile string) (*Editor, error) {
@@ -44,6 +47,7 @@ func NewEditor(version *Version, srcFile string) (*Editor, error) {
 		srcHandle: srcHandle,
 		toSave:    false,
 		segments:  list.New(),
+		rgCache:   NewRegionCache(),
 	}
 
 	// load the segments
@@ -62,7 +66,7 @@ func (e *Editor) loadSegments() error {
 
 	_, _, iErr := IterateSegments(e.srcHandle, true, func(l string) {
 		// do nothing here
-	}, nil, func(seg *Segment) error {
+	}, nil, e.rgCache.Region, func(seg *Segment) error {
 		// version check
 		if len(seg.StartIP) != e.verison.Bytes {
 			return fmt.Errorf("invalid ip segment(%s expected)", e.verison.Name)
@@ -100,9 +104,6 @@ func (e *Editor) loadSegments() error {
 	// to Keep the entire data continuous.
 	last = nil
 	for _, seg := range segments {
-		if err := seg.After(last); err != nil {
-		}
-
 		if last == nil {
 			if IPCompare(seg.StartIP, e.verison.Min) > 0 {
 				e.segments.PushBack(&Segment{
@@ -182,7 +183,7 @@ func (e *Editor) Slice(offset int, size int) []*Segment {
 }
 
 func (e *Editor) Put(ip string, cb func(newSeg *Segment, oldList []*Segment) []*Segment) (int, int, error) {
-	seg, err := SegmentFrom(ip)
+	seg, err := SegmentFrom(ip, e.rgCache.Region)
 	if err != nil {
 		return 0, 0, err
 	}
@@ -350,7 +351,7 @@ func (e *Editor) PutFile(src string, cb func(newSeg *Segment, oldList []*Segment
 	var oldRows, newRows = 0, 0
 	_, _, iErr := IterateSegments(handle, true, func(l string) {
 		// do nothing here
-	}, nil, func(seg *Segment) error {
+	}, nil, NewRegion, func(seg *Segment) error {
 		o, n, err := e.PutSegment(seg, cb)
 		if err == nil {
 			oldRows += o

maker/golang/xdb/maker.go

@@ -84,6 +84,7 @@ type Maker struct {
 	indexPolicy IndexPolicy
 	segments    []*Segment
 	regionPool  map[string]uint32
+	regionCache *RegionCache
 	vectorIndex []byte
 }
 
@@ -118,6 +119,7 @@ func NewMaker(version *Version, policy IndexPolicy, srcFile string, dstFile stri
 		indexPolicy: policy,
 		segments:    []*Segment{},
 		regionPool:  map[string]uint32{},
+		regionCache: NewRegionCache(),
 		vectorIndex: make([]byte, VectorIndexLength),
 	}, nil
 }
@@ -173,7 +175,7 @@ func (m *Maker) loadSegments() error {
 	}, func(region string) (string, error) {
 		// apply the field filter
 		return RegionFiltering(region, m.fields)
-	}, func(seg *Segment) error {
+	}, m.regionCache.Region, func(seg *Segment) error {
 		// ip version check
 		if len(seg.StartIP) != m.version.Bytes {
 			return fmt.Errorf("invalid ip segment(%s expected)", m.version.Name)

maker/golang/xdb/processor.go

@@ -26,6 +26,9 @@ type Processor struct {
 
 	fields   []int
 	segments []*Segment
+
+	// region cache
+	rgCache *RegionCache
 }
 
 func NewProcessor(srcFile string, dstFile string, fields []int,
@@ -55,6 +58,7 @@ func NewProcessor(srcFile string, dstFile string, fields []int,
 		fields: fields,
 
 		segments: []*Segment{},
+		rgCache:  NewRegionCache(),
 	}, nil
 }
 
@@ -94,7 +98,7 @@ func (p *Processor) loadSegments() error {
 		}
 
 		return RegionFiltering(region, p.fields)
-	}, func(seg *Segment) error {
+	}, p.rgCache.Region, func(seg *Segment) error {
 		// check the continuity of the data segment
 		// if err := seg.AfterCheck(last); err != nil {
 		// 	return err

maker/golang/xdb/region.go

@@ -10,39 +10,14 @@ import (
 // 1, content cache.
 // 2, util functions
 
-// global cache map
+// --- region
-var rcLock sync.Mutex
-var regionCache = map[string]*Region{}
 
 type Region struct {
 	Str    string   // region string
 	fields []string // region fields
 }
 
-var EmptyRegion = CacheRegion("")
+var EmptyRegion = NewRegion("")
-
-// Create or get the region from the global cache.
-// And it is a thread-safe implementation.
-func CacheRegion(str string) *Region {
-	// check the cache and return it directly
-	// if there is a cache available
-	rcLock.Lock()
-	defer rcLock.Unlock()
-
-	region, ok := regionCache[str]
-	if ok {
-		return region
-	}
-
-	// cache the new region
-	region = &Region{
-		Str:    str,
-		fields: nil,
-	}
-
-	regionCache[str] = region
-	return region
-}
 
 // Create a new region without checking cache info
 func NewRegion(str string) *Region {
@@ -87,12 +62,10 @@ func (r *Region) Filtering(fields []int) (*Region, error) {
 		sb = append(sb, fs[idx])
 	}
 
-	new := CacheRegion(strings.Join(sb, "|"))
+	return &Region{
-	if new.fields == nil {
+		Str:    strings.Join(sb, "|"),
-		new.fields = sb
+		fields: sb,
-	}
+	}, nil
-
-	return new, nil
 }
 
 // Equal check ptr (share the same region cache) or the Str is the same.
@@ -107,3 +80,50 @@ func (r *Region) IsEmpty() bool {
 func (r *Region) String() string {
 	return r.Str
 }
+
+// ---
+// --- region cache
+
+type RegionCache struct {
+	lock  sync.Mutex
+	cache map[string]*Region
+}
+
+func NewRegionCache() *RegionCache {
+	return &RegionCache{
+		lock:  sync.Mutex{},
+		cache: make(map[string]*Region),
+	}
+}
+
+func (rc *RegionCache) Region(str string) *Region {
+	rc.lock.Lock()
+	defer rc.lock.Unlock()
+
+	region, ok := rc.cache[str]
+	if ok {
+		return region
+	}
+
+	// cache the new region
+	region = &Region{
+		Str:    str,
+		fields: nil,
+	}
+
+	rc.cache[str] = region
+	return region
+}
+
+func (rc *RegionCache) Swap(r *Region) *Region {
+	return rc.Region(r.Str)
+}
+
+func (rc *RegionCache) Clean() {
+	rc.lock.Lock()
+	defer rc.lock.Unlock()
+
+	for k := range rc.cache {
+		delete(rc.cache, k)
+	}
+}

maker/golang/xdb/segment.go

@@ -15,7 +15,7 @@ type Segment struct {
 	Region  *Region
 }
 
-func SegmentFrom(seg string) (*Segment, error) {
+func SegmentFrom(seg string, cRegion func(string) *Region) (*Segment, error) {
 	var ps = strings.SplitN(strings.TrimSpace(seg), "|", 3)
 	if len(ps) != 3 {
 		return nil, fmt.Errorf("invalid ip segment `%s`", seg)
@@ -38,7 +38,7 @@ func SegmentFrom(seg string) (*Segment, error) {
 	return &Segment{
 		StartIP: sip,
 		EndIP:   eip,
-		Region:  CacheRegion(ps[2]),
+		Region:  cRegion(ps[2]),
 	}, nil
 }
 

maker/golang/xdb/util.go

@@ -140,7 +140,7 @@ func IPMiddle(sip, eip []byte) ([]byte, error) {
 	return IPHalf(buf), nil
 }
 
-func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) {
+func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), cRegion func(string) *Region, done func(seg *Segment) error) (int, int, error) {
 	var last *Segment = nil
 	var totalCount, mergeCount = 0, 0
 	var scanner = bufio.NewScanner(handle)
@@ -193,14 +193,14 @@ func IterateSegments(handle *os.File, autoMerge bool, before func(l string), fil
 		if filter != nil {
 			region, err = filter(ps[2])
 			if err != nil {
-				return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
+				return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", region, err)
 			}
 		}
 
 		var seg = &Segment{
 			StartIP: sip,
 			EndIP:   eip,
-			Region:  CacheRegion(region),
+			Region:  cRegion(region),
 		}
 
 		// check and automatic merging the Consecutive Segments, which means:

maker/golang/xdb/util_test.go

@@ -201,7 +201,7 @@ func TestSplitSegmentV4(t *testing.T) {
 	// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
 	// var str = "29.0.0.0|29.34.191.255|美国|0|0|0|0"
 	var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0"
-	seg, err := SegmentFrom(str)
+	seg, err := SegmentFrom(str, NewRegion)
 	if err != nil {
 		t.Fatalf("failed to parser segment '%s': %s", str, err)
 	}
@@ -220,7 +220,7 @@ func TestSplitSegmentV4(t *testing.T) {
 
 func TestRegionFiltering(t *testing.T) {
 	var line = "2001:1203:31:8000::|2001:1203:31:bfff:ffff:ffff:ffff:ffff||墨西哥|瓜纳华托州||||专线用户|"
-	seg, err := SegmentFrom(line)
+	seg, err := SegmentFrom(line, NewRegion)
 	if err != nil {
 		t.Fatalf("failed to parse segment '%s': %s", line, err)
 	}
@@ -235,7 +235,7 @@ func TestRegionFiltering(t *testing.T) {
 
 func TestSplitSegmentV6(t *testing.T) {
 	var str = "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff||瑞士|弗里堡州||||专线用户|IANA"
-	seg, err := SegmentFrom(str)
+	seg, err := SegmentFrom(str, NewRegion)
 	if err != nil {
 		t.Fatalf("failed to parser segment '%s': %s", str, err)
 	}
@@ -260,7 +260,7 @@ func TestIterateSegments(t *testing.T) {
 
 	_, _, _ = IterateSegments(handle, true, func(l string) {
 		// fmt.Printf("load segment: `%s`\n", l)
-	}, nil, func(seg *Segment) error {
+	}, nil, NewRegion, func(seg *Segment) error {
 		fmt.Printf("get segment: `%s`\n", seg)
 		return nil
 	})