Merge pull request #478 from lionsoul2014/golang_maker_region_mgr

Golang maker region struct and cache impl

maker/golang/cmd/bench.go

@@ -90,12 +90,16 @@ func Bench(sCmd string) {
 		fmt.Printf("failed to open source text file: %s\n", err)
 		return
 	}
-
 	defer handle.Close()
 
+	var rgCache = xdb.NewRegionCache()
+	defer rgCache.Clean()
+
 	var count, errCount, tStart = 0, 0, time.Now()
 	slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile)
-	_, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error {
+	_, _, iErr := xdb.IterateSegments(handle, false, func(l string) {
+		// do thing here
+	}, nil, rgCache.Region, func(seg *xdb.Segment) error {
 		var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
 		slog.Debug("try to bench", "segment", l)
 		// mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)
@@ -109,7 +113,7 @@ func Bench(sCmd string) {
 
 			// check the region info
 			count++
-			if r != seg.Region {
+			if r != seg.Region.Str {
 				errCount++
 				slog.Error(" --[Failed] region not match", "src", r, "dst", seg.Region)
 				if !ignoreError {

maker/golang/xdb/editor.go

@@ -24,6 +24,9 @@ type Editor struct {
 
 	// segments list
 	segments *list.List
+
+	// region cache
+	rgCache *RegionCache
 }
 
 func NewEditor(version *Version, srcFile string) (*Editor, error) {
@@ -44,6 +47,7 @@ func NewEditor(version *Version, srcFile string) (*Editor, error) {
 		srcHandle: srcHandle,
 		toSave:    false,
 		segments:  list.New(),
+		rgCache:   NewRegionCache(),
 	}
 
 	// load the segments
@@ -54,6 +58,10 @@ func NewEditor(version *Version, srcFile string) (*Editor, error) {
 	return e, nil
 }
 
+func (e *Editor) Region(str string) *Region {
+	return e.rgCache.Region(str)
+}
+
 // Load all the segments from the source file
 func (e *Editor) loadSegments() error {
 	var last *Segment = nil
@@ -62,7 +70,7 @@ func (e *Editor) loadSegments() error {
 
 	_, _, iErr := IterateSegments(e.srcHandle, true, func(l string) {
 		// do nothing here
-	}, nil, func(seg *Segment) error {
+	}, nil, e.Region, func(seg *Segment) error {
 		// version check
 		if len(seg.StartIP) != e.verison.Bytes {
 			return fmt.Errorf("invalid ip segment(%s expected)", e.verison.Name)
@@ -100,15 +108,12 @@ func (e *Editor) loadSegments() error {
 	// to Keep the entire data continuous.
 	last = nil
 	for _, seg := range segments {
-		if err := seg.After(last); err != nil {
-		}
-
 		if last == nil {
 			if IPCompare(seg.StartIP, e.verison.Min) > 0 {
 				e.segments.PushBack(&Segment{
 					StartIP: e.verison.Min,
 					EndIP:   IPSubOne(seg.StartIP),
-					Region:  "",
+					Region:  EmptyRegion,
 				})
 			}
 		} else if err := seg.RightBehind(last); err == nil {
@@ -121,7 +126,7 @@ func (e *Editor) loadSegments() error {
 			e.segments.PushBack(&Segment{
 				StartIP: IPAddOne(last.EndIP),
 				EndIP:   IPSubOne(seg.StartIP),
-				Region:  "",
+				Region:  EmptyRegion,
 			})
 		}
 
@@ -138,7 +143,7 @@ func (e *Editor) loadSegments() error {
 			e.segments.PushBack(&Segment{
 				StartIP: IPAddOne(back.Value.(*Segment).EndIP),
 				EndIP:   e.verison.Max,
-				Region:  "",
+				Region:  EmptyRegion,
 			})
 		}
 	}
@@ -182,7 +187,7 @@ func (e *Editor) Slice(offset int, size int) []*Segment {
 }
 
 func (e *Editor) Put(ip string, cb func(newSeg *Segment, oldList []*Segment) []*Segment) (int, int, error) {
-	seg, err := SegmentFrom(ip)
+	seg, err := SegmentFrom(ip, e.rgCache.Region)
 	if err != nil {
 		return 0, 0, err
 	}
@@ -350,7 +355,7 @@ func (e *Editor) PutFile(src string, cb func(newSeg *Segment, oldList []*Segment
 	var oldRows, newRows = 0, 0
 	_, _, iErr := IterateSegments(handle, true, func(l string) {
 		// do nothing here
-	}, nil, func(seg *Segment) error {
+	}, nil, NewRegion, func(seg *Segment) error {
 		o, n, err := e.PutSegment(seg, cb)
 		if err == nil {
 			oldRows += o
@@ -395,7 +400,7 @@ func (e *Editor) SaveToFile(dstFile string) error {
 		}
 
 		// ignore the padded or empty segment
-		if s.Region == "" {
+		if s.Region.IsEmpty() {
 			continue
 		}
 
@@ -415,4 +420,5 @@ func (e *Editor) SaveToFile(dstFile string) error {
 
 func (e *Editor) Close() {
 	_ = e.srcHandle.Close()
+	e.rgCache.Clean()
 }

maker/golang/xdb/maker.go

@@ -84,6 +84,7 @@ type Maker struct {
 	indexPolicy IndexPolicy
 	segments    []*Segment
 	regionPool  map[string]uint32
+	regionCache *RegionCache
 	vectorIndex []byte
 }
 
@@ -118,6 +119,7 @@ func NewMaker(version *Version, policy IndexPolicy, srcFile string, dstFile stri
 		indexPolicy: policy,
 		segments:    []*Segment{},
 		regionPool:  map[string]uint32{},
+		regionCache: NewRegionCache(),
 		vectorIndex: make([]byte, VectorIndexLength),
 	}, nil
 }
@@ -173,7 +175,7 @@ func (m *Maker) loadSegments() error {
 	}, func(region string) (string, error) {
 		// apply the field filter
 		return RegionFiltering(region, m.fields)
-	}, func(seg *Segment) error {
+	}, m.regionCache.Region, func(seg *Segment) error {
 		// ip version check
 		if len(seg.StartIP) != m.version.Bytes {
 			return fmt.Errorf("invalid ip segment(%s expected)", m.version.Name)
@@ -276,13 +278,13 @@ func (m *Maker) Start() error {
 	slog.Info("try to write the data block ... ")
 	for _, seg := range m.segments {
 		slog.Debug("try to write", "region", seg.Region)
-		ptr, has := m.regionPool[seg.Region]
+		ptr, has := m.regionPool[seg.Region.Str]
 		if has {
 			slog.Debug(" --[Cached]", "ptr=", ptr)
 			continue
 		}
 
-		var region = []byte(seg.Region)
+		var region = []byte(seg.Region.Str)
 		if len(region) > 0xFFFF {
 			return fmt.Errorf("too long region info `%s`: should be less than %d bytes", seg.Region, 0xFFFF)
 		}
@@ -303,7 +305,7 @@ func (m *Maker) Start() error {
 			return fmt.Errorf("write region '%s': %w", seg.Region, err)
 		}
 
-		m.regionPool[seg.Region] = uint32(pos)
+		m.regionPool[seg.Region.Str] = uint32(pos)
 		slog.Debug(" --[Added] with", "ptr", pos)
 	}
 
@@ -312,14 +314,14 @@ func (m *Maker) Start() error {
 	var indexBuff = make([]byte, m.version.SegmentIndexSize)
 	var counter, startIndexPtr, endIndexPtr = 0, int64(-1), int64(-1)
 	for _, seg := range m.segments {
-		dataPtr, has := m.regionPool[seg.Region]
+		dataPtr, has := m.regionPool[seg.Region.Str]
 		if !has {
 			return fmt.Errorf("missing ptr cache for region `%s`", seg.Region)
 		}
 
 		// @Note: data length should be the length of bytes.
 		// this works fine because of the string feature (byte sequence) of golang.
-		var dataLen = len(seg.Region)
+		var dataLen = len(seg.Region.Str)
 		if dataLen < 1 {
 			// @TODO: could this even be a case ?
 			// 	return fmt.Errorf("empty region info for segment '%s'", seg)
@@ -412,5 +414,7 @@ func (m *Maker) End() error {
 		return err
 	}
 
+	m.regionCache.Clean()
+
 	return nil
 }

maker/golang/xdb/processor.go

@@ -26,6 +26,9 @@ type Processor struct {
 
 	fields   []int
 	segments []*Segment
+
+	// region cache
+	rgCache *RegionCache
 }
 
 func NewProcessor(srcFile string, dstFile string, fields []int,
@@ -55,6 +58,7 @@ func NewProcessor(srcFile string, dstFile string, fields []int,
 		fields: fields,
 
 		segments: []*Segment{},
+		rgCache:  NewRegionCache(),
 	}, nil
 }
 
@@ -94,7 +98,7 @@ func (p *Processor) loadSegments() error {
 		}
 
 		return RegionFiltering(region, p.fields)
-	}, func(seg *Segment) error {
+	}, p.rgCache.Region, func(seg *Segment) error {
 		// check the continuity of the data segment
 		// if err := seg.AfterCheck(last); err != nil {
 		// 	return err
@@ -152,5 +156,7 @@ func (p *Processor) End() error {
 		return err
 	}
 
+	p.rgCache.Clean()
+
 	return nil
 }

maker/golang/xdb/region.go

@@ -0,0 +1,129 @@
+package xdb
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+)
+
+// region manager with:
+// 1, content cache.
+// 2, util functions
+
+// --- region
+
+type Region struct {
+	Str    string   // region string
+	fields []string // region fields
+}
+
+var EmptyRegion = NewRegion("")
+
+// Create a new region without checking cache info
+func NewRegion(str string) *Region {
+	return &Region{
+		Str:    str,
+		fields: nil,
+	}
+}
+
+func (r *Region) Fields() []string {
+	if r.fields == nil {
+		r.fields = strings.Split(r.Str, "|")
+	}
+
+	return r.fields
+}
+
+func (r *Region) Join(sep string) string {
+	if sep == "|" {
+		return r.Str
+	}
+
+	return strings.Join(r.Fields(), sep)
+}
+
+func (r *Region) Filtering(fields []int) (*Region, error) {
+	if len(fields) == 0 {
+		return r, nil
+	}
+
+	fs := r.Fields()
+	var sb []string
+	for _, idx := range fields {
+		if idx < 0 {
+			return r, fmt.Errorf("negative filter index %d", idx)
+		}
+
+		if idx >= len(fs) {
+			return r, fmt.Errorf("field index %d exceeded the max length of %d", idx, len(fs))
+		}
+
+		sb = append(sb, fs[idx])
+	}
+
+	return &Region{
+		Str:    strings.Join(sb, "|"),
+		fields: sb,
+	}, nil
+}
+
+// Equal check ptr (share the same region cache) or the Str is the same.
+func (r *Region) Equal(dst *Region) bool {
+	return (r == dst || r.Str == dst.Str)
+}
+
+func (r *Region) IsEmpty() bool {
+	return r.Str == ""
+}
+
+func (r *Region) String() string {
+	return r.Str
+}
+
+// ---
+// --- region cache
+
+type RegionCache struct {
+	lock  sync.Mutex
+	cache map[string]*Region
+}
+
+func NewRegionCache() *RegionCache {
+	return &RegionCache{
+		lock:  sync.Mutex{},
+		cache: make(map[string]*Region),
+	}
+}
+
+func (rc *RegionCache) Region(str string) *Region {
+	rc.lock.Lock()
+	defer rc.lock.Unlock()
+
+	region, ok := rc.cache[str]
+	if ok {
+		return region
+	}
+
+	// cache the new region
+	region = &Region{
+		Str:    str,
+		fields: nil,
+	}
+
+	rc.cache[str] = region
+	return region
+}
+
+func (rc *RegionCache) Swap(r *Region) *Region {
+	return rc.Region(r.Str)
+}
+
+func (rc *RegionCache) Clean() {
+	rc.lock.Lock()
+	defer rc.lock.Unlock()
+
+	for k := range rc.cache {
+		delete(rc.cache, k)
+	}
+}

maker/golang/xdb/segment.go

@@ -12,10 +12,10 @@ import (
 type Segment struct {
 	StartIP []byte
 	EndIP   []byte
-	Region  string
+	Region  *Region
 }
 
-func SegmentFrom(seg string) (*Segment, error) {
+func SegmentFrom(seg string, cRegion func(string) *Region) (*Segment, error) {
 	var ps = strings.SplitN(strings.TrimSpace(seg), "|", 3)
 	if len(ps) != 3 {
 		return nil, fmt.Errorf("invalid ip segment `%s`", seg)
@@ -38,7 +38,7 @@ func SegmentFrom(seg string) (*Segment, error) {
 	return &Segment{
 		StartIP: sip,
 		EndIP:   eip,
-		Region:  ps[2],
+		Region:  cRegion(ps[2]),
 	}, nil
 }
 

maker/golang/xdb/util.go

@@ -140,7 +140,7 @@ func IPMiddle(sip, eip []byte) ([]byte, error) {
 	return IPHalf(buf), nil
 }
 
-func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) {
+func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), cRegion func(string) *Region, done func(seg *Segment) error) (int, int, error) {
 	var last *Segment = nil
 	var totalCount, mergeCount = 0, 0
 	var scanner = bufio.NewScanner(handle)
@@ -193,14 +193,14 @@ func IterateSegments(handle *os.File, autoMerge bool, before func(l string), fil
 		if filter != nil {
 			region, err = filter(ps[2])
 			if err != nil {
-				return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
+				return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", region, err)
 			}
 		}
 
 		var seg = &Segment{
 			StartIP: sip,
 			EndIP:   eip,
-			Region:  region,
+			Region:  cRegion(region),
 		}
 
 		// check and automatic merging the Consecutive Segments, which means:
@@ -209,7 +209,7 @@ func IterateSegments(handle *os.File, autoMerge bool, before func(l string), fil
 		if last == nil {
 			last = seg
 			continue
-		} else if autoMerge && last.Region == seg.Region {
+		} else if autoMerge && last.Region.Equal(seg.Region) {
 			if err = seg.RightBehind(last); err == nil {
 				mergeCount++
 				last.EndIP = seg.EndIP
@@ -266,7 +266,7 @@ func MergeSegments(segList []*Segment) []*Segment {
 		if last == nil {
 			last = seg
 			continue
-		} else if last.Region == seg.Region {
+		} else if last.Region.Equal(seg.Region) {
 			if err = seg.RightBehind(last); err == nil {
 				last.EndIP = seg.EndIP
 				continue

maker/golang/xdb/util_test.go

@@ -201,7 +201,7 @@ func TestSplitSegmentV4(t *testing.T) {
 	// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
 	// var str = "29.0.0.0|29.34.191.255|美国|0|0|0|0"
 	var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0"
-	seg, err := SegmentFrom(str)
+	seg, err := SegmentFrom(str, NewRegion)
 	if err != nil {
 		t.Fatalf("failed to parser segment '%s': %s", str, err)
 	}
@@ -220,12 +220,12 @@ func TestSplitSegmentV4(t *testing.T) {
 
 func TestRegionFiltering(t *testing.T) {
 	var line = "2001:1203:31:8000::|2001:1203:31:bfff:ffff:ffff:ffff:ffff||墨西哥|瓜纳华托州||||专线用户|"
-	seg, err := SegmentFrom(line)
+	seg, err := SegmentFrom(line, NewRegion)
 	if err != nil {
 		t.Fatalf("failed to parse segment '%s': %s", line, err)
 	}
 
-	fReg, err := RegionFiltering(seg.Region, []int{1, 2, 4, 6})
+	fReg, err := seg.Region.Filtering([]int{1, 2, 4, 6})
 	if err != nil {
 		t.Fatalf("failed to filter region '%s': %s", seg.Region, err)
 	}
@@ -235,7 +235,7 @@ func TestRegionFiltering(t *testing.T) {
 
 func TestSplitSegmentV6(t *testing.T) {
 	var str = "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff||瑞士|弗里堡州||||专线用户|IANA"
-	seg, err := SegmentFrom(str)
+	seg, err := SegmentFrom(str, NewRegion)
 	if err != nil {
 		t.Fatalf("failed to parser segment '%s': %s", str, err)
 	}
@@ -260,7 +260,7 @@ func TestIterateSegments(t *testing.T) {
 
 	_, _, _ = IterateSegments(handle, true, func(l string) {
 		// fmt.Printf("load segment: `%s`\n", l)
-	}, nil, func(seg *Segment) error {
+	}, nil, NewRegion, func(seg *Segment) error {
 		fmt.Printf("get segment: `%s`\n", seg)
 		return nil
 	})