xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use the Region cache

This commit is contained in:
lionsoul2014 2026-04-29 15:52:40 +08:00
parent 5b10989ac2
commit 787bd7ae07
8 changed files with 81 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
maker/golang/cmd/bench.go
View File

@ -90,12 +90,14 @@ func Bench(sCmd string) {
fmt.Printf("failed to open source text file: %s\n", err)
return
}
defer handle.Close()
var rgCache = xdb.NewRegionCache()
var count, errCount, tStart = 0, 0, time.Now()
slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile)
_, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error {
_, _, iErr := xdb.IterateSegments(handle, false, func(l string) {
// do thing here
}, nil, rgCache.Region, func(seg *xdb.Segment) error {
var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region)
slog.Debug("try to bench", "segment", l)
// mip := xdb.IPMiddle(seg.StartIP, seg.EndIP)

13
maker/golang/xdb/editor.go
View File

@ -24,6 +24,9 @@ type Editor struct {
// segments list
segments *list.List
// region cache
rgCache *RegionCache
}
func NewEditor(version *Version, srcFile string) (*Editor, error) {
@ -44,6 +47,7 @@ func NewEditor(version *Version, srcFile string) (*Editor, error) {
srcHandle: srcHandle,
toSave: false,
segments: list.New(),
rgCache: NewRegionCache(),
}
// load the segments
@ -62,7 +66,7 @@ func (e *Editor) loadSegments() error {
_, _, iErr := IterateSegments(e.srcHandle, true, func(l string) {
// do nothing here
}, nil, func(seg *Segment) error {
}, nil, e.rgCache.Region, func(seg *Segment) error {
// version check
if len(seg.StartIP) != e.verison.Bytes {
return fmt.Errorf("invalid ip segment(%s expected)", e.verison.Name)
@ -100,9 +104,6 @@ func (e *Editor) loadSegments() error {
// to Keep the entire data continuous.
last = nil
for _, seg := range segments {
if err := seg.After(last); err != nil {
}
if last == nil {
if IPCompare(seg.StartIP, e.verison.Min) > 0 {
e.segments.PushBack(&Segment{
@ -182,7 +183,7 @@ func (e *Editor) Slice(offset int, size int) []*Segment {
}
func (e *Editor) Put(ip string, cb func(newSeg *Segment, oldList []*Segment) []*Segment) (int, int, error) {
seg, err := SegmentFrom(ip)
seg, err := SegmentFrom(ip, e.rgCache.Region)
if err != nil {
return 0, 0, err
}
@ -350,7 +351,7 @@ func (e *Editor) PutFile(src string, cb func(newSeg *Segment, oldList []*Segment
var oldRows, newRows = 0, 0
_, _, iErr := IterateSegments(handle, true, func(l string) {
// do nothing here
}, nil, func(seg *Segment) error {
}, nil, NewRegion, func(seg *Segment) error {
o, n, err := e.PutSegment(seg, cb)
if err == nil {
oldRows += o

4
maker/golang/xdb/maker.go
View File

@ -84,6 +84,7 @@ type Maker struct {
indexPolicy IndexPolicy
segments []*Segment
regionPool map[string]uint32
regionCache *RegionCache
vectorIndex []byte
}
@ -118,6 +119,7 @@ func NewMaker(version *Version, policy IndexPolicy, srcFile string, dstFile stri
indexPolicy: policy,
segments: []*Segment{},
regionPool: map[string]uint32{},
regionCache: NewRegionCache(),
vectorIndex: make([]byte, VectorIndexLength),
}, nil
}
@ -173,7 +175,7 @@ func (m *Maker) loadSegments() error {
}, func(region string) (string, error) {
// apply the field filter
return RegionFiltering(region, m.fields)
}, func(seg *Segment) error {
}, m.regionCache.Region, func(seg *Segment) error {
// ip version check
if len(seg.StartIP) != m.version.Bytes {
return fmt.Errorf("invalid ip segment(%s expected)", m.version.Name)

6
maker/golang/xdb/processor.go
View File

@ -26,6 +26,9 @@ type Processor struct {
fields []int
segments []*Segment
// region cache
rgCache *RegionCache
}
func NewProcessor(srcFile string, dstFile string, fields []int,
@ -55,6 +58,7 @@ func NewProcessor(srcFile string, dstFile string, fields []int,
fields: fields,
segments: []*Segment{},
rgCache: NewRegionCache(),
}, nil
}
@ -94,7 +98,7 @@ func (p *Processor) loadSegments() error {
}
return RegionFiltering(region, p.fields)
}, func(seg *Segment) error {
}, p.rgCache.Region, func(seg *Segment) error {
// check the continuity of the data segment
// if err := seg.AfterCheck(last); err != nil {
// return err

86
maker/golang/xdb/region.go
View File

@ -10,39 +10,14 @@ import (
// 1, content cache.
// 2, util functions
// global cache map
var rcLock sync.Mutex
var regionCache = map[string]*Region{}
// --- region
type Region struct {
Str string // region string
fields []string // region fields
}
var EmptyRegion = CacheRegion("")
// Create or get the region from the global cache.
// And it is a thread-safe implementation.
func CacheRegion(str string) *Region {
// check the cache and return it directly
// if there is a cache available
rcLock.Lock()
defer rcLock.Unlock()
region, ok := regionCache[str]
if ok {
return region
}
// cache the new region
region = &Region{
Str: str,
fields: nil,
}
regionCache[str] = region
return region
}
var EmptyRegion = NewRegion("")
// Create a new region without checking cache info
func NewRegion(str string) *Region {
@ -87,12 +62,10 @@ func (r *Region) Filtering(fields []int) (*Region, error) {
sb = append(sb, fs[idx])
}
new := CacheRegion(strings.Join(sb, "|"))
if new.fields == nil {
new.fields = sb
}
return new, nil
return &Region{
Str: strings.Join(sb, "|"),
fields: sb,
}, nil
}
// Equal check ptr (share the same region cache) or the Str is the same.
@ -107,3 +80,50 @@ func (r *Region) IsEmpty() bool {
func (r *Region) String() string {
return r.Str
}
// ---
// --- region cache
type RegionCache struct {
lock sync.Mutex
cache map[string]*Region
}
func NewRegionCache() *RegionCache {
return &RegionCache{
lock: sync.Mutex{},
cache: make(map[string]*Region),
}
}
func (rc *RegionCache) Region(str string) *Region {
rc.lock.Lock()
defer rc.lock.Unlock()
region, ok := rc.cache[str]
if ok {
return region
}
// cache the new region
region = &Region{
Str: str,
fields: nil,
}
rc.cache[str] = region
return region
}
func (rc *RegionCache) Swap(r *Region) *Region {
return rc.Region(r.Str)
}
func (rc *RegionCache) Clean() {
rc.lock.Lock()
defer rc.lock.Unlock()
for k := range rc.cache {
delete(rc.cache, k)
}
}

4
maker/golang/xdb/segment.go
View File

@ -15,7 +15,7 @@ type Segment struct {
Region *Region
}
func SegmentFrom(seg string) (*Segment, error) {
func SegmentFrom(seg string, cRegion func(string) *Region) (*Segment, error) {
var ps = strings.SplitN(strings.TrimSpace(seg), "|", 3)
if len(ps) != 3 {
return nil, fmt.Errorf("invalid ip segment `%s`", seg)
@ -38,7 +38,7 @@ func SegmentFrom(seg string) (*Segment, error) {
return &Segment{
StartIP: sip,
EndIP: eip,
Region: CacheRegion(ps[2]),
Region: cRegion(ps[2]),
}, nil
}

6
maker/golang/xdb/util.go
View File

@ -140,7 +140,7 @@ func IPMiddle(sip, eip []byte) ([]byte, error) {
return IPHalf(buf), nil
}
func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) {
func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), cRegion func(string) *Region, done func(seg *Segment) error) (int, int, error) {
var last *Segment = nil
var totalCount, mergeCount = 0, 0
var scanner = bufio.NewScanner(handle)
@ -193,14 +193,14 @@ func IterateSegments(handle *os.File, autoMerge bool, before func(l string), fil
if filter != nil {
region, err = filter(ps[2])
if err != nil {
return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err)
return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", region, err)
}
}
var seg = &Segment{
StartIP: sip,
EndIP: eip,
Region: CacheRegion(region),
Region: cRegion(region),
}
// check and automatic merging the Consecutive Segments, which means:

8
maker/golang/xdb/util_test.go
View File

@ -201,7 +201,7 @@ func TestSplitSegmentV4(t *testing.T) {
// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
// var str = "29.0.0.0|29.34.191.255|美国|0|0|0|0"
var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0"
seg, err := SegmentFrom(str)
seg, err := SegmentFrom(str, NewRegion)
if err != nil {
t.Fatalf("failed to parser segment '%s': %s", str, err)
}
@ -220,7 +220,7 @@ func TestSplitSegmentV4(t *testing.T) {
func TestRegionFiltering(t *testing.T) {
var line = "2001:1203:31:8000::|2001:1203:31:bfff:ffff:ffff:ffff:ffff||墨西哥|瓜纳华托州||||专线用户|"
seg, err := SegmentFrom(line)
seg, err := SegmentFrom(line, NewRegion)
if err != nil {
t.Fatalf("failed to parse segment '%s': %s", line, err)
}
@ -235,7 +235,7 @@ func TestRegionFiltering(t *testing.T) {
func TestSplitSegmentV6(t *testing.T) {
var str = "fec0::|ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff||瑞士|弗里堡州||||专线用户|IANA"
seg, err := SegmentFrom(str)
seg, err := SegmentFrom(str, NewRegion)
if err != nil {
t.Fatalf("failed to parser segment '%s': %s", str, err)
}
@ -260,7 +260,7 @@ func TestIterateSegments(t *testing.T) {
_, _, _ = IterateSegments(handle, true, func(l string) {
// fmt.Printf("load segment: `%s`\n", l)
}, nil, func(seg *Segment) error {
}, nil, NewRegion, func(seg *Segment) error {
fmt.Printf("get segment: `%s`\n", seg)
return nil
})