diff --git a/binding/golang/ReadMe.md b/binding/golang/ReadMe.md index 84d9ae2..4222cf6 100644 --- a/binding/golang/ReadMe.md +++ b/binding/golang/ReadMe.md @@ -7,9 +7,53 @@ go get github.com/lionsoul2014/ip2region/binding/golang ``` +### 关于查询服务 +从 `3.11.0` 版本开始提供了一个双协议兼容且并发安全的 `Ip2Region` 查询服务，建议优先使用该方式来进行查询调用，具体使用方式如下： +```go +import "github.com/lionsoul2014/ip2region/binding/golang/service" + +// 1, 创建 v4 的配置：指定缓存策略和 v4 的 xdb 文件路径 +// 参数1： 缓存策略, options: service.NoCache / service.VIndexCache / service.BufferCache +// 参数2: xdb 文件路径 +// 参数3: 初始化的查询器数量 +v4Config, err := service.NewV4Config(service.VIndexCache, "ip2region v4 xdb path", 20) +if err != nil { + return fmt.Errorf("failed to create v4 config: %s", err) +} + +// 2, 创建 v6 的配置：指定缓存策略和 v6 的 xdb 文件路径 +v6Config, err := service.NewV6Config(service.VIndexCache, "ip2region v6 xdb path", 20) +if err != nil { + return fmt.Errorf("failed to create v6 config: %s", err) +} + +// 3，通过上述配置创建 Ip2Region 查询服务 +ip2region, err := service.NewIp2Region(v4Config, v6Config) +if err != nil { + return fmt.Errorf("failed to create ip2region service: %s", err) +} + +// 4，导出 ip2region 服务进行双版本的IP地址的并发查询，例如： +var err error +v4Region, err := ip2region.SearchByStr("113.92.157.29") // 进行 IPv4 查询 +v6Region, err := ip2region.SearchByStr("240e:3b7:3272:d8d0:db09:c067:8d59:539e") // 进行 IPv6 查询 + + +// 5，在服务需要关闭的时候，同时关闭 ip2region 查询服务 +ip2region.Close() +``` + +##### `Ip2Region` 查询备注: +1. 该查询服务的 API 并发安全且同时支持 IPv4 和 Ipv6 的地址，内部实现会自动判断。 +2. v4 和 v6 的配置需要单独创建，可以给 v4 和 v6 设置使用不同的缓存策略，也可以指定其中一个为 `nil` 则该版本的 IP 地址查询都会返回 `""`。 +3. 请结合您的项目的并发数设置一个合适的查询器数量，这个值在运行过程中是固定的，每次查询会从池子里租借一个查询器来完成查询操作，查询完成后再归还回去，如果租借的时候池子已经空了则等待直到有可用的查询器来完成查询服务。 +4. 如果配置设置的缓存策略为 `service.BufferCache` 即 `全内存缓存` 则默认会使用单实例的内存查询器，该实现天生并发安全，此时指定的查询器数量无效。 +5. 如果 `Ip2Region` 查询器在提供服务期间，调用 Close 默认会最大等待 10 秒钟来等待尽量多的查询器归还，也可以调用 `CloseTimeout` 来自定义最长等待时间。 + + ### 关于查询 API 定位信息查询 API 原型为： -```golang +```go SearchByStr(string) (string, error) Search([]byte) (string, error) ``` @@ -17,7 +61,7 @@ Search([]byte) (string, error) ### 关于 IPv4 / IPv6 该 xdb 查询客户端实现同时支持对 IPv4 和 IPv6 的查询，使用方式如下： -```golang +```go // 如果是 IPv4: 设置 xdb 路径为 v4 的 xdb 文件，IP版本指定为 xdb.IPv4 dbPath := "../../data/ip2region_v4.xdb" // 或者你的 ipv4 xdb 的路径 version := xdb.IPv4 @@ -33,7 +77,7 @@ version = xdb.IPv6 ### 文件验证 建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 -```golang +```go err := xdb.VerifyFromFile(dbPath) if err != nil { // err 包含的验证的错误 @@ -45,7 +89,7 @@ if err != nil { ### 完全基于文件的查询 -```golang +```go import ( "fmt" "github.com/lionsoul2014/ip2region/binding/golang/xdb" @@ -82,7 +126,7 @@ func main() { ### 缓存 `VectorIndex` 索引 可以预先加载 `vectorIndex` 缓存，然后做成全局变量，每次创建 searcher 的时候使用全局的 `vectorIndex`，可以减少一次固定的 IO 操作从而加速查询，减少系统 io 压力。 -```golang +```go // 1、从 dbPath 加载 VectorIndex 缓存，把下述 vIndex 变量全局到内存里面。 vIndex, err := xdb.LoadVectorIndexFromFile(dbPath) if err != nil { @@ -103,7 +147,7 @@ if err != nil { ### 缓存整个 `xdb` 数据 可以预先加载整个 ip2region.xdb 到内存，完全基于内存查询，类似于之前的 memory search 查询。 -```golang +```go // 1、从 dbPath 加载整个 xdb 到内存 cBuff, err := xdb.LoadContentFromFile(dbPath) if err != nil { diff --git a/binding/golang/service/config.go b/binding/golang/service/config.go new file mode 100644 index 0000000..fd97485 --- /dev/null +++ b/binding/golang/service/config.go @@ -0,0 +1,153 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package service + +import ( + "fmt" + "os" + + "github.com/lionsoul2014/ip2region/binding/golang/xdb" +) + +// --- +// Ip2Region service config +// +// @Author Lion +// @Date 2025/12/03 + +const ( + NoCache = 0 + VIndexCache = 1 + BufferCache = 2 +) + +type Config struct { + cachePolicy int + ipVersion *xdb.Version + + // xdb file path + xdbPath string + header *xdb.Header + + // buffers + vIndex []byte + cBuffer []byte + + searchers int +} + +func NewV4Config(cachePolicy int, xdbPath string, searchers int) (*Config, error) { + return newConfig(cachePolicy, xdb.IPv4, xdbPath, searchers) +} + +func NewV6Config(cachePolicy int, xdbPath string, searchers int) (*Config, error) { + return newConfig(cachePolicy, xdb.IPv6, xdbPath, searchers) +} + +func newConfig(cachePolicy int, ipVersion *xdb.Version, xdbPath string, searchers int) (*Config, error) { + if searchers < 1 { + return nil, fmt.Errorf("searchers=%d, > 0 expected", searchers) + } + + // open the xdb binary file + handle, err := os.OpenFile(xdbPath, os.O_RDONLY, 0600) + if err != nil { + return nil, err + } + + // 1, verify the xdb + err = xdb.Verify(handle) + if err != nil { + return nil, err + } + + // 2, load the header + header, err := xdb.LoadHeader(handle) + if err != nil { + return nil, err + } + + // verify the ip version + xIpVersion, err := xdb.VersionFromHeader(header) + if err != nil { + return nil, err + } + + if xIpVersion.Id != ipVersion.Id { + return nil, fmt.Errorf("ip verison not match: xdb file %s with ip version=%s, as %s expected", xdbPath, xIpVersion.Name, ipVersion.Name) + } + + // 3, check and load the vector index buffer + var vIndex []byte = nil + if cachePolicy == VIndexCache { + vIndex, err = xdb.LoadVectorIndex(handle) + if err != nil { + return nil, err + } + } + + // 4, check and load the content buffer + var cBuffer []byte = nil + if cachePolicy == BufferCache { + cBuffer, err = xdb.LoadContent(handle) + if err != nil { + return nil, err + } + } + + return &Config{ + cachePolicy: cachePolicy, + ipVersion: ipVersion, + + xdbPath: xdbPath, + header: header, + + vIndex: vIndex, + cBuffer: cBuffer, + + searchers: searchers, + }, nil +} + +func (c *Config) String() string { + vIndex := "null" + if c.vIndex != nil { + vIndex = fmt.Sprintf("{bytes:%d}", len(c.vIndex)) + } + + cBuffer := "null" + if c.cBuffer != nil { + cBuffer = fmt.Sprintf("{bytes:%d}", len(c.cBuffer)) + } + + return fmt.Sprintf( + "{cache_policy:%d, version:%s, xdb_path:%s, header:%s, v_index:%s, c_buffer:%s}", + c.cachePolicy, c.ipVersion.String(), c.xdbPath, c.header.String(), vIndex, cBuffer, + ) +} + +func (c *Config) CachePolicy() int { + return c.cachePolicy +} + +func (c *Config) IPVersion() *xdb.Version { + return c.ipVersion +} + +func (c *Config) Header() *xdb.Header { + return c.header +} + +func (c *Config) VIndex() []byte { + return c.vIndex +} + +func (c *Config) CBuffer() []byte { + return c.cBuffer +} + +func (c *Config) Searchers() int { + return c.searchers +} diff --git a/binding/golang/service/config_test.go b/binding/golang/service/config_test.go new file mode 100644 index 0000000..b90022e --- /dev/null +++ b/binding/golang/service/config_test.go @@ -0,0 +1,37 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package service + +import ( + "fmt" + "testing" +) + +func TestV4Config(t *testing.T) { + v4Config, err := NewV4Config(VIndexCache, "../../../data/ip2region_v4.xdb", 10) + if err != nil { + t.Errorf("failed to new v4 config: %s", err) + return + } + + v4BufferConfig, err := NewV4Config(BufferCache, "../../../data/ip2region_v4.xdb", 10) + if err != nil { + t.Errorf("failed to new v4 config: %s", err) + return + } + + fmt.Printf("v4Config: %s\n", v4Config) + fmt.Printf("v4BufferConfig: %s\n", v4BufferConfig) +} + +func TestV6Config(t *testing.T) { + v6Config, err := NewV6Config(NoCache, "../../../data/ip2region_v6.xdb", 10) + if err != nil { + t.Errorf("failed to new v6 config: %s", err) + return + } + + fmt.Printf("v6Config: %s\n", v6Config) +} diff --git a/binding/golang/service/ip2region.go b/binding/golang/service/ip2region.go new file mode 100644 index 0000000..e89438d --- /dev/null +++ b/binding/golang/service/ip2region.go @@ -0,0 +1,182 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package service + +import ( + "fmt" + "time" + + "github.com/lionsoul2014/ip2region/binding/golang/xdb" +) + +// --- +// Ip2Region service +// 1. Unified query interface to IPv4 and IPv6 address. +// 2. Concurrency search support. +// +// @Author Lion +// @Date 2025/12/05 + +type Ip2Region struct { + // v4 pool for cache policy vIndex or NoCache + v4Pool *SearcherPool + + // v4 xdb searcher for full in-memeory search + v4InMemSearcher *xdb.Searcher + + // v6 pool for cache policy vIndex or NoCache:w + v6Pool *SearcherPool + + // v6 xdb searcher for full in-memeory search + v6InMemSearcher *xdb.Searcher +} + +// create a new Ip2Region service with specified v4 and v6 config. +// set it to nil to disabled the specified search for the specified version. +func NewIp2Region(v4Config *Config, v6Config *Config) (*Ip2Region, error) { + var err error + + // check and init the v4 pool or in-memory searcher + var v4Pool *SearcherPool + var v4InMemSearcher *xdb.Searcher + if v4Config == nil { + // with IPv4 disabled ? + v4Pool = nil + v4InMemSearcher = nil + } else if v4Config.cachePolicy == BufferCache { + v4Pool = nil + v4InMemSearcher, err = xdb.NewWithBuffer(v4Config.ipVersion, v4Config.cBuffer) + if err != nil { + return nil, fmt.Errorf("failed to create v4 in-memory searcher: %w", err) + } + } else { + v4InMemSearcher = nil + v4Pool, err = NewSearcherPool(v4Config) + if err != nil { + return nil, fmt.Errorf("failed to create v4 searcher pool: %w", err) + } + } + + // check and init the v6 pool or in-memory searcher + var v6Pool *SearcherPool + var v6InMemSearcher *xdb.Searcher + if v6Config == nil { + v6Pool = nil + v6InMemSearcher = nil + } else if v6Config.cachePolicy == BufferCache { + v6Pool = nil + v6InMemSearcher, err = xdb.NewWithBuffer(v6Config.ipVersion, v6Config.cBuffer) + if err != nil { + return nil, fmt.Errorf("failed to create v6 in-memory searcher: %w", err) + } + } else { + v6InMemSearcher = nil + v6Pool, err = NewSearcherPool(v6Config) + if err != nil { + return nil, fmt.Errorf("failed to create v6 in-memeory searcher pool: %w", err) + } + } + + return &Ip2Region{ + v4Pool: v4Pool, + v4InMemSearcher: v4InMemSearcher, + + v6Pool: v6Pool, + v6InMemSearcher: v6InMemSearcher, + }, nil +} + +// create the ip2region search service with the specified v4 & v6 xdb path. +// with default cache policy VIndexCache and default searchers = 20 +func NewIp2RegionWithPath(v4XdbPath string, v6XdbPath string) (*Ip2Region, error) { + var err error + + // create v4 config with default config items + var v4Config *Config + if v4XdbPath == "" { + v4Config = nil + } else { + v4Config, err = NewV4Config(VIndexCache, v4XdbPath, 20) + if err != nil { + return nil, fmt.Errorf("failed to create v4 config: %w", err) + } + } + + // create v6 config with default config items + var v6Config *Config + if v6XdbPath == "" { + v6Config = nil + } else { + v6Config, err = NewV6Config(VIndexCache, v6XdbPath, 20) + if err != nil { + return nil, fmt.Errorf("failed to create v6 config: %w", err) + } + } + + return NewIp2Region(v4Config, v6Config) +} + +func (ip2r *Ip2Region) SearchByStr(ipStr string) (string, error) { + ipBytes, err := xdb.ParseIP(ipStr) + if err != nil { + return "", err + } + + return ip2r.Search(ipBytes) +} + +func (ip2r *Ip2Region) Search(ipBytes []byte) (string, error) { + if l := len(ipBytes); l == 4 { + return ip2r.v4Search(ipBytes) + } else if l == 16 { + return ip2r.v6Search(ipBytes) + } else { + return "", fmt.Errorf("invalid byte ip address with len=%d", l) + } +} + +func (ip2r *Ip2Region) v4Search(ipBytes []byte) (string, error) { + if ip2r.v4InMemSearcher != nil { + return ip2r.v4InMemSearcher.Search(ipBytes) + } + + // v4 search is disabled + if ip2r.v4Pool == nil { + return "", nil + } + + v4Searcher := ip2r.v4Pool.BorrowSearcher() + defer ip2r.v4Pool.ReturnSearcher(v4Searcher) + return v4Searcher.Search(ipBytes) +} + +func (ip2r *Ip2Region) v6Search(ipBytes []byte) (string, error) { + if ip2r.v6InMemSearcher != nil { + return ip2r.v6InMemSearcher.Search(ipBytes) + } + + // v6 search is disabled + if ip2r.v6Pool == nil { + return "", nil + } + + v6Searcher := ip2r.v6Pool.BorrowSearcher() + defer ip2r.v6Pool.ReturnSearcher(v6Searcher) + return v6Searcher.Search(ipBytes) +} + +func (ip2r *Ip2Region) Close() { + ip2r.CloseTimeout(time.Second * 10) +} + +func (ip2r *Ip2Region) CloseTimeout(d time.Duration) { + if ip2r.v4Pool != nil { + ip2r.v4Pool.CloseTimeout(d) + } + + if ip2r.v6Pool != nil { + ip2r.v6Pool.CloseTimeout(d) + } +} diff --git a/binding/golang/service/ip2region_test.go b/binding/golang/service/ip2region_test.go new file mode 100644 index 0000000..4d84351 --- /dev/null +++ b/binding/golang/service/ip2region_test.go @@ -0,0 +1,273 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package service + +import ( + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/lionsoul2014/ip2region/binding/golang/xdb" +) + +func TestConfigCreate(t *testing.T) { + v4Config, err := NewV4Config(VIndexCache, "../../../data/ip2region_v4.xdb", 10) + if err != nil { + t.Fatalf("failed to create v4 config: %s", err) + } + + v6Config, err := NewV6Config(VIndexCache, "../../../data/ip2region_v6.xdb", 10) + if err != nil { + t.Fatalf("failed to create v6 config: %s", err) + } + + ip2region, err := NewIp2Region(v4Config, v6Config) + if err != nil { + t.Fatalf("failed to create ip2region service: %s", err) + } + + v4Bytes, err := xdb.ParseIP("219.133.110.197") + if err != nil { + t.Fatal("invalid ipv4 address") + } + + v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") + if err != nil { + t.Fatalf("invalid ipv6 address") + } + + for i := 0; i < 20; i++ { + v4Bytes = xdb.IPAddOne(v4Bytes) + v6Bytes = xdb.IPAddOne(v6Bytes) + v4Region, err := ip2region.Search(v4Bytes) + if err != nil { + t.Fatalf("failed to search(%s): %s", xdb.IP2String(v4Bytes), err) + } + + v6Region, err := ip2region.Search(v6Bytes) + if err != nil { + t.Fatalf("failed to search(%s): %s", xdb.IP2String(v6Bytes), err) + } + + fmt.Printf( + "%2d->search(%s)=%s, search(%s)=%s\n", + i, xdb.IP2String(v4Bytes), v4Region, xdb.IP2String(v6Bytes), v6Region, + ) + } + + ip2region.Close() + fmt.Print("ip2region closed gracefully") +} + +func TestPathCreate(t *testing.T) { + ip2region, err := NewIp2RegionWithPath("../../../data/ip2region_v4.xdb", "../../../data/ip2region_v6.xdb") + if err != nil { + t.Fatalf("failed to create ip2region with path: %s", err) + } + + v4Bytes, err := xdb.ParseIP("219.133.110.197") + if err != nil { + t.Fatal("invalid ipv4 address") + } + + v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") + if err != nil { + t.Fatalf("invalid ipv6 address") + } + + for i := 0; i < 20; i++ { + v4Bytes = xdb.IPAddOne(v4Bytes) + v6Bytes = xdb.IPAddOne(v6Bytes) + v4Region, err := ip2region.Search(v4Bytes) + if err != nil { + t.Fatalf("failed to search(%s): %s", xdb.IP2String(v4Bytes), err) + } + + v6Region, err := ip2region.Search(v6Bytes) + if err != nil { + t.Fatalf("failed to search(%s): %s", xdb.IP2String(v6Bytes), err) + } + + fmt.Printf( + "%2d->search(%s)=%s, search(%s)=%s\n", + i, xdb.IP2String(v4Bytes), v4Region, xdb.IP2String(v6Bytes), v6Region, + ) + } + + ip2region.Close() + fmt.Print("ip2region closed gracefully") +} + +func TestInMemSearch(t *testing.T) { + v4Config, err := NewV4Config(BufferCache, "../../../data/ip2region_v4.xdb", 10) + if err != nil { + t.Fatalf("failed to create v4 config: %s", err) + } + + v6Config, err := NewV6Config(BufferCache, "../../../data/ip2region_v6.xdb", 10) + if err != nil { + t.Fatalf("failed to create v6 config: %s", err) + } + + ip2region, err := NewIp2Region(v4Config, v6Config) + if err != nil { + t.Fatalf("failed to create ip2region service: %s", err) + } + + v4Bytes, err := xdb.ParseIP("219.133.110.197") + if err != nil { + t.Fatal("invalid ipv4 address") + } + + v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") + if err != nil { + t.Fatalf("invalid ipv6 address") + } + + for i := 0; i < 20; i++ { + v4Bytes = xdb.IPAddOne(v4Bytes) + v6Bytes = xdb.IPAddOne(v6Bytes) + v4Region, err := ip2region.Search(v4Bytes) + if err != nil { + t.Fatalf("failed to search(%s): %s", xdb.IP2String(v4Bytes), err) + } + + v6Region, err := ip2region.Search(v6Bytes) + if err != nil { + t.Fatalf("failed to search(%s): %s", xdb.IP2String(v6Bytes), err) + } + + fmt.Printf( + "%2d->search(%s)=%s, search(%s)=%s\n", + i, xdb.IP2String(v4Bytes), v4Region, xdb.IP2String(v6Bytes), v6Region, + ) + } + + ip2region.Close() + fmt.Print("ip2region closed gracefully") +} + +func TestV4Only(t *testing.T) { + v4Config, err := NewV4Config(NoCache, "../../../data/ip2region_v4.xdb", 10) + if err != nil { + t.Fatalf("failed to create v4 config: %s", err) + } + + ip2region, err := NewIp2Region(v4Config, nil) + if err != nil { + t.Fatalf("failed to create ip2region service: %s", err) + } + + v4Bytes, err := xdb.ParseIP("219.133.110.197") + if err != nil { + t.Fatal("invalid ipv4 address") + } + + v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") + if err != nil { + t.Fatalf("invalid ipv6 address") + } + + for i := 0; i < 10; i++ { + v4Bytes = xdb.IPAddOne(v4Bytes) + v6Bytes = xdb.IPAddOne(v6Bytes) + v4Region, err := ip2region.Search(v4Bytes) + if err != nil { + t.Fatalf("failed to search(%s): %s", xdb.IP2String(v4Bytes), err) + } + + v6Region, err := ip2region.Search(v6Bytes) + if err != nil { + t.Fatalf("failed to search(%s): %s", xdb.IP2String(v6Bytes), err) + } + + fmt.Printf( + "%2d->search(%s)=%s, search(%s)=%s\n", + i, xdb.IP2String(v4Bytes), v4Region, xdb.IP2String(v6Bytes), v6Region, + ) + } + + ip2region.Close() + fmt.Print("ip2region closed gracefully") +} + +func TestConcurrentCall(t *testing.T) { + v4Config, err := NewV4Config(VIndexCache, "../../../data/ip2region_v4.xdb", 15) + if err != nil { + t.Fatalf("failed to create v4 config: %s", err) + } + + v6Config, err := NewV6Config(VIndexCache, "../../../data/ip2region_v6.xdb", 15) + if err != nil { + t.Fatalf("failed to create v6 config: %s", err) + } + + v4Bytes, err := xdb.ParseIP("219.133.110.197") + if err != nil { + t.Fatal("invalid ipv4 address") + } + + v6Bytes, err := xdb.ParseIP("240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6") + if err != nil { + t.Fatalf("invalid ipv6 address") + } + + fmt.Printf("v4Config: %s\n", v4Config) + fmt.Printf("v6Config: %s\n", v6Config) + ip2region, err := NewIp2Region(v4Config, v6Config) + if err != nil { + t.Fatalf("failed to create ip2region service: %s", err) + } + + coroutines := 100 + var wg sync.WaitGroup + var count int64 = 0 + tStart := time.Now() + for i := 0; i < coroutines; i++ { + wg.Add(1) + go func() { + var ipBytes []byte + for i := 0; i < 5000; i++ { + if i%2 == 0 { + ipBytes = v4Bytes + } else { + ipBytes = v6Bytes + } + + region, err := ip2region.Search(ipBytes) + if err != nil { + fmt.Printf("Error: failed to search(%s): %s", xdb.IP2String(ipBytes), err) + break + } + + if l := len(ipBytes); l == 4 { + if region != "中国|广东省|深圳市|电信" { + fmt.Print("Error: region not equals") + break + } + } else { + if region != "中国|广东省|深圳市|家庭宽带" { + fmt.Print("Error: region not equals") + break + } + } + + atomic.AddInt64(&count, 1) + } + + // mark all searches finished for this coroutine + wg.Done() + }() + } + + // wait for all the searches to finished + wg.Wait() + costs := time.Since(tStart) + fmt.Printf("%d searches finished in %s, avg took: %s\n", count, costs, costs/time.Duration(count)) + ip2region.Close() + fmt.Print("ip2region closed gracefully") +} diff --git a/binding/golang/service/searcher_pool.go b/binding/golang/service/searcher_pool.go new file mode 100644 index 0000000..44cca41 --- /dev/null +++ b/binding/golang/service/searcher_pool.go @@ -0,0 +1,113 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package service + +import ( + "fmt" + "sync/atomic" + "time" + + "github.com/lionsoul2014/ip2region/binding/golang/xdb" +) + +// --- +// ip2region searcher pool +// +// @Author Lion +// @Date 2025/12/03 + +type SearcherPool struct { + // config + config *Config + + // searcher pool + pool chan *xdb.Searcher + + // for pool close + closing chan struct{} + + // searcher number that was loaned out + loanCount int32 +} + +func NewSearcherPool(config *Config) (*SearcherPool, error) { + if config.searchers < 1 { + return nil, fmt.Errorf("config.searchers must > 0") + } + + pool := make(chan *xdb.Searcher, config.searchers+1) + // check and create all the searchers + for i := 0; i < config.searchers; i++ { + searcher, err := xdb.NewSearcher(config.ipVersion, config.xdbPath, config.vIndex, config.cBuffer) + if err != nil { + return nil, fmt.Errorf("failed to create the %dth searcher: %w", i+1, err) + } + + // push the search to the pool + pool <- searcher + } + + return &SearcherPool{ + config: config, + pool: pool, + + closing: make(chan struct{}, 1), + loanCount: 0, + }, nil +} + +// get the loaned count +func (sp *SearcherPool) LoanCount() int { + return int(atomic.LoadInt32(&sp.loanCount)) +} + +func (sp *SearcherPool) BorrowSearcher() *xdb.Searcher { + // @Note: still accept searcher borrow while closing + s := <-sp.pool + atomic.AddInt32(&sp.loanCount, 1) + return s +} + +func (sp *SearcherPool) ReturnSearcher(searcher *xdb.Searcher) { + select { + case <-sp.closing: + // manually close the searcher + searcher.Close() + + // decrease the loan count + atomic.AddInt32(&sp.loanCount, -1) + default: + // return the searcher + sp.pool <- searcher + atomic.AddInt32(&sp.loanCount, -1) + } +} + +func (sp *SearcherPool) Close() { + sp.CloseTimeout(time.Second * 10) +} + +func (sp *SearcherPool) CloseTimeout(d time.Duration) { + close(sp.closing) + for { + timeout := false + select { + case s := <-sp.pool: + s.Close() + case <-time.After(d): + // check if all the loaned searchers was closed + timeout = true + } + + lc, left := sp.LoanCount(), len(sp.pool) + if left == 0 && lc == 0 { + break + } + + if timeout { + break + } + } +} diff --git a/binding/golang/service/searcher_pool_test.go b/binding/golang/service/searcher_pool_test.go new file mode 100644 index 0000000..af2503c --- /dev/null +++ b/binding/golang/service/searcher_pool_test.go @@ -0,0 +1,70 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package service + +import ( + "fmt" + "testing" +) + +func TestV4SearcherPool(t *testing.T) { + v4Config, err := NewV4Config(VIndexCache, "../../../data/ip2region_v4.xdb", 5) + if err != nil { + t.Fatalf("failed to new v4 config: %s", err) + } + + searcherPool, err := NewSearcherPool(v4Config) + if err != nil { + t.Fatalf("failed to create searcher pool: %s", err) + } + + ipString := "219.133.110.197" + for i := 0; i < 20; i++ { + searcher := searcherPool.BorrowSearcher() + region, err := searcher.SearchByStr(ipString) + if err != nil { + t.Fatalf("failed to search(%s): %s", ipString, err) + } + + fmt.Printf("%2d->search(%s)=%s\n", i, ipString, region) + searcherPool.ReturnSearcher(searcher) + } + + // borrow one at last for Close timeout wait testing ONLY + // searcherPool.BorrowSearcher() + + // close the searcher pool + searcherPool.Close() +} + +func TestV6SearcherPool(t *testing.T) { + v6Config, err := NewV6Config(VIndexCache, "../../../data/ip2region_v6.xdb", 5) + if err != nil { + t.Fatalf("failed to new v6 config: %s", err) + } + + searcherPool, err := NewSearcherPool(v6Config) + if err != nil { + t.Fatalf("failed to create searcher pool: %s", err) + } + + ipString := "240e:3b7:3275:f090:d2a3:7d1a:dd90:c3b6" + for i := 0; i < 20; i++ { + searcher := searcherPool.BorrowSearcher() + region, err := searcher.SearchByStr(ipString) + if err != nil { + t.Fatalf("failed to search(%s): %s", ipString, err) + } + + fmt.Printf("%2d->search(%s)=%s\n", i, ipString, region) + searcherPool.ReturnSearcher(searcher) + } + + // borrow one at last for Close timeout wait testing ONLY + // searcherPool.BorrowSearcher() + + // close the searcher pool + searcherPool.Close() +} diff --git a/binding/golang/xdb/header.go b/binding/golang/xdb/header.go index 2f758b9..0bba957 100644 --- a/binding/golang/xdb/header.go +++ b/binding/golang/xdb/header.go @@ -76,3 +76,10 @@ func NewHeader(input []byte) (*Header, error) { RuntimePtrBytes: int(binary.LittleEndian.Uint16(input[18:])), }, nil } + +func (h *Header) String() string { + return fmt.Sprintf( + "{version:%d, index_policy:%d, created_at:%d, start_index_ptr:%d, end_index_ptr:%d, ip_version:%d, runtime_ptr_bytes:%d}", + h.Version, h.IndexPolicy, h.CreatedAt, h.StartIndexPtr, h.EndIndexPtr, h.IPVersion, h.RuntimePtrBytes, + ) +} diff --git a/binding/golang/xdb/searcher.go b/binding/golang/xdb/searcher.go index 185a264..7ab111e 100644 --- a/binding/golang/xdb/searcher.go +++ b/binding/golang/xdb/searcher.go @@ -33,7 +33,19 @@ type Searcher struct { contentBuff []byte } -func baseNew(version *Version, dbFile string, vIndex []byte, cBuff []byte) (*Searcher, error) { +func NewWithFileOnly(version *Version, dbFile string) (*Searcher, error) { + return NewSearcher(version, dbFile, nil, nil) +} + +func NewWithVectorIndex(version *Version, dbFile string, vIndex []byte) (*Searcher, error) { + return NewSearcher(version, dbFile, vIndex, nil) +} + +func NewWithBuffer(version *Version, cBuff []byte) (*Searcher, error) { + return NewSearcher(version, "", nil, cBuff) +} + +func NewSearcher(version *Version, dbFile string, vIndex []byte, cBuff []byte) (*Searcher, error) { var err error // content buff first @@ -58,23 +70,11 @@ func baseNew(version *Version, dbFile string, vIndex []byte, cBuff []byte) (*Sea }, nil } -func NewWithFileOnly(version *Version, dbFile string) (*Searcher, error) { - return baseNew(version, dbFile, nil, nil) -} - -func NewWithVectorIndex(version *Version, dbFile string, vIndex []byte) (*Searcher, error) { - return baseNew(version, dbFile, vIndex, nil) -} - -func NewWithBuffer(version *Version, cBuff []byte) (*Searcher, error) { - return baseNew(version, "", nil, cBuff) -} - func (s *Searcher) Close() { if s.handle != nil { err := s.handle.Close() if err != nil { - return + // do error log here ? } } } diff --git a/binding/golang/xdb/util.go b/binding/golang/xdb/util.go index 50eaf92..4b48113 100644 --- a/binding/golang/xdb/util.go +++ b/binding/golang/xdb/util.go @@ -55,6 +55,33 @@ func IPCompare(ip1, ip2 []byte) int { return bytes.Compare(ip1, ip2) } +func IPAddOne(ip []byte) []byte { + var r = make([]byte, len(ip)) + copy(r, ip) + for i := len(ip) - 1; i >= 0; i-- { + r[i]++ + if r[i] != 0 { // No overflow + break + } + } + + return r +} + +func IPSubOne(ip []byte) []byte { + var r = make([]byte, len(ip)) + copy(r, ip) + for i := len(ip) - 1; i >= 0; i-- { + if r[i] != 0 { // No borrow needed + r[i]-- + break + } + r[i] = 0xFF // borrow from the next byte + } + + return r +} + // Verify if the current Searcher could be used to search the specified xdb file. // Why do we need this check ? // The future features of the xdb impl may cause the current searcher not able to work properly. diff --git a/binding/golang/xdb/version.go b/binding/golang/xdb/version.go index 972dad2..c086772 100644 --- a/binding/golang/xdb/version.go +++ b/binding/golang/xdb/version.go @@ -20,6 +20,13 @@ type Version struct { IPCompare func([]byte, []byte) int } +func (v *Version) String() string { + return fmt.Sprintf( + "{id:%d, name:%s, bytes:%d, segment_index_size:%d}", + v.Id, v.Name, v.Bytes, v.SegmentIndexSize, + ) +} + const ( IPv4VersionNo = 4 IPv6VersionNo = 6 diff --git a/binding/java/ReadMe.md b/binding/java/ReadMe.md index 44af011..8327ad2 100644 --- a/binding/java/ReadMe.md +++ b/binding/java/ReadMe.md @@ -7,13 +7,16 @@ org.lionsoul ip2region - 3.2.1 + 3.2.2 ``` ### 关于查询服务 从 `3.2.0` 版本开始提供了一个双协议兼容且并发安全的 `Ip2Region` 查询服务，**建议优先使用该方式来进行查询调用**，具体使用方式如下： ```java +import org.lionsoul.ip2region.service.Config; +import org.lionsoul.ip2region.service.Ip2Region; + // 1, 创建 v4 的配置：指定缓存策略和 v4 的 xdb 文件路径 final Config v4Config = Config.custom() .setCachePolicy(Config.VIndexCache) // 指定缓存策略: NoCache / VIndexCache / BufferCache diff --git a/binding/java/pom.xml b/binding/java/pom.xml index 0b563d6..35f74b1 100644 --- a/binding/java/pom.xml +++ b/binding/java/pom.xml @@ -4,7 +4,7 @@ org.lionsoul ip2region - 3.2.1 + 3.2.2 jar ip2region diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/Config.java b/binding/java/src/main/java/org/lionsoul/ip2region/service/Config.java similarity index 98% rename from binding/java/src/main/java/org/lionsoul/ip2region/Config.java rename to binding/java/src/main/java/org/lionsoul/ip2region/service/Config.java index 3012ef8..96babb6 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/Config.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/service/Config.java @@ -1,7 +1,7 @@ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. -package org.lionsoul.ip2region; +package org.lionsoul.ip2region.service; import java.io.IOException; diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/ConfigBuilder.java b/binding/java/src/main/java/org/lionsoul/ip2region/service/ConfigBuilder.java similarity index 98% rename from binding/java/src/main/java/org/lionsoul/ip2region/ConfigBuilder.java rename to binding/java/src/main/java/org/lionsoul/ip2region/service/ConfigBuilder.java index d17f1e5..dd838bd 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/ConfigBuilder.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/service/ConfigBuilder.java @@ -2,7 +2,7 @@ // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. -package org.lionsoul.ip2region; +package org.lionsoul.ip2region.service; import java.io.IOException; import java.io.RandomAccessFile; diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/Ip2Region.java b/binding/java/src/main/java/org/lionsoul/ip2region/service/Ip2Region.java similarity index 99% rename from binding/java/src/main/java/org/lionsoul/ip2region/Ip2Region.java rename to binding/java/src/main/java/org/lionsoul/ip2region/service/Ip2Region.java index 03f5de1..a6fc03a 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/Ip2Region.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/service/Ip2Region.java @@ -2,7 +2,7 @@ // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. -package org.lionsoul.ip2region; +package org.lionsoul.ip2region.service; import java.io.IOException; diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/SearcherPool.java b/binding/java/src/main/java/org/lionsoul/ip2region/service/SearcherPool.java similarity index 95% rename from binding/java/src/main/java/org/lionsoul/ip2region/SearcherPool.java rename to binding/java/src/main/java/org/lionsoul/ip2region/service/SearcherPool.java index 46f07da..48c7473 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/SearcherPool.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/service/SearcherPool.java @@ -2,7 +2,7 @@ // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. -package org.lionsoul.ip2region; +package org.lionsoul.ip2region.service; import java.io.IOException; import java.util.Iterator; @@ -67,6 +67,13 @@ public class SearcherPool { return this; } + public int getLoanCount() { + lock.lock(); + int lc = this.loanCount; + lock.unlock(); + return lc; + } + public Searcher borrowSearcher() throws InterruptedException { lock.lock(); try { diff --git a/binding/java/src/test/java/org/lionsoul/ip2region/ConfigTest.java b/binding/java/src/test/java/org/lionsoul/ip2region/service/ConfigTest.java similarity index 97% rename from binding/java/src/test/java/org/lionsoul/ip2region/ConfigTest.java rename to binding/java/src/test/java/org/lionsoul/ip2region/service/ConfigTest.java index bc4496c..56a5b66 100644 --- a/binding/java/src/test/java/org/lionsoul/ip2region/ConfigTest.java +++ b/binding/java/src/test/java/org/lionsoul/ip2region/service/ConfigTest.java @@ -1,4 +1,4 @@ -package org.lionsoul.ip2region; +package org.lionsoul.ip2region.service; import java.io.IOException; import java.security.CodeSource; diff --git a/binding/java/src/test/java/org/lionsoul/ip2region/Ip2RegionTest.java b/binding/java/src/test/java/org/lionsoul/ip2region/service/Ip2RegionTest.java similarity index 92% rename from binding/java/src/test/java/org/lionsoul/ip2region/Ip2RegionTest.java rename to binding/java/src/test/java/org/lionsoul/ip2region/service/Ip2RegionTest.java index cc80273..d0fcab7 100644 --- a/binding/java/src/test/java/org/lionsoul/ip2region/Ip2RegionTest.java +++ b/binding/java/src/test/java/org/lionsoul/ip2region/service/Ip2RegionTest.java @@ -1,9 +1,10 @@ -package org.lionsoul.ip2region; +package org.lionsoul.ip2region.service; import static org.junit.Assert.assertEquals; import java.io.IOException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Test; import org.lionsoul.ip2region.xdb.InetAddressException; @@ -104,15 +105,16 @@ public class Ip2RegionTest { byte[] v4Bytes = Util.parseIP("113.92.157.29"); byte[] v6Bytes = Util.parseIP("240e:3b7:3272:d8d0:db09:c067:8d59:539e"); - final int threads = 50; + final int threads = 100; final Ip2Region ip2Region = Ip2Region.create(v4Config, v6Config); final CountDownLatch latch = new CountDownLatch(threads); - final long startTime = System.currentTimeMillis(); + final AtomicInteger count = new AtomicInteger(0); + final long tStart = System.nanoTime(); for (int i = 0; i < threads; i++) { final Runnable t = new Runnable() { @Override public void run() { - for (int i = 0; i < 2000; i++) { + for (int i = 0; i < 5000; i++) { final byte[] ipBytes = i % 2 == 0 ? v4Bytes : v6Bytes; try { final String region = ip2Region.search(ipBytes); @@ -124,6 +126,8 @@ public class Ip2RegionTest { } catch (InetAddressException | IOException | InterruptedException e) { log.errorf("failed to search(%s): %s", Util.ipToString(ipBytes), e.getMessage()); } + + count.incrementAndGet(); } latch.countDown(); @@ -133,8 +137,8 @@ public class Ip2RegionTest { } latch.await(); - final long costs = System.currentTimeMillis() - startTime; - log.debugf("all search finished in %dms", costs); + final long costs = System.nanoTime() - tStart; + log.debugf("%d searches finished in %dms, avg took: %dµs", count.get(), costs / 1000_000, costs / count.get() / 1000); ip2Region.close(); log.debugf("ip2region closed gracefully"); } diff --git a/binding/java/src/test/java/org/lionsoul/ip2region/SearcherPoolTest.java b/binding/java/src/test/java/org/lionsoul/ip2region/service/SearcherPoolTest.java similarity index 98% rename from binding/java/src/test/java/org/lionsoul/ip2region/SearcherPoolTest.java rename to binding/java/src/test/java/org/lionsoul/ip2region/service/SearcherPoolTest.java index bfeafe0..5d07761 100644 --- a/binding/java/src/test/java/org/lionsoul/ip2region/SearcherPoolTest.java +++ b/binding/java/src/test/java/org/lionsoul/ip2region/service/SearcherPoolTest.java @@ -1,4 +1,4 @@ -package org.lionsoul.ip2region; +package org.lionsoul.ip2region.service; import org.junit.Test; import org.lionsoul.ip2region.xdb.Log;