add lua xdb searcher
This commit is contained in:
Lion
parent
f25ef646f5
commit
4934080ed0
|
|
@ -1,7 +1,153 @@
|
|||
# ip2region lua 查询客户端实现
|
||||
# ip2region xdb lua 查询客户端实现
|
||||
|
||||
#### 备注：请优先使用 lua_c 扩展 xdb 查询客户端，性能比纯 lua 实现的要快很多！！！
|
||||
|
||||
|
||||
# 使用方式
|
||||
|
||||
### 完全基于文件的查询
|
||||
```lua
|
||||
local xdb = require("xdb_searcher")
|
||||
|
||||
-- 1、从 db_path 创建基于文件的 xdb 查询对象
|
||||
local db_path = "ip2region.xdb file path"
|
||||
local searcher, err = xdb.new_with_file_only(db_path)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create searcher: %s", err))
|
||||
return
|
||||
end
|
||||
|
||||
-- 2、调用查询 API 进行查询
|
||||
local ip_str = "1.2.3.4"
|
||||
local s_time = xdb.now()
|
||||
region, err = searcher:search(ip_str)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||
return
|
||||
end
|
||||
|
||||
-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象
|
||||
|
||||
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
|
||||
```
|
||||
|
||||
### 缓存 `VectorIndex` 索引
|
||||
|
||||
如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。
|
||||
```lua
|
||||
local xdb = require("xdb_searcher")
|
||||
|
||||
local db_path = "ip2region.xdb file path"
|
||||
|
||||
-- 1、从指定的 db_path 加载 VectorIndex 缓存，把下述的 v_index 对象做成全局变量。
|
||||
-- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。
|
||||
v_index, err = xdb.load_vector_index(db_path)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to load vector index from '%s'", db_path))
|
||||
return
|
||||
end
|
||||
|
||||
-- 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。
|
||||
searcher, err = xdb.new_with_vector_index(db_path, v_index)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create vector index searcher: %s", err))
|
||||
return
|
||||
end
|
||||
|
||||
-- 3、调用查询 API
|
||||
local ip_str = "1.2.3.4"
|
||||
local s_time = xdb.now()
|
||||
region, err = searcher:search(ip_str)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||
return
|
||||
end
|
||||
|
||||
-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象
|
||||
|
||||
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
|
||||
```
|
||||
|
||||
### 缓存整个 `xdb` 数据
|
||||
|
||||
如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。
|
||||
```lua
|
||||
local xdb = require("xdb_searcher")
|
||||
|
||||
local db_path = "ip2region.xdb file path"
|
||||
|
||||
-- 1、从指定的 db_path 加载整个 xdb 到内存。
|
||||
-- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。
|
||||
content = xdb.load_content(db_path)
|
||||
if content == nil then
|
||||
print(string.format("failed to load xdb content from '%s'", db_path))
|
||||
return
|
||||
end
|
||||
|
||||
-- 2、使用全局的 content 创建带完全基于内存的查询对象。
|
||||
searcher, err = xdb.new_with_buffer(content)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create content buffer searcher: %s", err))
|
||||
return
|
||||
end
|
||||
|
||||
-- 3、调用查询 API
|
||||
local ip_str = "1.2.3.4"
|
||||
local s_time = xdb.now()
|
||||
region, err = searcher:search(ip_str)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to search(%s): %s", ip_str, err))
|
||||
return
|
||||
end
|
||||
|
||||
-- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。
|
||||
-- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。
|
||||
|
||||
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
|
||||
```
|
||||
|
||||
|
||||
# 查询测试
|
||||
|
||||
通过 `search_test.lua` 脚本来进行查询测试：
|
||||
```bash
|
||||
➜ lua git:(lua_binding) ✗ lua search_test.lua
|
||||
lua search_test.lua [command options]
|
||||
options:
|
||||
--db string ip2region binary xdb file path
|
||||
--cache-policy string cache policy: file/vectorIndex/content
|
||||
```
|
||||
|
||||
例如：使用默认的 data/ip2region.xdb 进行查询测试：
|
||||
```bash
|
||||
➜ lua git:(lua_binding) ✗ lua search_test.lua --db=../../data/ip2region.xdb --cache-policy=vectorIndex
|
||||
ip2region xdb searcher test program, cachePolicy: vectorIndex
|
||||
type 'quit' to exit
|
||||
ip2region>> 1.2.3.4
|
||||
{region: 美国|0|华盛顿|0|谷歌, io_count: 7, took: 0μs}
|
||||
ip2region>>
|
||||
```
|
||||
|
||||
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
|
||||
|
||||
|
||||
# bench 测试
|
||||
|
||||
通过 `bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能：
|
||||
```bash
|
||||
➜ lua git:(lua_binding) ✗ lua bench_test.lua
|
||||
lua bench_test.lua [command options]
|
||||
options:
|
||||
--db string ip2region binary xdb file path
|
||||
--src string source ip text file path
|
||||
--cache-policy string cache policy: file/vectorIndex/content
|
||||
```
|
||||
|
||||
例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试：
|
||||
```bash
|
||||
➜ lua git:(lua_binding) ✗ lua bench_test.lua --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=vectorIndex
|
||||
Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 29.000 s, cost: 7.899 μs/op}
|
||||
```
|
||||
|
||||
可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。
|
||||
@Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。
|
||||
|
|
|
|||
|
|
@ -0,0 +1,175 @@
|
|||
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||
-- Use of this source code is governed by a Apache2.0-style
|
||||
-- license that can be found in the LICENSE file.
|
||||
--
|
||||
-- ---
|
||||
-- @Author Lion <chenxin619315@gmail.com>
|
||||
-- @Date 2022/06/30
|
||||
|
||||
-- set the package to load the current xdb_searcher.so
|
||||
package.path = "./?.lua" .. package.path
|
||||
package.cpath = "./?.so" .. package.cpath
|
||||
local xdb = require("xdb_searcher")
|
||||
|
||||
function printHelp()
|
||||
print("lua bench_test.lua [command options]")
|
||||
print("options: ")
|
||||
print(" --db string ip2region binary xdb file path")
|
||||
print(" --src string source ip text file path")
|
||||
print(" --cache-policy string cache policy: file/vectorIndex/content")
|
||||
end
|
||||
|
||||
if #arg < 2 then
|
||||
printHelp(arg)
|
||||
return
|
||||
end
|
||||
|
||||
-- parser the command line args
|
||||
local dbFile, srcFile = "", ""
|
||||
local cachePolicy = "vectorIndex"
|
||||
for _, r in ipairs(arg) do
|
||||
if string.len(r) < 5 then
|
||||
goto continue
|
||||
end
|
||||
|
||||
if string.sub(r, 1, 2) ~= "--" then
|
||||
goto continue
|
||||
end
|
||||
|
||||
for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do
|
||||
if k == "db" then
|
||||
dbFile = v
|
||||
elseif k == "src" then
|
||||
srcFile = v
|
||||
elseif k == "cache-policy" then
|
||||
cachePolicy = v
|
||||
else
|
||||
print(string.format("undefined option `%s`", r))
|
||||
return
|
||||
end
|
||||
|
||||
-- break the match iterate
|
||||
break
|
||||
end
|
||||
|
||||
-- continue this loop
|
||||
::continue::
|
||||
end
|
||||
|
||||
-- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy))
|
||||
if string.len(dbFile) < 2 or string.len(srcFile) < 2 then
|
||||
printHelp()
|
||||
return
|
||||
end
|
||||
|
||||
-- create the searcher based on the cache-policy
|
||||
local searcher, v_index, content
|
||||
if cachePolicy == "file" then
|
||||
searcher, err = xdb.new_with_file_only(dbFile)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create searcher: %s", err))
|
||||
return
|
||||
end
|
||||
elseif cachePolicy == "vectorIndex" then
|
||||
v_index, err = xdb.load_vector_index(dbFile)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to load vector index: %s", err))
|
||||
return
|
||||
end
|
||||
|
||||
searcher, err = xdb.new_with_vector_index(dbFile, v_index)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create vector index searcher: %s", err))
|
||||
return
|
||||
end
|
||||
elseif cachePolicy == "content" then
|
||||
content, err = xdb.load_content(dbFile)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to load xdb content: %s", err))
|
||||
return
|
||||
end
|
||||
|
||||
searcher, err = xdb.new_with_buffer(content)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create content buffer searcher: %s", err))
|
||||
return
|
||||
end
|
||||
else
|
||||
print(string.format("undefined cache-policy `%s`", cachePolicy))
|
||||
return
|
||||
end
|
||||
|
||||
-- do the bench test
|
||||
local handle = io.open(srcFile, "r")
|
||||
if handle == nil then
|
||||
print(string.format("failed to open src text file `%s`", handle))
|
||||
return
|
||||
end
|
||||
|
||||
local lines = handle:lines()
|
||||
local sip_str, eip_str, s_region, region = "", "", "", ""
|
||||
local sip, mip, eip, err = 0, 0, 0, 0
|
||||
local count, t_time, c_time = 0, 0, 0
|
||||
local s_time = xdb.now()
|
||||
for l in lines do
|
||||
if string.len(l) < 1 then
|
||||
goto continue
|
||||
end
|
||||
|
||||
for v1, v2, v3 in string.gmatch(l, "([%d%.]+)|([%d%.]+)|([^\n]+)") do
|
||||
-- print(sip_str, eip_str, region)
|
||||
sip_str = v1
|
||||
eip_str = v2
|
||||
s_region = v3
|
||||
break
|
||||
end
|
||||
|
||||
sip, err = xdb.check_ip(sip_str)
|
||||
if err ~= nil then
|
||||
print(string.format("invalid start ip `%s`", sip_str))
|
||||
return
|
||||
end
|
||||
|
||||
eip, err = xdb.check_ip(eip_str)
|
||||
if err ~= nil then
|
||||
print(string.format("invalid end ip `%s`", sip_str))
|
||||
return
|
||||
end
|
||||
|
||||
if sip > eip then
|
||||
print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str))
|
||||
return
|
||||
end
|
||||
|
||||
mip = (sip + eip) >> 1
|
||||
for _, ip in ipairs({sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) do
|
||||
t_time = xdb.now()
|
||||
region, err = searcher:search(ip)
|
||||
c_time = c_time + xdb.now() - t_time
|
||||
if err ~= nil then
|
||||
print(string.format("failed to search ip `%s`", xdb.long2ip(ip)))
|
||||
return
|
||||
end
|
||||
|
||||
-- check the region
|
||||
if region ~= s_region then
|
||||
print(string.format("failed search(%s) with (%s != %s)\n", xdb.long2ip(ip), region, s_region))
|
||||
return
|
||||
end
|
||||
|
||||
count = count + 1
|
||||
end
|
||||
|
||||
::continue::
|
||||
end
|
||||
|
||||
-- resource cleanup
|
||||
searcher:close()
|
||||
|
||||
-- print the stats
|
||||
local avg_costs = 0
|
||||
if count > 0 then
|
||||
avg_costs = c_time / count
|
||||
end
|
||||
print(string.format("Bench finished, {cachePolicy: %s, total: %d, took: %.3f s, cost: %.3f μs/op}",
|
||||
cachePolicy, count, (xdb.now() - s_time)/1e6, c_time / count))
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||
-- Use of this source code is governed by a Apache2.0-style
|
||||
-- license that can be found in the LICENSE file.
|
||||
--
|
||||
-- ---
|
||||
-- @Author Lion <chenxin619315@gmail.com>
|
||||
-- @Date 2022/06/30
|
||||
|
||||
-- set the package to load the current xdb_searcher.so
|
||||
package.path = "./?.lua" .. package.path
|
||||
package.cpath = "./?.so" .. package.cpath
|
||||
local xdb = require("xdb_searcher")
|
||||
|
||||
function printHelp()
|
||||
print("lua search_test.lua [command options]")
|
||||
print("options: ")
|
||||
print(" --db string ip2region binary xdb file path")
|
||||
print(" --cache-policy string cache policy: file/vectorIndex/content")
|
||||
end
|
||||
|
||||
if #arg < 2 then
|
||||
printHelp(arg)
|
||||
return
|
||||
end
|
||||
|
||||
-- parser the command line args
|
||||
local dbFile = ""
|
||||
local cachePolicy = "vectorIndex"
|
||||
for _, r in ipairs(arg) do
|
||||
if string.len(r) < 5 then
|
||||
goto continue
|
||||
end
|
||||
|
||||
if string.sub(r, 1, 2) ~= "--" then
|
||||
goto continue
|
||||
end
|
||||
|
||||
for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do
|
||||
if k == "db" then
|
||||
dbFile = v
|
||||
elseif k == "cache-policy" then
|
||||
cachePolicy = v
|
||||
else
|
||||
print(string.format("undefined option `%s`", r))
|
||||
return
|
||||
end
|
||||
|
||||
-- break the match iterate
|
||||
break
|
||||
end
|
||||
|
||||
-- continue this loop
|
||||
::continue::
|
||||
end
|
||||
|
||||
-- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy))
|
||||
if string.len(dbFile) < 2 then
|
||||
printHelp()
|
||||
return
|
||||
end
|
||||
|
||||
-- create the searcher based on the cache-policy
|
||||
local searcher, v_index, content
|
||||
if cachePolicy == "file" then
|
||||
searcher, err = xdb.new_with_file_only(dbFile)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create searcher: %s", err))
|
||||
return
|
||||
end
|
||||
elseif cachePolicy == "vectorIndex" then
|
||||
v_index, err = xdb.load_vector_index(dbFile)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to load vector index: %s", err))
|
||||
return
|
||||
end
|
||||
|
||||
searcher, err = xdb.new_with_vector_index(dbFile, v_index)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create vector index searcher: %s", err))
|
||||
return
|
||||
end
|
||||
elseif cachePolicy == "content" then
|
||||
content, err = xdb.load_content(dbFile)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to load xdb content: %s", err))
|
||||
return
|
||||
end
|
||||
|
||||
searcher, err = xdb.new_with_buffer(content)
|
||||
if err ~= nil then
|
||||
print(string.format("failed to create content buffer searcher: %s", err))
|
||||
return
|
||||
end
|
||||
else
|
||||
print(string.format("undefined cache-policy `%s`", cachePolicy))
|
||||
return
|
||||
end
|
||||
|
||||
-- do the search
|
||||
print(string.format([[
|
||||
ip2region xdb searcher test program, cachePolicy: %s
|
||||
type 'quit' to exit]], cachePolicy))
|
||||
local region, err = "", nil
|
||||
local ip_int, s_time, c_time = 0, 0, 0
|
||||
while ( true ) do
|
||||
io.write("ip2region>> ");
|
||||
io.input(io.stdin);
|
||||
local line = io.read();
|
||||
if (line == nil) then
|
||||
break
|
||||
end
|
||||
|
||||
if ( line == "quit" ) then
|
||||
break
|
||||
end
|
||||
|
||||
ip_int, err = xdb.check_ip(line)
|
||||
if err ~= nil then
|
||||
print(string.format("invalid ip address `%s`", line))
|
||||
goto continue
|
||||
end
|
||||
|
||||
-- do the search
|
||||
s_time = xdb.now()
|
||||
region, err = searcher:search(line)
|
||||
if err ~= nil then
|
||||
print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count()))
|
||||
else
|
||||
c_time = xdb.now() - s_time
|
||||
print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time))
|
||||
end
|
||||
|
||||
::continue::
|
||||
end
|
||||
|
||||
-- resource cleanup
|
||||
searcher:close()
|
||||
|
|
@ -49,11 +49,11 @@ function newBase(dbPath, vIndex, cBuffer)
|
|||
obj.vector_index = vIndex
|
||||
obj.handle = io.open(dbPath, "r")
|
||||
if obj.handle == nil then
|
||||
error(string.format("failed to open xdb file `%s`", dbPath), 2)
|
||||
return nil, string.format("failed to open xdb file `%s`", dbPath)
|
||||
end
|
||||
end
|
||||
|
||||
return obj
|
||||
return obj, nil
|
||||
end
|
||||
|
||||
function _M.new_with_file_only(dbPath)
|
||||
|
|
@ -84,25 +84,32 @@ function _M:search(ip_src)
|
|||
end
|
||||
elseif t ~= "number" then
|
||||
return "", "invalid number or string ip"
|
||||
else
|
||||
-- use the original value
|
||||
ip = ip_src
|
||||
end
|
||||
|
||||
-- reset the global counter
|
||||
-- and global resource local cache
|
||||
self.io_count = 0
|
||||
local vector_index = self.vector_index
|
||||
local content_buff = self.content_buff
|
||||
local read_data = self.read
|
||||
|
||||
-- locate the segment index based on the vector index
|
||||
local il0 = (ip >> 24) & 0xFF
|
||||
local il1 = (ip >> 16) & 0xFF
|
||||
local idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize
|
||||
local s_ptr, e_ptr = 0, 0
|
||||
if self.vector_index ~= nil then
|
||||
s_ptr = getLong(self.vector_index, idx + 1)
|
||||
e_ptr = getLong(self.vector_index, idx + 5)
|
||||
elseif self.content_buff ~= nil then
|
||||
s_ptr = getLong(self.content_buff, HeaderInfoLength + idx + 1)
|
||||
e_ptr = getLong(self.content_buff, HeaderInfoLength + idx + 5)
|
||||
if vector_index ~= nil then
|
||||
s_ptr = getLong(vector_index, idx + 1)
|
||||
e_ptr = getLong(vector_index, idx + 5)
|
||||
elseif content_buff ~= nil then
|
||||
s_ptr = getLong(content_buff, HeaderInfoLength + idx + 1)
|
||||
e_ptr = getLong(content_buff, HeaderInfoLength + idx + 5)
|
||||
else
|
||||
-- load from the file
|
||||
buff, err = self:read(HeaderInfoLength + idx, SegmentIndexSize)
|
||||
buff, err = read_data(self, HeaderInfoLength + idx, SegmentIndexSize)
|
||||
if err ~= nil then
|
||||
return "", string.format("read buffer: %s", err)
|
||||
end
|
||||
|
|
@ -121,7 +128,7 @@ function _M:search(ip_src)
|
|||
p = s_ptr + m * SegmentIndexSize
|
||||
|
||||
-- read the segment index
|
||||
buff, err = self:read(p, SegmentIndexSize)
|
||||
buff, err = read_data(self, p, SegmentIndexSize)
|
||||
if err ~= nil then
|
||||
return "", string.format("read segment index at %d", p)
|
||||
end
|
||||
|
|
@ -148,7 +155,7 @@ function _M:search(ip_src)
|
|||
end
|
||||
|
||||
-- load and return the region data
|
||||
buff, err = self:read(data_ptr, data_len)
|
||||
buff, err = read_data(self, data_ptr, data_len)
|
||||
if err ~= nil then
|
||||
return "", string.format("read data at %d:%d", data_ptr, data_len)
|
||||
end
|
||||
|
|
@ -156,22 +163,27 @@ function _M:search(ip_src)
|
|||
return buff, nil
|
||||
end
|
||||
|
||||
|
||||
-- read specified bytes from the specified index
|
||||
|
||||
function _M:read(offset, length)
|
||||
-- local cache
|
||||
local content_buff = self.content_buff
|
||||
local handle = self.handle
|
||||
|
||||
-- check the in-memory buffer first
|
||||
if self.content_buff ~= nil then
|
||||
return string.sub(self.content_buff, offset + 1, offset + 1 + length), nil
|
||||
if content_buff ~= nil then
|
||||
return string.sub(content_buff, offset + 1, offset + length), nil
|
||||
end
|
||||
|
||||
-- read from the file
|
||||
local r = self.handle:seek("set", offset)
|
||||
local r = handle:seek("set", offset)
|
||||
if r == nil then
|
||||
return nil, string.format("seek to offset %d", offset)
|
||||
end
|
||||
|
||||
self.io_count = self.io_count + 1
|
||||
local buff = self.handle:read(length)
|
||||
local buff = handle:read(length)
|
||||
if buff == nil then
|
||||
return nil, string.format("read %d bytes", length)
|
||||
end
|
||||
|
|
@ -295,8 +307,9 @@ function _M.long2ip(ip)
|
|||
return string.format("%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8 ) & 0xFF, ip & 0xFF)
|
||||
end
|
||||
|
||||
-- this is a bit weird, but we have to better choice for now
|
||||
function _M.now()
|
||||
return 0
|
||||
return os.time() * 1e6
|
||||
end
|
||||
|
||||
-- End of util functions
|
||||
|
|
|
|||
Loading…
Reference in New Issue