From 08e11220a5af357524723b5f607ef7079760ea6e Mon Sep 17 00:00:00 2001 From: Lion Date: Wed, 6 Jul 2022 12:01:16 +0800 Subject: [PATCH] add lua xdb searcher --- binding/lua/ReadMe.md | 148 ++++++++++++++++++++++++++++- binding/lua/bench_test.lua | 175 +++++++++++++++++++++++++++++++++++ binding/lua/search_test.lua | 137 +++++++++++++++++++++++++++ binding/lua/xdb_searcher.lua | 45 +++++---- 4 files changed, 488 insertions(+), 17 deletions(-) create mode 100644 binding/lua/bench_test.lua create mode 100644 binding/lua/search_test.lua diff --git a/binding/lua/ReadMe.md b/binding/lua/ReadMe.md index 1b46687..c3a97a3 100644 --- a/binding/lua/ReadMe.md +++ b/binding/lua/ReadMe.md @@ -1,7 +1,153 @@ -# ip2region lua 查询客户端实现 +# ip2region xdb lua 查询客户端实现 + +#### 备注：请优先使用 lua_c 扩展 xdb 查询客户端，性能比纯 lua 实现的要快很多！！！ + # 使用方式 +### 完全基于文件的查询 +```lua +local xdb = require("xdb_searcher") + +-- 1、从 db_path 创建基于文件的 xdb 查询对象 +local db_path = "ip2region.xdb file path" +local searcher, err = xdb.new_with_file_only(db_path) +if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return +end + +-- 2、调用查询 API 进行查询 +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + +### 缓存 `VectorIndex` 索引 + +如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。 +```lua +local xdb = require("xdb_searcher") + +local db_path = "ip2region.xdb file path" + +-- 1、从指定的 db_path 加载 VectorIndex 缓存，把下述的 v_index 对象做成全局变量。 +-- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。 +v_index, err = xdb.load_vector_index(db_path) +if err ~= nil then + print(string.format("failed to load vector index from '%s'", db_path)) + return +end + +-- 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。 +searcher, err = xdb.new_with_vector_index(db_path, v_index) +if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return +end + +-- 3、调用查询 API +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + +### 缓存整个 `xdb` 数据 + +如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。 +```lua +local xdb = require("xdb_searcher") + +local db_path = "ip2region.xdb file path" + +-- 1、从指定的 db_path 加载整个 xdb 到内存。 +-- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。 +content = xdb.load_content(db_path) +if content == nil then + print(string.format("failed to load xdb content from '%s'", db_path)) + return +end + +-- 2、使用全局的 content 创建带完全基于内存的查询对象。 +searcher, err = xdb.new_with_buffer(content) +if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return +end + +-- 3、调用查询 API +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。 +-- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + + # 查询测试 +通过 `search_test.lua` 脚本来进行查询测试： +```bash +➜ lua git:(lua_binding) ✗ lua search_test.lua +lua search_test.lua [command options] +options: + --db string ip2region binary xdb file path + --cache-policy string cache policy: file/vectorIndex/content +``` + +例如：使用默认的 data/ip2region.xdb 进行查询测试： +```bash +➜ lua git:(lua_binding) ✗ lua search_test.lua --db=../../data/ip2region.xdb --cache-policy=vectorIndex +ip2region xdb searcher test program, cachePolicy: vectorIndex +type 'quit' to exit +ip2region>> 1.2.3.4 +{region: 美国|0|华盛顿|0|谷歌, io_count: 7, took: 0μs} +ip2region>> +``` + +输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。 + + # bench 测试 + +通过 `bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能： +```bash +➜ lua git:(lua_binding) ✗ lua bench_test.lua +lua bench_test.lua [command options] +options: + --db string ip2region binary xdb file path + --src string source ip text file path + --cache-policy string cache policy: file/vectorIndex/content +``` + +例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试： +```bash +➜ lua git:(lua_binding) ✗ lua bench_test.lua --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=vectorIndex +Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 29.000 s, cost: 7.899 μs/op} +``` + +可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。 +@Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。 diff --git a/binding/lua/bench_test.lua b/binding/lua/bench_test.lua new file mode 100644 index 0000000..91c9010 --- /dev/null +++ b/binding/lua/bench_test.lua @@ -0,0 +1,175 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/06/30 + +-- set the package to load the current xdb_searcher.so +package.path = "./?.lua" .. package.path +package.cpath = "./?.so" .. package.cpath +local xdb = require("xdb_searcher") + +function printHelp() + print("lua bench_test.lua [command options]") + print("options: ") + print(" --db string ip2region binary xdb file path") + print(" --src string source ip text file path") + print(" --cache-policy string cache policy: file/vectorIndex/content") +end + +if #arg < 2 then + printHelp(arg) + return +end + +-- parser the command line args +local dbFile, srcFile = "", "" +local cachePolicy = "vectorIndex" +for _, r in ipairs(arg) do + if string.len(r) < 5 then + goto continue + end + + if string.sub(r, 1, 2) ~= "--" then + goto continue + end + + for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do + if k == "db" then + dbFile = v + elseif k == "src" then + srcFile = v + elseif k == "cache-policy" then + cachePolicy = v + else + print(string.format("undefined option `%s`", r)) + return + end + + -- break the match iterate + break + end + + -- continue this loop + ::continue:: +end + +-- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy)) +if string.len(dbFile) < 2 or string.len(srcFile) < 2 then + printHelp() + return +end + +-- create the searcher based on the cache-policy +local searcher, v_index, content +if cachePolicy == "file" then + searcher, err = xdb.new_with_file_only(dbFile) + if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return + end +elseif cachePolicy == "vectorIndex" then + v_index, err = xdb.load_vector_index(dbFile) + if err ~= nil then + print(string.format("failed to load vector index: %s", err)) + return + end + + searcher, err = xdb.new_with_vector_index(dbFile, v_index) + if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return + end +elseif cachePolicy == "content" then + content, err = xdb.load_content(dbFile) + if err ~= nil then + print(string.format("failed to load xdb content: %s", err)) + return + end + + searcher, err = xdb.new_with_buffer(content) + if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return + end +else + print(string.format("undefined cache-policy `%s`", cachePolicy)) + return +end + +-- do the bench test +local handle = io.open(srcFile, "r") +if handle == nil then + print(string.format("failed to open src text file `%s`", handle)) + return +end + +local lines = handle:lines() +local sip_str, eip_str, s_region, region = "", "", "", "" +local sip, mip, eip, err = 0, 0, 0, 0 +local count, t_time, c_time = 0, 0, 0 +local s_time = xdb.now() +for l in lines do + if string.len(l) < 1 then + goto continue + end + + for v1, v2, v3 in string.gmatch(l, "([%d%.]+)|([%d%.]+)|([^\n]+)") do + -- print(sip_str, eip_str, region) + sip_str = v1 + eip_str = v2 + s_region = v3 + break + end + + sip, err = xdb.check_ip(sip_str) + if err ~= nil then + print(string.format("invalid start ip `%s`", sip_str)) + return + end + + eip, err = xdb.check_ip(eip_str) + if err ~= nil then + print(string.format("invalid end ip `%s`", sip_str)) + return + end + + if sip > eip then + print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str)) + return + end + + mip = (sip + eip) >> 1 + for _, ip in ipairs({sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) do + t_time = xdb.now() + region, err = searcher:search(ip) + c_time = c_time + xdb.now() - t_time + if err ~= nil then + print(string.format("failed to search ip `%s`", xdb.long2ip(ip))) + return + end + + -- check the region + if region ~= s_region then + print(string.format("failed search(%s) with (%s != %s)\n", xdb.long2ip(ip), region, s_region)) + return + end + + count = count + 1 + end + + ::continue:: +end + +-- resource cleanup +searcher:close() + +-- print the stats +local avg_costs = 0 +if count > 0 then + avg_costs = c_time / count +end +print(string.format("Bench finished, {cachePolicy: %s, total: %d, took: %.3f s, cost: %.3f μs/op}", + cachePolicy, count, (xdb.now() - s_time)/1e6, c_time / count)) diff --git a/binding/lua/search_test.lua b/binding/lua/search_test.lua new file mode 100644 index 0000000..24363a5 --- /dev/null +++ b/binding/lua/search_test.lua @@ -0,0 +1,137 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/06/30 + +-- set the package to load the current xdb_searcher.so +package.path = "./?.lua" .. package.path +package.cpath = "./?.so" .. package.cpath +local xdb = require("xdb_searcher") + +function printHelp() + print("lua search_test.lua [command options]") + print("options: ") + print(" --db string ip2region binary xdb file path") + print(" --cache-policy string cache policy: file/vectorIndex/content") +end + +if #arg < 2 then + printHelp(arg) + return +end + +-- parser the command line args +local dbFile = "" +local cachePolicy = "vectorIndex" +for _, r in ipairs(arg) do + if string.len(r) < 5 then + goto continue + end + + if string.sub(r, 1, 2) ~= "--" then + goto continue + end + + for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do + if k == "db" then + dbFile = v + elseif k == "cache-policy" then + cachePolicy = v + else + print(string.format("undefined option `%s`", r)) + return + end + + -- break the match iterate + break + end + + -- continue this loop + ::continue:: +end + +-- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy)) +if string.len(dbFile) < 2 then + printHelp() + return +end + +-- create the searcher based on the cache-policy +local searcher, v_index, content +if cachePolicy == "file" then + searcher, err = xdb.new_with_file_only(dbFile) + if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return + end +elseif cachePolicy == "vectorIndex" then + v_index, err = xdb.load_vector_index(dbFile) + if err ~= nil then + print(string.format("failed to load vector index: %s", err)) + return + end + + searcher, err = xdb.new_with_vector_index(dbFile, v_index) + if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return + end +elseif cachePolicy == "content" then + content, err = xdb.load_content(dbFile) + if err ~= nil then + print(string.format("failed to load xdb content: %s", err)) + return + end + + searcher, err = xdb.new_with_buffer(content) + if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return + end +else + print(string.format("undefined cache-policy `%s`", cachePolicy)) + return +end + +-- do the search +print(string.format([[ +ip2region xdb searcher test program, cachePolicy: %s +type 'quit' to exit]], cachePolicy)) +local region, err = "", nil +local ip_int, s_time, c_time = 0, 0, 0 +while ( true ) do + io.write("ip2region>> "); + io.input(io.stdin); + local line = io.read(); + if (line == nil) then + break + end + + if ( line == "quit" ) then + break + end + + ip_int, err = xdb.check_ip(line) + if err ~= nil then + print(string.format("invalid ip address `%s`", line)) + goto continue + end + + -- do the search + s_time = xdb.now() + region, err = searcher:search(line) + if err ~= nil then + print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count())) + else + c_time = xdb.now() - s_time + print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time)) + end + + ::continue:: +end + +-- resource cleanup +searcher:close() \ No newline at end of file diff --git a/binding/lua/xdb_searcher.lua b/binding/lua/xdb_searcher.lua index 0e3a3e4..289b414 100644 --- a/binding/lua/xdb_searcher.lua +++ b/binding/lua/xdb_searcher.lua @@ -49,11 +49,11 @@ function newBase(dbPath, vIndex, cBuffer) obj.vector_index = vIndex obj.handle = io.open(dbPath, "r") if obj.handle == nil then - error(string.format("failed to open xdb file `%s`", dbPath), 2) + return nil, string.format("failed to open xdb file `%s`", dbPath) end end - return obj + return obj, nil end function _M.new_with_file_only(dbPath) @@ -84,25 +84,32 @@ function _M:search(ip_src) end elseif t ~= "number" then return "", "invalid number or string ip" + else + -- use the original value + ip = ip_src end -- reset the global counter + -- and global resource local cache self.io_count = 0 + local vector_index = self.vector_index + local content_buff = self.content_buff + local read_data = self.read -- locate the segment index based on the vector index local il0 = (ip >> 24) & 0xFF local il1 = (ip >> 16) & 0xFF local idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize local s_ptr, e_ptr = 0, 0 - if self.vector_index ~= nil then - s_ptr = getLong(self.vector_index, idx + 1) - e_ptr = getLong(self.vector_index, idx + 5) - elseif self.content_buff ~= nil then - s_ptr = getLong(self.content_buff, HeaderInfoLength + idx + 1) - e_ptr = getLong(self.content_buff, HeaderInfoLength + idx + 5) + if vector_index ~= nil then + s_ptr = getLong(vector_index, idx + 1) + e_ptr = getLong(vector_index, idx + 5) + elseif content_buff ~= nil then + s_ptr = getLong(content_buff, HeaderInfoLength + idx + 1) + e_ptr = getLong(content_buff, HeaderInfoLength + idx + 5) else -- load from the file - buff, err = self:read(HeaderInfoLength + idx, SegmentIndexSize) + buff, err = read_data(self, HeaderInfoLength + idx, SegmentIndexSize) if err ~= nil then return "", string.format("read buffer: %s", err) end @@ -121,7 +128,7 @@ function _M:search(ip_src) p = s_ptr + m * SegmentIndexSize -- read the segment index - buff, err = self:read(p, SegmentIndexSize) + buff, err = read_data(self, p, SegmentIndexSize) if err ~= nil then return "", string.format("read segment index at %d", p) end @@ -148,7 +155,7 @@ function _M:search(ip_src) end -- load and return the region data - buff, err = self:read(data_ptr, data_len) + buff, err = read_data(self, data_ptr, data_len) if err ~= nil then return "", string.format("read data at %d:%d", data_ptr, data_len) end @@ -156,22 +163,27 @@ function _M:search(ip_src) return buff, nil end + -- read specified bytes from the specified index function _M:read(offset, length) + -- local cache + local content_buff = self.content_buff + local handle = self.handle + -- check the in-memory buffer first - if self.content_buff ~= nil then - return string.sub(self.content_buff, offset + 1, offset + 1 + length), nil + if content_buff ~= nil then + return string.sub(content_buff, offset + 1, offset + length), nil end -- read from the file - local r = self.handle:seek("set", offset) + local r = handle:seek("set", offset) if r == nil then return nil, string.format("seek to offset %d", offset) end self.io_count = self.io_count + 1 - local buff = self.handle:read(length) + local buff = handle:read(length) if buff == nil then return nil, string.format("read %d bytes", length) end @@ -295,8 +307,9 @@ function _M.long2ip(ip) return string.format("%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8 ) & 0xFF, ip & 0xFF) end +-- this is a bit weird, but we have to better choice for now function _M.now() - return 0 + return os.time() * 1e6 end -- End of util functions