add lua xdb searcher

binding/lua/ReadMe.md

@@ -1,7 +1,153 @@
-# ip2region lua 查询客户端实现
+# ip2region xdb lua 查询客户端实现
+
+#### 备注：请优先使用 lua_c 扩展 xdb 查询客户端，性能比纯 lua 实现的要快很多！！！
+
 
 # 使用方式
 
+### 完全基于文件的查询
+```lua
+local xdb = require("xdb_searcher")
+
+-- 1、从 db_path 创建基于文件的 xdb 查询对象
+local db_path = "ip2region.xdb file path"
+local searcher, err = xdb.new_with_file_only(db_path)
+if err ~= nil then
+    print(string.format("failed to create searcher: %s", err))
+    return
+end
+
+-- 2、调用查询 API 进行查询
+local ip_str = "1.2.3.4"
+local s_time = xdb.now()
+region, err = searcher:search(ip_str)
+if err ~= nil then
+    print(string.format("failed to search(%s): %s", ip_str, err))
+    return
+end
+
+-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象
+
+print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
+```
+
+### 缓存 `VectorIndex` 索引
+
+如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。
+```lua
+local xdb = require("xdb_searcher")
+
+local db_path = "ip2region.xdb file path"
+
+-- 1、从指定的 db_path 加载 VectorIndex 缓存，把下述的 v_index 对象做成全局变量。
+-- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。
+v_index, err = xdb.load_vector_index(db_path)
+if err ~= nil then
+    print(string.format("failed to load vector index from '%s'", db_path))
+    return
+end
+
+-- 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。
+searcher, err = xdb.new_with_vector_index(db_path, v_index)
+if err ~= nil then
+    print(string.format("failed to create vector index searcher: %s", err))
+    return
+end
+
+-- 3、调用查询 API 
+local ip_str = "1.2.3.4"
+local s_time = xdb.now()
+region, err = searcher:search(ip_str)
+if err ~= nil then
+    print(string.format("failed to search(%s): %s", ip_str, err))
+    return
+end
+
+-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象
+
+print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
+```
+
+### 缓存整个 `xdb` 数据
+
+如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。
+```lua
+local xdb = require("xdb_searcher")
+
+local db_path = "ip2region.xdb file path"
+
+-- 1、从指定的 db_path 加载整个 xdb 到内存。
+-- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。
+content = xdb.load_content(db_path)
+if content == nil then
+    print(string.format("failed to load xdb content from '%s'", db_path))
+    return
+end
+
+-- 2、使用全局的 content 创建带完全基于内存的查询对象。
+searcher, err = xdb.new_with_buffer(content)
+if err ~= nil then
+    print(string.format("failed to create content buffer searcher: %s", err))
+    return
+end
+
+-- 3、调用查询 API 
+local ip_str = "1.2.3.4"
+local s_time = xdb.now()
+region, err = searcher:search(ip_str)
+if err ~= nil then
+    print(string.format("failed to search(%s): %s", ip_str, err))
+    return
+end
+
+-- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。
+-- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。
+
+print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
+```
+
+
 # 查询测试
 
+通过 `search_test.lua` 脚本来进行查询测试：
+```bash
+➜  lua git:(lua_binding) ✗ lua search_test.lua
+lua search_test.lua [command options]
+options:
+ --db string             ip2region binary xdb file path
+ --cache-policy string   cache policy: file/vectorIndex/content
+```
+
+例如：使用默认的 data/ip2region.xdb 进行查询测试：
+```bash
+➜  lua git:(lua_binding) ✗ lua search_test.lua --db=../../data/ip2region.xdb --cache-policy=vectorIndex
+ip2region xdb searcher test program, cachePolicy: vectorIndex
+type 'quit' to exit
+ip2region>> 1.2.3.4
+{region: 美国|0|华盛顿|0|谷歌, io_count: 7, took: 0μs}
+ip2region>> 
+```
+
+输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
+
+
 # bench 测试
+
+通过 `bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能：
+```bash
+➜  lua git:(lua_binding) ✗ lua bench_test.lua 
+lua bench_test.lua [command options]
+options: 
+ --db string             ip2region binary xdb file path
+ --src string            source ip text file path
+ --cache-policy string   cache policy: file/vectorIndex/content
+```
+
+例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试：
+```bash
+➜  lua git:(lua_binding) ✗ lua bench_test.lua --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=vectorIndex
+Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 29.000 s, cost: 7.899 μs/op}
+```
+
+可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。
+@Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。

binding/lua/bench_test.lua

@@ -0,0 +1,175 @@
+-- Copyright 2022 The Ip2Region Authors. All rights reserved.
+-- Use of this source code is governed by a Apache2.0-style
+-- license that can be found in the LICENSE file.
+--
+-- ---
+-- @Author Lion <chenxin619315@gmail.com>
+-- @Date   2022/06/30
+
+-- set the package to load the current xdb_searcher.so
+package.path = "./?.lua" .. package.path
+package.cpath = "./?.so" .. package.cpath
+local xdb = require("xdb_searcher")
+
+function printHelp()
+    print("lua bench_test.lua [command options]")
+    print("options: ")
+    print(" --db string             ip2region binary xdb file path")
+    print(" --src string            source ip text file path")
+    print(" --cache-policy string   cache policy: file/vectorIndex/content")
+end
+
+if #arg < 2 then
+    printHelp(arg)
+    return
+end
+
+-- parser the command line args
+local dbFile, srcFile = "", ""
+local cachePolicy = "vectorIndex"
+for _, r in ipairs(arg) do
+    if string.len(r) < 5 then
+        goto continue
+    end
+
+    if string.sub(r, 1, 2) ~= "--" then
+        goto continue
+    end
+
+    for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do
+        if k == "db" then
+            dbFile = v
+        elseif k == "src" then
+            srcFile = v
+        elseif k == "cache-policy" then
+            cachePolicy = v
+        else
+            print(string.format("undefined option `%s`", r))
+            return
+        end
+
+        -- break the match iterate
+        break
+    end
+
+    -- continue this loop
+    ::continue::
+end
+
+-- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy))
+if string.len(dbFile) < 2 or string.len(srcFile) < 2 then
+    printHelp()
+    return
+end
+
+-- create the searcher based on the cache-policy
+local searcher, v_index, content
+if cachePolicy == "file" then
+    searcher, err = xdb.new_with_file_only(dbFile)
+    if err ~= nil then
+        print(string.format("failed to create searcher: %s", err))
+        return
+    end
+elseif cachePolicy == "vectorIndex" then
+    v_index, err = xdb.load_vector_index(dbFile)
+    if err ~= nil then
+        print(string.format("failed to load vector index: %s", err))
+        return
+    end
+
+    searcher, err = xdb.new_with_vector_index(dbFile, v_index)
+    if err ~= nil then
+        print(string.format("failed to create vector index searcher: %s", err))
+        return
+    end
+elseif cachePolicy == "content" then
+    content, err = xdb.load_content(dbFile)
+    if err ~= nil then
+        print(string.format("failed to load xdb content: %s", err))
+        return
+    end
+
+    searcher, err = xdb.new_with_buffer(content)
+    if err ~= nil then
+        print(string.format("failed to create content buffer searcher: %s", err))
+        return
+    end
+else
+    print(string.format("undefined cache-policy `%s`", cachePolicy))
+    return
+end
+
+-- do the bench test
+local handle = io.open(srcFile, "r")
+if handle == nil then
+    print(string.format("failed to open src text file `%s`", handle))
+    return
+end
+
+local lines = handle:lines()
+local sip_str, eip_str, s_region, region = "", "", "", ""
+local sip, mip, eip, err = 0, 0, 0, 0
+local count, t_time, c_time = 0, 0, 0
+local s_time = xdb.now()
+for l in lines do
+    if string.len(l) < 1 then
+        goto continue
+    end
+
+    for v1, v2, v3 in string.gmatch(l, "([%d%.]+)|([%d%.]+)|([^\n]+)") do
+        -- print(sip_str, eip_str, region)
+        sip_str = v1
+        eip_str = v2
+        s_region = v3
+        break
+    end
+
+    sip, err = xdb.check_ip(sip_str)
+    if err ~= nil then
+        print(string.format("invalid start ip `%s`", sip_str))
+        return
+    end
+
+    eip, err = xdb.check_ip(eip_str)
+    if err ~= nil then
+        print(string.format("invalid end ip `%s`", sip_str))
+        return
+    end
+
+    if sip > eip then
+        print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str))
+        return
+    end
+
+    mip = (sip + eip) >> 1
+    for _, ip in ipairs({sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) do
+        t_time = xdb.now()
+        region, err = searcher:search(ip)
+        c_time = c_time + xdb.now() - t_time
+        if err ~= nil then
+            print(string.format("failed to search ip `%s`", xdb.long2ip(ip)))
+            return
+        end
+
+        -- check the region
+        if region ~= s_region then
+            print(string.format("failed search(%s) with (%s != %s)\n", xdb.long2ip(ip), region, s_region))
+            return
+        end
+
+        count = count + 1
+    end
+
+    ::continue::
+end
+
+-- resource cleanup
+searcher:close()
+
+-- print the stats
+local avg_costs = 0
+if count > 0 then
+    avg_costs = c_time / count
+end
+print(string.format("Bench finished, {cachePolicy: %s, total: %d, took: %.3f s, cost: %.3f μs/op}",
+         cachePolicy, count, (xdb.now() - s_time)/1e6, c_time / count))

binding/lua/search_test.lua

@@ -0,0 +1,137 @@
+-- Copyright 2022 The Ip2Region Authors. All rights reserved.
+-- Use of this source code is governed by a Apache2.0-style
+-- license that can be found in the LICENSE file.
+--
+-- ---
+-- @Author Lion <chenxin619315@gmail.com>
+-- @Date   2022/06/30
+
+-- set the package to load the current xdb_searcher.so
+package.path = "./?.lua" .. package.path
+package.cpath = "./?.so" .. package.cpath
+local xdb = require("xdb_searcher")
+
+function printHelp()
+    print("lua search_test.lua [command options]")
+    print("options: ")
+    print(" --db string             ip2region binary xdb file path")
+    print(" --cache-policy string   cache policy: file/vectorIndex/content")
+end
+
+if #arg < 2 then
+    printHelp(arg)
+    return
+end
+
+-- parser the command line args
+local dbFile = ""
+local cachePolicy = "vectorIndex"
+for _, r in ipairs(arg) do
+    if string.len(r) < 5 then
+        goto continue
+    end
+
+    if string.sub(r, 1, 2) ~= "--" then
+        goto continue
+    end
+
+    for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do
+        if k == "db" then
+            dbFile = v
+        elseif k == "cache-policy" then
+            cachePolicy = v
+        else
+            print(string.format("undefined option `%s`", r))
+            return
+        end
+
+        -- break the match iterate
+        break
+    end
+
+    -- continue this loop
+    ::continue::
+end
+
+-- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy))
+if string.len(dbFile) < 2 then
+    printHelp()
+    return
+end
+
+-- create the searcher based on the cache-policy
+local searcher, v_index, content
+if cachePolicy == "file" then
+    searcher, err = xdb.new_with_file_only(dbFile)
+    if err ~= nil then
+        print(string.format("failed to create searcher: %s", err))
+        return
+    end
+elseif cachePolicy == "vectorIndex" then
+    v_index, err = xdb.load_vector_index(dbFile)
+    if err ~= nil then
+        print(string.format("failed to load vector index: %s", err))
+        return
+    end
+
+    searcher, err = xdb.new_with_vector_index(dbFile, v_index)
+    if err ~= nil then
+        print(string.format("failed to create vector index searcher: %s", err))
+        return
+    end
+elseif cachePolicy == "content" then
+    content, err = xdb.load_content(dbFile)
+    if err ~= nil then
+        print(string.format("failed to load xdb content: %s", err))
+        return
+    end
+
+    searcher, err = xdb.new_with_buffer(content)
+    if err ~= nil then
+        print(string.format("failed to create content buffer searcher: %s", err))
+        return
+    end
+else
+    print(string.format("undefined cache-policy `%s`", cachePolicy))
+    return
+end
+
+-- do the search
+print(string.format([[
+ip2region xdb searcher test program, cachePolicy: %s
+type 'quit' to exit]], cachePolicy))
+local region, err = "", nil
+local ip_int, s_time, c_time =  0, 0, 0
+while ( true ) do
+    io.write("ip2region>> ");
+    io.input(io.stdin);
+    local line = io.read();
+    if (line == nil) then
+        break
+    end
+
+    if ( line == "quit" ) then
+        break
+    end
+
+    ip_int, err = xdb.check_ip(line)
+    if err ~= nil then
+        print(string.format("invalid ip address `%s`", line))
+        goto continue
+    end
+
+    -- do the search
+    s_time = xdb.now()
+    region, err = searcher:search(line)
+    if err ~= nil then
+        print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count()))
+    else
+        c_time = xdb.now() - s_time
+        print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time))
+    end
+
+    ::continue::
+end
+
+-- resource cleanup
+searcher:close()

binding/lua/xdb_searcher.lua

@@ -49,11 +49,11 @@ function newBase(dbPath, vIndex, cBuffer)
         obj.vector_index = vIndex
         obj.handle = io.open(dbPath, "r")
         if obj.handle == nil then
-            error(string.format("failed to open xdb file `%s`", dbPath), 2)
+            return nil, string.format("failed to open xdb file `%s`", dbPath)
         end
     end
 
-    return obj
+    return obj, nil
 end
 
 function _M.new_with_file_only(dbPath)
@@ -84,25 +84,32 @@ function _M:search(ip_src)
         end
     elseif t ~= "number" then
         return "", "invalid number or string ip"
+    else
+        -- use the original value
+        ip = ip_src
     end
 
     -- reset the global counter
+    -- and global resource local cache
     self.io_count = 0
+    local vector_index = self.vector_index
+    local content_buff = self.content_buff
+    local read_data = self.read
 
     -- locate the segment index based on the vector index
     local il0 = (ip >> 24) & 0xFF
     local il1 = (ip >> 16) & 0xFF
     local idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize
     local s_ptr, e_ptr = 0, 0
-    if self.vector_index ~= nil then
+    if vector_index ~= nil then
-        s_ptr = getLong(self.vector_index, idx + 1)
+        s_ptr = getLong(vector_index, idx + 1)
-        e_ptr = getLong(self.vector_index, idx + 5)
+        e_ptr = getLong(vector_index, idx + 5)
-    elseif self.content_buff ~= nil then
+    elseif content_buff ~= nil then
-        s_ptr = getLong(self.content_buff, HeaderInfoLength + idx + 1)
+        s_ptr = getLong(content_buff, HeaderInfoLength + idx + 1)
-        e_ptr = getLong(self.content_buff, HeaderInfoLength + idx + 5)
+        e_ptr = getLong(content_buff, HeaderInfoLength + idx + 5)
     else
         -- load from the file
-        buff, err = self:read(HeaderInfoLength + idx, SegmentIndexSize)
+        buff, err = read_data(self, HeaderInfoLength + idx, SegmentIndexSize)
         if err ~= nil then
             return "", string.format("read buffer: %s", err)
         end
@@ -121,7 +128,7 @@ function _M:search(ip_src)
         p = s_ptr + m * SegmentIndexSize
 
         -- read the segment index
-        buff, err = self:read(p, SegmentIndexSize)
+        buff, err = read_data(self, p, SegmentIndexSize)
         if err ~= nil then
             return "", string.format("read segment index at %d", p)
         end
@@ -148,7 +155,7 @@ function _M:search(ip_src)
     end
 
     -- load and return the region data
-    buff, err = self:read(data_ptr, data_len)
+    buff, err = read_data(self, data_ptr, data_len)
     if err ~= nil then
         return "", string.format("read data at %d:%d", data_ptr, data_len)
     end
@@ -156,22 +163,27 @@ function _M:search(ip_src)
     return buff, nil
 end
 
+
 -- read specified bytes from the specified index
 
 function _M:read(offset, length)
+    -- local cache
+    local content_buff = self.content_buff
+    local handle = self.handle
+
     -- check the in-memory buffer first
-    if self.content_buff ~= nil then
+    if content_buff ~= nil then
-        return string.sub(self.content_buff, offset + 1, offset + 1 + length), nil
+        return string.sub(content_buff, offset + 1, offset + length), nil
     end
 
     -- read from the file
-    local r = self.handle:seek("set", offset)
+    local r = handle:seek("set", offset)
     if r == nil then
         return nil, string.format("seek to offset %d", offset)
     end
 
     self.io_count = self.io_count + 1
-    local buff = self.handle:read(length)
+    local buff = handle:read(length)
     if buff == nil then
         return nil, string.format("read %d bytes", length)
     end
@@ -295,8 +307,9 @@ function _M.long2ip(ip)
     return string.format("%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8 ) & 0xFF, ip & 0xFF)
 end
 
+-- this is a bit weird, but we have to better choice for now
 function _M.now()
-    return 0
+    return os.time() * 1e6
 end
 
 -- End of util functions