xdb lua searcher framework

binding/lua/util_test.lua

@@ -0,0 +1,102 @@
+-- Copyright 2022 The Ip2Region Authors. All rights reserved.
+-- Use of this source code is governed by a Apache2.0-style
+-- license that can be found in the LICENSE file.
+--
+-- ---
+-- @Author Lion <chenxin619315@gmail.com>
+-- @Date   2022/07/05
+
+-- set the package path
+package.path = "./?.lua"
+package.cpath = "./?.so"
+
+local xdb = require("xdb_searcher")
+
+---- ip checking testing
+print("--- testing check_ip and long2ip ... ")
+local ip_list = {
+    "1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0",
+    "256.7.12.9", "12.56.78.320", "32.12.45.192", "222.221.220.219",
+    "192.168.1.101 ", "132.96.12.98a", "x23.12.2.12"
+}
+
+local s_time = xdb.now()
+for _, ip_src in ipairs(ip_list) do
+    ip, err = xdb.check_ip(ip_src)
+    if err ~= nil then
+        print(string.format("invalid ip address `%s`: %s", ip_src, err))
+    else
+        ip_dst = xdb.long2ip(ip)
+        io.write(string.format("long(%-15s)=%10d, long2ip(%-10d)=%-15s", ip_src, ip, ip, ip_dst))
+        if ip_src ~= ip_dst then
+            print(" --[Failed]")
+        else
+            print(" --[Ok]")
+        end
+    end
+end
+
+---- buffer loading test
+print("\n--- testing load header ... ")
+header, err = xdb.load_header("../../data/ip2region.xdb")
+if err ~= nil then
+    print("failed to load header: ", err)
+else
+    print(string.format("xdb header buffer `%s` loaded", header))
+
+    local tpl = [[
+header: {
+    version: %d
+    index_policy: %d
+    created_at: %d
+    start_index_ptr: %d
+    end_index_ptr: %d
+}]]
+
+    local t = header:to_table()
+    print(string.format(tpl,
+            t["version"], t["index_policy"], t["created_at"], t["start_index_ptr"], t["end_index_ptr"])
+    )
+end
+
+
+print("\n--- testing load vector index ... ")
+v_index, err = xdb.load_vector_index("../../data/ip2region.xdb")
+if err ~= nil then
+    print("failed to load vector index: ", err)
+else
+    print(string.format("xdb vector index buffer `%s` loaded, info={name=%s, type=%d, length=%d}",
+            v_index, v_index:name(), v_index:type(), v_index:length()))
+    v_index:close()
+end
+
+
+print("\n--- testing load content buffer ... ")
+c_buffer, err = xdb.load_content("../../data/ip2region.xdb")
+if err ~= nil then
+    print("failed to load content: ", err)
+else
+    print(string.format("xdb content buffer `%s` loaded, info={name=%s, type=%d, length=%d}",
+            c_buffer, c_buffer:name(), c_buffer:type(), c_buffer:length()))
+    c_buffer:close();
+end
+
+
+print("\n--- testing search ... ")
+local ip_str = "1.2.3.4"
+searcher, err = xdb.new_with_file_only("../../data/ip2region.xdb")
+local t_start = xdb.now()
+region, err = searcher:search(ip_str)
+if err ~= nil then
+    print(string.format("search(%s) failed: %s", ip_str, err))
+else
+    local c_time = xdb.now() - t_start
+    print(string.format("search(%s): {region=%s, io_count: %d, took: %dμs, err=%s}",
+            ip_str, region, searcher:get_io_count(), c_time, err))
+    print(string.format("searcher.tostring=%s", searcher))
+end
+searcher:close()
+
+
+print("")
+print(string.format("all tests done, elapsed %d μs", xdb.now() - s_time))

binding/lua/xdb_searcher.lua

@@ -0,0 +1,172 @@
+-- Copyright 2022 The Ip2Region Authors. All rights reserved.
+-- Use of this source code is governed by a Apache2.0-style
+-- license that can be found in the LICENSE file.
+--
+-- ---
+-- @Author Lion <chenxin619315@gmail.com>
+-- @Date   2022/07/05
+
+-- constants define
+local HeaderInfoLength = 256
+local VectorIndexRows  = 256
+local VectorIndexCols  = 256
+local VectorIndexSize  = 8
+local SegmentIndexSize = 14
+local _M = {
+    -- xdb file handle
+    handle = nil,
+
+    -- header info
+    header = nil,
+    io_count = 0,
+
+    -- vector index
+    vector_index = nil,
+
+    -- xdb content buffer
+    content_buff = nil
+}
+
+-- index and to string attribute set
+_M.__index = _M
+_M.__tostring = function(self)
+    return "xdb searcher object"
+end
+
+
+-- --- construct functions
+function newBase(dbPath, vIndex, cBuffer)
+    local obj = setmetatable({}, _M)
+    if cBuffer ~= nil then
+        obj.io_count = 0
+        obj.vector_index = nil
+        obj.content_buff = cBuffer
+    else
+        obj.io_count = 0
+        obj.vector_index = vIndex
+        obj.handle = io.open(dbPath, "r")
+        if obj.handle == nil then
+            error(string.format("failed to open xdb file `%s`", dbPath), 2)
+        end
+    end
+
+    return obj
+end
+
+function _M.new_with_file_only(dbPath)
+    return newBase(dbPath, nil, nil)
+end
+
+function _M.new_with_vector_index(dbPath, vIndex)
+    return newBase(dbPath, vIndex, nil)
+end
+
+function _M.new_with_buffer(cBuffer)
+    return newBase(nil, nil, cBuffer)
+end
+
+-- End of constructors
+
+-- object api impl
+-- must call via ':'
+function _M:search(ip_src)
+    -- check and convert string ip to long ip
+    local t, ip = type(ip_src), 0
+    if t == nil then
+        return "", string.format("invalid ip address `%s`", ip_src)
+    elseif t == "string" then
+        ip, err = self.check_ip(ip_src)
+        if err ~= nil then
+            return "", string.format("check ip `%s`: %s", ip_src, err)
+        end
+    elseif t ~= "number" then
+        return "", "invalid number or string ip"
+    end
+
+    -- reset the global counter
+    self.io_count = 0
+
+    -- locate the segment index based on the vector index
+    local il0 = (ip >> 24) & 0xFF
+    local il1 = (ip >> 16) & 0xFF
+    local idx = il0 * VectorIndexCols * SegmentIndexSize + il1 * SegmentIndexSize
+    local s_ptr, e_ptr = 0, 0
+    if self.vector_index ~= nil then
+        s_ptr = getLong(self.vector_index, idx)
+        e_ptr = getLong(self.vector_index, idx + 4)
+    elseif self.content_buff ~= nil then
+        s_ptr = getLong(self.content_buff, HeaderInfoLength + idx)
+        e_ptr = getLong(self.content_buff, HeaderInfoLength + idx + 4)
+    else
+        -- load from the file
+    end
+
+    return "", "not implemented yet"
+end
+
+function _M:get_io_count()
+    return self.io_count
+end
+
+function _M:close()
+    if self.handle ~= nil then
+        self.handle:close()
+    end
+end
+-- End of search api
+
+
+-- static util functions
+function _M.load_header(dbPath)
+    return nil, "not implemented yet"
+end
+
+function _M.load_vector_index(dbPath)
+    return nil, "not implemented yet"
+end
+
+function _M.load_content(dbPath)
+    return nil, "not implemented yet"
+end
+
+function _M.check_ip(ip_str)
+    local ip, id = 0, 1
+    local offset_arr = {24, 16, 8, 0}
+    for p in string.gmatch(ip_str..".", "([%d]+)%.") do
+        -- match pattern checking
+        if p == nil then
+            return 0, "err=1"
+        end
+
+        -- count checking
+        if id > 4 then
+            return 0, "err=1"
+        end
+
+        -- value checking
+        v = tonumber(p)
+        if v > 255 then
+            return 0, "err=2"
+        end
+
+        ip = ip | (v << offset_arr[id])
+        id = id + 1
+    end
+
+    if id ~= 5 then
+        return 0, "err=1"
+    end
+
+    return ip, nil
+end
+
+function _M.long2ip(ip)
+    return string.format("%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8 ) & 0xFF, ip & 0xFF)
+end
+
+function _M.now()
+    return 0
+end
+-- End of util functions
+
+return _M