From 5b52484cf8bc03755b2ae8820c35b5f77d4fc8ee Mon Sep 17 00:00:00 2001 From: Lion Date: Tue, 5 Jul 2022 14:06:01 +0800 Subject: [PATCH] xdb lua searcher framework --- binding/lua/util_test.lua | 102 +++++++++++++++++++++ binding/lua/xdb_searcher.lua | 172 +++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 binding/lua/util_test.lua create mode 100644 binding/lua/xdb_searcher.lua diff --git a/binding/lua/util_test.lua b/binding/lua/util_test.lua new file mode 100644 index 0000000..fa75143 --- /dev/null +++ b/binding/lua/util_test.lua @@ -0,0 +1,102 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/07/05 + +-- set the package path +package.path = "./?.lua" +package.cpath = "./?.so" + +local xdb = require("xdb_searcher") + +---- ip checking testing +print("--- testing check_ip and long2ip ... ") +local ip_list = { + "1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0", + "256.7.12.9", "12.56.78.320", "32.12.45.192", "222.221.220.219", + "192.168.1.101 ", "132.96.12.98a", "x23.12.2.12" +} + +local s_time = xdb.now() +for _, ip_src in ipairs(ip_list) do + ip, err = xdb.check_ip(ip_src) + if err ~= nil then + print(string.format("invalid ip address `%s`: %s", ip_src, err)) + else + ip_dst = xdb.long2ip(ip) + io.write(string.format("long(%-15s)=%10d, long2ip(%-10d)=%-15s", ip_src, ip, ip, ip_dst)) + if ip_src ~= ip_dst then + print(" --[Failed]") + else + print(" --[Ok]") + end + end +end + +---- buffer loading test +print("\n--- testing load header ... ") +header, err = xdb.load_header("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load header: ", err) +else + print(string.format("xdb header buffer `%s` loaded", header)) + + local tpl = [[ +header: { + version: %d + index_policy: %d + created_at: %d + start_index_ptr: %d + end_index_ptr: %d +}]] + + local t = header:to_table() + print(string.format(tpl, + t["version"], t["index_policy"], t["created_at"], t["start_index_ptr"], t["end_index_ptr"]) + ) +end + + +print("\n--- testing load vector index ... ") +v_index, err = xdb.load_vector_index("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load vector index: ", err) +else + print(string.format("xdb vector index buffer `%s` loaded, info={name=%s, type=%d, length=%d}", + v_index, v_index:name(), v_index:type(), v_index:length())) + v_index:close() +end + + +print("\n--- testing load content buffer ... ") +c_buffer, err = xdb.load_content("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load content: ", err) +else + print(string.format("xdb content buffer `%s` loaded, info={name=%s, type=%d, length=%d}", + c_buffer, c_buffer:name(), c_buffer:type(), c_buffer:length())) + c_buffer:close(); +end + + +print("\n--- testing search ... ") +local ip_str = "1.2.3.4" +searcher, err = xdb.new_with_file_only("../../data/ip2region.xdb") +local t_start = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("search(%s) failed: %s", ip_str, err)) +else + local c_time = xdb.now() - t_start + print(string.format("search(%s): {region=%s, io_count: %d, took: %dμs, err=%s}", + ip_str, region, searcher:get_io_count(), c_time, err)) + print(string.format("searcher.tostring=%s", searcher)) +end +searcher:close() + + +print("") +print(string.format("all tests done, elapsed %d μs", xdb.now() - s_time)) \ No newline at end of file diff --git a/binding/lua/xdb_searcher.lua b/binding/lua/xdb_searcher.lua new file mode 100644 index 0000000..0bb21d2 --- /dev/null +++ b/binding/lua/xdb_searcher.lua @@ -0,0 +1,172 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/07/05 + +-- constants define +local HeaderInfoLength = 256 +local VectorIndexRows = 256 +local VectorIndexCols = 256 +local VectorIndexSize = 8 +local SegmentIndexSize = 14 +local _M = { + -- xdb file handle + handle = nil, + + -- header info + header = nil, + io_count = 0, + + -- vector index + vector_index = nil, + + -- xdb content buffer + content_buff = nil +} + +-- index and to string attribute set +_M.__index = _M +_M.__tostring = function(self) + return "xdb searcher object" +end + + +-- --- construct functions +function newBase(dbPath, vIndex, cBuffer) + local obj = setmetatable({}, _M) + if cBuffer ~= nil then + obj.io_count = 0 + obj.vector_index = nil + obj.content_buff = cBuffer + else + obj.io_count = 0 + obj.vector_index = vIndex + obj.handle = io.open(dbPath, "r") + if obj.handle == nil then + error(string.format("failed to open xdb file `%s`", dbPath), 2) + end + end + + return obj +end + +function _M.new_with_file_only(dbPath) + return newBase(dbPath, nil, nil) +end + +function _M.new_with_vector_index(dbPath, vIndex) + return newBase(dbPath, vIndex, nil) +end + +function _M.new_with_buffer(cBuffer) + return newBase(nil, nil, cBuffer) +end + +-- End of constructors + +-- object api impl +-- must call via ':' +function _M:search(ip_src) + -- check and convert string ip to long ip + local t, ip = type(ip_src), 0 + if t == nil then + return "", string.format("invalid ip address `%s`", ip_src) + elseif t == "string" then + ip, err = self.check_ip(ip_src) + if err ~= nil then + return "", string.format("check ip `%s`: %s", ip_src, err) + end + elseif t ~= "number" then + return "", "invalid number or string ip" + end + + -- reset the global counter + self.io_count = 0 + + -- locate the segment index based on the vector index + local il0 = (ip >> 24) & 0xFF + local il1 = (ip >> 16) & 0xFF + local idx = il0 * VectorIndexCols * SegmentIndexSize + il1 * SegmentIndexSize + local s_ptr, e_ptr = 0, 0 + if self.vector_index ~= nil then + s_ptr = getLong(self.vector_index, idx) + e_ptr = getLong(self.vector_index, idx + 4) + elseif self.content_buff ~= nil then + s_ptr = getLong(self.content_buff, HeaderInfoLength + idx) + e_ptr = getLong(self.content_buff, HeaderInfoLength + idx + 4) + else + -- load from the file + end + + return "", "not implemented yet" +end + +function _M:get_io_count() + return self.io_count +end + +function _M:close() + if self.handle ~= nil then + self.handle:close() + end +end +-- End of search api + + +-- static util functions +function _M.load_header(dbPath) + return nil, "not implemented yet" +end + +function _M.load_vector_index(dbPath) + return nil, "not implemented yet" +end + +function _M.load_content(dbPath) + return nil, "not implemented yet" +end + +function _M.check_ip(ip_str) + local ip, id = 0, 1 + local offset_arr = {24, 16, 8, 0} + for p in string.gmatch(ip_str..".", "([%d]+)%.") do + -- match pattern checking + if p == nil then + return 0, "err=1" + end + + -- count checking + if id > 4 then + return 0, "err=1" + end + + -- value checking + v = tonumber(p) + if v > 255 then + return 0, "err=2" + end + + ip = ip | (v << offset_arr[id]) + id = id + 1 + end + + if id ~= 5 then + return 0, "err=1" + end + + return ip, nil +end + +function _M.long2ip(ip) + return string.format("%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8 ) & 0xFF, ip & 0xFF) +end + +function _M.now() + return 0 +end +-- End of util functions + +return _M \ No newline at end of file