From 8f6effd8eb7e7013d700b1eb4392126c0bb7f05a Mon Sep 17 00:00:00 2001 From: lion Date: Mon, 20 Oct 2025 22:26:15 +0800 Subject: [PATCH] verify api and verify the xdb before search or bench test --- binding/lua/ReadMe.md | 17 +++++++ binding/lua/bench_test.lua | 25 ++++++++- binding/lua/search_test.lua | 25 ++++++++- binding/lua/util_test.lua | 19 ++++++- binding/lua/xdb_searcher.lua | 98 +++++++++++++++++++++++++++++------- 5 files changed, 159 insertions(+), 25 deletions(-) diff --git a/binding/lua/ReadMe.md b/binding/lua/ReadMe.md index 379c0d1..f279827 100644 --- a/binding/lua/ReadMe.md +++ b/binding/lua/ReadMe.md @@ -21,6 +21,23 @@ local version = xdb.IPv6 -- 备注：以下演示直接使用 dbPath 和 version 变量 ``` +### 文件验证 +建议您主动去验证 xdb 文件的适用性，因为后期的一些新功能可能会导致目前的 Searcher 版本无法适用你使用的 xdb 文件，验证可以避免运行过程中的一些不可预测的错误。 你不需要每次都去验证，例如在服务启动的时候，或者手动调用命令验证确认版本匹配即可，不要在每次创建的 Searcher 的时候运行验证，这样会影响查询的响应速度，尤其是高并发的使用场景。 +```lua +local xdb = require('xdb_searcher') + +local err = xdb.verify(dbPath); +if err ~= nil then + -- 适用性验证失败！！！ + -- 当前查询客户端实现不适用于 dbPath 指定的 xdb 文件的查询. + -- 应该停止启动服务，使用合适的 xdb 文件或者升级到适合 dbPath 的 Searcher 实现。 + print(string.format("binding is not applicable for xdb file '%s': %s", dbPath, err)) + return +end + +-- 验证通过，当前使用的 Searcher 可以安全的用于对 dbPath 指向的 xdb 的查询操作 +``` + ### 完全基于文件的查询 ```lua local xdb = require("xdb_searcher") diff --git a/binding/lua/bench_test.lua b/binding/lua/bench_test.lua index 0fded45..cb2c017 100644 --- a/binding/lua/bench_test.lua +++ b/binding/lua/bench_test.lua @@ -62,19 +62,40 @@ if string.len(dbFile) < 2 or string.len(srcFile) < 2 then return end --- load the header and define the ip version -local header, err = xdb.load_header(dbFile) +-- open the dbFile +local handle, closer, err = xdb.open_file(dbFile, "rb") if err ~= nil then + print(string.format("failed to open %s: %s", dbFile, err)) + return +end + +-- verify the xdb +err = xdb.verify(handle) +if err ~= nil then + closer() + print(string.format("verify(%s): %s", dbFile, err)) + return +end + +-- load the header and define the ip version +local header, err = xdb.load_header(handle) +if err ~= nil then + closer() print(string.format("failed to load the header: %s", err)) return end local version, err = xdb.version_from_header(header) if err ~= nil then + closer() print(string.format("failed to detect version from header: %s", err)) return end +-- file close +closer("bench_test") + + -- create the searcher based on the cache-policy local searcher, v_index, content if cachePolicy == "file" then diff --git a/binding/lua/search_test.lua b/binding/lua/search_test.lua index a7c1b77..f133997 100644 --- a/binding/lua/search_test.lua +++ b/binding/lua/search_test.lua @@ -59,19 +59,40 @@ if string.len(dbFile) < 2 then return end --- load the header and define the ip version -local header, err = xdb.load_header(dbFile) +-- open the dbFile +local handle, closer, err = xdb.open_file(dbFile, "rb") if err ~= nil then + print(string.format("failed to open %s: %s", dbFile, err)) + return +end + +-- verify the xdb +err = xdb.verify(handle) +if err ~= nil then + closer("xdb_verify") + print(string.format("verify(%s): %s", dbFile, err)) + return +end + +-- load the header and define the ip version +local header, err = xdb.load_header(handle) +if err ~= nil then + closer() print(string.format("failed to load the header: %s", err)) return end local version, err = xdb.version_from_header(header) if err ~= nil then + closer() print(string.format("failed to detect version from header: %s", err)) return end +-- file close +closer("search_test") + + -- create the searcher based on the cache-policy local searcher, v_index, content if cachePolicy == "file" then diff --git a/binding/lua/util_test.lua b/binding/lua/util_test.lua index cd8d072..0d2b8db 100644 --- a/binding/lua/util_test.lua +++ b/binding/lua/util_test.lua @@ -97,7 +97,7 @@ function test_load_vector_index() if err ~= nil then print("failed to load vector index: ", err) else - print("xdb vector index buffer loaded") + print("xdb vector index buffer loaded, length=", #v_index) end end @@ -106,10 +106,25 @@ function test_load_content() if err ~= nil then print("failed to load content: ", err) else - print("xdb content buffer loaded") + print("xdb content buffer loaded, length=", #c_buffer) end end +function test_verify() + local xdb_files = { + "../../data/ip2region_v4.xdb", + "../../data/ip2region_v6.xdb" + } + + for _, path in ipairs(xdb_files) do + local err = xdb.verify(path) + if err ~= nil then + print(string.format("verify(%s): %s", path, err)) + else + print(string.format("verify(%s): Ok", path)) + end + end +end function test_ip_search() local test_list = { diff --git a/binding/lua/xdb_searcher.lua b/binding/lua/xdb_searcher.lua index 9fc9c9c..5105c5c 100644 --- a/binding/lua/xdb_searcher.lua +++ b/binding/lua/xdb_searcher.lua @@ -232,25 +232,45 @@ end -- static util functions +function xdb.open_file(db_path, mode) + local t, handle = type(db_path), nil + local _closer = nil + if t == "userdata" then + handle = db_path -- file handle + _closer = function(caller) end + elseif t == "string" then + handle = io.open(db_path, mode) + if handle == nil then + return nil, nil, string.format("failed to open xdb file `%s`", db_path) + end + + _closer = function(caller) + handle:close() + end + end + + return handle, _closer, nil +end + function xdb.load_header(db_path) - local handle = io.open(db_path, "r") - if handle == nil then - return nil, string.format("failed to open xdb file `%s`", db_path) + local handle, closer, err = xdb.open_file(db_path, "rb") + if err ~= nil then + return nil, err end local r = handle:seek("set", 0) if r == nil then - handle:close() + closer("load_header") return nil, "failed to seek to 0" end local c = handle:read(header_info_length) if c == nil then - handle:close() + closer("load_header") return nil, string.format("failed to read %d bytes", header_info_length) end - handle:close() + closer("load_header") return { ["version"] = le_get_uint16(c, 1), ["index_policy"] = le_get_uint16(c, 3), @@ -267,43 +287,83 @@ function xdb.load_header(db_path) end function xdb.load_vector_index(db_path) - local handle = io.open(db_path, "r") - if handle == nil then - return nil, string.format("failed to open xdb file `%s`", db_path) + local handle, closer, err = xdb.open_file(db_path, "rb") + if err ~= nil then + return nil, err end local r = handle:seek("set", header_info_length) if r == nil then - handle:close() + closer("load_vector_index") return nil, string.format("failed to seek to %d", header_info_length) end local c = handle:read(vector_index_length) if c == nil then - handle:close() + closer("load_vector_index") return nil, string.format("failed to read %d bytes", vector_index_length) end - handle:close() + closer("load_vector_index") return c, nil end function xdb.load_content(db_path) - local handle = io.open(db_path, "r") - if handle == nil then - return nil, string.format("failed to open xdb file `%s`", db_path) - end - + local handle, closer, err = xdb.open_file(db_path, "rb") local c = handle:read("*a") if c == nil then - handle:close() + closer("load_content") return nil, string.format("failed to read xdb content") end - handle:close() + closer("load_content") return c, nil end +-- Verify if the current Searcher could be used to search the specified xdb file. +-- Why do we need this check ? +-- The future features of the xdb impl may cause the current searcher not able to work properly. +-- +-- @Note: You Just need to check this ONCE when the service starts +-- Or use another process (eg, A command) to check once Just to confirm the suitability. +function xdb.verify(db_path) + local handle, closer, err = xdb.open_file(db_path, "rb") + if err ~= nil then + return err + end + + -- load the header from handle + local header, err = xdb.load_header(handle) + if err ~= nil then + closer() + return string.format("failed to load header: %s", err) + end + + -- get the runtime ptr bytes + local runtime_ptr_bytes = 0 + if header.version == xdb_structure_20 then + runtime_ptr_bytes = 4 + elseif header.version == xdb_structure_30 then + runtime_ptr_bytes = header.runtime_ptr_bytes + else + closer() + return string.format("invalid structure version %d", header.version); + end + + -- 1, confirm the xdb file size + -- to ensure that the maximum file pointer does not overflow + local max_file_ptr = ((1 << (runtime_ptr_bytes * 8)) & 0xFFFFFFFFFFFFFFFF) - 1 + local _file_bytes = handle:seek("end", 0) + -- print("max_file_ptr=", max_file_ptr, "_file_bytes", _file_bytes) + if _file_bytes > max_file_ptr then + closer() + return string.format("xdb file exceeds the maximum supported bytes: %d", max_file_ptr); + end + + closer() + return nil +end + -- -- parse ip string function split(str, sep)