fix(erlang): unify v4 pool name and stop caching errors
This commit is contained in:
parent
445eb4cb8e
commit
25513cb246
|
|
@ -133,6 +133,10 @@ Finished in 0.150 seconds
|
||||||
|
|
||||||
Both IPv4 and IPv6 benchmarks share the same script. Run it with the desired IP version:
|
Both IPv4 and IPv6 benchmarks share the same script. Run it with the desired IP version:
|
||||||
|
|
||||||
|
> `cold` = first pass over the source file: each IP triggers a real search and the
|
||||||
|
> result is written into the ETS cache. `warm` = second pass over the same list,
|
||||||
|
> where every lookup is served directly from the ETS cache.
|
||||||
|
|
||||||
```
|
```
|
||||||
$ cd benchmarks/
|
$ cd benchmarks/
|
||||||
$ sh xdb-benchmark.sh ipv4
|
$ sh xdb-benchmark.sh ipv4
|
||||||
|
|
|
||||||
|
|
@ -110,6 +110,9 @@ Finished in 0.074 seconds
|
||||||
|
|
||||||
IPv4 与 IPv6 共用一个脚本,通过参数指定版本:
|
IPv4 与 IPv6 共用一个脚本,通过参数指定版本:
|
||||||
|
|
||||||
|
> `cold` = 第一次遍历源文件:每个 IP 都会触发真实查询,并把结果写入 ETS 缓存。
|
||||||
|
> `warm` = 第二次遍历同一列表,所有查询都直接命中 ETS 缓存。
|
||||||
|
|
||||||
```
|
```
|
||||||
$ cd benchmarks/
|
$ cd benchmarks/
|
||||||
$ sh xdb-benchmark.sh ipv4
|
$ sh xdb-benchmark.sh ipv4
|
||||||
|
|
|
||||||
|
|
@ -9,15 +9,6 @@
|
||||||
-define(XDB_VECTOR_INDEX_SIZE, 8).
|
-define(XDB_VECTOR_INDEX_SIZE, 8).
|
||||||
-define(XDB_VECTOR_INDEX_COUNT, (16#10000)). %% 256*256
|
-define(XDB_VECTOR_INDEX_COUNT, (16#10000)). %% 256*256
|
||||||
|
|
||||||
%% xdb header field offsets (bytes)
|
|
||||||
-define(XDB_HEADER_VERSION_OFFSET, 0).
|
|
||||||
-define(XDB_HEADER_INDEX_POLICY_OFFSET, 2).
|
|
||||||
-define(XDB_HEADER_CREATED_AT_OFFSET, 4).
|
|
||||||
-define(XDB_HEADER_START_INDEX_PTR_OFFSET, 8).
|
|
||||||
-define(XDB_HEADER_END_INDEX_PTR_OFFSET, 12).
|
|
||||||
-define(XDB_HEADER_IP_VERSION_OFFSET, 16).
|
|
||||||
-define(XDB_HEADER_RUNTIME_PTR_BYTES_OFFSET, 18).
|
|
||||||
|
|
||||||
%% IP versions as stored in xdb header
|
%% IP versions as stored in xdb header
|
||||||
-define(IP_VERSION_4, 4).
|
-define(IP_VERSION_4, 4).
|
||||||
-define(IP_VERSION_6, 6).
|
-define(IP_VERSION_6, 6).
|
||||||
|
|
@ -34,9 +25,18 @@
|
||||||
-define(IP2REGION_CACHE_V4, ets_ip2region_cache_v4).
|
-define(IP2REGION_CACHE_V4, ets_ip2region_cache_v4).
|
||||||
-define(IP2REGION_CACHE_V6, ets_ip2region_cache_v6).
|
-define(IP2REGION_CACHE_V6, ets_ip2region_cache_v6).
|
||||||
|
|
||||||
%% Per-version pool names. ?IP2REGION_POOL is kept as the legacy v4 pool name.
|
%% Per-version pool names.
|
||||||
-define(IP2REGION_POOL, ip2region_pool).
|
|
||||||
-define(IP2REGION_POOL_V4, ip2region_pool_v4).
|
-define(IP2REGION_POOL_V4, ip2region_pool_v4).
|
||||||
-define(IP2REGION_POOL_V6, ip2region_pool_v6).
|
-define(IP2REGION_POOL_V6, ip2region_pool_v6).
|
||||||
|
|
||||||
|
-record(xdb_header, {
|
||||||
|
version :: non_neg_integer(),
|
||||||
|
index_policy :: non_neg_integer(),
|
||||||
|
created_at :: non_neg_integer(),
|
||||||
|
start_index_ptr :: non_neg_integer(),
|
||||||
|
end_index_ptr :: non_neg_integer(),
|
||||||
|
ip_version :: non_neg_integer(),
|
||||||
|
runtime_ptr_bytes :: non_neg_integer()
|
||||||
|
}).
|
||||||
|
|
||||||
-endif.
|
-endif.
|
||||||
|
|
|
||||||
|
|
@ -58,16 +58,9 @@ ensure_table(Name, Opts) ->
|
||||||
pool_child_specs() ->
|
pool_child_specs() ->
|
||||||
{ok, DbConfig} = application:get_env(db),
|
{ok, DbConfig} = application:get_env(db),
|
||||||
{ok, PoolArgsCfg} = application:get_env(poolargs),
|
{ok, PoolArgsCfg} = application:get_env(poolargs),
|
||||||
Versions = [Version || {Version, _File} <- DbConfig],
|
|
||||||
UseLegacyName = (Versions == [ipv4]),
|
|
||||||
V4PoolName = case UseLegacyName of
|
|
||||||
true -> ?IP2REGION_POOL;
|
|
||||||
false -> ?IP2REGION_POOL_V4
|
|
||||||
end,
|
|
||||||
ok = application:set_env(?APP_NAME, v4_pool_name, V4PoolName),
|
|
||||||
lists:foldl(
|
lists:foldl(
|
||||||
fun({ipv4, File}, Acc) ->
|
fun({ipv4, File}, Acc) ->
|
||||||
[make_pool_spec(V4PoolName, ipv4, File, PoolArgsCfg) | Acc];
|
[make_pool_spec(?IP2REGION_POOL_V4, ipv4, File, PoolArgsCfg) | Acc];
|
||||||
({ipv6, File}, Acc) ->
|
({ipv6, File}, Acc) ->
|
||||||
[make_pool_spec(?IP2REGION_POOL_V6, ipv6, File, PoolArgsCfg) | Acc];
|
[make_pool_spec(?IP2REGION_POOL_V6, ipv6, File, PoolArgsCfg) | Acc];
|
||||||
(_, Acc) ->
|
(_, Acc) ->
|
||||||
|
|
|
||||||
|
|
@ -134,6 +134,7 @@ do_info(Info, State) ->
|
||||||
error_logger:error_report(io:format("unknown info: ~p", [Info])),
|
error_logger:error_report(io:format("unknown info: ~p", [Info])),
|
||||||
{noreply, State}.
|
{noreply, State}.
|
||||||
|
|
||||||
|
-spec resolve_version(#xdb_header{}) -> {ok, ipv4 | ipv6} | {error, term()}.
|
||||||
resolve_version(Header) ->
|
resolve_version(Header) ->
|
||||||
case ip2region_xdb:header_version(Header) of
|
case ip2region_xdb:header_version(Header) of
|
||||||
2 -> {ok, ipv4};
|
2 -> {ok, ipv4};
|
||||||
|
|
@ -147,6 +148,10 @@ resolve_version(Header) ->
|
||||||
end.
|
end.
|
||||||
|
|
||||||
load_vector_index(IoDevice, Version) ->
|
load_vector_index(IoDevice, Version) ->
|
||||||
|
%% The vector index ETS table is global and shared by all pool workers.
|
||||||
|
%% Only the first worker (or the first worker after a restart) populates it.
|
||||||
|
%% Subsequent workers skip the file:read here; search_ip/2 uses file:pread/3,
|
||||||
|
%% so the current file pointer position does not affect correctness.
|
||||||
Table = ip2region_xdb:vector_index_table(Version),
|
Table = ip2region_xdb:vector_index_table(Version),
|
||||||
case ets:info(Table, size) of
|
case ets:info(Table, size) of
|
||||||
undefined ->
|
undefined ->
|
||||||
|
|
|
||||||
|
|
@ -18,16 +18,6 @@
|
||||||
|
|
||||||
-include("ip2region.hrl").
|
-include("ip2region.hrl").
|
||||||
|
|
||||||
-record(xdb_header, {
|
|
||||||
version :: non_neg_integer(),
|
|
||||||
index_policy :: non_neg_integer(),
|
|
||||||
created_at :: non_neg_integer(),
|
|
||||||
start_index_ptr :: non_neg_integer(),
|
|
||||||
end_index_ptr :: non_neg_integer(),
|
|
||||||
ip_version :: non_neg_integer(),
|
|
||||||
runtime_ptr_bytes :: non_neg_integer()
|
|
||||||
}).
|
|
||||||
|
|
||||||
-spec parse_header(binary()) -> {ok, #xdb_header{}} | {error, invalid_header}.
|
-spec parse_header(binary()) -> {ok, #xdb_header{}} | {error, invalid_header}.
|
||||||
parse_header(Bin) when is_binary(Bin), byte_size(Bin) >= 20 ->
|
parse_header(Bin) when is_binary(Bin), byte_size(Bin) >= 20 ->
|
||||||
<<Version:16/little, IndexPolicy:16/little, CreatedAt:32/little,
|
<<Version:16/little, IndexPolicy:16/little, CreatedAt:32/little,
|
||||||
|
|
@ -57,11 +47,14 @@ header_runtime_ptr_bytes(#xdb_header{runtime_ptr_bytes = B}) -> B.
|
||||||
segment_index_size(ipv4) -> ?XDB_SEGMENT_INDEX_SIZE_V4;
|
segment_index_size(ipv4) -> ?XDB_SEGMENT_INDEX_SIZE_V4;
|
||||||
segment_index_size(ipv6) -> ?XDB_SEGMENT_INDEX_SIZE_V6.
|
segment_index_size(ipv6) -> ?XDB_SEGMENT_INDEX_SIZE_V6.
|
||||||
|
|
||||||
|
-spec vector_index_table(ipv4 | ipv6) -> atom().
|
||||||
vector_index_table(ipv4) -> ?XDB_VECTOR_INDEX_V4;
|
vector_index_table(ipv4) -> ?XDB_VECTOR_INDEX_V4;
|
||||||
vector_index_table(ipv6) -> ?XDB_VECTOR_INDEX_V6.
|
vector_index_table(ipv6) -> ?XDB_VECTOR_INDEX_V6.
|
||||||
|
|
||||||
|
-spec segment_index_table(ipv4 | ipv6) -> atom().
|
||||||
segment_index_table(ipv4) -> ?XDB_SEGMENT_INDEX_V4;
|
segment_index_table(ipv4) -> ?XDB_SEGMENT_INDEX_V4;
|
||||||
segment_index_table(ipv6) -> ?XDB_SEGMENT_INDEX_V6.
|
segment_index_table(ipv6) -> ?XDB_SEGMENT_INDEX_V6.
|
||||||
|
|
||||||
|
-spec cache_table(ipv4 | ipv6) -> atom().
|
||||||
cache_table(ipv4) -> ?IP2REGION_CACHE_V4;
|
cache_table(ipv4) -> ?IP2REGION_CACHE_V4;
|
||||||
cache_table(ipv6) -> ?IP2REGION_CACHE_V6.
|
cache_table(ipv6) -> ?IP2REGION_CACHE_V6.
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@
|
||||||
search(Ip) ->
|
search(Ip) ->
|
||||||
case ip2region_util:ip_to_bytes(Ip) of
|
case ip2region_util:ip_to_bytes(Ip) of
|
||||||
{ok, ipv4, IpBin} ->
|
{ok, ipv4, IpBin} ->
|
||||||
do_search(v4_pool(), ipv4, IpBin);
|
do_search(?IP2REGION_POOL_V4, ipv4, IpBin);
|
||||||
{ok, ipv6, IpBin} ->
|
{ok, ipv6, IpBin} ->
|
||||||
do_search(?IP2REGION_POOL_V6, ipv6, IpBin);
|
do_search(?IP2REGION_POOL_V6, ipv6, IpBin);
|
||||||
Ret ->
|
Ret ->
|
||||||
|
|
@ -31,16 +31,13 @@ do_search(PoolName, Version, IpBin) ->
|
||||||
_ ->
|
_ ->
|
||||||
Worker = poolboy:checkout(PoolName, true, infinity),
|
Worker = poolboy:checkout(PoolName, true, infinity),
|
||||||
try
|
try
|
||||||
Region = ip2region_worker:search(Worker, IpBin),
|
case ip2region_worker:search(Worker, IpBin) of
|
||||||
ets:insert(CacheTable, {IpBin, Region}),
|
{error, _} = Err -> Err;
|
||||||
Region
|
Region ->
|
||||||
|
ets:insert(CacheTable, {IpBin, Region}),
|
||||||
|
Region
|
||||||
|
end
|
||||||
after
|
after
|
||||||
poolboy:checkin(PoolName, Worker)
|
poolboy:checkin(PoolName, Worker)
|
||||||
end
|
end
|
||||||
end.
|
end.
|
||||||
|
|
||||||
v4_pool() ->
|
|
||||||
case application:get_env(?APP_NAME, v4_pool_name) of
|
|
||||||
{ok, PoolName} -> PoolName;
|
|
||||||
undefined -> ?IP2REGION_POOL
|
|
||||||
end.
|
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,10 @@ pools_started_test() ->
|
||||||
application:stop(ip2region),
|
application:stop(ip2region),
|
||||||
application:unload(ip2region),
|
application:unload(ip2region),
|
||||||
ok = application:load(ip2region),
|
ok = application:load(ip2region),
|
||||||
|
%% NOTE: rebar3 compiles tests under _build/test/lib/ip2region via a
|
||||||
|
%% symlink back to the source tree, so ?FILE resolves to the original
|
||||||
|
%% binding/erlang/test/... path. If rebar3 is configured to copy
|
||||||
|
%% instead of symlink, this path traversal must be adjusted.
|
||||||
TestDir = filename:dirname(?FILE),
|
TestDir = filename:dirname(?FILE),
|
||||||
ErlangDir = filename:dirname(TestDir),
|
ErlangDir = filename:dirname(TestDir),
|
||||||
BindingDir = filename:dirname(ErlangDir),
|
BindingDir = filename:dirname(ErlangDir),
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue