diff --git a/.gitignore b/.gitignore index ae58d2e..37ea030 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,6 @@ target #vscode .vscode build + +# git worktrees +.worktrees/ diff --git a/binding/erlang/Makefile b/binding/erlang/Makefile new file mode 100644 index 0000000..d71c6d4 --- /dev/null +++ b/binding/erlang/Makefile @@ -0,0 +1,18 @@ +.PHONY: all compile test clean bench-v4 bench-v6 + +all: compile + +compile: + rebar3 compile + +test: + rebar3 eunit + +clean: + rebar3 clean + +bench-v4: + sh benchmarks/xdb-benchmark.sh ipv4 + +bench-v6: + sh benchmarks/xdb-benchmark.sh ipv6 diff --git a/binding/erlang/README.md b/binding/erlang/README.md index 08b9999..15cbe74 100644 --- a/binding/erlang/README.md +++ b/binding/erlang/README.md @@ -1,10 +1,10 @@ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) -# ip2region erlang query client +# ip2region Erlang query client ### Introduction -This binding implements the xdb query client in Erlang, based on the Erlang OTP Application. The query logic is implemented by the `ip2region_worker` worker process, supporting multiple worker processes for load balancing. +This binding implements the xdb query client in `Erlang`, based on the `Erlang OTP Application`. The query logic is implemented by the `ip2region_worker` worker process, supporting multiple worker processes for load balancing. ### Application Configuration @@ -15,13 +15,35 @@ The configurable parameters for this application are in `ip2region.app.src`, as {poolargs, [ {size, 1}, %% Default number of worker processes {max_overflow, 5} %% Maximum number of worker processes + ]}, + {db, [ + {ipv4, "ip2region.xdb"} %% Default IPv4 xdb file ]} ]} ``` +### Dual-stack configuration (IPv4 + IPv6) + +To enable IPv6 queries, add the `ipv6` entry to the `db` list and place both xdb files under `priv/`: + +```erlang + {env,[ + {poolargs, [ + {size, 1}, + {max_overflow, 5} + ]}, + {db, [ + {ipv4, "ip2region.xdb"}, + {ipv6, "ip2region_v6.xdb"} + ]} + ]} +``` + +The `xdb:search/1` interface automatically detects IPv4 and IPv6 inputs and routes them to the correct worker pool. + ### Compile -``` +```bash $ rebar3 compile ``` @@ -29,113 +51,130 @@ $ rebar3 compile Place the xdb file in the `priv` directory, then start the Erlang node: -``` +```bash $ rebar3 shell ``` -Call the `xdb:search/1` interface in the Erlang shell to query IP address information. This interface supports IP addresses represented as list strings, binary strings, tuples, and integers, as follows: +Call the `xdb:search/1` interface in the Erlang shell to query IP address information. This interface supports IP addresses represented as list strings, binary strings, tuples, and integers: -``` +```erlang 1> xdb:search("1.0.8.0"). -[20013,22269,124,48,124,24191,19996,30465,124,24191,24030, - 24066,124,30005,20449] +[20013,22269,124,24191,19996,30465,124,24191,24030,24066, + 124,20013,22269,30005,20449,124,67,78] 2> 3> io:format("~ts~n", [xdb:search("1.0.8.0")]). -中国|0|广东省|广州市|电信 -io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]). -中国|0|广东省|广州市|电信 -4> io:format("~ts~n", [xdb:search({1,0,8,0})]). -中国|0|广东省|广州市|电信 +中国|广东省|广州市|中国电信|CN +4> io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]). +中国|广东省|广州市|中国电信|CN +5> io:format("~ts~n", [xdb:search({1,0,8,0})]). +中国|广东省|广州市|中国电信|CN 6> io:format("~ts~n", [xdb:search(16779264)]). -中国|0|广东省|广州市|电信 +中国|广东省|广州市|中国电信|CN +``` + +With dual-stack enabled, IPv6 addresses are supported in the same way: + +```erlang +1> io:format("~ts~n", [xdb:search("2001:4860:4860::8888")]). +United States|Florida|Miami|Google LLC|US +2> io:format("~ts~n", [xdb:search(<<"2001:4860:4860::8888">>)]). +United States|Florida|Miami|Google LLC|US +3> io:format("~ts~n", [xdb:search({8193,18528,18528,0,0,0,0,34952})]). +United States|Florida|Miami|Google LLC|US ``` ### Usage -* Add the dependency in `rebar.config` +- Add the dependency in `rebar.config` -``` +```erlang {deps, [ ip2region ]}. ``` -* Start the ip2region Application +- Start the ip2region Application -``` -...... - -application:ensure_started(ip2region), - -...... +```erlang +{ok, _} = application:ensure_all_started(ip2region). ``` -* Call the `xdb:search/1` interface to query IP information +- Call the `xdb:search/1` interface to query IP information -``` -...... - -ip2region:search("1.0.8.0"), - -...... +```erlang +xdb:search("1.0.8.0"). ``` ### Unit Test -``` +```bash $ rebar3 eunit ===> Verifying dependencies... ===> Analyzing applications... ===> Compiling ip2region ===> Performing EUnit tests... -=INFO REPORT==== 17-Jan-2023::11:52:59.920155 === -XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb +=INFO REPORT==== 28-Jun-2026::04:53:28 === +XdbFile:/Users/nana/Documents/code/ip2region/.worktrees/erlang-ipv6/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb .... -Finished in 0.074 seconds -4 tests, 0 failures +Finished in 0.192 seconds +63 tests, 0 failures ``` ### Benchmark -``` +Both IPv4 and IPv6 benchmarks share the same script. Run it with the desired IP version: + +> `cold` = first pass over the source file: each IP triggers a real search and the +> result is written into the ETS cache. `warm` = second pass over the same list, +> where every lookup is served directly from the ETS cache. + +```bash $ cd benchmarks/ -$ sh xdb-benchmark.sh -===> Verifying dependencies... -===> Analyzing applications... -===> Compiling ip2region -Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] - -Eshell V12.3.2.2 (abort with ^G) -1> =INFO REPORT==== 17-Jan-2023::11:37:35.631095 === -XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/default/lib/ip2region/priv/ip2region.xdb - -===> Booted ip2region -===> Evaluating: "xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()." -CPU info: -model name : AMD EPYC 7K62 48-Core Processor -cache size : 512 KB -cpu MHz : 2595.124 -bogomips : 5190.24 -cores/threads : 2 - -Erlang info: -system_version:Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] -load test data use 4.835593s - -start run benchmark tests - -search from file: -ip count:683844, -total time: 28.201699s, -search 24248.326315375536 times per second, -use 41.23995969841075 micro second per search - -search from cache: -ip count:683844, -total time: 0.671801s, -search 1017926.4395259906 times per second, -use 0.9823892583688677 micro second per search - -benchmark test finish +$ sh xdb-benchmark.sh ipv4 +``` + +For IPv6: + +```bash +$ sh xdb-benchmark.sh ipv6 +``` + +Or use the Makefile targets from the `binding/erlang` directory: + +```bash +$ make bench-v4 +$ make bench-v6 +``` + +#### IPv4 benchmark example + +```bash +System: + CPU : Apple M4 + Cores : 10 cores / 10 threads + Erlang : Erlang/OTP 29 [erts-17.0.2] [source] [64-bit] [smp:10:10] [ds:10:10:10] [async-threads:1] [jit] [dtrace] + Loaded : 487169 IPs in 1.335 s + +Benchmarks: + cold total= 9.601s count= 487169 qps= 50740.66 avg= 0.019708 ms/op (19.708 us/op) + warm total= 0.160s count= 487169 qps= 3053164.29 avg= 0.000328 ms/op ( 0.328 us/op) + +Done. +``` + +#### IPv6 benchmark example + +```bash +System: + CPU : Apple M4 + Cores : 10 cores / 10 threads + Erlang : Erlang/OTP 29 [erts-17.0.2] [source] [64-bit] [smp:10:10] [ds:10:10:10] [async-threads:1] [jit] [dtrace] + Loaded : 638953 IPs in 2.949 s + +Benchmarks: + cold total= 20.504s count= 638953 qps= 31162.52 avg= 0.032090 ms/op (32.090 us/op) + warm total= 0.444s count= 638953 qps= 1437781.56 avg= 0.000696 ms/op ( 0.696 us/op) + +Done. ``` diff --git a/binding/erlang/README_zh.md b/binding/erlang/README_zh.md index 0ebc6d6..bfea45b 100644 --- a/binding/erlang/README_zh.md +++ b/binding/erlang/README_zh.md @@ -1,128 +1,179 @@ :globe_with_meridians: [中文简体](README_zh.md) | [English](README.md) -# ip2region erlang 查询客户端 +# ip2region Erlang 查询客户端 ### 简介 -该bingding以erlang语言实现xdb查询客户端,基于Erlang OTP Application,查询逻辑由ip2region_worker工作进程实现,支持配多个工作进程来进行负载均衡。 + +该 bingding 以 `Erlang` 语言实现 xdb 查询客户端,基于 `Erlang OTP Application`,查询逻辑由 `ip2region_worker` 工作进程实现,支持配多个工作进程来进行负载均衡。 ### 应用配置 -该应用可配置的参数在ip2region.app.src中,如下: -``` erlang + +该应用可配置的参数在 `ip2region.app.src` 中, 如下: + +```erlang {env,[ {poolargs, [ {size, 1}, %% 工作进程默认数量 {max_overflow, 5} %% 工作进程最大数量 + ]}, + {db, [ + {ipv4, "ip2region.xdb"} %% 默认 IPv4 xdb 文件 ]} ]} ``` +### 双栈配置(IPv4 + IPv6) + +如需启用 IPv6 查询,在 `db` 列表中加入 `ipv6` 项,并将两个 xdb 文件放到 `priv/` 目录下: + +```erlang + {env,[ + {poolargs, [ + {size, 1}, + {max_overflow, 5} + ]}, + {db, [ + {ipv4, "ip2region.xdb"}, + {ipv6, "ip2region_v6.xdb"} + ]} + ]} +``` + +`xdb:search/1` 接口会自动识别 IPv4 与 IPv6 输入,并将其路由到对应的工作进程池。 + ### 编译 -``` +```bash $ rebar3 compile ``` ### 运行 -将xdb文件放到priv目录下,然后启动erlang节点: -``` + +将 xdb 文件放到 `priv` 目录下,然后启动 Erlang 节点: + +```bash $ rebar3 shell ``` -在erlang shell中调用xdb:search/1接口查询Ip地址信息, 该接口支持以list格式字符串、binary格式字符串、tuple和整数表示的IP地址,如下: -``` + +在 Erlang shell 中调用 `xdb:search/1` 接口查询 IP 地址信息。该接口支持 list 字符串、binary 字符串、tuple 和整数表示的 IP 地址: + +```erlang 1> xdb:search("1.0.8.0"). -[20013,22269,124,48,124,24191,19996,30465,124,24191,24030, - 24066,124,30005,20449] +[20013,22269,124,24191,19996,30465,124,24191,24030,24066, + 124,20013,22269,30005,20449,124,67,78] 2> 3> io:format("~ts~n", [xdb:search("1.0.8.0")]). -中国|0|广东省|广州市|电信 -io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]). -中国|0|广东省|广州市|电信 -4> io:format("~ts~n", [xdb:search({1,0,8,0})]). -中国|0|广东省|广州市|电信 +中国|广东省|广州市|中国电信|CN +4> io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]). +中国|广东省|广州市|中国电信|CN +5> io:format("~ts~n", [xdb:search({1,0,8,0})]). +中国|广东省|广州市|中国电信|CN 6> io:format("~ts~n", [xdb:search(16779264)]). -中国|0|广东省|广州市|电信 +中国|广东省|广州市|中国电信|CN +``` + +启用双栈后,IPv6 地址也按同样方式支持: + +```erlang +1> io:format("~ts~n", [xdb:search("2001:4860:4860::8888")]). +United States|Florida|Miami|Google LLC|US +2> io:format("~ts~n", [xdb:search(<<"2001:4860:4860::8888">>)]). +United States|Florida|Miami|Google LLC|US +3> io:format("~ts~n", [xdb:search({8193,18528,18528,0,0,0,0,34952})]). +United States|Florida|Miami|Google LLC|US ``` ### 使用方法 -* 在rebar.config中引入依赖 -``` + +- 在 `rebar.config` 中引入依赖 + +```erlang {deps, [ ip2region ]}. ``` -* 启动ip2region Application -``` -...... -application:ensure_started(ip2region), +- 启动 ip2region Application -...... +```erlang +{ok, _} = application:ensure_all_started(ip2region). ``` -* 调用xdb:search/1接口查询IP信息 -``` -...... +- 调用 `xdb:search/1` 接口查询 IP 信息 -ip2region:search("1.0.8.0"), - -...... +```erlang +xdb:search("1.0.8.0"). ``` ### 单元测试 -``` +```bash $ rebar3 eunit ===> Verifying dependencies... ===> Analyzing applications... ===> Compiling ip2region ===> Performing EUnit tests... -=INFO REPORT==== 17-Jan-2023::11:52:59.920155 === -XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb +=INFO REPORT==== 28-Jun-2026::04:53:28 === +XdbFile:/Users/nana/Documents/code/ip2region/.worktrees/erlang-ipv6/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb .... -Finished in 0.074 seconds -4 tests, 0 failures +Finished in 0.192 seconds +63 tests, 0 failures ``` ### 基准测试 -``` + +IPv4 与 IPv6 共用一个脚本,通过参数指定版本: + +> `cold` = 第一次遍历源文件:每个 IP 都会触发真实查询,并把结果写入 ETS 缓存。 +> `warm` = 第二次遍历同一列表,所有查询都直接命中 ETS 缓存。 + +```bash $ cd benchmarks/ -$ sh xdb-benchmark.sh -===> Verifying dependencies... -===> Analyzing applications... -===> Compiling ip2region -Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] - -Eshell V12.3.2.2 (abort with ^G) -1> =INFO REPORT==== 17-Jan-2023::11:37:35.631095 === -XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/default/lib/ip2region/priv/ip2region.xdb - -===> Booted ip2region -===> Evaluating: "xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()." -CPU info: -model name : AMD EPYC 7K62 48-Core Processor -cache size : 512 KB -cpu MHz : 2595.124 -bogomips : 5190.24 -cores/threads : 2 - -Erlang info: -system_version:Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] -load test data use 4.835593s - -start run benchmark tests - -search from file: -ip count:683844, -total time: 28.201699s, -search 24248.326315375536 times per second, -use 41.23995969841075 micro second per search - -search from cache: -ip count:683844, -total time: 0.671801s, -search 1017926.4395259906 times per second, -use 0.9823892583688677 micro second per search - -benchmark test finish +$ sh xdb-benchmark.sh ipv4 +``` + +IPv6: + +```bash +$ sh xdb-benchmark.sh ipv6 +``` + +也可以在 `binding/erlang` 目录直接用 Makefile: + +```bash +$ make bench-v4 +$ make bench-v6 +``` + +#### IPv4 基准测试示例 + +```bash +System: + CPU : Apple M4 + Cores : 10 cores / 10 threads + Erlang : Erlang/OTP 29 [erts-17.0.2] [source] [64-bit] [smp:10:10] [ds:10:10:10] [async-threads:1] [jit] [dtrace] + Loaded : 487169 IPs in 1.335 s + +Benchmarks: + cold total= 9.601s count= 487169 qps= 50740.66 avg= 0.019708 ms/op (19.708 us/op) + warm total= 0.160s count= 487169 qps= 3053164.29 avg= 0.000328 ms/op ( 0.328 us/op) + +Done. +``` + +#### IPv6 基准测试示例 + +```bash +System: + CPU : Apple M4 + Cores : 10 cores / 10 threads + Erlang : Erlang/OTP 29 [erts-17.0.2] [source] [64-bit] [smp:10:10] [ds:10:10:10] [async-threads:1] [jit] [dtrace] + Loaded : 638953 IPs in 2.949 s + +Benchmarks: + cold total= 20.504s count= 638953 qps= 31162.52 avg= 0.032090 ms/op (32.090 us/op) + warm total= 0.444s count= 638953 qps= 1437781.56 avg= 0.000696 ms/op ( 0.696 us/op) + +Done. ``` diff --git a/binding/erlang/benchmarks/xdb-benchmark.sh b/binding/erlang/benchmarks/xdb-benchmark.sh index 40d13f3..77b647b 100755 --- a/binding/erlang/benchmarks/xdb-benchmark.sh +++ b/binding/erlang/benchmarks/xdb-benchmark.sh @@ -1,5 +1,37 @@ #!/bin/bash +set -e -cd .. +# Run from binding/erlang so paths and rebar3 artifacts are correct. +cd "$(dirname "$0")/.." -rebar3 shell --eval="xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()." +VERSION="${1:-ipv4}" +PROJECT_ROOT="$(cd ../.. && pwd)" + +if [ "$VERSION" = "ipv6" ]; then + DB_FILE="${PROJECT_ROOT}/data/ip2region_v6.xdb" + DATA_FILE="${PROJECT_ROOT}/data/ipv6_source.txt" + DB_CONFIG="[{ipv6, \"${DB_FILE}\"}]" +elif [ "$VERSION" = "ipv4" ]; then + DATA_FILE="${PROJECT_ROOT}/data/ipv4_source.txt" + DB_CONFIG="[{ipv4, \"ip2region.xdb\"}]" +else + echo "Usage: $0 [ipv4|ipv6]" + exit 1 +fi + +rebar3 compile + +# shellcheck disable=SC2089 +EVAL="application:load(ip2region), \ +application:set_env(ip2region, db, ${DB_CONFIG}), \ +application:set_env(ip2region, poolargs, [{size,4},{max_overflow,8}]), \ +{ok,_}=application:ensure_all_started(ip2region), \ +xdb_benchmark:main(\"${DATA_FILE}\"), \ +init:stop()." + +# shellcheck disable=SC2090 +erl -pa _build/default/lib/poolboy/ebin \ + -pa _build/default/lib/ip2region/ebin \ + -kernel logger_level error \ + -noshell \ + -eval "${EVAL}" diff --git a/binding/erlang/include/ip2region.hrl b/binding/erlang/include/ip2region.hrl index a0e91e7..b08a7f4 100644 --- a/binding/erlang/include/ip2region.hrl +++ b/binding/erlang/include/ip2region.hrl @@ -4,23 +4,39 @@ -define(NONE, none). -define(APP_NAME, ip2region). --define(XDB_VECTOR_INDEX, ets_xdb_vector_index). --define(XDB_SEGMENT_INDEX, ets_xdb_segement_index). --define(IP2REGION_CACHE, ets_ip2region_cache). - - -define(XDB_HEADER_SIZE, 256). -define(XDB_VECTOR_COLS, 256). --define(XDB_VECTOR_INDEX_SIZE, 8). +-define(XDB_VECTOR_INDEX_SIZE, 8). -define(XDB_VECTOR_INDEX_COUNT, (16#10000)). %% 256*256 --define(XDB_SEGMENT_INDEX_SIZE, 14). +%% IP versions as stored in xdb header +-define(IP_VERSION_4, 4). +-define(IP_VERSION_6, 6). --define(IP2REGION_POOL, ip2region_pool). +%% Segment index sizes (bytes): start_ip + end_ip + data_len + data_ptr +-define(XDB_SEGMENT_INDEX_SIZE_V4, 14). %% 4 + 4 + 2 + 4 +-define(XDB_SEGMENT_INDEX_SIZE_V6, 38). %% 16 + 16 + 2 + 4 + +%% Per-version ETS tables +-define(XDB_VECTOR_INDEX_V4, ets_xdb_vector_index_v4). +-define(XDB_VECTOR_INDEX_V6, ets_xdb_vector_index_v6). +-define(XDB_SEGMENT_INDEX_V4, ets_xdb_segment_index_v4). +-define(XDB_SEGMENT_INDEX_V6, ets_xdb_segment_index_v6). +-define(IP2REGION_CACHE_V4, ets_ip2region_cache_v4). +-define(IP2REGION_CACHE_V6, ets_ip2region_cache_v6). + +%% Per-version pool names. +-define(IP2REGION_POOL_V4, ip2region_pool_v4). +-define(IP2REGION_POOL_V6, ip2region_pool_v6). + +-record(xdb_header, { + version :: non_neg_integer(), + index_policy :: non_neg_integer(), + created_at :: non_neg_integer(), + start_index_ptr :: non_neg_integer(), + end_index_ptr :: non_neg_integer(), + ip_version :: non_neg_integer(), + runtime_ptr_bytes :: non_neg_integer() +}). --ifndef(IF). --define(IF(C, T, F), case (C) of true -> (T); false -> (F) end). --define(IF(C, T), ?IF(C, T, skip)). -endif. - --endif. \ No newline at end of file diff --git a/binding/erlang/rebar.config b/binding/erlang/rebar.config index 076f916..dd1affa 100644 --- a/binding/erlang/rebar.config +++ b/binding/erlang/rebar.config @@ -1,6 +1,6 @@ {erl_opts, [ - debug_info, - export_all, + debug_info, + export_all, nowarn_export_all ]}. @@ -18,7 +18,7 @@ {ex_doc, [ {extras, ["README.md"]}, {main, "README.md"}, - {source_url, "https://github.com/leihua996/ip2region/tree/master/binding/erlang"} + {source_url, "https://github.com/lionsoul2014/ip2region/tree/master/binding/erlang"} ]}. {hex, [{doc, ex_doc}]}. diff --git a/binding/erlang/src/ip2region.app.src b/binding/erlang/src/ip2region.app.src index 4de369c..9188b34 100644 --- a/binding/erlang/src/ip2region.app.src +++ b/binding/erlang/src/ip2region.app.src @@ -5,16 +5,20 @@ {mod, {ip2region_app, []}}, {applications, [kernel, - stdlib + stdlib, + poolboy ]}, {env,[ {poolargs, [ {size, 1}, {max_overflow, 5} + ]}, + {db, [ + {ipv4, "ip2region.xdb"} ]} ]}, {modules, []}, {licenses, ["Apache-2.0"]}, - {links, [{"Github", "https://github.com/leihua996/ip2region/tree/master/binding/erlang"}]} + {links, [{"Github", "https://github.com/lionsoul2014/ip2region/tree/master/binding/erlang"}]} ]}. diff --git a/binding/erlang/src/ip2region_sup.erl b/binding/erlang/src/ip2region_sup.erl index 304bc98..7b07742 100644 --- a/binding/erlang/src/ip2region_sup.erl +++ b/binding/erlang/src/ip2region_sup.erl @@ -2,7 +2,7 @@ %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. -%% +%% %% @doc ip2region top level supervisor. %% @end %%%------------------------------------------------------------------- @@ -18,7 +18,6 @@ start_link() -> {ok, SupPid} = supervisor:start_link({local, ?SERVER}, ?MODULE, []), - {ok, _PoolPid} = start_ip2region_pool(SupPid), {ok, SupPid}. %% sup_flags() = #{strategy => strategy(), % optional @@ -35,21 +34,45 @@ init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 5}, - ChildSpecs = [], + ChildSpecs = pool_child_specs(), {ok, {SupFlags, ChildSpecs}}. %% internal functions -%% +%% create_table() -> Opts = [named_table, set, public, {read_concurrency, true}, {keypos, 1}], - ets:new(?XDB_VECTOR_INDEX, Opts), - ets:new(?XDB_SEGMENT_INDEX, Opts), - ets:new(?IP2REGION_CACHE, Opts). + %% Version-specific tables for dual-stack support + ensure_table(?XDB_VECTOR_INDEX_V4, Opts), + ensure_table(?XDB_VECTOR_INDEX_V6, Opts), + ensure_table(?XDB_SEGMENT_INDEX_V4, Opts), + ensure_table(?XDB_SEGMENT_INDEX_V6, Opts), + ensure_table(?IP2REGION_CACHE_V4, Opts), + ensure_table(?IP2REGION_CACHE_V6, Opts). -start_ip2region_pool(Sup) -> +ensure_table(Name, Opts) -> + case ets:whereis(Name) of + undefined -> ets:new(Name, Opts); + _ -> ok + end. + +pool_child_specs() -> + {ok, DbConfig} = application:get_env(db), {ok, PoolArgsCfg} = application:get_env(poolargs), - PoolName = ?IP2REGION_POOL, - PoolArgs = [{strategy, fifo}, {name, {local, PoolName}}, {worker_module, ip2region_worker} | PoolArgsCfg], - WorkerArgs = [], - ChildSpecs = poolboy:child_spec(PoolName, PoolArgs, WorkerArgs), - supervisor:start_child(Sup, ChildSpecs). \ No newline at end of file + lists:foldl( + fun({ipv4, File}, Acc) -> + [make_pool_spec(?IP2REGION_POOL_V4, ipv4, File, PoolArgsCfg) | Acc]; + ({ipv6, File}, Acc) -> + [make_pool_spec(?IP2REGION_POOL_V6, ipv6, File, PoolArgsCfg) | Acc]; + (_, Acc) -> + Acc + end, [], DbConfig). + +make_pool_spec(PoolName, Version, File, PoolArgsCfg) -> + PoolArgs = [ + {strategy, fifo}, + {name, {local, PoolName}}, + {worker_module, ip2region_worker} + | PoolArgsCfg + ], + WorkerArgs = [{xdb_file, File}, {expected_version, Version}], + poolboy:child_spec(PoolName, PoolArgs, WorkerArgs). diff --git a/binding/erlang/src/ip2region_util.erl b/binding/erlang/src/ip2region_util.erl index 0359b85..d84dd56 100644 --- a/binding/erlang/src/ip2region_util.erl +++ b/binding/erlang/src/ip2region_util.erl @@ -2,25 +2,70 @@ %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. -%% -%% @doc -%% ip2region utils +%% +%% @doc +%% ip2region utils: IPv4/IPv6 parsing and version detection. %% @end %%%------------------------------------------------------------------- -module(ip2region_util). --export([ipv4_to_n/1]). +-export([ipv4_to_n/1, ip_version/1, ip_to_bytes/1]). -ipv4_to_n(IntIp) when is_integer(IntIp) -> IntIp; -ipv4_to_n({A, B, C, D}) -> - <> = <>, - N; -ipv4_to_n(Ip) when is_binary(Ip) -> - ipv4_to_n(binary_to_list(Ip)); -ipv4_to_n(Ip) when is_list(Ip) -> +-define(IS_UINT8(V), is_integer(V), V >= 0, V =< 255). +-define(IS_UINT16(V), is_integer(V), V >= 0, V =< 65535). + +-spec ip_version(Ip :: tuple() | list() | binary() | integer()) -> + ipv4 | ipv6 | {error, atom()}. +ip_version(Ip) when is_integer(Ip), Ip >= 0, Ip =< 16#FFFFFFFF -> ipv4; +ip_version({A, B, C, D}) when ?IS_UINT8(A), ?IS_UINT8(B), ?IS_UINT8(C), ?IS_UINT8(D) -> ipv4; +ip_version({A, B, C, D, E, F, G, H}) when ?IS_UINT16(A), ?IS_UINT16(B), ?IS_UINT16(C), ?IS_UINT16(D), + ?IS_UINT16(E), ?IS_UINT16(F), ?IS_UINT16(G), ?IS_UINT16(H) -> ipv6; +ip_version(Ip) when is_binary(Ip) -> + ip_version(binary_to_list(Ip)); +ip_version(Ip) when is_list(Ip) -> case inet_parse:address(Ip) of - {ok, Addr} -> - ipv4_to_n(Addr); + {ok, {_, _, _, _}} -> ipv4; + {ok, {_, _, _, _, _, _, _, _}} -> ipv6; + _ -> {error, bad_ip_format} + end; +ip_version(_) -> + {error, bad_ip_format}. + +-spec ip_to_bytes(Ip :: tuple() | list() | binary() | integer()) -> + {ok, ipv4 | ipv6, binary()} | {error, atom()}. +ip_to_bytes(Ip) when is_integer(Ip), Ip >= 0, Ip =< 16#FFFFFFFF -> + {ok, ipv4, <>}; +ip_to_bytes({A, B, C, D}) when ?IS_UINT8(A), ?IS_UINT8(B), ?IS_UINT8(C), ?IS_UINT8(D) -> + {ok, ipv4, <>}; +ip_to_bytes({A, B, C, D, E, F, G, H}) when ?IS_UINT16(A), ?IS_UINT16(B), ?IS_UINT16(C), ?IS_UINT16(D), + ?IS_UINT16(E), ?IS_UINT16(F), ?IS_UINT16(G), ?IS_UINT16(H) -> + {ok, ipv6, <>}; +ip_to_bytes(Ip) when is_binary(Ip) -> + ip_to_bytes(binary_to_list(Ip)); +ip_to_bytes(Ip) when is_list(Ip) -> + case inet_parse:address(Ip) of + {ok, {A, B, C, D}} -> + {ok, ipv4, <>}; + {ok, {A, B, C, D, E, F, G, H}} -> + {ok, ipv6, <>}; _ -> {error, bad_ip_format} - end. \ No newline at end of file + end; +ip_to_bytes(_) -> + {error, bad_ip_format}. + +-spec ipv4_to_n(Ip :: tuple() | list() | binary() | integer()) -> + non_neg_integer() | {error, atom()}. +ipv4_to_n(IntIp) when is_integer(IntIp), IntIp >= 0, IntIp =< 16#FFFFFFFF -> IntIp; +ipv4_to_n({A, B, C, D}) when ?IS_UINT8(A), ?IS_UINT8(B), ?IS_UINT8(C), ?IS_UINT8(D) -> + <> = <>, + N; +ipv4_to_n(Ip) when is_binary(Ip) -> + ipv4_to_n(binary_to_list(Ip)); +ipv4_to_n(Ip) when is_list(Ip) -> + case ip_to_bytes(Ip) of + {ok, ipv4, <>} -> N; + _ -> {error, bad_ip_format} + end; +ipv4_to_n(_) -> + {error, bad_ip_format}. diff --git a/binding/erlang/src/ip2region_worker.erl b/binding/erlang/src/ip2region_worker.erl index 64d039c..90d9152 100644 --- a/binding/erlang/src/ip2region_worker.erl +++ b/binding/erlang/src/ip2region_worker.erl @@ -2,9 +2,9 @@ %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. -%% -%% @doc -%% ip2region xdb client worker +%% +%% @doc +%% ip2region xdb client worker, now version-aware (IPv4/IPv6). %% @end %%%------------------------------------------------------------------- -module(ip2region_worker). @@ -15,7 +15,11 @@ -export([start/1, stop/1, start_link/1]). -export([search/2]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --record(state, {xdb_fd}). + +-record(state, { + xdb_fd, + version :: ipv4 | ipv6 +}). %%========================================== %% API @@ -28,7 +32,6 @@ start_link(Args) -> Opts = [{spawn_opt, [{min_heap_size, 6000}]}], gen_server:start_link(?MODULE, Args, Opts). - stop(Pid) -> gen_server:call(Pid, stop). @@ -38,26 +41,46 @@ search(Pid, Ip) -> %%========================================== %% gen_server callbacks %% ========================================= -init(_Args) -> +init(Args) -> process_flag(trap_exit, true), - AppName = + AppName = case application:get_application() of {ok, AName} -> AName; _ -> ?APP_NAME end, PrivDir = code:priv_dir(AppName), - XdbFileName = filename:join([PrivDir, "ip2region.xdb"]), + XdbFileName = + case proplists:get_value(xdb_file, Args) of + undefined -> filename:join([PrivDir, "ip2region.xdb"]); + Path -> + case filename:pathtype(Path) of + absolute -> Path; + _ -> filename:join([PrivDir, Path]) + end + end, error_logger:info_report(io_lib:format("XdbFile:~s~n", [XdbFileName])), {ok, IoDevice} = file:open(XdbFileName, [read, binary]), - load_vector_index(IoDevice), - {ok, #state{xdb_fd = IoDevice}}. + {ok, HeaderBin} = file:read(IoDevice, ?XDB_HEADER_SIZE), + {ok, Header} = ip2region_xdb:parse_header(HeaderBin), + ExpectedVersion = proplists:get_value(expected_version, Args), + case resolve_version(Header) of + {ok, Version} when ExpectedVersion =:= undefined; ExpectedVersion =:= Version -> + load_vector_index(IoDevice, Version), + {ok, #state{xdb_fd = IoDevice, version = Version}}; + {ok, Version} -> + file:close(IoDevice), + {stop, {xdb_version_mismatch, ExpectedVersion, Version}}; + {error, Reason} -> + file:close(IoDevice), + {stop, Reason} + end. handle_call(Request, From, State) -> try do_call(Request, From, State) catch Class:Error:Stacktrace -> - error_logger:error_report(io_lib:format("~p handle call error, Req:~p ~p, stacktrace:~p~n", + error_logger:error_report(io_lib:format("~p handle call error, Req:~p ~p, stacktrace:~p~n", [?MODULE, Request, {Class, Error}, Stacktrace])), {reply, {error, {Class, Error}}, State} end. @@ -67,7 +90,7 @@ handle_cast(Msg, State) -> do_cast(Msg, State) catch Class:Error:Stacktrace -> - error_logger:error_report(io_lib:format("~p handle cast error, Msg:~p, ~p, stacktrace:~w~n", + error_logger:error_report(io_lib:format("~p handle cast error, Msg:~p, ~p, stacktrace:~w~n", [?MODULE, Msg, {Class, Error}, Stacktrace])), {noreply, State} end. @@ -77,7 +100,7 @@ handle_info(Info, State) -> do_info(Info, State) catch Class:Error:Stacktrace -> - error_logger:error_report(io_lib:format("~p handle info error, Info:~p, ~p, stacktrace:~p~n", + error_logger:error_report(io_lib:format("~p handle info error, Info:~p, ~p, stacktrace:~p~n", [?MODULE, Info, {Class, Error}, Stacktrace])), {noreply, State} end. @@ -85,22 +108,19 @@ handle_info(Info, State) -> terminate(_Reason, State) -> #state{xdb_fd = XdbFd} = State, case is_pid(XdbFd) of - true -> - file:close(XdbFd); - _ -> - skip + true -> file:close(XdbFd); + _ -> skip end, ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. - %%========================================== %% Internal function %% ========================================= do_call({search, Ip}, _From, #state{xdb_fd = IoDevice} = State) -> - Reply = search_ip(IoDevice, Ip), + Reply = search_ip(IoDevice, Ip, State), {reply, Reply, State}; do_call(stop, _From, State) -> @@ -108,7 +128,7 @@ do_call(stop, _From, State) -> do_call(Request, From, State) -> error_logger:error_report(io_lib:format("unknown request: ~p, from:~p", [Request, From])), - {noreply, State}. + {reply, {error, unknown_request}, State}. do_cast(Msg, State) -> error_logger:error_report(io_lib:format("unknown msg: ~p", [Msg])), @@ -117,67 +137,105 @@ do_cast(Msg, State) -> do_info(Info, State) -> error_logger:error_report(io:format("unknown info: ~p", [Info])), {noreply, State}. - -load_vector_index(IoDevice) -> - Key = ip2region_header_loaded, - case persistent_term:get(Key, false) of - true -> ok; - _ -> - {ok, <<_Header:?XDB_HEADER_SIZE/binary, VectorIndexBin/binary>> } = - file:read(IoDevice, ?XDB_HEADER_SIZE + ?XDB_VECTOR_INDEX_COUNT*8), - load_vector_index_aux(VectorIndexBin, 0), - persistent_term:put(Key, true) +-spec resolve_version(#xdb_header{}) -> {ok, ipv4 | ipv6} | {error, term()}. +resolve_version(Header) -> + case ip2region_xdb:header_version(Header) of + 2 -> {ok, ipv4}; + 3 -> + case ip2region_xdb:header_ip_version(Header) of + ?IP_VERSION_4 -> {ok, ipv4}; + ?IP_VERSION_6 -> {ok, ipv6}; + V -> {error, {invalid_xdb_ip_version, V}} + end; + V -> {error, {invalid_xdb_version, V}} end. -load_vector_index_aux(<<>>, _Index) -> ok; -load_vector_index_aux(<>, Index) -> - Term = {Index, SPtr, EPtr}, - ets:insert(?XDB_VECTOR_INDEX, Term), - load_vector_index_aux(VectorIndexBin, Index + 1). - - -search_ip(IoDevice, Ip) -> - IntIp = ip2region_util:ipv4_to_n(Ip), - case ets:lookup(?IP2REGION_CACHE, IntIp) of - [{_IntIp, RegionInfo}] -> - RegionInfo; +load_vector_index(IoDevice, Version) -> + %% The vector index ETS table is global and shared by all pool workers. + %% Only the first worker (or the first worker after a restart) populates it. + %% Subsequent workers skip the file:read here; search_ip/2 uses file:pread/3, + %% so the current file pointer position does not affect correctness. + Table = ip2region_xdb:vector_index_table(Version), + case ets:info(Table, size) of + ?XDB_VECTOR_INDEX_COUNT -> + ok; + undefined -> + Opts = [named_table, set, public, {read_concurrency, true}, {keypos, 1}], + ets:new(Table, Opts), + load_vector_index_data(IoDevice, Table); _ -> - <> = <>, - VectorIdx = A * ?XDB_VECTOR_COLS + B, - [{_, SPtr, EPtr}] = ets:lookup(?XDB_VECTOR_INDEX, VectorIdx), - RegionInfo = search_ip(IoDevice, IntIp, SPtr, EPtr, 0, (EPtr - SPtr) div ?XDB_SEGMENT_INDEX_SIZE), - ets:insert_new(?IP2REGION_CACHE, {IntIp, RegionInfo}), - RegionInfo + %% A previous worker may have crashed midway; reload to be safe. + load_vector_index_data(IoDevice, Table) end. -search_ip(IoDevice, IntIp, SPtr, EPtr, Low, High) when Low =< High -> +load_vector_index_data(IoDevice, Table) -> + {ok, VectorIndexBin} = + file:read(IoDevice, ?XDB_VECTOR_INDEX_COUNT * ?XDB_VECTOR_INDEX_SIZE), + load_vector_index_aux(VectorIndexBin, 0, Table). + +load_vector_index_aux(<<>>, _Index, _Table) -> ok; +load_vector_index_aux(<>, Index, Table) -> + ets:insert(Table, {Index, SPtr, EPtr}), + load_vector_index_aux(VectorIndexBin, Index + 1, Table). + +search_ip(IoDevice, Ip, #state{version = Version}) -> + SegSize = ip2region_xdb:segment_index_size(Version), + VectorTable = ip2region_xdb:vector_index_table(Version), + SegmentTable = ip2region_xdb:segment_index_table(Version), + <> = Ip, + VectorIdx = A * ?XDB_VECTOR_COLS + B, + [{_, SPtr, EPtr}] = ets:lookup(VectorTable, VectorIdx), + search_ip(IoDevice, Ip, SPtr, EPtr, 0, + (EPtr - SPtr) div SegSize, SegSize, Version, SegmentTable). + +search_ip(IoDevice, Ip, SPtr, EPtr, Low, High, SegSize, Version, SegmentTable) when Low =< High -> Middle = (Low + High) bsr 1, - SPtr2 = SPtr + Middle * ?XDB_SEGMENT_INDEX_SIZE, - {SIp, EIp, DataLen, DataPtr} = read_segement_index(IoDevice, SPtr2), - if - IntIp < SIp -> - search_ip(IoDevice, IntIp, SPtr, EPtr, Low, Middle - 1); - IntIp > EIp -> - search_ip(IoDevice, IntIp, SPtr, EPtr, Middle + 1, High); - true -> + SPtr2 = SPtr + Middle * SegSize, + {SIp, EIp, DataLen, DataPtr} = read_segment_index(IoDevice, SPtr2, SegSize, SegmentTable), + case ip_in_range(Ip, SIp, EIp, Version) of + below -> + search_ip(IoDevice, Ip, SPtr, EPtr, Low, Middle - 1, SegSize, Version, SegmentTable); + above -> + search_ip(IoDevice, Ip, SPtr, EPtr, Middle + 1, High, SegSize, Version, SegmentTable); + inside -> {ok, DataBin} = read_file(IoDevice, DataPtr, DataLen), unicode:characters_to_nfc_list(DataBin) end; -search_ip(_IoDevice, _IntIp, _SPtr, _EPtr, _Low, _High) -> +search_ip(_IoDevice, _Ip, _SPtr, _EPtr, _Low, _High, _SegSize, _Version, _SegmentTable) -> {error, unknown}. -read_file(IoDevice, Position, DataLength) -> - file:position(IoDevice, {bof, Position}), - file:read(IoDevice, DataLength). +ip_in_range(Ip, SIp, EIp, ipv4) -> + <> = Ip, + if + InputInt < SIp -> below; + InputInt > EIp -> above; + true -> inside + end; +ip_in_range(Ip, SIp, EIp, ipv6) -> + if + Ip < SIp -> below; + Ip > EIp -> above; + true -> inside + end. -read_segement_index(IoDevice, SPtr) -> - case ets:lookup(?XDB_SEGMENT_INDEX, SPtr) of +read_file(IoDevice, Position, DataLength) -> + file:pread(IoDevice, Position, DataLength). + +read_segment_index(IoDevice, SPtr, SegSize, SegmentTable) -> + case ets:lookup(SegmentTable, SPtr) of [{_SPtr, SIp, EIp, DataLen, DataPtr}] -> {SIp, EIp, DataLen, DataPtr}; _ -> - {ok, <>} = - read_file(IoDevice, SPtr, ?XDB_SEGMENT_INDEX_SIZE), - ets:insert_new(?XDB_SEGMENT_INDEX, {SPtr, SIp, EIp, DataLen, DataPtr}), + {ok, Bin} = read_file(IoDevice, SPtr, SegSize), + {SIp, EIp, DataLen, DataPtr} = decode_segment_index(Bin, SegSize), + ets:insert_new(SegmentTable, {SPtr, SIp, EIp, DataLen, DataPtr}), {SIp, EIp, DataLen, DataPtr} - end. \ No newline at end of file + end. + +decode_segment_index(Bin, ?XDB_SEGMENT_INDEX_SIZE_V4) -> + <> = Bin, + {SIp, EIp, DataLen, DataPtr}; +decode_segment_index(Bin, ?XDB_SEGMENT_INDEX_SIZE_V6) -> + <> = Bin, + {SIp, EIp, DataLen, DataPtr}. diff --git a/binding/erlang/src/ip2region_xdb.erl b/binding/erlang/src/ip2region_xdb.erl new file mode 100644 index 0000000..383f318 --- /dev/null +++ b/binding/erlang/src/ip2region_xdb.erl @@ -0,0 +1,60 @@ +%%%------------------------------------------------------------------- +%% Copyright 2022 The Ip2Region Authors. All rights reserved. +%% Use of this source code is governed by a Apache2.0-style +%% license that can be found in the LICENSE file. +%% +%% @doc +%% ip2region xdb binary format helpers. +%% @end +%%%------------------------------------------------------------------- +-module(ip2region_xdb). + +-export([parse_header/1]). +-export([header_version/1, header_index_policy/1, header_created_at/1, + header_start_index_ptr/1, header_end_index_ptr/1, + header_ip_version/1, header_runtime_ptr_bytes/1]). +-export([segment_index_size/1]). +-export([vector_index_table/1, segment_index_table/1, cache_table/1]). + +-include("ip2region.hrl"). + +-spec parse_header(binary()) -> {ok, #xdb_header{}} | {error, invalid_header}. +parse_header(Bin) when is_binary(Bin), byte_size(Bin) >= 20 -> + <> = Bin, + {ok, #xdb_header{ + version = Version, + index_policy = IndexPolicy, + created_at = CreatedAt, + start_index_ptr = StartIndexPtr, + end_index_ptr = EndIndexPtr, + ip_version = IpVersion, + runtime_ptr_bytes = RuntimePtrBytes + }}; +parse_header(_) -> + {error, invalid_header}. + +header_version(#xdb_header{version = V}) -> V. +header_index_policy(#xdb_header{index_policy = P}) -> P. +header_created_at(#xdb_header{created_at = T}) -> T. +header_start_index_ptr(#xdb_header{start_index_ptr = P}) -> P. +header_end_index_ptr(#xdb_header{end_index_ptr = P}) -> P. +header_ip_version(#xdb_header{ip_version = V}) -> V. +header_runtime_ptr_bytes(#xdb_header{runtime_ptr_bytes = B}) -> B. + +-spec segment_index_size(ipv4 | ipv6) -> pos_integer(). +segment_index_size(ipv4) -> ?XDB_SEGMENT_INDEX_SIZE_V4; +segment_index_size(ipv6) -> ?XDB_SEGMENT_INDEX_SIZE_V6. + +-spec vector_index_table(ipv4 | ipv6) -> atom(). +vector_index_table(ipv4) -> ?XDB_VECTOR_INDEX_V4; +vector_index_table(ipv6) -> ?XDB_VECTOR_INDEX_V6. + +-spec segment_index_table(ipv4 | ipv6) -> atom(). +segment_index_table(ipv4) -> ?XDB_SEGMENT_INDEX_V4; +segment_index_table(ipv6) -> ?XDB_SEGMENT_INDEX_V6. + +-spec cache_table(ipv4 | ipv6) -> atom(). +cache_table(ipv4) -> ?IP2REGION_CACHE_V4; +cache_table(ipv6) -> ?IP2REGION_CACHE_V6. diff --git a/binding/erlang/src/xdb.erl b/binding/erlang/src/xdb.erl index 2b98ac6..66b1c0d 100644 --- a/binding/erlang/src/xdb.erl +++ b/binding/erlang/src/xdb.erl @@ -1,11 +1,10 @@ - %%%------------------------------------------------------------------- %% Copyright 2022 The Ip2Region Authors. All rights reserved. %% Use of this source code is governed by a Apache2.0-style %% license that can be found in the LICENSE file. -%% -%% @doc -%% ip2region xdb client search api +%% +%% @doc +%% ip2region xdb client search api with IPv4/IPv6 auto-routing. %% @end %%%------------------------------------------------------------------- -module(xdb). @@ -13,20 +12,37 @@ -export([search/1]). --spec search(Ip :: tuple() | list() | binary()) -> Result :: binary | {error, Reason::atom()}. -search(Ip) when is_integer(Ip); is_list(Ip); is_tuple(Ip); is_binary(Ip) -> - case ip2region_util:ipv4_to_n(Ip) of - IntIp when is_integer(IntIp) -> - case ets:lookup(?IP2REGION_CACHE, IntIp) of - [{_IntIp, Region}] -> Region; - _ -> - Worker = poolboy:checkout(?IP2REGION_POOL, true, infinity), - try - ip2region_worker:search(Worker, IntIp) - after - poolboy:checkin(?IP2REGION_POOL, Worker) - end - end; - Ret -> - Ret +-spec search(Ip :: tuple() | list() | binary() | integer()) -> + Result :: list() | {error, term()}. +search(Ip) -> + case ip2region_util:ip_to_bytes(Ip) of + {ok, ipv4, IpBin} -> + do_search(?IP2REGION_POOL_V4, ipv4, IpBin); + {ok, ipv6, IpBin} -> + do_search(?IP2REGION_POOL_V6, ipv6, IpBin); + Ret -> + Ret + end. + +do_search(PoolName, Version, IpBin) -> + CacheTable = ip2region_xdb:cache_table(Version), + case ets:lookup(CacheTable, IpBin) of + [{_, Region}] -> Region; + _ -> + case whereis(PoolName) of + undefined -> + {error, pool_not_configured}; + _ -> + Worker = poolboy:checkout(PoolName, true, infinity), + try + case ip2region_worker:search(Worker, IpBin) of + {error, _} = Err -> Err; + Region -> + ets:insert(CacheTable, {IpBin, Region}), + Region + end + after + poolboy:checkin(PoolName, Worker) + end + end end. diff --git a/binding/erlang/src/xdb_benchmark.erl b/binding/erlang/src/xdb_benchmark.erl index e52801c..36d94d8 100644 --- a/binding/erlang/src/xdb_benchmark.erl +++ b/binding/erlang/src/xdb_benchmark.erl @@ -11,29 +11,84 @@ -export([main/1]). main(DataFile) -> - application:ensure_started(ip2region), + %% Keep benchmark output clean while still surfacing real errors. + _ = logger:set_handler_config(default, level, error), + _ = logger:set_primary_config(level, error), + {ok, _} = application:ensure_all_started(ip2region), show_hw_sw_info(), IpList = load_test_data(DataFile), run(IpList). show_hw_sw_info() -> - io:format("CPU info:~n", []), - io:format("~s", [os:cmd("egrep '^model name' /proc/cpuinfo | head -1")]), - io:format("~s", [os:cmd("egrep '^cache' /proc/cpuinfo | head -1")]), - io:format("~s", [os:cmd("egrep '^cpu MHz' /proc/cpuinfo | head -1")]), - io:format("~s", [os:cmd("egrep '^bogomips' /proc/cpuinfo | head -1")]), - io:format("cores/threads : ~s~n", [os:cmd("egrep -c '^processor' /proc/cpuinfo")]), - io:format("Erlang info:~n", []), - io:format("system_version:~s", [erlang:system_info(system_version)]), + {Model, Clock, Cores, Threads} = cpu_info(), + io:format("~nSystem:~n", []), + io:format(" CPU : ~s", [Model]), + case Clock of + "" -> io:format("~n", []); + _ -> io:format(" @ ~s~n", [Clock]) + end, + io:format(" Cores : ~s cores / ~s threads~n", [Cores, Threads]), + io:format(" Erlang : ~s~n", [string:trim(erlang:system_info(system_version))]), ok. +cpu_info() -> + case os:type() of + {unix, darwin} -> + Model = sysctl("machdep.cpu.brand_string"), + Clock = first_non_empty([ + format_clock(sysctl("hw.cpufrequency")), + format_clock(sysctl("hw.perflevel0.frequency")), + format_clock(sysctl("hw.perflevel1.frequency")) + ]), + Cores = sysctl("hw.physicalcpu"), + Threads = sysctl("hw.logicalcpu"), + {Model, Clock, Cores, Threads}; + {unix, linux} -> + Model = linux_cpu_field("model name"), + Clock = format_clock_mhz(linux_cpu_field("cpu MHz")), + Cores = string:trim(os:cmd("grep -c '^processor' /proc/cpuinfo 2>/dev/null")), + Threads = Cores, + {Model, Clock, Cores, Threads}; + _ -> + {"unknown", "", "?", "?"} + end. + +sysctl(Key) -> + string:trim(os:cmd("sysctl -n " ++ Key ++ " 2>/dev/null")). + +linux_cpu_field(Key) -> + Cmd = "grep -m1 '^" ++ Key ++ "' /proc/cpuinfo 2>/dev/null | cut -d: -f2- | sed 's/^ *//'", + string:trim(os:cmd(Cmd)). + +first_non_empty(["" | Rest]) -> first_non_empty(Rest); +first_non_empty([Val | _]) -> Val; +first_non_empty([]) -> "". + +format_clock(HzStr) -> + case string:to_integer(string:trim(HzStr)) of + {ok, Hz, _} when Hz > 1000000000 -> + lists:flatten(io_lib:format("~.2f GHz", [Hz / 1000000000])); + {ok, Hz, _} when Hz > 1000000 -> + lists:flatten(io_lib:format("~.2f MHz", [Hz / 1000000])); + _ -> + "" + end. + +format_clock_mhz(MhzStr) -> + case string:to_float(string:trim(MhzStr)) of + {ok, Mhz, _} -> + lists:flatten(io_lib:format("~.3f GHz", [Mhz / 1000])); + _ -> + "" + end. + load_test_data(DataFile) -> {ok, Fd} = file:open(DataFile, [read]), T0 = os:timestamp(), IpList = load_test_data(Fd, []), T1 = os:timestamp(), Sec = timer:now_diff(T1, T0) / 1000000, - io:format("load test data use ~ps~n", [Sec]), + io:format(" Loaded : ~p IPs in ~.3f s~n", [length(IpList), Sec]), IpList. load_test_data(Fd, IpList) -> @@ -46,30 +101,30 @@ load_test_data(Fd, IpList) -> load_test_data(Fd, IpList) end; _ -> - file:close(Fd), + file:close(Fd), IpList end. run(IpList) -> garbage_collect(), - io:format("~nstart run benchmark tests~n", []), - io:format("~nsearch from file:~n", []), - run_test(IpList), - io:format("~nsearch from cache:~n", []), - run_test(IpList), - io:format("~nbenchmark test finish~n", []). + io:format("~nBenchmarks:~n", []), + run_test("cold", IpList), + run_test("warm", IpList), + io:format("~nDone.~n", []). -run_test(IpList) -> +run_test(Label, IpList) -> T0 = os:timestamp(), run_test_aux(IpList), T1 = os:timestamp(), Sec = timer:now_diff(T1, T0) / 1000000, - IpCount = length(IpList), - io:format("ip count:~p,~ntotal time: ~ps,~nsearch ~p times per second,~nuse ~p micro second per search~n", - [IpCount, Sec, IpCount / Sec, Sec * 1000000/IpCount]). + Count = length(IpList), + Qps = Count / Sec, + MsOp = Sec * 1000 / Count, + UsOp = Sec * 1000000 / Count, + io:format(" ~-8s total=~7.3fs count=~7w qps=~12.2f avg=~9.6f ms/op (~6.3f us/op)~n", + [Label, Sec, Count, Qps, MsOp, UsOp]). run_test_aux([]) -> ok; run_test_aux([Ip | Tail]) -> xdb:search(Ip), run_test_aux(Tail). - diff --git a/binding/erlang/test/ip2region_sup_test.erl b/binding/erlang/test/ip2region_sup_test.erl new file mode 100644 index 0000000..9c86e12 --- /dev/null +++ b/binding/erlang/test/ip2region_sup_test.erl @@ -0,0 +1,8 @@ +-module(ip2region_sup_test). +-include_lib("eunit/include/eunit.hrl"). +-include("ip2region.hrl"). + +pools_started_test() -> + {ok, _} = ip2region_test_util:start_app(), + ?assert(is_pid(whereis(?IP2REGION_POOL_V4))), + ?assert(is_pid(whereis(?IP2REGION_POOL_V6))). diff --git a/binding/erlang/test/ip2region_test_util.erl b/binding/erlang/test/ip2region_test_util.erl new file mode 100644 index 0000000..4e7417b --- /dev/null +++ b/binding/erlang/test/ip2region_test_util.erl @@ -0,0 +1,51 @@ +-module(ip2region_test_util). + +-export([ + repo_root/0, + v6_xdb_path/0, + default_db_config/0, + reset_app/0, + load_app/0, + set_db_config/1, + start_app/0, + start_app/1, + stop_app/0 +]). + +repo_root() -> + TestDir = filename:dirname(?FILE), + ErlangDir = filename:dirname(TestDir), + BindingDir = filename:dirname(ErlangDir), + filename:dirname(BindingDir). + +v6_xdb_path() -> + filename:join([repo_root(), "data", "ip2region_v6.xdb"]). + +default_db_config() -> + [ + {ipv4, "ip2region.xdb"}, + {ipv6, v6_xdb_path()} + ]. + +reset_app() -> + try application:stop(ip2region) catch _:_ -> ok end, + try application:unload(ip2region) catch _:_ -> ok end, + ok. + +load_app() -> + ok = application:load(ip2region). + +set_db_config(Config) -> + ok = application:set_env(ip2region, db, Config). + +start_app() -> + start_app(default_db_config()). + +start_app(Config) -> + reset_app(), + load_app(), + set_db_config(Config), + application:ensure_all_started(ip2region). + +stop_app() -> + application:stop(ip2region). diff --git a/binding/erlang/test/ip2region_util_test.erl b/binding/erlang/test/ip2region_util_test.erl new file mode 100644 index 0000000..85880de --- /dev/null +++ b/binding/erlang/test/ip2region_util_test.erl @@ -0,0 +1,66 @@ +-module(ip2region_util_test). +-include_lib("eunit/include/eunit.hrl"). + +ipv4_inputs_test_() -> + [ + ?_assertEqual({ok, ipv4, <<1,0,8,0>>}, ip2region_util:ip_to_bytes("1.0.8.0")), + ?_assertEqual({ok, ipv4, <<1,0,8,0>>}, ip2region_util:ip_to_bytes(<<"1.0.8.0">>)), + ?_assertEqual({ok, ipv4, <<1,0,8,0>>}, ip2region_util:ip_to_bytes({1,0,8,0})), + ?_assertEqual({ok, ipv4, <<1,0,8,0>>}, ip2region_util:ip_to_bytes(16779264)), + ?_assertEqual(ipv4, ip2region_util:ip_version("1.0.8.0")), + ?_assertEqual(ipv4, ip2region_util:ip_version({1,0,8,0})) + ]. + +ipv6_inputs_test_() -> + [ + ?_assertEqual({ok, ipv6, <<0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1>>}, + ip2region_util:ip_to_bytes("::1")), + ?_assertEqual({ok, ipv6, <<0,0,0,0,0,0,0,0,0,0,255,255,1,0,8,0>>}, + ip2region_util:ip_to_bytes("::ffff:1.0.8.0")), + ?_assertEqual({ok, ipv6, <<0,0,0,0,0,0,0,0,0,0,255,255,1,0,8,0>>}, + ip2region_util:ip_to_bytes(<<"::ffff:1.0.8.0">>)), + ?_assertEqual({ok, ipv6, <<0,0,0,0,0,0,0,0,0,0,255,255,1,0,8,0>>}, + ip2region_util:ip_to_bytes({0,0,0,0,0,65535,256,2048})), + ?_assertEqual(ipv6, ip2region_util:ip_version("::1")), + ?_assertEqual(ipv6, ip2region_util:ip_version({0,0,0,0,0,0,0,1})) + ]. + +invalid_inputs_test_() -> + [ + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes("xxx.0.8.0")), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes("::ggg")), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes({1,2,3})), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes({1,2,3,4,5,6,7})), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes(atom)), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes(-1)), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes(16#100000000)), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes({256, 0, 0, 1})), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_to_bytes({-1, 0, 0, 0})), + ?_assertEqual({error, bad_ip_format}, + ip2region_util:ip_to_bytes({0, 0, 0, 0, 0, 0, 0, 65536})), + ?_assertEqual({error, bad_ip_format}, + ip2region_util:ip_to_bytes({0, 0, 0, 0, 0, 0, 0, -1})), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ipv4_to_n(-1)), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ipv4_to_n(16#100000000)), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ipv4_to_n({256, 0, 0, 1})) + ]. + +ip_version_test_() -> + [ + ?_assertEqual(ipv4, ip2region_util:ip_version(<<"1.0.8.0">>)), + ?_assertEqual(ipv6, ip2region_util:ip_version(<<"::1">>)), + ?_assertEqual(ipv4, ip2region_util:ip_version(16779264 + 1)), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version("::ggg")), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version({1,2,3})), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version({256,0,0,1})), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version({0,0,0,0,0,0,0,65536})), + ?_assertEqual({error, bad_ip_format}, ip2region_util:ip_version(16#100000000)) + ]. + +legacy_ipv4_to_n_test_() -> + [ + ?_assertEqual(16779264, ip2region_util:ipv4_to_n("1.0.8.0")), + ?_assertEqual(16779264, ip2region_util:ipv4_to_n(<<"1.0.8.0">>)), + ?_assertEqual(16779264, ip2region_util:ipv4_to_n({1,0,8,0})), + ?_assertEqual(16779264, ip2region_util:ipv4_to_n(16779264)) + ]. diff --git a/binding/erlang/test/ip2region_worker_test.erl b/binding/erlang/test/ip2region_worker_test.erl new file mode 100644 index 0000000..5a2417c --- /dev/null +++ b/binding/erlang/test/ip2region_worker_test.erl @@ -0,0 +1,13 @@ +-module(ip2region_worker_test). +-include_lib("eunit/include/eunit.hrl"). +-include("ip2region.hrl"). + +worker_search_v4_binary_test() -> + ip2region_sup:create_table(), + {ok, Pid} = ip2region_worker:start_link([{xdb_file, "ip2region.xdb"}]), + try + Region = ip2region_worker:search(Pid, <<1,0,8,0>>), + ?assert(is_list(Region)) + after + ip2region_worker:stop(Pid) + end. diff --git a/binding/erlang/test/ip2region_xdb_test.erl b/binding/erlang/test/ip2region_xdb_test.erl new file mode 100644 index 0000000..cf3084c --- /dev/null +++ b/binding/erlang/test/ip2region_xdb_test.erl @@ -0,0 +1,43 @@ +-module(ip2region_xdb_test). + +-include_lib("eunit/include/eunit.hrl"). +-include("ip2region.hrl"). + +valid_v4_header_test() -> + HeaderBin = <<2:16/little, 1:16/little, 0:32/little, + 256:32/little, 1024:32/little, + ?IP_VERSION_4:16/little, 4:16/little>>, + {ok, Header} = ip2region_xdb:parse_header(HeaderBin), + ?assertEqual(2, ip2region_xdb:header_version(Header)), + ?assertEqual(1, ip2region_xdb:header_index_policy(Header)), + ?assertEqual(256, ip2region_xdb:header_start_index_ptr(Header)), + ?assertEqual(1024, ip2region_xdb:header_end_index_ptr(Header)), + ?assertEqual(?IP_VERSION_4, ip2region_xdb:header_ip_version(Header)), + ?assertEqual(4, ip2region_xdb:header_runtime_ptr_bytes(Header)). + +valid_v6_header_test() -> + HeaderBin = <<3:16/little, 1:16/little, 0:32/little, + 256:32/little, 1024:32/little, + ?IP_VERSION_6:16/little, 4:16/little>>, + {ok, Header} = ip2region_xdb:parse_header(HeaderBin), + ?assertEqual(3, ip2region_xdb:header_version(Header)), + ?assertEqual(?IP_VERSION_6, ip2region_xdb:header_ip_version(Header)). + +invalid_header_test() -> + ?assertEqual({error, invalid_header}, ip2region_xdb:parse_header(<<0:128>>)). + +table_helpers_test_() -> + [ + ?_assertEqual(?XDB_VECTOR_INDEX_V4, ip2region_xdb:vector_index_table(ipv4)), + ?_assertEqual(?XDB_VECTOR_INDEX_V6, ip2region_xdb:vector_index_table(ipv6)), + ?_assertEqual(?XDB_SEGMENT_INDEX_V4, ip2region_xdb:segment_index_table(ipv4)), + ?_assertEqual(?XDB_SEGMENT_INDEX_V6, ip2region_xdb:segment_index_table(ipv6)), + ?_assertEqual(?IP2REGION_CACHE_V4, ip2region_xdb:cache_table(ipv4)), + ?_assertEqual(?IP2REGION_CACHE_V6, ip2region_xdb:cache_table(ipv6)) + ]. + +segment_index_size_test_() -> + [ + ?_assertEqual(14, ip2region_xdb:segment_index_size(ipv4)), + ?_assertEqual(38, ip2region_xdb:segment_index_size(ipv6)) + ]. diff --git a/binding/erlang/test/xdb_test.erl b/binding/erlang/test/xdb_test.erl index 85adcac..5388f6e 100644 --- a/binding/erlang/test/xdb_test.erl +++ b/binding/erlang/test/xdb_test.erl @@ -2,9 +2,11 @@ -include_lib("eunit/include/eunit.hrl"). +-define(IPV6_RESULT, "United States|Florida|Miami|Google LLC|US"). + search_test_() -> - application:ensure_started(ip2region), - A = "中国|0|广东省|广州市|电信", + {ok, _} = ip2region_test_util:start_app(), + A = "中国|广东省|广州市|中国电信|CN", Region0 = xdb:search("1.0.8.0"), Region1 = xdb:search(<<"1.0.8.0">>), Region2 = xdb:search({1,0,8,0}), @@ -14,5 +16,39 @@ search_test_() -> ?_assert(A =:= Region1), ?_assert(A =:= Region2), ?_assert({error, bad_ip_format} =:= Region3) + ]. - ]. \ No newline at end of file +ipv6_search_test_() -> + {ok, _} = ip2region_test_util:start_app(), + [ + ?_assert(?IPV6_RESULT =:= xdb:search("2001:4860:4860::8888")), + ?_assert(?IPV6_RESULT =:= xdb:search(<<"2001:4860:4860::8888">>)), + ?_assert(?IPV6_RESULT =:= xdb:search({8193, 18528, 18528, 0, 0, 0, 0, 34952})) + ]. + +invalid_search_test_() -> + {ok, _} = ip2region_test_util:start_app(), + [ + ?_assertEqual({error, bad_ip_format}, xdb:search("xxx.0.8.0")), + ?_assertEqual({error, bad_ip_format}, xdb:search("::ggg")), + ?_assertEqual({error, bad_ip_format}, xdb:search({1,2,3})) + ]. + +ipv6_pool_not_configured_test() -> + {ok, _} = ip2region_test_util:start_app([{ipv4, "ip2region.xdb"}]), + ?assertEqual({error, pool_not_configured}, xdb:search("2001:4860:4860::8888")). + +xdb_version_mismatch_test() -> + {ok, _} = ip2region_test_util:start_app(), + V4File = filename:join([ + ip2region_test_util:repo_root(), "binding", "erlang", "priv", "ip2region.xdb" + ]), + error_logger:tty(false), + try + ?assertEqual( + {error, {xdb_version_mismatch, ipv6, ipv4}}, + ip2region_worker:start([{xdb_file, V4File}, {expected_version, ipv6}]) + ) + after + error_logger:tty(true) + end.