diff --git a/.gitignore b/.gitignore index a276f0f..0743c23 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,7 @@ target /maker/golang/dbmaker /maker/golang/xdb_maker /maker/golang/golang + +#erlang +/binding/erlang/_build +/binding/erlang/doc diff --git a/binding/erlang/README.md b/binding/erlang/README.md new file mode 100644 index 0000000..56db5c2 --- /dev/null +++ b/binding/erlang/README.md @@ -0,0 +1,129 @@ +# ip2region xdb erlang 查询客户端 + +### 简介 +该bingding以erlang语言实现xdb查询客户端,基于Erlang OTP Application,查询逻辑由ip2region_worker工作进程实现,支持配多个工作进程来进行负载均衡。 + +### 应用配置 +该应用可配置的参数在ip2region.app.src中,如下: +``` erlang + {env,[ + {poolargs, [ + {size, 1}, %% 工作进程默认数量 + {max_overflow, 5} %% 工作进程最大数量 + ]} + ]} +``` + +### 编译 + +``` +$ rebar3 compile +``` +### 运行 +将xdb文件放到priv目录下,然后启动erlang节点: +``` +$ rebar3 shell + +``` +在erlang shell中调用xdb:search/1接口查询Ip地址信息, 该接口支持以list格式字符串、bianry格式字符串、tuple和整数表示的IP地址,如下: +``` +1> xdb:search("1.0.8.0"). +[20013,22269,124,48,124,24191,19996,30465,124,24191,24030, + 24066,124,30005,20449] +2> +3> io:format("~ts~n", [xdb:search("1.0.8.0")]). +中国|0|广东省|广州市|电信 +io:format("~ts~n", [xdb:search(<<"1.0.8.0">>)]). +中国|0|广东省|广州市|电信 +4> io:format("~ts~n", [xdb:search({1,0,8,0})]). +中国|0|广东省|广州市|电信 +6> io:format("~ts~n", [xdb:search(16779264)]). +中国|0|广东省|广州市|电信 +``` + +### 使用方法 +* 在rebar.config中引入依赖 +``` +{deps, [ + ip2region +]}. + +``` +* 启动ip2region Application +``` +...... + +application:ensure_started(ip2region), + +...... +``` + +* 调用xdb:search/1接口查询IP信息 +``` +...... + +ip2region:search("1.0.8.0"), + +...... +``` + +### 单元测试 + +``` +$ rebar3 eunit +===> Verifying dependencies... +===> Analyzing applications... +===> Compiling ip2region +===> Performing EUnit tests... +=INFO REPORT==== 17-Jan-2023::11:52:59.920155 === +XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/test/lib/ip2region/priv/ip2region.xdb + +.... +Finished in 0.074 seconds +4 tests, 0 failures + +``` + +### 基准测试 +``` +$ cd benchmarks/ +$ sh xdb-benchmark.sh +===> Verifying dependencies... +===> Analyzing applications... +===> Compiling ip2region +Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] + +Eshell V12.3.2.2 (abort with ^G) +1> =INFO REPORT==== 17-Jan-2023::11:37:35.631095 === +XdbFile:/home/admin/erl-workspace/ip2region/binding/erlang/_build/default/lib/ip2region/priv/ip2region.xdb + +===> Booted ip2region +===> Evaluating: "xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()." +CPU info: +model name : AMD EPYC 7K62 48-Core Processor +cache size : 512 KB +cpu MHz : 2595.124 +bogomips : 5190.24 +cores/threads : 2 + +Erlang info: +system_version:Erlang/OTP 24 [erts-12.3.2.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:1] [jit] +load test data use 4.835593s + +start run benchmark tests + +search from file: +ip count:683844, +total time: 28.201699s, +search 24248.326315375536 times per second, +use 41.23995969841075 micro second per search + +search from cache: +ip count:683844, +total time: 0.671801s, +search 1017926.4395259906 times per second, +use 0.9823892583688677 micro second per search + +benchmark test finish + +``` diff --git a/binding/erlang/benchmarks/xdb-benchmark.sh b/binding/erlang/benchmarks/xdb-benchmark.sh new file mode 100755 index 0000000..40d13f3 --- /dev/null +++ b/binding/erlang/benchmarks/xdb-benchmark.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd .. + +rebar3 shell --eval="xdb_benchmark:main(\"../../data/ip.merge.txt\"), init:stop()." diff --git a/binding/erlang/include/ip2region.hrl b/binding/erlang/include/ip2region.hrl new file mode 100644 index 0000000..a0e91e7 --- /dev/null +++ b/binding/erlang/include/ip2region.hrl @@ -0,0 +1,26 @@ +-ifndef(IP2REGION_HRL). +-define(IP2REGION_HRL, true). + +-define(NONE, none). +-define(APP_NAME, ip2region). + +-define(XDB_VECTOR_INDEX, ets_xdb_vector_index). +-define(XDB_SEGMENT_INDEX, ets_xdb_segement_index). +-define(IP2REGION_CACHE, ets_ip2region_cache). + + +-define(XDB_HEADER_SIZE, 256). +-define(XDB_VECTOR_COLS, 256). +-define(XDB_VECTOR_INDEX_SIZE, 8). +-define(XDB_VECTOR_INDEX_COUNT, (16#10000)). %% 256*256 + +-define(XDB_SEGMENT_INDEX_SIZE, 14). + +-define(IP2REGION_POOL, ip2region_pool). + +-ifndef(IF). +-define(IF(C, T, F), case (C) of true -> (T); false -> (F) end). +-define(IF(C, T), ?IF(C, T, skip)). +-endif. + +-endif. \ No newline at end of file diff --git a/binding/erlang/priv/dummy b/binding/erlang/priv/dummy new file mode 100644 index 0000000..e69de29 diff --git a/binding/erlang/rebar.config b/binding/erlang/rebar.config new file mode 100644 index 0000000..076f916 --- /dev/null +++ b/binding/erlang/rebar.config @@ -0,0 +1,24 @@ +{erl_opts, [ + debug_info, + export_all, + nowarn_export_all +]}. + +{plugins, [rebar3_hex, rebar3_ex_doc]}. + +{deps, [ + poolboy +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [ip2region]} +]}. + +{ex_doc, [ + {extras, ["README.md"]}, + {main, "README.md"}, + {source_url, "https://github.com/leihua996/ip2region/tree/master/binding/erlang"} +]}. + +{hex, [{doc, ex_doc}]}. diff --git a/binding/erlang/rebar.lock b/binding/erlang/rebar.lock new file mode 100644 index 0000000..1ab8da9 --- /dev/null +++ b/binding/erlang/rebar.lock @@ -0,0 +1,8 @@ +{"1.2.0", +[{<<"poolboy">>,{pkg,<<"poolboy">>,<<"1.5.2">>},0}]}. +[ +{pkg_hash,[ + {<<"poolboy">>, <<"392B007A1693A64540CEAD79830443ABF5762F5D30CF50BC95CB2C1AAAFA006B">>}]}, +{pkg_hash_ext,[ + {<<"poolboy">>, <<"DAD79704CE5440F3D5A3681C8590B9DC25D1A561E8F5A9C995281012860901E3">>}]} +]. diff --git a/binding/erlang/src/ip2region.app.src b/binding/erlang/src/ip2region.app.src new file mode 100644 index 0000000..4de369c --- /dev/null +++ b/binding/erlang/src/ip2region.app.src @@ -0,0 +1,20 @@ +{application, ip2region, + [{description, "ip2region xdb client application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {ip2region_app, []}}, + {applications, + [kernel, + stdlib + ]}, + {env,[ + {poolargs, [ + {size, 1}, + {max_overflow, 5} + ]} + ]}, + {modules, []}, + + {licenses, ["Apache-2.0"]}, + {links, [{"Github", "https://github.com/leihua996/ip2region/tree/master/binding/erlang"}]} + ]}. diff --git a/binding/erlang/src/ip2region_app.erl b/binding/erlang/src/ip2region_app.erl new file mode 100644 index 0000000..20bb1cb --- /dev/null +++ b/binding/erlang/src/ip2region_app.erl @@ -0,0 +1,22 @@ +%%%------------------------------------------------------------------- +%% Copyright 2022 The Ip2Region Authors. All rights reserved. +%% Use of this source code is governed by a Apache2.0-style +%% license that can be found in the LICENSE file. +%% +%% @doc +%% @end +%%%------------------------------------------------------------------- + +-module(ip2region_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + ip2region_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/binding/erlang/src/ip2region_sup.erl b/binding/erlang/src/ip2region_sup.erl new file mode 100644 index 0000000..304bc98 --- /dev/null +++ b/binding/erlang/src/ip2region_sup.erl @@ -0,0 +1,55 @@ +%%%------------------------------------------------------------------- +%% Copyright 2022 The Ip2Region Authors. All rights reserved. +%% Use of this source code is governed by a Apache2.0-style +%% license that can be found in the LICENSE file. +%% +%% @doc ip2region top level supervisor. +%% @end +%%%------------------------------------------------------------------- +-module(ip2region_sup). +-behaviour(supervisor). +-include("ip2region.hrl"). + +-export([start_link/0]). + +-export([init/1, create_table/0]). + +-define(SERVER, ?MODULE). + +start_link() -> + {ok, SupPid} = supervisor:start_link({local, ?SERVER}, ?MODULE, []), + {ok, _PoolPid} = start_ip2region_pool(SupPid), + {ok, SupPid}. + +%% sup_flags() = #{strategy => strategy(), % optional +%% intensity => non_neg_integer(), % optional +%% period => pos_integer()} % optional +%% child_spec() = #{id => child_id(), % mandatory +%% start => mfargs(), % mandatory +%% restart => restart(), % optional +%% shutdown => shutdown(), % optional +%% type => worker(), % optional +%% modules => modules()} % optional +init([]) -> + create_table(), + SupFlags = #{strategy => one_for_one, + intensity => 10, + period => 5}, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions +%% +create_table() -> + Opts = [named_table, set, public, {read_concurrency, true}, {keypos, 1}], + ets:new(?XDB_VECTOR_INDEX, Opts), + ets:new(?XDB_SEGMENT_INDEX, Opts), + ets:new(?IP2REGION_CACHE, Opts). + +start_ip2region_pool(Sup) -> + {ok, PoolArgsCfg} = application:get_env(poolargs), + PoolName = ?IP2REGION_POOL, + PoolArgs = [{strategy, fifo}, {name, {local, PoolName}}, {worker_module, ip2region_worker} | PoolArgsCfg], + WorkerArgs = [], + ChildSpecs = poolboy:child_spec(PoolName, PoolArgs, WorkerArgs), + supervisor:start_child(Sup, ChildSpecs). \ No newline at end of file diff --git a/binding/erlang/src/ip2region_util.erl b/binding/erlang/src/ip2region_util.erl new file mode 100644 index 0000000..0359b85 --- /dev/null +++ b/binding/erlang/src/ip2region_util.erl @@ -0,0 +1,26 @@ +%%%------------------------------------------------------------------- +%% Copyright 2022 The Ip2Region Authors. All rights reserved. +%% Use of this source code is governed by a Apache2.0-style +%% license that can be found in the LICENSE file. +%% +%% @doc +%% ip2region utils +%% @end +%%%------------------------------------------------------------------- +-module(ip2region_util). +-export([ipv4_to_n/1]). + + +ipv4_to_n(IntIp) when is_integer(IntIp) -> IntIp; +ipv4_to_n({A, B, C, D}) -> + <> = <>, + N; +ipv4_to_n(Ip) when is_binary(Ip) -> + ipv4_to_n(binary_to_list(Ip)); +ipv4_to_n(Ip) when is_list(Ip) -> + case inet_parse:address(Ip) of + {ok, Addr} -> + ipv4_to_n(Addr); + _ -> + {error, bad_ip_format} + end. \ No newline at end of file diff --git a/binding/erlang/src/ip2region_worker.erl b/binding/erlang/src/ip2region_worker.erl new file mode 100644 index 0000000..64d039c --- /dev/null +++ b/binding/erlang/src/ip2region_worker.erl @@ -0,0 +1,183 @@ +%%%------------------------------------------------------------------- +%% Copyright 2022 The Ip2Region Authors. All rights reserved. +%% Use of this source code is governed by a Apache2.0-style +%% license that can be found in the LICENSE file. +%% +%% @doc +%% ip2region xdb client worker +%% @end +%%%------------------------------------------------------------------- +-module(ip2region_worker). +-behaviour(gen_server). +-include("ip2region.hrl"). + +%% API +-export([start/1, stop/1, start_link/1]). +-export([search/2]). +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-record(state, {xdb_fd}). + +%%========================================== +%% API +%% ========================================= +start(Args) -> + Opts = [{spawn_opt, [{min_heap_size, 6000}]}], + gen_server:start(?MODULE, Args, Opts). + +start_link(Args) -> + Opts = [{spawn_opt, [{min_heap_size, 6000}]}], + gen_server:start_link(?MODULE, Args, Opts). + + +stop(Pid) -> + gen_server:call(Pid, stop). + +search(Pid, Ip) -> + gen_server:call(Pid, {search, Ip}). + +%%========================================== +%% gen_server callbacks +%% ========================================= +init(_Args) -> + process_flag(trap_exit, true), + AppName = + case application:get_application() of + {ok, AName} -> AName; + _ -> ?APP_NAME + end, + PrivDir = code:priv_dir(AppName), + XdbFileName = filename:join([PrivDir, "ip2region.xdb"]), + error_logger:info_report(io_lib:format("XdbFile:~s~n", [XdbFileName])), + {ok, IoDevice} = file:open(XdbFileName, [read, binary]), + load_vector_index(IoDevice), + {ok, #state{xdb_fd = IoDevice}}. + +handle_call(Request, From, State) -> + try + do_call(Request, From, State) + catch + Class:Error:Stacktrace -> + error_logger:error_report(io_lib:format("~p handle call error, Req:~p ~p, stacktrace:~p~n", + [?MODULE, Request, {Class, Error}, Stacktrace])), + {reply, {error, {Class, Error}}, State} + end. + +handle_cast(Msg, State) -> + try + do_cast(Msg, State) + catch + Class:Error:Stacktrace -> + error_logger:error_report(io_lib:format("~p handle cast error, Msg:~p, ~p, stacktrace:~w~n", + [?MODULE, Msg, {Class, Error}, Stacktrace])), + {noreply, State} + end. + +handle_info(Info, State) -> + try + do_info(Info, State) + catch + Class:Error:Stacktrace -> + error_logger:error_report(io_lib:format("~p handle info error, Info:~p, ~p, stacktrace:~p~n", + [?MODULE, Info, {Class, Error}, Stacktrace])), + {noreply, State} + end. + +terminate(_Reason, State) -> + #state{xdb_fd = XdbFd} = State, + case is_pid(XdbFd) of + true -> + file:close(XdbFd); + _ -> + skip + end, + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + + +%%========================================== +%% Internal function +%% ========================================= +do_call({search, Ip}, _From, #state{xdb_fd = IoDevice} = State) -> + Reply = search_ip(IoDevice, Ip), + {reply, Reply, State}; + +do_call(stop, _From, State) -> + {stop, normal, stopped, State}; + +do_call(Request, From, State) -> + error_logger:error_report(io_lib:format("unknown request: ~p, from:~p", [Request, From])), + {noreply, State}. + +do_cast(Msg, State) -> + error_logger:error_report(io_lib:format("unknown msg: ~p", [Msg])), + {noreply, State}. + +do_info(Info, State) -> + error_logger:error_report(io:format("unknown info: ~p", [Info])), + {noreply, State}. + + +load_vector_index(IoDevice) -> + Key = ip2region_header_loaded, + case persistent_term:get(Key, false) of + true -> ok; + _ -> + {ok, <<_Header:?XDB_HEADER_SIZE/binary, VectorIndexBin/binary>> } = + file:read(IoDevice, ?XDB_HEADER_SIZE + ?XDB_VECTOR_INDEX_COUNT*8), + load_vector_index_aux(VectorIndexBin, 0), + persistent_term:put(Key, true) + end. + +load_vector_index_aux(<<>>, _Index) -> ok; +load_vector_index_aux(<>, Index) -> + Term = {Index, SPtr, EPtr}, + ets:insert(?XDB_VECTOR_INDEX, Term), + load_vector_index_aux(VectorIndexBin, Index + 1). + + +search_ip(IoDevice, Ip) -> + IntIp = ip2region_util:ipv4_to_n(Ip), + case ets:lookup(?IP2REGION_CACHE, IntIp) of + [{_IntIp, RegionInfo}] -> + RegionInfo; + _ -> + <> = <>, + VectorIdx = A * ?XDB_VECTOR_COLS + B, + [{_, SPtr, EPtr}] = ets:lookup(?XDB_VECTOR_INDEX, VectorIdx), + RegionInfo = search_ip(IoDevice, IntIp, SPtr, EPtr, 0, (EPtr - SPtr) div ?XDB_SEGMENT_INDEX_SIZE), + ets:insert_new(?IP2REGION_CACHE, {IntIp, RegionInfo}), + RegionInfo + end. + +search_ip(IoDevice, IntIp, SPtr, EPtr, Low, High) when Low =< High -> + Middle = (Low + High) bsr 1, + SPtr2 = SPtr + Middle * ?XDB_SEGMENT_INDEX_SIZE, + {SIp, EIp, DataLen, DataPtr} = read_segement_index(IoDevice, SPtr2), + if + IntIp < SIp -> + search_ip(IoDevice, IntIp, SPtr, EPtr, Low, Middle - 1); + IntIp > EIp -> + search_ip(IoDevice, IntIp, SPtr, EPtr, Middle + 1, High); + true -> + {ok, DataBin} = read_file(IoDevice, DataPtr, DataLen), + unicode:characters_to_nfc_list(DataBin) + end; +search_ip(_IoDevice, _IntIp, _SPtr, _EPtr, _Low, _High) -> + {error, unknown}. + +read_file(IoDevice, Position, DataLength) -> + file:position(IoDevice, {bof, Position}), + file:read(IoDevice, DataLength). + +read_segement_index(IoDevice, SPtr) -> + case ets:lookup(?XDB_SEGMENT_INDEX, SPtr) of + [{_SPtr, SIp, EIp, DataLen, DataPtr}] -> + {SIp, EIp, DataLen, DataPtr}; + _ -> + {ok, <>} = + read_file(IoDevice, SPtr, ?XDB_SEGMENT_INDEX_SIZE), + ets:insert_new(?XDB_SEGMENT_INDEX, {SPtr, SIp, EIp, DataLen, DataPtr}), + {SIp, EIp, DataLen, DataPtr} + end. \ No newline at end of file diff --git a/binding/erlang/src/xdb.erl b/binding/erlang/src/xdb.erl new file mode 100644 index 0000000..2b98ac6 --- /dev/null +++ b/binding/erlang/src/xdb.erl @@ -0,0 +1,32 @@ + +%%%------------------------------------------------------------------- +%% Copyright 2022 The Ip2Region Authors. All rights reserved. +%% Use of this source code is governed by a Apache2.0-style +%% license that can be found in the LICENSE file. +%% +%% @doc +%% ip2region xdb client search api +%% @end +%%%------------------------------------------------------------------- +-module(xdb). +-include("ip2region.hrl"). + +-export([search/1]). + +-spec search(Ip :: tuple() | list() | binary()) -> Result :: binary | {error, Reason::atom()}. +search(Ip) when is_integer(Ip); is_list(Ip); is_tuple(Ip); is_binary(Ip) -> + case ip2region_util:ipv4_to_n(Ip) of + IntIp when is_integer(IntIp) -> + case ets:lookup(?IP2REGION_CACHE, IntIp) of + [{_IntIp, Region}] -> Region; + _ -> + Worker = poolboy:checkout(?IP2REGION_POOL, true, infinity), + try + ip2region_worker:search(Worker, IntIp) + after + poolboy:checkin(?IP2REGION_POOL, Worker) + end + end; + Ret -> + Ret + end. diff --git a/binding/erlang/src/xdb_benchmark.erl b/binding/erlang/src/xdb_benchmark.erl new file mode 100644 index 0000000..e52801c --- /dev/null +++ b/binding/erlang/src/xdb_benchmark.erl @@ -0,0 +1,75 @@ +%%%------------------------------------------------------------------- +%% Copyright 2022 The Ip2Region Authors. All rights reserved. +%% Use of this source code is governed by a Apache2.0-style +%% license that can be found in the LICENSE file. +%% +%% @doc +%% ip2region xdb client benchmark test +%% @end +%%%------------------------------------------------------------------- +-module(xdb_benchmark). +-export([main/1]). + +main(DataFile) -> + application:ensure_started(ip2region), + show_hw_sw_info(), + IpList = load_test_data(DataFile), + run(IpList). + +show_hw_sw_info() -> + io:format("CPU info:~n", []), + io:format("~s", [os:cmd("egrep '^model name' /proc/cpuinfo | head -1")]), + io:format("~s", [os:cmd("egrep '^cache' /proc/cpuinfo | head -1")]), + io:format("~s", [os:cmd("egrep '^cpu MHz' /proc/cpuinfo | head -1")]), + io:format("~s", [os:cmd("egrep '^bogomips' /proc/cpuinfo | head -1")]), + io:format("cores/threads : ~s~n", [os:cmd("egrep -c '^processor' /proc/cpuinfo")]), + io:format("Erlang info:~n", []), + io:format("system_version:~s", [erlang:system_info(system_version)]), + ok. + +load_test_data(DataFile) -> + {ok, Fd} = file:open(DataFile, [read]), + T0 = os:timestamp(), + IpList = load_test_data(Fd, []), + T1 = os:timestamp(), + Sec = timer:now_diff(T1, T0) / 1000000, + io:format("load test data use ~ps~n", [Sec]), + IpList. + +load_test_data(Fd, IpList) -> + case file:read_line(Fd) of + {ok, Ip} -> + case string:tokens(unicode:characters_to_list(Ip), "|") of + [SIp | _Tail] -> + load_test_data(Fd, [string:trim(SIp)| IpList]); + _ -> + load_test_data(Fd, IpList) + end; + _ -> + file:close(Fd), + IpList + end. + +run(IpList) -> + garbage_collect(), + io:format("~nstart run benchmark tests~n", []), + io:format("~nsearch from file:~n", []), + run_test(IpList), + io:format("~nsearch from cache:~n", []), + run_test(IpList), + io:format("~nbenchmark test finish~n", []). + +run_test(IpList) -> + T0 = os:timestamp(), + run_test_aux(IpList), + T1 = os:timestamp(), + Sec = timer:now_diff(T1, T0) / 1000000, + IpCount = length(IpList), + io:format("ip count:~p,~ntotal time: ~ps,~nsearch ~p times per second,~nuse ~p micro second per search~n", + [IpCount, Sec, IpCount / Sec, Sec * 1000000/IpCount]). + +run_test_aux([]) -> ok; +run_test_aux([Ip | Tail]) -> + xdb:search(Ip), + run_test_aux(Tail). + diff --git a/binding/erlang/test/xdb_test.erl b/binding/erlang/test/xdb_test.erl new file mode 100644 index 0000000..85adcac --- /dev/null +++ b/binding/erlang/test/xdb_test.erl @@ -0,0 +1,18 @@ +-module(xdb_test). + +-include_lib("eunit/include/eunit.hrl"). + +search_test_() -> + application:ensure_started(ip2region), + A = "中国|0|广东省|广州市|电信", + Region0 = xdb:search("1.0.8.0"), + Region1 = xdb:search(<<"1.0.8.0">>), + Region2 = xdb:search({1,0,8,0}), + Region3 = xdb:search("xxx.0.8.0"), + [ + ?_assert(A =:= Region0), + ?_assert(A =:= Region1), + ?_assert(A =:= Region2), + ?_assert({error, bad_ip_format} =:= Region3) + + ]. \ No newline at end of file