From cc876c93f13fecd4ba943fbfc40bc60509a2bf57 Mon Sep 17 00:00:00 2001 From: gongzhengyang Date: Tue, 20 Dec 2022 14:58:10 +0800 Subject: [PATCH] feat: finish all jobs --- binding/rust/Makefile | 12 -- binding/rust/ReadMe.md | 156 ++++++++++++++++++-------- binding/rust/bin/src/cmd.rs | 7 +- binding/rust/bin/src/main.rs | 10 +- binding/rust/search/benches/search.rs | 12 +- binding/rust/search/src/lib.rs | 18 ++- 6 files changed, 133 insertions(+), 82 deletions(-) delete mode 100644 binding/rust/Makefile diff --git a/binding/rust/Makefile b/binding/rust/Makefile deleted file mode 100644 index b73a6ba..0000000 --- a/binding/rust/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# ip2region rust binding makefile -all: build -.PHONY: all - -build: - cargo build -r -test: - cargo test -bench: - cargo bench -clean: - find ./ -name xdb_searcher | xargs rm -f \ No newline at end of file diff --git a/binding/rust/ReadMe.md b/binding/rust/ReadMe.md index 91eb254..4ed7e77 100644 --- a/binding/rust/ReadMe.md +++ b/binding/rust/ReadMe.md @@ -1,79 +1,137 @@ -# ip2region xdb rust 查询客户端实现 +# `ip2region xdb rust` 查询客户端实现 # 使用方式 ### 缓存整个 `xdb` 数据 -可以预先加载整个 ip2region.xdb 到内存，完全基于内存查询，类似于之前的 memory search 查询。 -```golang -// 1、从 dbPath 加载整个 xdb 到内存 -cBuff, err := LoadContentFromFile(dbPath) -if err != nil { - fmt.Printf("failed to load content from `%s`: %s\n", dbPath, err) - return -} +预先加载整个` ip2region.xdb` 到内存，完全基于内存查询，该方式线程安全，采用`once_cell::sync::OnceCell`，只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread` -// 2、用全局的 cBuff 创建完全基于内存的查询对象。 -searcher, err := xdb.NewWithBuffer(cBuff) -if err != nil { - fmt.Printf("failed to create searcher with content: %s\n", err) - return -} +配置`Cargo.toml`如下 -// 备注：并发使用，用整个 xdb 缓存创建的 searcher 对象可以安全用于并发。 +```toml +[dependencies] +search = { git = "https://github.com/lionsoul2014/ip2region.git", branch = "master" } +# 如果要在异步环境下使用，需要加上如下依赖 +tokio = { version = "1", features = ["full"]} ``` -# 编译测试程序 +编写`main.rs` + +**需要使用`XDB_FILEPATH`指定`ip2region.xdb`文件的路径**，该参数可以使用相对路径或者绝对路径，如果使用相对路径报错，请修改为绝对路径 + +```rust +use std::env; + +#[tokio::main] +async fn main() { + env::set_var( + "XDB_FILEPATH", + "../data/ip2region.xdb", + ); + // search_by_ip的参数可以是u32类型，字符串IP类型，字符串数字类型 + for i in 1..5 { + tokio::spawn(async move { + // u32 + println!("{:?}", search::search_by_ip(i)); + }); + } + // ip str + println!("{:?}", search::search_by_ip("1.0.1.0")); + + // u32 str + let ip_u32 = 1 << 24 | 1 << 8; + println!("{:?}", search::search_by_ip(ip_u32.to_string().as_str())); +} +``` + +进行测试 + +```shell +$ cargo run +init xdb searcher at ../data/ip2region.xdb +Ok("中国|0|福建省|福州市|电信") +Ok("中国|0|福建省|福州市|电信") +Ok("0|0|0|内网IP|内网IP") +Ok("0|0|0|内网IP|内网IP") +Ok("0|0|0|内网IP|内网IP") +Ok("0|0|0|内网IP|内网IP") +``` + +# 编译程序 + +通过如下方式编译得到 `ip2region` 可执行程序 + +切换到 `rust binding` 根目录，执行如下命令 -通过如下方式编译得到 xdb_searcher 可执行程序： ```bash -# 切换到 golang binding 根目录 -make +➜ cargo build -r ``` +生成的二进制文件会在`./target/release/ip2region`位置 # 查询测试 -通过 `xdb_searcher search` 命令来测试 ip2region.xdb 的查询： +通过 `./target/release/ip2region` 命令来测试查询 ``` -➜ golang git:(v2.0_xdb) ./xdb_searcher search -./xdb_searcher search [command options] -options: - --db string ip2region binary xdb file path - --cache-policy string cache policy: file/vectorIndex/content +➜ ./target/release/ip2region --help +you can set environment XDB_FILEPATH=../data/ip2region or just use --xdb in command +Usage: ip2region [OPTIONS] +Options: + --xdb + the xdb filepath, you can set this field like ../data/ip2region.xdb + -h, --help + Print help information (use `-h` for a summary) + -V, --version + Print version information ``` -例如：使用默认的 data/ip2region.xdb 进行查询测试 +命令行指定参数进行查询测试，输入 `ip` 地址或者一个`u32`类型的数字进行查询即可，输入 `quit` 退出测试程序 ```bash -➜ golang git:(v2.0_xdb) ✗ ./xdb_searcher search --db=../../data/ip2region.xdb +➜ ./target/release/ip2region --xdb=../../data/ip2region.xdb +init xdb searcher at ../../data/ip2region.xdb ip2region xdb searcher test program, type `quit` to exit -ip2region>> 1.2.3.4 -{region:美国|0|华盛顿|0|谷歌, took:101.57µs} +ip2region>> 1.1.1.1 +region: Ok("澳大利亚|0|0|0|0"), took: 4.227µs +ip2region>> 2.2.2.2 +region: Ok("法国|0|0|0|橘子电信"), took: 4.495µs +ip2region>> 222222222 +region: Ok("美国|0|康涅狄格|0|0"), took: 4.048µs ``` -输入 ip 地址进行查询即可，输入 quit 退出测试程序。可以设置 `cache-policy` 为 file/vectorIndex/content 来测试不同的查询缓存机制。 +或者使用环境变量 - -# bench 测试 - -通过 `xdb_searcher bench` 命令来进行自动 bench 测试，一方面确保程序和 `xdb` 文件都没有错误，另一方面通过大量的查询得到平均查询性能： -```bash -➜ golang git:(v2.0_xdb) ./xdb_searcher bench -./xdb_searcher bench [command options] -options: - --db string ip2region binary xdb file path - --src string source ip text file path - --cache-policy string cache policy: file/vectorIndex/content +```shell +➜ XDB_FILEPATH=../../data/ip2region.xdb ./target/release/ip2region +init xdb searcher at ../../data/ip2region.xdb +ip2region xdb searcher test program, type `quit` to exit +ip2region>> 2.2.2.2 +region: Ok("法国|0|0|0|橘子电信"), took: 4.458µs +ip2region>> 4.4.4.5 +region: Ok("美国|0|0|0|Level3"), took: 4.847µs ``` -例如：通过 data/ip2region.xdb 和 data/ip.merge.txt 进行 bench 测试： -```bash -➜ golang git:(v2.0_xdb) ✗ ./xdb_searcher bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt -Bench finished, {total: 3417955, took: 28.211578339s, cost: 8253 ns/op} +# 单元测试 + +```shell +➜ XDB_FILEPATH=../../../data/ip2region.xdb cargo test ``` -可以设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 不同缓存实现机制的效率。 +# `bench` 测试 -*请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件*。 +通过 `cargo bench` 命令来进行自动 `bench` 测试，一方面确保程序和 `xdb` 文件都没有错误，另一方面通过大量的查询得到平均查询性能 -bench 程序会逐行读取 `src` 指定的源IP文件，然后每个 IP 段选取 5 个固定位置的 IP 进行测试，以确保查询的 region 信息和原始的 region 信息是相同。测试途中没有调试信息的输出，有错误会打印错误信息并且终止运行，所以看到 `Bench finished` 就表示 bench 成功了，cost 是表示每次查询操作的平均时间(ns)。 +在不同机器上面的测试性能时间是不一样的，如下是在机器`CPU`是`Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz`，内存`DDR4 32G`下面的测试结果 + +```shell +➜ XDB_FILEPATH=../../../data/ip2region.xdb cargo bench +// --snip-- + Running benches/search.rs (target/release/deps/search-9614305a566885c4) +Benchmarking ip_search_bench: Warming up for 3.0000 sinit xdb searcher at ../../../data/ip2region.xdb +ip_search_bench time: [120.84 ns 122.91 ns 125.20 ns] + change: [-3.3346% -1.2786% +0.8027%] (p = 0.23 > 0.05) + No change in performance detected. +Found 6 outliers among 100 measurements (6.00%) + 6 (6.00%) high mild +``` + +可以看到上面的`ip_search_bench time`一行的参数表示是左右值分别显示置信区间的下限和上限，中间值显示 `Criterion.rs` 对基准程序每次迭代所用时间的最佳估计 diff --git a/binding/rust/bin/src/cmd.rs b/binding/rust/bin/src/cmd.rs index 74217d9..05c3912 100644 --- a/binding/rust/bin/src/cmd.rs +++ b/binding/rust/bin/src/cmd.rs @@ -4,6 +4,11 @@ pub fn get_matches() -> ArgMatches { Command::new("ip2region") .version("0.1") .about("ip2region bin program") - .arg(arg!(--xdb "the xdb filepath, you can set this field like ../data/ip2region.xdb")) + .long_about( + "you can set environment XDB_FILEPATH=../data/ip2region or just use --xdb in command", + ) + .arg( + arg!(--xdb "the xdb filepath, you can set this field like ../data/ip2region.xdb"), + ) .get_matches() } diff --git a/binding/rust/bin/src/main.rs b/binding/rust/bin/src/main.rs index 2279db4..2e94c19 100644 --- a/binding/rust/bin/src/main.rs +++ b/binding/rust/bin/src/main.rs @@ -7,23 +7,25 @@ mod cmd; fn main() { env::var("XDB_FILEPATH").unwrap_or_else(|_| { let matches = cmd::get_matches(); - let xdb_filepath = matches.get_one::("xdb").unwrap(); + let xdb_filepath = matches + .get_one::("xdb") + .expect("you must use --xdb in command or set XDB_FILEPATH environment"); env::set_var("XDB_FILEPATH", xdb_filepath); xdb_filepath.to_owned() }); + search::global_searcher(); - println!("ip2region xdb searcher test program, type `quit` to exit"); + println!("ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit"); loop { print!("ip2region>> "); std::io::stdout().flush().unwrap(); let mut line = String::new(); std::io::stdin().read_line(&mut line).unwrap(); if line.contains("quit") { - break + break; } let now = Instant::now(); let result = search::search_by_ip(line.trim()); println!("region: {:?}, took: {:?}", result, now.elapsed()); } - } diff --git a/binding/rust/search/benches/search.rs b/binding/rust/search/benches/search.rs index 192113e..8d9e85b 100644 --- a/binding/rust/search/benches/search.rs +++ b/binding/rust/search/benches/search.rs @@ -1,13 +1,15 @@ -use criterion::{Criterion, criterion_group, criterion_main}; +use criterion::{criterion_group, criterion_main, Criterion}; use rand; use search::search_by_ip; fn ip_search_benchmark(c: &mut Criterion) { - c.bench_function("ip_search_bench", |b| b.iter(|| { - let ip = rand::random::(); - search_by_ip(ip).unwrap(); - })); + c.bench_function("ip_search_bench", |b| { + b.iter(|| { + let ip = rand::random::(); + search_by_ip(ip).unwrap(); + }) + }); } criterion_group!(benches, ip_search_benchmark); diff --git a/binding/rust/search/src/lib.rs b/binding/rust/search/src/lib.rs index 09dd867..e3b8084 100644 --- a/binding/rust/search/src/lib.rs +++ b/binding/rust/search/src/lib.rs @@ -1,11 +1,11 @@ mod ip_value; +use std::env; use std::error::Error; +use std::fmt; +use std::fmt::Formatter; use std::fs::File; use std::io::Read; -use std::fmt; -use std::env; -use std::fmt::Formatter; use once_cell::sync::OnceCell; @@ -17,8 +17,6 @@ const VECTOR_INDEX_COLS: u32 = 256; const VECTOR_INDEX_SIZE: u32 = 8; const SEGMENT_INDEX_SIZE: usize = 14; -const DEFAULT_XDB_FILEPATH: &str = "../../../data/ip2region.xdb"; - pub struct Searcher { pub buffer: Vec, } @@ -26,8 +24,7 @@ pub struct Searcher { pub fn global_searcher() -> &'static Searcher { static SEARCHER: OnceCell = OnceCell::new(); SEARCHER.get_or_init(|| { - let xdp_filepath = env::var("XDB_FILEPATH") - .unwrap_or_else(|_| DEFAULT_XDB_FILEPATH.to_owned()); + let xdp_filepath = env::var("XDB_FILEPATH").expect("you must set XDB_FILEPATH for search"); println!("init xdb searcher at {xdp_filepath}"); Searcher::new(xdp_filepath.as_str()).unwrap() }) @@ -40,8 +37,8 @@ impl fmt::Display for Searcher { } pub fn search_by_ip(ip: T) -> Result> - where - T: ToUIntIP, +where + T: ToUIntIP, { let changed_value = ip.to_u32_ip()?; search_by_ip_u32(changed_value) @@ -117,8 +114,7 @@ mod tests { search_by_ip("2.0.0.0").unwrap(); search_by_ip("32").unwrap(); search_by_ip(32).unwrap(); - search_by_ip(Ipv4Addr::from_str("1.1.1.1").unwrap()) - .unwrap(); + search_by_ip(Ipv4Addr::from_str("1.1.1.1").unwrap()).unwrap(); } /// test find ip correct use the file ip.test.txt in ../../data