From 76345e67b0009a542d9d95550e4024647143a7f4 Mon Sep 17 00:00:00 2001 From: gongzhengyang Date: Wed, 21 Dec 2022 16:38:37 +0800 Subject: [PATCH 1/7] perf(add more benchmark and change function calls): --- binding/rust/bin/src/main.rs | 9 +- binding/rust/search/benches/search.rs | 37 +++++-- binding/rust/search/src/lib.rs | 136 +++++++++++++++----------- 3 files changed, 115 insertions(+), 67 deletions(-) diff --git a/binding/rust/bin/src/main.rs b/binding/rust/bin/src/main.rs index 2e94c19..ee4d154 100644 --- a/binding/rust/bin/src/main.rs +++ b/binding/rust/bin/src/main.rs @@ -8,10 +8,11 @@ fn main() { env::var("XDB_FILEPATH").unwrap_or_else(|_| { let matches = cmd::get_matches(); let xdb_filepath = matches - .get_one::("xdb") - .expect("you must use --xdb in command or set XDB_FILEPATH environment"); - env::set_var("XDB_FILEPATH", xdb_filepath); - xdb_filepath.to_owned() + .get_one::("xdb"); + if xdb_filepath.is_some() { + env::set_var("XDB_FILEPATH", xdb_filepath.unwrap()); + } + "".to_owned() }); search::global_searcher(); diff --git a/binding/rust/search/benches/search.rs b/binding/rust/search/benches/search.rs index 8d9e85b..f1bfef3 100644 --- a/binding/rust/search/benches/search.rs +++ b/binding/rust/search/benches/search.rs @@ -1,16 +1,41 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rand; -use search::search_by_ip; +use search::{buffer_value, get_block_by_size, get_start_end_ptr, global_searcher, search_by_ip}; -fn ip_search_benchmark(c: &mut Criterion) { +fn ip_search_bench(c: &mut Criterion) { c.bench_function("ip_search_bench", |b| { b.iter(|| { - let ip = rand::random::(); - search_by_ip(ip).unwrap(); + search_by_ip(rand::random::()).unwrap(); }) }); } -criterion_group!(benches, ip_search_benchmark); +fn buffer_value_bench(c: &mut Criterion) { + c.bench_function("buffer_value", |b| { + b.iter(|| { + let offset = rand::random::(); + let length = rand::random::(); + buffer_value(offset as usize, length as usize); + }); + }); +} + +fn get_block_by_size_bench(c: &mut Criterion) { + c.bench_function("get_block_by_size", |b| { + b.iter(||{ + get_block_by_size(&global_searcher().buffer, rand::random::() as usize, 4); + }) + }); +} + +fn get_start_end_ptr_bench(c: &mut Criterion) { + c.bench_function("get_start_end_ptr", |b| { + b.iter(|| { + get_start_end_ptr(rand::random::()); + }) + }); +} + +criterion_group!(benches, ip_search_bench, buffer_value_bench, get_block_by_size_bench, get_start_end_ptr_bench); criterion_main!(benches); diff --git a/binding/rust/search/src/lib.rs b/binding/rust/search/src/lib.rs index e3b8084..ca80e92 100644 --- a/binding/rust/search/src/lib.rs +++ b/binding/rust/search/src/lib.rs @@ -1,32 +1,56 @@ -mod ip_value; - use std::env; use std::error::Error; use std::fmt; use std::fmt::Formatter; use std::fs::File; use std::io::Read; +use std::path::Path; use once_cell::sync::OnceCell; use ip_value::ToUIntIP; -const HEADER_INFO_LENGTH: u32 = 256; -// const VECTOR_INDEX_ROWS: u32 = 256; -const VECTOR_INDEX_COLS: u32 = 256; -const VECTOR_INDEX_SIZE: u32 = 8; +mod ip_value; + +const HEADER_INFO_LENGTH: usize = 256; +const VECTOR_INDEX_COLS: usize = 256; +const VECTOR_INDEX_SIZE: usize = 8; const SEGMENT_INDEX_SIZE: usize = 14; +/// store the xdb file in memory totally pub struct Searcher { pub buffer: Vec, } +impl Searcher { + /// you can set the XDB_FILEPATH + /// or super dir has data dir with the file ip2region.xdb + /// it will check ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb + pub fn new() -> Result> { + let xdb_filepath = env::var("XDB_FILEPATH") + .unwrap_or_else(|_| { + let prefix = "../".to_owned(); + for recurse in 1..4 { + let filepath = prefix.repeat(recurse) + "data/ip2region.xdb"; + if Path::new(filepath.as_str()).exists() { + return filepath + } + }; + panic!("you must set XDB_FILEPATH or put file in ../data/ip2region.xdb") + }); + println!("load xdb searcher file at {xdb_filepath}"); + let mut f = File::open(xdb_filepath)?; + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer)?; + Ok(Self { buffer }) + } +} + +/// global init searcher thread safely pub fn global_searcher() -> &'static Searcher { static SEARCHER: OnceCell = OnceCell::new(); SEARCHER.get_or_init(|| { - let xdp_filepath = env::var("XDB_FILEPATH").expect("you must set XDB_FILEPATH for search"); - println!("init xdb searcher at {xdp_filepath}"); - Searcher::new(xdp_filepath.as_str()).unwrap() + Searcher::new().unwrap() }) } @@ -36,91 +60,89 @@ impl fmt::Display for Searcher { } } +pub fn get_start_end_ptr(ip: u32) -> (usize, usize) { + let il0= ((ip >> 24) & 0xFF) as usize; + let il1 = ((ip >> 16) & 0xFF) as usize; + let idx = VECTOR_INDEX_SIZE * (il0 * VECTOR_INDEX_COLS + il1); + let start_point = HEADER_INFO_LENGTH + idx; + + let start_ptr = get_block_by_size(&global_searcher().buffer, start_point, 4); + let end_ptr = get_block_by_size(&global_searcher().buffer, start_point + 4, 4); + (start_ptr, end_ptr) +} + +/// check https://mp.weixin.qq.com/s/ndjzu0BgaeBmDOCw5aqHUg for details pub fn search_by_ip(ip: T) -> Result> where T: ToUIntIP, { - let changed_value = ip.to_u32_ip()?; - search_by_ip_u32(changed_value) -} - -pub fn search_by_ip_u32(ip: u32) -> Result> { - let il0 = (ip >> 24) & 0xFF; - let il1 = (ip >> 16) & 0xFF; - let idx = VECTOR_INDEX_SIZE * (il0 * VECTOR_INDEX_COLS + il1); - - let start_point = (HEADER_INFO_LENGTH + idx) as usize; - let buffer = &global_searcher().buffer; - let start_ptr = get_u32(buffer, start_point); - let end_ptr = get_u32(buffer, start_point + 4); + let ip = ip.to_u32_ip()?; + let (start_ptr, end_ptr) = get_start_end_ptr(ip); let mut left: usize = 0; - let mut right: usize = ((end_ptr - start_ptr) as usize) / SEGMENT_INDEX_SIZE; + let mut right: usize = (end_ptr - start_ptr) / SEGMENT_INDEX_SIZE; + while left <= right { let mid = (left + right) >> 1; - let offset = (start_ptr as usize) + mid * SEGMENT_INDEX_SIZE; + let offset = &start_ptr + mid * SEGMENT_INDEX_SIZE; let buffer_ip_value = buffer_value(offset, SEGMENT_INDEX_SIZE); - let start_ip = get_u32(buffer_ip_value, 0); - if ip < start_ip { + let start_ip = get_block_by_size(&buffer_ip_value, 0, 4); + if &ip < &(start_ip as u32) { right = mid - 1; - } else if ip > get_u32(buffer_ip_value, 4) { + } else if &ip > &(get_block_by_size(&buffer_ip_value, 4, 4) as u32) { left = mid + 1; } else { - let length = (buffer_ip_value[8] as usize & 0x000000FF) - | (buffer_ip_value[9] as usize & 0x0000FF00); - - let offset = get_u32(buffer_ip_value, 10); - let result = buffer_value(offset as usize, length) - .iter() - .map(|x| x.to_owned()) - .collect::>(); - return Ok(String::from_utf8(result)?); + let data_length = get_block_by_size(&buffer_ip_value, 8, 2); + let data_offset = get_block_by_size(&buffer_ip_value, 10, 4); + let result = String::from_utf8( + buffer_value(data_offset, data_length) + .to_vec()); + return Ok(result?); } } Err("not matched".into()) } +pub fn start_end_buffer_value(bytes: &[u8], offset: usize, length: usize) -> &[u8] { + &bytes[offset..offset+length] +} + pub fn buffer_value(offset: usize, length: usize) -> &'static [u8] { &global_searcher().buffer[offset..offset + length] } -impl Searcher { - pub fn new(filepath: &str) -> Result> { - let mut f = File::open(filepath)?; - let mut buffer = Vec::new(); - f.read_to_end(&mut buffer)?; - Ok(Self { buffer }) +#[inline] +pub fn get_block_by_size(bytes: &[T], offset: usize, length: usize) -> usize +where + T: Clone, + usize: From, +{ + let mut result: usize = 0; + for (index, value) in bytes[offset..offset+length].iter().enumerate() { + result |= usize::from(value.clone()) << (index*8); } -} - -fn get_u32(bytes: &[u8], offset: usize) -> u32 { - (bytes[offset] as u32) & 0x000000FF - | ((bytes[offset + 1] as u32) << 8) & 0x0000FF00 - | ((bytes[offset + 2] as u32) << 16) & 0x00FF0000 - | ((bytes[offset + 3] as u32) << 24) & 0xFF000000 + result } #[cfg(test)] mod tests { - use super::*; use std::net::Ipv4Addr; use std::str::FromStr; use std::thread; - const TEST_IP_FILEPATH: &str = "../../../data/ip.test.txt"; + use super::*; ///test all types find correct #[test] - fn test_search_by_ip() { + fn test_multi_type_ip() { search_by_ip("2.0.0.0").unwrap(); search_by_ip("32").unwrap(); - search_by_ip(32).unwrap(); + search_by_ip(4294408949).unwrap(); search_by_ip(Ipv4Addr::from_str("1.1.1.1").unwrap()).unwrap(); } - /// test find ip correct use the file ip.test.txt in ../../data #[test] - fn test_random_choose_ip() { - let mut file = File::open(TEST_IP_FILEPATH).unwrap(); + fn test_match_all_ip_correct() { + let mut file = File::open("../../../data/ip.test.txt").unwrap(); let mut contents = String::new(); file.read_to_string(&mut contents).unwrap(); for line in contents.split("\n") { @@ -138,7 +160,7 @@ mod tests { } #[test] - fn test_multi_thread() { + fn test_multi_thread_only_load_xdb_once() { let handle = thread::spawn(|| { let result = search_by_ip("2.2.2.2").unwrap(); println!("ip search in spawn: {result}"); From 5702ca92591da2b175be16a0ad10f0c29d6e39a5 Mon Sep 17 00:00:00 2001 From: gongzhengyang Date: Thu, 22 Dec 2022 18:27:01 +0800 Subject: [PATCH 2/7] feat: change package name --- binding/rust/Cargo.toml | 2 +- binding/rust/ReadMe.md | 16 +++- binding/rust/{bin => example}/Cargo.toml | 6 +- binding/rust/{bin => example}/src/cmd.rs | 0 binding/rust/{bin => example}/src/main.rs | 10 +- .../rust/{search => ip2region2}/Cargo.toml | 2 +- .../{search => ip2region2}/benches/search.rs | 26 ++++-- .../{search => ip2region2}/src/ip_value.rs | 0 .../rust/{search => ip2region2}/src/lib.rs | 91 ++++--------------- binding/rust/ip2region2/src/searcher.rs | 66 ++++++++++++++ 10 files changed, 125 insertions(+), 94 deletions(-) rename binding/rust/{bin => example}/Cargo.toml (77%) rename binding/rust/{bin => example}/src/cmd.rs (100%) rename binding/rust/{bin => example}/src/main.rs (71%) rename binding/rust/{search => ip2region2}/Cargo.toml (94%) rename binding/rust/{search => ip2region2}/benches/search.rs (56%) rename binding/rust/{search => ip2region2}/src/ip_value.rs (100%) rename binding/rust/{search => ip2region2}/src/lib.rs (51%) create mode 100644 binding/rust/ip2region2/src/searcher.rs diff --git a/binding/rust/Cargo.toml b/binding/rust/Cargo.toml index d455ea1..433c81d 100644 --- a/binding/rust/Cargo.toml +++ b/binding/rust/Cargo.toml @@ -1,2 +1,2 @@ [workspace] -members = ["bin", "search"] +members = ["example", "ip2region2"] diff --git a/binding/rust/ReadMe.md b/binding/rust/ReadMe.md index 4ed7e77..30e4b0c 100644 --- a/binding/rust/ReadMe.md +++ b/binding/rust/ReadMe.md @@ -2,10 +2,6 @@ # 使用方式 -### 缓存整个 `xdb` 数据 - -预先加载整个` ip2region.xdb` 到内存，完全基于内存查询，该方式线程安全，采用`once_cell::sync::OnceCell`，只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread` - 配置`Cargo.toml`如下 ```toml @@ -15,6 +11,14 @@ search = { git = "https://github.com/lionsoul2014/ip2region.git", branch = "mast tokio = { version = "1", features = ["full"]} ``` +程序启动的时候是没加载文件，这个程序占用内存`1M`左右 + +一旦开始执行查询，`ip2region.xdb`文件会直接加载到内存，程序占用内存`12M`左右 + +预先加载整个` ip2region.xdb` 到内存，完全基于内存查询，该方式线程安全，采用`once_cell::sync::OnceCell`，只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread` + +### 缓存整个 `xdb` 数据 + 编写`main.rs` **需要使用`XDB_FILEPATH`指定`ip2region.xdb`文件的路径**，该参数可以使用相对路径或者绝对路径，如果使用相对路径报错，请修改为绝对路径 @@ -28,6 +32,10 @@ async fn main() { "XDB_FILEPATH", "../data/ip2region.xdb", ); + //可以调用如下直接加载文件 + // search::global_searcher(); + + // search_by_ip的参数可以是u32类型，字符串IP类型，字符串数字类型 for i in 1..5 { tokio::spawn(async move { diff --git a/binding/rust/bin/Cargo.toml b/binding/rust/example/Cargo.toml similarity index 77% rename from binding/rust/bin/Cargo.toml rename to binding/rust/example/Cargo.toml index 57c59df..b19caf7 100644 --- a/binding/rust/bin/Cargo.toml +++ b/binding/rust/example/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "ip2region" -default-run = "ip2region" +name = "example" +default-run = "example" version = "0.1.0" edition = "2021" rust-version = "1.66.0" @@ -10,5 +10,5 @@ license = "Apache-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -search = { path = "../search" } +ip2region2 = { path = "../ip2region2" } clap = { version = "4.0" } diff --git a/binding/rust/bin/src/cmd.rs b/binding/rust/example/src/cmd.rs similarity index 100% rename from binding/rust/bin/src/cmd.rs rename to binding/rust/example/src/cmd.rs diff --git a/binding/rust/bin/src/main.rs b/binding/rust/example/src/main.rs similarity index 71% rename from binding/rust/bin/src/main.rs rename to binding/rust/example/src/main.rs index ee4d154..4d2b97f 100644 --- a/binding/rust/bin/src/main.rs +++ b/binding/rust/example/src/main.rs @@ -7,15 +7,13 @@ mod cmd; fn main() { env::var("XDB_FILEPATH").unwrap_or_else(|_| { let matches = cmd::get_matches(); - let xdb_filepath = matches - .get_one::("xdb"); - if xdb_filepath.is_some() { - env::set_var("XDB_FILEPATH", xdb_filepath.unwrap()); + if let Some(xdb_filepath) = matches.get_one::("xdb") { + env::set_var("XDB_FILEPATH", xdb_filepath); } "".to_owned() }); - search::global_searcher(); + ip2region2::global_searcher(); println!("ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit"); loop { print!("ip2region>> "); @@ -26,7 +24,7 @@ fn main() { break; } let now = Instant::now(); - let result = search::search_by_ip(line.trim()); + let result = ip2region2::search_by_ip(line.trim()); println!("region: {:?}, took: {:?}", result, now.elapsed()); } } diff --git a/binding/rust/search/Cargo.toml b/binding/rust/ip2region2/Cargo.toml similarity index 94% rename from binding/rust/search/Cargo.toml rename to binding/rust/ip2region2/Cargo.toml index 142fd5c..6aa3475 100644 --- a/binding/rust/search/Cargo.toml +++ b/binding/rust/ip2region2/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "search" +name = "ip2region2" version = "0.1.0" edition = "2021" rust-version = "1.66.0" diff --git a/binding/rust/search/benches/search.rs b/binding/rust/ip2region2/benches/search.rs similarity index 56% rename from binding/rust/search/benches/search.rs rename to binding/rust/ip2region2/benches/search.rs index f1bfef3..b3a8411 100644 --- a/binding/rust/search/benches/search.rs +++ b/binding/rust/ip2region2/benches/search.rs @@ -1,7 +1,7 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use criterion::{criterion_group, criterion_main, Criterion}; use rand; -use search::{buffer_value, get_block_by_size, get_start_end_ptr, global_searcher, search_by_ip}; +use ip2region2::{buffer_value, get_block_by_size, get_start_end_ptr, global_searcher, search_by_ip}; fn ip_search_bench(c: &mut Criterion) { c.bench_function("ip_search_bench", |b| { @@ -23,19 +23,29 @@ fn buffer_value_bench(c: &mut Criterion) { fn get_block_by_size_bench(c: &mut Criterion) { c.bench_function("get_block_by_size", |b| { - b.iter(||{ - get_block_by_size(&global_searcher().buffer, rand::random::() as usize, 4); + b.iter(|| { + get_block_by_size( + &global_searcher().buffer(), + rand::random::() as usize, + 4, + ); }) }); } fn get_start_end_ptr_bench(c: &mut Criterion) { c.bench_function("get_start_end_ptr", |b| { - b.iter(|| { - get_start_end_ptr(rand::random::()); - }) + b.iter(|| { + get_start_end_ptr(rand::random::()); + }) }); } -criterion_group!(benches, ip_search_bench, buffer_value_bench, get_block_by_size_bench, get_start_end_ptr_bench); +criterion_group!( + benches, + ip_search_bench, + buffer_value_bench, + get_block_by_size_bench, + get_start_end_ptr_bench +); criterion_main!(benches); diff --git a/binding/rust/search/src/ip_value.rs b/binding/rust/ip2region2/src/ip_value.rs similarity index 100% rename from binding/rust/search/src/ip_value.rs rename to binding/rust/ip2region2/src/ip_value.rs diff --git a/binding/rust/search/src/lib.rs b/binding/rust/ip2region2/src/lib.rs similarity index 51% rename from binding/rust/search/src/lib.rs rename to binding/rust/ip2region2/src/lib.rs index ca80e92..97f5a95 100644 --- a/binding/rust/search/src/lib.rs +++ b/binding/rust/ip2region2/src/lib.rs @@ -1,80 +1,33 @@ -use std::env; use std::error::Error; -use std::fmt; -use std::fmt::Formatter; -use std::fs::File; -use std::io::Read; -use std::path::Path; - -use once_cell::sync::OnceCell; - +use std::fmt::Display; use ip_value::ToUIntIP; mod ip_value; +mod searcher; + +pub use searcher::global_searcher; const HEADER_INFO_LENGTH: usize = 256; const VECTOR_INDEX_COLS: usize = 256; const VECTOR_INDEX_SIZE: usize = 8; const SEGMENT_INDEX_SIZE: usize = 14; -/// store the xdb file in memory totally -pub struct Searcher { - pub buffer: Vec, -} - -impl Searcher { - /// you can set the XDB_FILEPATH - /// or super dir has data dir with the file ip2region.xdb - /// it will check ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb - pub fn new() -> Result> { - let xdb_filepath = env::var("XDB_FILEPATH") - .unwrap_or_else(|_| { - let prefix = "../".to_owned(); - for recurse in 1..4 { - let filepath = prefix.repeat(recurse) + "data/ip2region.xdb"; - if Path::new(filepath.as_str()).exists() { - return filepath - } - }; - panic!("you must set XDB_FILEPATH or put file in ../data/ip2region.xdb") - }); - println!("load xdb searcher file at {xdb_filepath}"); - let mut f = File::open(xdb_filepath)?; - let mut buffer = Vec::new(); - f.read_to_end(&mut buffer)?; - Ok(Self { buffer }) - } -} - -/// global init searcher thread safely -pub fn global_searcher() -> &'static Searcher { - static SEARCHER: OnceCell = OnceCell::new(); - SEARCHER.get_or_init(|| { - Searcher::new().unwrap() - }) -} - -impl fmt::Display for Searcher { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "searcher_with_len {}", self.buffer.len()) - } -} pub fn get_start_end_ptr(ip: u32) -> (usize, usize) { - let il0= ((ip >> 24) & 0xFF) as usize; + let il0 = ((ip >> 24) & 0xFF) as usize; let il1 = ((ip >> 16) & 0xFF) as usize; let idx = VECTOR_INDEX_SIZE * (il0 * VECTOR_INDEX_COLS + il1); let start_point = HEADER_INFO_LENGTH + idx; - let start_ptr = get_block_by_size(&global_searcher().buffer, start_point, 4); - let end_ptr = get_block_by_size(&global_searcher().buffer, start_point + 4, 4); + let start_ptr = get_block_by_size(global_searcher().buffer(), start_point, 4); + let end_ptr = get_block_by_size(global_searcher().buffer(), start_point + 4, 4); (start_ptr, end_ptr) } /// check https://mp.weixin.qq.com/s/ndjzu0BgaeBmDOCw5aqHUg for details pub fn search_by_ip(ip: T) -> Result> where - T: ToUIntIP, + T: ToUIntIP + Display, { let ip = ip.to_u32_ip()?; let (start_ptr, end_ptr) = get_start_end_ptr(ip); @@ -83,31 +36,25 @@ where while left <= right { let mid = (left + right) >> 1; - let offset = &start_ptr + mid * SEGMENT_INDEX_SIZE; + let offset = start_ptr + mid * SEGMENT_INDEX_SIZE; let buffer_ip_value = buffer_value(offset, SEGMENT_INDEX_SIZE); - let start_ip = get_block_by_size(&buffer_ip_value, 0, 4); - if &ip < &(start_ip as u32) { + let start_ip = get_block_by_size(buffer_ip_value, 0, 4); + if ip < (start_ip as u32) { right = mid - 1; - } else if &ip > &(get_block_by_size(&buffer_ip_value, 4, 4) as u32) { + } else if ip > (get_block_by_size(buffer_ip_value, 4, 4) as u32) { left = mid + 1; } else { - let data_length = get_block_by_size(&buffer_ip_value, 8, 2); - let data_offset = get_block_by_size(&buffer_ip_value, 10, 4); - let result = String::from_utf8( - buffer_value(data_offset, data_length) - .to_vec()); + let data_length = get_block_by_size(buffer_ip_value, 8, 2); + let data_offset = get_block_by_size(buffer_ip_value, 10, 4); + let result = String::from_utf8(buffer_value(data_offset, data_length).to_vec()); return Ok(result?); } } Err("not matched".into()) } -pub fn start_end_buffer_value(bytes: &[u8], offset: usize, length: usize) -> &[u8] { - &bytes[offset..offset+length] -} - pub fn buffer_value(offset: usize, length: usize) -> &'static [u8] { - &global_searcher().buffer[offset..offset + length] + &global_searcher().buffer()[offset..offset + length] } #[inline] @@ -117,8 +64,8 @@ where usize: From, { let mut result: usize = 0; - for (index, value) in bytes[offset..offset+length].iter().enumerate() { - result |= usize::from(value.clone()) << (index*8); + for (index, value) in bytes[offset..offset + length].iter().enumerate() { + result |= usize::from(value.clone()) << (index * 8); } result } @@ -128,6 +75,8 @@ mod tests { use std::net::Ipv4Addr; use std::str::FromStr; use std::thread; + use std::fs::File; + use std::io::Read; use super::*; diff --git a/binding/rust/ip2region2/src/searcher.rs b/binding/rust/ip2region2/src/searcher.rs new file mode 100644 index 0000000..abb64d9 --- /dev/null +++ b/binding/rust/ip2region2/src/searcher.rs @@ -0,0 +1,66 @@ +use std::env; +use std::error::Error; +use std::fmt; +use std::fmt::{Display, Formatter}; +use std::fs::File; +use std::io::Read; +use std::path::Path; + +use once_cell::sync::OnceCell; + +pub enum CachePolicy { + Never, + VecIndex, + Full, +} + +/// store the xdb file in memory totally +pub struct Searcher { + vec_cache: Vec, + full_cache: Vec, +} + +impl Searcher { + pub fn new(xdb_filepath: Option<&str>, cache_policy: Option) -> Result> { + let xdb_filepath = xdb_filepath.unwrap_or_else(|_| { + Searcher::default_detect_xdb_file().unwrap().as_str() + }); + println!("load xdb searcher file at {xdb_filepath}"); + let mut f = File::open(xdb_filepath)?; + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer)?; + Ok(Self { buffer }) + } + + /// it will check ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb + fn default_detect_xdb_file() -> Result> { + let prefix = "../".to_owned(); + for recurse in 1..4 { + let filepath = prefix.repeat(recurse) + "data/ip2region.xdb"; + if Path::new(filepath.as_str()).exists() { + return Ok(filepath); + } + } + Err("default filepath not find the xdb file, so you must set xdb_filepath".into()) + } + + pub fn buffer(&self) -> &Vec { + self.full_cache.as_ref() + } + + pub fn vec_cache(&self) -> &Vec { + self.vec_cache.as_ref() + } +} + +/// global init searcher thread safely +pub fn global_searcher() -> &'static Searcher { + static SEARCHER: OnceCell = OnceCell::new(); + SEARCHER.get_or_init(|| Searcher::new().unwrap()) +} + +impl Display for Searcher { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "searcher_with_len {}", self.buffer.len()) + } +} From 7e7a61243784d3891ae0afbd746932cd3f112132 Mon Sep 17 00:00:00 2001 From: gongzhengyang Date: Fri, 23 Dec 2022 12:02:12 +0800 Subject: [PATCH 3/7] =?UTF-8?q?feat:=20=E8=B0=83=E6=95=B4search=E7=9A=84?= =?UTF-8?q?=E6=95=B4=E4=BD=93=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/rust/ip2region2/Cargo.toml | 2 + binding/rust/ip2region2/src/lib.rs | 120 +------------- binding/rust/ip2region2/src/searcher.rs | 205 +++++++++++++++++++----- 3 files changed, 166 insertions(+), 161 deletions(-) diff --git a/binding/rust/ip2region2/Cargo.toml b/binding/rust/ip2region2/Cargo.toml index 6aa3475..11e5a02 100644 --- a/binding/rust/ip2region2/Cargo.toml +++ b/binding/rust/ip2region2/Cargo.toml @@ -10,6 +10,8 @@ license = "Apache-2.0" [dependencies] once_cell = "1.16" +tracing = "0.1" +#tracing-subscriber = "0.2" [dev-dependencies] criterion = "0.4" diff --git a/binding/rust/ip2region2/src/lib.rs b/binding/rust/ip2region2/src/lib.rs index 97f5a95..c307f85 100644 --- a/binding/rust/ip2region2/src/lib.rs +++ b/binding/rust/ip2region2/src/lib.rs @@ -1,121 +1,3 @@ -use std::error::Error; -use std::fmt::Display; -use ip_value::ToUIntIP; - mod ip_value; +pub use self::ip_value::ToUIntIP; mod searcher; - -pub use searcher::global_searcher; - -const HEADER_INFO_LENGTH: usize = 256; -const VECTOR_INDEX_COLS: usize = 256; -const VECTOR_INDEX_SIZE: usize = 8; -const SEGMENT_INDEX_SIZE: usize = 14; - - -pub fn get_start_end_ptr(ip: u32) -> (usize, usize) { - let il0 = ((ip >> 24) & 0xFF) as usize; - let il1 = ((ip >> 16) & 0xFF) as usize; - let idx = VECTOR_INDEX_SIZE * (il0 * VECTOR_INDEX_COLS + il1); - let start_point = HEADER_INFO_LENGTH + idx; - - let start_ptr = get_block_by_size(global_searcher().buffer(), start_point, 4); - let end_ptr = get_block_by_size(global_searcher().buffer(), start_point + 4, 4); - (start_ptr, end_ptr) -} - -/// check https://mp.weixin.qq.com/s/ndjzu0BgaeBmDOCw5aqHUg for details -pub fn search_by_ip(ip: T) -> Result> -where - T: ToUIntIP + Display, -{ - let ip = ip.to_u32_ip()?; - let (start_ptr, end_ptr) = get_start_end_ptr(ip); - let mut left: usize = 0; - let mut right: usize = (end_ptr - start_ptr) / SEGMENT_INDEX_SIZE; - - while left <= right { - let mid = (left + right) >> 1; - let offset = start_ptr + mid * SEGMENT_INDEX_SIZE; - let buffer_ip_value = buffer_value(offset, SEGMENT_INDEX_SIZE); - let start_ip = get_block_by_size(buffer_ip_value, 0, 4); - if ip < (start_ip as u32) { - right = mid - 1; - } else if ip > (get_block_by_size(buffer_ip_value, 4, 4) as u32) { - left = mid + 1; - } else { - let data_length = get_block_by_size(buffer_ip_value, 8, 2); - let data_offset = get_block_by_size(buffer_ip_value, 10, 4); - let result = String::from_utf8(buffer_value(data_offset, data_length).to_vec()); - return Ok(result?); - } - } - Err("not matched".into()) -} - -pub fn buffer_value(offset: usize, length: usize) -> &'static [u8] { - &global_searcher().buffer()[offset..offset + length] -} - -#[inline] -pub fn get_block_by_size(bytes: &[T], offset: usize, length: usize) -> usize -where - T: Clone, - usize: From, -{ - let mut result: usize = 0; - for (index, value) in bytes[offset..offset + length].iter().enumerate() { - result |= usize::from(value.clone()) << (index * 8); - } - result -} - -#[cfg(test)] -mod tests { - use std::net::Ipv4Addr; - use std::str::FromStr; - use std::thread; - use std::fs::File; - use std::io::Read; - - use super::*; - - ///test all types find correct - #[test] - fn test_multi_type_ip() { - search_by_ip("2.0.0.0").unwrap(); - search_by_ip("32").unwrap(); - search_by_ip(4294408949).unwrap(); - search_by_ip(Ipv4Addr::from_str("1.1.1.1").unwrap()).unwrap(); - } - - #[test] - fn test_match_all_ip_correct() { - let mut file = File::open("../../../data/ip.test.txt").unwrap(); - let mut contents = String::new(); - file.read_to_string(&mut contents).unwrap(); - for line in contents.split("\n") { - if !line.contains("|") { - continue; - } - let ip_test_line = line.splitn(3, "|").collect::>(); - let start_ip = Ipv4Addr::from_str(ip_test_line[0]).unwrap(); - let end_ip = Ipv4Addr::from_str(ip_test_line[1]).unwrap(); - for value in u32::from(start_ip)..u32::from(end_ip) + 1 { - let result = search_by_ip(value).unwrap(); - assert_eq!(result.as_str(), ip_test_line[2]) - } - } - } - - #[test] - fn test_multi_thread_only_load_xdb_once() { - let handle = thread::spawn(|| { - let result = search_by_ip("2.2.2.2").unwrap(); - println!("ip search in spawn: {result}"); - }); - let r = search_by_ip("1.1.1.1").unwrap(); - println!("ip search in main thread: {r}"); - handle.join().unwrap(); - } -} diff --git a/binding/rust/ip2region2/src/searcher.rs b/binding/rust/ip2region2/src/searcher.rs index abb64d9..8e4b4a6 100644 --- a/binding/rust/ip2region2/src/searcher.rs +++ b/binding/rust/ip2region2/src/searcher.rs @@ -1,4 +1,3 @@ -use std::env; use std::error::Error; use std::fmt; use std::fmt::{Display, Formatter}; @@ -8,59 +7,181 @@ use std::path::Path; use once_cell::sync::OnceCell; +use crate::ToUIntIP; + +const HEADER_INFO_LENGTH: usize = 256; +const VECTOR_INDEX_COLS: usize = 256; +const VECTOR_INDEX_SIZE: usize = 8; +const SEGMENT_INDEX_SIZE: usize = 14; +const VECTOR_INDEX_LENGTH: usize = 512 * 1024; + +const XDB_FILEPATH_ENV: &str = "XDB_FILEPATH"; +const CACHE_POLICY_ENV: &str = "CACHE_POLICY"; + +#[derive(Debug, Copy, Clone, PartialEq)] pub enum CachePolicy { - Never, + Never=1, VecIndex, Full, } -/// store the xdb file in memory totally -pub struct Searcher { - vec_cache: Vec, - full_cache: Vec, +/// check https://mp.weixin.qq.com/s/ndjzu0BgaeBmDOCw5aqHUg for details +pub fn search_by_ip(ip: T) -> Result> + where + T: ToUIntIP + Display, +{ + let ip = ip.to_u32_ip()?; + let (start_ptr, end_ptr) = get_start_end_ptr(ip); + let mut left: usize = 0; + let mut right: usize = (end_ptr - start_ptr) / SEGMENT_INDEX_SIZE; + + while left <= right { + let mid = (left + right) >> 1; + let offset = start_ptr + mid * SEGMENT_INDEX_SIZE; + let buffer_ip_value = &get_full_cache()[offset..offset+SEGMENT_INDEX_SIZE]; + let start_ip = get_block_by_size(buffer_ip_value, 0, 4); + if ip < (start_ip as u32) { + right = mid - 1; + } else if ip > (get_block_by_size(buffer_ip_value, 4, 4) as u32) { + left = mid + 1; + } else { + let data_length = get_block_by_size(buffer_ip_value, 8, 2); + let data_offset = get_block_by_size(buffer_ip_value, 10, 4); + let result = String::from_utf8(get_full_cache()[data_offset..(data_offset + data_length)].to_vec()); + return Ok(result?); + } + } + Err("not matched".into()) } -impl Searcher { - pub fn new(xdb_filepath: Option<&str>, cache_policy: Option) -> Result> { - let xdb_filepath = xdb_filepath.unwrap_or_else(|_| { - Searcher::default_detect_xdb_file().unwrap().as_str() - }); - println!("load xdb searcher file at {xdb_filepath}"); - let mut f = File::open(xdb_filepath)?; - let mut buffer = Vec::new(); - f.read_to_end(&mut buffer)?; - Ok(Self { buffer }) +pub fn get_start_end_ptr(ip: u32) -> (usize, usize) { + let il0 = ((ip >> 24) & 0xFF) as usize; + let il1 = ((ip >> 16) & 0xFF) as usize; + let idx = VECTOR_INDEX_SIZE * (il0 * VECTOR_INDEX_COLS + il1); + let start_point = idx; + let vector_cache = get_vector_index_cache(); + let start_ptr = get_block_by_size( vector_cache, start_point, 4); + let end_ptr = get_block_by_size(vector_cache, start_point + 4, 4); + (start_ptr, end_ptr) +} + +/// it will check ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb +fn default_detect_xdb_file() -> Result> { + let prefix = "../".to_owned(); + for recurse in 1..4 { + let filepath = prefix.repeat(recurse) + "data/ip2region.xdb"; + if Path::new(filepath.as_str()).exists() { + return Ok(filepath); + } + } + Err("default filepath not find the xdb file, so you must set xdb_filepath".into()) +} + +#[inline] +pub fn get_block_by_size(bytes: &[u8], offset: usize, length: usize) -> usize +{ + let mut result: usize = 0; + for (index, value) in bytes[offset..offset + length].iter().enumerate() { + result |= usize::from(value.clone()) << (index * 8); + } + result +} + +fn set_log_level() { + let rust_log_key = "RUST_LOG"; + std::env::var(rust_log_key).unwrap_or_else(|_| { + std::env::set_var(rust_log_key, "INFO"); + std::env::var(rust_log_key).unwrap() + }); +} + +pub fn searcher_init(xdb_filepath: Option, cache_policy: Option) { + set_log_level(); + let xdb_filepath = xdb_filepath.unwrap_or_else(|| { + default_detect_xdb_file().unwrap() + }); + std::env::set_var(XDB_FILEPATH_ENV, xdb_filepath.as_str()); + if let Some(policy) = cache_policy { + std::env::set_var(CACHE_POLICY_ENV, policy); + return; + } + std::env::set_var(CACHE_POLICY_ENV, CachePolicy::Full); + +} + +fn get_vector_index_cache() -> &'static [u8] { + let full_cache: &'static Vec = get_full_cache(); + &full_cache[HEADER_INFO_LENGTH..(HEADER_INFO_LENGTH + VECTOR_INDEX_LENGTH)] +} + +fn load_file() -> Vec{ + let xdb_filepath = std::env::var("XDB_FILEPATH").unwrap(); + tracing::debug!("load xdb searcher file at {} ", xdb_filepath); + let mut f = File::open(xdb_filepath).expect("file open error"); + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer).expect("load file error"); + buffer +} + +fn get_full_cache() -> &'static Vec { + let cache_policy = std::env::var(CACHE_POLICY_ENV).unwrap(); + if cache_policy == CachePolicy::Full { + static CACHE: OnceCell> = OnceCell::new(); + return CACHE.get_or_init(|| load_file()) + } + &load_file() +} + +#[cfg(test)] +mod tests { + use std::net::Ipv4Addr; + use std::str::FromStr; + use std::thread; + use std::fs::File; + use std::io::Read; + + use super::*; + + ///test all types find correct + #[test] + fn test_multi_type_ip() { + searcher_init(None, None); + + search_by_ip("2.0.0.0").unwrap(); + search_by_ip("32").unwrap(); + search_by_ip(4294408949).unwrap(); + search_by_ip(Ipv4Addr::from_str("1.1.1.1").unwrap()).unwrap(); } - /// it will check ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb - fn default_detect_xdb_file() -> Result> { - let prefix = "../".to_owned(); - for recurse in 1..4 { - let filepath = prefix.repeat(recurse) + "data/ip2region.xdb"; - if Path::new(filepath.as_str()).exists() { - return Ok(filepath); + #[test] + fn test_match_all_ip_correct() { + searcher_init(None, None); + let mut file = File::open("../../../data/ip.test.txt").unwrap(); + let mut contents = String::new(); + file.read_to_string(&mut contents).unwrap(); + for line in contents.split("\n") { + if !line.contains("|") { + continue; + } + let ip_test_line = line.splitn(3, "|").collect::>(); + let start_ip = Ipv4Addr::from_str(ip_test_line[0]).unwrap(); + let end_ip = Ipv4Addr::from_str(ip_test_line[1]).unwrap(); + for value in u32::from(start_ip)..u32::from(end_ip) + 1 { + let result = search_by_ip(value).unwrap(); + assert_eq!(result.as_str(), ip_test_line[2]) } } - Err("default filepath not find the xdb file, so you must set xdb_filepath".into()) } - pub fn buffer(&self) -> &Vec { - self.full_cache.as_ref() - } - - pub fn vec_cache(&self) -> &Vec { - self.vec_cache.as_ref() - } -} - -/// global init searcher thread safely -pub fn global_searcher() -> &'static Searcher { - static SEARCHER: OnceCell = OnceCell::new(); - SEARCHER.get_or_init(|| Searcher::new().unwrap()) -} - -impl Display for Searcher { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "searcher_with_len {}", self.buffer.len()) + #[test] + fn test_multi_thread_only_load_xdb_once() { + searcher_init(None, None); + let handle = thread::spawn(|| { + let result =search_by_ip("2.2.2.2").unwrap(); + println!("ip search in spawn: {result}"); + }); + let r = search_by_ip("1.1.1.1").unwrap(); + println!("ip search in main thread: {r}"); + handle.join().unwrap(); } } From 654080b1d06470a42d21c2d8632bd0af17b9d456 Mon Sep 17 00:00:00 2001 From: gongzhengyang Date: Fri, 23 Dec 2022 17:06:14 +0800 Subject: [PATCH 4/7] feat: change searcher fn --- binding/rust/example/Cargo.toml | 2 + binding/rust/example/src/main.rs | 28 +++++--- binding/rust/ip2region2/Cargo.toml | 2 +- binding/rust/ip2region2/benches/search.rs | 54 +++++++++------ binding/rust/ip2region2/src/lib.rs | 3 +- binding/rust/ip2region2/src/searcher.rs | 81 ++++++++--------------- 6 files changed, 84 insertions(+), 86 deletions(-) diff --git a/binding/rust/example/Cargo.toml b/binding/rust/example/Cargo.toml index b19caf7..3d0072f 100644 --- a/binding/rust/example/Cargo.toml +++ b/binding/rust/example/Cargo.toml @@ -12,3 +12,5 @@ license = "Apache-2.0" [dependencies] ip2region2 = { path = "../ip2region2" } clap = { version = "4.0" } +tracing = "0.1" +tracing-subscriber = "0.2.0" diff --git a/binding/rust/example/src/main.rs b/binding/rust/example/src/main.rs index 4d2b97f..2bfab9d 100644 --- a/binding/rust/example/src/main.rs +++ b/binding/rust/example/src/main.rs @@ -1,19 +1,29 @@ -use std::env; use std::io::Write; use std::time::Instant; +use ip2region2::{searcher_init, search_by_ip}; + mod cmd; fn main() { - env::var("XDB_FILEPATH").unwrap_or_else(|_| { - let matches = cmd::get_matches(); - if let Some(xdb_filepath) = matches.get_one::("xdb") { - env::set_var("XDB_FILEPATH", xdb_filepath); - } - "".to_owned() + /// set rust log level + let rust_log_key = "RUST_LOG"; + std::env::var(rust_log_key).unwrap_or_else(|_| { + std::env::set_var(rust_log_key, "INFO"); + std::env::var(rust_log_key).unwrap() }); + tracing_subscriber::fmt::init(); + + /// init default xdb_filepath config + /// if value if None, if will detect xdb file on ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb if exists + let matches = cmd::get_matches(); + if let Some(xdb_filepath) = matches.get_one::("xdb") { + searcher_init(Some(xdb_filepath.to_owned())) + } else { + searcher_init(None); + } + - ip2region2::global_searcher(); println!("ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit"); loop { print!("ip2region>> "); @@ -24,7 +34,7 @@ fn main() { break; } let now = Instant::now(); - let result = ip2region2::search_by_ip(line.trim()); + let result = search_by_ip(line.trim()); println!("region: {:?}, took: {:?}", result, now.elapsed()); } } diff --git a/binding/rust/ip2region2/Cargo.toml b/binding/rust/ip2region2/Cargo.toml index 11e5a02..a4d51bb 100644 --- a/binding/rust/ip2region2/Cargo.toml +++ b/binding/rust/ip2region2/Cargo.toml @@ -11,7 +11,7 @@ license = "Apache-2.0" [dependencies] once_cell = "1.16" tracing = "0.1" -#tracing-subscriber = "0.2" +tracing-subscriber = "0.2.0" [dev-dependencies] criterion = "0.4" diff --git a/binding/rust/ip2region2/benches/search.rs b/binding/rust/ip2region2/benches/search.rs index b3a8411..ff8bed1 100644 --- a/binding/rust/ip2region2/benches/search.rs +++ b/binding/rust/ip2region2/benches/search.rs @@ -1,42 +1,51 @@ -use criterion::{criterion_group, criterion_main, Criterion}; +use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rand; -use ip2region2::{buffer_value, get_block_by_size, get_start_end_ptr, global_searcher, search_by_ip}; +use ip2region2::searcher::{ + get_block_by_size, get_full_cache, get_vector_index_cache, + search_by_ip, searcher_init, get_int_block_value +}; fn ip_search_bench(c: &mut Criterion) { c.bench_function("ip_search_bench", |b| { + searcher_init(None); b.iter(|| { search_by_ip(rand::random::()).unwrap(); }) }); } -fn buffer_value_bench(c: &mut Criterion) { - c.bench_function("buffer_value", |b| { - b.iter(|| { - let offset = rand::random::(); - let length = rand::random::(); - buffer_value(offset as usize, length as usize); - }); - }); -} - fn get_block_by_size_bench(c: &mut Criterion) { c.bench_function("get_block_by_size", |b| { b.iter(|| { - get_block_by_size( - &global_searcher().buffer(), - rand::random::() as usize, - 4, - ); + black_box(get_block_by_size(get_full_cache(), + rand::random::() as usize, + 4)); }) }); } -fn get_start_end_ptr_bench(c: &mut Criterion) { - c.bench_function("get_start_end_ptr", |b| { +fn get_int_block_bench(c: &mut Criterion) { + c.bench_function("get_int_block_bench", |b| { b.iter(|| { - get_start_end_ptr(rand::random::()); + black_box(get_int_block_value(get_full_cache(), + rand::random::() as usize)); + }) + }); +} + +fn get_full_cache_bench(c: &mut Criterion) { + c.bench_function("get_full_cache", |b| { + b.iter(|| { + black_box(get_full_cache()); + }) + }); +} + +fn get_vec_index_cache_bench(c: &mut Criterion) { + c.bench_function("get_vec_index_cache", |b| { + b.iter(|| { + black_box(get_vector_index_cache()); }) }); } @@ -44,8 +53,9 @@ fn get_start_end_ptr_bench(c: &mut Criterion) { criterion_group!( benches, ip_search_bench, - buffer_value_bench, + get_int_block_bench, get_block_by_size_bench, - get_start_end_ptr_bench + get_full_cache_bench, + get_vec_index_cache_bench, ); criterion_main!(benches); diff --git a/binding/rust/ip2region2/src/lib.rs b/binding/rust/ip2region2/src/lib.rs index c307f85..350f1f2 100644 --- a/binding/rust/ip2region2/src/lib.rs +++ b/binding/rust/ip2region2/src/lib.rs @@ -1,3 +1,4 @@ mod ip_value; pub use self::ip_value::ToUIntIP; -mod searcher; +pub mod searcher; +pub use searcher::{search_by_ip, searcher_init}; diff --git a/binding/rust/ip2region2/src/searcher.rs b/binding/rust/ip2region2/src/searcher.rs index 8e4b4a6..d31fd6c 100644 --- a/binding/rust/ip2region2/src/searcher.rs +++ b/binding/rust/ip2region2/src/searcher.rs @@ -1,6 +1,5 @@ use std::error::Error; -use std::fmt; -use std::fmt::{Display, Formatter}; +use std::fmt::Display; use std::fs::File; use std::io::Read; use std::path::Path; @@ -16,19 +15,13 @@ const SEGMENT_INDEX_SIZE: usize = 14; const VECTOR_INDEX_LENGTH: usize = 512 * 1024; const XDB_FILEPATH_ENV: &str = "XDB_FILEPATH"; -const CACHE_POLICY_ENV: &str = "CACHE_POLICY"; -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum CachePolicy { - Never=1, - VecIndex, - Full, -} +static CACHE: OnceCell> = OnceCell::new(); /// check https://mp.weixin.qq.com/s/ndjzu0BgaeBmDOCw5aqHUg for details pub fn search_by_ip(ip: T) -> Result> - where - T: ToUIntIP + Display, +where + T: ToUIntIP + Display, { let ip = ip.to_u32_ip()?; let (start_ptr, end_ptr) = get_start_end_ptr(ip); @@ -38,7 +31,7 @@ pub fn search_by_ip(ip: T) -> Result> while left <= right { let mid = (left + right) >> 1; let offset = start_ptr + mid * SEGMENT_INDEX_SIZE; - let buffer_ip_value = &get_full_cache()[offset..offset+SEGMENT_INDEX_SIZE]; + let buffer_ip_value = &get_full_cache()[offset..offset + SEGMENT_INDEX_SIZE]; let start_ip = get_block_by_size(buffer_ip_value, 0, 4); if ip < (start_ip as u32) { right = mid - 1; @@ -47,7 +40,9 @@ pub fn search_by_ip(ip: T) -> Result> } else { let data_length = get_block_by_size(buffer_ip_value, 8, 2); let data_offset = get_block_by_size(buffer_ip_value, 10, 4); - let result = String::from_utf8(get_full_cache()[data_offset..(data_offset + data_length)].to_vec()); + let result = String::from_utf8( + get_full_cache()[data_offset..(data_offset + data_length)].to_vec(), + ); return Ok(result?); } } @@ -60,7 +55,7 @@ pub fn get_start_end_ptr(ip: u32) -> (usize, usize) { let idx = VECTOR_INDEX_SIZE * (il0 * VECTOR_INDEX_COLS + il1); let start_point = idx; let vector_cache = get_vector_index_cache(); - let start_ptr = get_block_by_size( vector_cache, start_point, 4); + let start_ptr = get_block_by_size(vector_cache, start_point, 4); let end_ptr = get_block_by_size(vector_cache, start_point + 4, 4); (start_ptr, end_ptr) } @@ -78,44 +73,29 @@ fn default_detect_xdb_file() -> Result> { } #[inline] -pub fn get_block_by_size(bytes: &[u8], offset: usize, length: usize) -> usize -{ +pub fn get_block_by_size(bytes: &[u8], offset: usize, length: usize) -> usize { let mut result: usize = 0; for (index, value) in bytes[offset..offset + length].iter().enumerate() { - result |= usize::from(value.clone()) << (index * 8); + result += usize::from(*value) << (index << 3); } result } -fn set_log_level() { - let rust_log_key = "RUST_LOG"; - std::env::var(rust_log_key).unwrap_or_else(|_| { - std::env::set_var(rust_log_key, "INFO"); - std::env::var(rust_log_key).unwrap() - }); +pub fn searcher_init(xdb_filepath: Option) +{ + let xdb_filepath = xdb_filepath.unwrap_or_else(|| default_detect_xdb_file().unwrap()); + std::env::set_var(XDB_FILEPATH_ENV, xdb_filepath); + CACHE.get_or_init(load_file); } -pub fn searcher_init(xdb_filepath: Option, cache_policy: Option) { - set_log_level(); - let xdb_filepath = xdb_filepath.unwrap_or_else(|| { - default_detect_xdb_file().unwrap() - }); - std::env::set_var(XDB_FILEPATH_ENV, xdb_filepath.as_str()); - if let Some(policy) = cache_policy { - std::env::set_var(CACHE_POLICY_ENV, policy); - return; - } - std::env::set_var(CACHE_POLICY_ENV, CachePolicy::Full); - -} - -fn get_vector_index_cache() -> &'static [u8] { +pub fn get_vector_index_cache() -> &'static [u8] { let full_cache: &'static Vec = get_full_cache(); &full_cache[HEADER_INFO_LENGTH..(HEADER_INFO_LENGTH + VECTOR_INDEX_LENGTH)] } -fn load_file() -> Vec{ - let xdb_filepath = std::env::var("XDB_FILEPATH").unwrap(); +fn load_file() -> Vec { + let xdb_filepath = + std::env::var("XDB_FILEPATH").unwrap_or_else(|_| default_detect_xdb_file().unwrap()); tracing::debug!("load xdb searcher file at {} ", xdb_filepath); let mut f = File::open(xdb_filepath).expect("file open error"); let mut buffer = Vec::new(); @@ -123,29 +103,24 @@ fn load_file() -> Vec{ buffer } -fn get_full_cache() -> &'static Vec { - let cache_policy = std::env::var(CACHE_POLICY_ENV).unwrap(); - if cache_policy == CachePolicy::Full { - static CACHE: OnceCell> = OnceCell::new(); - return CACHE.get_or_init(|| load_file()) - } - &load_file() +pub fn get_full_cache() -> &'static Vec { + CACHE.get_or_init(load_file) } #[cfg(test)] mod tests { + use std::fs::File; + use std::io::Read; use std::net::Ipv4Addr; use std::str::FromStr; use std::thread; - use std::fs::File; - use std::io::Read; use super::*; ///test all types find correct #[test] fn test_multi_type_ip() { - searcher_init(None, None); + searcher_init(None); search_by_ip("2.0.0.0").unwrap(); search_by_ip("32").unwrap(); @@ -155,7 +130,7 @@ mod tests { #[test] fn test_match_all_ip_correct() { - searcher_init(None, None); + searcher_init(None); let mut file = File::open("../../../data/ip.test.txt").unwrap(); let mut contents = String::new(); file.read_to_string(&mut contents).unwrap(); @@ -175,9 +150,9 @@ mod tests { #[test] fn test_multi_thread_only_load_xdb_once() { - searcher_init(None, None); + searcher_init(None); let handle = thread::spawn(|| { - let result =search_by_ip("2.2.2.2").unwrap(); + let result = search_by_ip("2.2.2.2").unwrap(); println!("ip search in spawn: {result}"); }); let r = search_by_ip("1.1.1.1").unwrap(); From f10b8971e11875f32a7cbb1c3431d4aaf70e0fcc Mon Sep 17 00:00:00 2001 From: gongzhengyang Date: Fri, 23 Dec 2022 18:39:58 +0800 Subject: [PATCH 5/7] feat: add example subcommands --- binding/rust/example/src/cmd.rs | 25 +++++-- binding/rust/example/src/main.rs | 90 +++++++++++++++++++---- binding/rust/ip2region2/benches/search.rs | 18 +++-- binding/rust/ip2region2/src/searcher.rs | 15 +++- 4 files changed, 120 insertions(+), 28 deletions(-) diff --git a/binding/rust/example/src/cmd.rs b/binding/rust/example/src/cmd.rs index 05c3912..98402a4 100644 --- a/binding/rust/example/src/cmd.rs +++ b/binding/rust/example/src/cmd.rs @@ -1,14 +1,27 @@ -use clap::{arg, ArgMatches, Command}; +use clap::{Arg, ArgMatches, Command}; pub fn get_matches() -> ArgMatches { + let db_arg = Arg::new("db") + .long("db") + .help("the xdb filepath, you can set this field like \ + ../data/ip2region.xdb,if you dont set,\ + if will detect xdb file on ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb if exists"); + Command::new("ip2region") .version("0.1") .about("ip2region bin program") - .long_about( - "you can set environment XDB_FILEPATH=../data/ip2region or just use --xdb in command", - ) - .arg( - arg!(--xdb "the xdb filepath, you can set this field like ../data/ip2region.xdb"), + .long_about("you can set --db in command to specific the xdb filepath, default run query") + .subcommand(Command::new("query").about("query test").arg(&db_arg)) + .subcommand( + Command::new("bench") + .about("bench test") + .arg( + Arg::new("src") + .long("src") + .help("set this to specific source bench file") + .required(true), + ) + .arg(&db_arg), ) .get_matches() } diff --git a/binding/rust/example/src/main.rs b/binding/rust/example/src/main.rs index 2bfab9d..bf5c874 100644 --- a/binding/rust/example/src/main.rs +++ b/binding/rust/example/src/main.rs @@ -1,29 +1,67 @@ +extern crate core; + +use clap::ArgMatches; +use std::fs::File; +use std::io::Read; use std::io::Write; +use std::net::Ipv4Addr; +use std::str::FromStr; use std::time::Instant; -use ip2region2::{searcher_init, search_by_ip}; +use ip2region2::{search_by_ip, searcher_init}; mod cmd; -fn main() { - /// set rust log level +/// set rust log level, if you don`t want print log, you can skip this +fn log_init() { let rust_log_key = "RUST_LOG"; std::env::var(rust_log_key).unwrap_or_else(|_| { std::env::set_var(rust_log_key, "INFO"); std::env::var(rust_log_key).unwrap() }); tracing_subscriber::fmt::init(); +} - /// init default xdb_filepath config - /// if value if None, if will detect xdb file on ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb if exists - let matches = cmd::get_matches(); - if let Some(xdb_filepath) = matches.get_one::("xdb") { - searcher_init(Some(xdb_filepath.to_owned())) - } else { - searcher_init(None); +fn bench_test(src_filepath: &str) { + let now = Instant::now(); + let mut count = 0; + let mut file = File::open(src_filepath).unwrap(); + let mut contents = String::new(); + file.read_to_string(&mut contents).unwrap(); + for line in contents.split('\n') { + if !line.contains('|') { + continue; + } + let ip_test_line = line.splitn(3, '|').collect::>(); + let start_ip = Ipv4Addr::from_str(ip_test_line[0]).unwrap(); + let end_ip = Ipv4Addr::from_str(ip_test_line[1]).unwrap(); + if end_ip < start_ip { + panic!("start ip({start_ip}) should not be greater than end ip({end_ip})") + } + let start_ip = u32::from(start_ip); + let end_ip = u32::from(end_ip); + let mid_ip = (((start_ip as u64) + (end_ip as u64)) >> 1) as u32; + for ip in [ + start_ip, + ((start_ip as u64 + mid_ip as u64) >> 1) as u32, + mid_ip, + ((mid_ip as u64 + end_ip as u64) >> 1) as u32, + end_ip, + ] { + search_by_ip(ip).unwrap(); + count += 1; + } } + println!( + "Bench finished, total: {count},\ + took: {:?} ,\ + cost: {:?}/op", + now.elapsed(), + now.elapsed() / count + ) +} - +fn query_test() { println!("ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit"); loop { print!("ip2region>> "); @@ -33,8 +71,34 @@ fn main() { if line.contains("quit") { break; } + let line = line.trim(); let now = Instant::now(); - let result = search_by_ip(line.trim()); - println!("region: {:?}, took: {:?}", result, now.elapsed()); + let result = search_by_ip(line); + let cost = now.elapsed(); + println!("region: {result:?}, took: {cost:?}", ); + } +} + +fn matches_for_searcher(matches: &ArgMatches) { + if let Some(xdb_filepath) = matches.get_one::("db") { + searcher_init(Some(xdb_filepath.to_owned())) + } else { + searcher_init(None); + } +} + +fn main() { + log_init(); + let matches = cmd::get_matches(); + if let Some(sub_matches) = matches.subcommand_matches("bench") { + matches_for_searcher(sub_matches); + let src_filepath = sub_matches.get_one::("src").unwrap(); + + bench_test(src_filepath); + } + + if let Some(sub_matches) = matches.subcommand_matches("query") { + matches_for_searcher(sub_matches); + query_test() } } diff --git a/binding/rust/ip2region2/benches/search.rs b/binding/rust/ip2region2/benches/search.rs index ff8bed1..3854502 100644 --- a/binding/rust/ip2region2/benches/search.rs +++ b/binding/rust/ip2region2/benches/search.rs @@ -2,8 +2,8 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rand; use ip2region2::searcher::{ - get_block_by_size, get_full_cache, get_vector_index_cache, - search_by_ip, searcher_init, get_int_block_value + get_block_by_size, get_full_cache, get_int_block_value, get_vector_index_cache, search_by_ip, + searcher_init, }; fn ip_search_bench(c: &mut Criterion) { @@ -18,9 +18,11 @@ fn ip_search_bench(c: &mut Criterion) { fn get_block_by_size_bench(c: &mut Criterion) { c.bench_function("get_block_by_size", |b| { b.iter(|| { - black_box(get_block_by_size(get_full_cache(), - rand::random::() as usize, - 4)); + black_box(get_block_by_size( + get_full_cache(), + rand::random::() as usize, + 4, + )); }) }); } @@ -28,8 +30,10 @@ fn get_block_by_size_bench(c: &mut Criterion) { fn get_int_block_bench(c: &mut Criterion) { c.bench_function("get_int_block_bench", |b| { b.iter(|| { - black_box(get_int_block_value(get_full_cache(), - rand::random::() as usize)); + black_box(get_int_block_value( + get_full_cache(), + rand::random::() as usize, + )); }) }); } diff --git a/binding/rust/ip2region2/src/searcher.rs b/binding/rust/ip2region2/src/searcher.rs index d31fd6c..f7dc20b 100644 --- a/binding/rust/ip2region2/src/searcher.rs +++ b/binding/rust/ip2region2/src/searcher.rs @@ -81,8 +81,7 @@ pub fn get_block_by_size(bytes: &[u8], offset: usize, length: usize) -> usize { result } -pub fn searcher_init(xdb_filepath: Option) -{ +pub fn searcher_init(xdb_filepath: Option) { let xdb_filepath = xdb_filepath.unwrap_or_else(|| default_detect_xdb_file().unwrap()); std::env::set_var(XDB_FILEPATH_ENV, xdb_filepath); CACHE.get_or_init(load_file); @@ -159,4 +158,16 @@ mod tests { println!("ip search in main thread: {r}"); handle.join().unwrap(); } + + #[test] + fn test_multi_searcher_init() { + for _ in 0..5 { + thread::spawn(|| { + searcher_init(None); + }); + } + searcher_init(None); + searcher_init(Some(String::from("test"))); + search_by_ip(123).unwrap(); + } } From 6c9fdc5d26bd6451b99f0208ed667def67276139 Mon Sep 17 00:00:00 2001 From: gongzhengyang Date: Sat, 24 Dec 2022 10:39:39 +0800 Subject: [PATCH 6/7] feat: add bench and update tracing version --- binding/rust/ReadMe.md | 42 +++++++++++++++++++++--------- binding/rust/example/Cargo.toml | 6 ++--- binding/rust/example/src/main.rs | 12 ++++++--- binding/rust/ip2region2/Cargo.toml | 2 +- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/binding/rust/ReadMe.md b/binding/rust/ReadMe.md index 30e4b0c..53b368e 100644 --- a/binding/rust/ReadMe.md +++ b/binding/rust/ReadMe.md @@ -2,27 +2,23 @@ # 使用方式 +使用`cargo`新建一个项目，如`cargo new ip-test` + 配置`Cargo.toml`如下 ```toml [dependencies] search = { git = "https://github.com/lionsoul2014/ip2region.git", branch = "master" } -# 如果要在异步环境下使用，需要加上如下依赖 -tokio = { version = "1", features = ["full"]} ``` -程序启动的时候是没加载文件，这个程序占用内存`1M`左右 +`ip2region.xdb`文件会直接加载到内存，程序占用内存`13M`左右 -一旦开始执行查询，`ip2region.xdb`文件会直接加载到内存，程序占用内存`12M`左右 - -预先加载整个` ip2region.xdb` 到内存，完全基于内存查询，该方式线程安全，采用`once_cell::sync::OnceCell`，只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread` +预先加载整个` ip2region.xdb` 到内存，完全基于内存查询，只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread` ### 缓存整个 `xdb` 数据 编写`main.rs` -**需要使用`XDB_FILEPATH`指定`ip2region.xdb`文件的路径**，该参数可以使用相对路径或者绝对路径，如果使用相对路径报错，请修改为绝对路径 - ```rust use std::env; @@ -65,17 +61,39 @@ Ok("0|0|0|内网IP|内网IP") Ok("0|0|0|内网IP|内网IP") ``` +# `binding/rust`路径下面的结构说明 + +`ip2region2` + +- 包含了`ip`到`region`的函数调用库 +- 里面包含了单元测试和`benchmark`测试 + +`example` + +- 包含了命令行可执行文件生成的源码程序 +- 作为一个用于`rust`的开发集成例子 + +开始编译之后会生成如下 + +`target` + +- 文件夹存放编译之后的文件以及编译产生的临时文件与缓存 + +`Cargo.lock` + +- 固定`rust`第三方库的版本 + +这个些编译生成的文件全部在`.gitignore`中有标识，不会被提交 + # 编译程序 -通过如下方式编译得到 `ip2region` 可执行程序 - -切换到 `rust binding` 根目录，执行如下命令 +切换到 `binding/rust` 路径，执行如下命令 ```bash ➜ cargo build -r ``` -生成的二进制文件会在`./target/release/ip2region`位置 +生成的二进制文件会在`./target/release/rust-example`位置 # 查询测试 diff --git a/binding/rust/example/Cargo.toml b/binding/rust/example/Cargo.toml index 3d0072f..952f0c2 100644 --- a/binding/rust/example/Cargo.toml +++ b/binding/rust/example/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "example" -default-run = "example" +name = "rust-example" +default-run = "rust-example" version = "0.1.0" edition = "2021" rust-version = "1.66.0" @@ -13,4 +13,4 @@ license = "Apache-2.0" ip2region2 = { path = "../ip2region2" } clap = { version = "4.0" } tracing = "0.1" -tracing-subscriber = "0.2.0" +tracing-subscriber = "0.3.14" diff --git a/binding/rust/example/src/main.rs b/binding/rust/example/src/main.rs index bf5c874..cced30c 100644 --- a/binding/rust/example/src/main.rs +++ b/binding/rust/example/src/main.rs @@ -1,6 +1,5 @@ extern crate core; -use clap::ArgMatches; use std::fs::File; use std::io::Read; use std::io::Write; @@ -8,6 +7,8 @@ use std::net::Ipv4Addr; use std::str::FromStr; use std::time::Instant; +use clap::ArgMatches; + use ip2region2::{search_by_ip, searcher_init}; mod cmd; @@ -28,11 +29,15 @@ fn bench_test(src_filepath: &str) { let mut file = File::open(src_filepath).unwrap(); let mut contents = String::new(); file.read_to_string(&mut contents).unwrap(); + for line in contents.split('\n') { if !line.contains('|') { continue; } let ip_test_line = line.splitn(3, '|').collect::>(); + if ip_test_line.len() != 3 { + panic!("this line {line} don`t have enough `|` for spilt"); + } let start_ip = Ipv4Addr::from_str(ip_test_line[0]).unwrap(); let end_ip = Ipv4Addr::from_str(ip_test_line[1]).unwrap(); if end_ip < start_ip { @@ -48,7 +53,8 @@ fn bench_test(src_filepath: &str) { ((mid_ip as u64 + end_ip as u64) >> 1) as u32, end_ip, ] { - search_by_ip(ip).unwrap(); + let result = search_by_ip(ip).unwrap(); + assert_eq!(result.as_str(), ip_test_line[2]); count += 1; } } @@ -75,7 +81,7 @@ fn query_test() { let now = Instant::now(); let result = search_by_ip(line); let cost = now.elapsed(); - println!("region: {result:?}, took: {cost:?}", ); + println!("region: {result:?}, took: {cost:?}",); } } diff --git a/binding/rust/ip2region2/Cargo.toml b/binding/rust/ip2region2/Cargo.toml index a4d51bb..07c7b1b 100644 --- a/binding/rust/ip2region2/Cargo.toml +++ b/binding/rust/ip2region2/Cargo.toml @@ -11,7 +11,7 @@ license = "Apache-2.0" [dependencies] once_cell = "1.16" tracing = "0.1" -tracing-subscriber = "0.2.0" +tracing-subscriber = "0.3.14" [dev-dependencies] criterion = "0.4" From 8d606affffacb451568e76a21fcc0d0d30844f8a Mon Sep 17 00:00:00 2001 From: gongzhengyang Date: Sat, 24 Dec 2022 13:17:29 +0800 Subject: [PATCH 7/7] docs: finish the binding/rust Readme.md --- binding/rust/ReadMe.md | 362 +++++++++++++++++----- binding/rust/ip2region2/benches/search.rs | 27 +- 2 files changed, 286 insertions(+), 103 deletions(-) diff --git a/binding/rust/ReadMe.md b/binding/rust/ReadMe.md index 53b368e..e43a385 100644 --- a/binding/rust/ReadMe.md +++ b/binding/rust/ReadMe.md @@ -1,72 +1,192 @@ # `ip2region xdb rust` 查询客户端实现 +# 实现效果 + +得益于`xdb`数据存储格式设计以及`rust`编译器的高度代码优化 + +- 实现单核`CPU`下接近每秒千万级别的查询，如果是4核8线这样的`CPU`，采用`tokio`异步运行时，可以达到接近每秒4千万查询速度，查询速度取决于`CPU`物理核睿频频率 +- 达到查询稳定在`100-150ns/op` +- `ip2region.xdb`文件会直接加载到内存，整个程序运行时候占用内存`13M`左右，即使是多线程或者异步运行时下面的高并发查询也是稳定在这个内存大小 +- 只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread` + +# 缓存方式说明 + +由于基于文件的查询以及缓存`VectorIndex`索引在并发较高(比如每秒上百并发)的情况下，每次查询都会从磁盘加载`ip2region.xdb`文件进入内存，由此会产生很高的磁盘`IO`以及极大的内存占用，所以决定做一次减法，不对这两种缓存进行开发，只提供缓存整个`xdb`文件的方式，以此实现最小的并发查询内存开销以及极限`CPU`性能压榨 + # 使用方式 -使用`cargo`新建一个项目，如`cargo new ip-test` +使用`cargo`新建一个项目，比如`cargo new ip-test` -配置`Cargo.toml`如下 +同时把`ip2region.xdb`文件也移动到该项目根路径下，或者不移动，下面示例编译的时候注意调整`xdb_filepath`的参数值 + +配置`Cargo.toml`的`[dependencies]`如下 ```toml [dependencies] search = { git = "https://github.com/lionsoul2014/ip2region.git", branch = "master" } +# 用于生成随机数 +rand = "0.8" +# 用于初始化日志打印 +tracing = "0.1" +tracing-subscriber = "0.3.14" +# 异步运行时 +tokio = { version = "1", features = ["full"]} ``` -`ip2region.xdb`文件会直接加载到内存，程序占用内存`13M`左右 - -预先加载整个` ip2region.xdb` 到内存，完全基于内存查询，只会加载一次数据，多线程安全，可以自由使用`tokio`异步运行时或者标准库的多线程`std::thread` - -### 缓存整个 `xdb` 数据 +### 基本使用示例 编写`main.rs` ```rust -use std::env; +use std::net::Ipv4Addr; +use std::thread; +use std::time::{Duration, Instant}; -#[tokio::main] -async fn main() { - env::set_var( - "XDB_FILEPATH", - "../data/ip2region.xdb", - ); - //可以调用如下直接加载文件 - // search::global_searcher(); - - - // search_by_ip的参数可以是u32类型，字符串IP类型，字符串数字类型 +use ip2region2::{search_by_ip, searcher_init}; + +fn main() { + // 配置输出日志信息 + tracing_subscriber::fmt::init(); + + // 初始化加载xdb文件 + let xdb_filepath = "./ip2region.xdb"; + searcher_init(Some(xdb_filepath.to_owned())); + // 如果../data或者../../data或者../../../data下面有对应的ip2region.xdb文件 + // 初始化函数可以直接调用如下 + // searcher_init(None); + + println!("\n测试多类型查询"); + println!("{}", search_by_ip("9999999").unwrap()); + println!("{}", search_by_ip("1.0.1.0").unwrap()); + println!("{}", search_by_ip(9999999).unwrap()); + println!("{:?}", search_by_ip(Ipv4Addr::from(3333333))); + + println!("\n测试多线程初始化以及多线程查询"); for i in 1..5 { - tokio::spawn(async move { - // u32 - println!("{:?}", search::search_by_ip(i)); + thread::spawn(move || { + // 再次初始化是没什么效果的 + searcher_init(Some(xdb_filepath.to_owned())); + println!("in thread {i} {:?}", search_by_ip(rand::random::())); }); } - // ip str - println!("{:?}", search::search_by_ip("1.0.1.0")); + // 等待多线程执行结束 + thread::sleep(Duration::from_secs(1)); - // u32 str - let ip_u32 = 1 << 24 | 1 << 8; - println!("{:?}", search::search_by_ip(ip_u32.to_string().as_str())); + let count = 10_000_000; + print!("\n计算千万数据总共耗时: "); + let now = Instant::now(); + for _ in 0..count { + search_by_ip(rand::random::()).unwrap(); + } + println!("{:?}, ave: {:?}", now.elapsed(), now.elapsed()/count); + + print!("\n计算千万数据，每次迭代都统计耗时的总共耗时: "); + let mut total = Duration::from_secs(0); + for _ in 0..count { + let current = Instant::now(); + search_by_ip(rand::random::()).unwrap(); + total += current.elapsed(); + } + println!("{:?}, ave: {:?}", total, total/count); + + print!("\n计算空迭代耗时: "); + let now = Instant::now(); + for _ in 0..count {} + println!("{:?}", now.elapsed()); } ``` -进行测试 +进行测试如下，需要指定`RUST_LOG`参数打印日志 ```shell -$ cargo run -init xdb searcher at ../data/ip2region.xdb -Ok("中国|0|福建省|福州市|电信") -Ok("中国|0|福建省|福州市|电信") -Ok("0|0|0|内网IP|内网IP") -Ok("0|0|0|内网IP|内网IP") -Ok("0|0|0|内网IP|内网IP") +➜ RUST_LOG=debug cargo run -r + Compiling ip-test v0.1.0 (/home/gong/rust-work/ip-test) + Finished release [optimized] target(s) in 0.27s + Running `target/release/ip-test` +2022-12-24T03:31:22.921958Z DEBUG ip2region2::searcher: load xdb searcher file at ./ip2region.xdb + +测试多类型查询 +0|0|0|内网IP|内网IP +中国|0|福建省|福州市|电信 +0|0|0|内网IP|内网IP Ok("0|0|0|内网IP|内网IP") + +测试多线程初始化以及多线程查询 +in thread 1 Ok("美国|0|新墨西哥|0|康卡斯特") +in thread 3 Ok("0|0|0|内网IP|内网IP") +in thread 2 Ok("土耳其|0|Ankara|0|0") +in thread 4 Ok("爱尔兰|0|Cork|0|0") + +计算千万数据总共耗时: 1.176241972s, ave: 117ns + +计算千万数据，每次迭代都统计耗时的总共耗时: 1.407956755s, ave: 140ns + +计算空迭代耗时: 40ns +``` + +### `tokio`使用示例 + +```rust +use std::time::Instant; + +use tokio::sync::mpsc; + +use ip2region2::{search_by_ip, searcher_init}; + +#[tokio::main] +async fn main() { + // 配置输出debug 信息 + tracing_subscriber::fmt::init(); + searcher_init(Some("./ip2region.xdb".to_owned())); + let main_now = Instant::now(); + let (tx, mut rx) = mpsc::channel(10); + for i in 0..6 { + let tx = tx.clone(); + tokio::spawn(async move { + let now = Instant::now(); + let count = 10_000_000; + for _ in 0..count { + search_by_ip(rand::random::()).unwrap(); + } + let message = format!( + "tokio spawn {i} over cost: {:?}, ave: {:?}", + now.elapsed(), + now.elapsed() / count + ); + tx.send(message).await.unwrap(); + }); + } + drop(tx); + while let Some(message) = rx.recv().await { + println!("{}", message); + } + println!("总共耗时: {:?}", main_now.elapsed()); +} +``` + +开始执行测试 + +```shell +$ RUST_LOG=debug cargo run -r + Compiling ip-test v0.1.0 (/home/gong/rust-work/ip-test) + Finished release [optimized] target(s) in 0.51s + Running `target/release/ip-test` +2022-12-24T04:05:32.876664Z DEBUG ip2region2::searcher: load xdb searcher file at ./ip2region.xdb +tokio spawn 4 over cost: 1.133448675s, ave: 113ns +tokio spawn 2 over cost: 1.133938619s, ave: 113ns +tokio spawn 1 over cost: 1.136872027s, ave: 113ns +tokio spawn 5 over cost: 1.173464286s, ave: 117ns +tokio spawn 0 over cost: 1.197527014s, ave: 119ns +tokio spawn 3 over cost: 1.26446099s, ave: 126ns +总共耗时: 1.264631935s ``` # `binding/rust`路径下面的结构说明 `ip2region2` -- 包含了`ip`到`region`的函数调用库 -- 里面包含了单元测试和`benchmark`测试 +- 封装了`ip`到`region`的函数 +- 里面包含单元测试和`benchmark`测试 `example` @@ -83,81 +203,157 @@ Ok("0|0|0|内网IP|内网IP") - 固定`rust`第三方库的版本 -这个些编译生成的文件全部在`.gitignore`中有标识，不会被提交 +编译生成的文件全部在`.gitignore`中有标识，不会被提交 # 编译程序 -切换到 `binding/rust` 路径，执行如下命令 +切换到 `ip2region/binding/rust` 路径，执行如下命令 ```bash -➜ cargo build -r +$ cargo build -r ``` 生成的二进制文件会在`./target/release/rust-example`位置 # 查询测试 -通过 `./target/release/ip2region` 命令来测试查询 -``` -➜ ./target/release/ip2region --help -you can set environment XDB_FILEPATH=../data/ip2region or just use --xdb in command -Usage: ip2region [OPTIONS] +切换到 `ip2region/binding/rust` 路径，执行如下命令 + +`help`输出如下 + +```shell +$ ./target/release/rust-example query --help +query test + +Usage: rust-example query [OPTIONS] + Options: - --xdb - the xdb filepath, you can set this field like ../data/ip2region.xdb - -h, --help - Print help information (use `-h` for a summary) - -V, --version - Print version information + --db the xdb filepath, you can set this field like ../data/ip2region.xdb,if you dont set,if will detect xdb file on ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb if exists + -h, --help Print help information ``` -命令行指定参数进行查询测试，输入 `ip` 地址或者一个`u32`类型的数字进行查询即可，输入 `quit` 退出测试程序 -```bash -➜ ./target/release/ip2region --xdb=../../data/ip2region.xdb -init xdb searcher at ../../data/ip2region.xdb -ip2region xdb searcher test program, type `quit` to exit +执行测试，使用默认`data/ip2region.txt` + +```shell +$ ./target/release/rust-example query --db=../../data/ip2region.xdb +ip2region xdb searcher test program, type `quit` or `Ctrl + c` to exit +ip2region>> 123123123 +region: Ok("美国|0|0|0|0"), took: 4.94µs ip2region>> 1.1.1.1 -region: Ok("澳大利亚|0|0|0|0"), took: 4.227µs +region: Ok("澳大利亚|0|0|0|0"), took: 2.057µs ip2region>> 2.2.2.2 -region: Ok("法国|0|0|0|橘子电信"), took: 4.495µs -ip2region>> 222222222 -region: Ok("美国|0|康涅狄格|0|0"), took: 4.048µs +region: Ok("法国|0|0|0|橘子电信"), took: 4.294µs +ip2region>> ``` -或者使用环境变量 +这边发现每次查询的消耗时间都超过`1µs`，和开头所说的纳秒级查询不一致啊，这个是由于`rust`的标准库封装的`use std::time::Instant`对象是调用系统底层函数实现的，导致会有微秒级别的误差 -```shell -➜ XDB_FILEPATH=../../data/ip2region.xdb ./target/release/ip2region -init xdb searcher at ../../data/ip2region.xdb -ip2region xdb searcher test program, type `quit` to exit -ip2region>> 2.2.2.2 -region: Ok("法国|0|0|0|橘子电信"), took: 4.458µs -ip2region>> 4.4.4.5 -region: Ok("美国|0|0|0|Level3"), took: 4.847µs +可以试着找一个新的项目 + +在`main.rs`写入如下 + +```rust +use std::thread; +use std::time::{Instant, Duration}; + +fn main() { + let now = Instant::now(); + thread::sleep(Duration::from_secs(3)); + println!("{:?}", now.elapsed()); +} ``` -# 单元测试 +执行命令如下，发现毫秒级别是没什么问题都，微秒和纳秒上面是存在误差的，详情可以参考[`rust`标准库文档的`time::Instant`章](https://rustwiki.org/zh-CN/std/time/struct.Instant.html) ```shell -➜ XDB_FILEPATH=../../../data/ip2region.xdb cargo test +$ cargo run -r + Finished release [optimized] target(s) in 0.03s + Running `target/release/ip-test` +3.000197389s ``` -# `bench` 测试 +# `bench`测试 -通过 `cargo bench` 命令来进行自动 `bench` 测试，一方面确保程序和 `xdb` 文件都没有错误，另一方面通过大量的查询得到平均查询性能 +测试平均性能 -在不同机器上面的测试性能时间是不一样的，如下是在机器`CPU`是`Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz`，内存`DDR4 32G`下面的测试结果 +切换到 `ip2region/binding/rust` 路径，执行如下命令 + +`help`输出如下 ```shell -➜ XDB_FILEPATH=../../../data/ip2region.xdb cargo bench +$ ./target/release/rust-example bench --help +bench test + +Usage: rust-example bench [OPTIONS] --src + +Options: + --src set this to specific source bench file + --db the xdb filepath, you can set this field like ../data/ip2region.xdb,if you dont set,if will detect xdb file on ../data/ip2region.xdb, ../../data/ip2region.xdb, ../../../data/ip2region.xdb if exists + -h, --help Print help information +``` + +使用默认的`ip2region`和`ip.merge.txt` + +```shell +$ ./target/release/rust-example bench --src=../../data/ip.merge.txt --db=../../data/ip2region.xdb +Bench finished, total: 3419220,took: 519.820535ms ,cost: 152ns/op +``` + +# `binding/rust`后续维护须知 + +`bingd/rust`编写了单元测试，后续开发需要保证单元测试正常 + +切换到 `ip2region/binding/rust` 路径，执行如下命令 + +```shell +$ cargo test +``` + +需要保证查询速度不会有大幅降低，希望有朝一日，远方的朋友可以再优化一下，实现几十纳秒级别的查询速度 + +下面是`ip2region2`库的第一版`benchmark`结果 + +重点关注如下 + +`search_by_ip_bench ` + +- 查询`ip`的实际调用函数 + +`get_block_by_size_bench` + +- 获取并且计算偏移值，和其他`binding`下的实现的`getLong`和`getShort`相似 +- 该函数会被`search_by_ip`多次调用，所以被标注为`#[inline]`使用内联优化，以此来消除函数调用产生的压栈开销 + +```shell +$ cargo bench +// --snip--- +search_by_ip_bench time: [116.99 ns 119.52 ns 122.31 ns] + change: [-8.1930% -5.8295% -3.3029%] (p = 0.00 < 0.05) + Performance has improved. +Found 4 outliers among 100 measurements (4.00%) + 4 (4.00%) high mild + +get_block_by_size_bench time: [5.2388 ns 5.2784 ns 5.3229 ns] + change: [-6.2649% -4.3559% -2.5539%] (p = 0.00 < 0.05) + Performance has improved. +Found 7 outliers among 100 measurements (7.00%) + 3 (3.00%) high mild + 4 (4.00%) high severe + +get_full_cache_bench time: [1.4800 ns 1.5034 ns 1.5325 ns] + change: [-17.984% -13.664% -9.1681%] (p = 0.00 < 0.05) + Performance has improved. +Found 15 outliers among 100 measurements (15.00%) + 3 (3.00%) high mild + 12 (12.00%) high severe + +get_vec_index_cache_bench + time: [1.7578 ns 1.7757 ns 1.7961 ns] + change: [-7.5169% -4.5088% -1.7147%] (p = 0.00 < 0.05) + Performance has improved. +Found 5 outliers among 100 measurements (5.00%) + 3 (3.00%) high mild + 2 (2.00%) high severe // --snip-- - Running benches/search.rs (target/release/deps/search-9614305a566885c4) -Benchmarking ip_search_bench: Warming up for 3.0000 sinit xdb searcher at ../../../data/ip2region.xdb -ip_search_bench time: [120.84 ns 122.91 ns 125.20 ns] - change: [-3.3346% -1.2786% +0.8027%] (p = 0.23 > 0.05) - No change in performance detected. -Found 6 outliers among 100 measurements (6.00%) - 6 (6.00%) high mild ``` -可以看到上面的`ip_search_bench time`一行的参数表示是左右值分别显示置信区间的下限和上限，中间值显示 `Criterion.rs` 对基准程序每次迭代所用时间的最佳估计 diff --git a/binding/rust/ip2region2/benches/search.rs b/binding/rust/ip2region2/benches/search.rs index 3854502..52051f1 100644 --- a/binding/rust/ip2region2/benches/search.rs +++ b/binding/rust/ip2region2/benches/search.rs @@ -2,12 +2,11 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rand; use ip2region2::searcher::{ - get_block_by_size, get_full_cache, get_int_block_value, get_vector_index_cache, search_by_ip, - searcher_init, + get_block_by_size, get_full_cache, get_vector_index_cache, search_by_ip, searcher_init, }; -fn ip_search_bench(c: &mut Criterion) { - c.bench_function("ip_search_bench", |b| { +fn search_by_ip_bench(c: &mut Criterion) { + c.bench_function("search_by_ip_bench", |b| { searcher_init(None); b.iter(|| { search_by_ip(rand::random::()).unwrap(); @@ -16,7 +15,7 @@ fn ip_search_bench(c: &mut Criterion) { } fn get_block_by_size_bench(c: &mut Criterion) { - c.bench_function("get_block_by_size", |b| { + c.bench_function("get_block_by_size_bench", |b| { b.iter(|| { black_box(get_block_by_size( get_full_cache(), @@ -27,19 +26,8 @@ fn get_block_by_size_bench(c: &mut Criterion) { }); } -fn get_int_block_bench(c: &mut Criterion) { - c.bench_function("get_int_block_bench", |b| { - b.iter(|| { - black_box(get_int_block_value( - get_full_cache(), - rand::random::() as usize, - )); - }) - }); -} - fn get_full_cache_bench(c: &mut Criterion) { - c.bench_function("get_full_cache", |b| { + c.bench_function("get_full_cache_bench", |b| { b.iter(|| { black_box(get_full_cache()); }) @@ -47,7 +35,7 @@ fn get_full_cache_bench(c: &mut Criterion) { } fn get_vec_index_cache_bench(c: &mut Criterion) { - c.bench_function("get_vec_index_cache", |b| { + c.bench_function("get_vec_index_cache_bench", |b| { b.iter(|| { black_box(get_vector_index_cache()); }) @@ -56,8 +44,7 @@ fn get_vec_index_cache_bench(c: &mut Criterion) { criterion_group!( benches, - ip_search_bench, - get_int_block_bench, + search_by_ip_bench, get_block_by_size_bench, get_full_cache_bench, get_vec_index_cache_bench,