Merge pull request #1 from lionsoul2014/master

update to newest
This commit is contained in:
Rocher 2018-07-17 15:32:37 +08:00 committed by GitHub
commit 05bc534d87
30 changed files with 149735 additions and 106177 deletions

11
.gitignore vendored
View File

@ -16,9 +16,20 @@ META-INF/
# vim swp file #
*.swp
.idea
# binding
/binding/java/classes/
/binding/java/doc/
/binding/java/target/
/binding/java/*.jar
/binding/c/testSearcher
# rust
Cargo.lock
target

2
Cargo.toml Normal file
View File

@ -0,0 +1,2 @@
[workspace]
members = ["binding/rust", "binding/rust/example"]

View File

@ -53,6 +53,14 @@ example result:
36.149.160.55
```
node:
```
> let ip2region = require('./ip2region.js');
> let dbService = ip2region.create('./ip2region.db');
> dbService.binarySearchSync(' 101.105.35.57')
{ city: 0, region: '中国|0|广东|深圳|鹏博士' }
```
java:
```shell
cd binding/java

26
binding/c/Makefile Normal file
View File

@ -0,0 +1,26 @@
A = @
CFLAGS= -g
CC= gcc
LIBS= -I./
LDFLAGS=
RM = rm -f
MV = mv
testSearcher: ip2region.o
$(A) $(CC) $(CFLAGS) $(LDFLAGS) $(LIBS) -o testSearcher testSearcher.c ip2region.c
$(A) $(RM) ip2region.o testSearcher.o
db:
$(A) cd ../../;\
export LANG=en_US.UTF-8;\
java -jar dbMaker-1.2.2.jar -src ./data/ip.merge.txt -region ./data/global_region.csv -dst .;\
$(MV) ./ip2region.db ./binding/c/;\
cd ./binding/c
all: db testSearcher
clean:
$(A) $(RM) testSearcher
.PHONY: all clean testSearcher db

View File

@ -1,6 +1,6 @@
###golang 实现ip地址查询
### golang 实现ip地址查询
####获取
#### 获取
```
go get github.com/mohong122/ip2region/binding/golang
@ -9,7 +9,7 @@ go get github.com/mohong122/ip2region/binding/golang
####使用
#### 使用
```golang
@ -17,7 +17,7 @@ package main
import (
"fmt"
"github.com/mohong122/ip2region/binding/golang"
"github.com/mohong122/ip2region/binding/golang/ip2region"
)
func main() {
@ -38,7 +38,7 @@ func main() {
```
####返回对象
#### 返回对象
```golang
type IpInfo struct {
CityId int64
@ -64,7 +64,7 @@ BenchmarkBinarySearch-4| 30000 | 42680 ns/op
```
cd /binging/golang
go run main.go ../../data/ip2Region.db
go run main.go ../../data/ip2region.db
Or

View File

@ -2,7 +2,7 @@ package main
import (
"os"
"github.com/mohong122/ip2region/binding/golang"
"github.com/mohong122/ip2region/binding/golang/ip2region"
"bufio"
"fmt"
"strings"

View File

@ -0,0 +1,4 @@
{
"singleQuote": true,
"tabWidth": 4
}

View File

@ -1,29 +1,29 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`binarySearch 1`] = `
exports[`ip2region binarySearch 1`] = `
Object {
"city": 2163,
"region": "中国|华南|广东省|深圳市|阿里云",
"region": "中国|0|广东省|深圳市|阿里云",
}
`;
exports[`binarySearch 2`] = `
exports[`ip2region binarySearch 2`] = `
Object {
"city": 0,
"region": "未分配或者内网IP|0|0|0|0",
"region": "0|0|0|内网IP|内网IP",
}
`;
exports[`should query 1`] = `
exports[`ip2region should query 1`] = `
Object {
"city": 2163,
"region": "中国|华南|广东省|深圳市|阿里云",
"region": "中国|0|广东省|深圳市|阿里云",
}
`;
exports[`should query 2`] = `
exports[`ip2region should query 2`] = `
Object {
"city": 0,
"region": "未分配或者内网IP|0|0|0|0",
"region": "0|0|0|内网IP|内网IP",
}
`;

View File

@ -4,274 +4,301 @@
* project: https://github.com/lionsoul2014/ip2region
*
* @author dongyado<dongyado@gmail.com>
* */
var fs = require('fs');
var ipbase = [16777216, 65536, 256, 1]; // for ip2long
var ip2region = {};
ip2region.db_file_path = null;
ip2region.db_fd = null;
var totalBlocks = 0;
var firstIndexPtr = 0;
var lastIndexPtr = 0;
var superBlock = new Buffer(8);
var indexBlockLength = 12;
var totalHeaderLength = 8192;
* @author leeching<leeching.fx@gmail.com>
*/
const fs = require('fs');
const IP_BASE = [16777216, 65536, 256, 1];
const INDEX_BLOCK_LENGTH = 12;
const TOTAL_HEADER_LENGTH = 8192;
/**
* binary search synchronized
* */
ip2region.binarySearchSync = function(ip)
{
var low = 0;
var mid = 0;
var high = totalBlocks;
var dataPos = 0;
var pos = 0;
var sip = 0;
var eip = 0;
var indexBuffer = new Buffer(12);
* Convert ip to long (xxx.xxx.xxx.xxx to a integer)
*
* @param {string} ip
* @return {number} long value
*/
function ip2long(ip) {
const arr = ip.split('.');
if (arr.length !== 4) {
throw new Error('invalid ip');
}
return arr.reduce((val, n, i) => {
n = Number(n);
if (!Number.isInteger(n) || n < 0 || n > 255) {
throw new Error('invalid ip');
}
return val + IP_BASE[i] * n;
}, 0);
}
if( typeof(ip) == 'string' ) ip = ip2long(ip);
/**
* Get long value from buffer with specified offset
*
* @param {Buffer} buffer
* @param {number} offset
* @return {number} long value
*/
function getLong(buffer, offset) {
const val =
(buffer[offset] & 0x000000ff) |
((buffer[offset + 1] << 8) & 0x0000ff00) |
((buffer[offset + 2] << 16) & 0x00ff0000) |
((buffer[offset + 3] << 24) & 0xff000000);
return val < 0 ? val >>> 0 : val;
}
// binary search
while( low <= high ) {
mid = ((low + high) >> 1);
pos = firstIndexPtr + mid * indexBlockLength;
fs.readSync(this.db_fd, indexBuffer, 0, indexBlockLength, pos);
sip = getLong(indexBuffer, 0);
/**
* @typedef {Object} SearchResult
* @property {number} city
* @property {string} region
*/
//console.log( ' sip : ' + sip + ' eip : ' + eip );
if ( ip < sip) {
high = mid - 1;
class IP2Region {
static create(dbPath) {
const oldInstance = IP2Region._instances.get(dbPath);
if (oldInstance) {
return oldInstance;
} else {
eip = getLong(indexBuffer, 4);
if ( ip > eip ) {
const instance = new IP2Region({ dbPath });
IP2Region._instances.set(dbPath, instance);
return instance;
}
}
/**
* For backward compatibility
*/
static destroy() {
IP2Region._instances.forEach(([key, instance]) => {
instance.destroy();
});
}
constructor(options = {}) {
const { dbPath } = options;
if (!dbPath || !fs.existsSync(dbPath)) {
throw new Error(`[ip2region] db file not exists : ${dbPath}`);
}
try {
this.dbFd = fs.openSync(dbPath, 'r');
} catch (e) {
throw new Error(
`[ip2region] Can not open ip2region.db file , path: ${dbPath}`
);
}
IP2Region._instances.set((this.dbPath = dbPath), this);
this.totalBlocks = this.firstIndexPtr = this.lastIndexPtr = 0;
this.calcTotalBlocks();
this.headerIndexBuffer = new Buffer(TOTAL_HEADER_LENGTH);
this.headerSip = [];
this.headerPtr = [];
this.headerLen = 0;
this.prepareHeader();
}
/**
* @public
*/
destroy() {
fs.closeSync(ip2rObj.dbFd);
IP2Region._instances.delete(this.dbPath);
}
/**
* @public
* @param {string} ip
* @return {SearchResult}
*/
binarySearchSync(ip) {
ip = ip2long(ip);
let low = 0;
let mid = 0;
let high = this.totalBlocks;
let dataPos = 0;
let pos = 0;
let sip = 0;
let eip = 0;
const indexBuffer = new Buffer(12);
// binary search
while (low <= high) {
mid = (low + high) >> 1;
pos = this.firstIndexPtr + mid * INDEX_BLOCK_LENGTH;
fs.readSync(this.dbFd, indexBuffer, 0, INDEX_BLOCK_LENGTH, pos);
sip = getLong(indexBuffer, 0);
if (ip < sip) {
high = mid - 1;
} else {
eip = getLong(indexBuffer, 4);
if (ip > eip) {
low = mid + 1;
} else {
dataPos = getLong(indexBuffer, 8);
break;
}
}
}
return this.readData(dataPos);
}
/**
* @public
* @param {string} ip
* @return {SearchResult}
*/
btreeSearchSync(ip) {
ip = ip2long(ip);
// first search (in header index)
let low = 0;
let mid = 0;
let high = this.headerLen;
let sptr = 0;
let eptr = 0;
while (low <= high) {
mid = (low + high) >> 1;
if (ip == this.headerSip[mid]) {
if (mid > 0) {
sptr = this.headerPtr[mid - 1];
eptr = this.headerPtr[mid];
} else {
sptr = this.headerPtr[mid];
eptr = this.headerPtr[mid + 1];
}
break;
}
if (ip < this.headerSip[mid]) {
if (mid == 0) {
sptr = this.headerPtr[mid];
eptr = this.headerPtr[mid + 1];
break;
} else if (ip > this.headerSip[mid - 1]) {
sptr = this.headerPtr[mid - 1];
eptr = this.headerPtr[mid];
break;
}
high = mid - 1;
} else {
if (mid == this.headerLen - 1) {
sptr = this.headerPtr[mid - 1];
eptr = this.headerPtr[mid];
break;
} else if (ip <= this.headerSip[mid + 1]) {
sptr = this.headerPtr[mid];
eptr = this.headerPtr[mid + 1];
break;
}
low = mid + 1;
}
}
// match nothing
if (sptr == 0) return null;
// second search (in index)
const blockLen = eptr - sptr;
const blockBuffer = new Buffer(blockLen + INDEX_BLOCK_LENGTH);
fs.readSync(
this.dbFd,
blockBuffer,
0,
blockLen + INDEX_BLOCK_LENGTH,
sptr
);
low = 0;
high = blockLen / INDEX_BLOCK_LENGTH;
let p = 0;
let sip = 0;
let eip = 0;
let dataPtr = 0;
while (low <= high) {
mid = (low + high) >> 1;
p = mid * INDEX_BLOCK_LENGTH;
sip = getLong(blockBuffer, p);
if (ip < sip) {
high = mid - 1;
} else {
dataPos = getLong(indexBuffer, 8);
break;
eip = getLong(blockBuffer, p + 4);
if (ip > eip) {
low = mid + 1;
} else {
dataPtr = getLong(blockBuffer, p + 8);
break;
}
}
}
return this.readData(dataPtr);
}
/**
* @private
*/
calcTotalBlocks() {
const superBlock = new Buffer(8);
fs.readSync(this.dbFd, superBlock, 0, 8, 0);
this.firstIndexPtr = getLong(superBlock, 0);
this.lastIndexPtr = getLong(superBlock, 4);
this.totalBlocks =
(this.lastIndexPtr - this.firstIndexPtr) / INDEX_BLOCK_LENGTH + 1;
}
/**
* @private
*/
prepareHeader() {
fs.readSync(
this.dbFd,
this.headerIndexBuffer,
0,
TOTAL_HEADER_LENGTH,
8
);
for (let i = 0; i < TOTAL_HEADER_LENGTH; i += 8) {
const startIp = getLong(this.headerIndexBuffer, i);
const dataPtr = getLong(this.headerIndexBuffer, i + 4);
if (dataPtr == 0) break;
this.headerSip.push(startIp);
this.headerPtr.push(dataPtr);
this.headerLen++; // header index size count
}
}
// read data
if (dataPos == 0) return null;
var dataLen = ((dataPos >> 24) & 0xFF);
var dataPos = (dataPos & 0x00FFFFFF);
var dataBuffer = new Buffer(dataLen);
/**
* @private
* @param {number} dataPos
* @return {SearchResult}
*/
readData(dataPos) {
if (dataPos == 0) return null;
const dataLen = (dataPos >> 24) & 0xff;
dataPos = dataPos & 0x00ffffff;
const dataBuffer = new Buffer(dataLen);
fs.readSync(this.db_fd, dataBuffer, 0, dataLen, dataPos);
fs.readSync(this.dbFd, dataBuffer, 0, dataLen, dataPos);
var city_id = getLong(dataBuffer, 0);
var data = dataBuffer.toString('utf8', 4, dataLen);
const city = getLong(dataBuffer, 0);
const region = dataBuffer.toString('utf8', 4, dataLen);
//console.log(city_id);
//console.log(data);
return { city: city_id, region: data };
return { city, region };
}
}
IP2Region._instances = new Map();
var headerSip = null;
var headerPtr = 0;
var headerLen = 0;
/**
* btree search synchronized
* */
ip2region.btreeSearchSync = function(ip)
{
var indexBlockBuffer = new Buffer(indexBlockLength);
var headerIndexBuffer = new Buffer(totalHeaderLength);
if( typeof(ip) == 'string' ) ip = ip2long(ip);
var i = 0;
// header index handler
if (headerSip == null) {
fs.readSync(this.db_fd, headerIndexBuffer, 0, totalHeaderLength, 8);
headerSip = new Array();
headerPtr = new Array();
var startIp = 0;
var dataPtr = 0;
for ( i = 0; i < totalHeaderLength; i += 8) {
startIp = getLong(headerIndexBuffer, i);
dataPtr = getLong(headerIndexBuffer, i + 4);
if ( dataPtr == 0) break;
headerSip.push(startIp);
headerPtr.push(dataPtr);
headerLen++; // header index size count
}
}
// first search (in header index)
var low = 0;
var mid = 0;
var high = headerLen;
var sptr = 0;
var eptr = 0;
while(low <= high) {
mid = ((low + high) >> 1);
if (ip == headerSip[mid]) {
if ( m > 0) {
sptr = headerPtr[mid - 1];
eptr = headerPtr[mid];
} else {
sptr = headerPtr[mid];
eptr = headerPtr[mid + 1];
}
break;
}
if ( ip < headerSip[mid]) {
if (mid == 0) {
sptr = headerPtr[mid];
eptr = headerPtr[mid + 1];
break;
} else if ( ip > headerSip[mid - 1]) {
sptr = headerPtr[mid - 1];
eptr = headerPtr[mid];
break;
}
high = mid - 1;
} else {
if ( mid == headerLen - 1) {
sptr = headerPtr[mid - 1];
eptr = headerPtr[mid];
break;
} else if ( ip <= headerSip[mid + 1]) {
sptr = headerPtr[mid];
eptr = headerPtr[mid + 1];
break;
}
low = mid + 1;
}
}
// match nothing
if (sptr == 0) return null;
// second search (in index)
var blockLen = eptr - sptr;
var blockBuffer = new Buffer(blockLen + indexBlockLength);
fs.readSync(this.db_fd, blockBuffer, 0, blockLen + indexBlockLength, sptr);
low = 0;
high = blockLen / indexBlockLength;
var p = 0;
var sip = 0;
var eip = 0;
var dataPtr = 0;
while(low <= high) {
mid = ((low + high) >> 1);
p = mid * indexBlockLength;
sip = getLong(blockBuffer, p);
if (ip < sip) {
high = mid - 1;
} else {
eip = getLong(blockBuffer, p + 4);
if (ip > eip) {
low = mid + 1;
} else {
dataPtr = getLong(blockBuffer, p + 8);
break;
}
}
}
// read data
if (dataPtr == 0) return null;
var dataLen = ((dataPtr >> 24) & 0xFF);
var dataPtr = (dataPtr & 0x00FFFFFF);
var dataBuffer = new Buffer(dataLen);
fs.readSync(this.db_fd, dataBuffer, 0, dataLen, dataPtr);
var city_id = getLong(dataBuffer, 0);
var data = dataBuffer.toString('utf8', 4, dataLen);
//console.log(city_id);
//console.log(data);
return { city: city_id, region: data };
}
/**
* convert ip to long (xxx.xxx.xxx.xxx to a integer)
* */
function ip2long(ip)
{
var val = 0;
ip.split('.').forEach(function(ele, i){
val += ipbase[i] * ele;
});
return val;
}
/**
* get long value from buffer with specified offset
* */
function getLong(buffer, offset)
{
var val = (
(buffer[offset] & 0x000000FF) |
((buffer[offset + 1] << 8) & 0x0000FF00) |
((buffer[offset + 2] << 16) & 0x00FF0000) |
((buffer[offset + 3] << 24) & 0xFF000000)
);
// convert to unsigned int
if (val < 0) {
val = val >>> 0;
}
return val;
}
exports.create = function(db_path)
{
if (typeof(db_path) == "undefined" || fs.exists(db_path) ) {
throw("[ip2region] db file not exists : " + db_path);
}
ip2region.db_file_path = db_path;
try {
ip2region.db_fd = fs.openSync(ip2region.db_file_path, 'r');
} catch(e) {
throw("[ip2region] Can not open ip2region.db file , path : "
+ ip2region.db_file_path);
}
// init basic search environment
if (totalBlocks == 0) {
fs.readSync(ip2region.db_fd, superBlock, 0, 8, 0);
firstIndexPtr = getLong(superBlock, 0);
lastIndexPtr = getLong(superBlock, 4);
totalBlocks = (lastIndexPtr - firstIndexPtr)
/ indexBlockLength + 1;
}
return ip2region;
}
exports.destroy = function(ip2rObj)
{
ip2rObj.db_file_path = null;
fs.closeSync(ip2rObj.db_fd);
}
module.exports = IP2Region;

View File

@ -1,15 +1,24 @@
'use strict';
const path = require('path');
const IP2Region = require('./ip2region');
const ip = require('./ip2region');
describe('ip2region', () => {
let instance;
const query = ip.create('../../data/ip2region.db');
beforeAll(() => {
instance = IP2Region.create(path.join(__dirname, '../../data/ip2region.db'));
});
it('should query', () => {
expect(query.btreeSearchSync('120.24.78.68')).toMatchSnapshot();
expect(query.btreeSearchSync('10.10.10.10')).toMatchSnapshot();
});
it('binarySearch', () => {
expect(query.binarySearchSync('120.24.78.68')).toMatchSnapshot();
expect(query.binarySearchSync('10.10.10.10')).toMatchSnapshot();
afterAll(() => {
instance.destroy();
});
test('should query', () => {
expect(instance.btreeSearchSync('120.24.78.68')).toMatchSnapshot();
expect(instance.btreeSearchSync('10.10.10.10')).toMatchSnapshot();
});
test('binarySearch', () => {
expect(instance.binarySearchSync('120.24.78.68')).toMatchSnapshot();
expect(instance.binarySearchSync('10.10.10.10')).toMatchSnapshot();
});
});

View File

@ -106,7 +106,7 @@ class Ip2Region
}
/**
* get the data block throught the specifield ip address or long ip numeric with binary search algorithm
* get the data block through the specified ip address or long ip numeric with binary search algorithm
*
* @param ip
* @return mixed Array or NULL for any error
@ -174,7 +174,7 @@ class Ip2Region
}
/**
* get the data block associated with the specifield ip with b-tree search algorithm
* get the data block associated with the specified ip with b-tree search algorithm
* @Note: not thread safe
*
* @param ip

View File

@ -1,29 +1,29 @@
# ip2region 的PHP拓展(php5版本)
# ip2region client - PHP扩展(php5版本)
### 安装步骤
* git clone https://github.com/lionsoul2016/ip2region.git
* git clone https://github.com/lionsoul2014/ip2region.git
* cd ip2region
* cp binding/php_extension/php5/ip2region 到 php source code 的ext目录下
* cp binding/c/下面所有的文件到 php source code 的ext/ip2region/lib 目录下
* 在ext/ip2region下运行
* cp binding/php_extension/php5/ip2region 到 php source code 的 ext 目录下
* cp binding/c/ 里面所有的文件到 php source code 的 ext/ip2region/lib 目录下
* 在 ext/ip2region 下,运行
phpize
./configure
make && sudo make install
* 配置ip2region.ini 指定db_file路径(cli/fpm)
* 配置 ip2region.ini 指定 db_file 路径,(cli/fpm)
extension=ip2region.so
ip2region.db_file=/path/to/ip2region.db
ip2region.db 在项目根目录下的data文件夹下
默认 ip2region.db 在项目根目录下的 data 文件夹下,如有改动,请修改 ip2region.ini 的 db_file
* 测试
在ext/ip2region/下运行
ext/ip2region/ 下运行
php ip2region.php
### 使用
参考ip2region/ip2region.php
参考当前目录下的 ip2region.php

View File

@ -67,10 +67,10 @@ void search(
if ( res == 1 )
{
add_assoc_long( *return_value, "cityId", (*_block).city_id);
add_assoc_long( *return_value, "city_id", (*_block).city_id);
add_assoc_string( *return_value, "region", (*_block).region, 1);
} else {
add_assoc_long( *return_value, "cityId", 0);
add_assoc_long( *return_value, "city_id", 0);
add_assoc_string( *return_value, "region", "[Error] Search Failed! Please check the path of ip2region db file.", 1);
}
}

View File

@ -1,29 +1,29 @@
# ip2region 的PHP拓展(php7版本)
# ip2region client - PHP扩展(php7版本)
### 安装步骤
* git clone https://github.com/lionsoul2016/ip2region.git
* git clone https://github.com/lionsoul2014/ip2region.git
* cd ip2region
* cp binding/php_extension/php5/ip2region 到 php source code 的ext目录下
* cp binding/c/下面所有的文件到 php source code 的ext/ip2region/lib 目录下
* 在ext/ip2region下运行
* cp binding/php_extension/php7/ip2region 到 php source code 的 ext 目录下
* cp binding/c/ 里面所有的文件到 php source code 的 ext/ip2region/lib 目录下
* 在 ext/ip2region 下,运行
phpize
./configure
make && sudo make install
* 配置ip2region.ini 指定db_file路径(cli/fpm)
* 配置 ip2region.ini 指定 db_file 路径,(cli/fpm)
extension=ip2region.so
ip2region.db_file=/path/to/ip2region.db
ip2region.db 在项目根目录下的data文件夹下
默认 ip2region.db 在项目根目录下的 data 文件夹下,如有改动,请修改 ip2region.ini 的 db_file
* 测试
在ext/ip2region/下运行
ext/ip2region/ 下运行
php ip2region.php
### 使用
参考ip2region/ip2region.php
参考当前目录下的 ip2region.php

View File

@ -67,12 +67,12 @@ void search(
if ( res == 1 )
{
add_assoc_long( *return_value, "cityId", (*_block).city_id);
add_assoc_long( *return_value, "city_id", (*_block).city_id);
//add_assoc_string( *return_value, "region", (*_block).region, 1);
// for phpng
add_assoc_string( *return_value, "region", (*_block).region);
} else {
add_assoc_long( *return_value, "cityId", 0);
add_assoc_long( *return_value, "city_id", 0);
//add_assoc_string( *return_value, "region", "[Error] Search Failed! Please check the path of ip2region db file.", 1);
// for phpng
add_assoc_string( *return_value, "region", "[Error] Search Failed! Please check the path of ip2region db file.");

17
binding/rust/Cargo.toml Normal file
View File

@ -0,0 +1,17 @@
[package]
name = "ip2region"
version = "0.2.0"
authors = ["biluohc <biluohc@qq.com>"]
include = ["./*", "../../data/ip2region.db", "../../Cargo.toml"]
[[bin]]
name="ip2region"
path="src/main.rs"
[dependencies]
lazy_static ={ version = "^1", optional = true }
[features]
lazy = ["lazy_static"]

View File

@ -0,0 +1,13 @@
[package]
name = "example"
version = "0.1.0"
authors = ["biluohc <biluohc@qq.com>"]
[dependencies]
[dependencies.ip2region]
path = "../"
# git = "https://github.com/lionsoul2014/ip2region"
# git = "https://github.com/biluohc/ip2region"
version = "*"
features = ["lazy"]

View File

@ -0,0 +1,114 @@
extern crate ip2region;
use ip2region::*;
use std::env;
use std::time::Instant;
// cargo run --release ../../../data/ip2region.db
static IPS: &'static [&'_ str] = &[
"117.136.105.202",
"47.95.47.253",
"127.0.0.1",
"10.0.0.1",
"1.1.1.1",
];
fn main() {
lazy();
overview();
}
fn lazy() {
for ip in IPS {
let start = Instant::now();
let res = memory_search(ip);
let end = start.elapsed().subsec_micros();
println!("lazy__ {:06} microseconds: {:?}", end, res);
let start = Instant::now();
let ip_addr = ip.parse().unwrap();
let res2 = memory_search_ip(&ip_addr);
let end = start.elapsed().subsec_micros();
println!("lazy__ {:06} microseconds: {:?}", end, res2);
if res.is_ok() && res2.is_ok() {
assert_eq!(res.unwrap(), res2.unwrap());
} else if res.is_err() && res2.is_err() {
} else {
panic!("not EQ")
}
}
}
fn overview() {
let args = env::args().skip(1).collect::<Vec<String>>();
let db_path = &args[0];
let mut ip2 = Ip2Region::new(db_path).unwrap();
let ip2o = ip2.to_owned().unwrap();
for ip in IPS {
// mem
let start = Instant::now();
let res = ip2o.memory_search(ip);
let end = start.elapsed().subsec_micros();
println!("memory {:06} microseconds: {:?}", end, res);
let start = Instant::now();
let ip_addr = ip.parse().unwrap();
let res2 = ip2o.memory_search_ip(&ip_addr);
let end = start.elapsed().subsec_micros();
println!("memory {:06} microseconds: {:?}", end, res2);
if res.is_ok() && res2.is_ok() {
assert_eq!(res.unwrap(), res2.unwrap());
} else if res.is_err() && res2.is_err() {
} else {
panic!("not EQ")
}
// binary
let start = Instant::now();
let res = ip2.binary_search(ip);
let end = start.elapsed().subsec_micros();
println!("binary {:06} microseconds: {:?}", end, res);
let start = Instant::now();
let ip_addr = ip.parse().unwrap();
let res2 = ip2.binary_search_ip(&ip_addr);
let end = start.elapsed().subsec_micros();
println!("binary {:06} microseconds: {:?}", end, res2);
if res.is_ok() && res2.is_ok() {
assert_eq!(res.unwrap(), res2.unwrap());
} else if res.is_err() && res2.is_err() {
} else {
panic!("not EQ")
}
// btree
let start = Instant::now();
let res = ip2.btree_search(ip);
let end = start.elapsed().subsec_micros();
println!("btree {:06} microseconds: {:?}", end, res);
let start = Instant::now();
let ip_addr = ip.parse().unwrap();
let res2 = ip2.btree_search_ip(&ip_addr);
let end = start.elapsed().subsec_micros();
println!("btree_ {:06} microseconds: {:?}", end, res2);
if res.is_ok() && res2.is_ok() {
assert_eq!(res.unwrap(), res2.unwrap());
} else if res.is_err() && res2.is_err() {
} else {
panic!("not EQ")
}
// \n
println!();
}
}

36
binding/rust/readme.md Normal file
View File

@ -0,0 +1,36 @@
## Rust 客户端
## 用法
Demo`example` 目录里
API文档 `cargo doc --features lazy --open` 可以看到所有。
运行测试: `cargo test --features lazy`
### 添加依赖
注意 `#` 是 toml格式的注释文件前三行的 `path``git` 是引用包的方式,`path` 是按路径引用包,`git` 是按git项目地址按实际情况选一个。
```toml
[dependencies.ip2region]
git = "https://github.com/lionsoul2014/ip2region"
# git = "https://github.com/biluohc/ip2region"
# path = "../"
version = "*"
# features = ["lazy"]
```
### 代码
查看 `example/src/main.rs`
### `lazy` feature 把 DB 直接打包进二进制
取消上面 toml 的 `# features = ["lazy"]` 行的注释即可使用,其 api 是 `memory_search``memory_search_ip`
只是目前 DB 足有3.2M,还是有些感人的。
关键的一行是 `features = ["lazy"]` ,不需要则可以注释或者删掉。

7
binding/rust/src/db.rs Normal file
View File

@ -0,0 +1,7 @@
pub const DB_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../data/ip2region.db");
#[cfg(feature = "lazy")]
pub static DB_BYTES: &'static [u8] = include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../data/ip2region.db"
));

44
binding/rust/src/error.rs Normal file
View File

@ -0,0 +1,44 @@
use std::{self, io, net, str};
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug)]
pub enum Error {
Io(io::Error),
Utf8(str::Utf8Error),
Addr(net::AddrParseError),
/// `224.0.0.0` ~ `239.255.255.255`
///
// `ff00::/8`
IpIsMulticast,
/// `0.0.0.0`
IpIsUnspecified,
/// `127.0.0.0/8`
IpIsLoopback,
///1. `10.0.0.0/8`
///
///2. `172.16.0.0/12`
///
///3. `192.168.0.0/16`
IpIsPrivate,
/// Unsupport Ipv6 Now
UnsupportIpv6,
NotFound,
}
impl From<io::Error> for Error {
fn from(e: io::Error) -> Self {
Error::Io(e)
}
}
impl From<str::Utf8Error> for Error {
fn from(e: str::Utf8Error) -> Self {
Error::Utf8(e)
}
}
impl From<net::AddrParseError> for Error {
fn from(e: net::AddrParseError) -> Self {
Error::Addr(e)
}
}

20
binding/rust/src/lazy.rs Normal file
View File

@ -0,0 +1,20 @@
use super::DB_BYTES;
lazy_static! {
static ref OWNED_IP_2_REGION: OwnedIp2Region = {
OwnedIp2Region {
db_bin_bytes: Cow::Borrowed(DB_BYTES),
first_index_ptr: get_u32(&DB_BYTES[..], 0),
total_blocks: (get_u32(&DB_BYTES[..], 4) - get_u32(&DB_BYTES[..], 0))
/ INDEX_BLOCK_LENGTH + 1,
}
};
}
pub fn memory_search<S: AsRef<str>>(ip_str: S) -> Result<IpInfo<'static>> {
OWNED_IP_2_REGION.memory_search(ip_str)
}
pub fn memory_search_ip(ip_addr: &IpAddr) -> Result<IpInfo> {
OWNED_IP_2_REGION.memory_search_ip(ip_addr)
}

375
binding/rust/src/lib.rs Normal file
View File

@ -0,0 +1,375 @@
#[cfg(feature = "lazy")]
#[macro_use]
extern crate lazy_static;
use std::cell::RefCell;
use std::fs::File;
use std::io::{self, Read, Seek, SeekFrom};
use std::net::IpAddr;
use std::{fmt, str};
mod db;
mod error;
pub use error::{Error, Result};
mod owned;
#[doc(hidden)]
pub use db::DB_PATH;
pub use owned::{OwnedIp2Region, OwnedIpInfo};
#[cfg(feature = "lazy")]
use db::DB_BYTES;
#[cfg(feature = "lazy")]
pub use owned::{memory_search, memory_search_ip};
const INDEX_BLOCK_LENGTH: u32 = 12;
const TOTAL_HEADER_LENGTH: usize = 8192;
thread_local!(static BUF: RefCell<[u8;256]> = RefCell::new([0;256]));
thread_local!(static BUF_BTREE: RefCell<[u8;TOTAL_HEADER_LENGTH]> = RefCell::new([0;TOTAL_HEADER_LENGTH]));
#[allow(non_snake_case)]
#[derive(Debug, Default, Clone, PartialEq)]
pub struct IpInfo<'a> {
pub city_id: u32,
pub country: &'a str,
pub region: &'a str,
pub province: &'a str,
pub city: &'a str,
pub ISP: &'a str,
}
impl<'a> IpInfo<'a> {
fn new(city_id: u32, fields: &[&'a str]) -> Self {
if fields.len() < 5 {
panic!(format!("invlid fields: {:?}", fields));
}
IpInfo {
country: fields[0],
region: fields[1],
province: fields[2],
city: fields[3],
ISP: fields[4],
city_id,
}
}
pub fn to_owned(&self) -> OwnedIpInfo {
OwnedIpInfo {
city_id: self.city_id,
country: self.country.to_owned(),
region: self.region.to_owned(),
province: self.province.to_owned(),
city: self.city.to_owned(),
ISP: self.ISP.to_owned(),
}
}
}
impl<'a> fmt::Display for IpInfo<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"{}|{}|{}|{}|{}|{}",
self.city_id, self.country, self.region, self.province, self.city, self.ISP
)
}
}
fn get_ip_info(city_id: u32, line: &[u8]) -> Result<IpInfo> {
let str = str::from_utf8(line)?;
let fields = str.split('|').collect::<Vec<&str>>();
Ok(IpInfo::new(city_id, &fields[..]))
}
fn get_u32(bytes: &[u8], offset: u32) -> u32 {
let offset = offset as usize;
let tmp = (bytes[offset] as i64) & 0x000000FF
| ((bytes[offset + 1] as i64) << 8) & 0x0000FF00
| ((bytes[offset + 2] as i64) << 16) & 0x00FF0000
| ((bytes[offset + 3] as i64) << 24) & 0xFF000000;
tmp as u32
}
fn ip2u32(ip: &IpAddr) -> Result<u32> {
if ip.is_ipv6() {
return Err(Error::UnsupportIpv6);
}
if ip.is_unspecified() {
return Err(Error::IpIsUnspecified);
}
if ip.is_loopback() {
return Err(Error::IpIsLoopback);
}
if ip.is_multicast() {
return Err(Error::IpIsMulticast);
}
match ip {
IpAddr::V4(v4) => {
if v4.is_private() {
return Err(Error::IpIsPrivate);
}
let mut sum: u32 = 0;
for (i, n) in v4.octets().iter().enumerate() {
sum += (*n as u32) << 24 - 8 * i;
}
return Ok(sum);
}
IpAddr::V6(_v6) => unreachable!(),
}
}
pub struct Ip2Region {
// db file handler
db_file: File,
//header block info
header_sip: Vec<u32>,
header_ptr: Vec<u32>,
header_len: u32,
// super block index info
first_index_ptr: u32,
last_index_ptr: u32,
total_blocks: u32,
}
impl Ip2Region {
pub fn new(path: &str) -> io::Result<Self> {
let file = File::open(path)?;
Ok(Ip2Region {
db_file: file,
header_sip: Vec::new(),
header_ptr: Vec::new(),
header_len: 0,
first_index_ptr: 0,
last_index_ptr: 0,
total_blocks: 0,
})
}
pub fn to_owned(&mut self) -> Result<OwnedIp2Region> {
OwnedIp2Region::new2(&mut self.db_file).map_err(Error::Io)
}
pub fn binary_search<S: AsRef<str>>(&mut self, ip_str: S) -> Result<OwnedIpInfo> {
let ip = ip_str.as_ref().parse::<IpAddr>()?;
self.binary_search_ip(&ip)
}
pub fn binary_search_ip(&mut self, ip_addr: &IpAddr) -> Result<OwnedIpInfo> {
BUF.with(|buf| {
let mut buf = buf.borrow_mut();
if self.total_blocks == 0 {
self.db_file.seek(SeekFrom::Start(0))?;
self.db_file.read_exact(&mut buf[..8])?;
self.first_index_ptr = get_u32(&buf[..8], 0);
self.last_index_ptr = get_u32(&buf[..8], 4);
self.total_blocks =
(self.last_index_ptr - self.first_index_ptr) / INDEX_BLOCK_LENGTH + 1;
}
let ip = ip2u32(ip_addr)?;
let mut h = self.total_blocks;
let (mut data_ptr, mut l) = (0u32, 0u32);
while l <= h {
let m = (l + h) >> 1;
let p = self.first_index_ptr + m * INDEX_BLOCK_LENGTH;
self.db_file.seek(SeekFrom::Start(p as u64))?;
self.db_file
.read_exact(&mut buf[0..INDEX_BLOCK_LENGTH as usize])?;
let sip = get_u32(&buf[..INDEX_BLOCK_LENGTH as usize], 0);
if ip < sip {
h = m - 1;
} else {
let eip = get_u32(&buf[..INDEX_BLOCK_LENGTH as usize], 4);
if ip > eip {
l = m + 1;
} else {
data_ptr = get_u32(&buf[..INDEX_BLOCK_LENGTH as usize], 8);
break;
}
}
}
if data_ptr == 0 {
Err(Error::NotFound)?;
}
let data_len = (data_ptr >> 24) & 0xff;
data_ptr = data_ptr & 0x00FFFFFF;
self.db_file.seek(SeekFrom::Start(data_ptr as u64))?;
self.db_file.read_exact(&mut buf[0..data_len as usize])?;
get_ip_info(
get_u32(&buf[..data_len as usize], 0),
&buf[4..data_len as usize],
).map(|i| i.to_owned())
})
}
pub fn btree_search<S: AsRef<str>>(&mut self, ip_str: S) -> Result<OwnedIpInfo> {
let ip = ip_str.as_ref().parse::<IpAddr>()?;
self.btree_search_ip(&ip)
}
pub fn btree_search_ip(&mut self, ip_addr: &IpAddr) -> Result<OwnedIpInfo> {
BUF_BTREE.with(|buf| {
let mut buf = buf.borrow_mut();
if self.header_len == 0 {
self.db_file.seek(SeekFrom::Start(8))?;
self.db_file.read_exact(&mut buf[0..TOTAL_HEADER_LENGTH])?;
let (mut i, mut idx) = (0, 0);
while i < TOTAL_HEADER_LENGTH {
let sip = get_u32(&buf[0..TOTAL_HEADER_LENGTH], i as u32);
let idx_ptr = get_u32(&buf[0..TOTAL_HEADER_LENGTH], i as u32 + 4);
if idx_ptr == 0 {
break;
}
self.header_sip.push(sip);
self.header_ptr.push(idx_ptr);
i += 8;
idx += 1;
}
self.header_len = idx
}
let ip = ip2u32(ip_addr)?;
let mut h = self.header_len;
let (mut sptr, mut eptr, mut l) = (0u32, 0u32, 0u32);
while l <= h {
let m = (l + h) >> 1;
if m < self.header_len {
if ip == self.header_sip[m as usize] {
if m > 0 {
sptr = self.header_ptr[m as usize - 1];
eptr = self.header_ptr[m as usize];
} else {
sptr = self.header_ptr[m as usize];
eptr = self.header_ptr[m as usize + 1];
}
break;
}
if ip < self.header_sip[m as usize] {
if m == 0 {
sptr = self.header_ptr[m as usize];
eptr = self.header_ptr[m as usize + 1];
break;
} else if ip > self.header_sip[m as usize - 1] {
sptr = self.header_ptr[m as usize - 1];
eptr = self.header_ptr[m as usize];
break;
}
h = m - 1
} else {
if m == self.header_len - 1 {
println!("m/hl: {}/{}", m, self.header_len);
sptr = self.header_ptr[m as usize - 1];
eptr = self.header_ptr[m as usize];
break;
} else if ip <= self.header_sip[m as usize + 1] {
sptr = self.header_ptr[m as usize];
eptr = self.header_ptr[m as usize + 1];
break;
}
l = m + 1
}
}
}
if sptr == 0 {
Err(Error::NotFound)?;
}
let block_len = eptr - sptr;
self.db_file.seek(SeekFrom::Start(sptr as u64))?;
let buf_size = (block_len + INDEX_BLOCK_LENGTH) as usize;
self.db_file.read_exact(&mut buf[..buf_size])?;
let mut data_ptr = 0;
h = block_len / INDEX_BLOCK_LENGTH;
l = 0;
while l <= h {
let m = (l + h) >> 1;
let p = m * INDEX_BLOCK_LENGTH;
let sip = get_u32(&buf[..buf_size], p);
if ip < sip {
h = m - 1;
} else {
let eip = get_u32(&buf[..buf_size], p + 4);
if ip > eip {
l = m + 1;
} else {
data_ptr = get_u32(&buf[..buf_size], p + 8);
break;
}
}
}
if data_ptr == 0 {
Err(Error::NotFound)?;
}
let data_len = (data_ptr >> 24) & 0xff;
data_ptr = data_ptr & 0x00FFFFFF;
self.db_file.seek(SeekFrom::Start(data_ptr as u64))?;
self.db_file.read_exact(&mut buf[0..data_len as usize])?;
get_ip_info(
get_u32(&buf[..data_len as usize], 0),
&buf[4..data_len as usize],
).map(|i| i.to_owned())
})
}
}
// cargo test --features lazy -- --nocapture
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn it_works() {
let mut ip2 = Ip2Region::new(DB_PATH).unwrap();
let ip2o = ip2.to_owned().unwrap();
for ip in vec![
"117.136.105.202",
"47.95.47.253",
"127.0.0.1",
"10.0.0.1",
"1.1.1.1",
] {
#[cfg(feature = "lazy")]
{
println!("lzay__: {:?}", memory_search(ip));
if ip2o.memory_search(ip).is_ok() {
assert_eq!(ip2o.memory_search(ip).unwrap(), memory_search(ip).unwrap());
} else {
assert!(memory_search(ip).is_err());
}
}
println!("memory: {:?}", ip2o.memory_search(ip));
println!("binary: {:?}", ip2.binary_search(ip));
println!("btree : {:?}", ip2.btree_search(ip));
if ip2o.memory_search(ip).is_ok() {
assert_eq!(
ip2o.memory_search(ip).unwrap().to_owned(),
ip2.binary_search(ip).unwrap()
);
assert_eq!(
ip2o.memory_search(ip).unwrap().to_owned(),
ip2.btree_search(ip).unwrap()
);
} else {
assert!(ip2.binary_search(ip).is_err());
assert!(ip2.btree_search(ip).is_err());
}
println!();
}
}
}

130
binding/rust/src/main.rs Normal file
View File

@ -0,0 +1,130 @@
extern crate ip2region;
use ip2region::*;
use std::env;
use std::io::{self, *};
use std::time::Instant;
// cargo run --release --features lazy
fn main() {
let args = env::args().skip(1).collect::<Vec<String>>();
println!(
r#"ip2region cli test
+-----------------------------------------------------------------+
| ip2region [db_file] [alrogrithm]
| format : [ip] [alrogrithm]'
| overview: cargo run --release"
| usage: cargo run --release -- [db_file] [alrogrithm]"
| exit: quit or exit or Ctrl+C
+-----------------------------------------------------------------+`"#
);
if !args.is_empty() {
let db_path = if args[0] != "." { &args[0] } else { DB_PATH };
let mut ip2 = Ip2Region::new(db_path).unwrap();
let ip2o = ip2.to_owned().unwrap();
let alg = if args.len() > 1 {
args[1].to_lowercase()
} else {
"memory".to_lowercase()
};
let mut buf = String::with_capacity(256);
loop {
buf.clear();
print!("ip2region>>");
io::stdout().flush().unwrap();
let line = match io::stdin().read_line(&mut buf) {
Ok(_) => buf.trim(),
Err(e) => panic!("[Fatal]: Read String from Stdin Error: {:?}", e),
};
if line.is_empty() {
continue;
}
if line == "quit" || line == "exit" {
println!("[Info]: Thanks for your use, Bye.");
break;
}
let ip_alg = line
.split_whitespace()
.filter(|s| !s.is_empty())
.collect::<Vec<&str>>();
let alg = if ip_alg.len() > 1 {
ip_alg[1].to_owned()
} else {
alg.clone()
};
let start = Instant::now();
let res = match alg.as_str() {
"memory" => ip2o.memory_search(ip_alg[0]).map(|o| o.to_owned()),
"binary" => ip2.binary_search(ip_alg[0]),
"btree" | "b-tree" => ip2.btree_search(ip_alg[0]),
miss => {
eprintln!("Not have the Algorithm: {:?}", miss);
continue;
}
};
let end = start.elapsed().subsec_micros();
match res {
Ok(i) => {
println!("[{:6} {:06} microseconds]: {}", alg, end, i);
}
Err(e) => {
eprintln!("[Error]: {:?}", e);
}
};
}
} else {
overview()
}
}
fn overview() {
let mut ip2 = Ip2Region::new(DB_PATH).unwrap();
let ip2o = ip2.to_owned().unwrap();
for ip in &[
"117.136.105.202",
"47.95.47.253",
"127.0.0.1",
"10.0.0.1",
"1.1.1.1",
] {
let start = Instant::now();
let res = ip2o.memory_search(ip);
let end = start.elapsed().subsec_micros();
println!("memory {:06} microseconds: {:?}", end, res);
#[cfg(feature = "lazy")]
{
let start = Instant::now();
let res = memory_search(ip);
let end = start.elapsed().subsec_micros();
println!("lazy__ {:06} microseconds: {:?}", end, res);
}
let start = Instant::now();
let res = ip2.binary_search(ip);
let end = start.elapsed().subsec_micros();
println!("binary {:06} microseconds: {:?}", end, res);
let start = Instant::now();
let res = ip2.btree_search(ip);
let end = start.elapsed().subsec_micros();
println!("btree {:06} microseconds: {:?}", end, res);
println!();
}
}

106
binding/rust/src/owned.rs Normal file
View File

@ -0,0 +1,106 @@
use super::*;
use std::borrow::Cow;
#[cfg(feature = "lazy")]
include!("lazy.rs");
#[allow(non_snake_case)]
#[derive(Debug, Default, Clone, PartialEq)]
pub struct OwnedIpInfo {
pub city_id: u32,
pub country: String,
pub region: String,
pub province: String,
pub city: String,
pub ISP: String,
}
impl OwnedIpInfo {
pub fn as_ref<'a>(&'a self) -> IpInfo<'a> {
IpInfo {
city_id: self.city_id,
country: &self.country,
region: &self.region,
province: &self.province,
city: &self.city,
ISP: &self.ISP,
}
}
}
impl fmt::Display for OwnedIpInfo {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", self.as_ref())
}
}
pub struct OwnedIp2Region {
// super block index info
first_index_ptr: u32,
// last_index_ptr: u32,
total_blocks: u32,
db_bin_bytes: Cow<'static, [u8]>,
}
impl OwnedIp2Region {
pub fn new(path: &str) -> io::Result<Self> {
let mut file = File::open(path)?;
Self::new2(&mut file)
}
pub(crate) fn new2(file: &mut File) -> io::Result<Self> {
let file_size = file.metadata()?.len();
let mut bytes = Vec::with_capacity(file_size as usize);
file.read_to_end(&mut bytes)?;
let first_index_ptr = get_u32(&bytes[..], 0);
let last_index_ptr = get_u32(&bytes[..], 4);
let total_blocks = (last_index_ptr - first_index_ptr) / INDEX_BLOCK_LENGTH + 1;
let db_bin_bytes = Cow::Owned(bytes);
Ok(OwnedIp2Region {
first_index_ptr,
total_blocks,
db_bin_bytes,
})
}
pub fn memory_search<S: AsRef<str>>(&self, ip_str: S) -> Result<IpInfo> {
let ip = ip_str.as_ref().parse()?;
self.memory_search_ip(&ip)
}
pub fn memory_search_ip(&self, ip_addr: &IpAddr) -> Result<IpInfo> {
let ip = ip2u32(ip_addr)?;
let mut h = self.total_blocks;
let (mut data_ptr, mut l) = (0u32, 0u32);
while l <= h {
let m = (l + h) >> 1;
let p = self.first_index_ptr + m * INDEX_BLOCK_LENGTH;
let sip = get_u32(&self.db_bin_bytes[..], p);
if ip < sip {
h = m - 1;
} else {
let eip = get_u32(&self.db_bin_bytes[..], p + 4);
if ip > eip {
l = m + 1;
} else {
data_ptr = get_u32(&self.db_bin_bytes[..], p + 8);
break;
}
}
}
if data_ptr == 0 {
Err(Error::NotFound)?;
}
let data_len = (data_ptr >> 24) & 0xff;
data_ptr = data_ptr & 0x00FFFFFF;
get_ip_info(
get_u32(&self.db_bin_bytes[..], data_ptr),
&self.db_bin_bytes[(data_ptr + 4) as usize..(data_ptr + data_len) as usize],
)
}
}

File diff suppressed because it is too large Load Diff

Binary file not shown.