xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'lionsoul2014:master' into python_dev

This commit is contained in:
厉害的花花 2022-07-08 16:10:57 +08:00 committed by GitHub
parent d8a960ef83 c1e98d5336
commit 3eadc47a19
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 2299 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -37,10 +37,14 @@ META-INF/
/binding/java/target/
/binding/java/*.jar
# clang
/binding/c/xdb_searcher
/binding/c/util_test
/binding/c/cmake-build-debug
# lua/luc_c
/binding/lua_c/cmake-build-debug
# golang
/binding/golang/searcher
/binding/golang/xdb_searcher

6
ReadMe.md
View File

@ -37,9 +37,9 @@ API 介绍，使用文档和测试程序请参考对应 `searcher` 查询客户
| [golang](binding/golang) | golang 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) |
| [php](binding/php) | php 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) |
| [java](binding/java) | java 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) |
| [lua](binding/lua) | lua 查询客户端实现 | 开发中... | [Lion](https://github.com/lionsoul2014) |
| [lua](binding/lua) | lua 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) |
| [c](binding/c) | ANSC c 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) |
| [lua_c](binding/lua_c) | lua 查询客户端实现 | 开发中... | [Lion](https://github.com/lionsoul2014) |
| [lua_c](binding/lua_c) | lua 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) |
| [rust](binding/rust) | rust 查询客户端实现 | 开发中... | [Lion](https://github.com/lionsoul2014) |
| [python](binding/python) | python 查询客户端实现 | 待开始 | 待确定 |
| [nodejs](binding/nodejs) | nodejs 查询客户端实现 | 待开始 | 待确定 |
@ -64,7 +64,7 @@ API 介绍，使用文档和测试程序请参考对应 `maker` 生成程序下
# 并发查询必读
全部查询客户端的 search 接口都 <b>不是</b> 并发安全的实现，不同进程/线程/携程需要通过创建不同的查询对象来安全使用，并发量很大的情况下，基于文件查询的方式可能会导致打开文件数过多的错误，请修改内核的最大允许打开文件数(fs.file-max=一个更高的值)，或者将整个xdb加载到内存进行安全并发使用。
全部查询客户端的 search 接口都 <b>不是</b> 并发安全的实现，不同进程/线程/协程需要通过创建不同的查询对象来安全使用，并发量很大的情况下，基于文件查询的方式可能会导致打开文件数过多的错误，请修改内核的最大允许打开文件数(fs.file-max=一个更高的值)，或者将整个xdb加载到内存进行安全并发使用。
# 相关备注

8
binding/c/ReadMe.md
View File

@ -49,7 +49,7 @@ int main(int argc, char *argv[]) {
int main(int argc, char *argv[]) {
char *db_path = "ip2region.xdb file path";
char *v_index;
xdb_vector_index_t *v_index;
xdb_searcher_t searcher;
char region_buffer[256], ip_buffer[16], *ip = "1.2.3.4";
long s_time;
@ -84,7 +84,7 @@ int main(int argc, char *argv[]) {
// 4、关闭 xdb 查询器，如果是要关闭服务，也需要释放 v_index 的内存。
xdb_close(&searcher);
xdb_free(v_index);
xdb_close_vector_index(v_index);
return 0;
}
```
@ -98,7 +98,7 @@ int main(int argc, char *argv[]) {
int main(int argc, char *argv[]) {
char *db_path = "ip2region.xdb file path";
char *c_buffer;
xdb_content_t *c_buffer;
xdb_searcher_t searcher;
char region_buffer[256], ip_buffer[16], *ip = "1.2.3.4";
long s_time;
@ -132,7 +132,7 @@ int main(int argc, char *argv[]) {
// 4、关闭 xdb 查询器，关闭服务的时候需要释放 c_buffer 的内存。
xdb_close(&searcher);
xdb_free(c_buffer);
xdb_close_content(c_buffer);
return 0;
}
```

8
binding/c/main.c
View File

@ -11,8 +11,8 @@
struct searcher_test_entry {
xdb_searcher_t searcher;
char *v_index;
char *c_buffer;
xdb_vector_index_t *v_index;
xdb_content_t *c_buffer;
};
typedef struct searcher_test_entry searcher_test_t;
@ -64,13 +64,13 @@ void destroy_searcher_test(searcher_test_t *test) {
// check and free the vector index
if (test->v_index != NULL) {
xdb_free(test->v_index);
xdb_close_vector_index(test->v_index);
test->v_index = NULL;
}
// check and free the content buffer
if (test->c_buffer != NULL) {
xdb_free(test->c_buffer);
xdb_close_content(test->c_buffer);
test->c_buffer = NULL;
}
}

30
binding/c/util_test.c
View File

@ -37,10 +37,9 @@ void test_check_ip() {
}
void test_load_header() {
xdb_header_t header;
int err = xdb_load_header_from_file("../../data/ip2region.xdb", &header);
if (err != 0) {
printf("failed to load header with errcode=%d\n", err);
xdb_header_t *header = xdb_load_header_from_file("../../data/ip2region.xdb");
if (header == NULL) {
printf("failed to load header");
} else {
printf("header loaded: {\n"
" version: %d, \n"
@ -48,33 +47,36 @@ void test_load_header() {
" created_at: %u, \n"
" start_index_ptr: %d, \n"
" end_index_ptr: %d\n"
" length: %d\n"
"}\n",
header.version, header.index_policy, header.created_at,
header.start_index_ptr, header.end_index_ptr
header->version, header->index_policy, header->created_at,
header->start_index_ptr, header->end_index_ptr, header->length
);
}
xdb_close_header(header);
}
void test_load_vector_index() {
char *ptr = xdb_load_vector_index_from_file("../../data/ip2region.xdb");
if (ptr == NULL) {
xdb_vector_index_t *v_index = xdb_load_vector_index_from_file("../../data/ip2region.xdb");
if (v_index == NULL) {
printf("failed to load vector index from file\n");
} else {
printf("vector index loaded from file\n");
printf("vector index loaded from file, length=%d\n", v_index->length);
}
xdb_free(ptr);
xdb_close_vector_index(v_index);
}
void test_load_content() {
char *ptr = xdb_load_content_from_file("../../data/ip2region.xdb");
if (ptr == NULL) {
xdb_content_t *content = xdb_load_content_from_file("../../data/ip2region.xdb");
if (content == NULL) {
printf("failed to load content from file\n");
} else {
printf("content loaded from file\n");
printf("content loaded from file, length=%d\n", content->length);
}
xdb_free(ptr);
xdb_close_content(content);
}
// valgrind --tool=memcheck --leak-check=full ./a.out

156
binding/c/xdb_searcher.c
View File

@ -12,13 +12,13 @@
// internal function prototype define
XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length);
XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const char *vIndex, const char *cBuff) {
XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) {
memset(xdb, 0x00, sizeof(xdb_searcher_t));
// check the content buffer first
if (cBuff != NULL) {
xdb->vector_index = NULL;
xdb->content_buff = cBuff;
if (c_buffer != NULL) {
xdb->v_index = NULL;
xdb->content = c_buffer;
return 0;
}
@ -29,27 +29,29 @@ XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const ch
}
xdb->handle = handle;
xdb->vector_index = vIndex;
xdb->v_index = v_index;
return 0;
}
// xdb searcher new api define
XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, char *db_path) {
XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) {
return xdb_new_base(xdb, db_path, NULL, NULL);
}
XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, char *db_path, char *vIndex) {
return xdb_new_base(xdb, db_path, vIndex, NULL);
XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) {
return xdb_new_base(xdb, db_path, v_index, NULL);
}
XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, char *c_buffer) {
XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) {
return xdb_new_base(xdb, NULL, NULL, c_buffer);
}
XDB_PUBLIC(void) xdb_close(xdb_searcher_t *xdb) {
XDB_PUBLIC(void) xdb_close(void *ptr) {
xdb_searcher_t *xdb = (xdb_searcher_t *) ptr;
if (xdb->handle != NULL) {
fclose(xdb->handle);
xdb->handle = NULL;
}
}
@ -77,12 +79,12 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu
il0 = ((int) (ip >> 24)) & 0xFF;
il1 = ((int) (ip >> 16)) & 0xFF;
idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size;
if (xdb->vector_index != NULL) {
s_ptr = xdb_get_uint(xdb->vector_index, idx);
e_ptr = xdb_get_uint(xdb->vector_index, idx + 4);
} else if (xdb->content_buff != NULL) {
s_ptr = xdb_get_uint(xdb->content_buff, xdb_header_info_length + idx);
e_ptr = xdb_get_uint(xdb->content_buff, xdb_header_info_length + idx + 4);
if (xdb->v_index != NULL) {
s_ptr = xdb_get_uint(xdb->v_index->buffer, idx);
e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4);
} else if (xdb->content != NULL) {
s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx);
e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4);
} else {
err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer));
if (err != 0) {
@ -146,8 +148,8 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu
XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) {
// check the xdb content cache first
if (xdb->content_buff != NULL) {
memcpy(buffer, xdb->content_buff + offset, length);
if (xdb->content != NULL) {
memcpy(buffer, xdb->content->buffer + offset, length);
return 0;
}
@ -171,39 +173,59 @@ XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) {
// --- buffer load util functions
XDB_PUBLIC(int) xdb_load_header(FILE *handle, xdb_header_t *header) {
char buffer[256];
XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *handle) {
xdb_header_t *header;
unsigned int size = xdb_header_info_length;
if (fseek(handle, 0, SEEK_SET) == -1) {
return 1;
// entry alloc
header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t));
if (header == NULL) {
return NULL;
}
if (fread(buffer, 1, 256, handle) != 256) {
return 2;
if (fseek(handle, 0, SEEK_SET) == -1) {
xdb_free(header);
return NULL;
}
if (fread(header->buffer, 1,size, handle) != size) {
xdb_free(header);
return NULL;
}
// fill the fields
header->version = (unsigned short) xdb_get_ushort(buffer, 0);
header->index_policy = (unsigned short) xdb_get_ushort(buffer, 2);
header->created_at = xdb_get_uint(buffer, 4);
header->start_index_ptr = xdb_get_uint(buffer, 8);
header->end_index_ptr = xdb_get_uint(buffer,12);
header->length = size;
header->version = (unsigned short) xdb_get_ushort(header->buffer, 0);
header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2);
header->created_at = xdb_get_uint(header->buffer, 4);
header->start_index_ptr = xdb_get_uint(header->buffer, 8);
header->end_index_ptr = xdb_get_uint(header->buffer,12);
return 0;
return header;
}
XDB_PUBLIC(int) xdb_load_header_from_file(char *db_path, xdb_header_t *header) {
XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *db_path) {
FILE *handle = fopen(db_path, "r");
if (handle == NULL) {
return 10;
return NULL;
}
return xdb_load_header(handle, header);
return xdb_load_header(handle);
}
XDB_PUBLIC(char *) xdb_load_vector_index(FILE *handle) {
char *ptr = NULL;
int size = xdb_vector_index_length;
XDB_PUBLIC(void) xdb_close_header(void *ptr) {
xdb_header_t *header = (xdb_header_t *) ptr;
if (header->length > 0) {
header->length = 0;
xdb_free(header);
}
}
// --- vector index
XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *handle) {
xdb_vector_index_t *v_index;
unsigned int size = xdb_vector_index_length;
// seek to the vector index offset
if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) {
@ -211,20 +233,21 @@ XDB_PUBLIC(char *) xdb_load_vector_index(FILE *handle) {
}
// do the buffer read
ptr = (char *) xdb_malloc(size);
if (ptr == NULL) {
v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t));
if (v_index == NULL) {
return NULL;
}
if (fread(ptr, 1, size, handle) != size) {
xdb_free(ptr);
v_index->length = size;
if (fread(v_index->buffer, 1, size, handle) != size) {
xdb_free(v_index);
return NULL;
}
return ptr;
return v_index;
}
XDB_PUBLIC(char *) xdb_load_vector_index_from_file(char *db_path) {
XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *db_path) {
FILE *handle = fopen(db_path, "r");
if (handle == NULL) {
return NULL;
@ -233,36 +256,55 @@ XDB_PUBLIC(char *) xdb_load_vector_index_from_file(char *db_path) {
return xdb_load_vector_index(handle);
}
XDB_PUBLIC(char *) xdb_load_content(FILE *handle) {
long filesize;
char *ptr = NULL;
XDB_PUBLIC(void) xdb_close_vector_index(void *ptr) {
xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr;
if (v_index->length > 0) {
v_index->length = 0;
xdb_free(v_index);
}
}
// --- content buffer
XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *handle) {
unsigned int size;
xdb_content_t *content;
char *ptr;
// determine the file size
if (fseek(handle, 0, SEEK_END) == -1) {
return NULL;
}
filesize = ftell(handle);
size = (unsigned int) ftell(handle);
if (fseek(handle, 0, SEEK_SET) == -1) {
return NULL;
}
// do the file read
ptr = (char *) xdb_malloc(filesize);
if (ptr == NULL) {
content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t));
if (content == NULL) {
return NULL;
}
// do the buffer alloc
content->buffer = (char *) xdb_malloc(size);
if (content->buffer == NULL) {
xdb_free(content);
return NULL;
}
// read the content into the buffer
if (fread(ptr, 1, filesize, handle) != filesize) {
content->length = size;
if (fread(content->buffer, 1, size, handle) != size) {
xdb_free(ptr);
return NULL;
}
return ptr;
return content;
}
XDB_PUBLIC(char *) xdb_load_content_from_file(char *db_path) {
XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *db_path) {
FILE *handle = fopen(db_path, "r");
if (handle == NULL) {
return NULL;
@ -271,6 +313,16 @@ XDB_PUBLIC(char *) xdb_load_content_from_file(char *db_path) {
return xdb_load_content(handle);
}
XDB_PUBLIC(void) xdb_close_content(void *ptr) {
xdb_content_t *content = (xdb_content_t *) ptr;
if (content->length > 0) {
content->length = 0;
xdb_free(content->buffer);
content->buffer = NULL;
xdb_free(content);
}
}
// --- End
// get unsigned long (4bytes) from a specified buffer start from the specified offset
@ -339,4 +391,4 @@ XDB_PUBLIC(long) xdb_now() {
struct timeval c_time;
gettimeofday(&c_time, NULL);
return c_time.tv_sec * (int)1e6 + c_time.tv_usec;
}
}

91
binding/c/xdb_searcher.h
View File

@ -38,6 +38,59 @@
#define xdb_vector_index_length 524288
// --- buffer load util functions
// use the following buffer struct to wrap the binary buffer data
// since the buffer data could not be operated with the string API.
struct xdb_header {
unsigned short version;
unsigned short index_policy;
unsigned int created_at;
unsigned int start_index_ptr;
unsigned int end_index_ptr;
// the original buffer
unsigned int length;
char buffer[xdb_header_info_length];
};
typedef struct xdb_header xdb_header_t;
XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *);
XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *);
XDB_PUBLIC(void) xdb_close_header(void *);
// --- vector index buffer
struct xdb_vector_index {
unsigned int length;
char buffer[xdb_vector_index_length];
};
typedef struct xdb_vector_index xdb_vector_index_t;
XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *);
XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *);
XDB_PUBLIC(void) xdb_close_vector_index(void *);
// --- content buffer
struct xdb_content {
unsigned int length;
char *buffer;
};
typedef struct xdb_content xdb_content_t;
XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *);
XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *);
XDB_PUBLIC(void) xdb_close_content(void *);
// --- End buffer load
// xdb searcher structure
struct xdb_searcher_entry {
FILE *handle;
@ -49,22 +102,22 @@ struct xdb_searcher_entry {
// vector index buffer cache.
// preload the vector index will reduce the number of IO operations
// thus speedup the search process.
const char *vector_index;
const xdb_vector_index_t *v_index;
// content buffer.
// cache the whole xdb content.
const char *content_buff;
const xdb_content_t *content;
};
typedef struct xdb_searcher_entry xdb_searcher_t;
// xdb searcher new api define
XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, char *);
XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, const char *);
XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, char *, char *);
XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *);
XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, char *);
XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *);
XDB_PUBLIC(void) xdb_close(xdb_searcher_t *);
XDB_PUBLIC(void) xdb_close(void *);
// xdb searcher search api define
XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t);
@ -74,32 +127,6 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, unsigned int, char *, size_t);
XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *);
// --- buffer load util functions
struct xdb_header {
unsigned short version;
unsigned short index_policy;
unsigned int created_at;
unsigned int start_index_ptr;
unsigned int end_index_ptr;
};
typedef struct xdb_header xdb_header_t;
XDB_PUBLIC(int) xdb_load_header(FILE *, xdb_header_t *);
XDB_PUBLIC(int) xdb_load_header_from_file(char *, xdb_header_t *);
XDB_PUBLIC(char *) xdb_load_vector_index(FILE *);
XDB_PUBLIC(char *) xdb_load_vector_index_from_file(char *);
XDB_PUBLIC(char *) xdb_load_content(FILE *);
XDB_PUBLIC(char *) xdb_load_content_from_file(char *);
// --- End buffer load
// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian
XDB_PUBLIC(unsigned int) xdb_get_uint(const char *, int);

16
binding/golang/xdb/util.go
View File

@ -9,20 +9,21 @@
package xdb
import (
"encoding/binary"
"fmt"
"os"
"strconv"
"strings"
)
var shiftIndex = []int{24, 16, 8, 0}
func CheckIP(ip string) (uint32, error) {
var ps = strings.Split(ip, ".")
if len(ps) != 4 {
return 0, fmt.Errorf("invalid ip address `%s`", ip)
}
var buff = make([]byte, 4)
var val = uint32(0)
for i, s := range ps {
d, err := strconv.Atoi(s)
if err != nil {
@ -33,20 +34,15 @@ func CheckIP(ip string) (uint32, error) {
return 0, fmt.Errorf("the %dth part `%s` should be an integer bettween 0 and 255", i, s)
}
buff[i] = byte(d)
val |= uint32(d) << shiftIndex[i]
}
// convert the ip to integer
return binary.BigEndian.Uint32(buff), nil
return val, nil
}
func Long2IP(ip uint32) string {
var buff = make([]string, 4)
buff[0] = fmt.Sprintf("%d", (ip>>24)&0xFF)
buff[1] = fmt.Sprintf("%d", (ip>>16)&0xFF)
buff[2] = fmt.Sprintf("%d", (ip>>8)&0xFF)
buff[3] = fmt.Sprintf("%d", (ip>>0)&0xFF)
return strings.Join(buff, ".")
return fmt.Sprintf("%d.%d.%d.%d", (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, ip&0xFF)
}
func MidIP(sip uint32, eip uint32) uint32 {

30
binding/golang/xdb/util_test.go
View File

@ -9,11 +9,41 @@
package xdb
import (
"encoding/binary"
"fmt"
"net"
"testing"
"time"
)
func TestCheckIP(t *testing.T) {
var str = "29.34.191.255"
ip, err := CheckIP(str)
if err != nil {
t.Errorf("check ip `%s`: %s\n", str, err)
}
netIP := net.ParseIP(str).To4()
if netIP == nil {
t.Fatalf("parse ip `%s` failed", str)
}
u32 := binary.BigEndian.Uint32(netIP)
fmt.Printf("checkip: %d, parseip: %d, isEqual: %v\n", ip, u32, ip == u32)
}
func TestLong2IP(t *testing.T) {
var str = "29.34.191.255"
netIP := net.ParseIP(str).To4()
if netIP == nil {
t.Fatalf("parse ip `%s` failed", str)
}
u32 := binary.BigEndian.Uint32(netIP)
ipStr := Long2IP(u32)
fmt.Printf("originIP: %s, Long2IP: %s, isEqual: %v\n", str, ipStr, ipStr == str)
}
func TestLoadVectorIndex(t *testing.T) {
vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region.xdb")
if err != nil {

10
binding/java/ReadMe.md
View File

@ -7,7 +7,7 @@
<dependency>
<groupId>org.lionsoul</groupId>
<artifactId>ip2region</artifactId>
<version>2.6.2</version>
<version>2.6.4</version>
</dependency>
```
@ -34,7 +34,7 @@ public class SearcherTest {
try {
String ip = "1.2.3.4";
long sTime = System.nanoTime();
String region = searcher.searchByStr(ip);
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
@ -80,7 +80,7 @@ public class SearcherTest {
try {
String ip = "1.2.3.4";
long sTime = System.nanoTime();
String region = searcher.searchByStr(ip);
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
@ -126,14 +126,14 @@ public class SearcherTest {
try {
String ip = "1.2.3.4";
long sTime = System.nanoTime();
String region = searcher.searchByStr(ip);
String region = searcher.search(ip);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {
System.out.printf("failed to search(%s): %s\n", ip, e);
}
// 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去垮线程访问。
// 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。
}
}
```

2
binding/java/pom.xml
View File

@ -4,7 +4,7 @@
<groupId>org.lionsoul</groupId>
<artifactId>ip2region</artifactId>
<version>2.6.2</version>
<version>2.6.4</version>
<packaging>jar</packaging>
<name>ip2region</name>

2
binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java
View File

@ -89,7 +89,7 @@ public class SearchTest {
try {
double sTime = System.nanoTime();
String region = searcher.searchByStr(line);
String region = searcher.search(line);
long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime));
System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost);
} catch (Exception e) {

2
binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java
View File

@ -15,6 +15,7 @@ public class Header {
public final int createdAt;
public final int startIndexPtr;
public final int endIndexPtr;
public final byte[] buffer;
public Header(byte[] buff) {
assert buff.length >= 16;
@ -23,6 +24,7 @@ public class Header {
createdAt = Searcher.getInt(buff, 4);
startIndexPtr = Searcher.getInt(buff, 8);
endIndexPtr = Searcher.getInt(buff, 12);
buffer = buff;
}
@Override public String toString() {

4
binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java
View File

@ -71,7 +71,7 @@ public class Searcher {
return ioCount;
}
public String searchByStr(String ipStr) throws Exception {
public String search(String ipStr) throws Exception {
long ip = checkIP(ipStr);
return search(ip);
}
@ -135,7 +135,7 @@ public class Searcher {
// load and return the region data
final byte[] regionBuff = new byte[dataLen];
read(dataPtr, regionBuff);
return new String(regionBuff);
return new String(regionBuff, "utf-8");
}
protected void read(int offset, byte[] buffer) throws IOException {

148
binding/lua/ReadMe.md
View File

@ -1,7 +1,153 @@
# ip2region lua 查询客户端实现
# ip2region xdb lua 查询客户端实现
#### 备注：请优先使用 lua_c 扩展 xdb 查询客户端，性能比纯 lua 实现的要快很多！！！
# 使用方式
### 完全基于文件的查询
```lua
local xdb = require("xdb_searcher")
-- 1、从 db_path 创建基于文件的 xdb 查询对象
local db_path = "ip2region.xdb file path"
local searcher, err = xdb.new_with_file_only(db_path)
if err ~= nil then
print(string.format("failed to create searcher: %s", err))
return
end
-- 2、调用查询 API 进行查询
local ip_str = "1.2.3.4"
local s_time = xdb.now()
region, err = searcher:search(ip_str)
if err ~= nil then
print(string.format("failed to search(%s): %s", ip_str, err))
return
end
-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
```
### 缓存 `VectorIndex` 索引
如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。
```lua
local xdb = require("xdb_searcher")
local db_path = "ip2region.xdb file path"
-- 1、从指定的 db_path 加载 VectorIndex 缓存，把下述的 v_index 对象做成全局变量。
-- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。
v_index, err = xdb.load_vector_index(db_path)
if err ~= nil then
print(string.format("failed to load vector index from '%s'", db_path))
return
end
-- 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。
searcher, err = xdb.new_with_vector_index(db_path, v_index)
if err ~= nil then
print(string.format("failed to create vector index searcher: %s", err))
return
end
-- 3、调用查询 API
local ip_str = "1.2.3.4"
local s_time = xdb.now()
region, err = searcher:search(ip_str)
if err ~= nil then
print(string.format("failed to search(%s): %s", ip_str, err))
return
end
-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
```
### 缓存整个 `xdb` 数据
如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。
```lua
local xdb = require("xdb_searcher")
local db_path = "ip2region.xdb file path"
-- 1、从指定的 db_path 加载整个 xdb 到内存。
-- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。
content = xdb.load_content(db_path)
if content == nil then
print(string.format("failed to load xdb content from '%s'", db_path))
return
end
-- 2、使用全局的 content 创建带完全基于内存的查询对象。
searcher, err = xdb.new_with_buffer(content)
if err ~= nil then
print(string.format("failed to create content buffer searcher: %s", err))
return
end
-- 3、调用查询 API
local ip_str = "1.2.3.4"
local s_time = xdb.now()
region, err = searcher:search(ip_str)
if err ~= nil then
print(string.format("failed to search(%s): %s", ip_str, err))
return
end
-- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。
-- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
```
# 查询测试
通过 `search_test.lua` 脚本来进行查询测试：
```bash
➜ lua git:(lua_binding) ✗ lua search_test.lua
lua search_test.lua [command options]
options:
--db string ip2region binary xdb file path
--cache-policy string cache policy: file/vectorIndex/content
```
例如：使用默认的 data/ip2region.xdb 进行查询测试：
```bash
➜ lua git:(lua_binding) ✗ lua search_test.lua --db=../../data/ip2region.xdb --cache-policy=vectorIndex
ip2region xdb searcher test program, cachePolicy: vectorIndex
type 'quit' to exit
ip2region>> 1.2.3.4
{region: 美国|0|华盛顿|0|谷歌, io_count: 7, took: 0μs}
ip2region>>
```
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
# bench 测试
通过 `bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能：
```bash
➜ lua git:(lua_binding) ✗ lua bench_test.lua
lua bench_test.lua [command options]
options:
--db string ip2region binary xdb file path
--src string source ip text file path
--cache-policy string cache policy: file/vectorIndex/content
```
例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试：
```bash
➜ lua git:(lua_binding) ✗ lua bench_test.lua --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=vectorIndex
Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 29.000 s, cost: 7.899 μs/op}
```
可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。
@Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。

175
binding/lua/bench_test.lua Normal file
View File

@ -0,0 +1,175 @@
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
-- Use of this source code is governed by a Apache2.0-style
-- license that can be found in the LICENSE file.
--
-- ---
-- @Author Lion <chenxin619315@gmail.com>
-- @Date 2022/06/30
-- set the package to load the current xdb_searcher.so
package.path = "./?.lua" .. package.path
package.cpath = "./?.so" .. package.cpath
local xdb = require("xdb_searcher")
function printHelp()
print("lua bench_test.lua [command options]")
print("options: ")
print(" --db string ip2region binary xdb file path")
print(" --src string source ip text file path")
print(" --cache-policy string cache policy: file/vectorIndex/content")
end
if #arg < 2 then
printHelp(arg)
return
end
-- parser the command line args
local dbFile, srcFile = "", ""
local cachePolicy = "vectorIndex"
for _, r in ipairs(arg) do
if string.len(r) < 5 then
goto continue
end
if string.sub(r, 1, 2) ~= "--" then
goto continue
end
for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do
if k == "db" then
dbFile = v
elseif k == "src" then
srcFile = v
elseif k == "cache-policy" then
cachePolicy = v
else
print(string.format("undefined option `%s`", r))
return
end
-- break the match iterate
break
end
-- continue this loop
::continue::
end
-- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy))
if string.len(dbFile) < 2 or string.len(srcFile) < 2 then
printHelp()
return
end
-- create the searcher based on the cache-policy
local searcher, v_index, content
if cachePolicy == "file" then
searcher, err = xdb.new_with_file_only(dbFile)
if err ~= nil then
print(string.format("failed to create searcher: %s", err))
return
end
elseif cachePolicy == "vectorIndex" then
v_index, err = xdb.load_vector_index(dbFile)
if err ~= nil then
print(string.format("failed to load vector index: %s", err))
return
end
searcher, err = xdb.new_with_vector_index(dbFile, v_index)
if err ~= nil then
print(string.format("failed to create vector index searcher: %s", err))
return
end
elseif cachePolicy == "content" then
content, err = xdb.load_content(dbFile)
if err ~= nil then
print(string.format("failed to load xdb content: %s", err))
return
end
searcher, err = xdb.new_with_buffer(content)
if err ~= nil then
print(string.format("failed to create content buffer searcher: %s", err))
return
end
else
print(string.format("undefined cache-policy `%s`", cachePolicy))
return
end
-- do the bench test
local handle = io.open(srcFile, "r")
if handle == nil then
print(string.format("failed to open src text file `%s`", handle))
return
end
local lines = handle:lines()
local sip_str, eip_str, s_region, region = "", "", "", ""
local sip, mip, eip, err = 0, 0, 0, 0
local count, t_time, c_time = 0, 0, 0
local s_time = xdb.now()
for l in lines do
if string.len(l) < 1 then
goto continue
end
for v1, v2, v3 in string.gmatch(l, "([%d%.]+)|([%d%.]+)|([^\n]+)") do
-- print(sip_str, eip_str, region)
sip_str = v1
eip_str = v2
s_region = v3
break
end
sip, err = xdb.check_ip(sip_str)
if err ~= nil then
print(string.format("invalid start ip `%s`", sip_str))
return
end
eip, err = xdb.check_ip(eip_str)
if err ~= nil then
print(string.format("invalid end ip `%s`", sip_str))
return
end
if sip > eip then
print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str))
return
end
mip = (sip + eip) >> 1
for _, ip in ipairs({sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) do
t_time = xdb.now()
region, err = searcher:search(ip)
c_time = c_time + xdb.now() - t_time
if err ~= nil then
print(string.format("failed to search ip `%s`", xdb.long2ip(ip)))
return
end
-- check the region
if region ~= s_region then
print(string.format("failed search(%s) with (%s != %s)\n", xdb.long2ip(ip), region, s_region))
return
end
count = count + 1
end
::continue::
end
-- resource cleanup
searcher:close()
-- print the stats
local avg_costs = 0
if count > 0 then
avg_costs = c_time / count
end
print(string.format("Bench finished, {cachePolicy: %s, total: %d, took: %.3f s, cost: %.3f μs/op}",
cachePolicy, count, (xdb.now() - s_time)/1e6, c_time / count))

137
binding/lua/search_test.lua Normal file
View File

@ -0,0 +1,137 @@
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
-- Use of this source code is governed by a Apache2.0-style
-- license that can be found in the LICENSE file.
--
-- ---
-- @Author Lion <chenxin619315@gmail.com>
-- @Date 2022/06/30
-- set the package to load the current xdb_searcher.so
package.path = "./?.lua" .. package.path
package.cpath = "./?.so" .. package.cpath
local xdb = require("xdb_searcher")
function printHelp()
print("lua search_test.lua [command options]")
print("options: ")
print(" --db string ip2region binary xdb file path")
print(" --cache-policy string cache policy: file/vectorIndex/content")
end
if #arg < 2 then
printHelp(arg)
return
end
-- parser the command line args
local dbFile = ""
local cachePolicy = "vectorIndex"
for _, r in ipairs(arg) do
if string.len(r) < 5 then
goto continue
end
if string.sub(r, 1, 2) ~= "--" then
goto continue
end
for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do
if k == "db" then
dbFile = v
elseif k == "cache-policy" then
cachePolicy = v
else
print(string.format("undefined option `%s`", r))
return
end
-- break the match iterate
break
end
-- continue this loop
::continue::
end
-- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy))
if string.len(dbFile) < 2 then
printHelp()
return
end
-- create the searcher based on the cache-policy
local searcher, v_index, content
if cachePolicy == "file" then
searcher, err = xdb.new_with_file_only(dbFile)
if err ~= nil then
print(string.format("failed to create searcher: %s", err))
return
end
elseif cachePolicy == "vectorIndex" then
v_index, err = xdb.load_vector_index(dbFile)
if err ~= nil then
print(string.format("failed to load vector index: %s", err))
return
end
searcher, err = xdb.new_with_vector_index(dbFile, v_index)
if err ~= nil then
print(string.format("failed to create vector index searcher: %s", err))
return
end
elseif cachePolicy == "content" then
content, err = xdb.load_content(dbFile)
if err ~= nil then
print(string.format("failed to load xdb content: %s", err))
return
end
searcher, err = xdb.new_with_buffer(content)
if err ~= nil then
print(string.format("failed to create content buffer searcher: %s", err))
return
end
else
print(string.format("undefined cache-policy `%s`", cachePolicy))
return
end
-- do the search
print(string.format([[
ip2region xdb searcher test program, cachePolicy: %s
type 'quit' to exit]], cachePolicy))
local region, err = "", nil
local ip_int, s_time, c_time = 0, 0, 0
while ( true ) do
io.write("ip2region>> ");
io.input(io.stdin);
local line = io.read();
if (line == nil) then
break
end
if ( line == "quit" ) then
break
end
ip_int, err = xdb.check_ip(line)
if err ~= nil then
print(string.format("invalid ip address `%s`", line))
goto continue
end
-- do the search
s_time = xdb.now()
region, err = searcher:search(line)
if err ~= nil then
print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count()))
else
c_time = xdb.now() - s_time
print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time))
end
::continue::
end
-- resource cleanup
searcher:close()

98
binding/lua/util_test.lua Normal file
View File

@ -0,0 +1,98 @@
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
-- Use of this source code is governed by a Apache2.0-style
-- license that can be found in the LICENSE file.
--
-- ---
-- @Author Lion <chenxin619315@gmail.com>
-- @Date 2022/07/05
-- set the package path
package.path = "./?.lua"
package.cpath = "./?.so"
local xdb = require("xdb_searcher")
---- ip checking testing
print("--- testing check_ip and long2ip ... ")
local ip_list = {
"1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0",
"256.7.12.9", "12.56.78.320", "32.12.45.192", "222.221.220.219",
"192.168.1.101 ", "132.96.12.98a", "x23.12.2.12"
}
local s_time = xdb.now()
for _, ip_src in ipairs(ip_list) do
ip, err = xdb.check_ip(ip_src)
if err ~= nil then
print(string.format("invalid ip address `%s`: %s", ip_src, err))
else
ip_dst = xdb.long2ip(ip)
io.write(string.format("long(%-15s)=%10d, long2ip(%-10d)=%-15s", ip_src, ip, ip, ip_dst))
if ip_src ~= ip_dst then
print(" --[Failed]")
else
print(" --[Ok]")
end
end
end
---- buffer loading test
print("\n--- testing load header ... ")
header, err = xdb.load_header("../../data/ip2region.xdb")
if err ~= nil then
print("failed to load header: ", err)
else
print("xdb header buffer loaded")
local tpl = [[
header: {
version: %d
index_policy: %d
created_at: %d
start_index_ptr: %d
end_index_ptr: %d
}]]
print(string.format(tpl,
header["version"], header["index_policy"],
header["created_at"], header["start_index_ptr"], header["end_index_ptr"])
)
end
print("\n--- testing load vector index ... ")
v_index, err = xdb.load_vector_index("../../data/ip2region.xdb")
if err ~= nil then
print("failed to load vector index: ", err)
else
print("xdb vector index buffer loaded")
end
print("\n--- testing load content buffer ... ")
c_buffer, err = xdb.load_content("../../data/ip2region.xdb")
if err ~= nil then
print("failed to load content: ", err)
else
print("xdb content buffer loaded")
end
print("\n--- testing search ... ")
local ip_str = "1.2.3.4"
searcher, err = xdb.new_with_file_only("../../data/ip2region.xdb")
local t_start = xdb.now()
region, err = searcher:search(ip_str)
if err ~= nil then
print(string.format("search(%s) failed: %s", ip_str, err))
else
local c_time = xdb.now() - t_start
print(string.format("search(%s): {region=%s, io_count: %d, took: %dμs, err=%s}",
ip_str, region, searcher:get_io_count(), c_time, err))
print(string.format("searcher.tostring=%s", searcher))
end
searcher:close()
print("")
print(string.format("all tests done, elapsed %d μs", xdb.now() - s_time))

333
binding/lua/xdb_searcher.lua Normal file
View File

@ -0,0 +1,333 @@
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
-- Use of this source code is governed by a Apache2.0-style
-- license that can be found in the LICENSE file.
--
-- ---
-- @Author Lion <chenxin619315@gmail.com>
-- @Date 2022/07/05
-- constants define
local HeaderInfoLength = 256
local VectorIndexRows = 256
local VectorIndexCols = 256
local VectorIndexSize = 8
local SegmentIndexSize = 14
local VectorIndexLength = 524288
local _M = {
-- xdb file handle
handle = nil,
-- header info
header = nil,
io_count = 0,
-- vector index
vector_index = nil,
-- xdb content buffer
content_buff = nil
}
-- index and to string attribute set
_M.__index = _M
_M.__tostring = function(self)
return "xdb searcher object (lua)"
end
-- construct functions
function newBase(dbPath, vIndex, cBuffer)
local obj = setmetatable({}, _M)
if cBuffer ~= nil then
obj.io_count = 0
obj.vector_index = nil
obj.content_buff = cBuffer
else
obj.io_count = 0
obj.vector_index = vIndex
obj.handle = io.open(dbPath, "r")
if obj.handle == nil then
return nil, string.format("failed to open xdb file `%s`", dbPath)
end
end
return obj, nil
end
function _M.new_with_file_only(dbPath)
return newBase(dbPath, nil, nil)
end
function _M.new_with_vector_index(dbPath, vIndex)
return newBase(dbPath, vIndex, nil)
end
function _M.new_with_buffer(cBuffer)
return newBase(nil, nil, cBuffer)
end
-- End of constructors
-- object api impl, must call via ':'
function _M:search(ip_src)
-- check and convert string ip to long ip
local t, ip = type(ip_src), 0
if t == nil then
return "", string.format("invalid ip address `%s`", ip_src)
elseif t == "string" then
ip, err = self.check_ip(ip_src)
if err ~= nil then
return "", string.format("check ip `%s`: %s", ip_src, err)
end
elseif t ~= "number" then
return "", "invalid number or string ip"
else
-- use the original value
ip = ip_src
end
-- reset the global counter
-- and global resource local cache
self.io_count = 0
local vector_index = self.vector_index
local content_buff = self.content_buff
local read_data = self.read
-- locate the segment index based on the vector index
local il0 = (ip >> 24) & 0xFF
local il1 = (ip >> 16) & 0xFF
local idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize
local s_ptr, e_ptr = 0, 0
if vector_index ~= nil then
s_ptr = getLong(vector_index, idx + 1)
e_ptr = getLong(vector_index, idx + 5)
elseif content_buff ~= nil then
s_ptr = getLong(content_buff, HeaderInfoLength + idx + 1)
e_ptr = getLong(content_buff, HeaderInfoLength + idx + 5)
else
-- load from the file
buff, err = read_data(self, HeaderInfoLength + idx, SegmentIndexSize)
if err ~= nil then
return "", string.format("read buffer: %s", err)
end
s_ptr = getLong(buff, 1)
e_ptr = getLong(buff, 5)
end
-- print(string.format("s_ptr: %d, e_ptr: %d", s_ptr, e_ptr))
-- binary search to get the data
local data_ptr, data_len, p = 0, 0, 0
local sip, eip, err, buff = 0, 0, ""
local l, m, h = 0, 0, (e_ptr - s_ptr) / SegmentIndexSize
while l <= h do
m = (l + h) >> 1
p = s_ptr + m * SegmentIndexSize
-- read the segment index
buff, err = read_data(self, p, SegmentIndexSize)
if err ~= nil then
return "", string.format("read segment index at %d", p)
end
sip = getLong(buff, 1)
if ip < sip then
h = m - 1
else
eip = getLong(buff, 5)
if ip > eip then
l = m + 1
else
data_len = getShort(buff, 9)
data_ptr = getLong(buff, 11)
break
end
end
end
-- matching nothing interception
-- print(string.format("data_len=%d, data_ptr=%d", data_len, data_ptr))
if data_len == 0 then
return "", nil
end
-- load and return the region data
buff, err = read_data(self, data_ptr, data_len)
if err ~= nil then
return "", string.format("read data at %d:%d", data_ptr, data_len)
end
return buff, nil
end
-- read specified bytes from the specified index
function _M:read(offset, length)
-- local cache
local content_buff = self.content_buff
local handle = self.handle
-- check the in-memory buffer first
if content_buff ~= nil then
return string.sub(content_buff, offset + 1, offset + length), nil
end
-- read from the file
local r = handle:seek("set", offset)
if r == nil then
return nil, string.format("seek to offset %d", offset)
end
self.io_count = self.io_count + 1
local buff = handle:read(length)
if buff == nil then
return nil, string.format("read %d bytes", length)
end
return buff, nil
end
function _M:get_io_count()
return self.io_count
end
function _M:close()
if self.handle ~= nil then
self.handle:close()
end
end
-- End of search api
-- static util functions
function _M.load_header(dbPath)
local handle = io.open(dbPath, "r")
if handle == nil then
return nil, string.format("failed to open xdb file `%s`", dbPath)
end
local r = handle:seek("set", 0)
if r == nil then
handle:close()
return nil, "failed to seek to 0"
end
local c = handle:read(HeaderInfoLength)
if c == nil then
handle:close()
return nil, string.format("failed to read %d bytes", HeaderInfoLength)
end
handle:close()
return {
["version"] = getShort(c, 1),
["index_policy"] = getShort(c, 3),
["created_at"] = getLong(c, 5),
["start_index_ptr"] = getLong(c, 9),
["end_index_ptr"] = getLong(c, 13),
["raw_data"] = c
}, nil
end
function _M.load_vector_index(dbPath)
local handle = io.open(dbPath, "r")
if handle == nil then
return nil, string.format("failed to open xdb file `%s`", dbPath)
end
local r = handle:seek("set", HeaderInfoLength)
if r == nil then
handle:close()
return nil, string.format("failed to seek to %d", HeaderInfoLength)
end
local c = handle:read(VectorIndexLength)
if c == nil then
handle:close()
return nil, string.format("failed to read %d bytes", VectorIndexLength)
end
handle:close()
return c, nil
end
function _M.load_content(dbPath)
local handle = io.open(dbPath, "r")
if handle == nil then
return nil, string.format("failed to open xdb file `%s`", dbPath)
end
local c = handle:read("*a")
if c == nil then
return nil, string.format("failed to read xdb content")
end
handle:close()
return c, nil
end
function _M.check_ip(ip_str)
local ip, id, v = 0, 1, 0
local offset_arr = {24, 16, 8, 0}
for p in string.gmatch(ip_str..".", "([%d]+)%.") do
-- match pattern checking
if p == nil then
return 0, "err=1"
end
-- count checking
if id > 4 then
return 0, "err=1"
end
-- value checking
v = tonumber(p)
if v > 255 then
return 0, "err=2"
end
ip = ip | (v << offset_arr[id])
id = id + 1
end
if id ~= 5 then
return 0, "err=1"
end
return ip, nil
end
function _M.long2ip(ip)
return string.format("%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8 ) & 0xFF, ip & 0xFF)
end
-- this is a bit weird, but we have to better choice for now
function _M.now()
return os.time() * 1e6
end
-- End of util functions
--internal function to get a integer from a binary string
function getLong(buff, idx)
local i1 = (string.byte(string.sub(buff, idx, idx)))
local i2 = (string.byte(string.sub(buff, idx+1, idx+1)) << 8)
local i3 = (string.byte(string.sub(buff, idx+2, idx+2)) << 16)
local i4 = (string.byte(string.sub(buff, idx+3, idx+3)) << 24)
return (i1 | i2 | i3 | i4)
end
function getShort(buff, idx)
local i1 = (string.byte(string.sub(buff, idx, idx)))
local i2 = (string.byte(string.sub(buff, idx+1, idx+1)) << 8)
return (i1 | i2)
end
return _M

16
binding/lua_c/Makefile Normal file
View File

@ -0,0 +1,16 @@
LuaVersion = 5.3
LIB_DIR = /usr/local/share/lua/$(LuaVersion)
all: ../c/xdb_searcher.h ../c/xdb_searcher.c xdb_searcher.c
gcc -g -O2 -I../c/ -I/usr/include/lua$(LuaVersion) ../c/xdb_searcher.c xdb_searcher.c -fPIC -shared -o xdb_searcher.so
install:
sudo mkdir -p $(LIB_DIR); \
sudo cp xdb_searcher.so $(LIB_DIR);\
echo "install xdb searcher to $(LIB_DIR) successfully.";\
clean:
find . -name \*.so | xargs rm -f
find . -name \*.o | xargs rm -f
.PHONY: clean

157
binding/lua_c/ReadMe.md
View File

@ -1,7 +1,162 @@
# ip2region lua c 扩展查询客户端实现
# ip2region xdb lua c 扩展查询客户端实现
# 编译安装
通过如下方式来编译安装该扩展：
```bash
# cd 到 lua_c binding 的根目录
make
sudo make install
```
备注：Makefile 里面的 `LuaVersion` 变量代表你本地环境的 lua 的版本，默认为 5.3，如果你的版本不是 5.3，记得先修改为和你本地 lua 版本一致的版本号。
# 使用方式
### 完全基于文件的查询
```lua
local xdb = require("xdb_searcher")
-- 1、从 db_path 创建基于文件的 xdb 查询对象
local db_path = "ip2region.xdb file path"
local searcher, err = xdb.new_with_file_only(db_path)
if err ~= nil then
print(string.format("failed to create searcher: %s", err))
return
end
-- 2、调用查询 API 进行查询
local ip_str = "1.2.3.4"
local s_time = xdb.now()
region, err = searcher:search(ip_str)
if err ~= nil then
print(string.format("failed to search(%s): %s", ip_str, err))
return
end
-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
```
### 缓存 `VectorIndex` 索引
如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。
```lua
local xdb = require("xdb_searcher")
local db_path = "ip2region.xdb file path"
-- 1、从指定的 db_path 加载 VectorIndex 缓存，把下述的 v_index 对象做成全局变量。
-- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。
v_index, err = xdb.load_vector_index(db_path)
if err ~= nil then
print(string.format("failed to load vector index from '%s'", db_path))
return
end
-- 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。
searcher, err = xdb.new_with_vector_index(db_path, v_index)
if err ~= nil then
print(string.format("failed to create vector index searcher: %s", err))
return
end
-- 3、调用查询 API
local ip_str = "1.2.3.4"
local s_time = xdb.now()
region, err = searcher:search(ip_str)
if err ~= nil then
print(string.format("failed to search(%s): %s", ip_str, err))
return
end
-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
```
### 缓存整个 `xdb` 数据
如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。
```lua
local xdb = require("xdb_searcher")
local db_path = "ip2region.xdb file path"
-- 1、从指定的 db_path 加载整个 xdb 到内存。
-- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。
content = xdb.load_content(db_path)
if content == nil then
print(string.format("failed to load xdb content from '%s'", db_path))
return
end
-- 2、使用全局的 content 创建带完全基于内存的查询对象。
searcher, err = xdb.new_with_buffer(content)
if err ~= nil then
print(string.format("failed to create content buffer searcher: %s", err))
return
end
-- 3、调用查询 API
local ip_str = "1.2.3.4"
local s_time = xdb.now()
region, err = searcher:search(ip_str)
if err ~= nil then
print(string.format("failed to search(%s): %s", ip_str, err))
return
end
-- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。
-- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。
print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time))
```
# 查询测试
通过 `search_test.lua` 脚本来进行查询测试：
```bash
➜ lua_c git:(lua_binding) ✗ lua ./search_test.lua
lua search_test.lua [command options]
options:
--db string ip2region binary xdb file path
--cache-policy string cache policy: file/vectorIndex/content
```
例如：使用默认的 data/ip2region.xdb 进行查询测试：
```bash
➜ lua_c git:(lua_binding) ✗ lua ./search_test.lua --db=../../data/ip2region.xdb --cache-policy=vectorIndex
ip2region xdb searcher test program, cachePolicy: vectorIndex
type 'quit' to exit
ip2region>> 1.2.3.4
{region: 美国|0|华盛顿|0|谷歌, io_count: 7, took: 15μs}
ip2region>>
```
输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。
# bench 测试
通过 `bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能：
```bash
➜ lua_c git:(lua_binding) ✗ lua ./bench_test.lua
lua bench_test.lua [command options]
options:
--db string ip2region binary xdb file path
--src string source ip text file path
--cache-policy string cache policy: file/vectorIndex/content
```
例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试：
```bash
➜ lua_c git:(lua_binding) ✗ lua ./bench_test.lua --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=vectorIndex
Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 5.865 s, cost: 1.399 μs/op}
```
可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。
@Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。

181
binding/lua_c/bench_test.lua Normal file
View File

@ -0,0 +1,181 @@
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
-- Use of this source code is governed by a Apache2.0-style
-- license that can be found in the LICENSE file.
--
-- ---
-- @Author Lion <chenxin619315@gmail.com>
-- @Date 2022/06/30
-- set the package to load the current xdb_searcher.so
package.path = "./?.lua" .. package.path
package.cpath = "./?.so" .. package.cpath
local xdb = require("xdb_searcher")
function printHelp()
print("lua bench_test.lua [command options]")
print("options: ")
print(" --db string ip2region binary xdb file path")
print(" --src string source ip text file path")
print(" --cache-policy string cache policy: file/vectorIndex/content")
end
if #arg < 2 then
printHelp(arg)
return
end
-- parser the command line args
local dbFile, srcFile = "", ""
local cachePolicy = "vectorIndex"
for _, r in ipairs(arg) do
if string.len(r) < 5 then
goto continue
end
if string.sub(r, 1, 2) ~= "--" then
goto continue
end
for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do
if k == "db" then
dbFile = v
elseif k == "src" then
srcFile = v
elseif k == "cache-policy" then
cachePolicy = v
else
print(string.format("undefined option `%s`", r))
return
end
-- break the match iterate
break
end
-- continue this loop
::continue::
end
-- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy))
if string.len(dbFile) < 2 or string.len(srcFile) < 2 then
printHelp()
return
end
-- create the searcher based on the cache-policy
local searcher, v_index, content
if cachePolicy == "file" then
searcher, err = xdb.new_with_file_only(dbFile)
if err ~= nil then
print(string.format("failed to create searcher: %s", err))
return
end
elseif cachePolicy == "vectorIndex" then
v_index, err = xdb.load_vector_index(dbFile)
if err ~= nil then
print(string.format("failed to load vector index: %s", err))
return
end
searcher, err = xdb.new_with_vector_index(dbFile, v_index)
if err ~= nil then
print(string.format("failed to create vector index searcher: %s", err))
return
end
elseif cachePolicy == "content" then
content, err = xdb.load_content(dbFile)
if err ~= nil then
print(string.format("failed to load xdb content from '%s'", dbFile))
return
end
searcher, err = xdb.new_with_buffer(content)
if err ~= nil then
print(string.format("failed to create content buffer searcher: %s", err))
return
end
else
print(string.format("undefined cache-policy `%s`", cachePolicy))
return
end
-- do the bench test
local handle = io.open(srcFile, "r")
if handle == nil then
print(string.format("failed to open src text file `%s`", handle))
return
end
local lines = handle:lines()
local sip_str, eip_str, s_region, region = "", "", "", ""
local sip, mip, eip, err = 0, 0, 0, 0
local count, t_time, c_time = 0, 0, 0
local s_time = xdb.now()
for l in lines do
if string.len(l) < 1 then
goto continue
end
for v1, v2, v3 in string.gmatch(l, "([%d%.]+)|([%d%.]+)|([^\n]+)") do
-- print(sip_str, eip_str, region)
sip_str = v1
eip_str = v2
s_region = v3
break
end
sip, err = xdb.check_ip(sip_str)
if err ~= nil then
print(string.format("invalid start ip `%s`", sip_str))
return
end
eip, err = xdb.check_ip(eip_str)
if err ~= nil then
print(string.format("invalid end ip `%s`", sip_str))
return
end
if sip > eip then
print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str))
return
end
mip = (sip + eip) >> 1
for _, ip in ipairs({sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) do
t_time = xdb.now()
region, err = searcher:search(ip)
c_time = c_time + xdb.now() - t_time
if err ~= nil then
print(string.format("failed to search ip `%s`", xdb.long2ip(ip)))
return
end
-- check the region
if region ~= s_region then
printf(string.format("failed search(%s) with (%s != %s)\n", xdb.long2ip(ip), region, s_region))
return
end
count = count + 1
end
::continue::
end
-- resource cleanup
searcher:close()
if v_index ~= nil then
v_index:close()
end
if content ~= nil then
content:close()
end
-- print the stats
local avg_costs = 0
if count > 0 then
avg_costs = c_time / count
end
print(string.format("Bench finished, {cachePolicy: %s, total: %d, took: %.3f s, cost: %.3f μs/op}",
cachePolicy, count, (xdb.now() - s_time)/1e6, c_time / count))

143
binding/lua_c/search_test.lua Normal file
View File

@ -0,0 +1,143 @@
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
-- Use of this source code is governed by a Apache2.0-style
-- license that can be found in the LICENSE file.
--
-- ---
-- @Author Lion <chenxin619315@gmail.com>
-- @Date 2022/06/30
-- set the package to load the current xdb_searcher.so
package.path = "./?.lua" .. package.path
package.cpath = "./?.so" .. package.cpath
local xdb = require("xdb_searcher")
function printHelp()
print("lua search_test.lua [command options]")
print("options: ")
print(" --db string ip2region binary xdb file path")
print(" --cache-policy string cache policy: file/vectorIndex/content")
end
if #arg < 2 then
printHelp(arg)
return
end
-- parser the command line args
local dbFile = ""
local cachePolicy = "vectorIndex"
for _, r in ipairs(arg) do
if string.len(r) < 5 then
goto continue
end
if string.sub(r, 1, 2) ~= "--" then
goto continue
end
for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do
if k == "db" then
dbFile = v
elseif k == "cache-policy" then
cachePolicy = v
else
print(string.format("undefined option `%s`", r))
return
end
-- break the match iterate
break
end
-- continue this loop
::continue::
end
-- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy))
if string.len(dbFile) < 2 then
printHelp()
return
end
-- create the searcher based on the cache-policy
local searcher, v_index, content
if cachePolicy == "file" then
searcher, err = xdb.new_with_file_only(dbFile)
if err ~= nil then
print(string.format("failed to create searcher: %s", err))
return
end
elseif cachePolicy == "vectorIndex" then
v_index, err = xdb.load_vector_index(dbFile)
if err ~= nil then
print(string.format("failed to load vector index: %s", err))
return
end
searcher, err = xdb.new_with_vector_index(dbFile, v_index)
if err ~= nil then
print(string.format("failed to create vector index searcher: %s", err))
return
end
elseif cachePolicy == "content" then
content, err = xdb.load_content(dbFile)
if err ~= nil then
print(string.format("failed to load xdb content from '%s'", dbFile))
return
end
searcher, err = xdb.new_with_buffer(content)
if err ~= nil then
print(string.format("failed to create content buffer searcher: %s", err))
return
end
else
print(string.format("undefined cache-policy `%s`", cachePolicy))
return
end
-- do the search
print(string.format([[
ip2region xdb searcher test program, cachePolicy: %s
type 'quit' to exit]], cachePolicy))
local region, err = "", nil
local ip_int, s_time, c_time = 0, 0, 0
while ( true ) do
io.write("ip2region>> ");
io.input(io.stdin);
local line = io.read();
if (line == nil) then
break
end
if ( line == "quit" ) then
break
end
ip_int, err = xdb.check_ip(line)
if err ~= nil then
print(string.format("invalid ip address `%s`", line))
goto continue
end
-- do the search
s_time = xdb.now()
region, err = searcher:search(line)
if err ~= nil then
print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count()))
else
c_time = xdb.now() - s_time
print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time))
end
::continue::
end
-- resource cleanup
searcher:close()
if v_index ~= nil then
v_index:close()
end
if content ~= nil then
content:close()
end

97
binding/lua_c/util_test.lua Normal file
View File

@ -0,0 +1,97 @@
-- Copyright 2022 The Ip2Region Authors. All rights reserved.
-- Use of this source code is governed by a Apache2.0-style
-- license that can be found in the LICENSE file.
--
-- ---
-- @Author Lion <chenxin619315@gmail.com>
-- @Date 2022/06/30
-- set the package to load the current xdb_searcher.so
package.path = "./?.lua" .. package.path
package.cpath = "./?.so" .. package.cpath
local xdb = require("xdb_searcher")
---- ip checking testing
print("--- testing check_ip and long2ip ... ")
local ip_list = {
"1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0",
"256.7.12.9", "12.56.78.320", "32.12.45.192", "222.221.220.219",
"192.168.1.101 ", "132.96.12.98a", "x23.12.2.12"
}
local s_time = xdb.now()
for _, ip_src in ipairs(ip_list) do
ip, err = xdb.check_ip(ip_src)
if err ~= nil then
print(string.format("invalid ip address `%s`: %s", ip_src, err))
else
ip_dst = xdb.long2ip(ip)
io.write(string.format("long(%-15s)=%10d, long2ip(%-10d)=%-15s", ip_src, ip, ip, ip_dst))
if ip_src ~= ip_dst then
print(" --[Failed]")
else
print(" --[Ok]")
end
end
end
---- buffer loading test
print("\n--- testing load header ... ")
header, err = xdb.load_header("../../data/ip2region.xdb")
if err ~= nil then
print("failed to load header: ", err)
else
print(string.format("xdb header buffer `%s` loaded", header))
local tpl = [[
header: {
version: %d
index_policy: %d
created_at: %d
start_index_ptr: %d
end_index_ptr: %d
}]]
local t = header:to_table()
print(string.format(tpl,
t["version"], t["index_policy"], t["created_at"], t["start_index_ptr"], t["end_index_ptr"])
)
end
print("\n--- testing load vector index ... ")
v_index, err = xdb.load_vector_index("../../data/ip2region.xdb")
if err ~= nil then
print("failed to load vector index: ", err)
else
print(string.format("xdb vector index buffer `%s` loaded, info={name=%s, type=%d, length=%d}",
v_index, v_index:name(), v_index:type(), v_index:length()))
v_index:close()
end
print("\n--- testing load content buffer ... ")
c_buffer, err = xdb.load_content("../../data/ip2region.xdb")
if err ~= nil then
print("failed to load content: ", err)
else
print(string.format("xdb content buffer `%s` loaded, info={name=%s, type=%d, length=%d}",
c_buffer, c_buffer:name(), c_buffer:type(), c_buffer:length()))
c_buffer:close();
end
print("\n--- testing search ... ")
local ip_str = "1.2.3.4"
searcher, err = xdb.new_with_file_only("../../data/ip2region.xdb")
local t_start = xdb.now()
region, err = searcher:search(ip_str)
local c_time = xdb.now() - t_start
print(string.format("search(%s): {region=%s, io_count: %d, took: %dμs, err=%s}",
ip_str, region, searcher:get_io_count(), c_time, err))
print(string.format("searcher.tostring=%s", searcher))
searcher:close()
print("")
print(string.format("all tests done, elapsed %d μs", xdb.now() - s_time))

519
binding/lua_c/xdb_searcher.c Normal file
View File

@ -0,0 +1,519 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// ---
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/30
#include "stdio.h"
#include "lua.h"
#include "lauxlib.h"
#include "../c/xdb_searcher.h"
#define XDB_SEARCHER_METATABLE_NAME "xdb_searcher_mt"
#define XDB_BUFFER_METATABLE_NAME "xdb_buffer_mt"
#define xdb_header_buffer 1
#define xdb_vector_index_buffer 2
#define xdb_content_buffer 3
// --- xdb buffer interface impl
struct xdb_buffer_entry {
int type; // buffer type
char *name; // buffer name
void *ptr; // buffer ptr
void (*closer) (void *);
};
typedef struct xdb_buffer_entry xdb_buffer_t;
static int lua_xdb_buffer_name(lua_State *L) {
xdb_buffer_t *buffer;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'");
buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME);
lua_pushstring(L, buffer->name);
return 1;
}
static int lua_xdb_buffer_type(lua_State *L) {
xdb_buffer_t *buffer;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'");
buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME);
lua_pushinteger(L, buffer->type);
return 1;
}
static int lua_xdb_buffer_to_table(lua_State *L) {
xdb_buffer_t *buffer;
xdb_header_t *header;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'");
buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME);
lua_newtable(L);
if (buffer->type == xdb_header_buffer) {
header = (xdb_header_t *) buffer->ptr;
lua_pushinteger(L, header->version);
lua_setfield(L, -2, "version");
lua_pushinteger(L, header->index_policy);
lua_setfield(L, -2, "index_policy");
lua_pushinteger(L, header->created_at);
lua_setfield(L, -2, "created_at");
lua_pushinteger(L, header->start_index_ptr);
lua_setfield(L, -2, "start_index_ptr");
lua_pushinteger(L, header->end_index_ptr);
lua_setfield(L, -2, "end_index_ptr");
} else {
// do nothing for now
}
return 1;
}
static int lua_xdb_buffer_length(lua_State *L) {
xdb_buffer_t *buffer;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'");
buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME);
if (buffer->type == xdb_header_buffer) {
lua_pushinteger(L, ((xdb_header_t *) buffer->ptr)->length);
} else if (buffer->type == xdb_vector_index_buffer) {
lua_pushinteger(L, ((xdb_vector_index_t *) buffer->ptr)->length);
} else if (buffer->type == xdb_content_buffer) {
lua_pushinteger(L, ((xdb_content_t *) buffer->ptr)->length);
} else {
lua_pushinteger(L, -1);
}
return 1;
}
static int lua_xdb_buffer_tostring(lua_State *L) {
xdb_buffer_t *buffer;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'");
buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME);
lua_pushfstring(L, "xdb %s buffer object {name: %s, type: %d}", buffer->name, buffer->name, buffer->type);
return 1;
}
static int lua_xdb_buffer_close(lua_State *L) {
xdb_buffer_t *buffer;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'");
buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME);
// check and call the closer
if (buffer->closer != NULL) {
buffer->closer(buffer->ptr);
buffer->closer = NULL;
}
return 0;
}
// module method define, should be access via ':'
static const struct luaL_Reg xdb_buffer_methods[] = {
{"name", lua_xdb_buffer_name},
{"type", lua_xdb_buffer_type},
{"length", lua_xdb_buffer_length},
{"to_table", lua_xdb_buffer_to_table},
{"close", lua_xdb_buffer_close},
{"__gc", lua_xdb_buffer_close},
{"__tostring", lua_xdb_buffer_tostring},
{NULL, NULL},
};
// --- End of xdb buffer
// --- buffer util function
static int lua_xdb_load_header_from_file(lua_State *L) {
const char *db_path;
xdb_header_t *header;
xdb_buffer_t *buffer;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and the xdb file path expected");
db_path = luaL_checkstring(L, 1);
header = xdb_load_header_from_file(db_path);
if (header == NULL) {
lua_pushnil(L);
lua_pushfstring(L, "load header from `%s`", db_path);
return 2;
}
// alloc the buffer.
buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t));
if (buffer == NULL) {
lua_pushnil(L);
lua_pushfstring(L, "failed to alloc xdb buffer entry");
return 2;
}
// init the buffer
buffer->type = xdb_header_buffer;
buffer->name = "header";
buffer->ptr = header;
buffer->closer = xdb_close_header;
// set the metatable of the header buffer object and push onto the stack
luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME);
lua_setmetatable(L, -2);
lua_pushnil(L);
return 2;
}
static int lua_xdb_load_vector_index_from_file(lua_State *L) {
const char *db_path;
xdb_vector_index_t *v_index;
xdb_buffer_t *buffer;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and the xdb path expected");
db_path = luaL_checkstring(L, 1);
v_index = xdb_load_vector_index_from_file(db_path);
if (v_index == NULL) {
lua_pushnil(L);
lua_pushfstring(L, "load vector index from `%s`", db_path);
return 2;
}
// alloc the buffer.
buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t));
if (buffer == NULL) {
lua_pushnil(L);
lua_pushstring(L, "failed to alloc xdb buffer entry");
return 2;
}
// init the buffer
buffer->type = xdb_vector_index_buffer;
buffer->name = "v_index";
buffer->ptr = v_index;
buffer->closer = xdb_close_vector_index;
// set the metatable of the header buffer object and push onto the stack
luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME);
lua_setmetatable(L, -2);
lua_pushnil(L);
return 2;
}
static int lua_xdb_load_content_from_file(lua_State *L) {
const char *db_path;
xdb_content_t *content;
xdb_buffer_t *buffer;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and xdb path expected");
db_path = luaL_checkstring(L, 1);
content = xdb_load_content_from_file(db_path);
if (content == NULL) {
lua_pushnil(L);
lua_pushfstring(L, "load xdb content from `%s`", db_path);
return 2;
}
// alloc the buffer.
buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t));
if (buffer == NULL) {
lua_pushnil(L);
lua_pushstring(L, "failed to alloc xdb buffer entry");
return 2;
}
// init the buffer
buffer->type = xdb_content_buffer;
buffer->name = "content";
buffer->ptr = content;
buffer->closer = xdb_close_content;
// set the metatable of the header buffer object and push onto the stack
luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME);
lua_setmetatable(L, -2);
lua_pushnil(L);
return 2;
}
// --- End of buffer api
// --- xdb searcher api
static int lua_xdb_new_with_file_only(lua_State *L) {
int err;
xdb_searcher_t *searcher;
const char *db_path = NULL;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and xdb file path expected");
// check the db path
db_path = luaL_checkstring(L, 1);
// alloc for the searcher
searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t));
if (searcher == NULL) {
return luaL_error(L, "failed to alloc xdb searcher entry");
}
// init the xdb searcher
err = xdb_new_with_file_only(searcher, db_path);
if (err != 0) {
lua_pushnil(L);
lua_pushfstring(L, "init xdb searcher on `%s`: errcode=%d", db_path, err);
return 2;
}
// push the metatable onto the stack and
// set it as the metatable of the current searcher
luaL_getmetatable(L, XDB_SEARCHER_METATABLE_NAME);
lua_setmetatable(L, -2);
lua_pushnil(L);
return 2;
}
static int lua_xdb_new_with_vector_index(lua_State *L) {
xdb_searcher_t *searcher;
xdb_buffer_t *xBuffer;
const char *db_path;
int err;
luaL_argcheck(L, lua_gettop(L) == 2, 1, "call via '.', xdb file path and vector index buffer expected");
// check the db path and vector index buffer
db_path = luaL_checkstring(L, 1);
xBuffer = luaL_checkudata(L, 2, XDB_BUFFER_METATABLE_NAME);
if (xBuffer->type != xdb_vector_index_buffer) {
return luaL_error(L, "invalid vector index buffer");
}
// alloc the searcher
searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t));
if (searcher == NULL) {
return luaL_error(L, "failed to alloc xdb searcher entry");
}
// init the xdb searcher
err = xdb_new_with_vector_index(searcher, db_path, (xdb_vector_index_t *) xBuffer->ptr);
if (err != 0) {
lua_pushnil(L);
lua_pushfstring(L, "init vector index cached xdb searcher on `%s` with errcode=%d", db_path, err);
return 2;
}
// push the metatable onto the stack and
// set it as the metatable of the current searcher
luaL_getmetatable(L, XDB_SEARCHER_METATABLE_NAME);
lua_setmetatable(L, -2);
lua_pushnil(L);
return 2;
}
static int lua_xdb_new_with_buffer(lua_State *L) {
xdb_searcher_t *searcher;
xdb_buffer_t *xBuffer;
int err;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and xdb content buffer expected");
xBuffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME);
if (xBuffer->type != xdb_content_buffer) {
return luaL_error(L, "invalid xdb content buffer");
}
// alloc the searcher
searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t));
if (searcher == NULL) {
return luaL_error(L, "failed to alloc xdb searcher entry");
}
// init the xdb searcher
err = xdb_new_with_buffer(searcher, (xdb_content_t *) xBuffer->ptr);
if (err != 0) {
lua_pushnil(L);
lua_pushfstring(L, "init content cached xdb searcher with errcode=%d", err);
return 2;
}
// push the metatable onto the stack and
// set it as the metatable of the current searcher
luaL_getmetatable(L, XDB_SEARCHER_METATABLE_NAME);
lua_setmetatable(L, -2);
lua_pushnil(L);
return 2;
}
static int lua_xdb_close(lua_State *L) {
xdb_searcher_t *searcher;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'");
searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_SEARCHER_METATABLE_NAME);
if (searcher == NULL) {
return luaL_error(L, "broken xdb searcher instance");
}
xdb_close(searcher);
return 0;
}
static int lua_xdb_search(lua_State *L) {
int err;
const char *ip_str;
unsigned int ip_int;
char region_buffer[1024] = {'\0'};
xdb_searcher_t *searcher;
luaL_argcheck(L, lua_gettop(L) == 2, 2, "call via ':' and string or long ip address expected");
// get the searcher
searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_SEARCHER_METATABLE_NAME);
// input ip type checking
if (lua_isinteger(L, 2)) {
ip_int = lua_tointeger(L, 2);
} else if (lua_isstring(L, 2)) {
ip_str = lua_tostring(L, 2);
err = xdb_check_ip(ip_str, &ip_int);
if (err != 0) {
lua_pushnil(L);
lua_pushfstring(L, "invalid string ip `%s`: errcode=%d", ip_str, err);
return 2;
}
} else {
return luaL_error(L, "input ip not integer or string");
}
// do the search
err = xdb_search(searcher, ip_int, region_buffer, sizeof(region_buffer));
if (err != 0) {
lua_pushinteger(L, err);
lua_pushfstring(L, "err=%d", err);
} else {
lua_pushstring(L, region_buffer);
lua_pushnil(L);
}
return 2;
}
static int lua_xdb_get_io_count(lua_State *L) {
xdb_searcher_t *searcher;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':' or xdb searcher was broken");
searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_SEARCHER_METATABLE_NAME);
lua_pushinteger(L, xdb_get_io_count(searcher));
return 1;
}
static int lua_xdb_tostring(lua_State *L) {
lua_pushliteral(L, "xdb searcher object");
return 1;
}
// -- static util function
static int lua_xdb_check_ip(lua_State *L) {
int err;
unsigned int ip;
const char *ip_str;
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and string ip expected, eg: 1.2.3.4");
ip_str = luaL_checkstring(L, 1);
err = xdb_check_ip(ip_str, &ip);
if (err != 0) {
lua_pushinteger(L, 0);
lua_pushfstring(L, "err=%d", err);
} else {
lua_pushinteger(L, ip);
lua_pushnil(L);
}
return 2;
}
static int lua_xdb_long2ip(lua_State *L) {
unsigned int ip;
char ip_buff[16] = {'\0'};
luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and long ip expected");
ip = luaL_checkinteger(L, 1);
xdb_long2ip(ip, ip_buff);
lua_pushstring(L, ip_buff);
return 1;
}
static int lua_xdb_now(lua_State *L) {
lua_pushinteger(L, xdb_now());
return 1;
}
// --- End of static util
// module method define, should be access via ':'
static const struct luaL_Reg xdb_searcher_methods[] = {
{"search", lua_xdb_search},
{"get_io_count",lua_xdb_get_io_count},
{"close", lua_xdb_close},
{"__gc", lua_xdb_close},
{"__tostring", lua_xdb_tostring},
{NULL, NULL},
};
// module function define, should be access via '.'
static const struct luaL_Reg xdb_searcher_functions[] = {
{"new_with_file_only", lua_xdb_new_with_file_only},
{"new_with_vector_index", lua_xdb_new_with_vector_index},
{"new_with_buffer", lua_xdb_new_with_buffer},
{"load_header", lua_xdb_load_header_from_file},
{"load_vector_index", lua_xdb_load_vector_index_from_file},
{"load_content", lua_xdb_load_content_from_file},
{"check_ip", lua_xdb_check_ip},
{"long2ip", lua_xdb_long2ip},
{"now", lua_xdb_now},
{NULL, NULL}
};
// module register function
int luaopen_xdb_searcher(lua_State *L)
{
// register the constants
lua_pushinteger(L, xdb_header_buffer);
lua_setglobal(L, "header_buffer");
lua_pushinteger(L, xdb_vector_index_buffer);
lua_setglobal(L, "v_index_buffer");
lua_pushinteger(L, xdb_content_buffer);
lua_setglobal(L, "content_buffer");
// create a metatable for xdb buffer object
luaL_newmetatable(L, XDB_BUFFER_METATABLE_NAME);
lua_pushvalue(L, -1);
lua_setfield(L, -2, "__index");
luaL_setfuncs(L, xdb_buffer_methods, 0);
// create a metatable for xdb searcher object
luaL_newmetatable(L, XDB_SEARCHER_METATABLE_NAME);
lua_pushvalue(L, -1);
lua_setfield(L, -2, "__index");
luaL_setfuncs(L, xdb_searcher_methods, 0);
luaL_setfuncs(L, xdb_searcher_functions, 0);
return 1;
}

2
binding/php/ReadMe.md
View File

@ -2,6 +2,8 @@
# 使用方式
第三方 composer 地址: [https://github.com/chinayin/ip2region-core-php](https://github.com/chinayin/ip2region-core-php)
### 完全基于文件的查询
```php
$dbFile = "ip2region.xdb file path";

18
maker/golang/make.bat Normal file
View File

@ -0,0 +1,18 @@
::ip2region golang maker makefile in windows
@echo off
if [%1] == [] goto:build
if %1==clean (
call:clean
) else if %1==build (
call:build
)
exit /b 0
:build
go build -o xdb_maker.exe
exit /b 0
:clean
del/f/s/q xdb_maker.exe

18
maker/golang/xdb/util.go
View File

@ -5,7 +5,6 @@
package xdb
import (
"encoding/binary"
"fmt"
"strconv"
"strings"
@ -13,13 +12,15 @@ import (
// Util function
var shiftIndex = []int{24, 16, 8, 0}
func CheckIP(ip string) (uint32, error) {
var ps = strings.Split(ip, ".")
if len(ps) != 4 {
return 0, fmt.Errorf("invalid ip address `%s`", ip)
}
var buff = make([]byte, 4)
var val uint32
for i, s := range ps {
d, err := strconv.Atoi(s)
if err != nil {
@ -30,21 +31,14 @@ func CheckIP(ip string) (uint32, error) {
return 0, fmt.Errorf("the %dth part `%s` should be an integer bettween 0 and 255", i, s)
}
buff[i] = byte(d)
val |= uint32(d) << shiftIndex[i]
}
// convert the ip to integer
return binary.BigEndian.Uint32(buff), nil
return val, nil
}
func Long2IP(ip uint32) string {
var buff = make([]string, 4)
buff[0] = fmt.Sprintf("%d", (ip>>24)&0xFF)
buff[1] = fmt.Sprintf("%d", (ip>>16)&0xFF)
buff[2] = fmt.Sprintf("%d", (ip>>8)&0xFF)
buff[3] = fmt.Sprintf("%d", (ip>>0)&0xFF)
return strings.Join(buff, ".")
return fmt.Sprintf("%d.%d.%d.%d", (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, (ip>>0)&0xFF)
}
func MidIP(sip uint32, eip uint32) uint32 {

30
maker/golang/xdb/util_test.go
View File

@ -5,10 +5,40 @@
package xdb
import (
"encoding/binary"
"fmt"
"net"
"testing"
)
func TestCheckIP(t *testing.T) {
var str = "29.34.191.255"
ip, err := CheckIP(str)
if err != nil {
t.Errorf("check ip `%s`: %s\n", str, err)
}
netIP := net.ParseIP(str).To4()
if netIP == nil {
t.Fatalf("parse ip `%s` failed", str)
}
u32 := binary.BigEndian.Uint32(netIP)
fmt.Printf("checkip: %d, parseip: %d, isEqual: %v\n", ip, u32, ip == u32)
}
func TestLong2IP(t *testing.T) {
var str = "29.34.191.255"
netIP := net.ParseIP(str).To4()
if netIP == nil {
t.Fatalf("parse ip `%s` failed", str)
}
u32 := binary.BigEndian.Uint32(netIP)
ipStr := Long2IP(u32)
fmt.Printf("originIP: %s, Long2IP: %s, isEqual: %v\n", str, ipStr, ipStr == str)
}
func TestSplitSegment(t *testing.T) {
// var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信"
// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"