diff --git a/.gitignore b/.gitignore index a167fe3..ce243fa 100644 --- a/.gitignore +++ b/.gitignore @@ -37,10 +37,14 @@ META-INF/ /binding/java/target/ /binding/java/*.jar +# clang /binding/c/xdb_searcher /binding/c/util_test /binding/c/cmake-build-debug +# lua/luc_c +/binding/lua_c/cmake-build-debug + # golang /binding/golang/searcher /binding/golang/xdb_searcher diff --git a/ReadMe.md b/ReadMe.md index dfac983..3ccba55 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -37,9 +37,9 @@ API 介绍，使用文档和测试程序请参考对应 `searcher` 查询客户 | [golang](binding/golang) | golang 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) | | [php](binding/php) | php 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) | | [java](binding/java) | java 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) | -| [lua](binding/lua) | lua 查询客户端实现 | 开发中... | [Lion](https://github.com/lionsoul2014) | +| [lua](binding/lua) | lua 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) | | [c](binding/c) | ANSC c 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) | -| [lua_c](binding/lua_c) | lua 查询客户端实现 | 开发中... | [Lion](https://github.com/lionsoul2014) | +| [lua_c](binding/lua_c) | lua 查询客户端实现 | 已完成 | [Lion](https://github.com/lionsoul2014) | | [rust](binding/rust) | rust 查询客户端实现 | 开发中... | [Lion](https://github.com/lionsoul2014) | | [python](binding/python) | python 查询客户端实现 | 待开始 | 待确定 | | [nodejs](binding/nodejs) | nodejs 查询客户端实现 | 待开始 | 待确定 | @@ -64,7 +64,7 @@ API 介绍，使用文档和测试程序请参考对应 `maker` 生成程序下 # 并发查询必读 -全部查询客户端的 search 接口都 不是 并发安全的实现，不同进程/线程/携程需要通过创建不同的查询对象来安全使用，并发量很大的情况下，基于文件查询的方式可能会导致打开文件数过多的错误，请修改内核的最大允许打开文件数(fs.file-max=一个更高的值)，或者将整个xdb加载到内存进行安全并发使用。 +全部查询客户端的 search 接口都 不是 并发安全的实现，不同进程/线程/协程需要通过创建不同的查询对象来安全使用，并发量很大的情况下，基于文件查询的方式可能会导致打开文件数过多的错误，请修改内核的最大允许打开文件数(fs.file-max=一个更高的值)，或者将整个xdb加载到内存进行安全并发使用。 # 相关备注 diff --git a/binding/c/ReadMe.md b/binding/c/ReadMe.md index 0fb5117..591f905 100644 --- a/binding/c/ReadMe.md +++ b/binding/c/ReadMe.md @@ -49,7 +49,7 @@ int main(int argc, char *argv[]) { int main(int argc, char *argv[]) { char *db_path = "ip2region.xdb file path"; - char *v_index; + xdb_vector_index_t *v_index; xdb_searcher_t searcher; char region_buffer[256], ip_buffer[16], *ip = "1.2.3.4"; long s_time; @@ -84,7 +84,7 @@ int main(int argc, char *argv[]) { // 4、关闭 xdb 查询器，如果是要关闭服务，也需要释放 v_index 的内存。 xdb_close(&searcher); - xdb_free(v_index); + xdb_close_vector_index(v_index); return 0; } ``` @@ -98,7 +98,7 @@ int main(int argc, char *argv[]) { int main(int argc, char *argv[]) { char *db_path = "ip2region.xdb file path"; - char *c_buffer; + xdb_content_t *c_buffer; xdb_searcher_t searcher; char region_buffer[256], ip_buffer[16], *ip = "1.2.3.4"; long s_time; @@ -132,7 +132,7 @@ int main(int argc, char *argv[]) { // 4、关闭 xdb 查询器，关闭服务的时候需要释放 c_buffer 的内存。 xdb_close(&searcher); - xdb_free(c_buffer); + xdb_close_content(c_buffer); return 0; } ``` diff --git a/binding/c/main.c b/binding/c/main.c index 309ba85..49a0f13 100644 --- a/binding/c/main.c +++ b/binding/c/main.c @@ -11,8 +11,8 @@ struct searcher_test_entry { xdb_searcher_t searcher; - char *v_index; - char *c_buffer; + xdb_vector_index_t *v_index; + xdb_content_t *c_buffer; }; typedef struct searcher_test_entry searcher_test_t; @@ -64,13 +64,13 @@ void destroy_searcher_test(searcher_test_t *test) { // check and free the vector index if (test->v_index != NULL) { - xdb_free(test->v_index); + xdb_close_vector_index(test->v_index); test->v_index = NULL; } // check and free the content buffer if (test->c_buffer != NULL) { - xdb_free(test->c_buffer); + xdb_close_content(test->c_buffer); test->c_buffer = NULL; } } diff --git a/binding/c/util_test.c b/binding/c/util_test.c index 3a35230..f6828d2 100644 --- a/binding/c/util_test.c +++ b/binding/c/util_test.c @@ -37,10 +37,9 @@ void test_check_ip() { } void test_load_header() { - xdb_header_t header; - int err = xdb_load_header_from_file("../../data/ip2region.xdb", &header); - if (err != 0) { - printf("failed to load header with errcode=%d\n", err); + xdb_header_t *header = xdb_load_header_from_file("../../data/ip2region.xdb"); + if (header == NULL) { + printf("failed to load header"); } else { printf("header loaded: {\n" " version: %d, \n" @@ -48,33 +47,36 @@ void test_load_header() { " created_at: %u, \n" " start_index_ptr: %d, \n" " end_index_ptr: %d\n" + " length: %d\n" "}\n", - header.version, header.index_policy, header.created_at, - header.start_index_ptr, header.end_index_ptr + header->version, header->index_policy, header->created_at, + header->start_index_ptr, header->end_index_ptr, header->length ); } + + xdb_close_header(header); } void test_load_vector_index() { - char *ptr = xdb_load_vector_index_from_file("../../data/ip2region.xdb"); - if (ptr == NULL) { + xdb_vector_index_t *v_index = xdb_load_vector_index_from_file("../../data/ip2region.xdb"); + if (v_index == NULL) { printf("failed to load vector index from file\n"); } else { - printf("vector index loaded from file\n"); + printf("vector index loaded from file, length=%d\n", v_index->length); } - xdb_free(ptr); + xdb_close_vector_index(v_index); } void test_load_content() { - char *ptr = xdb_load_content_from_file("../../data/ip2region.xdb"); - if (ptr == NULL) { + xdb_content_t *content = xdb_load_content_from_file("../../data/ip2region.xdb"); + if (content == NULL) { printf("failed to load content from file\n"); } else { - printf("content loaded from file\n"); + printf("content loaded from file, length=%d\n", content->length); } - xdb_free(ptr); + xdb_close_content(content); } // valgrind --tool=memcheck --leak-check=full ./a.out diff --git a/binding/c/xdb_searcher.c b/binding/c/xdb_searcher.c index 452aadd..6f8b0d2 100644 --- a/binding/c/xdb_searcher.c +++ b/binding/c/xdb_searcher.c @@ -12,13 +12,13 @@ // internal function prototype define XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length); -XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const char *vIndex, const char *cBuff) { +XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { memset(xdb, 0x00, sizeof(xdb_searcher_t)); // check the content buffer first - if (cBuff != NULL) { - xdb->vector_index = NULL; - xdb->content_buff = cBuff; + if (c_buffer != NULL) { + xdb->v_index = NULL; + xdb->content = c_buffer; return 0; } @@ -29,27 +29,29 @@ XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const ch } xdb->handle = handle; - xdb->vector_index = vIndex; + xdb->v_index = v_index; return 0; } // xdb searcher new api define -XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, char *db_path) { +XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) { return xdb_new_base(xdb, db_path, NULL, NULL); } -XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, char *db_path, char *vIndex) { - return xdb_new_base(xdb, db_path, vIndex, NULL); +XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) { + return xdb_new_base(xdb, db_path, v_index, NULL); } -XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, char *c_buffer) { +XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) { return xdb_new_base(xdb, NULL, NULL, c_buffer); } -XDB_PUBLIC(void) xdb_close(xdb_searcher_t *xdb) { +XDB_PUBLIC(void) xdb_close(void *ptr) { + xdb_searcher_t *xdb = (xdb_searcher_t *) ptr; if (xdb->handle != NULL) { fclose(xdb->handle); + xdb->handle = NULL; } } @@ -77,12 +79,12 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu il0 = ((int) (ip >> 24)) & 0xFF; il1 = ((int) (ip >> 16)) & 0xFF; idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; - if (xdb->vector_index != NULL) { - s_ptr = xdb_get_uint(xdb->vector_index, idx); - e_ptr = xdb_get_uint(xdb->vector_index, idx + 4); - } else if (xdb->content_buff != NULL) { - s_ptr = xdb_get_uint(xdb->content_buff, xdb_header_info_length + idx); - e_ptr = xdb_get_uint(xdb->content_buff, xdb_header_info_length + idx + 4); + if (xdb->v_index != NULL) { + s_ptr = xdb_get_uint(xdb->v_index->buffer, idx); + e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4); + } else if (xdb->content != NULL) { + s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx); + e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4); } else { err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); if (err != 0) { @@ -146,8 +148,8 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) { // check the xdb content cache first - if (xdb->content_buff != NULL) { - memcpy(buffer, xdb->content_buff + offset, length); + if (xdb->content != NULL) { + memcpy(buffer, xdb->content->buffer + offset, length); return 0; } @@ -171,39 +173,59 @@ XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) { // --- buffer load util functions -XDB_PUBLIC(int) xdb_load_header(FILE *handle, xdb_header_t *header) { - char buffer[256]; +XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *handle) { + xdb_header_t *header; + unsigned int size = xdb_header_info_length; - if (fseek(handle, 0, SEEK_SET) == -1) { - return 1; + // entry alloc + header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t)); + if (header == NULL) { + return NULL; } - if (fread(buffer, 1, 256, handle) != 256) { - return 2; + if (fseek(handle, 0, SEEK_SET) == -1) { + xdb_free(header); + return NULL; + } + + if (fread(header->buffer, 1,size, handle) != size) { + xdb_free(header); + return NULL; } // fill the fields - header->version = (unsigned short) xdb_get_ushort(buffer, 0); - header->index_policy = (unsigned short) xdb_get_ushort(buffer, 2); - header->created_at = xdb_get_uint(buffer, 4); - header->start_index_ptr = xdb_get_uint(buffer, 8); - header->end_index_ptr = xdb_get_uint(buffer,12); + header->length = size; + header->version = (unsigned short) xdb_get_ushort(header->buffer, 0); + header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2); + header->created_at = xdb_get_uint(header->buffer, 4); + header->start_index_ptr = xdb_get_uint(header->buffer, 8); + header->end_index_ptr = xdb_get_uint(header->buffer,12); - return 0; + return header; } -XDB_PUBLIC(int) xdb_load_header_from_file(char *db_path, xdb_header_t *header) { +XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *db_path) { FILE *handle = fopen(db_path, "r"); if (handle == NULL) { - return 10; + return NULL; } - return xdb_load_header(handle, header); + return xdb_load_header(handle); } -XDB_PUBLIC(char *) xdb_load_vector_index(FILE *handle) { - char *ptr = NULL; - int size = xdb_vector_index_length; +XDB_PUBLIC(void) xdb_close_header(void *ptr) { + xdb_header_t *header = (xdb_header_t *) ptr; + if (header->length > 0) { + header->length = 0; + xdb_free(header); + } +} + +// --- vector index + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *handle) { + xdb_vector_index_t *v_index; + unsigned int size = xdb_vector_index_length; // seek to the vector index offset if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) { @@ -211,20 +233,21 @@ XDB_PUBLIC(char *) xdb_load_vector_index(FILE *handle) { } // do the buffer read - ptr = (char *) xdb_malloc(size); - if (ptr == NULL) { + v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t)); + if (v_index == NULL) { return NULL; } - if (fread(ptr, 1, size, handle) != size) { - xdb_free(ptr); + v_index->length = size; + if (fread(v_index->buffer, 1, size, handle) != size) { + xdb_free(v_index); return NULL; } - return ptr; + return v_index; } -XDB_PUBLIC(char *) xdb_load_vector_index_from_file(char *db_path) { +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *db_path) { FILE *handle = fopen(db_path, "r"); if (handle == NULL) { return NULL; @@ -233,36 +256,55 @@ XDB_PUBLIC(char *) xdb_load_vector_index_from_file(char *db_path) { return xdb_load_vector_index(handle); } -XDB_PUBLIC(char *) xdb_load_content(FILE *handle) { - long filesize; - char *ptr = NULL; +XDB_PUBLIC(void) xdb_close_vector_index(void *ptr) { + xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr; + if (v_index->length > 0) { + v_index->length = 0; + xdb_free(v_index); + } +} + +// --- content buffer + +XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *handle) { + unsigned int size; + xdb_content_t *content; + char *ptr; // determine the file size if (fseek(handle, 0, SEEK_END) == -1) { return NULL; } - filesize = ftell(handle); + size = (unsigned int) ftell(handle); if (fseek(handle, 0, SEEK_SET) == -1) { return NULL; } // do the file read - ptr = (char *) xdb_malloc(filesize); - if (ptr == NULL) { + content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t)); + if (content == NULL) { + return NULL; + } + + // do the buffer alloc + content->buffer = (char *) xdb_malloc(size); + if (content->buffer == NULL) { + xdb_free(content); return NULL; } // read the content into the buffer - if (fread(ptr, 1, filesize, handle) != filesize) { + content->length = size; + if (fread(content->buffer, 1, size, handle) != size) { xdb_free(ptr); return NULL; } - return ptr; + return content; } -XDB_PUBLIC(char *) xdb_load_content_from_file(char *db_path) { +XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *db_path) { FILE *handle = fopen(db_path, "r"); if (handle == NULL) { return NULL; @@ -271,6 +313,16 @@ XDB_PUBLIC(char *) xdb_load_content_from_file(char *db_path) { return xdb_load_content(handle); } +XDB_PUBLIC(void) xdb_close_content(void *ptr) { + xdb_content_t *content = (xdb_content_t *) ptr; + if (content->length > 0) { + content->length = 0; + xdb_free(content->buffer); + content->buffer = NULL; + xdb_free(content); + } +} + // --- End // get unsigned long (4bytes) from a specified buffer start from the specified offset @@ -339,4 +391,4 @@ XDB_PUBLIC(long) xdb_now() { struct timeval c_time; gettimeofday(&c_time, NULL); return c_time.tv_sec * (int)1e6 + c_time.tv_usec; -} \ No newline at end of file +} diff --git a/binding/c/xdb_searcher.h b/binding/c/xdb_searcher.h index 13b7725..12808bc 100644 --- a/binding/c/xdb_searcher.h +++ b/binding/c/xdb_searcher.h @@ -38,6 +38,59 @@ #define xdb_vector_index_length 524288 +// --- buffer load util functions + +// use the following buffer struct to wrap the binary buffer data +// since the buffer data could not be operated with the string API. +struct xdb_header { + unsigned short version; + unsigned short index_policy; + unsigned int created_at; + unsigned int start_index_ptr; + unsigned int end_index_ptr; + + // the original buffer + unsigned int length; + char buffer[xdb_header_info_length]; +}; +typedef struct xdb_header xdb_header_t; + +XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *); + +XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_header(void *); + + +// --- vector index buffer +struct xdb_vector_index { + unsigned int length; + char buffer[xdb_vector_index_length]; +}; +typedef struct xdb_vector_index xdb_vector_index_t; + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *); + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_vector_index(void *); + + +// --- content buffer +struct xdb_content { + unsigned int length; + char *buffer; +}; +typedef struct xdb_content xdb_content_t; + +XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *); + +XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_content(void *); + +// --- End buffer load + // xdb searcher structure struct xdb_searcher_entry { FILE *handle; @@ -49,22 +102,22 @@ struct xdb_searcher_entry { // vector index buffer cache. // preload the vector index will reduce the number of IO operations // thus speedup the search process. - const char *vector_index; + const xdb_vector_index_t *v_index; // content buffer. // cache the whole xdb content. - const char *content_buff; + const xdb_content_t *content; }; typedef struct xdb_searcher_entry xdb_searcher_t; // xdb searcher new api define -XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, char *); +XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, const char *); -XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, char *, char *); +XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *); -XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, char *); +XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *); -XDB_PUBLIC(void) xdb_close(xdb_searcher_t *); +XDB_PUBLIC(void) xdb_close(void *); // xdb searcher search api define XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t); @@ -74,32 +127,6 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, unsigned int, char *, size_t); XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *); -// --- buffer load util functions - -struct xdb_header { - unsigned short version; - unsigned short index_policy; - unsigned int created_at; - unsigned int start_index_ptr; - unsigned int end_index_ptr; -}; -typedef struct xdb_header xdb_header_t; - -XDB_PUBLIC(int) xdb_load_header(FILE *, xdb_header_t *); - -XDB_PUBLIC(int) xdb_load_header_from_file(char *, xdb_header_t *); - -XDB_PUBLIC(char *) xdb_load_vector_index(FILE *); - -XDB_PUBLIC(char *) xdb_load_vector_index_from_file(char *); - -XDB_PUBLIC(char *) xdb_load_content(FILE *); - -XDB_PUBLIC(char *) xdb_load_content_from_file(char *); - -// --- End buffer load - - // get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian XDB_PUBLIC(unsigned int) xdb_get_uint(const char *, int); diff --git a/binding/golang/xdb/util.go b/binding/golang/xdb/util.go index 49eb20f..7cb4cb9 100644 --- a/binding/golang/xdb/util.go +++ b/binding/golang/xdb/util.go @@ -9,20 +9,21 @@ package xdb import ( - "encoding/binary" "fmt" "os" "strconv" "strings" ) +var shiftIndex = []int{24, 16, 8, 0} + func CheckIP(ip string) (uint32, error) { var ps = strings.Split(ip, ".") if len(ps) != 4 { return 0, fmt.Errorf("invalid ip address `%s`", ip) } - var buff = make([]byte, 4) + var val = uint32(0) for i, s := range ps { d, err := strconv.Atoi(s) if err != nil { @@ -33,20 +34,15 @@ func CheckIP(ip string) (uint32, error) { return 0, fmt.Errorf("the %dth part `%s` should be an integer bettween 0 and 255", i, s) } - buff[i] = byte(d) + val |= uint32(d) << shiftIndex[i] } // convert the ip to integer - return binary.BigEndian.Uint32(buff), nil + return val, nil } func Long2IP(ip uint32) string { - var buff = make([]string, 4) - buff[0] = fmt.Sprintf("%d", (ip>>24)&0xFF) - buff[1] = fmt.Sprintf("%d", (ip>>16)&0xFF) - buff[2] = fmt.Sprintf("%d", (ip>>8)&0xFF) - buff[3] = fmt.Sprintf("%d", (ip>>0)&0xFF) - return strings.Join(buff, ".") + return fmt.Sprintf("%d.%d.%d.%d", (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, ip&0xFF) } func MidIP(sip uint32, eip uint32) uint32 { diff --git a/binding/golang/xdb/util_test.go b/binding/golang/xdb/util_test.go index 18624a1..bea7711 100644 --- a/binding/golang/xdb/util_test.go +++ b/binding/golang/xdb/util_test.go @@ -9,11 +9,41 @@ package xdb import ( + "encoding/binary" "fmt" + "net" "testing" "time" ) +func TestCheckIP(t *testing.T) { + var str = "29.34.191.255" + ip, err := CheckIP(str) + if err != nil { + t.Errorf("check ip `%s`: %s\n", str, err) + } + + netIP := net.ParseIP(str).To4() + if netIP == nil { + t.Fatalf("parse ip `%s` failed", str) + } + + u32 := binary.BigEndian.Uint32(netIP) + fmt.Printf("checkip: %d, parseip: %d, isEqual: %v\n", ip, u32, ip == u32) +} + +func TestLong2IP(t *testing.T) { + var str = "29.34.191.255" + netIP := net.ParseIP(str).To4() + if netIP == nil { + t.Fatalf("parse ip `%s` failed", str) + } + + u32 := binary.BigEndian.Uint32(netIP) + ipStr := Long2IP(u32) + fmt.Printf("originIP: %s, Long2IP: %s, isEqual: %v\n", str, ipStr, ipStr == str) +} + func TestLoadVectorIndex(t *testing.T) { vIndex, err := LoadVectorIndexFromFile("../../../data/ip2region.xdb") if err != nil { diff --git a/binding/java/ReadMe.md b/binding/java/ReadMe.md index 35dc7e8..f6e06f7 100644 --- a/binding/java/ReadMe.md +++ b/binding/java/ReadMe.md @@ -7,7 +7,7 @@ org.lionsoul ip2region - 2.6.2 + 2.6.4 ``` @@ -34,7 +34,7 @@ public class SearcherTest { try { String ip = "1.2.3.4"; long sTime = System.nanoTime(); - String region = searcher.searchByStr(ip); + String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { @@ -80,7 +80,7 @@ public class SearcherTest { try { String ip = "1.2.3.4"; long sTime = System.nanoTime(); - String region = searcher.searchByStr(ip); + String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { @@ -126,14 +126,14 @@ public class SearcherTest { try { String ip = "1.2.3.4"; long sTime = System.nanoTime(); - String region = searcher.searchByStr(ip); + String region = searcher.search(ip); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } - // 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去垮线程访问。 + // 备注：并发使用，用整个 xdb 数据缓存创建的查询对象可以安全的用于并发，也就是你可以把这个 searcher 对象做成全局对象去跨线程访问。 } } ``` diff --git a/binding/java/pom.xml b/binding/java/pom.xml index 60b69b5..5b2ad96 100644 --- a/binding/java/pom.xml +++ b/binding/java/pom.xml @@ -4,7 +4,7 @@ org.lionsoul ip2region - 2.6.2 + 2.6.4 jar ip2region diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java b/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java index 179cb9f..5c70266 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java @@ -89,7 +89,7 @@ public class SearchTest { try { double sTime = System.nanoTime(); - String region = searcher.searchByStr(line); + String region = searcher.search(line); long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java index 8239639..8ff0680 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java @@ -15,6 +15,7 @@ public class Header { public final int createdAt; public final int startIndexPtr; public final int endIndexPtr; + public final byte[] buffer; public Header(byte[] buff) { assert buff.length >= 16; @@ -23,6 +24,7 @@ public class Header { createdAt = Searcher.getInt(buff, 4); startIndexPtr = Searcher.getInt(buff, 8); endIndexPtr = Searcher.getInt(buff, 12); + buffer = buff; } @Override public String toString() { diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java index 869b288..5aecbc8 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java @@ -71,7 +71,7 @@ public class Searcher { return ioCount; } - public String searchByStr(String ipStr) throws Exception { + public String search(String ipStr) throws Exception { long ip = checkIP(ipStr); return search(ip); } @@ -135,7 +135,7 @@ public class Searcher { // load and return the region data final byte[] regionBuff = new byte[dataLen]; read(dataPtr, regionBuff); - return new String(regionBuff); + return new String(regionBuff, "utf-8"); } protected void read(int offset, byte[] buffer) throws IOException { diff --git a/binding/lua/ReadMe.md b/binding/lua/ReadMe.md index 1b46687..c3a97a3 100644 --- a/binding/lua/ReadMe.md +++ b/binding/lua/ReadMe.md @@ -1,7 +1,153 @@ -# ip2region lua 查询客户端实现 +# ip2region xdb lua 查询客户端实现 + +#### 备注：请优先使用 lua_c 扩展 xdb 查询客户端，性能比纯 lua 实现的要快很多！！！ + # 使用方式 +### 完全基于文件的查询 +```lua +local xdb = require("xdb_searcher") + +-- 1、从 db_path 创建基于文件的 xdb 查询对象 +local db_path = "ip2region.xdb file path" +local searcher, err = xdb.new_with_file_only(db_path) +if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return +end + +-- 2、调用查询 API 进行查询 +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + +### 缓存 `VectorIndex` 索引 + +如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。 +```lua +local xdb = require("xdb_searcher") + +local db_path = "ip2region.xdb file path" + +-- 1、从指定的 db_path 加载 VectorIndex 缓存，把下述的 v_index 对象做成全局变量。 +-- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。 +v_index, err = xdb.load_vector_index(db_path) +if err ~= nil then + print(string.format("failed to load vector index from '%s'", db_path)) + return +end + +-- 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。 +searcher, err = xdb.new_with_vector_index(db_path, v_index) +if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return +end + +-- 3、调用查询 API +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + +### 缓存整个 `xdb` 数据 + +如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。 +```lua +local xdb = require("xdb_searcher") + +local db_path = "ip2region.xdb file path" + +-- 1、从指定的 db_path 加载整个 xdb 到内存。 +-- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。 +content = xdb.load_content(db_path) +if content == nil then + print(string.format("failed to load xdb content from '%s'", db_path)) + return +end + +-- 2、使用全局的 content 创建带完全基于内存的查询对象。 +searcher, err = xdb.new_with_buffer(content) +if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return +end + +-- 3、调用查询 API +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。 +-- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + + # 查询测试 +通过 `search_test.lua` 脚本来进行查询测试： +```bash +➜ lua git:(lua_binding) ✗ lua search_test.lua +lua search_test.lua [command options] +options: + --db string ip2region binary xdb file path + --cache-policy string cache policy: file/vectorIndex/content +``` + +例如：使用默认的 data/ip2region.xdb 进行查询测试： +```bash +➜ lua git:(lua_binding) ✗ lua search_test.lua --db=../../data/ip2region.xdb --cache-policy=vectorIndex +ip2region xdb searcher test program, cachePolicy: vectorIndex +type 'quit' to exit +ip2region>> 1.2.3.4 +{region: 美国|0|华盛顿|0|谷歌, io_count: 7, took: 0μs} +ip2region>> +``` + +输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。 + + # bench 测试 + +通过 `bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能： +```bash +➜ lua git:(lua_binding) ✗ lua bench_test.lua +lua bench_test.lua [command options] +options: + --db string ip2region binary xdb file path + --src string source ip text file path + --cache-policy string cache policy: file/vectorIndex/content +``` + +例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试： +```bash +➜ lua git:(lua_binding) ✗ lua bench_test.lua --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=vectorIndex +Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 29.000 s, cost: 7.899 μs/op} +``` + +可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。 +@Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。 diff --git a/binding/lua/bench_test.lua b/binding/lua/bench_test.lua new file mode 100644 index 0000000..91c9010 --- /dev/null +++ b/binding/lua/bench_test.lua @@ -0,0 +1,175 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/06/30 + +-- set the package to load the current xdb_searcher.so +package.path = "./?.lua" .. package.path +package.cpath = "./?.so" .. package.cpath +local xdb = require("xdb_searcher") + +function printHelp() + print("lua bench_test.lua [command options]") + print("options: ") + print(" --db string ip2region binary xdb file path") + print(" --src string source ip text file path") + print(" --cache-policy string cache policy: file/vectorIndex/content") +end + +if #arg < 2 then + printHelp(arg) + return +end + +-- parser the command line args +local dbFile, srcFile = "", "" +local cachePolicy = "vectorIndex" +for _, r in ipairs(arg) do + if string.len(r) < 5 then + goto continue + end + + if string.sub(r, 1, 2) ~= "--" then + goto continue + end + + for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do + if k == "db" then + dbFile = v + elseif k == "src" then + srcFile = v + elseif k == "cache-policy" then + cachePolicy = v + else + print(string.format("undefined option `%s`", r)) + return + end + + -- break the match iterate + break + end + + -- continue this loop + ::continue:: +end + +-- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy)) +if string.len(dbFile) < 2 or string.len(srcFile) < 2 then + printHelp() + return +end + +-- create the searcher based on the cache-policy +local searcher, v_index, content +if cachePolicy == "file" then + searcher, err = xdb.new_with_file_only(dbFile) + if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return + end +elseif cachePolicy == "vectorIndex" then + v_index, err = xdb.load_vector_index(dbFile) + if err ~= nil then + print(string.format("failed to load vector index: %s", err)) + return + end + + searcher, err = xdb.new_with_vector_index(dbFile, v_index) + if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return + end +elseif cachePolicy == "content" then + content, err = xdb.load_content(dbFile) + if err ~= nil then + print(string.format("failed to load xdb content: %s", err)) + return + end + + searcher, err = xdb.new_with_buffer(content) + if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return + end +else + print(string.format("undefined cache-policy `%s`", cachePolicy)) + return +end + +-- do the bench test +local handle = io.open(srcFile, "r") +if handle == nil then + print(string.format("failed to open src text file `%s`", handle)) + return +end + +local lines = handle:lines() +local sip_str, eip_str, s_region, region = "", "", "", "" +local sip, mip, eip, err = 0, 0, 0, 0 +local count, t_time, c_time = 0, 0, 0 +local s_time = xdb.now() +for l in lines do + if string.len(l) < 1 then + goto continue + end + + for v1, v2, v3 in string.gmatch(l, "([%d%.]+)|([%d%.]+)|([^\n]+)") do + -- print(sip_str, eip_str, region) + sip_str = v1 + eip_str = v2 + s_region = v3 + break + end + + sip, err = xdb.check_ip(sip_str) + if err ~= nil then + print(string.format("invalid start ip `%s`", sip_str)) + return + end + + eip, err = xdb.check_ip(eip_str) + if err ~= nil then + print(string.format("invalid end ip `%s`", sip_str)) + return + end + + if sip > eip then + print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str)) + return + end + + mip = (sip + eip) >> 1 + for _, ip in ipairs({sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) do + t_time = xdb.now() + region, err = searcher:search(ip) + c_time = c_time + xdb.now() - t_time + if err ~= nil then + print(string.format("failed to search ip `%s`", xdb.long2ip(ip))) + return + end + + -- check the region + if region ~= s_region then + print(string.format("failed search(%s) with (%s != %s)\n", xdb.long2ip(ip), region, s_region)) + return + end + + count = count + 1 + end + + ::continue:: +end + +-- resource cleanup +searcher:close() + +-- print the stats +local avg_costs = 0 +if count > 0 then + avg_costs = c_time / count +end +print(string.format("Bench finished, {cachePolicy: %s, total: %d, took: %.3f s, cost: %.3f μs/op}", + cachePolicy, count, (xdb.now() - s_time)/1e6, c_time / count)) diff --git a/binding/lua/search_test.lua b/binding/lua/search_test.lua new file mode 100644 index 0000000..24363a5 --- /dev/null +++ b/binding/lua/search_test.lua @@ -0,0 +1,137 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/06/30 + +-- set the package to load the current xdb_searcher.so +package.path = "./?.lua" .. package.path +package.cpath = "./?.so" .. package.cpath +local xdb = require("xdb_searcher") + +function printHelp() + print("lua search_test.lua [command options]") + print("options: ") + print(" --db string ip2region binary xdb file path") + print(" --cache-policy string cache policy: file/vectorIndex/content") +end + +if #arg < 2 then + printHelp(arg) + return +end + +-- parser the command line args +local dbFile = "" +local cachePolicy = "vectorIndex" +for _, r in ipairs(arg) do + if string.len(r) < 5 then + goto continue + end + + if string.sub(r, 1, 2) ~= "--" then + goto continue + end + + for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do + if k == "db" then + dbFile = v + elseif k == "cache-policy" then + cachePolicy = v + else + print(string.format("undefined option `%s`", r)) + return + end + + -- break the match iterate + break + end + + -- continue this loop + ::continue:: +end + +-- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy)) +if string.len(dbFile) < 2 then + printHelp() + return +end + +-- create the searcher based on the cache-policy +local searcher, v_index, content +if cachePolicy == "file" then + searcher, err = xdb.new_with_file_only(dbFile) + if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return + end +elseif cachePolicy == "vectorIndex" then + v_index, err = xdb.load_vector_index(dbFile) + if err ~= nil then + print(string.format("failed to load vector index: %s", err)) + return + end + + searcher, err = xdb.new_with_vector_index(dbFile, v_index) + if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return + end +elseif cachePolicy == "content" then + content, err = xdb.load_content(dbFile) + if err ~= nil then + print(string.format("failed to load xdb content: %s", err)) + return + end + + searcher, err = xdb.new_with_buffer(content) + if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return + end +else + print(string.format("undefined cache-policy `%s`", cachePolicy)) + return +end + +-- do the search +print(string.format([[ +ip2region xdb searcher test program, cachePolicy: %s +type 'quit' to exit]], cachePolicy)) +local region, err = "", nil +local ip_int, s_time, c_time = 0, 0, 0 +while ( true ) do + io.write("ip2region>> "); + io.input(io.stdin); + local line = io.read(); + if (line == nil) then + break + end + + if ( line == "quit" ) then + break + end + + ip_int, err = xdb.check_ip(line) + if err ~= nil then + print(string.format("invalid ip address `%s`", line)) + goto continue + end + + -- do the search + s_time = xdb.now() + region, err = searcher:search(line) + if err ~= nil then + print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count())) + else + c_time = xdb.now() - s_time + print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time)) + end + + ::continue:: +end + +-- resource cleanup +searcher:close() \ No newline at end of file diff --git a/binding/lua/util_test.lua b/binding/lua/util_test.lua new file mode 100644 index 0000000..a3df35e --- /dev/null +++ b/binding/lua/util_test.lua @@ -0,0 +1,98 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/07/05 + +-- set the package path +package.path = "./?.lua" +package.cpath = "./?.so" + +local xdb = require("xdb_searcher") + +---- ip checking testing +print("--- testing check_ip and long2ip ... ") +local ip_list = { + "1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0", + "256.7.12.9", "12.56.78.320", "32.12.45.192", "222.221.220.219", + "192.168.1.101 ", "132.96.12.98a", "x23.12.2.12" +} + +local s_time = xdb.now() +for _, ip_src in ipairs(ip_list) do + ip, err = xdb.check_ip(ip_src) + if err ~= nil then + print(string.format("invalid ip address `%s`: %s", ip_src, err)) + else + ip_dst = xdb.long2ip(ip) + io.write(string.format("long(%-15s)=%10d, long2ip(%-10d)=%-15s", ip_src, ip, ip, ip_dst)) + if ip_src ~= ip_dst then + print(" --[Failed]") + else + print(" --[Ok]") + end + end +end + +---- buffer loading test +print("\n--- testing load header ... ") +header, err = xdb.load_header("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load header: ", err) +else + print("xdb header buffer loaded") + + local tpl = [[ +header: { + version: %d + index_policy: %d + created_at: %d + start_index_ptr: %d + end_index_ptr: %d +}]] + + print(string.format(tpl, + header["version"], header["index_policy"], + header["created_at"], header["start_index_ptr"], header["end_index_ptr"]) + ) +end + + +print("\n--- testing load vector index ... ") +v_index, err = xdb.load_vector_index("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load vector index: ", err) +else + print("xdb vector index buffer loaded") +end + + +print("\n--- testing load content buffer ... ") +c_buffer, err = xdb.load_content("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load content: ", err) +else + print("xdb content buffer loaded") +end + + +print("\n--- testing search ... ") +local ip_str = "1.2.3.4" +searcher, err = xdb.new_with_file_only("../../data/ip2region.xdb") +local t_start = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("search(%s) failed: %s", ip_str, err)) +else + local c_time = xdb.now() - t_start + print(string.format("search(%s): {region=%s, io_count: %d, took: %dμs, err=%s}", + ip_str, region, searcher:get_io_count(), c_time, err)) + print(string.format("searcher.tostring=%s", searcher)) +end +searcher:close() + + +print("") +print(string.format("all tests done, elapsed %d μs", xdb.now() - s_time)) \ No newline at end of file diff --git a/binding/lua/xdb_searcher.lua b/binding/lua/xdb_searcher.lua new file mode 100644 index 0000000..289b414 --- /dev/null +++ b/binding/lua/xdb_searcher.lua @@ -0,0 +1,333 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/07/05 + +-- constants define +local HeaderInfoLength = 256 +local VectorIndexRows = 256 +local VectorIndexCols = 256 +local VectorIndexSize = 8 +local SegmentIndexSize = 14 +local VectorIndexLength = 524288 + +local _M = { + -- xdb file handle + handle = nil, + + -- header info + header = nil, + io_count = 0, + + -- vector index + vector_index = nil, + + -- xdb content buffer + content_buff = nil +} + +-- index and to string attribute set +_M.__index = _M +_M.__tostring = function(self) + return "xdb searcher object (lua)" +end + + +-- construct functions + +function newBase(dbPath, vIndex, cBuffer) + local obj = setmetatable({}, _M) + if cBuffer ~= nil then + obj.io_count = 0 + obj.vector_index = nil + obj.content_buff = cBuffer + else + obj.io_count = 0 + obj.vector_index = vIndex + obj.handle = io.open(dbPath, "r") + if obj.handle == nil then + return nil, string.format("failed to open xdb file `%s`", dbPath) + end + end + + return obj, nil +end + +function _M.new_with_file_only(dbPath) + return newBase(dbPath, nil, nil) +end + +function _M.new_with_vector_index(dbPath, vIndex) + return newBase(dbPath, vIndex, nil) +end + +function _M.new_with_buffer(cBuffer) + return newBase(nil, nil, cBuffer) +end + +-- End of constructors + +-- object api impl, must call via ':' + +function _M:search(ip_src) + -- check and convert string ip to long ip + local t, ip = type(ip_src), 0 + if t == nil then + return "", string.format("invalid ip address `%s`", ip_src) + elseif t == "string" then + ip, err = self.check_ip(ip_src) + if err ~= nil then + return "", string.format("check ip `%s`: %s", ip_src, err) + end + elseif t ~= "number" then + return "", "invalid number or string ip" + else + -- use the original value + ip = ip_src + end + + -- reset the global counter + -- and global resource local cache + self.io_count = 0 + local vector_index = self.vector_index + local content_buff = self.content_buff + local read_data = self.read + + -- locate the segment index based on the vector index + local il0 = (ip >> 24) & 0xFF + local il1 = (ip >> 16) & 0xFF + local idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize + local s_ptr, e_ptr = 0, 0 + if vector_index ~= nil then + s_ptr = getLong(vector_index, idx + 1) + e_ptr = getLong(vector_index, idx + 5) + elseif content_buff ~= nil then + s_ptr = getLong(content_buff, HeaderInfoLength + idx + 1) + e_ptr = getLong(content_buff, HeaderInfoLength + idx + 5) + else + -- load from the file + buff, err = read_data(self, HeaderInfoLength + idx, SegmentIndexSize) + if err ~= nil then + return "", string.format("read buffer: %s", err) + end + + s_ptr = getLong(buff, 1) + e_ptr = getLong(buff, 5) + end + + -- print(string.format("s_ptr: %d, e_ptr: %d", s_ptr, e_ptr)) + -- binary search to get the data + local data_ptr, data_len, p = 0, 0, 0 + local sip, eip, err, buff = 0, 0, "" + local l, m, h = 0, 0, (e_ptr - s_ptr) / SegmentIndexSize + while l <= h do + m = (l + h) >> 1 + p = s_ptr + m * SegmentIndexSize + + -- read the segment index + buff, err = read_data(self, p, SegmentIndexSize) + if err ~= nil then + return "", string.format("read segment index at %d", p) + end + + sip = getLong(buff, 1) + if ip < sip then + h = m - 1 + else + eip = getLong(buff, 5) + if ip > eip then + l = m + 1 + else + data_len = getShort(buff, 9) + data_ptr = getLong(buff, 11) + break + end + end + end + + -- matching nothing interception + -- print(string.format("data_len=%d, data_ptr=%d", data_len, data_ptr)) + if data_len == 0 then + return "", nil + end + + -- load and return the region data + buff, err = read_data(self, data_ptr, data_len) + if err ~= nil then + return "", string.format("read data at %d:%d", data_ptr, data_len) + end + + return buff, nil +end + + +-- read specified bytes from the specified index + +function _M:read(offset, length) + -- local cache + local content_buff = self.content_buff + local handle = self.handle + + -- check the in-memory buffer first + if content_buff ~= nil then + return string.sub(content_buff, offset + 1, offset + length), nil + end + + -- read from the file + local r = handle:seek("set", offset) + if r == nil then + return nil, string.format("seek to offset %d", offset) + end + + self.io_count = self.io_count + 1 + local buff = handle:read(length) + if buff == nil then + return nil, string.format("read %d bytes", length) + end + + return buff, nil +end + +function _M:get_io_count() + return self.io_count +end + +function _M:close() + if self.handle ~= nil then + self.handle:close() + end +end + +-- End of search api + + +-- static util functions + +function _M.load_header(dbPath) + local handle = io.open(dbPath, "r") + if handle == nil then + return nil, string.format("failed to open xdb file `%s`", dbPath) + end + + local r = handle:seek("set", 0) + if r == nil then + handle:close() + return nil, "failed to seek to 0" + end + + local c = handle:read(HeaderInfoLength) + if c == nil then + handle:close() + return nil, string.format("failed to read %d bytes", HeaderInfoLength) + end + + handle:close() + return { + ["version"] = getShort(c, 1), + ["index_policy"] = getShort(c, 3), + ["created_at"] = getLong(c, 5), + ["start_index_ptr"] = getLong(c, 9), + ["end_index_ptr"] = getLong(c, 13), + ["raw_data"] = c + }, nil +end + +function _M.load_vector_index(dbPath) + local handle = io.open(dbPath, "r") + if handle == nil then + return nil, string.format("failed to open xdb file `%s`", dbPath) + end + + local r = handle:seek("set", HeaderInfoLength) + if r == nil then + handle:close() + return nil, string.format("failed to seek to %d", HeaderInfoLength) + end + + local c = handle:read(VectorIndexLength) + if c == nil then + handle:close() + return nil, string.format("failed to read %d bytes", VectorIndexLength) + end + + handle:close() + return c, nil +end + +function _M.load_content(dbPath) + local handle = io.open(dbPath, "r") + if handle == nil then + return nil, string.format("failed to open xdb file `%s`", dbPath) + end + + local c = handle:read("*a") + if c == nil then + return nil, string.format("failed to read xdb content") + end + + handle:close() + return c, nil +end + +function _M.check_ip(ip_str) + local ip, id, v = 0, 1, 0 + local offset_arr = {24, 16, 8, 0} + for p in string.gmatch(ip_str..".", "([%d]+)%.") do + -- match pattern checking + if p == nil then + return 0, "err=1" + end + + -- count checking + if id > 4 then + return 0, "err=1" + end + + -- value checking + v = tonumber(p) + if v > 255 then + return 0, "err=2" + end + + ip = ip | (v << offset_arr[id]) + id = id + 1 + end + + if id ~= 5 then + return 0, "err=1" + end + + return ip, nil +end + +function _M.long2ip(ip) + return string.format("%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8 ) & 0xFF, ip & 0xFF) +end + +-- this is a bit weird, but we have to better choice for now +function _M.now() + return os.time() * 1e6 +end + +-- End of util functions + +--internal function to get a integer from a binary string + +function getLong(buff, idx) + local i1 = (string.byte(string.sub(buff, idx, idx))) + local i2 = (string.byte(string.sub(buff, idx+1, idx+1)) << 8) + local i3 = (string.byte(string.sub(buff, idx+2, idx+2)) << 16) + local i4 = (string.byte(string.sub(buff, idx+3, idx+3)) << 24) + return (i1 | i2 | i3 | i4) +end + +function getShort(buff, idx) + local i1 = (string.byte(string.sub(buff, idx, idx))) + local i2 = (string.byte(string.sub(buff, idx+1, idx+1)) << 8) + return (i1 | i2) +end + +return _M \ No newline at end of file diff --git a/binding/lua_c/Makefile b/binding/lua_c/Makefile new file mode 100644 index 0000000..cd0f1ff --- /dev/null +++ b/binding/lua_c/Makefile @@ -0,0 +1,16 @@ +LuaVersion = 5.3 +LIB_DIR = /usr/local/share/lua/$(LuaVersion) + +all: ../c/xdb_searcher.h ../c/xdb_searcher.c xdb_searcher.c + gcc -g -O2 -I../c/ -I/usr/include/lua$(LuaVersion) ../c/xdb_searcher.c xdb_searcher.c -fPIC -shared -o xdb_searcher.so + +install: + sudo mkdir -p $(LIB_DIR); \ + sudo cp xdb_searcher.so $(LIB_DIR);\ + echo "install xdb searcher to $(LIB_DIR) successfully.";\ + +clean: + find . -name \*.so | xargs rm -f + find . -name \*.o | xargs rm -f + +.PHONY: clean \ No newline at end of file diff --git a/binding/lua_c/ReadMe.md b/binding/lua_c/ReadMe.md index b830ebe..f1f211e 100644 --- a/binding/lua_c/ReadMe.md +++ b/binding/lua_c/ReadMe.md @@ -1,7 +1,162 @@ -# ip2region lua c 扩展查询客户端实现 +# ip2region xdb lua c 扩展查询客户端实现 + +# 编译安装 + +通过如下方式来编译安装该扩展： +```bash +# cd 到 lua_c binding 的根目录 +make +sudo make install +``` + +备注：Makefile 里面的 `LuaVersion` 变量代表你本地环境的 lua 的版本，默认为 5.3，如果你的版本不是 5.3，记得先修改为和你本地 lua 版本一致的版本号。 + # 使用方式 +### 完全基于文件的查询 +```lua +local xdb = require("xdb_searcher") + +-- 1、从 db_path 创建基于文件的 xdb 查询对象 +local db_path = "ip2region.xdb file path" +local searcher, err = xdb.new_with_file_only(db_path) +if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return +end + +-- 2、调用查询 API 进行查询 +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + +### 缓存 `VectorIndex` 索引 + +如果你的 `lua` 母环境支持，可以预先加载 vectorIndex 缓存，然后做成全局变量，每次创建 Searcher 的时候使用全局的 vectorIndex，可以减少一次固定的 IO 操作从而加速查询，减少 io 压力。 +```lua +local xdb = require("xdb_searcher") + +local db_path = "ip2region.xdb file path" + +-- 1、从指定的 db_path 加载 VectorIndex 缓存，把下述的 v_index 对象做成全局变量。 +-- vectorIndex 加载一次即可，建议在服务启动的时候加载为全局对象。 +v_index, err = xdb.load_vector_index(db_path) +if err ~= nil then + print(string.format("failed to load vector index from '%s'", db_path)) + return +end + +-- 2、使用全局的 v_index 创建带 VectorIndex 缓存的查询对象。 +searcher, err = xdb.new_with_vector_index(db_path, v_index) +if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return +end + +-- 3、调用查询 API +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，每个协程需要创建单独的 xdb 查询对象，但是共享全局的 v_index 对象 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + +### 缓存整个 `xdb` 数据 + +如果你的 `lua` 母环境支持，可以预先加载整个 xdb 的数据到内存，这样可以实现完全基于内存的查询，类似之前的 memory search 查询。 +```lua +local xdb = require("xdb_searcher") + +local db_path = "ip2region.xdb file path" + +-- 1、从指定的 db_path 加载整个 xdb 到内存。 +-- xdb内容加载一次即可，建议在服务启动的时候加载为全局对象。 +content = xdb.load_content(db_path) +if content == nil then + print(string.format("failed to load xdb content from '%s'", db_path)) + return +end + +-- 2、使用全局的 content 创建带完全基于内存的查询对象。 +searcher, err = xdb.new_with_buffer(content) +if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return +end + +-- 3、调用查询 API +local ip_str = "1.2.3.4" +local s_time = xdb.now() +region, err = searcher:search(ip_str) +if err ~= nil then + print(string.format("failed to search(%s): %s", ip_str, err)) + return +end + +-- 备注：并发使用，用 xdb 整个缓存创建的查询对象可以安全的用于并发。 +-- 建议在服务启动的时候创建好全局的 searcher 对象，然后全局并发使用。 + +print(string.format("{region: %s, took: %.5f μs}", region, xdb.now() - s_time)) +``` + + # 查询测试 +通过 `search_test.lua` 脚本来进行查询测试： +```bash +➜ lua_c git:(lua_binding) ✗ lua ./search_test.lua +lua search_test.lua [command options] +options: + --db string ip2region binary xdb file path + --cache-policy string cache policy: file/vectorIndex/content +``` + +例如：使用默认的 data/ip2region.xdb 进行查询测试： +```bash +➜ lua_c git:(lua_binding) ✗ lua ./search_test.lua --db=../../data/ip2region.xdb --cache-policy=vectorIndex +ip2region xdb searcher test program, cachePolicy: vectorIndex +type 'quit' to exit +ip2region>> 1.2.3.4 +{region: 美国|0|华盛顿|0|谷歌, io_count: 7, took: 15μs} +ip2region>> +``` + +输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。 + + # bench 测试 + +通过 `bench_test.lua` 脚本来进行自动 bench 测试，一方面确保 `xdb` 文件没有错误，另一方面通过大量的查询测试平均查询性能： +```bash +➜ lua_c git:(lua_binding) ✗ lua ./bench_test.lua +lua bench_test.lua [command options] +options: + --db string ip2region binary xdb file path + --src string source ip text file path + --cache-policy string cache policy: file/vectorIndex/content +``` + +例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试： +```bash +➜ lua_c git:(lua_binding) ✗ lua ./bench_test.lua --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=vectorIndex +Bench finished, {cachePolicy: vectorIndex, total: 3417955, took: 5.865 s, cost: 1.399 μs/op} +``` + +可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。 +@Note：请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。 diff --git a/binding/lua_c/bench_test.lua b/binding/lua_c/bench_test.lua new file mode 100644 index 0000000..af1df2c --- /dev/null +++ b/binding/lua_c/bench_test.lua @@ -0,0 +1,181 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/06/30 + +-- set the package to load the current xdb_searcher.so +package.path = "./?.lua" .. package.path +package.cpath = "./?.so" .. package.cpath +local xdb = require("xdb_searcher") + +function printHelp() + print("lua bench_test.lua [command options]") + print("options: ") + print(" --db string ip2region binary xdb file path") + print(" --src string source ip text file path") + print(" --cache-policy string cache policy: file/vectorIndex/content") +end + +if #arg < 2 then + printHelp(arg) + return +end + +-- parser the command line args +local dbFile, srcFile = "", "" +local cachePolicy = "vectorIndex" +for _, r in ipairs(arg) do + if string.len(r) < 5 then + goto continue + end + + if string.sub(r, 1, 2) ~= "--" then + goto continue + end + + for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do + if k == "db" then + dbFile = v + elseif k == "src" then + srcFile = v + elseif k == "cache-policy" then + cachePolicy = v + else + print(string.format("undefined option `%s`", r)) + return + end + + -- break the match iterate + break + end + + -- continue this loop + ::continue:: +end + +-- print(string.format("dbFile=%s, srcFile=%s, cachePolicy=%s", dbFile, srcFile, cachePolicy)) +if string.len(dbFile) < 2 or string.len(srcFile) < 2 then + printHelp() + return +end + +-- create the searcher based on the cache-policy +local searcher, v_index, content +if cachePolicy == "file" then + searcher, err = xdb.new_with_file_only(dbFile) + if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return + end +elseif cachePolicy == "vectorIndex" then + v_index, err = xdb.load_vector_index(dbFile) + if err ~= nil then + print(string.format("failed to load vector index: %s", err)) + return + end + + searcher, err = xdb.new_with_vector_index(dbFile, v_index) + if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return + end +elseif cachePolicy == "content" then + content, err = xdb.load_content(dbFile) + if err ~= nil then + print(string.format("failed to load xdb content from '%s'", dbFile)) + return + end + + searcher, err = xdb.new_with_buffer(content) + if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return + end +else + print(string.format("undefined cache-policy `%s`", cachePolicy)) + return +end + +-- do the bench test +local handle = io.open(srcFile, "r") +if handle == nil then + print(string.format("failed to open src text file `%s`", handle)) + return +end + +local lines = handle:lines() +local sip_str, eip_str, s_region, region = "", "", "", "" +local sip, mip, eip, err = 0, 0, 0, 0 +local count, t_time, c_time = 0, 0, 0 +local s_time = xdb.now() +for l in lines do + if string.len(l) < 1 then + goto continue + end + + for v1, v2, v3 in string.gmatch(l, "([%d%.]+)|([%d%.]+)|([^\n]+)") do + -- print(sip_str, eip_str, region) + sip_str = v1 + eip_str = v2 + s_region = v3 + break + end + + sip, err = xdb.check_ip(sip_str) + if err ~= nil then + print(string.format("invalid start ip `%s`", sip_str)) + return + end + + eip, err = xdb.check_ip(eip_str) + if err ~= nil then + print(string.format("invalid end ip `%s`", sip_str)) + return + end + + if sip > eip then + print(string.format("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str)) + return + end + + mip = (sip + eip) >> 1 + for _, ip in ipairs({sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) do + t_time = xdb.now() + region, err = searcher:search(ip) + c_time = c_time + xdb.now() - t_time + if err ~= nil then + print(string.format("failed to search ip `%s`", xdb.long2ip(ip))) + return + end + + -- check the region + if region ~= s_region then + printf(string.format("failed search(%s) with (%s != %s)\n", xdb.long2ip(ip), region, s_region)) + return + end + + count = count + 1 + end + + ::continue:: +end + +-- resource cleanup +searcher:close() +if v_index ~= nil then + v_index:close() +end +if content ~= nil then + content:close() +end + +-- print the stats +local avg_costs = 0 +if count > 0 then + avg_costs = c_time / count +end +print(string.format("Bench finished, {cachePolicy: %s, total: %d, took: %.3f s, cost: %.3f μs/op}", + cachePolicy, count, (xdb.now() - s_time)/1e6, c_time / count)) \ No newline at end of file diff --git a/binding/lua_c/search_test.lua b/binding/lua_c/search_test.lua new file mode 100644 index 0000000..0622d59 --- /dev/null +++ b/binding/lua_c/search_test.lua @@ -0,0 +1,143 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/06/30 + +-- set the package to load the current xdb_searcher.so +package.path = "./?.lua" .. package.path +package.cpath = "./?.so" .. package.cpath +local xdb = require("xdb_searcher") + +function printHelp() + print("lua search_test.lua [command options]") + print("options: ") + print(" --db string ip2region binary xdb file path") + print(" --cache-policy string cache policy: file/vectorIndex/content") +end + +if #arg < 2 then + printHelp(arg) + return +end + +-- parser the command line args +local dbFile = "" +local cachePolicy = "vectorIndex" +for _, r in ipairs(arg) do + if string.len(r) < 5 then + goto continue + end + + if string.sub(r, 1, 2) ~= "--" then + goto continue + end + + for k, v in string.gmatch(string.sub(r, 3), "([^=]+)=([^%s]+)") do + if k == "db" then + dbFile = v + elseif k == "cache-policy" then + cachePolicy = v + else + print(string.format("undefined option `%s`", r)) + return + end + + -- break the match iterate + break + end + + -- continue this loop + ::continue:: +end + +-- print(string.format("dbFile=%s, cachePolicy=%s", dbFile, cachePolicy)) +if string.len(dbFile) < 2 then + printHelp() + return +end + +-- create the searcher based on the cache-policy +local searcher, v_index, content +if cachePolicy == "file" then + searcher, err = xdb.new_with_file_only(dbFile) + if err ~= nil then + print(string.format("failed to create searcher: %s", err)) + return + end +elseif cachePolicy == "vectorIndex" then + v_index, err = xdb.load_vector_index(dbFile) + if err ~= nil then + print(string.format("failed to load vector index: %s", err)) + return + end + + searcher, err = xdb.new_with_vector_index(dbFile, v_index) + if err ~= nil then + print(string.format("failed to create vector index searcher: %s", err)) + return + end +elseif cachePolicy == "content" then + content, err = xdb.load_content(dbFile) + if err ~= nil then + print(string.format("failed to load xdb content from '%s'", dbFile)) + return + end + + searcher, err = xdb.new_with_buffer(content) + if err ~= nil then + print(string.format("failed to create content buffer searcher: %s", err)) + return + end +else + print(string.format("undefined cache-policy `%s`", cachePolicy)) + return +end + +-- do the search +print(string.format([[ +ip2region xdb searcher test program, cachePolicy: %s +type 'quit' to exit]], cachePolicy)) +local region, err = "", nil +local ip_int, s_time, c_time = 0, 0, 0 +while ( true ) do + io.write("ip2region>> "); + io.input(io.stdin); + local line = io.read(); + if (line == nil) then + break + end + + if ( line == "quit" ) then + break + end + + ip_int, err = xdb.check_ip(line) + if err ~= nil then + print(string.format("invalid ip address `%s`", line)) + goto continue + end + + -- do the search + s_time = xdb.now() + region, err = searcher:search(line) + if err ~= nil then + print(string.format("{err: %s, io_count: %d}", err, searcher:get_io_count())) + else + c_time = xdb.now() - s_time + print(string.format("{region: %s, io_count: %d, took: %dμs}", region, searcher:get_io_count(), c_time)) + end + + ::continue:: +end + +-- resource cleanup +searcher:close() +if v_index ~= nil then + v_index:close() +end +if content ~= nil then + content:close() +end \ No newline at end of file diff --git a/binding/lua_c/util_test.lua b/binding/lua_c/util_test.lua new file mode 100644 index 0000000..5a481c5 --- /dev/null +++ b/binding/lua_c/util_test.lua @@ -0,0 +1,97 @@ +-- Copyright 2022 The Ip2Region Authors. All rights reserved. +-- Use of this source code is governed by a Apache2.0-style +-- license that can be found in the LICENSE file. +-- +-- --- +-- @Author Lion +-- @Date 2022/06/30 + +-- set the package to load the current xdb_searcher.so +package.path = "./?.lua" .. package.path +package.cpath = "./?.so" .. package.cpath +local xdb = require("xdb_searcher") + +---- ip checking testing +print("--- testing check_ip and long2ip ... ") +local ip_list = { + "1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0", + "256.7.12.9", "12.56.78.320", "32.12.45.192", "222.221.220.219", + "192.168.1.101 ", "132.96.12.98a", "x23.12.2.12" +} + +local s_time = xdb.now() +for _, ip_src in ipairs(ip_list) do + ip, err = xdb.check_ip(ip_src) + if err ~= nil then + print(string.format("invalid ip address `%s`: %s", ip_src, err)) + else + ip_dst = xdb.long2ip(ip) + io.write(string.format("long(%-15s)=%10d, long2ip(%-10d)=%-15s", ip_src, ip, ip, ip_dst)) + if ip_src ~= ip_dst then + print(" --[Failed]") + else + print(" --[Ok]") + end + end +end + +---- buffer loading test +print("\n--- testing load header ... ") +header, err = xdb.load_header("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load header: ", err) +else + print(string.format("xdb header buffer `%s` loaded", header)) + + local tpl = [[ +header: { + version: %d + index_policy: %d + created_at: %d + start_index_ptr: %d + end_index_ptr: %d +}]] + + local t = header:to_table() + print(string.format(tpl, + t["version"], t["index_policy"], t["created_at"], t["start_index_ptr"], t["end_index_ptr"]) + ) +end + + +print("\n--- testing load vector index ... ") +v_index, err = xdb.load_vector_index("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load vector index: ", err) +else + print(string.format("xdb vector index buffer `%s` loaded, info={name=%s, type=%d, length=%d}", + v_index, v_index:name(), v_index:type(), v_index:length())) + v_index:close() +end + + +print("\n--- testing load content buffer ... ") +c_buffer, err = xdb.load_content("../../data/ip2region.xdb") +if err ~= nil then + print("failed to load content: ", err) +else + print(string.format("xdb content buffer `%s` loaded, info={name=%s, type=%d, length=%d}", + c_buffer, c_buffer:name(), c_buffer:type(), c_buffer:length())) + c_buffer:close(); +end + + +print("\n--- testing search ... ") +local ip_str = "1.2.3.4" +searcher, err = xdb.new_with_file_only("../../data/ip2region.xdb") +local t_start = xdb.now() +region, err = searcher:search(ip_str) +local c_time = xdb.now() - t_start +print(string.format("search(%s): {region=%s, io_count: %d, took: %dμs, err=%s}", + ip_str, region, searcher:get_io_count(), c_time, err)) +print(string.format("searcher.tostring=%s", searcher)) +searcher:close() + + +print("") +print(string.format("all tests done, elapsed %d μs", xdb.now() - s_time)) \ No newline at end of file diff --git a/binding/lua_c/xdb_searcher.c b/binding/lua_c/xdb_searcher.c new file mode 100644 index 0000000..59268ad --- /dev/null +++ b/binding/lua_c/xdb_searcher.c @@ -0,0 +1,519 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +// --- +// @Author Lion +// @Date 2022/06/30 + + +#include "stdio.h" +#include "lua.h" +#include "lauxlib.h" +#include "../c/xdb_searcher.h" + +#define XDB_SEARCHER_METATABLE_NAME "xdb_searcher_mt" +#define XDB_BUFFER_METATABLE_NAME "xdb_buffer_mt" + +#define xdb_header_buffer 1 +#define xdb_vector_index_buffer 2 +#define xdb_content_buffer 3 + + +// --- xdb buffer interface impl + +struct xdb_buffer_entry { + int type; // buffer type + char *name; // buffer name + void *ptr; // buffer ptr + void (*closer) (void *); +}; +typedef struct xdb_buffer_entry xdb_buffer_t; + +static int lua_xdb_buffer_name(lua_State *L) { + xdb_buffer_t *buffer; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); + buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); + + lua_pushstring(L, buffer->name); + return 1; +} + +static int lua_xdb_buffer_type(lua_State *L) { + xdb_buffer_t *buffer; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); + buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); + + lua_pushinteger(L, buffer->type); + return 1; +} + +static int lua_xdb_buffer_to_table(lua_State *L) { + xdb_buffer_t *buffer; + xdb_header_t *header; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); + buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); + + lua_newtable(L); + if (buffer->type == xdb_header_buffer) { + header = (xdb_header_t *) buffer->ptr; + lua_pushinteger(L, header->version); + lua_setfield(L, -2, "version"); + + lua_pushinteger(L, header->index_policy); + lua_setfield(L, -2, "index_policy"); + + lua_pushinteger(L, header->created_at); + lua_setfield(L, -2, "created_at"); + + lua_pushinteger(L, header->start_index_ptr); + lua_setfield(L, -2, "start_index_ptr"); + + lua_pushinteger(L, header->end_index_ptr); + lua_setfield(L, -2, "end_index_ptr"); + } else { + // do nothing for now + } + + return 1; +} + +static int lua_xdb_buffer_length(lua_State *L) { + xdb_buffer_t *buffer; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); + buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); + + if (buffer->type == xdb_header_buffer) { + lua_pushinteger(L, ((xdb_header_t *) buffer->ptr)->length); + } else if (buffer->type == xdb_vector_index_buffer) { + lua_pushinteger(L, ((xdb_vector_index_t *) buffer->ptr)->length); + } else if (buffer->type == xdb_content_buffer) { + lua_pushinteger(L, ((xdb_content_t *) buffer->ptr)->length); + } else { + lua_pushinteger(L, -1); + } + + return 1; +} + +static int lua_xdb_buffer_tostring(lua_State *L) { + xdb_buffer_t *buffer; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); + buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); + + lua_pushfstring(L, "xdb %s buffer object {name: %s, type: %d}", buffer->name, buffer->name, buffer->type); + return 1; +} + +static int lua_xdb_buffer_close(lua_State *L) { + xdb_buffer_t *buffer; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); + buffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); + + // check and call the closer + if (buffer->closer != NULL) { + buffer->closer(buffer->ptr); + buffer->closer = NULL; + } + + return 0; +} + +// module method define, should be access via ':' +static const struct luaL_Reg xdb_buffer_methods[] = { + {"name", lua_xdb_buffer_name}, + {"type", lua_xdb_buffer_type}, + {"length", lua_xdb_buffer_length}, + {"to_table", lua_xdb_buffer_to_table}, + {"close", lua_xdb_buffer_close}, + {"__gc", lua_xdb_buffer_close}, + {"__tostring", lua_xdb_buffer_tostring}, + {NULL, NULL}, +}; + +// --- End of xdb buffer + + +// --- buffer util function + +static int lua_xdb_load_header_from_file(lua_State *L) { + const char *db_path; + xdb_header_t *header; + xdb_buffer_t *buffer; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and the xdb file path expected"); + db_path = luaL_checkstring(L, 1); + header = xdb_load_header_from_file(db_path); + if (header == NULL) { + lua_pushnil(L); + lua_pushfstring(L, "load header from `%s`", db_path); + return 2; + } + + // alloc the buffer. + buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t)); + if (buffer == NULL) { + lua_pushnil(L); + lua_pushfstring(L, "failed to alloc xdb buffer entry"); + return 2; + } + + // init the buffer + buffer->type = xdb_header_buffer; + buffer->name = "header"; + buffer->ptr = header; + buffer->closer = xdb_close_header; + + // set the metatable of the header buffer object and push onto the stack + luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME); + lua_setmetatable(L, -2); + lua_pushnil(L); + + return 2; +} + +static int lua_xdb_load_vector_index_from_file(lua_State *L) { + const char *db_path; + xdb_vector_index_t *v_index; + xdb_buffer_t *buffer; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and the xdb path expected"); + db_path = luaL_checkstring(L, 1); + v_index = xdb_load_vector_index_from_file(db_path); + if (v_index == NULL) { + lua_pushnil(L); + lua_pushfstring(L, "load vector index from `%s`", db_path); + return 2; + } + + // alloc the buffer. + buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t)); + if (buffer == NULL) { + lua_pushnil(L); + lua_pushstring(L, "failed to alloc xdb buffer entry"); + return 2; + } + + // init the buffer + buffer->type = xdb_vector_index_buffer; + buffer->name = "v_index"; + buffer->ptr = v_index; + buffer->closer = xdb_close_vector_index; + + // set the metatable of the header buffer object and push onto the stack + luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME); + lua_setmetatable(L, -2); + lua_pushnil(L); + + return 2; +} + +static int lua_xdb_load_content_from_file(lua_State *L) { + const char *db_path; + xdb_content_t *content; + xdb_buffer_t *buffer; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and xdb path expected"); + db_path = luaL_checkstring(L, 1); + content = xdb_load_content_from_file(db_path); + if (content == NULL) { + lua_pushnil(L); + lua_pushfstring(L, "load xdb content from `%s`", db_path); + return 2; + } + + // alloc the buffer. + buffer = (xdb_buffer_t *) lua_newuserdata(L, sizeof(xdb_buffer_t)); + if (buffer == NULL) { + lua_pushnil(L); + lua_pushstring(L, "failed to alloc xdb buffer entry"); + return 2; + } + + // init the buffer + buffer->type = xdb_content_buffer; + buffer->name = "content"; + buffer->ptr = content; + buffer->closer = xdb_close_content; + + // set the metatable of the header buffer object and push onto the stack + luaL_getmetatable(L, XDB_BUFFER_METATABLE_NAME); + lua_setmetatable(L, -2); + lua_pushnil(L); + + return 2; +} + +// --- End of buffer api + + +// --- xdb searcher api + +static int lua_xdb_new_with_file_only(lua_State *L) { + int err; + xdb_searcher_t *searcher; + const char *db_path = NULL; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and xdb file path expected"); + + // check the db path + db_path = luaL_checkstring(L, 1); + + // alloc for the searcher + searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t)); + if (searcher == NULL) { + return luaL_error(L, "failed to alloc xdb searcher entry"); + } + + // init the xdb searcher + err = xdb_new_with_file_only(searcher, db_path); + if (err != 0) { + lua_pushnil(L); + lua_pushfstring(L, "init xdb searcher on `%s`: errcode=%d", db_path, err); + return 2; + } + + // push the metatable onto the stack and + // set it as the metatable of the current searcher + luaL_getmetatable(L, XDB_SEARCHER_METATABLE_NAME); + lua_setmetatable(L, -2); + lua_pushnil(L); + + return 2; +} + +static int lua_xdb_new_with_vector_index(lua_State *L) { + xdb_searcher_t *searcher; + xdb_buffer_t *xBuffer; + const char *db_path; + int err; + + luaL_argcheck(L, lua_gettop(L) == 2, 1, "call via '.', xdb file path and vector index buffer expected"); + + // check the db path and vector index buffer + db_path = luaL_checkstring(L, 1); + xBuffer = luaL_checkudata(L, 2, XDB_BUFFER_METATABLE_NAME); + if (xBuffer->type != xdb_vector_index_buffer) { + return luaL_error(L, "invalid vector index buffer"); + } + + // alloc the searcher + searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t)); + if (searcher == NULL) { + return luaL_error(L, "failed to alloc xdb searcher entry"); + } + + // init the xdb searcher + err = xdb_new_with_vector_index(searcher, db_path, (xdb_vector_index_t *) xBuffer->ptr); + if (err != 0) { + lua_pushnil(L); + lua_pushfstring(L, "init vector index cached xdb searcher on `%s` with errcode=%d", db_path, err); + return 2; + } + + // push the metatable onto the stack and + // set it as the metatable of the current searcher + luaL_getmetatable(L, XDB_SEARCHER_METATABLE_NAME); + lua_setmetatable(L, -2); + lua_pushnil(L); + + return 2; +} + +static int lua_xdb_new_with_buffer(lua_State *L) { + xdb_searcher_t *searcher; + xdb_buffer_t *xBuffer; + int err; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and xdb content buffer expected"); + xBuffer = (xdb_buffer_t *) luaL_checkudata(L, 1, XDB_BUFFER_METATABLE_NAME); + if (xBuffer->type != xdb_content_buffer) { + return luaL_error(L, "invalid xdb content buffer"); + } + + // alloc the searcher + searcher = (xdb_searcher_t *) lua_newuserdata(L, sizeof(xdb_searcher_t)); + if (searcher == NULL) { + return luaL_error(L, "failed to alloc xdb searcher entry"); + } + + // init the xdb searcher + err = xdb_new_with_buffer(searcher, (xdb_content_t *) xBuffer->ptr); + if (err != 0) { + lua_pushnil(L); + lua_pushfstring(L, "init content cached xdb searcher with errcode=%d", err); + return 2; + } + + // push the metatable onto the stack and + // set it as the metatable of the current searcher + luaL_getmetatable(L, XDB_SEARCHER_METATABLE_NAME); + lua_setmetatable(L, -2); + lua_pushnil(L); + + return 2; +} + +static int lua_xdb_close(lua_State *L) { + xdb_searcher_t *searcher; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':'"); + searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_SEARCHER_METATABLE_NAME); + if (searcher == NULL) { + return luaL_error(L, "broken xdb searcher instance"); + } + + xdb_close(searcher); + return 0; +} + +static int lua_xdb_search(lua_State *L) { + int err; + const char *ip_str; + unsigned int ip_int; + char region_buffer[1024] = {'\0'}; + xdb_searcher_t *searcher; + + luaL_argcheck(L, lua_gettop(L) == 2, 2, "call via ':' and string or long ip address expected"); + + // get the searcher + searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_SEARCHER_METATABLE_NAME); + + // input ip type checking + if (lua_isinteger(L, 2)) { + ip_int = lua_tointeger(L, 2); + } else if (lua_isstring(L, 2)) { + ip_str = lua_tostring(L, 2); + err = xdb_check_ip(ip_str, &ip_int); + if (err != 0) { + lua_pushnil(L); + lua_pushfstring(L, "invalid string ip `%s`: errcode=%d", ip_str, err); + return 2; + } + } else { + return luaL_error(L, "input ip not integer or string"); + } + + // do the search + err = xdb_search(searcher, ip_int, region_buffer, sizeof(region_buffer)); + if (err != 0) { + lua_pushinteger(L, err); + lua_pushfstring(L, "err=%d", err); + } else { + lua_pushstring(L, region_buffer); + lua_pushnil(L); + } + + return 2; +} + +static int lua_xdb_get_io_count(lua_State *L) { + xdb_searcher_t *searcher; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via ':' or xdb searcher was broken"); + searcher = (xdb_searcher_t *) luaL_checkudata(L, 1, XDB_SEARCHER_METATABLE_NAME); + lua_pushinteger(L, xdb_get_io_count(searcher)); + return 1; +} + +static int lua_xdb_tostring(lua_State *L) { + lua_pushliteral(L, "xdb searcher object"); + return 1; +} + +// -- static util function + +static int lua_xdb_check_ip(lua_State *L) { + int err; + unsigned int ip; + const char *ip_str; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and string ip expected, eg: 1.2.3.4"); + ip_str = luaL_checkstring(L, 1); + err = xdb_check_ip(ip_str, &ip); + if (err != 0) { + lua_pushinteger(L, 0); + lua_pushfstring(L, "err=%d", err); + } else { + lua_pushinteger(L, ip); + lua_pushnil(L); + } + + return 2; +} + +static int lua_xdb_long2ip(lua_State *L) { + unsigned int ip; + char ip_buff[16] = {'\0'}; + + luaL_argcheck(L, lua_gettop(L) == 1, 1, "call via '.' and long ip expected"); + ip = luaL_checkinteger(L, 1); + xdb_long2ip(ip, ip_buff); + lua_pushstring(L, ip_buff); + + return 1; +} + +static int lua_xdb_now(lua_State *L) { + lua_pushinteger(L, xdb_now()); + return 1; +} + +// --- End of static util + +// module method define, should be access via ':' +static const struct luaL_Reg xdb_searcher_methods[] = { + {"search", lua_xdb_search}, + {"get_io_count",lua_xdb_get_io_count}, + {"close", lua_xdb_close}, + {"__gc", lua_xdb_close}, + {"__tostring", lua_xdb_tostring}, + {NULL, NULL}, +}; + +// module function define, should be access via '.' +static const struct luaL_Reg xdb_searcher_functions[] = { + {"new_with_file_only", lua_xdb_new_with_file_only}, + {"new_with_vector_index", lua_xdb_new_with_vector_index}, + {"new_with_buffer", lua_xdb_new_with_buffer}, + {"load_header", lua_xdb_load_header_from_file}, + {"load_vector_index", lua_xdb_load_vector_index_from_file}, + {"load_content", lua_xdb_load_content_from_file}, + {"check_ip", lua_xdb_check_ip}, + {"long2ip", lua_xdb_long2ip}, + {"now", lua_xdb_now}, + {NULL, NULL} +}; + +// module register function +int luaopen_xdb_searcher(lua_State *L) +{ + // register the constants + lua_pushinteger(L, xdb_header_buffer); + lua_setglobal(L, "header_buffer"); + lua_pushinteger(L, xdb_vector_index_buffer); + lua_setglobal(L, "v_index_buffer"); + lua_pushinteger(L, xdb_content_buffer); + lua_setglobal(L, "content_buffer"); + + // create a metatable for xdb buffer object + luaL_newmetatable(L, XDB_BUFFER_METATABLE_NAME); + lua_pushvalue(L, -1); + lua_setfield(L, -2, "__index"); + luaL_setfuncs(L, xdb_buffer_methods, 0); + + // create a metatable for xdb searcher object + luaL_newmetatable(L, XDB_SEARCHER_METATABLE_NAME); + lua_pushvalue(L, -1); + lua_setfield(L, -2, "__index"); + luaL_setfuncs(L, xdb_searcher_methods, 0); + luaL_setfuncs(L, xdb_searcher_functions, 0); + + return 1; +} \ No newline at end of file diff --git a/binding/php/ReadMe.md b/binding/php/ReadMe.md index c8080e8..02802e6 100644 --- a/binding/php/ReadMe.md +++ b/binding/php/ReadMe.md @@ -2,6 +2,8 @@ # 使用方式 +第三方 composer 地址: [https://github.com/chinayin/ip2region-core-php](https://github.com/chinayin/ip2region-core-php) + ### 完全基于文件的查询 ```php $dbFile = "ip2region.xdb file path"; diff --git a/maker/golang/make.bat b/maker/golang/make.bat new file mode 100644 index 0000000..28ae394 --- /dev/null +++ b/maker/golang/make.bat @@ -0,0 +1,18 @@ +::ip2region golang maker makefile in windows +@echo off + +if [%1] == [] goto:build + +if %1==clean ( + call:clean +) else if %1==build ( + call:build +) +exit /b 0 + +:build +go build -o xdb_maker.exe +exit /b 0 + +:clean +del/f/s/q xdb_maker.exe diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index 96debc8..df01c5b 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -5,7 +5,6 @@ package xdb import ( - "encoding/binary" "fmt" "strconv" "strings" @@ -13,13 +12,15 @@ import ( // Util function +var shiftIndex = []int{24, 16, 8, 0} + func CheckIP(ip string) (uint32, error) { var ps = strings.Split(ip, ".") if len(ps) != 4 { return 0, fmt.Errorf("invalid ip address `%s`", ip) } - var buff = make([]byte, 4) + var val uint32 for i, s := range ps { d, err := strconv.Atoi(s) if err != nil { @@ -30,21 +31,14 @@ func CheckIP(ip string) (uint32, error) { return 0, fmt.Errorf("the %dth part `%s` should be an integer bettween 0 and 255", i, s) } - buff[i] = byte(d) + val |= uint32(d) << shiftIndex[i] } - // convert the ip to integer - - return binary.BigEndian.Uint32(buff), nil + return val, nil } func Long2IP(ip uint32) string { - var buff = make([]string, 4) - buff[0] = fmt.Sprintf("%d", (ip>>24)&0xFF) - buff[1] = fmt.Sprintf("%d", (ip>>16)&0xFF) - buff[2] = fmt.Sprintf("%d", (ip>>8)&0xFF) - buff[3] = fmt.Sprintf("%d", (ip>>0)&0xFF) - return strings.Join(buff, ".") + return fmt.Sprintf("%d.%d.%d.%d", (ip>>24)&0xFF, (ip>>16)&0xFF, (ip>>8)&0xFF, (ip>>0)&0xFF) } func MidIP(sip uint32, eip uint32) uint32 { diff --git a/maker/golang/xdb/util_test.go b/maker/golang/xdb/util_test.go index 0de34dc..7e918c1 100644 --- a/maker/golang/xdb/util_test.go +++ b/maker/golang/xdb/util_test.go @@ -5,10 +5,40 @@ package xdb import ( + "encoding/binary" "fmt" + "net" "testing" ) +func TestCheckIP(t *testing.T) { + var str = "29.34.191.255" + ip, err := CheckIP(str) + if err != nil { + t.Errorf("check ip `%s`: %s\n", str, err) + } + + netIP := net.ParseIP(str).To4() + if netIP == nil { + t.Fatalf("parse ip `%s` failed", str) + } + + u32 := binary.BigEndian.Uint32(netIP) + fmt.Printf("checkip: %d, parseip: %d, isEqual: %v\n", ip, u32, ip == u32) +} + +func TestLong2IP(t *testing.T) { + var str = "29.34.191.255" + netIP := net.ParseIP(str).To4() + if netIP == nil { + t.Fatalf("parse ip `%s` failed", str) + } + + u32 := binary.BigEndian.Uint32(netIP) + ipStr := Long2IP(u32) + fmt.Printf("originIP: %s, Long2IP: %s, isEqual: %v\n", str, ipStr, ipStr == str) +} + func TestSplitSegment(t *testing.T) { // var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信" // var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"