From 288859f7aef6ca0746ff8ca5ec9c33d0eb4bf2d2 Mon Sep 17 00:00:00 2001 From: Lion Date: Fri, 1 Jul 2022 11:57:45 +0800 Subject: [PATCH] wrap the header/vector/content buffer --- binding/c/main.c | 8 +-- binding/c/util_test.c | 30 ++++---- binding/c/xdb_searcher.c | 145 ++++++++++++++++++++++++++------------- binding/c/xdb_searcher.h | 87 +++++++++++++++-------- 4 files changed, 173 insertions(+), 97 deletions(-) diff --git a/binding/c/main.c b/binding/c/main.c index 309ba85..49a0f13 100644 --- a/binding/c/main.c +++ b/binding/c/main.c @@ -11,8 +11,8 @@ struct searcher_test_entry { xdb_searcher_t searcher; - char *v_index; - char *c_buffer; + xdb_vector_index_t *v_index; + xdb_content_t *c_buffer; }; typedef struct searcher_test_entry searcher_test_t; @@ -64,13 +64,13 @@ void destroy_searcher_test(searcher_test_t *test) { // check and free the vector index if (test->v_index != NULL) { - xdb_free(test->v_index); + xdb_close_vector_index(test->v_index); test->v_index = NULL; } // check and free the content buffer if (test->c_buffer != NULL) { - xdb_free(test->c_buffer); + xdb_close_content(test->c_buffer); test->c_buffer = NULL; } } diff --git a/binding/c/util_test.c b/binding/c/util_test.c index 3a35230..f6828d2 100644 --- a/binding/c/util_test.c +++ b/binding/c/util_test.c @@ -37,10 +37,9 @@ void test_check_ip() { } void test_load_header() { - xdb_header_t header; - int err = xdb_load_header_from_file("../../data/ip2region.xdb", &header); - if (err != 0) { - printf("failed to load header with errcode=%d\n", err); + xdb_header_t *header = xdb_load_header_from_file("../../data/ip2region.xdb"); + if (header == NULL) { + printf("failed to load header"); } else { printf("header loaded: {\n" " version: %d, \n" @@ -48,33 +47,36 @@ void test_load_header() { " created_at: %u, \n" " start_index_ptr: %d, \n" " end_index_ptr: %d\n" + " length: %d\n" "}\n", - header.version, header.index_policy, header.created_at, - header.start_index_ptr, header.end_index_ptr + header->version, header->index_policy, header->created_at, + header->start_index_ptr, header->end_index_ptr, header->length ); } + + xdb_close_header(header); } void test_load_vector_index() { - char *ptr = xdb_load_vector_index_from_file("../../data/ip2region.xdb"); - if (ptr == NULL) { + xdb_vector_index_t *v_index = xdb_load_vector_index_from_file("../../data/ip2region.xdb"); + if (v_index == NULL) { printf("failed to load vector index from file\n"); } else { - printf("vector index loaded from file\n"); + printf("vector index loaded from file, length=%d\n", v_index->length); } - xdb_free(ptr); + xdb_close_vector_index(v_index); } void test_load_content() { - char *ptr = xdb_load_content_from_file("../../data/ip2region.xdb"); - if (ptr == NULL) { + xdb_content_t *content = xdb_load_content_from_file("../../data/ip2region.xdb"); + if (content == NULL) { printf("failed to load content from file\n"); } else { - printf("content loaded from file\n"); + printf("content loaded from file, length=%d\n", content->length); } - xdb_free(ptr); + xdb_close_content(content); } // valgrind --tool=memcheck --leak-check=full ./a.out diff --git a/binding/c/xdb_searcher.c b/binding/c/xdb_searcher.c index 9a3fc62..efc8edc 100644 --- a/binding/c/xdb_searcher.c +++ b/binding/c/xdb_searcher.c @@ -12,13 +12,13 @@ // internal function prototype define XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length); -XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const char *vIndex, const char *cBuff) { +XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { memset(xdb, 0x00, sizeof(xdb_searcher_t)); // check the content buffer first - if (cBuff != NULL) { - xdb->vector_index = NULL; - xdb->content_buff = cBuff; + if (c_buffer != NULL) { + xdb->v_index = NULL; + xdb->content = c_buffer; return 0; } @@ -29,7 +29,7 @@ XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const ch } xdb->handle = handle; - xdb->vector_index = vIndex; + xdb->v_index = v_index; return 0; } @@ -39,11 +39,11 @@ XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) return xdb_new_base(xdb, db_path, NULL, NULL); } -XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const char *vIndex) { - return xdb_new_base(xdb, db_path, vIndex, NULL); +XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) { + return xdb_new_base(xdb, db_path, v_index, NULL); } -XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, const char *c_buffer) { +XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) { return xdb_new_base(xdb, NULL, NULL, c_buffer); } @@ -78,12 +78,12 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu il0 = ((int) (ip >> 24)) & 0xFF; il1 = ((int) (ip >> 16)) & 0xFF; idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; - if (xdb->vector_index != NULL) { - s_ptr = xdb_get_uint(xdb->vector_index, idx); - e_ptr = xdb_get_uint(xdb->vector_index, idx + 4); - } else if (xdb->content_buff != NULL) { - s_ptr = xdb_get_uint(xdb->content_buff, xdb_header_info_length + idx); - e_ptr = xdb_get_uint(xdb->content_buff, xdb_header_info_length + idx + 4); + if (xdb->v_index != NULL) { + s_ptr = xdb_get_uint(xdb->v_index->buffer, idx); + e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4); + } else if (xdb->content != NULL) { + s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx); + e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4); } else { err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); if (err != 0) { @@ -147,8 +147,8 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) { // check the xdb content cache first - if (xdb->content_buff != NULL) { - memcpy(buffer, xdb->content_buff + offset, length); + if (xdb->content != NULL) { + memcpy(buffer, xdb->content->buffer + offset, length); return 0; } @@ -172,39 +172,58 @@ XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) { // --- buffer load util functions -XDB_PUBLIC(int) xdb_load_header(FILE *handle, xdb_header_t *header) { - char buffer[256]; +XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *handle) { + xdb_header_t *header; + unsigned int size = xdb_header_info_length; - if (fseek(handle, 0, SEEK_SET) == -1) { - return 1; + // entry alloc + header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t)); + if (header == NULL) { + return NULL; } - if (fread(buffer, 1, 256, handle) != 256) { - return 2; + if (fseek(handle, 0, SEEK_SET) == -1) { + xdb_free(header); + return NULL; + } + + if (fread(header->buffer, 1,size, handle) != size) { + xdb_free(header); + return NULL; } // fill the fields - header->version = (unsigned short) xdb_get_ushort(buffer, 0); - header->index_policy = (unsigned short) xdb_get_ushort(buffer, 2); - header->created_at = xdb_get_uint(buffer, 4); - header->start_index_ptr = xdb_get_uint(buffer, 8); - header->end_index_ptr = xdb_get_uint(buffer,12); + header->length = size; + header->version = (unsigned short) xdb_get_ushort(header->buffer, 0); + header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2); + header->created_at = xdb_get_uint(header->buffer, 4); + header->start_index_ptr = xdb_get_uint(header->buffer, 8); + header->end_index_ptr = xdb_get_uint(header->buffer,12); - return 0; + return header; } -XDB_PUBLIC(int) xdb_load_header_from_file(const char *db_path, xdb_header_t *header) { +XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *db_path) { FILE *handle = fopen(db_path, "r"); if (handle == NULL) { - return 10; + return NULL; } - return xdb_load_header(handle, header); + return xdb_load_header(handle); } -XDB_PUBLIC(char *) xdb_load_vector_index(FILE *handle) { - char *ptr = NULL; - int size = xdb_vector_index_length; +XDB_PUBLIC(void) xdb_close_header(xdb_header_t *header) { + if (header->length > 0) { + header->length = 0; + xdb_free(header); + } +} + +// --- vector index + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *handle) { + xdb_vector_index_t *v_index; + unsigned int size = xdb_vector_index_length; // seek to the vector index offset if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) { @@ -212,20 +231,21 @@ XDB_PUBLIC(char *) xdb_load_vector_index(FILE *handle) { } // do the buffer read - ptr = (char *) xdb_malloc(size); - if (ptr == NULL) { + v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t)); + if (v_index == NULL) { return NULL; } - if (fread(ptr, 1, size, handle) != size) { - xdb_free(ptr); + v_index->length = size; + if (fread(v_index->buffer, 1, size, handle) != size) { + xdb_free(v_index); return NULL; } - return ptr; + return v_index; } -XDB_PUBLIC(char *) xdb_load_vector_index_from_file(const char *db_path) { +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *db_path) { FILE *handle = fopen(db_path, "r"); if (handle == NULL) { return NULL; @@ -234,36 +254,54 @@ XDB_PUBLIC(char *) xdb_load_vector_index_from_file(const char *db_path) { return xdb_load_vector_index(handle); } -XDB_PUBLIC(char *) xdb_load_content(FILE *handle) { - long filesize; - char *ptr = NULL; +XDB_PUBLIC(void) xdb_close_vector_index(xdb_vector_index_t *v_index) { + if (v_index->length > 0) { + v_index->length = 0; + xdb_free(v_index); + } +} + +// --- content buffer + +XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *handle) { + unsigned int size; + xdb_content_t *content; + char *ptr; // determine the file size if (fseek(handle, 0, SEEK_END) == -1) { return NULL; } - filesize = ftell(handle); + size = (unsigned int) ftell(handle); if (fseek(handle, 0, SEEK_SET) == -1) { return NULL; } // do the file read - ptr = (char *) xdb_malloc(filesize); - if (ptr == NULL) { + content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t)); + if (content == NULL) { + return NULL; + } + + // do the buffer alloc + content->buffer = (char *) xdb_malloc(size); + if (content->buffer == NULL) { + xdb_free(content); return NULL; } // read the content into the buffer - if (fread(ptr, 1, filesize, handle) != filesize) { + content->length = size; + if (fread(content->buffer, 1, size, handle) != size) { xdb_free(ptr); return NULL; } - return ptr; + return content; } -XDB_PUBLIC(char *) xdb_load_content_from_file(const char *db_path) { +XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *db_path) { FILE *handle = fopen(db_path, "r"); if (handle == NULL) { return NULL; @@ -272,6 +310,15 @@ XDB_PUBLIC(char *) xdb_load_content_from_file(const char *db_path) { return xdb_load_content(handle); } +XDB_PUBLIC(void) xdb_close_content(xdb_content_t *content) { + if (content->length > 0) { + content->length = 0; + xdb_free(content->buffer); + content->buffer = NULL; + xdb_free(content); + } +} + // --- End // get unsigned long (4bytes) from a specified buffer start from the specified offset diff --git a/binding/c/xdb_searcher.h b/binding/c/xdb_searcher.h index 167e121..5f8427c 100644 --- a/binding/c/xdb_searcher.h +++ b/binding/c/xdb_searcher.h @@ -38,6 +38,59 @@ #define xdb_vector_index_length 524288 +// --- buffer load util functions + +// use the following buffer struct to wrap the binary buffer data +// since the buffer data could not be operated with the string API. +struct xdb_header { + unsigned short version; + unsigned short index_policy; + unsigned int created_at; + unsigned int start_index_ptr; + unsigned int end_index_ptr; + + // the original buffer + unsigned int length; + char buffer[xdb_header_info_length]; +}; +typedef struct xdb_header xdb_header_t; + +XDB_PUBLIC(xdb_header_t *) xdb_load_header(FILE *); + +XDB_PUBLIC(xdb_header_t *) xdb_load_header_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_header(xdb_header_t *); + + +// --- vector index buffer +struct xdb_vector_index { + unsigned int length; + char buffer[xdb_vector_index_length]; +}; +typedef struct xdb_vector_index xdb_vector_index_t; + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index(FILE *); + +XDB_PUBLIC(xdb_vector_index_t *) xdb_load_vector_index_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_vector_index(xdb_vector_index_t *); + + +// --- content buffer +struct xdb_content { + unsigned int length; + char *buffer; +}; +typedef struct xdb_content xdb_content_t; + +XDB_PUBLIC(xdb_content_t *) xdb_load_content(FILE *); + +XDB_PUBLIC(xdb_content_t *) xdb_load_content_from_file(const char *); + +XDB_PUBLIC(void) xdb_close_content(xdb_content_t *); + +// --- End buffer load + // xdb searcher structure struct xdb_searcher_entry { FILE *handle; @@ -49,20 +102,20 @@ struct xdb_searcher_entry { // vector index buffer cache. // preload the vector index will reduce the number of IO operations // thus speedup the search process. - const char *vector_index; + const xdb_vector_index_t *v_index; // content buffer. // cache the whole xdb content. - const char *content_buff; + const xdb_content_t *content; }; typedef struct xdb_searcher_entry xdb_searcher_t; // xdb searcher new api define XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, const char *); -XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, const char *, const char *); +XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *); -XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, const char *); +XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *); XDB_PUBLIC(void) xdb_close(xdb_searcher_t *); @@ -74,32 +127,6 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, unsigned int, char *, size_t); XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *); -// --- buffer load util functions - -struct xdb_header { - unsigned short version; - unsigned short index_policy; - unsigned int created_at; - unsigned int start_index_ptr; - unsigned int end_index_ptr; -}; -typedef struct xdb_header xdb_header_t; - -XDB_PUBLIC(int) xdb_load_header(FILE *, xdb_header_t *); - -XDB_PUBLIC(int) xdb_load_header_from_file(const char *, xdb_header_t *); - -XDB_PUBLIC(char *) xdb_load_vector_index(FILE *); - -XDB_PUBLIC(char *) xdb_load_vector_index_from_file(const char *); - -XDB_PUBLIC(char *) xdb_load_content(FILE *); - -XDB_PUBLIC(char *) xdb_load_content_from_file(const char *); - -// --- End buffer load - - // get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian XDB_PUBLIC(unsigned int) xdb_get_uint(const char *, int);