From 654959686f95001c93aef1ba1ae54743c7a4bed4 Mon Sep 17 00:00:00 2001 From: lion Date: Tue, 28 Jun 2022 16:28:16 +0800 Subject: [PATCH] buffer load functions are ready --- binding/c/util_test.c | 68 ++++++++++++++++++++-- binding/c/xdb_searcher.c | 118 +++++++++++++++++++++++++++++++-------- binding/c/xdb_searcher.h | 37 +++++++----- 3 files changed, 183 insertions(+), 40 deletions(-) diff --git a/binding/c/util_test.c b/binding/c/util_test.c index 2c502b9..1d51750 100644 --- a/binding/c/util_test.c +++ b/binding/c/util_test.c @@ -9,11 +9,11 @@ #include "stdio.h" #include "xdb_searcher.h" -int main(int argc, char *argv[]) { +void test_check_ip() { char *ip_list[] = { - "1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0", - "256.7.12.9", "12.56.78.320", "32.12.45.192", "222.221.220.219", - "192.168.1.101 ", "132.96.12.98a", "x23.12.2.12" + "1.2.3.4", "192.168.2.3", "120.24.78.129", "255.255.255.0", + "256.7.12.9", "12.56.78.320", "32.12.45.192", "222.221.220.219", + "192.168.1.101 ", "132.96.12.98a", "x23.12.2.12" }; int errcode, i; @@ -34,6 +34,66 @@ int main(int argc, char *argv[]) { printf(" --[Ok]\n"); } } +} + +void test_load_header() { + xdb_header_t header; + int err = xdb_load_header_from_file("../../data/ip2region.xdb", &header); + if (err != 0) { + printf("failed to load header with errcode=%d\n", err); + } else { + printf("header loaded: {\n" + " version: %d, \n" + " index_policy: %d, \n" + " created_at: %u, \n" + " start_index_ptr: %d, \n" + " end_index_ptr: %d\n" + "}\n", + header.version, header.index_policy, header.created_at, + header.start_index_ptr, header.end_index_ptr + ); + } +} + +void test_load_vector_index() { + char *ptr = xdb_load_vector_index_from_file("../../data/ip2region.xdb"); + if (ptr == NULL) { + printf("failed to load vector index from file\n"); + } else { + printf("vector index loaded from file\n"); + } + + xdb_free(ptr); +} + +void test_load_content() { + char *ptr = xdb_load_content_from_file("../../data/ip2region.xdb"); + if (ptr == NULL) { + printf("failed to load content from file\n"); + } else { + printf("content loaded from file\n"); + } + + xdb_free(ptr); +} + +// valgrind --tool=memcheck --leak-check=full ./a.out +int main(int argc, char *argv[]) { + printf("test check ip ... \n"); + test_check_ip(); + printf("|--done\n\n"); + + printf("test load header ... \n"); + test_load_header(); + printf("|--done\n\n"); + + printf("test load vector index ... \n"); + test_load_vector_index(); + printf("|--done\n\n"); + + printf("test load content ... \n"); + test_load_content(); + printf("|--done\n\n"); return 0; } \ No newline at end of file diff --git a/binding/c/xdb_searcher.c b/binding/c/xdb_searcher.c index f7de603..69c3602 100644 --- a/binding/c/xdb_searcher.c +++ b/binding/c/xdb_searcher.c @@ -67,7 +67,7 @@ XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, ch XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) { int il0, il1, idx, err, l, h, m, data_len; unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr; - char vector_buffer[VectorIndexSize], segment_buffer[SegmentIndexSize]; + char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size]; // reset the io counter xdb->io_count = 0; @@ -75,15 +75,15 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu // locate the segment index block based on the vector index il0 = (ip >> 24) & 0xFF; il1 = (ip >> 16) & 0xFF; - idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; + idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; if (xdb->vector_index != NULL) { s_ptr = get_unsigned_int(xdb->vector_index, idx); e_ptr = get_unsigned_int(xdb->vector_index, idx + 4); } else if (xdb->content_buff != NULL) { - s_ptr = get_unsigned_int(xdb->content_buff, HeaderInfoLength + idx); - e_ptr = get_unsigned_int(xdb->content_buff, HeaderInfoLength + idx + 4); + s_ptr = get_unsigned_int(xdb->content_buff, xdb_header_info_length + idx); + e_ptr = get_unsigned_int(xdb->content_buff, xdb_header_info_length + idx + 4); } else { - err = read(xdb, HeaderInfoLength + idx, vector_buffer, sizeof(vector_buffer)); + err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); if (err != 0) { return 10 + err; } @@ -96,10 +96,10 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu // binary search to get the final region info data_len = 0, data_ptr = 0; - l = 0, h = (e_ptr - s_ptr) / SegmentIndexSize; + l = 0, h = (e_ptr - s_ptr) / xdb_segment_index_size; while (l <= h) { m = (l + h) >> 1; - p = s_ptr + m * SegmentIndexSize; + p = s_ptr + m * xdb_segment_index_size; // read the segment index item err = read(xdb, p, segment_buffer, sizeof(segment_buffer)); @@ -152,13 +152,11 @@ XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t len } // seek to the offset - int errcode = fseek(xdb->handle, offset, SEEK_SET); - if (errcode == -1) { + if (fseek(xdb->handle, offset, SEEK_SET) == -1) { return 1; } - int r_size = fread(buffer, 1, length, xdb->handle); - if (r_size != 1) { + if (fread(buffer, 1, length, xdb->handle) != -1) { return 2; } @@ -172,28 +170,104 @@ XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) { // --- buffer load util functions -XDB_PUBLIC(int) xdb_load_header(FILE *handle, char *buffer, size_t length) { +XDB_PUBLIC(int) xdb_load_header(FILE *handle, xdb_header_t *header) { + char buffer[256]; + + if (fseek(handle, 0, SEEK_SET) == -1) { + return 1; + } + + if (fread(buffer, 1, 256, handle) != 256) { + return 2; + } + + // fill the fields + header->version = (unsigned short) get_unsigned_short(buffer, 0); + header->index_policy = (unsigned short) get_unsigned_short(buffer, 2); + header->created_at = get_unsigned_int(buffer, 4); + header->start_index_ptr = get_unsigned_int(buffer, 8); + header->end_index_ptr = get_unsigned_int(buffer,12); + return 0; } -XDB_PUBLIC(int) xdb_load_header_from_file(char *dbPath, char *buffer, size_t length) { - return 0; +XDB_PUBLIC(int) xdb_load_header_from_file(char *db_path, xdb_header_t *header) { + FILE *handle = fopen(db_path, "r"); + if (handle == NULL) { + return 10; + } + + return xdb_load_header(handle, header); } -XDB_PUBLIC(int) xdb_load_vector_index(FILE *handle, char *buffer, size_t length) { - return 0; +XDB_PUBLIC(char *) xdb_load_vector_index(FILE *handle) { + char *ptr = NULL; + int size = xdb_vector_index_length; + + // seek to the vector index offset + if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) { + return NULL; + } + + // do the buffer read + ptr = (char *) xdb_malloc(size); + if (ptr == NULL) { + return NULL; + } + + if (fread(ptr, 1, size, handle) != size) { + xdb_free(ptr); + return NULL; + } + + return ptr; } -XDB_PUBLIC(int) xdb_load_vector_index_from_file(char *dbPath, char *buffer, size_t length) { - return 0; +XDB_PUBLIC(char *) xdb_load_vector_index_from_file(char *db_path) { + FILE *handle = fopen(db_path, "r"); + if (handle == NULL) { + return NULL; + } + + return xdb_load_vector_index(handle); } -XDB_PUBLIC(int) xdb_load_content(FILE *handle, char *buffer, size_t length) { - return 0; +XDB_PUBLIC(char *) xdb_load_content(FILE *handle) { + long filesize; + char *ptr = NULL; + + // determine the file size + if (fseek(handle, 0, SEEK_END) == -1) { + return NULL; + } + + filesize = ftell(handle); + if (fseek(handle, 0, SEEK_SET) == -1) { + return NULL; + } + + // do the file read + ptr = (char *) xdb_malloc(filesize); + if (ptr == NULL) { + return NULL; + } + + // read the content into the buffer + if (fread(ptr, 1, filesize, handle) != filesize) { + xdb_free(ptr); + return NULL; + } + + return ptr; } -XDB_PUBLIC(int) xdb_load_content_from_file(char *handle, char *buffer, size_t length) { - return 0; +XDB_PUBLIC(char *) xdb_load_content_from_file(char *db_path) { + FILE *handle = fopen(db_path, "r"); + if (handle == NULL) { + return NULL; + } + + return xdb_load_content(handle); } // --- End diff --git a/binding/c/xdb_searcher.h b/binding/c/xdb_searcher.h index 8070e05..a71ed2b 100644 --- a/binding/c/xdb_searcher.h +++ b/binding/c/xdb_searcher.h @@ -34,16 +34,16 @@ do { \ #define xdb_calloc( _blocks, _bytes ) calloc( _blocks, _bytes ) #define xdb_malloc( _bytes ) malloc( _bytes ) #define xdb_free( _ptr ) free( _ptr ) -#define field_size(type, field) sizeof(((type *)0)->field) -#define sf_field_size(type, field) (sizeof(((type *)0)->field) - 1) - // public constants define -#define HeaderInfoLength 256 -#define VectorIndexRows 256 -#define VectorIndexCols 256 -#define VectorIndexSize 8 -#define SegmentIndexSize 14 +#define xdb_header_info_length 256 +#define xdb_vector_index_rows 256 +#define xdb_vector_index_cols 256 +#define xdb_vector_index_size 8 +#define xdb_segment_index_size 14 + +// cache of vector_index_row × vector_index_rows × vector_index_size +#define xdb_vector_index_length 524288 // xdb searcher structure @@ -84,17 +84,26 @@ XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *); // --- buffer load util functions -XDB_PUBLIC(int) xdb_load_header(FILE *, char *, size_t); +struct xdb_header { + unsigned short version; + unsigned short index_policy; + unsigned int created_at; + unsigned int start_index_ptr; + unsigned int end_index_ptr; +}; +typedef struct xdb_header xdb_header_t; -XDB_PUBLIC(int) xdb_load_header_from_file(char *, char *, size_t); +XDB_PUBLIC(int) xdb_load_header(FILE *, xdb_header_t *); -XDB_PUBLIC(int) xdb_load_vector_index(FILE *, char *, size_t); +XDB_PUBLIC(int) xdb_load_header_from_file(char *, xdb_header_t *); -XDB_PUBLIC(int) xdb_load_vector_index_from_file(char *, char *, size_t); +XDB_PUBLIC(char *) xdb_load_vector_index(FILE *); -XDB_PUBLIC(int) xdb_load_content(FILE *, char *, size_t); +XDB_PUBLIC(char *) xdb_load_vector_index_from_file(char *); -XDB_PUBLIC(int) xdb_load_content_from_file(char *, char *, size_t); +XDB_PUBLIC(char *) xdb_load_content(FILE *); + +XDB_PUBLIC(char *) xdb_load_content_from_file(char *); // --- End buffer load