diff --git a/binding/c/main.c b/binding/c/main.c index e252f48..309ba85 100644 --- a/binding/c/main.c +++ b/binding/c/main.c @@ -7,9 +7,74 @@ // @Date 2022/06/28 #include "stdio.h" -#include "sys/time.h" #include "xdb_searcher.h" +struct searcher_test_entry { + xdb_searcher_t searcher; + char *v_index; + char *c_buffer; +}; +typedef struct searcher_test_entry searcher_test_t; + +int init_searcher_test(searcher_test_t *test, char *db_path, char *cache_policy) { + int err; + test->v_index = NULL; + test->c_buffer = NULL; + + if (strcmp(cache_policy, "file") == 0) { + err = xdb_new_with_file_only(&test->searcher, db_path); + if (err != 0) { + printf("failed to create searcher with errcode=%d\n", err); + return 1; + } + } else if (strcmp(cache_policy, "vectorIndex") == 0) { + test->v_index = xdb_load_vector_index_from_file(db_path); + if (test->v_index == NULL) { + printf("failed to load vector index from `%s`\n", db_path); + return 2; + } + + err = xdb_new_with_vector_index(&test->searcher, db_path, test->v_index); + if (err != 0) { + printf("failed to create vector index cached searcher with errcode=%d\n", err); + return 3; + } + } else if (strcmp(cache_policy, "content") == 0) { + test->c_buffer = xdb_load_content_from_file(db_path); + if (test->c_buffer == NULL) { + printf("failed to load xdb content from `%s`\n", db_path); + return 4; + } + + err = xdb_new_with_buffer(&test->searcher, test->c_buffer); + if (err != 0) { + printf("failed to create content cached searcher with errcode=%d\n", err); + return 5; + } + } else { + printf("invalid cache policy `%s`, options: file/vectorIndex/content\n", cache_policy); + return 6; + } + + return 0; +} + +void destroy_searcher_test(searcher_test_t *test) { + xdb_close(&test->searcher); + + // check and free the vector index + if (test->v_index != NULL) { + xdb_free(test->v_index); + test->v_index = NULL; + } + + // check and free the content buffer + if (test->c_buffer != NULL) { + xdb_free(test->c_buffer); + test->c_buffer = NULL; + } +} + //read a line from a command line. static char *get_line(FILE *fp, char *__dst) { register int c; @@ -43,7 +108,8 @@ void test_search(int argc, char *argv[]) { // for search long s_time, c_time; unsigned int ip; - char line[256] = {'\0'}, region[256] = {'\0'}; + char line[512] = {'\0'}, region[512] = {'\0'}; + searcher_test_t test; for (i = 2; i < argc; i++) { r = argv[i]; @@ -87,41 +153,9 @@ void test_search(int argc, char *argv[]) { } // printf("db_file=%s, cache_policy=%s\n", db_file, cache_policy); - - xdb_searcher_t searcher; - char *vIndex = NULL, *cBuffer = NULL; - if (strcmp(cache_policy, "file") == 0) { - err = xdb_new_with_file_only(&searcher, db_file); - if (err != 0) { - printf("failed to create searcher with errcode=%d\n", err); - return; - } - } else if (strcmp(cache_policy, "vectorIndex") == 0) { - vIndex = xdb_load_vector_index_from_file(db_file); - if (vIndex == NULL) { - printf("failed to load vector index from `%s`\n", db_file); - return; - } - - err = xdb_new_with_vector_index(&searcher, db_file, vIndex); - if (err != 0) { - printf("failed to create vector index cached searcher with errcode=%d\n", err); - return; - } - } else if (strcmp(cache_policy, "content") == 0) { - cBuffer = xdb_load_content_from_file(db_file); - if (cBuffer == NULL) { - printf("failed to load xdb content from `%s`\n", db_file); - return; - } - - err = xdb_new_with_buffer(&searcher, cBuffer); - if (err != 0) { - printf("failed to create content cached searcher with errcode=%d\n", err); - return; - } - } else { - printf("invalid cache policy `%s`, options: file/vectorIndex/content\n", cache_policy); + err = init_searcher_test(&test, db_file, cache_policy); + if (err != 0) { + // init program will print the error reasons; return; } @@ -138,35 +172,38 @@ void test_search(int argc, char *argv[]) { break; } - if (check_ip(line, &ip) != 0) { + if (xdb_check_ip(line, &ip) != 0) { printf("invalid ip address `%s`\n", line); continue; } s_time = xdb_now(); - err = xdb_search(&searcher, ip, region, sizeof(region)); + err = xdb_search(&test.searcher, ip, region, sizeof(region)); if (err != 0) { - printf("{err: %d, io_count: %d}\n", err, xdb_get_io_count(&searcher)); + printf("{err: %d, io_count: %d}\n", err, xdb_get_io_count(&test.searcher)); } else { c_time = xdb_now() - s_time; - printf("{region: %s, io_count: %d, took: %ld μs}\n", region, xdb_get_io_count(&searcher), c_time); + printf("{region: %s, io_count: %d, took: %ld μs}\n", region, xdb_get_io_count(&test.searcher), c_time); } } - xdb_close(&searcher); - if (vIndex != NULL) { - xdb_free(vIndex); - } - if(cBuffer != NULL) { - xdb_free(cBuffer); - } + destroy_searcher_test(&test); printf("searcher test program exited, thanks for trying\n"); } void test_bench(int argc, char *argv[]) { - int i, n; + int i, n, err; char *r, key[33] = {'\0'}, val[256] = {'\0'}; char db_file[256] = {'\0'}, src_file[256] = {'\0'}, cache_policy[16] = {"vectorIndex"}; + + FILE *handle; + char line[1024] = {'\0'}, sip_str[16] = {'\0'}, eip_str[16] = {'\0'}; + char src_region[512] = {'\0'}, region_buffer[512] = {'\0'}; + unsigned int sip, eip, mip, ip_list[5]; + int count = 0, took; + long s_time, t_time, c_time = 0; + searcher_test_t test; + for (i = 2; i < argc; i++) { r = argv[i]; if (strlen(r) < 5) { @@ -209,7 +246,76 @@ void test_bench(int argc, char *argv[]) { return; } - printf("db_file=%s, src_file=%s, cache_policy=%s\n", db_file, src_file, cache_policy); + // printf("db_file=%s, src_file=%s, cache_policy=%s\n", db_file, src_file, cache_policy); + s_time = xdb_now(); + err = init_searcher_test(&test, db_file, cache_policy); + if (err != 0) { + // the init function will print the details; + return; + } + + // open the source file + handle = fopen(src_file, "r"); + if (handle == NULL) { + printf("failed to open source text file `%s`\n", src_file); + return; + } + + while(fgets(line, sizeof(line), handle) != NULL) { + n = sscanf(line, "%15[^|]|%15[^|]|%511[^\n]", sip_str, eip_str, src_region); + if (n != 3) { + printf("invalid ip segment line `%s`\n", line); + return; + } + + if (xdb_check_ip(sip_str, &sip) != 0) { + printf("invalid start ip `%s`\n", sip_str); + return; + } + + if (xdb_check_ip(eip_str, &eip) != 0) { + printf("invalid end ip `%s`\n", sip_str); + return; + } + + if (sip > eip) { + printf("start ip(%s) should not be greater than end ip(%s)\n", sip_str, eip_str); + return; + } + + mip = xdb_mip(sip, eip); + ip_list[0] = sip; + ip_list[1] = xdb_mip(sip, mip); + ip_list[2] = mip; + ip_list[3] = xdb_mip(mip, eip); + ip_list[4] = eip; + for (i = 0; i < 5; i++) { + t_time = xdb_now(); + err = xdb_search(&test.searcher, ip_list[i], region_buffer, sizeof(region_buffer)); + if (err != 0) { + xdb_long2ip(ip_list[i], sip_str); + printf("failed to search ip `%s` with errno=%d\n", sip_str, err); + return; + } + + c_time += xdb_now() - t_time; + + // check the region info + if (strcmp(region_buffer, src_region) != 0) { + xdb_long2ip(ip_list[i], sip_str); + printf("failed to search(%s) with (%s != %s)\n", sip_str, region_buffer, src_region); + return; + } + + count++; + } + }; + + took = xdb_now() - s_time; + destroy_searcher_test(&test); + fclose(handle); + printf("Bench finished, {cache_policy: %s, total: %d, took: %.3fs, cost: %d μs/op}\n", + cache_policy, count, took/1e6, count == 0 ? 0 : (int)(c_time/count)); } int main(int argc, char *argv[]) { diff --git a/binding/c/util_test.c b/binding/c/util_test.c index 1d51750..3a35230 100644 --- a/binding/c/util_test.c +++ b/binding/c/util_test.c @@ -20,13 +20,13 @@ void test_check_ip() { unsigned int ip; char ip_buff[16] = {'\0'}; for (i = 0; i < 11; i++) { - errcode = check_ip(ip_list[i], &ip); + errcode = xdb_check_ip(ip_list[i], &ip); if (errcode != 0) { printf("invalid ip address `%s`\n", ip_list[i]); continue; } - long2ip(ip, ip_buff); + xdb_long2ip(ip, ip_buff); printf("long(%-15s)=%-10u, long2ip(%-10u)=%-15s", ip_list[i], ip, ip, ip_buff); if (strcmp(ip_list[i], ip_buff) != 0) { printf(" --[Failed]\n"); diff --git a/binding/c/xdb_searcher.c b/binding/c/xdb_searcher.c index 7ec46a9..452aadd 100644 --- a/binding/c/xdb_searcher.c +++ b/binding/c/xdb_searcher.c @@ -57,7 +57,7 @@ XDB_PUBLIC(void) xdb_close(xdb_searcher_t *xdb) { XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) { unsigned int ip = 0; - int errcode = check_ip(str_ip, &ip); + int errcode = xdb_check_ip(str_ip, &ip); if (errcode != 0) { return 10 + errcode; } else { @@ -78,19 +78,19 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu il1 = ((int) (ip >> 16)) & 0xFF; idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; if (xdb->vector_index != NULL) { - s_ptr = get_unsigned_int(xdb->vector_index, idx); - e_ptr = get_unsigned_int(xdb->vector_index, idx + 4); + s_ptr = xdb_get_uint(xdb->vector_index, idx); + e_ptr = xdb_get_uint(xdb->vector_index, idx + 4); } else if (xdb->content_buff != NULL) { - s_ptr = get_unsigned_int(xdb->content_buff, xdb_header_info_length + idx); - e_ptr = get_unsigned_int(xdb->content_buff, xdb_header_info_length + idx + 4); + s_ptr = xdb_get_uint(xdb->content_buff, xdb_header_info_length + idx); + e_ptr = xdb_get_uint(xdb->content_buff, xdb_header_info_length + idx + 4); } else { err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); if (err != 0) { return 10 + err; } - s_ptr = get_unsigned_int(vector_buffer, 0); - e_ptr = get_unsigned_int(vector_buffer, 4); + s_ptr = xdb_get_uint(vector_buffer, 0); + e_ptr = xdb_get_uint(vector_buffer, 4); } // printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr); @@ -108,16 +108,16 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu } // decode the data fields as needed - sip = get_unsigned_int(segment_buffer, 0); + sip = xdb_get_uint(segment_buffer, 0); if (ip < sip) { h = m - 1; } else { - eip = get_unsigned_int(segment_buffer, 4); + eip = xdb_get_uint(segment_buffer, 4); if (ip > eip) { l = m + 1; } else { - data_len = get_unsigned_short(segment_buffer, 8); - data_ptr = get_unsigned_int(segment_buffer, 10); + data_len = xdb_get_ushort(segment_buffer, 8); + data_ptr = xdb_get_uint(segment_buffer, 10); break; } } @@ -183,11 +183,11 @@ XDB_PUBLIC(int) xdb_load_header(FILE *handle, xdb_header_t *header) { } // fill the fields - header->version = (unsigned short) get_unsigned_short(buffer, 0); - header->index_policy = (unsigned short) get_unsigned_short(buffer, 2); - header->created_at = get_unsigned_int(buffer, 4); - header->start_index_ptr = get_unsigned_int(buffer, 8); - header->end_index_ptr = get_unsigned_int(buffer,12); + header->version = (unsigned short) xdb_get_ushort(buffer, 0); + header->index_policy = (unsigned short) xdb_get_ushort(buffer, 2); + header->created_at = xdb_get_uint(buffer, 4); + header->start_index_ptr = xdb_get_uint(buffer, 8); + header->end_index_ptr = xdb_get_uint(buffer,12); return 0; } @@ -274,7 +274,7 @@ XDB_PUBLIC(char *) xdb_load_content_from_file(char *db_path) { // --- End // get unsigned long (4bytes) from a specified buffer start from the specified offset -XDB_PUBLIC(unsigned int) get_unsigned_int(const char *buffer, int offset) { +XDB_PUBLIC(unsigned int) xdb_get_uint(const char *buffer, int offset) { return ( ((buffer[offset ]) & 0x000000FF) | ((buffer[offset+1] << 8) & 0x0000FF00) | @@ -284,7 +284,7 @@ XDB_PUBLIC(unsigned int) get_unsigned_int(const char *buffer, int offset) { } // get unsigned short (2bytes) from a specified buffer start from the specified offset -XDB_PUBLIC(int) get_unsigned_short(const char *buffer, int offset) { +XDB_PUBLIC(int) xdb_get_ushort(const char *buffer, int offset) { return ( ((buffer[offset ]) & 0x000000FF) | ((buffer[offset+1] << 8) & 0x0000FF00) @@ -293,7 +293,7 @@ XDB_PUBLIC(int) get_unsigned_short(const char *buffer, int offset) { // string ip to unsigned int static int shiftIndex[4] = {24, 16, 8, 0}; -XDB_PUBLIC(int) check_ip(const char *src_ip, unsigned int *dst_ip) { +XDB_PUBLIC(int) xdb_check_ip(const char *src_ip, unsigned int *dst_ip) { char c; int i, n, ip = 0; const char *ptr = src_ip; @@ -326,10 +326,15 @@ XDB_PUBLIC(int) check_ip(const char *src_ip, unsigned int *dst_ip) { } // unsigned int ip to string ip -XDB_PUBLIC(void) long2ip(unsigned int ip, char *buffer) { +XDB_PUBLIC(void) xdb_long2ip(unsigned int ip, char *buffer) { sprintf(buffer, "%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF); } +// get the middle ip of a and b +XDB_PUBLIC(unsigned int) xdb_mip(unsigned long a, unsigned long b) { + return (unsigned int) ((a + b) >> 1); +} + XDB_PUBLIC(long) xdb_now() { struct timeval c_time; gettimeofday(&c_time, NULL); diff --git a/binding/c/xdb_searcher.h b/binding/c/xdb_searcher.h index 665f5d9..13b7725 100644 --- a/binding/c/xdb_searcher.h +++ b/binding/c/xdb_searcher.h @@ -101,16 +101,19 @@ XDB_PUBLIC(char *) xdb_load_content_from_file(char *); // get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian -XDB_PUBLIC(unsigned int) get_unsigned_int(const char *, int); +XDB_PUBLIC(unsigned int) xdb_get_uint(const char *, int); // get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian -XDB_PUBLIC(int) get_unsigned_short(const char *, int); +XDB_PUBLIC(int) xdb_get_ushort(const char *, int); // check the specified string ip and convert it to an unsigned int -XDB_PUBLIC(int) check_ip(const char *, unsigned int *); +XDB_PUBLIC(int) xdb_check_ip(const char *, unsigned int *); // unsigned int ip to string ip -XDB_PUBLIC(void) long2ip(unsigned int, char *); +XDB_PUBLIC(void) xdb_long2ip(unsigned int, char *); + +// get the middle ip of a and b +XDB_PUBLIC(unsigned int) xdb_mip(unsigned long, unsigned long); // get the current time in microseconds XDB_PUBLIC(long) xdb_now();