try to apply the bytes_ip_t for the search

This commit is contained in:
lion 2025-09-17 14:43:28 +08:00
parent 4cff9eb6ee
commit 1575e581cd
6 changed files with 122 additions and 48 deletions

View File

@ -18,11 +18,12 @@ typedef struct searcher_test_entry searcher_test_t;
int init_searcher_test(searcher_test_t *test, char *db_path, char *cache_policy) { int init_searcher_test(searcher_test_t *test, char *db_path, char *cache_policy) {
int err; int err;
xdb_ip_version_t *version = XDB_IPv4;
test->v_index = NULL; test->v_index = NULL;
test->c_buffer = NULL; test->c_buffer = NULL;
if (strcmp(cache_policy, "file") == 0) { if (strcmp(cache_policy, "file") == 0) {
err = xdb_new_with_file_only(&test->searcher, db_path); err = xdb_new_with_file_only(version, &test->searcher, db_path);
if (err != 0) { if (err != 0) {
printf("failed to create searcher with errcode=%d\n", err); printf("failed to create searcher with errcode=%d\n", err);
return 1; return 1;
@ -34,7 +35,7 @@ int init_searcher_test(searcher_test_t *test, char *db_path, char *cache_policy)
return 2; return 2;
} }
err = xdb_new_with_vector_index(&test->searcher, db_path, test->v_index); err = xdb_new_with_vector_index(version, &test->searcher, db_path, test->v_index);
if (err != 0) { if (err != 0) {
printf("failed to create vector index cached searcher with errcode=%d\n", err); printf("failed to create vector index cached searcher with errcode=%d\n", err);
return 3; return 3;
@ -46,7 +47,7 @@ int init_searcher_test(searcher_test_t *test, char *db_path, char *cache_policy)
return 4; return 4;
} }
err = xdb_new_with_buffer(&test->searcher, test->c_buffer); err = xdb_new_with_buffer(version, &test->searcher, test->c_buffer);
if (err != 0) { if (err != 0) {
printf("failed to create content cached searcher with errcode=%d\n", err); printf("failed to create content cached searcher with errcode=%d\n", err);
return 5; return 5;

Binary file not shown.

View File

@ -68,8 +68,8 @@ void test_parse_ip() {
}; };
int errcode; int errcode;
bytes_ip_t ip_bytes[17] = {'\0'}; bytes_ip_t ip_bytes[16] = {'\0'};
string_ip_t ip_string[INET6_ADDRSTRLEN + 1] = {'\0'}; string_ip_t ip_string[INET6_ADDRSTRLEN] = {'\0'};
// init the sock env (for windows) // init the sock env (for windows)
if ((errcode = xdb_init_winsock()) != 0) { if ((errcode = xdb_init_winsock()) != 0) {

View File

@ -46,9 +46,6 @@
#define xdb_ipv6_version_no 6 #define xdb_ipv6_version_no 6
#define xdb_ipv4_bytes 4 #define xdb_ipv4_bytes 4
#define xdb_ipv6_bytes 16 #define xdb_ipv6_bytes 16
#define XDB_IPV4 4
#define XDB_IPv6 6
// cache of vector_index_row × vector_index_rows × vector_index_size // cache of vector_index_row × vector_index_rows × vector_index_size
#define xdb_vector_index_length 524288 #define xdb_vector_index_length 524288
@ -108,6 +105,26 @@ XDB_PUBLIC(int) xdb_ip_sub_compare(const bytes_ip_t *, size_t, const char *, int
// --- END xdb utils // --- END xdb utils
// --- ip version
#define XDB_IPv4 (xdb_version_ipv4())
#define XDB_IPv6 (xdb_version_ipv6())
typedef int (* ip_compare_fn_t) (const bytes_ip_t *, size_t, const char *, int);
struct xdb_ip_version_entry {
int id; // version id
char *name; // version name
int bytes; // ip bytes number
int segment_index_size; // segment index size in bytes
// function to compare two ips
ip_compare_fn_t ip_compare;
};
typedef struct xdb_ip_version_entry xdb_ip_version_t;
XDB_PUBLIC(xdb_ip_version_t *) xdb_version_ipv4();
XDB_PUBLIC(xdb_ip_version_t *) xdb_version_ipv6();
// --- END ip version
// --- xdb buffer functions // --- xdb buffer functions
@ -170,6 +187,10 @@ XDB_PUBLIC(void) xdb_free_content(void *);
// xdb searcher structure // xdb searcher structure
struct xdb_searcher_entry { struct xdb_searcher_entry {
// ip version
xdb_ip_version_t *version;
// xdb file handle
FILE *handle; FILE *handle;
// header info // header info
@ -188,18 +209,20 @@ struct xdb_searcher_entry {
typedef struct xdb_searcher_entry xdb_searcher_t; typedef struct xdb_searcher_entry xdb_searcher_t;
// xdb searcher new api define // xdb searcher new api define
XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *, const char *); XDB_PUBLIC(int) xdb_new_with_file_only(xdb_ip_version_t *, xdb_searcher_t *, const char *);
XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *); XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_ip_version_t *, xdb_searcher_t *, const char *, const xdb_vector_index_t *);
XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *); XDB_PUBLIC(int) xdb_new_with_buffer(xdb_ip_version_t *, xdb_searcher_t *, const xdb_content_t *);
XDB_PUBLIC(void) xdb_close(void *); XDB_PUBLIC(void) xdb_close(void *);
// xdb searcher search api define // xdb searcher search api define
XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t); XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t);
XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, unsigned int, char *, size_t); XDB_PUBLIC(int) xdb_search(xdb_searcher_t *, const bytes_ip_t *, int, char *, size_t);
XDB_PUBLIC(xdb_ip_version_t *) xdb_get_ip_version(xdb_searcher_t *);
XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *); XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *);

View File

@ -11,9 +11,12 @@
// internal function prototype define // internal function prototype define
XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length); XDB_PRIVATE(int) read(xdb_searcher_t *, long offset, char *, size_t length);
XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { XDB_PRIVATE(int) xdb_new_base(xdb_ip_version_t *version, xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) {
memset(xdb, 0x00, sizeof(xdb_searcher_t)); memset(xdb, 0x00, sizeof(xdb_searcher_t));
// set the version
xdb->version = version;
// check the content buffer first // check the content buffer first
if (c_buffer != NULL) { if (c_buffer != NULL) {
xdb->v_index = NULL; xdb->v_index = NULL;
@ -34,16 +37,16 @@ XDB_PRIVATE(int) xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xd
} }
// xdb searcher new api define // xdb searcher new api define
XDB_PUBLIC(int) xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) { XDB_PUBLIC(int) xdb_new_with_file_only(xdb_ip_version_t *version, xdb_searcher_t *xdb, const char *db_path) {
return xdb_new_base(xdb, db_path, NULL, NULL); return xdb_new_base(version, xdb, db_path, NULL, NULL);
} }
XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) { XDB_PUBLIC(int) xdb_new_with_vector_index(xdb_ip_version_t *version, xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) {
return xdb_new_base(xdb, db_path, v_index, NULL); return xdb_new_base(version, xdb, db_path, v_index, NULL);
} }
XDB_PUBLIC(int) xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) { XDB_PUBLIC(int) xdb_new_with_buffer(xdb_ip_version_t *version, xdb_searcher_t *xdb, const xdb_content_t *c_buffer) {
return xdb_new_base(xdb, NULL, NULL, c_buffer); return xdb_new_base(version, xdb, NULL, NULL, c_buffer);
} }
XDB_PUBLIC(void) xdb_close(void *ptr) { XDB_PUBLIC(void) xdb_close(void *ptr) {
@ -56,27 +59,38 @@ XDB_PUBLIC(void) xdb_close(void *ptr) {
// --- xdb searcher search api define // --- xdb searcher search api define
XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) { XDB_PUBLIC(int) xdb_search_by_string(xdb_searcher_t *xdb, const char *ip_string, char *region_buffer, size_t length) {
unsigned int ip = 0; bytes_ip_t ip_bytes[16] = {'\0'};
int errcode = xdb_check_ip(str_ip, &ip); int versionNo = xdb_parse_ip(ip_string, ip_bytes, sizeof(ip_bytes));
if (errcode != 0) { if (versionNo == -1) {
return 10 + errcode; return 10;
} else { } else {
return xdb_search(xdb, ip, region_buffer, length); return xdb_search(xdb, ip_bytes, versionNo, region_buffer, length);
} }
} }
XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) { XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, const bytes_ip_t *ip_bytes, int byte_count, char *region_buffer, size_t length) {
int il0, il1, idx, err, l, h, m, data_len; int il0, il1, idx, err, data_len, bytes, d_bytes;
register int seg_index_size, l, h, m;
unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr; unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr;
char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size]; char vector_buffer[xdb_vector_index_size];
char *segment_buffer = NULL;
// reset the io counter // ip version check
if (byte_count != xdb->version->bytes) {
return -1;
}
// some resets
err = 0;
data_len = 0;
bytes = byte_count;
d_bytes = byte_count << 1;
xdb->io_count = 0; xdb->io_count = 0;
// locate the segment index block based on the vector index // locate the segment index block based on the vector index
il0 = ((int) (ip >> 24)) & 0xFF; il0 = (int) (ip_bytes[0]);
il1 = ((int) (ip >> 16)) & 0xFF; il1 = (int) (ip_bytes[1]);
idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size;
if (xdb->v_index != NULL) { if (xdb->v_index != NULL) {
s_ptr = xdb_le_get_uint32(xdb->v_index->buffer, idx); s_ptr = xdb_le_get_uint32(xdb->v_index->buffer, idx);
@ -96,53 +110,65 @@ XDB_PUBLIC(int) xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_bu
// printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr); // printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr);
// binary search to get the final region info // binary search to get the final region info
seg_index_size = xdb->version->segment_index_size;
segment_buffer = xdb_malloc(seg_index_size);
if (segment_buffer == NULL) {
return -2;
}
data_len = 0, data_ptr = 0; data_len = 0, data_ptr = 0;
l = 0, h = ((int) (e_ptr - s_ptr)) / xdb_segment_index_size; l = 0, h = ((int) (e_ptr - s_ptr)) / seg_index_size;
while (l <= h) { while (l <= h) {
m = (l + h) >> 1; m = (l + h) >> 1;
p = s_ptr + m * xdb_segment_index_size; p = s_ptr + m * seg_index_size;
// read the segment index item // read the segment index item
err = read(xdb, p, segment_buffer, sizeof(segment_buffer)); err = read(xdb, p, segment_buffer, sizeof(segment_buffer));
if (err != 0) { if (err != 0) {
return 20 + err; err += 20;
goto done;
} }
// decode the data fields as needed // decode the data fields as needed
sip = xdb_le_get_uint32(segment_buffer, 0); sip = xdb_le_get_uint32(segment_buffer, 0);
if (ip < sip) { if (xdb_ip_sub_compare(ip_bytes, byte_count, segment_buffer, 0) < 0) {
h = m - 1; h = m - 1;
} else { } else if (xdb_ip_sub_compare(ip_bytes, byte_count, segment_buffer, bytes) > 0) {
eip = xdb_le_get_uint32(segment_buffer, 4);
if (ip > eip) {
l = m + 1; l = m + 1;
} else { } else {
data_len = xdb_le_get_uint16(segment_buffer, 8); data_len = xdb_le_get_uint16(segment_buffer, d_bytes);
data_ptr = xdb_le_get_uint32(segment_buffer, 10); data_ptr = xdb_le_get_uint32(segment_buffer, d_bytes + 2);
break; break;
} }
} }
}
// printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr); // printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr);
if (data_len == 0) { if (data_len == 0) {
region_buffer[0] = '\0'; goto done;
return 0;
} }
// buffer length checking // buffer length checking
if (data_len >= (int) length) { if (data_len >= (int) length) {
return 1; err = 1;
goto done;
} }
err = read(xdb, data_ptr, region_buffer, data_len); err = read(xdb, data_ptr, region_buffer, data_len);
if (err != 0) { if (err != 0) {
return 30 + err; err += 30;
goto done;
}
done:
// checn and free the segment buffer
if (segment_buffer != NULL) {
xdb_free(segment_buffer);
segment_buffer = NULL;
} }
// auto append a NULL-end // auto append a NULL-end
region_buffer[data_len] = '\0'; region_buffer[data_len] = '\0';
return 0; return err;
} }
XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) { XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) {
@ -165,6 +191,10 @@ XDB_PRIVATE(int) read(xdb_searcher_t *xdb, long offset, char *buffer, size_t len
return 0; return 0;
} }
XDB_PUBLIC(xdb_ip_version_t *) xdb_get_ip_version(xdb_searcher_t *xdb) {
return xdb->version;
}
XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) { XDB_PUBLIC(int) xdb_get_io_count(xdb_searcher_t *xdb) {
return xdb->io_count; return xdb->io_count;
} }

View File

@ -389,3 +389,23 @@ XDB_PUBLIC(void) xdb_free_content(void *ptr) {
} }
// --- End // --- End
// --- ip version
static xdb_ip_version_t _ip_version_list[] = {
// 14 = 4 + 4 + 2 + 4
{xdb_ipv4_version_no, "IPv4", xdb_ipv4_bytes, 14, NULL},
// 38 = 16 + 16 + 2 + 4
{xdb_ipv6_version_no, "IPv6", xdb_ipv6_bytes, 38, NULL},
// END
{0, NULL, 0, 0, NULL}
};
XDB_PUBLIC(xdb_ip_version_t *) xdb_version_ipv4() {
return &_ip_version_list[0];
}
XDB_PUBLIC(xdb_ip_version_t *) xdb_version_ipv6() {
return &_ip_version_list[1];
}