From b04e477b996a40a7e728483c4fc067270845f133 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 17:46:38 +0800 Subject: [PATCH 01/16] nginx-impl init --- .gitignore | 3 + binding/nginx/LICENSE | 21 + binding/nginx/README.md | 14 + binding/nginx/config | 14 + binding/nginx/src/ngx_http_ip2region_module.c | 147 +++++++ binding/nginx/src/ngx_http_ip2region_module.h | 18 + binding/nginx/src/xdb_searcher.c | 402 ++++++++++++++++++ binding/nginx/src/xdb_searcher.h | 127 ++++++ binding/nginx/t/http_ua_parser.t | 27 ++ 9 files changed, 773 insertions(+) create mode 100644 binding/nginx/LICENSE create mode 100644 binding/nginx/README.md create mode 100644 binding/nginx/config create mode 100644 binding/nginx/src/ngx_http_ip2region_module.c create mode 100644 binding/nginx/src/ngx_http_ip2region_module.h create mode 100644 binding/nginx/src/xdb_searcher.c create mode 100644 binding/nginx/src/xdb_searcher.h create mode 100644 binding/nginx/t/http_ua_parser.t diff --git a/.gitignore b/.gitignore index 0743c23..baa7e0e 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,6 @@ target #erlang /binding/erlang/_build /binding/erlang/doc + +#vscode +.vscode diff --git a/binding/nginx/LICENSE b/binding/nginx/LICENSE new file mode 100644 index 0000000..e39f9f3 --- /dev/null +++ b/binding/nginx/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Wu Jian Ping + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/binding/nginx/README.md b/binding/nginx/README.md new file mode 100644 index 0000000..15c2ec2 --- /dev/null +++ b/binding/nginx/README.md @@ -0,0 +1,14 @@ +# nginx-ip2region + +## 编译 + +```shell +$ ./configure \ + --prefix=/Users/wujp/Documents/workspaces/open-source/nginx-1.23.4/.nginx \ + --add-module=/Users/wujp/Documents/workspaces/open-source/ip2region/binding/nginx + +$ make +$ make install +``` + +Made with ♥ by Wu Jian Ping diff --git a/binding/nginx/config b/binding/nginx/config new file mode 100644 index 0000000..9d8f5f7 --- /dev/null +++ b/binding/nginx/config @@ -0,0 +1,14 @@ +ngx_addon_name=ngx_ip2region_module + +NGX_IP2REGION_SRCS=" \ + $ngx_addon_dir/src/ngx_http_ip2region_module.c \ + $ngx_addon_dir/src/xdb_searcher.c \ + " + +NGX_IP2REGION_DEPS=" \ + " + +HTTP_MODULES="$HTTP_MODULES ngx_http_ip2region_module" +# STREAM_MODULES="ngx_stream_ip2region_module $STREAM_MODULES" +NGX_ADDON_SRCS="$NGX_ADDON_SRCS $NGX_IP2REGION_SRCS" +NGX_ADDON_DEPS="$NGX_ADDON_DEPS $NGX_IP2REGION_DEPS" diff --git a/binding/nginx/src/ngx_http_ip2region_module.c b/binding/nginx/src/ngx_http_ip2region_module.c new file mode 100644 index 0000000..5010cc6 --- /dev/null +++ b/binding/nginx/src/ngx_http_ip2region_module.c @@ -0,0 +1,147 @@ +/* + * Created by Wu Jian Ping on - 2023/03/20. + */ + +#include "ngx_http_ip2region_module.h" + +static ngx_int_t ngx_http_ip2region_add_variables(ngx_conf_t *cf); + +static void *ngx_http_ip2region_create_conf(ngx_conf_t *cf); +static void ngx_http_ip2region_cleanup(void *data); +static char *ngx_http_ip2region_processor(ngx_conf_t *cf, + ngx_command_t *cmd, void *conf); + +static ngx_int_t ngx_http_ip2region_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); + +static ngx_http_module_t ngx_http_ip2region_ctx = { + ngx_http_ip2region_add_variables, /* pre configuration */ + NULL, /* post configuration */ + ngx_http_ip2region_create_conf, /* create main configuration */ + NULL, /* init main configuration */ + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +static ngx_command_t ngx_http_ip2region_commands[] = { + { ngx_string("ip2region"), + NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1, + ngx_http_ip2region_processor, + NGX_HTTP_MAIN_CONF_OFFSET, + 0, + NULL }, + + ngx_null_command +}; + + +/* ngx_module_t is required, otherwise failed at complie time */ +ngx_module_t ngx_http_ip2region_module = { + NGX_MODULE_V1, + &ngx_http_ip2region_ctx, /* module context */ + ngx_http_ip2region_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_http_variable_t ngx_http_ip2region_vars[] = { + { ngx_string("ip2region"), NULL, + ngx_http_ip2region_variable, + 0, 0, 0 }, + + ngx_http_null_variable +}; + + +static char * +ngx_http_ip2region_processor(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf) +{ + // ngx_http_ip2region_conf_t *uacf = conf; + // ngx_str_t *value; + + // if (uacf->ip2region) { + // return "is duplicate"; + // } + + // value = cf->args->elts; + + // uacf->ip2region = open_ip2region_regexes_yaml((char *)value[1].data); + + // if(uacf->ip2region == NULL) { + // ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + // "open_ip2region_regexes_yaml(\"%V\") failed", &value[1]); + // return NGX_CONF_ERROR; + // } + + return NGX_CONF_OK; +} + + +static void * +ngx_http_ip2region_create_conf(ngx_conf_t *cf) +{ + ngx_pool_cleanup_t *cln; + ngx_http_ip2region_conf_t *conf; + + conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_ip2region_conf_t)); + if (conf == NULL) { + return NULL; + } + + cln = ngx_pool_cleanup_add(cf->pool, 0); + if (cln == NULL) { + return NULL; + } + + cln->handler = ngx_http_ip2region_cleanup; + cln->data = conf; + + return conf; +} + + +static ngx_int_t +ngx_http_ip2region_add_variables(ngx_conf_t *cf) +{ + ngx_http_variable_t *var; + ngx_http_variable_t *v; + + /* the last ngx_http_variable_t's name is ngx_null_string and it's name's len is 0*/ + for (v = ngx_http_ip2region_vars; v->name.len; v++) { + var = ngx_http_add_variable(cf, &v->name, v->flags); + if (var == NULL) { + return NGX_ERROR; + } + + var->get_handler = v->get_handler; + var->data = v->data; + } + return NGX_OK; +} + +static ngx_int_t +ngx_http_ip2region_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + v->not_found = 1; + return NGX_OK; +} + + +static void +ngx_http_ip2region_cleanup(void *data) +{ + // ngx_http_ip2region_conf_t *uacf = data; +} \ No newline at end of file diff --git a/binding/nginx/src/ngx_http_ip2region_module.h b/binding/nginx/src/ngx_http_ip2region_module.h new file mode 100644 index 0000000..7d504de --- /dev/null +++ b/binding/nginx/src/ngx_http_ip2region_module.h @@ -0,0 +1,18 @@ +#ifndef __NGX_HTTP_IP2REGION_MODULE_INCLUDED__ +#define __NGX_HTTP_IP2REGION_MODULE_INCLUDED__ + +#include +#include +#include + +typedef struct { + // xdb_searcher_t searcher; + // xdb_vector_index_t *v_index; + // xdb_content_t *c_buffer; +} ip2region_searcher_t; + +typedef struct { + ip2region_searcher_t *ip2region_searcher; +} ngx_http_ip2region_conf_t; + +#endif diff --git a/binding/nginx/src/xdb_searcher.c b/binding/nginx/src/xdb_searcher.c new file mode 100644 index 0000000..e895972 --- /dev/null +++ b/binding/nginx/src/xdb_searcher.c @@ -0,0 +1,402 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +// --- +// @Author Lion +// @Date 2022/06/27 + +#include "sys/time.h" +#include "xdb_searcher.h" + +// internal function prototype define +static inline int read(xdb_searcher_t *, long offset, char *, size_t length); + +static inline int xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { + memset(xdb, 0x00, sizeof(xdb_searcher_t)); + + // check the content buffer first + if (c_buffer != NULL) { + xdb->v_index = NULL; + xdb->content = c_buffer; + return 0; + } + + // open the xdb binary file + FILE *handle = fopen(db_path, "rb"); + if (handle == NULL) { + return 1; + } + + xdb->handle = handle; + xdb->v_index = v_index; + + return 0; +} + +// xdb searcher new api define +int xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) { + return xdb_new_base(xdb, db_path, NULL, NULL); +} + +int xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) { + return xdb_new_base(xdb, db_path, v_index, NULL); +} + +int xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) { + return xdb_new_base(xdb, NULL, NULL, c_buffer); +} + +void xdb_close(void *ptr) { + xdb_searcher_t *xdb = (xdb_searcher_t *) ptr; + if (xdb->handle != NULL) { + fclose(xdb->handle); + xdb->handle = NULL; + } +} + +// --- xdb searcher search api define + +int xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) { + unsigned int ip = 0; + int errcode = xdb_check_ip(str_ip, &ip); + if (errcode != 0) { + return 10 + errcode; + } else { + return xdb_search(xdb, ip, region_buffer, length); + } +} + +int xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) { + int il0, il1, idx, err, l, h, m; + unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr; + size_t data_len; + char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size]; + + // reset the io counter + xdb->io_count = 0; + + // locate the segment index block based on the vector index + il0 = ((int) (ip >> 24)) & 0xFF; + il1 = ((int) (ip >> 16)) & 0xFF; + idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; + if (xdb->v_index != NULL) { + s_ptr = xdb_get_uint(xdb->v_index->buffer, idx); + e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4); + } else if (xdb->content != NULL) { + s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx); + e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4); + } else { + err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); + if (err != 0) { + return 10 + err; + } + + s_ptr = xdb_get_uint(vector_buffer, 0); + e_ptr = xdb_get_uint(vector_buffer, 4); + } + + // printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr); + // binary search to get the final region info + data_len = 0, data_ptr = 0; + l = 0, h = ((int) (e_ptr - s_ptr)) / xdb_segment_index_size; + while (l <= h) { + m = (l + h) >> 1; + p = s_ptr + m * xdb_segment_index_size; + + // read the segment index item + err = read(xdb, p, segment_buffer, sizeof(segment_buffer)); + if (err != 0) { + return 20 + err; + } + + // decode the data fields as needed + sip = xdb_get_uint(segment_buffer, 0); + if (ip < sip) { + h = m - 1; + } else { + eip = xdb_get_uint(segment_buffer, 4); + if (ip > eip) { + l = m + 1; + } else { + data_len = xdb_get_ushort(segment_buffer, 8); + data_ptr = xdb_get_uint(segment_buffer, 10); + break; + } + } + } + + // printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr); + if (data_len == 0) { + region_buffer[0] = '\0'; + return 0; + } + + // buffer length checking + if (data_len >= length) { + return 1; + } + + err = read(xdb, data_ptr, region_buffer, data_len); + if (err != 0) { + return 30 + err; + } + + // auto append a NULL-end + region_buffer[data_len] = '\0'; + return 0; +} + +static inline int read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) { + // check the xdb content cache first + if (xdb->content != NULL) { + memcpy(buffer, xdb->content->buffer + offset, length); + return 0; + } + + // seek to the offset + if (fseek(xdb->handle, offset, SEEK_SET) == -1) { + return 1; + } + + xdb->io_count++; + if (fread(buffer, 1, length, xdb->handle) != length) { + return 2; + } + + return 0; +} + +int xdb_get_io_count(xdb_searcher_t *xdb) { + return xdb->io_count; +} + + +// --- buffer load util functions +xdb_header_t* xdb_load_header(FILE *handle) { + xdb_header_t *header; + unsigned int size = xdb_header_info_length; + + // entry alloc + header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t)); + if (header == NULL) { + return NULL; + } + + if (fseek(handle, 0, SEEK_SET) == -1) { + xdb_free(header); + return NULL; + } + + if (fread(header->buffer, 1,size, handle) != size) { + xdb_free(header); + return NULL; + } + + // fill the fields + header->length = size; + header->version = (unsigned short) xdb_get_ushort(header->buffer, 0); + header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2); + header->created_at = xdb_get_uint(header->buffer, 4); + header->start_index_ptr = xdb_get_uint(header->buffer, 8); + header->end_index_ptr = xdb_get_uint(header->buffer,12); + + return header; +} + +xdb_header_t* xdb_load_header_from_file(const char *db_path) { + xdb_header_t *header; + FILE *handle = fopen(db_path, "rb"); + if (handle == NULL) { + return NULL; + } + + header = xdb_load_header(handle); + fclose(handle); + return header; +} + +void xdb_close_header(void *ptr) { + xdb_header_t *header = (xdb_header_t *) ptr; + if (header->length > 0) { + header->length = 0; + xdb_free(header); + } +} + +// --- vector index + +xdb_vector_index_t* xdb_load_vector_index(FILE *handle) { + xdb_vector_index_t *v_index; + unsigned int size = xdb_vector_index_length; + + // seek to the vector index offset + if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) { + return NULL; + } + + // do the buffer read + v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t)); + if (v_index == NULL) { + return NULL; + } + + v_index->length = size; + if (fread(v_index->buffer, 1, size, handle) != size) { + xdb_free(v_index); + return NULL; + } + + return v_index; +} + +xdb_vector_index_t* xdb_load_vector_index_from_file(const char *db_path) { + xdb_vector_index_t *v_index; + FILE *handle = fopen(db_path, "rb"); + if (handle == NULL) { + return NULL; + } + + v_index = xdb_load_vector_index(handle); + fclose(handle); + return v_index; +} + +void xdb_close_vector_index(void *ptr) { + xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr; + if (v_index->length > 0) { + v_index->length = 0; + xdb_free(v_index); + } +} + +// --- content buffer + +xdb_content_t * xdb_load_content(FILE *handle) { + unsigned int size; + xdb_content_t *content; + + // determine the file size + if (fseek(handle, 0, SEEK_END) == -1) { + return NULL; + } + + size = (unsigned int) ftell(handle); + if (fseek(handle, 0, SEEK_SET) == -1) { + return NULL; + } + + // do the file read + content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t)); + if (content == NULL) { + return NULL; + } + + // do the buffer alloc + content->buffer = (char *) xdb_malloc(size); + if (content->buffer == NULL) { + xdb_free(content); + return NULL; + } + + // read the content into the buffer + content->length = size; + if (fread(content->buffer, 1, size, handle) != size) { + xdb_free(content); + return NULL; + } + + return content; +} + +xdb_content_t* xdb_load_content_from_file(const char *db_path) { + xdb_content_t *content; + FILE *handle = fopen(db_path, "rb"); + if (handle == NULL) { + return NULL; + } + + content = xdb_load_content(handle); + fclose(handle); + return content; +} + +void xdb_close_content(void *ptr) { + xdb_content_t *content = (xdb_content_t *) ptr; + if (content->length > 0) { + content->length = 0; + xdb_free(content->buffer); + content->buffer = NULL; + xdb_free(content); + } +} + +// --- End + +// get unsigned long (4bytes) from a specified buffer start from the specified offset +unsigned int xdb_get_uint(const char *buffer, int offset) { + return ( + ((buffer[offset ]) & 0x000000FF) | + ((buffer[offset+1] << 8) & 0x0000FF00) | + ((buffer[offset+2] << 16) & 0x00FF0000) | + ((buffer[offset+3] << 24) & 0xFF000000) + ); +} + +// get unsigned short (2bytes) from a specified buffer start from the specified offset +int xdb_get_ushort(const char *buffer, int offset) { + return ( + ((buffer[offset ]) & 0x000000FF) | + ((buffer[offset+1] << 8) & 0x0000FF00) + ); +} + +// string ip to unsigned int +static int shiftIndex[4] = {24, 16, 8, 0}; +int xdb_check_ip(const char *src_ip, unsigned int *dst_ip) { + char c; + int i, n, ip = 0; + const char *ptr = src_ip; + for (i = 0; i < 4; i++) { + n = 0; + while (1) { + c = *ptr; + ptr++; + if (c >= '0' && c <= '9') { + n *= 10; + n += c - '0'; + } else if ((i < 3 && c == '.') || i == 3) { + // stopping at the '.' but ignore the tailing chars + // after the 3rd one (auto clean the tailing none-integer ?). + break; + } else { + return 1; + } + } + + if (n > 0xFF) { + return 2; + } + + ip |= (n << shiftIndex[i]); + } + + *dst_ip = ip; + return 0; +} + +// unsigned int ip to string ip +void xdb_long2ip(unsigned int ip, char *buffer) { + sprintf(buffer, "%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF); +} + +// get the middle ip of a and b +unsigned int xdb_mip(unsigned long a, unsigned long b) { + return (unsigned int) ((a + b) >> 1); +} + +long xdb_now() { + struct timeval c_time; + gettimeofday(&c_time, NULL); + return c_time.tv_sec * (int)1e6 + c_time.tv_usec; +} diff --git a/binding/nginx/src/xdb_searcher.h b/binding/nginx/src/xdb_searcher.h new file mode 100644 index 0000000..ca553dc --- /dev/null +++ b/binding/nginx/src/xdb_searcher.h @@ -0,0 +1,127 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +// --- +// @Author Lion +// @Date 2022/06/27 + +#ifndef C_XDB_SEARCHER_H +#define C_XDB_SEARCHER_H + +#include +#include +#include + +#define xdb_calloc( _blocks, _bytes ) calloc( _blocks, _bytes ) +#define xdb_malloc( _bytes ) malloc( _bytes ) +#define xdb_free( _ptr ) free( _ptr ) + +// public constants define +#define xdb_header_info_length 256 +#define xdb_vector_index_rows 256 +#define xdb_vector_index_cols 256 +#define xdb_vector_index_size 8 +#define xdb_segment_index_size 14 + +// cache of vector_index_row × vector_index_rows × vector_index_size +#define xdb_vector_index_length 524288 + +typedef struct { + unsigned short version; + unsigned short index_policy; + unsigned int created_at; + unsigned int start_index_ptr; + unsigned int end_index_ptr; + + // the original buffer + unsigned int length; + char buffer[xdb_header_info_length]; +} xdb_header_t; + +xdb_header_t* xdb_load_header(FILE *); + +xdb_header_t *xdb_load_header_from_file(const char *); + +void xdb_close_header(void *); + +// --- vector index buffer +struct xdb_vector_index { + unsigned int length; + char buffer[xdb_vector_index_length]; +}; +typedef struct xdb_vector_index xdb_vector_index_t; + +xdb_vector_index_t* xdb_load_vector_index(FILE *); + +xdb_vector_index_t * xdb_load_vector_index_from_file(const char *); + +void xdb_close_vector_index(void *); + +// --- content buffer +typedef struct { + unsigned int length; + char *buffer; +} xdb_content_t; + +xdb_content_t* xdb_load_content(FILE *); + +xdb_content_t* xdb_load_content_from_file(const char *); + +void xdb_close_content(void *); + +// --- End buffer load + +// xdb searcher structure +typedef struct xdb_searcher_entry { + FILE *handle; + + // header info + const char *header; + int io_count; + + // vector index buffer cache. + // preload the vector index will reduce the number of IO operations + // thus speedup the search process. + const xdb_vector_index_t *v_index; + + // content buffer. + // cache the whole xdb content. + const xdb_content_t *content; +} xdb_searcher_t; + +// xdb searcher new api define +int xdb_new_with_file_only(xdb_searcher_t *, const char *); + +int xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *); + +int xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *); + +void xdb_close(void *); + +// xdb searcher search api define +int xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t); + +int xdb_search(xdb_searcher_t *, unsigned int, char *, size_t); + +int xdb_get_io_count(xdb_searcher_t *); + +// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian +unsigned int xdb_get_uint(const char *, int); + +// get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian +int xdb_get_ushort(const char *, int); + +// check the specified string ip and convert it to an unsigned int +int xdb_check_ip(const char *, unsigned int *); + +// unsigned int ip to string ip +void xdb_long2ip(unsigned int, char *); + +// get the middle ip of a and b +unsigned int xdb_mip(unsigned long, unsigned long); + +// get the current time in microseconds +long xdb_now(); + +#endif //C_XDB_SEARCHER_H diff --git a/binding/nginx/t/http_ua_parser.t b/binding/nginx/t/http_ua_parser.t new file mode 100644 index 0000000..7b16856 --- /dev/null +++ b/binding/nginx/t/http_ua_parser.t @@ -0,0 +1,27 @@ +use lib 'lib'; +use Test::Nginx::Socket; # 'no_plan'; + +repeat_each(2); + +plan tests => repeat_each() * 124; + +no_long_string(); +#no_diff; + +run_tests(); + +__DATA__ + +=== TEST 1: set request header at client side +--- config + location /foo { + echo $http_x_foo; + } +--- request + GET /foo +--- more_headers +X-Foo: blah +--- response_headers +! X-Foo +--- response_body +blah From 1d09953acb964f0366315672673fc6a8c90414d4 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 21:03:53 +0800 Subject: [PATCH 02/16] nginx implementation first cut --- binding/nginx/src/ngx_http_ip2region_module.c | 234 ++++++++++++++++-- binding/nginx/src/ngx_http_ip2region_module.h | 11 +- binding/nginx/src/xdb_searcher.c | 3 + binding/nginx/src/xdb_searcher.h | 3 + 4 files changed, 230 insertions(+), 21 deletions(-) diff --git a/binding/nginx/src/ngx_http_ip2region_module.c b/binding/nginx/src/ngx_http_ip2region_module.c index 5010cc6..ac148f3 100644 --- a/binding/nginx/src/ngx_http_ip2region_module.c +++ b/binding/nginx/src/ngx_http_ip2region_module.c @@ -1,9 +1,11 @@ /* - * Created by Wu Jian Ping on - 2023/03/20. + * Created by Wu Jian Ping on - 2023/03/30. */ #include "ngx_http_ip2region_module.h" +static ngx_int_t ngx_http_ua_parser_test_full_name(char *name); + static ngx_int_t ngx_http_ip2region_add_variables(ngx_conf_t *cf); static void *ngx_http_ip2region_create_conf(ngx_conf_t *cf); @@ -28,7 +30,7 @@ static ngx_http_module_t ngx_http_ip2region_ctx = { static ngx_command_t ngx_http_ip2region_commands[] = { { ngx_string("ip2region"), - NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1, + NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE12, ngx_http_ip2region_processor, NGX_HTTP_MAIN_CONF_OFFSET, 0, @@ -68,22 +70,95 @@ static char * ngx_http_ip2region_processor(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { - // ngx_http_ip2region_conf_t *uacf = conf; - // ngx_str_t *value; + ngx_http_ip2region_conf_t *ip2region_cf; + ip2region_searcher_t *ip2region_searcher; + char *db_name, *cache_policy; + ngx_str_t *value; + int err; + char *db_path; + size_t len; + ip2region_cf = conf; - // if (uacf->ip2region) { - // return "is duplicate"; - // } + if (ip2region_cf->ip2region_searcher) { + return "is duplicate"; + } - // value = cf->args->elts; + value = cf->args->elts; - // uacf->ip2region = open_ip2region_regexes_yaml((char *)value[1].data); + db_name = (char *)value[1].data; - // if(uacf->ip2region == NULL) { - // ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - // "open_ip2region_regexes_yaml(\"%V\") failed", &value[1]); - // return NGX_CONF_ERROR; - // } + // default cache_policy: content + if(cf->args->nelts == 2) { + cache_policy = "content"; + } else { + cache_policy = (char *)value[2].data; + } + + if(ngx_http_ua_parser_test_full_name(db_name) == NGX_OK) { + db_path = db_name; + } else { // relative path to conf directory + len = ngx_cycle->conf_prefix.len + strlen(db_name) + 1; + db_path = malloc(len); + memset(db_path, '\0', len); + memcpy(db_path, ngx_cycle->conf_prefix.data, ngx_cycle->conf_prefix.len); + strcat(db_path, db_name); + } + + ip2region_searcher = ngx_palloc(cf->pool, sizeof(ip2region_searcher_t)); + + if(ip2region_searcher == NULL) { + return NGX_CONF_ERROR; + } + + ip2region_searcher->v_index = NULL; + ip2region_searcher->c_buffer = NULL; + + if (strcmp(cache_policy, "file") == 0) { + err = xdb_new_with_file_only(&ip2region_searcher->searcher, db_path); + if (err != 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to create searcher: %s", db_path); + return NGX_CONF_ERROR; + } + } else if (strcmp(cache_policy, "vectorIndex") == 0) { + ip2region_searcher->v_index = xdb_load_vector_index_from_file(db_path); + if (ip2region_searcher->v_index == NULL) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to load vector index from: %s", db_path); + return NGX_CONF_ERROR; + } + + err = xdb_new_with_vector_index(&ip2region_searcher->searcher, db_path, ip2region_searcher->v_index); + if (err != 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to create vector index cached searcher: %s", db_path); + return NGX_CONF_ERROR; + } + } else if (strcmp(cache_policy, "content") == 0) { + ip2region_searcher->c_buffer = xdb_load_content_from_file(db_path); + if (ip2region_searcher->c_buffer == NULL) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to load xdb content: %s", db_path); + return NGX_CONF_ERROR; + } + + err = xdb_new_with_buffer(&ip2region_searcher->searcher, ip2region_searcher->c_buffer); + if (err != 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "failed to create content cached searcher: %s", db_path); + return NGX_CONF_ERROR; + } + } else { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid cache policy `%V`, options: file/vectorIndex/content", &value[2]); + return NGX_CONF_ERROR; + } + + ip2region_cf->ip2region_searcher = ip2region_searcher; + + if(ngx_http_ua_parser_test_full_name(db_name) != NGX_OK) { + free(db_path); + } return NGX_CONF_OK; } @@ -117,8 +192,7 @@ ngx_http_ip2region_add_variables(ngx_conf_t *cf) { ngx_http_variable_t *var; ngx_http_variable_t *v; - - /* the last ngx_http_variable_t's name is ngx_null_string and it's name's len is 0*/ + for (v = ngx_http_ip2region_vars; v->name.len; v++) { var = ngx_http_add_variable(cf, &v->name, v->flags); if (var == NULL) { @@ -131,10 +205,66 @@ ngx_http_ip2region_add_variables(ngx_conf_t *cf) return NGX_OK; } + static ngx_int_t ngx_http_ip2region_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data) { + ngx_http_ip2region_conf_t *ip2region_conf; + struct sockaddr_in *sin; + char region[512] = {'\0'}; + int err; + unsigned int ip; + +#if (NGX_HAVE_INET6) + u_char *p; + in_addr_t addr; + struct sockaddr_in6 *sin6; +#endif + + ip2region_conf = ngx_http_get_module_main_conf(r, ngx_http_ip2region_module); + + if (ip2region_conf->ip2region_searcher != NULL) { + + switch (r->connection->sockaddr->sa_family) { + case AF_INET: + sin = (struct sockaddr_in *) r->connection->sockaddr; + ip = htonl(sin->sin_addr.s_addr); + err = xdb_search(&ip2region_conf->ip2region_searcher->searcher, ip, region, sizeof(region)); + if (err == 0) { + v->data = (unsigned char *)region; + v->len = strlen(region); + return NGX_OK; + } + break; + +#if (NGX_HAVE_INET6) + + case AF_INET6: + sin6 = (struct sockaddr_in6 *) r->connection->sockaddr; + p = sin6->sin6_addr.s6_addr; + + if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) { + addr = p[12] << 24; + addr += p[13] << 16; + addr += p[14] << 8; + addr += p[15]; + + ip = htonl(addr); + err = xdb_search(&ip2region_conf->ip2region_searcher->searcher, ip, region, sizeof(region)); + if (err == 0) { + v->data = (unsigned char *)region; + v->len = strlen(region); + return NGX_OK; + } + } + break; + +#endif + + } + } + v->not_found = 1; return NGX_OK; } @@ -143,5 +273,73 @@ ngx_http_ip2region_variable(ngx_http_request_t *r, static void ngx_http_ip2region_cleanup(void *data) { - // ngx_http_ip2region_conf_t *uacf = data; -} \ No newline at end of file + ngx_http_ip2region_conf_t *ip2region_conf = data; + + if(ip2region_conf->ip2region_searcher != NULL) { + xdb_close(&ip2region_conf->ip2region_searcher->searcher); + + // check and free the vector index + if (ip2region_conf->ip2region_searcher->v_index != NULL) { + xdb_close_vector_index(ip2region_conf->ip2region_searcher->v_index); + ip2region_conf->ip2region_searcher->v_index = NULL; + } + + // check and free the content buffer + if (ip2region_conf->ip2region_searcher->c_buffer != NULL) { + xdb_close_content(ip2region_conf->ip2region_searcher->c_buffer); + ip2region_conf->ip2region_searcher->c_buffer = NULL; + } + + ip2region_conf->ip2region_searcher = NULL; + } +} + + +static ngx_int_t +ngx_http_ua_parser_test_full_name(char *name) +{ +#if (NGX_WIN32) + u_char c0, c1; + + c0 = name[0]; + + if (strlen(name) < 2) { + if (c0 == '/') { + return 2; + } + + return NGX_DECLINED; + } + + c1 = name[1]; + + if (c1 == ':') { + c0 |= 0x20; + + if ((c0 >= 'a' && c0 <= 'z')) { + return NGX_OK; + } + + return NGX_DECLINED; + } + + if (c1 == '/') { + return NGX_OK; + } + + if (c0 == '/') { + return 2; + } + + return NGX_DECLINED; + +#else + + if (name[0] == '/') { + return NGX_OK; + } + + return NGX_DECLINED; + +#endif +} diff --git a/binding/nginx/src/ngx_http_ip2region_module.h b/binding/nginx/src/ngx_http_ip2region_module.h index 7d504de..90bb485 100644 --- a/binding/nginx/src/ngx_http_ip2region_module.h +++ b/binding/nginx/src/ngx_http_ip2region_module.h @@ -1,14 +1,19 @@ +/* + * Created by Wu Jian Ping on - 2023/03/30. + */ + #ifndef __NGX_HTTP_IP2REGION_MODULE_INCLUDED__ #define __NGX_HTTP_IP2REGION_MODULE_INCLUDED__ #include #include #include +#include "xdb_searcher.h" typedef struct { - // xdb_searcher_t searcher; - // xdb_vector_index_t *v_index; - // xdb_content_t *c_buffer; + xdb_searcher_t searcher; + xdb_vector_index_t *v_index; + xdb_content_t *c_buffer; } ip2region_searcher_t; typedef struct { diff --git a/binding/nginx/src/xdb_searcher.c b/binding/nginx/src/xdb_searcher.c index e895972..e14ff42 100644 --- a/binding/nginx/src/xdb_searcher.c +++ b/binding/nginx/src/xdb_searcher.c @@ -6,6 +6,9 @@ // @Author Lion // @Date 2022/06/27 +// @Modified By Wu Jian Ping +// @Date 2023/03/30 + #include "sys/time.h" #include "xdb_searcher.h" diff --git a/binding/nginx/src/xdb_searcher.h b/binding/nginx/src/xdb_searcher.h index ca553dc..4656b49 100644 --- a/binding/nginx/src/xdb_searcher.h +++ b/binding/nginx/src/xdb_searcher.h @@ -6,6 +6,9 @@ // @Author Lion // @Date 2022/06/27 +// @Modified By Wu Jian Ping +// @Date 2023/03/30 + #ifndef C_XDB_SEARCHER_H #define C_XDB_SEARCHER_H From 9578720cd1ad05272e81ea3c5455af76843519a7 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 21:34:06 +0800 Subject: [PATCH 03/16] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/nginx/README.md | 32 +++++++++++++++++-- binding/nginx/src/ngx_http_ip2region_module.c | 6 ++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index 15c2ec2..be91805 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -4,11 +4,39 @@ ```shell $ ./configure \ - --prefix=/Users/wujp/Documents/workspaces/open-source/nginx-1.23.4/.nginx \ - --add-module=/Users/wujp/Documents/workspaces/open-source/ip2region/binding/nginx + --prefix=$(PWD)/../build \ + --add-module=$(PWD)/../ip2region/binding/nginx $ make $ make install ``` +```nginx +... +http { + + log_format json_access_log escape=json '{' + '"remote_addr": "$remote_addr", ' + '"region": "$ip2region", ' + '"http_x_forwarded_for": "$http_x_forwarded_for"' + '}'; + + access_log logs/access.log json_access_log; + + # 设置xdb文件路径 + ip2region ip2region.xdb; + + server { + listen 80; + server_name localhost; + + location / { + root html; + index index.html index.htm; + } + } +} + +``` + Made with ♥ by Wu Jian Ping diff --git a/binding/nginx/src/ngx_http_ip2region_module.c b/binding/nginx/src/ngx_http_ip2region_module.c index ac148f3..3795006 100644 --- a/binding/nginx/src/ngx_http_ip2region_module.c +++ b/binding/nginx/src/ngx_http_ip2region_module.c @@ -10,7 +10,7 @@ static ngx_int_t ngx_http_ip2region_add_variables(ngx_conf_t *cf); static void *ngx_http_ip2region_create_conf(ngx_conf_t *cf); static void ngx_http_ip2region_cleanup(void *data); -static char *ngx_http_ip2region_processor(ngx_conf_t *cf, +static char *ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static ngx_int_t ngx_http_ip2region_variable(ngx_http_request_t *r, @@ -31,7 +31,7 @@ static ngx_http_module_t ngx_http_ip2region_ctx = { static ngx_command_t ngx_http_ip2region_commands[] = { { ngx_string("ip2region"), NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE12, - ngx_http_ip2region_processor, + ngx_http_ip2region_init, NGX_HTTP_MAIN_CONF_OFFSET, 0, NULL }, @@ -67,7 +67,7 @@ static ngx_http_variable_t ngx_http_ip2region_vars[] = { static char * -ngx_http_ip2region_processor(ngx_conf_t *cf, ngx_command_t *cmd, +ngx_http_ip2region_init(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_http_ip2region_conf_t *ip2region_cf; From d96d54bbaa934ae9b1c3f11b417e0902b42b3c1d Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 21:58:40 +0800 Subject: [PATCH 04/16] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/nginx/README.md | 14 ++++++------ binding/nginx/config | 22 +++++++++++++------ binding/nginx/src/ngx_http_ip2region_module.c | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index be91805..90dd834 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -15,16 +15,16 @@ $ make install ... http { - log_format json_access_log escape=json '{' - '"remote_addr": "$remote_addr", ' - '"region": "$ip2region", ' - '"http_x_forwarded_for": "$http_x_forwarded_for"' - '}'; + log_format main escape=json '{' + '"remote_addr": "$remote_addr", ' + '"region": "$ip2region", ' + '"http_x_forwarded_for": "$http_x_forwarded_for"' + '}'; - access_log logs/access.log json_access_log; + access_log logs/access.log main; # 设置xdb文件路径 - ip2region ip2region.xdb; + ip2region_db ip2region.xdb; server { listen 80; diff --git a/binding/nginx/config b/binding/nginx/config index 9d8f5f7..08fe212 100644 --- a/binding/nginx/config +++ b/binding/nginx/config @@ -1,14 +1,22 @@ -ngx_addon_name=ngx_ip2region_module +ngx_addon_name=ngx_http_ip2region_module -NGX_IP2REGION_SRCS=" \ +NGX_HTTP_IP2REGION_SRCS=" \ $ngx_addon_dir/src/ngx_http_ip2region_module.c \ $ngx_addon_dir/src/xdb_searcher.c \ " -NGX_IP2REGION_DEPS=" \ +NGX_HTTP_IP2REGION_DEPS=" \ " -HTTP_MODULES="$HTTP_MODULES ngx_http_ip2region_module" -# STREAM_MODULES="ngx_stream_ip2region_module $STREAM_MODULES" -NGX_ADDON_SRCS="$NGX_ADDON_SRCS $NGX_IP2REGION_SRCS" -NGX_ADDON_DEPS="$NGX_ADDON_DEPS $NGX_IP2REGION_DEPS" +if test -n "$ngx_module_link"; then + ngx_module_type=HTTP + ngx_module_name=$ngx_addon_name + ngx_module_deps="$NGX_HTTP_IP2REGION_DEPS" + ngx_module_srcs="$NGX_HTTP_IP2REGION_SRCS" + + . auto/module +else + HTTP_MODULES="$HTTP_MODULES $ngx_addon_name" + NGX_ADDON_DEPS="$NGX_ADDON_DEPS $NGX_HTTP_IP2REGION_DEPS" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $NGX_HTTP_IP2REGION_SRCS" +fi \ No newline at end of file diff --git a/binding/nginx/src/ngx_http_ip2region_module.c b/binding/nginx/src/ngx_http_ip2region_module.c index 3795006..aa3e3e7 100644 --- a/binding/nginx/src/ngx_http_ip2region_module.c +++ b/binding/nginx/src/ngx_http_ip2region_module.c @@ -29,7 +29,7 @@ static ngx_http_module_t ngx_http_ip2region_ctx = { static ngx_command_t ngx_http_ip2region_commands[] = { - { ngx_string("ip2region"), + { ngx_string("ip2region_db"), NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE12, ngx_http_ip2region_init, NGX_HTTP_MAIN_CONF_OFFSET, From 0bc0d9ea87f8f34c2df9428763eb3d30ca2ddf5a Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 22:08:49 +0800 Subject: [PATCH 05/16] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/nginx/README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index 90dd834..152a022 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -3,14 +3,24 @@ ## 编译 ```shell +$ mkdir ip2region +$ wget https://nginx.org/download/nginx-1.23.4.tar.gz +$ tar -zxf nginx-1.23.4.tar.gz && rm -rf nginx-1.23.4.tar.gz +$ git clone https://github.com/lionsoul2014/ip2region.git +$ cd nginx-1.23.4 $ ./configure \ - --prefix=$(PWD)/../build \ --add-module=$(PWD)/../ip2region/binding/nginx - $ make $ make install ``` +## Nginx 配置 + +> Syntax: `ip2region_db xdb_file_path` [cache_policy 可选]; +> Context: http + +cache_policy: `file` or `vectorIndex` or `content`, default: `content` + ```nginx ... http { @@ -39,4 +49,8 @@ http { ``` +$ ./configure \ + --prefix=$(PWD)/../build \ + --add-module=$(PWD)/../ip2region/binding/nginx + Made with ♥ by Wu Jian Ping From a8571d4a993775dcb2e40b1a6c9cc583f3483c64 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 22:11:55 +0800 Subject: [PATCH 06/16] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/nginx/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index 152a022..0be617d 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -1,9 +1,10 @@ # nginx-ip2region -## 编译 +## Build ```shell -$ mkdir ip2region +$ mkdir -p workspace +$ cd workspace $ wget https://nginx.org/download/nginx-1.23.4.tar.gz $ tar -zxf nginx-1.23.4.tar.gz && rm -rf nginx-1.23.4.tar.gz $ git clone https://github.com/lionsoul2014/ip2region.git @@ -14,9 +15,9 @@ $ make $ make install ``` -## Nginx 配置 +## Nginx conf -> Syntax: `ip2region_db xdb_file_path` [cache_policy 可选]; +> Syntax: `ip2region_db xdb_file_path [cache_policy 可选]`; > Context: http cache_policy: `file` or `vectorIndex` or `content`, default: `content` @@ -33,7 +34,7 @@ http { access_log logs/access.log main; - # 设置xdb文件路径 + # set xdb file path ip2region_db ip2region.xdb; server { From e8595bf525ebd3175c2fe7ebe1f026c7a3cad63d Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 22:12:37 +0800 Subject: [PATCH 07/16] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/nginx/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index 0be617d..f5a83fe 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -50,8 +50,4 @@ http { ``` -$ ./configure \ - --prefix=$(PWD)/../build \ - --add-module=$(PWD)/../ip2region/binding/nginx - Made with ♥ by Wu Jian Ping From 0b5148be767e814fb443aac10fedd5d9b7672ed6 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 22:13:03 +0800 Subject: [PATCH 08/16] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/nginx/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index f5a83fe..fa2304d 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -1,6 +1,6 @@ # nginx-ip2region -## Build +## build ```shell $ mkdir -p workspace @@ -15,7 +15,7 @@ $ make $ make install ``` -## Nginx conf +## nginx conf > Syntax: `ip2region_db xdb_file_path [cache_policy 可选]`; > Context: http From 6607c9e61c3b8da352526b3a1bb33a122e6d6ede Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 22:15:41 +0800 Subject: [PATCH 09/16] . --- binding/nginx/t/{http_ua_parser.t => http_ip2region.t} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename binding/nginx/t/{http_ua_parser.t => http_ip2region.t} (100%) diff --git a/binding/nginx/t/http_ua_parser.t b/binding/nginx/t/http_ip2region.t similarity index 100% rename from binding/nginx/t/http_ua_parser.t rename to binding/nginx/t/http_ip2region.t From 885c4d815fedc7216deb2f86154afc85c3c74ae5 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 22:16:56 +0800 Subject: [PATCH 10/16] . --- binding/nginx/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index fa2304d..9aa1bd0 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -17,7 +17,7 @@ $ make install ## nginx conf -> Syntax: `ip2region_db xdb_file_path [cache_policy 可选]`; +> Syntax: `ip2region_db xdb_file_path [cache_policy Optional]`; > Context: http cache_policy: `file` or `vectorIndex` or `content`, default: `content` From 09b05a6026fae83d8c87cf60420642b3a4cdb96d Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Thu, 30 Mar 2023 22:19:42 +0800 Subject: [PATCH 11/16] . --- binding/nginx/src/ngx_http_ip2region_module.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/binding/nginx/src/ngx_http_ip2region_module.h b/binding/nginx/src/ngx_http_ip2region_module.h index 90bb485..7bfbb7c 100644 --- a/binding/nginx/src/ngx_http_ip2region_module.h +++ b/binding/nginx/src/ngx_http_ip2region_module.h @@ -2,8 +2,8 @@ * Created by Wu Jian Ping on - 2023/03/30. */ -#ifndef __NGX_HTTP_IP2REGION_MODULE_INCLUDED__ -#define __NGX_HTTP_IP2REGION_MODULE_INCLUDED__ +#ifndef __NGX_HTTP_IP2REGION_MODULE_H_INCLUDED__ +#define __NGX_HTTP_IP2REGION_MODULE_H_INCLUDED__ #include #include @@ -11,9 +11,9 @@ #include "xdb_searcher.h" typedef struct { - xdb_searcher_t searcher; - xdb_vector_index_t *v_index; - xdb_content_t *c_buffer; + xdb_searcher_t searcher; + xdb_vector_index_t *v_index; + xdb_content_t *c_buffer; } ip2region_searcher_t; typedef struct { From fed6788fe4e4fd2fb32786fada0b4c1c37e664d1 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Fri, 31 Mar 2023 00:28:13 +0800 Subject: [PATCH 12/16] enhance readme --- binding/nginx/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index 9aa1bd0..de50b7d 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -22,6 +22,8 @@ $ make install cache_policy: `file` or `vectorIndex` or `content`, default: `content` +Edit `nginx.conf` add `ip2region_db` directive + ```nginx ... http { @@ -50,4 +52,14 @@ http { ``` +Copy `ip2region.xdb` to `nginx/config` folder, then restart nginx, the `region` data stored in `ip2region` variable + +nginx access log sample + +```log +{"remote_addr": "127.0.0.1", "region": "0|0|0|内网IP|内网IP", "http_x_forwarded_for": ""} +{"remote_addr": "127.0.0.1", "region": "0|0|0|内网IP|内网IP", "http_x_forwarded_for": ""} + +``` + Made with ♥ by Wu Jian Ping From 4ddd785bf0a03159ae5dec8ae9cf6fba17bfc03c Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Fri, 31 Mar 2023 00:32:05 +0800 Subject: [PATCH 13/16] cleanup --- binding/nginx/README.md | 3 +++ binding/nginx/src/xdb_searcher.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index de50b7d..659cff1 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -38,6 +38,9 @@ http { # set xdb file path ip2region_db ip2region.xdb; + # ip2region_db ip2region.xdb vectorIndex; + # ip2region_db ip2region.xdb file; + # ip2region_db ip2region.xdb content; server { listen 80; diff --git a/binding/nginx/src/xdb_searcher.c b/binding/nginx/src/xdb_searcher.c index e14ff42..d763b38 100644 --- a/binding/nginx/src/xdb_searcher.c +++ b/binding/nginx/src/xdb_searcher.c @@ -13,7 +13,7 @@ #include "xdb_searcher.h" // internal function prototype define -static inline int read(xdb_searcher_t *, long offset, char *, size_t length); +static int read(xdb_searcher_t *, long offset, char *, size_t length); static inline int xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { memset(xdb, 0x00, sizeof(xdb_searcher_t)); From a406f0e6369d576f6a4223f87e47d2a6af66b976 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Fri, 31 Mar 2023 18:14:24 +0800 Subject: [PATCH 14/16] =?UTF-8?q?1.=20c=20binding=20=E6=94=AF=E6=8C=81MAC?= =?UTF-8?q?=E6=9E=84=E5=BB=BA=202.=20c=20binding=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=9E=84=E5=BB=BA=E9=9D=99=E6=80=81=E5=BA=93(nginx=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E9=9C=80=E8=A6=81=E7=94=A8=E5=88=B0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + binding/c/Makefile | 10 + binding/c/xdb_searcher.h | 2 +- binding/nginx/README.md | 15 +- binding/nginx/config | 4 +- binding/nginx/src/ngx_http_ip2region_module.h | 2 +- binding/nginx/src/xdb_searcher.c | 405 ------------------ binding/nginx/src/xdb_searcher.h | 130 ------ 8 files changed, 27 insertions(+), 542 deletions(-) delete mode 100644 binding/nginx/src/xdb_searcher.c delete mode 100644 binding/nginx/src/xdb_searcher.h diff --git a/.gitignore b/.gitignore index baa7e0e..33056e3 100644 --- a/.gitignore +++ b/.gitignore @@ -88,3 +88,4 @@ target #vscode .vscode +build diff --git a/binding/c/Makefile b/binding/c/Makefile index e444283..ea9523b 100644 --- a/binding/c/Makefile +++ b/binding/c/Makefile @@ -6,9 +6,19 @@ xdb_searcher: xdb_searcher.h xdb_searcher.c main.c util_test: xdb_searcher.h xdb_searcher.c util_test.c gcc -g -O2 -I./ xdb_searcher.c util_test.c -o util_test +xdb_searcher.o: xdb_searcher.c + gcc -c -o xdb_searcher.o xdb_searcher.c + +xdb_searcher_lib: xdb_searcher.o + mkdir -p build/lib + mkdir -p build/include + ar -rc build/lib/libxdb_searcher.a `find . -name *.o` + cp xdb_searcher.h build/include + clean: find ./ -name \*.o | xargs rm -f find ./ -name util_test | xargs rm -f find ./ -name xdb_searcher | xargs rm -f + rm -rf build .PHONY: all clean xdb_searcher util_test \ No newline at end of file diff --git a/binding/c/xdb_searcher.h b/binding/c/xdb_searcher.h index 12808bc..8be9429 100644 --- a/binding/c/xdb_searcher.h +++ b/binding/c/xdb_searcher.h @@ -17,7 +17,7 @@ # define XDB_PUBLIC(type) extern __declspec(dllexport) type # define XDB_PRIVATE(type) static type # define XDB_WINDOWS -#elif ( defined(linux) || defined(_UNIX) ) +#elif ( defined(linux) || defined(_UNIX) || defined(__APPLE__) ) # define XDB_PUBLIC(type) extern type # define XDB_PRIVATE(type) static inline type # define XDB_LINUX diff --git a/binding/nginx/README.md b/binding/nginx/README.md index 659cff1..76c51ff 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -8,9 +8,13 @@ $ cd workspace $ wget https://nginx.org/download/nginx-1.23.4.tar.gz $ tar -zxf nginx-1.23.4.tar.gz && rm -rf nginx-1.23.4.tar.gz $ git clone https://github.com/lionsoul2014/ip2region.git -$ cd nginx-1.23.4 -$ ./configure \ - --add-module=$(PWD)/../ip2region/binding/nginx +$ cd ip2region/binding/c +$ make xdb_searcher_lib +$ cd ../../../nginx-1.23.4 +$ ./configure \ + --add-module=$(PWD)/../ip2region/binding/nginx \ + --with-cc-opt="-I $(PWD)/../ip2region/binding/c/build/include" \ + --with-ld-opt="-L$(PWD)/../ip2region/binding/c/build/lib" $ make $ make install ``` @@ -65,4 +69,9 @@ nginx access log sample ``` +./configure \ + --prefix=$(PWD)/../build/ \ + --add-module=$(PWD)/../ip2region/binding/nginx \ + --with-http_realip_module + Made with ♥ by Wu Jian Ping diff --git a/binding/nginx/config b/binding/nginx/config index 08fe212..1da0253 100644 --- a/binding/nginx/config +++ b/binding/nginx/config @@ -1,8 +1,7 @@ ngx_addon_name=ngx_http_ip2region_module NGX_HTTP_IP2REGION_SRCS=" \ - $ngx_addon_dir/src/ngx_http_ip2region_module.c \ - $ngx_addon_dir/src/xdb_searcher.c \ + $ngx_addon_dir/src/ngx_http_ip2region_module.c \ " NGX_HTTP_IP2REGION_DEPS=" \ @@ -13,6 +12,7 @@ if test -n "$ngx_module_link"; then ngx_module_name=$ngx_addon_name ngx_module_deps="$NGX_HTTP_IP2REGION_DEPS" ngx_module_srcs="$NGX_HTTP_IP2REGION_SRCS" + ngx_module_libs="-lxdb_searcher" . auto/module else diff --git a/binding/nginx/src/ngx_http_ip2region_module.h b/binding/nginx/src/ngx_http_ip2region_module.h index 7bfbb7c..8861cee 100644 --- a/binding/nginx/src/ngx_http_ip2region_module.h +++ b/binding/nginx/src/ngx_http_ip2region_module.h @@ -8,7 +8,7 @@ #include #include #include -#include "xdb_searcher.h" +#include typedef struct { xdb_searcher_t searcher; diff --git a/binding/nginx/src/xdb_searcher.c b/binding/nginx/src/xdb_searcher.c deleted file mode 100644 index d763b38..0000000 --- a/binding/nginx/src/xdb_searcher.c +++ /dev/null @@ -1,405 +0,0 @@ -// Copyright 2022 The Ip2Region Authors. All rights reserved. -// Use of this source code is governed by a Apache2.0-style -// license that can be found in the LICENSE file. - -// --- -// @Author Lion -// @Date 2022/06/27 - -// @Modified By Wu Jian Ping -// @Date 2023/03/30 - -#include "sys/time.h" -#include "xdb_searcher.h" - -// internal function prototype define -static int read(xdb_searcher_t *, long offset, char *, size_t length); - -static inline int xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) { - memset(xdb, 0x00, sizeof(xdb_searcher_t)); - - // check the content buffer first - if (c_buffer != NULL) { - xdb->v_index = NULL; - xdb->content = c_buffer; - return 0; - } - - // open the xdb binary file - FILE *handle = fopen(db_path, "rb"); - if (handle == NULL) { - return 1; - } - - xdb->handle = handle; - xdb->v_index = v_index; - - return 0; -} - -// xdb searcher new api define -int xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) { - return xdb_new_base(xdb, db_path, NULL, NULL); -} - -int xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) { - return xdb_new_base(xdb, db_path, v_index, NULL); -} - -int xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) { - return xdb_new_base(xdb, NULL, NULL, c_buffer); -} - -void xdb_close(void *ptr) { - xdb_searcher_t *xdb = (xdb_searcher_t *) ptr; - if (xdb->handle != NULL) { - fclose(xdb->handle); - xdb->handle = NULL; - } -} - -// --- xdb searcher search api define - -int xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) { - unsigned int ip = 0; - int errcode = xdb_check_ip(str_ip, &ip); - if (errcode != 0) { - return 10 + errcode; - } else { - return xdb_search(xdb, ip, region_buffer, length); - } -} - -int xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) { - int il0, il1, idx, err, l, h, m; - unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr; - size_t data_len; - char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size]; - - // reset the io counter - xdb->io_count = 0; - - // locate the segment index block based on the vector index - il0 = ((int) (ip >> 24)) & 0xFF; - il1 = ((int) (ip >> 16)) & 0xFF; - idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size; - if (xdb->v_index != NULL) { - s_ptr = xdb_get_uint(xdb->v_index->buffer, idx); - e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4); - } else if (xdb->content != NULL) { - s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx); - e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4); - } else { - err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer)); - if (err != 0) { - return 10 + err; - } - - s_ptr = xdb_get_uint(vector_buffer, 0); - e_ptr = xdb_get_uint(vector_buffer, 4); - } - - // printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr); - // binary search to get the final region info - data_len = 0, data_ptr = 0; - l = 0, h = ((int) (e_ptr - s_ptr)) / xdb_segment_index_size; - while (l <= h) { - m = (l + h) >> 1; - p = s_ptr + m * xdb_segment_index_size; - - // read the segment index item - err = read(xdb, p, segment_buffer, sizeof(segment_buffer)); - if (err != 0) { - return 20 + err; - } - - // decode the data fields as needed - sip = xdb_get_uint(segment_buffer, 0); - if (ip < sip) { - h = m - 1; - } else { - eip = xdb_get_uint(segment_buffer, 4); - if (ip > eip) { - l = m + 1; - } else { - data_len = xdb_get_ushort(segment_buffer, 8); - data_ptr = xdb_get_uint(segment_buffer, 10); - break; - } - } - } - - // printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr); - if (data_len == 0) { - region_buffer[0] = '\0'; - return 0; - } - - // buffer length checking - if (data_len >= length) { - return 1; - } - - err = read(xdb, data_ptr, region_buffer, data_len); - if (err != 0) { - return 30 + err; - } - - // auto append a NULL-end - region_buffer[data_len] = '\0'; - return 0; -} - -static inline int read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) { - // check the xdb content cache first - if (xdb->content != NULL) { - memcpy(buffer, xdb->content->buffer + offset, length); - return 0; - } - - // seek to the offset - if (fseek(xdb->handle, offset, SEEK_SET) == -1) { - return 1; - } - - xdb->io_count++; - if (fread(buffer, 1, length, xdb->handle) != length) { - return 2; - } - - return 0; -} - -int xdb_get_io_count(xdb_searcher_t *xdb) { - return xdb->io_count; -} - - -// --- buffer load util functions -xdb_header_t* xdb_load_header(FILE *handle) { - xdb_header_t *header; - unsigned int size = xdb_header_info_length; - - // entry alloc - header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t)); - if (header == NULL) { - return NULL; - } - - if (fseek(handle, 0, SEEK_SET) == -1) { - xdb_free(header); - return NULL; - } - - if (fread(header->buffer, 1,size, handle) != size) { - xdb_free(header); - return NULL; - } - - // fill the fields - header->length = size; - header->version = (unsigned short) xdb_get_ushort(header->buffer, 0); - header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2); - header->created_at = xdb_get_uint(header->buffer, 4); - header->start_index_ptr = xdb_get_uint(header->buffer, 8); - header->end_index_ptr = xdb_get_uint(header->buffer,12); - - return header; -} - -xdb_header_t* xdb_load_header_from_file(const char *db_path) { - xdb_header_t *header; - FILE *handle = fopen(db_path, "rb"); - if (handle == NULL) { - return NULL; - } - - header = xdb_load_header(handle); - fclose(handle); - return header; -} - -void xdb_close_header(void *ptr) { - xdb_header_t *header = (xdb_header_t *) ptr; - if (header->length > 0) { - header->length = 0; - xdb_free(header); - } -} - -// --- vector index - -xdb_vector_index_t* xdb_load_vector_index(FILE *handle) { - xdb_vector_index_t *v_index; - unsigned int size = xdb_vector_index_length; - - // seek to the vector index offset - if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) { - return NULL; - } - - // do the buffer read - v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t)); - if (v_index == NULL) { - return NULL; - } - - v_index->length = size; - if (fread(v_index->buffer, 1, size, handle) != size) { - xdb_free(v_index); - return NULL; - } - - return v_index; -} - -xdb_vector_index_t* xdb_load_vector_index_from_file(const char *db_path) { - xdb_vector_index_t *v_index; - FILE *handle = fopen(db_path, "rb"); - if (handle == NULL) { - return NULL; - } - - v_index = xdb_load_vector_index(handle); - fclose(handle); - return v_index; -} - -void xdb_close_vector_index(void *ptr) { - xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr; - if (v_index->length > 0) { - v_index->length = 0; - xdb_free(v_index); - } -} - -// --- content buffer - -xdb_content_t * xdb_load_content(FILE *handle) { - unsigned int size; - xdb_content_t *content; - - // determine the file size - if (fseek(handle, 0, SEEK_END) == -1) { - return NULL; - } - - size = (unsigned int) ftell(handle); - if (fseek(handle, 0, SEEK_SET) == -1) { - return NULL; - } - - // do the file read - content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t)); - if (content == NULL) { - return NULL; - } - - // do the buffer alloc - content->buffer = (char *) xdb_malloc(size); - if (content->buffer == NULL) { - xdb_free(content); - return NULL; - } - - // read the content into the buffer - content->length = size; - if (fread(content->buffer, 1, size, handle) != size) { - xdb_free(content); - return NULL; - } - - return content; -} - -xdb_content_t* xdb_load_content_from_file(const char *db_path) { - xdb_content_t *content; - FILE *handle = fopen(db_path, "rb"); - if (handle == NULL) { - return NULL; - } - - content = xdb_load_content(handle); - fclose(handle); - return content; -} - -void xdb_close_content(void *ptr) { - xdb_content_t *content = (xdb_content_t *) ptr; - if (content->length > 0) { - content->length = 0; - xdb_free(content->buffer); - content->buffer = NULL; - xdb_free(content); - } -} - -// --- End - -// get unsigned long (4bytes) from a specified buffer start from the specified offset -unsigned int xdb_get_uint(const char *buffer, int offset) { - return ( - ((buffer[offset ]) & 0x000000FF) | - ((buffer[offset+1] << 8) & 0x0000FF00) | - ((buffer[offset+2] << 16) & 0x00FF0000) | - ((buffer[offset+3] << 24) & 0xFF000000) - ); -} - -// get unsigned short (2bytes) from a specified buffer start from the specified offset -int xdb_get_ushort(const char *buffer, int offset) { - return ( - ((buffer[offset ]) & 0x000000FF) | - ((buffer[offset+1] << 8) & 0x0000FF00) - ); -} - -// string ip to unsigned int -static int shiftIndex[4] = {24, 16, 8, 0}; -int xdb_check_ip(const char *src_ip, unsigned int *dst_ip) { - char c; - int i, n, ip = 0; - const char *ptr = src_ip; - for (i = 0; i < 4; i++) { - n = 0; - while (1) { - c = *ptr; - ptr++; - if (c >= '0' && c <= '9') { - n *= 10; - n += c - '0'; - } else if ((i < 3 && c == '.') || i == 3) { - // stopping at the '.' but ignore the tailing chars - // after the 3rd one (auto clean the tailing none-integer ?). - break; - } else { - return 1; - } - } - - if (n > 0xFF) { - return 2; - } - - ip |= (n << shiftIndex[i]); - } - - *dst_ip = ip; - return 0; -} - -// unsigned int ip to string ip -void xdb_long2ip(unsigned int ip, char *buffer) { - sprintf(buffer, "%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF); -} - -// get the middle ip of a and b -unsigned int xdb_mip(unsigned long a, unsigned long b) { - return (unsigned int) ((a + b) >> 1); -} - -long xdb_now() { - struct timeval c_time; - gettimeofday(&c_time, NULL); - return c_time.tv_sec * (int)1e6 + c_time.tv_usec; -} diff --git a/binding/nginx/src/xdb_searcher.h b/binding/nginx/src/xdb_searcher.h deleted file mode 100644 index 4656b49..0000000 --- a/binding/nginx/src/xdb_searcher.h +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2022 The Ip2Region Authors. All rights reserved. -// Use of this source code is governed by a Apache2.0-style -// license that can be found in the LICENSE file. - -// --- -// @Author Lion -// @Date 2022/06/27 - -// @Modified By Wu Jian Ping -// @Date 2023/03/30 - -#ifndef C_XDB_SEARCHER_H -#define C_XDB_SEARCHER_H - -#include -#include -#include - -#define xdb_calloc( _blocks, _bytes ) calloc( _blocks, _bytes ) -#define xdb_malloc( _bytes ) malloc( _bytes ) -#define xdb_free( _ptr ) free( _ptr ) - -// public constants define -#define xdb_header_info_length 256 -#define xdb_vector_index_rows 256 -#define xdb_vector_index_cols 256 -#define xdb_vector_index_size 8 -#define xdb_segment_index_size 14 - -// cache of vector_index_row × vector_index_rows × vector_index_size -#define xdb_vector_index_length 524288 - -typedef struct { - unsigned short version; - unsigned short index_policy; - unsigned int created_at; - unsigned int start_index_ptr; - unsigned int end_index_ptr; - - // the original buffer - unsigned int length; - char buffer[xdb_header_info_length]; -} xdb_header_t; - -xdb_header_t* xdb_load_header(FILE *); - -xdb_header_t *xdb_load_header_from_file(const char *); - -void xdb_close_header(void *); - -// --- vector index buffer -struct xdb_vector_index { - unsigned int length; - char buffer[xdb_vector_index_length]; -}; -typedef struct xdb_vector_index xdb_vector_index_t; - -xdb_vector_index_t* xdb_load_vector_index(FILE *); - -xdb_vector_index_t * xdb_load_vector_index_from_file(const char *); - -void xdb_close_vector_index(void *); - -// --- content buffer -typedef struct { - unsigned int length; - char *buffer; -} xdb_content_t; - -xdb_content_t* xdb_load_content(FILE *); - -xdb_content_t* xdb_load_content_from_file(const char *); - -void xdb_close_content(void *); - -// --- End buffer load - -// xdb searcher structure -typedef struct xdb_searcher_entry { - FILE *handle; - - // header info - const char *header; - int io_count; - - // vector index buffer cache. - // preload the vector index will reduce the number of IO operations - // thus speedup the search process. - const xdb_vector_index_t *v_index; - - // content buffer. - // cache the whole xdb content. - const xdb_content_t *content; -} xdb_searcher_t; - -// xdb searcher new api define -int xdb_new_with_file_only(xdb_searcher_t *, const char *); - -int xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *); - -int xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *); - -void xdb_close(void *); - -// xdb searcher search api define -int xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t); - -int xdb_search(xdb_searcher_t *, unsigned int, char *, size_t); - -int xdb_get_io_count(xdb_searcher_t *); - -// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian -unsigned int xdb_get_uint(const char *, int); - -// get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian -int xdb_get_ushort(const char *, int); - -// check the specified string ip and convert it to an unsigned int -int xdb_check_ip(const char *, unsigned int *); - -// unsigned int ip to string ip -void xdb_long2ip(unsigned int, char *); - -// get the middle ip of a and b -unsigned int xdb_mip(unsigned long, unsigned long); - -// get the current time in microseconds -long xdb_now(); - -#endif //C_XDB_SEARCHER_H From 4e588db3decf9a596fc6fe4720bd8cbb89d27c9e Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Fri, 31 Mar 2023 18:19:25 +0800 Subject: [PATCH 15/16] =?UTF-8?q?=E7=A7=BB=E9=99=A4LICENSE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/nginx/LICENSE | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 binding/nginx/LICENSE diff --git a/binding/nginx/LICENSE b/binding/nginx/LICENSE deleted file mode 100644 index e39f9f3..0000000 --- a/binding/nginx/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2023 Wu Jian Ping - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. From 4697640fbc1fc362e083e09cf6ea8c7f9c90a5f4 Mon Sep 17 00:00:00 2001 From: Wu Jian Ping Date: Fri, 31 Mar 2023 18:22:54 +0800 Subject: [PATCH 16/16] cleanup --- binding/nginx/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/binding/nginx/README.md b/binding/nginx/README.md index 76c51ff..e71f9c0 100644 --- a/binding/nginx/README.md +++ b/binding/nginx/README.md @@ -69,9 +69,4 @@ nginx access log sample ``` -./configure \ - --prefix=$(PWD)/../build/ \ - --add-module=$(PWD)/../ip2region/binding/nginx \ - --with-http_realip_module - Made with ♥ by Wu Jian Ping