xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nginx-impl init

This commit is contained in:
Wu Jian Ping 2023-03-30 17:46:38 +08:00
parent 138eea8849
commit b04e477b99
9 changed files with 773 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -85,3 +85,6 @@ target
#erlang #erlang
/binding/erlang/_build /binding/erlang/_build
/binding/erlang/doc /binding/erlang/doc
#vscode
.vscode

21
binding/nginx/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2023 Wu Jian Ping
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

14
binding/nginx/README.md Normal file
View File

@ -0,0 +1,14 @@
# nginx-ip2region
## 编译
```shell
$ ./configure \
--prefix=/Users/wujp/Documents/workspaces/open-source/nginx-1.23.4/.nginx \
--add-module=/Users/wujp/Documents/workspaces/open-source/ip2region/binding/nginx
$ make
$ make install
```
Made with ♥ by Wu Jian Ping

14
binding/nginx/config Normal file
View File

@ -0,0 +1,14 @@
ngx_addon_name=ngx_ip2region_module
NGX_IP2REGION_SRCS=" \
$ngx_addon_dir/src/ngx_http_ip2region_module.c \
$ngx_addon_dir/src/xdb_searcher.c \
"
NGX_IP2REGION_DEPS=" \
"
HTTP_MODULES="$HTTP_MODULES ngx_http_ip2region_module"
# STREAM_MODULES="ngx_stream_ip2region_module $STREAM_MODULES"
NGX_ADDON_SRCS="$NGX_ADDON_SRCS $NGX_IP2REGION_SRCS"
NGX_ADDON_DEPS="$NGX_ADDON_DEPS $NGX_IP2REGION_DEPS"

147
binding/nginx/src/ngx_http_ip2region_module.c Normal file
View File

@ -0,0 +1,147 @@
/*
* Created by Wu Jian Ping on - 2023/03/20.
*/
#include "ngx_http_ip2region_module.h"
static ngx_int_t ngx_http_ip2region_add_variables(ngx_conf_t *cf);
static void *ngx_http_ip2region_create_conf(ngx_conf_t *cf);
static void ngx_http_ip2region_cleanup(void *data);
static char *ngx_http_ip2region_processor(ngx_conf_t *cf,
ngx_command_t *cmd, void *conf);
static ngx_int_t ngx_http_ip2region_variable(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
static ngx_http_module_t ngx_http_ip2region_ctx = {
ngx_http_ip2region_add_variables, /* pre configuration */
NULL, /* post configuration */
ngx_http_ip2region_create_conf, /* create main configuration */
NULL, /* init main configuration */
NULL, /* create server configuration */
NULL, /* merge server configuration */
NULL, /* create location configuration */
NULL /* merge location configuration */
};
static ngx_command_t ngx_http_ip2region_commands[] = {
{ ngx_string("ip2region"),
NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1,
ngx_http_ip2region_processor,
NGX_HTTP_MAIN_CONF_OFFSET,
0,
NULL },
ngx_null_command
};
/* ngx_module_t is required, otherwise failed at complie time */
ngx_module_t ngx_http_ip2region_module = {
NGX_MODULE_V1,
&ngx_http_ip2region_ctx, /* module context */
ngx_http_ip2region_commands, /* module directives */
NGX_HTTP_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
static ngx_http_variable_t ngx_http_ip2region_vars[] = {
{ ngx_string("ip2region"), NULL,
ngx_http_ip2region_variable,
0, 0, 0 },
ngx_http_null_variable
};
static char *
ngx_http_ip2region_processor(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf)
{
// ngx_http_ip2region_conf_t *uacf = conf;
// ngx_str_t *value;
// if (uacf->ip2region) {
// return "is duplicate";
// }
// value = cf->args->elts;
// uacf->ip2region = open_ip2region_regexes_yaml((char *)value[1].data);
// if(uacf->ip2region == NULL) {
// ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
// "open_ip2region_regexes_yaml(\"%V\") failed", &value[1]);
// return NGX_CONF_ERROR;
// }
return NGX_CONF_OK;
}
static void *
ngx_http_ip2region_create_conf(ngx_conf_t *cf)
{
ngx_pool_cleanup_t *cln;
ngx_http_ip2region_conf_t *conf;
conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_ip2region_conf_t));
if (conf == NULL) {
return NULL;
}
cln = ngx_pool_cleanup_add(cf->pool, 0);
if (cln == NULL) {
return NULL;
}
cln->handler = ngx_http_ip2region_cleanup;
cln->data = conf;
return conf;
}
static ngx_int_t
ngx_http_ip2region_add_variables(ngx_conf_t *cf)
{
ngx_http_variable_t *var;
ngx_http_variable_t *v;
/* the last ngx_http_variable_t's name is ngx_null_string and it's name's len is 0*/
for (v = ngx_http_ip2region_vars; v->name.len; v++) {
var = ngx_http_add_variable(cf, &v->name, v->flags);
if (var == NULL) {
return NGX_ERROR;
}
var->get_handler = v->get_handler;
var->data = v->data;
}
return NGX_OK;
}
static ngx_int_t
ngx_http_ip2region_variable(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data)
{
v->not_found = 1;
return NGX_OK;
}
static void
ngx_http_ip2region_cleanup(void *data)
{
// ngx_http_ip2region_conf_t *uacf = data;
}

18
binding/nginx/src/ngx_http_ip2region_module.h Normal file
View File

@ -0,0 +1,18 @@
#ifndef __NGX_HTTP_IP2REGION_MODULE_INCLUDED__
#define __NGX_HTTP_IP2REGION_MODULE_INCLUDED__
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
typedef struct {
// xdb_searcher_t searcher;
// xdb_vector_index_t *v_index;
// xdb_content_t *c_buffer;
} ip2region_searcher_t;
typedef struct {
ip2region_searcher_t *ip2region_searcher;
} ngx_http_ip2region_conf_t;
#endif

402
binding/nginx/src/xdb_searcher.c Normal file
View File

@ -0,0 +1,402 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// ---
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/27
#include "sys/time.h"
#include "xdb_searcher.h"
// internal function prototype define
static inline int read(xdb_searcher_t *, long offset, char *, size_t length);
static inline int xdb_new_base(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index, const xdb_content_t *c_buffer) {
memset(xdb, 0x00, sizeof(xdb_searcher_t));
// check the content buffer first
if (c_buffer != NULL) {
xdb->v_index = NULL;
xdb->content = c_buffer;
return 0;
}
// open the xdb binary file
FILE *handle = fopen(db_path, "rb");
if (handle == NULL) {
return 1;
}
xdb->handle = handle;
xdb->v_index = v_index;
return 0;
}
// xdb searcher new api define
int xdb_new_with_file_only(xdb_searcher_t *xdb, const char *db_path) {
return xdb_new_base(xdb, db_path, NULL, NULL);
}
int xdb_new_with_vector_index(xdb_searcher_t *xdb, const char *db_path, const xdb_vector_index_t *v_index) {
return xdb_new_base(xdb, db_path, v_index, NULL);
}
int xdb_new_with_buffer(xdb_searcher_t *xdb, const xdb_content_t *c_buffer) {
return xdb_new_base(xdb, NULL, NULL, c_buffer);
}
void xdb_close(void *ptr) {
xdb_searcher_t *xdb = (xdb_searcher_t *) ptr;
if (xdb->handle != NULL) {
fclose(xdb->handle);
xdb->handle = NULL;
}
}
// --- xdb searcher search api define
int xdb_search_by_string(xdb_searcher_t *xdb, const char *str_ip, char *region_buffer, size_t length) {
unsigned int ip = 0;
int errcode = xdb_check_ip(str_ip, &ip);
if (errcode != 0) {
return 10 + errcode;
} else {
return xdb_search(xdb, ip, region_buffer, length);
}
}
int xdb_search(xdb_searcher_t *xdb, unsigned int ip, char *region_buffer, size_t length) {
int il0, il1, idx, err, l, h, m;
unsigned int s_ptr, e_ptr, p, sip, eip, data_ptr;
size_t data_len;
char vector_buffer[xdb_vector_index_size], segment_buffer[xdb_segment_index_size];
// reset the io counter
xdb->io_count = 0;
// locate the segment index block based on the vector index
il0 = ((int) (ip >> 24)) & 0xFF;
il1 = ((int) (ip >> 16)) & 0xFF;
idx = il0 * xdb_vector_index_cols * xdb_vector_index_size + il1 * xdb_vector_index_size;
if (xdb->v_index != NULL) {
s_ptr = xdb_get_uint(xdb->v_index->buffer, idx);
e_ptr = xdb_get_uint(xdb->v_index->buffer, idx + 4);
} else if (xdb->content != NULL) {
s_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx);
e_ptr = xdb_get_uint(xdb->content->buffer, xdb_header_info_length + idx + 4);
} else {
err = read(xdb, xdb_header_info_length + idx, vector_buffer, sizeof(vector_buffer));
if (err != 0) {
return 10 + err;
}
s_ptr = xdb_get_uint(vector_buffer, 0);
e_ptr = xdb_get_uint(vector_buffer, 4);
}
// printf("s_ptr=%u, e_ptr=%u\n", s_ptr, e_ptr);
// binary search to get the final region info
data_len = 0, data_ptr = 0;
l = 0, h = ((int) (e_ptr - s_ptr)) / xdb_segment_index_size;
while (l <= h) {
m = (l + h) >> 1;
p = s_ptr + m * xdb_segment_index_size;
// read the segment index item
err = read(xdb, p, segment_buffer, sizeof(segment_buffer));
if (err != 0) {
return 20 + err;
}
// decode the data fields as needed
sip = xdb_get_uint(segment_buffer, 0);
if (ip < sip) {
h = m - 1;
} else {
eip = xdb_get_uint(segment_buffer, 4);
if (ip > eip) {
l = m + 1;
} else {
data_len = xdb_get_ushort(segment_buffer, 8);
data_ptr = xdb_get_uint(segment_buffer, 10);
break;
}
}
}
// printf("data_len=%u, data_ptr=%u\n", data_len, data_ptr);
if (data_len == 0) {
region_buffer[0] = '\0';
return 0;
}
// buffer length checking
if (data_len >= length) {
return 1;
}
err = read(xdb, data_ptr, region_buffer, data_len);
if (err != 0) {
return 30 + err;
}
// auto append a NULL-end
region_buffer[data_len] = '\0';
return 0;
}
static inline int read(xdb_searcher_t *xdb, long offset, char *buffer, size_t length) {
// check the xdb content cache first
if (xdb->content != NULL) {
memcpy(buffer, xdb->content->buffer + offset, length);
return 0;
}
// seek to the offset
if (fseek(xdb->handle, offset, SEEK_SET) == -1) {
return 1;
}
xdb->io_count++;
if (fread(buffer, 1, length, xdb->handle) != length) {
return 2;
}
return 0;
}
int xdb_get_io_count(xdb_searcher_t *xdb) {
return xdb->io_count;
}
// --- buffer load util functions
xdb_header_t* xdb_load_header(FILE *handle) {
xdb_header_t *header;
unsigned int size = xdb_header_info_length;
// entry alloc
header = (xdb_header_t *) xdb_malloc(sizeof(xdb_header_t));
if (header == NULL) {
return NULL;
}
if (fseek(handle, 0, SEEK_SET) == -1) {
xdb_free(header);
return NULL;
}
if (fread(header->buffer, 1,size, handle) != size) {
xdb_free(header);
return NULL;
}
// fill the fields
header->length = size;
header->version = (unsigned short) xdb_get_ushort(header->buffer, 0);
header->index_policy = (unsigned short) xdb_get_ushort(header->buffer, 2);
header->created_at = xdb_get_uint(header->buffer, 4);
header->start_index_ptr = xdb_get_uint(header->buffer, 8);
header->end_index_ptr = xdb_get_uint(header->buffer,12);
return header;
}
xdb_header_t* xdb_load_header_from_file(const char *db_path) {
xdb_header_t *header;
FILE *handle = fopen(db_path, "rb");
if (handle == NULL) {
return NULL;
}
header = xdb_load_header(handle);
fclose(handle);
return header;
}
void xdb_close_header(void *ptr) {
xdb_header_t *header = (xdb_header_t *) ptr;
if (header->length > 0) {
header->length = 0;
xdb_free(header);
}
}
// --- vector index
xdb_vector_index_t* xdb_load_vector_index(FILE *handle) {
xdb_vector_index_t *v_index;
unsigned int size = xdb_vector_index_length;
// seek to the vector index offset
if (fseek(handle, xdb_header_info_length, SEEK_SET) == -1) {
return NULL;
}
// do the buffer read
v_index = (xdb_vector_index_t *) xdb_malloc(sizeof(xdb_vector_index_t));
if (v_index == NULL) {
return NULL;
}
v_index->length = size;
if (fread(v_index->buffer, 1, size, handle) != size) {
xdb_free(v_index);
return NULL;
}
return v_index;
}
xdb_vector_index_t* xdb_load_vector_index_from_file(const char *db_path) {
xdb_vector_index_t *v_index;
FILE *handle = fopen(db_path, "rb");
if (handle == NULL) {
return NULL;
}
v_index = xdb_load_vector_index(handle);
fclose(handle);
return v_index;
}
void xdb_close_vector_index(void *ptr) {
xdb_vector_index_t *v_index = (xdb_vector_index_t *) ptr;
if (v_index->length > 0) {
v_index->length = 0;
xdb_free(v_index);
}
}
// --- content buffer
xdb_content_t * xdb_load_content(FILE *handle) {
unsigned int size;
xdb_content_t *content;
// determine the file size
if (fseek(handle, 0, SEEK_END) == -1) {
return NULL;
}
size = (unsigned int) ftell(handle);
if (fseek(handle, 0, SEEK_SET) == -1) {
return NULL;
}
// do the file read
content = (xdb_content_t *) xdb_malloc(sizeof(xdb_content_t));
if (content == NULL) {
return NULL;
}
// do the buffer alloc
content->buffer = (char *) xdb_malloc(size);
if (content->buffer == NULL) {
xdb_free(content);
return NULL;
}
// read the content into the buffer
content->length = size;
if (fread(content->buffer, 1, size, handle) != size) {
xdb_free(content);
return NULL;
}
return content;
}
xdb_content_t* xdb_load_content_from_file(const char *db_path) {
xdb_content_t *content;
FILE *handle = fopen(db_path, "rb");
if (handle == NULL) {
return NULL;
}
content = xdb_load_content(handle);
fclose(handle);
return content;
}
void xdb_close_content(void *ptr) {
xdb_content_t *content = (xdb_content_t *) ptr;
if (content->length > 0) {
content->length = 0;
xdb_free(content->buffer);
content->buffer = NULL;
xdb_free(content);
}
}
// --- End
// get unsigned long (4bytes) from a specified buffer start from the specified offset
unsigned int xdb_get_uint(const char *buffer, int offset) {
return (
((buffer[offset ]) & 0x000000FF) |
((buffer[offset+1] << 8) & 0x0000FF00) |
((buffer[offset+2] << 16) & 0x00FF0000) |
((buffer[offset+3] << 24) & 0xFF000000)
);
}
// get unsigned short (2bytes) from a specified buffer start from the specified offset
int xdb_get_ushort(const char *buffer, int offset) {
return (
((buffer[offset ]) & 0x000000FF) |
((buffer[offset+1] << 8) & 0x0000FF00)
);
}
// string ip to unsigned int
static int shiftIndex[4] = {24, 16, 8, 0};
int xdb_check_ip(const char *src_ip, unsigned int *dst_ip) {
char c;
int i, n, ip = 0;
const char *ptr = src_ip;
for (i = 0; i < 4; i++) {
n = 0;
while (1) {
c = *ptr;
ptr++;
if (c >= '0' && c <= '9') {
n *= 10;
n += c - '0';
} else if ((i < 3 && c == '.') || i == 3) {
// stopping at the '.' but ignore the tailing chars
// after the 3rd one (auto clean the tailing none-integer ?).
break;
} else {
return 1;
}
}
if (n > 0xFF) {
return 2;
}
ip |= (n << shiftIndex[i]);
}
*dst_ip = ip;
return 0;
}
// unsigned int ip to string ip
void xdb_long2ip(unsigned int ip, char *buffer) {
sprintf(buffer, "%d.%d.%d.%d", (ip >> 24) & 0xFF, (ip >> 16) & 0xFF, (ip >> 8) & 0xFF, ip & 0xFF);
}
// get the middle ip of a and b
unsigned int xdb_mip(unsigned long a, unsigned long b) {
return (unsigned int) ((a + b) >> 1);
}
long xdb_now() {
struct timeval c_time;
gettimeofday(&c_time, NULL);
return c_time.tv_sec * (int)1e6 + c_time.tv_usec;
}

127
binding/nginx/src/xdb_searcher.h Normal file
View File

@ -0,0 +1,127 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
// ---
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/06/27
#ifndef C_XDB_SEARCHER_H
#define C_XDB_SEARCHER_H
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define xdb_calloc( _blocks, _bytes ) calloc( _blocks, _bytes )
#define xdb_malloc( _bytes ) malloc( _bytes )
#define xdb_free( _ptr ) free( _ptr )
// public constants define
#define xdb_header_info_length 256
#define xdb_vector_index_rows 256
#define xdb_vector_index_cols 256
#define xdb_vector_index_size 8
#define xdb_segment_index_size 14
// cache of vector_index_row × vector_index_rows × vector_index_size
#define xdb_vector_index_length 524288
typedef struct {
unsigned short version;
unsigned short index_policy;
unsigned int created_at;
unsigned int start_index_ptr;
unsigned int end_index_ptr;
// the original buffer
unsigned int length;
char buffer[xdb_header_info_length];
} xdb_header_t;
xdb_header_t* xdb_load_header(FILE *);
xdb_header_t *xdb_load_header_from_file(const char *);
void xdb_close_header(void *);
// --- vector index buffer
struct xdb_vector_index {
unsigned int length;
char buffer[xdb_vector_index_length];
};
typedef struct xdb_vector_index xdb_vector_index_t;
xdb_vector_index_t* xdb_load_vector_index(FILE *);
xdb_vector_index_t * xdb_load_vector_index_from_file(const char *);
void xdb_close_vector_index(void *);
// --- content buffer
typedef struct {
unsigned int length;
char *buffer;
} xdb_content_t;
xdb_content_t* xdb_load_content(FILE *);
xdb_content_t* xdb_load_content_from_file(const char *);
void xdb_close_content(void *);
// --- End buffer load
// xdb searcher structure
typedef struct xdb_searcher_entry {
FILE *handle;
// header info
const char *header;
int io_count;
// vector index buffer cache.
// preload the vector index will reduce the number of IO operations
// thus speedup the search process.
const xdb_vector_index_t *v_index;
// content buffer.
// cache the whole xdb content.
const xdb_content_t *content;
} xdb_searcher_t;
// xdb searcher new api define
int xdb_new_with_file_only(xdb_searcher_t *, const char *);
int xdb_new_with_vector_index(xdb_searcher_t *, const char *, const xdb_vector_index_t *);
int xdb_new_with_buffer(xdb_searcher_t *, const xdb_content_t *);
void xdb_close(void *);
// xdb searcher search api define
int xdb_search_by_string(xdb_searcher_t *, const char *, char *, size_t);
int xdb_search(xdb_searcher_t *, unsigned int, char *, size_t);
int xdb_get_io_count(xdb_searcher_t *);
// get unsigned long (4bytes) from a specified buffer start from the specified offset with little-endian
unsigned int xdb_get_uint(const char *, int);
// get unsigned short (2bytes) from a specified buffer start from the specified offset with little-endian
int xdb_get_ushort(const char *, int);
// check the specified string ip and convert it to an unsigned int
int xdb_check_ip(const char *, unsigned int *);
// unsigned int ip to string ip
void xdb_long2ip(unsigned int, char *);
// get the middle ip of a and b
unsigned int xdb_mip(unsigned long, unsigned long);
// get the current time in microseconds
long xdb_now();
#endif //C_XDB_SEARCHER_H

27
binding/nginx/t/http_ua_parser.t Normal file
View File

@ -0,0 +1,27 @@
use lib 'lib';
use Test::Nginx::Socket; # 'no_plan';
repeat_each(2);
plan tests => repeat_each() * 124;
no_long_string();
#no_diff;
run_tests();
__DATA__
=== TEST 1: set request header at client side
--- config
location /foo {
echo $http_x_foo;
}
--- request
GET /foo
--- more_headers
X-Foo: blah
--- response_headers
! X-Foo
--- response_body
blah