xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #245 from wujjpp/v2.0-for-nodejs 

xdb searcher for nodejs
This commit is contained in:
Lion 2022-07-25 13:23:52 +08:00 committed by GitHub
parent 3163e9300f f2a7485a56
commit c80d2ffb82
11 changed files with 711 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -9,6 +9,7 @@
*.xdb
*.iml
META-INF/
.DS_Store
# Binary Files #
*.jar
@ -71,6 +72,9 @@ target
/v1.0/binding/nodejs/coverage
/binding/nodejs/tests/unitTests/__snapshots__
/binding/nodejs/coverage
/binding/nodejs/node_modules
/binding/nodejs/.nyc_output
/binging/nodejs/package-lock.json
# maker
## golang

17
binding/nodejs/.eslintrc.js Normal file
View File

@ -0,0 +1,17 @@
module.exports = {
env: {
browser: true,
commonjs: true,
es2021: true
},
parserOptions: {
ecmaVersion: 'latest'
},
extends: [
'standard'
],
globals: {
describe: true,
it: true
}
}

5
binding/nodejs/.mocharc.json Normal file
View File

@ -0,0 +1,5 @@
{
"reporter": "spec",
"timeout": 60000,
"exit": true
}

5
binding/nodejs/.nycrc.json Normal file
View File

@ -0,0 +1,5 @@
{
"all": true,
"report-dir": "./coverage/",
"reporter": ["text", "html"]
}

154
binding/nodejs/ReadMe.md
View File

@ -1,7 +1,155 @@
# ip2region nodejs 查询客户端实现
# 使用方式
## 使用方式
# 查询测试
### 完全基于文件的查询
# bench 测试
```javascript
// 导入包
const Searcher = require('.')
// 指定ip2region数据文件路径
const dbPath = 'ip2region.xdb file path'
try {
// 创建searcher对象
const searcher = Searcher.newWithFileOnly(dbPath)
// 查询
const data = await searcher.search('218.4.167.70')
// data: {region: '中国|0|江苏省|苏州市|电信', ioCount: 3, took: 1.342389}
} catch(e) {
console.log(e)
}
```
### 缓存 `VectorIndex` 索引
```javascript
// 导入包
const Searcher = require('.')
// 指定ip2region数据文件路径
const dbPath = 'ip2region.xdb file path'
try {
// 同步读取vectorIndex
const vectorIndex = Searcher.loadVectorIndexFromFile(dbPath)
// 创建searcher对象
const searcher = Searcher.newWithVectorIndex(dbPath, vectorIndex)
// 查询 await 或 promise均可
const data = await searcher.search('218.4.167.70')
// data: {region: '中国|0|江苏省|苏州市|电信', ioCount: 2, took: 0.402874}
} catch(e) {
console.log(e)
}
```
### 缓存整个 `xdb` 数据
```javascript
// 导入包
const Searcher = require('.')
// 指定ip2region数据文件路径
const dbPath = 'ip2region.xdb file path'
try {
// 同步读取buffer
const buffer = Searcher.loadContentFromFile(dbPath)
// 创建searcher对象
const searcher = Searcher.newWithVectorIndex(buffer)
// 查询 await 或 promise均可
const data = await searcher.search('218.4.167.70')
// data: {region:'中国|0|江苏省|苏州市|电信', ioCount: 0, took: 0.063833}
} catch(e) {
console.log(e)
}
```
## 查询测试
可以通过 `node ./tests/test.app.js` 命令来测试查询：
```shell
➜ nodejs git:(v2.0-for-nodejs) ✗ node ./tests/test.app.js --help
usage: Usage node test.app.js <agrs>
ip2region test app
optional arguments:
-h, --help show this help message and exit
-d DB, --db DB ip2region binary xdb file path, default: ../../data/ip2region.xdb
-c CACHE_POLICY, --cache-policy CACHE_POLICY
cache policy: file/vectorIndex/content, default: content
```
例如：使用默认的 data/ip2region.xdb 文件进行查询测试：
```shell
➜ nodejs git:(v2.0-for-nodejs) ✗ node ./tests/test.app.js
parameters:
dbPath: ../../data/ip2region.xdb
cache-policy: content
type 'quit' to exit
ip2region>> 1.2.3.4
{ region: '美国|0|华盛顿|0|谷歌', ioCount: 0, took: 54.606261 }
ip2region>>
```
输入 ip 即可进行查询测试，也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。
## bench 测试
```shell
➜ nodejs git:(v2.0-for-nodejs) ✗ node ./tests/bench.app.js --help
usage: Usage node test.app.js [command options]
ip2region benchmark app
optional arguments:
-h, --help show this help message and exit
--db DB ip2region binary xdb file path, default: ../../data/ip2region.xdb
--src SRC source ip text file path, default: ../../data/ip.merge.txt
--cache-policy CACHE_POLICY
cache policy: file/vectorIndex/content, default: content
```
例如：通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 文件进行 bench 测试：
```shell
➜ nodejs git:(v2.0-for-nodejs) ✗ node ./tests/bench.app.js
options:
dbPath: ../../data/ip2region.xdb
src: ../../data/ip2region.xdb
cache-policy: content
Bench finished, {cachePolicy: content, total: 3417955, took: 20.591887765s, cost: 6.02462225658325μs/op}
```
可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。
>Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。
## 单元测试及覆盖率结果
```shell
➜ nodejs git:(v2.0-for-nodejs) ✗ npm run coverage
...
ip2region
✔ #newWithFileOnly and search
✔ #newWithVectorIndex and search
✔ #newWithBuffer and search
3 passing (6ms)
----------|---------|----------|---------|---------|----------------------------------
File | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s
----------|---------|----------|---------|---------|----------------------------------
All files | 91.58 | 60.71 | 100 | 91.58 |
index.js | 91.58 | 60.71 | 100 | 91.58 | 61,75,90,146,152,187,193,207,215
----------|---------|----------|---------|---------|----------------------------------
```
Made with ♥ by Wu Jian Ping

201
binding/nodejs/index.js Normal file
View File

@ -0,0 +1,201 @@
/*
* Created by Wu Jian Ping on - 2022/07/22.
*/
const fs = require('fs')
const VectorIndexSize = 8
const VectorIndexCols = 256
const VectorIndexLength = 256 * 256 * (4 + 4)
const SegmentIndexSize = 14
const IP_REGEX = /^((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])$/
const getStartEndPtr = Symbol('#getStartEndPtr')
const getBuffer = Symbol('#getBuffer')
const openFilePromise = Symbol('#openFilePromise')
class Searcher {
constructor (dbFile, vectorIndex, buffer) {
this._dbFile = dbFile
this._vectorIndex = vectorIndex
this._buffer = buffer
if (this._buffer) {
this._vectorIndex = this._buffer.subarray(256, 256 + VectorIndexLength)
}
}
async [getStartEndPtr] (idx, fd, ioStatus) {
if (this._vectorIndex) {
const sPtr = this._vectorIndex.readUInt32LE(idx)
const ePtr = this._vectorIndex.readUInt32LE(idx + 4)
return { sPtr, ePtr }
} else {
const buf = await this[getBuffer](256 + idx, 8, fd, ioStatus)
const sPtr = buf.readUInt32LE()
const ePtr = buf.readUInt32LE(4)
return { sPtr, ePtr }
}
}
async [getBuffer] (offset, length, fd, ioStatus) {
if (this._buffer) {
return this._buffer.subarray(offset, offset + length)
} else {
const buf = Buffer.alloc(length)
return new Promise((resolve, reject) => {
ioStatus.ioCount += 1
fs.read(fd, buf, 0, length, offset, (err) => {
if (err) {
reject(err)
} else {
resolve(buf)
}
})
})
}
}
[openFilePromise] (fileName) {
return new Promise((resolve, reject) => {
fs.open(fileName, 'r', (err, fd) => {
if (err) {
reject(err)
} else {
resolve(fd)
}
})
})
}
async search (ip) {
const startTime = process.hrtime()
const ioStatus = {
ioCount: 0
}
if (!isValidIp(ip)) {
throw new Error(`IP: ${ip} is invalid`)
}
let fd = null
if (!this._buffer) {
fd = await this[openFilePromise](this._dbFile)
}
const ps = ip.split('.')
const i0 = parseInt(ps[0])
const i1 = parseInt(ps[1])
const i2 = parseInt(ps[2])
const i3 = parseInt(ps[3])
const ipInt = i0 * 256 * 256 * 256 + i1 * 256 * 256 + i2 * 256 + i3
const idx = i0 * VectorIndexCols * VectorIndexSize + i1 * VectorIndexSize
const { sPtr, ePtr } = await this[getStartEndPtr](idx, fd, ioStatus)
let l = 0
let h = (ePtr - sPtr) / SegmentIndexSize
let result = null
while (l <= h) {
const m = (l + h) >> 1
const p = sPtr + m * SegmentIndexSize
const buff = await this[getBuffer](p, SegmentIndexSize, fd, ioStatus)
const sip = buff.readUInt32LE(0)
if (ipInt < sip) {
h = m - 1
} else {
const eip = buff.readUInt32LE(4)
if (ipInt > eip) {
l = m + 1
} else {
const dataLen = buff.readUInt16LE(8)
const dataPtr = buff.readUInt32LE(10)
const data = await this[getBuffer](dataPtr, dataLen, fd, ioStatus)
result = data.toString('utf-8')
break
}
}
}
if (fd) {
fs.close(fd)
}
const diff = process.hrtime(startTime)
const took = (diff[0] * 1e9 + diff[1]) / 1e3
return { region: result, ioCount: ioStatus.ioCount, took }
}
}
const _checkFile = dbPath => {
try {
fs.accessSync(dbPath, fs.constants.F_OK)
} catch (err) {
throw new Error(`${dbPath} ${err ? 'does not exist' : 'exists'}`)
}
try {
fs.accessSync(dbPath, fs.constants.R_OK)
} catch (err) {
throw new Error(`${dbPath} ${err ? 'is not readable' : 'is readable'}`)
}
}
const isValidIp = ip => {
return IP_REGEX.test(ip)
}
const newWithFileOnly = dbPath => {
_checkFile(dbPath)
return new Searcher(dbPath, null, null)
}
const newWithVectorIndex = (dbPath, vectorIndex) => {
_checkFile(dbPath)
if (!Buffer.isBuffer(vectorIndex)) {
throw new Error('vectorIndex is invalid')
}
return new Searcher(dbPath, vectorIndex, null)
}
const newWithBuffer = buffer => {
if (!Buffer.isBuffer(buffer)) {
throw new Error('buffer is invalid')
}
return new Searcher(null, null, buffer)
}
const loadVectorIndexFromFile = dbPath => {
const fd = fs.openSync(dbPath, 'r')
const buffer = Buffer.alloc(VectorIndexLength)
fs.readSync(fd, buffer, 0, VectorIndexLength, 256)
fs.close(fd)
return buffer
}
const loadContentFromFile = dbPath => {
const stats = fs.statSync(dbPath)
const buffer = Buffer.alloc(stats.size)
const fd = fs.openSync(dbPath, 'r')
fs.readSync(fd, buffer, 0, stats.size, 0)
fs.close(fd)
return buffer
}
module.exports = {
isValidIp,
loadVectorIndexFromFile,
loadContentFromFile,
newWithFileOnly,
newWithVectorIndex,
newWithBuffer
}

32
binding/nodejs/package.json Normal file
View File

@ -0,0 +1,32 @@
{
"name": "node-ip2region",
"version": "2.0.0",
"description": "official nodejs client of ip2region",
"main": "index.js",
"scripts": {
"lint": "eslint ./index.js",
"test": "mocha ./tests/function.test.js",
"coverage": "nyc npm run test",
"benchmark": "node ./tests/benchmark.js"
},
"author": "Wu Jian Ping",
"license": "ISC",
"devDependencies": {
"@types/chai": "^4.3.1",
"@types/node": "^18.0.6",
"argparse": "^2.0.1",
"benchmark": "^2.1.4",
"chai": "^4.3.6",
"eslint": "^8.20.0",
"eslint-config-standard": "^17.0.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-n": "^15.2.4",
"eslint-plugin-promise": "^6.0.0",
"linebyline": "^1.3.0",
"mocha": "^10.0.0",
"nyc": "^15.1.0"
},
"engines": {
"node": ">=8.0.0"
}
}

146
binding/nodejs/tests/bench.app.js Normal file
View File

@ -0,0 +1,146 @@
/*
* Created by Wu Jian Ping on - 2022/07/22.
*/
const Searcher = require('..')
const { ArgumentParser } = require('argparse')
const readline = require('linebyline')
// 处理输入参数
const parser = new ArgumentParser({
add_help: true,
description: 'ip2region benchmark app',
prog: 'node test.app.js',
usage: 'Usage %(prog)s [command options]'
})
parser.add_argument('--db', { help: 'ip2region binary xdb file path, default: ../../data/ip2region.xdb' })
parser.add_argument('--src', { help: 'source ip text file path, default: ../../data/ip.merge.txt' })
parser.add_argument('--cache-policy', { help: 'cache policy: file/vectorIndex/content, default: content' })
const args = parser.parse_args()
const dbPath = args.db || '../../data/ip2region.xdb'
const src = args.src || '../../data/ip.merge.txt'
const cachePolicy = args.cache_policy || 'content'
// 创建searcher对象
const createSearcher = () => {
let searcher = null
let vectorIndex = null
let buffer = null
switch (cachePolicy) {
case 'file':
searcher = Searcher.newWithFileOnly(dbPath)
break
case 'vectorIndex':
vectorIndex = Searcher.loadVectorIndexFromFile(dbPath)
searcher = Searcher.newWithVectorIndex(dbPath, vectorIndex)
break
default:
buffer = Searcher.loadContentFromFile(dbPath)
searcher = Searcher.newWithBuffer(buffer)
}
console.log('options: ')
console.log(` dbPath: ${dbPath}`)
console.log(` src: ${dbPath}`)
console.log(` cache-policy: ${cachePolicy}`)
console.log('')
return searcher
}
const ipToInt = ip => {
// 切割IP
const ps = ip.split('.')
// 将各段转成int
const i0 = parseInt(ps[0])
const i1 = parseInt(ps[1])
const i2 = parseInt(ps[2])
const i3 = parseInt(ps[3])
// 假如使用移位操作的话，这边可能产生负数
return i0 * 256 * 256 * 256 + i1 * 256 * 256 + i2 * 256 + i3
}
const intToIp = ip => {
const i0 = Math.floor(ip / (256 * 256 * 256))
const i1 = Math.floor((ip % (256 * 256 * 256)) / (256 * 256))
const i2 = Math.floor((ip % (256 * 256)) / 256)
const i3 = ip % 256
return `${i0}.${i1}.${i2}.${i3}`
}
const searcher = createSearcher()
// 开始时间
const startTime = process.hrtime()
let total = 0
// 程序主入口
const main = async () => {
const rl = readline(src)
rl
.on('line', async (line, lineCount, byteCount) => {
try {
const list = line.split('|')
const sip = list[0]
const eip = list[1]
if (!Searcher.isValidIp(sip)) {
throw new Error(`IP: ${sip} is invalid`)
}
if (!Searcher.isValidIp(eip)) {
throw new Error(`IP: ${eip} is invalid`)
}
const sipInt = ipToInt(sip)
const eipInt = ipToInt(eip)
if (sipInt > eipInt) {
throw new Error(`start ip(${sip}) should not be greater than end ip(${eip})`)
}
const mipInt = Math.floor((sipInt + eipInt) / 2)
const mip = intToIp(mipInt)
const mipLeftInt = Math.floor((sipInt + mipInt) / 2)
const mipLeft = intToIp(mipLeftInt)
const mipRightInt = Math.floor((mipInt + eipInt) / 2)
const mipRight = intToIp(mipRightInt)
const arr = [sip, mipLeft, mip, mipRight, eip]
for (let i = 0; i < arr.length; ++i) {
const target = arr[i]
const info = await searcher.search(target)
const region = list.slice(2, list.length).join('|')
// check the region info
if (region !== info.region) {
throw new Error(`failed search(${mip}) with (${region} != ${info.region})`)
}
total++
}
} catch (err) {
console.log(err)
process.exit(1)
}
})
.on('error', err => {
console.log(err)
process.exit(1)
})
}
process.on('exit', code => {
if (code === 0) {
// 这边只算个总时间就够了
const diff = process.hrtime(startTime)
const totalInNS = diff[0] * 1e9 + diff[1]
console.log(`Bench finished, {cachePolicy: ${cachePolicy}, total: ${total}, took: ${totalInNS / 1e9}s, cost: ${total === 0 ? 0 : (totalInNS / 1e3) / total}μs/op}`)
}
})
main()

38
binding/nodejs/tests/benchmark.js Normal file
View File

@ -0,0 +1,38 @@
/*
* Created by Wu Jian Ping on - 2022/07/22.
*/
const Benchmark = require('benchmark')
const path = require('path')
const Searcher = require('..')
const dbPath = path.join(__dirname, '..', '..', '..', 'data', 'ip2region.xdb')
const buffer = Searcher.loadContentFromFile(dbPath)
const searcher1 = Searcher.newWithBuffer(buffer)
const vectorIndex = Searcher.loadVectorIndexFromFile(dbPath)
const searcher2 = Searcher.newWithVectorIndex(dbPath, vectorIndex)
const searcher3 = Searcher.newWithFileOnly(dbPath)
const suite = new Benchmark.Suite()
suite
.add('#缓存整个xdb数据【搜索218.4.167.70】', async () => {
const ip = '218.4.167.70'
return searcher1.search(ip)
})
.add('#缓存VectorIndex索引【搜索218.4.167.70】', async () => {
const ip = '218.4.167.70'
return searcher2.search(ip)
})
.add('#完全基于文件的查询【搜索218.4.167.70】', async () => {
const ip = '218.4.167.70'
return searcher3.search(ip)
})
.on('cycle', function (event) {
console.log(String(event.target)) // eslint-disable-line
})
.on('complete', function () {
console.log('Fastest is ' + this.filter('fastest').map('name')) // eslint-disable-line
})
.run({ async: true })

33
binding/nodejs/tests/function.test.js Normal file
View File

@ -0,0 +1,33 @@
/*
* Created by Wu Jian Ping on - 2022/07/22.
*/
const { expect } = require('chai')
const path = require('path')
const Searcher = require('..')
const dbPath = path.join(__dirname, '..', '..', '..', 'data', 'ip2region.xdb')
const buffer = Searcher.loadContentFromFile(dbPath)
const searcher1 = Searcher.newWithBuffer(buffer)
const vectorIndex = Searcher.loadVectorIndexFromFile(dbPath)
const searcher2 = Searcher.newWithVectorIndex(dbPath, vectorIndex)
const searcher3 = Searcher.newWithFileOnly(dbPath)
describe('ip2region', () => {
it('#newWithFileOnly and search', async () => {
const d = await searcher3.search('218.4.167.70')
expect(d.region).equal('中国|0|江苏省|苏州市|电信')
})
it('#newWithVectorIndex and search', async () => {
const d = await searcher2.search('218.4.167.70')
expect(d.region).equal('中国|0|江苏省|苏州市|电信')
})
it('#newWithBuffer and search', async () => {
const d = await searcher1.search('218.4.167.70')
expect(d.region).equal('中国|0|江苏省|苏州市|电信')
})
})

79
binding/nodejs/tests/test.app.js Normal file
View File

@ -0,0 +1,79 @@
/*
* Created by Wu Jian Ping on - 2022/07/22.
*/
const Searcher = require('../')
const { ArgumentParser } = require('argparse')
// 处理输入参数
const parser = new ArgumentParser({
add_help: true,
description: 'ip2region test app',
prog: 'node test.app.js',
usage: 'Usage %(prog)s [command options]'
})
parser.add_argument('--db', { help: 'ip2region binary xdb file path, default: ../../data/ip2region.xdb' })
parser.add_argument('--cache-policy', { help: 'cache policy: file/vectorIndex/content, default: content' })
const args = parser.parse_args()
const dbPath = args.db || '../../data/ip2region.xdb'
const cachePolicy = args.cache_policy || 'content'
// 创建searcher对象
const createSearcher = () => {
let searcher = null
let vectorIndex = null
let buffer = null
switch (cachePolicy) {
case 'file':
searcher = Searcher.newWithFileOnly(dbPath)
break
case 'vectorIndex':
vectorIndex = Searcher.loadVectorIndexFromFile(dbPath)
searcher = Searcher.newWithVectorIndex(dbPath, vectorIndex)
break
default:
buffer = Searcher.loadContentFromFile(dbPath)
searcher = Searcher.newWithBuffer(buffer)
}
console.log('options: ')
console.log(` dbPath: ${dbPath}`)
console.log(` cache-policy: ${cachePolicy}`)
console.log('')
return searcher
}
// 从控制台读取用户一行输入
const readlineSync = () => {
return new Promise((resolve, reject) => {
process.stdin.resume()
process.stdin.on('data', data => {
process.stdin.pause()
resolve(data.toString('utf-8'))
})
})
}
const searcher = createSearcher()
const main = async () => {
console.log('type \'quit\' to exit')
while (true) {
process.stdout.write('ip2region>> ')
const ip = (await readlineSync()).trim()
if (ip === 'quit') {
process.exit(0)
} else {
try {
const response = await searcher.search(ip)
console.log(response)
} catch (err) {
console.log(err)
}
}
}
}
main()