diff --git a/.gitignore b/.gitignore index ce243fa..a276f0f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.xdb *.iml META-INF/ +.DS_Store # Binary Files # *.jar @@ -71,6 +72,9 @@ target /v1.0/binding/nodejs/coverage /binding/nodejs/tests/unitTests/__snapshots__ /binding/nodejs/coverage +/binding/nodejs/node_modules +/binding/nodejs/.nyc_output +/binging/nodejs/package-lock.json # maker ## golang diff --git a/binding/nodejs/.eslintrc.js b/binding/nodejs/.eslintrc.js new file mode 100644 index 0000000..11a8bcc --- /dev/null +++ b/binding/nodejs/.eslintrc.js @@ -0,0 +1,17 @@ +module.exports = { + env: { + browser: true, + commonjs: true, + es2021: true + }, + parserOptions: { + ecmaVersion: 'latest' + }, + extends: [ + 'standard' + ], + globals: { + describe: true, + it: true + } +} diff --git a/binding/nodejs/.mocharc.json b/binding/nodejs/.mocharc.json new file mode 100644 index 0000000..bf668ea --- /dev/null +++ b/binding/nodejs/.mocharc.json @@ -0,0 +1,5 @@ +{ + "reporter": "spec", + "timeout": 60000, + "exit": true +} \ No newline at end of file diff --git a/binding/nodejs/.nycrc.json b/binding/nodejs/.nycrc.json new file mode 100644 index 0000000..8fb5c3d --- /dev/null +++ b/binding/nodejs/.nycrc.json @@ -0,0 +1,5 @@ +{ + "all": true, + "report-dir": "./coverage/", + "reporter": ["text", "html"] +} \ No newline at end of file diff --git a/binding/nodejs/ReadMe.md b/binding/nodejs/ReadMe.md index 85829b5..83d7204 100644 --- a/binding/nodejs/ReadMe.md +++ b/binding/nodejs/ReadMe.md @@ -1,7 +1,155 @@ # ip2region nodejs 查询客户端实现 -# 使用方式 +## 使用方式 -# 查询测试 +### 完全基于文件的查询 -# bench 测试 +```javascript +// 导入包 +const Searcher = require('.') +// 指定ip2region数据文件路径 +const dbPath = 'ip2region.xdb file path' + +try { + // 创建searcher对象 + const searcher = Searcher.newWithFileOnly(dbPath) + // 查询 + const data = await searcher.search('218.4.167.70') + // data: {region: '中国|0|江苏省|苏州市|电信', ioCount: 3, took: 1.342389} +} catch(e) { + console.log(e) +} + +``` + +### 缓存 `VectorIndex` 索引 + +```javascript +// 导入包 +const Searcher = require('.') +// 指定ip2region数据文件路径 +const dbPath = 'ip2region.xdb file path' + +try { + // 同步读取vectorIndex + const vectorIndex = Searcher.loadVectorIndexFromFile(dbPath) + // 创建searcher对象 + const searcher = Searcher.newWithVectorIndex(dbPath, vectorIndex) + // 查询 await 或 promise均可 + const data = await searcher.search('218.4.167.70') + // data: {region: '中国|0|江苏省|苏州市|电信', ioCount: 2, took: 0.402874} +} catch(e) { + console.log(e) +} +``` + +### 缓存整个 `xdb` 数据 + +```javascript +// 导入包 +const Searcher = require('.') +// 指定ip2region数据文件路径 +const dbPath = 'ip2region.xdb file path' + +try { + // 同步读取buffer + const buffer = Searcher.loadContentFromFile(dbPath) + // 创建searcher对象 + const searcher = Searcher.newWithVectorIndex(buffer) + // 查询 await 或 promise均可 + const data = await searcher.search('218.4.167.70') + // data: {region:'中国|0|江苏省|苏州市|电信', ioCount: 0, took: 0.063833} +} catch(e) { + console.log(e) +} +``` + +## 查询测试 + +可以通过 `node ./tests/test.app.js` 命令来测试查询: + +```shell +➜ nodejs git:(v2.0-for-nodejs) ✗ node ./tests/test.app.js --help +usage: Usage node test.app.js + +ip2region test app + +optional arguments: + -h, --help show this help message and exit + -d DB, --db DB ip2region binary xdb file path, default: ../../data/ip2region.xdb + -c CACHE_POLICY, --cache-policy CACHE_POLICY + cache policy: file/vectorIndex/content, default: content +``` + +例如:使用默认的 data/ip2region.xdb 文件进行查询测试: + +```shell +➜ nodejs git:(v2.0-for-nodejs) ✗ node ./tests/test.app.js +parameters: + dbPath: ../../data/ip2region.xdb + cache-policy: content + +type 'quit' to exit +ip2region>> 1.2.3.4 +{ region: '美国|0|华盛顿|0|谷歌', ioCount: 0, took: 54.606261 } +ip2region>> +``` + +输入 ip 即可进行查询测试,也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的查询效果。 + +## bench 测试 + +```shell +➜ nodejs git:(v2.0-for-nodejs) ✗ node ./tests/bench.app.js --help +usage: Usage node test.app.js [command options] + +ip2region benchmark app + +optional arguments: + -h, --help show this help message and exit + --db DB ip2region binary xdb file path, default: ../../data/ip2region.xdb + --src SRC source ip text file path, default: ../../data/ip.merge.txt + --cache-policy CACHE_POLICY + cache policy: file/vectorIndex/content, default: content + +``` + +例如:通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 文件进行 bench 测试: + +```shell +➜ nodejs git:(v2.0-for-nodejs) ✗ node ./tests/bench.app.js +options: + dbPath: ../../data/ip2region.xdb + src: ../../data/ip2region.xdb + cache-policy: content + +Bench finished, {cachePolicy: content, total: 3417955, took: 20.591887765s, cost: 6.02462225658325μs/op} +``` + +可以通过分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效果。 +>Note: 注意 bench 使用的 src 文件要是生成对应 xdb 文件相同的源文件。 + +## 单元测试及覆盖率结果 + +```shell +➜ nodejs git:(v2.0-for-nodejs) ✗ npm run coverage + +... + + ip2region + ✔ #newWithFileOnly and search + ✔ #newWithVectorIndex and search + ✔ #newWithBuffer and search + + + 3 passing (6ms) + +----------|---------|----------|---------|---------|---------------------------------- +File | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s +----------|---------|----------|---------|---------|---------------------------------- +All files | 91.58 | 60.71 | 100 | 91.58 | + index.js | 91.58 | 60.71 | 100 | 91.58 | 61,75,90,146,152,187,193,207,215 +----------|---------|----------|---------|---------|---------------------------------- +``` + +Made with ♥ by Wu Jian Ping diff --git a/binding/nodejs/index.js b/binding/nodejs/index.js new file mode 100644 index 0000000..0ed4896 --- /dev/null +++ b/binding/nodejs/index.js @@ -0,0 +1,201 @@ +/* + * Created by Wu Jian Ping on - 2022/07/22. + */ + +const fs = require('fs') + +const VectorIndexSize = 8 +const VectorIndexCols = 256 +const VectorIndexLength = 256 * 256 * (4 + 4) +const SegmentIndexSize = 14 +const IP_REGEX = /^((\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])$/ + +const getStartEndPtr = Symbol('#getStartEndPtr') +const getBuffer = Symbol('#getBuffer') +const openFilePromise = Symbol('#openFilePromise') + +class Searcher { + constructor (dbFile, vectorIndex, buffer) { + this._dbFile = dbFile + this._vectorIndex = vectorIndex + this._buffer = buffer + + if (this._buffer) { + this._vectorIndex = this._buffer.subarray(256, 256 + VectorIndexLength) + } + } + + async [getStartEndPtr] (idx, fd, ioStatus) { + if (this._vectorIndex) { + const sPtr = this._vectorIndex.readUInt32LE(idx) + const ePtr = this._vectorIndex.readUInt32LE(idx + 4) + return { sPtr, ePtr } + } else { + const buf = await this[getBuffer](256 + idx, 8, fd, ioStatus) + const sPtr = buf.readUInt32LE() + const ePtr = buf.readUInt32LE(4) + return { sPtr, ePtr } + } + } + + async [getBuffer] (offset, length, fd, ioStatus) { + if (this._buffer) { + return this._buffer.subarray(offset, offset + length) + } else { + const buf = Buffer.alloc(length) + return new Promise((resolve, reject) => { + ioStatus.ioCount += 1 + fs.read(fd, buf, 0, length, offset, (err) => { + if (err) { + reject(err) + } else { + resolve(buf) + } + }) + }) + } + } + + [openFilePromise] (fileName) { + return new Promise((resolve, reject) => { + fs.open(fileName, 'r', (err, fd) => { + if (err) { + reject(err) + } else { + resolve(fd) + } + }) + }) + } + + async search (ip) { + const startTime = process.hrtime() + const ioStatus = { + ioCount: 0 + } + + if (!isValidIp(ip)) { + throw new Error(`IP: ${ip} is invalid`) + } + + let fd = null + + if (!this._buffer) { + fd = await this[openFilePromise](this._dbFile) + } + + const ps = ip.split('.') + const i0 = parseInt(ps[0]) + const i1 = parseInt(ps[1]) + const i2 = parseInt(ps[2]) + const i3 = parseInt(ps[3]) + + const ipInt = i0 * 256 * 256 * 256 + i1 * 256 * 256 + i2 * 256 + i3 + const idx = i0 * VectorIndexCols * VectorIndexSize + i1 * VectorIndexSize + const { sPtr, ePtr } = await this[getStartEndPtr](idx, fd, ioStatus) + let l = 0 + let h = (ePtr - sPtr) / SegmentIndexSize + let result = null + + while (l <= h) { + const m = (l + h) >> 1 + + const p = sPtr + m * SegmentIndexSize + + const buff = await this[getBuffer](p, SegmentIndexSize, fd, ioStatus) + + const sip = buff.readUInt32LE(0) + + if (ipInt < sip) { + h = m - 1 + } else { + const eip = buff.readUInt32LE(4) + if (ipInt > eip) { + l = m + 1 + } else { + const dataLen = buff.readUInt16LE(8) + const dataPtr = buff.readUInt32LE(10) + const data = await this[getBuffer](dataPtr, dataLen, fd, ioStatus) + result = data.toString('utf-8') + break + } + } + } + if (fd) { + fs.close(fd) + } + + const diff = process.hrtime(startTime) + + const took = (diff[0] * 1e9 + diff[1]) / 1e3 + return { region: result, ioCount: ioStatus.ioCount, took } + } +} + +const _checkFile = dbPath => { + try { + fs.accessSync(dbPath, fs.constants.F_OK) + } catch (err) { + throw new Error(`${dbPath} ${err ? 'does not exist' : 'exists'}`) + } + + try { + fs.accessSync(dbPath, fs.constants.R_OK) + } catch (err) { + throw new Error(`${dbPath} ${err ? 'is not readable' : 'is readable'}`) + } +} + +const isValidIp = ip => { + return IP_REGEX.test(ip) +} + +const newWithFileOnly = dbPath => { + _checkFile(dbPath) + + return new Searcher(dbPath, null, null) +} + +const newWithVectorIndex = (dbPath, vectorIndex) => { + _checkFile(dbPath) + + if (!Buffer.isBuffer(vectorIndex)) { + throw new Error('vectorIndex is invalid') + } + + return new Searcher(dbPath, vectorIndex, null) +} + +const newWithBuffer = buffer => { + if (!Buffer.isBuffer(buffer)) { + throw new Error('buffer is invalid') + } + + return new Searcher(null, null, buffer) +} + +const loadVectorIndexFromFile = dbPath => { + const fd = fs.openSync(dbPath, 'r') + const buffer = Buffer.alloc(VectorIndexLength) + fs.readSync(fd, buffer, 0, VectorIndexLength, 256) + fs.close(fd) + return buffer +} + +const loadContentFromFile = dbPath => { + const stats = fs.statSync(dbPath) + const buffer = Buffer.alloc(stats.size) + const fd = fs.openSync(dbPath, 'r') + fs.readSync(fd, buffer, 0, stats.size, 0) + fs.close(fd) + return buffer +} + +module.exports = { + isValidIp, + loadVectorIndexFromFile, + loadContentFromFile, + newWithFileOnly, + newWithVectorIndex, + newWithBuffer +} diff --git a/binding/nodejs/package.json b/binding/nodejs/package.json new file mode 100644 index 0000000..7228876 --- /dev/null +++ b/binding/nodejs/package.json @@ -0,0 +1,32 @@ +{ + "name": "node-ip2region", + "version": "2.0.0", + "description": "official nodejs client of ip2region", + "main": "index.js", + "scripts": { + "lint": "eslint ./index.js", + "test": "mocha ./tests/function.test.js", + "coverage": "nyc npm run test", + "benchmark": "node ./tests/benchmark.js" + }, + "author": "Wu Jian Ping", + "license": "ISC", + "devDependencies": { + "@types/chai": "^4.3.1", + "@types/node": "^18.0.6", + "argparse": "^2.0.1", + "benchmark": "^2.1.4", + "chai": "^4.3.6", + "eslint": "^8.20.0", + "eslint-config-standard": "^17.0.0", + "eslint-plugin-import": "^2.26.0", + "eslint-plugin-n": "^15.2.4", + "eslint-plugin-promise": "^6.0.0", + "linebyline": "^1.3.0", + "mocha": "^10.0.0", + "nyc": "^15.1.0" + }, + "engines": { + "node": ">=8.0.0" + } +} \ No newline at end of file diff --git a/binding/nodejs/tests/bench.app.js b/binding/nodejs/tests/bench.app.js new file mode 100644 index 0000000..b1b1aeb --- /dev/null +++ b/binding/nodejs/tests/bench.app.js @@ -0,0 +1,146 @@ +/* + * Created by Wu Jian Ping on - 2022/07/22. + */ + +const Searcher = require('..') +const { ArgumentParser } = require('argparse') +const readline = require('linebyline') + +// 处理输入参数 +const parser = new ArgumentParser({ + add_help: true, + description: 'ip2region benchmark app', + prog: 'node test.app.js', + usage: 'Usage %(prog)s [command options]' +}) + +parser.add_argument('--db', { help: 'ip2region binary xdb file path, default: ../../data/ip2region.xdb' }) +parser.add_argument('--src', { help: 'source ip text file path, default: ../../data/ip.merge.txt' }) +parser.add_argument('--cache-policy', { help: 'cache policy: file/vectorIndex/content, default: content' }) + +const args = parser.parse_args() +const dbPath = args.db || '../../data/ip2region.xdb' +const src = args.src || '../../data/ip.merge.txt' +const cachePolicy = args.cache_policy || 'content' + +// 创建searcher对象 +const createSearcher = () => { + let searcher = null + let vectorIndex = null + let buffer = null + + switch (cachePolicy) { + case 'file': + searcher = Searcher.newWithFileOnly(dbPath) + break + case 'vectorIndex': + vectorIndex = Searcher.loadVectorIndexFromFile(dbPath) + searcher = Searcher.newWithVectorIndex(dbPath, vectorIndex) + break + default: + buffer = Searcher.loadContentFromFile(dbPath) + searcher = Searcher.newWithBuffer(buffer) + } + console.log('options: ') + console.log(` dbPath: ${dbPath}`) + console.log(` src: ${dbPath}`) + console.log(` cache-policy: ${cachePolicy}`) + console.log('') + return searcher +} + +const ipToInt = ip => { + // 切割IP + const ps = ip.split('.') + // 将各段转成int + const i0 = parseInt(ps[0]) + const i1 = parseInt(ps[1]) + const i2 = parseInt(ps[2]) + const i3 = parseInt(ps[3]) + + // 假如使用移位操作的话,这边可能产生负数 + return i0 * 256 * 256 * 256 + i1 * 256 * 256 + i2 * 256 + i3 +} + +const intToIp = ip => { + const i0 = Math.floor(ip / (256 * 256 * 256)) + const i1 = Math.floor((ip % (256 * 256 * 256)) / (256 * 256)) + const i2 = Math.floor((ip % (256 * 256)) / 256) + const i3 = ip % 256 + + return `${i0}.${i1}.${i2}.${i3}` +} + +const searcher = createSearcher() + +// 开始时间 +const startTime = process.hrtime() +let total = 0 + +// 程序主入口 +const main = async () => { + const rl = readline(src) + rl + .on('line', async (line, lineCount, byteCount) => { + try { + const list = line.split('|') + const sip = list[0] + const eip = list[1] + + if (!Searcher.isValidIp(sip)) { + throw new Error(`IP: ${sip} is invalid`) + } + if (!Searcher.isValidIp(eip)) { + throw new Error(`IP: ${eip} is invalid`) + } + + const sipInt = ipToInt(sip) + const eipInt = ipToInt(eip) + + if (sipInt > eipInt) { + throw new Error(`start ip(${sip}) should not be greater than end ip(${eip})`) + } + + const mipInt = Math.floor((sipInt + eipInt) / 2) + const mip = intToIp(mipInt) + + const mipLeftInt = Math.floor((sipInt + mipInt) / 2) + const mipLeft = intToIp(mipLeftInt) + + const mipRightInt = Math.floor((mipInt + eipInt) / 2) + const mipRight = intToIp(mipRightInt) + + const arr = [sip, mipLeft, mip, mipRight, eip] + + for (let i = 0; i < arr.length; ++i) { + const target = arr[i] + const info = await searcher.search(target) + + const region = list.slice(2, list.length).join('|') + // check the region info + if (region !== info.region) { + throw new Error(`failed search(${mip}) with (${region} != ${info.region})`) + } + total++ + } + } catch (err) { + console.log(err) + process.exit(1) + } + }) + .on('error', err => { + console.log(err) + process.exit(1) + }) +} + +process.on('exit', code => { + if (code === 0) { + // 这边只算个总时间就够了 + const diff = process.hrtime(startTime) + const totalInNS = diff[0] * 1e9 + diff[1] + console.log(`Bench finished, {cachePolicy: ${cachePolicy}, total: ${total}, took: ${totalInNS / 1e9}s, cost: ${total === 0 ? 0 : (totalInNS / 1e3) / total}μs/op}`) + } +}) + +main() diff --git a/binding/nodejs/tests/benchmark.js b/binding/nodejs/tests/benchmark.js new file mode 100644 index 0000000..fef7ea7 --- /dev/null +++ b/binding/nodejs/tests/benchmark.js @@ -0,0 +1,38 @@ +/* + * Created by Wu Jian Ping on - 2022/07/22. + */ + +const Benchmark = require('benchmark') +const path = require('path') +const Searcher = require('..') + +const dbPath = path.join(__dirname, '..', '..', '..', 'data', 'ip2region.xdb') +const buffer = Searcher.loadContentFromFile(dbPath) +const searcher1 = Searcher.newWithBuffer(buffer) + +const vectorIndex = Searcher.loadVectorIndexFromFile(dbPath) +const searcher2 = Searcher.newWithVectorIndex(dbPath, vectorIndex) + +const searcher3 = Searcher.newWithFileOnly(dbPath) + +const suite = new Benchmark.Suite() +suite + .add('#缓存整个xdb数据【搜索218.4.167.70】', async () => { + const ip = '218.4.167.70' + return searcher1.search(ip) + }) + .add('#缓存VectorIndex索引【搜索218.4.167.70】', async () => { + const ip = '218.4.167.70' + return searcher2.search(ip) + }) + .add('#完全基于文件的查询【搜索218.4.167.70】', async () => { + const ip = '218.4.167.70' + return searcher3.search(ip) + }) + .on('cycle', function (event) { + console.log(String(event.target)) // eslint-disable-line + }) + .on('complete', function () { + console.log('Fastest is ' + this.filter('fastest').map('name')) // eslint-disable-line + }) + .run({ async: true }) diff --git a/binding/nodejs/tests/function.test.js b/binding/nodejs/tests/function.test.js new file mode 100644 index 0000000..bdfd524 --- /dev/null +++ b/binding/nodejs/tests/function.test.js @@ -0,0 +1,33 @@ +/* + * Created by Wu Jian Ping on - 2022/07/22. + */ + +const { expect } = require('chai') +const path = require('path') +const Searcher = require('..') + +const dbPath = path.join(__dirname, '..', '..', '..', 'data', 'ip2region.xdb') +const buffer = Searcher.loadContentFromFile(dbPath) +const searcher1 = Searcher.newWithBuffer(buffer) + +const vectorIndex = Searcher.loadVectorIndexFromFile(dbPath) +const searcher2 = Searcher.newWithVectorIndex(dbPath, vectorIndex) + +const searcher3 = Searcher.newWithFileOnly(dbPath) + +describe('ip2region', () => { + it('#newWithFileOnly and search', async () => { + const d = await searcher3.search('218.4.167.70') + expect(d.region).equal('中国|0|江苏省|苏州市|电信') + }) + + it('#newWithVectorIndex and search', async () => { + const d = await searcher2.search('218.4.167.70') + expect(d.region).equal('中国|0|江苏省|苏州市|电信') + }) + + it('#newWithBuffer and search', async () => { + const d = await searcher1.search('218.4.167.70') + expect(d.region).equal('中国|0|江苏省|苏州市|电信') + }) +}) diff --git a/binding/nodejs/tests/test.app.js b/binding/nodejs/tests/test.app.js new file mode 100644 index 0000000..26abbd0 --- /dev/null +++ b/binding/nodejs/tests/test.app.js @@ -0,0 +1,79 @@ +/* + * Created by Wu Jian Ping on - 2022/07/22. + */ + +const Searcher = require('../') +const { ArgumentParser } = require('argparse') + +// 处理输入参数 +const parser = new ArgumentParser({ + add_help: true, + description: 'ip2region test app', + prog: 'node test.app.js', + usage: 'Usage %(prog)s [command options]' +}) + +parser.add_argument('--db', { help: 'ip2region binary xdb file path, default: ../../data/ip2region.xdb' }) +parser.add_argument('--cache-policy', { help: 'cache policy: file/vectorIndex/content, default: content' }) + +const args = parser.parse_args() +const dbPath = args.db || '../../data/ip2region.xdb' +const cachePolicy = args.cache_policy || 'content' + +// 创建searcher对象 +const createSearcher = () => { + let searcher = null + let vectorIndex = null + let buffer = null + + switch (cachePolicy) { + case 'file': + searcher = Searcher.newWithFileOnly(dbPath) + break + case 'vectorIndex': + vectorIndex = Searcher.loadVectorIndexFromFile(dbPath) + searcher = Searcher.newWithVectorIndex(dbPath, vectorIndex) + break + default: + buffer = Searcher.loadContentFromFile(dbPath) + searcher = Searcher.newWithBuffer(buffer) + } + console.log('options: ') + console.log(` dbPath: ${dbPath}`) + console.log(` cache-policy: ${cachePolicy}`) + console.log('') + return searcher +} + +// 从控制台读取用户一行输入 +const readlineSync = () => { + return new Promise((resolve, reject) => { + process.stdin.resume() + process.stdin.on('data', data => { + process.stdin.pause() + resolve(data.toString('utf-8')) + }) + }) +} + +const searcher = createSearcher() + +const main = async () => { + console.log('type \'quit\' to exit') + while (true) { + process.stdout.write('ip2region>> ') + const ip = (await readlineSync()).trim() + if (ip === 'quit') { + process.exit(0) + } else { + try { + const response = await searcher.search(ip) + console.log(response) + } catch (err) { + console.log(err) + } + } + } +} + +main()