diff --git a/binding/python/ReadMe.md b/binding/python/ReadMe.md deleted file mode 100644 index 71778e6..0000000 --- a/binding/python/ReadMe.md +++ /dev/null @@ -1,114 +0,0 @@ -# ip2region python 查询客户端实现 - -# 使用方式 - -### 完全基于文件的查询 - -```python -from xdbSearcher import XdbSearcher - -def searchWithFile(): - # 1. 创建查询对象 - dbPath = "../../data/ip2region_v4.xdb" - searcher = XdbSearcher(dbfile=dbPath) - - # 2. 执行查询 - ip = "1.2.3.4" - region_str = searcher.searchByIPStr(ip) - print(region_str) - - # 3. 关闭searcher - searcher.close() -``` - -### 缓存 `VectorIndex` 索引 - -我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据,然后全局缓存,每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作,从而加速查询,减少 IO 压力。 - -```python -from xdbSearcher import XdbSearcher - -def searchWithVectorIndex(): - # 1. 预先加载整个 xdb - dbPath = "../../data/ip2region_v4.xdb" - vi = XdbSearcher.loadVectorIndexFromFile(dbfile=dbPath) - - # 2. 使用上面的缓存创建查询对象, 同时也要加载 xdb 文件 - searcher = XdbSearcher(dbfile=dbPath, vectorIndex=vi) - - # 3. 执行查询 - ip = "1.2.3.4" - region_str = searcher.search(ip) - print(region_str) - - # 4. 关闭searcher - searcher.close() -``` - -### 缓存整个 `xdb` 数据 - -我们也可以预先加载整个 xdb 文件的数据到内存,然后基于这个数据创建查询对象来实现完全基于文件的查询,类似之前的 memory search。 - -```python -from xdbSearcher import XdbSearcher - -def searchWithContent(): - # 1. 预先加载整个 xdb - dbPath = "../../data/ip2region_v4.xdb"; - cb = XdbSearcher.loadContentFromFile(dbfile=dbPath) - - # 2. 仅需要使用上面的全文件缓存创建查询对象, 不需要传源 xdb 文件 - searcher = XdbSearcher(contentBuff=cb) - - # 3. 执行查询 - ip = "1.2.3.4" - region_str = searcher.search(ip) - print(region_str) - - # 4. 关闭searcher - searcher.close() -``` -# 查询测试 - -通过 `search_test.py` 脚本来进行查询测试: -```bash -➜ python git:(python_dev) ✗ python3 ./search_test.py -python3 search_test.py [command options] -options: - --db string ip2region binary xdb file path - --cache-policy string cache policy: file/vectorIndex/content -``` - -例如:使用默认的 data/ip2region_v4.xdb 进行查询测试: -```bash -➜ python git:(python_dev) ✗ python3 ./search_test.py --db=../../data/ip2region_v4.xdb --cache-policy=content -ip2region xdb searcher test program, cachePolicy: content -type 'quit' to exit -ip2region>> 1.2.3.4 -region :美国|0|华盛顿|0|谷歌 , took 0.0689 ms -ip2region>> quit -searcher test program exited, thanks for trying -``` - -输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。 - -# bench 测试 - -通过 `bench_test.py` 脚本来进行自动 bench 测试,一方面确保 `xdb` 文件没有错误,另一方面通过大量的查询测试平均查询性能: -```bash -➜ python git:(python_dev) ✗ python3 ./bench_test.py -python bench_test.py [command options] -options: - --db string ip2region binary xdb file path - --src string source ip text file path - --cache-policy string cache policy: file/vectorIndex/content -``` - -例如:通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试: -```bash -➜ python git:(python_dev) ✗ python3 ./bench_test.py --db=../../data/ip2region_v4.xdb --src=../../data/ipv4_source.txt --cache-policy=content -Bench finished, [cachePolicy: content, total: 3417955, took: 34.93 s, cost: 0.0094 ms/op] -``` - -可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。 -@Note:请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。 \ No newline at end of file diff --git a/binding/python/bench_test.py b/binding/python/bench_test.py deleted file mode 100644 index 2c1c8c3..0000000 --- a/binding/python/bench_test.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2022 The Ip2Region Authors. All rights reserved. -# Use of this source code is governed by a Apache2.0-style -# license that can be found in the LICENSE file. -# -# Created by luckydog on 2022/7/1. -# Copyright © 2022年 luckydog. All rights reserved. -# -# from ast import main -import io - -from xdbSearcher import XdbSearcher -import argparse -import time -import sys - - -def printHelp(): - print("python bench_test.py [command options]") - print("options: ") - print(" --db string ip2region binary xdb file path") - print(" --src string source ip text file path") - print(" --cache-policy string cache policy: file/vectorIndex/content") - -def trim(string): - if string[:1] != ' ' and string[-1:] != ' ': - return string - elif string[:1] == ' ': - return trim(string[1:]) - else: - return trim(string[:-1]) - -def start_bench(dbFile="", srcFile="", cachePolicy="vectorIndex"): - if cachePolicy == "file": - try: - searcher = XdbSearcher(dbfile=dbFile) - except Exception as err: - print(err) - return - elif cachePolicy == "vectorIndex": - try: - vi = XdbSearcher.loadVectorIndexFromFile(dbfile=dbFile) - if vi is None: - print(f"failed to load vector index from {dbFile}\n") - searcher = XdbSearcher(dbfile=dbFile, vectorIndex=vi) - - except Exception as err: - print(err) - return - else: - try: - cb = XdbSearcher.loadContentFromFile(dbfile=dbFile) - if cb is None: - print(f"failed to load xdb content from {dbFile}\n") - searcher = XdbSearcher(contentBuff=cb) - - except Exception as err: - print(err) - return - # do the bench test - - try: - count = 0 - costs = 0 - sTime = time.time() - f = io.open(srcFile, "rb") - while True: - line = trim(f.readline(1024)).decode("utf-8").replace("\n", "") - if len(line) < 1: - break - - ps = line.split("|",2) - if len(ps) != 3: - print(f"invalid ip segment line :{line}") - return - sip = XdbSearcher.ip2long(None, ps[0]) - eip = XdbSearcher.ip2long(None, ps[1]) - - if sip > eip: - print(f"start ip({ps[0]}) should not be greater than end ip({ps[1]})") - return - - mip = (sip + eip) >> 1 - - for ip in [sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip]: - try: - cTime = time.time() - region = searcher.search(ip) - costs = costs + (time.time() - cTime) - except Exception as error: - print(f"failed to search ip :{ip}") - return - - if region is None: - print(f"failed to search ip :{ip}") - return - if region != ps[2]: - print(f"failed search({ip}) with ({region} != {ps[2]})") - return - count = count + 1 - - # close the searcher at last - f.close() - searcher.close() - print(f"Bench finished, {{cachePolicy: {cachePolicy}, total: {count}, took: {round(time.time() - sTime, 2)} s, cost: {round(costs/count*1000, 4)} ms/op}}") - except Exception as err: - print(f"failed to open source text file :{err}") - return - - -if __name__ == '__main__': - if len(sys.argv) < 2: - printHelp() - exit(0) - parse = argparse.ArgumentParser() - parse.add_argument("--db", help="ip2region binary xdb file path") - parse.add_argument("--src", help="source ip text file path") - parse.add_argument("--cache-policy", choices=["file", "vectorIndex", "content"], - help="cache policy: file/vectorIndex/content") - args = parse.parse_args() - start_bench(dbFile=args.db, srcFile=args.src, cachePolicy=args.cache_policy) diff --git a/binding/python/iptest.py b/binding/python/iptest.py deleted file mode 100644 index d11dee3..0000000 --- a/binding/python/iptest.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2022 The Ip2Region Authors. All rights reserved. -# Use of this source code is governed by a Apache2.0-style -# license that can be found in the LICENSE file. -# - -from xdbSearcher import XdbSearcher - -def searchWithFile(): - # 1. 创建查询对象 - dbPath = "../../data/ip2region_v4.xdb" - searcher = XdbSearcher(dbfile=dbPath) - - # 2. 执行查询 - ip = "1.2.3.4" - region_str = searcher.searchByIPStr(ip) - print(region_str) - - # 3. 关闭searcher - searcher.close() - -def searchWithVectorIndex(): - # 1. 预先加载整个 xdb - dbPath = "../../data/ip2region_v4.xdb" - vi = XdbSearcher.loadVectorIndexFromFile(dbfile=dbPath) - - # 2. 使用上面的缓存创建查询对象, 同时也要加载 xdb 文件 - searcher = XdbSearcher(dbfile=dbPath, vectorIndex=vi) - - # 3. 执行查询 - ip = "1.2.3.4" - region_str = searcher.search(ip) - print(region_str) - - # 4. 关闭searcher - searcher.close() - -def searchWithContent(): - # 1. 预先加载整个 xdb - dbPath = "../../data/ip2region_v4.xdb"; - cb = XdbSearcher.loadContentFromFile(dbfile=dbPath) - - # 2. 仅需要使用上面的全文件缓存创建查询对象, 不需要传源 xdb 文件 - searcher = XdbSearcher(contentBuff=cb) - - # 3. 执行查询 - ip = "1.2.3.4" - region_str = searcher.search(ip) - print(region_str) - - # 4. 关闭searcher - searcher.close() - - -if __name__ == '__main__': - searchWithContent() - \ No newline at end of file diff --git a/binding/python/search_test.py b/binding/python/search_test.py deleted file mode 100644 index 742a2d9..0000000 --- a/binding/python/search_test.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2022 The Ip2Region Authors. All rights reserved. -# Use of this source code is governed by a Apache2.0-style -# license that can be found in the LICENSE file. -# -# Created by luckydog on 2022/7/1. -# Copyright © 2022年 luckydog. All rights reserved. -# - -from xdbSearcher import XdbSearcher -import argparse -import time -import sys - -def printHelp(): - print("python3 search_test.py [command options]") - print("options: ") - print(" --db string ip2region binary xdb file path") - print(" --cache-policy string cache policy: file/vectorIndex/content") - - -def trim(string): - if string[:1] != ' ' and string[-1:] != ' ': - return string - elif string[:1] == ' ': - return trim(string[1:]) - else: - return trim(string[:-1]) - - -def start_search(dbFile="", cachePolicy="vectorIndex"): - if cachePolicy == "file": - try: - searcher = XdbSearcher(dbfile=dbFile) - except Exception as err: - print(err) - return - elif cachePolicy == "vectorIndex": - try: - vi = XdbSearcher.loadVectorIndexFromFile(dbfile=dbFile) - if vi is None: - print(f"failed to load vector index from {dbFile}\n") - searcher = XdbSearcher(dbfile=dbFile, vectorIndex=vi) - - except Exception as err: - print(err) - return - else: - try: - cb = XdbSearcher.loadContentFromFile(dbfile=dbFile) - if cb is None: - print(f"failed to load xdb content from {dbFile}\n") - searcher = XdbSearcher(contentBuff=cb) - - except Exception as err: - print(err) - return - - # 开始的提示 - print(f"ip2region xdb searcher test program, cachePolicy: {cachePolicy}\ntype 'quit' to exit") - while True: - line = trim(input("ip2region>> ")) - # print(f"{line}") - - if len(line) < 2: - continue - if line == "quit": - break - - if not XdbSearcher.isip(None, ip=line): - print("Error: invalid ip address") - continue - start = time.time() - - try: - region_str = searcher.searchByIPStr(line) - except Exception as error: - print(error) - return - - print(f"{{region: {region_str} , took: {round((time.time()-start)*1000.00, 4)} ms}}") - # quit - searcher.close() - print("searcher test program exited, thanks for trying") - - -if __name__ == '__main__': - if len(sys.argv) < 2: - printHelp() - exit(0) - parse = argparse.ArgumentParser() - parse.add_argument("--db", help="ip2region binary xdb file path") - parse.add_argument("--cache-policy", choices=["file", "vectorIndex", "content"], - help="cache policy: file/vectorIndex/content") - args = parse.parse_args() - start_search(dbFile=args.db, cachePolicy=args.cache_policy) diff --git a/binding/python/xdbSearcher.py b/binding/python/xdbSearcher.py deleted file mode 100644 index 8f04e3f..0000000 --- a/binding/python/xdbSearcher.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2022 The Ip2Region Authors. All rights reserved. -# Use of this source code is governed by a Apache2.0-style -# license that can be found in the LICENSE file. -# -# Created by luckydog on 2022/6/29. -# Copyright © 2022年 luckydog. All rights reserved. -# - -import socket -import struct -import io -import sys - - -# xdb默认参数 -HeaderInfoLength = 256 -VectorIndexRows = 256 -VectorIndexCols = 256 -VectorIndexSize = 8 -SegmentIndexSize = 14 - - -class XdbSearcher(object): - __f = None - - # the minimal memory allocation. - vectorIndex = None - # 整个读取xdb,保存在内存中 - contentBuff = None - - @staticmethod - def loadVectorIndexFromFile(dbfile): - try: - f = io.open(dbfile, "rb") - f.seek(HeaderInfoLength) - vi_len = VectorIndexRows * VectorIndexCols * SegmentIndexSize - vector_data = f.read(vi_len) - f.close() - return vector_data - except IOError as e: - print("[Error]: %s" % e) - - @staticmethod - def loadContentFromFile(dbfile): - try: - f = io.open(dbfile, "rb") - all_data = f.read() - f.close() - return all_data - except IOError as e: - print("[Error]: %s" % e) - - def __init__(self, dbfile=None, vectorIndex=None, contentBuff=None): - self.initDatabase(dbfile, vectorIndex, contentBuff) - - def search(self, ip): - if isinstance(ip, str): - if not ip.isdigit(): ip = self.ip2long(ip) - return self.searchByIPLong(ip) - else: - return self.searchByIPLong(ip) - - def searchByIPStr(self, ip): - if not ip.isdigit(): ip = self.ip2long(ip) - return self.searchByIPLong(ip) - - def searchByIPLong(self, ip): - # locate the segment index block based on the vector index - sPtr = ePtr = 0 - il0 = (int)((ip >> 24) & 0xFF) - il1 = (int)((ip >> 16) & 0xFF) - idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize - - if self.vectorIndex is not None: - sPtr = self.getLong(self.vectorIndex, idx) - ePtr = self.getLong(self.vectorIndex, idx + 4) - elif self.contentBuff is not None: - sPtr = self.getLong(self.contentBuff, HeaderInfoLength + idx) - ePtr = self.getLong(self.contentBuff, HeaderInfoLength + idx + 4) - else: - self.__f.seek(HeaderInfoLength + idx) - buffer_ptr = self.__f.read(8) - sPtr = self.getLong(buffer_ptr, 0) - ePtr = self.getLong(buffer_ptr, 4) - - # binary search the segment index block to get the region info - dataLen = dataPtr = int(-1) - l = int(0) - h = int((ePtr - sPtr) / SegmentIndexSize) - while l <= h: - m = int((l + h) >> 1) - p = int(sPtr + m * SegmentIndexSize) - # read the segment index - buffer_sip = self.readBuffer(p, SegmentIndexSize) - sip = self.getLong(buffer_sip, 0) - if ip < sip: - h = m - 1 - else: - eip = self.getLong(buffer_sip, 4) - if ip > eip: - l = m + 1 - else: - dataLen = self.getInt2(buffer_sip, 8) - dataPtr = self.getLong(buffer_sip, 10) - break - - # empty match interception - if dataPtr < 0: - return "" - - buffer_string = self.readBuffer(dataPtr, dataLen) - return_string = buffer_string.decode("utf-8") - return return_string - - def readBuffer(self, offset, length): - buffer = None - # check the in-memory buffer first - if self.contentBuff is not None: - buffer = self.contentBuff[offset:offset + length] - return buffer - - # read from the file handle - if self.__f is not None: - self.__f.seek(offset) - buffer = self.__f.read(length) - return buffer - - def initDatabase(self, dbfile, vi, cb): - """ - " initialize the database for search - " param: dbFile, vectorIndex, contentBuff - """ - try: - if cb is not None: - self.__f = None - self.vectorIndex = None - self.contentBuff = cb - else: - self.__f = io.open(dbfile, "rb") - self.vectorIndex = vi - except IOError as e: - print("[Error]: %s" % e) - sys.exit() - - def ip2long(self, ip): - _ip = socket.inet_aton(ip) - return struct.unpack("!L", _ip)[0] - - def isip(self, ip): - p = ip.split(".") - - if len(p) != 4: return False - for pp in p: - if not pp.isdigit(): return False - if len(pp) > 3: return False - if int(pp) > 255: return False - return True - - def getLong(self, b, offset): - if len(b[offset:offset + 4]) == 4: - return struct.unpack('I', b[offset:offset + 4])[0] - return 0 - - def getInt2(self, b, offset): - return ((b[offset] & 0x000000FF) | (b[offset+1] << 8)) - - def close(self): - if self.__f is not None: - self.__f.close() - self.vectorIndex = None - self.contentBuff = None - - -if __name__ == '__main__': - ip_array = [ - "1.2.3.4", - "192.168.1.1" - ] - # 1. 缓存 - dbPath = "./data/ip2region_v4.xdb"; - cb = XdbSearcher.loadContentFromFile(dbfile=dbPath) - - # 2. 创建查询对象 - searcher = XdbSearcher(contentBuff=cb) - - # 3. 执行查询 - # ip = "1.2.3.4" - for ip in ip_array: - region_str = searcher.searchByIPStr(ip) - print(region_str) - searcher.close()