diff --git a/binding/python/ReadMe.md b/binding/python/ReadMe.md index f6dce47..837dc4d 100644 --- a/binding/python/ReadMe.md +++ b/binding/python/ReadMe.md @@ -2,6 +2,113 @@ # 使用方式 +### 完全基于文件的查询 + +```python +from xdbSearcher import XdbSearcher + +def searchWithFile(): + # 1. 创建查询对象 + dbPath = "../../data/ip2region.xdb" + searcher = XdbSearcher(dbfile=dbPath) + + # 2. 执行查询 + ip = "1.2.3.4" + region_str = searcher.searchByIPStr(ip) + print(region_str) + + # 3. 关闭searcher + searcher.close() +``` + +### 缓存 `VectorIndex` 索引 + +我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据,然后全局缓存,每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作,从而加速查询,减少 IO 压力。 + +```python +from xdbSearcher import XdbSearcher + +def searchWithVectorIndex(): + # 1. 预先加载整个 xdb + dbPath = "../../data/ip2region.xdb" + vi = XdbSearcher.loadVectorIndexFromFile(dbfile=dbPath) + + # 2. 使用上面的缓存创建查询对象, 同时也要加载 xdb 文件 + searcher = XdbSearcher(dbfile=dbPath, vectorIndex=vi) + + # 3. 执行查询 + ip = "1.2.3.4" + region_str = searcher.search(ip) + print(region_str) + + # 4. 关闭searcher + searcher.close() +``` + +### 缓存整个 `xdb` 数据 + +我们也可以预先加载整个 ip2region.xdb 的数据到内存,然后基于这个数据创建查询对象来实现完全基于文件的查询,类似之前的 memory search。 + +```python +from xdbSearcher import XdbSearcher + +def searchWithContent(): + # 1. 预先加载整个 xdb + dbPath = "../../data/ip2region.xdb"; + cb = XdbSearcher.loadContentFromFile(dbfile=dbPath) + + # 2. 仅需要使用上面的全文件缓存创建查询对象, 不需要传源 xdb 文件 + searcher = XdbSearcher(contentBuff=cb) + + # 3. 执行查询 + ip = "1.2.3.4" + region_str = searcher.search(ip) + print(region_str) + + # 4. 关闭searcher + searcher.close() +``` # 查询测试 +通过 `search_test.py` 脚本来进行查询测试: +```bash +➜ python git:(python_dev) ✗ python3 ./search_test.py +python3 search_test.py [command options] +options: + --db string ip2region binary xdb file path + --cache-policy string cache policy: file/vectorIndex/content +``` + +例如:使用默认的 data/ip2region.xdb 进行查询测试: +```bash +➜ python git:(python_dev) ✗ python3 ./search_test.py --db=../../data/ip2region.xdb --cache-policy=content +ip2region xdb searcher test program, cachePolicy: content +type 'quit' to exit +ip2region>> 1.2.3.4 +region :美国|0|华盛顿|0|谷歌 , took 0.0689 ms +ip2region>> quit +searcher test program exited, thanks for trying +``` + +输入 ip 即可进行查询测试。也可以分别设置 `cache-policy` 为 file/vectorIndex/content 来测试三种不同缓存实现的效率。 + # bench 测试 + +通过 `bench_test.py` 脚本来进行自动 bench 测试,一方面确保 `xdb` 文件没有错误,另一方面通过大量的查询测试平均查询性能: +```bash +➜ python git:(python_dev) ✗ python3 ./bench_test.py +python bench_test.py [command options] +options: + --db string ip2region binary xdb file path + --src string source ip text file path + --cache-policy string cache policy: file/vectorIndex/content +``` + +例如:通过默认的 data/ip2region.xdb 和 data/ip.merge.txt 来进行 bench 测试: +```bash +➜ python git:(python_dev) ✗ python3 ./bench_test.py --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt --cache-policy=content +Bench finished, [cachePolicy: content, total: 3417955, took: 34.93 s, cost: 0.0094 ms/op] +``` + +可以通过设置 `cache-policy` 参数来分别测试 file/vectorIndex/content 三种不同的缓存实现的的性能。 +@Note:请注意 bench 使用的 src 文件需要是生成对应的 xdb 文件的相同的源文件。 \ No newline at end of file diff --git a/binding/python/bench_test.py b/binding/python/bench_test.py new file mode 100644 index 0000000..be5469f --- /dev/null +++ b/binding/python/bench_test.py @@ -0,0 +1,120 @@ +# +# bench_test.py +# bench_test +# +# Created by luckydog on 2022/7/1. +# Copyright © 2022年 luckydog. All rights reserved. +# +from ast import main +import io + +from xdbSearcher import XdbSearcher +import argparse +import time +import sys + + +def printHelp(): + print("python bench_test.py [command options]") + print("options: ") + print(" --db string ip2region binary xdb file path") + print(" --src string source ip text file path") + print(" --cache-policy string cache policy: file/vectorIndex/content") + +def trim(string): + if string[:1] != ' ' and string[-1:] != ' ': + return string + elif string[:1] == ' ': + return trim(string[1:]) + else: + return trim(string[:-1]) + +def start_bench(dbFile="", srcFile="", cachePolicy="vectorIndex"): + if cachePolicy == "file": + try: + searcher = XdbSearcher(dbfile=dbFile) + except Exception as err: + print(err) + return + elif cachePolicy == "vectorIndex": + try: + vi = XdbSearcher.loadVectorIndexFromFile(dbfile=dbFile) + if vi is None: + print(f"failed to load vector index from {dbFile}\n") + searcher = XdbSearcher(dbfile=dbFile, vectorIndex=vi) + + except Exception as err: + print(err) + return + else: + try: + cb = XdbSearcher.loadContentFromFile(dbfile=dbFile) + if cb is None: + print(f"failed to load xdb content from {dbFile}\n") + searcher = XdbSearcher(contentBuff=cb) + + except Exception as err: + print(err) + return + # do the bench test + + try: + count = 0 + costs = 0 + sTime = time.time() + f = io.open(srcFile, "rb") + while True: + line = trim(f.readline(1024)).decode("utf-8").replace("\n", "") + if len(line) < 1: + break + + ps = line.split("|",2) + if len(ps) != 3: + print(f"invalid ip segment line :{line}") + return + sip = XdbSearcher.ip2long(None, ps[0]) + eip = XdbSearcher.ip2long(None, ps[1]) + + if sip > eip: + print(f"start ip({ps[0]}) should not be greater than end ip({ps[1]})") + return + + mip = (sip + eip) >> 1 + + for ip in [sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip]: + try: + cTime = time.time() + region = searcher.search(ip) + costs = costs + (time.time() - cTime) + except Exception as error: + print(f"failed to search ip :{ip}") + return + + if region is None: + print(f"failed to search ip :{ip}") + return + if region != ps[2]: + print(f"failed search({ip}) with ({region} != {ps[2]})") + return + count = count + 1 + + # close the searcher at last + f.close() + searcher.close() + print(f"Bench finished, [cachePolicy: {cachePolicy}, total: {count}, took: {round(time.time() - sTime, 2)} s, cost: {round(costs/count*1000, 4)} ms/op]") + except Exception as err: + print(f"failed to open source text file :{err}") + return + + +if __name__ == '__main__': + if len(sys.argv) < 2: + printHelp() + exit(0) + parse = argparse.ArgumentParser() + parse.add_argument("--db", help="ip2region binary xdb file path") + parse.add_argument("--src", help="source ip text file path") + parse.add_argument("--cache-policy", choices=["file", "vectorIndex", "content"], + help="cache policy: file/vectorIndex/content") + args = parse.parse_args() + start_bench(dbFile=args.db, srcFile=args.src, cachePolicy=args.cache_policy) diff --git a/binding/python/iptest.py b/binding/python/iptest.py new file mode 100644 index 0000000..428b283 --- /dev/null +++ b/binding/python/iptest.py @@ -0,0 +1,52 @@ + +from xdbSearcher import XdbSearcher + +def searchWithFile(): + # 1. 创建查询对象 + dbPath = "../../data/ip2region.xdb" + searcher = XdbSearcher(dbfile=dbPath) + + # 2. 执行查询 + ip = "1.2.3.4" + region_str = searcher.searchByIPStr(ip) + print(region_str) + + # 3. 关闭searcher + searcher.close() + +def searchWithVectorIndex(): + # 1. 预先加载整个 xdb + dbPath = "../../data/ip2region.xdb" + vi = XdbSearcher.loadVectorIndexFromFile(dbfile=dbPath) + + # 2. 使用上面的缓存创建查询对象, 同时也要加载 xdb 文件 + searcher = XdbSearcher(dbfile=dbPath, vectorIndex=vi) + + # 3. 执行查询 + ip = "1.2.3.4" + region_str = searcher.search(ip) + print(region_str) + + # 4. 关闭searcher + searcher.close() + +def searchWithContent(): + # 1. 预先加载整个 xdb + dbPath = "../../data/ip2region.xdb"; + cb = XdbSearcher.loadContentFromFile(dbfile=dbPath) + + # 2. 仅需要使用上面的全文件缓存创建查询对象, 不需要传源 xdb 文件 + searcher = XdbSearcher(contentBuff=cb) + + # 3. 执行查询 + ip = "1.2.3.4" + region_str = searcher.search(ip) + print(region_str) + + # 4. 关闭searcher + searcher.close() + + +if __name__ == '__main__': + searchWithContent() + \ No newline at end of file diff --git a/binding/python/search_test.py b/binding/python/search_test.py new file mode 100644 index 0000000..12a316a --- /dev/null +++ b/binding/python/search_test.py @@ -0,0 +1,95 @@ +# +# search_test.py +# search_test +# +# Created by luckydog on 2022/7/1. +# Copyright © 2022年 luckydog. All rights reserved. +# + +from xdbSearcher import XdbSearcher +import argparse +import time +import sys + +def printHelp(): + print("python3 search_test.py [command options]") + print("options: ") + print(" --db string ip2region binary xdb file path") + print(" --cache-policy string cache policy: file/vectorIndex/content") + + +def trim(string): + if string[:1] != ' ' and string[-1:] != ' ': + return string + elif string[:1] == ' ': + return trim(string[1:]) + else: + return trim(string[:-1]) + + +def start_search(dbFile="", cachePolicy="vectorIndex"): + if cachePolicy == "file": + try: + searcher = XdbSearcher(dbfile=dbFile) + except Exception as err: + print(err) + return + elif cachePolicy == "vectorIndex": + try: + vi = XdbSearcher.loadVectorIndexFromFile(dbfile=dbFile) + if vi is None: + print(f"failed to load vector index from {dbFile}\n") + searcher = XdbSearcher(dbfile=dbFile, vectorIndex=vi) + + except Exception as err: + print(err) + return + else: + try: + cb = XdbSearcher.loadContentFromFile(dbfile=dbFile) + if cb is None: + print(f"failed to load xdb content from {dbFile}\n") + searcher = XdbSearcher(contentBuff=cb) + + except Exception as err: + print(err) + return + + # 开始的提示 + print(f"ip2region xdb searcher test program, cachePolicy: {cachePolicy}\ntype 'quit' to exit") + while True: + line = trim(input("ip2region>> ")) + # print(f"{line}") + + if len(line) < 2: + continue + if line == "quit": + break + + if not XdbSearcher.isip(None, ip=line): + print("Error: invalid ip address") + continue + start = time.time() + + try: + region_str = searcher.searchByIPStr(line) + except Exception as error: + print(error) + return + + print(f"region :{region_str} , took {round((time.time()-start)*1000.00, 4)} ms") + # quit + searcher.close() + print("searcher test program exited, thanks for trying") + + +if __name__ == '__main__': + if len(sys.argv) < 2: + printHelp() + exit(0) + parse = argparse.ArgumentParser() + parse.add_argument("--db", help="ip2region binary xdb file path") + parse.add_argument("--cache-policy", choices=["file", "vectorIndex", "content"], + help="cache policy: file/vectorIndex/content") + args = parse.parse_args() + start_search(dbFile=args.db, cachePolicy=args.cache_policy) diff --git a/binding/python/xdbSearcher.py b/binding/python/xdbSearcher.py new file mode 100644 index 0000000..eddc847 --- /dev/null +++ b/binding/python/xdbSearcher.py @@ -0,0 +1,191 @@ +# +# ip2Region.py +# ip2Region +# +# Created by luckydog on 2022/6/29. +# Copyright © 2022年 luckydog. All rights reserved. +# + +import socket +import struct +import io +import sys + + +# xdb默认参数 +HeaderInfoLength = 256 +VectorIndexRows = 256 +VectorIndexCols = 256 +VectorIndexSize = 8 +SegmentIndexSize = 14 + + +class XdbSearcher(object): + __f = None + + # the minimal memory allocation. + vectorIndex = None + # 整个读取xdb,保存在内存中 + contentBuff = None + + @staticmethod + def loadVectorIndexFromFile(dbfile): + try: + f = io.open(dbfile, "rb") + f.seek(HeaderInfoLength) + vi_len = VectorIndexRows * VectorIndexCols * SegmentIndexSize + vector_data = f.read(vi_len) + f.close() + return vector_data + except IOError as e: + print("[Error]: %s" % e) + + @staticmethod + def loadContentFromFile(dbfile): + try: + f = io.open(dbfile, "rb") + all_data = f.read() + f.close() + return all_data + except IOError as e: + print("[Error]: %s" % e) + + def __init__(self, dbfile=None, vectorIndex=None, contentBuff=None): + self.initDatabase(dbfile, vectorIndex, contentBuff) + + def search(self, ip): + if isinstance(ip, str): + if not ip.isdigit(): ip = self.ip2long(ip) + return self.searchByIPLong(ip) + else: + return self.searchByIPLong(ip) + + def searchByIPStr(self, ip): + if not ip.isdigit(): ip = self.ip2long(ip) + return self.searchByIPLong(ip) + + def searchByIPLong(self, ip): + # locate the segment index block based on the vector index + sPtr = ePtr = 0 + il0 = (int)((ip >> 24) & 0xFF) + il1 = (int)((ip >> 16) & 0xFF) + idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize + + if self.vectorIndex is not None: + sPtr = self.getLong(self.vectorIndex, idx) + ePtr = self.getLong(self.vectorIndex, idx + 4) + elif self.contentBuff is not None: + sPtr = self.getLong(self.contentBuff, HeaderInfoLength + idx) + ePtr = self.getLong(self.contentBuff, HeaderInfoLength + idx + 4) + else: + self.__f.seek(HeaderInfoLength + idx) + buffer_ptr = self.__f.read(8) + sPtr = self.getLong(buffer_ptr, 0) + ePtr = self.getLong(buffer_ptr, 4) + + # binary search the segment index block to get the region info + dataLen = dataPtr = int(-1) + l = int(0) + h = int((ePtr - sPtr) / SegmentIndexSize) + while l <= h: + m = int((l + h) >> 1) + p = int(sPtr + m * SegmentIndexSize) + # read the segment index + buffer_sip = self.readBuffer(p, SegmentIndexSize) + sip = self.getLong(buffer_sip, 0) + if ip < sip: + h = m - 1 + else: + eip = self.getLong(buffer_sip, 4) + if ip > eip: + l = m + 1 + else: + dataLen = self.getInt2(buffer_sip, 8) + dataPtr = self.getLong(buffer_sip, 10) + break + + # empty match interception + if dataPtr < 0: + return "" + + buffer_string = self.readBuffer(dataPtr, dataLen) + return_string = buffer_string.decode("utf-8") + return return_string + + def readBuffer(self, offset, length): + buffer = None + # check the in-memory buffer first + if self.contentBuff is not None: + buffer = self.contentBuff[offset:offset + length] + return buffer + + # read from the file handle + if self.__f is not None: + self.__f.seek(offset) + buffer = self.__f.read(length) + return buffer + + def initDatabase(self, dbfile, vi, cb): + """ + " initialize the database for search + " param: dbFile, vectorIndex, contentBuff + """ + try: + if cb is not None: + self.__f = None + self.vectorIndex = None + self.contentBuff = cb + else: + self.__f = io.open(dbfile, "rb") + self.vectorIndex = vi + except IOError as e: + print("[Error]: %s" % e) + sys.exit() + + def ip2long(self, ip): + _ip = socket.inet_aton(ip) + return struct.unpack("!L", _ip)[0] + + def isip(self, ip): + p = ip.split(".") + + if len(p) != 4: return False + for pp in p: + if not pp.isdigit(): return False + if len(pp) > 3: return False + if int(pp) > 255: return False + return True + + def getLong(self, b, offset): + if len(b[offset:offset + 4]) == 4: + return struct.unpack('I', b[offset:offset + 4])[0] + return 0 + + def getInt2(self, b, offset): + return ((b[offset] & 0x000000FF) | (b[offset+1] & 0x0000FF00)) + + def close(self): + if self.__f is not None: + self.__f.close() + self.vectorIndex = None + self.contentBuff = None + + +if __name__ == '__main__': + ip_array = [ + "1.2.3.4", + "192.168.1.1" + ] + # 1. 缓存 + dbPath = "./data/ip2region.xdb"; + cb = Searcher.loadContentFromFile(dbfile=dbPath) + + # 2. 创建查询对象 + searcher = Searcher(contentBuff=cb) + + # 3. 执行查询 + # ip = "1.2.3.4" + for ip in ip_array: + region_str = searcher.searchByIPStr(ip) + print(region_str) + searcher.close() \ No newline at end of file