From 894fdc39249a34a0cbed97507ea4d3f8ab1b9c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8E=89=E5=AE=B3=E7=9A=84=E8=8A=B1=E8=8A=B1?= <117415792@qq.com> Date: Thu, 30 Jun 2022 14:33:48 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20python=20=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 厉害的花花 <117415792@qq.com> --- binding/python/ReadMe.md | 67 +++++++++++++ binding/python/ip2Region.py | 182 ++++++++++++++++++++++++++++++++++++ binding/python/iptest.py | 18 ++++ 3 files changed, 267 insertions(+) create mode 100644 binding/python/ip2Region.py create mode 100644 binding/python/iptest.py diff --git a/binding/python/ReadMe.md b/binding/python/ReadMe.md index f6dce47..1d45816 100644 --- a/binding/python/ReadMe.md +++ b/binding/python/ReadMe.md @@ -2,6 +2,73 @@ # 使用方式 +### 完全基于文件的查询 + +```python +import ip2Region + +if __name__ == '__main__': + # 1. 创建查询对象 + dbPath = "./data/ip2region.xdb"; + searcher = ip2Region.Ip2Region(dbfile=dbPath) + + # 2. 执行查询 + ip = "1.2.3.4" + region_str = searcher.searchByIPStr(ip) + print(region_str) + + # 3. 关闭searcher + searcher.close() +``` + +### 缓存 `VectorIndex` 索引 + +我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。 + +```python +import ip2Region + +if __name__ == '__main__': + # 1. 预先加载 VectorIndex 缓存 + dbPath = "./data/ip2region.xdb"; + vi = ip2Region.Ip2Region.loadVectorIndexFromFile(dbfile=dbPath) + + # 2. 使用上面的缓存创建查询对象, 同时也要加载 xdb 文件 + searcher = ip2Region.Ip2Region(dbfile=dbPath, vectorIndex=vi) + + # 3. 执行查询 + ip = "1.2.3.4" + region_str = searcher.searchByIPStr(ip) + print(region_str) + + # 4. 关闭searcher + searcher.close() +``` + +### 缓存整个 `xdb` 数据 + +我们也可以预先加载整个 ip2region.xdb 的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。 + +```python +import ip2Region + +if __name__ == '__main__': + # 1. 预先加载整个 xdb + dbPath = "./data/ip2region.xdb"; + cb = ip2Region.Ip2Region.loadContentFromFile(dbfile=dbPath) + + # 2. 仅需要使用上面的全文件缓存创建查询对象, 不需要传源 xdb 文件 + searcher = ip2Region.Ip2Region(contentBuff=cb) + + # 3. 执行查询 + ip = "1.2.3.4" + region_str = searcher.searchByIPStr(ip) + print(region_str) + + # 4. 关闭searcher + searcher.close() + +``` # 查询测试 # bench 测试 diff --git a/binding/python/ip2Region.py b/binding/python/ip2Region.py new file mode 100644 index 0000000..3443e17 --- /dev/null +++ b/binding/python/ip2Region.py @@ -0,0 +1,182 @@ +# +# ip2Region.py +# ip2Region +# +# Created by luckydog on 2022/6/29. +# Copyright © 2022年 luckydog. All rights reserved. +# + +import socket +import struct +import io +import sys + + +# xdb默认参数 +HeaderInfoLength = 256 +VectorIndexRows = 256 +VectorIndexCols = 256 +VectorIndexSize = 8 +SegmentIndexSize = 14 + + +class Ip2Region(object): + __f = None + + # the minimal memory allocation. + vectorIndex = None + # 整个读取xdb，保存在内存中 + contentBuff = None + + @staticmethod + def loadVectorIndexFromFile(dbfile): + try: + f = io.open(dbfile, "rb") + f.seek(HeaderInfoLength) + vi_len = VectorIndexRows * VectorIndexCols * SegmentIndexSize + vector_data = f.read(vi_len) + f.close() + return vector_data + except IOError as e: + print("[Error]: %s" % e) + + @staticmethod + def loadContentFromFile(dbfile): + try: + f = io.open(dbfile, "rb") + all_data = f.read() + f.close() + return all_data + except IOError as e: + print("[Error]: %s" % e) + + def __init__(self, dbfile=None, vectorIndex=None, contentBuff=None): + self.initDatabase(dbfile, vectorIndex, contentBuff) + + def searchByIPStr(self, ip): + if not ip.isdigit(): ip = self.ip2long(ip) + + # locate the segment index block based on the vector index + sPtr = ePtr = 0 + il0 = (int)((ip >> 24) & 0xFF) + il1 = (int)((ip >> 16) & 0xFF) + idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize + + if self.vectorIndex is not None: + sPtr = self.getLong(self.vectorIndex, idx) + ePtr = self.getLong(self.vectorIndex, idx + 4) + elif self.contentBuff is not None: + sPtr = self.getLong(self.contentBuff, HeaderInfoLength + idx) + ePtr = self.getLong(self.contentBuff, HeaderInfoLength + idx + 4) + else: + self.__f.seek(HeaderInfoLength + idx) + buffer_ptr = self.__f.read(8) + sPtr = self.getLong(buffer_ptr, 0) + ePtr = self.getLong(buffer_ptr, 4) + + # binary search the segment index block to get the region info + dataLen = dataPtr = int(-1) + l = int(0) + h = int((ePtr - sPtr) / SegmentIndexSize) + while l <= h: + m = int((l + h) >> 1) + p = int(sPtr + m * SegmentIndexSize) + # read the segment index + buffer_sip = self.readBuffer(p, SegmentIndexSize) + sip = self.getLong(buffer_sip, 0) + if ip < sip: + h = m - 1 + else: + eip = self.getLong(buffer_sip, 4) + if ip > eip: + l = m + 1 + else: + dataLen = self.getInt2(buffer_sip, 8) + dataPtr = self.getLong(buffer_sip, 10) + break + + # empty match interception + if dataPtr < 0: + return "" + + buffer_string = self.readBuffer(dataPtr, dataLen) + return_string = buffer_string.decode("utf-8") + return return_string + + def readBuffer(self, offset, length): + buffer = None + # check the in-memory buffer first + if self.contentBuff is not None: + buffer = self.contentBuff[offset:offset + length] + return buffer + + # read from the file handle + if self.__f is not None: + self.__f.seek(offset) + buffer = self.__f.read(length) + return buffer + + def initDatabase(self, dbfile, vi, cb): + """ + " initialize the database for search + " param: dbFile, vectorIndex, contentBuff + """ + try: + if cb is not None: + self.__f = None + self.vectorIndex = None + self.contentBuff = cb + else: + self.__f = io.open(dbfile, "rb") + self.vectorIndex = vi + except IOError as e: + print("[Error]: %s" % e) + sys.exit() + + def ip2long(self, ip): + _ip = socket.inet_aton(ip) + return struct.unpack("!L", _ip)[0] + + def isip(self, ip): + p = ip.split(".") + + if len(p) != 4: return False + for pp in p: + if not pp.isdigit(): return False + if len(pp) > 3: return False + if int(pp) > 255: return False + return True + + def getLong(self, b, offset): + if len(b[offset:offset + 4]) == 4: + return struct.unpack('I', b[offset:offset + 4])[0] + return 0 + + def getInt2(self, b, offset): + return ((b[offset] & 0x000000FF) | (b[offset+1] & 0x0000FF00)) + + def close(self): + if self.__f is not None: + self.__f.close() + self.vectorIndex = None + self.contentBuff = None + + +if __name__ == '__main__': + ip_array = [ + "1.2.3.4", + "192.168.1.1" + ] + # 1. 缓存 + dbPath = "./data/ip2region.xdb"; + cb = Ip2Region.loadContentFromFile(dbfile=dbPath) + + # 2. 创建查询对象 + searcher = Ip2Region(contentBuff=cb) + + # 3. 执行查询 + ip = "1.2.3.4" + for ip in ip_array: + region_str = searcher.searchByIPStr(ip) + print(region_str) + searcher.close() \ No newline at end of file diff --git a/binding/python/iptest.py b/binding/python/iptest.py new file mode 100644 index 0000000..2883aa0 --- /dev/null +++ b/binding/python/iptest.py @@ -0,0 +1,18 @@ +import ip2Region + +if __name__ == '__main__': + # 1. 预先加载整个 xdb + dbPath = "./data/ip2region.xdb"; + # vi = ip2Region.Ip2Region.loadVectorIndexFromFile(dbfile="./data/ip2region.xdb") + cb = ip2Region.Ip2Region.loadContentFromFile(dbfile=dbPath) + + # 2. 仅需要使用上面的全文件缓存创建查询对象, 不需要传源 xdb 文件 + searcher = ip2Region.Ip2Region(contentBuff=cb) + + # 3. 执行查询 + ip = "1.2.3.4" + region_str = searcher.searchByIPStr(ip) + print(region_str) + + # 4. 关闭searcher + searcher.close() \ No newline at end of file