增加 python 查询实现

Signed-off-by: 厉害的花花 <117415792@qq.com>
2022-06-30 14:33:48 +08:00 · 2022-06-30 14:33:48 +08:00 · 78b999b338
parent 8cc0c6057d
commit 78b999b338
3 changed files with 267 additions and 0 deletions
--- a/binding/python/ReadMe.md
+++ b/binding/python/ReadMe.md
@ -2,6 +2,73 @@

 # 使用方式

+### 完全基于文件的查询
+
+```python
+import ip2Region
+
+if __name__ == '__main__':
+    # 1. 创建查询对象
+    dbPath = "./data/ip2region.xdb";
+    searcher = ip2Region.Ip2Region(dbfile=dbPath)
+    
+    # 2. 执行查询
+    ip = "1.2.3.4"
+    region_str = searcher.searchByIPStr(ip)
+    print(region_str)
+    
+    # 3. 关闭searcher
+    searcher.close()
+```
+
+### 缓存 `VectorIndex` 索引
+
+我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。
+
+```python
+import ip2Region
+
+if __name__ == '__main__':
+    # 1. 预先加载 VectorIndex 缓存
+    dbPath = "./data/ip2region.xdb";
+    vi = ip2Region.Ip2Region.loadVectorIndexFromFile(dbfile=dbPath)
+
+    # 2. 使用上面的缓存创建查询对象, 同时也要加载 xdb 文件
+    searcher = ip2Region.Ip2Region(dbfile=dbPath, vectorIndex=vi)
+    
+    # 3. 执行查询
+    ip = "1.2.3.4"
+    region_str = searcher.searchByIPStr(ip)
+    print(region_str)
+    
+    # 4. 关闭searcher
+    searcher.close()
+```
+
+### 缓存整个 `xdb` 数据
+
+我们也可以预先加载整个 ip2region.xdb 的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。
+
+```python
+import ip2Region
+
+if __name__ == '__main__':
+    # 1. 预先加载整个 xdb
+    dbPath = "./data/ip2region.xdb";
+    cb = ip2Region.Ip2Region.loadContentFromFile(dbfile=dbPath)
+    
+    # 2. 仅需要使用上面的全文件缓存创建查询对象, 不需要传源 xdb 文件
+    searcher = ip2Region.Ip2Region(contentBuff=cb)
+    
+    # 3. 执行查询
+    ip = "1.2.3.4"
+    region_str = searcher.searchByIPStr(ip)
+    print(region_str)
+    
+    # 4. 关闭searcher
+    searcher.close()
+
+```
 # 查询测试

 # bench 测试
--- a/binding/python/ip2Region.py
+++ b/binding/python/ip2Region.py
@ -0,0 +1,182 @@
+#
+#  ip2Region.py
+#  ip2Region
+#
+#  Created by luckydog on 2022/6/29.
+#  Copyright © 2022年 luckydog. All rights reserved.
+#
+
+import socket
+import struct
+import io
+import sys
+
+
+# xdb默认参数
+HeaderInfoLength = 256
+VectorIndexRows = 256
+VectorIndexCols = 256
+VectorIndexSize = 8
+SegmentIndexSize = 14
+
+
+class Ip2Region(object):
+    __f = None
+
+    # the minimal memory allocation.
+    vectorIndex = None
+    # 整个读取xdb，保存在内存中
+    contentBuff = None
+
+    @staticmethod
+    def loadVectorIndexFromFile(dbfile):
+        try:
+            f = io.open(dbfile, "rb")
+            f.seek(HeaderInfoLength)
+            vi_len = VectorIndexRows * VectorIndexCols * SegmentIndexSize
+            vector_data = f.read(vi_len)
+            f.close()
+            return vector_data
+        except IOError as e:
+            print("[Error]: %s" % e)
+
+    @staticmethod
+    def loadContentFromFile(dbfile):
+        try:
+            f = io.open(dbfile, "rb")
+            all_data = f.read()
+            f.close()
+            return all_data
+        except IOError as e:
+            print("[Error]: %s" % e)
+
+    def __init__(self, dbfile=None, vectorIndex=None, contentBuff=None):
+        self.initDatabase(dbfile, vectorIndex, contentBuff)
+
+    def searchByIPStr(self, ip):
+        if not ip.isdigit(): ip = self.ip2long(ip)
+
+        # locate the segment index block based on the vector index
+        sPtr = ePtr = 0
+        il0 = (int)((ip >> 24) & 0xFF)
+        il1 = (int)((ip >> 16) & 0xFF)
+        idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize
+
+        if self.vectorIndex is not None:
+            sPtr = self.getLong(self.vectorIndex, idx)
+            ePtr = self.getLong(self.vectorIndex, idx + 4)
+        elif self.contentBuff is not None:
+            sPtr = self.getLong(self.contentBuff, HeaderInfoLength + idx)
+            ePtr = self.getLong(self.contentBuff, HeaderInfoLength + idx + 4)
+        else:
+            self.__f.seek(HeaderInfoLength + idx)
+            buffer_ptr = self.__f.read(8)
+            sPtr = self.getLong(buffer_ptr, 0)
+            ePtr = self.getLong(buffer_ptr, 4)
+
+        # binary search the segment index block to get the region info
+        dataLen = dataPtr = int(-1)
+        l = int(0)
+        h = int((ePtr - sPtr) / SegmentIndexSize)
+        while l <= h:
+            m = int((l + h) >> 1)
+            p = int(sPtr + m * SegmentIndexSize)
+            # read the segment index
+            buffer_sip = self.readBuffer(p, SegmentIndexSize)
+            sip = self.getLong(buffer_sip, 0)
+            if ip < sip:
+                h = m - 1
+            else:
+                eip = self.getLong(buffer_sip, 4)
+                if ip > eip:
+                    l = m + 1
+                else:
+                    dataLen = self.getInt2(buffer_sip, 8)
+                    dataPtr = self.getLong(buffer_sip, 10)
+                    break
+
+        # empty match interception
+        if dataPtr < 0:
+            return ""
+
+        buffer_string = self.readBuffer(dataPtr, dataLen)
+        return_string = buffer_string.decode("utf-8")
+        return return_string
+
+    def readBuffer(self, offset, length):
+        buffer = None
+        # check the in-memory buffer first
+        if self.contentBuff is not None:
+            buffer = self.contentBuff[offset:offset + length]
+            return buffer
+
+        # read from the file handle
+        if self.__f is not None:
+            self.__f.seek(offset)
+            buffer = self.__f.read(length)
+        return buffer
+
+    def initDatabase(self, dbfile, vi, cb):
+        """
+        " initialize the database for search
+        " param: dbFile, vectorIndex, contentBuff
+        """
+        try:
+            if cb is not None:
+                self.__f = None
+                self.vectorIndex = None
+                self.contentBuff = cb
+            else:
+                self.__f = io.open(dbfile, "rb")
+                self.vectorIndex = vi
+        except IOError as e:
+            print("[Error]: %s" % e)
+            sys.exit()
+
+    def ip2long(self, ip):
+        _ip = socket.inet_aton(ip)
+        return struct.unpack("!L", _ip)[0]
+
+    def isip(self, ip):
+        p = ip.split(".")
+
+        if len(p) != 4: return False
+        for pp in p:
+            if not pp.isdigit(): return False
+            if len(pp) > 3: return False
+            if int(pp) > 255: return False
+        return True
+
+    def getLong(self, b, offset):
+        if len(b[offset:offset + 4]) == 4:
+            return struct.unpack('I', b[offset:offset + 4])[0]
+        return 0
+
+    def getInt2(self, b, offset):
+        return ((b[offset] & 0x000000FF) | (b[offset+1] & 0x0000FF00))
+
+    def close(self):
+        if self.__f is not None:
+            self.__f.close()
+        self.vectorIndex = None
+        self.contentBuff = None
+
+
+if __name__ == '__main__':
+    ip_array = [
+        "1.2.3.4",
+        "192.168.1.1"
+    ]
+    # 1. 缓存
+    dbPath = "./data/ip2region.xdb";
+    cb = Ip2Region.loadContentFromFile(dbfile=dbPath)
+    
+    # 2. 创建查询对象
+    searcher = Ip2Region(contentBuff=cb)
+    
+    # 3. 执行查询
+    ip = "1.2.3.4"
+    for ip in ip_array:
+        region_str = searcher.searchByIPStr(ip)
+        print(region_str)
+    searcher.close()
--- a/binding/python/iptest.py
+++ b/binding/python/iptest.py
@ -0,0 +1,18 @@
+import ip2Region
+
+if __name__ == '__main__':
+    # 1. 预先加载整个 xdb
+    dbPath = "./data/ip2region.xdb";
+    # vi = ip2Region.Ip2Region.loadVectorIndexFromFile(dbfile="./data/ip2region.xdb")
+    cb = ip2Region.Ip2Region.loadContentFromFile(dbfile=dbPath)
+    
+    # 2. 仅需要使用上面的全文件缓存创建查询对象, 不需要传源 xdb 文件
+    searcher = ip2Region.Ip2Region(contentBuff=cb)
+    
+    # 3. 执行查询
+    ip = "1.2.3.4"
+    region_str = searcher.searchByIPStr(ip)
+    print(region_str)
+    
+    # 4. 关闭searcher
+    searcher.close()