From caad8bad88ab1921d62603029f4ac3deeee86009 Mon Sep 17 00:00:00 2001 From: koma Date: Wed, 13 Jul 2016 15:54:08 +0800 Subject: [PATCH] add memory search alrogrithm and optimize script --- binding/python/ip2Region.py | 134 ++++++++++++++++++++++----------- binding/python/testSearcher.py | 12 ++- 2 files changed, 99 insertions(+), 47 deletions(-) diff --git a/binding/python/ip2Region.py b/binding/python/ip2Region.py index 351fb9b..100566c 100644 --- a/binding/python/ip2Region.py +++ b/binding/python/ip2Region.py @@ -10,11 +10,52 @@ import struct, io, socket, sys class Ip2Region(object): __headerSip = [] __headerPtr = [] - __f = None + __f = None + __sPtr = 0 + __indexLen = 0 + __dbBinStr = '' def __init__(self, dbfile): self.initDatabase(dbfile) + def memorySearch(self, ip): + """ + " memory search method + " param: ip + """ + if not ip.isdigit(): ip = self.ip2long(ip) + + if self.__dbBinStr == '': + self.__dbBinStr = self.__f.read() #read all the contents in file + self.__sPtr = self.getLong(self.__dbBinStr, 0) + endPtr = self.getLong(self.__dbBinStr, 4) + self.__indexLen = endPtr - self.__sPtr + + startPtr = self.__sPtr + indexLen = self.__indexLen + dbBinStr = self.__dbBinStr + + l, h, mixPtr = (0, int(indexLen/12), 0) + while l <= h: + m = int((l+h)/2) + ptr = startPtr + m*12 + + sip = self.getLong(dbBinStr, ptr) + eip = self.getLong(dbBinStr, ptr+4) + + if ip >= sip: + if ip > eip: + l = m + 1 + else: + mixPtr = self.getLong(dbBinStr, ptr+8) + break; + else: + h = m - 1 + + if mixPtr == 0: return "N2" + + return self.returnData(mixPtr) + def binarySearch(self, ip): """ " binary search method @@ -22,12 +63,16 @@ class Ip2Region(object): """ if not ip.isdigit(): ip = self.ip2long(ip) - self.__f.seek(0) - b = self.__f.read(8) - startPtr = self.getLong(b, 0) - endPtr = self.getLong(b, 4) + if self.__indexLen < 1: + self.__f.seek(0) + b = self.__f.read(8) + self.__sPtr = self.getLong(b, 0) + endPtr = self.getLong(b, 4) + self.__indexLen = endPtr - self.__sPtr + + startPtr = self.__sPtr + indexLen = self.__indexLen - indexLen = endPtr - startPtr self.__f.seek(startPtr) b = self.__f.read(indexLen+12) @@ -37,7 +82,7 @@ class Ip2Region(object): ptr = startPtr + m*12 self.__f.seek(ptr) - b = self.__f.read(12) + b = self.__f.read(12) sip = self.getLong(b, 0) eip = self.getLong(b, 4) @@ -52,15 +97,7 @@ class Ip2Region(object): if mixPtr == 0: return "N2" - dataPtr = mixPtr & 0x00FFFFFFL - dataLen = (mixPtr >> 24) & 0xFF - - self.__f.seek(dataPtr) - data = self.__f.read(dataLen) - return { - "city_id": self.getLong(data, 0), - "region" : data[4:] - } + return self.returnData(mixPtr) def btreeSearch(self, ip): """ @@ -69,7 +106,23 @@ class Ip2Region(object): """ if not ip.isdigit(): ip = self.ip2long(ip) - headerLen = len(self.__headerSip) - 1 + if len(self.__headerSip) < 1: + #pass the super block + self.__f.seek(8) + #read the header block + b = self.__f.read(4086) + #parse the header block + sip = None + ptr = None + for i in range(0, len(b)-1, 8): + sip = self.getLong(b, i) + ptr = self.getLong(b, i+4) + if ptr == 0: + break + self.__headerSip.append(sip) + self.__headerPtr.append(ptr) + + headerLen = len(self.__headerSip) - 1 l, h, sptr, eptr = (0, headerLen, 0, 0) while l <= h: m = int((l+h)/2) @@ -129,15 +182,7 @@ class Ip2Region(object): if mixPtr == 0: return "N2" - dataPtr = mixPtr & 0x00FFFFFFL - dataLen = (mixPtr >> 24) & 0xFF - - self.__f.seek(dataPtr) - data = self.__f.read(dataLen) - return { - "city_id": self.getLong(data, 0), - "region" : data[4:] - } + return self.returnData(mixPtr) def initDatabase(self, dbfile): """ @@ -146,27 +191,29 @@ class Ip2Region(object): """ try: self.__f = io.open(dbfile, "rb") - #pass the super block - self.__f.seek(8) - #read the header block - b = self.__f.read(4086) - #parse the header block - sip = None - ptr = None - for i in range(0, len(b)-1, 8): - sip = self.getLong(b, i) - ptr = self.getLong(b, i+4) - if ptr == 0: - break - self.__headerSip.append(sip) - self.__headerPtr.append(ptr) - except IOError, e: print "[Error]: ", e sys.exit() + def returnData(self, dsptr): + """ + " get ip data from db file by data start ptr + " param: dsptr + """ + dataPtr = dsptr & 0x00FFFFFFL + dataLen = (dsptr >> 24) & 0xFF + + self.__f.seek(dataPtr) + data = self.__f.read(dataLen) + + return { + "city_id": self.getLong(data, 0), + "region" : data[4:] + } + def ip2long(self, ip): _ip = socket.inet_aton(ip) + return struct.unpack("!L", _ip)[0] def isip(self, ip): @@ -174,15 +221,16 @@ class Ip2Region(object): if len(p) != 4 : return False for pp in p: - if not pp.isdigit(): return False + if not pp.isdigit() : return False if len(pp) > 3 : return False - if int(pp) > 255 : return False + if int(pp) > 255 : return False return True def getLong(self, b, offset): if len( b[offset:offset+4] ) == 4: return struct.unpack('I', b[offset:offset+4])[0] + return 0 def close(self): diff --git a/binding/python/testSearcher.py b/binding/python/testSearcher.py index 9ecbd17..a2fa528 100644 --- a/binding/python/testSearcher.py +++ b/binding/python/testSearcher.py @@ -31,11 +31,13 @@ def testSearch(): algorithm = sys.argv[2] if algorithm == "binary": method = 2 + elif algorithm == "memory": + method = 3 print "initializing %s..." % (algorithm) print "+----------------------------------+" - print "| ip2region test script |" - print "| Author: komazhang@foxmail.com |" + print "| ip2region test program |" + print "| Author: chenxin619315@gmail.com. |" print "| Type 'quit' to exit program |" print "+----------------------------------+" @@ -60,12 +62,14 @@ def testSearch(): sTime = time.time() * 1000 if method == 1: data = searcher.btreeSearch(line) - else: + elif method == 2: data = searcher.binarySearch(line) + else: + data = searcher.memorySearch(line) eTime = time.time() * 1000 if isinstance(data, dict): - print "[Return]: %s|%s in %f millseconds" % (data["city_id"], data["region"], eTime-sTime) + print "%s|%s in %f millseconds" % (data["city_id"], data["region"], eTime-sTime) else: print "[Error]: ", data