diff --git a/binding/python/beachmark.py b/binding/python/beachmark.py new file mode 100644 index 0000000..533746f --- /dev/null +++ b/binding/python/beachmark.py @@ -0,0 +1,49 @@ +#-*- coding:utf-8 -*- +""" +" ip2region python seacher client module benchmark test file +" +" Autho: koma +" Date : 2018-10-04 +""" +import threading +import time + +from ip2Region import Ip2Region + +class BeachmarkThread(threading.Thread): + __searcher = None + __lock = None + + def __init__(self, searcher, lock): + self.__searcher = searcher + self.__lock = lock + threading.Thread.__init__(self) + + def run(self): + self.__lock.acquire() + try: + sTime = time.time() * 1000 + data = self.__searcher.memorySearch("49.220.138.233") + eTime = time.time() * 1000 + print("%s|%s in %5f millseconds" % (data["city_id"], data["region"].decode('utf-8'), eTime - sTime)) + finally: + self.__lock.release() + +if __name__ == "__main__": + threads = [] + searcher = Ip2Region("/data/ip2region/data/ip2region.db") + lock = threading.Lock() + + for i in range(10000): + t = BeachmarkThread(searcher, lock) + threads.append(t) + + sTime = time.time() * 1000 + for t in threads: + t.start() + + for t in threads: + t.join() + eTime = time.time() * 1000 + + print("Beachmark done: %5f" % (eTime - sTime)) diff --git a/binding/python/ip2Region.py b/binding/python/ip2Region.py index f30dc4b..7e7d699 100644 --- a/binding/python/ip2Region.py +++ b/binding/python/ip2Region.py @@ -8,12 +8,17 @@ import struct, io, socket, sys class Ip2Region(object): - __headerSip = [] - __headerPtr = [] - __f = None - __sPtr = 0 - __indexLen = 0 - __dbBinStr = '' + __INDEX_BLOCK_LENGTH = 12 + __TOTAL_HEADER_LENGTH = 8192 + + __f = None + __headerSip = [] + __headerPtr = [] + __headerLen = 0 + __indexSPtr = 0 + __indexLPtr = 0 + __indexCount = 0 + __dbBinStr = '' def __init__(self, dbfile): self.initDatabase(dbfile) @@ -26,35 +31,30 @@ class Ip2Region(object): if not ip.isdigit(): ip = self.ip2long(ip) if self.__dbBinStr == '': - self.__dbBinStr = self.__f.read() #read all the contents in file - self.__sPtr = self.getLong(self.__dbBinStr, 0) - endPtr = self.getLong(self.__dbBinStr, 4) - self.__indexLen = endPtr - self.__sPtr + self.__dbBinStr = self.__f.read() #read all the contents in file + self.__indexSPtr = self.getLong(self.__dbBinStr, 0) + self.__indexLPtr = self.getLong(self.__dbBinStr, 4) + self.__indexCount = int((self.__indexLPtr - self.__indexSPtr)/self.__INDEX_BLOCK_LENGTH)+1 - startPtr = self.__sPtr - indexLen = self.__indexLen - dbBinStr = self.__dbBinStr - - l, h, mixPtr = (0, int(indexLen/12), 0) + l, h, dataPtr = (0, self.__indexCount, 0) while l <= h: - m = int((l+h)/2) - ptr = startPtr + m*12 + m = int((l+h) >> 1) + p = self.__indexSPtr + m*self.__INDEX_BLOCK_LENGTH + sip = self.getLong(self.__dbBinStr, p) - sip = self.getLong(dbBinStr, ptr) - eip = self.getLong(dbBinStr, ptr+4) - - if ip >= sip: - if ip > eip: - l = m + 1 - else: - mixPtr = self.getLong(dbBinStr, ptr+8) - break; + if ip < sip: + h = m -1 else: - h = m - 1 + eip = self.getLong(self.__dbBinStr, p+4) + if ip > eip: + l = m + 1; + else: + dataPtr = self.getLong(self.__dbBinStr, p+8) + break - if mixPtr == 0: return "N2" + if dataPtr == 0: raise Exception("Data pointer not found") - return self.returnData(mixPtr) + return self.returnData(dataPtr) def binarySearch(self, ip): """ @@ -63,41 +63,34 @@ class Ip2Region(object): """ if not ip.isdigit(): ip = self.ip2long(ip) - if self.__indexLen < 1: + if self.__indexCount == 0: self.__f.seek(0) - b = self.__f.read(8) - self.__sPtr = self.getLong(b, 0) - endPtr = self.getLong(b, 4) - self.__indexLen = endPtr - self.__sPtr + superBlock = self.__f.read(8) + self.__indexSPtr = self.getLong(superBlock, 0) + self.__indexLPtr = self.getLong(superBlock, 4) + self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1 - startPtr = self.__sPtr - indexLen = self.__indexLen - - self.__f.seek(startPtr) - b = self.__f.read(indexLen+12) - - l, h, mixPtr = (0, int(indexLen/12), 0) + l, h, dataPtr = (0, self.__indexCount, 0) while l <= h: - m = int((l+h)/2) - ptr = startPtr + m*12 - self.__f.seek(ptr) + m = int((l+h) >> 1) + p = m*self.__INDEX_BLOCK_LENGTH - b = self.__f.read(12) - sip = self.getLong(b, 0) - eip = self.getLong(b, 4) - - if ip >= sip: + self.__f.seek(self.__indexSPtr+p) + buffer = self.__f.read(self.__INDEX_BLOCK_LENGTH) + sip = self.getLong(buffer, 0) + if ip < sip: + h = m - 1 + else: + eip = self.getLong(buffer, 4) if ip > eip: l = m + 1 else: - mixPtr = self.getLong(b, 8) - break; - else: - h = m - 1 + dataPtr = self.getLong(buffer, 8) + break - if mixPtr == 0: return "N2" + if dataPtr == 0: raise Exception("Data pointer not found") - return self.returnData(mixPtr) + return self.returnData(dataPtr) def btreeSearch(self, ip): """ @@ -107,82 +100,81 @@ class Ip2Region(object): if not ip.isdigit(): ip = self.ip2long(ip) if len(self.__headerSip) < 1: + headerLen = 0 #pass the super block self.__f.seek(8) #read the header block - b = self.__f.read(8192) + b = self.__f.read(self.__TOTAL_HEADER_LENGTH) #parse the header block - sip = None - ptr = None - for i in range(0, len(b)-1, 8): + for i in range(0, len(b), 8): sip = self.getLong(b, i) ptr = self.getLong(b, i+4) if ptr == 0: break self.__headerSip.append(sip) self.__headerPtr.append(ptr) + headerLen += 1 + self.__headerLen = headerLen - headerLen = len(self.__headerSip) - 1 - l, h, sptr, eptr = (0, headerLen, 0, 0) + l, h, sptr, eptr = (0, self.__headerLen, 0, 0) while l <= h: - m = int((l+h)/2) + m = int((l+h) >> 1) if ip == self.__headerSip[m]: if m > 0: sptr = self.__headerPtr[m-1] eptr = self.__headerPtr[m] - break; else: sptr = self.__headerPtr[m] eptr = self.__headerPtr[m+1] - break; + break - if ip > self.__headerSip[m]: - if m == headerLen: - sptr = self.__headerPtr[m-1] - eptr = self.__headerPtr[m] - break; - elif ip < self.__headerSip[m+1]: - sptr = self.__headerPtr[m] - eptr = self.__headerPtr[m+1] - break; - - l = m + 1 - else: + if ip < self.__headerSip[m]: if m == 0: sptr = self.__headerPtr[m] eptr = self.__headerPtr[m+1] - break; + break elif ip > self.__headerSip[m-1]: sptr = self.__headerPtr[m-1] eptr = self.__headerPtr[m] - break; - + break h = m - 1 + else: + if m == self.__headerLen - 1: + sptr = self.__headerPtr[m-1] + eptr = self.__headerPtr[m] + break + elif ip <= self.__headerSip[m+1]: + sptr = self.__headerPtr[m] + eptr = self.__headerPtr[m+1] + break + l = m + 1 - if sptr == 0: return "N1" + if sptr == 0: raise Exception("Index pointer not found") indexLen = eptr - sptr self.__f.seek(sptr) - b = self.__f.read(indexLen + 12) + index = self.__f.read(indexLen + self.__INDEX_BLOCK_LENGTH) - l, h, mixPtr = (0, int(indexLen/12), 0) + l, h, dataPrt = (0, int(indexLen/self.__INDEX_BLOCK_LENGTH), 0) while l <= h: - m = int((l+h)/2) - offset = m * 12 + m = int((l+h) >> 1) + offset = int(m * self.__INDEX_BLOCK_LENGTH) + sip = self.getLong(index, offset) - if ip >= self.getLong(b, offset): - if ip > self.getLong(b, offset+4): - l = m + 1 - else: - mixPtr = self.getLong(b, offset+8) - break; - else: + if ip < sip: h = m - 1 + else: + eip = self.getLong(index, offset+4) + if ip > eip: + l = m + 1; + else: + dataPrt = self.getLong(index, offset+8) + break - if mixPtr == 0: return "N2" + if dataPrt == 0: raise Exception("Data pointer not found") - return self.returnData(mixPtr) + return self.returnData(dataPrt) def initDatabase(self, dbfile): """ @@ -191,18 +183,18 @@ class Ip2Region(object): """ try: self.__f = io.open(dbfile, "rb") - except IOError, e: - print "[Error]: ", e + except IOError as e: + print("[Error]: %s" % e) sys.exit() - def returnData(self, dsptr): + def returnData(self, dataPtr): """ " get ip data from db file by data start ptr " param: dsptr """ - dataPtr = dsptr & 0x00FFFFFFL - dataLen = (dsptr >> 24) & 0xFF - + dataLen = (dataPtr >> 24) & 0xFF + dataPtr = dataPtr & 0x00FFFFFF + self.__f.seek(dataPtr) data = self.__f.read(dataLen) @@ -213,7 +205,6 @@ class Ip2Region(object): def ip2long(self, ip): _ip = socket.inet_aton(ip) - return struct.unpack("!L", _ip)[0] def isip(self, ip): @@ -228,13 +219,14 @@ class Ip2Region(object): return True def getLong(self, b, offset): - if len( b[offset:offset+4] ) == 4: + if len(b[offset:offset+4]) == 4: return struct.unpack('I', b[offset:offset+4])[0] - return 0 def close(self): - self.__headerSip = None + if self.__f != None: + self.__f.close() + + self.__dbBinStr = None self.__headerPtr = None - self.__f.close() - self.__f = None + self.__headerSip = None diff --git a/binding/python/testSearcher.py b/binding/python/testSearcher.py index bc5726a..f8cde3c 100644 --- a/binding/python/testSearcher.py +++ b/binding/python/testSearcher.py @@ -6,72 +6,76 @@ " Date : 2015-11-06 """ import struct, sys, os, time +from platform import python_version + from ip2Region import Ip2Region def testSearch(): """ " ip2region test function """ - llen = len(sys.argv) + argLen = len(sys.argv) + version = python_version() + algorithms = ["binary", "b-tree", "memory"] - if llen < 2: - print "Usage: python testSearcher.py [ip2region db file] [alrogrithm]" - print "Algorithm: binary or b-tree" + if argLen < 2: + print("Usage: python testSearcher.py [ip2region db file] [alrogrithm]") + print("Algorithm: %s" % ", ".join(algorithms)) return 0 - dbFile = sys.argv[1] - method = 1 - algorithm = "b-tree" - + dbFile = sys.argv[1] + if (not os.path.isfile(dbFile)) or (not os.path.exists(dbFile)): - print "[Error]: Specified db file is not exists." + print("[Error]: Specified db file is not exists.") return 0 - if llen > 2: + if argLen > 2: algorithm = sys.argv[2] - if algorithm == "binary": - method = 2 - elif algorithm == "memory": - method = 3 + try: + algorithms.index(algorithm) + except Exception as e: + algorithm = "b-tree" - print "initializing %s..." % (algorithm) - print "+----------------------------------+" - print "| ip2region test program |" - print "| Author: chenxin619315@gmail.com. |" - print "| Type 'quit' to exit program |" - print "+----------------------------------+" + print("initializing %s..." % (algorithm)) + print("+----------------------------------+") + print("| ip2region test program |") + print("| Author: chenxin619315@gmail.com. |") + print("| Type 'quit' to exit program |") + print("+----------------------------------+") - searcher = Ip2Region(dbFile); + searcher = Ip2Region(dbFile) while True: - line = raw_input("ip2region>> ") + if version[:1] == "2": + line = raw_input("ip2region>> ") + else: + line = input("ip2region>> ") line = line.strip() if line == "": - print "[Error]: Invalid ip address." + print("[Error]: Invalid ip address.") continue if line == "quit": - print "[Info]: Thanks for your use, Bye." + print("[Info]: Thanks for your use, Bye.") break if not searcher.isip(line): - print "[Error]: Invalid ip address." + print("[Error]: Invalid ip address.") continue - sTime = time.time() * 1000 - if method == 1: - data = searcher.btreeSearch(line) - elif method == 2: - data = searcher.binarySearch(line) - else: - data = searcher.memorySearch(line) - eTime = time.time() * 1000 - - if isinstance(data, dict): - print "%s|%s in %f millseconds" % (data["city_id"], data["region"], eTime-sTime) - else: - print "[Error]: ", data + try: + sTime = time.time()*1000 + if algorithm == "binary": + data = searcher.binarySearch(line) + elif algorithm == "memory": + data = searcher.memorySearch(line) + else: + data = searcher.btreeSearch(line) + eTime = time.time()*1000 + print("%s|%s in %5f millseconds" % (data["city_id"], data["region"].decode('utf-8'), eTime - sTime)) + except Exception as e: + print("[Error]: %s" % e) searcher.close() diff --git a/binding/python3/ip2Region.py b/binding/python3/ip2Region.py deleted file mode 100644 index 1636434..0000000 --- a/binding/python3/ip2Region.py +++ /dev/null @@ -1,240 +0,0 @@ -#-*- coding:utf-8 -*- -""" -" ip2region python seacher client module -" -" Author: koma -" Date : 2015-11-06 -""" -import struct, io, socket, sys - -class Ip2Region(object): - __headerSip = [] - __headerPtr = [] - __f = None - __sPtr = 0 - __indexLen = 0 - __dbBinStr = '' - - def __init__(self, dbfile): - self.initDatabase(dbfile) - - def memorySearch(self, ip): - """ - " memory search method - " param: ip - """ - if not ip.isdigit(): ip = self.ip2long(ip) - - if self.__dbBinStr == '': - self.__dbBinStr = self.__f.read() #read all the contents in file - self.__sPtr = self.getLong(self.__dbBinStr, 0) - endPtr = self.getLong(self.__dbBinStr, 4) - self.__indexLen = endPtr - self.__sPtr - - startPtr = self.__sPtr - indexLen = self.__indexLen - dbBinStr = self.__dbBinStr - - l, h, mixPtr = (0, int(indexLen/12), 0) - while l <= h: - m = int((l+h)/2) - ptr = startPtr + m*12 - - sip = self.getLong(dbBinStr, ptr) - eip = self.getLong(dbBinStr, ptr+4) - - if ip >= sip: - if ip > eip: - l = m + 1 - else: - mixPtr = self.getLong(dbBinStr, ptr+8) - break; - else: - h = m - 1 - - if mixPtr == 0: return "N2" - - return self.returnData(mixPtr) - - def binarySearch(self, ip): - """ - " binary search method - " param: ip - """ - if not ip.isdigit(): ip = self.ip2long(ip) - - if self.__indexLen < 1: - self.__f.seek(0) - b = self.__f.read(8) - self.__sPtr = self.getLong(b, 0) - endPtr = self.getLong(b, 4) - self.__indexLen = endPtr - self.__sPtr - - startPtr = self.__sPtr - indexLen = self.__indexLen - - self.__f.seek(startPtr) - b = self.__f.read(indexLen+12) - - l, h, mixPtr = (0, int(indexLen/12), 0) - while l <= h: - m = int((l+h)/2) - ptr = startPtr + m*12 - self.__f.seek(ptr) - - b = self.__f.read(12) - sip = self.getLong(b, 0) - eip = self.getLong(b, 4) - - if ip >= sip: - if ip > eip: - l = m + 1 - else: - mixPtr = self.getLong(b, 8) - break; - else: - h = m - 1 - - if mixPtr == 0: return "N2" - - return self.returnData(mixPtr) - - def btreeSearch(self, ip): - """ - " b-tree search method - " param: ip - """ - if not ip.isdigit(): ip = self.ip2long(ip) - - if len(self.__headerSip) < 1: - #pass the super block - self.__f.seek(8) - #read the header block - b = self.__f.read(8192) - #parse the header block - sip = None - ptr = None - for i in range(0, len(b)-1, 8): - sip = self.getLong(b, i) - ptr = self.getLong(b, i+4) - if ptr == 0: - break - self.__headerSip.append(sip) - self.__headerPtr.append(ptr) - - headerLen = len(self.__headerSip) - 1 - l, h, sptr, eptr = (0, headerLen, 0, 0) - while l <= h: - m = int((l+h)/2) - - if ip == self.__headerSip[m]: - if m > 0: - sptr = self.__headerPtr[m-1] - eptr = self.__headerPtr[m] - break; - else: - sptr = self.__headerPtr[m] - eptr = self.__headerPtr[m+1] - break; - - if ip > self.__headerSip[m]: - if m == headerLen: - sptr = self.__headerPtr[m-1] - eptr = self.__headerPtr[m] - break; - elif ip < self.__headerSip[m+1]: - sptr = self.__headerPtr[m] - eptr = self.__headerPtr[m+1] - break; - - l = m + 1 - else: - if m == 0: - sptr = self.__headerPtr[m] - eptr = self.__headerPtr[m+1] - break; - elif ip > self.__headerSip[m-1]: - sptr = self.__headerPtr[m-1] - eptr = self.__headerPtr[m] - break; - - h = m - 1 - - if sptr == 0: return "N1" - - indexLen = eptr - sptr - self.__f.seek(sptr) - b = self.__f.read(indexLen + 12) - - l, h, mixPtr = (0, int(indexLen/12), 0) - while l <= h: - m = int((l+h)/2) - offset = m * 12 - - if ip >= self.getLong(b, offset): - if ip > self.getLong(b, offset+4): - l = m + 1 - else: - mixPtr = self.getLong(b, offset+8) - break; - else: - h = m - 1 - - if mixPtr == 0: return "N2" - - return self.returnData(mixPtr) - - def initDatabase(self, dbfile): - """ - " initialize the database for search - " param: dbFile - """ - try: - self.__f = io.open(dbfile, "rb") - except IOError as e: - print ("[Error]: ", e) - sys.exit() - - def returnData(self, dsptr): - """ - " get ip data from db file by data start ptr - " param: dsptr - """ - dataPtr = dsptr & 0x00FFFFFF - dataLen = (dsptr >> 24) & 0xFF - - self.__f.seek(dataPtr) - data = self.__f.read(dataLen) - - return { - "city_id": self.getLong(data, 0), - "region" : data[4:] - } - - def ip2long(self, ip): - _ip = socket.inet_aton(ip) - - return struct.unpack("!L", _ip)[0] - - def isip(self, ip): - p = ip.split(".") - - if len(p) != 4 : return False - for pp in p: - if not pp.isdigit() : return False - if len(pp) > 3 : return False - if int(pp) > 255 : return False - - return True - - def getLong(self, b, offset): - if len( b[offset:offset+4] ) == 4: - return struct.unpack('I', b[offset:offset+4])[0] - - return 0 - - def close(self): - self.__headerSip = None - self.__headerPtr = None - self.__f.close() - self.__f = None diff --git a/binding/python3/testSearcher.py b/binding/python3/testSearcher.py deleted file mode 100644 index fd36f10..0000000 --- a/binding/python3/testSearcher.py +++ /dev/null @@ -1,79 +0,0 @@ -#-*- coding:utf-8 -*- -""" -" ip2region python seacher client module -" -" Autho: koma -" Date : 2015-11-06 -""" -import struct, sys, os, time -from ip2Region import Ip2Region - -def testSearch(): - """ - " ip2region test function - """ - llen = len(sys.argv) - - if llen < 2: - print ("Usage: python testSearcher.py [ip2region db file] [alrogrithm]") - print ("Algorithm: binary or b-tree") - return 0 - - dbFile = sys.argv[1] - method = 1 - algorithm = "b-tree" - - if (not os.path.isfile(dbFile)) or (not os.path.exists(dbFile)): - print ("[Error]: Specified db file is not exists.") - return 0 - - if llen > 2: - algorithm = sys.argv[2] - if algorithm == "binary": - method = 2 - elif algorithm == "memory": - method = 3 - - print ("initializing %s..."% algorithm) - print ("+----------------------------------+") - print ("| ip2region test program |") - print ("| Author: chenxin619315@gmail.com. |") - print ("| Type 'quit' or 'exit' to exit program |") - print ("+----------------------------------+") - - searcher = Ip2Region(dbFile); - - while True: - line = input("ip2region>> ") - line = line.strip() - - if line == "": - print ("[Error]: Invalid ip address.") - continue - - if line == "quit" or line == "exit" : - print ("[Info]: Thanks for your use, Bye.") - break - - if not searcher.isip(line): - print ("[Error]: Invalid ip address.") - continue - - sTime = time.time() * 1000 - if method == 1: - data = searcher.btreeSearch(line) - elif method == 2: - data = searcher.binarySearch(line) - else: - data = searcher.memorySearch(line) - eTime = time.time() * 1000 - - if isinstance(data, dict): - print ("%s|%s in %f millseconds" % (data["city_id"], data["region"].decode('utf-8'), eTime-sTime)) - else: - print ("[Error]: ", data) - - searcher.close() - -if __name__ == "__main__": - testSearch()