xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

重构python客户端，提升查询性能，兼容python2.x,3.x

This commit is contained in:
Koma 2018-10-04 11:39:49 +08:00
parent 1701f49e45
commit a584ef9caf
5 changed files with 188 additions and 462 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
binding/python/beachmark.py Normal file
View File

@ -0,0 +1,49 @@
#-*- coding:utf-8 -*-
"""
" ip2region python seacher client module benchmark test file
"
" Autho: koma<komazhang@foxmail.com>
" Date : 2018-10-04
"""
import threading
import time
from ip2Region import Ip2Region
class BeachmarkThread(threading.Thread):
__searcher = None
__lock = None
def __init__(self, searcher, lock):
self.__searcher = searcher
self.__lock = lock
threading.Thread.__init__(self)
def run(self):
self.__lock.acquire()
try:
sTime = time.time() * 1000
data = self.__searcher.memorySearch("49.220.138.233")
eTime = time.time() * 1000
print("%s|%s in %5f millseconds" % (data["city_id"], data["region"].decode('utf-8'), eTime - sTime))
finally:
self.__lock.release()
if __name__ == "__main__":
threads = []
searcher = Ip2Region("/data/ip2region/data/ip2region.db")
lock = threading.Lock()
for i in range(10000):
t = BeachmarkThread(searcher, lock)
threads.append(t)
sTime = time.time() * 1000
for t in threads:
t.start()
for t in threads:
t.join()
eTime = time.time() * 1000
print("Beachmark done: %5f" % (eTime - sTime))

202
binding/python/ip2Region.py
View File

@ -8,12 +8,17 @@
import struct, io, socket, sys import struct, io, socket, sys
class Ip2Region(object): class Ip2Region(object):
__headerSip = [] __INDEX_BLOCK_LENGTH = 12
__headerPtr = [] __TOTAL_HEADER_LENGTH = 8192
__f = None
__sPtr = 0 __f = None
__indexLen = 0 __headerSip = []
__dbBinStr = '' __headerPtr = []
__headerLen = 0
__indexSPtr = 0
__indexLPtr = 0
__indexCount = 0
__dbBinStr = ''
def __init__(self, dbfile): def __init__(self, dbfile):
self.initDatabase(dbfile) self.initDatabase(dbfile)
@ -26,35 +31,30 @@ class Ip2Region(object):
if not ip.isdigit(): ip = self.ip2long(ip) if not ip.isdigit(): ip = self.ip2long(ip)
if self.__dbBinStr == '': if self.__dbBinStr == '':
self.__dbBinStr = self.__f.read() #read all the contents in file self.__dbBinStr = self.__f.read() #read all the contents in file
self.__sPtr = self.getLong(self.__dbBinStr, 0) self.__indexSPtr = self.getLong(self.__dbBinStr, 0)
endPtr = self.getLong(self.__dbBinStr, 4) self.__indexLPtr = self.getLong(self.__dbBinStr, 4)
self.__indexLen = endPtr - self.__sPtr self.__indexCount = int((self.__indexLPtr - self.__indexSPtr)/self.__INDEX_BLOCK_LENGTH)+1
startPtr = self.__sPtr l, h, dataPtr = (0, self.__indexCount, 0)
indexLen = self.__indexLen
dbBinStr = self.__dbBinStr
l, h, mixPtr = (0, int(indexLen/12), 0)
while l <= h: while l <= h:
m = int((l+h)/2) m = int((l+h) >> 1)
ptr = startPtr + m*12 p = self.__indexSPtr + m*self.__INDEX_BLOCK_LENGTH
sip = self.getLong(self.__dbBinStr, p)
sip = self.getLong(dbBinStr, ptr) if ip < sip:
eip = self.getLong(dbBinStr, ptr+4) h = m -1
if ip >= sip:
if ip > eip:
l = m + 1
else:
mixPtr = self.getLong(dbBinStr, ptr+8)
break;
else: else:
h = m - 1 eip = self.getLong(self.__dbBinStr, p+4)
if ip > eip:
l = m + 1;
else:
dataPtr = self.getLong(self.__dbBinStr, p+8)
break
if mixPtr == 0: return "N2" if dataPtr == 0: raise Exception("Data pointer not found")
return self.returnData(mixPtr) return self.returnData(dataPtr)
def binarySearch(self, ip): def binarySearch(self, ip):
""" """
@ -63,41 +63,34 @@ class Ip2Region(object):
""" """
if not ip.isdigit(): ip = self.ip2long(ip) if not ip.isdigit(): ip = self.ip2long(ip)
if self.__indexLen < 1: if self.__indexCount == 0:
self.__f.seek(0) self.__f.seek(0)
b = self.__f.read(8) superBlock = self.__f.read(8)
self.__sPtr = self.getLong(b, 0) self.__indexSPtr = self.getLong(superBlock, 0)
endPtr = self.getLong(b, 4) self.__indexLPtr = self.getLong(superBlock, 4)
self.__indexLen = endPtr - self.__sPtr self.__indexCount = int((self.__indexLPtr - self.__indexSPtr) / self.__INDEX_BLOCK_LENGTH) + 1
startPtr = self.__sPtr l, h, dataPtr = (0, self.__indexCount, 0)
indexLen = self.__indexLen
self.__f.seek(startPtr)
b = self.__f.read(indexLen+12)
l, h, mixPtr = (0, int(indexLen/12), 0)
while l <= h: while l <= h:
m = int((l+h)/2) m = int((l+h) >> 1)
ptr = startPtr + m*12 p = m*self.__INDEX_BLOCK_LENGTH
self.__f.seek(ptr)
b = self.__f.read(12) self.__f.seek(self.__indexSPtr+p)
sip = self.getLong(b, 0) buffer = self.__f.read(self.__INDEX_BLOCK_LENGTH)
eip = self.getLong(b, 4) sip = self.getLong(buffer, 0)
if ip < sip:
if ip >= sip: h = m - 1
else:
eip = self.getLong(buffer, 4)
if ip > eip: if ip > eip:
l = m + 1 l = m + 1
else: else:
mixPtr = self.getLong(b, 8) dataPtr = self.getLong(buffer, 8)
break; break
else:
h = m - 1
if mixPtr == 0: return "N2" if dataPtr == 0: raise Exception("Data pointer not found")
return self.returnData(mixPtr) return self.returnData(dataPtr)
def btreeSearch(self, ip): def btreeSearch(self, ip):
""" """
@ -107,82 +100,81 @@ class Ip2Region(object):
if not ip.isdigit(): ip = self.ip2long(ip) if not ip.isdigit(): ip = self.ip2long(ip)
if len(self.__headerSip) < 1: if len(self.__headerSip) < 1:
headerLen = 0
#pass the super block #pass the super block
self.__f.seek(8) self.__f.seek(8)
#read the header block #read the header block
b = self.__f.read(8192) b = self.__f.read(self.__TOTAL_HEADER_LENGTH)
#parse the header block #parse the header block
sip = None for i in range(0, len(b), 8):
ptr = None
for i in range(0, len(b)-1, 8):
sip = self.getLong(b, i) sip = self.getLong(b, i)
ptr = self.getLong(b, i+4) ptr = self.getLong(b, i+4)
if ptr == 0: if ptr == 0:
break break
self.__headerSip.append(sip) self.__headerSip.append(sip)
self.__headerPtr.append(ptr) self.__headerPtr.append(ptr)
headerLen += 1
self.__headerLen = headerLen
headerLen = len(self.__headerSip) - 1 l, h, sptr, eptr = (0, self.__headerLen, 0, 0)
l, h, sptr, eptr = (0, headerLen, 0, 0)
while l <= h: while l <= h:
m = int((l+h)/2) m = int((l+h) >> 1)
if ip == self.__headerSip[m]: if ip == self.__headerSip[m]:
if m > 0: if m > 0:
sptr = self.__headerPtr[m-1] sptr = self.__headerPtr[m-1]
eptr = self.__headerPtr[m] eptr = self.__headerPtr[m]
break;
else: else:
sptr = self.__headerPtr[m] sptr = self.__headerPtr[m]
eptr = self.__headerPtr[m+1] eptr = self.__headerPtr[m+1]
break; break
if ip > self.__headerSip[m]: if ip < self.__headerSip[m]:
if m == headerLen:
sptr = self.__headerPtr[m-1]
eptr = self.__headerPtr[m]
break;
elif ip < self.__headerSip[m+1]:
sptr = self.__headerPtr[m]
eptr = self.__headerPtr[m+1]
break;
l = m + 1
else:
if m == 0: if m == 0:
sptr = self.__headerPtr[m] sptr = self.__headerPtr[m]
eptr = self.__headerPtr[m+1] eptr = self.__headerPtr[m+1]
break; break
elif ip > self.__headerSip[m-1]: elif ip > self.__headerSip[m-1]:
sptr = self.__headerPtr[m-1] sptr = self.__headerPtr[m-1]
eptr = self.__headerPtr[m] eptr = self.__headerPtr[m]
break; break
h = m - 1 h = m - 1
else:
if m == self.__headerLen - 1:
sptr = self.__headerPtr[m-1]
eptr = self.__headerPtr[m]
break
elif ip <= self.__headerSip[m+1]:
sptr = self.__headerPtr[m]
eptr = self.__headerPtr[m+1]
break
l = m + 1
if sptr == 0: return "N1" if sptr == 0: raise Exception("Index pointer not found")
indexLen = eptr - sptr indexLen = eptr - sptr
self.__f.seek(sptr) self.__f.seek(sptr)
b = self.__f.read(indexLen + 12) index = self.__f.read(indexLen + self.__INDEX_BLOCK_LENGTH)
l, h, mixPtr = (0, int(indexLen/12), 0) l, h, dataPrt = (0, int(indexLen/self.__INDEX_BLOCK_LENGTH), 0)
while l <= h: while l <= h:
m = int((l+h)/2) m = int((l+h) >> 1)
offset = m * 12 offset = int(m * self.__INDEX_BLOCK_LENGTH)
sip = self.getLong(index, offset)
if ip >= self.getLong(b, offset): if ip < sip:
if ip > self.getLong(b, offset+4):
l = m + 1
else:
mixPtr = self.getLong(b, offset+8)
break;
else:
h = m - 1 h = m - 1
else:
eip = self.getLong(index, offset+4)
if ip > eip:
l = m + 1;
else:
dataPrt = self.getLong(index, offset+8)
break
if mixPtr == 0: return "N2" if dataPrt == 0: raise Exception("Data pointer not found")
return self.returnData(mixPtr) return self.returnData(dataPrt)
def initDatabase(self, dbfile): def initDatabase(self, dbfile):
""" """
@ -191,18 +183,18 @@ class Ip2Region(object):
""" """
try: try:
self.__f = io.open(dbfile, "rb") self.__f = io.open(dbfile, "rb")
except IOError, e: except IOError as e:
print "[Error]: ", e print("[Error]: %s" % e)
sys.exit() sys.exit()
def returnData(self, dsptr): def returnData(self, dataPtr):
""" """
" get ip data from db file by data start ptr " get ip data from db file by data start ptr
" param: dsptr " param: dsptr
""" """
dataPtr = dsptr & 0x00FFFFFFL dataLen = (dataPtr >> 24) & 0xFF
dataLen = (dsptr >> 24) & 0xFF dataPtr = dataPtr & 0x00FFFFFF
self.__f.seek(dataPtr) self.__f.seek(dataPtr)
data = self.__f.read(dataLen) data = self.__f.read(dataLen)
@ -213,7 +205,6 @@ class Ip2Region(object):
def ip2long(self, ip): def ip2long(self, ip):
_ip = socket.inet_aton(ip) _ip = socket.inet_aton(ip)
return struct.unpack("!L", _ip)[0] return struct.unpack("!L", _ip)[0]
def isip(self, ip): def isip(self, ip):
@ -228,13 +219,14 @@ class Ip2Region(object):
return True return True
def getLong(self, b, offset): def getLong(self, b, offset):
if len( b[offset:offset+4] ) == 4: if len(b[offset:offset+4]) == 4:
return struct.unpack('I', b[offset:offset+4])[0] return struct.unpack('I', b[offset:offset+4])[0]
return 0 return 0
def close(self): def close(self):
self.__headerSip = None if self.__f != None:
self.__f.close()
self.__dbBinStr = None
self.__headerPtr = None self.__headerPtr = None
self.__f.close() self.__headerSip = None
self.__f = None

80
binding/python/testSearcher.py
View File

@ -6,72 +6,76 @@
" Date : 2015-11-06 " Date : 2015-11-06
""" """
import struct, sys, os, time import struct, sys, os, time
from platform import python_version
from ip2Region import Ip2Region from ip2Region import Ip2Region
def testSearch(): def testSearch():
""" """
" ip2region test function " ip2region test function
""" """
llen = len(sys.argv) argLen = len(sys.argv)
version = python_version()
algorithms = ["binary", "b-tree", "memory"]
if llen < 2: if argLen < 2:
print "Usage: python testSearcher.py [ip2region db file] [alrogrithm]" print("Usage: python testSearcher.py [ip2region db file] [alrogrithm]")
print "Algorithm: binary or b-tree" print("Algorithm: %s" % ", ".join(algorithms))
return 0 return 0
dbFile = sys.argv[1] dbFile = sys.argv[1]
method = 1
algorithm = "b-tree"
if (not os.path.isfile(dbFile)) or (not os.path.exists(dbFile)): if (not os.path.isfile(dbFile)) or (not os.path.exists(dbFile)):
print "[Error]: Specified db file is not exists." print("[Error]: Specified db file is not exists.")
return 0 return 0
if llen > 2: if argLen > 2:
algorithm = sys.argv[2] algorithm = sys.argv[2]
if algorithm == "binary": try:
method = 2 algorithms.index(algorithm)
elif algorithm == "memory": except Exception as e:
method = 3 algorithm = "b-tree"
print "initializing %s..." % (algorithm) print("initializing %s..." % (algorithm))
print "+----------------------------------+" print("+----------------------------------+")
print "| ip2region test program |" print("| ip2region test program |")
print "| Author: chenxin619315@gmail.com. |" print("| Author: chenxin619315@gmail.com. |")
print "| Type 'quit' to exit program |" print("| Type 'quit' to exit program |")
print "+----------------------------------+" print("+----------------------------------+")
searcher = Ip2Region(dbFile); searcher = Ip2Region(dbFile)
while True: while True:
line = raw_input("ip2region>> ") if version[:1] == "2":
line = raw_input("ip2region>> ")
else:
line = input("ip2region>> ")
line = line.strip() line = line.strip()
if line == "": if line == "":
print "[Error]: Invalid ip address." print("[Error]: Invalid ip address.")
continue continue
if line == "quit": if line == "quit":
print "[Info]: Thanks for your use, Bye." print("[Info]: Thanks for your use, Bye.")
break break
if not searcher.isip(line): if not searcher.isip(line):
print "[Error]: Invalid ip address." print("[Error]: Invalid ip address.")
continue continue
sTime = time.time() * 1000 try:
if method == 1: sTime = time.time()*1000
data = searcher.btreeSearch(line) if algorithm == "binary":
elif method == 2: data = searcher.binarySearch(line)
data = searcher.binarySearch(line) elif algorithm == "memory":
else: data = searcher.memorySearch(line)
data = searcher.memorySearch(line) else:
eTime = time.time() * 1000 data = searcher.btreeSearch(line)
eTime = time.time()*1000
if isinstance(data, dict): print("%s|%s in %5f millseconds" % (data["city_id"], data["region"].decode('utf-8'), eTime - sTime))
print "%s|%s in %f millseconds" % (data["city_id"], data["region"], eTime-sTime) except Exception as e:
else: print("[Error]: %s" % e)
print "[Error]: ", data
searcher.close() searcher.close()

240
binding/python3/ip2Region.py
View File

@ -1,240 +0,0 @@
#-*- coding:utf-8 -*-
"""
" ip2region python seacher client module
"
" Author: koma<komazhang@foxmail.com>
" Date : 2015-11-06
"""
import struct, io, socket, sys
class Ip2Region(object):
__headerSip = []
__headerPtr = []
__f = None
__sPtr = 0
__indexLen = 0
__dbBinStr = ''
def __init__(self, dbfile):
self.initDatabase(dbfile)
def memorySearch(self, ip):
"""
" memory search method
" param: ip
"""
if not ip.isdigit(): ip = self.ip2long(ip)
if self.__dbBinStr == '':
self.__dbBinStr = self.__f.read() #read all the contents in file
self.__sPtr = self.getLong(self.__dbBinStr, 0)
endPtr = self.getLong(self.__dbBinStr, 4)
self.__indexLen = endPtr - self.__sPtr
startPtr = self.__sPtr
indexLen = self.__indexLen
dbBinStr = self.__dbBinStr
l, h, mixPtr = (0, int(indexLen/12), 0)
while l <= h:
m = int((l+h)/2)
ptr = startPtr + m*12
sip = self.getLong(dbBinStr, ptr)
eip = self.getLong(dbBinStr, ptr+4)
if ip >= sip:
if ip > eip:
l = m + 1
else:
mixPtr = self.getLong(dbBinStr, ptr+8)
break;
else:
h = m - 1
if mixPtr == 0: return "N2"
return self.returnData(mixPtr)
def binarySearch(self, ip):
"""
" binary search method
" param: ip
"""
if not ip.isdigit(): ip = self.ip2long(ip)
if self.__indexLen < 1:
self.__f.seek(0)
b = self.__f.read(8)
self.__sPtr = self.getLong(b, 0)
endPtr = self.getLong(b, 4)
self.__indexLen = endPtr - self.__sPtr
startPtr = self.__sPtr
indexLen = self.__indexLen
self.__f.seek(startPtr)
b = self.__f.read(indexLen+12)
l, h, mixPtr = (0, int(indexLen/12), 0)
while l <= h:
m = int((l+h)/2)
ptr = startPtr + m*12
self.__f.seek(ptr)
b = self.__f.read(12)
sip = self.getLong(b, 0)
eip = self.getLong(b, 4)
if ip >= sip:
if ip > eip:
l = m + 1
else:
mixPtr = self.getLong(b, 8)
break;
else:
h = m - 1
if mixPtr == 0: return "N2"
return self.returnData(mixPtr)
def btreeSearch(self, ip):
"""
" b-tree search method
" param: ip
"""
if not ip.isdigit(): ip = self.ip2long(ip)
if len(self.__headerSip) < 1:
#pass the super block
self.__f.seek(8)
#read the header block
b = self.__f.read(8192)
#parse the header block
sip = None
ptr = None
for i in range(0, len(b)-1, 8):
sip = self.getLong(b, i)
ptr = self.getLong(b, i+4)
if ptr == 0:
break
self.__headerSip.append(sip)
self.__headerPtr.append(ptr)
headerLen = len(self.__headerSip) - 1
l, h, sptr, eptr = (0, headerLen, 0, 0)
while l <= h:
m = int((l+h)/2)
if ip == self.__headerSip[m]:
if m > 0:
sptr = self.__headerPtr[m-1]
eptr = self.__headerPtr[m]
break;
else:
sptr = self.__headerPtr[m]
eptr = self.__headerPtr[m+1]
break;
if ip > self.__headerSip[m]:
if m == headerLen:
sptr = self.__headerPtr[m-1]
eptr = self.__headerPtr[m]
break;
elif ip < self.__headerSip[m+1]:
sptr = self.__headerPtr[m]
eptr = self.__headerPtr[m+1]
break;
l = m + 1
else:
if m == 0:
sptr = self.__headerPtr[m]
eptr = self.__headerPtr[m+1]
break;
elif ip > self.__headerSip[m-1]:
sptr = self.__headerPtr[m-1]
eptr = self.__headerPtr[m]
break;
h = m - 1
if sptr == 0: return "N1"
indexLen = eptr - sptr
self.__f.seek(sptr)
b = self.__f.read(indexLen + 12)
l, h, mixPtr = (0, int(indexLen/12), 0)
while l <= h:
m = int((l+h)/2)
offset = m * 12
if ip >= self.getLong(b, offset):
if ip > self.getLong(b, offset+4):
l = m + 1
else:
mixPtr = self.getLong(b, offset+8)
break;
else:
h = m - 1
if mixPtr == 0: return "N2"
return self.returnData(mixPtr)
def initDatabase(self, dbfile):
"""
" initialize the database for search
" param: dbFile
"""
try:
self.__f = io.open(dbfile, "rb")
except IOError as e:
print ("[Error]: ", e)
sys.exit()
def returnData(self, dsptr):
"""
" get ip data from db file by data start ptr
" param: dsptr
"""
dataPtr = dsptr & 0x00FFFFFF
dataLen = (dsptr >> 24) & 0xFF
self.__f.seek(dataPtr)
data = self.__f.read(dataLen)
return {
"city_id": self.getLong(data, 0),
"region" : data[4:]
}
def ip2long(self, ip):
_ip = socket.inet_aton(ip)
return struct.unpack("!L", _ip)[0]
def isip(self, ip):
p = ip.split(".")
if len(p) != 4 : return False
for pp in p:
if not pp.isdigit() : return False
if len(pp) > 3 : return False
if int(pp) > 255 : return False
return True
def getLong(self, b, offset):
if len( b[offset:offset+4] ) == 4:
return struct.unpack('I', b[offset:offset+4])[0]
return 0
def close(self):
self.__headerSip = None
self.__headerPtr = None
self.__f.close()
self.__f = None

79
binding/python3/testSearcher.py
View File

@ -1,79 +0,0 @@
#-*- coding:utf-8 -*-
"""
" ip2region python seacher client module
"
" Autho: koma<komazhang@foxmail.com>
" Date : 2015-11-06
"""
import struct, sys, os, time
from ip2Region import Ip2Region
def testSearch():
"""
" ip2region test function
"""
llen = len(sys.argv)
if llen < 2:
print ("Usage: python testSearcher.py [ip2region db file] [alrogrithm]")
print ("Algorithm: binary or b-tree")
return 0
dbFile = sys.argv[1]
method = 1
algorithm = "b-tree"
if (not os.path.isfile(dbFile)) or (not os.path.exists(dbFile)):
print ("[Error]: Specified db file is not exists.")
return 0
if llen > 2:
algorithm = sys.argv[2]
if algorithm == "binary":
method = 2
elif algorithm == "memory":
method = 3
print ("initializing %s..."% algorithm)
print ("+----------------------------------+")
print ("| ip2region test program |")
print ("| Author: chenxin619315@gmail.com. |")
print ("| Type 'quit' or 'exit' to exit program |")
print ("+----------------------------------+")
searcher = Ip2Region(dbFile);
while True:
line = input("ip2region>> ")
line = line.strip()
if line == "":
print ("[Error]: Invalid ip address.")
continue
if line == "quit" or line == "exit" :
print ("[Info]: Thanks for your use, Bye.")
break
if not searcher.isip(line):
print ("[Error]: Invalid ip address.")
continue
sTime = time.time() * 1000
if method == 1:
data = searcher.btreeSearch(line)
elif method == 2:
data = searcher.binarySearch(line)
else:
data = searcher.memorySearch(line)
eTime = time.time() * 1000
if isinstance(data, dict):
print ("%s|%s in %f millseconds" % (data["city_id"], data["region"].decode('utf-8'), eTime-sTime))
else:
print ("[Error]: ", data)
searcher.close()
if __name__ == "__main__":
testSearch()