xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add memory search alrogrithm and optimize script

This commit is contained in:
koma 2016-07-13 15:54:08 +08:00
parent 503e95026f
commit caad8bad88
2 changed files with 99 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
binding/python/ip2Region.py
View File

@ -10,11 +10,52 @@ import struct, io, socket, sys
class Ip2Region(object):
__headerSip = []
__headerPtr = []
__f = None
__f = None
__sPtr = 0
__indexLen = 0
__dbBinStr = ''
def __init__(self, dbfile):
self.initDatabase(dbfile)
def memorySearch(self, ip):
"""
" memory search method
" param: ip
"""
if not ip.isdigit(): ip = self.ip2long(ip)
if self.__dbBinStr == '':
self.__dbBinStr = self.__f.read() #read all the contents in file
self.__sPtr = self.getLong(self.__dbBinStr, 0)
endPtr = self.getLong(self.__dbBinStr, 4)
self.__indexLen = endPtr - self.__sPtr
startPtr = self.__sPtr
indexLen = self.__indexLen
dbBinStr = self.__dbBinStr
l, h, mixPtr = (0, int(indexLen/12), 0)
while l <= h:
m = int((l+h)/2)
ptr = startPtr + m*12
sip = self.getLong(dbBinStr, ptr)
eip = self.getLong(dbBinStr, ptr+4)
if ip >= sip:
if ip > eip:
l = m + 1
else:
mixPtr = self.getLong(dbBinStr, ptr+8)
break;
else:
h = m - 1
if mixPtr == 0: return "N2"
return self.returnData(mixPtr)
def binarySearch(self, ip):
"""
" binary search method
@ -22,12 +63,16 @@ class Ip2Region(object):
"""
if not ip.isdigit(): ip = self.ip2long(ip)
self.__f.seek(0)
b = self.__f.read(8)
startPtr = self.getLong(b, 0)
endPtr = self.getLong(b, 4)
if self.__indexLen < 1:
self.__f.seek(0)
b = self.__f.read(8)
self.__sPtr = self.getLong(b, 0)
endPtr = self.getLong(b, 4)
self.__indexLen = endPtr - self.__sPtr
startPtr = self.__sPtr
indexLen = self.__indexLen
indexLen = endPtr - startPtr
self.__f.seek(startPtr)
b = self.__f.read(indexLen+12)
@ -37,7 +82,7 @@ class Ip2Region(object):
ptr = startPtr + m*12
self.__f.seek(ptr)
b = self.__f.read(12)
b = self.__f.read(12)
sip = self.getLong(b, 0)
eip = self.getLong(b, 4)
@ -52,15 +97,7 @@ class Ip2Region(object):
if mixPtr == 0: return "N2"
dataPtr = mixPtr & 0x00FFFFFFL
dataLen = (mixPtr >> 24) & 0xFF
self.__f.seek(dataPtr)
data = self.__f.read(dataLen)
return {
"city_id": self.getLong(data, 0),
"region" : data[4:]
}
return self.returnData(mixPtr)
def btreeSearch(self, ip):
"""
@ -69,7 +106,23 @@ class Ip2Region(object):
"""
if not ip.isdigit(): ip = self.ip2long(ip)
headerLen = len(self.__headerSip) - 1
if len(self.__headerSip) < 1:
#pass the super block
self.__f.seek(8)
#read the header block
b = self.__f.read(4086)
#parse the header block
sip = None
ptr = None
for i in range(0, len(b)-1, 8):
sip = self.getLong(b, i)
ptr = self.getLong(b, i+4)
if ptr == 0:
break
self.__headerSip.append(sip)
self.__headerPtr.append(ptr)
headerLen = len(self.__headerSip) - 1
l, h, sptr, eptr = (0, headerLen, 0, 0)
while l <= h:
m = int((l+h)/2)
@ -129,15 +182,7 @@ class Ip2Region(object):
if mixPtr == 0: return "N2"
dataPtr = mixPtr & 0x00FFFFFFL
dataLen = (mixPtr >> 24) & 0xFF
self.__f.seek(dataPtr)
data = self.__f.read(dataLen)
return {
"city_id": self.getLong(data, 0),
"region" : data[4:]
}
return self.returnData(mixPtr)
def initDatabase(self, dbfile):
"""
@ -146,27 +191,29 @@ class Ip2Region(object):
"""
try:
self.__f = io.open(dbfile, "rb")
#pass the super block
self.__f.seek(8)
#read the header block
b = self.__f.read(4086)
#parse the header block
sip = None
ptr = None
for i in range(0, len(b)-1, 8):
sip = self.getLong(b, i)
ptr = self.getLong(b, i+4)
if ptr == 0:
break
self.__headerSip.append(sip)
self.__headerPtr.append(ptr)
except IOError, e:
print "[Error]: ", e
sys.exit()
def returnData(self, dsptr):
"""
" get ip data from db file by data start ptr
" param: dsptr
"""
dataPtr = dsptr & 0x00FFFFFFL
dataLen = (dsptr >> 24) & 0xFF
self.__f.seek(dataPtr)
data = self.__f.read(dataLen)
return {
"city_id": self.getLong(data, 0),
"region" : data[4:]
}
def ip2long(self, ip):
_ip = socket.inet_aton(ip)
return struct.unpack("!L", _ip)[0]
def isip(self, ip):
@ -174,15 +221,16 @@ class Ip2Region(object):
if len(p) != 4 : return False
for pp in p:
if not pp.isdigit(): return False
if not pp.isdigit() : return False
if len(pp) > 3 : return False
if int(pp) > 255 : return False
if int(pp) > 255 : return False
return True
def getLong(self, b, offset):
if len( b[offset:offset+4] ) == 4:
return struct.unpack('I', b[offset:offset+4])[0]
return 0
def close(self):

12
binding/python/testSearcher.py
View File

@ -31,11 +31,13 @@ def testSearch():
algorithm = sys.argv[2]
if algorithm == "binary":
method = 2
elif algorithm == "memory":
method = 3
print "initializing %s..." % (algorithm)
print "+----------------------------------+"
print "| ip2region test script |"
print "| Author: komazhang@foxmail.com |"
print "| ip2region test program |"
print "| Author: chenxin619315@gmail.com. |"
print "| Type 'quit' to exit program |"
print "+----------------------------------+"
@ -60,12 +62,14 @@ def testSearch():
sTime = time.time() * 1000
if method == 1:
data = searcher.btreeSearch(line)
else:
elif method == 2:
data = searcher.binarySearch(line)
else:
data = searcher.memorySearch(line)
eTime = time.time() * 1000
if isinstance(data, dict):
print "[Return]: %s|%s in %f millseconds" % (data["city_id"], data["region"], eTime-sTime)
print "%s|%s in %f millseconds" % (data["city_id"], data["region"], eTime-sTime)
else:
print "[Error]: ", data