Standardized code and add comments
This commit is contained in:
linyufeng
parent
50a852828a
commit
f300f7eaca
|
|
@ -42,7 +42,7 @@ func TestLong2IP(t *testing.T) {
|
||||||
func TestSplitSegment(t *testing.T) {
|
func TestSplitSegment(t *testing.T) {
|
||||||
// var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信"
|
// var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信"
|
||||||
// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
|
// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
|
||||||
var str = "28.201.224.0|29.34.191.255|美国|0|0|0|0"
|
var str = "116.31.76.0|117.21.79.49|region"
|
||||||
seg, err := SegmentFrom(str)
|
seg, err := SegmentFrom(str)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to parser segment '%s': %s", str, err)
|
t.Fatalf("failed to parser segment '%s': %s", str, err)
|
||||||
|
|
|
||||||
|
|
@ -10,8 +10,6 @@ ip2region xdb maker
|
||||||
main.py [command] [command options]
|
main.py [command] [command options]
|
||||||
Command:
|
Command:
|
||||||
gen generate the binary db file
|
gen generate the binary db file
|
||||||
search binary xdb search test
|
|
||||||
bench binary xdb bench test
|
|
||||||
```
|
```
|
||||||
|
|
||||||
# `xdb` 数据生成
|
# `xdb` 数据生成
|
||||||
|
|
@ -35,56 +33,6 @@ options:
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
# `xdb` 数据查询
|
# `xdb` 数据查询 和 bench 测试
|
||||||
|
|
||||||
通过 `python main.py search` 命令来测试查询输入的 ip：
|
基于xdb格式的查询功能和测试见 [ip2region binding](https://github.com/lionsoul2014/ip2region/tree/master/binding)
|
||||||
```
|
|
||||||
➜ python git:(v2.0_xdb) ✗ python main.py search
|
|
||||||
main.py search [command options]
|
|
||||||
options:
|
|
||||||
--db string ip2region binary xdb file path
|
|
||||||
```
|
|
||||||
|
|
||||||
例如，使用自带的 xdb 文件来运行查询测试：
|
|
||||||
```
|
|
||||||
➜ python git:(v2.0_xdb) ✗ python main.py search --db=./ip2region.xdb
|
|
||||||
ip2region xdb search test program, commands:
|
|
||||||
loadIndex : load the vector index for search speedup.
|
|
||||||
clearIndex: clear the vector index.
|
|
||||||
quit : exit the test program
|
|
||||||
ip2region>> 117.148.181.111
|
|
||||||
[region:中国|0|浙江省|嘉兴市|移动, took:0s]
|
|
||||||
ip2region>> 120.196.20.28
|
|
||||||
[region:中国|0|广东省|茂名市|移动, took:0s]
|
|
||||||
ip2region>> 81.33.22.150
|
|
||||||
[region:西班牙|0|马德里|马德里|西班牙电信, took:0s]
|
|
||||||
ip2region>>
|
|
||||||
```
|
|
||||||
|
|
||||||
# bench 测试
|
|
||||||
|
|
||||||
如果你自主生成了 `xdb` 文件，请确保运行如下的 `python main.py bench` 命令来确保生成的的 `xdb` 文件的正确性：
|
|
||||||
```
|
|
||||||
➜ python git:(v2.0_xdb) ✗ python main.py bench
|
|
||||||
main.py bench [command options]
|
|
||||||
options:
|
|
||||||
--db string ip2region binary xdb file path
|
|
||||||
--src string source ip text file path
|
|
||||||
--ignore-error bool keep going if bench failed
|
|
||||||
```
|
|
||||||
|
|
||||||
例如：使用 data/ip.merge.txt 源文件来 bench 测试 data/ip2region.xdb 这个 xdb 文件：
|
|
||||||
```
|
|
||||||
➜ python git:(v2.0_xdb) ✗ python main.py bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt
|
|
||||||
# 会看到一堆输出，看到类似如下的数据表示 bench 测试通过了，否则就会报错
|
|
||||||
...
|
|
||||||
try to bench segment: `{}` 224.0.0.0|255.255.255.255|0|0|0|内网IP|内网IP
|
|
||||||
|-try to bench ip '224.0.0.0' ... --[Ok]
|
|
||||||
|-try to bench ip '231.255.255.255' ... --[Ok]
|
|
||||||
|-try to bench ip '239.255.255.255' ... --[Ok]
|
|
||||||
|-try to bench ip '247.255.255.255' ... --[Ok]
|
|
||||||
|-try to bench ip '255.255.255.255' ... --[Ok]
|
|
||||||
Bench finished, [count: 3417955, failed: 0, took: 88.061s]
|
|
||||||
```
|
|
||||||
*请注意 bench 测试使用的 `src` 文件需要是对应的生成 ip2region.xdb 的源文件相同*。
|
|
||||||
如果运行过程中有错误会立马停止运行，也可以执行 --ignore-error=true 参数来忽略错误，在最后看 failed 的统计结果。
|
|
||||||
|
|
@ -1,34 +1,35 @@
|
||||||
# Created by leolin49 on 2022/7/7.
|
# Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||||
# Copyright (C) 2022 leolin49. All rights reserved.
|
# Use of this source code is governed by a Apache2.0-style
|
||||||
|
# license that can be found in the LICENSE file.
|
||||||
|
#
|
||||||
|
# Author: leolin49 <leolin49@foxmail.com>
|
||||||
|
#
|
||||||
import logging
|
import logging
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
import xdb.maker as mk
|
import xdb.maker as mk
|
||||||
import xdb.index as idx
|
import xdb.index as idx
|
||||||
import xdb.searcher as sc
|
|
||||||
import xdb.util as util
|
|
||||||
|
|
||||||
# format log
|
# Format log
|
||||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s-%(name)s-%(lineno)s-%(levelname)s - %(message)s')
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s-%(name)s-%(lineno)s-%(levelname)s - %(message)s",
|
||||||
|
)
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def print_help():
|
def print_help():
|
||||||
print("ip2region xdb maker")
|
print("ip2region xdb python maker")
|
||||||
print("{} [command] [command options]".format(sys.argv[0]))
|
print("{} [command] [command options]".format(sys.argv[0]))
|
||||||
print("Command: ")
|
print("Command: ")
|
||||||
print(" gen generate the binary db file")
|
print(" gen generate the binary db file")
|
||||||
print(" search binary xdb search test")
|
|
||||||
print(" bench binary xdb bench test")
|
|
||||||
|
|
||||||
|
|
||||||
def gen_db():
|
def gen_db():
|
||||||
src_file, dst_file = "", ""
|
src_file, dst_file = "", ""
|
||||||
index_policy = idx.VectorIndexPolicy
|
index_policy = idx.Vector_Index_Policy
|
||||||
# check input argv
|
# Check input parameters
|
||||||
for i in range(2, len(sys.argv)):
|
for i in range(2, len(sys.argv)):
|
||||||
r = sys.argv[i]
|
r = sys.argv[i]
|
||||||
if len(r) < 5:
|
if len(r) < 5:
|
||||||
|
|
@ -40,11 +41,11 @@ def gen_db():
|
||||||
print("missing = for args pair '{}'".format(r))
|
print("missing = for args pair '{}'".format(r))
|
||||||
return
|
return
|
||||||
if r[2:s_idx] == "src":
|
if r[2:s_idx] == "src":
|
||||||
src_file = r[s_idx+1:]
|
src_file = r[s_idx + 1:]
|
||||||
elif r[2:s_idx] == "dst":
|
elif r[2:s_idx] == "dst":
|
||||||
dst_file = r[s_idx+1:]
|
dst_file = r[s_idx + 1:]
|
||||||
elif r[2:s_idx] == "index":
|
elif r[2:s_idx] == "index":
|
||||||
index_policy = idx.index_policy_from_string(r[s_idx+1:])
|
index_policy = idx.index_policy_from_string(r[s_idx + 1:])
|
||||||
else:
|
else:
|
||||||
print("undefined option `{}`".format(r))
|
print("undefined option `{}`".format(r))
|
||||||
return
|
return
|
||||||
|
|
@ -56,142 +57,17 @@ def gen_db():
|
||||||
return
|
return
|
||||||
|
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
# make the binary file
|
# Make the binary file
|
||||||
maker = mk.new_maker(index_policy, src_file, dst_file)
|
maker = mk.new_maker(index_policy, src_file, dst_file)
|
||||||
maker.init()
|
maker.init()
|
||||||
maker.start()
|
maker.start()
|
||||||
maker.end()
|
maker.end()
|
||||||
|
|
||||||
logging.info("Done, elapsed: {:.0f}m{:.0f}s".format((time.time() - start_time) / 60, (time.time() - start_time) % 60))
|
logging.info(
|
||||||
|
"Done, elapsed: {:.0f}m{:.0f}s".format(
|
||||||
|
(time.time() - start_time) / 60, (time.time() - start_time) % 60
|
||||||
def test_search():
|
|
||||||
db_file = ""
|
|
||||||
for i in range(2, len(sys.argv)):
|
|
||||||
r = sys.argv[i]
|
|
||||||
if len(r) < 5:
|
|
||||||
continue
|
|
||||||
if not r.startswith("--"):
|
|
||||||
continue
|
|
||||||
e_idx = r.index("=")
|
|
||||||
if e_idx < 0:
|
|
||||||
print("missing = for args pair '{}'".format(r))
|
|
||||||
return
|
|
||||||
if r[2:e_idx] == "db":
|
|
||||||
db_file = r[e_idx+1:]
|
|
||||||
else:
|
|
||||||
print("undefined option `{}`".format(r))
|
|
||||||
return
|
|
||||||
if db_file == "":
|
|
||||||
print("{} search [command options]".format(sys.argv[0]))
|
|
||||||
print("options:")
|
|
||||||
print(" --db string ip2region binary xdb file path")
|
|
||||||
return
|
|
||||||
cb = sc.XdbSearcher.loadContentFromFile(dbfile=db_file)
|
|
||||||
searcher = sc.XdbSearcher(contentBuff=cb)
|
|
||||||
print("ip2region xdb search test program, commands:\nloadIndex : load the vector index for search "
|
|
||||||
"speedup.\nclearIndex: clear the vector index.\nquit : exit the test program")
|
|
||||||
while True:
|
|
||||||
print("ip2region>> ", end="")
|
|
||||||
line = input()
|
|
||||||
|
|
||||||
# command interception and execution
|
|
||||||
if line == "loadIndex":
|
|
||||||
searcher.loadVectorIndexFromFile(dbfile=db_file)
|
|
||||||
print("vector index cached")
|
|
||||||
continue
|
|
||||||
elif line == "clearIndex":
|
|
||||||
# FIXME need to add 'clearVectorIndex' method in searcher
|
|
||||||
print("vector index cleared")
|
|
||||||
continue
|
|
||||||
elif line == "quit":
|
|
||||||
break
|
|
||||||
|
|
||||||
ip = util.checkip(line)
|
|
||||||
if ip == -1:
|
|
||||||
print("invalid ip address `{}`".format(line))
|
|
||||||
continue
|
|
||||||
|
|
||||||
s_tm = datetime.now()
|
|
||||||
region = searcher.search(ip)
|
|
||||||
# TODO calculate io count in `searcher.search` method
|
|
||||||
print("\x1b[0;32m[region:{}, took:{:.0f}s]\x1b[0m".format(
|
|
||||||
region, (datetime.now().microsecond - s_tm.microsecond) / 1000)
|
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def test_bench():
|
|
||||||
db_file, src_file = "", ""
|
|
||||||
ignore_error = False
|
|
||||||
for i in range(2, len(sys.argv)):
|
|
||||||
r = sys.argv[i]
|
|
||||||
if len(r) < 5:
|
|
||||||
continue
|
|
||||||
if not r.startswith("--"):
|
|
||||||
continue
|
|
||||||
s_idx = r.index("=")
|
|
||||||
if s_idx < 0:
|
|
||||||
print("missing = for args pair '{}'".format(r))
|
|
||||||
return
|
|
||||||
if r[2:s_idx] == "db":
|
|
||||||
db_file = r[s_idx + 1:]
|
|
||||||
elif r[2:s_idx] == "src":
|
|
||||||
src_file = r[s_idx + 1:]
|
|
||||||
elif r[2:s_idx] == "ignore-error":
|
|
||||||
v = r[s_idx + 1:]
|
|
||||||
if v == "true" or v == "1":
|
|
||||||
ignore_error = True
|
|
||||||
elif v == "false" or v == "0":
|
|
||||||
ignore_error = False
|
|
||||||
else:
|
|
||||||
print("invalid value for ignore-error option, could be false/0 or true/1")
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
print("undefined option `{}`".format(r))
|
|
||||||
return
|
|
||||||
|
|
||||||
if db_file == "" or src_file == "":
|
|
||||||
print("{} bench [command options]".format(sys.argv[0]))
|
|
||||||
print("options:")
|
|
||||||
print(" --db string ip2region binary xdb file path")
|
|
||||||
print(" --src string source ip text file path")
|
|
||||||
print(" --ignore-error bool keep going if bench failed")
|
|
||||||
return
|
|
||||||
|
|
||||||
cb = sc.XdbSearcher.loadContentFromFile(dbfile=db_file)
|
|
||||||
searcher = sc.XdbSearcher(contentBuff=cb)
|
|
||||||
cnt, err_cnt, s_tm = 0, 0, time.time()
|
|
||||||
with open(src_file, 'r', encoding="utf-8") as f:
|
|
||||||
lines = f.read().splitlines()
|
|
||||||
for line in lines:
|
|
||||||
ps = line.split("|", maxsplit=2)
|
|
||||||
if len(ps) != 3:
|
|
||||||
print("invalid ip segment line `{}`".format(line))
|
|
||||||
return
|
|
||||||
sip = util.checkip(ps[0])
|
|
||||||
if sip == -1:
|
|
||||||
print("invalid ip address `{}`".format(line))
|
|
||||||
return
|
|
||||||
eip = util.checkip(ps[1])
|
|
||||||
if eip == -1:
|
|
||||||
print("invalid ip address `{}`".format(line))
|
|
||||||
return
|
|
||||||
print("try to bench segment: `{}`", line)
|
|
||||||
mip = util.mid_ip(sip, eip)
|
|
||||||
for ip in [sip, util.mid_ip(sip, mip), mip, util.mid_ip(mip, eip), eip]:
|
|
||||||
print("|-try to bench ip '{}' ...".format(util.long2ip(ip)), end="")
|
|
||||||
region = searcher.search(ip)
|
|
||||||
|
|
||||||
# check the region info
|
|
||||||
cnt += 1
|
|
||||||
if region != ps[2]:
|
|
||||||
err_cnt += 1
|
|
||||||
print(" --[Failed] ({} != {})".format(region, ps[2]))
|
|
||||||
if not ignore_error:
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
print(" --[Ok]")
|
|
||||||
print("Bench finished, [count: {}, failed: {}, took: {:.3f}s]".format(cnt, err_cnt, time.time() - s_tm))
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
@ -202,13 +78,9 @@ def main():
|
||||||
cmd = sys.argv[1].lower()
|
cmd = sys.argv[1].lower()
|
||||||
if cmd == "gen":
|
if cmd == "gen":
|
||||||
gen_db()
|
gen_db()
|
||||||
elif cmd == "search":
|
|
||||||
test_search()
|
|
||||||
elif cmd == "bench":
|
|
||||||
test_bench()
|
|
||||||
else:
|
else:
|
||||||
print_help()
|
print_help()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|
|
||||||
|
|
@ -1,2 +1,6 @@
|
||||||
# Created by leolin49 on 2022/7/7.
|
# Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||||
# Copyright (C) 2022 leolin49. All rights reserved.
|
# Use of this source code is governed by a Apache2.0-style
|
||||||
|
# license that can be found in the LICENSE file.
|
||||||
|
#
|
||||||
|
# Author: leolin49 <leolin49@foxmail.com>
|
||||||
|
#
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,24 @@
|
||||||
# Created by leolin49 on 2022/7/7.
|
# Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||||
# Copyright (C) 2022 leolin49. All rights reserved.
|
# Use of this source code is governed by a Apache2.0-style
|
||||||
|
# license that can be found in the LICENSE file.
|
||||||
|
#
|
||||||
|
# Author: leolin49 <leolin49@foxmail.com>
|
||||||
|
#
|
||||||
import struct
|
import struct
|
||||||
|
|
||||||
VectorIndexPolicy = 1
|
Vector_Index_Policy = 1
|
||||||
BTreeIndexPolicy = 2
|
BTree_Index_Policy = 2
|
||||||
SegmentIndexBlockSize = 14
|
|
||||||
|
|
||||||
|
|
||||||
def index_policy_from_string(s: str) -> int:
|
def index_policy_from_string(s: str) -> int:
|
||||||
sl = s.lower()
|
sl = s.lower()
|
||||||
if sl == "vector":
|
if sl == "vector":
|
||||||
return VectorIndexPolicy
|
return Vector_Index_Policy
|
||||||
elif sl == "btree":
|
elif sl == "btree":
|
||||||
return BTreeIndexPolicy
|
return BTree_Index_Policy
|
||||||
else:
|
else:
|
||||||
print("invalid policy `{}`, used default vector index".format(s))
|
print("invalid policy `{}`, used default vector index".format(s))
|
||||||
return VectorIndexPolicy
|
return Vector_Index_Policy
|
||||||
|
|
||||||
|
|
||||||
class VectorIndexBlock:
|
class VectorIndexBlock:
|
||||||
|
|
@ -26,11 +29,14 @@ class VectorIndexBlock:
|
||||||
self.first_ptr = fp
|
self.first_ptr = fp
|
||||||
self.last_ptr = lp
|
self.last_ptr = lp
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return "FirstPtr: {}, LastPrt: {}".format(self.first_ptr, self.last_ptr)
|
||||||
|
|
||||||
def encode(self) -> bytes:
|
def encode(self) -> bytes:
|
||||||
return struct.pack("<II", self.first_ptr, self.last_ptr)
|
return struct.pack("<II", self.first_ptr, self.last_ptr)
|
||||||
|
|
||||||
def string(self) -> str:
|
|
||||||
return "FirstPtr: {}, LastPrt: {}".format(self.first_ptr, self.last_ptr)
|
Segment_Index_Block_Size = 14
|
||||||
|
|
||||||
|
|
||||||
class SegmentIndexBlock:
|
class SegmentIndexBlock:
|
||||||
|
|
@ -45,8 +51,12 @@ class SegmentIndexBlock:
|
||||||
self.data_len = dl
|
self.data_len = dl
|
||||||
self.data_ptr = dp
|
self.data_ptr = dp
|
||||||
|
|
||||||
def encode(self) -> bytes:
|
def __str__(self):
|
||||||
return struct.pack("<IIHI", self.start_ip, self.end_ip, self.data_len, self.data_ptr)
|
return "{sip: {}, eip: {}, len: {}, ptr: {}}".format(
|
||||||
|
self.start_ip, self.end_ip, self.data_len, self.data_ptr
|
||||||
|
)
|
||||||
|
|
||||||
def string(self) -> str:
|
def encode(self) -> bytes:
|
||||||
return "{sip: {}, eip: {}, len: {}, ptr: {}}".format(self.start_ip, self.end_ip, self.data_len, self.data_ptr)
|
return struct.pack(
|
||||||
|
"<IIHI", self.start_ip, self.end_ip, self.data_len, self.data_ptr
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
# Created by leolin49 on 2022/7/7.
|
# Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||||
# Copyright (C) 2022 leolin49. All rights reserved.
|
# Use of this source code is governed by a Apache2.0-style
|
||||||
|
# license that can be found in the LICENSE file.
|
||||||
|
#
|
||||||
|
# Author: leolin49 <leolin49@foxmail.com>
|
||||||
#
|
#
|
||||||
# ----
|
# ----
|
||||||
# ip2region database v2.0 structure
|
# ip2region database v2.0 structure
|
||||||
|
|
@ -36,40 +39,39 @@
|
||||||
#
|
#
|
||||||
# data entry structure:
|
# data entry structure:
|
||||||
# +--------------------+-----------------------+
|
# +--------------------+-----------------------+
|
||||||
# | 2bytes (for desc) | dynamic length |
|
# | 2bytes (for desc) | dynamic length |
|
||||||
# +--------------------+-----------------------+
|
# +--------------------+-----------------------+
|
||||||
# data length whatever in bytes
|
# data length whatever in bytes
|
||||||
#
|
#
|
||||||
# index entry structure
|
# index entry structure
|
||||||
# +------------+-----------+---------------+------------+
|
# +------------+-----------+---------------+------------+
|
||||||
# | 4bytes | 4bytes | 2bytes | 4 bytes |
|
# | 4bytes | 4bytes | 2bytes | 4 bytes |
|
||||||
# +------------+-----------+---------------+------------+
|
# +------------+-----------+---------------+------------+
|
||||||
# start ip end ip data length data ptr
|
# start ip end ip data length data ptr
|
||||||
import os
|
|
||||||
import struct
|
|
||||||
import sys
|
|
||||||
sys.path.append(os.path.realpath(os.path.dirname(os.path.realpath(__file__))))
|
|
||||||
import logging
|
import logging
|
||||||
|
import struct
|
||||||
import time
|
import time
|
||||||
import segment as seg
|
import sys
|
||||||
import index as idx
|
|
||||||
import util
|
import xdb.segment as seg
|
||||||
|
import xdb.index as idx
|
||||||
|
import xdb.util as util
|
||||||
|
|
||||||
|
|
||||||
VersionNo = 2
|
Version_No = 2
|
||||||
HeaderInfoLength = 256
|
Header_Info_Length = 256
|
||||||
VectorIndexRows = 256
|
Vector_Index_Rows = 256
|
||||||
VectorIndexCols = 256
|
Vector_Index_Cols = 256
|
||||||
VectorIndexSize = 8
|
Vector_Index_Size = 8
|
||||||
VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize
|
Vector_Index_Length = Vector_Index_Rows * Vector_Index_Cols * Vector_Index_Size
|
||||||
|
|
||||||
|
|
||||||
class Maker:
|
class Maker:
|
||||||
src_handle = None
|
src_handle = None
|
||||||
dst_handle = None
|
dst_handle = None
|
||||||
index_policy = 0
|
index_policy = idx.Vector_Index_Policy
|
||||||
segments = []
|
segments = None
|
||||||
region_pool = {}
|
region_pool = None
|
||||||
vector_index = None
|
vector_index = None
|
||||||
|
|
||||||
def __init__(self, sh, dh, ip, sg, rp, vi):
|
def __init__(self, sh, dh, ip, sg, rp, vi):
|
||||||
|
|
@ -83,30 +85,32 @@ class Maker:
|
||||||
def init(self):
|
def init(self):
|
||||||
"""
|
"""
|
||||||
Init the `xdb` binary file.
|
Init the `xdb` binary file.
|
||||||
1. init the file header
|
1. Init the file header
|
||||||
2. load all the segments
|
2. Load all the segments
|
||||||
"""
|
"""
|
||||||
self.init_db_header()
|
self.init_db_header()
|
||||||
self.load_segments()
|
self.load_segments()
|
||||||
|
|
||||||
def init_db_header(self):
|
def init_db_header(self):
|
||||||
"""Init and write the file header to the destination xdb file."""
|
"""
|
||||||
|
Init and write the file header to the destination xdb file.
|
||||||
|
"""
|
||||||
logging.info("try to init the db header ... ")
|
logging.info("try to init the db header ... ")
|
||||||
self.src_handle.seek(0, 0)
|
self.src_handle.seek(0, 0)
|
||||||
|
|
||||||
header = bytearray([0]*256)
|
# Make and write the header space
|
||||||
# make and write the header space
|
header = bytearray([0] * 256)
|
||||||
# 1. version number
|
# 1. Version number
|
||||||
header[0:2] = VersionNo.to_bytes(2, byteorder="little")
|
header[0:2] = Version_No.to_bytes(2, byteorder="little")
|
||||||
# 2. index policy code
|
# 2. Index policy code
|
||||||
header[2:4] = int(self.index_policy).to_bytes(2, byteorder="little")
|
header[2:4] = int(self.index_policy).to_bytes(2, byteorder="little")
|
||||||
# 3. generate unix timestamp
|
# 3. Generate unix timestamp
|
||||||
header[4:8] = int(time.time()).to_bytes(4, byteorder="little")
|
header[4:8] = int(time.time()).to_bytes(4, byteorder="little")
|
||||||
# 4. index block start ptr
|
# 4. Index block start ptr
|
||||||
header[8:12] = int(0).to_bytes(4, byteorder="little")
|
header[8:12] = int(0).to_bytes(4, byteorder="little")
|
||||||
# 5. index block end ptr
|
# 5. Index block end ptr
|
||||||
header[12:16] = int(0).to_bytes(4, byteorder="little")
|
header[12:16] = int(0).to_bytes(4, byteorder="little")
|
||||||
# write header buffer to file
|
# Write header buffer to file
|
||||||
self.dst_handle.write(header)
|
self.dst_handle.write(header)
|
||||||
|
|
||||||
def load_segments(self) -> list:
|
def load_segments(self) -> list:
|
||||||
|
|
@ -125,122 +129,158 @@ class Maker:
|
||||||
if len(ps) != 3:
|
if len(ps) != 3:
|
||||||
logging.error("invalid ip segment line `{}`".format(line))
|
logging.error("invalid ip segment line `{}`".format(line))
|
||||||
return []
|
return []
|
||||||
sip = util.checkip(ps[0])
|
sip = util.check_ip(ps[0])
|
||||||
if sip == -1:
|
if sip == -1:
|
||||||
logging.error("invalid ip address `{}`".format(line))
|
logging.error(
|
||||||
|
"invalid ip address `{}` in line `{}`".format(ps[0], line)
|
||||||
|
)
|
||||||
return []
|
return []
|
||||||
eip = util.checkip(ps[1])
|
eip = util.check_ip(ps[1])
|
||||||
if eip == -1:
|
if eip == -1:
|
||||||
logging.error("invalid ip address `{}`".format(line))
|
logging.error(
|
||||||
|
"invalid ip address `{}` in line `{}`".format(ps[1], line)
|
||||||
|
)
|
||||||
return []
|
return []
|
||||||
if sip > eip:
|
if sip > eip:
|
||||||
logging.error("start ip({}) should not be greater than end ip({})".format(ps[0], ps[1]))
|
logging.error(
|
||||||
|
"start ip({}) should not be greater than end ip({})".format(
|
||||||
|
ps[0], ps[1]
|
||||||
|
)
|
||||||
|
)
|
||||||
return []
|
return []
|
||||||
if len(ps[2]) < 1:
|
if len(ps[2]) < 1:
|
||||||
logging.error("empty region info in segment line `{}`".format(line))
|
logging.error("empty region info in segment line `{}`".format(line))
|
||||||
return []
|
return []
|
||||||
segment = seg.Segment(sip=sip, eip=eip, reg=ps[2])
|
|
||||||
|
|
||||||
# check the continuity of data segment
|
segment = seg.Segment(sip=sip, eip=eip, reg=ps[2])
|
||||||
|
# Check the continuity of data segment
|
||||||
if last is not None:
|
if last is not None:
|
||||||
if last.end_ip + 1 != segment.start_ip:
|
if last.end_ip + 1 != segment.start_ip:
|
||||||
logging.error("discontinuous data segment: last.eip+1({})!=seg.sip({}, {})".format(sip, eip, ps[0]))
|
logging.error(
|
||||||
|
"discontinuous data segment: last.eip+1({})!=seg.sip({}, {})".format(
|
||||||
|
sip, eip, ps[0]
|
||||||
|
)
|
||||||
|
)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
self.segments.append(segment)
|
self.segments.append(segment)
|
||||||
last = segment
|
last = segment
|
||||||
logging.info("all segments loaded, length: {}, elapsed: {}".format(len(self.segments), time.time() - s_tm))
|
logging.info(
|
||||||
|
"all segments loaded, length: {}, elapsed: {}".format(
|
||||||
|
len(self.segments), time.time() - s_tm
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def set_vector_index(self, ip, ptr):
|
def set_vector_index(self, ip, ptr):
|
||||||
|
"""
|
||||||
|
Init and refresh the vector index based on the IP pre-two bytes.
|
||||||
|
"""
|
||||||
row, col = (ip >> 24) & 0xFF, (ip >> 16) & 0xFF
|
row, col = (ip >> 24) & 0xFF, (ip >> 16) & 0xFF
|
||||||
vi_block = self.vector_index[row][col]
|
vi_block = self.vector_index[row][col]
|
||||||
if vi_block.first_ptr == 0:
|
if vi_block.first_ptr == 0:
|
||||||
vi_block.first_ptr = ptr
|
vi_block.first_ptr = ptr
|
||||||
vi_block.last_ptr = ptr + idx.SegmentIndexBlockSize
|
vi_block.last_ptr = ptr + idx.Segment_Index_Block_Size
|
||||||
else:
|
else:
|
||||||
vi_block.last_ptr = ptr + idx.SegmentIndexBlockSize
|
vi_block.last_ptr = ptr + idx.Segment_Index_Block_Size
|
||||||
self.vector_index[row][col] = vi_block
|
self.vector_index[row][col] = vi_block
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
"""Start to make the 'xdb' binary file."""
|
"""
|
||||||
|
Start to make the 'xdb' binary file.
|
||||||
|
"""
|
||||||
if len(self.segments) < 1:
|
if len(self.segments) < 1:
|
||||||
logging.error("empty segment list")
|
logging.error("empty segment list")
|
||||||
return
|
return
|
||||||
|
|
||||||
# 1. write all the region/data to the binary file
|
# 1. Write all the region/data to the binary file
|
||||||
self.dst_handle.seek(HeaderInfoLength+VectorIndexLength, 0)
|
self.dst_handle.seek(Header_Info_Length + Vector_Index_Length, 0)
|
||||||
|
|
||||||
logging.info("try to write the data block ... ")
|
logging.info("try to write the data block ... ")
|
||||||
for s in self.segments:
|
for s in self.segments:
|
||||||
logging.info("try to write region '{}'...".format(s.region))
|
logging.info("try to write region '{}'...".format(s.region))
|
||||||
if s.region in self.region_pool:
|
if s.region in self.region_pool:
|
||||||
logging.info(" --[Cached] with ptr={}".format(self.region_pool[s.region]))
|
logging.info(
|
||||||
|
" --[Cached] with ptr={}".format(self.region_pool[s.region])
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
region = bytes(s.region, encoding="utf-8")
|
region = bytes(s.region, encoding="utf-8")
|
||||||
if len(region) > 0xFFFF:
|
if len(region) > 0xFFFF:
|
||||||
logging.error("too long region info `{}`: should be less than {} bytes".format(s.region, 0xFFFF))
|
logging.error(
|
||||||
|
"too long region info `{}`: should be less than {} bytes".format(
|
||||||
|
s.region, 0xFFFF
|
||||||
|
)
|
||||||
|
)
|
||||||
return
|
return
|
||||||
|
# Get the first ptr of the next region
|
||||||
# get the first ptr of the next region
|
|
||||||
pos = self.dst_handle.seek(0, 1)
|
pos = self.dst_handle.seek(0, 1)
|
||||||
logging.info("{} {} {}".format(pos, region, s.region))
|
logging.info("{} {} {}".format(pos, region, s.region))
|
||||||
self.dst_handle.write(region)
|
self.dst_handle.write(region)
|
||||||
self.region_pool[s.region] = pos
|
self.region_pool[s.region] = pos
|
||||||
logging.info(" --[Added] with ptr={}".format(pos))
|
logging.info(" --[Added] with ptr={}".format(pos))
|
||||||
# 2. write the index block and cache the super index block
|
# 2. Write the index block and cache the super index block
|
||||||
logging.info("try to write the segment index block ... ")
|
logging.info("try to write the segment index block ... ")
|
||||||
counter, start_index_ptr, end_index_ptr = 0, -1, -1
|
counter, start_index_ptr, end_index_ptr = 0, -1, -1
|
||||||
for sg in self.segments:
|
for sg in self.segments:
|
||||||
data_ptr = -1
|
if sg.region not in self.region_pool:
|
||||||
if sg.region in self.region_pool:
|
|
||||||
data_ptr = self.region_pool[sg.region]
|
|
||||||
else:
|
|
||||||
logging.error("missing ptr cache for region `{}`".format(sg.region))
|
logging.error("missing ptr cache for region `{}`".format(sg.region))
|
||||||
return
|
return
|
||||||
|
|
||||||
data_len = len(bytes(sg.region, encoding="utf-8"))
|
data_len = len(bytes(sg.region, encoding="utf-8"))
|
||||||
if data_len < 1:
|
if data_len < 1:
|
||||||
logging.error("empty region info for segment '{}'".format(sg.region))
|
logging.error("empty region info for segment '{}'".format(sg.region))
|
||||||
return
|
return
|
||||||
|
|
||||||
seg_list = sg.split()
|
seg_list = sg.split()
|
||||||
logging.info("try to index segment({} split) {} ...".format(len(seg_list), sg.string()))
|
logging.info(
|
||||||
|
"try to index segment({} split) {} ...".format(len(seg_list), sg)
|
||||||
|
)
|
||||||
for s in seg_list:
|
for s in seg_list:
|
||||||
pos = self.dst_handle.seek(0, 1)
|
pos = self.dst_handle.seek(0, 1)
|
||||||
|
|
||||||
s_index = idx.SegmentIndexBlock(
|
s_index = idx.SegmentIndexBlock(
|
||||||
sip=s.start_ip, eip=s.end_ip, dl=data_len, dp=data_ptr
|
sip=s.start_ip,
|
||||||
|
eip=s.end_ip,
|
||||||
|
dl=data_len,
|
||||||
|
dp=self.region_pool[sg.region],
|
||||||
)
|
)
|
||||||
self.dst_handle.write(s_index.encode())
|
self.dst_handle.write(s_index.encode())
|
||||||
logging.info("|-segment index: {}, ptr: {}, segment: {}".format(counter, pos, s.string()))
|
logging.info(
|
||||||
|
"|-segment index: {}, ptr: {}, segment: {}".format(counter, pos, s)
|
||||||
|
)
|
||||||
self.set_vector_index(s.start_ip, pos)
|
self.set_vector_index(s.start_ip, pos)
|
||||||
counter += 1
|
counter += 1
|
||||||
|
|
||||||
# check and record the start index ptr
|
# Check and record the start index ptr
|
||||||
if start_index_ptr == -1:
|
if start_index_ptr == -1:
|
||||||
start_index_ptr = pos
|
start_index_ptr = pos
|
||||||
end_index_ptr = pos
|
end_index_ptr = pos
|
||||||
|
|
||||||
# synchronized the vector index block
|
# 3. Synchronized the vector index block
|
||||||
logging.info("try to write the vector index block ... ")
|
logging.info("try to write the vector index block ... ")
|
||||||
self.dst_handle.seek(HeaderInfoLength, 0)
|
self.dst_handle.seek(Header_Info_Length, 0)
|
||||||
for i in range(0, len(self.vector_index)):
|
for i in range(0, len(self.vector_index)):
|
||||||
for j in range(0, len(self.vector_index[i])):
|
for j in range(0, len(self.vector_index[i])):
|
||||||
vi = self.vector_index[i][j]
|
vi = self.vector_index[i][j]
|
||||||
self.dst_handle.write(vi.encode())
|
self.dst_handle.write(vi.encode())
|
||||||
|
|
||||||
# synchronized the segment index info
|
# 4. Synchronized the segment index info
|
||||||
logging.info("try to write the segment index ptr ... ")
|
logging.info("try to write the segment index ptr ... ")
|
||||||
buff = struct.pack("<II", start_index_ptr, end_index_ptr)
|
buff = struct.pack("<II", start_index_ptr, end_index_ptr)
|
||||||
self.dst_handle.seek(8, 0)
|
self.dst_handle.seek(8, 0)
|
||||||
self.dst_handle.write(buff)
|
self.dst_handle.write(buff)
|
||||||
|
|
||||||
logging.info("write done, dataBlocks: {}, indexBlocks: ({}, {}), indexPtr: ({}, {})".format(
|
logging.info(
|
||||||
len(self.region_pool), len(self.segments), counter, start_index_ptr, end_index_ptr
|
"write done, dataBlocks: {}, indexBlocks: ({}, {}), indexPtr: ({}, {})".format(
|
||||||
))
|
len(self.region_pool),
|
||||||
|
len(self.segments),
|
||||||
|
counter,
|
||||||
|
start_index_ptr,
|
||||||
|
end_index_ptr,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def end(self):
|
def end(self):
|
||||||
"""End of make the 'xdb' binary file."""
|
"""
|
||||||
|
End of make the 'xdb' binary file.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
self.src_handle.close()
|
self.src_handle.close()
|
||||||
self.dst_handle.close()
|
self.dst_handle.close()
|
||||||
|
|
@ -250,20 +290,27 @@ class Maker:
|
||||||
|
|
||||||
|
|
||||||
def new_maker(policy: int, srcfile: str, dstfile: str) -> Maker:
|
def new_maker(policy: int, srcfile: str, dstfile: str) -> Maker:
|
||||||
"""Create a xdb Maker to make the xdb binary file
|
"""
|
||||||
|
Create a xdb Maker to make the xdb binary file
|
||||||
:param policy: index algorithm code 1:vector, 2:b-tree
|
:param policy: index algorithm code 1:vector, 2:b-tree
|
||||||
:param srcfile: source ip text file path
|
:param srcfile: source ip text file path
|
||||||
:param dstfile: destination binary xdb file path
|
:param dstfile: destination binary xdb file path
|
||||||
:return: the 'xdb' Maker
|
:return: the 'xdb' Maker
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
sh = open(srcfile, mode='r', encoding='utf-8')
|
sh = open(srcfile, mode="r", encoding="utf-8")
|
||||||
dh = open(dstfile, mode='wb')
|
dh = open(dstfile, mode="wb")
|
||||||
return Maker(
|
return Maker(
|
||||||
sh=sh, dh=dh, ip=policy, sg=[], rp={},
|
sh=sh,
|
||||||
vi=[[idx.VectorIndexBlock() for _ in range(VectorIndexRows)] for _ in range(VectorIndexCols)],
|
dh=dh,
|
||||||
|
ip=policy,
|
||||||
|
sg=[],
|
||||||
|
rp={},
|
||||||
|
vi=[
|
||||||
|
[idx.VectorIndexBlock() for _ in range(Vector_Index_Rows)]
|
||||||
|
for _ in range(Vector_Index_Cols)
|
||||||
|
],
|
||||||
)
|
)
|
||||||
except IOError as e:
|
except IOError as e:
|
||||||
logging.error(e)
|
logging.error(e)
|
||||||
sys.exit()
|
sys.exit()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,161 +0,0 @@
|
||||||
import socket
|
|
||||||
import struct
|
|
||||||
import io
|
|
||||||
import sys
|
|
||||||
|
|
||||||
HeaderInfoLength = 256
|
|
||||||
VectorIndexRows = 256
|
|
||||||
VectorIndexCols = 256
|
|
||||||
VectorIndexSize = 8
|
|
||||||
SegmentIndexSize = 14
|
|
||||||
|
|
||||||
|
|
||||||
class XdbSearcher(object):
|
|
||||||
__f = None
|
|
||||||
|
|
||||||
# the minimal memory allocation.
|
|
||||||
vectorIndex = None
|
|
||||||
# 整个读取xdb，保存在内存中
|
|
||||||
contentBuff = None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def loadVectorIndexFromFile(dbfile):
|
|
||||||
try:
|
|
||||||
f = io.open(dbfile, "rb")
|
|
||||||
f.seek(HeaderInfoLength)
|
|
||||||
vi_len = VectorIndexRows * VectorIndexCols * SegmentIndexSize
|
|
||||||
vector_data = f.read(vi_len)
|
|
||||||
f.close()
|
|
||||||
return vector_data
|
|
||||||
except IOError as e:
|
|
||||||
print("[Error]: %s" % e)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def loadContentFromFile(dbfile):
|
|
||||||
try:
|
|
||||||
f = io.open(dbfile, "rb")
|
|
||||||
all_data = f.read()
|
|
||||||
f.close()
|
|
||||||
return all_data
|
|
||||||
except IOError as e:
|
|
||||||
print("[Error]: %s" % e)
|
|
||||||
|
|
||||||
def __init__(self, dbfile=None, vectorIndex=None, contentBuff=None):
|
|
||||||
self.initDatabase(dbfile, vectorIndex, contentBuff)
|
|
||||||
|
|
||||||
def search(self, ip):
|
|
||||||
if isinstance(ip, str):
|
|
||||||
if not ip.isdigit(): ip = self.ip2long(ip)
|
|
||||||
return self.searchByIPLong(ip)
|
|
||||||
else:
|
|
||||||
return self.searchByIPLong(ip)
|
|
||||||
|
|
||||||
def searchByIPStr(self, ip):
|
|
||||||
if not ip.isdigit(): ip = self.ip2long(ip)
|
|
||||||
return self.searchByIPLong(ip)
|
|
||||||
|
|
||||||
def searchByIPLong(self, ip):
|
|
||||||
# locate the segment index block based on the vector index
|
|
||||||
sPtr = ePtr = 0
|
|
||||||
il0 = (int)((ip >> 24) & 0xFF)
|
|
||||||
il1 = (int)((ip >> 16) & 0xFF)
|
|
||||||
idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize
|
|
||||||
|
|
||||||
if self.vectorIndex is not None:
|
|
||||||
sPtr = self.getLong(self.vectorIndex, idx)
|
|
||||||
ePtr = self.getLong(self.vectorIndex, idx + 4)
|
|
||||||
elif self.contentBuff is not None:
|
|
||||||
sPtr = self.getLong(self.contentBuff, HeaderInfoLength + idx)
|
|
||||||
ePtr = self.getLong(self.contentBuff, HeaderInfoLength + idx + 4)
|
|
||||||
else:
|
|
||||||
self.__f.seek(HeaderInfoLength + idx)
|
|
||||||
buffer_ptr = self.__f.read(8)
|
|
||||||
sPtr = self.getLong(buffer_ptr, 0)
|
|
||||||
ePtr = self.getLong(buffer_ptr, 4)
|
|
||||||
|
|
||||||
# binary search the segment index block to get the region info
|
|
||||||
dataLen = dataPtr = int(-1)
|
|
||||||
l = int(0)
|
|
||||||
h = int((ePtr - sPtr) / SegmentIndexSize)
|
|
||||||
while l <= h:
|
|
||||||
m = int((l + h) >> 1)
|
|
||||||
p = int(sPtr + m * SegmentIndexSize)
|
|
||||||
# read the segment index
|
|
||||||
buffer_sip = self.readBuffer(p, SegmentIndexSize)
|
|
||||||
sip = self.getLong(buffer_sip, 0)
|
|
||||||
if ip < sip:
|
|
||||||
h = m - 1
|
|
||||||
else:
|
|
||||||
eip = self.getLong(buffer_sip, 4)
|
|
||||||
if ip > eip:
|
|
||||||
l = m + 1
|
|
||||||
else:
|
|
||||||
dataLen = self.getInt2(buffer_sip, 8)
|
|
||||||
dataPtr = self.getLong(buffer_sip, 10)
|
|
||||||
break
|
|
||||||
|
|
||||||
# empty match interception
|
|
||||||
if dataPtr < 0:
|
|
||||||
return ""
|
|
||||||
|
|
||||||
buffer_string = self.readBuffer(dataPtr, dataLen)
|
|
||||||
return_string = buffer_string.decode("utf-8")
|
|
||||||
return return_string
|
|
||||||
|
|
||||||
def readBuffer(self, offset, length):
|
|
||||||
buffer = None
|
|
||||||
# check the in-memory buffer first
|
|
||||||
if self.contentBuff is not None:
|
|
||||||
buffer = self.contentBuff[offset:offset + length]
|
|
||||||
return buffer
|
|
||||||
|
|
||||||
# read from the file handle
|
|
||||||
if self.__f is not None:
|
|
||||||
self.__f.seek(offset)
|
|
||||||
buffer = self.__f.read(length)
|
|
||||||
return buffer
|
|
||||||
|
|
||||||
def initDatabase(self, dbfile, vi, cb):
|
|
||||||
"""
|
|
||||||
" initialize the database for search
|
|
||||||
" param: dbFile, vectorIndex, contentBuff
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
if cb is not None:
|
|
||||||
self.__f = None
|
|
||||||
self.vectorIndex = None
|
|
||||||
self.contentBuff = cb
|
|
||||||
else:
|
|
||||||
self.__f = io.open(dbfile, "rb")
|
|
||||||
self.vectorIndex = vi
|
|
||||||
except IOError as e:
|
|
||||||
print("[Error]: %s" % e)
|
|
||||||
sys.exit()
|
|
||||||
|
|
||||||
def ip2long(self, ip):
|
|
||||||
_ip = socket.inet_aton(ip)
|
|
||||||
return struct.unpack("!L", _ip)[0]
|
|
||||||
|
|
||||||
def isip(self, ip):
|
|
||||||
p = ip.split(".")
|
|
||||||
|
|
||||||
if len(p) != 4: return False
|
|
||||||
for pp in p:
|
|
||||||
if not pp.isdigit(): return False
|
|
||||||
if len(pp) > 3: return False
|
|
||||||
if int(pp) > 255: return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
def getLong(self, b, offset):
|
|
||||||
if len(b[offset:offset + 4]) == 4:
|
|
||||||
return struct.unpack('I', b[offset:offset + 4])[0]
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def getInt2(self, b, offset):
|
|
||||||
return ((b[offset] & 0x000000FF) | (b[offset + 1] & 0x0000FF00))
|
|
||||||
|
|
||||||
def close(self):
|
|
||||||
if self.__f is not None:
|
|
||||||
self.__f.close()
|
|
||||||
self.vectorIndex = None
|
|
||||||
self.contentBuff = None
|
|
||||||
|
|
@ -1,6 +1,10 @@
|
||||||
# Created by leolin49 on 2022/7/7.
|
# Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||||
# Copyright (C) 2022 leolin49. All rights reserved.
|
# Use of this source code is governed by a Apache2.0-style
|
||||||
import util
|
# license that can be found in the LICENSE file.
|
||||||
|
#
|
||||||
|
# Author: leolin49 <leolin49@foxmail.com>
|
||||||
|
#
|
||||||
|
import xdb.util as util
|
||||||
|
|
||||||
|
|
||||||
class Segment:
|
class Segment:
|
||||||
|
|
@ -12,9 +16,30 @@ class Segment:
|
||||||
self.start_ip, self.end_ip = sip, eip
|
self.start_ip, self.end_ip = sip, eip
|
||||||
self.region = reg
|
self.region = reg
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return "{}|{}|{}".format(
|
||||||
|
util.long2ip(self.start_ip), util.long2ip(self.end_ip), self.region
|
||||||
|
)
|
||||||
|
|
||||||
def split(self) -> list:
|
def split(self) -> list:
|
||||||
"""Split the segment based on the pre-two bytes."""
|
"""
|
||||||
# 1, split the segment with the first byte
|
Split the segment based on the pre-two bytes.
|
||||||
|
:return: the list of segment ofter split
|
||||||
|
"""
|
||||||
|
# Example:
|
||||||
|
# split the segment "116.31.76.0|117.21.79.49|region"
|
||||||
|
#
|
||||||
|
# Return the list with segments:
|
||||||
|
# 116.31.76.0 | 116.31.255.255 | region
|
||||||
|
# 116.32.0.0 | 116.32.255.255 | region
|
||||||
|
# ... | ... | region
|
||||||
|
# 116.255.0.0 | 116.255.255.255 | region
|
||||||
|
# 117.0.0.0 | 117.0.255.255 | region
|
||||||
|
# 117.1.0.0 | 117.1.255.255 | region
|
||||||
|
# ... | ... | region
|
||||||
|
# 117.21.0.0 | 117.21.79.49 | region
|
||||||
|
|
||||||
|
# 1. Split the segment with the first byte
|
||||||
t_list_1 = []
|
t_list_1 = []
|
||||||
s_byte_1, e_byte_1 = (self.start_ip >> 24) & 0xFF, (self.end_ip >> 24) & 0xFF
|
s_byte_1, e_byte_1 = (self.start_ip >> 24) & 0xFF, (self.end_ip >> 24) & 0xFF
|
||||||
n_sip = self.start_ip
|
n_sip = self.start_ip
|
||||||
|
|
@ -25,11 +50,10 @@ class Segment:
|
||||||
n_sip = (i + 1) << 24
|
n_sip = (i + 1) << 24
|
||||||
else:
|
else:
|
||||||
eip = self.end_ip
|
eip = self.end_ip
|
||||||
|
# Append the new segment (maybe)
|
||||||
# append the new segment (maybe)
|
|
||||||
t_list_1.append(Segment(sip, eip))
|
t_list_1.append(Segment(sip, eip))
|
||||||
|
|
||||||
# 2, split the segments with the second byte
|
# 2. Split the segments with the second byte
|
||||||
t_list_2 = []
|
t_list_2 = []
|
||||||
for s in t_list_1:
|
for s in t_list_1:
|
||||||
base = s.start_ip & 0xFF000000
|
base = s.start_ip & 0xFF000000
|
||||||
|
|
@ -42,28 +66,5 @@ class Segment:
|
||||||
n_sip = 0
|
n_sip = 0
|
||||||
else:
|
else:
|
||||||
eip = self.end_ip
|
eip = self.end_ip
|
||||||
|
|
||||||
t_list_2.append(Segment(sip, eip, self.region))
|
t_list_2.append(Segment(sip, eip, self.region))
|
||||||
|
|
||||||
return t_list_2
|
return t_list_2
|
||||||
|
|
||||||
def string(self) -> str:
|
|
||||||
return util.long2ip(self.start_ip) + "|" + util.long2ip(self.end_ip) + "|" + self.region
|
|
||||||
|
|
||||||
|
|
||||||
def segment_from(seg: str) -> Segment:
|
|
||||||
segment = Segment()
|
|
||||||
ps = seg.split("|", 3)
|
|
||||||
if len(ps) != 3:
|
|
||||||
return segment
|
|
||||||
|
|
||||||
sip = util.checkip(ps[0])
|
|
||||||
if sip == -1:
|
|
||||||
return segment
|
|
||||||
eip = util.checkip(ps[1])
|
|
||||||
if eip == -1:
|
|
||||||
return segment
|
|
||||||
|
|
||||||
segment.start_ip, segment.end_ip = sip, eip
|
|
||||||
segment.region = ps[2]
|
|
||||||
return segment
|
|
||||||
|
|
|
||||||
|
|
@ -1,42 +1,47 @@
|
||||||
# Created by leolin49 on 2022/7/7.
|
# Copyright 2022 The Ip2Region Authors. All rights reserved.
|
||||||
# Copyright (C) 2022 leolin49. All rights reserved.
|
# Use of this source code is governed by a Apache2.0-style
|
||||||
|
# license that can be found in the LICENSE file.
|
||||||
shift_index = (24, 16, 8, 0)
|
#
|
||||||
# Util function
|
# Author: leolin49 <leolin49@foxmail.com>
|
||||||
|
#
|
||||||
|
_SHIFT_INDEX = (24, 16, 8, 0)
|
||||||
|
|
||||||
|
|
||||||
def checkip(ip: str) -> int:
|
def check_ip(ip: str) -> int:
|
||||||
"""Convert ip string to integer."""
|
"""
|
||||||
|
Convert ip string to integer.
|
||||||
|
Return -1 if ip is not the correct ipv4 address.
|
||||||
|
"""
|
||||||
if not is_ipv4(ip):
|
if not is_ipv4(ip):
|
||||||
return -1
|
return -1
|
||||||
ps = ip.split(".")
|
ps = ip.split(".")
|
||||||
if len(ps) != 4:
|
|
||||||
return 0
|
|
||||||
val = 0
|
val = 0
|
||||||
for i in range(len(ps)):
|
for i in range(len(ps)):
|
||||||
d = int(ps[i])
|
d = int(ps[i])
|
||||||
if d < 0 or d > 255:
|
val |= d << _SHIFT_INDEX[i]
|
||||||
return 0
|
|
||||||
val |= d << shift_index[i]
|
|
||||||
return val
|
return val
|
||||||
|
|
||||||
|
|
||||||
def long2ip(num: int) -> str:
|
def long2ip(num: int) -> str:
|
||||||
"""Convert integer to ip string."""
|
"""
|
||||||
return "{}.{}.{}.{}".format((num >> 24) & 0xFF, (num >> 16) & 0xFF, (num >> 8) & 0xFF, num & 0xFF)
|
Convert integer to ip string.
|
||||||
|
Return empty string if the num greater than UINT32_MAX or less than 0.
|
||||||
|
"""
|
||||||
def mid_ip(sip: int, eip: int):
|
if num < 0 or num > 0xFFFFFFFF:
|
||||||
"""Get the middle ip between sip and eip."""
|
return ""
|
||||||
return (sip + eip) >> 1
|
return "{}.{}.{}.{}".format(
|
||||||
|
(num >> 24) & 0xFF, (num >> 16) & 0xFF, (num >> 8) & 0xFF, num & 0xFF
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def is_ipv4(ip: str) -> bool:
|
def is_ipv4(ip: str) -> bool:
|
||||||
"""Determine whether it is an ipv4 address."""
|
"""
|
||||||
p = ip.split(".")
|
Determine whether it is an ipv4 address.
|
||||||
if len(p) != 4:
|
"""
|
||||||
|
ps = ip.split(".")
|
||||||
|
if len(ps) != 4:
|
||||||
return False
|
return False
|
||||||
for pp in p:
|
for p in ps:
|
||||||
if not pp.isdigit() or len(pp) > 3 or int(pp) > 255:
|
if not p.isdigit() or len(p) > 3 or (int(p) < 0 or int(p) > 255):
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue