xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'java_xdb_maker'

This commit is contained in:
lion 2022-07-15 20:29:04 +08:00
parent 3b22a015a3 58d9daa85b
commit 9b60bd1254
9 changed files with 987 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
maker/java/ReadMe.md
View File

@ -1,7 +1,36 @@
# ip2region xdb java 生成实现 # ip2region xdb java 生成实现
# 编译安装
通过 maven 来编译可运行 jar 程序：
```bash
# cd 到 maker/java 根目录
mvn clean compile package
```
然会会在当前目录的 target 目录下得到一个 ip2region-maker-{version}.jar 的打包文件。
# 数据生成 # 数据生成
# 数据查询 通过 `java -jar ip2region-maker-{version}.jar` 来生成 ip2region.xdb 二进制文件：
```bash
➜ java git:(java_xdb_maker) ✗ java -jar ./target/ip2region-maker-1.0.0.jar
ip2region xdb maker
java -jar ip2region-maker-{version}.jar [command options]
options:
--src string source ip text file path
--dst string destination binary xdb file path
```
# bench 测试 例如，通过默认的 data/ip.merge.txt 原数据，在当前目录生成一个 ip2region.xdb 二进制文件：
```bash
# 会看到一堆输出，最终会看到如下输出表示运行成功
...
2022-07-15 20:21:29 INFO org.lionsoul.ip2region.xdb.Maker try to write the vector index block ...
2022-07-15 20:21:29 INFO org.lionsoul.ip2region.xdb.Maker try to write the segment index ptr ...
2022-07-15 20:21:29 INFO org.lionsoul.ip2region.xdb.Maker write done, dataBlocks: 13804, indexBlocks: (683591, 720221), indexPtr: (982904, 11065984)
2022-07-15 20:21:29 INFO org.lionsoul.ip2region.MakerTest Done, elapsed: 50 s
```
# 数据 查询/bench 测试
已经完成开发的 [binding](../../binding/) 都有查询和 bench 测试程序以及使用文档，你可以使用你熟悉的语言的 searcher 进行查询测试或者bench测试，来确认数据的正确性和完整性。

198
maker/java/pom.xml Normal file
View File

@ -0,0 +1,198 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.lionsoul</groupId>
<artifactId>ip2region-maker</artifactId>
<version>1.0.0</version>
<packaging>jar</packaging>
<name>ip2region</name>
<url>https://github.com/lionsoul2014/ip2region</url>
<description>Open source offline internet address db manager framework and locator</description>
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
<url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<url>git@github.com:lionsoul2014/ip2region.git</url>
<connection>scm:git:git@github.com:lionsoul2014/ip2region.git</connection>
<developerConnection>scm:git:git@github.com:lionsoul2014/ip2region.git</developerConnection>
</scm>
<developers>
<developer>
<id>lionsoul</id>
<name>chenxin</name>
<email>chenxin619315@gmail.com</email>
</developer>
</developers>
<issueManagement>
<url>https://github.com/lionsoul2014/ip2region/issues</url>
<system>Github issues</system>
</issueManagement>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.1.2</version>
<executions>
<execution>
<id>attach-sources</id>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9</version>
<executions>
<execution>
<id>attach-javadocs</id>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
<configuration>
<additionalparam>${javadoc.opts}</additionalparam>
</configuration>
</execution>
</executions>
<configuration>
<failOnError>false</failOnError>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>1.4</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.lionsoul.ip2region.MakerTest</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>6</source>
<target>6</target>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>java8-doclint-disabled</id>
<activation>
<jdk>[1.8,)</jdk>
</activation>
<properties>
<javadoc.opts>-Xdoclint:none</javadoc.opts>
</properties>
</profile>
<profile>
<id>release</id>
<build>
<plugins>
<!-- Source -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<!-- Javadoc -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
<configuration>
<additionalparam>${javadoc.opts}</additionalparam>
</configuration>
</execution>
</executions>
</plugin>
<!-- GPG -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.5</version>
<executions>
<execution>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<distributionManagement>
<snapshotRepository>
<id>oss</id>
<url>https://oss.sonatype.org/content/repositories/snapshots/</url>
</snapshotRepository>
<repository>
<id>oss</id>
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
</profile>
</profiles>
</project>

90
maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java Normal file
View File

@ -0,0 +1,90 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/12
package org.lionsoul.ip2region;
import org.lionsoul.ip2region.xdb.IndexPolicy;
import org.lionsoul.ip2region.xdb.Log;
import org.lionsoul.ip2region.xdb.Maker;
public class MakerTest {
public final static Log log = Log.getLogger(MakerTest.class);
public static void printHelp(String[] args) {
System.out.println("ip2region xdb maker");
System.out.println("java -jar ip2region-maker-{version}.jar [command options]");
System.out.println("options:");
System.out.println(" --src string source ip text file path");
System.out.println(" --dst string destination binary xdb file path");
}
public static void genDb(String[] args) throws Exception {
String srcFile = "", dstFile = "";
int indexPolicy = IndexPolicy.Vector;
for (final String r : args) {
if (r.length() < 5) {
continue;
}
if (r.indexOf("--") != 0) {
continue;
}
int sIdx = r.indexOf('=');
if (sIdx < 0) {
System.out.printf("missing = for args pair `%s`\n", r);
return;
}
String key = r.substring(2, sIdx);
String val = r.substring(sIdx + 1);
// System.out.printf("key=%s, val=%s\n", key, val);
if ("src".equals(key)) {
srcFile = val;
} else if ("dst".equals(key)) {
dstFile = val;
} else if ("index".equals(key)) {
try {
indexPolicy = IndexPolicy.parse(val);
} catch (Exception e) {
System.out.println("parse policy " + e);
}
} else {
System.out.printf("undefined option `%s`\n", r);
return;
}
}
if (srcFile.length() < 1 || dstFile.length() < 1) {
printHelp(args);
return;
}
long tStart = System.currentTimeMillis();
Maker maker = new Maker(indexPolicy, srcFile, dstFile);
maker.init();
maker.start();
maker.end();
log.infof("Done, elapsed: %d s", (System.currentTimeMillis() - tStart) / 1000);
}
public static void main(String[] args) {
if (args.length < 1) {
printHelp(args);
return;
}
try {
genDb(args);
} catch (Exception e) {
System.out.printf("failed running genDb: %s\n", e);
}
}
}

75
maker/java/src/main/java/org/lionsoul/ip2region/UtilTest.java Normal file
View File

@ -0,0 +1,75 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/15
package org.lionsoul.ip2region;
import org.lionsoul.ip2region.xdb.IndexPolicy;
import org.lionsoul.ip2region.xdb.Log;
import org.lionsoul.ip2region.xdb.Segment;
import org.lionsoul.ip2region.xdb.Util;
public class UtilTest {
public static final Log log = Log.getLogger(UtilTest.class);
public static void testIndexPolicy() {
final String[] inputs = {"vector", "btree", "VecTor", "BTree", "abc"};
for (String str : inputs) {
try {
int policy = IndexPolicy.parse(str);
log.infof("parse(%s)=%d", str, policy);
} catch (Exception e) {
log.errorf("parse index policy `%s`: %s", str, e.getMessage());
}
}
}
public static void testIPUtil() {
final String ipStr = "1.2.3.4";
try {
long ip = Util.checkIP(ipStr);
log.infof("checkIP(%s)=%d, validate: %s, long2ip(%d)=%s", ipStr, ip, ip == 16909060 ? "true" : "false", ip, Util.long2ip(ip));
} catch (Exception e) {
log.errorf("failed to check ip `%s`: %s", ipStr, e.getMessage());
}
}
public static void testSegment() {
final String[] t_segs = {
"1.1.0.0|1.3.3.24|中国|广东|深圳|电信",
"0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP",
"28.201.224.0|29.34.191.255|美国|0|0|0|0"
};
for (String str : t_segs) {
try {
final Segment seg = Segment.parse(str);
log.infof("segment(%s)=%s, split: ", str, seg.toString());
for (Segment s : seg.split()) {
log.debugf(s.toString());
}
} catch (Exception e) {
log.errorf("failed to parse segment `%s`: %s", str, e.getMessage());
}
}
}
public static void main(String[] args) {
System.out.println("+-- testing index policy");
testIndexPolicy();
System.out.println();
System.out.println("+-- testing ip util");
testIPUtil();
System.out.println();
System.out.println("+-- testing segments");
testSegment();
System.out.println();
}
}

25
maker/java/src/main/java/org/lionsoul/ip2region/xdb/IndexPolicy.java Normal file
View File

@ -0,0 +1,25 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/14
package org.lionsoul.ip2region.xdb;
public class IndexPolicy {
public static final int Vector = 1;
public static final int BTree = 2;
// parser the index policy from string
public static int parse(String policy) throws Exception {
String v = policy.toLowerCase();
if ("vector".equals(v)) {
return Vector;
} else if ("btree".equals(v)) {
return BTree;
} else {
throw new Exception("unknown index policy `"+policy+"`");
}
}
}

116
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Log.java Normal file
View File

@ -0,0 +1,116 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/14
package org.lionsoul.ip2region.xdb;
import java.text.SimpleDateFormat;
import java.util.Date;
// simple log implementation
public class Log {
/* Log level constants define */
public static final int DEBUG = 0;
public static final int INFO = 1;
public static final int WARN = 2;
public static final int ERROR = 3;
// level name
public static final String[] level_string = new String[] {
"DEBUG",
"INFO",
"WARN",
"ERROR"
};
public final Class<?> baseClass;
private static int level;
public Log(Class<?> baseClass) {
this.baseClass = baseClass;
}
public static Log getLogger(Class<?> baseClass) {
return new Log(baseClass);
}
public String format(int level, String format, Object... args) {
// append the datetime
final StringBuilder sb = new StringBuilder();
final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sb.append(String.format("%s %-5s ", sdf.format(new Date()), level_string[level]));
// append the class name
sb.append(baseClass.getName()).append(' ');
sb.append(String.format(format, args));
return sb.toString();
}
public void printf(int level, String format, Object... args) {
if (level < DEBUG || level > ERROR) {
throw new IndexOutOfBoundsException("invalid level index " + level);
}
// level filter
if (level < Log.level) {
return;
}
System.out.println(format(level, format, args));
System.out.flush();
}
public String getDebugf(String format, Object... args) {
return format(DEBUG, format, args);
}
public void debugf(String format, Object... args) {
printf(DEBUG, format, args);
}
public String getInfof(String format, Object... args) {
return format(INFO, format, args);
}
public void infof(String format, Object... args) {
printf(INFO, format, args);
}
public String getWarnf(String format, Object... args) {
return format(WARN, format, args);
}
public void warnf(String format, Object... args) {
printf(WARN, format, args);
}
public String getErrorf(String format, Object... args) {
return format(ERROR, format, args);
}
public void errorf(String format, Object... args) {
printf(ERROR, format, args);
}
public static void setLevel(int level) {
Log.level = level;
}
public static void setLevel(String level) {
String v = level.toLowerCase();
if ("debug".equals(v)) {
Log.level = DEBUG;
} else if ("info".equals(v)) {
Log.level = INFO;
} else if ("warn".equals(v)) {
Log.level = WARN;
} else if ("error".equals(v)) {
Log.level = ERROR;
}
}
}

295
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java Normal file
View File

@ -0,0 +1,295 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/12
// --- Ip2Region v2.0 data structure
//
// +----------------+--------------------------+---------------+--------------+
// | header space | vector speed up index | data payload | block index |
// +----------------+--------------------------+---------------+--------------+
// | 256 bytes | 512 KiB (fixed) | dynamic size | dynamic size |
// +----------------+--------------------------+---------------+--------------+
//
// 1. padding space : for header info like block index ptr, version, release date eg ... or any other temporary needs.
// -- 2bytes: version number, different version means structure update, it fixed to 2 for now
// -- 2bytes: index algorithm code.
// -- 4bytes: generate unix timestamp (version)
// -- 4bytes: index block start ptr
// -- 4bytes: index block end ptr
//
//
// 2. data block : region or whatever data info.
// 3. segment index block : binary index block.
// 4. vector index block : fixed index info for block index search speed up.
// space structure table:
// -- 0 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block
// -- 1 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block
// -- 2 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block
// -- ...
// -- 255 -> | 1rt super block | 2nd super block | 3rd super block | ... | 255th super block
//
//
// super block structure:
// +-----------------------+----------------------+
// | first index block ptr | last index block ptr |
// +-----------------------+----------------------+
//
// data entry structure:
// +--------------------+-----------------------+
// | 2bytes (for desc) | dynamic length |
// +--------------------+-----------------------+
// data length whatever in bytes
//
// index entry structure
// +------------+-----------+---------------+------------+
// | 4bytes | 4bytes | 2bytes | 4 bytes |
// +------------+-----------+---------------+------------+
// start ip end ip data length data ptr
package org.lionsoul.ip2region.xdb;
import javax.xml.crypto.Data;
import java.io.*;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
public class Maker {
// constants define
public static final int VersionNo = 2;
public static final int HeaderInfoLength = 256;
public static final int VectorIndexRows = 256;
public static final int VectorIndexCols = 256;
public static final int VectorIndexSize = 8;
public static final int SegmentIndexSize = 14;
public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
private static final Log log = Log.getLogger(Maker.class);
// source text file handle
private final File srcFile;
private final List<Segment> segments;
private final Charset bytesCharset;
// destination binary file handle
private final RandomAccessFile dstHandle;
// index policy
private final int indexPolicy;
// region pool
private final Map<String, DataEntry> regionPool;
// vector index raw bytes
private final byte[] vectorIndex;
public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException {
this.srcFile = new File(srcFile);
if (!this.srcFile.exists()) {
throw new FileNotFoundException("source text file `" +srcFile+ "` not found");
}
this.bytesCharset = Charset.forName("utf-8");
this.segments = new LinkedList<Segment>();
this.dstHandle = new RandomAccessFile(dstFile, "rw");
this.indexPolicy = policy;
this.regionPool = new HashMap<String, DataEntry>();
this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0
}
// init the header of the target xdb binary file
private void initHeader() throws IOException {
log.infof("try to init the db header ... ");
dstHandle.seek(0);
// make and write the header space
final byte[] header = new byte[HeaderInfoLength];
// encode the data
Util.write(header, 0, VersionNo, 2);
Util.write(header, 2, indexPolicy, 2);
Util.write(header, 4, System.currentTimeMillis() / 1000, 4);
Util.write(header, 8, 0, 4); // start index ptr
Util.write(header, 12, 0, 4); // end index ptr
dstHandle.write(header);
}
// load all the segments
private void loadSegments() throws Exception {
log.infof("try to load the segments ... ");
long tStart = System.currentTimeMillis();
Segment last = null;
String line;
final FileInputStream fis = new FileInputStream(srcFile);
final BufferedReader br = new BufferedReader(new InputStreamReader(fis, bytesCharset));
while ((line = br.readLine()) != null) {
log.infof("load segment `%s`", line);
final String[] ps = line.split("\\|", 3);
if (ps.length != 3) {
br.close();
throw new Exception("invalid ip segment line `"+ps[0]+"`");
}
long sip = Util.checkIP(ps[0]);
long eip = Util.checkIP(ps[1]);
if (sip > eip) {
br.close();
throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")");
}
if (ps[2].length() < 1) {
br.close();
throw new Exception("empty region info in segment line `"+ps[2]+"`");
}
// check the continuity of the data segment
if (last != null) {
if (last.endIP + 1 != sip) {
br.close();
throw new Exception("discontinuous data segment: last.eip+1("+sip+") != seg.sip("+eip+", "+ps[0]+")");
}
}
Segment seg = new Segment(sip, eip, ps[2]);
segments.add(seg);
last = seg;
}
br.close();
log.infof("all segments loaded, length: %d, elapsed: %d ms", segments.size(), System.currentTimeMillis() - tStart);
}
// init the maker
public void init() throws Exception {
// init the db header
initHeader();
// load all the segments
loadSegments();
}
// set the vector index info of the specified ip
private void setVectorIndex(long ip, long ptr) {
int il0 = (int) ((ip >> 24) & 0xFF);
int il1 = (int) ((ip >> 16) & 0xFF);
int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
long sPtr = Util.getIntLong(vectorIndex, idx);
if (sPtr == 0) {
Util.write(vectorIndex, idx, ptr, 4);
Util.write(vectorIndex, idx + 4, ptr, 4);
} else {
Util.write(vectorIndex, idx + 4, ptr, 4);
}
}
// start to make the binary file
public void start() throws Exception {
if (segments.size() == 0) {
throw new Exception("empty segment list");
}
// 1, write all the region/data to the binary file
dstHandle.seek(HeaderInfoLength + VectorIndexLength);
log.infof("try to write the data block ... ");
for (Segment seg : segments) {
log.infof("try to write region `%s` ... ", seg.region);
DataEntry e = regionPool.get(seg.region);
if (e != null) {
log.infof(" --[Cached] with ptr=%d", e.ptr);
continue;
}
// get the utf-8 bytes of the region info
byte[] regionBuff = seg.region.getBytes(bytesCharset);
if (regionBuff.length < 1) {
throw new Exception("empty region info for segment `"+seg+"`");
} else if (regionBuff.length > 0xFFFF) {
throw new Exception("too long region info `"+seg.region+"`: should be less than 65535 bytes");
}
// record the current ptr
long pos = dstHandle.getFilePointer();
dstHandle.write(regionBuff);
// record the mapping
regionPool.put(seg.region, new DataEntry(regionBuff.length, pos));
log.infof(" --[Added] with ptr=%d", pos);
}
// 2, write the index block cache the super index block
log.infof("try to write the segment index block ... ");
int counter = 0;
long startIndexPtr = -1, endIndexPtr = -1;
byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4
for (Segment seg : segments) {
// we need the region ptr
DataEntry e = regionPool.get(seg.region);
if (e == null) {
throw new Exception("missing ptr cache for region `"+seg.region+"`");
}
List<Segment> segList = seg.split();
log.infof("try to index segment(%d splits) %s ... ", segList.size(), seg);
for (Segment s : segList) {
long pos = dstHandle.getFilePointer();
// encode the segment index info
Util.write(indexBuff, 0, s.startIP, 4);
Util.write(indexBuff, 4, s.endIP, 4);
Util.write(indexBuff, 8, e.length, 2);
Util.write(indexBuff, 10, e.ptr, 4);
dstHandle.write(indexBuff);
log.infof("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s);
setVectorIndex(s.startIP, pos);
counter++;
// check and record the start index ptr
if (startIndexPtr == -1) {
startIndexPtr = pos;
}
endIndexPtr = pos;
}
}
// 3, synchronize the vector index block
log.infof("try to write the vector index block ... ");
dstHandle.seek(HeaderInfoLength);
dstHandle.write(vectorIndex);
// 4, synchronize the segment index info
log.infof("try to write the segment index ptr ... ");
Util.write(indexBuff, 0, startIndexPtr, 4);
Util.write(indexBuff, 4, endIndexPtr, 4);
dstHandle.seek(8);
dstHandle.write(indexBuff, 0, 8);
log.infof("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)",
regionPool.size(), segments.size(), counter, startIndexPtr, endIndexPtr);
}
// end the make, do the resource clean up
public void end() throws IOException {
this.dstHandle.close();
}
private static class DataEntry {
long ptr;
int length; // in bytes
DataEntry(int length, long ptr) {
this.length = length;
this.ptr = ptr;
}
}
}

85
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Segment.java Normal file
View File

@ -0,0 +1,85 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/14
package org.lionsoul.ip2region.xdb;
import java.util.ArrayList;
import java.util.List;
public class Segment {
public final long startIP;
public final long endIP;
public final String region;
// parser the Segment from an input string
public static Segment parse(String input) throws Exception {
String[] ps = input.split("\\|", 3);
if (ps.length != 3) {
throw new Exception("invalid ip segment `"+input+"`");
}
long sip = Util.checkIP(ps[0]);
long eip = Util.checkIP(ps[1]);
if (sip > eip) {
throw new Exception("start ip `"+ps[0]+"` should not be greater than end ip `"+ps[1]+"`");
}
return new Segment(sip, eip, ps[2]);
}
public Segment(long startIP, long endIP, String region) {
this.startIP = startIP;
this.endIP = endIP;
this.region = region;
}
// split the current segment for vector index
public List<Segment> split() {
long sByte1 = (int) ((startIP >> 24) & 0xFF);
long eByte1 = (int) ((endIP >> 24) & 0xFF);
long nSip = startIP;
final List<Segment> tList = new ArrayList<Segment>();
for (long i = sByte1; i <= eByte1; i++) {
long sip = (i << 24) | (nSip & 0xFFFFFF);
long eip = (i << 24) | 0xFFFFFF;
if (eip < endIP) {
nSip = (i + 1) << 24;
} else {
eip = endIP;
}
// append the new segment
tList.add(new Segment(sip, eip, null));
}
// 2, split the segments with the second byte
final List<Segment> segList = new ArrayList<Segment>();
for (Segment seg : tList) {
long base = seg.startIP & 0xFF000000;
long tSip = seg.startIP;
long sb2 = (seg.startIP >> 16) & 0xFF;
long eb2 = (seg.endIP >> 16) & 0xFF;
for (long i = sb2; i <= eb2; i++) {
long sip = base | (i << 16) | (tSip & 0xFFFF);
long eip = base | (i << 16) | 0xFFFF;
if (eip < seg.endIP) {
tSip = 0;
} else {
eip = seg.endIP;
}
segList.add(new Segment(sip, eip, region));
}
}
return segList;
}
@Override public String toString() {
return Util.long2ip(startIP) + "|" + Util.long2ip(endIP) + "|" + region;
}
}

72
maker/java/src/main/java/org/lionsoul/ip2region/xdb/Util.java Normal file
View File

@ -0,0 +1,72 @@
// Copyright 2022 The Ip2Region Authors. All rights reserved.
// Use of this source code is governed by a Apache2.0-style
// license that can be found in the LICENSE file.
//
// @Author Lion <chenxin619315@gmail.com>
// @Date 2022/07/14
package org.lionsoul.ip2region.xdb;
public class Util
{
// write specified bytes into a byte array start from offset
public static void write( byte[] b, int offset, long v, int bytes) {
for ( int i = 0; i < bytes; i++ ) {
b[offset++] = (byte)((v >>> (8 * i)) & 0xFF);
}
}
// write a int to a byte array
public static void writeIntLong(byte[] b, int offset, long v) {
b[offset++] = (byte)((v ) & 0xFF);
b[offset++] = (byte)((v >> 8) & 0xFF);
b[offset++] = (byte)((v >> 16) & 0xFF);
b[offset ] = (byte)((v >> 24) & 0xFF);
}
// get an int from a byte array start from the specified offset
public static long getIntLong(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FFL)) |
((b[offset++] << 8) & 0x0000FF00L) |
((b[offset++] << 16) & 0x00FF0000L) |
((b[offset ] << 24) & 0xFF000000L)
);
}
public static int getInt2(byte[] b, int offset) {
return (
(b[offset++] & 0x000000FF) |
(b[offset ] & 0x0000FF00)
);
}
/* long int to ip string */
public static String long2ip( long ip ) {
return String.valueOf((ip >> 24) & 0xFF) + '.' +
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
}
public static final byte[] shiftIndex = {24, 16, 8, 0};
/* check the specified ip address */
public static long checkIP(String ip) throws Exception {
String[] ps = ip.split("\\.");
if (ps.length != 4) {
throw new Exception("invalid ip address `" + ip + "`");
}
long ipDst = 0;
for (int i = 0; i < ps.length; i++) {
int val = Integer.parseInt(ps[i]);
if (val > 255) {
throw new Exception("ip part `"+ps[i]+"` should be less then 256");
}
ipDst |= ((long) val << shiftIndex[i]);
}
return ipDst & 0xFFFFFFFFL;
}
}