From 061f2f8c91f637e7a5f4e909ac24ab9f6646c4a0 Mon Sep 17 00:00:00 2001 From: Lion Date: Fri, 15 Jul 2022 19:05:42 +0800 Subject: [PATCH] maker and maker test implemented --- maker/java/pom.xml | 14 +- .../org/lionsoul/ip2region/MakerTest.java | 76 +++++++- .../org/lionsoul/ip2region/xdb/Maker.java | 179 ++++++++++++++++-- 3 files changed, 253 insertions(+), 16 deletions(-) diff --git a/maker/java/pom.xml b/maker/java/pom.xml index 391aef7..1bf6653 100644 --- a/maker/java/pom.xml +++ b/maker/java/pom.xml @@ -3,8 +3,8 @@ 4.0.0 org.lionsoul - ip2region - 2.6.4 + ip2region-maker + 1.0.0 jar ip2region @@ -47,7 +47,7 @@ junit junit - 4.13.1 + 4.13.2 test @@ -110,6 +110,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 6 + 6 + + diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java index 017facd..414a2da 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java @@ -7,10 +7,84 @@ package org.lionsoul.ip2region; +import org.lionsoul.ip2region.xdb.IndexPolicy; +import org.lionsoul.ip2region.xdb.Log; +import org.lionsoul.ip2region.xdb.Maker; + public class MakerTest { + public final static Log log = Log.getLogger(MakerTest.class); + + public static void printHelp(String[] args) { + System.out.println("ip2region xdb maker"); + System.out.println("java -jar ip2region-maker-{version}.jar [command options]"); + System.out.println("options:"); + System.out.println(" --src string source ip text file path"); + System.out.println(" --dst string destination binary xdb file path"); + } + + public static void genDb(String[] args) throws Exception { + String srcFile = "", dstFile = ""; + int indexPolicy = IndexPolicy.Vector; + for (final String r : args) { + if (r.length() < 5) { + continue; + } + + if (r.indexOf("--") != 0) { + continue; + } + + int sIdx = r.indexOf('='); + if (sIdx < 0) { + System.out.printf("missing = for args pair `%s`\n", r); + return; + } + + String key = r.substring(2, sIdx); + String val = r.substring(sIdx + 1); + // System.out.printf("key=%s, val=%s\n", key, val); + if ("src".equals(key)) { + srcFile = val; + } else if ("dst".equals(key)) { + dstFile = val; + } else if ("index".equals(key)) { + try { + indexPolicy = IndexPolicy.parse(val); + } catch (Exception e) { + System.out.println("parse policy " + e); + } + } else { + System.out.printf("undefined option `%s`\n", r); + return; + } + } + + if (srcFile.length() < 1 || dstFile.length() < 1) { + printHelp(args); + return; + } + + long tStart = System.currentTimeMillis(); + Maker maker = new Maker(indexPolicy, srcFile, dstFile); + maker.init(); + maker.start(); + maker.end(); + + log.infof("Done, elapsed: %d s\n", (System.currentTimeMillis() - tStart) / 1000); + } + public static void main(String[] args) { - System.out.println("xdb maker"); + if (args.length < 1) { + printHelp(args); + return; + } + + try { + genDb(args); + } catch (Exception e) { + System.out.printf("failed running genDb: %s\n", e); + } } } diff --git a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java index d86f427..42108de 100644 --- a/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java +++ b/maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java @@ -51,11 +51,12 @@ package org.lionsoul.ip2region.xdb; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; +import javax.xml.crypto.Data; +import java.io.*; +import java.nio.charset.Charset; import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; import java.util.Map; public class Maker { @@ -66,11 +67,14 @@ public class Maker { public static final int VectorIndexCols = 256; public static final int VectorIndexSize = 8; public static final int SegmentIndexSize = 14; + public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize; private static final Log log = Log.getLogger(Maker.class); // source text file handle private final File srcFile; + private final List segments; + private final Charset bytesCharset; // destination binary file handle private final RandomAccessFile dstHandle; @@ -79,10 +83,10 @@ public class Maker { private final int indexPolicy; // region pool - private final Map regionPool; + private final Map regionPool; // vector index raw bytes - private byte[] vectorIndex; + private final byte[] vectorIndex; public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException { this.srcFile = new File(srcFile); @@ -90,9 +94,12 @@ public class Maker { throw new FileNotFoundException("source text file `" +srcFile+ "` not found"); } - this.dstHandle = new RandomAccessFile(dstFile, "r"); + this.bytesCharset = Charset.forName("utf-8"); + this.segments = new LinkedList(); + this.dstHandle = new RandomAccessFile(dstFile, "rw"); this.indexPolicy = policy; - this.regionPool = new HashMap(); + this.regionPool = new HashMap(); + this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0 } // init the header of the target xdb binary file @@ -114,12 +121,53 @@ public class Maker { } // load all the segments - private void loadSegments() { + private void loadSegments() throws Exception { + log.infof("try to load the segments ... "); + long tStart = System.currentTimeMillis(); + Segment last = null; + String line; + final FileInputStream fis = new FileInputStream(srcFile); + final BufferedReader br = new BufferedReader(new InputStreamReader(fis, bytesCharset)); + while ((line = br.readLine()) != null) { + log.infof("load segment `%s`", line); + final String[] ps = line.split("\\|", 3); + if (ps.length != 3) { + br.close(); + throw new Exception("invalid ip segment line `"+ps[0]+"`"); + } + + long sip = Util.checkIP(ps[0]); + long eip = Util.checkIP(ps[1]); + if (sip > eip) { + br.close(); + throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")"); + } + + if (ps[2].length() < 1) { + br.close(); + throw new Exception("empty region info in segment line `"+ps[2]+"`"); + } + + // check the continuity of the data segment + if (last != null) { + if (last.endIP + 1 != sip) { + br.close(); + throw new Exception("discontinuous data segment: last.eip+1("+sip+") != seg.sip("+eip+", "+ps[0]+")"); + } + } + + Segment seg = new Segment(sip, eip, ps[2]); + segments.add(seg); + last = seg; + } + + br.close(); + log.infof("all segments loaded, length: %d, elapsed: %d ms", segments.size(), System.currentTimeMillis() - tStart); } // init the maker - public void init() throws IOException { + public void init() throws Exception { // init the db header initHeader(); @@ -127,9 +175,106 @@ public class Maker { loadSegments(); } - // start to make the binary file - public void make() { + // set the vector index info of the specified ip + private void setVectorIndex(long ip, long ptr) { + int il0 = (int) ((ip >> 24) & 0xFF); + int il1 = (int) ((ip >> 16) & 0xFF); + int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; + long sPtr = Util.getIntLong(vectorIndex, idx); + if (sPtr == 0) { + Util.write(vectorIndex, idx, ptr, 4); + Util.write(vectorIndex, idx + 4, ptr, 4); + } else { + Util.write(vectorIndex, idx + 4, ptr, 4); + } + } + // start to make the binary file + public void start() throws Exception { + if (segments.size() == 0) { + throw new Exception("empty segment list"); + } + + // 1, write all the region/data to the binary file + dstHandle.seek(HeaderInfoLength + VectorIndexLength); + + log.infof("try to write the data block ... "); + for (Segment seg : segments) { + log.infof("try to write region `%s` ... ", seg.region); + DataEntry e = regionPool.get(seg.region); + if (e != null) { + log.infof(" --[Cached] with ptr=%d", e.ptr); + continue; + } + + // get the utf-8 bytes of the region info + byte[] regionBuff = seg.region.getBytes(bytesCharset); + if (regionBuff.length < 1) { + throw new Exception("empty region info for segment `"+seg+"`"); + } else if (regionBuff.length > 0xFFFF) { + throw new Exception("too long region info `"+seg.region+"`: should be less than 65535 bytes"); + } + + // record the current ptr + long pos = dstHandle.getFilePointer(); + dstHandle.write(regionBuff); + + // record the mapping + regionPool.put(seg.region, new DataEntry(regionBuff.length, pos)); + log.infof(" --[Added] with ptr=%d", pos); + } + + // 2, write the index block cache the super index block + log.infof("try to write the segment index block ... "); + int counter = 0; + long startIndexPtr = -1, endIndexPtr = -1; + byte[] indexBuff = new byte[SegmentIndexSize]; // 4 + 4 + 2 + 4 + for (Segment seg : segments) { + // we need the region ptr + DataEntry e = regionPool.get(seg.region); + if (e == null) { + throw new Exception("missing ptr cache for region `"+seg.region+"`"); + } + + List segList = seg.split(); + log.infof("try to index segment(%d splits) %s ... ", segList.size(), seg); + for (Segment s : segList) { + long pos = dstHandle.getFilePointer(); + + // encode the segment index info + Util.write(indexBuff, 0, s.startIP, 4); + Util.write(indexBuff, 4, s.endIP, 4); + Util.write(indexBuff, 8, e.length, 2); + Util.write(indexBuff, 10, e.ptr, 4); + dstHandle.write(indexBuff); + + log.infof("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s); + setVectorIndex(s.startIP, pos); + counter++; + + // check and record the start index ptr + if (startIndexPtr == -1) { + startIndexPtr = pos; + } + + endIndexPtr = pos; + } + } + + // 3, synchronize the vector index block + log.infof("try to write the vector index block ... "); + dstHandle.seek(HeaderInfoLength); + dstHandle.write(vectorIndex); + + // 4, synchronize the segment index info + log.infof("try to write the segment index ptr ... "); + Util.write(indexBuff, 0, startIndexPtr, 4); + Util.write(indexBuff, 4, endIndexPtr, 4); + dstHandle.seek(8); + dstHandle.write(indexBuff, 0, 8); + + log.infof("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)", + regionPool.size(), segments.size(), counter, startIndexPtr, endIndexPtr); } // end the make, do the resource clean up @@ -137,4 +282,14 @@ public class Maker { this.dstHandle.close(); } + private static class DataEntry { + long ptr; + int length; // in bytes + + DataEntry(int length, long ptr) { + this.length = length; + this.ptr = ptr; + } + } + }