maker and maker test implemented

maker/java/pom.xml

@@ -3,8 +3,8 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>org.lionsoul</groupId>
-    <artifactId>ip2region</artifactId>
+    <artifactId>ip2region-maker</artifactId>
-    <version>2.6.4</version>
+    <version>1.0.0</version>
     <packaging>jar</packaging>
 
     <name>ip2region</name>
@@ -47,7 +47,7 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.13.1</version>
+            <version>4.13.2</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -110,6 +110,14 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>6</source>
+                    <target>6</target>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 

maker/java/src/main/java/org/lionsoul/ip2region/MakerTest.java

@@ -7,10 +7,84 @@
 
 package org.lionsoul.ip2region;
 
+import org.lionsoul.ip2region.xdb.IndexPolicy;
+import org.lionsoul.ip2region.xdb.Log;
+import org.lionsoul.ip2region.xdb.Maker;
+
 public class MakerTest {
 
+    public final static Log log = Log.getLogger(MakerTest.class);
+
+    public static void printHelp(String[] args) {
+        System.out.println("ip2region xdb maker");
+        System.out.println("java -jar ip2region-maker-{version}.jar [command options]");
+        System.out.println("options:");
+        System.out.println(" --src string    source ip text file path");
+        System.out.println(" --dst string    destination binary xdb file path");
+    }
+
+    public static void genDb(String[] args) throws Exception {
+        String srcFile = "", dstFile = "";
+        int indexPolicy = IndexPolicy.Vector;
+        for (final String r : args) {
+            if (r.length() < 5) {
+                continue;
+            }
+
+            if (r.indexOf("--") != 0) {
+                continue;
+            }
+
+            int sIdx = r.indexOf('=');
+            if (sIdx < 0) {
+                System.out.printf("missing = for args pair `%s`\n", r);
+                return;
+            }
+
+            String key = r.substring(2, sIdx);
+            String val = r.substring(sIdx + 1);
+            // System.out.printf("key=%s, val=%s\n", key, val);
+            if ("src".equals(key)) {
+                srcFile = val;
+            } else if ("dst".equals(key)) {
+                dstFile = val;
+            } else if ("index".equals(key)) {
+                try {
+                    indexPolicy = IndexPolicy.parse(val);
+                } catch (Exception e) {
+                    System.out.println("parse policy " + e);
+                }
+            } else {
+                System.out.printf("undefined option `%s`\n", r);
+                return;
+            }
+        }
+
+        if (srcFile.length() < 1 || dstFile.length() < 1) {
+            printHelp(args);
+            return;
+        }
+
+        long tStart = System.currentTimeMillis();
+        Maker maker = new Maker(indexPolicy, srcFile, dstFile);
+        maker.init();
+        maker.start();
+        maker.end();
+
+        log.infof("Done, elapsed: %d s\n", (System.currentTimeMillis() - tStart) / 1000);
+    }
+
     public static void main(String[] args) {
-        System.out.println("xdb maker");
+        if (args.length < 1) {
+            printHelp(args);
+            return;
+        }
+
+        try {
+            genDb(args);
+        } catch (Exception e) {
+            System.out.printf("failed running genDb: %s\n", e);
+        }
     }
 
 }

maker/java/src/main/java/org/lionsoul/ip2region/xdb/Maker.java

@@ -51,11 +51,12 @@
 
 package org.lionsoul.ip2region.xdb;
 
-import java.io.File;
+import javax.xml.crypto.Data;
-import java.io.FileNotFoundException;
+import java.io.*;
-import java.io.IOException;
+import java.nio.charset.Charset;
-import java.io.RandomAccessFile;
 import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 
 public class Maker {
@@ -66,11 +67,14 @@ public class Maker {
     public static final int VectorIndexCols = 256;
     public static final int VectorIndexSize = 8;
     public static final int SegmentIndexSize = 14;
+    public static final int VectorIndexLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
 
     private static final Log log = Log.getLogger(Maker.class);
 
     // source text file handle
     private final File srcFile;
+    private final List<Segment> segments;
+    private final Charset bytesCharset;
 
     // destination binary file handle
     private final RandomAccessFile dstHandle;
@@ -79,10 +83,10 @@ public class Maker {
     private final int indexPolicy;
 
     // region pool
-    private final Map<String, Long> regionPool;
+    private final Map<String, DataEntry> regionPool;
 
     // vector index raw bytes
-    private byte[] vectorIndex;
+    private final byte[] vectorIndex;
 
     public Maker(int policy, String srcFile, String dstFile) throws FileNotFoundException {
         this.srcFile = new File(srcFile);
@@ -90,9 +94,12 @@ public class Maker {
             throw new FileNotFoundException("source text file `" +srcFile+ "` not found");
         }
 
-        this.dstHandle = new RandomAccessFile(dstFile, "r");
+        this.bytesCharset = Charset.forName("utf-8");
+        this.segments = new LinkedList<Segment>();
+        this.dstHandle = new RandomAccessFile(dstFile, "rw");
         this.indexPolicy = policy;
-        this.regionPool = new HashMap<String, Long>();
+        this.regionPool = new HashMap<String, DataEntry>();
+        this.vectorIndex = new byte[VectorIndexLength]; // all filled with 0
     }
 
     // init the header of the target xdb binary file
@@ -114,12 +121,53 @@ public class Maker {
     }
 
     // load all the segments
-    private void loadSegments() {
+    private void loadSegments() throws Exception {
+        log.infof("try to load the segments ... ");
+        long tStart = System.currentTimeMillis();
+        Segment last = null;
+        String line;
 
+        final FileInputStream fis = new FileInputStream(srcFile);
+        final BufferedReader br = new BufferedReader(new InputStreamReader(fis, bytesCharset));
+        while ((line = br.readLine()) != null) {
+            log.infof("load segment `%s`", line);
+            final String[] ps = line.split("\\|", 3);
+            if (ps.length != 3) {
+                br.close();
+                throw new Exception("invalid ip segment line `"+ps[0]+"`");
+            }
+
+            long sip = Util.checkIP(ps[0]);
+            long eip = Util.checkIP(ps[1]);
+            if (sip > eip) {
+                br.close();
+                throw new Exception("start ip("+ps[0]+") should not be greater than end ip("+ps[1]+")");
+            }
+
+            if (ps[2].length() < 1) {
+                br.close();
+                throw new Exception("empty region info in segment line `"+ps[2]+"`");
+            }
+
+            // check the continuity of the data segment
+            if (last != null) {
+                if (last.endIP + 1 != sip) {
+                    br.close();
+                    throw new Exception("discontinuous data segment: last.eip+1("+sip+") != seg.sip("+eip+", "+ps[0]+")");
+                }
+            }
+
+            Segment seg = new Segment(sip, eip, ps[2]);
+            segments.add(seg);
+            last = seg;
+        }
+
+        br.close();
+        log.infof("all segments loaded, length: %d, elapsed: %d ms", segments.size(), System.currentTimeMillis() - tStart);
     }
 
     // init the maker
-    public void init() throws IOException {
+    public void init() throws Exception {
         // init the db header
         initHeader();
 
@@ -127,9 +175,106 @@ public class Maker {
         loadSegments();
     }
 
-    // start to make the binary file
+    // set the vector index info of the specified ip
-    public void make() {
+    private void setVectorIndex(long ip, long ptr) {
+        int il0 = (int) ((ip >> 24) & 0xFF);
+        int il1 = (int) ((ip >> 16) & 0xFF);
+        int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
+        long sPtr = Util.getIntLong(vectorIndex, idx);
+        if (sPtr == 0) {
+            Util.write(vectorIndex, idx, ptr, 4);
+            Util.write(vectorIndex, idx + 4, ptr, 4);
+        } else {
+            Util.write(vectorIndex, idx + 4, ptr, 4);
+        }
+    }
 
+    // start to make the binary file
+    public void start() throws Exception {
+        if (segments.size() == 0) {
+            throw new Exception("empty segment list");
+        }
+
+        // 1, write all the region/data to the binary file
+        dstHandle.seek(HeaderInfoLength + VectorIndexLength);
+
+        log.infof("try to write the data block ... ");
+        for (Segment seg : segments) {
+            log.infof("try to write region `%s` ... ", seg.region);
+            DataEntry e = regionPool.get(seg.region);
+            if (e != null) {
+                log.infof(" --[Cached] with ptr=%d", e.ptr);
+                continue;
+            }
+
+            // get the utf-8 bytes of the region info
+            byte[] regionBuff = seg.region.getBytes(bytesCharset);
+            if (regionBuff.length < 1) {
+                throw new Exception("empty region info for segment `"+seg+"`");
+            } else if (regionBuff.length > 0xFFFF) {
+                throw new Exception("too long region info `"+seg.region+"`: should be less than 65535 bytes");
+            }
+
+            // record the current ptr
+            long pos = dstHandle.getFilePointer();
+            dstHandle.write(regionBuff);
+
+            // record the mapping
+            regionPool.put(seg.region, new DataEntry(regionBuff.length, pos));
+            log.infof(" --[Added] with ptr=%d", pos);
+        }
+
+        // 2, write the index block cache the super index block
+        log.infof("try to write the segment index block ... ");
+        int counter = 0;
+        long startIndexPtr = -1, endIndexPtr = -1;
+        byte[] indexBuff = new byte[SegmentIndexSize];    // 4 + 4 + 2 + 4
+        for (Segment seg : segments) {
+            // we need the region ptr
+            DataEntry e = regionPool.get(seg.region);
+            if (e == null) {
+                throw new Exception("missing ptr cache for region `"+seg.region+"`");
+            }
+
+            List<Segment> segList = seg.split();
+            log.infof("try to index segment(%d splits) %s ... ", segList.size(), seg);
+            for (Segment s : segList) {
+                long pos = dstHandle.getFilePointer();
+
+                // encode the segment index info
+                Util.write(indexBuff,  0, s.startIP, 4);
+                Util.write(indexBuff,  4, s.endIP, 4);
+                Util.write(indexBuff,  8, e.length, 2);
+                Util.write(indexBuff, 10, e.ptr, 4);
+                dstHandle.write(indexBuff);
+
+                log.infof("|-segment index: %d, ptr: %d, segment: %s", counter, pos, s);
+                setVectorIndex(s.startIP, pos);
+                counter++;
+
+                // check and record the start index ptr
+                if (startIndexPtr == -1) {
+                    startIndexPtr = pos;
+                }
+
+                endIndexPtr = pos;
+            }
+        }
+
+        // 3, synchronize the vector index block
+        log.infof("try to write the vector index block ... ");
+        dstHandle.seek(HeaderInfoLength);
+        dstHandle.write(vectorIndex);
+
+        // 4, synchronize the segment index info
+        log.infof("try to write the segment index ptr ... ");
+        Util.write(indexBuff, 0, startIndexPtr, 4);
+        Util.write(indexBuff, 4, endIndexPtr, 4);
+        dstHandle.seek(8);
+        dstHandle.write(indexBuff, 0, 8);
+
+        log.infof("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)",
+            regionPool.size(), segments.size(), counter, startIndexPtr, endIndexPtr);
     }
 
     // end the make, do the resource clean up
@@ -137,4 +282,14 @@ public class Maker {
         this.dstHandle.close();
     }
 
+    private static class DataEntry {
+        long ptr;
+        int length; // in bytes
+
+        DataEntry(int length, long ptr) {
+            this.length = length;
+            this.ptr = ptr;
+        }
+    }
+
 }