From fb8e0ad9c999a421d741f7b63ba55a04bcfd7eeb Mon Sep 17 00:00:00 2001 From: jiahao <1250322798@qq.com> Date: Thu, 10 Aug 2023 14:09:48 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=8E=A8:=20building\java=20=E4=B8=AD?= =?UTF-8?q?=20Searcher=20=E5=AE=9E=E7=8E=B0=20Closeable=20=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- binding/java/ReadMe.md | 55 ++-- binding/java/pom.xml | 10 + .../org/lionsoul/ip2region/SearchTest.java | 163 +++++------ .../java/org/lionsoul/ip2region/UtilTest.java | 2 +- .../org/lionsoul/ip2region/xdb/Header.java | 15 +- .../org/lionsoul/ip2region/xdb/Searcher.java | 270 +++++++++--------- 6 files changed, 257 insertions(+), 258 deletions(-) diff --git a/binding/java/ReadMe.md b/binding/java/ReadMe.md index 8f8e7fc..44231f4 100644 --- a/binding/java/ReadMe.md +++ b/binding/java/ReadMe.md @@ -15,35 +15,29 @@ ```java import org.lionsoul.ip2region.xdb.Searcher; + import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { - // 1、创建 searcher 对象 String dbPath = "ip2region.xdb file path"; - Searcher searcher = null; - try { - searcher = Searcher.newWithFileOnly(dbPath); - } catch (IOException e) { - System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e); - return; - } + String ip = "1.2.3.4"; - // 2、查询 - try { - String ip = "1.2.3.4"; + // 1、创建 searcher 对象 + try (Searcher searcher = Searcher.newWithFileOnly(dbPath)) { + // 2、查询 long sTime = System.nanoTime(); String region = searcher.search(ip); - long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); + long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); + } catch (IOException e) { + System.out.printf("failed to create searcher with `%s`: %s\n", dbPath, e); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } + // 3、关闭资源（这里通过 try-with-resources 自动关闭） - // 3、关闭资源 - searcher.close(); - // 备注：并发使用，每个线程需要创建一个独立的 searcher 对象单独使用。 } } @@ -54,12 +48,14 @@ public class SearcherTest { 我们可以提前从 `xdb` 文件中加载出来 `VectorIndex` 数据，然后全局缓存，每次创建 Searcher 对象的时候使用全局的 VectorIndex 缓存可以减少一次固定的 IO 操作，从而加速查询，减少 IO 压力。 ```java import org.lionsoul.ip2region.xdb.Searcher; + import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { String dbPath = "ip2region.xdb file path"; + String ip = "1.2.3.4"; // 1、从 dbPath 中预先加载 VectorIndex 缓存，并且把这个得到的数据作为全局变量，后续反复使用。 byte[] vIndex; @@ -71,27 +67,17 @@ public class SearcherTest { } // 2、使用全局的 vIndex 创建带 VectorIndex 缓存的查询对象。 - Searcher searcher; - try { - searcher = Searcher.newWithVectorIndex(dbPath, vIndex); - } catch (Exception e) { - System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e); - return; - } - - // 3、查询 - try { - String ip = "1.2.3.4"; + try (Searcher searcher = Searcher.newWithVectorIndex(dbPath, vIndex)) { long sTime = System.nanoTime(); String region = searcher.search(ip); - long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); + long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); + } catch (IOException e) { + System.out.printf("failed to create vectorIndex cached searcher with `%s`: %s\n", dbPath, e); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } - - // 4、关闭资源 - searcher.close(); + // 3、关闭资源（这里通过 try-with-resources 自动关闭） // 备注：每个线程需要单独创建一个独立的 Searcher 对象，但是都共享全局的制度 vIndex 缓存。 } @@ -103,12 +89,14 @@ public class SearcherTest { 我们也可以预先加载整个 ip2region.xdb 的数据到内存，然后基于这个数据创建查询对象来实现完全基于文件的查询，类似之前的 memory search。 ```java import org.lionsoul.ip2region.xdb.Searcher; + import java.io.*; import java.util.concurrent.TimeUnit; public class SearcherTest { public static void main(String[] args) { String dbPath = "ip2region.xdb file path"; + String ip = "1.2.3.4"; // 1、从 dbPath 加载整个 xdb 到内存。 byte[] cBuff; @@ -123,22 +111,21 @@ public class SearcherTest { Searcher searcher; try { searcher = Searcher.newWithBuffer(cBuff); - } catch (Exception e) { + } catch (IOException e) { System.out.printf("failed to create content cached searcher: %s\n", e); return; } // 3、查询 try { - String ip = "1.2.3.4"; long sTime = System.nanoTime(); String region = searcher.search(ip); - long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); + long cost = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - sTime); System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); } catch (Exception e) { System.out.printf("failed to search(%s): %s\n", ip, e); } - + // 4、关闭资源 - 该 searcher 对象可以安全用于并发，等整个服务关闭的时候再关闭 searcher // searcher.close(); diff --git a/binding/java/pom.xml b/binding/java/pom.xml index 43fe239..7ad2492 100644 --- a/binding/java/pom.xml +++ b/binding/java/pom.xml @@ -41,6 +41,7 @@ UTF-8 UTF-8 + 1.8 1.6 1.6 @@ -112,6 +113,15 @@ + + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + + diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java b/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java index 5c70266..e139d57 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/SearchTest.java @@ -8,7 +8,10 @@ package org.lionsoul.ip2region; import org.lionsoul.ip2region.xdb.Searcher; -import java.io.*; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; import java.util.concurrent.TimeUnit; public class SearchTest { @@ -65,7 +68,7 @@ public class SearchTest { } } - if (dbPath.length() < 1) { + if (dbPath.isEmpty()) { System.out.print("java -jar ip2region-{version}.jar search [command options]\n"); System.out.print("options:\n"); System.out.print(" --db string ip2region binary xdb file path\n"); @@ -73,32 +76,30 @@ public class SearchTest { return; } - Searcher searcher = createSearcher(dbPath, cachePolicy); - final BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); - System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy); - while ( true ) { - System.out.print("ip2region>> "); - String line = reader.readLine().trim(); - if ( line.length() < 2 ) { - continue; - } + try (Searcher searcher = createSearcher(dbPath, cachePolicy); + BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) { + System.out.printf("ip2region xdb searcher test program, cachePolicy: %s\ntype 'quit' to exit\n", cachePolicy); + while (true) { + System.out.print("ip2region>> "); + String line = reader.readLine().trim(); + if (line.length() < 2) { + continue; + } - if ( line.equalsIgnoreCase("quit") ) { - break; - } + if ("quit".equalsIgnoreCase(line)) { + break; + } - try { - double sTime = System.nanoTime(); - String region = searcher.search(line); - long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); - System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); - } catch (Exception e) { - System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount()); + try { + double sTime = System.nanoTime(); + String region = searcher.search(line); + long cost = TimeUnit.NANOSECONDS.toMicros((long) (System.nanoTime() - sTime)); + System.out.printf("{region: %s, ioCount: %d, took: %d μs}\n", region, searcher.getIOCount(), cost); + } catch (Exception e) { + System.out.printf("{err: %s, ioCount: %d}\n", e, searcher.getIOCount()); + } } } - - reader.close(); - searcher.close(); System.out.println("searcher test program exited, thanks for trying"); } @@ -121,19 +122,23 @@ public class SearchTest { String key = r.substring(2, sIdx); String val = r.substring(sIdx + 1); - if ("db".equals(key)) { - dbPath = val; - } else if ("src".equals(key)) { - srcPath = val; - } else if ("cache-policy".equals(key)) { - cachePolicy = val; - } else { - System.out.printf("undefined option `%s`\n", r); - return; + switch (key) { + case "db": + dbPath = val; + break; + case "src": + srcPath = val; + break; + case "cache-policy": + cachePolicy = val; + break; + default: + System.out.printf("undefined option `%s`\n", r); + return; } } - if (dbPath.length() < 1 || srcPath.length() < 1) { + if (dbPath.isEmpty() || srcPath.isEmpty()) { System.out.print("java -jar ip2region-{version}.jar bench [command options]\n"); System.out.print("options:\n"); System.out.print(" --db string ip2region binary xdb file path\n"); @@ -142,61 +147,59 @@ public class SearchTest { return; } - Searcher searcher = createSearcher(dbPath, cachePolicy); long count = 0, costs = 0, tStart = System.nanoTime(); - String line; - final BufferedReader reader = new BufferedReader(new FileReader(srcPath)); - while ((line = reader.readLine()) != null) { - String l = line.trim(); - String[] ps = l.split("\\|", 3); - if (ps.length != 3) { - System.out.printf("invalid ip segment `%s`\n", l); - return; - } - - long sip; - try { - sip = Searcher.checkIP(ps[0]); - } catch (Exception e) { - System.out.printf("check start ip `%s`: %s\n", ps[0], e); - return; - } - - long eip; - try { - eip = Searcher.checkIP(ps[1]); - } catch (Exception e) { - System.out.printf("check end ip `%s`: %s\n", ps[1], e); - return; - } - - if (sip > eip) { - System.out.printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]); - return; - } - - long mip = (sip + eip) >> 1; - for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) { - long sTime = System.nanoTime(); - String region = searcher.search(ip); - costs += System.nanoTime() - sTime; - - // check the region info - if (!ps[2].equals(region)) { - System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]); + try (Searcher searcher = createSearcher(dbPath, cachePolicy); + BufferedReader reader = new BufferedReader(new FileReader(srcPath))) { + String line; + while ((line = reader.readLine()) != null) { + String l = line.trim(); + String[] ps = l.split("\\|", 3); + if (ps.length != 3) { + System.out.printf("invalid ip segment `%s`\n", l); return; } - count++; + long sip; + try { + sip = Searcher.checkIP(ps[0]); + } catch (Exception e) { + System.out.printf("check start ip `%s`: %s\n", ps[0], e); + return; + } + + long eip; + try { + eip = Searcher.checkIP(ps[1]); + } catch (Exception e) { + System.out.printf("check end ip `%s`: %s\n", ps[1], e); + return; + } + + if (sip > eip) { + System.out.printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]); + return; + } + + long mip = (sip + eip) >> 1; + for (final long ip : new long[]{sip, (sip + mip) >> 1, mip, (mip + eip) >> 1, eip}) { + long sTime = System.nanoTime(); + String region = searcher.search(ip); + costs += System.nanoTime() - sTime; + + // check the region info + if (!ps[2].equals(region)) { + System.out.printf("failed search(%s) with (%s != %s)\n", Searcher.long2ip(ip), region, ps[2]); + return; + } + + count++; + } } } - - reader.close(); - searcher.close(); long took = System.nanoTime() - tStart; System.out.printf("Bench finished, {cachePolicy: %s, total: %d, took: %ds, cost: %d μs/op}\n", cachePolicy, count, TimeUnit.NANOSECONDS.toSeconds(took), - count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs/count)); + count == 0 ? 0 : TimeUnit.NANOSECONDS.toMicros(costs / count)); } public static void main(String[] args) { diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java b/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java index aba7779..06e806c 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/UtilTest.java @@ -12,7 +12,7 @@ public class UtilTest { public static void testIP2Long() { String ip = "1.2.3.4"; - long ipAddr = 0; + long ipAddr; try { ipAddr = Searcher.checkIP(ip); } catch (Exception e) { diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java index a55cedc..b38f918 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Header.java @@ -24,13 +24,14 @@ public class Header { buffer = buff; } - @Override public String toString() { + @Override + public String toString() { return "{" + - "Version: " + version + ',' + - "IndexPolicy: " + indexPolicy + ',' + - "CreatedAt: " + createdAt + ',' + - "StartIndexPtr: " + startIndexPtr + ',' + - "EndIndexPtr: " + endIndexPtr + - '}'; + "Version: " + version + ',' + + "IndexPolicy: " + indexPolicy + ',' + + "CreatedAt: " + createdAt + ',' + + "StartIndexPtr: " + startIndexPtr + ',' + + "EndIndexPtr: " + endIndexPtr + + '}'; } } diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java index f441036..67e111e 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java @@ -9,22 +9,21 @@ package org.lionsoul.ip2region.xdb; // @Date 2022/06/23 +import java.io.Closeable; import java.io.IOException; import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; -public class Searcher { +public class Searcher implements Closeable { // constant defined copied from the xdb maker public static final int HeaderInfoLength = 256; - public static final int VectorIndexRows = 256; - public static final int VectorIndexCols = 256; - public static final int VectorIndexSize = 8; + public static final int VectorIndexRows = 256; + public static final int VectorIndexCols = 256; + public static final int VectorIndexSize = 8; public static final int SegmentIndexSize = 14; - + public static final byte[] shiftIndex = {24, 16, 8, 0}; // random access file handle for file based search private final RandomAccessFile handle; - - private int ioCount = 0; - // vector index. // use the byte[] instead of VectorIndex entry array to keep // the minimal memory allocation. @@ -34,20 +33,7 @@ public class Searcher { private final byte[] contentBuff; // --- static method to create searchers - - public static Searcher newWithFileOnly(String dbPath) throws IOException { - return new Searcher(dbPath, null, null); - } - - public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException { - return new Searcher(dbPath, vectorIndex, null); - } - - public static Searcher newWithBuffer(byte[] cBuff) throws IOException { - return new Searcher(null, null, cBuff); - } - - // --- End of creator + private int ioCount = 0; public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException { if (cBuff != null) { @@ -61,6 +47,130 @@ public class Searcher { } } + public static Searcher newWithFileOnly(String dbPath) throws IOException { + return new Searcher(dbPath, null, null); + } + + // --- End of creator + + public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException { + return new Searcher(dbPath, vectorIndex, null); + } + + public static Searcher newWithBuffer(byte[] cBuff) throws IOException { + return new Searcher(null, null, cBuff); + } + + public static Header loadHeader(RandomAccessFile handle) throws IOException { + handle.seek(0); + final byte[] buff = new byte[HeaderInfoLength]; + handle.read(buff); + return new Header(buff); + } + + public static Header loadHeaderFromFile(String dbPath) throws IOException { + final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); + final Header header = loadHeader(handle); + handle.close(); + return header; + } + + public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException { + handle.seek(HeaderInfoLength); + int len = VectorIndexRows * VectorIndexCols * VectorIndexSize; + final byte[] buff = new byte[len]; + int rLen = handle.read(buff); + if (rLen != len) { + throw new IOException("incomplete read: read bytes should be " + len); + } + + return buff; + } + + public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException { + final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); + final byte[] vIndex = loadVectorIndex(handle); + handle.close(); + return vIndex; + } + + // --- static cache util function + + public static byte[] loadContent(RandomAccessFile handle) throws IOException { + handle.seek(0); + final byte[] buff = new byte[(int) handle.length()]; + int rLen = handle.read(buff); + if (rLen != buff.length) { + throw new IOException("incomplete read: read bytes should be " + buff.length); + } + + return buff; + } + + public static byte[] loadContentFromFile(String dbPath) throws IOException { + final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); + final byte[] content = loadContent(handle); + handle.close(); + return content; + } + + /* get an int from a byte array start from the specified offset */ + public static long getIntLong(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FFL)) | + ((b[offset++] << 8) & 0x0000FF00L) | + ((b[offset++] << 16) & 0x00FF0000L) | + ((b[offset] << 24) & 0xFF000000L) + ); + } + + public static int getInt(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FF)) | + ((b[offset++] << 8) & 0x0000FF00) | + ((b[offset++] << 16) & 0x00FF0000) | + ((b[offset] << 24) & 0xFF000000) + ); + } + + public static int getInt2(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FF)) | + ((b[offset] << 8) & 0x0000FF00) + ); + } + + /* long int to ip string */ + public static String long2ip(long ip) { + return String.valueOf((ip >> 24) & 0xFF) + '.' + + ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); + } + + // --- End cache load util function + + // --- static util method + + /* check the specified ip address */ + public static long checkIP(String ip) throws Exception { + String[] ps = ip.split("\\."); + if (ps.length != 4) { + throw new Exception("invalid ip address `" + ip + "`"); + } + + long ipDst = 0; + for (int i = 0; i < ps.length; i++) { + int val = Integer.parseInt(ps[i]); + if (val > 255) { + throw new Exception("ip part `" + ps[i] + "` should be less then 256"); + } + + ipDst |= ((long) val << shiftIndex[i]); + } + + return ipDst & 0xFFFFFFFFL; + } + + @Override public void close() throws IOException { if (this.handle != null) { this.handle.close(); @@ -135,7 +245,7 @@ public class Searcher { // load and return the region data final byte[] regionBuff = new byte[dataLen]; read(dataPtr, regionBuff); - return new String(regionBuff, "utf-8"); + return new String(regionBuff, StandardCharsets.UTF_8); } protected void read(int offset, byte[] buffer) throws IOException { @@ -157,116 +267,4 @@ public class Searcher { } } - // --- static cache util function - - public static Header loadHeader(RandomAccessFile handle) throws IOException { - handle.seek(0); - final byte[] buff = new byte[HeaderInfoLength]; - handle.read(buff); - return new Header(buff); - } - - public static Header loadHeaderFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final Header header = loadHeader(handle); - handle.close(); - return header; - } - - public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException { - handle.seek(HeaderInfoLength); - int len = VectorIndexRows * VectorIndexCols * VectorIndexSize; - final byte[] buff = new byte[len]; - int rLen = handle.read(buff); - if (rLen != len) { - throw new IOException("incomplete read: read bytes should be " + len); - } - - return buff; - } - - public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final byte[] vIndex = loadVectorIndex(handle); - handle.close(); - return vIndex; - } - - public static byte[] loadContent(RandomAccessFile handle) throws IOException { - handle.seek(0); - final byte[] buff = new byte[(int) handle.length()]; - int rLen = handle.read(buff); - if (rLen != buff.length) { - throw new IOException("incomplete read: read bytes should be " + buff.length); - } - - return buff; - } - - public static byte[] loadContentFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final byte[] content = loadContent(handle); - handle.close(); - return content; - } - - // --- End cache load util function - - // --- static util method - - /* get an int from a byte array start from the specified offset */ - public static long getIntLong(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FFL)) | - ((b[offset++] << 8) & 0x0000FF00L) | - ((b[offset++] << 16) & 0x00FF0000L) | - ((b[offset ] << 24) & 0xFF000000L) - ); - } - - public static int getInt(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FF)) | - ((b[offset++] << 8) & 0x0000FF00) | - ((b[offset++] << 16) & 0x00FF0000) | - ((b[offset ] << 24) & 0xFF000000) - ); - } - - public static int getInt2(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FF)) | - ((b[offset ] << 8) & 0x0000FF00) - ); - } - - /* long int to ip string */ - public static String long2ip( long ip ) - { - return String.valueOf((ip >> 24) & 0xFF) + '.' + - ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); - } - - public static final byte[] shiftIndex = {24, 16, 8, 0}; - - /* check the specified ip address */ - public static long checkIP(String ip) throws Exception { - String[] ps = ip.split("\\."); - if (ps.length != 4) { - throw new Exception("invalid ip address `" + ip + "`"); - } - - long ipDst = 0; - for (int i = 0; i < ps.length; i++) { - int val = Integer.parseInt(ps[i]); - if (val > 255) { - throw new Exception("ip part `"+ps[i]+"` should be less then 256"); - } - - ipDst |= ((long) val << shiftIndex[i]); - } - - return ipDst & 0xFFFFFFFFL; - } - -} \ No newline at end of file +} From 83f69936b8b28605eee0c16fa71060b1f2360eb4 Mon Sep 17 00:00:00 2001 From: jiahao <1250322798@qq.com> Date: Thu, 10 Aug 2023 14:46:44 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=8E=A8:=20=E6=96=B9=E6=B3=95=E9=A1=BA?= =?UTF-8?q?=E5=BA=8F=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/lionsoul/ip2region/xdb/Searcher.java | 258 +++++++++--------- 1 file changed, 130 insertions(+), 128 deletions(-) diff --git a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java index 67e111e..4912042 100644 --- a/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java +++ b/binding/java/src/main/java/org/lionsoul/ip2region/xdb/Searcher.java @@ -17,13 +17,16 @@ import java.nio.charset.StandardCharsets; public class Searcher implements Closeable { // constant defined copied from the xdb maker public static final int HeaderInfoLength = 256; - public static final int VectorIndexRows = 256; - public static final int VectorIndexCols = 256; - public static final int VectorIndexSize = 8; + public static final int VectorIndexRows = 256; + public static final int VectorIndexCols = 256; + public static final int VectorIndexSize = 8; public static final int SegmentIndexSize = 14; - public static final byte[] shiftIndex = {24, 16, 8, 0}; + // random access file handle for file based search private final RandomAccessFile handle; + + private int ioCount = 0; + // vector index. // use the byte[] instead of VectorIndex entry array to keep // the minimal memory allocation. @@ -33,7 +36,20 @@ public class Searcher implements Closeable { private final byte[] contentBuff; // --- static method to create searchers - private int ioCount = 0; + + public static Searcher newWithFileOnly(String dbPath) throws IOException { + return new Searcher(dbPath, null, null); + } + + public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException { + return new Searcher(dbPath, vectorIndex, null); + } + + public static Searcher newWithBuffer(byte[] cBuff) throws IOException { + return new Searcher(null, null, cBuff); + } + + // --- End of creator public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException { if (cBuff != null) { @@ -47,129 +63,6 @@ public class Searcher implements Closeable { } } - public static Searcher newWithFileOnly(String dbPath) throws IOException { - return new Searcher(dbPath, null, null); - } - - // --- End of creator - - public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException { - return new Searcher(dbPath, vectorIndex, null); - } - - public static Searcher newWithBuffer(byte[] cBuff) throws IOException { - return new Searcher(null, null, cBuff); - } - - public static Header loadHeader(RandomAccessFile handle) throws IOException { - handle.seek(0); - final byte[] buff = new byte[HeaderInfoLength]; - handle.read(buff); - return new Header(buff); - } - - public static Header loadHeaderFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final Header header = loadHeader(handle); - handle.close(); - return header; - } - - public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException { - handle.seek(HeaderInfoLength); - int len = VectorIndexRows * VectorIndexCols * VectorIndexSize; - final byte[] buff = new byte[len]; - int rLen = handle.read(buff); - if (rLen != len) { - throw new IOException("incomplete read: read bytes should be " + len); - } - - return buff; - } - - public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final byte[] vIndex = loadVectorIndex(handle); - handle.close(); - return vIndex; - } - - // --- static cache util function - - public static byte[] loadContent(RandomAccessFile handle) throws IOException { - handle.seek(0); - final byte[] buff = new byte[(int) handle.length()]; - int rLen = handle.read(buff); - if (rLen != buff.length) { - throw new IOException("incomplete read: read bytes should be " + buff.length); - } - - return buff; - } - - public static byte[] loadContentFromFile(String dbPath) throws IOException { - final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); - final byte[] content = loadContent(handle); - handle.close(); - return content; - } - - /* get an int from a byte array start from the specified offset */ - public static long getIntLong(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FFL)) | - ((b[offset++] << 8) & 0x0000FF00L) | - ((b[offset++] << 16) & 0x00FF0000L) | - ((b[offset] << 24) & 0xFF000000L) - ); - } - - public static int getInt(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FF)) | - ((b[offset++] << 8) & 0x0000FF00) | - ((b[offset++] << 16) & 0x00FF0000) | - ((b[offset] << 24) & 0xFF000000) - ); - } - - public static int getInt2(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FF)) | - ((b[offset] << 8) & 0x0000FF00) - ); - } - - /* long int to ip string */ - public static String long2ip(long ip) { - return String.valueOf((ip >> 24) & 0xFF) + '.' + - ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); - } - - // --- End cache load util function - - // --- static util method - - /* check the specified ip address */ - public static long checkIP(String ip) throws Exception { - String[] ps = ip.split("\\."); - if (ps.length != 4) { - throw new Exception("invalid ip address `" + ip + "`"); - } - - long ipDst = 0; - for (int i = 0; i < ps.length; i++) { - int val = Integer.parseInt(ps[i]); - if (val > 255) { - throw new Exception("ip part `" + ps[i] + "` should be less then 256"); - } - - ipDst |= ((long) val << shiftIndex[i]); - } - - return ipDst & 0xFFFFFFFFL; - } - @Override public void close() throws IOException { if (this.handle != null) { @@ -267,4 +160,113 @@ public class Searcher implements Closeable { } } + // --- static cache util function + + public static Header loadHeader(RandomAccessFile handle) throws IOException { + handle.seek(0); + final byte[] buff = new byte[HeaderInfoLength]; + handle.read(buff); + return new Header(buff); + } + + public static Header loadHeaderFromFile(String dbPath) throws IOException { + try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) { + return loadHeader(handle); + } + } + + public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException { + handle.seek(HeaderInfoLength); + int len = VectorIndexRows * VectorIndexCols * VectorIndexSize; + final byte[] buff = new byte[len]; + int rLen = handle.read(buff); + if (rLen != len) { + throw new IOException("incomplete read: read bytes should be " + len); + } + + return buff; + } + + public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException { + try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) { + return loadVectorIndex(handle); + } + } + + public static byte[] loadContent(RandomAccessFile handle) throws IOException { + handle.seek(0); + final byte[] buff = new byte[(int) handle.length()]; + int rLen = handle.read(buff); + if (rLen != buff.length) { + throw new IOException("incomplete read: read bytes should be " + buff.length); + } + + return buff; + } + + public static byte[] loadContentFromFile(String dbPath) throws IOException { + try (RandomAccessFile handle = new RandomAccessFile(dbPath, "r")) { + return loadContent(handle); + } + } + + // --- End cache load util function + + // --- static util method + + /* get an int from a byte array start from the specified offset */ + public static long getIntLong(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FFL)) | + ((b[offset++] << 8) & 0x0000FF00L) | + ((b[offset++] << 16) & 0x00FF0000L) | + ((b[offset ] << 24) & 0xFF000000L) + ); + } + + public static int getInt(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FF)) | + ((b[offset++] << 8) & 0x0000FF00) | + ((b[offset++] << 16) & 0x00FF0000) | + ((b[offset ] << 24) & 0xFF000000) + ); + } + + public static int getInt2(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FF)) | + ((b[offset ] << 8) & 0x0000FF00) + ); + } + + /* long int to ip string */ + public static String long2ip( long ip ) + { + return String.valueOf((ip >> 24) & 0xFF) + '.' + + ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); + } + + public static final byte[] shiftIndex = {24, 16, 8, 0}; + + /* check the specified ip address */ + public static long checkIP(String ip) throws Exception { + String[] ps = ip.split("\\."); + if (ps.length != 4) { + throw new Exception("invalid ip address `" + ip + "`"); + } + + long ipDst = 0; + for (int i = 0; i < ps.length; i++) { + int val = Integer.parseInt(ps[i]); + if (val > 255) { + throw new Exception("ip part `"+ps[i]+"` should be less then 256"); + } + + ipDst |= ((long) val << shiftIndex[i]); + } + + return ipDst & 0xFFFFFFFFL; + } + }