From e13f9b4a56543b53d51ce87baaa29d7d0c865507 Mon Sep 17 00:00:00 2001 From: Killer Date: Wed, 10 Jun 2020 18:09:57 +0800 Subject: [PATCH] add DbSearcher --- maker/csharp/DbMaker/DbSearcher.cs | 462 +++++++++++++++++++++++++++++ maker/csharp/DbMaker/Program.cs | 1 - maker/csharp/DbMaker/Util.cs | 12 +- 3 files changed, 470 insertions(+), 5 deletions(-) create mode 100644 maker/csharp/DbMaker/DbSearcher.cs diff --git a/maker/csharp/DbMaker/DbSearcher.cs b/maker/csharp/DbMaker/DbSearcher.cs new file mode 100644 index 0000000..d60ad76 --- /dev/null +++ b/maker/csharp/DbMaker/DbSearcher.cs @@ -0,0 +1,462 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace DbMaker +{ + /// + /// ip db searcher class (Not thread safe) + /// + public class DbSearcher + { + public const int BTREE_ALGORITHM = 1; + public const int BINARY_ALGORITHM = 2; + public const int MEMORY_ALGORITYM = 3; + + /** + * db config + */ + private DbConfig dbConfig = null; + + /** + * db file access handler + */ + private Stream raf = null; + + /** + * header blocks buffer + */ + private long[] HeaderSip = null; + private int[] HeaderPtr = null; + private int headerLength; + + /** + * super blocks info + */ + private long firstIndexPtr = 0; + private long lastIndexPtr = 0; + private int totalIndexBlocks = 0; + + /** + * for memory mode + * the original db binary string + */ + private byte[] dbBinStr = null; + + /** + * construct class + * + * @param bdConfig + * @param dbFile + * @throws FileNotFoundException + */ + public DbSearcher(DbConfig dbConfig, String dbFile) + { + this.dbConfig = dbConfig; + raf = File.OpenRead(dbFile); //new RandomAccessFile(dbFile, "r"); + } + + /** + * construct method with self-define std ip2region bianry string support + * Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL + * + * @param dbConfig + * @param dbBinStr + */ + public DbSearcher(DbConfig dbConfig, byte[] dbBinStr) + { + this.dbConfig = dbConfig; + this.dbBinStr = dbBinStr; + + firstIndexPtr = Util.getIntLong(dbBinStr, 0); + lastIndexPtr = Util.getIntLong(dbBinStr, 4); + totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / IndexBlock.GetIndexBlockLength()) + 1; + } + + /** + * get the region with a int ip address with memory binary search algorithm + * + * @param ip + * @throws IOException + */ + public DataBlock memorySearch(long ip) + { + int blen = IndexBlock.GetIndexBlockLength(); + if (dbBinStr == null) + { + dbBinStr = new byte[(int)raf.Length]; + raf.Seek(0L, SeekOrigin.Begin); + //raf.readFully(dbBinStr, 0, dbBinStr.length); + raf.Read(dbBinStr, 0, dbBinStr.Length); + //initialize the global vars + firstIndexPtr = Util.getIntLong(dbBinStr, 0); + lastIndexPtr = Util.getIntLong(dbBinStr, 4); + totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1; + } + + //search the index blocks to define the data + int l = 0, h = totalIndexBlocks; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + int p = (int)(firstIndexPtr + m * blen); + + sip = Util.getIntLong(dbBinStr, p); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(dbBinStr, p + 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(dbBinStr, p + 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //get the data + int dataLen = (int)((dataptr >> 24) & 0xFF); + int dataPtr = (int)((dataptr & 0x00FFFFFF)); + int city_id = (int)Util.getIntLong(dbBinStr, dataPtr); + //String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8"); + var region = Encoding.UTF8.GetString(dbBinStr, dataPtr + 4, dataLen - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region throught the ip address with memory binary search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException +*/ + public DataBlock memorySearch(String ip) + { + return memorySearch(Util.ip2long(ip)); + } + + + /** + * get by index ptr + * + * @param indexPtr + * @throws IOException +*/ + public DataBlock getByIndexPtr(long ptr) + { + raf.Seek(ptr, SeekOrigin.Begin); + byte[] + buffer = new byte[12]; + //raf.readFully(buffer, 0, buffer.length); + raf.Read(buffer, 0, buffer.Length); + //long startIp = Util.getIntLong(buffer, 0); + //long endIp = Util.getIntLong(buffer, 4); + long extra = Util.getIntLong(buffer, 8); + + int dataLen = (int)((extra >> 24) & 0xFF); + int dataPtr = (int)((extra & 0x00FFFFFF)); + + raf.Seek(dataPtr, SeekOrigin.Begin); + byte[] data = new byte[dataLen]; + //raf.readFully(data, 0, data.length); + raf.Read(data, 0, data.Length); + int city_id = (int)Util.getIntLong(data, 0); + //String region = new String(data, 4, data.length - 4, "UTF-8"); + String region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region with a int ip address with b-tree algorithm + * + * @param ip + * @throws IOException +*/ + public DataBlock btreeSearch(long ip) + { + //check and load the header + if (HeaderSip == null) + { + //raf.seek(8L); //pass the super block + raf.Seek(8, SeekOrigin.Begin); + byte[] b = new byte[dbConfig.TotalHeaderSize]; + // byte[] b = new byte[4096]; + //raf.readFully(b, 0, b.length); + raf.Read(b, 0, b.Length); + + //fill the header + int len = b.Length >> 3, idx = 0; //b.lenght / 8 + HeaderSip = new long[len]; + HeaderPtr = new int[len]; + long startIp; + long xDataPtr = 0; + for (int i = 0; i < b.Length; i += 8) + { + startIp = Util.getIntLong(b, i); + xDataPtr = Util.getIntLong(b, i + 4); + if (xDataPtr == 0) break; + + HeaderSip[idx] = startIp; + HeaderPtr[idx] = (int)xDataPtr; + idx++; + } + + headerLength = idx; + } + + //1. define the index block with the binary search + if (ip == HeaderSip[0]) + { + return getByIndexPtr(HeaderPtr[0]); + } + else if (ip == HeaderSip[headerLength - 1]) + { + return getByIndexPtr(HeaderPtr[headerLength - 1]); + } + + int l = 0, h = headerLength, sptr = 0, eptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + + //perfetc matched, just return it + if (ip == HeaderSip[m]) + { + if (m > 0) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + } + else + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + } + + break; + } + + //less then the middle value + if (ip < HeaderSip[m]) + { + if (m == 0) + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + break; + } + else if (ip > HeaderSip[m - 1]) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + break; + } + h = m - 1; + } + else + { + if (m == headerLength - 1) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + break; + } + else if (ip <= HeaderSip[m + 1]) + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + break; + } + l = m + 1; + } + } + + //match nothing just stop it + if (sptr == 0) return null; + + //2. search the index blocks to define the data + int blockLen = eptr - sptr, blen = IndexBlock.GetIndexBlockLength(); + byte[] + iBuffer = new byte[blockLen + blen]; //include the right border block + //raf.seek(sptr); + //raf.readFully(iBuffer, 0, iBuffer.length); + raf.Seek(sptr, SeekOrigin.Begin); + raf.Read(iBuffer, 0, iBuffer.Length); + + l = 0; h = blockLen / blen; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + int p = m * blen; + sip = Util.getIntLong(iBuffer, p); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(iBuffer, p + 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(iBuffer, p + 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //3. get the data + int dataLen = (int)((dataptr >> 24) & 0xFF); + int dataPtr = (int)((dataptr & 0x00FFFFFF)); + + //raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + // raf.readFully(data, 0, data.length); + raf.Seek(dataPtr, SeekOrigin.Begin); + raf.Read(data, 0, data.Length); + int city_id = (int)Util.getIntLong(data, 0); + //String region = new String(data, 4, data.length - 4, "UTF-8"); + var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region throught the ip address with b-tree search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock btreeSearch(String ip) + { + return btreeSearch(Util.ip2long(ip)); + } + + /** + * get the region with a int ip address with binary search algorithm + * + * @param ip + * @throws IOException +*/ + public DataBlock binarySearch(long ip) + { + int blen = IndexBlock.GetIndexBlockLength(); + if (totalIndexBlocks == 0) + { + //raf.seek(0L); + byte[] superBytes = new byte[8]; + //raf.readFully(superBytes, 0, superBytes.length); + raf.Seek(0, SeekOrigin.Begin); + raf.Read(superBytes, 0, superBytes.Length); + //initialize the global vars + firstIndexPtr = Util.getIntLong(superBytes, 0); + lastIndexPtr = Util.getIntLong(superBytes, 4); + totalIndexBlocks = (int)((lastIndexPtr - firstIndexPtr) / blen) + 1; + } + + //search the index blocks to define the data + int l = 0, h = totalIndexBlocks; + byte[] + buffer = new byte[blen]; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + //raf.seek(firstIndexPtr + m * blen); //set the file pointer + //raf.readFully(buffer, 0, buffer.length); + raf.Seek(firstIndexPtr + m * blen, SeekOrigin.Begin); + raf.Read(buffer, 0, buffer.Length); + sip = Util.getIntLong(buffer, 0); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(buffer, 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(buffer, 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //get the data + int dataLen = (int)((dataptr >> 24) & 0xFF); + int dataPtr = (int)((dataptr & 0x00FFFFFF)); + + //raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + //raf.readFully(data, 0, data.length); + raf.Seek(dataPtr, SeekOrigin.Begin); + raf.Read(data, 0, data.Length); + + int city_id = (int)Util.getIntLong(data, 0); + //String region = new String(data, 4, data.length - 4, "UTF-8"); + var region = Encoding.UTF8.GetString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region throught the ip address with binary search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock binarySearch(String ip) + { + return binarySearch(Util.ip2long(ip)); + } + + /** + * get the db config + * + * @return DbConfig +*/ + public DbConfig getDbConfig() + { + return dbConfig; + } + + /** + * close the db + * + * @throws IOException +*/ + public void close() + { + HeaderSip = null; //let gc do its work + HeaderPtr = null; + dbBinStr = null; + raf.Close(); + } + } +} diff --git a/maker/csharp/DbMaker/Program.cs b/maker/csharp/DbMaker/Program.cs index 97138e2..e6d0dd7 100644 --- a/maker/csharp/DbMaker/Program.cs +++ b/maker/csharp/DbMaker/Program.cs @@ -6,7 +6,6 @@ namespace DbMaker { static void Main(string[] args) { - Console.WriteLine("Hello World!"); } } } diff --git a/maker/csharp/DbMaker/Util.cs b/maker/csharp/DbMaker/Util.cs index 60e493e..54f6e3a 100644 --- a/maker/csharp/DbMaker/Util.cs +++ b/maker/csharp/DbMaker/Util.cs @@ -95,10 +95,14 @@ namespace DbMaker */ public static long ip2long(String ip) { - IPAddress ipaddress = IPAddress.Parse(ip); - byte[] addbuffer = ipaddress.GetAddressBytes(); - Array.Reverse(addbuffer); - return BitConverter.ToInt64(addbuffer, 0); + ip = ip.Trim(); + String[] ips = ip.Split('.'); + long ip1 = Int64.Parse(ips[0]); + long ip2 = Int64.Parse(ips[1]); + long ip3 = Int64.Parse(ips[2]); + long ip4 = Int64.Parse(ips[3]); + long ip2long = 1L * ip1 * 256 * 256 * 256 + ip2 * 256 * 256 + ip3 * 256 + ip4; + return ip2long; } /**