From 7f5ec337d08e063b34b3e20788a2bdce8449652b Mon Sep 17 00:00:00 2001 From: Alan Date: Wed, 26 Jul 2023 21:25:34 +0800 Subject: [PATCH] feat:support for concurrent file-based queries --- .../csharp/IP2Region.Net.BenchMark/Program.cs | 12 +- .../IP2Region.Net/Abstractions/ISearcher.cs | 20 +++ binding/csharp/IP2Region.Net/CHANGELOG.md | 16 +++ .../csharp/IP2Region.Net/IP2Region.Net.csproj | 19 +-- .../Abstractions/AbstractCacheStrategy.cs | 64 +++++++++ .../Internal/CacheStrategyFactory.cs | 31 ++++ .../Internal/ContentCacheStrategy.cs | 32 +++++ .../Internal/FileCacheStrategy.cs | 22 +++ .../Internal/VectorIndexCacheStrategy.cs | 26 ++++ .../csharp/IP2Region.Net/XDB/CachePolicy.cs | 6 +- binding/csharp/IP2Region.Net/XDB/ISearcher.cs | 14 -- binding/csharp/IP2Region.Net/XDB/Searcher.cs | 134 ++++-------------- binding/csharp/IP2Region.Net/XDB/Util.cs | 6 +- 13 files changed, 259 insertions(+), 143 deletions(-) create mode 100644 binding/csharp/IP2Region.Net/Abstractions/ISearcher.cs create mode 100644 binding/csharp/IP2Region.Net/CHANGELOG.md create mode 100644 binding/csharp/IP2Region.Net/Internal/Abstractions/AbstractCacheStrategy.cs create mode 100644 binding/csharp/IP2Region.Net/Internal/CacheStrategyFactory.cs create mode 100644 binding/csharp/IP2Region.Net/Internal/ContentCacheStrategy.cs create mode 100644 binding/csharp/IP2Region.Net/Internal/FileCacheStrategy.cs create mode 100644 binding/csharp/IP2Region.Net/Internal/VectorIndexCacheStrategy.cs delete mode 100644 binding/csharp/IP2Region.Net/XDB/ISearcher.cs diff --git a/binding/csharp/IP2Region.Net.BenchMark/Program.cs b/binding/csharp/IP2Region.Net.BenchMark/Program.cs index 5b3ebc7..c8c14e6 100644 --- a/binding/csharp/IP2Region.Net.BenchMark/Program.cs +++ b/binding/csharp/IP2Region.Net.BenchMark/Program.cs @@ -1,13 +1,16 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Running; +using IP2Region.Net.Abstractions; using IP2Region.Net.XDB; BenchmarkRunner.Run(typeof(Program).Assembly); public class CachePolicyCompare { - private readonly Searcher _contentSearcher = new Searcher(CachePolicy.Content); - private readonly Searcher _vectorSearcher = new Searcher(CachePolicy.VectorIndex); + private static readonly string XdbPath = Path.Combine(AppContext.BaseDirectory, "IP2Region", "ip2region.xdb"); + private readonly ISearcher _contentSearcher = new Searcher(CachePolicy.Content, XdbPath); + private readonly ISearcher _vectorSearcher = new Searcher(CachePolicy.VectorIndex,XdbPath); + private readonly ISearcher _fileSearcher = new Searcher(CachePolicy.File,XdbPath); private readonly string _testIpAddress = "114.114.114.114"; @@ -18,4 +21,9 @@ public class CachePolicyCompare [Benchmark] [BenchmarkCategory(nameof(CachePolicy.VectorIndex))] public void CachePolicy_VectorIndex() => _vectorSearcher.Search(_testIpAddress); + + + [Benchmark] + [BenchmarkCategory(nameof(CachePolicy.File))] + public void CachePolicy_File() => _fileSearcher.Search(_testIpAddress); } \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/Abstractions/ISearcher.cs b/binding/csharp/IP2Region.Net/Abstractions/ISearcher.cs new file mode 100644 index 0000000..ffd64b9 --- /dev/null +++ b/binding/csharp/IP2Region.Net/Abstractions/ISearcher.cs @@ -0,0 +1,20 @@ +// Copyright 2023 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// @Author Alan +// @Date 2023/07/25 + +using System.Net; + +namespace IP2Region.Net.Abstractions; + +public interface ISearcher +{ + string? Search(string ipStr); + + string? Search(IPAddress ipAddress); + + string? Search(uint ipAddress); + + int IoCount { get; } +} \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/CHANGELOG.md b/binding/csharp/IP2Region.Net/CHANGELOG.md new file mode 100644 index 0000000..59b9e5d --- /dev/null +++ b/binding/csharp/IP2Region.Net/CHANGELOG.md @@ -0,0 +1,16 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [Unreleased] + +## [2.0.0] - 2023-07-26 + +### Removed +- Remove nuget include xdb file +- Searcher cache policy default parameters +- Searcher xdb file path default parameters + +### Added +- Dependent file query policies CachePolicy.VectorIndex, CachePolicy.File support thread-safe concurrent queries +- Dramatically optimizes overall performance \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/IP2Region.Net.csproj b/binding/csharp/IP2Region.Net/IP2Region.Net.csproj index 5cefdb3..d3f7740 100644 --- a/binding/csharp/IP2Region.Net/IP2Region.Net.csproj +++ b/binding/csharp/IP2Region.Net/IP2Region.Net.csproj @@ -2,32 +2,23 @@ IP2Region.Net - 1.0.2 + 2.0.0 IP2Region.Net Alan Lee Apache-2.0 README.md https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp - ip2region 是一个离线IP地址定位库，极速响应，微秒级查询 - IP2Region - - + Please refer to CHANGELOG.md for details + .NET client library for ip2region + IP2Region GeoIP IPSearch + git enable enable net6.0;netstandard2.1 10.0 - - - - - - - PreserveNewest - - diff --git a/binding/csharp/IP2Region.Net/Internal/Abstractions/AbstractCacheStrategy.cs b/binding/csharp/IP2Region.Net/Internal/Abstractions/AbstractCacheStrategy.cs new file mode 100644 index 0000000..13d19e5 --- /dev/null +++ b/binding/csharp/IP2Region.Net/Internal/Abstractions/AbstractCacheStrategy.cs @@ -0,0 +1,64 @@ +// Copyright 2023 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// @Author Alan +// @Date 2023/07/25 + +using System.Buffers; + +namespace IP2Region.Net.Internal.Abstractions; + +internal abstract class AbstractCacheStrategy +{ + protected const int HeaderInfoLength = 256; + protected const int VectorIndexRows = 256; + protected const int VectorIndexCols = 256; + protected const int VectorIndexSize = 8; + + protected readonly FileStream XdbFileStream; + private const int BufferSize = 4096; + + internal int IoCount { get; private set; } + + protected AbstractCacheStrategy(string xdbPath) + { + XdbFileStream = new FileStream(xdbPath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize, + useAsync: true); + } + + protected int GetVectorIndexStartPos(uint ip) + { + var il0 = ip >> 24 & 0xFF; + var il1 = ip >> 16 & 0xFF; + var idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; + return (int)idx; + } + + internal abstract ReadOnlyMemory GetVectorIndex(uint ip); + + internal virtual ReadOnlyMemory GetData(int offset, int length) + { + byte[] buffer = ArrayPool.Shared.Rent(length); + int totalBytesRead = 0; + try + { + XdbFileStream.Seek(offset, SeekOrigin.Begin); + + int bytesRead; + do + { + int bytesToRead = Math.Min(BufferSize, length - totalBytesRead); + bytesRead = XdbFileStream.Read(buffer, totalBytesRead, bytesToRead); + totalBytesRead += bytesRead; + + IoCount++; + } while (bytesRead > 0 && totalBytesRead < length); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + + return new ReadOnlyMemory(buffer, 0, totalBytesRead); + } +} \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/Internal/CacheStrategyFactory.cs b/binding/csharp/IP2Region.Net/Internal/CacheStrategyFactory.cs new file mode 100644 index 0000000..14e8eb0 --- /dev/null +++ b/binding/csharp/IP2Region.Net/Internal/CacheStrategyFactory.cs @@ -0,0 +1,31 @@ +// Copyright 2023 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// @Author Alan +// @Date 2023/07/25 + +using IP2Region.Net.Internal.Abstractions; +using IP2Region.Net.XDB; + +namespace IP2Region.Net.Internal; + +internal class CacheStrategyFactory +{ + private readonly string _xdbPath; + + public CacheStrategyFactory(string xdbPath) + { + _xdbPath = xdbPath; + } + + public AbstractCacheStrategy CreateCacheStrategy(CachePolicy cachePolicy) + { + return cachePolicy switch + { + CachePolicy.Content => new ContentCacheStrategy(_xdbPath), + CachePolicy.VectorIndex => new VectorIndexCacheStrategy(_xdbPath), + CachePolicy.File => new FileCacheStrategy(_xdbPath), + _ => throw new ArgumentException(nameof(cachePolicy)) + }; + } +} \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/Internal/ContentCacheStrategy.cs b/binding/csharp/IP2Region.Net/Internal/ContentCacheStrategy.cs new file mode 100644 index 0000000..e3094e1 --- /dev/null +++ b/binding/csharp/IP2Region.Net/Internal/ContentCacheStrategy.cs @@ -0,0 +1,32 @@ +// Copyright 2023 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// @Author Alan +// @Date 2023/07/25 + +using IP2Region.Net.Internal.Abstractions; + +namespace IP2Region.Net.Internal; + +internal class ContentCacheStrategy : AbstractCacheStrategy +{ + private readonly ReadOnlyMemory _cacheData; + + public ContentCacheStrategy(string xdbPath) : base(xdbPath) + { + _cacheData = base.GetData(0, (int)XdbFileStream.Length); + XdbFileStream.Close(); + XdbFileStream.Dispose(); + } + + internal override ReadOnlyMemory GetVectorIndex(uint ip) + { + int idx = GetVectorIndexStartPos(ip); + return _cacheData.Slice(HeaderInfoLength + idx, VectorIndexSize); + } + + internal override ReadOnlyMemory GetData(int offset, int length) + { + return _cacheData.Slice(offset, length); + } +} \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/Internal/FileCacheStrategy.cs b/binding/csharp/IP2Region.Net/Internal/FileCacheStrategy.cs new file mode 100644 index 0000000..31f90f3 --- /dev/null +++ b/binding/csharp/IP2Region.Net/Internal/FileCacheStrategy.cs @@ -0,0 +1,22 @@ +// Copyright 2023 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// @Author Alan +// @Date 2023/07/25 + +using IP2Region.Net.Internal.Abstractions; + +namespace IP2Region.Net.Internal; + +internal class FileCacheStrategy : AbstractCacheStrategy +{ + public FileCacheStrategy(string xdbPath) : base(xdbPath) + { + } + + internal override ReadOnlyMemory GetVectorIndex(uint ip) + { + var idx = GetVectorIndexStartPos(ip); + return GetData(HeaderInfoLength + idx, VectorIndexSize); + } +} \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/Internal/VectorIndexCacheStrategy.cs b/binding/csharp/IP2Region.Net/Internal/VectorIndexCacheStrategy.cs new file mode 100644 index 0000000..f1d8ae6 --- /dev/null +++ b/binding/csharp/IP2Region.Net/Internal/VectorIndexCacheStrategy.cs @@ -0,0 +1,26 @@ +// Copyright 2023 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. +// @Author Alan +// @Date 2023/07/25 + +using IP2Region.Net.Internal.Abstractions; + +namespace IP2Region.Net.Internal; + +internal class VectorIndexCacheStrategy : AbstractCacheStrategy +{ + private readonly ReadOnlyMemory _vectorIndex; + + public VectorIndexCacheStrategy(string xdbPath) : base(xdbPath) + { + var vectorLength = VectorIndexRows * VectorIndexCols * VectorIndexSize; + _vectorIndex = base.GetData(HeaderInfoLength, vectorLength); + } + + internal override ReadOnlyMemory GetVectorIndex(uint ip) + { + var idx = GetVectorIndexStartPos(ip); + return _vectorIndex.Slice(idx, VectorIndexSize); + } +} \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/XDB/CachePolicy.cs b/binding/csharp/IP2Region.Net/XDB/CachePolicy.cs index e84560a..0f4d035 100644 --- a/binding/csharp/IP2Region.Net/XDB/CachePolicy.cs +++ b/binding/csharp/IP2Region.Net/XDB/CachePolicy.cs @@ -3,15 +3,15 @@ namespace IP2Region.Net.XDB; public enum CachePolicy { /// - /// no cache , not thread safe! + /// no cache /// File, /// - /// cache vector index , reduce the number of IO operations , not thread safe! + /// cache vector index , reduce the number of IO operations /// VectorIndex, /// - /// default cache policy , cache whole xdb file , thread safe + /// default cache policy , cache whole xdb file /// Content } \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/XDB/ISearcher.cs b/binding/csharp/IP2Region.Net/XDB/ISearcher.cs deleted file mode 100644 index ce34759..0000000 --- a/binding/csharp/IP2Region.Net/XDB/ISearcher.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.Net; - -namespace IP2Region.Net.XDB; - -public interface ISearcher -{ - string? Search(string ipStr); - - string? Search(IPAddress ipAddress); - - string? Search(uint ipAddress); - - int IoCount { get; } -} \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/XDB/Searcher.cs b/binding/csharp/IP2Region.Net/XDB/Searcher.cs index b90088c..d6bcc1d 100644 --- a/binding/csharp/IP2Region.Net/XDB/Searcher.cs +++ b/binding/csharp/IP2Region.Net/XDB/Searcher.cs @@ -1,60 +1,29 @@ -// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Copyright 2023 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. // @Author Alan Lee -// @Date 2022/09/06 +// @Date 2023/07/23 using System.Net; +using System.Runtime.InteropServices; using System.Text; +using IP2Region.Net.Abstractions; +using IP2Region.Net.Internal; +using IP2Region.Net.Internal.Abstractions; namespace IP2Region.Net.XDB; public class Searcher : ISearcher { - const int HeaderInfoLength = 256; - const int VectorIndexRows = 256; - const int VectorIndexCols = 256; - const int VectorIndexSize = 8; const int SegmentIndexSize = 14; - private readonly byte[]? _vectorIndex; - private readonly byte[]? _contentBuff; - private readonly FileStream _contentStream; - private readonly CachePolicy _cachePolicy; - public int IoCount { get; private set; } + private readonly AbstractCacheStrategy _cacheStrategy; + public int IoCount => _cacheStrategy.IoCount; - public Searcher(CachePolicy cachePolicy = CachePolicy.Content, string? dbPath = null) + public Searcher(CachePolicy cachePolicy, string dbPath) { - if (string.IsNullOrEmpty(dbPath)) - { - dbPath = Path.Combine(AppContext.BaseDirectory, "Data", "ip2region.xdb"); - } - - _contentStream = File.OpenRead(dbPath); - _cachePolicy = cachePolicy; - - switch (_cachePolicy) - { - case CachePolicy.Content: - using (var stream = new MemoryStream()) - { - _contentStream.CopyTo(stream); - _contentBuff = stream.ToArray(); - } - - break; - case CachePolicy.VectorIndex: - var vectorLength = VectorIndexRows * VectorIndexCols * VectorIndexSize; - _vectorIndex = new byte[vectorLength]; - Read(HeaderInfoLength, _vectorIndex); - break; - } - } - - ~Searcher() - { - _contentStream.Close(); - _contentStream.Dispose(); + var factory = new CacheStrategyFactory(dbPath); + _cacheStrategy = factory.CreateCacheStrategy(cachePolicy); } public string? Search(string ipStr) @@ -71,62 +40,37 @@ public class Searcher : ISearcher public string? Search(uint ip) { - var il0 = ip >> 24 & 0xFF; - var il1 = ip >> 16 & 0xFF; - var idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; - - uint sPtr = 0, ePtr = 0; - - switch (_cachePolicy) - { - case CachePolicy.VectorIndex: - sPtr = BitConverter.ToUInt32(_vectorIndex.AsSpan()[(int)idx..]); - ePtr = BitConverter.ToUInt32(_vectorIndex.AsSpan()[((int)idx + 4)..]); - break; - case CachePolicy.Content: - sPtr = BitConverter.ToUInt32(_contentBuff.AsSpan()[(HeaderInfoLength + (int)idx)..]); - ePtr = BitConverter.ToUInt32(_contentBuff.AsSpan()[(HeaderInfoLength + (int)idx + 4)..]); - break; - case CachePolicy.File: - var buff = new byte[VectorIndexSize]; - Read((int)(idx + HeaderInfoLength), buff); - sPtr = BitConverter.ToUInt32(buff); - ePtr = BitConverter.ToUInt32(buff.AsSpan()[4..]); - break; - } - + var index = _cacheStrategy.GetVectorIndex(ip); + uint sPtr = MemoryMarshal.Read(index.Span); + uint ePtr = MemoryMarshal.Read(index.Span[4..]); var dataLen = 0; uint dataPtr = 0; - var l = 0; - var h = (int)((ePtr - sPtr) / SegmentIndexSize); - var buffer = new byte[SegmentIndexSize]; + uint l = 0; + uint h = (ePtr -sPtr) / SegmentIndexSize; while (l <= h) { var mid = Util.GetMidIp(l, h); var pos = sPtr + mid * SegmentIndexSize; - Read((int)pos, buffer); - var sip = BitConverter.ToUInt32(buffer); + var buffer = _cacheStrategy.GetData((int)pos, SegmentIndexSize); + uint sip = MemoryMarshal.Read(buffer.Span); + uint eip = MemoryMarshal.Read(buffer.Span[4..]); if (ip < sip) { h = mid - 1; } + else if (ip > eip) + { + l = mid + 1; + } else { - var eip = BitConverter.ToUInt32(buffer.AsSpan()[4..]); - if (ip > eip) - { - l = mid + 1; - } - else - { - dataLen = BitConverter.ToUInt16(buffer.AsSpan()[8..]); - dataPtr = BitConverter.ToUInt32(buffer.AsSpan()[10..]); - break; - } + dataLen = MemoryMarshal.Read(buffer.Span[8..]); + dataPtr = MemoryMarshal.Read(buffer.Span[10..]); + break; } } @@ -135,29 +79,7 @@ public class Searcher : ISearcher return default; } - var regionBuff = new byte[dataLen]; - Read((int)dataPtr, regionBuff); - return Encoding.UTF8.GetString(regionBuff); - } - - private void Read(int offset, byte[] buff) - { - switch (_cachePolicy) - { - case CachePolicy.Content: - _contentBuff.AsSpan()[offset..(offset + buff.Length)].CopyTo(buff); - break; - default: - _contentStream.Seek(offset, SeekOrigin.Begin); - IoCount++; - - var rLen = _contentStream.Read(buff); - if (rLen != buff.Length) - { - throw new IOException($"incomplete read: readed bytes should be {buff.Length}"); - } - - break; - } + var regionBuff = _cacheStrategy.GetData((int)dataPtr,dataLen); + return Encoding.UTF8.GetString(regionBuff.Span); } } \ No newline at end of file diff --git a/binding/csharp/IP2Region.Net/XDB/Util.cs b/binding/csharp/IP2Region.Net/XDB/Util.cs index a3581e0..1075377 100644 --- a/binding/csharp/IP2Region.Net/XDB/Util.cs +++ b/binding/csharp/IP2Region.Net/XDB/Util.cs @@ -1,4 +1,5 @@ using System.Net; +using System.Runtime.InteropServices; namespace IP2Region.Net.XDB; @@ -14,12 +15,9 @@ public static class Util { byte[] bytes = ipAddress.GetAddressBytes(); Array.Reverse(bytes); - return BitConverter.ToUInt32(bytes, 0); + return MemoryMarshal.Read(bytes); } public static uint GetMidIp(uint x, uint y) => (x & y) + ((x ^ y) >> 1); - - public static int GetMidIp(int x, int y) - => (x & y) + ((x ^ y) >> 1); } \ No newline at end of file