Merge pull request #310 from malus2077/master
c# client support VectorIndex, File concurrent queries
This commit is contained in:
commit
a72a6b8282
|
|
@ -16,4 +16,11 @@
|
||||||
<ProjectReference Include="..\IP2Region.Net\IP2Region.Net.csproj" />
|
<ProjectReference Include="..\IP2Region.Net\IP2Region.Net.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<Content Include="..\..\..\data\ip2region.xdb">
|
||||||
|
<Link>IP2Region/ip2region.xdb</Link>
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</Content>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,16 @@
|
||||||
using BenchmarkDotNet.Attributes;
|
using BenchmarkDotNet.Attributes;
|
||||||
using BenchmarkDotNet.Running;
|
using BenchmarkDotNet.Running;
|
||||||
|
using IP2Region.Net.Abstractions;
|
||||||
using IP2Region.Net.XDB;
|
using IP2Region.Net.XDB;
|
||||||
|
|
||||||
BenchmarkRunner.Run(typeof(Program).Assembly);
|
BenchmarkRunner.Run(typeof(Program).Assembly);
|
||||||
|
|
||||||
public class CachePolicyCompare
|
public class CachePolicyCompare
|
||||||
{
|
{
|
||||||
private readonly Searcher _contentSearcher = new Searcher(CachePolicy.Content);
|
private static readonly string XdbPath = Path.Combine(AppContext.BaseDirectory, "IP2Region", "ip2region.xdb");
|
||||||
private readonly Searcher _vectorSearcher = new Searcher(CachePolicy.VectorIndex);
|
private readonly ISearcher _contentSearcher = new Searcher(CachePolicy.Content, XdbPath);
|
||||||
|
private readonly ISearcher _vectorSearcher = new Searcher(CachePolicy.VectorIndex,XdbPath);
|
||||||
|
private readonly ISearcher _fileSearcher = new Searcher(CachePolicy.File,XdbPath);
|
||||||
|
|
||||||
private readonly string _testIpAddress = "114.114.114.114";
|
private readonly string _testIpAddress = "114.114.114.114";
|
||||||
|
|
||||||
|
|
@ -18,4 +21,9 @@ public class CachePolicyCompare
|
||||||
[Benchmark]
|
[Benchmark]
|
||||||
[BenchmarkCategory(nameof(CachePolicy.VectorIndex))]
|
[BenchmarkCategory(nameof(CachePolicy.VectorIndex))]
|
||||||
public void CachePolicy_VectorIndex() => _vectorSearcher.Search(_testIpAddress);
|
public void CachePolicy_VectorIndex() => _vectorSearcher.Search(_testIpAddress);
|
||||||
|
|
||||||
|
|
||||||
|
[Benchmark]
|
||||||
|
[BenchmarkCategory(nameof(CachePolicy.File))]
|
||||||
|
public void CachePolicy_File() => _fileSearcher.Search(_testIpAddress);
|
||||||
}
|
}
|
||||||
|
|
@ -21,13 +21,14 @@
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Folder Include="TestData" />
|
<Content Include="..\..\..\data\ip.merge.txt">
|
||||||
</ItemGroup>
|
<Link>TestData/ip.merge.txt</Link>
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<None Update="TestData\ip.merge.txt">
|
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
</None>
|
</Content>
|
||||||
|
<Content Include="..\..\..\data\ip2region.xdb">
|
||||||
|
<Link>TestData/ip2region.xdb</Link>
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</Content>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,8 @@ namespace IP2Region.Net.Test;
|
||||||
[TestFixture]
|
[TestFixture]
|
||||||
public class SearcherTest
|
public class SearcherTest
|
||||||
{
|
{
|
||||||
|
private readonly string _xdbPath = Path.Combine(AppContext.BaseDirectory, "TestData", "ip2region.xdb");
|
||||||
|
|
||||||
public static IEnumerable<string> Ips()
|
public static IEnumerable<string> Ips()
|
||||||
{
|
{
|
||||||
yield return "114.114.114.114";
|
yield return "114.114.114.114";
|
||||||
|
|
@ -15,25 +17,28 @@ public class SearcherTest
|
||||||
}
|
}
|
||||||
|
|
||||||
[TestCaseSource(nameof(Ips))]
|
[TestCaseSource(nameof(Ips))]
|
||||||
|
[Parallelizable(ParallelScope.All)]
|
||||||
public void TestSearchCacheContent(string ip)
|
public void TestSearchCacheContent(string ip)
|
||||||
{
|
{
|
||||||
var contentSearcher = new Searcher(CachePolicy.Content);
|
var contentSearcher = new Searcher(CachePolicy.Content,_xdbPath);
|
||||||
var region = contentSearcher.Search(ip);
|
var region = contentSearcher.Search(ip);
|
||||||
Console.WriteLine(region);
|
Console.WriteLine(region);
|
||||||
}
|
}
|
||||||
|
|
||||||
[TestCaseSource(nameof(Ips))]
|
[TestCaseSource(nameof(Ips))]
|
||||||
|
[Parallelizable(ParallelScope.All)]
|
||||||
public void TestSearchCacheVector(string ip)
|
public void TestSearchCacheVector(string ip)
|
||||||
{
|
{
|
||||||
var vectorSearcher = new Searcher(CachePolicy.VectorIndex);
|
var vectorSearcher = new Searcher(CachePolicy.VectorIndex,_xdbPath);
|
||||||
var region = vectorSearcher.Search(ip);
|
var region = vectorSearcher.Search(ip);
|
||||||
Console.WriteLine(region);
|
Console.WriteLine(region);
|
||||||
}
|
}
|
||||||
|
|
||||||
[TestCaseSource(nameof(Ips))]
|
[TestCaseSource(nameof(Ips))]
|
||||||
|
[Parallelizable(ParallelScope.All)]
|
||||||
public void TestSearchCacheFile(string ip)
|
public void TestSearchCacheFile(string ip)
|
||||||
{
|
{
|
||||||
var fileSearcher = new Searcher(CachePolicy.File);
|
var fileSearcher = new Searcher(CachePolicy.File,_xdbPath);
|
||||||
var region = fileSearcher.Search(ip);
|
var region = fileSearcher.Search(ip);
|
||||||
Console.WriteLine(region);
|
Console.WriteLine(region);
|
||||||
}
|
}
|
||||||
|
|
@ -43,7 +48,7 @@ public class SearcherTest
|
||||||
[TestCase(CachePolicy.File)]
|
[TestCase(CachePolicy.File)]
|
||||||
public void TestBenchSearch(CachePolicy cachePolicy)
|
public void TestBenchSearch(CachePolicy cachePolicy)
|
||||||
{
|
{
|
||||||
Searcher searcher = new Searcher(cachePolicy);
|
Searcher searcher = new Searcher(cachePolicy,_xdbPath);
|
||||||
var srcPath = Path.Combine(AppContext.BaseDirectory, "TestData", "ip.merge.txt");
|
var srcPath = Path.Combine(AppContext.BaseDirectory, "TestData", "ip.merge.txt");
|
||||||
|
|
||||||
foreach (var line in File.ReadLines(srcPath))
|
foreach (var line in File.ReadLines(srcPath))
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,20 @@
|
||||||
|
// Copyright 2023 The Ip2Region Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a Apache2.0-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
// @Author Alan <lzh.shap@gmail.com>
|
||||||
|
// @Date 2023/07/25
|
||||||
|
|
||||||
|
using System.Net;
|
||||||
|
|
||||||
|
namespace IP2Region.Net.Abstractions;
|
||||||
|
|
||||||
|
public interface ISearcher
|
||||||
|
{
|
||||||
|
string? Search(string ipStr);
|
||||||
|
|
||||||
|
string? Search(IPAddress ipAddress);
|
||||||
|
|
||||||
|
string? Search(uint ipAddress);
|
||||||
|
|
||||||
|
int IoCount { get; }
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,16 @@
|
||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [2.0.0] - 2023-07-26
|
||||||
|
|
||||||
|
### Removed
|
||||||
|
- Remove nuget include xdb file
|
||||||
|
- Searcher cache policy default parameters
|
||||||
|
- Searcher xdb file path default parameters
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Dependent file query policies CachePolicy.VectorIndex, CachePolicy.File support thread-safe concurrent queries
|
||||||
|
- Dramatically optimizes overall performance
|
||||||
|
|
@ -2,32 +2,23 @@
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<id>IP2Region.Net</id>
|
<id>IP2Region.Net</id>
|
||||||
<version>1.0.2</version>
|
<version>2.0.0</version>
|
||||||
<title>IP2Region.Net</title>
|
<title>IP2Region.Net</title>
|
||||||
<authors>Alan Lee</authors>
|
<authors>Alan Lee</authors>
|
||||||
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
|
<PackageLicenseExpression>Apache-2.0</PackageLicenseExpression>
|
||||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||||
<PackageProjectUrl>https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp</PackageProjectUrl>
|
<PackageProjectUrl>https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp</PackageProjectUrl>
|
||||||
<RepositoryUrl>https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp</RepositoryUrl>
|
<RepositoryUrl>https://github.com/lionsoul2014/ip2region/tree/master/binding/csharp</RepositoryUrl>
|
||||||
<Description>ip2region 是一个离线IP地址定位库,极速响应,微秒级查询</Description>
|
<PackageReleaseNotes>Please refer to CHANGELOG.md for details</PackageReleaseNotes>
|
||||||
<tags>IP2Region</tags>
|
<Description>.NET client library for ip2region</Description>
|
||||||
|
<PackageTags>IP2Region GeoIP IPSearch</PackageTags>
|
||||||
|
<RepositoryType>git</RepositoryType>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<TargetFrameworks>net6.0;netstandard2.1</TargetFrameworks>
|
<TargetFrameworks>net6.0;netstandard2.1</TargetFrameworks>
|
||||||
<LangVersion>10.0</LangVersion>
|
<LangVersion>10.0</LangVersion>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<Folder Include="Data" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<None Update="Data\ip2region.xdb">
|
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
|
||||||
</None>
|
|
||||||
<None Include="..\README.md" Pack="true" PackagePath="\" />
|
<None Include="..\README.md" Pack="true" PackagePath="\" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
// Copyright 2023 The Ip2Region Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a Apache2.0-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
// @Author Alan <lzh.shap@gmail.com>
|
||||||
|
// @Date 2023/07/25
|
||||||
|
|
||||||
|
using System.Buffers;
|
||||||
|
|
||||||
|
namespace IP2Region.Net.Internal.Abstractions;
|
||||||
|
|
||||||
|
internal abstract class AbstractCacheStrategy
|
||||||
|
{
|
||||||
|
protected const int HeaderInfoLength = 256;
|
||||||
|
protected const int VectorIndexRows = 256;
|
||||||
|
protected const int VectorIndexCols = 256;
|
||||||
|
protected const int VectorIndexSize = 8;
|
||||||
|
|
||||||
|
protected readonly FileStream XdbFileStream;
|
||||||
|
private const int BufferSize = 4096;
|
||||||
|
|
||||||
|
internal int IoCount { get; private set; }
|
||||||
|
|
||||||
|
protected AbstractCacheStrategy(string xdbPath)
|
||||||
|
{
|
||||||
|
XdbFileStream = new FileStream(xdbPath, FileMode.Open, FileAccess.Read, FileShare.Read, BufferSize,
|
||||||
|
useAsync: true);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected int GetVectorIndexStartPos(uint ip)
|
||||||
|
{
|
||||||
|
var il0 = ip >> 24 & 0xFF;
|
||||||
|
var il1 = ip >> 16 & 0xFF;
|
||||||
|
var idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
|
||||||
|
return (int)idx;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal abstract ReadOnlyMemory<byte> GetVectorIndex(uint ip);
|
||||||
|
|
||||||
|
internal virtual ReadOnlyMemory<byte> GetData(int offset, int length)
|
||||||
|
{
|
||||||
|
byte[] buffer = ArrayPool<byte>.Shared.Rent(length);
|
||||||
|
int totalBytesRead = 0;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
XdbFileStream.Seek(offset, SeekOrigin.Begin);
|
||||||
|
|
||||||
|
int bytesRead;
|
||||||
|
do
|
||||||
|
{
|
||||||
|
int bytesToRead = Math.Min(BufferSize, length - totalBytesRead);
|
||||||
|
bytesRead = XdbFileStream.Read(buffer, totalBytesRead, bytesToRead);
|
||||||
|
totalBytesRead += bytesRead;
|
||||||
|
|
||||||
|
IoCount++;
|
||||||
|
} while (bytesRead > 0 && totalBytesRead < length);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
ArrayPool<byte>.Shared.Return(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new ReadOnlyMemory<byte>(buffer, 0, totalBytesRead);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
// Copyright 2023 The Ip2Region Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a Apache2.0-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
// @Author Alan <lzh.shap@gmail.com>
|
||||||
|
// @Date 2023/07/25
|
||||||
|
|
||||||
|
using IP2Region.Net.Internal.Abstractions;
|
||||||
|
using IP2Region.Net.XDB;
|
||||||
|
|
||||||
|
namespace IP2Region.Net.Internal;
|
||||||
|
|
||||||
|
internal class CacheStrategyFactory
|
||||||
|
{
|
||||||
|
private readonly string _xdbPath;
|
||||||
|
|
||||||
|
public CacheStrategyFactory(string xdbPath)
|
||||||
|
{
|
||||||
|
_xdbPath = xdbPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
public AbstractCacheStrategy CreateCacheStrategy(CachePolicy cachePolicy)
|
||||||
|
{
|
||||||
|
return cachePolicy switch
|
||||||
|
{
|
||||||
|
CachePolicy.Content => new ContentCacheStrategy(_xdbPath),
|
||||||
|
CachePolicy.VectorIndex => new VectorIndexCacheStrategy(_xdbPath),
|
||||||
|
CachePolicy.File => new FileCacheStrategy(_xdbPath),
|
||||||
|
_ => throw new ArgumentException(nameof(cachePolicy))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
// Copyright 2023 The Ip2Region Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a Apache2.0-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
// @Author Alan <lzh.shap@gmail.com>
|
||||||
|
// @Date 2023/07/25
|
||||||
|
|
||||||
|
using IP2Region.Net.Internal.Abstractions;
|
||||||
|
|
||||||
|
namespace IP2Region.Net.Internal;
|
||||||
|
|
||||||
|
internal class ContentCacheStrategy : AbstractCacheStrategy
|
||||||
|
{
|
||||||
|
private readonly ReadOnlyMemory<byte> _cacheData;
|
||||||
|
|
||||||
|
public ContentCacheStrategy(string xdbPath) : base(xdbPath)
|
||||||
|
{
|
||||||
|
_cacheData = base.GetData(0, (int)XdbFileStream.Length);
|
||||||
|
XdbFileStream.Close();
|
||||||
|
XdbFileStream.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
internal override ReadOnlyMemory<byte> GetVectorIndex(uint ip)
|
||||||
|
{
|
||||||
|
int idx = GetVectorIndexStartPos(ip);
|
||||||
|
return _cacheData.Slice(HeaderInfoLength + idx, VectorIndexSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal override ReadOnlyMemory<byte> GetData(int offset, int length)
|
||||||
|
{
|
||||||
|
return _cacheData.Slice(offset, length);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
// Copyright 2023 The Ip2Region Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a Apache2.0-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
// @Author Alan <lzh.shap@gmail.com>
|
||||||
|
// @Date 2023/07/25
|
||||||
|
|
||||||
|
using IP2Region.Net.Internal.Abstractions;
|
||||||
|
|
||||||
|
namespace IP2Region.Net.Internal;
|
||||||
|
|
||||||
|
internal class FileCacheStrategy : AbstractCacheStrategy
|
||||||
|
{
|
||||||
|
public FileCacheStrategy(string xdbPath) : base(xdbPath)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
internal override ReadOnlyMemory<byte> GetVectorIndex(uint ip)
|
||||||
|
{
|
||||||
|
var idx = GetVectorIndexStartPos(ip);
|
||||||
|
return GetData(HeaderInfoLength + idx, VectorIndexSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
// Copyright 2023 The Ip2Region Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a Apache2.0-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
// @Author Alan <lzh.shap@gmail.com>
|
||||||
|
// @Date 2023/07/25
|
||||||
|
|
||||||
|
using IP2Region.Net.Internal.Abstractions;
|
||||||
|
|
||||||
|
namespace IP2Region.Net.Internal;
|
||||||
|
|
||||||
|
internal class VectorIndexCacheStrategy : AbstractCacheStrategy
|
||||||
|
{
|
||||||
|
private readonly ReadOnlyMemory<byte> _vectorIndex;
|
||||||
|
|
||||||
|
public VectorIndexCacheStrategy(string xdbPath) : base(xdbPath)
|
||||||
|
{
|
||||||
|
var vectorLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
||||||
|
_vectorIndex = base.GetData(HeaderInfoLength, vectorLength);
|
||||||
|
}
|
||||||
|
|
||||||
|
internal override ReadOnlyMemory<byte> GetVectorIndex(uint ip)
|
||||||
|
{
|
||||||
|
var idx = GetVectorIndexStartPos(ip);
|
||||||
|
return _vectorIndex.Slice(idx, VectorIndexSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -3,15 +3,15 @@ namespace IP2Region.Net.XDB;
|
||||||
public enum CachePolicy
|
public enum CachePolicy
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// no cache , not thread safe!
|
/// no cache
|
||||||
/// </summary>
|
/// </summary>
|
||||||
File,
|
File,
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// cache vector index , reduce the number of IO operations , not thread safe!
|
/// cache vector index , reduce the number of IO operations
|
||||||
/// </summary>
|
/// </summary>
|
||||||
VectorIndex,
|
VectorIndex,
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// default cache policy , cache whole xdb file , thread safe
|
/// default cache policy , cache whole xdb file
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Content
|
Content
|
||||||
}
|
}
|
||||||
|
|
@ -1,14 +0,0 @@
|
||||||
using System.Net;
|
|
||||||
|
|
||||||
namespace IP2Region.Net.XDB;
|
|
||||||
|
|
||||||
public interface ISearcher
|
|
||||||
{
|
|
||||||
string? Search(string ipStr);
|
|
||||||
|
|
||||||
string? Search(IPAddress ipAddress);
|
|
||||||
|
|
||||||
string? Search(uint ipAddress);
|
|
||||||
|
|
||||||
int IoCount { get; }
|
|
||||||
}
|
|
||||||
|
|
@ -1,60 +1,29 @@
|
||||||
// Copyright 2022 The Ip2Region Authors. All rights reserved.
|
// Copyright 2023 The Ip2Region Authors. All rights reserved.
|
||||||
// Use of this source code is governed by a Apache2.0-style
|
// Use of this source code is governed by a Apache2.0-style
|
||||||
// license that can be found in the LICENSE file.
|
// license that can be found in the LICENSE file.
|
||||||
// @Author Alan Lee <lzh.shap@gmail.com>
|
// @Author Alan Lee <lzh.shap@gmail.com>
|
||||||
// @Date 2022/09/06
|
// @Date 2023/07/23
|
||||||
|
|
||||||
using System.Net;
|
using System.Net;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
using IP2Region.Net.Abstractions;
|
||||||
|
using IP2Region.Net.Internal;
|
||||||
|
using IP2Region.Net.Internal.Abstractions;
|
||||||
|
|
||||||
namespace IP2Region.Net.XDB;
|
namespace IP2Region.Net.XDB;
|
||||||
|
|
||||||
public class Searcher : ISearcher
|
public class Searcher : ISearcher
|
||||||
{
|
{
|
||||||
const int HeaderInfoLength = 256;
|
|
||||||
const int VectorIndexRows = 256;
|
|
||||||
const int VectorIndexCols = 256;
|
|
||||||
const int VectorIndexSize = 8;
|
|
||||||
const int SegmentIndexSize = 14;
|
const int SegmentIndexSize = 14;
|
||||||
|
|
||||||
private readonly byte[]? _vectorIndex;
|
private readonly AbstractCacheStrategy _cacheStrategy;
|
||||||
private readonly byte[]? _contentBuff;
|
public int IoCount => _cacheStrategy.IoCount;
|
||||||
private readonly FileStream _contentStream;
|
|
||||||
private readonly CachePolicy _cachePolicy;
|
|
||||||
public int IoCount { get; private set; }
|
|
||||||
|
|
||||||
public Searcher(CachePolicy cachePolicy = CachePolicy.Content, string? dbPath = null)
|
public Searcher(CachePolicy cachePolicy, string dbPath)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrEmpty(dbPath))
|
var factory = new CacheStrategyFactory(dbPath);
|
||||||
{
|
_cacheStrategy = factory.CreateCacheStrategy(cachePolicy);
|
||||||
dbPath = Path.Combine(AppContext.BaseDirectory, "Data", "ip2region.xdb");
|
|
||||||
}
|
|
||||||
|
|
||||||
_contentStream = File.OpenRead(dbPath);
|
|
||||||
_cachePolicy = cachePolicy;
|
|
||||||
|
|
||||||
switch (_cachePolicy)
|
|
||||||
{
|
|
||||||
case CachePolicy.Content:
|
|
||||||
using (var stream = new MemoryStream())
|
|
||||||
{
|
|
||||||
_contentStream.CopyTo(stream);
|
|
||||||
_contentBuff = stream.ToArray();
|
|
||||||
}
|
|
||||||
|
|
||||||
break;
|
|
||||||
case CachePolicy.VectorIndex:
|
|
||||||
var vectorLength = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
|
||||||
_vectorIndex = new byte[vectorLength];
|
|
||||||
Read(HeaderInfoLength, _vectorIndex);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
~Searcher()
|
|
||||||
{
|
|
||||||
_contentStream.Close();
|
|
||||||
_contentStream.Dispose();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public string? Search(string ipStr)
|
public string? Search(string ipStr)
|
||||||
|
|
@ -71,62 +40,37 @@ public class Searcher : ISearcher
|
||||||
|
|
||||||
public string? Search(uint ip)
|
public string? Search(uint ip)
|
||||||
{
|
{
|
||||||
var il0 = ip >> 24 & 0xFF;
|
var index = _cacheStrategy.GetVectorIndex(ip);
|
||||||
var il1 = ip >> 16 & 0xFF;
|
uint sPtr = MemoryMarshal.Read<uint>(index.Span);
|
||||||
var idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
|
uint ePtr = MemoryMarshal.Read<uint>(index.Span[4..]);
|
||||||
|
|
||||||
uint sPtr = 0, ePtr = 0;
|
|
||||||
|
|
||||||
switch (_cachePolicy)
|
|
||||||
{
|
|
||||||
case CachePolicy.VectorIndex:
|
|
||||||
sPtr = BitConverter.ToUInt32(_vectorIndex.AsSpan()[(int)idx..]);
|
|
||||||
ePtr = BitConverter.ToUInt32(_vectorIndex.AsSpan()[((int)idx + 4)..]);
|
|
||||||
break;
|
|
||||||
case CachePolicy.Content:
|
|
||||||
sPtr = BitConverter.ToUInt32(_contentBuff.AsSpan()[(HeaderInfoLength + (int)idx)..]);
|
|
||||||
ePtr = BitConverter.ToUInt32(_contentBuff.AsSpan()[(HeaderInfoLength + (int)idx + 4)..]);
|
|
||||||
break;
|
|
||||||
case CachePolicy.File:
|
|
||||||
var buff = new byte[VectorIndexSize];
|
|
||||||
Read((int)(idx + HeaderInfoLength), buff);
|
|
||||||
sPtr = BitConverter.ToUInt32(buff);
|
|
||||||
ePtr = BitConverter.ToUInt32(buff.AsSpan()[4..]);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
var dataLen = 0;
|
var dataLen = 0;
|
||||||
uint dataPtr = 0;
|
uint dataPtr = 0;
|
||||||
var l = 0;
|
uint l = 0;
|
||||||
var h = (int)((ePtr - sPtr) / SegmentIndexSize);
|
uint h = (ePtr -sPtr) / SegmentIndexSize;
|
||||||
var buffer = new byte[SegmentIndexSize];
|
|
||||||
|
|
||||||
while (l <= h)
|
while (l <= h)
|
||||||
{
|
{
|
||||||
var mid = Util.GetMidIp(l, h);
|
var mid = Util.GetMidIp(l, h);
|
||||||
var pos = sPtr + mid * SegmentIndexSize;
|
var pos = sPtr + mid * SegmentIndexSize;
|
||||||
|
|
||||||
Read((int)pos, buffer);
|
var buffer = _cacheStrategy.GetData((int)pos, SegmentIndexSize);
|
||||||
var sip = BitConverter.ToUInt32(buffer);
|
uint sip = MemoryMarshal.Read<uint>(buffer.Span);
|
||||||
|
uint eip = MemoryMarshal.Read<uint>(buffer.Span[4..]);
|
||||||
|
|
||||||
if (ip < sip)
|
if (ip < sip)
|
||||||
{
|
{
|
||||||
h = mid - 1;
|
h = mid - 1;
|
||||||
}
|
}
|
||||||
|
else if (ip > eip)
|
||||||
|
{
|
||||||
|
l = mid + 1;
|
||||||
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
var eip = BitConverter.ToUInt32(buffer.AsSpan()[4..]);
|
dataLen = MemoryMarshal.Read<ushort>(buffer.Span[8..]);
|
||||||
if (ip > eip)
|
dataPtr = MemoryMarshal.Read<uint>(buffer.Span[10..]);
|
||||||
{
|
break;
|
||||||
l = mid + 1;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
dataLen = BitConverter.ToUInt16(buffer.AsSpan()[8..]);
|
|
||||||
dataPtr = BitConverter.ToUInt32(buffer.AsSpan()[10..]);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -135,29 +79,7 @@ public class Searcher : ISearcher
|
||||||
return default;
|
return default;
|
||||||
}
|
}
|
||||||
|
|
||||||
var regionBuff = new byte[dataLen];
|
var regionBuff = _cacheStrategy.GetData((int)dataPtr,dataLen);
|
||||||
Read((int)dataPtr, regionBuff);
|
return Encoding.UTF8.GetString(regionBuff.Span);
|
||||||
return Encoding.UTF8.GetString(regionBuff);
|
|
||||||
}
|
|
||||||
|
|
||||||
private void Read(int offset, byte[] buff)
|
|
||||||
{
|
|
||||||
switch (_cachePolicy)
|
|
||||||
{
|
|
||||||
case CachePolicy.Content:
|
|
||||||
_contentBuff.AsSpan()[offset..(offset + buff.Length)].CopyTo(buff);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
_contentStream.Seek(offset, SeekOrigin.Begin);
|
|
||||||
IoCount++;
|
|
||||||
|
|
||||||
var rLen = _contentStream.Read(buff);
|
|
||||||
if (rLen != buff.Length)
|
|
||||||
{
|
|
||||||
throw new IOException($"incomplete read: readed bytes should be {buff.Length}");
|
|
||||||
}
|
|
||||||
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
using System.Net;
|
using System.Net;
|
||||||
|
using System.Runtime.InteropServices;
|
||||||
|
|
||||||
namespace IP2Region.Net.XDB;
|
namespace IP2Region.Net.XDB;
|
||||||
|
|
||||||
|
|
@ -14,12 +15,9 @@ public static class Util
|
||||||
{
|
{
|
||||||
byte[] bytes = ipAddress.GetAddressBytes();
|
byte[] bytes = ipAddress.GetAddressBytes();
|
||||||
Array.Reverse(bytes);
|
Array.Reverse(bytes);
|
||||||
return BitConverter.ToUInt32(bytes, 0);
|
return MemoryMarshal.Read<uint>(bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static uint GetMidIp(uint x, uint y)
|
public static uint GetMidIp(uint x, uint y)
|
||||||
=> (x & y) + ((x ^ y) >> 1);
|
=> (x & y) + ((x ^ y) >> 1);
|
||||||
|
|
||||||
public static int GetMidIp(int x, int y)
|
|
||||||
=> (x & y) + ((x ^ y) >> 1);
|
|
||||||
}
|
}
|
||||||
|
|
@ -1,12 +1,11 @@
|
||||||
# IP2Region.Net
|
# IP2Region.Net
|
||||||
|
|
||||||
IP2Region c# xdb client
|
.NET client library for IP2Region
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
Install the package with [NuGet](https://www.nuget.org/packages/IP2Region.Net)
|
Install the package with [NuGet](https://www.nuget.org/packages/IP2Region.Net)
|
||||||
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
Install-Package IP2Region.Net
|
Install-Package IP2Region.Net
|
||||||
```
|
```
|
||||||
|
|
@ -14,44 +13,43 @@ Install-Package IP2Region.Net
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
```csharp
|
```csharp
|
||||||
|
using IP2Region.Net.Abstractions;
|
||||||
using IP2Region.Net.XDB;
|
using IP2Region.Net.XDB;
|
||||||
|
|
||||||
//use default db and cache whole xdb file
|
ISearcher searcher = new Searcher(CachePolicy , "your xdb file path");
|
||||||
Searcher searcher = new Searcher();
|
|
||||||
searcher.Search("ipaddress value");
|
|
||||||
|
|
||||||
/*
|
|
||||||
* custom cache policy and xdb file path
|
|
||||||
* CachePolicy.Content default cache policy , cache whole xdb file , thread safe
|
|
||||||
* CachePolicy.VectorIndex cache vector index , reduce the number of IO operations , not thread safe!
|
|
||||||
* CachePolicy.File no cache , not thread safe!
|
|
||||||
*/
|
|
||||||
Searcher searcher = new Searcher(CachePolicy.File, "your xdb file path");
|
|
||||||
|
|
||||||
```
|
```
|
||||||
|
### Cache Policy Description
|
||||||
|
| Cache Policy | Description | Thread Safe |
|
||||||
|
|-------------------------|------------------------------------------------------------------------------------------------------------|-------------|
|
||||||
|
| CachePolicy.Content | Cache the entire `xdb` data. | Yes |
|
||||||
|
| CachePolicy.VectorIndex | Cache `vecotorIndex` to speed up queries and reduce system io pressure by reducing one fixed IO operation. | Yes |
|
||||||
|
| CachePolicy.File | Completely file-based queries | Yes |
|
||||||
|
### XDB File Description
|
||||||
|
Generate using [maker](https://github.com/lionsoul2014/ip2region/tree/master/maker/csharp), or [download](https://github.com/lionsoul2014/ip2region/blob/master/data/ip2region.xdb) pre-generated xdb files
|
||||||
|
|
||||||
## ASP.NET Core Usage
|
## ASP.NET Core Usage
|
||||||
|
|
||||||
```csharp
|
```csharp
|
||||||
services.AddSingleton<ISearcher,Searcher>();
|
services.AddSingleton<ISearcher>(new Searcher(CachePolicy , "your xdb file path"));
|
||||||
```
|
```
|
||||||
|
|
||||||
## Performance
|
## Performance
|
||||||
|
|
||||||
``` ini
|
``` ini
|
||||||
|
|
||||||
BenchmarkDotNet=v0.13.2, OS=Windows 11 (10.0.22000.856/21H2)
|
BenchmarkDotNet=v0.13.2, OS=macOS 13.4.1 (c) (22F770820d) [Darwin 22.5.0]
|
||||||
AMD Ryzen 5 3550H with Radeon Vega Mobile Gfx, 1 CPU, 8 logical and 4 physical cores
|
Apple M1, 1 CPU, 8 logical and 8 physical cores
|
||||||
.NET SDK=6.0.400
|
.NET SDK=7.0.306
|
||||||
[Host] : .NET 6.0.8 (6.0.822.36306), X64 RyuJIT AVX2
|
[Host] : .NET 6.0.20 (6.0.2023.32017), Arm64 RyuJIT AdvSIMD
|
||||||
DefaultJob : .NET 6.0.8 (6.0.822.36306), X64 RyuJIT AVX2
|
DefaultJob : .NET 6.0.20 (6.0.2023.32017), Arm64 RyuJIT AdvSIMD
|
||||||
|
|
||||||
|
|
||||||
```
|
```
|
||||||
| Method | Mean | Error | StdDev |
|
| Method | Mean | Error | StdDev |
|
||||||
|------------------------ |------------:|----------:|----------:|
|
|-------------------------|-----------:|---------:|---------:|
|
||||||
| CachePolicy_Content | 224.6 ns | 4.44 ns | 7.41 ns |
|
| CachePolicy_Content | 155.7 ns | 0.46 ns | 0.39 ns |
|
||||||
| CachePolicy_VectorIndex | 11,648.4 ns | 231.98 ns | 457.91 ns |
|
| CachePolicy_File | 2,186.8 ns | 34.27 ns | 32.06 ns |
|
||||||
|
| CachePolicy_VectorIndex | 1,570.3 ns | 27.53 ns | 22.99 ns |
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -146,9 +146,9 @@ namespace IP2RegionMaker.XDB
|
||||||
{
|
{
|
||||||
Console.WriteLine($"try to write region {seg.Region}");
|
Console.WriteLine($"try to write region {seg.Region}");
|
||||||
|
|
||||||
if (_regionPool.ContainsKey(seg.Region))
|
if (_regionPool.TryGetValue(seg.Region, out var value))
|
||||||
{
|
{
|
||||||
Console.WriteLine($"--[Cached] with ptr={_regionPool[seg.Region]}");
|
Console.WriteLine($"--[Cached] with ptr={value}");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue