xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #159 from coderbusy/master 

add C# maker
This commit is contained in:
Lion 2020-10-10 15:27:42 +08:00 committed by GitHub
parent e17286e324 9c70364080
commit fc39a114c1
14 changed files with 1913 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

341
maker/csharp/.gitignore vendored Normal file
View File

@ -0,0 +1,341 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
##
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
# User-specific files
*.rsuser
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
[Aa][Rr][Mm]/
[Aa][Rr][Mm]64/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
# Visual Studio 2015/2017 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# Visual Studio 2017 auto generated files
Generated\ Files/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUNIT
*.VisualState.xml
TestResult.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# Benchmark Results
BenchmarkDotNet.Artifacts/
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
# StyleCop
StyleCopReport.xml
# Files built by Visual Studio
*_i.c
*_p.c
*_h.h
*.ilk
*.meta
*.obj
*.iobj
*.pch
*.pdb
*.ipdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*_wpftmp.csproj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# Visual Studio Trace Files
*.e2e
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# JustCode is a .NET coding add-in
.JustCode
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# AxoCover is a Code Coverage Tool
.axoCover/*
!.axoCover/settings.json
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# Note: Comment the next line if you want to checkin your web deploy settings,
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# The packages folder can be ignored because of Package Restore
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
*.appx
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!?*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# ASP.NET Core default setup: bower directory is configured as wwwroot/lib/ and bower restore is true
**/wwwroot/lib/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
ServiceFabricBackup/
*.rptproj.bak
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
*.rptproj.rsuser
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# JetBrains Rider
.idea/
*.sln.iml
# CodeRush personal settings
.cr/personal
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Tabs Studio
*.tss
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
*.binlog
# NVidia Nsight GPU debugger configuration file
*.nvuser
# MFractors (Xamarin productivity tool) working folder
.mfractor/
# Local History for Visual Studio
.localhistory/
# BeatPulse healthcheck temp database
healthchecksdb

25
maker/csharp/DbMaker.sln Normal file
View File

@ -0,0 +1,25 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29102.190
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DbMaker", "DbMaker\DbMaker.csproj", "{724424E3-EC3C-4819-A29C-AA7FD7F2357F}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {135F4345-E533-41E0-ACA9-7F345351555D}
EndGlobalSection
EndGlobal

89
maker/csharp/DbMaker/DataBlock.cs Normal file
View File

@ -0,0 +1,89 @@
using System;
using System.Text;
namespace DbMaker
{
/**
* data block class
*
* @author chenxin<chenxin619315@gmail.com>
*/
public class DataBlock
{
/**
* city id
*/
private int city_id;
/**
* region address
*/
private String region;
/**
* region ptr in the db file
*/
private int dataPtr;
/**
* construct method
*
* @param city_id
* @param region region string
* @param dataPtr data ptr
*/
public DataBlock(int city_id, String region, int dataPtr)
{
this.city_id = city_id;
this.region = region;
this.dataPtr = dataPtr;
}
public DataBlock(int city_id, String region) : this(city_id, region, 0)
{
//this(city_id, region, 0);
}
public int getCityId()
{
return city_id;
}
public DataBlock setCityId(int city_id)
{
this.city_id = city_id;
return this;
}
public String getRegion()
{
return region;
}
public DataBlock setRegion(String region)
{
this.region = region;
return this;
}
public int getDataPtr()
{
return dataPtr;
}
public DataBlock setDataPtr(int dataPtr)
{
this.dataPtr = dataPtr;
return this;
}
public override String ToString()
{
StringBuilder sb = new StringBuilder();
sb.Append(city_id).Append('|').Append(region).Append('|').Append(dataPtr);
return sb.ToString();
}
}
}

67
maker/csharp/DbMaker/DbConfig.cs Normal file
View File

@ -0,0 +1,67 @@
namespace DbMaker
{
/**
* database configuration class
*
* @author chenxin<chenxin619315@gmail.com>
*/
public class DbConfig
{
/**
* total header data block size
*/
private int totalHeaderSize;
/**
* max index data block size
* u should always choice the fastest read block size
*/
private int indexBlockSize;
/**
* construct method
*
* @param totalHeaderSize
* @throws DbMakerConfigException
*/
public DbConfig(int totalHeaderSize)
{
if ((totalHeaderSize % 8) != 0)
{
throw new DbMakerConfigException("totalHeaderSize must be times of 8");
}
this.totalHeaderSize = totalHeaderSize;
this.indexBlockSize = 8192; //4 * 2048
}
public DbConfig() : this(8 * 2048)
{
//this(8 * 2048);
}
public int getTotalHeaderSize()
{
return totalHeaderSize;
}
public DbConfig setTotalHeaderSize(int totalHeaderSize)
{
this.totalHeaderSize = totalHeaderSize;
return this;
}
public int getIndexBlockSize()
{
return indexBlockSize;
}
public DbConfig setIndexBlockSize(int dataBlockSize)
{
this.indexBlockSize = dataBlockSize;
return this;
}
}
}

458
maker/csharp/DbMaker/DbMaker.cs Normal file
View File

@ -0,0 +1,458 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using DbMaker;
namespace DbMaker
{
/**
* fast ip db maker
*
* db struct:
* 1. header part
* 1): super part:
* +------------+-----------+
* | 4 bytes | 4 bytes |
* +------------+-----------+
* start index ptr, end index ptr
*
* 2): b-tree index part
* +------------+-----------+-----------+-----------+
* | 4bytes | 4bytes | 4bytes | 4bytes | ...
* +------------+-----------+-----------+-----------+
* start ip ptr index ptr
*
* 2. data part:
* +------------+-----------------------+
* | 2bytes | dynamic length |
* +------------+-----------------------+
* data length city_id|Country|Province|Area|City|ISP
*
* 3. index part: (ip range)
* +------------+-----------+---------------+
* | 4bytes | 4bytes | 4bytes |
* +------------+-----------+---------------+
* start ip end ip 3 byte data ptr & 1 byte data length
*
* @author chenxin<chenxin619315@gmail.com>
*/
public class DbMaker
{
/**
* db config
*/
private DbConfig dbConfig;
/**
* ip source file path
*/
private FileStream ipSrcFile;
/**
* buffer
*/
private List<IndexBlock> indexPool;
private List<HeaderBlock> headerPool;
/**
* global region Id mapping data
*/
private FileStream globalRegionFile = null;
private Dictionary<String, int> globalRegionMap = null;
/**
* region and data ptr mapping data
*/
private Dictionary<String, DataBlock> regionPtrPool = null;
/**
* construct method
*
* @param config
* @param ipSrcFile tb source ip file
* @param globalRegionFile global_region.csv file offer by lion
* @throws DbMakerConfigException
* @throws IOException
*/
public DbMaker(
DbConfig config,
String ipSrcFile,
String globalRegionFile)
{
this.dbConfig = config;
this.ipSrcFile = File.OpenRead(ipSrcFile); //new File(ipSrcFile);
this.globalRegionMap = new Dictionary<string, int>(); //new HashMap<String, Integer>();
this.regionPtrPool = new Dictionary<string, DataBlock>(); //new HashMap<String, DataBlock>();
if (globalRegionFile != null)
{
this.globalRegionFile = File.OpenRead(globalRegionFile); //new File(globalRegionFile);
}
//if ( this.ipSrcFile.exists() == false ) {
if (!File.Exists(ipSrcFile))
{
throw new IOException("Error: Invalid file path " + ipSrcFile);
}
}
/**
* initialize the db file
*
* @param raf
* @throws IOException
*/
private void initDbFile(RandomAccessFile raf)
{
//1. zero fill the header part
raf.seek(0L);
raf.write(new byte[8]); //super block
raf.write(new byte[dbConfig.getTotalHeaderSize()]); //header block
headerPool = new List<HeaderBlock>(); //new LinkedList<HeaderBlock>();
indexPool = new List<IndexBlock>(); //new LinkedList<IndexBlock>();
}
/**
* make the Db file
*
* @param dbFile target output file path
* @throws IOException
*/
public void make(String dbFile)
{
//check and load the gloabl region
if (globalRegionFile != null)
{
Console.WriteLine("+-Try to load the global region data ...");
StreamReader greader = new StreamReader((globalRegionFile));
String gline = null;
while (!greader.EndOfStream)
{
gline = greader.ReadLine();// != null
if (String.IsNullOrWhiteSpace(gline))
{
continue;
}
String[] p = gline.Split(",");
if (p.Length != 5) continue;
//push the mapping
globalRegionMap.put(p[2], Int32.Parse(p[0]));
}
greader.Close();
Console.WriteLine("|--[Ok]");
}
//alloc the header size
StreamReader reader = new StreamReader(this.ipSrcFile);
RandomAccessFile raf = new RandomAccessFile(dbFile);
//init the db file
initDbFile(raf);
Console.WriteLine("+-Db file initialized.");
//analysis main loop
Console.WriteLine("+-Try to write the data blocks ... ");
String line = null;
while (!reader.EndOfStream)
{
//line = reader.ReadLine()
line = reader.ReadLine();
if (String.IsNullOrWhiteSpace(line))
{
continue;
}
line = line.Trim();
if (line.Length == 0) continue;
if (line[0] == '#') continue;
//1. get the start ip
int sIdx = 0, eIdx = 0;
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue;
String startIp = line.Substring(sIdx, eIdx);
//2. get the end ip
sIdx = eIdx + 1;
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue;
String endIp = line.Substring(sIdx, eIdx - sIdx);
//3. get the region
sIdx = eIdx + 1;
String region = line.Substring(sIdx);
Console.WriteLine("+-Try to process item " + line);
addDataBlock(raf, startIp, endIp, region);
Console.WriteLine("|--[Ok]");
}
Console.WriteLine("|--Data block flushed!");
Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n");
//write the index bytes
Console.WriteLine("+-Try to write index blocks ... ");
//record the start block
IndexBlock indexBlock = null;
HeaderBlock hb = null;
indexBlock = indexPool.First();
long indexStartIp = indexBlock.getStartIp(),
indexStratPtr = raf.getFilePointer(), indexEndPtr;
headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr)));
int blockLength = IndexBlock.getIndexBlockLength();
int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1;
//Iterator<IndexBlock> indexIt = indexPool.iterator();
//while ( indexIt.hasNext() ) {
foreach (var block in indexPool)
{
if (++counter >= shotCounter)
{
hb = new HeaderBlock(
block.getStartIp(),
(int)raf.getFilePointer()
);
headerPool.add(hb);
counter = 0;
}
//write the buffer
raf.write(block.getBytes());
}
//record the end block
if (counter > 0)
{
indexBlock = indexPool.Last();
hb = new HeaderBlock(
indexBlock.getStartIp(),
((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength()
);
headerPool.add(hb);
}
indexEndPtr = raf.getFilePointer();
Console.WriteLine("|--[Ok]");
//write the super blocks
Console.WriteLine("+-Try to write the super blocks ... ");
raf.seek(0L); //reset the file pointer
byte[] superBuffer = new byte[8];
Util.writeIntLong(superBuffer, 0, indexStratPtr);
Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength);
raf.write(superBuffer);
Console.WriteLine("|--[Ok]");
//write the header blocks
Console.WriteLine("+-Try to write the header blocks ... ");
//Iterator<HeaderBlock> headerIt = headerPool.iterator();
//while ( headerIt.hasNext() ) {
// HeaderBlock headerBlock = headerIt.next();
// raf.write(headerBlock.getBytes());
//}
foreach (var headerBlock in headerPool)
{
raf.write(headerBlock.getBytes());
}
//write the copyright and the release timestamp info
Console.WriteLine("+-Try to write the copyright and release date info ... ");
raf.seek(raf.length());
//Calendar cal = Calendar.getInstance();
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
var date = DateTime.Now;
String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd");
var timespan = new DateTimeOffset(date).ToUnixTimeSeconds();
raf.write((int)(timespan)); //the unix timestamp
raf.write(copyright.getBytes("UTF-8"));
Console.WriteLine("|--[Ok]");
reader.Close();
raf.close();
}
/**
* internal method to add a new data block record
*
* @param raf
* @param startIp
* @param endIp
* @param region data
*/
private void addDataBlock(
RandomAccessFile raf,
String startIp, String endIp, String region)
{
byte[] data = region.getBytes("UTF-8");
int dataPtr = 0;
/*byte[] city = new byte[4];
int city_id = getCityId(region);
Util.writeIntLong(city, 0, city_id);
dataPtr = (int)raf.getFilePointer();
raf.write(city);
raf.write(data);*/
//check region ptr pool first
if (regionPtrPool.containsKey(region))
{
DataBlock dataBlock = regionPtrPool.get(region);
dataPtr = dataBlock.getDataPtr();
Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region);
}
else
{
byte[] city = new byte[4];
int city_id = getCityId(region);
Util.writeIntLong(city, 0, city_id);
dataPtr = (int)raf.getFilePointer();
raf.write(city);
raf.write(data);
regionPtrPool.put(region, new DataBlock(city_id, region, dataPtr));
}
//add the data index blocks
IndexBlock ib = new IndexBlock(
Util.ip2long(startIp),
Util.ip2long(endIp),
dataPtr,
data.Length + 4 //4 bytes for the city id
);
indexPool.add(ib);
}
/**
* get the city id base on the global region data
*
* @param region
* @return int
*/
public int getCityId(String region)
{
String[] p = region.Split("|");
if (p.Length != 5) return 0;
String key = null;
int? intv = null;
for (int i = 3; i >= 0; i--)
{
if (p[i].Equals("0")) continue;
if (i == 3
&& p[i].IndexOf("省直辖县级") > -1)
{
key = p[2] + p[3];
}
else
{
key = p[i];
}
if (!globalRegionMap.ContainsKey(key))
{
return 0;
}
intv = globalRegionMap.get(key);
//if (intv == null) return 0;
return intv.Value;
}
return 0;
}
public DbConfig getDbConfig()
{
return dbConfig;
}
public DbMaker setDbConfig(DbConfig dbConfig)
{
this.dbConfig = dbConfig;
return this;
}
public FileStream getIpSrcFile()
{
return ipSrcFile;
}
public DbMaker setIpSrcFile(FileStream ipSrcFile)
{
this.ipSrcFile = ipSrcFile;
return this;
}
/**
* make this directly a runnable application
* interface to make the database file
*/
public static void Main(String[] args)
{
String dstDir = "./data/";
String[] path = new String[] { null, null };
for (int i = 0; i < args.Length; i++)
{
if (args[i].Equals("-src"))
{
path[0] = args[++i];
}
else if (args[i].Equals("-region"))
{
path[1] = args[++i];
}
else if (args[i].Equals("-dst"))
{
dstDir = args[++i];
}
}
for (int i = 0; i < path.Length; i++)
{
if (path[i] == null)
{
Console.WriteLine("Usage: DbMaker "
+ "-src [source text file path] "
+ "-region [global region file path]");
Console.WriteLine("eg: DbMaker "
+ "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv");
//System.exit(0);
Environment.Exit(0);
}
}
//check and stdlize the destination directory
if (!dstDir.EndsWith("/"))
{
dstDir = dstDir + "/";
}
try
{
if (!Directory.Exists(dstDir))
{
Directory.CreateDirectory(dstDir);
}
DbConfig config = new DbConfig();
DbMaker dbMaker = new DbMaker(config, path[0], path[1]);
dbMaker.make(dstDir + "ip2region.db");
}
catch (DbMakerConfigException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
catch (IOException e)
{
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}

8
maker/csharp/DbMaker/DbMaker.csproj Normal file
View File

@ -0,0 +1,8 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
</Project>

12
maker/csharp/DbMaker/DbMakerConfigException.cs Normal file
View File

@ -0,0 +1,12 @@
using System;
namespace DbMaker
{
public class DbMakerConfigException : Exception
{
public DbMakerConfigException(String message) : base(message)
{
}
}
}

463
maker/csharp/DbMaker/DbSearcher.cs Normal file
View File

@ -0,0 +1,463 @@
using System;
using System.Text;
namespace DbMaker
{
/**
* ip db searcher class (Not thread safe)
*
* @author chenxin<chenxin619315@gmail.com>
*/
public class DbSearcher : IDisposable
{
public const int BTREE_ALGORITHM = 1;
public const int BINARY_ALGORITHM = 2;
public const int MEMORY_ALGORITYM = 3;
/**
* db config
*/
private DbConfig dbConfig = null;
/**
* db file access handler
*/
private RandomAccessFile raf = null;
/**
* header blocks buffer
*/
private long[] HeaderSip = null;
private int[] HeaderPtr = null;
private int headerLength;
/**
* super blocks info
*/
private long firstIndexPtr = 0;
private long lastIndexPtr = 0;
private int totalIndexBlocks = 0;
/**
* for memory mode
* the original db binary string
*/
private byte[] dbBinStr = null;
/**
* construct class
*
* @param bdConfig
* @param dbFile
* @throws FileNotFoundException
*/
public DbSearcher(DbConfig dbConfig, String dbFile)
{
this.dbConfig = dbConfig;
raf = new RandomAccessFile(dbFile);
}
/**
* construct method with self-define std ip2region bianry string support
* Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL
*
* @param dbConfig
* @param dbBinStr
*/
public DbSearcher(DbConfig dbConfig, byte[] dbBinStr)
{
this.dbConfig = dbConfig;
this.dbBinStr = dbBinStr;
firstIndexPtr = Util.getIntLong(dbBinStr, 0);
lastIndexPtr = Util.getIntLong(dbBinStr, 4);
totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / IndexBlock.getIndexBlockLength()) + 1;
}
/**
* get the region with a int ip address with memory binary search algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock memorySearch(long ip)
{
int blen = IndexBlock.getIndexBlockLength();
if (dbBinStr == null)
{
dbBinStr = new byte[(int) raf.length()];
raf.seek(0L);
raf.readFully(dbBinStr, 0, dbBinStr.Length);
//initialize the global vars
firstIndexPtr = Util.getIntLong(dbBinStr, 0);
lastIndexPtr = Util.getIntLong(dbBinStr, 4);
totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1;
}
//search the index blocks to define the data
int l = 0, h = totalIndexBlocks;
long sip, eip, dataptr = 0;
while (l <= h)
{
int m = (l + h) >> 1;
int p = (int) (firstIndexPtr + m * blen);
sip = Util.getIntLong(dbBinStr, p);
if (ip < sip)
{
h = m - 1;
}
else
{
eip = Util.getIntLong(dbBinStr, p + 4);
if (ip > eip)
{
l = m + 1;
}
else
{
dataptr = Util.getIntLong(dbBinStr, p + 8);
break;
}
}
}
//not matched
if (dataptr == 0) return null;
//get the data
int dataLen = (int) ((dataptr >> 24) & 0xFF);
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
int city_id = (int) Util.getIntLong(dbBinStr, dataPtr);
//String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8");
var region = ReadString(dbBinStr, dataPtr + 4, dataLen - 4);
return new DataBlock(city_id, region, dataPtr);
}
private string ReadString(byte[] bytes, int dataPtr, int dataLen)
{
return Encoding.UTF8.GetString(bytes, dataPtr, dataLen);
}
/**
* get the region throught the ip address with memory binary search algorithm
*
* @param ip
* @return DataBlock
* @throws IOException
*/
public DataBlock memorySearch(String ip)
{
return memorySearch(Util.ip2long(ip));
}
/**
* get by index ptr
*
* @param indexPtr
* @throws IOException
*/
public DataBlock getByIndexPtr(long ptr)
{
raf.seek(ptr);
byte[] buffer = new byte[12];
raf.readFully(buffer, 0, buffer.Length);
//long startIp = Util.getIntLong(buffer, 0);
//long endIp = Util.getIntLong(buffer, 4);
long extra = Util.getIntLong(buffer, 8);
int dataLen = (int) ((extra >> 24) & 0xFF);
int dataPtr = (int) ((extra & 0x00FFFFFF));
raf.seek(dataPtr);
byte[] data = new byte[dataLen];
raf.readFully(data, 0, data.Length);
int city_id = (int) Util.getIntLong(data, 0);
//String region = new String(data, 4, data.Length - 4, "UTF-8");
var region = ReadString(data, 4, data.Length - 4);
return new DataBlock(city_id, region, dataPtr);
}
/**
* get the region with a int ip address with b-tree algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock btreeSearch(long ip)
{
//check and load the header
if (HeaderSip == null)
{
raf.seek(8L); //pass the super block
byte[] b = new byte[dbConfig.getTotalHeaderSize()];
// byte[] b = new byte[4096];
raf.readFully(b, 0, b.Length);
//fill the header
int len = b.Length >> 3, idx = 0; //b.lenght / 8
HeaderSip = new long[len];
HeaderPtr = new int [len];
long startIp, xDataPtr;
for (int i = 0; i < b.Length; i += 8)
{
startIp = Util.getIntLong(b, i);
xDataPtr = Util.getIntLong(b, i + 4);
if (xDataPtr == 0) break;
HeaderSip[idx] = startIp;
HeaderPtr[idx] = (int) xDataPtr;
idx++;
}
headerLength = idx;
}
//1. define the index block with the binary search
if (ip == HeaderSip[0])
{
return getByIndexPtr(HeaderPtr[0]);
}
else if (ip == HeaderSip[headerLength - 1])
{
return getByIndexPtr(HeaderPtr[headerLength - 1]);
}
int l = 0, h = headerLength, sptr = 0, eptr = 0;
while (l <= h)
{
int m = (l + h) >> 1;
//perfetc matched, just return it
if (ip == HeaderSip[m])
{
if (m > 0)
{
sptr = HeaderPtr[m - 1];
eptr = HeaderPtr[m];
}
else
{
sptr = HeaderPtr[m];
eptr = HeaderPtr[m + 1];
}
break;
}
//less then the middle value
if (ip < HeaderSip[m])
{
if (m == 0)
{
sptr = HeaderPtr[m];
eptr = HeaderPtr[m + 1];
break;
}
else if (ip > HeaderSip[m - 1])
{
sptr = HeaderPtr[m - 1];
eptr = HeaderPtr[m];
break;
}
h = m - 1;
}
else
{
if (m == headerLength - 1)
{
sptr = HeaderPtr[m - 1];
eptr = HeaderPtr[m];
break;
}
else if (ip <= HeaderSip[m + 1])
{
sptr = HeaderPtr[m];
eptr = HeaderPtr[m + 1];
break;
}
l = m + 1;
}
}
//match nothing just stop it
if (sptr == 0) return null;
//2. search the index blocks to define the data
int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength();
byte[] iBuffer = new byte[blockLen + blen]; //include the right border block
raf.seek(sptr);
raf.readFully(iBuffer, 0, iBuffer.Length);
l = 0;
h = blockLen / blen;
long sip, eip, dataptr = 0;
while (l <= h)
{
int m = (l + h) >> 1;
int p = m * blen;
sip = Util.getIntLong(iBuffer, p);
if (ip < sip)
{
h = m - 1;
}
else
{
eip = Util.getIntLong(iBuffer, p + 4);
if (ip > eip)
{
l = m + 1;
}
else
{
dataptr = Util.getIntLong(iBuffer, p + 8);
break;
}
}
}
//not matched
if (dataptr == 0) return null;
//3. get the data
int dataLen = (int) ((dataptr >> 24) & 0xFF);
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
raf.seek(dataPtr);
byte[] data = new byte[dataLen];
raf.readFully(data, 0, data.Length);
int city_id = (int) Util.getIntLong(data, 0);
//String region = new String(data, 4, data.Length - 4, "UTF-8");
var region = ReadString(data, 4, data.Length - 4);
return new DataBlock(city_id, region, dataPtr);
}
/**
* get the region throught the ip address with b-tree search algorithm
*
* @param ip
* @return DataBlock
* @throws IOException
*/
public DataBlock btreeSearch(String ip)
{
return btreeSearch(Util.ip2long(ip));
}
/**
* get the region with a int ip address with binary search algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock binarySearch(long ip)
{
int blen = IndexBlock.getIndexBlockLength();
if (totalIndexBlocks == 0)
{
raf.seek(0L);
byte[] superBytes = new byte[8];
raf.readFully(superBytes, 0, superBytes.Length);
//initialize the global vars
firstIndexPtr = Util.getIntLong(superBytes, 0);
lastIndexPtr = Util.getIntLong(superBytes, 4);
totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1;
}
//search the index blocks to define the data
int l = 0, h = totalIndexBlocks;
byte[] buffer = new byte[blen];
long sip, eip, dataptr = 0;
while (l <= h)
{
int m = (l + h) >> 1;
raf.seek(firstIndexPtr + m * blen); //set the file pointer
raf.readFully(buffer, 0, buffer.Length);
sip = Util.getIntLong(buffer, 0);
if (ip < sip)
{
h = m - 1;
}
else
{
eip = Util.getIntLong(buffer, 4);
if (ip > eip)
{
l = m + 1;
}
else
{
dataptr = Util.getIntLong(buffer, 8);
break;
}
}
}
//not matched
if (dataptr == 0) return null;
//get the data
int dataLen = (int) ((dataptr >> 24) & 0xFF);
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
raf.seek(dataPtr);
byte[] data = new byte[dataLen];
raf.readFully(data, 0, data.Length);
int city_id = (int) Util.getIntLong(data, 0);
//String region = new String(data, 4, data.Length - 4, "UTF-8");
var region = ReadString(data, 4, data.Length - 4);
return new DataBlock(city_id, region, dataPtr);
}
/**
* get the region throught the ip address with binary search algorithm
*
* @param ip
* @return DataBlock
* @throws IOException
*/
public DataBlock binarySearch(String ip)
{
return binarySearch(Util.ip2long(ip));
}
/**
* get the db config
*
* @return DbConfig
*/
public DbConfig getDbConfig()
{
return dbConfig;
}
/**
* close the db
*
* @throws IOException
*/
private void close()
{
HeaderSip = null; //let gc do its work
HeaderPtr = null;
dbBinStr = null;
raf.close();
}
public void Dispose()
{
close();
}
}
}

40
maker/csharp/DbMaker/Extensions.cs Normal file
View File

@ -0,0 +1,40 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace DbMaker
{
public static class Extensions
{
public static byte[] getBytes(this string str, string encode)
{
return Encoding.GetEncoding(encode).GetBytes(str);
}
public static Boolean containsKey<TKey, TValue>(this IDictionary<TKey, TValue> dic, TKey key)
{
return dic.ContainsKey(key);
}
public static TValue get<TKey, TValue>(this IDictionary<TKey, TValue> dic, TKey key)
{
return dic[key];
}
public static void put<TKey, TValue>(this IDictionary<TKey, TValue> dic, TKey key, TValue value)
{
//dic.Add(key,value);
dic[key] = value;
}
public static void add<TItem>(this IList<TItem> list, TItem item)
{
list.Add(item);
}
public static void printStackTrace(this Exception e)
{
Console.WriteLine(e);
}
}
}

70
maker/csharp/DbMaker/HeaderBlock.cs Normal file
View File

@ -0,0 +1,70 @@
namespace DbMaker
{
/**
* header block class
*
* @author chenxin<chenxin619315@gmail.com>
*/
public class HeaderBlock
{
/**
* index block start ip address
*/
private long indexStartIp;
/**
* ip address
*/
private int indexPtr;
public HeaderBlock(long indexStartIp, int indexPtr)
{
this.indexStartIp = indexStartIp;
this.indexPtr = indexPtr;
}
public long getIndexStartIp()
{
return indexStartIp;
}
public HeaderBlock setIndexStartIp(long indexStartIp)
{
this.indexStartIp = indexStartIp;
return this;
}
public int getIndexPtr()
{
return indexPtr;
}
public HeaderBlock setIndexPtr(int indexPtr)
{
this.indexPtr = indexPtr;
return this;
}
/**
* get the bytes for db storage
*
* @return byte[]
*/
public byte[] getBytes()
{
/*
* +------------+-----------+
* | 4bytes | 4bytes |
* +------------+-----------+
* start ip index ptr
*/
byte[] b = new byte[8];
Util.writeIntLong(b, 0, indexStartIp);
Util.writeIntLong(b, 4, indexPtr);
return b;
}
}
}

115
maker/csharp/DbMaker/IndexBlock.cs Normal file
View File

@ -0,0 +1,115 @@
namespace DbMaker
{
/**
* item index class
*
* @author chenxin<chenxin619315@gmail.com>
*/
public class IndexBlock
{
private static int LENGTH = 12;
/**
* start ip address
*/
private long startIp;
/**
* end ip address
*/
private long endIp;
/**
* data ptr and data length
*/
private int dataPtr;
/**
* data length
*/
private int dataLen;
public IndexBlock(long startIp, long endIp, int dataPtr, int dataLen)
{
this.startIp = startIp;
this.endIp = endIp;
this.dataPtr = dataPtr;
this.dataLen = dataLen;
}
public long getStartIp()
{
return startIp;
}
public IndexBlock setStartIp(long startIp)
{
this.startIp = startIp;
return this;
}
public long getEndIp()
{
return endIp;
}
public IndexBlock setEndIp(long endIp)
{
this.endIp = endIp;
return this;
}
public int getDataPtr()
{
return dataPtr;
}
public IndexBlock setDataPtr(int dataPtr)
{
this.dataPtr = dataPtr;
return this;
}
public int getDataLen()
{
return dataLen;
}
public IndexBlock setDataLen(int dataLen)
{
this.dataLen = dataLen;
return this;
}
public static int getIndexBlockLength()
{
return LENGTH;
}
/**
* get the bytes for storage
*
* @return byte[]
*/
public byte[] getBytes()
{
/*
* +------------+-----------+-----------+
* | 4bytes | 4bytes | 4bytes |
* +------------+-----------+-----------+
* start ip end ip data ptr + len
*/
byte[] b = new byte[12];
Util.writeIntLong(b, 0, startIp); //start ip
Util.writeIntLong(b, 4, endIp); //end ip
//write the data ptr and the length
long mix = dataPtr | ((dataLen << 24) & 0xFF000000L);
Util.writeIntLong(b, 8, mix);
return b;
}
}
}

8
maker/csharp/DbMaker/Properties/launchSettings.json Normal file
View File

@ -0,0 +1,8 @@
{
"profiles": {
"DbMaker": {
"commandName": "Project",
"commandLineArgs": " -src ip.merge.txt -region global_region.csv"
}
}
}

64
maker/csharp/DbMaker/RandomAccessFile.cs Normal file
View File

@ -0,0 +1,64 @@
using System;
using System.IO;
namespace DbMaker
{
public class RandomAccessFile : IDisposable
{
private readonly Stream _stream;
public RandomAccessFile(String file) : this(File.Open(file, FileMode.OpenOrCreate))
{
}
public RandomAccessFile(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
_stream = stream;
}
public void Dispose()
{
_stream?.Dispose();
}
public long length()
{
return _stream.Length;
}
public void seek(long position)
{
_stream.Seek(position, SeekOrigin.Begin);
}
public void readFully(byte[] dbBinStr, int offset, int count)
{
_stream.Read(dbBinStr, offset, count);
}
public void close()
{
_stream.Dispose();
}
public void write(byte[] bytes)
{
_stream.Write(bytes);
}
public void write(int i)
{
}
public long getFilePointer()
{
return _stream.Position;
}
}
}

153
maker/csharp/DbMaker/Util.cs Normal file
View File

@ -0,0 +1,153 @@
using System;
using System.Text;
namespace DbMaker
{
/**
* util class
*
* @author chenxin<chenxin619315@gmail.com>
*/
public class Util
{
/**
* write specfield bytes to a byte array start from offset
*
* @param b
* @param offset
* @param v
* @param bytes
*/
public static void write(byte[] b, int offset, long v, int bytes)
{
for (int i = 0; i < bytes; i++)
{
//b[offset++] = (byte)((v >>> (8 * i)) & 0xFF);
b[offset++] = (byte) ((v >> (8 * i)) & 0xFF);
}
}
/**
* write a int to a byte array
*
* @param b
* @param offet
* @param v
*/
public static void writeIntLong(byte[] b, int offset, long v)
{
b[offset++] = (byte) ((v >> 0) & 0xFF);
b[offset++] = (byte) ((v >> 8) & 0xFF);
b[offset++] = (byte) ((v >> 16) & 0xFF);
b[offset] = (byte) ((v >> 24) & 0xFF);
}
/**
* get a int from a byte array start from the specifiled offset
*
* @param b
* @param offset
*/
public static long getIntLong(byte[] b, int offset)
{
return (
((b[offset++] & 0x000000FFL)) |
((b[offset++] << 8) & 0x0000FF00L) |
((b[offset++] << 16) & 0x00FF0000L) |
((b[offset] << 24) & 0xFF000000L)
);
}
/**
* get a int from a byte array start from the specifield offset
*
* @param b
* @param offset
*/
public static int getInt3(byte[] b, int offset)
{
return (
(b[offset++] & 0x000000FF) |
(b[offset++] & 0x0000FF00) |
(b[offset] & 0x00FF0000)
);
}
public static int getInt2(byte[] b, int offset)
{
return (
(b[offset++] & 0x000000FF) |
(b[offset] & 0x0000FF00)
);
}
public static int getInt1(byte[] b, int offset)
{
return (
(b[offset] & 0x000000FF)
);
}
/**
* string ip to long ip
*
* @param ip
* @return long
*/
public static long ip2long(String ip)
{
ip = ip.Trim();
String[] ips = ip.Split('.');
long ip1 = Int64.Parse(ips[0]);
long ip2 = Int64.Parse(ips[1]);
long ip3 = Int64.Parse(ips[2]);
long ip4 = Int64.Parse(ips[3]);
long ip2long = 1L * ip1 * 256 * 256 * 256 + ip2 * 256 * 256 + ip3 * 256 + ip4;
return ip2long;
}
/**
* int to ip string
*
* @param ip
* @return string
*/
public static String long2ip(long ip)
{
StringBuilder sb = new StringBuilder();
sb
.Append((ip >> 24) & 0xFF).Append('.')
.Append((ip >> 16) & 0xFF).Append('.')
.Append((ip >> 8) & 0xFF).Append('.')
.Append((ip >> 0) & 0xFF);
return sb.ToString();
}
/**
* check the validate of the specifeld ip address
*
* @param ip
* @return boolean
*/
public static Boolean isIpAddress(String ip)
{
String[] p = ip.Split("\\.");
if (p.Length != 4) return false;
//for ( String pp : p ) {
//for ( String pp : p ) {
foreach (var pp in p)
{
if (pp.Length > 3) return false;
int val = Int32.Parse(pp); //Integer.valueOf(pp);
if (val > 255) return false;
}
return true;
}
}
}