diff --git a/maker/csharp/.gitignore b/maker/csharp/.gitignore new file mode 100644 index 0000000..3e8a155 --- /dev/null +++ b/maker/csharp/.gitignore @@ -0,0 +1,341 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ +# ASP.NET Core default setup: bower directory is configured as wwwroot/lib/ and bower restore is true +**/wwwroot/lib/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# BeatPulse healthcheck temp database +healthchecksdb diff --git a/maker/csharp/DbMaker.sln b/maker/csharp/DbMaker.sln new file mode 100644 index 0000000..bd53da2 --- /dev/null +++ b/maker/csharp/DbMaker.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29102.190 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DbMaker", "DbMaker\DbMaker.csproj", "{724424E3-EC3C-4819-A29C-AA7FD7F2357F}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {724424E3-EC3C-4819-A29C-AA7FD7F2357F}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {135F4345-E533-41E0-ACA9-7F345351555D} + EndGlobalSection +EndGlobal diff --git a/maker/csharp/DbMaker/DataBlock.cs b/maker/csharp/DbMaker/DataBlock.cs new file mode 100644 index 0000000..d33bf94 --- /dev/null +++ b/maker/csharp/DbMaker/DataBlock.cs @@ -0,0 +1,89 @@ +using System; +using System.Text; + +namespace DbMaker +{ + + /** + * data block class + * + * @author chenxin + */ + public class DataBlock + { + /** + * city id + */ + private int city_id; + + /** + * region address + */ + private String region; + + /** + * region ptr in the db file + */ + private int dataPtr; + + /** + * construct method + * + * @param city_id + * @param region region string + * @param dataPtr data ptr + */ + public DataBlock(int city_id, String region, int dataPtr) + { + this.city_id = city_id; + this.region = region; + this.dataPtr = dataPtr; + } + + public DataBlock(int city_id, String region) : this(city_id, region, 0) + { + //this(city_id, region, 0); + } + + public int getCityId() + { + return city_id; + } + + public DataBlock setCityId(int city_id) + { + this.city_id = city_id; + return this; + } + + public String getRegion() + { + return region; + } + + public DataBlock setRegion(String region) + { + this.region = region; + return this; + } + + public int getDataPtr() + { + return dataPtr; + } + + public DataBlock setDataPtr(int dataPtr) + { + this.dataPtr = dataPtr; + return this; + } + + public override String ToString() + { + StringBuilder sb = new StringBuilder(); + + sb.Append(city_id).Append('|').Append(region).Append('|').Append(dataPtr); + return sb.ToString(); + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/DbConfig.cs b/maker/csharp/DbMaker/DbConfig.cs new file mode 100644 index 0000000..56f0c58 --- /dev/null +++ b/maker/csharp/DbMaker/DbConfig.cs @@ -0,0 +1,67 @@ +namespace DbMaker +{ + + /** + * database configuration class + * + * @author chenxin + */ + public class DbConfig + { + /** + * total header data block size + */ + private int totalHeaderSize; + + /** + * max index data block size + * u should always choice the fastest read block size + */ + private int indexBlockSize; + + /** + * construct method + * + * @param totalHeaderSize + * @throws DbMakerConfigException + */ + public DbConfig(int totalHeaderSize) + { + if ((totalHeaderSize % 8) != 0) + { + throw new DbMakerConfigException("totalHeaderSize must be times of 8"); + } + + this.totalHeaderSize = totalHeaderSize; + this.indexBlockSize = 8192; //4 * 2048 + } + + public DbConfig() : this(8 * 2048) + { + //this(8 * 2048); + } + + public int getTotalHeaderSize() + { + return totalHeaderSize; + } + + public DbConfig setTotalHeaderSize(int totalHeaderSize) + { + this.totalHeaderSize = totalHeaderSize; + return this; + } + + public int getIndexBlockSize() + { + return indexBlockSize; + } + + public DbConfig setIndexBlockSize(int dataBlockSize) + { + this.indexBlockSize = dataBlockSize; + return this; + } + } + +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/DbMaker.cs b/maker/csharp/DbMaker/DbMaker.cs new file mode 100644 index 0000000..5e3dce5 --- /dev/null +++ b/maker/csharp/DbMaker/DbMaker.cs @@ -0,0 +1,458 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using DbMaker; + +namespace DbMaker +{ + + /** + * fast ip db maker + * + * db struct: + * 1. header part + * 1): super part: + * +------------+-----------+ + * | 4 bytes | 4 bytes | + * +------------+-----------+ + * start index ptr, end index ptr + * + * 2): b-tree index part + * +------------+-----------+-----------+-----------+ + * | 4bytes | 4bytes | 4bytes | 4bytes | ... + * +------------+-----------+-----------+-----------+ + * start ip ptr index ptr + * + * 2. data part: + * +------------+-----------------------+ + * | 2bytes | dynamic length | + * +------------+-----------------------+ + * data length city_id|Country|Province|Area|City|ISP + * + * 3. index part: (ip range) + * +------------+-----------+---------------+ + * | 4bytes | 4bytes | 4bytes | + * +------------+-----------+---------------+ + * start ip end ip 3 byte data ptr & 1 byte data length + * + * @author chenxin +*/ + public class DbMaker + { + /** + * db config + */ + private DbConfig dbConfig; + + /** + * ip source file path + */ + private FileStream ipSrcFile; + + /** + * buffer + */ + private List indexPool; + private List headerPool; + + /** + * global region Id mapping data + */ + private FileStream globalRegionFile = null; + private Dictionary globalRegionMap = null; + + /** + * region and data ptr mapping data + */ + private Dictionary regionPtrPool = null; + + /** + * construct method + * + * @param config + * @param ipSrcFile tb source ip file + * @param globalRegionFile global_region.csv file offer by lion + * @throws DbMakerConfigException + * @throws IOException + */ + public DbMaker( + DbConfig config, + String ipSrcFile, + String globalRegionFile) + { + this.dbConfig = config; + this.ipSrcFile = File.OpenRead(ipSrcFile); //new File(ipSrcFile); + this.globalRegionMap = new Dictionary(); //new HashMap(); + this.regionPtrPool = new Dictionary(); //new HashMap(); + if (globalRegionFile != null) + { + this.globalRegionFile = File.OpenRead(globalRegionFile); //new File(globalRegionFile); + } + + //if ( this.ipSrcFile.exists() == false ) { + if (!File.Exists(ipSrcFile)) + { + throw new IOException("Error: Invalid file path " + ipSrcFile); + } + } + + /** + * initialize the db file + * + * @param raf + * @throws IOException + */ + private void initDbFile(RandomAccessFile raf) + { + //1. zero fill the header part + raf.seek(0L); + raf.write(new byte[8]); //super block + raf.write(new byte[dbConfig.getTotalHeaderSize()]); //header block + + headerPool = new List(); //new LinkedList(); + indexPool = new List(); //new LinkedList(); + } + + /** + * make the Db file + * + * @param dbFile target output file path + * @throws IOException + */ + public void make(String dbFile) + { + //check and load the gloabl region + if (globalRegionFile != null) + { + Console.WriteLine("+-Try to load the global region data ..."); + StreamReader greader = new StreamReader((globalRegionFile)); + String gline = null; + while (!greader.EndOfStream) + { + gline = greader.ReadLine();// != null + if (String.IsNullOrWhiteSpace(gline)) + { + continue; + } + + String[] p = gline.Split(","); + if (p.Length != 5) continue; + + //push the mapping + globalRegionMap.put(p[2], Int32.Parse(p[0])); + } + + greader.Close(); + Console.WriteLine("|--[Ok]"); + } + + //alloc the header size + StreamReader reader = new StreamReader(this.ipSrcFile); + RandomAccessFile raf = new RandomAccessFile(dbFile); + + //init the db file + initDbFile(raf); + Console.WriteLine("+-Db file initialized."); + + //analysis main loop + Console.WriteLine("+-Try to write the data blocks ... "); + String line = null; + while (!reader.EndOfStream) + { + //line = reader.ReadLine() + line = reader.ReadLine(); + if (String.IsNullOrWhiteSpace(line)) + { + continue; + } + line = line.Trim(); + if (line.Length == 0) continue; + if (line[0] == '#') continue; + + //1. get the start ip + int sIdx = 0, eIdx = 0; + if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue; + String startIp = line.Substring(sIdx, eIdx); + + //2. get the end ip + sIdx = eIdx + 1; + if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue; + String endIp = line.Substring(sIdx, eIdx - sIdx); + + //3. get the region + sIdx = eIdx + 1; + String region = line.Substring(sIdx); + + Console.WriteLine("+-Try to process item " + line); + addDataBlock(raf, startIp, endIp, region); + Console.WriteLine("|--[Ok]"); + } + Console.WriteLine("|--Data block flushed!"); + Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n"); + + //write the index bytes + Console.WriteLine("+-Try to write index blocks ... "); + + //record the start block + IndexBlock indexBlock = null; + HeaderBlock hb = null; + indexBlock = indexPool.First(); + long indexStartIp = indexBlock.getStartIp(), + indexStratPtr = raf.getFilePointer(), indexEndPtr; + headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr))); + + int blockLength = IndexBlock.getIndexBlockLength(); + int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1; + //Iterator indexIt = indexPool.iterator(); + //while ( indexIt.hasNext() ) { + foreach (var block in indexPool) + { + if (++counter >= shotCounter) + { + hb = new HeaderBlock( + block.getStartIp(), + (int)raf.getFilePointer() + ); + + headerPool.add(hb); + counter = 0; + } + + //write the buffer + raf.write(block.getBytes()); + } + + //record the end block + if (counter > 0) + { + indexBlock = indexPool.Last(); + hb = new HeaderBlock( + indexBlock.getStartIp(), + ((int)raf.getFilePointer()) - IndexBlock.getIndexBlockLength() + ); + + headerPool.add(hb); + } + + indexEndPtr = raf.getFilePointer(); + Console.WriteLine("|--[Ok]"); + + //write the super blocks + Console.WriteLine("+-Try to write the super blocks ... "); + raf.seek(0L); //reset the file pointer + byte[] superBuffer = new byte[8]; + Util.writeIntLong(superBuffer, 0, indexStratPtr); + Util.writeIntLong(superBuffer, 4, indexEndPtr - blockLength); + raf.write(superBuffer); + Console.WriteLine("|--[Ok]"); + + //write the header blocks + Console.WriteLine("+-Try to write the header blocks ... "); + //Iterator headerIt = headerPool.iterator(); + //while ( headerIt.hasNext() ) { + // HeaderBlock headerBlock = headerIt.next(); + // raf.write(headerBlock.getBytes()); + //} + foreach (var headerBlock in headerPool) + { + raf.write(headerBlock.getBytes()); + } + + //write the copyright and the release timestamp info + Console.WriteLine("+-Try to write the copyright and release date info ... "); + raf.seek(raf.length()); + //Calendar cal = Calendar.getInstance(); + //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd"); + var date = DateTime.Now; + String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd"); + var timespan = new DateTimeOffset(date).ToUnixTimeSeconds(); + raf.write((int)(timespan)); //the unix timestamp + raf.write(copyright.getBytes("UTF-8")); + Console.WriteLine("|--[Ok]"); + + reader.Close(); + raf.close(); + } + + /** + * internal method to add a new data block record + * + * @param raf + * @param startIp + * @param endIp + * @param region data + */ + private void addDataBlock( + RandomAccessFile raf, + String startIp, String endIp, String region) + { + + byte[] data = region.getBytes("UTF-8"); + int dataPtr = 0; + + /*byte[] city = new byte[4]; + int city_id = getCityId(region); + Util.writeIntLong(city, 0, city_id); + dataPtr = (int)raf.getFilePointer(); + raf.write(city); + raf.write(data);*/ + + //check region ptr pool first + if (regionPtrPool.containsKey(region)) + { + DataBlock dataBlock = regionPtrPool.get(region); + dataPtr = dataBlock.getDataPtr(); + Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region); + } + else + { + byte[] city = new byte[4]; + int city_id = getCityId(region); + Util.writeIntLong(city, 0, city_id); + dataPtr = (int)raf.getFilePointer(); + raf.write(city); + raf.write(data); + + regionPtrPool.put(region, new DataBlock(city_id, region, dataPtr)); + } + + //add the data index blocks + IndexBlock ib = new IndexBlock( + Util.ip2long(startIp), + Util.ip2long(endIp), + dataPtr, + data.Length + 4 //4 bytes for the city id + ); + indexPool.add(ib); + } + + /** + * get the city id base on the global region data + * + * @param region + * @return int + */ + public int getCityId(String region) + { + String[] p = region.Split("|"); + if (p.Length != 5) return 0; + + String key = null; + int? intv = null; + for (int i = 3; i >= 0; i--) + { + if (p[i].Equals("0")) continue; + if (i == 3 + && p[i].IndexOf("省直辖县级") > -1) + { + key = p[2] + p[3]; + } + else + { + key = p[i]; + } + + if (!globalRegionMap.ContainsKey(key)) + { + return 0; + } + intv = globalRegionMap.get(key); + //if (intv == null) return 0; + return intv.Value; + } + + return 0; + } + + public DbConfig getDbConfig() + { + return dbConfig; + } + + public DbMaker setDbConfig(DbConfig dbConfig) + { + this.dbConfig = dbConfig; + return this; + } + + public FileStream getIpSrcFile() + { + return ipSrcFile; + } + + public DbMaker setIpSrcFile(FileStream ipSrcFile) + { + this.ipSrcFile = ipSrcFile; + return this; + } + + /** + * make this directly a runnable application + * interface to make the database file + */ + public static void Main(String[] args) + { + String dstDir = "./data/"; + String[] path = new String[] { null, null }; + for (int i = 0; i < args.Length; i++) + { + if (args[i].Equals("-src")) + { + path[0] = args[++i]; + } + else if (args[i].Equals("-region")) + { + path[1] = args[++i]; + } + else if (args[i].Equals("-dst")) + { + dstDir = args[++i]; + } + } + + for (int i = 0; i < path.Length; i++) + { + if (path[i] == null) + { + Console.WriteLine("Usage: DbMaker " + + "-src [source text file path] " + + "-region [global region file path]"); + Console.WriteLine("eg: DbMaker " + + "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv"); + //System.exit(0); + Environment.Exit(0); + } + } + + //check and stdlize the destination directory + if (!dstDir.EndsWith("/")) + { + dstDir = dstDir + "/"; + } + + try + { + if (!Directory.Exists(dstDir)) + { + Directory.CreateDirectory(dstDir); + } + DbConfig config = new DbConfig(); + DbMaker dbMaker = new DbMaker(config, path[0], path[1]); + dbMaker.make(dstDir + "ip2region.db"); + } + catch (DbMakerConfigException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/DbMaker.csproj b/maker/csharp/DbMaker/DbMaker.csproj new file mode 100644 index 0000000..2c0dacc --- /dev/null +++ b/maker/csharp/DbMaker/DbMaker.csproj @@ -0,0 +1,8 @@ + + + + Exe + netcoreapp3.1 + + + diff --git a/maker/csharp/DbMaker/DbMakerConfigException.cs b/maker/csharp/DbMaker/DbMakerConfigException.cs new file mode 100644 index 0000000..7e1399e --- /dev/null +++ b/maker/csharp/DbMaker/DbMakerConfigException.cs @@ -0,0 +1,12 @@ +using System; + +namespace DbMaker +{ + public class DbMakerConfigException : Exception + { + public DbMakerConfigException(String message) : base(message) + { + + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/DbSearcher.cs b/maker/csharp/DbMaker/DbSearcher.cs new file mode 100644 index 0000000..e8cb63b --- /dev/null +++ b/maker/csharp/DbMaker/DbSearcher.cs @@ -0,0 +1,463 @@ +using System; +using System.Text; + +namespace DbMaker +{ + + /** + * ip db searcher class (Not thread safe) + * + * @author chenxin +*/ + public class DbSearcher : IDisposable + { + public const int BTREE_ALGORITHM = 1; + public const int BINARY_ALGORITHM = 2; + public const int MEMORY_ALGORITYM = 3; + + /** + * db config + */ + private DbConfig dbConfig = null; + + /** + * db file access handler + */ + private RandomAccessFile raf = null; + + /** + * header blocks buffer + */ + private long[] HeaderSip = null; + + private int[] HeaderPtr = null; + private int headerLength; + + /** + * super blocks info + */ + private long firstIndexPtr = 0; + + private long lastIndexPtr = 0; + private int totalIndexBlocks = 0; + + /** + * for memory mode + * the original db binary string + */ + private byte[] dbBinStr = null; + + /** + * construct class + * + * @param bdConfig + * @param dbFile + * @throws FileNotFoundException + */ + public DbSearcher(DbConfig dbConfig, String dbFile) + { + this.dbConfig = dbConfig; + raf = new RandomAccessFile(dbFile); + } + + /** + * construct method with self-define std ip2region bianry string support + * Thanks to the issue from Wendal at https://gitee.com/lionsoul/ip2region/issues/IILFL + * + * @param dbConfig + * @param dbBinStr + */ + public DbSearcher(DbConfig dbConfig, byte[] dbBinStr) + { + this.dbConfig = dbConfig; + this.dbBinStr = dbBinStr; + + firstIndexPtr = Util.getIntLong(dbBinStr, 0); + lastIndexPtr = Util.getIntLong(dbBinStr, 4); + totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / IndexBlock.getIndexBlockLength()) + 1; + } + + /** + * get the region with a int ip address with memory binary search algorithm + * + * @param ip + * @throws IOException + */ + public DataBlock memorySearch(long ip) + { + int blen = IndexBlock.getIndexBlockLength(); + if (dbBinStr == null) + { + dbBinStr = new byte[(int) raf.length()]; + raf.seek(0L); + raf.readFully(dbBinStr, 0, dbBinStr.Length); + + //initialize the global vars + firstIndexPtr = Util.getIntLong(dbBinStr, 0); + lastIndexPtr = Util.getIntLong(dbBinStr, 4); + totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1; + } + + //search the index blocks to define the data + int l = 0, h = totalIndexBlocks; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + int p = (int) (firstIndexPtr + m * blen); + + sip = Util.getIntLong(dbBinStr, p); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(dbBinStr, p + 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(dbBinStr, p + 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //get the data + int dataLen = (int) ((dataptr >> 24) & 0xFF); + int dataPtr = (int) ((dataptr & 0x00FFFFFF)); + int city_id = (int) Util.getIntLong(dbBinStr, dataPtr); + //String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, "UTF-8"); + var region = ReadString(dbBinStr, dataPtr + 4, dataLen - 4); + return new DataBlock(city_id, region, dataPtr); + } + + private string ReadString(byte[] bytes, int dataPtr, int dataLen) + { + return Encoding.UTF8.GetString(bytes, dataPtr, dataLen); + } + + /** + * get the region throught the ip address with memory binary search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock memorySearch(String ip) + { + return memorySearch(Util.ip2long(ip)); + } + + + /** + * get by index ptr + * + * @param indexPtr + * @throws IOException + */ + public DataBlock getByIndexPtr(long ptr) + { + raf.seek(ptr); + byte[] buffer = new byte[12]; + raf.readFully(buffer, 0, buffer.Length); + //long startIp = Util.getIntLong(buffer, 0); + //long endIp = Util.getIntLong(buffer, 4); + long extra = Util.getIntLong(buffer, 8); + + int dataLen = (int) ((extra >> 24) & 0xFF); + int dataPtr = (int) ((extra & 0x00FFFFFF)); + + raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + raf.readFully(data, 0, data.Length); + + int city_id = (int) Util.getIntLong(data, 0); + //String region = new String(data, 4, data.Length - 4, "UTF-8"); + var region = ReadString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region with a int ip address with b-tree algorithm + * + * @param ip + * @throws IOException + */ + public DataBlock btreeSearch(long ip) + { + //check and load the header + if (HeaderSip == null) + { + raf.seek(8L); //pass the super block + byte[] b = new byte[dbConfig.getTotalHeaderSize()]; + // byte[] b = new byte[4096]; + raf.readFully(b, 0, b.Length); + + //fill the header + int len = b.Length >> 3, idx = 0; //b.lenght / 8 + HeaderSip = new long[len]; + HeaderPtr = new int [len]; + long startIp, xDataPtr; + for (int i = 0; i < b.Length; i += 8) + { + startIp = Util.getIntLong(b, i); + xDataPtr = Util.getIntLong(b, i + 4); + if (xDataPtr == 0) break; + + HeaderSip[idx] = startIp; + HeaderPtr[idx] = (int) xDataPtr; + idx++; + } + + headerLength = idx; + } + + //1. define the index block with the binary search + if (ip == HeaderSip[0]) + { + return getByIndexPtr(HeaderPtr[0]); + } + else if (ip == HeaderSip[headerLength - 1]) + { + return getByIndexPtr(HeaderPtr[headerLength - 1]); + } + + int l = 0, h = headerLength, sptr = 0, eptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + + //perfetc matched, just return it + if (ip == HeaderSip[m]) + { + if (m > 0) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + } + else + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + } + + break; + } + + //less then the middle value + if (ip < HeaderSip[m]) + { + if (m == 0) + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + break; + } + else if (ip > HeaderSip[m - 1]) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + break; + } + + h = m - 1; + } + else + { + if (m == headerLength - 1) + { + sptr = HeaderPtr[m - 1]; + eptr = HeaderPtr[m]; + break; + } + else if (ip <= HeaderSip[m + 1]) + { + sptr = HeaderPtr[m]; + eptr = HeaderPtr[m + 1]; + break; + } + + l = m + 1; + } + } + + //match nothing just stop it + if (sptr == 0) return null; + + //2. search the index blocks to define the data + int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength(); + byte[] iBuffer = new byte[blockLen + blen]; //include the right border block + raf.seek(sptr); + raf.readFully(iBuffer, 0, iBuffer.Length); + + l = 0; + h = blockLen / blen; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + int p = m * blen; + sip = Util.getIntLong(iBuffer, p); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(iBuffer, p + 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(iBuffer, p + 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //3. get the data + int dataLen = (int) ((dataptr >> 24) & 0xFF); + int dataPtr = (int) ((dataptr & 0x00FFFFFF)); + + raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + raf.readFully(data, 0, data.Length); + + int city_id = (int) Util.getIntLong(data, 0); + //String region = new String(data, 4, data.Length - 4, "UTF-8"); + var region = ReadString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region throught the ip address with b-tree search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock btreeSearch(String ip) + { + return btreeSearch(Util.ip2long(ip)); + } + + /** + * get the region with a int ip address with binary search algorithm + * + * @param ip + * @throws IOException + */ + public DataBlock binarySearch(long ip) + { + int blen = IndexBlock.getIndexBlockLength(); + if (totalIndexBlocks == 0) + { + raf.seek(0L); + byte[] superBytes = new byte[8]; + raf.readFully(superBytes, 0, superBytes.Length); + //initialize the global vars + firstIndexPtr = Util.getIntLong(superBytes, 0); + lastIndexPtr = Util.getIntLong(superBytes, 4); + totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1; + } + + //search the index blocks to define the data + int l = 0, h = totalIndexBlocks; + byte[] buffer = new byte[blen]; + long sip, eip, dataptr = 0; + while (l <= h) + { + int m = (l + h) >> 1; + raf.seek(firstIndexPtr + m * blen); //set the file pointer + raf.readFully(buffer, 0, buffer.Length); + sip = Util.getIntLong(buffer, 0); + if (ip < sip) + { + h = m - 1; + } + else + { + eip = Util.getIntLong(buffer, 4); + if (ip > eip) + { + l = m + 1; + } + else + { + dataptr = Util.getIntLong(buffer, 8); + break; + } + } + } + + //not matched + if (dataptr == 0) return null; + + //get the data + int dataLen = (int) ((dataptr >> 24) & 0xFF); + int dataPtr = (int) ((dataptr & 0x00FFFFFF)); + + raf.seek(dataPtr); + byte[] data = new byte[dataLen]; + raf.readFully(data, 0, data.Length); + + int city_id = (int) Util.getIntLong(data, 0); + //String region = new String(data, 4, data.Length - 4, "UTF-8"); + var region = ReadString(data, 4, data.Length - 4); + return new DataBlock(city_id, region, dataPtr); + } + + /** + * get the region throught the ip address with binary search algorithm + * + * @param ip + * @return DataBlock + * @throws IOException + */ + public DataBlock binarySearch(String ip) + { + return binarySearch(Util.ip2long(ip)); + } + + /** + * get the db config + * + * @return DbConfig + */ + public DbConfig getDbConfig() + { + return dbConfig; + } + + /** + * close the db + * + * @throws IOException + */ + private void close() + { + HeaderSip = null; //let gc do its work + HeaderPtr = null; + dbBinStr = null; + raf.close(); + } + + public void Dispose() + { + close(); + } + } + +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/Extensions.cs b/maker/csharp/DbMaker/Extensions.cs new file mode 100644 index 0000000..092c1f4 --- /dev/null +++ b/maker/csharp/DbMaker/Extensions.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DbMaker +{ + public static class Extensions + { + public static byte[] getBytes(this string str, string encode) + { + return Encoding.GetEncoding(encode).GetBytes(str); + } + + public static Boolean containsKey(this IDictionary dic, TKey key) + { + return dic.ContainsKey(key); + } + + public static TValue get(this IDictionary dic, TKey key) + { + return dic[key]; + } + + public static void put(this IDictionary dic, TKey key, TValue value) + { + //dic.Add(key,value); + dic[key] = value; + } + + public static void add(this IList list, TItem item) + { + list.Add(item); + } + + public static void printStackTrace(this Exception e) + { + Console.WriteLine(e); + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/HeaderBlock.cs b/maker/csharp/DbMaker/HeaderBlock.cs new file mode 100644 index 0000000..ac120fc --- /dev/null +++ b/maker/csharp/DbMaker/HeaderBlock.cs @@ -0,0 +1,70 @@ +namespace DbMaker +{ + + /** + * header block class + * + * @author chenxin +*/ + public class HeaderBlock + { + /** + * index block start ip address + */ + private long indexStartIp; + + /** + * ip address + */ + private int indexPtr; + + public HeaderBlock(long indexStartIp, int indexPtr) + { + this.indexStartIp = indexStartIp; + this.indexPtr = indexPtr; + } + + public long getIndexStartIp() + { + return indexStartIp; + } + + public HeaderBlock setIndexStartIp(long indexStartIp) + { + this.indexStartIp = indexStartIp; + return this; + } + + public int getIndexPtr() + { + return indexPtr; + } + + public HeaderBlock setIndexPtr(int indexPtr) + { + this.indexPtr = indexPtr; + return this; + } + + /** + * get the bytes for db storage + * + * @return byte[] + */ + public byte[] getBytes() + { + /* + * +------------+-----------+ + * | 4bytes | 4bytes | + * +------------+-----------+ + * start ip index ptr + */ + byte[] b = new byte[8]; + + Util.writeIntLong(b, 0, indexStartIp); + Util.writeIntLong(b, 4, indexPtr); + + return b; + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/IndexBlock.cs b/maker/csharp/DbMaker/IndexBlock.cs new file mode 100644 index 0000000..1d2c843 --- /dev/null +++ b/maker/csharp/DbMaker/IndexBlock.cs @@ -0,0 +1,115 @@ +namespace DbMaker +{ + + /** + * item index class + * + * @author chenxin + */ + public class IndexBlock + { + private static int LENGTH = 12; + + /** + * start ip address + */ + private long startIp; + + /** + * end ip address + */ + private long endIp; + + /** + * data ptr and data length + */ + private int dataPtr; + + /** + * data length + */ + private int dataLen; + + public IndexBlock(long startIp, long endIp, int dataPtr, int dataLen) + { + this.startIp = startIp; + this.endIp = endIp; + this.dataPtr = dataPtr; + this.dataLen = dataLen; + } + + public long getStartIp() + { + return startIp; + } + + public IndexBlock setStartIp(long startIp) + { + this.startIp = startIp; + return this; + } + + public long getEndIp() + { + return endIp; + } + + public IndexBlock setEndIp(long endIp) + { + this.endIp = endIp; + return this; + } + + public int getDataPtr() + { + return dataPtr; + } + + public IndexBlock setDataPtr(int dataPtr) + { + this.dataPtr = dataPtr; + return this; + } + + public int getDataLen() + { + return dataLen; + } + + public IndexBlock setDataLen(int dataLen) + { + this.dataLen = dataLen; + return this; + } + + public static int getIndexBlockLength() + { + return LENGTH; + } + + /** + * get the bytes for storage + * + * @return byte[] + */ + public byte[] getBytes() + { + /* + * +------------+-----------+-----------+ + * | 4bytes | 4bytes | 4bytes | + * +------------+-----------+-----------+ + * start ip end ip data ptr + len + */ + byte[] b = new byte[12]; + + Util.writeIntLong(b, 0, startIp); //start ip + Util.writeIntLong(b, 4, endIp); //end ip + + //write the data ptr and the length + long mix = dataPtr | ((dataLen << 24) & 0xFF000000L); + Util.writeIntLong(b, 8, mix); + + return b; + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/Properties/launchSettings.json b/maker/csharp/DbMaker/Properties/launchSettings.json new file mode 100644 index 0000000..7345383 --- /dev/null +++ b/maker/csharp/DbMaker/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "DbMaker": { + "commandName": "Project", + "commandLineArgs": " -src ip.merge.txt -region global_region.csv" + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/RandomAccessFile.cs b/maker/csharp/DbMaker/RandomAccessFile.cs new file mode 100644 index 0000000..585821e --- /dev/null +++ b/maker/csharp/DbMaker/RandomAccessFile.cs @@ -0,0 +1,64 @@ +using System; +using System.IO; + +namespace DbMaker +{ + public class RandomAccessFile : IDisposable + { + private readonly Stream _stream; + + public RandomAccessFile(String file) : this(File.Open(file, FileMode.OpenOrCreate)) + { + + } + + public RandomAccessFile(Stream stream) + { + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } + _stream = stream; + } + + public void Dispose() + { + _stream?.Dispose(); + } + + public long length() + { + return _stream.Length; + } + + public void seek(long position) + { + _stream.Seek(position, SeekOrigin.Begin); + } + + public void readFully(byte[] dbBinStr, int offset, int count) + { + _stream.Read(dbBinStr, offset, count); + } + + public void close() + { + _stream.Dispose(); + } + + public void write(byte[] bytes) + { + _stream.Write(bytes); + } + + public void write(int i) + { + + } + + public long getFilePointer() + { + return _stream.Position; + } + } +} \ No newline at end of file diff --git a/maker/csharp/DbMaker/Util.cs b/maker/csharp/DbMaker/Util.cs new file mode 100644 index 0000000..a92f8af --- /dev/null +++ b/maker/csharp/DbMaker/Util.cs @@ -0,0 +1,153 @@ +using System; +using System.Text; + +namespace DbMaker +{ + + + /** + * util class + * + * @author chenxin + */ + public class Util + { + /** + * write specfield bytes to a byte array start from offset + * + * @param b + * @param offset + * @param v + * @param bytes + */ + public static void write(byte[] b, int offset, long v, int bytes) + { + for (int i = 0; i < bytes; i++) + { + //b[offset++] = (byte)((v >>> (8 * i)) & 0xFF); + b[offset++] = (byte) ((v >> (8 * i)) & 0xFF); + } + } + + /** + * write a int to a byte array + * + * @param b + * @param offet + * @param v + */ + public static void writeIntLong(byte[] b, int offset, long v) + { + b[offset++] = (byte) ((v >> 0) & 0xFF); + b[offset++] = (byte) ((v >> 8) & 0xFF); + b[offset++] = (byte) ((v >> 16) & 0xFF); + b[offset] = (byte) ((v >> 24) & 0xFF); + } + + /** + * get a int from a byte array start from the specifiled offset + * + * @param b + * @param offset + */ + public static long getIntLong(byte[] b, int offset) + { + return ( + ((b[offset++] & 0x000000FFL)) | + ((b[offset++] << 8) & 0x0000FF00L) | + ((b[offset++] << 16) & 0x00FF0000L) | + ((b[offset] << 24) & 0xFF000000L) + ); + } + + /** + * get a int from a byte array start from the specifield offset + * + * @param b + * @param offset + */ + public static int getInt3(byte[] b, int offset) + { + return ( + (b[offset++] & 0x000000FF) | + (b[offset++] & 0x0000FF00) | + (b[offset] & 0x00FF0000) + ); + } + + public static int getInt2(byte[] b, int offset) + { + return ( + (b[offset++] & 0x000000FF) | + (b[offset] & 0x0000FF00) + ); + } + + public static int getInt1(byte[] b, int offset) + { + return ( + (b[offset] & 0x000000FF) + ); + } + + /** + * string ip to long ip + * + * @param ip + * @return long + */ + public static long ip2long(String ip) + { + ip = ip.Trim(); + String[] ips = ip.Split('.'); + long ip1 = Int64.Parse(ips[0]); + long ip2 = Int64.Parse(ips[1]); + long ip3 = Int64.Parse(ips[2]); + long ip4 = Int64.Parse(ips[3]); + long ip2long = 1L * ip1 * 256 * 256 * 256 + ip2 * 256 * 256 + ip3 * 256 + ip4; + return ip2long; + } + + /** + * int to ip string + * + * @param ip + * @return string + */ + public static String long2ip(long ip) + { + StringBuilder sb = new StringBuilder(); + + sb + .Append((ip >> 24) & 0xFF).Append('.') + .Append((ip >> 16) & 0xFF).Append('.') + .Append((ip >> 8) & 0xFF).Append('.') + .Append((ip >> 0) & 0xFF); + + return sb.ToString(); + } + + /** + * check the validate of the specifeld ip address + * + * @param ip + * @return boolean + */ + public static Boolean isIpAddress(String ip) + { + String[] p = ip.Split("\\."); + if (p.Length != 4) return false; + + //for ( String pp : p ) { + //for ( String pp : p ) { + foreach (var pp in p) + { + if (pp.Length > 3) return false; + int val = Int32.Parse(pp); //Integer.valueOf(pp); + if (val > 255) return false; + } + + return true; + } + } +} \ No newline at end of file