xinb
/
ip2region_private
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add CSharp Macker

This commit is contained in:
Soar360 2020-07-09 14:01:07 +08:00
parent c18377d4c1
commit 9c70364080
4 changed files with 452 additions and 408 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
maker/csharp/DbMaker/DbMaker.cs
View File

@ -9,7 +9,7 @@ using DbMaker;
namespace DbMaker namespace DbMaker
{ {
/** /**
* fast ip db maker * fast ip db maker
* *
* db struct: * db struct:
@ -40,8 +40,8 @@ namespace DbMaker
* *
* @author chenxin<chenxin619315@gmail.com> * @author chenxin<chenxin619315@gmail.com>
*/ */
public class DbMaker public class DbMaker
{ {
/** /**
* db config * db config
*/ */
@ -81,13 +81,13 @@ public class DbMaker
public DbMaker( public DbMaker(
DbConfig config, DbConfig config,
String ipSrcFile, String ipSrcFile,
String globalRegionFile ) String globalRegionFile)
{ {
this.dbConfig = config; this.dbConfig = config;
this.ipSrcFile = File.OpenRead(ipSrcFile); //new File(ipSrcFile); this.ipSrcFile = File.OpenRead(ipSrcFile); //new File(ipSrcFile);
this.globalRegionMap = new Dictionary<string, int>(); //new HashMap<String, Integer>(); this.globalRegionMap = new Dictionary<string, int>(); //new HashMap<String, Integer>();
this.regionPtrPool = new Dictionary<string, DataBlock>(); //new HashMap<String, DataBlock>(); this.regionPtrPool = new Dictionary<string, DataBlock>(); //new HashMap<String, DataBlock>();
if ( globalRegionFile != null ) if (globalRegionFile != null)
{ {
this.globalRegionFile = File.OpenRead(globalRegionFile); //new File(globalRegionFile); this.globalRegionFile = File.OpenRead(globalRegionFile); //new File(globalRegionFile);
} }
@ -105,7 +105,7 @@ public class DbMaker
* @param raf * @param raf
* @throws IOException * @throws IOException
*/ */
private void initDbFile( RandomAccessFile raf ) private void initDbFile(RandomAccessFile raf)
{ {
//1. zero fill the header part //1. zero fill the header part
raf.seek(0L); raf.seek(0L);
@ -125,11 +125,13 @@ public class DbMaker
public void make(String dbFile) public void make(String dbFile)
{ {
//check and load the gloabl region //check and load the gloabl region
if ( globalRegionFile != null ) { if (globalRegionFile != null)
{
Console.WriteLine("+-Try to load the global region data ..."); Console.WriteLine("+-Try to load the global region data ...");
StreamReader greader = new StreamReader((globalRegionFile)); StreamReader greader = new StreamReader((globalRegionFile));
String gline = null; String gline = null;
while (!greader.EndOfStream ) { while (!greader.EndOfStream)
{
gline = greader.ReadLine();// != null gline = greader.ReadLine();// != null
if (String.IsNullOrWhiteSpace(gline)) if (String.IsNullOrWhiteSpace(gline))
{ {
@ -137,7 +139,7 @@ public class DbMaker
} }
String[] p = gline.Split(","); String[] p = gline.Split(",");
if ( p.Length != 5 ) continue; if (p.Length != 5) continue;
//push the mapping //push the mapping
globalRegionMap.put(p[2], Int32.Parse(p[0])); globalRegionMap.put(p[2], Int32.Parse(p[0]));
@ -158,7 +160,8 @@ public class DbMaker
//analysis main loop //analysis main loop
Console.WriteLine("+-Try to write the data blocks ... "); Console.WriteLine("+-Try to write the data blocks ... ");
String line = null; String line = null;
while ( !reader.EndOfStream ) { while (!reader.EndOfStream)
{
//line = reader.ReadLine() //line = reader.ReadLine()
line = reader.ReadLine(); line = reader.ReadLine();
if (String.IsNullOrWhiteSpace(line)) if (String.IsNullOrWhiteSpace(line))
@ -166,18 +169,18 @@ public class DbMaker
continue; continue;
} }
line = line.Trim(); line = line.Trim();
if ( line.Length == 0 ) continue; if (line.Length == 0) continue;
if ( line[0] == '#' ) continue; if (line[0] == '#') continue;
//1. get the start ip //1. get the start ip
int sIdx = 0, eIdx = 0; int sIdx = 0, eIdx = 0;
if ( (eIdx = line.IndexOf('|', sIdx + 1)) == -1 ) continue; if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue;
String startIp = line.Substring(sIdx, eIdx); String startIp = line.Substring(sIdx, eIdx);
//2. get the end ip //2. get the end ip
sIdx = eIdx + 1; sIdx = eIdx + 1;
if ( (eIdx = line.IndexOf('|', sIdx + 1)) == -1 ) continue; if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue;
String endIp = line.Substring(sIdx, eIdx); String endIp = line.Substring(sIdx, eIdx - sIdx);
//3. get the region //3. get the region
sIdx = eIdx + 1; sIdx = eIdx + 1;
@ -188,7 +191,7 @@ public class DbMaker
Console.WriteLine("|--[Ok]"); Console.WriteLine("|--[Ok]");
} }
Console.WriteLine("|--Data block flushed!"); Console.WriteLine("|--Data block flushed!");
Console.WriteLine("|--Data file pointer: "+raf.getFilePointer()+"\n"); Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n");
//write the index bytes //write the index bytes
Console.WriteLine("+-Try to write index blocks ... "); Console.WriteLine("+-Try to write index blocks ... ");
@ -202,11 +205,13 @@ public class DbMaker
headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr))); headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr)));
int blockLength = IndexBlock.getIndexBlockLength(); int blockLength = IndexBlock.getIndexBlockLength();
int counter = 0, shotCounter = (dbConfig.getIndexBlockSize()/blockLength) - 1; int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1;
//Iterator<IndexBlock> indexIt = indexPool.iterator(); //Iterator<IndexBlock> indexIt = indexPool.iterator();
//while ( indexIt.hasNext() ) { //while ( indexIt.hasNext() ) {
foreach(var block in indexPool){ foreach (var block in indexPool)
if ( ++counter >= shotCounter ) { {
if (++counter >= shotCounter)
{
hb = new HeaderBlock( hb = new HeaderBlock(
block.getStartIp(), block.getStartIp(),
(int)raf.getFilePointer() (int)raf.getFilePointer()
@ -221,7 +226,8 @@ public class DbMaker
} }
//record the end block //record the end block
if ( counter > 0 ) { if (counter > 0)
{
indexBlock = indexPool.Last(); indexBlock = indexPool.Last();
hb = new HeaderBlock( hb = new HeaderBlock(
indexBlock.getStartIp(), indexBlock.getStartIp(),
@ -262,7 +268,7 @@ public class DbMaker
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd"); //SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
var date = DateTime.Now; var date = DateTime.Now;
String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd"); String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd");
var timespan= new DateTimeOffset(date).ToUnixTimeSeconds(); var timespan = new DateTimeOffset(date).ToUnixTimeSeconds();
raf.write((int)(timespan)); //the unix timestamp raf.write((int)(timespan)); //the unix timestamp
raf.write(copyright.getBytes("UTF-8")); raf.write(copyright.getBytes("UTF-8"));
Console.WriteLine("|--[Ok]"); Console.WriteLine("|--[Ok]");
@ -295,11 +301,14 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
raf.write(data);*/ raf.write(data);*/
//check region ptr pool first //check region ptr pool first
if ( regionPtrPool.containsKey(region) ) { if (regionPtrPool.containsKey(region))
{
DataBlock dataBlock = regionPtrPool.get(region); DataBlock dataBlock = regionPtrPool.get(region);
dataPtr = dataBlock.getDataPtr(); dataPtr = dataBlock.getDataPtr();
Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region); Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region);
} else { }
else
{
byte[] city = new byte[4]; byte[] city = new byte[4];
int city_id = getCityId(region); int city_id = getCityId(region);
Util.writeIntLong(city, 0, city_id); Util.writeIntLong(city, 0, city_id);
@ -326,24 +335,32 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
* @param region * @param region
* @return int * @return int
*/ */
public int getCityId( String region ) public int getCityId(String region)
{ {
String[] p = region.Split("\\|"); String[] p = region.Split("|");
if ( p.Length != 5 ) return 0; if (p.Length != 5) return 0;
String key = null; String key = null;
int? intv = null; int? intv = null;
for ( int i = 3; i >= 0; i-- ) { for (int i = 3; i >= 0; i--)
if ( p[i].Equals("0") ) continue; {
if ( i == 3 if (p[i].Equals("0")) continue;
&& p[i].IndexOf("省直辖县级") > -1 ) { if (i == 3
key = p[2]+p[3]; && p[i].IndexOf("省直辖县级") > -1)
} else { {
key = p[2] + p[3];
}
else
{
key = p[i]; key = p[i];
} }
if (!globalRegionMap.ContainsKey(key))
{
return 0;
}
intv = globalRegionMap.get(key); intv = globalRegionMap.get(key);
if ( intv == null ) return 0; //if (intv == null) return 0;
return intv.Value; return intv.Value;
} }
@ -379,23 +396,31 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
public static void Main(String[] args) public static void Main(String[] args)
{ {
String dstDir = "./data/"; String dstDir = "./data/";
String[] path = new String[]{null, null}; String[] path = new String[] { null, null };
for ( int i = 0; i < args.Length; i++ ) { for (int i = 0; i < args.Length; i++)
if ( args[i].Equals("-src") ) { {
if (args[i].Equals("-src"))
{
path[0] = args[++i]; path[0] = args[++i];
} else if ( args[i].Equals("-region") ) { }
else if (args[i].Equals("-region"))
{
path[1] = args[++i]; path[1] = args[++i];
} else if ( args[i].Equals("-dst") ) { }
else if (args[i].Equals("-dst"))
{
dstDir = args[++i]; dstDir = args[++i];
} }
} }
for ( int i = 0; i < path.Length; i++ ) { for (int i = 0; i < path.Length; i++)
if ( path[i] == null ) { {
Console.WriteLine("Usage: java -jar dbMaker.jar " if (path[i] == null)
{
Console.WriteLine("Usage: DbMaker "
+ "-src [source text file path] " + "-src [source text file path] "
+ "-region [global region file path]"); + "-region [global region file path]");
Console.WriteLine("eg: java -jar dbMaker.jar " Console.WriteLine("eg: DbMaker "
+ "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv"); + "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv");
//System.exit(0); //System.exit(0);
Environment.Exit(0); Environment.Exit(0);
@ -403,21 +428,31 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
} }
//check and stdlize the destination directory //check and stdlize the destination directory
if ( ! dstDir.EndsWith("/") ) { if (!dstDir.EndsWith("/"))
{
dstDir = dstDir + "/"; dstDir = dstDir + "/";
} }
try { try
{
if (!Directory.Exists(dstDir))
{
Directory.CreateDirectory(dstDir);
}
DbConfig config = new DbConfig(); DbConfig config = new DbConfig();
DbMaker dbMaker = new DbMaker(config, path[0], path[1]); DbMaker dbMaker = new DbMaker(config, path[0], path[1]);
dbMaker.make(dstDir + "ip2region.db"); dbMaker.make(dstDir + "ip2region.db");
} catch (DbMakerConfigException e) { }
catch (DbMakerConfigException e)
{
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (IOException e) { }
catch (IOException e)
{
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
} }

3
maker/csharp/DbMaker/Extensions.cs
View File

@ -23,7 +23,8 @@ namespace DbMaker
public static void put<TKey, TValue>(this IDictionary<TKey, TValue> dic, TKey key, TValue value) public static void put<TKey, TValue>(this IDictionary<TKey, TValue> dic, TKey key, TValue value)
{ {
dic.Add(key,value); //dic.Add(key,value);
dic[key] = value;
} }
public static void add<TItem>(this IList<TItem> list, TItem item) public static void add<TItem>(this IList<TItem> list, TItem item)

8
maker/csharp/DbMaker/Properties/launchSettings.json Normal file
View File

@ -0,0 +1,8 @@
{
"profiles": {
"DbMaker": {
"commandName": "Project",
"commandLineArgs": " -src ip.merge.txt -region global_region.csv"
}
}
}

2
maker/csharp/DbMaker/RandomAccessFile.cs
View File

@ -7,7 +7,7 @@ namespace DbMaker
{ {
private readonly Stream _stream; private readonly Stream _stream;
public RandomAccessFile(String file) : this(File.OpenRead(file)) public RandomAccessFile(String file) : this(File.Open(file, FileMode.OpenOrCreate))
{ {
} }