add CSharp Macker

This commit is contained in:
Soar360 2020-07-09 14:01:07 +08:00
parent 56a9fe834e
commit 9101c218d3
4 changed files with 452 additions and 408 deletions

View File

@ -125,11 +125,13 @@ public class DbMaker
public void make(String dbFile) public void make(String dbFile)
{ {
//check and load the gloabl region //check and load the gloabl region
if ( globalRegionFile != null ) { if (globalRegionFile != null)
{
Console.WriteLine("+-Try to load the global region data ..."); Console.WriteLine("+-Try to load the global region data ...");
StreamReader greader = new StreamReader((globalRegionFile)); StreamReader greader = new StreamReader((globalRegionFile));
String gline = null; String gline = null;
while (!greader.EndOfStream ) { while (!greader.EndOfStream)
{
gline = greader.ReadLine();// != null gline = greader.ReadLine();// != null
if (String.IsNullOrWhiteSpace(gline)) if (String.IsNullOrWhiteSpace(gline))
{ {
@ -158,7 +160,8 @@ public class DbMaker
//analysis main loop //analysis main loop
Console.WriteLine("+-Try to write the data blocks ... "); Console.WriteLine("+-Try to write the data blocks ... ");
String line = null; String line = null;
while ( !reader.EndOfStream ) { while (!reader.EndOfStream)
{
//line = reader.ReadLine() //line = reader.ReadLine()
line = reader.ReadLine(); line = reader.ReadLine();
if (String.IsNullOrWhiteSpace(line)) if (String.IsNullOrWhiteSpace(line))
@ -177,7 +180,7 @@ public class DbMaker
//2. get the end ip //2. get the end ip
sIdx = eIdx + 1; sIdx = eIdx + 1;
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue; if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue;
String endIp = line.Substring(sIdx, eIdx); String endIp = line.Substring(sIdx, eIdx - sIdx);
//3. get the region //3. get the region
sIdx = eIdx + 1; sIdx = eIdx + 1;
@ -205,8 +208,10 @@ public class DbMaker
int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1; int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1;
//Iterator<IndexBlock> indexIt = indexPool.iterator(); //Iterator<IndexBlock> indexIt = indexPool.iterator();
//while ( indexIt.hasNext() ) { //while ( indexIt.hasNext() ) {
foreach(var block in indexPool){ foreach (var block in indexPool)
if ( ++counter >= shotCounter ) { {
if (++counter >= shotCounter)
{
hb = new HeaderBlock( hb = new HeaderBlock(
block.getStartIp(), block.getStartIp(),
(int)raf.getFilePointer() (int)raf.getFilePointer()
@ -221,7 +226,8 @@ public class DbMaker
} }
//record the end block //record the end block
if ( counter > 0 ) { if (counter > 0)
{
indexBlock = indexPool.Last(); indexBlock = indexPool.Last();
hb = new HeaderBlock( hb = new HeaderBlock(
indexBlock.getStartIp(), indexBlock.getStartIp(),
@ -295,11 +301,14 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
raf.write(data);*/ raf.write(data);*/
//check region ptr pool first //check region ptr pool first
if ( regionPtrPool.containsKey(region) ) { if (regionPtrPool.containsKey(region))
{
DataBlock dataBlock = regionPtrPool.get(region); DataBlock dataBlock = regionPtrPool.get(region);
dataPtr = dataBlock.getDataPtr(); dataPtr = dataBlock.getDataPtr();
Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region); Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region);
} else { }
else
{
byte[] city = new byte[4]; byte[] city = new byte[4];
int city_id = getCityId(region); int city_id = getCityId(region);
Util.writeIntLong(city, 0, city_id); Util.writeIntLong(city, 0, city_id);
@ -328,22 +337,30 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
*/ */
public int getCityId(String region) public int getCityId(String region)
{ {
String[] p = region.Split("\\|"); String[] p = region.Split("|");
if (p.Length != 5) return 0; if (p.Length != 5) return 0;
String key = null; String key = null;
int? intv = null; int? intv = null;
for ( int i = 3; i >= 0; i-- ) { for (int i = 3; i >= 0; i--)
{
if (p[i].Equals("0")) continue; if (p[i].Equals("0")) continue;
if (i == 3 if (i == 3
&& p[i].IndexOf("省直辖县级") > -1 ) { && p[i].IndexOf("省直辖县级") > -1)
{
key = p[2] + p[3]; key = p[2] + p[3];
} else { }
else
{
key = p[i]; key = p[i];
} }
if (!globalRegionMap.ContainsKey(key))
{
return 0;
}
intv = globalRegionMap.get(key); intv = globalRegionMap.get(key);
if ( intv == null ) return 0; //if (intv == null) return 0;
return intv.Value; return intv.Value;
} }
@ -380,22 +397,30 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
{ {
String dstDir = "./data/"; String dstDir = "./data/";
String[] path = new String[] { null, null }; String[] path = new String[] { null, null };
for ( int i = 0; i < args.Length; i++ ) { for (int i = 0; i < args.Length; i++)
if ( args[i].Equals("-src") ) { {
if (args[i].Equals("-src"))
{
path[0] = args[++i]; path[0] = args[++i];
} else if ( args[i].Equals("-region") ) { }
else if (args[i].Equals("-region"))
{
path[1] = args[++i]; path[1] = args[++i];
} else if ( args[i].Equals("-dst") ) { }
else if (args[i].Equals("-dst"))
{
dstDir = args[++i]; dstDir = args[++i];
} }
} }
for ( int i = 0; i < path.Length; i++ ) { for (int i = 0; i < path.Length; i++)
if ( path[i] == null ) { {
Console.WriteLine("Usage: java -jar dbMaker.jar " if (path[i] == null)
{
Console.WriteLine("Usage: DbMaker "
+ "-src [source text file path] " + "-src [source text file path] "
+ "-region [global region file path]"); + "-region [global region file path]");
Console.WriteLine("eg: java -jar dbMaker.jar " Console.WriteLine("eg: DbMaker "
+ "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv"); + "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv");
//System.exit(0); //System.exit(0);
Environment.Exit(0); Environment.Exit(0);
@ -403,18 +428,28 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
} }
//check and stdlize the destination directory //check and stdlize the destination directory
if ( ! dstDir.EndsWith("/") ) { if (!dstDir.EndsWith("/"))
{
dstDir = dstDir + "/"; dstDir = dstDir + "/";
} }
try { try
{
if (!Directory.Exists(dstDir))
{
Directory.CreateDirectory(dstDir);
}
DbConfig config = new DbConfig(); DbConfig config = new DbConfig();
DbMaker dbMaker = new DbMaker(config, path[0], path[1]); DbMaker dbMaker = new DbMaker(config, path[0], path[1]);
dbMaker.make(dstDir + "ip2region.db"); dbMaker.make(dstDir + "ip2region.db");
} catch (DbMakerConfigException e) { }
catch (DbMakerConfigException e)
{
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} catch (IOException e) { }
catch (IOException e)
{
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }

View File

@ -23,7 +23,8 @@ namespace DbMaker
public static void put<TKey, TValue>(this IDictionary<TKey, TValue> dic, TKey key, TValue value) public static void put<TKey, TValue>(this IDictionary<TKey, TValue> dic, TKey key, TValue value)
{ {
dic.Add(key,value); //dic.Add(key,value);
dic[key] = value;
} }
public static void add<TItem>(this IList<TItem> list, TItem item) public static void add<TItem>(this IList<TItem> list, TItem item)

View File

@ -0,0 +1,8 @@
{
"profiles": {
"DbMaker": {
"commandName": "Project",
"commandLineArgs": " -src ip.merge.txt -region global_region.csv"
}
}
}

View File

@ -7,7 +7,7 @@ namespace DbMaker
{ {
private readonly Stream _stream; private readonly Stream _stream;
public RandomAccessFile(String file) : this(File.OpenRead(file)) public RandomAccessFile(String file) : this(File.Open(file, FileMode.OpenOrCreate))
{ {
} }