add CSharp Macker
This commit is contained in:
Soar360
parent
c18377d4c1
commit
9c70364080
|
|
@ -9,7 +9,7 @@ using DbMaker;
|
|||
namespace DbMaker
|
||||
{
|
||||
|
||||
/**
|
||||
/**
|
||||
* fast ip db maker
|
||||
*
|
||||
* db struct:
|
||||
|
|
@ -40,8 +40,8 @@ namespace DbMaker
|
|||
*
|
||||
* @author chenxin<chenxin619315@gmail.com>
|
||||
*/
|
||||
public class DbMaker
|
||||
{
|
||||
public class DbMaker
|
||||
{
|
||||
/**
|
||||
* db config
|
||||
*/
|
||||
|
|
@ -81,13 +81,13 @@ public class DbMaker
|
|||
public DbMaker(
|
||||
DbConfig config,
|
||||
String ipSrcFile,
|
||||
String globalRegionFile )
|
||||
String globalRegionFile)
|
||||
{
|
||||
this.dbConfig = config;
|
||||
this.ipSrcFile = File.OpenRead(ipSrcFile); //new File(ipSrcFile);
|
||||
this.globalRegionMap = new Dictionary<string, int>(); //new HashMap<String, Integer>();
|
||||
this.regionPtrPool = new Dictionary<string, DataBlock>(); //new HashMap<String, DataBlock>();
|
||||
if ( globalRegionFile != null )
|
||||
if (globalRegionFile != null)
|
||||
{
|
||||
this.globalRegionFile = File.OpenRead(globalRegionFile); //new File(globalRegionFile);
|
||||
}
|
||||
|
|
@ -105,7 +105,7 @@ public class DbMaker
|
|||
* @param raf
|
||||
* @throws IOException
|
||||
*/
|
||||
private void initDbFile( RandomAccessFile raf )
|
||||
private void initDbFile(RandomAccessFile raf)
|
||||
{
|
||||
//1. zero fill the header part
|
||||
raf.seek(0L);
|
||||
|
|
@ -125,11 +125,13 @@ public class DbMaker
|
|||
public void make(String dbFile)
|
||||
{
|
||||
//check and load the gloabl region
|
||||
if ( globalRegionFile != null ) {
|
||||
if (globalRegionFile != null)
|
||||
{
|
||||
Console.WriteLine("+-Try to load the global region data ...");
|
||||
StreamReader greader = new StreamReader((globalRegionFile));
|
||||
String gline = null;
|
||||
while (!greader.EndOfStream ) {
|
||||
while (!greader.EndOfStream)
|
||||
{
|
||||
gline = greader.ReadLine();// != null
|
||||
if (String.IsNullOrWhiteSpace(gline))
|
||||
{
|
||||
|
|
@ -137,7 +139,7 @@ public class DbMaker
|
|||
}
|
||||
|
||||
String[] p = gline.Split(",");
|
||||
if ( p.Length != 5 ) continue;
|
||||
if (p.Length != 5) continue;
|
||||
|
||||
//push the mapping
|
||||
globalRegionMap.put(p[2], Int32.Parse(p[0]));
|
||||
|
|
@ -158,7 +160,8 @@ public class DbMaker
|
|||
//analysis main loop
|
||||
Console.WriteLine("+-Try to write the data blocks ... ");
|
||||
String line = null;
|
||||
while ( !reader.EndOfStream ) {
|
||||
while (!reader.EndOfStream)
|
||||
{
|
||||
//line = reader.ReadLine()
|
||||
line = reader.ReadLine();
|
||||
if (String.IsNullOrWhiteSpace(line))
|
||||
|
|
@ -166,18 +169,18 @@ public class DbMaker
|
|||
continue;
|
||||
}
|
||||
line = line.Trim();
|
||||
if ( line.Length == 0 ) continue;
|
||||
if ( line[0] == '#' ) continue;
|
||||
if (line.Length == 0) continue;
|
||||
if (line[0] == '#') continue;
|
||||
|
||||
//1. get the start ip
|
||||
int sIdx = 0, eIdx = 0;
|
||||
if ( (eIdx = line.IndexOf('|', sIdx + 1)) == -1 ) continue;
|
||||
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue;
|
||||
String startIp = line.Substring(sIdx, eIdx);
|
||||
|
||||
//2. get the end ip
|
||||
sIdx = eIdx + 1;
|
||||
if ( (eIdx = line.IndexOf('|', sIdx + 1)) == -1 ) continue;
|
||||
String endIp = line.Substring(sIdx, eIdx);
|
||||
if ((eIdx = line.IndexOf('|', sIdx + 1)) == -1) continue;
|
||||
String endIp = line.Substring(sIdx, eIdx - sIdx);
|
||||
|
||||
//3. get the region
|
||||
sIdx = eIdx + 1;
|
||||
|
|
@ -188,7 +191,7 @@ public class DbMaker
|
|||
Console.WriteLine("|--[Ok]");
|
||||
}
|
||||
Console.WriteLine("|--Data block flushed!");
|
||||
Console.WriteLine("|--Data file pointer: "+raf.getFilePointer()+"\n");
|
||||
Console.WriteLine("|--Data file pointer: " + raf.getFilePointer() + "\n");
|
||||
|
||||
//write the index bytes
|
||||
Console.WriteLine("+-Try to write index blocks ... ");
|
||||
|
|
@ -202,11 +205,13 @@ public class DbMaker
|
|||
headerPool.add(new HeaderBlock(indexStartIp, (int)(indexStratPtr)));
|
||||
|
||||
int blockLength = IndexBlock.getIndexBlockLength();
|
||||
int counter = 0, shotCounter = (dbConfig.getIndexBlockSize()/blockLength) - 1;
|
||||
int counter = 0, shotCounter = (dbConfig.getIndexBlockSize() / blockLength) - 1;
|
||||
//Iterator<IndexBlock> indexIt = indexPool.iterator();
|
||||
//while ( indexIt.hasNext() ) {
|
||||
foreach(var block in indexPool){
|
||||
if ( ++counter >= shotCounter ) {
|
||||
foreach (var block in indexPool)
|
||||
{
|
||||
if (++counter >= shotCounter)
|
||||
{
|
||||
hb = new HeaderBlock(
|
||||
block.getStartIp(),
|
||||
(int)raf.getFilePointer()
|
||||
|
|
@ -221,7 +226,8 @@ public class DbMaker
|
|||
}
|
||||
|
||||
//record the end block
|
||||
if ( counter > 0 ) {
|
||||
if (counter > 0)
|
||||
{
|
||||
indexBlock = indexPool.Last();
|
||||
hb = new HeaderBlock(
|
||||
indexBlock.getStartIp(),
|
||||
|
|
@ -262,7 +268,7 @@ public class DbMaker
|
|||
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
|
||||
var date = DateTime.Now;
|
||||
String copyright = "Created by lionsoul at " + date.ToString("yyyy/MM/dd");
|
||||
var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
|
||||
var timespan = new DateTimeOffset(date).ToUnixTimeSeconds();
|
||||
raf.write((int)(timespan)); //the unix timestamp
|
||||
raf.write(copyright.getBytes("UTF-8"));
|
||||
Console.WriteLine("|--[Ok]");
|
||||
|
|
@ -295,11 +301,14 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
|
|||
raf.write(data);*/
|
||||
|
||||
//check region ptr pool first
|
||||
if ( regionPtrPool.containsKey(region) ) {
|
||||
if (regionPtrPool.containsKey(region))
|
||||
{
|
||||
DataBlock dataBlock = regionPtrPool.get(region);
|
||||
dataPtr = dataBlock.getDataPtr();
|
||||
Console.WriteLine("dataPtr: " + dataPtr + ", region: " + region);
|
||||
} else {
|
||||
}
|
||||
else
|
||||
{
|
||||
byte[] city = new byte[4];
|
||||
int city_id = getCityId(region);
|
||||
Util.writeIntLong(city, 0, city_id);
|
||||
|
|
@ -326,24 +335,32 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
|
|||
* @param region
|
||||
* @return int
|
||||
*/
|
||||
public int getCityId( String region )
|
||||
public int getCityId(String region)
|
||||
{
|
||||
String[] p = region.Split("\\|");
|
||||
if ( p.Length != 5 ) return 0;
|
||||
String[] p = region.Split("|");
|
||||
if (p.Length != 5) return 0;
|
||||
|
||||
String key = null;
|
||||
int? intv = null;
|
||||
for ( int i = 3; i >= 0; i-- ) {
|
||||
if ( p[i].Equals("0") ) continue;
|
||||
if ( i == 3
|
||||
&& p[i].IndexOf("省直辖县级") > -1 ) {
|
||||
key = p[2]+p[3];
|
||||
} else {
|
||||
for (int i = 3; i >= 0; i--)
|
||||
{
|
||||
if (p[i].Equals("0")) continue;
|
||||
if (i == 3
|
||||
&& p[i].IndexOf("省直辖县级") > -1)
|
||||
{
|
||||
key = p[2] + p[3];
|
||||
}
|
||||
else
|
||||
{
|
||||
key = p[i];
|
||||
}
|
||||
|
||||
if (!globalRegionMap.ContainsKey(key))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
intv = globalRegionMap.get(key);
|
||||
if ( intv == null ) return 0;
|
||||
//if (intv == null) return 0;
|
||||
return intv.Value;
|
||||
}
|
||||
|
||||
|
|
@ -379,23 +396,31 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
|
|||
public static void Main(String[] args)
|
||||
{
|
||||
String dstDir = "./data/";
|
||||
String[] path = new String[]{null, null};
|
||||
for ( int i = 0; i < args.Length; i++ ) {
|
||||
if ( args[i].Equals("-src") ) {
|
||||
String[] path = new String[] { null, null };
|
||||
for (int i = 0; i < args.Length; i++)
|
||||
{
|
||||
if (args[i].Equals("-src"))
|
||||
{
|
||||
path[0] = args[++i];
|
||||
} else if ( args[i].Equals("-region") ) {
|
||||
}
|
||||
else if (args[i].Equals("-region"))
|
||||
{
|
||||
path[1] = args[++i];
|
||||
} else if ( args[i].Equals("-dst") ) {
|
||||
}
|
||||
else if (args[i].Equals("-dst"))
|
||||
{
|
||||
dstDir = args[++i];
|
||||
}
|
||||
}
|
||||
|
||||
for ( int i = 0; i < path.Length; i++ ) {
|
||||
if ( path[i] == null ) {
|
||||
Console.WriteLine("Usage: java -jar dbMaker.jar "
|
||||
for (int i = 0; i < path.Length; i++)
|
||||
{
|
||||
if (path[i] == null)
|
||||
{
|
||||
Console.WriteLine("Usage: DbMaker "
|
||||
+ "-src [source text file path] "
|
||||
+ "-region [global region file path]");
|
||||
Console.WriteLine("eg: java -jar dbMaker.jar "
|
||||
Console.WriteLine("eg: DbMaker "
|
||||
+ "-src ./data/ip.merge.txt -region ./data/origin/global_region.csv");
|
||||
//System.exit(0);
|
||||
Environment.Exit(0);
|
||||
|
|
@ -403,21 +428,31 @@ var timespan= new DateTimeOffset(date).ToUnixTimeSeconds();
|
|||
}
|
||||
|
||||
//check and stdlize the destination directory
|
||||
if ( ! dstDir.EndsWith("/") ) {
|
||||
if (!dstDir.EndsWith("/"))
|
||||
{
|
||||
dstDir = dstDir + "/";
|
||||
}
|
||||
|
||||
try {
|
||||
try
|
||||
{
|
||||
if (!Directory.Exists(dstDir))
|
||||
{
|
||||
Directory.CreateDirectory(dstDir);
|
||||
}
|
||||
DbConfig config = new DbConfig();
|
||||
DbMaker dbMaker = new DbMaker(config, path[0], path[1]);
|
||||
dbMaker.make(dstDir + "ip2region.db");
|
||||
} catch (DbMakerConfigException e) {
|
||||
}
|
||||
catch (DbMakerConfigException e)
|
||||
{
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -23,7 +23,8 @@ namespace DbMaker
|
|||
|
||||
public static void put<TKey, TValue>(this IDictionary<TKey, TValue> dic, TKey key, TValue value)
|
||||
{
|
||||
dic.Add(key,value);
|
||||
//dic.Add(key,value);
|
||||
dic[key] = value;
|
||||
}
|
||||
|
||||
public static void add<TItem>(this IList<TItem> list, TItem item)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
{
|
||||
"profiles": {
|
||||
"DbMaker": {
|
||||
"commandName": "Project",
|
||||
"commandLineArgs": " -src ip.merge.txt -region global_region.csv"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7,7 +7,7 @@ namespace DbMaker
|
|||
{
|
||||
private readonly Stream _stream;
|
||||
|
||||
public RandomAccessFile(String file) : this(File.OpenRead(file))
|
||||
public RandomAccessFile(String file) : this(File.Open(file, FileMode.OpenOrCreate))
|
||||
{
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue