MongoDBのMapReduce統計キーワード周波数テスト
6587 ワード
テスト環境:windows xp
Operating System: Windows XP Professional (5.1, Build 2600) Service Pack 3 (2600.xpsp_sp3_gdr.101209-1647)
Language: Chinese (Regional Setting: Chinese)
Processor: Pentium(R) Dual-Core CPU E5500 @ 2.80GHz (2 CPUs)
Memory: 3292MB RAM
テスト結果:
1079844データ統計10957キーワードソート上位100レコード、総消費時間:308578ミリ秒
試験手順:
Operating System: Windows XP Professional (5.1, Build 2600) Service Pack 3 (2600.xpsp_sp3_gdr.101209-1647)
Language: Chinese (Regional Setting: Chinese)
Processor: Pentium(R) Dual-Core CPU E5500 @ 2.80GHz (2 CPUs)
Memory: 3292MB RAM
テスト結果:
1079844データ統計10957キーワードソート上位100レコード、総消費時間:308578ミリ秒
試験手順:
import java.io.BufferedReader;
import java.io.FileReader;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MapReduceCommand;
import com.mongodb.MapReduceOutput;
import com.mongodb.Mongo;
import com.mongodb.MongoException;
/**
*
*/
/**
* *
*/
public class Test4MongoDb {
/**
* @param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
//insertTestKeywordLog();
calculateSearchKeyword();
}
/**
*
*/
private static void insertTestKeywordLog() {
List<String> keyWordList = new ArrayList<String>();
try {
BufferedReader reader = new BufferedReader(new FileReader(
"d:\\pinyin.txt"));
String line = null;
Random ran = new Random(System.currentTimeMillis());
Mongo m;
int totalRows = 0;
long start = 0;
long end = 0;
m = new Mongo("localhost", 9999);
DB db = m.getDB("test");
DBCollection collection = db.getCollection("t_log");
String month = "02";
String year = "2010";
start = System.currentTimeMillis();
while ((line = reader.readLine()) != null) {
int insertCount = ran.nextInt(100);
if (insertCount == 0) {
insertCount = 1;
}
totalRows += insertCount;
for (int i = 0; i < insertCount; i++) {
DBObject record = new BasicDBObject();
record.put("id", System.currentTimeMillis());
record.put("keyword", line);
int day = ran.nextInt(28);
if (day == 0) {
day = 1;
}
record.put("dd", year + "-" + month
+ (day < 10 ? "-0" + day : "-" + day));
collection.save(record);
}
}
end = System.currentTimeMillis();
System.out.println("insert time =" + (end - start) + " row count="
+ totalRows);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
*
*/
private static void calculateSearchKeyword() {
long start = 0;
long end = 0;
Mongo m;
try {
m = new Mongo("localhost", 9999);
DB db = m.getDB("test");
DBCollection collection = db.getCollection("t_log");
DBObject newDB = new BasicDBObject();
newDB.put("max", 100000);
String collectionName = "t_log_result_"
+ System.currentTimeMillis();
DBCollection resultCollection = db.createCollection(collectionName,
newDB);
//
DBObject indexObject = new BasicDBObject();
indexObject.put("hitCount", -1);
resultCollection.createIndex(indexObject);
//DBCollection resultCollection = db.getCollection("t_log_result_"+System.currentTimeMillis());
start = System.currentTimeMillis();
DBObject dbKey = new BasicDBObject();
dbKey.put("dd", true);
//
DBObject condition = new BasicDBObject();
condition.put("dd", new BasicDBObject("$gte", "2010-02-01").append(
"$lte", "2010-02-28"));
// map
String map = "function() { key=this.keyword; "
+ " emit(key,{'count':1}); " + "}";
// reduce
String reduce = " function r( key, values ) { " + " var count=0;"
+ " for ( var i = 0; i < values.length; i++ ){"
+ " count += values[i].count;" + " }"
+ " return count;} ";
///Map<String,Object> scope = new HashMap<String,Object>();
MapReduceCommand mr = new MapReduceCommand(collection, map, reduce,
null, MapReduceCommand.OutputType.INLINE, condition);
int resultCount = 0;
MapReduceOutput out = collection.mapReduce(mr);
//
for (DBObject result : out.results()) {
Double value = null;
if (result.get("value") != null
&& result.get("value") instanceof DBObject) {
DBObject dbObj = (DBObject) result.get("value");
value = (Double) dbObj.get("count");
} else {
value = (Double) result.get("value");
}
String found = (String) result.get("_id");
DBObject keywordObject = new BasicDBObject();
keywordObject.put("hitCount", value);
keywordObject.put("keyword", found);
//
resultCollection.save(keywordObject);
resultCount++;
}
DBObject query = new BasicDBObject();
DBObject orderBy = new BasicDBObject();
orderBy.put("hitCount", -1);
// top100
DBCursor cursor = resultCollection.find().sort(orderBy).limit(100);
while (cursor.hasNext()) {
System.out.println(cursor.next());
}
end = System.currentTimeMillis();
System.out.println("total time =" + (end - start)
+ " total row count=" + resultCount);
} catch (UnknownHostException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (MongoException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}