大規模な分散システムアーキテクチャと設計実戦ノート8
5968 ワード
fourinoneで実現された整数読み書きに基づく億単位のソート
添付ファイルは生成された1億個の整数乱数のコードです
import com.fourinone.BeanContext;
public class ParkServerDemo
{
public static void main(String[] args)
{
BeanContext.startPark();
}
}
import com.fourinone.*;
import java.util.*;
import com.fourinone.ArrayAdapter.*;
//public class Worker extends MigrantWorker
public class Worker extends MigrantWorker
{
int total=25000000;//
int max=100000;//
int block=100000;//
int groups=16;//
int groupnum=max/groups;//
String path="d://tmp";//
Workman[] wms=null;
public Worker(){}
public Worker(int total,int max,int block,int groups,String path)
{
this.total=total;
this.max=max;
this.block=block;
this.groups=groups;
this.groupnum=max/groups;
this.path=path;
}
public WareHouse doTask(WareHouse wh)
{
int index=getSelfIndex();
int step=(Integer)wh.getObj("step");
if(wms==null)
{
wms=getWorkerAll();
}
WareHouse result=new WareHouse("ok",1);
//long begin=(new Date()).getTime();
if(step==1)
{
// 16
FileAdapter[] fas=new FileAdapter[groups];
for(int i=0;i<groups;i++)
fas[i]=new FileAdapter(path+"//"+index+"//"+i+"//data");
//
FileAdapter fa=new FileAdapter(path+"//"+index+"//data");
for(int n=0;n<(total/block);n++)
{
// 1000
int [] its=fa.getIntReader(n*block,block).readIntAll();
// ArrayList
ArrayList<ArrayList<Integer>> list=new ArrayList<ArrayList<Integer>> ();
for(int p=0;p<groups;p++)
{
list.add(new ArrayList<Integer>());
}
int temp=-1;
for(int m=0;m<block;m++)
{
temp=its[m]/groupnum;
//System.out.println(temp);
list.get(temp).add(its[m]);
}
for(int k=0;k<groups;k++)
{
fas[k].getIntWriter().writeListInt(list.get(k));
}
}
}
else if(step==2)
{
for(int i=0;i<16;i++)
{
FileAdapter fa=new FileAdapter(path+"//"+index+"//"+i+"//data");
int[] nums=fa.getIntReader().readIntAll();
if((i/4)!=index)
{
WareHouse out=new WareHouse();
out.put("i",i);
out.put("v",nums);
// (i/4) i
wms[i/4].receive(out);
}
fa.close();
}
}
else if(step==3)
{
int total=0;
for(int i=0;i<16;i++)
{
if((i/4)==index)
{
FileAdapter fa=new FileAdapter(path+"//"+index+"//"+i+"//data");
int [] nums=fa.getIntReader().readIntAll();
ListInt is=ArrayAdapter.getListInt();
is.sort(nums);
total+=nums.length;
FileAdapter tofa=new FileAdapter(path+"//"+index+"//"+i+".data");
tofa.getIntWriter().writeInt(nums);
System.out.println(" "+i+".data");
fa.close();
}
else
{
FileAdapter fa=new FileAdapter(path+"//"+index+"//"+i);
fa.delete();
fa.close();
}
}
result.setObj("total",total);
}
return result;
}
protected boolean receive(WareHouse inhouse)
{
Integer i=(Integer)inhouse.get("i");
int [] nums=(int [])inhouse.get("v");
int index=i/4;
FileAdapter fa=new FileAdapter(path+"//"+index+"//"+i+"//data");
fa.getIntWriter().writeInt(nums);
System.out.println(" "+i+" "+index+" ");
return true;
}
public static void main(String [] args)
{
Worker w=new Worker(Integer.parseInt(args[2]),Integer.parseInt(args[3]),Integer.parseInt(args[4]),Integer.parseInt(args[5]),args[6]);
w.waitWorking(args[0],Integer.parseInt(args[1]),"Worker");
}
}
import com.fourinone.Contractor;
import com.fourinone.WareHouse;
import com.fourinone.WorkerLocal;
import java.util.Date;
public class Ctor extends Contractor
{
public WareHouse giveTask(WareHouse wh)
{
WorkerLocal[] wks = getWaitingWorkers("Worker");
System.out.println("wks.length:"+wks.length+";"+wh);
int total=0;
System.out.println(" ");
wh.setObj("step", 1);//1:group;
doTaskBatch(wks, wh);
System.out.println(" ");
wh.setObj("step", 2);//2:merge;
doTaskBatch(wks, wh);
System.out.println(" ");
wh.setObj("step", 3);//3:sort
WareHouse[] hmarr = doTaskBatch(wks, wh);
for(int i=0;i<hmarr.length;i++){
Object num = hmarr[i].getObj("total");
if(num!=null)
total+=(Integer)num;
}
wh.setObj("total",total);
return wh;
}
public static void main(String[] args)
{
Ctor a = new Ctor();
WareHouse wh = new WareHouse();
long begin = (new Date()).getTime();
a.doProject(wh);
long end = (new Date()).getTime();
System.out.println("total:"+wh.getObj("total")+",time:"+(end-begin)/1000+"s");
}
}
添付ファイルは生成された1億個の整数乱数のコードです