Hadoop---入門のHDFSのJAVA API操作

JAR準備:
hadoop-2.8.0のshareディレクトリのjarパッケージをプロジェクトに追加します.

common下のhadoop-common-2.8.0.jar

common/libの下のjar

hdfs下のhadoop-hdfs-2.8.0.jar

hdfs/libのすべてのjar

 
例:

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.net.URI;
import java.util.Iterator;
import java.util.Map.Entry;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

//     hdfs，         
//     ，hdfs   api  jvm                 ：-DHADOOP_USER_NAME=hadoop
public class HdfsClientDemo {

	private FileSystem fs;
	private Configuration conf;

	@Before
	public void init() throws Exception{
		
		conf = new Configuration();
		conf.set("dfs.replication", "2");

		fs = FileSystem.get(new URI("hdfs://192.168.153.136:9000"),conf,"hadoop");
	}
	
	//   conf    
	@Test
	public void testConf(){
		Iterator> iterator = conf.iterator();
		while (iterator.hasNext()) {
			Entry ent = iterator.next();
			System.out.println(ent.getKey() + ": " + ent.getValue());
		}
	}
	
	//      HDFS    
	@Test
	public void testUpload() throws Exception {
		// （1）      IO         HDFS    
		//FileInputStream fileInputStream =new FileInputStream("e:/Spring MVC.docx");
		//FSDataOutputStream fsDataOutputStream = fs.create(new Path("/Spring MVC222.docx"));
		//IOUtils.copy(fileInputStream, fsDataOutputStream);
		
		// （2）   
		boolean windowsAbsolutePath = Path.isWindowsAbsolutePath("e:/Spring MVC.docx", true);
		System.out.println(windowsAbsolutePath);
		if (windowsAbsolutePath) {
			fs.copyFromLocalFile(new Path("e:/Spring MVC.docx"), new Path("/Spring MVC.docx"));
			fs.close();
		}
	}
	
	//  HDFS           
	@Test
	public void testDownload() throws Exception {
		//（1）         hadoop  ，    IO 
		//FSDataInputStream fsDataInputStream=fs.open(new Path("/spring/Spring MVC.docx"));
        //FileOutputStream fileOutputStream=new FileOutputStream("e:/Spring MVC.docx");
        //IOUtils.copy(fsDataInputStream, fileOutputStream);
		
        // （2）fs.copyToLocalFile(new Path("/Spring MVC1111.docx"), new Path("e:/"));        hadoop  ，        ：  1      hdfs     ，  4      java  API
		fs.copyToLocalFile(false,new Path("/Spring MVC1111.docx"), new Path("e:/"),true);
		fs.close();
	}
	
	
	
	//   HDFS  ，    /
	@Test
	public void makdirTest() throws Exception {
		boolean mkdirs = fs.mkdirs(new Path("/spring"));
		System.out.println(mkdirs);
	}
	
	@Test
	public void createTest() throws Exception{
		
	}
	
	//        
	@Test
	public void deleteTest() throws Exception{
		// fs.exists():      ，     
		boolean file_exists = fs.exists(new Path("/spring"));  // true
		// fs.isFile():      
		boolean file = fs.isFile(new Path("/Spring MVC.docx"));  // true
		// fs.isDirectory():            
		boolean directory1 = fs.isDirectory(new Path("/spring"));   // true
		boolean directory2 = fs.isDirectory(new Path("/Spring MVC.docx"));  // false
		
		System.out.println(file_exists);
		System.out.println(file);
		System.out.println(directory1);
		System.out.println(directory2);
		if (file_exists) {
			boolean delete = fs.delete(new Path("/spring"), true);//true，     
			System.out.println(delete);
		}
	}
	
	//          
	@Test
	public void listTest() throws Exception{
		
		FileStatus[] listStatus = fs.listStatus(new Path("/"));
		for (FileStatus fileStatus : listStatus) {
			System.err.println(fileStatus.getPath()+"================="+fileStatus.toString());
		}
		
		//          
		RemoteIterator listFiles = fs.listFiles(new Path("/"), true);
		while(listFiles.hasNext()){
			LocatedFileStatus fileStatus = listFiles.next();
			System.out.println("blocksize：" + fileStatus.getBlockSize());
			System.out.println("owner：" + fileStatus.getOwner());
			System.out.println("Replication：" + fileStatus.getReplication());
			System.out.println("Permission：" + fileStatus.getPermission());
			System.out.println("Path：" + fileStatus.getPath());
			System.out.println("FileName：" + fileStatus.getPath().getName());
			System.out.println("File Len：" + fileStatus.getLen());
			BlockLocation[] blockLocations = fileStatus.getBlockLocations();
			for (BlockLocation blockLocation : blockLocations) {
				System.out.println("      ：" + blockLocation.getOffset());
				System.out.println("   ：" + blockLocation.getLength());
				String[] hosts = blockLocation.getHosts();
				for (String datanode : hosts) {
					//       ，         3   ，   hadoop     dfs.replication 2   
					//         3    ？
					//      client Configuration hadoop hdfs-site.xml         ，   client      Configuration，  Configuration     dfs.replication   ，    3
					//         ，           conf.set("dfs.replication", "2");
					System.out.println("     ：" + datanode);
				}
			}
		}
		
	}
	
	
	//   hdfs      
	@Test
	public void testCat() throws Exception{
		FSDataInputStream in = fs.open(new Path("/Spring MVC.docx"));
		IOUtils.copy(in, System.out);
	}
	
}