Hadoopベースのデータ圧縮と解凍の例

データ圧縮
テストの入力データ(/input/kmeans/sample.txt)は次のとおりです.
1,1 2,2 3,3 -3,-3 -4,-4 -5,-5
出力結果(/output/compress.gz)は一連の圧縮文字列である

import java.io.IOException;
import java.io.InputStream;  
import java.io.OutputStream;  
  
import org.apache.hadoop.conf.Configuration;    
import org.apache.hadoop.fs.FileSystem;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IOUtils;  
import org.apache.hadoop.io.compress.CompressionCodec;  
import org.apache.hadoop.io.compress.CompressionOutputStream;  
import org.apache.hadoop.util.ReflectionUtils;  
  
import com.panky.tools.Tools;  
  
public class StreamCompressor {  
    public static void main(String[] args) throws IOException, ClassNotFoundException {  
        String codecClassname="org.apache.hadoop.io.compress.GzipCodec";  
        //   
        Class> codecClass=Class.forName(codecClassname);  
        Configuration configuration=new Tools().getTools();  
        FileSystem fileSystem=FileSystem.get(configuration);  
        InputStream inputStream=fileSystem.open(new Path("/input/kmeans/sample.txt"));  
        OutputStream outputStream=fileSystem.create(new Path("/output/compress.gz"));  
        // ReflectionUtil CompressionCodec   
        CompressionCodec compressionCodec=(CompressionCodec)ReflectionUtils.newInstance(codecClass, configuration);  
         // CompressionOutputStream System.out ，   
        CompressionOutputStream compressionOutputStream=compressionCodec.createOutputStream(outputStream);  
        IOUtils.copyBytes(inputStream, compressionOutputStream, 4096, false);  
        compressionOutputStream.finish();  
    }  
} 

データ解凍
入力データ(/output/compress.gz)は、データ圧縮の出力結果
出力結果(/output/compress):
1,1 2,2 3,3 -3,-3 -4,-4 -5,-5

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.CompressionInputStream;

import com.panky.tools.Tools;

public class FileDecompressor {
	public static void main(String[] args) throws IOException {
		// .gz ， GzipCodec 。
		String uriString="/output/compress.gz";
		Configuration configuration=new Tools().getTools();
		FileSystem fileSystem=FileSystem.get(configuration);
		CompressionCodecFactory factory=new CompressionCodecFactory(configuration);
		// getCodec() ，CompressionCodecFactory CompressionCodec 。
		CompressionCodec codec=factory.getCodec(new Path(uriString));
		if(codec==null){
			System.out.println("No codec found for"+uriString);
			System.exit(1);
		}
		// CompressionCodecFactory removeSuffix()    codec， 
		String outString=CompressionCodecFactory.removeSuffix(uriString, codec.getDefaultExtension());
		// createInputStream(InputStream in) CompressionInputStream 
		InputStream inputStream=codec.createInputStream(fileSystem.open(new Path(uriString)));
		// 
		//CompressionInputStream inputStream=codec.createInputStream(fileSystem.open(new Path(uriString)));
		OutputStream outputStream=fileSystem.create(new Path(outString));
		IOUtils.copyBytes(inputStream, outputStream, configuration);
		IOUtils.closeStream(inputStream);
		IOUtils.closeStream(outputStream);
	}
}