Javaマルチスレッドダウンロードテクノロジー実装
123475 ワード
マルチスレッドダウンロードマルチスレッドダウンロード技術は、簡単に言えば、ダウンロードするファイルをいくつかのブロックに分けて、異なるスレッドで各データのダウンロードタスクを担当します.
技術的要点RandomAccessFile:Javaでランダムアクセスファイルを実現するためのクラスhttp Rangeリクエストヘッダの具体的な考え方1、ファイルブロック.ファイルブロックサイズ(blockSize)=(ファイルサイズ+スレッド数-1)/スレッド数;2、各スレッドがダウンロードするファイルの開始と終了位置を決定する.ここで、各スレッド番号:0,1,2,3を仮定すると、最初のスレッドが担当するダウンロード位置は:0*blockSize-(0+1)blockSize-1である.2番目のスレッドが担当するダウンロード位置は、1 blockSize-(1+1)blockSize-1であり、i番目のスレッドが担当するダウンロード位置は、iblockSize-(i+1)blockSize-1である.すなわち、スレッド(id番号)ダウンロード開始位置start=idblock、スレッド(id番号)ダウンロード終了位置end=(id+1)*block-1、3、httpリクエストヘッダ、conn.setRequestProperty(「Range」、「bytes=」+start+「-」+end)を設定する.
コードは簡単なJavaマルチスレッドダウンロードコードを実現します.
パッケージマルチスレッドダウンロードファイルダウンロードはよく使われるモジュールで、後で呼び出すのに便利です.関連する開発技術は以下の通りである.
JDK 1.7 Eclipse Juno Maven 3 HttpClient 4.3.6工程目次構造は以下の通りである.
最後にクライアント呼び出しコード
ソースコードhttps://github.com/TiFG/FileDownloader
ソース:http://blog.csdn.net/top_code/article/details/50477486
技術的要点RandomAccessFile:Javaでランダムアクセスファイルを実現するためのクラスhttp Rangeリクエストヘッダの具体的な考え方1、ファイルブロック.ファイルブロックサイズ(blockSize)=(ファイルサイズ+スレッド数-1)/スレッド数;2、各スレッドがダウンロードするファイルの開始と終了位置を決定する.ここで、各スレッド番号:0,1,2,3を仮定すると、最初のスレッドが担当するダウンロード位置は:0*blockSize-(0+1)blockSize-1である.2番目のスレッドが担当するダウンロード位置は、1 blockSize-(1+1)blockSize-1であり、i番目のスレッドが担当するダウンロード位置は、iblockSize-(i+1)blockSize-1である.すなわち、スレッド(id番号)ダウンロード開始位置start=idblock、スレッド(id番号)ダウンロード終了位置end=(id+1)*block-1、3、httpリクエストヘッダ、conn.setRequestProperty(「Range」、「bytes=」+start+「-」+end)を設定する.
コードは簡単なJavaマルチスレッドダウンロードコードを実現します.
package com.ricky.java.test.download;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
public class Downloader {
private URL url; //
private File file; //
private static final int THREAD_AMOUNT = 8; //
private static final String DOWNLOAD_DIR_PATH = "D:/Download"; //
private int threadLen; //
public Downloader(String address, String filename) throws IOException { //
url = new URL(address);
File dir = new File(DOWNLOAD_DIR_PATH);
if(!dir.exists()){
dir.mkdirs();
}
file = new File(dir, filename);
}
public void download() throws IOException {
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(5000);
int totalLen = conn.getContentLength(); //
threadLen = (totalLen + THREAD_AMOUNT - 1) / THREAD_AMOUNT; //
System.out.println("totalLen="+totalLen+",threadLen:"+threadLen);
RandomAccessFile raf = new RandomAccessFile(file, "rws"); //
raf.setLength(totalLen); //
raf.close();
for (int i = 0; i < THREAD_AMOUNT; i++) // 3 ,
new DownloadThread(i).start();
}
private class DownloadThread extends Thread {
private int id;
public DownloadThread(int id) {
this.id = id;
}
public void run() {
int start = id * threadLen; //
int end = id * threadLen + threadLen - 1; //
System.out.println(" " + id + ": " + start + "-" + end);
try {
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(5000);
conn.setRequestProperty("Range", "bytes=" + start + "-" + end); //
InputStream in = conn.getInputStream();
RandomAccessFile raf = new RandomAccessFile(file, "rws");
raf.seek(start); //
byte[] buffer = new byte[1024];
int len;
while ((len = in.read(buffer)) != -1)
raf.write(buffer, 0, len);
raf.close();
System.out.println(" " + id + " ");
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static void main(String[] args) throws IOException {
String address = "http://dldir1.qq.com/qqfile/qq/QQ7.9/16621/QQ7.9.exe";
new Downloader(address, "QQ7.9.exe").download();
// String address = "http://api.t.dianping.com/n/api.xml?cityId=2";
// new Downloader(address, "2.xml").download();
}
}
パッケージマルチスレッドダウンロードファイルダウンロードはよく使われるモジュールで、後で呼び出すのに便利です.関連する開発技術は以下の通りである.
JDK 1.7 Eclipse Juno Maven 3 HttpClient 4.3.6工程目次構造は以下の通りである.
com.ricky.common.java.download.FileDownloader
package com.ricky.common.java.download;
import org.apache.log4j.Logger;
import com.ricky.common.java.download.config.FileDownloaderConfiguration;
/**
* Java
* @author Ricky Fung
*
*/
public class FileDownloader {
protected Logger mLogger = Logger.getLogger("devLog");
private volatile static FileDownloader fileDownloader;
private FileDownloaderEngine downloaderEngine;
private FileDownloaderConfiguration configuration;
public static FileDownloader getInstance(){
if(fileDownloader==null){
synchronized (FileDownloader.class) {
if(fileDownloader==null){
fileDownloader = new FileDownloader();
}
}
}
return fileDownloader;
}
protected FileDownloader(){
}
public synchronized void init(FileDownloaderConfiguration configuration){
if (configuration == null) {
throw new IllegalArgumentException("FileDownloader configuration can not be initialized with null");
}
if (this.configuration == null) {
mLogger.info("init FileDownloader");
downloaderEngine = new FileDownloaderEngine(configuration);
this.configuration = configuration;
}else{
mLogger.warn("Try to initialize FileDownloader which had already been initialized before.");
}
}
public boolean download(String url, String filename){
return downloaderEngine.download(url, filename);
}
public boolean isInited() {
return configuration != null;
}
public void destroy() {
if(downloaderEngine!=null){
downloaderEngine.close();
downloaderEngine = null;
}
}
}
com.ricky.common.java.download.config.FileDownloaderConfiguration
package com.ricky.common.java.download.config;
import java.io.File;
public class FileDownloaderConfiguration {
private final int connectTimeout;
private final int socketTimeout;
private final int maxRetryCount;
private final int coreThreadNum;
private final long requestBytesSize;
private final File downloadDestinationDir;
private FileDownloaderConfiguration(Builder builder) {
this.connectTimeout = builder.connectTimeout;
this.socketTimeout = builder.socketTimeout;
this.maxRetryCount = builder.maxRetryCount;
this.coreThreadNum = builder.coreThreadNum;
this.requestBytesSize = builder.requestBytesSize;
this.downloadDestinationDir = builder.downloadDestinationDir;
}
public int getConnectTimeout() {
return connectTimeout;
}
public int getSocketTimeout() {
return socketTimeout;
}
public int getMaxRetryCount() {
return maxRetryCount;
}
public int getCoreThreadNum() {
return coreThreadNum;
}
public long getRequestBytesSize() {
return requestBytesSize;
}
public File getDownloadDestinationDir() {
return downloadDestinationDir;
}
public static FileDownloaderConfiguration.Builder custom() {
return new Builder();
}
public static class Builder {
private int connectTimeout;
private int socketTimeout;
private int maxRetryCount;
private int coreThreadNum;
private long requestBytesSize;
private File downloadDestinationDir;
public Builder connectTimeout(int connectTimeout) {
this.connectTimeout = connectTimeout;
return this;
}
public Builder socketTimeout(int socketTimeout) {
this.socketTimeout = socketTimeout;
return this;
}
public Builder coreThreadNum(int coreThreadNum) {
this.coreThreadNum = coreThreadNum;
return this;
}
public Builder maxRetryCount(int maxRetryCount) {
this.maxRetryCount = maxRetryCount;
return this;
}
public Builder requestBytesSize(long requestBytesSize) {
this.requestBytesSize = requestBytesSize;
return this;
}
public Builder downloadDestinationDir(File downloadDestinationDir) {
this.downloadDestinationDir = downloadDestinationDir;
return this;
}
public FileDownloaderConfiguration build() {
initDefaultValue(this);
return new FileDownloaderConfiguration(this);
}
private void initDefaultValue(Builder builder) {
if(builder.connectTimeout<1){
builder.connectTimeout = 6*1000;
}
if(builder.socketTimeout<1){
builder.socketTimeout = 6*1000;
}
if(builder.maxRetryCount<1){
builder.maxRetryCount = 1;
}
if(builder.coreThreadNum<1){
builder.coreThreadNum = 3;
}
if(builder.requestBytesSize<1){
builder.requestBytesSize = 1024*128;
}
if(builder.downloadDestinationDir==null){
builder.downloadDestinationDir = new File("./");
}
}
}
}
com.ricky.common.java.download.FileDownloaderEngine
package com.ricky.common.java.download;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.BitSet;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.log4j.Logger;
import com.ricky.common.java.download.config.FileDownloaderConfiguration;
import com.ricky.common.java.download.job.DownloadWorker;
import com.ricky.common.java.download.job.Worker.DownloadListener;
public class FileDownloaderEngine {
protected Logger mLogger = Logger.getLogger("devLog");
private FileDownloaderConfiguration configuration;
private ExecutorService pool;
private HttpRequestImpl httpRequestImpl;
private File downloadDestinationDir;
private int coreThreadNum;
public FileDownloaderEngine(FileDownloaderConfiguration configuration){
this.configuration = configuration;
this.coreThreadNum = configuration.getCoreThreadNum();
this.httpRequestImpl = new HttpRequestImpl(this.configuration);
this.pool = Executors.newFixedThreadPool(this.configuration.getCoreThreadNum());
this.downloadDestinationDir = this.configuration.getDownloadDestinationDir();
if(!this.downloadDestinationDir.exists()){
this.downloadDestinationDir.mkdirs();
}
}
public boolean download(String url, String filename){
long start_time = System.currentTimeMillis();
mLogger.info(" ,url:"+url+",filename:"+filename);
long total_file_len = httpRequestImpl.getFileSize(url); //
if(total_file_len<1){
mLogger.warn(" ,url:"+url+",filename:"+filename);
return false;
}
final BitSet downloadIndicatorBitSet = new BitSet(coreThreadNum); //
File file = null;
try {
file = new File(downloadDestinationDir, filename);
RandomAccessFile raf = new RandomAccessFile(file, "rws"); //
raf.setLength(total_file_len); //
raf.close();
mLogger.info("create new file:"+file);
} catch (FileNotFoundException e) {
mLogger.error("create new file error", e);
} catch (IOException e) {
mLogger.error("create new file error", e);
}
if(file==null || !file.exists()){
mLogger.warn(" ,url:"+url+",filename:"+filename);
return false;
}
long thread_download_len = (total_file_len + coreThreadNum - 1) / coreThreadNum; //
mLogger.info("filename:"+filename+",total_file_len="+total_file_len+",coreThreadNum:"+coreThreadNum+",thread_download_len:"+thread_download_len);
CountDownLatch latch = new CountDownLatch(coreThreadNum);//
for (int i = 0; i < coreThreadNum; i++){
DownloadWorker worker = new DownloadWorker(i, url, thread_download_len, file, httpRequestImpl, latch);
worker.addListener(new DownloadListener() {
@Override
public void notify(int thread_id, String url, long start, long end,
boolean result, String msg) {
mLogger.info("thread_id:"+thread_id+" download result:"+result+",url->"+url);
modifyState(downloadIndicatorBitSet, thread_id);
}
});
pool.execute(worker);
}
try {
latch.await();
} catch (InterruptedException e) {
mLogger.error("CountDownLatch Interrupt", e);
}
mLogger.info(" ,url:"+url+", :"+((System.currentTimeMillis()-start_time)/1000)+"(s)");
return downloadIndicatorBitSet.cardinality()==coreThreadNum;
}
private synchronized void modifyState(BitSet bitSet, int index){
bitSet.set(index);
}
/** */
public void close(){
if(httpRequestImpl!=null){
httpRequestImpl.close();
httpRequestImpl = null;
}
if(pool!=null){
pool.shutdown();
pool = null;
}
}
}
com.ricky.common.java.download.job.DownloadWorker
package com.ricky.common.java.download.job;
import java.io.File;
import java.util.concurrent.CountDownLatch;
import org.apache.log4j.Logger;
import com.ricky.common.java.download.HttpRequestImpl;
import com.ricky.common.java.download.RetryFailedException;
public class DownloadWorker extends Worker {
protected Logger mLogger = Logger.getLogger("devLog");
private int id;
private String url;
private File file;
private long thread_download_len;
private CountDownLatch latch;
private HttpRequestImpl httpRequestImpl;
public DownloadWorker(int id, String url, long thread_download_len, File file, HttpRequestImpl httpRequestImpl, CountDownLatch latch) {
this.id = id;
this.url = url;
this.thread_download_len = thread_download_len;
this.file = file;
this.httpRequestImpl = httpRequestImpl;
this.latch = latch;
}
@Override
public void run() {
long start = id * thread_download_len; //
long end = id * thread_download_len + thread_download_len - 1; //
mLogger.info(" :" + id +" url:"+url+ ",range:" + start + "-" + end);
boolean result = false;
try {
httpRequestImpl.downloadPartFile(id, url, file, start, end);
result = true;
mLogger.info(" :" + id + " "+url+ " range[" + start + "-" + end+"] ");
} catch (RetryFailedException e) {
mLogger.error(" :" + id +" ", e);
}catch (Exception e) {
mLogger.error(" :" + id +" ", e);
}
if(listener!=null){
mLogger.info("notify FileDownloaderEngine download result");
listener.notify(id, url, start, end, result, "");
}
latch.countDown();
}
}
com.ricky.common.java.download.job.Worker
package com.ricky.common.java.download.job;
public abstract class Worker implements Runnable {
protected DownloadListener listener;
public void addListener(DownloadListener listener){
this.listener = listener;
}
public interface DownloadListener{
public void notify(int thread_id, String url, long start, long end, boolean result, String msg);
}
}
com.ricky.common.java.download.HttpRequestImpl
package com.ricky.common.java.download;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.log4j.Logger;
import com.ricky.common.java.download.config.FileDownloaderConfiguration;
import com.ricky.common.java.http.HttpClientManager;
public class HttpRequestImpl {
protected Logger mLogger = Logger.getLogger("devLog");
private int connectTimeout;
private int socketTimeout;
private int maxRetryCount;
private long requestBytesSize;
private CloseableHttpClient httpclient = HttpClientManager.getHttpClient();
public HttpRequestImpl(FileDownloaderConfiguration configuration){
connectTimeout = configuration.getConnectTimeout();
socketTimeout = configuration.getSocketTimeout();
maxRetryCount = configuration.getMaxRetryCount();
requestBytesSize = configuration.getRequestBytesSize();
}
public void downloadPartFile(int id, String url, File file, long start, long end){
RandomAccessFile raf = null;
try {
raf = new RandomAccessFile(file, "rws");
} catch (FileNotFoundException e) {
mLogger.error("file not found:"+file, e);
throw new IllegalArgumentException(e);
}
int retry = 0;
long pos = start;
while(pos<end){
long end_index = pos + requestBytesSize;
if(end_index>end){
end_index = end;
}
boolean success = false;
try {
success = requestByRange(url, raf, pos, end_index);
} catch (ClientProtocolException e) {
mLogger.error("download error,start:"+pos+",end:"+end_index, e);
}catch (IOException e) {
mLogger.error("download error,start:"+pos+",end:"+end_index, e);
}catch (Exception e) {
mLogger.error("download error,start:"+pos+",end:"+end_index, e);
}
// mLogger.info(" :" + id +",download url:"+url+",range:"+ pos + "-" + end_index+",success="+success );
if(success){
pos += requestBytesSize;
retry = 0;
}else{
if(retry < maxRetryCount){
retry++;
mLogger.warn(" :" + id +",url:"+url+",range:"+pos+","+end_index+" , "+retry+" ");
}else{
mLogger.warn(" :" + id +",url:"+url+",range:"+pos+","+end_index+" , !");
throw new RetryFailedException(" ");
}
}
}
}
private boolean requestByRange(String url, RandomAccessFile raf, long start, long end) throws ClientProtocolException, IOException {
HttpGet httpget = new HttpGet(url);
httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.152 Safari/537.36");
httpget.setHeader("Range", "bytes=" + start + "-" + end);
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.setSocketTimeout(socketTimeout)
.build();
httpget.setConfig(requestConfig);
CloseableHttpResponse response = null;
try {
response = httpclient.execute(httpget);
int code = response.getStatusLine().getStatusCode();
if(code==HttpStatus.SC_OK || code== HttpStatus.SC_PARTIAL_CONTENT){
HttpEntity entity = response.getEntity();
if (entity != null) {
InputStream in = entity.getContent();
raf.seek(start);//
byte[] buffer = new byte[1024];
int len;
while ((len = in.read(buffer)) != -1){
raf.write(buffer, 0, len);
}
return true;
}else{
mLogger.warn("response entity is null,url:"+url);
}
}else{
mLogger.warn("response error, code="+code+",url:"+url);
}
}finally {
IOUtils.closeQuietly(response);
}
return false;
}
public long getFileSize(String url){
int retry = 0;
long filesize = 0;
while(retry<maxRetryCount){
try {
filesize = getContentLength(url);
} catch (Exception e) {
mLogger.error("get File Size error", e);
}
if(filesize>0){
break;
}else{
retry++;
mLogger.warn("get File Size failed,retry:"+retry);
}
}
return filesize;
}
private long getContentLength(String url) throws ClientProtocolException, IOException{
HttpGet httpget = new HttpGet(url);
httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.152 Safari/537.36");
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(connectTimeout)
.setSocketTimeout(socketTimeout)
.build();
httpget.setConfig(requestConfig);
CloseableHttpResponse response = null;
try {
response = httpclient.execute(httpget);
int code = response.getStatusLine().getStatusCode();
if(code==HttpStatus.SC_OK){
HttpEntity entity = response.getEntity();
if (entity != null) {
return entity.getContentLength();
}
}else{
mLogger.warn("response code="+code);
}
}finally {
IOUtils.closeQuietly(response);
}
return -1;
}
public void close(){
if(httpclient!=null){
try {
httpclient.close();
} catch (IOException e) {
e.printStackTrace();
}
httpclient = null;
}
}
}
最後にクライアント呼び出しコード
package com.ricky.common.java;
import java.io.File;
import com.ricky.common.java.download.FileDownloader;
import com.ricky.common.java.download.config.FileDownloaderConfiguration;
public class FileDownloaderTest {
public static void main(String[] args) {
FileDownloader fileDownloader = FileDownloader.getInstance();
FileDownloaderConfiguration configuration = FileDownloaderConfiguration
.custom()
.coreThreadNum(5)
.downloadDestinationDir(new File("D:/Download"))
.build();
fileDownloader.init(configuration);
String url = "http://dldir1.qq.com/qqfile/qq/QQ7.9/16621/QQ7.9.exe";;
String filename = "QQ7.9.exe";
boolean result = fileDownloader.download(url, filename);
System.out.println("download result:"+result);
fileDownloader.destroy(); //close it when you not need
}
}
ソースコードhttps://github.com/TiFG/FileDownloader
ソース:http://blog.csdn.net/top_code/article/details/50477486