Java 文件多线程下载

最近在做文件下载这块的东西，研究了一下多线程文件下载这块的知识。这里只说一下原理，具体实现请看代码，已经写了注释了。

主要原理

         为了加快下载速度，每个文件固定N个线程来下载，然后每个线程负责下载该文件的某一部分，比如文件大小90M，用3个线程来下载，那么第一个线程负责下载文件的长度范围：0-30*1024*1024-1，第二个线程负责下载文件的长度范围：30*1024*1024-60*1024*1024-1，第三个线程负责下载文件的长度范围：60*1024*1024-90*1024*1024-1，3个线程下载完后就合成了整个文件。这里需要用到Http中的ContentLength和Range请求头，ContentLength对应文件的总长度，Range头用来请求文件某一子块的内容，例如：Range 0-10000，表示请求该文件0-10000字节的内容。

具体代码如下：

import java.io.File;  
import java.io.IOException;  
import java.io.InputStream;  
import java.io.RandomAccessFile;  
import java.util.zip.GZIPInputStream;  
  
import org.apache.http.Header;  
import org.apache.http.HttpResponse;  
import org.apache.http.HttpStatus;  
import org.apache.http.client.ClientProtocolException;  
import org.apache.http.client.HttpClient;  
import org.apache.http.client.methods.HttpGet;  
import org.apache.http.client.params.CookiePolicy;  
import org.apache.http.params.CoreConnectionPNames;  
  
import com.ricky.java.common.download.file.http.HttpClientManager;  
import com.ricky.java.common.download.file.util.Constants;  
  
public class Downloader {  
      
    private String url; // 目标地址  
    private File file;  // 本地文件  
    private static final int THREAD_AMOUNT = 3;                 // 线程数  
    private static final String DIR_PATH = "D:/download/file";      // 下载目录  
    private long threadLen;                                     // 每个线程下载多少  
  
    private HttpClient mHttpClient = HttpClientManager.getHttpClient();  
      
    public Downloader(String address) throws IOException {      // 通过构造函数传入下载地址  
        url = address;  
        file = new File(DIR_PATH, address.substring(address.lastIndexOf("?") + 1)+".xml");  
    }  
  
    public void download() throws IOException {  
          
        long totalLen = getContentLength(url);                          // 获取文件长度  
        threadLen = (totalLen + THREAD_AMOUNT - 1) / THREAD_AMOUNT;         // 计算每个线程要下载的长度  
          
        System.out.println("totalLen="+totalLen+"***threadLen="+threadLen);  
          
        RandomAccessFile raf = new RandomAccessFile(file, "rws");           // 在本地创建一个和服务端大小相同的文件  
        raf.setLength(totalLen);                                            // 设置文件的大小  
        raf.close();  
          
        for (int i = 0; i < THREAD_AMOUNT; i++)                              // 开启3条线程, 每个线程下载一部分数据到本地文件中  
            new DownloadThread(i).start();  
    }  
      
    public long getContentLength(String address) {  
        HttpGet httpget = null;  
        try {  
            httpget = new HttpGet(address);  
            httpget.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.79 Safari/537.1");  
            httpget.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");  
              
            httpget.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,Constants.SO_TIMEOUT);  
            httpget.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,Constants.CONNECTION_TIMEOUT);  
            httpget.getParams().setParameter("http.protocol.cookie-policy",CookiePolicy.BROWSER_COMPATIBILITY);  
              
            HttpResponse response = mHttpClient.execute(httpget);  
              
            int status = response.getStatusLine().getStatusCode();  
              
            if (status == HttpStatus.SC_OK) {  
                  
                return response.getEntity().getContentLength();  
            }  
        } catch (ClientProtocolException e) {  
            e.printStackTrace();  
        } catch (IOException e) {  
            e.printStackTrace();  
        }finally{  
            if(httpget!=null){  
                httpget.abort();  
            }  
        }  
        return 0;  
    }  
  
    private class DownloadThread extends Thread {  
        private int id;   
        public DownloadThread(int id) {  
            this.id = id;  
        }  
        public void run() {  
            long start = id * threadLen;                        // 起始位置  
            long end = id * threadLen + threadLen - 1;      // 结束位置  
            System.out.println("线程" + id + ": " + start + "-" + end);  
              
            HttpGet httpget = null;  
            try {  
                httpget = new HttpGet(url);  
                httpget.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.79 Safari/537.1");  
                httpget.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");  
                  
                httpget.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,Constants.SO_TIMEOUT);  
                httpget.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,Constants.CONNECTION_TIMEOUT);  
                httpget.getParams().setParameter("http.protocol.cookie-policy",CookiePolicy.BROWSER_COMPATIBILITY);  
                  
                HttpResponse response = mHttpClient.execute(httpget);  
                  
                int status = response.getStatusLine().getStatusCode();  
                  
                if (status == HttpStatus.SC_OK) {  
                    InputStream in = response.getEntity().getContent();  
                    Header contentEncoding = response.getFirstHeader("Content-Encoding");  
                    if (contentEncoding != null && contentEncoding.getValue().equalsIgnoreCase("gzip")) {  
                        System.out.println("gzip InputStream in post");  
                        in = new GZIPInputStream(in);  
                    }  
                      
                    RandomAccessFile raf = new RandomAccessFile(file, "rws");  
                    raf.seek(start);  
                      
                    byte[] buffer = new byte[1024];  
                    int len;  
                    while ((len = in.read(buffer)) != -1)  
                        raf.write(buffer, 0, len);  
                    raf.close();  
                      
                    System.out.println("线程" + id + "下载完毕");  
                }else{  
                    System.out.println("线程" + id + "请求失败,响应码="+status);  
                }  
                  
            } catch (ClientProtocolException e) {  
                e.printStackTrace();  
            } catch (IOException e) {  
                e.printStackTrace();  
            }finally{  
                if(httpget!=null){  
                    httpget.abort();  
                }  
            }  
        }  
    }  
      
    public static void main(String[] args) throws IOException {  
//      new Downloader("http://dldir1.qq.com/qqfile/qq/QQ6.2/12179/QQ6.2.exe").download();  
        new Downloader("http://api.t.dianping.com/n/api.xml?cityId=1").download();  
    }  
}  

HttpClientManager.java

  
import org.apache.http.client.HttpClient;  
import org.apache.http.conn.scheme.PlainSocketFactory;  
import org.apache.http.conn.scheme.Scheme;  
import org.apache.http.conn.scheme.SchemeRegistry;  
import org.apache.http.conn.ssl.SSLSocketFactory;  
import org.apache.http.impl.client.DefaultHttpClient;  
import org.apache.http.impl.conn.PoolingClientConnectionManager;  
import org.apache.http.params.BasicHttpParams;  
import org.apache.http.params.CoreConnectionPNames;  
import org.apache.http.params.HttpParams;  
  
import com.ricky.java.common.download.file.util.Constants;  
  
public class HttpClientManager {  
      
    private static HttpParams httpParams;  
    private static PoolingClientConnectionManager cm;  
  
    /** 
     * 最大连接数 
     */  
    public final static int MAX_TOTAL_CONNECTIONS = 200;  
    /** 
     * 每个路由最大连接数 
     */  
    public final static int MAX_ROUTE_CONNECTIONS = 300;  
      
  
    static {  
        SchemeRegistry schemeRegistry = new SchemeRegistry();  
        schemeRegistry.register(  
                new Scheme("http",80,PlainSocketFactory.getSocketFactory()));  
        schemeRegistry.register(  
                new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));  
          
        cm = new PoolingClientConnectionManager(schemeRegistry);  
        cm.setMaxTotal(MAX_TOTAL_CONNECTIONS);  
        cm.setDefaultMaxPerRoute(MAX_ROUTE_CONNECTIONS);  
          
        HttpParams params = new BasicHttpParams();  
        params.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,Constants.CONNECTION_TIMEOUT);  
        params.setParameter(CoreConnectionPNames.SO_TIMEOUT, Constants.SO_TIMEOUT);  
    }  
  
    public static HttpClient getHttpClient() {  
        return new DefaultHttpClient(cm, httpParams);  
    }  
  
}  

如果想实现文件断点下载的话，只需要在记录每个线程当前下载了多少长度的内容即可，可以将其持久化到文件或数据库中保存起来，然后线程开始下载的时候都先读取一下它当前下载了多少就OK了。

标签： ssl 代码 数据库

版权申明：本站文章部分自网络，如有侵权，请联系：west999com@outlook.com
特别注意：本站所有转载文章言论不代表本站观点！
本站所提供的图片等素材，版权归原作者所有，如需使用，请与原作者联系。