现在的位置: 首页 > 自动控制 > 工业·编程 > 正文

多线程断点续传研究(1)

2015-11-08 14:10 工业·编程 ⁄ 共 13419字 ⁄ 字号 评论 1 条

知道通过HttpWebRequest就可以进行多线程断点下载,是我不用考虑从Socket写起。

对于一个多线程断点续传程序,我大致认为只要考虑如下几点问题就行了。

1. 下载数据可以从给定位置进行;

2. 可以进行分块下载;

3. 记录下载位置,以供下次重新下载的时候使用。

通过对原文的阅读来看,发现以上问题的前两个已经实现。这样会使需要附加的操作会更简单些。

为了能记录每个线程下载的位置,我借用了最简单的方式,就是xml方式。

以下就是多线程下载的主文件内容:

//--------------------------- Download File class ---------------------------------------

//---File: clsNewDownloadFile

//---Description: The multi-download file class file using HttpWebRequest class

//---Author: Knight

//---Date: Aug.4, 2006

//---------------------------------------------------------------------------------------

//---------------------------{Download File class}---------------------------------------

namespace DownloadFile

{

using System;

using System.IO;

using System.Net;

using System.Text;

using System.Security;

using System.Threading;

using System.Collections.Specialized;

using System.Diagnostics;

using System.Data;

/// <summary>

/// Download file info

/// </summary>

public class DownloadFileInfo

{

private string _RequestURL;

private string _ResponseURL;

private string _FileName;

private int _TotalSize;

private int _BlockLeftSize;

public string RequestURL

{

get{ return _RequestURL; }

}

public string ResponseURL

{

get{ return _ResponseURL; }

}

public string FileName

{

get{ return _FileName; }

}

public int TotalSize

{

get{ return _TotalSize;}

set{ _TotalSize = value;}

}

public int BlockLeftSize

{

get{ return _BlockLeftSize;}

set{ _BlockLeftSize = value;}

}

public DownloadFileInfo(

string RequestURL,

string ResponseURL,

string FileName,

int TotalSize,

int BlockLeftSize )

{

this._RequestURL = RequestURL;

this._ResponseURL = ResponseURL;

this._FileName = FileName;

this._TotalSize = TotalSize;

this._BlockLeftSize = BlockLeftSize;

}

public DownloadFileInfo(

string RequestURL,

string ResponseURL,

string FileName ):

this( RequestURL, ResponseURL, FileName, 0, 0 )

{

}

}

/// <summary>

/// Download event arguments

/// </summary>

public class DownLoadEventArgs : System.EventArgs

{

private DownloadFileInfo _DownloadFileInfo;

private int _Position;

private int _ReadCount;

private int _ThreadNO;

public DownloadFileInfo DownFileInfo

{

get{ return _DownloadFileInfo; }

}

public int StartPosition

{

get{ return _Position; }

}

public int ReadCount

{

get{ return _ReadCount; }

}

public int ThreadNO

{

get{ return _ThreadNO; }

}

public DownLoadEventArgs(

DownloadFileInfo DownFileInfo,

int nStartPostion,

int nReadCount,

int nThreadNO )

{

this._DownloadFileInfo = DownFileInfo;

this._Position = nStartPostion;

this._ReadCount = nReadCount;

this._ThreadNO = nThreadNO;

}

}

public delegate void DataReceivedEventHandler( DownLoadEventArgs e );

public delegate void ThreadFinishedHandler();

/// <summary>

/// class for sub-download threads

/// </summary>

public class SubDownloadThread

{

private readonly DownloadFileInfo _FileInfo;

private int _ThreadNO;

private int _Position;

private int _BlockSizeLeft;

public event DataReceivedEventHandler DataReceived;

private ThreadFinishedHandler _Finished;

private bool _IsStopped;

private Thread thdDownload;

public SubDownloadThread(

DownloadFileInfo DownFileInfo,

int ThreadNO,

int Position,

int BlockSizeLeft,

ThreadFinishedHandler Finished )

{

this._FileInfo = DownFileInfo;

this._ThreadNO = ThreadNO;

this._Position = Position;

this._BlockSizeLeft = BlockSizeLeft;

this._Finished = Finished;

}

/// <summary>

/// Start to create thread to download file

/// </summary>

public void StartDownload()

{

thdDownload = new Thread( new ThreadStart( this.Download ) );

thdDownload.Start();

}

/// <summary>

/// Stop current download-thread

/// </summary>

public void Stop()

{

_IsStopped = true;

if( thdDownload.ThreadState == System.Threading.ThreadState.Running )

thdDownload.Join( 10 );

Debug.WriteLine( string.Format( "Thread NO:{0} is stopped!" ,_ThreadNO ) );

}

/// <summary>

/// Function for sub-thread

/// </summary>

private void Download()

{

HttpWebResponse hwrp = null;

try

{

Debug.WriteLine( string.Format( "Thread NO:{0} begins to download!" ,_ThreadNO ) );

// Creat request by url

HttpWebRequest hwrq = (HttpWebRequest) WebRequest.Create(

new Uri( this._FileInfo.RequestURL ));

// Go to download position

hwrq.AddRange( _Position );

// Get response from request

hwrp = (HttpWebResponse) hwrq.GetResponse();

}

catch (Exception e)

{

Debug.WriteLine( e.Message );

return;

}

// download and write data from

Debug.WriteLine( string.Format( "Thread NO:{0} call function named DownloadData!" ,

_ThreadNO ) );

DownloadData( hwrp );

}

/// <summary>

/// Download data through web request

/// </summary>

/// <param name="Response"></param>

private void DownloadData( WebResponse Response )

{

const int BUFFER_SIZE = 0x10000;//64k buffer size

byte[] bBuffer = new byte[ BUFFER_SIZE ];

Stream sReader = Response.GetResponseStream();

if( sReader == null ) return;

int nReadCount = 0;

while( !_IsStopped && this._BlockSizeLeft > 0 )

{

Debug.WriteLine( string.Format( "Thread NO:{0} reads data!" ,

_ThreadNO ) );

// Set read size

nReadCount = ( BUFFER_SIZE > this._BlockSizeLeft )

? this._BlockSizeLeft:BUFFER_SIZE ;//Get current read count

// Read data

int nRealReadCount = sReader.Read( bBuffer, 0, nReadCount );

if( nRealReadCount <= 0 ) break;

Debug.WriteLine( string.Format( "Thread NO:{0} writes data!" ,

_ThreadNO ) );

// Write data

using( FileStream sw = new FileStream(

this._FileInfo.FileName,

System.IO.FileMode.OpenOrCreate,

System.IO.FileAccess.ReadWrite, System.IO.FileShare.ReadWrite))

{

sw.Position = this._Position;

sw.Write( bBuffer, 0, nRealReadCount );

Debug.WriteLine( string.Format( "Thread NO:{0} writes {1} data!" ,

_ThreadNO, nRealReadCount ) );

sw.Flush();

sw.Close();

}

Debug.WriteLine( string.Format( "Thread NO:{0} send callback info!" ,

_ThreadNO ) );

// Call back

DataReceived( new DownLoadEventArgs( this._FileInfo,

this._Position,

nRealReadCount,

this._ThreadNO ) );

// Set position and block left

this._Position += nRealReadCount;

this._BlockSizeLeft -= nRealReadCount;

}

if( _IsStopped ) return;

Debug.WriteLine( string.Format( "Thread NO:{0} sends finish-info!" ,

_ThreadNO ) );

_Finished();

}

}

/// <summary>

/// Class for download thread

/// </summary>

public class DownloadThread

{

private DownloadFileInfo _FileInfo;

private SubDownloadThread[] subThreads;

private int nThreadFinishedCount;

private DataRow drFileInfo;

private DataTable dtThreadInfo;

public DataReceivedEventHandler DataReceived;

public event ThreadFinishedHandler Finished;

/// Class's Constructor

public DownloadThread(

string RequestURL,

string ResponseURL,

string FileName,

DataRow NewFileInfo,

int SubThreadNum )

{

// Create download file info

_FileInfo = new DownloadFileInfo( RequestURL, ResponseURL, FileName );

// Create request to get file info

bool blnMultiDownload = GetFileDownInfo();

// Create file info datarow

drFileInfo = NewFileInfo;

InitDataRow();

// Create subthreads and set these info in threadinfo table

if( SubThreadNum <= 0 ) SubThreadNum = 5;//Defualt value

//Not support to be multi-thread download or less than 64k

if( !blnMultiDownload || _FileInfo.TotalSize <= 0x10000 ) SubThreadNum = 1;

subThreads = new SubDownloadThread[SubThreadNum];

nThreadFinishedCount = 0;

CreateThreadTable( SubThreadNum );

}

public DownloadThread( DataRow DownloadFileInfo, DataTable ThreadInfos )

{

// Create download file info

drFileInfo = DownloadFileInfo;

_FileInfo = new DownloadFileInfo(

drFileInfo["RequestURL"].ToString(),

drFileInfo["ResponseURL"].ToString(),

drFileInfo["FileName"].ToString(),

Convert.ToInt32( drFileInfo["TotalSize"].ToString() ),

Convert.ToInt32( drFileInfo["BlockLeftSize"].ToString() ) );

// Create sub thread class objects

dtThreadInfo = ThreadInfos;

subThreads = new SubDownloadThread[ dtThreadInfo.Rows.Count ];

nThreadFinishedCount = 0;

}

/// {Class's Constructor}

/// <summary>

/// Start to download

/// </summary>

public void Start()

{

StartSubThreads();

}

/// <summary>

/// Create table to save thread info

/// </summary>

/// <param name="SubThreadNum"></param>

private void CreateThreadTable( int SubThreadNum )

{

int nThunkSize = _FileInfo.TotalSize / SubThreadNum;

dtThreadInfo = new DataTable("DownloadFileInfo");

dtThreadInfo.Columns.Add( "ThreadNO", typeof(int) );

dtThreadInfo.Columns.Add( "Position", typeof(int) );

dtThreadInfo.Columns.Add( "BlockLeftSize", typeof(int) );

DataRow drNew;

int i = 0;

for( ; i < SubThreadNum-1; i++ )

{

drNew = dtThreadInfo.NewRow();

drNew["ThreadNO"] = i;

drNew["Position"] = i * nThunkSize;

drNew["BlockLeftSize"] = nThunkSize;

dtThreadInfo.Rows.Add( drNew );

}

drNew = dtThreadInfo.NewRow();

drNew["ThreadNO"] = i;

drNew["Position"] = i * nThunkSize;

drNew["BlockLeftSize"] = _FileInfo.TotalSize - i * nThunkSize;

dtThreadInfo.Rows.Add( drNew );

dtThreadInfo.AcceptChanges();

}

/// <summary>

/// Start sub-threads to download file

/// </summary>

private void StartSubThreads()

{

foreach( DataRow dr in dtThreadInfo.Rows )

{

int ThreadNO = Convert.ToInt32( dr["ThreadNO"].ToString() );

if( Convert.ToInt32( dr["BlockLeftSize"].ToString() ) > 0 )

{

subThreads[ ThreadNO ] = new SubDownloadThread(

_FileInfo,

ThreadNO,

Convert.ToInt32( dr["Position"].ToString() ),

Convert.ToInt32( dr["BlockLeftSize"].ToString() ),

new ThreadFinishedHandler( this.DownloadFinished ) );

subThreads[ ThreadNO ].DataReceived += this.DataReceived;

subThreads[ ThreadNO ].StartDownload();

}

else

DownloadFinished();

}

}

/// <summary>

/// Save download file info

/// </summary>

private void InitDataRow()

{

drFileInfo.BeginEdit();

drFileInfo["RequestURL"] = _FileInfo.RequestURL;

drFileInfo["ResponseURL"] = _FileInfo.ResponseURL;

drFileInfo["FileName"] = _FileInfo.FileName;

drFileInfo["TotalSize"] = _FileInfo.TotalSize;

drFileInfo["BlockLeftSize"] = _FileInfo.BlockLeftSize;

drFileInfo["CreatedTime"] = DateTime.Now.ToString( "yyyyMMddHHmmss" );

drFileInfo.EndEdit();

drFileInfo.AcceptChanges();

}

/// <summary>

/// Check url to get download file info

/// </summary>

/// <returns></returns>

private bool GetFileDownInfo()

{

HttpWebRequest hwrq;

HttpWebResponse hwrp = null;

try

{

//Create request

hwrq = (HttpWebRequest) WebRequest.Create( _FileInfo.RequestURL );

hwrp = (HttpWebResponse) hwrq.GetResponse();

//Get file size info

long L = hwrp.ContentLength;

L = ((L == -1) || (L > 0x7fffffff)) ? ((long) 0x7fffffff) : L;

_FileInfo.TotalSize = (int)L;

_FileInfo.BlockLeftSize = _FileInfo.TotalSize;

//Check whether this url is supported to be multi-threads download

return (hwrp.Headers["Accept-Ranges"] != null

& hwrp.Headers["Accept-Ranges"] == "bytes");

}

catch (Exception e)

{

throw e;

}

}

/// <summary>

/// Update download thread info

/// </summary>

/// <param name="ThreadNO"></param>

/// <param name="Position"></param>

/// <param name="DownloadSize"></param>

public void UpdateDownloadInfo( int ThreadNO, int Position, int DownloadSize )

{

if( ThreadNO >= dtThreadInfo.Rows.Count ) return;

DataRow drThreadNO = dtThreadInfo.Rows[ThreadNO];

drThreadNO["Position"] = Position + DownloadSize;

drThreadNO["BlockLeftSize"] = Convert.ToInt32( drThreadNO["BlockLeftSize"].ToString() )

- DownloadSize;

drThreadNO.AcceptChanges();

drFileInfo["BlockLeftSize"] = Convert.ToInt32( drFileInfo["BlockLeftSize"].ToString() )

- DownloadSize;

}

/// <summary>

/// Stop download threads

/// </summary>

public void Stop()

{

for( int i = 0; i < subThreads.Length; i++ )

subThreads[i].Stop();

}

/// <summary>

/// Thread download finished

/// </summary>

private void DownloadFinished()

{

// Some download thread finished

nThreadFinishedCount++;

if( nThreadFinishedCount == subThreads.Length )

{

// All download threads finished

drFileInfo.Delete();

//drFileInfo.AcceptChanges();

Finished();

}

}

public DataTable ThreadInfo

{

get{ return dtThreadInfo; }

}

}

}

对于如上类的使用,首先需要创建DataSet以及DataTable来存下载文件的信息。

private DataTable dtFileInfos = null;

private DataSet dsFileInfo = null;

当系统第一次运行的时候,需要初始化此Table。

private void CreateTable()

{

dtFileInfos = new DataTable( "DownloadFileInfo" );

dtFileInfos.Columns.Add( "RequestURL", typeof( string ) );

dtFileInfos.Columns.Add( "ResponseURL", typeof( string ) );

dtFileInfos.Columns.Add( "FileName", typeof( string ) );

dtFileInfos.Columns.Add( "TotalSize", typeof( int ) );

dtFileInfos.Columns.Add( "BlockLeftSize", typeof( int ) );

dtFileInfos.Columns.Add( "CreatedTime", typeof( string ) );

dsFileInfo.Tables.Add( dtFileInfos );

}

进过如上的初始化后,就可以进行下载了,不过需要为DownlodThread初始化两个事件,就是在DataReceived以及DownloadFinished。

private void DataReceived( DownLoadEventArgs e )

{

myDownload.UpdateDownloadInfo( e.ThreadNO, e.StartPosition, e.ReadCount );

Debug.WriteLine( string.Format( "Thread NO:{0} read {2} bytes from {1} postion!" ,

e.ThreadNO,

e.StartPosition,

e.ReadCount ) );

}

private void Finished()

{

dtFileInfos.AcceptChanges();

dsFileInfo.WriteXml( FILE_NAME ); //your log file

MessageBox.Show( "Download finished" );

btnStop.Enabled = false;

btnRestart.Enabled = false;

btnDownload.Enabled = true;

}

开始一个新任务可以如下:

//Create new row to save download file info

DataRow dr = dtFileInfos.NewRow();

dtFileInfos.Rows.Add( dr );

//Create object

myDownload = new DownloadThread(

txtURL.Text,

"",

@"E:/Temp/" + DateTime.Now.ToString( "yyyyMMddHHmmss" ),

dr,

1 );

myDownload.Finished += new ThreadFinishedHandler( Finished );

myDownload.DataReceived = new DataReceivedEventHandler( DataReceived );

myDownload.Start();

暂停可以如下:

private void btnStop_Click(object sender, System.EventArgs e)

{

// Set stop message to all

myDownload.Stop();

// Save log file

dtFileInfos.AcceptChanges();

dsFileInfo.WriteXml( FILE_NAME );

foreach( DataRow dr in dtFileInfos.Rows )

{

DataSet dsThreadInfo = null;

DataTable dt = myDownload.ThreadInfo;

if( dt.DataSet == null )

{

dsThreadInfo = new DataSet();

dsThreadInfo.Tables.Add( myDownload.ThreadInfo );

}

else

dsThreadInfo = dt.DataSet;

dsThreadInfo.WriteXml( dr["CreatedTime"].ToString( ) + ".xml" );

}

}

重新开始可以如下:

private void btnRestart_Click(object sender, System.EventArgs e)

{

// Read thread info

DataSet dsThreadInfo = new DataSet();

foreach( DataRow dr in dtFileInfos.Rows )

{

dsThreadInfo.ReadXml( dr["CreatedTime"].ToString( ) + ".xml" );

}

if( dsThreadInfo.Tables.Count <= 0 ) return;

// Init download object

// Restart to download

myDownload = new DownloadThread( dtFileInfos.Rows[0],

dsThreadInfo.Tables[0] );

myDownload.Finished += new ThreadFinishedHandler( Finished );

myDownload.DataReceived = new DataReceivedEventHandler( DataReceived );

myDownload.Start();

btnStop.Enabled = true;

}

不过当程序关闭后,需要重新Load下载日志文件,如下:

const string FILE_NAME = "Download.Log";

private void LoadFile()

{

dsFileInfo = new DataSet();

if( !File.Exists( FILE_NAME ) )

{

CreateTable();

return;

}

dsFileInfo.ReadXml( FILE_NAME );

if( dsFileInfo.Tables.Count == 0 )

CreateTable();

else

dtFileInfos = dsFileInfo.Tables[ "DownloadFileInfo" ];

}

按照如上的步骤,基本上一个多线程断点续传的程序就完成。

不过,对于多线程来说,从方法引用文章来看,需要在服务器段进行某些设置。也就是说不光是要在下载端要做处理,还要在服务器端坐处理。但是我对此产生怀疑,这也是我迟迟没有发布这篇文章的原因。

目前,对于我所写的类来说,单线程断点续传已经没有问题。但是多线程进行操作的时候,第二个线程发送HttpWebRequest,无法获得请求。不过在此,我没有参照文章所提的方法对服务器端作处理,因为用FlashGet就可以多线程,所以有些怀疑是HttpWebRequest的问题。考虑到时间比较紧的原因,我没有再深究下去,毕竟我没有考虑用HttpWebRequest来实现这个多线程断点下载程序,可能从Socket去写要更好些。

不过通过写这段小程序,我也收获不小,有很多细节没考虑得很清楚,值得仔细考虑,大致如下。

1. 文件读取分块问题;

2. 当下载线程数增加或者减少的时候,如何继续下载一个文件;

3. 日志以及下载log地记录方式。

作者:knight94

目前有 1 条留言    访客:0 条, 博主:0 条 ,引用: 1 条

    外部的引用: 1 条

    • 多线程断点续传研究(2) | 求索阁

    给我留言

    留言无头像?