Skip to content

Feature/resumble upload ex #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
## CHANGE LOG


### v6.1.1
2014-02-25 issue [#41](https://github.com/qiniu/csharp-sdk/pull/41)

- 简化断点续上传,删除bput逻辑
- 增加断点续上传中块上传结果本地缓存特性
- 修复bug:>2.5GB文件上传失败

### v6.1.0

2014-02-18 issue [#37](https://github.com/qiniu/csharp-sdk/pull/37)
Expand Down
2 changes: 1 addition & 1 deletion Docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ public static void PutFile(string bucket, string key, string fname)
{
var policy = new PutPolicy(bucket, 3600);
string upToken = policy.Token();
PutExtra extra = new PutExtra { Bucket = bucket };
PutExtra extra = new PutExtra();
IOClient client = new IOClient();
client.PutFinished += new EventHandler<PutRet>((o, ret) => {
if (ret.OK)
Expand Down
98 changes: 41 additions & 57 deletions Qiniu/IO/Resumable/ResumablePut.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
#if NET40
Expand Down Expand Up @@ -64,16 +65,23 @@ public ResumablePutExtra Extra
set { extra = value; }
}

/// <summary>
/// Allow cache put result
/// </summary>
public bool AllowCache = true;

/// <summary>
/// 断点续传类
/// </summary>
/// <param name="putSetting"></param>
/// <param name="extra"></param>
public ResumablePut(Settings putSetting, ResumablePutExtra extra)
/// <param name="allowcache">true:允许上传结果在本地保存,这样当网络失去连接而再次重新上传时,对已经上传成功的快不需要再次上传</param>
public ResumablePut(Settings putSetting, ResumablePutExtra extra,bool allowcache = true)
{
extra.chunkSize = putSetting.ChunkSize;
this.putSetting = putSetting;
this.extra = extra;
this.AllowCache = allowcache;
}

/// <summary>
Expand All @@ -88,6 +96,13 @@ public CallRet PutFile(string upToken, string localFile, string key)
{
throw new Exception(string.Format("{0} does not exist", localFile));
}
string puttedFile = string.Empty;
Dictionary<int, BlkputRet> puttedBlk = new Dictionary<int, BlkputRet>();
if (this.AllowCache)
{
puttedFile = ResumbalePutHelper.GetPutHistroryFile(localFile);
puttedBlk = ResumbalePutHelper.GetHistory(puttedFile);
}
PutAuthClient client = new PutAuthClient(upToken);
CallRet ret;
using (FileStream fs = File.OpenRead(localFile))
Expand All @@ -97,26 +112,21 @@ public CallRet PutFile(string upToken, string localFile, string key)
chunks = fsize / extra.chunkSize + 1;
extra.Progresses = new BlkputRet[block_cnt];
//并行上传
#if NET35||NET20
for (int i = 0; i < block_cnt; i++)
{
#elif NET40
Parallel.For(0, block_cnt, (i) =>{
#endif

if (this.AllowCache && puttedBlk != null && puttedBlk.ContainsKey(i) && puttedBlk[i] != null)
{
Console.WriteLine(string.Format("block{0} has putted", i));
extra.Progresses[i] = puttedBlk[i];
continue;
}
int readLen = BLOCKSIZE;
if ((i + 1) * BLOCKSIZE > fsize)
readLen = (int)(fsize - i * BLOCKSIZE);
if ((long)(i + 1) * BLOCKSIZE > fsize)
readLen = (int)(fsize - (long)i * BLOCKSIZE);
byte[] byteBuf = new byte[readLen];
#if NET40
lock (fs)
{
#endif
fs.Seek(i * BLOCKSIZE, SeekOrigin.Begin);
fs.Seek((long)i * BLOCKSIZE, SeekOrigin.Begin);
fs.Read(byteBuf, 0, readLen);
#if NET40
}
#endif
//并行上传BLOCK
BlkputRet blkRet = ResumableBlockPut(client, byteBuf, i, readLen);
if (blkRet == null)
Expand All @@ -125,13 +135,13 @@ public CallRet PutFile(string upToken, string localFile, string key)
}
else
{
if (this.AllowCache)
{
ResumbalePutHelper.Append(puttedFile, i, blkRet);
}
extra.OnNotify(new PutNotifyEvent(i, readLen, extra.Progresses[i]));
}
#if NET35||NET20
}
#elif NET40
});
#endif
ret = Mkfile(client, key, fs.Length);
}
if (ret.OK)
Expand Down Expand Up @@ -181,56 +191,41 @@ private BlkputRet ResumableBlockPut(Client client, byte[] body, int blkIdex, int
uint crc32 = CRC32.CheckSumBytes(firstChunk);
for (int i = 0; i < putSetting.TryTimes; i++)
{
extra.Progresses[blkIdex] = Mkblock(client, firstChunk, body.Length);
if (extra.Progresses[blkIdex] == null || crc32 != extra.Progresses[blkIdex].crc32)
try
{
extra.Progresses[blkIdex] = Mkblock(client, firstChunk, body.Length);
}
catch (Exception ee)
{
if (i == (putSetting.TryTimes - 1))
{
return null;
throw ee;
}
System.Threading.Thread.Sleep(1000);
continue;
}
else
{
progress();
break;
}
}
}
#endregion

#region PutBlock
while (extra.Progresses[blkIdex].offset < blkSize)
{
bodyLength = (chunkSize < (blkSize - extra.Progresses[blkIdex].offset)) ? chunkSize : (int)(blkSize - extra.Progresses[blkIdex].offset);
byte[] chunk = new byte[bodyLength];
Array.Copy(body, extra.Progresses[blkIdex].offset, chunk, 0, bodyLength);
for (int i = 0; i < putSetting.TryTimes; i++)
{
extra.Progresses[blkIdex] = BlockPut(client, extra.Progresses[blkIdex], new MemoryStream(chunk), bodyLength);
if (extra.Progresses[blkIdex] == null)
if (extra.Progresses[blkIdex] == null || crc32 != extra.Progresses[blkIdex].crc32)
{
if (i == (putSetting.TryTimes - 1))
{
return null;
}
System.Threading.Thread.Sleep(1000);
continue;
}
else
{
uploadedChunks++;
if (Progress != null)
{
Progress((float)uploadedChunks / chunks);
}
progress();
break;
}
}
}
#endregion

return extra.Progresses[blkIdex];
}


private BlkputRet Mkblock(Client client, byte[] firstChunk, long blkSize)
{
string url = string.Format("{0}/mkblk/{1}", Config.UP_HOST, blkSize);
Expand All @@ -242,17 +237,6 @@ private BlkputRet Mkblock(Client client, byte[] firstChunk, long blkSize)
return null;
}

private BlkputRet BlockPut(Client client, BlkputRet ret, Stream body, long length)
{
string url = string.Format("{0}/bput/{1}/{2}", Config.UP_HOST, ret.ctx, ret.offset);
CallRet callRet = client.CallWithBinary(url, "application/octet-stream", body, length);
if (callRet.OK)
{
return QiniuJsonHelper.ToObject<BlkputRet>(callRet.Response);
}
return null;
}

private CallRet Mkfile(Client client, string key, long fsize)
{
StringBuilder urlBuilder = new StringBuilder();
Expand Down
87 changes: 87 additions & 0 deletions Qiniu/IO/Resumable/ResumablePutHelper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
using System;
using System.Collections.Generic;
using System.Security.Cryptography;
using System.IO;
using System.IO.Compression;
using Qiniu.IO.Resumable;
namespace Qiniu
{
public static class ResumbalePutHelper
{
public static Dictionary<int,BlkputRet> GetHistory(string filename)
{
return ParseHistory(filename);
}

/// <summary>
/// File Format:
/// i,offset,ctx/n
/// i,offset,ctx/n
/// </summary>
/// <param name="tempFile"></param>
public static Dictionary<int,BlkputRet> ParseHistory(string tempFile)
{
if (!File.Exists(tempFile))
{
return null;
}
string[] lines = File.ReadAllLines(tempFile);
Dictionary<int, BlkputRet> result = new Dictionary<int, BlkputRet>();
foreach (string line in lines)
{
string[] fields = line.Split(',');
BlkputRet ret = new BlkputRet();
ret.offset = ulong.Parse(fields[1]);
ret.ctx = fields[2];
int idx = int.Parse(fields[0]);
result.Add(idx, ret);
}
return result;
}

/// <summary>
///
/// </summary>
/// <param name="tempFileName"></param>
/// <param name="idx"></param>
/// <param name="ret"></param>
public static void Append(string tempFileName, int idx, BlkputRet ret)
{
string content = idx + "," + ret.offset + "," + ret.ctx + "\n";
File.AppendAllText(tempFileName, content);
}

/// <summary>
/// ��ȡ�ļ���SHA1ֵ
/// </summary>
/// <param name="filename"></param>
/// <returns>base64�����sha1ֵ</returns>
public static string GetFileBase64Sha1(string filename)
{
SHA1 sha1 = new SHA1CryptoServiceProvider();
using (Stream reader = System.IO.File.OpenRead(filename))
{
byte[] result = sha1.ComputeHash(reader);
return BitConverter.ToString(result);
}
}

/// <summary>
/// ��ȡ��ʱ�ļ���
/// </summary>
/// <param name="filename"></param>
/// <returns></returns>
public static string GetTempleFile(string filename)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

服务端sdk的断点续传不建议做持久化。

{
string tempPath = Path.GetTempPath();
return tempPath + filename;
}

public static string GetPutHistroryFile(string filename) {

string f = GetFileBase64Sha1(filename);
return GetTempleFile(f);
}
}
}

2 changes: 1 addition & 1 deletion Qiniu/IO/Resumable/ResumablePutRet.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,6 @@ public class BlkputRet
[JsonProperty("crc32")]
public UInt32 crc32;
[JsonProperty("offset")]
public UInt32 offset;
public ulong offset;
}
}
5 changes: 3 additions & 2 deletions Qiniu/IO/Resumable/Settings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,9 @@ public int TryTimes {
/// <param name="chunkSize">chunk大小,默认为4MB</param>
/// <param name="tryTimes">失败重试次数,默认为3</param>
public Settings (int chunkSize=1 << 22, int tryTimes=3)
{
this.chunkSize = chunkSize;
{
//取消chunk
this.chunkSize = 1 << 22;
this.tryTimes = tryTimes;
}
}
Expand Down
3 changes: 2 additions & 1 deletion Qiniu/Qiniu.2.0.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
<PlatformTarget>x86</PlatformTarget>
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand Down Expand Up @@ -53,6 +53,7 @@
<Compile Include="IO\PutExtra.cs" />
<Compile Include="IO\PutRet.cs" />
<Compile Include="IO\Resumable\ResumablePutExtra.cs" />
<Compile Include="IO\Resumable\ResumablePutHelper.cs" />
<Compile Include="IO\Resumable\Settings.cs" />
<Compile Include="RS\GetPolicy.cs" />
<Compile Include="RS\PutPolicy.cs" />
Expand Down
5 changes: 3 additions & 2 deletions Qiniu/Qiniu.4.0.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
<PlatformTarget>x86</PlatformTarget>
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
Expand Down Expand Up @@ -53,6 +53,7 @@
<Compile Include="IO\PutExtra.cs" />
<Compile Include="IO\PutRet.cs" />
<Compile Include="IO\Resumable\ResumablePutExtra.cs" />
<Compile Include="IO\Resumable\ResumablePutHelper.cs" />
<Compile Include="IO\Resumable\Settings.cs" />
<Compile Include="RS\GetPolicy.cs" />
<Compile Include="RS\PutPolicy.cs" />
Expand Down Expand Up @@ -102,4 +103,4 @@
<None Include="app.config" />
</ItemGroup>
<ItemGroup />
</Project>
</Project>
Binary file added bin/Qiniu.2.0.dll
Binary file not shown.
Binary file modified bin/Qiniu.4.0.dll
Binary file not shown.
4 changes: 2 additions & 2 deletions csharp-sdk.2.0.sln
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 11.00
# Visual Studio 2010
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SDK", "SDK", "{65603FA4-DDC3-4FD7-ADA3-7BCC2161A247}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu", "Qiniu\Qiniu.2.0.csproj", "{AD4EA9D1-11C2-4BF6-8A06-72A966BC1B80}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.2.0", "Qiniu\Qiniu.2.0.csproj", "{AD4EA9D1-11C2-4BF6-8A06-72A966BC1B80}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.Test", "Qiniu.Test\Qiniu.2.0.Test.csproj", "{95DC2A77-2344-4315-9F6F-334CC928459C}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.2.0.Test", "Qiniu.Test\Qiniu.2.0.Test.csproj", "{95DC2A77-2344-4315-9F6F-334CC928459C}"
EndProject
Global
GlobalSection(TestCaseManagementSettings) = postSolution
Expand Down
4 changes: 2 additions & 2 deletions csharp-sdk.4.0.sln
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 11.00
# Visual Studio 2010
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SDK", "SDK", "{65603FA4-DDC3-4FD7-ADA3-7BCC2161A247}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu", "Qiniu\Qiniu.4.0.csproj", "{AD4EA9D1-11C2-4BF6-8A06-72A966BC1B80}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.4.0", "Qiniu\Qiniu.4.0.csproj", "{AD4EA9D1-11C2-4BF6-8A06-72A966BC1B80}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.Test", "Qiniu.Test\Qiniu.4.0.Test.csproj", "{95DC2A77-2344-4315-9F6F-334CC928459C}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.4.0.Test", "Qiniu.Test\Qiniu.4.0.Test.csproj", "{95DC2A77-2344-4315-9F6F-334CC928459C}"
EndProject
Global
GlobalSection(TestCaseManagementSettings) = postSolution
Expand Down