diff --git a/CHANGELOG.md b/CHANGELOG.md index df7c3fb6..d156d45f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## CHANGE LOG + +### v6.1.1 +2014-02-25 issue [#41](https://github.com/qiniu/csharp-sdk/pull/41) + +- 简化断点续上传,删除bput逻辑 +- 增加断点续上传中块上传结果本地缓存特性 +- 修复bug:>2.5GB文件上传失败 + ### v6.1.0 2014-02-18 issue [#37](https://github.com/qiniu/csharp-sdk/pull/37) diff --git a/Docs/README.md b/Docs/README.md index 70a011e5..a8944883 100644 --- a/Docs/README.md +++ b/Docs/README.md @@ -424,7 +424,7 @@ public static void PutFile(string bucket, string key, string fname) { var policy = new PutPolicy(bucket, 3600); string upToken = policy.Token(); - PutExtra extra = new PutExtra { Bucket = bucket }; + PutExtra extra = new PutExtra(); IOClient client = new IOClient(); client.PutFinished += new EventHandler((o, ret) => { if (ret.OK) diff --git a/Qiniu/IO/Resumable/ResumablePut.cs b/Qiniu/IO/Resumable/ResumablePut.cs index 22893e0a..826df04b 100644 --- a/Qiniu/IO/Resumable/ResumablePut.cs +++ b/Qiniu/IO/Resumable/ResumablePut.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Text; #if NET40 @@ -64,16 +65,23 @@ public ResumablePutExtra Extra set { extra = value; } } + /// + /// Allow cache put result + /// + public bool AllowCache = true; + /// /// 断点续传类 /// /// /// - public ResumablePut(Settings putSetting, ResumablePutExtra extra) + /// true:允许上传结果在本地保存,这样当网络失去连接而再次重新上传时,对已经上传成功的快不需要再次上传 + public ResumablePut(Settings putSetting, ResumablePutExtra extra,bool allowcache = true) { extra.chunkSize = putSetting.ChunkSize; this.putSetting = putSetting; this.extra = extra; + this.AllowCache = allowcache; } /// @@ -88,6 +96,13 @@ public CallRet PutFile(string upToken, string localFile, string key) { throw new Exception(string.Format("{0} does not exist", localFile)); } + string puttedFile = string.Empty; + Dictionary puttedBlk = new Dictionary(); + if (this.AllowCache) + { + puttedFile = ResumbalePutHelper.GetPutHistroryFile(localFile); + puttedBlk = ResumbalePutHelper.GetHistory(puttedFile); + } PutAuthClient client = new PutAuthClient(upToken); CallRet ret; using (FileStream fs = File.OpenRead(localFile)) @@ -97,26 +112,21 @@ public CallRet PutFile(string upToken, string localFile, string key) chunks = fsize / extra.chunkSize + 1; extra.Progresses = new BlkputRet[block_cnt]; //并行上传 -#if NET35||NET20 for (int i = 0; i < block_cnt; i++) { -#elif NET40 - Parallel.For(0, block_cnt, (i) =>{ -#endif + if (this.AllowCache && puttedBlk != null && puttedBlk.ContainsKey(i) && puttedBlk[i] != null) + { + Console.WriteLine(string.Format("block{0} has putted", i)); + extra.Progresses[i] = puttedBlk[i]; + continue; + } int readLen = BLOCKSIZE; - if ((i + 1) * BLOCKSIZE > fsize) - readLen = (int)(fsize - i * BLOCKSIZE); + if ((long)(i + 1) * BLOCKSIZE > fsize) + readLen = (int)(fsize - (long)i * BLOCKSIZE); byte[] byteBuf = new byte[readLen]; -#if NET40 - lock (fs) - { -#endif - fs.Seek(i * BLOCKSIZE, SeekOrigin.Begin); + fs.Seek((long)i * BLOCKSIZE, SeekOrigin.Begin); fs.Read(byteBuf, 0, readLen); -#if NET40 - } -#endif //并行上传BLOCK BlkputRet blkRet = ResumableBlockPut(client, byteBuf, i, readLen); if (blkRet == null) @@ -125,13 +135,13 @@ public CallRet PutFile(string upToken, string localFile, string key) } else { + if (this.AllowCache) + { + ResumbalePutHelper.Append(puttedFile, i, blkRet); + } extra.OnNotify(new PutNotifyEvent(i, readLen, extra.Progresses[i])); } -#if NET35||NET20 } -#elif NET40 - }); -#endif ret = Mkfile(client, key, fs.Length); } if (ret.OK) @@ -181,56 +191,41 @@ private BlkputRet ResumableBlockPut(Client client, byte[] body, int blkIdex, int uint crc32 = CRC32.CheckSumBytes(firstChunk); for (int i = 0; i < putSetting.TryTimes; i++) { - extra.Progresses[blkIdex] = Mkblock(client, firstChunk, body.Length); - if (extra.Progresses[blkIdex] == null || crc32 != extra.Progresses[blkIdex].crc32) + try + { + extra.Progresses[blkIdex] = Mkblock(client, firstChunk, body.Length); + } + catch (Exception ee) { if (i == (putSetting.TryTimes - 1)) { - return null; + throw ee; } + System.Threading.Thread.Sleep(1000); continue; } - else - { - progress(); - break; - } - } - } - #endregion - - #region PutBlock - while (extra.Progresses[blkIdex].offset < blkSize) - { - bodyLength = (chunkSize < (blkSize - extra.Progresses[blkIdex].offset)) ? chunkSize : (int)(blkSize - extra.Progresses[blkIdex].offset); - byte[] chunk = new byte[bodyLength]; - Array.Copy(body, extra.Progresses[blkIdex].offset, chunk, 0, bodyLength); - for (int i = 0; i < putSetting.TryTimes; i++) - { - extra.Progresses[blkIdex] = BlockPut(client, extra.Progresses[blkIdex], new MemoryStream(chunk), bodyLength); - if (extra.Progresses[blkIdex] == null) + if (extra.Progresses[blkIdex] == null || crc32 != extra.Progresses[blkIdex].crc32) { if (i == (putSetting.TryTimes - 1)) { return null; } + System.Threading.Thread.Sleep(1000); continue; } else { - uploadedChunks++; - if (Progress != null) - { - Progress((float)uploadedChunks / chunks); - } + progress(); break; } } } #endregion + return extra.Progresses[blkIdex]; } + private BlkputRet Mkblock(Client client, byte[] firstChunk, long blkSize) { string url = string.Format("{0}/mkblk/{1}", Config.UP_HOST, blkSize); @@ -242,17 +237,6 @@ private BlkputRet Mkblock(Client client, byte[] firstChunk, long blkSize) return null; } - private BlkputRet BlockPut(Client client, BlkputRet ret, Stream body, long length) - { - string url = string.Format("{0}/bput/{1}/{2}", Config.UP_HOST, ret.ctx, ret.offset); - CallRet callRet = client.CallWithBinary(url, "application/octet-stream", body, length); - if (callRet.OK) - { - return QiniuJsonHelper.ToObject(callRet.Response); - } - return null; - } - private CallRet Mkfile(Client client, string key, long fsize) { StringBuilder urlBuilder = new StringBuilder(); diff --git a/Qiniu/IO/Resumable/ResumablePutHelper.cs b/Qiniu/IO/Resumable/ResumablePutHelper.cs new file mode 100644 index 00000000..20e2d5a9 --- /dev/null +++ b/Qiniu/IO/Resumable/ResumablePutHelper.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.IO; +using System.IO.Compression; +using Qiniu.IO.Resumable; +namespace Qiniu +{ + public static class ResumbalePutHelper + { + public static Dictionary GetHistory(string filename) + { + return ParseHistory(filename); + } + + /// + /// File Format: + /// i,offset,ctx/n + /// i,offset,ctx/n + /// + /// + public static Dictionary ParseHistory(string tempFile) + { + if (!File.Exists(tempFile)) + { + return null; + } + string[] lines = File.ReadAllLines(tempFile); + Dictionary result = new Dictionary(); + foreach (string line in lines) + { + string[] fields = line.Split(','); + BlkputRet ret = new BlkputRet(); + ret.offset = ulong.Parse(fields[1]); + ret.ctx = fields[2]; + int idx = int.Parse(fields[0]); + result.Add(idx, ret); + } + return result; + } + + /// + /// + /// + /// + /// + /// + public static void Append(string tempFileName, int idx, BlkputRet ret) + { + string content = idx + "," + ret.offset + "," + ret.ctx + "\n"; + File.AppendAllText(tempFileName, content); + } + + /// + /// ȡļSHA1ֵ + /// + /// + /// base64sha1ֵ + public static string GetFileBase64Sha1(string filename) + { + SHA1 sha1 = new SHA1CryptoServiceProvider(); + using (Stream reader = System.IO.File.OpenRead(filename)) + { + byte[] result = sha1.ComputeHash(reader); + return BitConverter.ToString(result); + } + } + + /// + /// ȡʱļ + /// + /// + /// + public static string GetTempleFile(string filename) + { + string tempPath = Path.GetTempPath(); + return tempPath + filename; + } + + public static string GetPutHistroryFile(string filename) { + + string f = GetFileBase64Sha1(filename); + return GetTempleFile(f); + } + } +} + diff --git a/Qiniu/IO/Resumable/ResumablePutRet.cs b/Qiniu/IO/Resumable/ResumablePutRet.cs index e7dfa6d0..a04828e2 100644 --- a/Qiniu/IO/Resumable/ResumablePutRet.cs +++ b/Qiniu/IO/Resumable/ResumablePutRet.cs @@ -13,6 +13,6 @@ public class BlkputRet [JsonProperty("crc32")] public UInt32 crc32; [JsonProperty("offset")] - public UInt32 offset; + public ulong offset; } } diff --git a/Qiniu/IO/Resumable/Settings.cs b/Qiniu/IO/Resumable/Settings.cs index 63331210..4c98e784 100644 --- a/Qiniu/IO/Resumable/Settings.cs +++ b/Qiniu/IO/Resumable/Settings.cs @@ -32,8 +32,9 @@ public int TryTimes { /// chunk大小,默认为4MB /// 失败重试次数,默认为3 public Settings (int chunkSize=1 << 22, int tryTimes=3) - { - this.chunkSize = chunkSize; + { + //取消chunk + this.chunkSize = 1 << 22; this.tryTimes = tryTimes; } } diff --git a/Qiniu/Qiniu.2.0.csproj b/Qiniu/Qiniu.2.0.csproj index 00539836..b5e9df3e 100644 --- a/Qiniu/Qiniu.2.0.csproj +++ b/Qiniu/Qiniu.2.0.csproj @@ -15,7 +15,7 @@ - x86 + AnyCPU true full false @@ -53,6 +53,7 @@ + diff --git a/Qiniu/Qiniu.4.0.csproj b/Qiniu/Qiniu.4.0.csproj index 1734a9af..43ffa972 100644 --- a/Qiniu/Qiniu.4.0.csproj +++ b/Qiniu/Qiniu.4.0.csproj @@ -15,7 +15,7 @@ - x86 + AnyCPU true full false @@ -53,6 +53,7 @@ + @@ -102,4 +103,4 @@ - + \ No newline at end of file diff --git a/bin/Qiniu.2.0.dll b/bin/Qiniu.2.0.dll new file mode 100755 index 00000000..aa4d0d98 Binary files /dev/null and b/bin/Qiniu.2.0.dll differ diff --git a/bin/Qiniu.4.0.dll b/bin/Qiniu.4.0.dll index 3263456b..dc9d0294 100755 Binary files a/bin/Qiniu.4.0.dll and b/bin/Qiniu.4.0.dll differ diff --git a/csharp-sdk.2.0.sln b/csharp-sdk.2.0.sln index f374e292..d3f062ea 100644 --- a/csharp-sdk.2.0.sln +++ b/csharp-sdk.2.0.sln @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 11.00 # Visual Studio 2010 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SDK", "SDK", "{65603FA4-DDC3-4FD7-ADA3-7BCC2161A247}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu", "Qiniu\Qiniu.2.0.csproj", "{AD4EA9D1-11C2-4BF6-8A06-72A966BC1B80}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.2.0", "Qiniu\Qiniu.2.0.csproj", "{AD4EA9D1-11C2-4BF6-8A06-72A966BC1B80}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.Test", "Qiniu.Test\Qiniu.2.0.Test.csproj", "{95DC2A77-2344-4315-9F6F-334CC928459C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.2.0.Test", "Qiniu.Test\Qiniu.2.0.Test.csproj", "{95DC2A77-2344-4315-9F6F-334CC928459C}" EndProject Global GlobalSection(TestCaseManagementSettings) = postSolution diff --git a/csharp-sdk.4.0.sln b/csharp-sdk.4.0.sln index a98455bc..7d25e03f 100644 --- a/csharp-sdk.4.0.sln +++ b/csharp-sdk.4.0.sln @@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 11.00 # Visual Studio 2010 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SDK", "SDK", "{65603FA4-DDC3-4FD7-ADA3-7BCC2161A247}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu", "Qiniu\Qiniu.4.0.csproj", "{AD4EA9D1-11C2-4BF6-8A06-72A966BC1B80}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.4.0", "Qiniu\Qiniu.4.0.csproj", "{AD4EA9D1-11C2-4BF6-8A06-72A966BC1B80}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.Test", "Qiniu.Test\Qiniu.4.0.Test.csproj", "{95DC2A77-2344-4315-9F6F-334CC928459C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Qiniu.4.0.Test", "Qiniu.Test\Qiniu.4.0.Test.csproj", "{95DC2A77-2344-4315-9F6F-334CC928459C}" EndProject Global GlobalSection(TestCaseManagementSettings) = postSolution