From 0b8b39dbd1ce86dfb3bc4b90502aab5e2272c005 Mon Sep 17 00:00:00 2001 From: Thomas Claudius Huber Date: Thu, 1 Nov 2018 15:07:48 +0100 Subject: [PATCH 01/15] Update README.md Fix little typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ba3035a3..3df337aa 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ Once you setup the storage blob context, you can start to use `WindowsAzure.Stor ```csharp // Setup the number of the concurrent operations TransferManager.Configurations.ParallelOperations = 64; -// Setup the transfer context and track the upoload progress +// Setup the transfer context and track the upload progress SingleTransferContext context = new SingleTransferContext(); context.ProgressHandler = new Progress((progress) => { From 8799c89833ce09c7087372dddd76312a55184ef0 Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Wed, 28 Nov 2018 14:52:47 +0800 Subject: [PATCH 02/15] Resolve issue of reporting MD5 mismatch issue when content MD5 checking is disabled. --- .../AzureBlobDirectoryLocation.cs | 19 +- .../AzureFileDirectoryLocation.cs | 22 +- lib/TransferJobs/AzureFileLocation.cs | 6 + test/DMLibTest/Cases/CheckContentMD5Test.cs | 313 +++++++++++++++++- 4 files changed, 349 insertions(+), 11 deletions(-) diff --git a/lib/TransferJobs/AzureBlobDirectoryLocation.cs b/lib/TransferJobs/AzureBlobDirectoryLocation.cs index 06042058..a02935d5 100644 --- a/lib/TransferJobs/AzureBlobDirectoryLocation.cs +++ b/lib/TransferJobs/AzureBlobDirectoryLocation.cs @@ -18,18 +18,25 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement #else [DataContract] #endif // BINARY_SERIALIZATION + [KnownType(typeof(SerializableBlobRequestOptions))] internal class AzureBlobDirectoryLocation : TransferLocation #if BINARY_SERIALIZATION , ISerializable #endif // BINARY_SERIALIZATION { private const string BlobDirName = "CloudBlobDir"; + private const string RequestOptionsName = "RequestOptions"; #if !BINARY_SERIALIZATION [DataMember] #endif private SerializableCloudBlobDirectory blobDirectorySerializer; +#if !BINARY_SERIALIZATION + [DataMember] +#endif + private SerializableRequestOptions requestOptions; + /// /// Initializes a new instance of the class. /// @@ -54,6 +61,7 @@ private AzureBlobDirectoryLocation(SerializationInfo info, StreamingContext cont } this.blobDirectorySerializer = (SerializableCloudBlobDirectory)info.GetValue(BlobDirName, typeof(SerializableCloudBlobDirectory)); + this.requestOptions = (SerializableBlobRequestOptions)info.GetValue(RequestOptionsName, typeof(SerializableBlobRequestOptions)); } #endif // BINARY_SERIALIZATION @@ -95,8 +103,14 @@ public CloudBlobDirectory BlobDirectory /// internal BlobRequestOptions BlobRequestOptions { - get; - set; + get + { + return (BlobRequestOptions)SerializableBlobRequestOptions.GetRequestOptions(this.requestOptions); + } + set + { + SerializableRequestOptions.SetRequestOptions(ref this.requestOptions, value); + } } #if BINARY_SERIALIZATION @@ -113,6 +127,7 @@ public void GetObjectData(SerializationInfo info, StreamingContext context) } info.AddValue(BlobDirName, this.blobDirectorySerializer, typeof(SerializableCloudBlobDirectory)); + info.AddValue(RequestOptionsName, this.requestOptions, typeof(SerializableBlobRequestOptions)); } #endif // BINARY_SERIALIZATION diff --git a/lib/TransferJobs/AzureFileDirectoryLocation.cs b/lib/TransferJobs/AzureFileDirectoryLocation.cs index 76c70a7a..6756949e 100644 --- a/lib/TransferJobs/AzureFileDirectoryLocation.cs +++ b/lib/TransferJobs/AzureFileDirectoryLocation.cs @@ -18,18 +18,25 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement #else [DataContract] #endif // BINARY_SERIALIZATION + [KnownType(typeof(SerializableFileRequestOptions))] internal class AzureFileDirectoryLocation : TransferLocation #if BINARY_SERIALIZATION , ISerializable #endif // BINARY_SERIALIZATION { private const string FileDirName = "FileDir"; - + private const string RequestOptionsName = "RequestOptions"; + #if !BINARY_SERIALIZATION [DataMember] #endif private SerializableCloudFileDirectory fileDirectorySerializer; +#if !BINARY_SERIALIZATION + [DataMember] +#endif + private SerializableRequestOptions requestOptions; + /// /// Initializes a new instance of the class. /// @@ -54,6 +61,7 @@ private AzureFileDirectoryLocation(SerializationInfo info, StreamingContext cont } this.fileDirectorySerializer = (SerializableCloudFileDirectory)info.GetValue(FileDirName, typeof(SerializableCloudFileDirectory)); + this.requestOptions = (SerializableRequestOptions)info.GetValue(RequestOptionsName, typeof(SerializableRequestOptions)); } #endif // BINARY_SERIALIZATION @@ -95,8 +103,15 @@ public CloudFileDirectory FileDirectory /// internal FileRequestOptions FileRequestOptions { - get; - set; + get + { + return (FileRequestOptions)SerializableRequestOptions.GetRequestOptions(this.requestOptions); + } + + set + { + SerializableRequestOptions.SetRequestOptions(ref this.requestOptions, value); + } } #if BINARY_SERIALIZATION @@ -113,6 +128,7 @@ public void GetObjectData(SerializationInfo info, StreamingContext context) } info.AddValue(FileDirName, this.fileDirectorySerializer, typeof(SerializableCloudFileDirectory)); + info.AddValue(RequestOptionsName, this.requestOptions, typeof(SerializableRequestOptions)); } #endif // BINARY_SERIALIZATION diff --git a/lib/TransferJobs/AzureFileLocation.cs b/lib/TransferJobs/AzureFileLocation.cs index ddf020fb..9b75fe08 100644 --- a/lib/TransferJobs/AzureFileLocation.cs +++ b/lib/TransferJobs/AzureFileLocation.cs @@ -30,13 +30,19 @@ internal class AzureFileLocation : TransferLocation private const string RequestOptionsName = "RequestOptions"; private const string ETagName = "ETag"; +#if !BINARY_SERIALIZATION [DataMember] +#endif private SerializableAccessCondition accessCondition; +#if !BINARY_SERIALIZATION [DataMember] +#endif private SerializableRequestOptions requestOptions; +#if !BINARY_SERIALIZATION [DataMember] +#endif private SerializableCloudFile fileSerializer; diff --git a/test/DMLibTest/Cases/CheckContentMD5Test.cs b/test/DMLibTest/Cases/CheckContentMD5Test.cs index a02e5de0..af097309 100644 --- a/test/DMLibTest/Cases/CheckContentMD5Test.cs +++ b/test/DMLibTest/Cases/CheckContentMD5Test.cs @@ -7,6 +7,8 @@ namespace DMLibTest.Cases { using System; using System.Collections.Generic; + using System.IO; + using System.Threading; using DMLibTestCodeGen; using Microsoft.VisualStudio.TestTools.UnitTesting; using Microsoft.WindowsAzure.Storage.DataMovement; @@ -147,15 +149,15 @@ public void TestDirectoryCheckContentMD5() TransferEventChecker eventChecker = new TransferEventChecker(); TransferContext context = new DirectoryTransferContext(); eventChecker.Apply(context); - + bool failureReported = false; context.FileFailed += (sender, args) => + { + if (args.Exception != null) { - if (args.Exception != null) - { - failureReported = args.Exception.Message.Contains(checkWrongMD5File); - } - }; + failureReported = args.Exception.Message.Contains(checkWrongMD5File); + } + }; ProgressChecker progressChecker = new ProgressChecker(4, totalSize, 3, 1, 0, totalSize); context.ProgressHandler = progressChecker.GetProgressHandler(); @@ -220,5 +222,304 @@ public void TestDirectoryCheckContentMD5() VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[0], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); } } + + [TestCategory(Tag.Function)] + [DMLibTestMethodSet(DMLibTestMethodSet.LocalDest)] + public void TestDirectoryCheckContentMD5Resume() + { + long fileSize = 5 * 1024; + int fileCountMulti = 32; + long totalSize = fileSize * 4 * fileCountMulti; + string wrongMD5 = "wrongMD5"; + + string checkWrongMD5File = "checkWrongMD5File"; + string checkCorrectMD5File = "checkCorrectMD5File"; + string notCheckWrongMD5File = "notCheckWrongMD5File"; + string notCheckCorrectMD5File = "notCheckCorrectMD5File"; + + // Prepare data for transfer items with checkMD5 + DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); + DirNode checkMD5Folder = new DirNode("checkMD5"); + for (int i = 0; i < fileCountMulti; ++i) + { + var wrongMD5FileNode = new FileNode($"{checkWrongMD5File}_{i}") + { + SizeInByte = fileSize, + MD5 = wrongMD5 + }; + + checkMD5Folder.AddFileNode(wrongMD5FileNode); + + DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{checkCorrectMD5File}_{i}", fileSize); + } + sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); + + // Prepare data for transfer items with disabling MD5 check + DirNode notCheckMD5Folder = new DirNode("notCheckMD5"); + + for (int i = 0; i < fileCountMulti; ++i) + { + var wrongMD5FileNode = new FileNode($"{notCheckWrongMD5File}_{i}") + { + SizeInByte = fileSize, + MD5 = wrongMD5 + }; + + notCheckMD5Folder.AddFileNode(wrongMD5FileNode); + + DMLibDataHelper.AddOneFileInBytes(notCheckMD5Folder, $"{notCheckCorrectMD5File}_{i}", fileSize); + } + + sourceDataInfo.RootNode.AddDirNode(notCheckMD5Folder); + + SourceAdaptor.GenerateData(sourceDataInfo); + + TransferEventChecker eventChecker = new TransferEventChecker(); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TransferContext context = new DirectoryTransferContext(); + eventChecker.Apply(context); + + ProgressChecker progressChecker = new ProgressChecker(4 * fileCountMulti, totalSize, 3 * fileCountMulti, null, 0, totalSize); + context.ProgressHandler = progressChecker.GetProgressHandler(); + List transferExceptions = new List(); + + TransferItem checkMD5Item = new TransferItem() + { + SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), + DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), + IsDirectoryTransfer = true, + SourceType = DMLibTestContext.SourceType, + DestType = DMLibTestContext.DestType, + IsServiceCopy = DMLibTestContext.IsAsync, + TransferContext = context, + Options = new DownloadDirectoryOptions() + { + DisableContentMD5Validation = false, + Recursive = true, + }, + CancellationToken = cancellationTokenSource.Token, + }; + + TransferItem notCheckMD5Item = new TransferItem() + { + SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), + DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, notCheckMD5Folder), + IsDirectoryTransfer = true, + SourceType = DMLibTestContext.SourceType, + DestType = DMLibTestContext.DestType, + IsServiceCopy = DMLibTestContext.IsAsync, + TransferContext = context, + Options = new DownloadDirectoryOptions() + { + DisableContentMD5Validation = true, + Recursive = true, + }, + CancellationToken = cancellationTokenSource.Token + }; + + var executionOption = new TestExecutionOptions(); + executionOption.AfterAllItemAdded = () => + { + // Wait until there are data transferred + progressChecker.DataTransferred.WaitOne(); + + // Cancel the transfer and store the second checkpoint + cancellationTokenSource.Cancel(); + }; + executionOption.LimitSpeed = true; + + var testResult = this.RunTransferItems(new List() { checkMD5Item, notCheckMD5Item }, executionOption); + + eventChecker = new TransferEventChecker(); + context = new DirectoryTransferContext(DMLibTestHelper.RandomReloadCheckpoint(context.LastCheckpoint)); + eventChecker.Apply(context); + + bool failureReported = false; + context.FileFailed += (sender, args) => + { + if (args.Exception != null) + { + failureReported = args.Exception.Message.Contains(checkWrongMD5File); + } + + transferExceptions.Add(args.Exception); + }; + + progressChecker.Reset(); + context.ProgressHandler = progressChecker.GetProgressHandler(); + + checkMD5Item = checkMD5Item.Clone(); + notCheckMD5Item = notCheckMD5Item.Clone(); + + checkMD5Item.TransferContext = context; + notCheckMD5Item.TransferContext = context; + + testResult = this.RunTransferItems(new List() { checkMD5Item, notCheckMD5Item }, new TestExecutionOptions()); + + DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); + DMLibDataInfo actualDataInfo = testResult.DataInfo; + for (int i = 0; i < fileCountMulti; ++i) + { + expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}"); + expectedDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}"); + actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{checkWrongMD5File}_{i}"); + actualDataInfo.RootNode.GetDirNode(notCheckMD5Folder.Name).DeleteFileNode($"{notCheckWrongMD5File}_{i}"); + } + + Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); + Test.Assert(failureReported, "Verify md5 check failure is reported."); + VerificationHelper.VerifyFinalProgress(progressChecker, 3 * fileCountMulti, 0, fileCountMulti); + + if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti) + { + Test.Error("Expect one exception but actually no exception is thrown."); + } + else + { + for (int i = 0; i < fileCountMulti; ++i) + { + VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); + } + } + } + + [TestCategory(Tag.Function)] + [DMLibTestMethodSet(DMLibTestMethodSet.LocalDest)] + public void TestDirectoryCheckContentMD5StreamResume() + { + TestDirectoryCheckContentMD5StreamResume(true); + TestDirectoryCheckContentMD5StreamResume(false); + } + + private void TestDirectoryCheckContentMD5StreamResume(bool checkMD5) + { + long fileSize = 5 * 1024; + int fileCountMulti = 32; + long totalSize = fileSize * 4 * fileCountMulti; + string wrongMD5 = "wrongMD5"; + + string wrongMD5File = "wrongMD5File"; + string correctMD5File = "correctMD5File"; + + // Prepare data for transfer items with checkMD5 + DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty); + DirNode checkMD5Folder = new DirNode(checkMD5 ? "checkMD5" : "notCheckMD5"); + for (int i = 0; i < fileCountMulti; ++i) + { + var wrongMD5FileNode = new FileNode($"{wrongMD5File}_{i}") + { + SizeInByte = fileSize, + MD5 = wrongMD5 + }; + + checkMD5Folder.AddFileNode(wrongMD5FileNode); + + DMLibDataHelper.AddOneFileInBytes(checkMD5Folder, $"{correctMD5File}_{i}", fileSize); + } + sourceDataInfo.RootNode.AddDirNode(checkMD5Folder); + + SourceAdaptor.GenerateData(sourceDataInfo); + DestAdaptor.Cleanup(); + + TransferEventChecker eventChecker = new TransferEventChecker(); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + using (var resumeStream = new MemoryStream()) + { + TransferContext context = new DirectoryTransferContext(resumeStream); + eventChecker.Apply(context); + + ProgressChecker progressChecker = new ProgressChecker(2 * fileCountMulti, + totalSize, checkMD5 ? fileCountMulti : 2 * fileCountMulti, null, 0, totalSize); + + context.ProgressHandler = progressChecker.GetProgressHandler(); + List transferExceptions = new List(); + + TransferItem checkMD5Item = new TransferItem() + { + SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), + DestObject = DestAdaptor.GetTransferObject(sourceDataInfo.RootPath, checkMD5Folder), + IsDirectoryTransfer = true, + SourceType = DMLibTestContext.SourceType, + DestType = DMLibTestContext.DestType, + IsServiceCopy = DMLibTestContext.IsAsync, + TransferContext = context, + Options = new DownloadDirectoryOptions() + { + DisableContentMD5Validation = !checkMD5, + Recursive = true, + }, + CancellationToken = cancellationTokenSource.Token, + }; + + var executionOption = new TestExecutionOptions(); + executionOption.AfterAllItemAdded = () => + { + // Wait until there are data transferred + progressChecker.DataTransferred.WaitOne(); + + // Cancel the transfer and store the second checkpoint + cancellationTokenSource.Cancel(); + }; + executionOption.LimitSpeed = true; + + var testResult = this.RunTransferItems(new List() { checkMD5Item }, executionOption); + + eventChecker = new TransferEventChecker(); + resumeStream.Position = 0; + context = new DirectoryTransferContext(resumeStream); + eventChecker.Apply(context); + + bool failureReported = false; + context.FileFailed += (sender, args) => + { + if (args.Exception != null) + { + failureReported = args.Exception.Message.Contains(wrongMD5File); + } + + transferExceptions.Add(args.Exception); + }; + + progressChecker.Reset(); + context.ProgressHandler = progressChecker.GetProgressHandler(); + + checkMD5Item = checkMD5Item.Clone(); + + checkMD5Item.TransferContext = context; + + testResult = this.RunTransferItems(new List() { checkMD5Item }, new TestExecutionOptions()); + + DMLibDataInfo expectedDataInfo = sourceDataInfo.Clone(); + DMLibDataInfo actualDataInfo = testResult.DataInfo; + for (int i = 0; i < fileCountMulti; ++i) + { + expectedDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}"); + actualDataInfo.RootNode.GetDirNode(checkMD5Folder.Name).DeleteFileNode($"{wrongMD5File}_{i}"); + } + + Test.Assert(DMLibDataHelper.Equals(expectedDataInfo, actualDataInfo), "Verify transfer result."); + Test.Assert(checkMD5 ? failureReported : !failureReported, "Verify md5 check failure is expected."); + VerificationHelper.VerifyFinalProgress(progressChecker, checkMD5 ? fileCountMulti : 2 * fileCountMulti, 0, checkMD5 ? fileCountMulti : 0); + + if (checkMD5) + { + if (testResult.Exceptions.Count != 0 || transferExceptions.Count != fileCountMulti) + { + Test.Error("Expect one exception but actually no exception is thrown."); + } + else + { + for (int i = 0; i < fileCountMulti; ++i) + { + VerificationHelper.VerifyExceptionErrorMessage(transferExceptions[i], new string[] { "The MD5 hash calculated from the downloaded data does not match the MD5 hash stored in the property of source" }); + } + } + } + else + { + Test.Assert(testResult.Exceptions.Count == 0, "Should no exception thrown out when disabling check md5"); + } + } + } } } From 5934d0fcea45c942f1c7c2d5b11651f5efec90c9 Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Sat, 29 Dec 2018 10:12:45 +0800 Subject: [PATCH 03/15] Upgrade to XSCL 9.4.2 --- lib/DataMovement.csproj | 25 ++- lib/Extensions/StorageCopyState.cs | 146 ++++++++++++++++++ lib/Resources.Designer.cs | 2 +- .../AsyncCopyController.cs | 15 +- .../BlobAsyncCopyController.cs | 7 +- .../FileAsyncCopyController.cs | 5 +- lib/TransferManager.cs | 21 --- lib/Transfer_RequestOptions.cs | 25 --- lib/packages.config | 4 + .../DMLibTest/CloudBlobContainerExtensions.cs | 2 +- netcore/DMLibTest/Constants.cs | 19 ++- netcore/DMLibTest/DMLibTest.csproj | 4 +- ...t.WindowsAzure.Storage.DataMovement.csproj | 4 +- test/DMLibTest/DMLibTest.csproj | 17 +- .../Framework/CloudObjectExtensions.cs | 6 - .../Framework/SharedAccessPermissions.cs | 6 - test/DMLibTest/Util/Helpers.cs | 13 +- test/DMLibTest/app.config | 2 +- test/DMLibTest/packages.config | 3 + test/DMLibTestCodeGen/App.config | 2 +- test/DMLibTestCodeGen/DMLibTestCodeGen.csproj | 2 +- test/DMTestLib/DMTestLib.csproj | 2 +- test/MsTestLib/MsTestLib.csproj | 2 +- ...icrosoft.Azure.Storage.DataMovement.nuspec | 6 +- 24 files changed, 236 insertions(+), 104 deletions(-) create mode 100644 lib/Extensions/StorageCopyState.cs diff --git a/lib/DataMovement.csproj b/lib/DataMovement.csproj index 73adf472..a69f84f5 100644 --- a/lib/DataMovement.csproj +++ b/lib/DataMovement.csproj @@ -9,11 +9,11 @@ Properties Microsoft.WindowsAzure.Storage.DataMovement Microsoft.WindowsAzure.Storage.DataMovement - v4.5 + v4.5.2 512 ..\ - CODE_ACCESS_SECURITY;BINARY_SERIALIZATION;REQUEST_EVENT_ARGS_EXPOSES_REQUEST;$(DefineConstants) + CODE_ACCESS_SECURITY;BINARY_SERIALIZATION;$(DefineConstants) true @@ -54,14 +54,19 @@ ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll + + ..\packages\Microsoft.Azure.Storage.Blob.9.4.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + + ..\packages\Microsoft.Azure.Storage.Common.9.4.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + + ..\packages\Microsoft.Azure.Storage.File.9.4.2\lib\net452\Microsoft.Azure.Storage.File.dll + ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll True - - ..\packages\WindowsAzure.Storage.9.3.2\lib\net45\Microsoft.WindowsAzure.Storage.dll - True - ..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll @@ -69,6 +74,7 @@ + @@ -79,6 +85,7 @@ + @@ -210,7 +217,9 @@ - + + Designer + @@ -220,4 +229,4 @@ - \ No newline at end of file + diff --git a/lib/Extensions/StorageCopyState.cs b/lib/Extensions/StorageCopyState.cs new file mode 100644 index 00000000..caa4ec49 --- /dev/null +++ b/lib/Extensions/StorageCopyState.cs @@ -0,0 +1,146 @@ +//------------------------------------------------------------------------------ +// +// Copyright (c) Microsoft Corporation +// +//------------------------------------------------------------------------------ + +namespace Microsoft.WindowsAzure.Storage.DataMovement.Extensions +{ + using System; + + internal enum StorageCopyStatus + { + // + // Summary: + // The copy status is invalid. + Invalid = 0, + // + // Summary: + // The copy operation is pending. + Pending = 1, + // + // Summary: + // The copy operation succeeded. + Success = 2, + // + // Summary: + // The copy operation has been aborted. + Aborted = 3, + // + // Summary: + // The copy operation encountered an error. + Failed = 4 + } + + internal class StorageCopyState + { + public StorageCopyState(Microsoft.WindowsAzure.Storage.Blob.CopyState blobCopyState) + { + this.CopyId = blobCopyState.CopyId; + this.CompletionTime = blobCopyState.CompletionTime; + this.SetStatus(blobCopyState.Status); + this.Source = blobCopyState.Source; + this.BytesCopied = blobCopyState.BytesCopied; + this.TotalBytes = blobCopyState.TotalBytes; + this.StatusDescription = blobCopyState.StatusDescription; + this.DestinationSnapshotTime = blobCopyState.DestinationSnapshotTime; + } + + public StorageCopyState(Microsoft.WindowsAzure.Storage.File.CopyState fileCopyState) + { + this.CopyId = fileCopyState.CopyId; + this.CompletionTime = fileCopyState.CompletionTime; + this.SetStatus(fileCopyState.Status); + this.Source = fileCopyState.Source; + this.BytesCopied = fileCopyState.BytesCopied; + this.TotalBytes = fileCopyState.TotalBytes; + this.StatusDescription = fileCopyState.StatusDescription; + this.DestinationSnapshotTime = fileCopyState.DestinationSnapshotTime; + } + + private void SetStatus(Microsoft.WindowsAzure.Storage.Blob.CopyStatus blobCopyStatus) + { + switch (blobCopyStatus) + { + case Blob.CopyStatus.Invalid: + this.Status = StorageCopyStatus.Invalid; + break; + case Blob.CopyStatus.Pending: + this.Status = StorageCopyStatus.Pending; + break; + case Blob.CopyStatus.Success: + this.Status = StorageCopyStatus.Success; + break; + case Blob.CopyStatus.Aborted: + this.Status = StorageCopyStatus.Aborted; + break; + case Blob.CopyStatus.Failed: + this.Status = StorageCopyStatus.Failed; + break; + default: + this.Status = StorageCopyStatus.Invalid; + break; + } + } + + private void SetStatus(Microsoft.WindowsAzure.Storage.File.CopyStatus fileCopyStatus) + { + switch (fileCopyStatus) + { + case File.CopyStatus.Invalid: + this.Status = StorageCopyStatus.Invalid; + break; + case File.CopyStatus.Pending: + this.Status = StorageCopyStatus.Pending; + break; + case File.CopyStatus.Success: + this.Status = StorageCopyStatus.Success; + break; + case File.CopyStatus.Aborted: + this.Status = StorageCopyStatus.Aborted; + break; + case File.CopyStatus.Failed: + this.Status = StorageCopyStatus.Failed; + break; + default: + this.Status = StorageCopyStatus.Invalid; + break; + } + } + + // + // Summary: + // Gets the ID of the copy operation. + public string CopyId { get; private set; } + // + // Summary: + // Gets the time the copy operation completed, and indicates whether completion + // was due to a successful copy, the cancelling of the operation, or a failure. + public DateTimeOffset? CompletionTime { get; private set; } + // + // Summary: + // Gets the status of the copy operation. + public StorageCopyStatus Status { get; private set; } + // + // Summary: + // Gets the source URI of a copy operation. + public Uri Source { get; private set; } + // + // Summary: + // Gets the number of bytes copied in the operation so far. + public long? BytesCopied { get; private set; } + // + // Summary: + // Gets the total number of bytes in the source of the copy. + public long? TotalBytes { get; private set; } + // + // Summary: + // Gets the description of the current status, if any. + public string StatusDescription { get; private set; } + // + // Summary: + // Gets the incremental destination snapshot time for the latest incremental copy, + // if present. + public DateTimeOffset? DestinationSnapshotTime { get; private set; } + } +} diff --git a/lib/Resources.Designer.cs b/lib/Resources.Designer.cs index f8a67705..20584d6b 100644 --- a/lib/Resources.Designer.cs +++ b/lib/Resources.Designer.cs @@ -19,7 +19,7 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources { diff --git a/lib/TransferControllers/AsyncCopyControllers/AsyncCopyController.cs b/lib/TransferControllers/AsyncCopyControllers/AsyncCopyController.cs index 9c190957..e43bd977 100644 --- a/lib/TransferControllers/AsyncCopyControllers/AsyncCopyController.cs +++ b/lib/TransferControllers/AsyncCopyControllers/AsyncCopyController.cs @@ -14,6 +14,7 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement.TransferControllers using System.Threading.Tasks; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.Blob.Protocol; + using Microsoft.WindowsAzure.Storage.DataMovement.Extensions; using Microsoft.WindowsAzure.Storage.File; internal abstract class AsyncCopyController : TransferControllerBase @@ -497,7 +498,7 @@ private bool HandleStartCopyResult(StorageException se) if (null != se.RequestInformation && BlobErrorCodeStrings.PendingCopyOperation == se.RequestInformation.ErrorCode) { - CopyState copyState = this.FetchCopyStateAsync().Result; + StorageCopyState copyState = this.FetchCopyStateAsync().Result; if (null == copyState) { @@ -564,7 +565,7 @@ private async Task GetCopyStateAsync() this.hasWork = false; this.StartCallbackHandler(); - CopyState copyState = null; + StorageCopyState copyState = null; try { @@ -593,7 +594,7 @@ private async Task GetCopyStateAsync() await this.HandleFetchCopyStateResultAsync(copyState); } - private async Task HandleFetchCopyStateResultAsync(CopyState copyState) + private async Task HandleFetchCopyStateResultAsync(StorageCopyState copyState) { if (null == copyState) { @@ -617,7 +618,7 @@ private async Task HandleFetchCopyStateResultAsync(CopyState copyState) Resources.MismatchFoundBetweenLocalAndServerCopyIdsException); } - if (CopyStatus.Success == copyState.Status) + if (StorageCopyStatus.Success == copyState.Status) { this.UpdateTransferProgress(copyState); @@ -632,7 +633,7 @@ private async Task HandleFetchCopyStateResultAsync(CopyState copyState) this.SetFinished(); } - else if (CopyStatus.Pending == copyState.Status) + else if (StorageCopyStatus.Pending == copyState.Status) { this.UpdateTransferProgress(copyState); @@ -657,7 +658,7 @@ private async Task HandleFetchCopyStateResultAsync(CopyState copyState) } } - private void UpdateTransferProgress(CopyState copyState) + private void UpdateTransferProgress(StorageCopyState copyState) { if (null != copyState && copyState.TotalBytes.HasValue) @@ -782,7 +783,7 @@ await sourceLocation.AzureFile.FetchAttributesAsync( protected abstract Task DoFetchDestAttributesAsync(); protected abstract Task DoStartCopyAsync(); protected abstract void DoHandleGetDestinationException(StorageException se); - protected abstract Task FetchCopyStateAsync(); + protected abstract Task FetchCopyStateAsync(); protected abstract Task SetAttributesAsync(SetAttributesCallbackAsync setAttributes); } } diff --git a/lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs b/lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs index 8831be1a..e276c591 100644 --- a/lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs +++ b/lib/TransferControllers/AsyncCopyControllers/BlobAsyncCopyController.cs @@ -14,6 +14,7 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement.TransferControllers using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.DataMovement; + using Microsoft.WindowsAzure.Storage.DataMovement.Extensions; /// /// Blob asynchronous copy. @@ -121,7 +122,7 @@ protected override Task DoStartCopyAsync() if (BlobType.BlockBlob == this.destBlob.BlobType) { return (this.destBlob as CloudBlockBlob).StartCopyAsync( - this.SourceFile.GenerateCopySourceFile(), + this.SourceFile.GenerateCopySourceUri(), null, destAccessCondition, Utils.GenerateBlobRequestOptions(this.destLocation.BlobRequestOptions), @@ -168,7 +169,7 @@ protected override void DoHandleGetDestinationException(StorageException se) } } - protected override async Task FetchCopyStateAsync() + protected override async Task FetchCopyStateAsync() { await this.destBlob.FetchAttributesAsync( Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition), @@ -176,7 +177,7 @@ await this.destBlob.FetchAttributesAsync( Utils.GenerateOperationContext(this.TransferContext), this.CancellationToken); - return this.destBlob.CopyState; + return new StorageCopyState(this.destBlob.CopyState); } protected override async Task SetAttributesAsync(SetAttributesCallbackAsync setCustomAttributes) diff --git a/lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs b/lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs index 0e752e80..9eb63ee4 100644 --- a/lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs +++ b/lib/TransferControllers/AsyncCopyControllers/FileAsyncCopyController.cs @@ -13,6 +13,7 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement.TransferControllers using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.DataMovement; + using Microsoft.WindowsAzure.Storage.DataMovement.Extensions; using Microsoft.WindowsAzure.Storage.File; /// @@ -118,7 +119,7 @@ protected override void DoHandleGetDestinationException(StorageException se) { } - protected override async Task FetchCopyStateAsync() + protected override async Task FetchCopyStateAsync() { await this.destFile.FetchAttributesAsync( Utils.GenerateConditionWithCustomerCondition(this.destLocation.AccessCondition), @@ -126,7 +127,7 @@ await this.destFile.FetchAttributesAsync( Utils.GenerateOperationContext(this.TransferContext), this.CancellationToken); - return this.destFile.CopyState; + return new StorageCopyState(this.destFile.CopyState); } protected override async Task SetAttributesAsync(SetAttributesCallbackAsync setCustomAttributes) diff --git a/lib/TransferManager.cs b/lib/TransferManager.cs index 622ca954..fd6f8275 100644 --- a/lib/TransferManager.cs +++ b/lib/TransferManager.cs @@ -37,27 +37,6 @@ public static class TransferManager /// private static ConcurrentDictionary allTransfers = new ConcurrentDictionary(); -#if REQUEST_EVENT_ARGS_EXPOSES_REQUEST // DNX profiles don't expose the HttpWebRequest through RequestEventArgs, so the user agent cannot be set - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1810:InitializeReferenceTypeStaticFieldsInline", Justification = "Performance")] - static TransferManager() - { - OperationContext.GlobalSendingRequest += (sender, args) => - { - // User agent string format: [UserAgentPrefix] - // e.g. Suppose UserAgentPrefix is "MyApp", the overall user agent string would be like - // 'MyApp DataMovement/0.5.0.0 WA-Storage/8.0.0 (.NET CLR 4.0.30319.34014; Win32NT 6.2.9200.0)' - string userAgent = Constants.UserAgent + " " + Microsoft.WindowsAzure.Storage.Shared.Protocol.Constants.HeaderConstants.UserAgent; - - if (!string.IsNullOrEmpty(configurations.UserAgentPrefix)) - { - userAgent = configurations.UserAgentPrefix + " " + userAgent; - } - - args.Request.UserAgent = userAgent; - }; - } -#endif // REQUEST_EVENT_ARGS_EXPOSES_REQUEST - /// /// Gets or sets the transfer configurations associated with the transfer manager /// diff --git a/lib/Transfer_RequestOptions.cs b/lib/Transfer_RequestOptions.cs index 33261164..9f1b20f3 100644 --- a/lib/Transfer_RequestOptions.cs +++ b/lib/Transfer_RequestOptions.cs @@ -12,7 +12,6 @@ namespace Microsoft.WindowsAzure.Storage.DataMovement using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.File; using Microsoft.WindowsAzure.Storage.RetryPolicies; - using Microsoft.WindowsAzure.Storage.Table; /// /// Defines default RequestOptions for every type of transfer job. @@ -174,30 +173,6 @@ internal static IRequestOptions CreateDefaultRequestOptions(TransferLocation loc return requestOptions; } - /// - /// Gets the default . - /// - /// The default - [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode", Justification = "It will be called in TableDataMovement project.")] - public static TableRequestOptions DefaultTableRequestOptions - { - get - { - IRetryPolicy defaultRetryPolicy = new TransferRetryPolicy( - retryPoliciesDefaultBackoff, - DefaultRetryCountXMsError, - DefaultRetryCountOtherError); - - return new TableRequestOptions - { - MaximumExecutionTime = DefaultMaximumExecutionTime, - RetryPolicy = defaultRetryPolicy, - ServerTimeout = DefaultServerTimeout, - PayloadFormat = TablePayloadFormat.Json - }; - } - } - /// /// Define retry policy used in blob transfer. /// diff --git a/lib/packages.config b/lib/packages.config index c368fe5a..eaf78266 100644 --- a/lib/packages.config +++ b/lib/packages.config @@ -1,10 +1,14 @@  + + + + \ No newline at end of file diff --git a/netcore/DMLibTest/CloudBlobContainerExtensions.cs b/netcore/DMLibTest/CloudBlobContainerExtensions.cs index 671fd28d..547a6f13 100644 --- a/netcore/DMLibTest/CloudBlobContainerExtensions.cs +++ b/netcore/DMLibTest/CloudBlobContainerExtensions.cs @@ -8,7 +8,7 @@ public static class CloudBlobContainerExtensions { public static void Create(this CloudBlobContainer container, BlobRequestOptions requestOptions = null, OperationContext operationContext = null) { - container.CreateAsync(requestOptions, operationContext).GetAwaiter().GetResult(); + container.CreateAsync(BlobContainerPublicAccessType.Off, requestOptions, operationContext).GetAwaiter().GetResult(); } public static bool CreateIfNotExists(this CloudBlobContainer container, BlobRequestOptions requestOptions = null, OperationContext operationContext = null) diff --git a/netcore/DMLibTest/Constants.cs b/netcore/DMLibTest/Constants.cs index 12e3b5eb..d2b5b872 100644 --- a/netcore/DMLibTest/Constants.cs +++ b/netcore/DMLibTest/Constants.cs @@ -1,7 +1,24 @@ -namespace DMLibTest +using System; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace DMLibTest { internal static class Collections { public const string Global = "global"; } + + internal static class RequestOptions + { + public static BlobRequestOptions DefaultBlobRequestOptions + { + get + { + return new BlobRequestOptions() + { + MaximumExecutionTime = TimeSpan.FromMinutes(15) + }; + } + } + } } diff --git a/netcore/DMLibTest/DMLibTest.csproj b/netcore/DMLibTest/DMLibTest.csproj index c2c55c95..730ba8eb 100644 --- a/netcore/DMLibTest/DMLibTest.csproj +++ b/netcore/DMLibTest/DMLibTest.csproj @@ -95,6 +95,9 @@ + + + @@ -106,7 +109,6 @@ - diff --git a/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj b/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj index 7afebe4d..6d1d9d22 100644 --- a/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj +++ b/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj @@ -35,10 +35,12 @@ + + + - diff --git a/test/DMLibTest/DMLibTest.csproj b/test/DMLibTest/DMLibTest.csproj index a0b9491f..d40df300 100644 --- a/test/DMLibTest/DMLibTest.csproj +++ b/test/DMLibTest/DMLibTest.csproj @@ -8,7 +8,7 @@ Properties DMLibTest DMLibTest - v4.5 + v4.5.2 512 {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 10.0 @@ -56,15 +56,20 @@ ..\..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll True + + ..\..\packages\Microsoft.Azure.Storage.Blob.9.4.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + + ..\..\packages\Microsoft.Azure.Storage.Common.9.4.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + + ..\..\packages\Microsoft.Azure.Storage.File.9.4.2\lib\net452\Microsoft.Azure.Storage.File.dll + ..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll True - - ..\..\packages\WindowsAzure.Storage.9.3.2\lib\net45\Microsoft.WindowsAzure.Storage.dll - True - ..\..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll @@ -225,4 +230,4 @@ - \ No newline at end of file + diff --git a/test/DMLibTest/Framework/CloudObjectExtensions.cs b/test/DMLibTest/Framework/CloudObjectExtensions.cs index 372f6da6..2e2fc453 100644 --- a/test/DMLibTest/Framework/CloudObjectExtensions.cs +++ b/test/DMLibTest/Framework/CloudObjectExtensions.cs @@ -9,7 +9,6 @@ namespace DMLibTest using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.File; using Microsoft.WindowsAzure.Storage.RetryPolicies; - using Microsoft.WindowsAzure.Storage.Table; internal static class CloudObjectExtensions { @@ -70,10 +69,5 @@ internal static class HelperConst { RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(90), 3), }; - - public static TableRequestOptions DefaultTableOptions = new TableRequestOptions - { - RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(90), 3), - }; } } diff --git a/test/DMLibTest/Framework/SharedAccessPermissions.cs b/test/DMLibTest/Framework/SharedAccessPermissions.cs index 43bbb125..b8a489fc 100644 --- a/test/DMLibTest/Framework/SharedAccessPermissions.cs +++ b/test/DMLibTest/Framework/SharedAccessPermissions.cs @@ -8,7 +8,6 @@ namespace DMLibTest using System; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.File; - using Microsoft.WindowsAzure.Storage.Table; [Flags] public enum SharedAccessPermissions @@ -39,11 +38,6 @@ public static SharedAccessFilePermissions ToFilePermissions(this SharedAccessPer return (SharedAccessFilePermissions)Enum.Parse(typeof(SharedAccessFilePermissions), sap.ToString()); } - public static SharedAccessTablePermissions ToTablePermissions(this SharedAccessPermissions sap) - { - return (SharedAccessTablePermissions)Enum.Parse(typeof(SharedAccessTablePermissions), sap.ToString()); - } - public static SharedAccessPermissions ToCommonPermissions(this Enum specificPermissions) { return (SharedAccessPermissions)Enum.Parse(typeof(SharedAccessPermissions), specificPermissions.ToString()); diff --git a/test/DMLibTest/Util/Helpers.cs b/test/DMLibTest/Util/Helpers.cs index b365c016..e75e5bb8 100644 --- a/test/DMLibTest/Util/Helpers.cs +++ b/test/DMLibTest/Util/Helpers.cs @@ -18,16 +18,13 @@ namespace DMLibTest using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; - + using DMLibTest.Framework; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Auth; using Microsoft.WindowsAzure.Storage.Blob; using Microsoft.WindowsAzure.Storage.File; using Microsoft.WindowsAzure.Storage.RetryPolicies; - using Microsoft.WindowsAzure.Storage.Table; using MS.Test.Common.MsTestLib; - - using DMLibTest.Framework; using StorageBlobType = Microsoft.WindowsAzure.Storage.Blob.BlobType; /// @@ -106,7 +103,7 @@ public static bool ListBlobs(string connectionString, string containerName, out public static void WaitForTakingEffect(dynamic cloudStorageClient) { Test.Assert( - cloudStorageClient is CloudBlobClient || cloudStorageClient is CloudTableClient || cloudStorageClient is CloudFileClient, + cloudStorageClient is CloudBlobClient || cloudStorageClient is CloudFileClient, "The argument should only be CloudStorageClient."); if (DMLibTestHelper.GetTestAgainst() != TestAgainst.PublicAzure) @@ -2690,7 +2687,7 @@ public bool CleanupContainer(string containerName) CloudBlobContainer container = BlobClient.GetContainerReference(containerName); if (!container.Exists()) return true; - IEnumerable blobs = container.ListBlobs(null, true, BlobListingDetails.All); + IEnumerable blobs = container.ListBlobs(null, true, BlobListingDetails.All, RequestOptions.DefaultBlobRequestOptions); if (blobs != null) { if (blobs.Count() > 500) @@ -2725,7 +2722,7 @@ public bool CleanupContainer(string containerName) } Thread.Sleep(5 * 1000); - if (container.ListBlobs(null, true, BlobListingDetails.All).Any()) + if (container.ListBlobs(null, true, BlobListingDetails.All, RequestOptions.DefaultBlobRequestOptions).Any()) { Test.Warn("The container hasn't been cleaned actually."); Test.Info("Trying to cleanup the container by recreating it..."); @@ -2736,7 +2733,7 @@ public bool CleanupContainer(string containerName) return true; } } - catch (Exception e) when (Helper.IsNotFoundStorageException(e) || Helper.IsConflictStorageException(e)) + catch (Exception e) when (Helper.IsNotFoundStorageException(e) || Helper.IsConflictStorageException(e) || e is OperationCanceledException) { return CleanupContainerByRecreateIt(containerName); } diff --git a/test/DMLibTest/app.config b/test/DMLibTest/app.config index e6418af2..e3bfb5a4 100644 --- a/test/DMLibTest/app.config +++ b/test/DMLibTest/app.config @@ -12,4 +12,4 @@ - + diff --git a/test/DMLibTest/packages.config b/test/DMLibTest/packages.config index 8fad87bd..c3684452 100644 --- a/test/DMLibTest/packages.config +++ b/test/DMLibTest/packages.config @@ -1,6 +1,9 @@  + + + diff --git a/test/DMLibTestCodeGen/App.config b/test/DMLibTestCodeGen/App.config index 53eed678..2e7446d6 100644 --- a/test/DMLibTestCodeGen/App.config +++ b/test/DMLibTestCodeGen/App.config @@ -1,7 +1,7 @@ - + diff --git a/test/DMLibTestCodeGen/DMLibTestCodeGen.csproj b/test/DMLibTestCodeGen/DMLibTestCodeGen.csproj index 19767bc5..5d502632 100644 --- a/test/DMLibTestCodeGen/DMLibTestCodeGen.csproj +++ b/test/DMLibTestCodeGen/DMLibTestCodeGen.csproj @@ -10,7 +10,7 @@ Properties DMLibTestCodeGen DMLibTestCodeGen - v4.5 + v4.5.2 512 ..\..\ true diff --git a/test/DMTestLib/DMTestLib.csproj b/test/DMTestLib/DMTestLib.csproj index e1d3c84f..a0890108 100644 --- a/test/DMTestLib/DMTestLib.csproj +++ b/test/DMTestLib/DMTestLib.csproj @@ -9,7 +9,7 @@ Properties DMTestLib DMTestLib - v4.5 + v4.5.2 512 diff --git a/test/MsTestLib/MsTestLib.csproj b/test/MsTestLib/MsTestLib.csproj index 2ef0e25a..531fe3a2 100644 --- a/test/MsTestLib/MsTestLib.csproj +++ b/test/MsTestLib/MsTestLib.csproj @@ -9,7 +9,7 @@ MS.Test.Common.MsTestLib MsTestLib Properties - v4.5 + v4.5.2 512 ..\ true diff --git a/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec b/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec index 3cc66138..6791b2f8 100644 --- a/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec +++ b/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec @@ -19,12 +19,14 @@ - + + - + + From eb2aeea9b4b2fc2e29fa52eef7077a7187d9baf9 Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Wed, 16 Jan 2019 10:22:07 +0800 Subject: [PATCH 04/15] Fix a test build issue. --- netcore/DMLibTest/Constants.cs | 14 -------------- test/DMLibTest/Util/DMLibTestConstants.cs | 15 +++++++++++++++ test/DMLibTest/Util/Helpers.cs | 4 ++-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/netcore/DMLibTest/Constants.cs b/netcore/DMLibTest/Constants.cs index d2b5b872..bacdd6c8 100644 --- a/netcore/DMLibTest/Constants.cs +++ b/netcore/DMLibTest/Constants.cs @@ -7,18 +7,4 @@ internal static class Collections { public const string Global = "global"; } - - internal static class RequestOptions - { - public static BlobRequestOptions DefaultBlobRequestOptions - { - get - { - return new BlobRequestOptions() - { - MaximumExecutionTime = TimeSpan.FromMinutes(15) - }; - } - } - } } diff --git a/test/DMLibTest/Util/DMLibTestConstants.cs b/test/DMLibTest/Util/DMLibTestConstants.cs index e496d6fa..c0cb681a 100644 --- a/test/DMLibTest/Util/DMLibTestConstants.cs +++ b/test/DMLibTest/Util/DMLibTestConstants.cs @@ -9,6 +9,7 @@ namespace DMLibTest using Microsoft.WindowsAzure.Storage.DataMovement; using MS.Test.Common.MsTestLib; using Microsoft.WindowsAzure.Storage.RetryPolicies; + using Microsoft.WindowsAzure.Storage.Blob; public static class Tag { @@ -118,5 +119,19 @@ public static int RecursiveFolderDepth return recursiveFolderDepth; } } + + internal static class RequestOptions + { + public static BlobRequestOptions DefaultBlobRequestOptions + { + get + { + return new BlobRequestOptions() + { + MaximumExecutionTime = TimeSpan.FromMinutes(15) + }; + } + } + } } } diff --git a/test/DMLibTest/Util/Helpers.cs b/test/DMLibTest/Util/Helpers.cs index e75e5bb8..59fe2eb7 100644 --- a/test/DMLibTest/Util/Helpers.cs +++ b/test/DMLibTest/Util/Helpers.cs @@ -2687,7 +2687,7 @@ public bool CleanupContainer(string containerName) CloudBlobContainer container = BlobClient.GetContainerReference(containerName); if (!container.Exists()) return true; - IEnumerable blobs = container.ListBlobs(null, true, BlobListingDetails.All, RequestOptions.DefaultBlobRequestOptions); + IEnumerable blobs = container.ListBlobs(null, true, BlobListingDetails.All, DMLibTestConstants.RequestOptions.DefaultBlobRequestOptions); if (blobs != null) { if (blobs.Count() > 500) @@ -2722,7 +2722,7 @@ public bool CleanupContainer(string containerName) } Thread.Sleep(5 * 1000); - if (container.ListBlobs(null, true, BlobListingDetails.All, RequestOptions.DefaultBlobRequestOptions).Any()) + if (container.ListBlobs(null, true, BlobListingDetails.All, DMLibTestConstants.RequestOptions.DefaultBlobRequestOptions).Any()) { Test.Warn("The container hasn't been cleaned actually."); Test.Info("Trying to cleanup the container by recreating it..."); From 60ea2f2dbe10533562f069dc9589389266a8ba7a Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Wed, 16 Jan 2019 10:22:25 +0800 Subject: [PATCH 05/15] Update DMLib version to 0.9.1 --- .../Microsoft.WindowsAzure.Storage.DataMovement.csproj | 2 +- tools/AssemblyInfo/SharedAssemblyInfo.cs | 4 ++-- tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec | 6 +++--- tools/nupkg/buildDebugNupkg.cmd | 8 ++++---- tools/nupkg/buildNupkg.cmd | 8 ++++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj b/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj index 6d1d9d22..de84f1e4 100644 --- a/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj +++ b/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj @@ -2,7 +2,7 @@ Microsoft.WindowsAzure.Storage.DataMovement Class Library - 0.9.0.0 + 0.9.1.0 Microsoft netstandard2.0 true diff --git a/tools/AssemblyInfo/SharedAssemblyInfo.cs b/tools/AssemblyInfo/SharedAssemblyInfo.cs index ed59dca2..b34a0919 100644 --- a/tools/AssemblyInfo/SharedAssemblyInfo.cs +++ b/tools/AssemblyInfo/SharedAssemblyInfo.cs @@ -12,8 +12,8 @@ using System.Resources; using System.Runtime.InteropServices; -[assembly: AssemblyVersion("0.9.0.0")] -[assembly: AssemblyFileVersion("0.9.0.0")] +[assembly: AssemblyVersion("0.9.1.0")] +[assembly: AssemblyFileVersion("0.9.1.0")] [assembly: AssemblyCompany("Microsoft")] [assembly: AssemblyProduct("Microsoft Azure Storage")] diff --git a/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec b/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec index 6791b2f8..d1abffee 100644 --- a/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec +++ b/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec @@ -2,7 +2,7 @@ Microsoft.Azure.Storage.DataMovement - 0.9.0 + 0.9.1 Microsoft Azure Storage Data Movement Library Microsoft Microsoft @@ -17,7 +17,7 @@ A client library designed for high-performance and scalable uploading, downloading and copying data to and from Microsoft Azure Blob and File Storage Microsoft, Azure, Storage, Blob, File, DataMovement, Upload, Download, Copy, High-Performance, Scalable, Reliable, windowsazureofficial - + @@ -32,7 +32,7 @@ - + diff --git a/tools/nupkg/buildDebugNupkg.cmd b/tools/nupkg/buildDebugNupkg.cmd index 66e287ad..7f5eb131 100644 --- a/tools/nupkg/buildDebugNupkg.cmd +++ b/tools/nupkg/buildDebugNupkg.cmd @@ -2,13 +2,13 @@ pushd %~dp0 rmdir /s /q .\workspace mkdir .\workspace copy .\Microsoft.Azure.Storage.DataMovement.nuspec .\workspace -mkdir .\workspace\lib\net45 +mkdir .\workspace\lib\net452 mkdir .\workspace\lib\netstandard2.0 pushd ..\.. del /q /f *.nupkg -copy .\lib\bin\Debug\Microsoft.WindowsAzure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\net45 -copy .\lib\bin\Debug\Microsoft.WindowsAzure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\net45 -copy .\lib\bin\Debug\Microsoft.WindowsAzure.Storage.DataMovement.XML .\tools\nupkg\workspace\lib\net45 +copy .\lib\bin\Debug\Microsoft.WindowsAzure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\net452 +copy .\lib\bin\Debug\Microsoft.WindowsAzure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\net452 +copy .\lib\bin\Debug\Microsoft.WindowsAzure.Storage.DataMovement.XML .\tools\nupkg\workspace\lib\net452 copy .\netcore\Microsoft.WindowsAzure.Storage.DataMovement\bin\Debug\netstandard2.0\Microsoft.WindowsAzure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\netstandard2.0 copy .\netcore\Microsoft.WindowsAzure.Storage.DataMovement\bin\Debug\netstandard2.0\Microsoft.WindowsAzure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\netstandard2.0 copy .\netcore\Microsoft.WindowsAzure.Storage.DataMovement\bin\Debug\netstandard2.0\Microsoft.WindowsAzure.Storage.DataMovement.xml .\tools\nupkg\workspace\lib\netstandard2.0 diff --git a/tools/nupkg/buildNupkg.cmd b/tools/nupkg/buildNupkg.cmd index efa7c645..da8706f2 100644 --- a/tools/nupkg/buildNupkg.cmd +++ b/tools/nupkg/buildNupkg.cmd @@ -2,13 +2,13 @@ pushd %~dp0 rmdir /s /q .\workspace mkdir .\workspace copy .\Microsoft.Azure.Storage.DataMovement.nuspec .\workspace -mkdir .\workspace\lib\net45 +mkdir .\workspace\lib\net452 mkdir .\workspace\lib\netstandard2.0 pushd ..\.. del /q /f *.nupkg -copy .\lib\bin\Release\Microsoft.WindowsAzure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\net45 -copy .\lib\bin\Release\Microsoft.WindowsAzure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\net45 -copy .\lib\bin\Release\Microsoft.WindowsAzure.Storage.DataMovement.XML .\tools\nupkg\workspace\lib\net45 +copy .\lib\bin\Release\Microsoft.WindowsAzure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\net452 +copy .\lib\bin\Release\Microsoft.WindowsAzure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\net452 +copy .\lib\bin\Release\Microsoft.WindowsAzure.Storage.DataMovement.XML .\tools\nupkg\workspace\lib\net452 copy .\netcore\Microsoft.WindowsAzure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.WindowsAzure.Storage.DataMovement.dll .\tools\nupkg\workspace\lib\netstandard2.0 copy .\netcore\Microsoft.WindowsAzure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.WindowsAzure.Storage.DataMovement.pdb .\tools\nupkg\workspace\lib\netstandard2.0 copy .\netcore\Microsoft.WindowsAzure.Storage.DataMovement\bin\Release\netstandard2.0\Microsoft.WindowsAzure.Storage.DataMovement.xml .\tools\nupkg\workspace\lib\netstandard2.0 From 5c6262cfed1672210c55662e029b757496ae7d73 Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Wed, 16 Jan 2019 11:58:34 +0800 Subject: [PATCH 06/15] Fix a code style analysis issue. --- lib/Extensions/StorageCopyState.cs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/lib/Extensions/StorageCopyState.cs b/lib/Extensions/StorageCopyState.cs index caa4ec49..44ac6b0a 100644 --- a/lib/Extensions/StorageCopyState.cs +++ b/lib/Extensions/StorageCopyState.cs @@ -37,25 +37,21 @@ internal class StorageCopyState public StorageCopyState(Microsoft.WindowsAzure.Storage.Blob.CopyState blobCopyState) { this.CopyId = blobCopyState.CopyId; - this.CompletionTime = blobCopyState.CompletionTime; this.SetStatus(blobCopyState.Status); this.Source = blobCopyState.Source; this.BytesCopied = blobCopyState.BytesCopied; this.TotalBytes = blobCopyState.TotalBytes; this.StatusDescription = blobCopyState.StatusDescription; - this.DestinationSnapshotTime = blobCopyState.DestinationSnapshotTime; } public StorageCopyState(Microsoft.WindowsAzure.Storage.File.CopyState fileCopyState) { this.CopyId = fileCopyState.CopyId; - this.CompletionTime = fileCopyState.CompletionTime; this.SetStatus(fileCopyState.Status); this.Source = fileCopyState.Source; this.BytesCopied = fileCopyState.BytesCopied; this.TotalBytes = fileCopyState.TotalBytes; this.StatusDescription = fileCopyState.StatusDescription; - this.DestinationSnapshotTime = fileCopyState.DestinationSnapshotTime; } private void SetStatus(Microsoft.WindowsAzure.Storage.Blob.CopyStatus blobCopyStatus) @@ -114,11 +110,6 @@ private void SetStatus(Microsoft.WindowsAzure.Storage.File.CopyStatus fileCopySt public string CopyId { get; private set; } // // Summary: - // Gets the time the copy operation completed, and indicates whether completion - // was due to a successful copy, the cancelling of the operation, or a failure. - public DateTimeOffset? CompletionTime { get; private set; } - // - // Summary: // Gets the status of the copy operation. public StorageCopyStatus Status { get; private set; } // @@ -137,10 +128,5 @@ private void SetStatus(Microsoft.WindowsAzure.Storage.File.CopyStatus fileCopySt // Summary: // Gets the description of the current status, if any. public string StatusDescription { get; private set; } - // - // Summary: - // Gets the incremental destination snapshot time for the latest incremental copy, - // if present. - public DateTimeOffset? DestinationSnapshotTime { get; private set; } } } From d40fc6ee40f1e44ccd874d22e23e21d3fc79a56d Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Wed, 16 Jan 2019 19:39:55 +0800 Subject: [PATCH 07/15] Add timeout to avoid hang when listing blobs and files/directories in test cases. --- test/DMLibTest/Cases/SnapshotTest.cs | 2 +- .../Framework/CloudObjectExtensions.cs | 2 ++ test/DMLibTest/Util/DMLibTestConstants.cs | 15 +------------- test/DMLibTest/Util/Helpers.cs | 20 +++++++++---------- 4 files changed, 14 insertions(+), 25 deletions(-) diff --git a/test/DMLibTest/Cases/SnapshotTest.cs b/test/DMLibTest/Cases/SnapshotTest.cs index 37dee162..ffdc9c19 100644 --- a/test/DMLibTest/Cases/SnapshotTest.cs +++ b/test/DMLibTest/Cases/SnapshotTest.cs @@ -323,7 +323,7 @@ public void TestFileShareSnapshotsCopyToBase() Test.Assert(task.Result.NumberOfFilesTransferred == fileCount, string.Format("Transferred file :{0} == {1}", task.Result.NumberOfFilesTransferred, fileCount)); // verify that Files in Share Snapshot is transferred - IEnumerable sourceFiles = SourceObject.ListFilesAndDirectories(); + IEnumerable sourceFiles = SourceObject.ListFilesAndDirectories(HelperConst.DefaultFileOptions); foreach (IListFileItem item in sourceFiles) { if (item is CloudFile) diff --git a/test/DMLibTest/Framework/CloudObjectExtensions.cs b/test/DMLibTest/Framework/CloudObjectExtensions.cs index 2e2fc453..295d501e 100644 --- a/test/DMLibTest/Framework/CloudObjectExtensions.cs +++ b/test/DMLibTest/Framework/CloudObjectExtensions.cs @@ -63,11 +63,13 @@ internal static class HelperConst public static BlobRequestOptions DefaultBlobOptions = new BlobRequestOptions { RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(90), 3), + MaximumExecutionTime = TimeSpan.FromMinutes(15) }; public static FileRequestOptions DefaultFileOptions = new FileRequestOptions { RetryPolicy = new LinearRetry(TimeSpan.FromSeconds(90), 3), + MaximumExecutionTime = TimeSpan.FromMinutes(15) }; } } diff --git a/test/DMLibTest/Util/DMLibTestConstants.cs b/test/DMLibTest/Util/DMLibTestConstants.cs index c0cb681a..24d1147f 100644 --- a/test/DMLibTest/Util/DMLibTestConstants.cs +++ b/test/DMLibTest/Util/DMLibTestConstants.cs @@ -10,6 +10,7 @@ namespace DMLibTest using MS.Test.Common.MsTestLib; using Microsoft.WindowsAzure.Storage.RetryPolicies; using Microsoft.WindowsAzure.Storage.Blob; + using Microsoft.WindowsAzure.Storage.File; public static class Tag { @@ -119,19 +120,5 @@ public static int RecursiveFolderDepth return recursiveFolderDepth; } } - - internal static class RequestOptions - { - public static BlobRequestOptions DefaultBlobRequestOptions - { - get - { - return new BlobRequestOptions() - { - MaximumExecutionTime = TimeSpan.FromMinutes(15) - }; - } - } - } } } diff --git a/test/DMLibTest/Util/Helpers.cs b/test/DMLibTest/Util/Helpers.cs index 59fe2eb7..75f8b6f5 100644 --- a/test/DMLibTest/Util/Helpers.cs +++ b/test/DMLibTest/Util/Helpers.cs @@ -82,7 +82,7 @@ public static bool ListBlobs(string connectionString, string containerName, out try { CloudBlobContainer container = BlobClient.GetContainerReference(containerName); - IEnumerable blobs = container.ListBlobs(null, true, BlobListingDetails.All); + IEnumerable blobs = container.ListBlobs(null, true, BlobListingDetails.All, HelperConst.DefaultBlobOptions); if (blobs != null) { foreach (CloudBlob blob in blobs) @@ -1972,7 +1972,7 @@ public static bool CompareCloudFileDirAndLocalDir(CloudFileDirectory dir, string localSubDirs.Add(Path.GetFileName(localSubDir)); } - foreach (IListFileItem item in dir.ListFilesAndDirectories()) + foreach (IListFileItem item in dir.ListFilesAndDirectories(HelperConst.DefaultFileOptions)) { if (item is CloudFile) { @@ -2117,7 +2117,7 @@ public IEnumerable EnumerateFiles(string shareName, string dirName, bool public static IEnumerable EnumerateFiles(CloudFileDirectory dir, bool recursive) { var folders = new List(); - foreach (IListFileItem item in dir.ListFilesAndDirectories()) + foreach (IListFileItem item in dir.ListFilesAndDirectories(HelperConst.DefaultFileOptions)) { if (item is CloudFile) { @@ -2163,7 +2163,7 @@ public IEnumerable EnumerateDirectories(string shareName, string dirName public static IEnumerable EnumerateDirectories(CloudFileDirectory dir, bool recursive) { List dirs = new List(); - foreach (IListFileItem item in dir.ListFilesAndDirectories()) + foreach (IListFileItem item in dir.ListFilesAndDirectories(HelperConst.DefaultFileOptions)) { if (item is CloudFileDirectory) { @@ -2232,7 +2232,7 @@ public bool CleanupFileDirectory(string shareName, string fileDirectoryName) } else { - if (root.ListFilesAndDirectories().Count() > 500) + if (root.ListFilesAndDirectories(HelperConst.DefaultFileOptions).Count() > 500) return CleanupFileShareByRecreateIt(shareName); } @@ -2242,7 +2242,7 @@ public bool CleanupFileDirectory(string shareName, string fileDirectoryName) public static void CleanupFileDirectory(CloudFileDirectory cloudDirectory) { - foreach (IListFileItem item in cloudDirectory.ListFilesAndDirectories()) + foreach (IListFileItem item in cloudDirectory.ListFilesAndDirectories(HelperConst.DefaultFileOptions)) { if (item is CloudFile) { @@ -2498,7 +2498,7 @@ public bool ListBlobs(string containerName, BlobListingDetails listingDetails, o try { CloudBlobContainer container = BlobClient.GetContainerReference(containerName); - IEnumerable blobs = container.ListBlobs(null, true, listingDetails); + IEnumerable blobs = container.ListBlobs(null, true, listingDetails, HelperConst.DefaultBlobOptions); if (blobs != null) { foreach (CloudBlob blob in blobs) @@ -2687,7 +2687,7 @@ public bool CleanupContainer(string containerName) CloudBlobContainer container = BlobClient.GetContainerReference(containerName); if (!container.Exists()) return true; - IEnumerable blobs = container.ListBlobs(null, true, BlobListingDetails.All, DMLibTestConstants.RequestOptions.DefaultBlobRequestOptions); + IEnumerable blobs = container.ListBlobs(null, true, BlobListingDetails.All, HelperConst.DefaultBlobOptions); if (blobs != null) { if (blobs.Count() > 500) @@ -2722,7 +2722,7 @@ public bool CleanupContainer(string containerName) } Thread.Sleep(5 * 1000); - if (container.ListBlobs(null, true, BlobListingDetails.All, DMLibTestConstants.RequestOptions.DefaultBlobRequestOptions).Any()) + if (container.ListBlobs(null, true, BlobListingDetails.All, HelperConst.DefaultBlobOptions).Any()) { Test.Warn("The container hasn't been cleaned actually."); Test.Info("Trying to cleanup the container by recreating it..."); @@ -3371,7 +3371,7 @@ public bool DeleteBlob(string containerName, string blobName) CloudBlobContainer container = BlobClient.GetContainerReference(containerName); if (container.Exists()) { - IEnumerable blobs = container.ListBlobs(blobName, true, BlobListingDetails.All); + IEnumerable blobs = container.ListBlobs(blobName, true, BlobListingDetails.All, HelperConst.DefaultBlobOptions); foreach (CloudBlob blob in blobs) { if (blob.Name == blobName) From 39af9f8beb3ac66e57c7ca2573a693e2b22258b1 Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Fri, 18 Jan 2019 11:30:31 +0800 Subject: [PATCH 08/15] Fix an incorrect block id issue when uploading non-seekable, non-fixed size stream to block blob. --- .../TransferReaders/StreamedReader.cs | 2 +- lib/TransferStatusHelpers/SharedTransferData.cs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/TransferControllers/TransferReaders/StreamedReader.cs b/lib/TransferControllers/TransferReaders/StreamedReader.cs index ae03370c..2dd42cb5 100644 --- a/lib/TransferControllers/TransferReaders/StreamedReader.cs +++ b/lib/TransferControllers/TransferReaders/StreamedReader.cs @@ -488,7 +488,7 @@ private void SetChunkFinish() // Should only get into this block once. if (-1 == this.SharedTransferData.TotalLength) { - this.SharedTransferData.TotalLength = this.readLength; + this.SharedTransferData.UpdateTotalLength(this.readLength); } this.state = State.Finished; diff --git a/lib/TransferStatusHelpers/SharedTransferData.cs b/lib/TransferStatusHelpers/SharedTransferData.cs index 5736d878..269b32b9 100644 --- a/lib/TransferStatusHelpers/SharedTransferData.cs +++ b/lib/TransferStatusHelpers/SharedTransferData.cs @@ -27,6 +27,15 @@ public long TotalLength } } + /// + /// Update totallength but don't update BlockSize accordingly. + /// + /// + public void UpdateTotalLength(long value) + { + this.totalLength = value; + } + public int BlockSize { get; set; } public int MemoryChunksRequiredEachTime { get; set; } From 932a9df2f3863cda62c83b196e94c78ea48dfc58 Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Fri, 25 Jan 2019 10:29:12 +0800 Subject: [PATCH 09/15] Fix test issue "notmatch" is invalid etag value. --- test/DMLibTest/Cases/AccessConditionTest.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/DMLibTest/Cases/AccessConditionTest.cs b/test/DMLibTest/Cases/AccessConditionTest.cs index 5d1997c4..49d7f3f0 100644 --- a/test/DMLibTest/Cases/AccessConditionTest.cs +++ b/test/DMLibTest/Cases/AccessConditionTest.cs @@ -81,7 +81,7 @@ public void TestDestAccessCondition() private void TestAccessCondition(SourceOrDest sourceOrDest) { - string eTag = "notmatch"; + string eTag = "\"notmatch\""; AccessCondition accessCondition = new AccessCondition() { IfMatchETag = eTag @@ -147,10 +147,10 @@ private void TestAccessCondition(SourceOrDest sourceOrDest) Exception exception = result.Exceptions[0]; #if DNXCORE50 VerificationHelper.VerifyTransferException(exception, TransferErrorCode.Unknown); - - // TODO: The InnerException is as expected but has a different message and HttpStatusCode - // compared to the desktop version of XSCL; is this an XSCL bug? - VerificationHelper.VerifyStorageException(exception.InnerException, 0, "The format of value 'notmatch' is invalid."); + + // Verify innner StorageException + VerificationHelper.VerifyStorageException(exception.InnerException, (int)HttpStatusCode.PreconditionFailed, + "The condition specified using HTTP conditional header(s) is not met."); #else VerificationHelper.VerifyTransferException(exception, TransferErrorCode.Unknown); From 5c66725017bd49407129efb223ecf49b624bf24c Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Fri, 1 Feb 2019 11:23:27 +0800 Subject: [PATCH 10/15] Resolve a race condition when uploading a non-fixed sized, non-seekable stream. --- .../TransferReaders/StreamedReader.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/TransferControllers/TransferReaders/StreamedReader.cs b/lib/TransferControllers/TransferReaders/StreamedReader.cs index 2dd42cb5..7331af13 100644 --- a/lib/TransferControllers/TransferReaders/StreamedReader.cs +++ b/lib/TransferControllers/TransferReaders/StreamedReader.cs @@ -449,6 +449,12 @@ private void ReadingDataHandler(ReadDataState asyncState, bool endofStream) if ((currentReadLength == this.SharedTransferData.TotalLength) || endofStream) { Interlocked.Exchange(ref this.readCompleted, 1); + + // Should only get into this block once. + if (-1 == this.SharedTransferData.TotalLength) + { + this.SharedTransferData.UpdateTotalLength(this.readLength); + } } if (endofStream && (-1 != this.SharedTransferData.TotalLength) && (currentReadLength != this.SharedTransferData.TotalLength)) @@ -485,12 +491,6 @@ private void SetChunkFinish() { if (0 == Interlocked.Exchange(ref this.setCompletionDone, 1)) { - // Should only get into this block once. - if (-1 == this.SharedTransferData.TotalLength) - { - this.SharedTransferData.UpdateTotalLength(this.readLength); - } - this.state = State.Finished; if (!this.md5HashStream.SucceededSeparateMd5Calculator) { From 6c78f9488a8fcaa10692cee480aa0deaf9328815 Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Wed, 13 Feb 2019 16:15:07 +0800 Subject: [PATCH 11/15] Update copyright to 2019 --- tools/AssemblyInfo/SharedAssemblyInfo.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/AssemblyInfo/SharedAssemblyInfo.cs b/tools/AssemblyInfo/SharedAssemblyInfo.cs index b34a0919..36116877 100644 --- a/tools/AssemblyInfo/SharedAssemblyInfo.cs +++ b/tools/AssemblyInfo/SharedAssemblyInfo.cs @@ -17,7 +17,7 @@ [assembly: AssemblyCompany("Microsoft")] [assembly: AssemblyProduct("Microsoft Azure Storage")] -[assembly: AssemblyCopyright("Copyright © 2018 Microsoft Corp.")] +[assembly: AssemblyCopyright("Copyright © 2019 Microsoft Corp.")] [assembly: AssemblyTrademark("Microsoft ® is a registered trademark of Microsoft Corporation.")] [assembly: AssemblyCulture("")] From 5bbf41668a451b4bb580f6628bd6a449488e647a Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Mon, 18 Feb 2019 10:36:10 +0800 Subject: [PATCH 12/15] Update version to 0.10.0 --- .../Microsoft.WindowsAzure.Storage.DataMovement.csproj | 2 +- tools/AssemblyInfo/SharedAssemblyInfo.cs | 4 ++-- tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj b/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj index de84f1e4..04d94596 100644 --- a/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj +++ b/netcore/Microsoft.WindowsAzure.Storage.DataMovement/Microsoft.WindowsAzure.Storage.DataMovement.csproj @@ -2,7 +2,7 @@ Microsoft.WindowsAzure.Storage.DataMovement Class Library - 0.9.1.0 + 0.10.0.0 Microsoft netstandard2.0 true diff --git a/tools/AssemblyInfo/SharedAssemblyInfo.cs b/tools/AssemblyInfo/SharedAssemblyInfo.cs index 36116877..7cc154f5 100644 --- a/tools/AssemblyInfo/SharedAssemblyInfo.cs +++ b/tools/AssemblyInfo/SharedAssemblyInfo.cs @@ -12,8 +12,8 @@ using System.Resources; using System.Runtime.InteropServices; -[assembly: AssemblyVersion("0.9.1.0")] -[assembly: AssemblyFileVersion("0.9.1.0")] +[assembly: AssemblyVersion("0.10.0.0")] +[assembly: AssemblyFileVersion("0.10.0.0")] [assembly: AssemblyCompany("Microsoft")] [assembly: AssemblyProduct("Microsoft Azure Storage")] diff --git a/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec b/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec index d1abffee..179813e4 100644 --- a/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec +++ b/tools/nupkg/Microsoft.Azure.Storage.DataMovement.nuspec @@ -2,7 +2,7 @@ Microsoft.Azure.Storage.DataMovement - 0.9.1 + 0.10.0 Microsoft Azure Storage Data Movement Library Microsoft Microsoft From 16ac255a4a5c5e66084b260029907a75c8f0331e Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Mon, 18 Feb 2019 10:58:04 +0800 Subject: [PATCH 13/15] Update changelog, readme and breakingchange --- BreakingChanges.txt | 6 ++++++ README.md | 15 ++++++++++----- changelog.txt | 10 ++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/BreakingChanges.txt b/BreakingChanges.txt index e2a4ce3e..d2a9a8a2 100644 --- a/BreakingChanges.txt +++ b/BreakingChanges.txt @@ -1,3 +1,9 @@ +Tracking Breaking Changes since 0.9.0 + + - Changed dependency from azure storage client library 9.3.2 to azure storage blob client library 9.4.2 and azure storage file client library 9.4.2 + To upgrade to DataMovement Library 0.10.0 from previous version, please uninstall azure storage client library package first to avoid compiling error + - For DataMovement Library on .Net Framework, the minimum required .Net Framework version changed from 4.5 to 4.5.2 + Tracking Breaking Changes since 0.8.1 - Change to overwrite destination's metadata with source's metadata in async copying instead of keeping destination's metadata diff --git a/README.md b/README.md index 3df337aa..83a843a7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Microsoft Azure Storage Data Movement Library (0.9.0) +# Microsoft Azure Storage Data Movement Library (0.10.0) The Microsoft Azure Storage Data Movement Library designed for high-performance uploading, downloading and copying Azure Storage Blob and File. This library is based on the core data movement framework that powers [AzCopy](https://azure.microsoft.com/documentation/articles/storage-use-azcopy/). @@ -30,7 +30,7 @@ For the best development experience, we recommend that developers use the offici ## Target Frameworks -- .NET Framework 4.5 or above +- .NET Framework 4.5.2 or above - Netstandard2.0 ## Requirements @@ -60,12 +60,17 @@ within your project you can also have them installed by the .NET package manager ## Dependencies -### Azure Storage Client Library +### Azure Storage Blob Client Library -This version depends on Azure Storage Client Library +This version depends on Azure Storage Blob Client Library -- [WindowsAzure.Storage](https://www.nuget.org/packages/WindowsAzure.Storage/) +- [Microsoft.Azure.Storage.Blob](https://www.nuget.org/packages/Microsoft.Azure.Storage.Blob/) +### Azure Storage File Client Library + +This version depends on Azure Storage File Client Library + +- [Microsoft.Azure.Storage.File](https://www.nuget.org/packages/Microsoft.Azure.Storage.File/) ## Code Samples diff --git a/changelog.txt b/changelog.txt index 45a26204..afc45753 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,13 @@ +2019.02.18 Version 0.10.0 + * All Services on both .Net Framework and .Net Core + - Changed dependency from azure storage client library 9.3.2 to azure storage blob client library 9.4.2 and azure storage file client library 9.4.2 + - Fixed issue of config to disable content MD5 checking in download doesn't take effect during resuming + - Fixed issue of hang when uploading from a non-fixed sized stream + * Blob Service on both .Net Framework and .Net Core + - Fixed issue of block id may not be correct when changing BlockSize during uploading from a non-fixed sized stream to block blob + * All Services on .Net Framework + - Minimum required .Net Framework version is changed from 4.5 to 4.5.2 + 2018.10.25 Version 0.9.0 * All Services on both .Net Framework and .Net Core - Upgrade azure storage client library to 9.3.2 From bc3845119775f9a410cfe2edc3c33ace20b6742a Mon Sep 17 00:00:00 2001 From: Emma Zhu Date: Tue, 26 Feb 2019 10:40:57 +0800 Subject: [PATCH 14/15] Update samples to reference to DMLib 0.10.0 --- .../DataMovementSamples/App.config | 6 ++--- .../DataMovementSamples.csproj | 19 +++++++++----- .../DataMovementSamples/packages.config | 6 +++-- .../DataMovementSamples.csproj | 3 ++- .../S3ToAzureSample/App.config | 6 ++--- .../S3ToAzureSample/S3ToAzureSample.csproj | 26 ++++++++++++------- .../S3ToAzureSample/packages.config | 6 +++-- 7 files changed, 45 insertions(+), 27 deletions(-) diff --git a/samples/DataMovementSamples/DataMovementSamples/App.config b/samples/DataMovementSamples/DataMovementSamples/App.config index 663df64a..3c381abd 100644 --- a/samples/DataMovementSamples/DataMovementSamples/App.config +++ b/samples/DataMovementSamples/DataMovementSamples/App.config @@ -1,10 +1,10 @@ - + - + - \ No newline at end of file + diff --git a/samples/DataMovementSamples/DataMovementSamples/DataMovementSamples.csproj b/samples/DataMovementSamples/DataMovementSamples/DataMovementSamples.csproj index 30bf585f..38ab14a4 100644 --- a/samples/DataMovementSamples/DataMovementSamples/DataMovementSamples.csproj +++ b/samples/DataMovementSamples/DataMovementSamples/DataMovementSamples.csproj @@ -9,8 +9,9 @@ Properties DataMovementSamples DataMovementSamples - v4.5 + v4.5.2 512 + AnyCPU @@ -35,14 +36,20 @@ ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll + + ..\packages\Microsoft.Azure.Storage.Blob.9.4.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + + ..\packages\Microsoft.Azure.Storage.Common.9.4.2\lib\net452\Microsoft.Azure.Storage.Common.dll + + + ..\packages\Microsoft.Azure.Storage.File.9.4.2\lib\net452\Microsoft.Azure.Storage.File.dll + ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll - - ..\packages\WindowsAzure.Storage.9.3.2\lib\net45\Microsoft.WindowsAzure.Storage.dll - - - ..\packages\Microsoft.Azure.Storage.DataMovement.0.9.0\lib\net45\Microsoft.WindowsAzure.Storage.DataMovement.dll + + ..\packages\Microsoft.Azure.Storage.DataMovement.0.10.0\lib\net452\Microsoft.WindowsAzure.Storage.DataMovement.dll ..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll diff --git a/samples/DataMovementSamples/DataMovementSamples/packages.config b/samples/DataMovementSamples/DataMovementSamples/packages.config index 68c2ab1b..6a95873b 100644 --- a/samples/DataMovementSamples/DataMovementSamples/packages.config +++ b/samples/DataMovementSamples/DataMovementSamples/packages.config @@ -1,12 +1,14 @@  - + + + + - \ No newline at end of file diff --git a/samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj b/samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj index 41755f2c..4164c994 100644 --- a/samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj +++ b/samples/DataMovementSamples/netcore/DataMovementSamples/DataMovementSamples.csproj @@ -24,7 +24,8 @@ - + + diff --git a/samples/S3ToAzureSample/S3ToAzureSample/App.config b/samples/S3ToAzureSample/S3ToAzureSample/App.config index 8ef6e405..469b89c0 100644 --- a/samples/S3ToAzureSample/S3ToAzureSample/App.config +++ b/samples/S3ToAzureSample/S3ToAzureSample/App.config @@ -1,7 +1,7 @@ - + - + @@ -11,4 +11,4 @@ --> - \ No newline at end of file + diff --git a/samples/S3ToAzureSample/S3ToAzureSample/S3ToAzureSample.csproj b/samples/S3ToAzureSample/S3ToAzureSample/S3ToAzureSample.csproj index c808351a..487eec13 100644 --- a/samples/S3ToAzureSample/S3ToAzureSample/S3ToAzureSample.csproj +++ b/samples/S3ToAzureSample/S3ToAzureSample/S3ToAzureSample.csproj @@ -9,8 +9,9 @@ Properties S3ToAzureSample S3ToAzureSample - v4.5 + v4.5.2 512 + AnyCPU @@ -43,17 +44,20 @@ ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - - ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll + + ..\packages\Microsoft.Azure.Storage.Blob.9.4.2\lib\net452\Microsoft.Azure.Storage.Blob.dll + + + ..\packages\Microsoft.Azure.Storage.Common.9.4.2\lib\net452\Microsoft.Azure.Storage.Common.dll - - ..\packages\WindowsAzure.Storage.9.3.2\lib\net45\Microsoft.WindowsAzure.Storage.dll + + ..\packages\Microsoft.Azure.Storage.File.9.4.2\lib\net452\Microsoft.Azure.Storage.File.dll - - ..\packages\Microsoft.Azure.Storage.DataMovement.0.9.0\lib\net45\Microsoft.WindowsAzure.Storage.DataMovement.dll + + ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll - - ..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll + + ..\packages\Microsoft.Azure.Storage.DataMovement.0.10.0\lib\net452\Microsoft.WindowsAzure.Storage.DataMovement.dll @@ -70,7 +74,9 @@ - + + Designer +