【Azure 存储服务】.NET7.0 示例代码之上传大文件到Azure Storage Blob (二)

问题描述

在上一篇博文(【Azure 存储服务】.NET7.0 示例代码之上传大文件到Azure Storage Blob (一)：https://www.cnblogs.com/lulight/p/17061631.html)中，介绍了第一种分片的方式上传文件。 本文章接着介绍第二种方式，使用 Microsoft.Azure.Storage.DataMovement 库中的 TransferManager.UploadAsync 通过并发的方式来上传大文件。

问题回答

第一步：添加 Microsoft.Azure.Storage.DataMovement

dotnet add package Microsoft.Azure.Storage.DataMovement

第二步：编写示例代码

        String storageConnectionString = "xxxxxxxxxxxxxxxxxxx";

        CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
        CloudBlobClient blobclient = account.CreateCloudBlobClient();
        CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123");
        await blobcontainer.CreateIfNotExistsAsync();

        // 获取文件路径
        string sourcePath = @"C:\home\bigfiles0120.zip";
        CloudBlockBlob docBlob = blobcontainer.GetBlockBlobReference("bigfiles-2");
        await docBlob.DeleteIfExistsAsync();

        // 设置并发操作的数量
        TransferManager.Configurations.ParallelOperations = 64;
        // 设置单块 blob 的大小，它必须在 4MB 到 100MB 之间，并且是 4MB 的倍数，默认情况下是 4MB
        TransferManager.Configurations.BlockSize = 64 * 1024 * 1024;
        // 设置传输上下文并跟踪上传进度
        var context = new SingleTransferContext();
        UploadOptions uploadOptions = new UploadOptions
        {
            DestinationAccessCondition = AccessCondition.GenerateIfExistsCondition()
        };
        context.ProgressHandler = new Progress<TransferStatus>(progress =>
        {
            //显示上传进度
            Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred);
        });
        // 使用 Stopwatch 查看上传所需时间
        var timer = System.Diagnostics.Stopwatch.StartNew();
        // 上传 Blob
        TransferManager.UploadAsync(sourcePath, docBlob, uploadOptions, context, CancellationToken.None).Wait();
        timer.Stop();
        Console.WriteLine("Time (millisecond):" + timer.ElapsedMilliseconds);
        Console.WriteLine("upload success");

第一种分片方式上传和第二步并发上传的代码执行对比：

全部代码

Program.cs

// See https://aka.ms/new-console-template for more information

Console.WriteLine("Hello, World! Start to upload big files...");

//第一种上传文件方法: Microsoft.WindowsAzure.Storage
Console.WriteLine("第一种上传文件方法: Microsoft.WindowsAzure.Storage");
await UploadMethodOne.WindowsAzureStorageUpload();

//第二种上传文件方法: Microsoft.Azure.Storage.DataMovement
Console.WriteLine("第二种上传文件方法: Microsoft.Azure.Storage.DataMovement");
await UploadMethodTwo.DataMovementUploadFiletoBlob();

Console.WriteLine("End!");

UploadMethodOne.cs

using Microsoft.Azure;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Microsoft.WindowsAzure.Storage.RetryPolicies;

public static class UploadMethodOne
{
    public static async Task WindowsAzureStorageUpload()
    {
        TimeSpan backOffPeriod = TimeSpan.FromSeconds(2);
        int retryCount = 1;
        //设置请求选项
        BlobRequestOptions requestoptions = new BlobRequestOptions()
        {
            SingleBlobUploadThresholdInBytes = 1024 * 1024 * 10, //10MB
            ParallelOperationThreadCount = 12,
            RetryPolicy = new ExponentialRetry(backOffPeriod, retryCount),
        };

        //String storageConnectionString = System.Environment.GetEnvironmentVariable("StorageConnectionString", EnvironmentVariableTarget.User);
        //Console.WriteLine("String account string : "+storageConnectionString);
        String storageConnectionString = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
        CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
        CloudBlobClient blobclient = account.CreateCloudBlobClient();
        //设置客户端默认请求选项
        blobclient.DefaultRequestOptions = requestoptions;
        CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123");

        await blobcontainer.CreateIfNotExistsAsync();
        //文件路径，文件大小
        string sourcePath = @"C:\home\bigfiles0120.zip";
        CloudBlockBlob blockblob = blobcontainer.GetBlockBlobReference("bigfiles-1");
        //设置单个块 Blob 的大小（分块方式）
        blockblob.StreamWriteSizeInBytes = 1024 * 1024 * 5;
        try
        {
            Console.WriteLine("uploading");
            //使用 Stopwatch 查看上传时间
            var timer = System.Diagnostics.Stopwatch.StartNew();
            using (var filestream = System.IO.File.OpenRead(sourcePath))
            {
                await blockblob.UploadFromStreamAsync(filestream);
            }
            timer.Stop();

            Console.WriteLine(timer.ElapsedMilliseconds);

            Console.WriteLine("Upload Successful, Time:" + timer.ElapsedMilliseconds);
        }
        catch (Exception e)
        {
            Console.WriteLine(e.Message);
        }

    }
}

UploadMethodTwo.cs

using Microsoft.Azure.Storage;
using Microsoft.Azure.Storage.Blob;
using Microsoft.Azure.Storage.DataMovement;
public static class UploadMethodTwo
{
    public async static Task DataMovementUploadFiletoBlob()
    {
        String storageConnectionString = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";

        CloudStorageAccount account = CloudStorageAccount.Parse(storageConnectionString);
        CloudBlobClient blobclient = account.CreateCloudBlobClient();
        CloudBlobContainer blobcontainer = blobclient.GetContainerReference("uploadfiles-123");
        await blobcontainer.CreateIfNotExistsAsync();

        // 获取文件路径
        string sourcePath = @"C:\home\bigfiles0120.zip";
        CloudBlockBlob docBlob = blobcontainer.GetBlockBlobReference("bigfiles-2");
        await docBlob.DeleteIfExistsAsync();

        // 设置并发操作的数量
        TransferManager.Configurations.ParallelOperations = 64;
        // 设置单块 blob 的大小，它必须在 4MB 到 100MB 之间，并且是 4MB 的倍数，默认情况下是 4MB
        TransferManager.Configurations.BlockSize = 64 * 1024 * 1024;
        // 设置传输上下文并跟踪上传进度
        var context = new SingleTransferContext();
        UploadOptions uploadOptions = new UploadOptions
        {
            DestinationAccessCondition = AccessCondition.GenerateIfExistsCondition()
        };
        context.ProgressHandler = new Progress<TransferStatus>(progress =>
        {
            //显示上传进度
            Console.WriteLine("Bytes uploaded: {0}", progress.BytesTransferred);
        });
        // 使用 Stopwatch 查看上传所需时间
        var timer = System.Diagnostics.Stopwatch.StartNew();
        // 上传 Blob
        TransferManager.UploadAsync(sourcePath, docBlob, uploadOptions, context, CancellationToken.None).Wait();
        timer.Stop();
        Console.WriteLine("Time (millisecond):" + timer.ElapsedMilliseconds);
        Console.WriteLine("upload success");
    }
}

参考资料

上传大文件到 Azure 存储块 Blob：https://docs.azure.cn/zh-cn/articles/azure-operations-guide/storage/aog-storage-blob-howto-upload-big-file-to-storage