node.js - Azure blockBlobURL.download() 对文件大小有限制吗？

Question

我正在使用 Azure 的 blockBlobURL.download() 下载图像，但只接收图像的顶部部分。从 Azure blob 下载到可读流的数量是否有限制？内容长度为 172628，存在属性 highWaterMark: 16384。这些有关系吗？

async function compareToBaseline(imageData, blobName, metadata){

  const baselineBlobName = "MacOSX10.12/chrome/initial"

  const containerURL = ContainerURL.fromServiceURL(serviceURL, "baselines")
  const blockBlobURL = BlockBlobURL.fromContainerURL(containerURL, baselineBlobName );
  let baseLineImage = await blockBlobURL.download(aborter, 0)

  baseLineImage = baseLineImage.originalResponse.readableStreamBody.read()
  console.log(baseLineImage.length);

  baseLineImage = new Buffer(baseLineImage, 'base64');
  await fs.writeFile('./newest.png', baseLineImage, 'binary', function(err){
    console.log('written');
  })
}

结果只是图像的顶部。

Answer 1

每次调用 Azure 存储服务的大小限制为 4 MB。如果您的文件大于 4 MB，则必须将其分成 block 。欲了解更多信息，请参阅Azure Storage scalability and performance targets .

这里是示例 C# 代码，用于以 1MB block 下载非常大的文件。它也是以性能为导向的。

 private static void DownloadLargeFile()
        {
            string connectionString = "connString"; //ConfigurationSettings.AppSettings["StorageConnectionString"]; //blob connection string
#pragma warning restore CS0618 // Type or member is obsolete
#pragma warning disable CS0618 // Type or member is obsolete
            string sourceContainerName = "quickstartblob"; //ConfigurationSettings.AppSettings["sourcecontainerName"]; //source blob container name            
#pragma warning restore CS0618 // Type or member is obsolete
            string sourceBlobFileName = "QuickStart1.txt"; //source blob name
            CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);
            var blobClient = account.CreateCloudBlobClient();
            var container = blobClient.GetContainerReference(sourceContainerName);
            var file = sourceBlobFileName;
            var blob = container.GetBlockBlobReference(file);
            //First fetch the size of the blob. We use this to create an empty file with size = blob's size
            blob.FetchAttributes();
            var blobSize = blob.Properties.Length;
            long blockSize = (1 * 1024 * 1024);//1 MB chunk;
            blockSize = Math.Min(blobSize, blockSize);
            //Create an empty file of blob size
            using (FileStream fs = new FileStream(file, FileMode.Create))//Create empty file.
            {
                fs.SetLength(blobSize);//Set its size
            }
            var blobRequestOptions = new BlobRequestOptions
            {
                RetryPolicy = new ExponentialRetry(TimeSpan.FromSeconds(5), 3),
                MaximumExecutionTime = TimeSpan.FromMinutes(60),
                ServerTimeout = TimeSpan.FromMinutes(60)
            };
            long currentPointer = 0;
            long bytesRemaining = blobSize;
            do
            {
                var bytesToFetch = Math.Min(blockSize, bytesRemaining);
                using (MemoryStream ms = new MemoryStream())
                {
                    //Download range (by default 1 MB)
                    blob.DownloadRangeToStream(ms, currentPointer, bytesToFetch, null, blobRequestOptions);
                    ms.Position = 0;
                    var contents = ms.ToArray();
                    using (var fs = new FileStream(file, FileMode.Open))//Open that file
                    {
                        fs.Position = currentPointer;//Move the cursor to the end of file.
                        fs.Write(contents, 0, contents.Length);//Write the contents to the end of file.
                    }
                    currentPointer += contents.Length;//Update pointer
                    bytesRemaining -= contents.Length;//Update bytes to fetch
                }
            }
            while (bytesRemaining > 0);
        }

在 Node.js 中类似下面的内容

var azure = require('azure-storage');
var fs = require('fs');

module.exports = function (context, input) {

context.done();

var accessKey = 'myaccesskey';
var storageAccount = 'mystorageaccount';
var containerName = 'mycontainer';

var blobService = azure.createBlobService(storageAccount, accessKey);

var recordName = "a_large_movie.mov";
var blobName = "standard/mov/" + recordName;

var blobSize;
var chunkSize = (1024 * 512) * 8; // I'm experimenting with this variable
var startPos = 0;
var fullPath = "D:/home/site/wwwroot/myAzureFunction/input/";
var blobProperties = blobService.getBlobProperties(containerName, blobName,   null, function (error, blob) {
    if (error) {
        throw error;
    }
    else    {
        blobSize = blob.contentLength;
        context.log('Registered length: ' + blobSize);
        fullPath = fullPath + recordName;
        console.log(fullPath);
        doDownload();
    }
}
);

function doDownload() {
var stream = fs.createWriteStream(fullPath, {flags: 'a'});
var endPos = startPos + chunkSize;
if (endPos > blobSize) {
    endPos = blobSize;
    context.log('Reached end of file endPos: ' + endPos);
}

context.log("Downloading " + (endPos - startPos) + " bytes starting from " + startPos + " marker.");

blobService.getBlobToStream(
    containerName, 
    blobName, 
    stream, 
    { 
        "rangeStart": startPos, 
        "rangeEnd": endPos-1 
    }, 
    function(error) {
        if (error) {
            throw error;
        }
        else if (!error) {
            startPos = endPos;
            if (startPos <= blobSize - 1) {
                doDownload();
            }
        }
    }
);
}

};

希望有帮助。