android - 在Android中合并多个Azure云 block blob

Question

我是 Azure Blob 云的新手。我基本上想将视频文件从我的 Android 应用程序上传到 Azure 云，但我不能，因为一旦大小达到 32MB，它就会停止抛出 OutOfMemory 异常。因此，我对如何解决这个问题进行了一些研究，并提出了一个解决方案，将文件分解为字节，然后将其作为多个 blob 上传。最后将它们编译成一个 blob。但我不知道该怎么做。我尝试使用 commitBlockList，但为此我无法获取每个 blob 的 Id。

try {
        // Setup the cloud storage account.
        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        int maxSize = 64 * Constants.MB;

        // Create a blob service client
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        blobClient.getDefaultRequestOptions().setSingleBlobPutThresholdInBytes(maxSize);
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        container.createIfNotExists();
        BlobContainerPermissions containerPermissions = new     BlobContainerPermissions();
        containerPermissions.setPublicAccess(BlobContainerPublicAccessType.CONTAINER);
        container.uploadPermissions(containerPermissions);
        CloudBlockBlob finalFile = container.getBlockBlobReference("1.jpg");
        CloudBlob b = container.getBlockBlobReference("temp");
        String Lease = b.getSnapshotID();
        b.uploadFromFile(URL);
        List<BlockEntry> blockEntryIterator = new ArrayList<>();
        blockEntryIterator.add(new BlockEntry(Lease));
        finalFile.commitBlockList(blockEntryIterator);
    } catch (Throwable t) {

    }

~~~更新~~~

我尝试将文件分成几个部分，但现在出现此错误“指定的 blob 或 block 内容无效”。 public void splitTest(String URL) 抛出 IOException、URISyntaxException、InvalidKeyException、StorageException {

    new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"STARTED");
    CloudBlockBlob blob = null;
    List<BlockEntry> blockList = null;
    try{
        // get file reference
        FileInputStream fs = new FileInputStream( URL );
        File sourceFile = new File( URL);

        // set counters
        long fileSize = sourceFile.length();
        int blockSize = 3 * (1024 * 1024); // 256K
        int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
        long bytesLeft = fileSize;
        int blockNumber = 0;
        long bytesRead = 0;

        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        String title = "Android_" + getFileNameFromUrl(URL);
        // get ref to the blob we are creating while uploading
        blob = container.getBlockBlobReference(title);
        blob.deleteIfExists();

        // list of all block ids we will be uploading - need it for the commit at the end
        blockList = new ArrayList<BlockEntry>();

        // loop through the file and upload chunks of the file to the blob
        while( bytesLeft > 0 ) {

            blockNumber++;
            // how much to read (only last chunk may be smaller)
            int bytesToRead = 0;
            if ( bytesLeft >= (long)blockSize ) {
                bytesToRead = blockSize;
            } else {
                bytesToRead = (int)bytesLeft;
            }

            // trace out progress
            float pctDone = ((float)blockNumber / (float)blockCount) * (float)100;


            // save block id in array (must be base64)
            String x = "";
            if(blockNumber<=9) {
                traceLine( "blockid: 000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid000" + blockNumber;
            }
            else if(blockNumber>=10 && blockNumber<=99){
                traceLine( "blockid: 00" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid00" + blockNumber;
            }
            else if(blockNumber>=100 && blockNumber<=999){
                traceLine( "blockid0: " + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid0" + blockNumber;
            }
            else if(blockNumber>=1000 && blockNumber<=9999){
                traceLine( "blockid: " + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "blockid" + blockNumber;
            }
            String blockId = Base64.encodeToString(x.getBytes(),Base64.DEFAULT).replace("\n","").toLowerCase();
            traceLine( "Base 64["+x+"] -> " + blockId);
            BlockEntry block = new BlockEntry(blockId);
            blockList.add(block);

            // upload block chunk to Azure Storage
            blob.uploadBlock( blockId, fs, (long)bytesToRead);

            // increment/decrement counters
            bytesRead += bytesToRead;
            bytesLeft -= bytesToRead;

        }
        fs.close();
        traceLine( "CommitBlockList. BytesUploaded: " + bytesRead);
        blob.commitBlockList(blockList);
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"UPLOAD COMPLETE");
        return;
    }
    catch (StorageException storageException) {
        traceLine("StorageException encountered: ");
        traceLine(storageException.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    } catch( IOException ex ) {
        traceLine( "IOException: " + ex );
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    } catch (Exception e) {
        traceLine("Exception encountered: ");
        traceLine(e.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        assert blockList != null;
        blob.commitBlockList(blockList);
        return;
    }
}

~~~工作更新~~~

对于任何想要重用此方法来打开文件并逐字节读取它的人都可以使用此方法。我可以上传 ~1GB 的文件，但之后会出现错误 500 并崩溃。如果有人有任何解决方案，请告诉我。

public boolean splitTest(String URL) throws IOException, URISyntaxException, InvalidKeyException, StorageException {
    new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"STARTED");
    CloudBlockBlob blob = null;
    List<BlockEntry> blockList = null;
    try{
        // get file reference
        FileInputStream fs = new FileInputStream(URL);
        File sourceFile = new File(URL);

        // set counters
        long fileSize = sourceFile.length();
        int blockSize = 512 * 1024; // 256K
        //int blockSize = 1 * (1024 * 1024); // 256K
        int blockCount = (int)((float)fileSize / (float)blockSize) + 1;
        long bytesLeft = fileSize;
        int blockNumber = 0;
        long bytesRead = 0;

        CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
        CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
        CloudBlobContainer container = blobClient.getContainerReference("testing");
        String title = "Android/Android_" + getFileNameFromUrl(URL).replace("\n","").replace(" ","_").replace("-","").toLowerCase();
        // get ref to the blob we are creating while uploading
        blob = container.getBlockBlobReference(title);
        traceLine("Title of blob -> " + title);

        if(blob.exists())
            blob.deleteIfExists();

        blob.setStreamWriteSizeInBytes(blockSize);
        // list of all block ids we will be uploading - need it for the commit at the end
        blockList = new ArrayList<>();

        // loop through the file and upload chunks of the file to the blob
        while( bytesLeft > 0 ) {
            // how much to read (only last chunk may be smaller)
            int bytesToRead = 0;
            if ( bytesLeft >= (long)blockSize ) {
                bytesToRead = blockSize;
            } else {
                bytesToRead = (int)bytesLeft;
            }

            // trace out progress
            float pctDone = ((float)blockNumber / (float)blockCount) * (float)100;


            // save block id in array (must be base64)
            String x = "";
            if(blockNumber<=9) {
                traceLine( "tempblobid0000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid0000" + blockNumber;
            }
            else if(blockNumber>=10 && blockNumber<=99){
                traceLine( "tempblobid000" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid000" + blockNumber;
            }
            else if(blockNumber>=100 && blockNumber<=999){
                traceLine( "tempblobid00" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid00" + blockNumber;
            }
            else if(blockNumber>=1000 && blockNumber<=9999){
                traceLine( "tempblobid0" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid0" + blockNumber;
            }
            else if(blockNumber>=10000 && blockNumber<=99999){
                traceLine( "tempblobid" + blockNumber + ". " + String.format("%.0f%%",pctDone) + " done.");
                x = "tempblobid" + blockNumber;
            }
            String blockId = Base64.encodeToString(x.getBytes(),Base64.NO_WRAP).replace("\n","").toLowerCase();
            traceLine( "Base 64["+ x +"] -> " + blockId);
            BlockEntry block = new BlockEntry(blockId);
            blockList.add(block);
            // upload block chunk to Azure Storage
            blob.uploadBlock( blockId, fs, (long)bytesToRead);
            notification2(a,pctDone);
            //a.update(pctDone);
            // increment/decrement counters
            bytesRead += bytesToRead;
            bytesLeft -= bytesToRead;
            blockNumber++;
        }
        fs.close();
        traceLine( "CommitBlockList. BytesUploaded: " + bytesRead + "\t total bytes -> " + fileSize + "\tBytes Left -> " + bytesLeft);
        blob.commitBlockList(blockList);
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"UPLOAD COMPLETE");
        return true;
    }
    catch (StorageException storageException) {
        traceLine("StorageException encountered: ");
        traceLine(storageException.getMessage());
        traceLine("HTTP Status code -> " + storageException.getHttpStatusCode());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    } catch( IOException ex ) {
        traceLine( "IOException: " + ex );
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    } catch (Exception e) {
        traceLine("Exception encountered: ");
        traceLine(e.getMessage());
        new ConversionNotificationSetup().sendNotification(a.getApplicationContext(),"FAILED");
        if (blob != null) {
            blob.commitBlockList(blockList);
        }
        return false;
    }

}

Answer 1

这取决于您使用的手机类型。如果您的 32MB 内存耗尽，那么您使用的是非常小的手机或正在使用许多其他进程。查看您的手机以及您有多少可用内存，就像 Gaurav 提到的和我的其他答案提到的那样，将阈值降低到该水平。 self 划分对于你想要做的事情并没有真正的帮助。

您要查看的两个设置是 blob 本身的 singleBlobPutThresholdInBytes 和 setStreamWriteSizeInBytes。 singleBlobPutThresholdInBytes 会影响我们开始分块与放置整个 blob 的时间，而 setStreamWriteSizeInBytes 会影响分块发生时 block 的大小。放置阈值默认为 64MB，写入大小默认为 4MB。请注意，如果减少写入大小，您可能无法获得最大块 blob 大小，因为 block blob 限制为 50k block 。尝试将 singleBlobPutThreshold 减少到您可以处理的内存量，直到 4MB - 如果它小于 4MB，您确实遇到了麻烦，并且还需要减少流写入大小。