c# - CompleteMultipartUploadResponse 中的 EntityTooSmall

Question

使用 .NET SDK v.1.5.21.0

我正在尝试上传一个大文件 (63Mb)，我正在按照以下示例进行操作:

http://docs.aws.amazon.com/AmazonS3/latest/dev/LLuploadFileDotNet.html

但是使用助手代替漏洞代码并使用 jQuery 文件上传

https://github.com/blueimp/jQuery-File-Upload/blob/master/basic-plus.html

我有的是:

string bucket = "mybucket";

long totalSize = long.Parse(context.Request.Headers["X-File-Size"]),
        maxChunkSize = long.Parse(context.Request.Headers["X-File-MaxChunkSize"]),
        uploadedBytes = long.Parse(context.Request.Headers["X-File-UloadedBytes"]),
        partNumber = uploadedBytes / maxChunkSize + 1,
        fileSize = partNumber * inputStream.Length;

bool lastPart = inputStream.Length < maxChunkSize;

// http://docs.aws.amazon.com/AmazonS3/latest/dev/LLuploadFileDotNet.html
if (partNumber == 1) // initialize upload
{
    iView.Utilities.Amazon_S3.S3MultipartUpload.InitializePartToCloud(fileName, bucket);
}

try
{
    // upload part
    iView.Utilities.Amazon_S3.S3MultipartUpload.UploadPartToCloud(fs, fileName, bucket, (int)partNumber, uploadedBytes, maxChunkSize);

    if (lastPart)
        // wrap it up and go home
        iView.Utilities.Amazon_S3.S3MultipartUpload.CompletePartToCloud(fileName, bucket);

}
catch (System.Exception ex)
{
    // Huston, we have a problem!
    //Console.WriteLine("Exception occurred: {0}", exception.Message);
    iView.Utilities.Amazon_S3.S3MultipartUpload.AbortPartToCloud(fileName, bucket);
}

和

public static class S3MultipartUpload
{
    private static string accessKey = System.Configuration.ConfigurationManager.AppSettings["AWSAccessKey"];
    private static string secretAccessKey = System.Configuration.ConfigurationManager.AppSettings["AWSSecretKey"];
    private static AmazonS3 client = Amazon.AWSClientFactory.CreateAmazonS3Client(accessKey, secretAccessKey);
    public static InitiateMultipartUploadResponse initResponse;
    public static List<UploadPartResponse> uploadResponses;

    public static void InitializePartToCloud(string destinationFilename, string destinationBucket)
    {
        // 1. Initialize.
        uploadResponses = new List<UploadPartResponse>();

        InitiateMultipartUploadRequest initRequest =
            new InitiateMultipartUploadRequest()
            .WithBucketName(destinationBucket)
            .WithKey(destinationFilename.TrimStart('/'));

        initResponse = client.InitiateMultipartUpload(initRequest);
    }
    public static void UploadPartToCloud(Stream fileStream, string destinationFilename, string destinationBucket, int partNumber, long uploadedBytes, long maxChunkedBytes)
    {
        // 2. Upload Parts.
        UploadPartRequest request = new UploadPartRequest()
            .WithBucketName(destinationBucket)
            .WithKey(destinationFilename.TrimStart('/'))
            .WithUploadId(initResponse.UploadId)
            .WithPartNumber(partNumber)
            .WithPartSize(maxChunkedBytes)
            .WithFilePosition(uploadedBytes)
            .WithInputStream(fileStream) as UploadPartRequest;

        uploadResponses.Add(client.UploadPart(request));
    }
    public static void CompletePartToCloud(string destinationFilename, string destinationBucket)
    {
        // Step 3: complete.
        CompleteMultipartUploadRequest compRequest =
            new CompleteMultipartUploadRequest()
            .WithBucketName(destinationBucket)
            .WithKey(destinationFilename.TrimStart('/'))
            .WithUploadId(initResponse.UploadId)
            .WithPartETags(uploadResponses);

        CompleteMultipartUploadResponse completeUploadResponse =
            client.CompleteMultipartUpload(compRequest);
    }
    public static void AbortPartToCloud(string destinationFilename, string destinationBucket)
    {
        // abort.
        client.AbortMultipartUpload(new AbortMultipartUploadRequest()
                .WithBucketName(destinationBucket)
                .WithKey(destinationFilename.TrimStart('/'))
                .WithUploadId(initResponse.UploadId));
    }
}

我的 maxChunckedSize 是 6Mb (6 * (1024*1024)) 因为我读到最小值为 5Mb...

为什么我收到 “您建议的上传小于允许的最小大小” 异常？我做错了什么？

错误是:

<Error>
  <Code>EntityTooSmall</Code>
  <Message>Your proposed upload is smaller than the minimum allowed size</Message>
  <ETag>d41d8cd98f00b204e9800998ecf8427e</ETag>
  <MinSizeAllowed>5242880</MinSizeAllowed>
  <ProposedSize>0</ProposedSize>
  <RequestId>C70E7A23C87CE5FC</RequestId>
  <HostId>pmhuMXdRBSaCDxsQTHzucV5eUNcDORvKY0L4ZLMRBz7Ch1DeMh7BtQ6mmfBCLPM2</HostId>
  <PartNumber>1</PartNumber>
</Error>

如果我传递流和流长度，我如何获得 ProposedSize？

Answer 1

这是一个有效的解决方案，适用于最新的 Amazon SDK(如今天:v.1.5.37.0)

Amazon S3 分段上传的工作方式如下:

1. 使用client.InitiateMultipartUpload(initRequest)初始化请求
2. 使用 client.UploadPart(request) 发送文件 block (循环直到结束)
3. 使用client.CompleteMultipartUpload(compRequest)完成请求
4. 如果出现任何问题，请记住处理客户端和请求，并使用 client.AbortMultipartUpload(abortMultipartUploadRequest)
触发中止命令

我将客户端保持在 Session 中，因为我们也需要为每个 block 上传使用它，保留现在用于完成该过程的 ETag。

---

您可以在 Amazon Docs 中看到一个示例和简单的方法本身，我最终得到了一个类来做所有事情，另外，我已经与可爱的 jQuery File Upload 整合在一起了。插件(以及下面的处理程序代码)。

S3MultipartUpload如下

public class S3MultipartUpload : IDisposable
{
    string accessKey = System.Configuration.ConfigurationManager.AppSettings.Get("AWSAccessKey");
    string secretAccessKey = System.Configuration.ConfigurationManager.AppSettings.Get("AWSSecretKey");

    AmazonS3 client;
    public string OriginalFilename { get; set; }
    public string DestinationFilename { get; set; }
    public string DestinationBucket { get; set; }

    public InitiateMultipartUploadResponse initResponse;
    public List<PartETag> uploadPartETags;
    public string UploadId { get; private set; }

    public S3MultipartUpload(string destinationFilename, string destinationBucket)
    {
        if (client == null)
        {
            System.Net.WebRequest.DefaultWebProxy = null; // disable proxy to make upload quicker

            client = Amazon.AWSClientFactory.CreateAmazonS3Client(accessKey, secretAccessKey, new AmazonS3Config()
            {
                RegionEndpoint = Amazon.RegionEndpoint.EUWest1,
                CommunicationProtocol = Protocol.HTTP
            });

            this.OriginalFilename = destinationFilename.TrimStart('/');
            this.DestinationFilename = string.Format("{0:yyyy}{0:MM}{0:dd}{0:HH}{0:mm}{0:ss}{0:fffff}_{1}", DateTime.UtcNow, this.OriginalFilename);
            this.DestinationBucket = destinationBucket;

            this.InitializePartToCloud();
        }
    }

    private void InitializePartToCloud()
    {
        // 1. Initialize.
        uploadPartETags = new List<PartETag>();

        InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest();
        initRequest.BucketName = this.DestinationBucket;
        initRequest.Key = this.DestinationFilename;

        // make it public
        initRequest.AddHeader("x-amz-acl", "public-read");

        initResponse = client.InitiateMultipartUpload(initRequest);
    }
    public void UploadPartToCloud(Stream fileStream, long uploadedBytes, long maxChunkedBytes)
    {
        int partNumber = uploadPartETags.Count() + 1; // current part

        // 2. Upload Parts.
        UploadPartRequest request = new UploadPartRequest();
        request.BucketName = this.DestinationBucket;
        request.Key = this.DestinationFilename;
        request.UploadId = initResponse.UploadId;
        request.PartNumber = partNumber;
        request.PartSize = fileStream.Length;
        //request.FilePosition = uploadedBytes // remove this line?
        request.InputStream = fileStream; // as UploadPartRequest;

        var up = client.UploadPart(request);
        uploadPartETags.Add(new PartETag() { ETag = up.ETag, PartNumber = partNumber });
    }
    public string CompletePartToCloud()
    {
        // Step 3: complete.
        CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest();
        compRequest.BucketName = this.DestinationBucket;
        compRequest.Key = this.DestinationFilename;
        compRequest.UploadId = initResponse.UploadId;
        compRequest.PartETags = uploadPartETags;

        string r = "Something went badly wrong";

        using (CompleteMultipartUploadResponse completeUploadResponse = client.CompleteMultipartUpload(compRequest))
            r = completeUploadResponse.ResponseXml;

        return r;
    }
    public void AbortPartToCloud()
    {
        // abort.
        client.AbortMultipartUpload(new AbortMultipartUploadRequest()
        {
            BucketName = this.DestinationBucket,
            Key = this.DestinationFilename,
            UploadId = initResponse.UploadId
        });
    }

    public void Dispose()
    {
        if (client != null) client.Dispose();
        if (initResponse != null) initResponse.Dispose();
    }
}

我使用 DestinationFilename 作为目标文件，这样我就可以避免使用相同的名称，但我会保留 OriginalFilename 以备日后需要。

使用 jQuery 文件上传插件，所有工作都在一个通用处理程序中进行，过程是这样的:

// Upload partial file
private void UploadPartialFile(string fileName, HttpContext context, List<FilesStatus> statuses)
{
    if (context.Request.Files.Count != 1)
        throw new HttpRequestValidationException("Attempt to upload chunked file containing more than one fragment per request");

    var inputStream = context.Request.Files[0].InputStream;
    string contentRange = context.Request.Headers["Content-Range"]; // "bytes 0-6291455/14130271"

    int fileSize = int.Parse(contentRange.Split('/')[1]);,
        maxChunkSize = int.Parse(context.Request.Headers["X-Max-Chunk-Size"]),
        uploadedBytes = int.Parse(contentRange.Replace("bytes ", "").Split('-')[0]);

    iView.Utilities.AWS.S3MultipartUpload s3Upload = null;

    try
    {

        // ######################################################################################
        // 1. Initialize Amazon S3 Client
        if (uploadedBytes == 0)
        {
            HttpContext.Current.Session["s3-upload"] = new iView.Utilities.AWS.S3MultipartUpload(fileName, awsBucket);

            s3Upload = (iView.Utilities.AWS.S3MultipartUpload)HttpContext.Current.Session["s3-upload"];
            string msg = System.String.Format("Upload started: {0} ({1:N0}Mb)", s3Upload.DestinationFilename, (fileSize / 1024));
            this.Log(msg);
        }

        // cast current session object
        if (s3Upload == null)
            s3Upload = (iView.Utilities.AWS.S3MultipartUpload)HttpContext.Current.Session["s3-upload"];

        // ######################################################################################
        // 2. Send Chunks
        s3Upload.UploadPartToCloud(inputStream, uploadedBytes, maxChunkSize);

        // ######################################################################################
        // 3. Complete Upload
        if (uploadedBytes + maxChunkSize > fileSize)
        {
            string completeRequest = s3Upload.CompletePartToCloud();
            this.Log(completeRequest); // log S3 response

            s3Upload.Dispose(); // dispose all objects
            HttpContext.Current.Session["s3-upload"] = null; // we don't need this anymore
        }

    }
    catch (System.Exception ex)
    {
        if (ex.InnerException != null)
            while (ex.InnerException != null)
                ex = ex.InnerException;

        this.Log(string.Format("{0}\n\n{1}", ex.Message, ex.StackTrace)); // log error

        s3Upload.AbortPartToCloud(); // abort current upload
        s3Upload.Dispose(); // dispose all objects

        statuses.Add(new FilesStatus(ex.Message));
        return;
    }

    statuses.Add(new FilesStatus(s3Upload.DestinationFilename, fileSize, ""));
}

请记住，要在通用处理程序中包含一个Session 对象，您需要实现IRequiresSessionState，这样您的处理程序将如下所示:

public class UploadHandlerSimple : IHttpHandler, IRequiresSessionState

在 fileupload.js 中(在 _initXHRData 下)我添加了一个名为 X-Max-Chunk-Size 的额外 header ，这样我就可以通过将其发送到亚马逊并计算它是否是上传文件的最后一部分。

---

随时发表评论并进行智能编辑以供所有人使用。