python - 使用fineuploader和django将文件上传到S3时出现签名错误消息

Question

更新:请参阅此消息的底部。这是python3/hmac版本问题。

我正在使用优秀的 uploader 、S3 和 django 1.11 设置文件上传系统。我设置了网址、模板和 View ，但以下是尝试上传文件时收到的(客户端)错误消息:

• [Fine Uploader 5.15.0] 0 的 POST 请求失败 - 响应代码 400
• [Fine Uploader 5.15.0] 尝试解析签名响应时出错:SyntaxError:JSON 解析错误:意外的标识符“无效”
• [Fine Uploader 5.15.0] 从服务器收到空响应或无效响应!
• [Fine Uploader 5.15.0] 策略签名失败。从服务器收到空的或无效的响应!

django 设置中需要良好的 uploader :

• AWS_CLIENT_SECRET_KEY = IAM 用户 secret 访问 key
• AWS_SERVER_PUBLIC_KEY = IAM 用户访问 key ID
• AWS_SERVER_SECRET_KEY = IAM 用户 secret 访问 key

我有我创建的 iam 用户的访问 key ID 和 secret 访问 key ，并按如上所示进行设置。 AWS_CLIENT_SECRET_KEY = AWS_SERVER_SECRET_KEY = IAM 用户 key 。我不确定这是否正确，也可能是问题所在，但我不知道 AWS_CLIENT_SECRET_KEY 和 AWS_SERVER_SECRET_KEY 之间有什么区别，以及如果不是 iam key ，则在哪里可以找到它们。

这是代码:

查看:

from django.shortcuts import render
from django.conf import settings
from django.http import HttpResponse, HttpRequest
from django.shortcuts import render
from django.views.decorators.http import require_http_methods
from django.views.decorators.csrf import csrf_exempt

import base64, hmac, hashlib, json, sys


import boto
from boto.s3.connection import Key, S3Connection

boto.set_stream_logger( 'boto' )
S3 = S3Connection( settings.AWS_SERVER_PUBLIC_KEY, settings.AWS_SERVER_SECRET_KEY )


def video_create_form( request ):
    return render( request, 'video_create_form_view.html' )


@csrf_exempt
def success_redirect_endpoint( request ):
    """ This is where the upload will snd a POST request after the
    file has been stored in S3.
    """
    return make_response( 200 )


@csrf_exempt
def handle_s3( request ):
    """ View which handles all POST and DELETE requests sent by Fine Uploader
    S3. You will need to adjust these paths/conditions based on your setup.
    """
    if request.method == "POST":
        return handle_POST( request )
    elif request.method == "DELETE":
        return handle_DELETE( request )
    else:
        return HttpResponse( status = 405 )


def handle_POST( request ):
    """ Handle S3 uploader POST requests here. For files <=5MiB this is a simple
    request to sign the policy document. For files >5MiB this is a request
    to sign the headers to start a multipart encoded request.
    """
    if request.POST.get( 'success', None ):
        return make_response( 200 )
    else:
        request_payload = json.loads( request.body )
        headers = request_payload.get( 'headers', None )
        if headers:
            # The presence of the 'headers' property in the request payload
            # means this is a request to sign a REST/multipart request
            # and NOT a policy document
            response_data = sign_headers( headers )
        else:
            if not is_valid_policy( request_payload ):
                return make_response( 400, { 'invalid': True } )
            response_data = sign_policy_document( request_payload )
        response_payload = json.dumps( response_data )
        return make_response( 200, response_payload )


def handle_DELETE( request ):
    """ Handle file deletion requests. For this, we use the Amazon Python SDK,
    boto.
    """
    if boto:
        bucket_name = request.REQUEST.get( 'bucket' )
        key_name = request.REQUEST.get( 'key' )
        aws_bucket = S3.get_bucket( bucket_name, validate = False )
        aws_key = Key( aws_bucket, key_name )
        aws_key.delete()
        return make_response( 200 )
    else:
        return make_response( 500 )


def make_response( status = 200, content = None ):
    """ Construct an HTTP response. Fine Uploader expects 'application/json'.
    """
    response = HttpResponse()
    response.status_code = status
    response[ 'Content-Type' ] = "application/json"
    response.content = content
    return response


def is_valid_policy( policy_document ):
    """ Verify the policy document has not been tampered with client-side
    before sending it off.
    """
    # bucket = settings.AWS_EXPECTED_BUCKET
    # parsed_max_size = settings.AWS_MAX_SIZE
    bucket = ''
    parsed_max_size = 0

    for condition in policy_document[ 'conditions' ]:
        if isinstance( condition, list ) and condition[ 0 ] == 'content-length-range':
            parsed_max_size = condition[ 2 ]
        else:
            if condition.get( 'bucket', None ):
                bucket = condition[ 'bucket' ]

    return bucket == settings.AWS_EXPECTED_BUCKET and parsed_max_size == settings.AWS_MAX_SIZE


def sign_policy_document( policy_document ):
    """ Sign and return the policy doucument for a simple upload.
    http://aws.amazon.com/articles/1434/#signyours3postform
    """
    policy = base64.b64encode( json.dumps( policy_document ) )
    signature = base64.b64encode(
            hmac.new( settings.AWS_CLIENT_SECRET_KEY, policy, hashlib.sha1 ).digest() )
    return {
        'policy'   : policy,
        'signature': signature
    }


def sign_headers( headers ):
    """ Sign and return the headers for a chunked upload. """
    return {
        'signature': base64.b64encode(
                hmac.new( settings.AWS_CLIENT_SECRET_KEY, headers, hashlib.sha1 ).digest() )
    }

模板:

{% load static %}

<!DOCTYPE html>
<html>
<head>
    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <link href="{% static "fine-uploader-gallery.css" %}" rel="stylesheet">
    <script src="{% static "s3.fine-uploader.js" %}"></script>


    <title>Fine Uploader Gallery UI</title>
</head>
<body>
<div id="uploader"></div>

<script type="text/template" id="qq-template">
    <div class="qq-uploader-selector qq-uploader qq-gallery"
         qq-drop-area-text="Drop files here">
        <div class="qq-total-progress-bar-container-selector qq-total-progress-bar-container">
            <div role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100"
                 class="qq-total-progress-bar-selector qq-progress-bar qq-total-progress-bar"></div>
        </div>
        <div class="qq-upload-drop-area-selector qq-upload-drop-area" qq-hide-dropzone>
            <span class="qq-upload-drop-area-text-selector"></span>
        </div>
        <div class="qq-upload-button-selector qq-upload-button">
            <div>Upload a file</div>
        </div>
        <span class="qq-drop-processing-selector qq-drop-processing">
                <span>Processing dropped files...</span>
                <span class="qq-drop-processing-spinner-selector qq-drop-processing-spinner"></span>
            </span>
        <ul class="qq-upload-list-selector qq-upload-list" role="region" aria-live="polite"
            aria-relevant="additions removals">
            <li>
                    <span role="status"
                          class="qq-upload-status-text-selector qq-upload-status-text"></span>
                <div class="qq-progress-bar-container-selector qq-progress-bar-container">
                    <div role="progressbar" aria-valuenow="0" aria-valuemin="0"
                         aria-valuemax="100"
                         class="qq-progress-bar-selector qq-progress-bar"></div>
                </div>
                <span class="qq-upload-spinner-selector qq-upload-spinner"></span>
                <div class="qq-thumbnail-wrapper">
                    <img class="qq-thumbnail-selector" qq-max-size="120" qq-server-scale>
                </div>
                <button type="button" class="qq-upload-cancel-selector qq-upload-cancel">X
                </button>
                <button type="button" class="qq-upload-retry-selector qq-upload-retry">
                    <span class="qq-btn qq-retry-icon" aria-label="Retry"></span>
                    Retry
                </button>

                <div class="qq-file-info">
                    <div class="qq-file-name">
                        <span class="qq-upload-file-selector qq-upload-file"></span>
                        <span class="qq-edit-filename-icon-selector qq-btn qq-edit-filename-icon"
                              aria-label="Edit filename"></span>
                    </div>
                    <input class="qq-edit-filename-selector qq-edit-filename" tabindex="0"
                           type="text">
                    <span class="qq-upload-size-selector qq-upload-size"></span>
                    <button type="button"
                            class="qq-btn qq-upload-delete-selector qq-upload-delete">
                        <span class="qq-btn qq-delete-icon" aria-label="Delete"></span>
                    </button>
                    <button type="button"
                            class="qq-btn qq-upload-pause-selector qq-upload-pause">
                        <span class="qq-btn qq-pause-icon" aria-label="Pause"></span>
                    </button>
                    <button type="button"
                            class="qq-btn qq-upload-continue-selector qq-upload-continue">
                        <span class="qq-btn qq-continue-icon" aria-label="Continue"></span>
                    </button>
                </div>
            </li>
        </ul>

        <dialog class="qq-alert-dialog-selector">
            <div class="qq-dialog-message-selector"></div>
            <div class="qq-dialog-buttons">
                <button type="button" class="qq-cancel-button-selector">Close</button>
            </div>
        </dialog>

        <dialog class="qq-confirm-dialog-selector">
            <div class="qq-dialog-message-selector"></div>
            <div class="qq-dialog-buttons">
                <button type="button" class="qq-cancel-button-selector">No</button>
                <button type="button" class="qq-ok-button-selector">Yes</button>
            </div>
        </dialog>

        <dialog class="qq-prompt-dialog-selector">
            <div class="qq-dialog-message-selector"></div>
            <input type="text">
            <div class="qq-dialog-buttons">
                <button type="button" class="qq-cancel-button-selector">Cancel</button>
                <button type="button" class="qq-ok-button-selector">Ok</button>
            </div>
        </dialog>
    </div>
</script>

<script>
    var uploader = new qq.s3.FineUploader( {
        debug        : true,
        element      : document.getElementById( 'uploader' ),
        request      : {
            endpoint : 'https://mybucketname.s3.amazonaws.com',
            accessKey: 'AK*******'
        },
        signature    : {
            endpoint: '/videos/s3/signature'
        },
        uploadSuccess: {
            endpoint: '/videos/s3/success'
        },
        iframeSupport: {
            localBlankPagePath: '/success.html'
        },
        retry        : {
            enableAuto: true // defaults to false
        },
        deleteFile   : {
            enabled : true,
            endpoint: '/videos/s3/delete'
        }
    } );
</script>
</body>
</html>

网址(导入到可能的网址文件中)

from django.conf.urls import url
from videos.controllers.video_create_controller import video_create_form, handle_s3, success_redirect_endpoint


urlpatterns = [
    url( r'^video-create-form/$', video_create_form, name = 'video_create_form' ),
    url( r'^s3/signature', handle_s3, name = "s3_signee" ),
    url( r'^s3/delete', handle_s3, name = 's3_delete' ),
    url( r'^s3/success', success_redirect_endpoint, name = "s3_succes_endpoint" )
]

设置

# Amazon variables. Be wary and don't hard-code your secret keys here. Rather,
# set them as environment variables, or read them from a file somehow.
AWS_CLIENT_SECRET_KEY = 'WDq/cy*****'
AWS_SERVER_PUBLIC_KEY = 'AK*****'
AWS_SERVER_SECRET_KEY = 'WDq/cy*****'

AWS_EXPECTED_BUCKET = 'mybucketname'
AWS_MAX_SIZE = 15000000

Cors政策

这似乎不是 aws 端设置问题，因为我可以通过其他方式将文件放入我的存储桶中。

<?xml version="1.0" encoding="UTF-8"?>
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CORSRule>
    <AllowedOrigin>*</AllowedOrigin>
    <AllowedMethod>GET</AllowedMethod>
    <AllowedMethod>HEAD</AllowedMethod>
    <AllowedMethod>PUT</AllowedMethod>
    <AllowedMethod>POST</AllowedMethod>
    <MaxAgeSeconds>3000</MaxAgeSeconds>
    <ExposeHeader>ETag</ExposeHeader>
    <AllowedHeader>*</AllowedHeader>
</CORSRule>
</CORSConfiguration>

更新:这似乎是 python 版本的问题:使用 FLASK 重现安装后，我能够收到有关 hmac 的错误消息:

 raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__)
TypeError: key: expected bytes or bytearray, but got 'str

使用 python 2.7 设置 venv 解决了这个问题，并且我让所有过程都正常工作。我正在调查这个问题，如果有人解决了请告知。

'

Answer 1

这是 View 代码，已完全更新以与 python 3 和 Boto3 配合使用。您可能需要运行

/Applications/Python\ 3.6/Install\ Certificates.command 

如果您使用的是 python 3.6 并且在删除文件时遇到 ssl 错误问题，请从 Mac 命令行进行操作。

所以这不是 aws 或权限问题...而是 python3 字节/字符串问题。此行还存在类型强制问题，始终返回 false:

    return bucket == settings.AWS_EXPECTED_BUCKET and parsed_max_size == settings.AWS_MAX_SIZE

这阻止了任何有意义的错误消息的显示。

from django.conf import settings
from django.http import HttpResponse
from django.shortcuts import render
from django.views.decorators.csrf import csrf_exempt

import base64, hmac, hashlib, json

import boto3

# Enforce session to inject credentials
session = boto3.Session(
        aws_access_key_id = settings.AWS_SERVER_PUBLIC_KEY,
        aws_secret_access_key = settings.AWS_SERVER_SECRET_KEY,
)
S3 = session.resource( 's3' )


def video_create_form( request ):
    return render( request, 'video_create_form_view.html' )


@csrf_exempt
def success_redirect_endpoint( request ):
    """ This is where the upload will send a POST request after the
    file has been stored in S3.
    """
    return make_response( 200 )


@csrf_exempt
def handle_s3( request ):
    """ View which handles all POST and DELETE requests sent by Fine Uploader
    S3. You will need to adjust these paths/conditions based on your setup.
    """
    if request.method == "POST":
        return handle_POST( request )
    elif request.method == "DELETE":
        return handle_DELETE( request )
    else:
        return HttpResponse( status = 405 )


def handle_POST( request ):
    """ Handle S3 uploader POST requests here. For files <=5MiB this is a simple
    request to sign the policy document. For files >5MiB this is a request
    to sign the headers to start a multipart encoded request.
    """
    class MyEncoder( json.JSONEncoder ):
        """Converts a dict of bytes to Json"""
        def default( self, obj ):
            if isinstance( obj, (bytes, bytearray) ):
                return obj.decode( "ASCII" )  # <- or any other encoding of your choice
            # Let the base class default method raise the TypeError
            return json.JSONEncoder.default( self, obj )

    if request.POST.get( 'success', None ):
        return make_response( 200 )
    else:
        request_payload = json.loads( request.body )
        headers = request_payload.get( 'headers', None )
        if headers:
            # The presence of the 'headers' property in the request payload
            # means this is a request to sign a REST/multipart request
            # and NOT a policy document
            response_data = sign_headers( headers )
        else:
            if not is_valid_policy( request_payload ):
                return make_response( 400, { 'invalid': True } )
            response_data = sign_policy_document( request_payload )
        response_payload = json.dumps( response_data, cls = MyEncoder )
        return make_response( 200, response_payload )


def handle_DELETE( request ):
    """ Handle file deletion requests. For this, we use the Amazon Python SDK, boto.
    """
    if boto3:
        bucket_name = request.GET.get( 'bucket' )
        key_name = request.GET.get( 'key' )
        S3.Object( bucket_name, key_name ).delete()

        return make_response( 200 )
    else:
        return make_response( 500 )


def make_response( status = 200, content = None ):
    """ Construct an HTTP response. Fine Uploader expects 'application/json'.
    """
    response = HttpResponse()
    response.status_code = status
    response[ 'Content-Type' ] = "application/json"
    response.content = content
    return response


def is_valid_policy( policy_document ):
    """ Verify the policy document has not been tampered with client-side
    before sending it off.
    """
    # bucket = settings.AWS_EXPECTED_BUCKET
    # parsed_max_size = settings.AWS_MAX_SIZE
    bucket = ''
    parsed_max_size = 0

    for condition in policy_document[ 'conditions' ]:
        if isinstance( condition, list ) and condition[ 0 ] == 'content-length-range':
            parsed_max_size = condition[ 2 ]
        else:
            if condition.get( 'bucket', None ):
                bucket = condition[ 'bucket' ]

    return bucket == settings.AWS_EXPECTED_BUCKET and int(
            parsed_max_size ) == settings.AWS_MAX_SIZE


def sign_policy_document( policy_document ):
    """ Sign and return the policy doucument for a simple upload.
    http://aws.amazon.com/articles/1434/#signyours3postform
    """
    policy_document_string = str.encode( str( policy_document ) )
    policy = base64.b64encode( policy_document_string )
    aws_secret_key = settings.AWS_CLIENT_SECRET_KEY
    secret_key = str.encode( aws_secret_key )

    signature = base64.b64encode(
            hmac.new( secret_key, policy, hashlib.sha1 ).digest() )
    return {
        'policy'   : policy,
        'signature': signature
    }


def sign_headers( headers ):
    """ Sign and return the headers for a chunked upload. """
    headers_bytes = bytearray( headers, 'utf-8' )  # hmac doesn't want unicode
    aws_client_secret = str.encode( settings.AWS_CLIENT_SECRET_KEY )
    return {
        'signature': base64.b64encode(
                hmac.new( aws_client_secret, headers_bytes, hashlib.sha1 ).digest() )
    }