python - 使用@tornado.web.stream_request_body上传文件

Question

我正在尝试使用 @tornado.web.stream_request_body 上传文件。

但是我在上传大文件时遇到问题。例如，当我上传大于 100 MB ( https://yadi.sk/i/rzLQ96pk3Tcef6 ) 的 PDF 文件时，它加载不正确并且无法在查看器中打开。

代码示例:

MAX_STREAMED_SIZE = 1024 * 1024 * 1024

@tornado.web.stream_request_body
class UploadHandler(tornado.web.RequestHandler):
    def prepare(self):
       self.request.connection.set_max_body_size(MAX_STREAMED_SIZE)
       self.f = open(os.path.join('files', '12322.pdf'), "w+b")

    def data_received(self, data):
       self.f.write(data)

    def post(self):
       self.f.close()
       print("upload completed")

问题的原因是什么？

Answer 1

试试这个。上传一个文件即可正常工作，不会占用大量内存

#! /usr/bin/env python
#-* coding: utf-8 -*

# Official packages

# 3rd-party Packages
import tornado.web

# Local Packages

# CONST
MB = 1024 * 1024
GB = 1024 * MB
TB = 1024 * GB
MAX_STREAMED_SIZE = 16*GB

# Class&Function Defination
@tornado.web.stream_request_body
class UploadHandler(tornado.web.RequestHandler):
    def initialize(self):
        self.bytes_read = 0
        self.meta = dict()
        self.receiver = self.get_receiver()

    # def prepare(self):
    """If no stream_request_body"""
    #     self.request.connection.set_max_body_size(MAX_STREAMED_SIZE)

    def data_received(self, chunk):
        self.receiver(chunk)

    def get_receiver(self):
        index = 0
        SEPARATE = b'\r\n'

        def receiver(chunk):
            nonlocal index
            if index == 0:
                index +=1
                split_chunk             = chunk.split(SEPARATE)
                self.meta['boundary']   = SEPARATE + split_chunk[0] + b'--' + SEPARATE
                self.meta['header']     = SEPARATE.join(split_chunk[0:3])
                self.meta['header']     += SEPARATE *2
                self.meta['filename']   = split_chunk[1].split(b'=')[-1].replace(b'"',b'').decode()

                chunk = chunk[len(self.meta['header']):] # Stream掐头
                import os
                self.fp = open(os.path.join('upload',self.meta['filename']), "wb")
                self.fp.write(chunk)
            else:
                self.fp.write(chunk)
        return receiver

    def post(self, *args, **kwargs):
        # Stream去尾
        self.meta['content_length'] = int(self.request.headers.get('Content-Length')) - \
                                      len(self.meta['header']) - \
                                      len(self.meta['boundary'])

        self.fp.seek(self.meta['content_length'], 0)
        self.fp.truncate()
        self.fp.close()
        self.finish('OK')

# Logic
if __name__ == '__main__':
    pass