Python Django 截断响应

Question

我创建了一个小型 Django 应用程序并将其部署在 Ubuntu 上， View .py

from django.shortcuts import render
import json
import psycopg2
import csv
from django.http import HttpResponse
from ratelimit.decorators import ratelimit
def create_connection():
    return psycopg2.connect(database="data",user="ag",host="localhost",port=5432,password='pass')

def execute_query(query, parameters):
    with create_connection() as connection:
        result = []
        cursor = connection.cursor()
        cursor.execute(query, parameters)
        for row in cursor:
            result.append(row)
        for i in range(len(result)):
            result[i] = list(result[i])
            for j in range(len(result[i])):
                result[i][j] = result[i][j] if result[i][j] else ""
        return result

@ratelimit(key='ip', rate='10/m')
def to_csv(request):
    j_body = json.loads(request.GET.get('data', ''))
    query = j_body['query']
    print(query)
    print("****************************************************************")
    parameters = j_body['params']
    headers = j_body['headers']
    rows = execute_query(query, parameters)
    response = HttpResponse(content_type='text_csv')
    response['Content-Disposition'] = 'attachment; filename="result.csv"'
    writer = csv.writer(response)
    writer.writerow(headers)
    writer.writerows(rows)
    return response

这个应用程序的主要目标 - 获取 sql 查询并将 csv 响应作为文件返回，该文件的大小可以超过 200 mb。我有一个问题 - 我的响应总是被截断为 696 kb(无论cursor.execute 结果中的行数如何)。有例子。您可以看到最后被截断的行。

服务器由“python3 manage.py runserver 0.0.0.0:8000”命令运行。请帮我解决我的问题。

已更新我用 Nginx 和 Gunicorn 重新部署了它，并尝试了此处建议的解决方案 ngnix + gunicorn throws truncated response body尽管如此，我仍然遇到同样的错误

我按照这篇文章修复了我的代码 - https://docs.djangoproject.com/en/1.10/howto/outputting-csv/#streaming-large-csv-files

#Also changed execute_query method, so it doesn't create redurant copy of response from database
def execute_query(query, parameters):
    with create_connection() as connection:
        result = []
        cursor = connection.cursor()
        cursor.execute(query, parameters)
        return cursor

class Echo(object):
    """An object that implements just the write method of the file-like
    interface.
    """
    def write(self, value):
        """Write the value by returning it, instead of storing in a buffer."""
        return value

@ratelimit(key='ip', rate='10/m')
def to_csv(request):
    try:
        j_body = json.loads(request.GET.get('data', ''))
        query = j_body['query']
        print("****************************************************************")
        parameters = j_body['params']
        headers = j_body['headers']
        cursor = execute_query(query, parameters)
        pseudo_buffer = Echo()
        writer = csv.writer(pseudo_buffer)
        writer.writerow(headers)
        print('Writing began')
        response = StreamingHttpResponse((writer.writerow(row) for row in cursor),
                                     content_type="text/csv")
        response['Content-Disposition'] = 'attachment; filename="result.csv"'
        print('Sending began')
        return response
    except:
        return HttpResponse('Error')

现在一切正常。

Answer 1

听起来您可能需要 StreamingHttpResponse 。你尝试过实现这个吗？您似乎遇到了某种缓冲/分块响应问题。可以使用gunicorn或nginx之类的东西来增加缓冲区大小，但这不是真正的解决方案。