python - 无法使用python在谷歌驱动器上上传大文件

Question

我正在尝试使用以下代码通过 Google API 将文件上传到 Google Drive

import httplib2
from apiclient import discovery
from httplib2 import Http
from oauth2client import file, client, tools
try:
    import argparse
    flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
    flags = None

SCOPES =['https://www.googleapis.com/auth/drive','https://www.googleapis.com/auth/drive.file','https://www.googleapis.com/auth/drive.appdata', 'https://www.googleapis.com/auth/drive.apps.readonly']
store = file.Storage('scope.json')
creds = store.get()
if not creds or creds.invalid:
    flow = client.flow_from_clientsecrets('client_secret.json', SCOPES)
    creds = tools.run_flow(flow, store, flags) if flags else tools.run(flow, store)
    DRIVE = discovery.build('drive', 'v3', http=creds.authorize(Http()))
else:
    credentials = creds
    http = credentials.authorize(httplib2.Http())
    DRIVE = discovery.build('drive', 'v3', http=http)

FILES = (
    ('/home/vkm/mayur/Demo_Google_API.zip', 'application/vmd.google-apps.document'),
)

for filename, mimeType in FILES:
    metadata = {'name': filename}
    if mimeType:
        metadata['mimeType'] = mimeType
    res = DRIVE.files().create(body=metadata, media_body=filename).execute()
    if res:
        print('Uploaded "%s" (%s)' % (filename, res['mimeType']))

我可以上传小文件，但是当我尝试使用 8GB 的​​文件时，它会出现 MemorryErro。请找到我收到的错误消息。

Traceback (most recent call last):
  File "demo.py", line 46, in <module>
    res = DRIVE.files().create(body=metadata, media_body=filename).execute()
  File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 853, in method
    payload = media_upload.getbytes(0, media_upload.size())
  File "/usr/local/lib/python2.7/dist-packages/googleapiclient/http.py", line 482, in getbytes
    return self._fd.read(length)
MemoryError

Answer 1

Vikram's comment在 mhawke's answer 中揭示了一个问题: next_chunk 需要根据以下的返回值调用:

request = DRIVE.files().create(body=metadata, media_body=media)

不在 request.execute() 的返回值上。

这是一段 Python 代码，我验证过它可以处理我的 Google 云端硬盘帐户中最大 10MB 的文件:

# Upload some file that just happens to be binary (we
# don't care about metadata, just upload it without
# translation):
the_file_to_upload = 'some_binary_file'
metadata = {'name': the_file_to_upload}
# Note the chunksize restrictions given in
# https://developers.google.com/api-client-library/python/guide/media_upload
media = MediaFileUpload(the_file_to_upload,
                        chunksize=1024 * 1024,
                        # Not sure whether or not this mimetypes is necessary:
                        mimetype='text/plain',
                        resumable=True)
request = drive_service.files().create(body=metadata, media_body=media)
response = None
while response is None:
    status, response = request.next_chunk()
    if status:
        print("Uploaded %d%%." % int(status.progress() * 100))
print("Upload of {} is complete.".format(the_file_to_upload))

这里是一段 Python 代码，它下载相同的文件，但下载到不同的文件，这样我就可以使用 sha1sum验证文件没有被进出 Google 云端硬盘更改。

# Verify downloading works without translation:
request = drive_service.files().get_media(fileId=response['id'])
# Use io.FileIO. Refer to:
# https://google.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaIoBaseDownload-class.html
out_filename = the_file_to_upload + ".out"
fh = io.FileIO(out_filename, mode='wb')
downloader = MediaIoBaseDownload(fh, request, chunksize=1024 * 1024)
done = False
while done is False:
    status, done = downloader.next_chunk()
    if status:
        print("Download %d%%." % int(status.progress() * 100))
print("Download Complete!")