google-drive-api - 提高每个用户的 Google Drive API 限制并不能阻止速率限制异常

Question

我正在处理一个使用 Drive API 将纯文本文件上传到 Google Drive 的过程。即使实际请求数远不及 API 控制台中为 Drive API 设置的每用户限制，该过程也经常遇到速率限制异常。事实上，设置每个用户的限制似乎不会影响我们收到异常的频率。是否有其他限制(除了每个用户的限制)来控制每秒可以发出的请求数？可以调整吗？

该过程对这些异常使用指数退避，因此操作最终会成功。我们每秒只发出大约 5 个左右的请求，并且每个用户的限制设置为 100。

Caused by: com.google.api.client.googleapis.json.GoogleJsonResponseException: 403 Forbidden
{
  "code" : 403,
  "errors" : [ {
    "domain" : "usageLimits",
    "message" : "Rate Limit Exceeded",
    "reason" : "rateLimitExceeded"
  } ],
  "message" : "Rate Limit Exceeded"
}

编辑:这是开发人员提供的代码的“简化”版本。我们正在使用具有域委派的服务帐户，如: https://developers.google.com/drive/delegation .

package com.seto.fs.daemon;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

import com.google.api.client.auth.oauth2.Credential;
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
import com.google.api.client.http.FileContent;
import com.google.api.client.http.HttpBackOffIOExceptionHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler;
import com.google.api.client.http.HttpBackOffUnsuccessfulResponseHandler.BackOffRequired;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpTransport;
import com.google.api.client.json.JsonFactory;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.client.testing.util.MockBackOff;
import com.google.api.client.util.DateTime;
import com.google.api.client.util.ExponentialBackOff;
import com.google.api.services.drive.Drive;
import com.google.api.services.drive.Drive.Files.Insert;
import com.google.api.services.drive.DriveScopes;
import com.google.api.services.drive.model.ChildList;
import com.google.api.services.drive.model.ChildReference;
import com.google.api.services.drive.model.File.Labels;
import com.google.api.services.drive.model.ParentReference;


public class Test {
    private static final int testFilesCount = 100;
    private static final int threadsCount = 3;
    private static final AtomicInteger rateLimitErrorsCount = new AtomicInteger(0);

    private static final String impersonatedUser = "<impersonatedUserEmail>";
    private static final String serviceAccountID = "<some-id>@developer.gserviceaccount.com";
    private static final String serviceAccountPK = "/path/to/<public_key_fingerprint>-privatekey.p12";

    public static void main(String[] args) throws Exception {
        // Create HTTP transport
        HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport();
        // Create JsonFactory
        final JsonFactory jsonFactory = new JacksonFactory();

        // Create Google credential for service account
        final Credential credential = new GoogleCredential.Builder()
                .setTransport(httpTransport)
                .setJsonFactory(jsonFactory)
                .setServiceAccountScopes(Arrays.asList(DriveScopes.DRIVE))
                .setServiceAccountUser(impersonatedUser)
                .setServiceAccountId(serviceAccountID)
                .setServiceAccountPrivateKeyFromP12File(new File(serviceAccountPK))
                .build();

        // Create Drive client
        final Drive drive = new Drive.Builder(httpTransport, jsonFactory, new HttpRequestInitializer() {
            public void initialize(HttpRequest request) throws IOException {
                request.setContentLoggingLimit(0);
                request.setCurlLoggingEnabled(false);

                // Authorization initialization
                credential.initialize(request);

                // Exponential Back-off for 5xx response and 403 rate limit exceeded error
                HttpBackOffUnsuccessfulResponseHandler serverErrorHandler
                    = new HttpBackOffUnsuccessfulResponseHandler(new ExponentialBackOff.Builder().build());
                serverErrorHandler.setBackOffRequired(new BackOffRequired() {
                    public boolean isRequired(HttpResponse response) {
                        return response.getStatusCode() / 100 == 5
                            || (response.getStatusCode() == 403 && isRateLimitExceeded(
                                    GoogleJsonResponseException.from(jsonFactory, response)));
                    }
                });
                request.setUnsuccessfulResponseHandler(serverErrorHandler);

                // Back-off for socket connection error
                MockBackOff backOff = new MockBackOff();
                backOff.setBackOffMillis(2000);
                backOff.setMaxTries(5);
                request.setIOExceptionHandler(new HttpBackOffIOExceptionHandler(backOff));
            }
        }).setApplicationName("GoogleDriveUploadFile/1.0").build();

        // Get root folder id
        final String rootFolderId = drive.about().get().execute().getRootFolderId();

        // Query all children under root folder
        ChildList result = drive.children().list(rootFolderId).execute();

        // Delete all children under root folder
        for (ChildReference child : result.getItems()) {
            System.out.println("Delete child: " + child.getId());
            drive.files().delete(child.getId()).execute();
        }

        // Create a drive folder
        com.google.api.services.drive.model.File folderMetadata
            = new com.google.api.services.drive.model.File();
        folderMetadata.setMimeType("application/vnd.google-apps.folder")
            .setParents(Arrays.asList(new ParentReference().setId(rootFolderId)))
            .setTitle("DriveTestFolder");
        final com.google.api.services.drive.model.File driveTestFolder = drive.files().insert(folderMetadata).execute();

        // Create test files
        final List<File> testFiles = Collections.synchronizedList(createTestFiles());

        // Run threads to upload files to drive
        List<Thread> threads = new ArrayList<Thread>();
        for (int i = 0; i < threadsCount; i++) {
            Thread thread = new Thread(new Runnable() {
                public void run() {
                    while (testFiles.size() > 0) {
                        try {
                            File testFile = testFiles.remove(0);

                            // The file meta data
                            com.google.api.services.drive.model.File fileMetadata =
                                new com.google.api.services.drive.model.File()
                                .setTitle(testFile.getName()).setParents(Arrays.asList(new ParentReference().setId(driveTestFolder.getId())))
                                .setLabels(new Labels().setRestricted(false)).setMimeType("text/plain")
                                .setModifiedDate(new DateTime(testFile.lastModified()))
                                .setDescription("folder:MyDrive " + testFile.getName());

                            // Insert to drive
                            FileContent fileContent = new FileContent("text/plain", testFile);
                            Insert insertFileCommand = drive.files().insert(fileMetadata, fileContent)
                                    .setUseContentAsIndexableText(true);
                            insertFileCommand.getMediaHttpUploader().setDirectUploadEnabled(true);

                            insertFileCommand.execute();

                            System.out.println(testFile.getName() + " is uploaded");
                        } catch (IOException e) {
                            e.printStackTrace();
                        } catch (IndexOutOfBoundsException e) {
                            // ignore
                        }
                    }
                }
            });
            threads.add(thread);
        }

        long startTime = System.currentTimeMillis();

        for (Thread thread : threads) {
            thread.start();
        }

        for (Thread thread : threads) {
            thread.join();
        }

        System.out.println("Total time spent: " + (System.currentTimeMillis() - startTime)
                + "ms for " + testFilesCount + " files with " + threadsCount + " threads");
        System.out.println("Rate limit errors hit: " + rateLimitErrorsCount.intValue());
    }

    private static List<File> createTestFiles() throws Exception {

        // Create test files directory
        File testFolder = new File("TestFiles");
        testFolder.mkdir();

        // Create test files
        List<File> testFiles = new ArrayList<File>();
        for (int i = 0; i < testFilesCount; i++) {
            File testFile = new File("TestFiles/" + i + ".txt");
            FileOutputStream fops = new FileOutputStream(testFile);
            fops.write(testFile.getAbsolutePath().getBytes());
            fops.close();
            testFiles.add(testFile);
        }
        return testFiles;
    }

    private static boolean isRateLimitExceeded(GoogleJsonResponseException ex) {
        boolean result = false;
        if (ex.getDetails() != null && ex.getDetails().getErrors() != null
                && ex.getDetails().getErrors().size() > 0) {
            String reason = ex.getDetails().getErrors().get(0).getReason();
            result = "rateLimitExceeded".equals(reason) || "userRateLimitExceeded".equals(reason);
            if (result) {
                rateLimitErrorsCount.incrementAndGet();
                System.err.println("Rate limit error");
            }
        }

        return result;
    }

}

编辑:当我们使用单个线程并在每次调用之间设置 500 毫秒的延迟时，我们遇到了这个异常。看起来不可能接近我们配置的每用户费率。即使是默认的每秒 10 个请求看起来也是不可能的。为什么？

Answer 1

上传有一个下限，并且是针对所有应用的每个用户。将看看我们是否可以在文档中发布限制。