python - 如何组合/加速多个 API 调用以提高性能？

Question

更新:我发现something that might be useful ，但我在弄清楚如何实现它方面仍然遇到了一些麻烦。如果我尝试像这样映射 get_data，我不确定如何将每次调用的结果分配给相应的变量。

parameters = [
    [service, profile_id, '30daysAgo', 'ga:browser', 'sessions::condition::ga:deviceCategory==desktop'],
    [service, profile_id, '60daysAgo', 'ga:browser', 'sessions::condition::ga:deviceCategory==desktop'],
    ...
    [service, profile_id, '90daysAgo', 'ga:browser,ga:browserVersion', 'sessions::condition::ga:deviceCategory==mobile']
]

with ThreadPoolExecutor(max_workers=4) as executor:
    executor.map(get_data, parameters)

<小时/>

我正在编写一个 Python 应用程序(使用 Google 分析 API)，该应用程序允许用户获取排名前 10 的桌面浏览器、按版本分割的桌面浏览器、移动浏览器以及用于访问给定内容的移动操作系统的报告过去 30、60 和 90 天内访问过的网站。截至目前，一切似乎都进展顺利。

然而，性能却无处不在。正在发出 12 个 API 请求 - 4 组数据中每组 3 个。有时应用程序需要大约 10 秒才能运行，有时则需要一分钟多一点。看起来这一切都取决于 API 的响应方式。所以我的问题是:有什么方法可以组合其中一些请求或以并发执行的方式安排它们？

我尝试寻找合并请求的方法，这样也许我只需对每组数据执行一个请求即可返回 30、60 和 90 天的信息，但我无法找到任何事物。至于同时请求，我只是不太确定如何去做类似的事情。我能找到的最接近的是 this question/answer ，但我不太明白有关批处理的答案。

相关代码如下:

def get_data(service, profile_id, days, dimensions, segment):
    return service.data().ga().get(
        ids='ga:' + profile_id,
        start_date=days,
        end_date='today',
        metrics='ga:sessions',
        dimensions=dimensions,
        sort='-ga:sessions',
        segment=segment,
        max_results=10).execute()


def get_results(service, profile_id):
    global glob_startdate
    global glob_months

    # get top 10 desktop browsers
    print("Getting top 10 desktop browsers...")
    data_1a = get_data(service, profile_id, '30daysAgo', 'ga:browser', 'sessions::condition::ga:deviceCategory==desktop')
    data_1b = get_data(service, profile_id, '60daysAgo', 'ga:browser', 'sessions::condition::ga:deviceCategory==desktop')
    data_1c = get_data(service, profile_id, '90daysAgo', 'ga:browser', 'sessions::condition::ga:deviceCategory==desktop')
    data1 = [data_1a, data_1b, data_1c]

    # get top 10 desktop browser versions
    print("Getting top 10 desktop browser versions...")
    data_2a = get_data(service, profile_id, '30daysAgo', 'ga:browser,ga:browserVersion', 'sessions::condition::ga:deviceCategory==desktop')
    data_2b = get_data(service, profile_id, '60daysAgo', 'ga:browser,ga:browserVersion', 'sessions::condition::ga:deviceCategory==desktop')
    data_2c = get_data(service, profile_id, '90daysAgo', 'ga:browser,ga:browserVersion', 'sessions::condition::ga:deviceCategory==desktop')
    data2 = [data_2a, data_2b, data_2c]

    # get top 10 mobile OS's
    print("Getting top 10 mobile OS's...")
    data_3a = get_data(service, profile_id, '30daysAgo', 'ga:operatingSystem,ga:operatingSystemVersion', 'sessions::condition::ga:deviceCategory==mobile')
    data_3b = get_data(service, profile_id, '60daysAgo', 'ga:operatingSystem,ga:operatingSystemVersion', 'sessions::condition::ga:deviceCategory==mobile')
    data_3c = get_data(service, profile_id, '90daysAgo', 'ga:operatingSystem,ga:operatingSystemVersion', 'sessions::condition::ga:deviceCategory==mobile')
    data3 = [data_3a, data_3b, data_3c]

    # get top 10 mobile browsers
    print("Getting top 10 mobile browsers...")
    data_4a = get_data(service, profile_id, '30daysAgo', 'ga:browser,ga:browserVersion', 'sessions::condition::ga:deviceCategory==mobile')
    data_4b = get_data(service, profile_id, '60daysAgo', 'ga:browser,ga:browserVersion', 'sessions::condition::ga:deviceCategory==mobile')
    data_4c = get_data(service, profile_id, '90daysAgo', 'ga:browser,ga:browserVersion', 'sessions::condition::ga:deviceCategory==mobile')
    data4 = [data_4a, data_4b, data_4c]

谢谢!

Answer 1

您可以batch由于 API quota and limits 一次最多 10 个请求.

from apiclient.http import BatchHttpRequest
import httplib2


def call_back(request_id, response, exception):
  """Do something with the response of each call"""
  pass

def get_request(service, profile_id, days, dimensions, segment):
   """Note I removed the execute() from the end of this method."""
   return service.data().ga().get(
     ids='ga:' + profile_id,
     start_date=days,
     end_date='today',
     metrics='ga:sessions',
     dimensions=dimensions,
     sort='-ga:sessions',
     segment=segment,
     max_results=10)

# Create a batch Http Request object
batch = BatchHttpRequest(callback=self.call_back)


# Construct your queries.
# get top 10 desktop browsers
print("Getting top 10 desktop browsers...")
request_1a = get_request(service, profile_id, '30daysAgo', 'ga:browser', 'sessions::condition::ga:deviceCategory==desktop')
request_1b = get_request(service, profile_id, '60daysAgo', 'ga:browser', 'sessions::condition::ga:deviceCategory==desktop')
request_1c = get_request(service, profile_id, '90daysAgo', 'ga:browser', 'sessions::condition::ga:deviceCategory==desktop')

for request in [request_1a, request_1b, request_1c]:
    batch.add(request)

batch.execute(http=httplib2.Http())