python - 使用 urllib2 检查响应

Question

我正在尝试通过使用 opencorporates api 增加页面计数器来访问页面。但问题是有时会出现无用的数据。例如在下面的 url 中 jurisdiction_code = ae_az 我得到的网页只显示这个:

{"api_version":"0.2","results":{"companies":[],"page":1,"per_page":26,"total_pages":0,"total_count":0}}

这在技术上是空的。如何检查此类数据并跳过此数据以转到下一个管辖区？

这是我的代码

import urllib2
import json,os

f = open('codes','r')
for line in f.readlines():
   id = line.strip('\n')
   url = 'http://api.opencorporates.com/v0.2/companies/search?q=&jurisdiction_code={0}&per_page=26&current_status=Active&page={1}?api_token=ab123cd45' 
   i = 0
   directory = id
   os.makedirs(directory)
   while True:
      i += 1
      req = urllib2.Request(url.format(id, i))
      print url.format(id,i)
      try:
        response = urllib2.urlopen(url.format(id, i))
      except urllib2.HTTPError, e:
        break
      content = response.read()
      fo = str(i) + '.json'    
      OUTFILE = os.path.join(directory, fo)
      with open(OUTFILE, 'w') as f:
        f.write(content)

Answer 1

解释您返回的响应(您已经知道它是 json)并检查您想要的数据是否存在。

...
content = response.read()
data = json.loads(content)
if not data.get('results', {}).get('companies'):
    break
...

这是您使用 Requests 编写的代码，并使用此处的答案。它远没有达到应有的健壮或干净，但展示了您可能想要采用的路径。速率限制是一个猜测，似乎不起作用。请记住将您的实际 API key 放入。

import json
import os
from time import sleep
import requests

url = 'http://api.opencorporates.com/v0.2/companies/search'
token = 'ab123cd45'
rate = 20  # seconds to wait after rate limited

with open('codes') as f:
    codes = [l.strip('\n') for l in f]


def get_page(code, page, **kwargs):
    params = {
        # 'api_token': token,
        'jurisdiction_code': code,
        'page': page,
    }
    params.update(kwargs)

    while True:
        r = requests.get(url, params=params)

        try:
            data = r.json()
        except ValueError:
            return None

        if 'error' in data:
            print data['error']['message']
            sleep(rate)
            continue

        return data['results']


def dump_page(code, page, data):
    with open(os.path.join(code, str(page) + '.json'), 'w') as f:
        json.dump(data, f)


for code in codes:
    try:
        os.makedirs(code)
    except os.error:
        pass

    data = get_page(code, 1)
    if data is None:
        continue

    dump_page(code, 1, data['companies'])

    for page in xrange(1, int(data.get('total_pages', 1))):
        data = get_page(code, page)
        if data is None:
            break

        dump_page(code, page, data['companies'])