python-3.x - 将上市公司数据的 HTML 读入 Dataframe

Question

当我尝试从纳斯达克读取上市公司数据时，我通常用于将简单的 HTML 数据导入数据帧的代码返回 IndexError: list index out of range 消息，但无法看到问题出在哪里。
我真的很感激一些帮助。

from bs4 import BeautifulSoup
import pandas as pd
import requests

url = 'https://www.nasdaq.com/market-activity/stocks/screener'

headers = {"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36"}

response = requests.get(url, headers=headers)

soup = BeautifulSoup(response.content, 'html.parser')

tables = soup.find_all('table', rules = 'all')

table = str(tables[0]) #cast table to string

df = pd.read_html(table, skiprows=2, flavor='bs4')[0]
print(df.head())

这是我想读入数据框的第一个 HTML 表......共 406 个。每个页面有 20 行公司数据。

Answer 1

该页面是使用 javascript 动态加载的，因此您无法通过请求读取加载它。使用浏览器中的 Developer 选项卡，您可以拦截生成数据的 xhr 链接及其所有参数，并捕获该数据。响应是一个 json 字符串，可以加载到 json 中以转换为字典，pandas 可以将其读取到数据帧中:

import requests
import json
import pandas as pd

cookies = {
    'AKA_A2': 'A',
    'ak_bmsc': '26AE62881D0C356A61469B75581CA47C1743FBD2BD1D000043A5905F1822AD4D~plb7aR8lcS0B182W3yyiLmqNhQixYZnITWijGoQj/spL0g901tCb0pELApCkydFuj4bGJCEiELfcYRtc2CCQ/tGJW8qmqP0CKaWoD69VKYrrz6hStjyz5KhVZnU2OmvmzWGxB4pu6azcXssTvZF6u9VBVnrWfgskZ93v5Bzjd2dFi4GNkm0wVoriKx3EF3a42osdrS2BerWewsuue0VxZ6yplfE4lqCpJSfjCY1ZWmfWw=',
    'entryUrl': 'https://www.nasdaq.com/market-activity/stocks/screener',
    'entryReferringURL': '',
    'bm_sv': 'D9BCD75B3FA5D7AEA644C7F771A79D5D~LRB0VO8GVndeCJY9Plht4U91bu3VCHeOYyDjNVBhBBWccsB0Qyp5kETWAi6k5W9yYuGoNpzShkqSAp2XiBV9yxtcojZ4T2srEkO++eu7cqWzDoxyGhY+5p/sdZqs1lgBTQq+U0yEnNugvWYB47+1cSKC4BmfG+38fuiAEeUGHR8=',
    'bm_mi': 'F10357CB7F1B64B05A3B062B4BF82D00~4IByO6LJHitJhKvPEZPQz9+rv5diETENZZAFqqe7zVEdPRlAFti/sVtnzwy+6DrelVLk66qeOjSIoq8f9FvJxw8yJQd0yuZ5pEv955P9BBA8JSJtJBefi6A8W2HM2SrhMfdz7GvxajupdlUd2UlNo6vp9rRO4dFDmwOyV0R1yq8UtTDDICzm/2+Dsb2e33u1ToU0Y0gPIIDVRSlahJjKrhYwbr3mTXx+JJoy6612KGlZEuWQnieh31FMu1Fu0NaOUxmK9/8I2kMNKNgJh0nOuA==',
}

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:81.0) Gecko/20100101 Firefox/81.0',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'en-US,en;q=0.5',
    'Connection': 'keep-alive',
    'Referer': 'https://www.nasdaq.com/market-activity/stocks/screener',
    'Cache-Control': 'max-age=0',
    'TE': 'Trailers',
}

params = (
    ('page', '1'),
    ('pageSize', '20'),
)

response = requests.get('https://www.nasdaq.com/api/v1/screener', headers=headers, params=params, cookies=cookies)

table=json.loads(response.text)
df = pd.DataFrame.from_dict(table)
new_df = pd.DataFrame(list(df['data']))
print(new_df)

输出:

ticker               company      marketCap marketCapGroup      sectorName  \
0    AAPL                 Apple  2037310823400           Mega      Technology   
1    MSFT             Microsoft  1624396702497           Mega      Technology   
2    AMZN                Amazon  1611367016163           Mega  Consumer Goods   
3    GOOG      Alphabet Class C  1058287004605           Mega      Technology   

等等。