python - 从动态Web表格中收集数据

Question

我想从带有动态表的网页中抓取数据。
该表包含有关火车行程的信息。
这是网站:
https://www.laerm-monitoring.de/zug/?mp=3/
我试图通过一个简单的挂载请求 session 来请求数据，但是我只得到基本的HTML数据，而表中没有该数据。

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504, 429),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

session = requests_retry_session()
response = session.get('https://www.laerm-monitoring.de/zug/?mp=3/')
response.content

如何正确执行此操作？

Answer 1

数据是从其他URL动态加载的。您可以使用此示例如何仅使用requests/beautifulsoup加载它:

import json
import requests
from bs4 import BeautifulSoup

data = {
    "sort": "Einfahrtzeit-desc",
    "page": "1",
    "pageSize": "10",
    "group": "",
    "filter": "",
    "__RequestVerificationToken": "",
    "locid": "1",
}

headers = {"X-Requested-With": "XMLHttpRequest"}

url = "https://www.laerm-monitoring.de/zug/"
api_url = "https://www.laerm-monitoring.de/zug/train_read"

with requests.Session() as s:
    soup = BeautifulSoup(s.get(url).content, "html.parser")
    data["__RequestVerificationToken"] = soup.select_one(
        '[name="__RequestVerificationToken"]'
    )["value"]
    data = s.post(api_url, data=data, headers=headers).json()

# pretty print the data
print(json.dumps(data, indent=4))

打印:

{
    "Data": [
        {
            "id": 2536954,
            "Einfahrtzeit": "2021-04-24T20:56:26.1703+02:00",
            "Gleis": 1,
            "Richtung": "Kiel",
            "Category": "PZ",
            "Zugkategorie": "Personenzug",
            "Vorbeifahrtdauer": 7.3,
            "Zugl\u00e4nge": 181.85884,
            "Geschwindigkeit": 115.57797,
            "Maximalpegel": 88.611084,
            "Vorbeifahrtpegel": 85.421326,
            "G\u00fcltig": "OK"
        },
        {
            "id": 2536944,
            "Einfahrtzeit": "2021-04-24T20:52:25.1703+02:00",
            "Gleis": 2,
            "Richtung": "Hamburg",
            "Category": "PZ",
            "Zugkategorie": "Personenzug",
            "Vorbeifahrtdauer": 6.3,
            "Zugl\u00e4nge": 211.10226,
            "Geschwindigkeit": 152.60104,
            "Maximalpegel": 91.81743,
            "Vorbeifahrtpegel": 87.95224,
            "G\u00fcltig": "OK"
        },
        {
            "id": 2536929,
            "Einfahrtzeit": "2021-04-24T20:44:31.4703+02:00",
            "Gleis": 1,
            "Richtung": "Kiel",
            "Category": "PZ",
            "Zugkategorie": "Personenzug",
            "Vorbeifahrtdauer": 5.3,
            "Zugl\u00e4nge": 104.69964,
            "Geschwindigkeit": 110.10052,
            "Maximalpegel": 82.100815,
            "Vorbeifahrtpegel": 79.98168,
            "G\u00fcltig": "OK"
        },
        {
            "id": 2536924,
            "Einfahrtzeit": "2021-04-24T20:42:30.3703+02:00",
            "Gleis": 1,
            "Richtung": "Kiel",
            "Category": "PZ",
            "Zugkategorie": "Personenzug",
            "Vorbeifahrtdauer": 2.9,
            "Zugl\u00e4nge": 49.305683,
            "Geschwindigkeit": 125.18,
            "Maximalpegel": 98.63289,
            "Vorbeifahrtpegel": 97.25019,
            "G\u00fcltig": "OK"
        },
        {
            "id": 2536925,
            "Einfahrtzeit": "2021-04-24T20:42:20.5703+02:00",
            "Gleis": 2,
            "Richtung": "Hamburg",
            "Category": "PZ",
            "Zugkategorie": "Personenzug",
            "Vorbeifahrtdauer": 0.0,
            "Zugl\u00e4nge": 0.0,
            "Geschwindigkeit": 0.0,
            "Maximalpegel": 0.0,
            "Vorbeifahrtpegel": 0.0,
            "G\u00fcltig": "-"
        },
        {
            "id": 2536911,
            "Einfahrtzeit": "2021-04-24T20:35:19.3703+02:00",
            "Gleis": 1,
            "Richtung": "Kiel",
            "Category": "PZ",
            "Zugkategorie": "Personenzug",
            "Vorbeifahrtdauer": 4.1,
            "Zugl\u00e4nge": 103.97647,
            "Geschwindigkeit": 132.2034,
            "Maximalpegel": 87.111984,
            "Vorbeifahrtpegel": 85.6776,
            "G\u00fcltig": "OK"
        },
        {
            "id": 2536907,
            "Einfahrtzeit": "2021-04-24T20:33:31.2703+02:00",
            "Gleis": 2,
            "Richtung": "Hamburg",
            "Category": "GZ",
            "Zugkategorie": "G\u00fcterzug",
            "Vorbeifahrtdauer": 23.8,
            "Zugl\u00e4nge": 583.19586,
            "Geschwindigkeit": 95.63598,
            "Maximalpegel": 88.02967,
            "Vorbeifahrtpegel": 85.02115,
            "G\u00fcltig": "OK"
        },
        {
            "id": 2536890,
            "Einfahrtzeit": "2021-04-24T20:25:36.1703+02:00",
            "Gleis": 2,
            "Richtung": "Hamburg",
            "Category": "PZ",
            "Zugkategorie": "Personenzug",
            "Vorbeifahrtdauer": 3.5,
            "Zugl\u00e4nge": 104.63446,
            "Geschwindigkeit": 160.47487,
            "Maximalpegel": 88.60612,
            "Vorbeifahrtpegel": 86.46721,
            "G\u00fcltig": "OK"
        },
        {
            "id": 2536882,
            "Einfahrtzeit": "2021-04-24T20:22:05.8703+02:00",
            "Gleis": 2,
            "Richtung": "Hamburg",
            "Category": "GZ",
            "Zugkategorie": "G\u00fcterzug",
            "Vorbeifahrtdauer": 26.6,
            "Zugl\u00e4nge": 653.52515,
            "Geschwindigkeit": 94.59859,
            "Maximalpegel": 91.9396,
            "Vorbeifahrtpegel": 85.50632,
            "G\u00fcltig": "OK"
        },
        {
            "id": 2536869,
            "Einfahrtzeit": "2021-04-24T20:16:24.3703+02:00",
            "Gleis": 1,
            "Richtung": "Kiel",
            "Category": "PZ",
            "Zugkategorie": "Personenzug",
            "Vorbeifahrtdauer": 3.3,
            "Zugl\u00e4nge": 87.8222,
            "Geschwindigkeit": 160.01207,
            "Maximalpegel": 91.3928,
            "Vorbeifahrtpegel": 89.54336,
            "G\u00fcltig": "OK"
        }
    ],
    "Total": 8657,
    "AggregateResults": null,
    "Errors": null
}