python - 使用 python selenium 单击查看联系人后，我无法获取相同的检查输出，从而无法抓取手机号码

Question

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.common.exceptions import TimeoutException

driver = webdriver.Chrome(r'C:\chromedriver.exe')

driver.get('https://www.gigadocs.com/hyderabad/dentist')

driver.find_element_by_xpath('//*[@id="listingTab"]/div[2]/div/div[1]/div[1]/div/div[2]/div[2]/ul/li[1]/span'). click()

soup = BeautifulSoup(driver.page_source,'html.parser')

mobile = soup.find('ul',class_='detailsList')
print(mobile)

我尝试单击“查看联系人”来抓取手机号码，但单击后，得到与“查看联系人”相同的输出。

Answer 1

你不需要 Selenium 的开销。该页面使用医生和诊所的 ID 发出 POST 请求来检索电话号码。您可以从初始页面中抓取这些 ID，然后模仿这些请求来获取电话号码。我使用医生 ID 作为字典的键，并使用电话号码更新值。

import requests
from bs4 import BeautifulSoup as bs

headers = {
    'User-Agent': 'Mozilla/5.0',
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'X-Requested-With': 'XMLHttpRequest'
}

data = {
  'doctorId': '3806',   #[data-doctor]
  'clinicId': '1519',   #[data-clinic]
  'clickSource': 'mobile'
}

with requests.Session() as s:
    s.headers = headers
    r = s.get('https://www.gigadocs.com/hyderabad/dentist')
    soup = bs(r.content, 'lxml')
    tel_numbers = {i['data-doctor']:i['data-clinic'] for i in soup.select('.appointmentBtn')}

    for k, v in tel_numbers.items():
        data['doctorId'] = k
        data['clinicId'] = v
        r = s.post('https://www.gigadocs.com/search/getmobilenumbers', data=data).json()
        tel_numbers[k] = r['mobile']

print(tel_numbers)