python - 使用 selenium 抓取多个页面

Question

我正在尝试抓取网页 https://stats.nba.com/players/traditional/?sort=PTS&dir=-1 。我知道我可以抓取第一页。现在，我的困境是，当我单击下一个玩家列表的按钮后，我该如何再次抓取它？第一次单击按钮后，此代码给我一个错误。

点击按钮后页面的链接不会改变。 table 上的机会。因此，单击按钮后，目标是再次抓取表格以获取更多信息。

点击按钮的行会抛出这种错误

StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
  (Session info: chrome=78.0.3904.108)
  (Driver info: chromedriver=2.41.578737 (49da6702b16031c40d63e5618de03a32ff6c197e),platform=Windows NT 10.0.18362 x86_64

driver = webdriver.Chrome(executable_path="./chromedriver/windows/chromedriver.exe")
driver.get(nba_players)
player = driver.find_elements_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[2]/div[1]/table/tbody/tr')
new_split = []
player_stats = []

for i in player:
    player_stats.append(i.text.split('\n'))
    for z in player_stats:
        new_split.append(z[2].split(' '))
#     button = driver.find_element_by_xpath('/html/body/main/div[2]/div/div[2]/div/div/nba-stat-table/div[1]/div/div/a[2]')
#     button.click()
#     time.sleep(120)

Answer 1

不要为此使用 Selenium(虽然可以，但有更好的方法)。直接去api获取数据:

import requests
import pandas as pd


url = 'https://stats.nba.com/stats/leaguedashplayerstats'

payload = {
'College': '',
'Conference': '',
'Country': '',
'DateFrom': '',
'DateTo': '',
'Division': '',
'DraftPick': '',
'DraftYear': '',
'GameScope': '',
'GameSegment': '',
'Height': '',
'LastNGames': '0',
'LeagueID': '00',
'Location': '',
'MeasureType': 'Base',
'Month': '0',
'OpponentTeamID': '0',
'Outcome': '',
'PORound': '0',
'PaceAdjust': 'N',
'PerMode': 'PerGame',
'Period': '0',
'PlayerExperience':'' ,
'PlayerPosition': '',
'PlusMinus': 'N',
'Rank': 'N',
'Season': '2019-20',
'SeasonSegment': '',
'SeasonType': 'Regular Season',
'ShotClockRange': '',
'StarterBench': '',
'TeamID': '0',
'TwoWay': '0',
'VsConference': '',
'VsDivision':'' ,
'Weight': ''}

jsonData = requests.get(url, params=payload).json()


cols = jsonData['resultSets'][0]['headers']
rows = jsonData['resultSets'][0]['rowSet']

df = pd.DataFrame(rows, columns=cols)

输出:

print (df.head(10).to_string())
   PLAYER_ID        PLAYER_NAME     TEAM_ID TEAM_ABBREVIATION   AGE  GP   W   L  W_PCT   MIN  FGM   FGA  FG_PCT  FG3M  FG3A  FG3_PCT  FTM  FTA  FT_PCT  OREB  DREB  REB  AST  TOV  STL  BLK  BLKA   PF  PFD   PTS  PLUS_MINUS  NBA_FANTASY_PTS  DD2  TD3  GP_RANK  W_RANK  L_RANK  W_PCT_RANK  MIN_RANK  FGM_RANK  FGA_RANK  FG_PCT_RANK  FG3M_RANK  FG3A_RANK  FG3_PCT_RANK  FTM_RANK  FTA_RANK  FT_PCT_RANK  OREB_RANK  DREB_RANK  REB_RANK  AST_RANK  TOV_RANK  STL_RANK  BLK_RANK  BLKA_RANK  PF_RANK  PFD_RANK  PTS_RANK  PLUS_MINUS_RANK  NBA_FANTASY_PTS_RANK  DD2_RANK  TD3_RANK  CFID            CFPARAMS
0     203932       Aaron Gordon  1610612753               ORL  24.0  24  11  13  0.458  31.4  5.0  11.8   0.423   1.2   4.0    0.305  2.3  3.2   0.711   1.7   5.2  6.9  2.9  1.5  0.7  0.5   0.7  2.1  3.0  13.5        -2.0             27.9    5    0      178     152     132         228        63       103        87          280        152        118           278        87        74          269         57         52        49       102       139       193       149         87      159        75       103              328                    86        37        13     5   203932,1610612753
1    1628988      Aaron Holiday  1610612754               IND  23.0  26  19   7  0.731  21.6  3.7   8.7   0.427   1.5   3.7    0.402  1.0  1.2   0.833   0.4   2.2  2.6  2.6  1.3  0.8  0.1   0.4  1.6  1.9   9.8         2.7             18.3    1    0       90      25     266          78       210       156       149          273        110        136            89       230       257          112        314        253       279       117       166       149       376        176      273       151       160               98                   207       109        13     5  1628988,1610612754
2    1627846        Abdel Nader  1610612760               OKC  26.0  19  10   9  0.526  17.7  2.6   5.2   0.495   1.2   2.6    0.440  1.2  1.4   0.852   0.3   1.6  1.9  0.6  1.0  0.4  0.4   0.1  1.5  1.1   7.5        -1.4             12.0    0    0      266     180     196         200       266       244       263          119        165        210            47       187       218           93        333        335       338       365       215       312       193        369      302       271       231              295                   297       172        13     5  1627846,1610612760
3    1629678  Admiral Schofield  1610612764               WAS  22.0  12   3   9  0.250   9.8  1.3   2.3   0.571   0.7   1.5    0.444  0.4  0.5   0.833   0.3   1.0  1.3  0.4  0.3  0.1  0.0   0.1  1.3  0.5   3.8         2.4              5.9    0    0      341     366     196         384       384       357       390           46        245        290            45       352       364          112        356        385       392       400       421       405       398        380      318       372       353              108                   397       172        13     5  1629678,1610612764
4     201143         Al Horford  1610612755               PHI  33.0  24  16   8  0.667  30.6  5.6  11.8   0.472   1.4   4.1    0.343  1.0  1.5   0.676   1.8   5.2  6.9  3.9  1.2  1.1  1.1   0.4  2.4  1.6  13.6         4.0             33.2    2    0      178      80     218         140        78        81        87          152        118        109           196       213       193          302         50         55        47        58       181        61        37        208      114       179        99               62                    55        78        13     5   201143,1610612755
5     202329    Al-Farouq Aminu  1610612753               ORL  29.0  18   7  11  0.389  21.1  1.4   4.8   0.291   0.5   2.0    0.250  1.1  1.6   0.655   1.3   3.5  4.8  1.2  0.9  1.0  0.4   0.2  1.5  1.6   4.3        -1.9             15.3    1    0      276     263     167         310       221       355       290          431        282        252           324       209       185          318         89        149       127       271       245        82       152        297      289       183       328              326                   258       109        13     5   202329,1610612753
6     202692         Alec Burks  1610612744               GSW  28.0  25   4  21  0.160  29.1  5.2  12.0   0.430   1.4   4.0    0.356  3.6  4.1   0.883   0.8   3.6  4.4  2.9  1.6  1.1  0.2   0.6  2.2  3.1  15.4        -6.1             27.6    1    0      136     341       6         437       104        95        85          261        115        114           173        41        55           60        186        134       151       100       110        63       271        105      147        72        79              440                    89       109        13     5   202692,1610612744
7    1627936        Alex Caruso  1610612747               LAL  25.0  26  23   3  0.885  20.5  2.1   4.7   0.447   0.8   2.2    0.357  0.7  1.0   0.680   0.2   2.0  2.2  1.8  1.0  1.1  0.3   0.4  1.6  1.7   5.7         5.7             14.1    0    0       90       8     372          27       225       281       293          211        231        245           169       287       288          299        365        281       313       176       215        72       251        198      267       176       282               32                   272       172        13     5  1627936,1610612747
8     203458           Alex Len  1610612737               ATL  26.0  27   6  21  0.222  17.9  3.3   6.1   0.539   0.3   1.3    0.257  1.3  1.9   0.686   1.6   3.4  5.0  1.1  1.0  0.5  0.7   0.5  2.1  1.5   8.2        -4.7             18.7    3    0       35     284       6         404       263       185       229           67        319        315           323       176       164          293         65        156       111       284       237       248        75        152      172       192       202              414                   194        59        13     5   203458,1610612737
9    1628035   Alfonzo McKinnie  1610612739               CLE  27.0  18   3  15  0.167  10.0  0.9   2.9   0.327   0.4   1.4    0.320  0.4  0.6   0.636   0.6   1.4  2.1  0.2  0.4  0.3  0.2   0.3  1.2  0.4   2.7        -1.9              6.5    0    0      276     366      89         431       378       396       372          409        298        306           254       356       349          333        231        359       329       432       379       322       316        230      344       394       388              322                   391       172        13     5  1628035,1610612739