r - 使用 rvest 将字段添加到已抓取的表列表中

Question

我已经成功地抓取了我想要的数据(在 SO 用户的帮助下)，但是我遗漏了每个抓取表中的数据代表谁的关键。所以我试图使用 mutate 添加一个名为 player 的字段，它与 player[[j]] 相同，但这在列表上不起作用。我已经阅读了 lapply 并尝试过但也没有成功。关于如何实现这一点有什么建议吗？

library(rvest)
library(plyr)
library(dplyr)
library(tidyr)


### get a list of players
page <- (0:18)
urls <- list()
for (i in 1:length(page)) {
  url<- paste0("https://www.mlssoccer.com/players?page=",page[i])
  urls[[i]] <- url
}

tbl <- list()
j <- 1
for (j in seq_along(urls)) {
  tbl[[j]] <- urls[[j]] %>%   
    read_html() %>% 
    html_nodes("a.name_link") %>%
    html_text()
  j <- j+1
  if (j == length(urls)) break
}

### join all of the names into one data frame
tbl <- ldply(tbl, data.frame)


player_tb<- as.data.frame(lapply(tbl, tolower))
colnames(player_tb) <- 'name'
player_table<- as.list(gsub(" ", "-", player_tb$name)) 
colnames(player_table) <- 'player'

#### using a list of players, get the game summary for each regular    season game, adding the player name to the table
pages<- list()
for( i in seq_along(player_table)) {
  page <- paste0("https://www.mlssoccer.com/players/",player_table[i])
  pages[[i]] <- page
}


player_stats <- list()
j <- 1
for (j in seq_along(pages)) {
  player_stats[[j]] <- pages[[j]] %>%   
    read_html() %>% 
    html_nodes("table") %>%
    html_table() %>%
    mutate(player = player)  ## this is the piece that fails
  j <- j+1                   
  if (j == length(pages)) break
}

t <- do.call(rbind, player_stats)

Answer 1

您可以尝试使用 purrr 包来避免 for 循环并加快速度

使用 purrr，您还可以safely、possibly 和 quietly 这些非常酷的功能。有些玩家没有统计数据，您的代码会失败。现在不会了

这个想法是在一个大数据框中收集所有统计数据，并有一个带有玩家名字的标识符列

library(rvest)
library(tidyverse)

# lets assume 3 pages only to do it quickly
page <- (0:2)

# no need to create a list. Just a vector
urls = paste0("https://www.mlssoccer.com/players?page=", page)

# define this function that collects the player's name from a url
get_the_names = function( url){
  url %>% 
    read_html() %>% 
    html_nodes("a.name_link") %>% 
    html_text()
}

# map the urls to the function that gets the names
players = map(urls, get_the_names) %>% 
  # turn into a single character vector
  unlist() %>% 
  # make lower case
  tolower() %>% 
  # replace the `space` to underscore
  str_replace_all(" ", "-")


# Now create a vector of player urls
player_urls = paste0("https://www.mlssoccer.com/players/", players )

# define a function that reads the 3rd table of the url
get_the_summary_stats <-  function(url){

  url %>% 
    read_html() %>% 
    html_nodes("table") %>% 
    html_table() %>% .[[3]] 
}

# lets read 3 players only to speed things up [otherwise it takes a significant amount of time to run...]
a_few_players = player_urls[1:3]

# get the stats 
tables = a_few_players %>% 
  # important step so I can name the rows I get in the table
  set_names() %>% 
  #map the player urls to the function that reads the 3rd table
  # note the `safely` wrap around the get_the_summary_stats' function
  # since there are players with no stats and causes an error (eg.brenden-aaronson )
  # the output will be a list of lists [result and error]
  map(., safely(get_the_summary_stats)) %>% 
  # collect only the `result` output (the table) INTO A DATA FRAME
  # There is also an `error` output
  # also, name each row with the players name
  map_df("result", .id = "player") %>% 
  #keep only the player name (remove the www.mls.... part)
  mutate(player = str_replace(player, "https://www.mlssoccer.com/players/", "")) %>% 
  as_tibble()

让我们看看我们得到了多少

  tables %>% count(player)

# A tibble: 2 x 2
  player                n
  <chr>             <int>
1 anatole-abang        81
2 saad-abdul-salaam   136

现在您可以按玩家名称过滤数据框

  tables %>% 
  filter(player == "anatole-abang")

# A tibble: 81 x 14
   player        Date       Match      Result Appearance  MINS     G     A  SHTS   SOG    FC    FS     Y     R
   <chr>         <chr>      <chr>      <chr>  <chr>      <int> <int> <int> <int> <int> <int> <int> <int> <int>
 1 anatole-abang 10/28/2018 ORL @ RBNY W 0-1  Unused Sub     0     0     0     0     0     0     0     0     0
 2 anatole-abang 10/21/2018 RBNY @ PHI W 1-0  Unused Sub     0     0     0     0     0     0     0     0     0
 3 anatole-abang 10/06/2018 RBNY @ SJ  W 3-1  Unused Sub     0     0     0     0     0     0     0     0     0
 4 anatole-abang 9/30/2018  ATL @ RBNY W 0-2  Unused Sub     0     0     0     0     0     0     0     0     0
 5 anatole-abang 9/22/2018  TOR @ RBNY W 0-2  Unused Sub     0     0     0     0     0     0     0     0     0
 6 anatole-abang 9/16/2018  RBNY @ DC  T 3-3  Unused Sub     0     0     0     0     0     0     0     0     0
 7 anatole-abang 9/01/2018  RBNY @ MTL L 0-3  Unused Sub     0     0     0     0     0     0     0     0     0
 8 anatole-abang 8/29/2018  HOU @ RBNY W 0-1  Unused Sub     0     0     0     0     0     0     0     0     0
 9 anatole-abang 8/26/2018  DC @ RBNY  W 0-1  Unused Sub     0     0     0     0     0     0     0     0     0
10 anatole-abang 8/22/2018  RBNY @ NYC T 1-1  Unused Sub     0     0     0     0     0     0     0     0     0
# ... with 71 more rows