javascript - 从html中提取数据

Question

我有一个 scraper 类，它从一个流行的赔率门户网站收集玩家。没有确切的名字，该网站使用简称。例如 Rafael -> R。幸运的是，它们可以在链接中以 slugified 形式找到 (nadal-rafael)

我做了一个方法processPlayers，尝试解决这个问题。它适用于更简单的情况，但如果玩家的名字中有连字符或她/他有两个名字，则会失败。

我写了一些测试来演示 html 结构和问题。

class Scraper {
  /**
   * Converts a html string to a cheerio object
   * @param {String} html The html string
   * @return {Object} The cheerio object
   */
  htmlToDom(html) {
    return cheerio.load(html)
  }
  /**
   * Gives back the number of parts if the name would slugify
   * It takes in account, that the name could contains hyphen
   * Leopold von Sacher-Masoch -> leopold-von-sacher-masoch
   * @param {Array} a_name The name splitted by space (' ')
   * @return {Integer} The length of the name
   */
  getNameLength(a_name) {
    let name = a_name.length > 1 ? a_name.join(' ') : a_name[0]
    return a_name.length + name.split('-').length - 1
  }

  capitalize(a_name) {
    let res = []
    a_name.forEach(str => {
      res.push(str.substr(0, 1).toUpperCase() + str.substr(1))
    })
    return res.join(' ')
  }

  processPlayers(players) {
    let link = players('a')
    let href = link.attr('href')
    let a_players = link.text().split(' - ')
    let a_href = href.split('/')
    let a_link = a_href[a_href.length - 2].split('-')
    let a_player1 = a_players[0].trim().split(' ')
    let a_player2 = a_players[1].trim().split(' ')
    let a_player1_lastName = a_player1.slice(0, -1)
    let a_player2_lastName = a_player2.slice(0, -1)
    let a_player1ShortFirstName = [a_player1[a_player1.length - 1]]
    let a_player2ShortFirstName = [a_player2[a_player2.length - 1]]

    let p1_lnLength = this.getNameLength(a_player1_lastName)
    let p1_fnLength = this.getNameLength(a_player1ShortFirstName)
    let p2_lnLength = this.getNameLength(a_player2_lastName)
    let p2_fnLength = this.getNameLength(a_player2ShortFirstName)
    let p1_length = p1_lnLength + p1_fnLength
    let p2_length = p2_lnLength + p2_fnLength
    let player1FirstName = this.capitalize(a_link.slice(p1_lnLength, p1_length))
    let player2FirstName = this.capitalize(a_link.slice(p1_length + p2_lnLength, p1_length + p2_length))

    return {
      p1: {
        firstName: player1FirstName,
        lastName: a_player1_lastName.join(' ')
      },
      p2: {
        firstName: player2FirstName,
        lastName: a_player2_lastName.join(' ')
      }
    }
  }
}

// test ===========================================================


test('simple case', function() {
  let playersCell = `
    <td>
      <a href="/t/pavlyuchenkova-anastasia-sorribes-tormo-sara/">
        <span>Pavlyuchenkova A.</span>
         - Sorribes Tormo S.
       </a>
    </td>
    `
  const scraper = new Scraper()
  const td = scraper.htmlToDom(playersCell)
  const players = scraper.processPlayers(td)

  equal(players.p1.firstName, 'Anastasia')
  equal(players.p1.lastName, 'Pavlyuchenkova')

  deepEqual(players.p2, {
    firstName: 'Sara',
    lastName: 'Sorribes Tormo',
  })
});

// =====================================================================

test('hyphen in last name', function() {
  let playersCell = `
    <td><a href="/t/kudermetova-veronika-duque-marino-mariana/">
      <span>Kudermetova V.</span> - Duque-Marino M.</a>
    </td>
    `
  const scraper = new Scraper()
  const td = scraper.htmlToDom(playersCell)
  const players = scraper.processPlayers(td)

  equal(players.p2.firstName, 'Mariana')
  equal(players.p2.lastName, 'Duque-Marino')
});

// =====================================================================

test('hyphen in first name', function() {
  let playersCell = `
    <td>
      <a href="/t/tsonga-jo-wilfried-mayer-florian/">
        <span>Tsonga J-W.</span>
        - Mayer F.
      </a>
    </td>
    `
  const scraper = new Scraper()
  const td = scraper.htmlToDom(playersCell)
  const players = scraper.processPlayers(td)

  equal(players.p1.firstName, 'Jo-Wilfried')
  equal(players.p1.lastName, 'Tsonga')
});

test('two first names', function() {
  let playersCell = `
      <td>
        <a href="/t/alexandrova-ekaterina-muguruza-blanco-garbine/">
        Alexandrova E. - <span>Muguruza</span> B. G.</a>
      </td>
    `
  const scraper = new Scraper()
  const td = scraper.htmlToDom(playersCell)
  const players = scraper.processPlayers(td)

  equal(players.p2.firstName, 'Blanco Garbine')
  equal(players.p2.lastName, 'Muguruza')
});

<!DOCTYPE html>
<html>

<head>
  <meta charset=utf-8 />
  <title></title>
  <script src="https://wzrd.in/standalone/cheerio@latest"></script>
  <link rel="StyleSheet" href="http://code.jquery.com/qunit/qunit-1.12.0.css" type="text/css">
</head>

<body>
  <div id="qunit"></div>
  <div id="qunit-fixture"></div>
  <script src="http://code.jquery.com/qunit/qunit-1.12.0.js" type="text/javascript">
  </script>
</body>

</html>

看起来 stackoverflow 的代码片段对 qunit 有一些问题。所以这是 JsBin 链接:

http://jsbin.com/tohafiqivo/1/edit?html,js,output

Answer 1

看起来问题是在“困难”情况下正确确定姓氏和名字。

您将文本拆分为两个全名，好的。
然后你把全名分成几个词，好吧。

然后你就犯了一个错误，假设 first name 总是最后一个词，其余的是 last name。

事实上，last name是所有不在缩写形式中的词，剩下的就是first name。

我已经通过这种方式解决了你的问题。

请在下面找到更新的 class Scraper(我已经删除了不再使用的功能):

class Scraper {
  /**
   * Converts a html string to a cheerio object
   * @param {String} html The html string
   * @return {Object} The cheerio object
   */
  htmlToDom(html) {
    return cheerio.load(html)
  }

  processPlayers(players) {
    let href = players('a').attr('href')
    let N = players('a').text().trim().split(/\s+-\s+/).map(n => {
        let r = new RegExp('.+('+n.replace(/-/g,'\\S+').replace(/\./g,'[^-]+').replace(/\s/g,'.')+').+','i')
        let p = {lastName: n.replace(/\s\S+\./g,'')}
        href.match(r).map(m => {
          r = new RegExp(p.lastName.replace(/\s/g,'.') + '-', 'i')
          m = m.replace(r,'').split(/-/).map(i => {return i.substring(0,1).toUpperCase() + i.substring(1)})
          p.firstName = n.split(p.lastName + ' ')[1].replace(/\./g,'').split('').map(l => {return m[0] && m[0].indexOf(l) === 0 ? m.shift() : l}).join('')
        })
        return p
      })
    return {p1: N[0], p2: N[1]}
  }
}

它通过了所有测试。对不起，如果它不是很可读。