python - 将 QuadgramCollat​​ionFinder 转换为 PentagramCollat​​ionFinder

Question

我尝试将 NLTK 工具包中的 QuadgramCollat​​ionFinde 类转换为 PentagramCollocationFinder one，即使用 5 个项目，不仅是 4 个项目，而且确实有效。

这是nltk类

import nltk
from nltk.compat import iteritems
from nltk.probability import FreqDist
from nltk.util import ngrams
from nltk.metrics import ContingencyMeasures, BigramAssocMeasures,TrigramAssocMeasures
from nltk.metrics.spearman import ranks_from_scores, spearman_correlation
from __future__ import print_function

class QuadgramCollocationFinder(AbstractCollocationFinder):
    """A tool for the finding and ranking of quadgram collocations or other association measures.
    It is often useful to use from_words() rather than constructing an instance directly.
    """
    default_ws = 4

    def __init__(self, word_fd, quadgram_fd, ii, iii, ixi, ixxi, iixi, ixii):
        """Construct a QuadgramCollocationFinder, given FreqDists for appearances of words,
        bigrams, trigrams, two words with one word and two words between them, three words
        with a word between them in both variations.
        """
        AbstractCollocationFinder.__init__(self, word_fd, quadgram_fd)
        self.iii = iii
        self.ii = ii
        self.ixi = ixi
        self.ixxi = ixxi
        self.iixi = iixi
        self.ixii = ixii


    def from_words(cls, words, window_size=4):
        if window_size < 4:
            raise ValueError("Specify window_size at least 4")
        ixxx = FreqDist()
        iiii = FreqDist()
        ii = FreqDist()
        iii = FreqDist()
        ixi = FreqDist()
        ixxi = FreqDist()
        iixi = FreqDist()
        ixii = FreqDist()

        for window in ngrams(words, window_size, pad_right=True):
            w1 = window[0]
            if w1 is None: 
                continue
            for w2, w3, w4 in _itertools.combinations(window[1:], 3):
                ixxx[w1] += 1
                if w2 is None:
                    continue
                ii[(w1, w2)] += 1
                if w3 is None:
                    continue
                iii[(w1, w2, w3)] += 1
                ixi[(w1, w3)] += 1
                if w4 is None:
                    continue
                iiii[(w1, w2, w3, w4)] += 1
                ixxi[(w1, w4)] += 1
                ixii[(w1, w3, w4)] += 1
                iixi[(w1, w2, w4)] += 1

        return cls(ixxx, iiii, ii, iii, ixi, ixxi, iixi, ixii)


    def score_ngram(self, score_fn, w1, w2, w3, w4):
        n_all = self.word_fd.N()
        n_iiii = self.ngram_fd[(w1, w2, w3, w4)]
        if not n_iiii:
            return
        n_iiix = self.iii[(w1, w2, w3)]
        n_xiii = self.iii[(w2, w3, w4)]
        n_iixi = self.iixi[(w1, w2, w4)]
        n_ixii = self.ixii[(w1, w3, w4)]

        n_iixx = self.ii[(w1, w2)]
        n_xxii = self.ii[(w3, w4)]
        n_xiix = self.ii[(w2, w3)]
        n_ixix = self.ixi[(w1, w3)]
        n_ixxi = self.ixxi[(w1, w4)]
        n_xixi = self.ixi[(w2, w4)]

        n_ixxx = self.word_fd[w1]
        n_xixx = self.word_fd[w2]
        n_xxix = self.word_fd[w3]
        n_xxxi = self.word_fd[w4]
        return score_fn(n_iiii,
                        (n_iiix, n_iixi, n_ixii, n_xiii),
                        (n_iixx, n_ixix, n_ixxi, n_xixi, n_xxii, n_xiix),
                        (n_ixxx, n_xixx, n_xxix, n_xxxi),
                        n_all)

这就是我的想法

class fivegramCollocationFinder(AbstractCollocationFinder):
"""A tool for the finding and ranking of quadgram collocations or other association measures.
It is often useful to use from_words() rather than constructing an instance directly.
"""
default_ws = 4

def __init__(self, word_fd, quingram_fd, ii, iii, ixi,iiii,ixxi, ixii, iixi, ixxxi, iixxi, ixixi, ixxii, iiixi, ixiii, iixii):
    """Construct a QuadgramCollocationFinder, given FreqDists for appearances of words,
    bigrams, trigrams, two words with one word and two words between them, three words
    with a word between them in both variations.
    """
    AbstractCollocationFinder.__init__(self, word_fd, quingram_fd)
    self.iiii = iiii
    self.iii = iii
    self.ii = ii
    self.ixi = ixi
    self.ixxi = ixxi
    self.iixi = iixi
    self.ixii = ixii
    self.ixxxi = ixxxi
    self.iixxi = iixxi
    self.ixixi = ixixi
    self.ixxii = ixxii
    self.iiixi = iiixi
    self.ixiii = ixiii
    self.iixii = iixii

@classmethod

def from_words(cls, words, window_size=5):
    if window_size < 5:
        raise ValueError("Specify window_size at least 4")
    ixxxx = FreqDist()
    ii =  FreqDist()
    iii = FreqDist()
    iiii = FreqDist()
    iiiii = FreqDist()
    ixi = FreqDist()
    ixxi = FreqDist()
    iixi = FreqDist()
    ixii = FreqDist()
    ixxxi = FreqDist()
    iixxi  = FreqDist()
    ixixi  = FreqDist()
    ixxii = FreqDist()
    iiixi = FreqDist()
    ixiii = FreqDist()
    iixii   = FreqDist()


    for window in ngrams(words, window_size, pad_right=True):
        w1 = window[0]
        if w1 is None: 
            continue
        for w2, w3, w4, w5 in _itertools.combinations(window[1:], 4):
            ixxxx[w1] += 1
            if w2 is None:
                continue
            ii[(w1, w2)] += 1
            if w3 is None:
                continue
            iii[(w1, w2, w3)] += 1
            ixi[(w1, w3)] += 1
            if w4 is None:
                continue
            iiii[(w1, w2, w3, w4)] += 1
            ixxi[(w1, w4)] += 1
            ixii[(w1, w3, w4)] += 1
            iixi[(w1, w2, w4)] += 1
            if w5 is None:
                continue
            iiiii[(w1, w2, w3, w4, w5)] += 1
            ixxxi[(w1, w5)] += 1
            iixxi[(w1, w2, w5)] += 1
            ixixi[(w1, w3, w5)] += 1
            ixxii[(w1, w4, w5)] += 1
            iiixi[(w1, w2, w3, w5)] +=1
            ixiii[(w1, w3, w4, w5)] +=1
            iixii[(w1, w2, w4, w5)] +=1


    return cls(ixxxx, ii,iii, ixi, ixxi, iixi, ixii,iiii, iiiii, ixxxi, iixxi, ixixi, ixxii, iiixi, ixiii, iixii)


def score_ngram(self, score_fn, w1, w2, w3, w4, w5):
    n_all = self.word_fd.N()
    n_iiiii = self.ngram_fd[(w1, w2, w3, w4, w5)]
    if not n_iiiii:
        return


    n_iiixi = self.iiixi[(w1, w2, w3, w5)]
    n_ixiii = self.ixiii[(w1, w3, w4, w5)]
    n_iixii = self.iixii[(w1, w2, w4, w5)]
    n_iiiix = self.iiii[(w1,w2,w3,w4)]
    n_xiiii = self.iiii[(w2,w3,w4,w5)]

    n_iixix = self.iixix[(w1, w2, w4)]
    n_iixxi = self.iixxi[(w1, w2, w5)]
    n_ixixi = self.ixixi[(w1, w3, w5)]
    n_ixxii = self.ixxii[(w1, w4, w5)]
    n_xiixi = self.xiixi[(w2, w3, w5)]
    n_xixii = self.xixii[(w2, w4, w5)]
    n_ixiix = self.ixiix[(w1, w3, w4)]
    n_iiixx = self.iii[(w1, w2, w3)]
    n_xiiix = self.iii[(w2, w3, w4)]
    n_xxiii = self.iii[(w3, w4, w5)]




    n_ixixx = self.ixixx[(w1, w3)]
    n_ixxix = self.ixxix[(w1, w4)]
    n_ixxxi = self.ixxxi[(w1, w5)]
    n_xixix = self.xixix[(w2, w4)]
    n_xixxi = self.xixxi[(w2, w5)]
    n_xxixi = self.xxixi[(w2, w5)]
    n_iixxx = self.ii[(w1, w2)]
    n_xiixx = self.ii[(w2, w3)]
    n_xxiix= self.ii[(w3, w4)]
    n_xxxii= self.ii[(w4, w5)]


    n_ixxxx = self.word_fd[w1]
    n_xixxx = self.word_fd[w2]
    n_xxixx = self.word_fd[w3]
    n_xxxix = self.word_fd[w4]
    n_xxxxi = self.word_fd[w5]

    return score_fn(n_iiiii,
                    (n_iiixi, n_ixiii, n_iixii, n_iiiix, n_xiiii), 
                    (n_iixix,n_iixxi, n_ixixi, n_ixxii, n_xiixi, n_xixii,n_ixiix, n_iiixx, n_xiiix, n_xxiii),
                    (n_ixixx, n_ixxix,n_ixxxi, n_xixix, n_xixxi, n_xxixi, n_iixxx, n_xiixx,n_xxiix,n_xxxii),
                    (n_ixxxx,n_xixxx, n_xxixx, n_xxxix, n_xxxxi),
                    n_all)

因此需要更新类以检测 PentagramCollocationFinder

有人可以帮忙吗？

Answer 1

构建模式似乎有些问题，所以这里有一些代码可以构建所有合法的 i-patterns，以及要使用的 n-patterns。

import collections

def make_ngram_ipatterns(n):
    """Make all needed patterns used by *gramCollocationFinder up to n words"""

    i_patterns = []

    for i in xrange(1, n+1):
        if i <= 2:
            i_patterns.append('i' * i)

        else:
            for j in xrange(2**(i-2)):
                 bin_str = '{0:0{1}b}'.format(j, i-2)
                 ix_pattern = bin_str.replace('0', 'x').replace('1', 'i')
                 i_patterns.append('i{}i'.format(ix_pattern))

    return i_patterns

def make_ngram_npatterns(n):
    """Make all needed n-patterings used by *gramCollocationFinder up to n words"""
    all_ipatterns = make_ngram_ipatterns(n)

    npatterns = []

    for ipattern in all_ipatterns:
         i_order = sum(c == 'i' for c in ipattern)
         i_length = len(ipattern)
         for j in xrange(n - i_length+1):
             npattern = 'n_{}{}{}'.format('x'* j,
                                           ipattern ,
                                           'x'* (n - i_length - j))

             npatterns.append((i_order, ipattern, npattern))

    return sorted(npatterns)


def main():

    n = 5

    all_ipatterns = make_ngram_ipatterns(n)

    print '\n'.join(make_ngram_ipatterns(n))

    for order, ipattern, npattern in make_ngram_npatterns(n):
         wparams = ', '.join('w{}'.format(i+1)
                                for i, c in enumerate(npattern[2:])
                                if c == 'i'
                            )
         print('order: {1:2}   ipattern: {2:{0}s}   npattern: {3}'
               ' ->  {3} = self.{2}({4})'.format(
                   n, order, ipattern, npattern, wparams))


if __name__ == '__main__':
    main()

n=5 的输出是:

i
ii
ixi
iii
ixxi
ixii
iixi
iiii
ixxxi
ixxii
ixixi
ixiii
iixxi
iixii
iiixi
iiiii
order:  1   ipattern: i       npattern: n_ixxxx ->  n_ixxxx = self.i(w1)
order:  1   ipattern: i       npattern: n_xixxx ->  n_xixxx = self.i(w2)
order:  1   ipattern: i       npattern: n_xxixx ->  n_xxixx = self.i(w3)
order:  1   ipattern: i       npattern: n_xxxix ->  n_xxxix = self.i(w4)
order:  1   ipattern: i       npattern: n_xxxxi ->  n_xxxxi = self.i(w5)
order:  2   ipattern: ii      npattern: n_iixxx ->  n_iixxx = self.ii(w1, w2)
order:  2   ipattern: ii      npattern: n_xiixx ->  n_xiixx = self.ii(w2, w3)
order:  2   ipattern: ii      npattern: n_xxiix ->  n_xxiix = self.ii(w3, w4)
order:  2   ipattern: ii      npattern: n_xxxii ->  n_xxxii = self.ii(w4, w5)
order:  2   ipattern: ixi     npattern: n_ixixx ->  n_ixixx = self.ixi(w1, w3)
order:  2   ipattern: ixi     npattern: n_xixix ->  n_xixix = self.ixi(w2, w4)
order:  2   ipattern: ixi     npattern: n_xxixi ->  n_xxixi = self.ixi(w3, w5)
order:  2   ipattern: ixxi    npattern: n_ixxix ->  n_ixxix = self.ixxi(w1, w4)
order:  2   ipattern: ixxi    npattern: n_xixxi ->  n_xixxi = self.ixxi(w2, w5)
order:  2   ipattern: ixxxi   npattern: n_ixxxi ->  n_ixxxi = self.ixxxi(w1, w5)
order:  3   ipattern: iii     npattern: n_iiixx ->  n_iiixx = self.iii(w1, w2, w3)
order:  3   ipattern: iii     npattern: n_xiiix ->  n_xiiix = self.iii(w2, w3, w4)
order:  3   ipattern: iii     npattern: n_xxiii ->  n_xxiii = self.iii(w3, w4, w5)
order:  3   ipattern: iixi    npattern: n_iixix ->  n_iixix = self.iixi(w1, w2, w4)
order:  3   ipattern: iixi    npattern: n_xiixi ->  n_xiixi = self.iixi(w2, w3, w5)
order:  3   ipattern: iixxi   npattern: n_iixxi ->  n_iixxi = self.iixxi(w1, w2, w5)
order:  3   ipattern: ixii    npattern: n_ixiix ->  n_ixiix = self.ixii(w1, w3, w4)
order:  3   ipattern: ixii    npattern: n_xixii ->  n_xixii = self.ixii(w2, w4, w5)
order:  3   ipattern: ixixi   npattern: n_ixixi ->  n_ixixi = self.ixixi(w1, w3, w5)
order:  3   ipattern: ixxii   npattern: n_ixxii ->  n_ixxii = self.ixxii(w1, w4, w5)
order:  4   ipattern: iiii    npattern: n_iiiix ->  n_iiiix = self.iiii(w1, w2, w3, w4)
order:  4   ipattern: iiii    npattern: n_xiiii ->  n_xiiii = self.iiii(w2, w3, w4, w5)
order:  4   ipattern: iiixi   npattern: n_iiixi ->  n_iiixi = self.iiixi(w1, w2, w3, w5)
order:  4   ipattern: iixii   npattern: n_iixii ->  n_iixii = self.iixii(w1, w2, w4, w5)
order:  4   ipattern: ixiii   npattern: n_ixiii ->  n_ixiii = self.ixiii(w1, w3, w4, w5)
order:  5   ipattern: iiiii   npattern: n_iiiii ->  n_iiiii = self.iiiii(w1, w2, w3, w4, w5)

改变到一个新的维度现在是使用和设置所有 i-patterns 为低阶类，替换 n-patterns，并将所有相同顺序的 n-patterns 整理到 score_fn() 集。

编辑:使用适当的 w#

完成了 n 模式的设置