Python机器学习: Using Multinomial Naive Bayes gives Multiprocessing exception

Question

我正在尝试预测某些文档的标签。每个文档可以有多个标签。这是我编写的示例程序

import pandas as pd
import pickle
import re
from sklearn.cross_validation import train_test_split
from sklearn.metrics.metrics import classification_report, accuracy_score, confusion_matrix
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB as MNB
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV

def Mytrain():
    pipeline = Pipeline([
    ('vect', TfidfVectorizer(stop_words='english',sublinear_tf=True)),
    ('clf', MNB())
    ])

    parameters = {
        'vect__max_df': (0.25, 0.5, 0.6, 0.7, 1.0),
        'vect__ngram_range': ((1, 1), (1, 2), (2,3), (1,3), (1,4), (1,5)),
        'vect__use_idf': (True, False),
        'clf__fit_prior': (True, False)
    }

    traindf = pickle.load(open("train.pkl","rb"))

    X, y = traindf['Data'], traindf['Tags'].as_matrix()

    Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, train_size=0.7)

    gridSearch = GridSearchCV(pipeline, parameters, n_jobs=3, verbose=1, scoring='accuracy')
    gridSearch.fit(Xtrain, ytrain)

    print ('best score: %0.3f' % gridSearch.best_score_)
    print ('best parameters set:')

    res = open("res.txt", 'w')
    res.write ('best parameters set:\n')
    bestParameters = gridSearch.best_estimator_.get_params()
    for paramName in sorted(parameters.keys()):
        print ('\t %s: %r' % (paramName, bestParameters[paramName]))
        res.write('\t %s: %r\n' % (paramName, bestParameters[paramName]))

    pickle.dump(bestParameters,open("bestParams.pkl","wb"))

    predictions = gridSearch.predict(Xtest)
    print ('Accuracy:', accuracy_score(ytest, predictions))
    print ('Confusion Matrix:', confusion_matrix(ytest, predictions))
    print ('Classification Report:', classification_report(ytest, predictions))

请注意，标签可以有多个值。现在我明白了

An unexpected error occurred while tokenizing input
The following traceback may be corrupted or invalid
The error message is: ('EOF in multi-line statement', (40, 0))

Traceback (most recent call last):
  File "X:\abc\predMNB.py", line 128, in <module>
    MNBdrill(fname,topn)
  File "X:\abc\predMNB.py", line 82, in MNBdrill
    gridSearch.fit(Xtrain, ytrain)
  File "X:\pqr\Anaconda2\lib\site-packages\sklearn\grid_search.py", line 732, in fit
    return self._fit(X, y, ParameterGrid(self.param_grid))
  File "X:\pqr\Anaconda2\lib\site-packages\sklearn\grid_search.py", line 505, in _fit
    for parameters in parameter_iterable
  File "X:\pqr\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 666, in __call__
    self.retrieve()
  File "X:\pqr\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 549, in retrieve
    raise exception_type(report)
sklearn.externals.joblib.my_exceptions.JoblibMemoryError: JoblibMemoryError

然后

Multiprocessing exception:
...........................................................................
X:\pqr\Anaconda2\lib\site-packages\sklearn\grid_search.py in fit(self=GridSearchCV(cv=None, error_score='raise',
     ..._func=None,
       scoring='accuracy', verbose=1), X=14151    text for document having t1,t2,t3,t4
Name: Content, dtype: object, y=array([u't1',u't2',u't3',u't4'], dtype=object))
    727         y : array-like, shape = [n_samples] or [n_samples, n_output], optional
    728             Target relative to X for classification or regression;
    729             None for unsupervised learning.
    730 
    731         """
--> 732         return self._fit(X, y, ParameterGrid(self.param_grid))
        self._fit = <bound method GridSearchCV._fit of GridSearchCV(...func=None,
       scoring='accuracy', verbose=1)>
        X = 14151    text for document having t1,t2,t3,t4
Name: Content, dtype: object
        y = array([u't1',u't2',u't3',u't4'], dtype=object)
        self.param_grid = {'clf__fit_prior': (True, False), 'vect__max_df': (0.25, 0.5, 0.6, 0.7, 1.0), 'vect__ngram_range': ((1, 1), (1, 2), (2, 3), (1, 3), (1, 4), (1, 5)), 'vect__use_idf': (True, False)}
    733 
    734 
    735 class RandomizedSearchCV(BaseSearchCV):
    736     """Randomized search on hyper parameters.

...........................................................................
X:\pqr\Anaconda2\lib\site-packages\sklearn\grid_search.py in _fit(self=GridSearchCV(cv=None, error_score='raise',
     ..._func=None,
       scoring='accuracy', verbose=1), X=14151    text for document having t1,t2,t3,t4
Name: Content, dtype: object, y=array([u't1',u't2',u't3',u't4'], dtype=object), parameter_iterable=<sklearn.grid_search.ParameterGrid object>)
    500         )(
    501             delayed(_fit_and_score)(clone(base_estimator), X, y, self.scorer_,
    502                                     train, test, self.verbose, parameters,
    503                                     self.fit_params, return_parameters=True,
    504                                     error_score=self.error_score)
--> 505                 for parameters in parameter_iterable
        parameters = undefined
        parameter_iterable = <sklearn.grid_search.ParameterGrid object>
    506                 for train, test in cv)
    507 
    508         # Out is a list of triplet: score, estimator, n_test_samples
    509         n_fits = len(out)

...........................................................................
X:\pqr\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in __call__(self=Parallel(n_jobs=3), iterable=<itertools.islice object>)
    661             if pre_dispatch == "all" or n_jobs == 1:
    662                 # The iterable was consumed all at once by the above for loop.
    663                 # No need to wait for async callbacks to trigger to
    664                 # consumption.
    665                 self._iterating = False
--> 666             self.retrieve()
        self.retrieve = <bound method Parallel.retrieve of Parallel(n_jobs=3)>
    667             # Make sure that we get a last message telling us we are done
    668             elapsed_time = time.time() - self._start_time
    669             self._print('Done %3i out of %3i | elapsed: %s finished',
    670                         (len(self._output),

    ---------------------------------------------------------------------------
    Sub-process traceback:
    ---------------------------------------------------------------------------
    MemoryError                                     

堆栈跟踪在此之后继续指向具有相同问题的其他方法。如果需要，我可以发布整个内容，但这就是我认为正在发生的事情

通知

scoring='accuracy', verbose=1), X=14151    text for document having t1,t2,t3,t4
Name: Content, dtype: object, y=array([u't1',u't2',u't3',u't4'], dtype=object))

由于有多个标签，这会导致问题吗？

还有，什么意思

多处理异常？

内存错误？

请帮我解决这个问题。

Answer 1

您有多少列车数据？

我最好的选择是，唯一的“真正”错误是MemoryError，即您在尝试训练分类器时使用所有可用的RAM，而所有其他奇怪的错误/回溯都是失败的后果内存分配。

在训练分类器时，您是否检查过您的可用内存？