- c - 在位数组中找到第一个零
- linux - Unix 显示有关匹配两种模式之一的文件的信息
- 正则表达式替换多个文件
- linux - 隐藏来自 xtrace 的命令
在 scikit learn 中使用 partial_fit 函数进行训练时,我在程序未终止的情况下收到以下错误,这怎么可能,即使经过训练的模型表现正确并提供正确的输出,这又是如何发生的?这有什么值得担心的吗?
/usr/lib/python2.7/dist-packages/sklearn/naive_bayes.py:207: RuntimeWarning: divide by zero encountered in log
self.class_log_prior_ = (np.log(self.class_count_)
我正在使用以下修改后的训练函数,因为我必须维护一个恒定的标签\类列表,因为 partial_fit 不允许在后续运行中添加新类\标签,每批训练数据中的先验类是相同的:
class MySklearnClassifier(SklearnClassifier):
def train(self, labeled_featuresets,classes=None, partial=True):
"""
Train (fit) the scikit-learn estimator.
:param labeled_featuresets: A list of ``(featureset, label)``
where each ``featureset`` is a dict mapping strings to either
numbers, booleans or strings.
"""
X, y = list(compat.izip(*labeled_featuresets))
X = self._vectorizer.fit_transform(X)
y = self._encoder.fit_transform(y)
if partial:
classes=self._encoder.fit_transform(list(set(classes)))
self._clf.partial_fit(X, y, classes=list(set(classes)))
else:
self._clf.fit(X, y)
return self
同样在第二次调用 partial_fit 时,它会抛出类 count=2000 的以下错误,调用 model = self.train(featureset, classes=labels,partial=partial) 时训练样本为 3592:
self._clf.partial_fit(X, y, classes=list(set(classes)))
File "/usr/lib/python2.7/dist-packages/sklearn/naive_bayes.py", line 277, in partial_fit
self._count(X, Y)
File "/usr/lib/python2.7/dist-packages/sklearn/naive_bayes.py", line 443, in _count
self.feature_count_ += safe_sparse_dot(Y.T, X)
ValueError: operands could not be broadcast together with shapes (2000,11430) (2000,10728) (2000,11430)
根据抛出的错误,我哪里出错了?这是否意味着我正在推送不正确的尺寸数据?我试过跟随,我现在打电话:
X = self._vectorizer.transform(X)
y = self._encoder.transform(y)
每次调用部分拟合。早些时候,我对每个 partialfit 调用都使用了 fittransform。这是正确的吗
class MySklearnClassifier(SklearnClassifier):
def train(self, labeled_featuresets, classes=None, partial=False):
"""
Train (fit) the scikit-learn estimator.
:param labeled_featuresets: A list of ``(featureset, label)``
where each ``featureset`` is a dict mapping strings to either
numbers, booleans or strings.
"""
X, y = list(compat.izip(*labeled_featuresets))
if partial:
classes = self._encoder.fit_transform(np.unique(classes))
X = self._vectorizer.transform(X)
y = self._encoder.transform(y)
self._clf.partial_fit(X, y, classes=list(set(classes)))
else:
X = self._vectorizer.fit_transform(X)
y = self._encoder.fit_transform(y)
self._clf.fit(X, y)
return self._clf
经过多次尝试,我能够让以下代码正常工作,通过考虑第一次调用,但我假设分类器腌制文件的大小会在每次迭代后增加,但我为每批获得相同大小的 pkl 文件这是不可能的:
class MySklearnClassifier(SklearnClassifier):
def train(self, labeled_featuresets, classes=None, partial=False,firstcall=True):
"""
Train (fit) the scikit-learn estimator.
:param labeled_featuresets: A list of ``(featureset, label)``
where each ``featureset`` is a dict mapping strings to either
numbers, booleans or strings.
"""
X, y = list(compat.izip(*labeled_featuresets))
if partial:
if firstcall:
classes = self._encoder.fit_transform(np.unique(classes))
X = self._vectorizer.fit_transform(X)
y = self._encoder.fit_transform(y)
self._clf.partial_fit(X, y, classes=classes)
else:
X = self._vectorizer.transform(X)
y = self._encoder.fit_transform(y)
self._clf.partial_fit(X, y)
else:
X = self._vectorizer.fit_transform(X)
y = self._encoder.fit_transform(y)
self._clf.fit(X, y)
return self
完整代码如下:
class postagger(ClassifierBasedTagger):
"""
A classifier based postagger.
"""
#MySklearnClassifier()
def __init__(self, feature_detector=None, train=None,estimator=None,
classifierinstance=None, backoff=None,
cutoff_prob=None, verbose=True):
if backoff is None:
self._taggers = [self]
else:
self._taggers = [self] + backoff._taggers
if estimator:
classifier = MySklearnClassifier(estimator=estimator)
#MySklearnClassifier.__init__(self, classifier)
elif classifierinstance:
classifier=classifierinstance
if feature_detector is not None:
self._feature_detector = feature_detector
# The feature detector function, used to generate a featureset
# or each token: feature_detector(tokens, index, history) -> featureset
self._cutoff_prob = cutoff_prob
"""Cutoff probability for tagging -- if the probability of the
most likely tag is less than this, then use backoff."""
self._classifier = classifier
"""The classifier used to choose a tag for each token."""
# if train and picklename:
# self._train(classifier_builder, picklename,tagged_corpus=train, ONLYERRORS=False,verbose=True,onlyfeatures=True ,LOADCONSTRUCTED=None)
def legacy_getfeatures(self, tagged_corpus=None, ONLYERRORS=False, existingfeaturesetfile=None, verbose=True,
labels=artlabels):
featureset = []
labels=artlabels
if not existingfeaturesetfile and tagged_corpus:
if ONLYERRORS:
classifier_corpus = open(tagged_corpus + '-ONLYERRORS.richfeature', 'w')
else:
classifier_corpus = open(tagged_corpus + '.richfeature', 'w')
if verbose:
print('Constructing featureset for training corpus for classifier.')
nlp = English()
#df=pandas.DataFrame()
store = HDFStore('featurestore.h5')
for sentence in sPickle.s_load(open(tagged_corpus,'r')):
untagged_words, tags, senindex = zip(*sentence)
doc = nlp(u' '.join(untagged_words))
# untagged_sentence, tags , rest = unpack_three(*zip(*sentence))
for index in range(len(sentence)):
if ONLYERRORS:
if tags[index] == '<!SAME!>' and random.random() < 0.05:
featureset = self.new_feature_detector(doc, index)
sPickle.s_dump_elt((featureset, tags[index]), classifier_corpus)
featureset['label']=tags[index]
featureset['senindex']=str(senindex[0])
featureset['wordindex']=index
df=pandas.DataFrame([featureset])
store.append('df',df,index=False,min_itemsize = 150)
# classifier_corpus.append((featureset, tags[index]))
elif tags[index] in labels:
featureset = self.new_feature_detector(doc, index)
sPickle.s_dump_elt((featureset, tags[index]), classifier_corpus)
featureset['label']=tags[index]
featureset['senindex']=str(senindex[0])
featureset['wordindex']=index
df=pandas.DataFrame([featureset])
store.append('df',df,index=False,min_itemsize = 150)
# classifier_corpus.append((featureset, tags[index]))
# else:
# for element in sPickle.s_load(open(existingfeaturesetfile, 'w')):
# featureset.append( element)
return tagged_corpus + '.richfeature'
def _train(self, featuresetdata, classifier_builder=MultinomialNB(), partial=False, batchsize=500):
"""
Build a new classifier, based on the given training data
*tagged_corpus*.
"""
#labels = set(cPickle.load(open(arguments['-k'], 'r')))
if partial==False:
print('Training classifier FULLMODE')
featureset = []
for element in sPickle.s_load(open(featuresetdata, 'r')):
featureset.append(element)
model = self._classifier.train(featureset, classes=artlabels, partial=False,firstcall=True)
print('Training complete, dumping')
try:
joblib.dump(model, str(featuresetdata) + '-FULLTRAIN ' + slugify(str(classifier_builder))[:10] +'.mpkl')
print "joblib dumped"
except:
print "joblib error"
cPickle.dump(model, open(str(featuresetdata) + '-FULLTRAIN ' + slugify(str(classifier_builder))[:10] +'.cmpkl', 'w'))
print('dumped')
return
#joblib.dump(self._classifier,str(datetime.datetime.now().hour)+'-naivebayes.pickle',compress=0)
print('Training classifier each batch of {} training points'.format(batchsize))
for i, batchelement in enumerate(batch(sPickle.s_load(open(featuresetdata, 'r')), batchsize)):
featureset = []
for element in batchelement:
featureset.append(element)
# model = super(postagger, self).train (featureset, partial)
# pdb.set_trace()
# featureset = [item for sublist in featureset for item in sublist]
trainsize = len(featureset)
print("submitting {} training points for training\neg last one:".format(trainsize))
for d, l in featureset:
if len(d) != 113:
print d
assert False
print featureset[-1]
# pdb.set_trace()
try:
if i==0:
model = self._classifier.train(featureset, classes=artlabels, partial=True,firstcall=True)
else:
model = self._classifier.train(featureset, classes=artlabels, partial=True,firstcall=False)
except:
type, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
print('Training for batch {} complete, dumping'.format(i))
cPickle.dump(model, open(
str(featuresetdata) + '-' + slugify(str(classifier_builder))[
:10] + 'UPDATED batch-{} of {} points.mpkl'.format(
i, trainsize), 'w'))
print('dumped')
#joblib.dump(self._classifier,str(datetime.datetime.now().hour)+'-naivebayes.pickle',compress=0)
def untag(self,tagged_sentence):
"""
Given a tagged sentence, return an untagged version of that
sentence. I.e., return a list containing the first element
of each tuple in *tagged_sentence*.
>>> from nltk.tag.util import untag
>>> untag([('John', 'NNP'), ('saw', 'VBD'), ('Mary', 'NNP')])
['John', 'saw', 'Mary']
"""
return [w[0] for w in tagged_sentence]
def evaluate(self, gold):
"""
Score the accuracy of the tagger against the gold standard.
Strip the tags from the gold standard text, retag it using
the tagger, then compute the accuracy score.
:type gold: list(list(tuple(str, str)))
:param gold: The list of tagged sentences to score the tagger on.
:rtype: float
"""
gold_tokens=[]
full_gold_tokens=[]
tagged_sents = self.tag_sents(self.untag(sent) for sent in gold)
for sentence in gold:#flatten the list
untagged_sentences, goldtags,type_feature,startpos_feature,sentence_feature,senindex_feature = zip(*sentence)
gold_tokens.extend(zip(untagged_sentences,goldtags))
full_gold_tokens.extend(zip( untagged_sentences, goldtags,type_feature,startpos_feature,sentence_feature,senindex_feature))
test_tokens = sum(tagged_sents, []) #flatten the list
getmismatch(gold_tokens,test_tokens,full_gold_tokens)
return accuracy(gold_tokens, test_tokens)
#
def new_feature_detector(self, tokens, index):
return getfeatures(tokens, index)
def tag_sents(self, sentences):
"""
Apply ``self.tag()`` to each element of *sentences*. I.e.:
return [self.tag(sent) for sent in sentences]
"""
return [self.tag(sent) for sent in sentences]
def tag(self, tokens):
# docs inherited from TaggerI
tags = []
for i in range(len(tokens)):
tags.append(self.tag_one(tokens, i))
return list(zip(tokens, tags))
def tag_one(self, tokens, index):
"""
Determine an appropriate tag for the specified token, and
return that tag. If this tagger is unable to determine a tag
for the specified token, then its backoff tagger is consulted.
:rtype: str
:type tokens: list
:param tokens: The list of words that are being tagged.
:type index: int
:param index: The index of the word whose tag should be
returned.
:type history: list(str)
:param history: A list of the tags for all words before *index*.
"""
tag = None
for tagger in self._taggers:
tag = tagger.choose_tag(tokens, index)
if tag is not None: break
return tag
def choose_tag(self, tokens, index):
# Use our feature detector to get the featureset.
featureset = self.new_feature_detector(tokens, index)
# Use the classifier to pick a tag. If a cutoff probability
# was specified, then check that the tag's probability is
# higher than that cutoff first; otherwise, return None.
if self._cutoff_prob is None:
return self._classifier.prob_classify_many([featureset])
#return self._classifier.classify_many([featureset])
pdist = self._classifier.prob_classify_many([featureset])
tag = pdist.max()
return tag if pdist.prob(tag) >= self._cutoff_prob else None
最佳答案
RuntimeWarning
您收到此警告是因为在 0 上调用了 np.log
:
In [6]: np.log(0)
/home/anaconda/envs/python34/lib/python3.4/site-packages/ipykernel/__main__.py:1: RuntimeWarning: divide by zero encountered in log
if __name__ == '__main__':
Out[6]: -inf
那是因为在你的一个调用中,一些类根本没有被表示(它们的计数为 0)因此 np.log
被调用为 0。你不必担心关于它。
I am using the following modified training function as I have to maintain a constant list of labels\classes as the partial_fit does not allow adding new classes\labels on subsequent runs , the class prior is same in each batch of training data
partial_fit
,您需要从一开始就传递标签/类列表,这是对的。MultinomialNB
等分类器的默认行为是它们先验拟合数据(基本上它们计算频率)。使用 partial_fit
时,它们将增量 执行此计算,以便您获得与使用单个 fit
调用相同的结果。 Also on the second call to partial_fit it throws following error for class count=2000 , and training samples are 3592 on calling model = self.train(featureset, classes=labels,partial=partial)
这里我们需要更多的细节。我很困惑 X
的形状是 (n_samples, n_features)
但在回溯中它看起来是 (2000,11430)
。这意味着 X
有 2000 个样本。
错误确实意味着您输入的尺寸不一致。我建议为每个 partial_fit
调用打印 X.shape
、y.shape
after vectorization。
此外,您不应该在为每个 partial_fit
调用转换 X
的矢量化器上调用 fit
或 fit_transform
:你应该适应它一次,然后只变换 X。这是为了确保你获得变换后的 X 的一致尺寸。
这是您告诉我们您正在使用的代码:
class MySklearnClassifier(SklearnClassifier):
def train(self, labeled_featuresets, classes=None, partial=False):
"""
Train (fit) the scikit-learn estimator.
:param labeled_featuresets: A list of ``(featureset, label)``
where each ``featureset`` is a dict mapping strings to either
numbers, booleans or strings.
"""
X, y = list(compat.izip(*labeled_featuresets))
if partial:
classes = self._encoder.fit_transform(np.unique(classes))
X = self._vectorizer.transform(X)
y = self._encoder.transform(y)
self._clf.partial_fit(X, y, classes=list(set(classes)))
else:
X = self._vectorizer.fit_transform(X)
y = self._encoder.fit_transform(y)
self._clf.fit(X, y)
return self._clf
据我所知,这并没有太大问题,但我们确实需要更多关于您如何在此处使用它的上下文。
吹毛求疵:我觉得如果将 classes
变量作为类属性会更清楚,因为对于每个 partial_fit
调用,此变量都需要相同。
如果您将不同的值传递给构造函数 classes
参数,您可能会在这里做错事。
可以帮助我们帮助您的更多信息:
_vectorizer
、_encoder
使用什么?您最终使用的是什么分类器?关于python - scikit learn中partial_fit遇到的错误,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/32697093/
我已经使用 vue-cli 两个星期了,直到今天一切正常。我在本地建立这个项目。 https://drive.google.com/open?id=0BwGw1zyyKjW7S3RYWXRaX24tQ
您好,我正在尝试使用 python 库 pytesseract 从图像中提取文本。请找到代码: from PIL import Image from pytesseract import image_
我的错误 /usr/bin/ld: errno: TLS definition in /lib/libc.so.6 section .tbss mismatches non-TLS reference
我已经训练了一个模型,我正在尝试使用 predict函数但它返回以下错误。 Error in contrasts<-(*tmp*, value = contr.funs[1 + isOF[nn]])
根据Microsoft DataConnectors的信息我想通过 this ODBC driver 创建一个从 PowerBi 到 PostgreSQL 的连接器使用直接查询。我重用了 Micros
我已经为 SoundManagement 创建了一个包,其中有一个扩展 MediaPlayer 的类。我希望全局控制这个变量。这是我的代码: package soundmanagement; impo
我在Heroku上部署了一个应用程序。我正在使用免费服务。 我经常收到以下错误消息。 PG::Error: ERROR: out of memory 如果刷新浏览器,就可以了。但是随后,它又随机发生
我正在运行 LAMP 服务器,这个 .htaccess 给我一个 500 错误。其作用是过滤关键字并重定向到相应的域名。 Options +FollowSymLinks RewriteEngine
我有两个驱动器 A 和 B。使用 python 脚本,我在“A”驱动器中创建一些文件,并运行 powerscript,该脚本以 1 秒的间隔将驱动器 A 中的所有文件复制到驱动器 B。 我在 powe
下面的函数一直返回这个错误信息。我认为可能是 double_precision 字段类型导致了这种情况,我尝试使用 CAST,但要么不是这样,要么我没有做对...帮助? 这是错误: ERROR: i
这个问题已经有答案了: Syntax error due to using a reserved word as a table or column name in MySQL (1 个回答) 已关闭
我的数据库有这个小问题。 我创建了一个表“articoli”,其中包含商品的品牌、型号和价格。 每篇文章都由一个 id (ID_ARTICOLO)` 定义,它是一个自动递增字段。 好吧,现在当我尝试插
我是新来的。我目前正在 DeVry 在线学习中级 C++ 编程。我们正在使用 C++ Primer Plus 这本书,到目前为止我一直做得很好。我的老师最近向我们扔了一个曲线球。我目前的任务是这样的:
这个问题在这里已经有了答案: What is an undefined reference/unresolved external symbol error and how do I fix it?
我的网站中有一段代码有问题;此错误仅发生在 Internet Explorer 7 中。 我没有在这里发布我所有的 HTML/CSS 标记,而是发布了网站的一个版本 here . 如您所见,我在列中有
如果尝试在 USB 设备上构建 node.js 应用程序时在我的树莓派上使用 npm 时遇到一些问题。 package.json 看起来像这样: { "name" : "node-todo",
在 Python 中,您有 None单例,在某些情况下表现得很奇怪: >>> a = None >>> type(a) >>> isinstance(a,None) Traceback (most
这是我的 build.gradle (Module:app) 文件: apply plugin: 'com.android.application' android { compileSdkV
我是 android 的新手,我的项目刚才编译和运行正常,但在我尝试实现抽屉导航后,它给了我这个错误 FAILURE: Build failed with an exception. What wen
谁能解释一下?我想我正在做一些非常愚蠢的事情,并且急切地等待着启蒙。 我得到这个输出: phpversion() == 7.2.25-1+0~20191128.32+debian8~1.gbp108
我是一名优秀的程序员,十分优秀!