machine-learning - 数据集在循环变量列表时抛出 KeyError

Question

我有一个变量列表，其值的编码方式使 Pandas 感到厌烦。例如:我有一个名为“Alley”的列，它有一个值列表，其中一个是 NA，它代表“No Alley”。然而，Pandas 将此解释为 NaN。为了解决这个问题，我使用 XX 等任意符号对所有 NaN 值进行编码。这些变量实际上没有空值/缺失值。这些只是 Pandas 误解其值的变量。我将它们收集在一个列表中:

na_data = ['Alley', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'FireplaceQu',
           'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'PoolQC', 'Fence', 'MiscFeature']

并将每个 NaN 读数替换为 XX:

for i in na_data:
    df[i] = df[i].fillna('XX')

这是我遇到的旧错误:

Traceback (most recent call last):
  File "C:\Users\security\AppData\Roaming\Python\Python37\site-packages\pandas\core\indexes\base.py", line 2657, in get_loc
    return self._engine.get_loc(key)
  File "pandas\_libs\index.pyx", line 108, in pandas._libs.index.IndexEngine.get_loc
  File "pandas\_libs\index.pyx", line 129, in pandas._libs.index.IndexEngine.get_loc
  File "pandas\_libs\index_class_helper.pxi", line 91, in pandas._libs.index.Int64Engine._check_type
KeyError: 'Alley'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:/Users/security/Downloads/AP/Boston-Kaggle/Model.py", line 67, in <module>
    print(feature_encoding(train, categorical_columns))
  File "C:/Users/security/Downloads/AP/Boston-Kaggle/Model.py", line 50, in feature_encoding
    df[i] = df[i].fillna('XX')
  File "C:\Users\security\AppData\Roaming\Python\Python37\site-packages\pandas\core\frame.py", line 2927, in __getitem__
    indexer = self.columns.get_loc(key)
  File "C:\Users\security\AppData\Roaming\Python\Python37\site-packages\pandas\core\indexes\base.py", line 2659, in get_loc
    return self._engine.get_loc(self._maybe_cast_indexer(key))
  File "pandas\_libs\index.pyx", line 108, in pandas._libs.index.IndexEngine.get_loc
  File "pandas\_libs\index.pyx", line 129, in pandas._libs.index.IndexEngine.get_loc
  File "pandas\_libs\index_class_helper.pxi", line 91, in pandas._libs.index.Int64Engine._check_type
KeyError: 'Alley'

变量Alley肯定存在于数据集中!我从数据集中复制/粘贴名称只是为了更好地衡量。

这是我的整个代码(已更新):

import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import train_test_split

train = pd.read_csv("https://raw.githubusercontent.com/oo92/Boston-Kaggle/master/train.csv")
test = pd.read_csv("https://raw.githubusercontent.com/oo92/Boston-Kaggle/master/test.csv")

categorical_columns = ['MSSubClass', 'MSZoning', 'LotShape', 'LandContour', 'LotConfig', 'Neighborhood', 'Condition1',
                       'Condition2', 'BldgType', 'HouseStyle', 'RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd',
                       'Foundation', 'Heating', 'Electrical', 'Functional', 'GarageType', 'PavedDrive', 'Fence',
                       'MiscFeature', 'SaleType', 'SaleCondition', 'Street', 'CentralAir', 'Utilities', 'ExterQual',
                       'LandSlope', 'ExterCond', 'HeatingQC', 'KitchenQual']

ranked_columns = ['Utilities', 'LandSlope', 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'BsmtExposure',
                  'BsmtFinType1', 'BsmtFinType2', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageQual', 'GarageCond',
                  'PoolQC', 'OverallQual', 'OverallCond']

numerical_columns = ['LotArea', 'LotFrontage', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2',
                     'BsmtUnfSF','TotalBsmtSF', '1stFlrSF', '2ndFlrSf', 'LowQualFinSF', 'GrLivArea', 'BsmtFullBath',
                     'BsmtHalfBath', 'FullBath', 'HalfBath', 'Bedroom', 'Kitchen', 'TotRmsAbvGrd', 'Fireplaces',
                     'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch',
                     '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal', 'MoSold', 'YrSold']

na_data = ['Alley', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'FireplaceQu',
           'GarageType', 'GarageFinish', 'GarageQual', 'GarageCond', 'PoolQC', 'Fence', 'MiscFeature']

for i in na_data:
    train[i] = train[i].fillna('XX')

#Replaced the NaN values of LotFrontage and MasVnrArea with the mean of their column
train['LotFrontage'] = train['LotFrontage'].fillna(train['LotFrontage'].mean())
train['MasVnrArea'] = train['MasVnrArea'].fillna(train['MasVnrArea'].mean())

concatenated_list = categorical_columns + na_data

# take one-hot encoding
OHE_sdf = pd.get_dummies(train[concatenated_list])

# drop the old categorical column from original df
train.drop(columns = categorical_columns, axis = 1, inplace = True)

# attach one-hot encoded columns to original data frame
train = pd.concat([train, OHE_sdf], axis = 1, ignore_index = False)

x_train, x_test, y_train, y_test = train_test_split(train, train['SalePrice'], test_size = 0.3, random_state = 42)

sel = SelectFromModel(RandomForestClassifier(n_estimators = 100), threshold = 300 * "mean")
sel.fit(x_train, y_train)
sel.get_support()

selected_feat = x_train.columns[sel.get_support()]

print(selected_feat())

这是新错误:

Traceback (most recent call last):
  File "/home/onur/Documents/Boston-Kaggle/Model.py", line 49, in <module>
    sel.fit(x_train, y_train)
  File "/opt/anaconda/envs/lib/python3.7/site-packages/sklearn/feature_selection/from_model.py", line 196, in fit
    self.estimator_.fit(X, y, **fit_params)
  File "/opt/anaconda/envs/lib/python3.7/site-packages/sklearn/ensemble/forest.py", line 249, in fit
    X = check_array(X, accept_sparse="csc", dtype=DTYPE)
  File "/opt/anaconda/envs/lib/python3.7/site-packages/sklearn/utils/validation.py", line 496, in check_array
    array = np.asarray(array, dtype=dtype, order=order)
  File "/opt/anaconda/envs/lib/python3.7/site-packages/numpy/core/_asarray.py", line 85, in asarray
    return array(a, dtype, copy=False, order=order)
ValueError: could not convert string to float: 'XX'

Answer 1

您将数据连接到错误的轴上

df = pd.concat([df, OHE_sdf], axis = 1, ignore_index = True)
# Should be
df = pd.concat([df, OHE_sdf], axis = 0, ignore_index = True)

但是，这将导致另一个错误，因为您对 na_columns 中列出的某些列进行了热编码，例如 Garage_Type 已被编码为多个列，每个列对应一个潜在值，因此它不再存在，因此无法替换其 nan 值。

编辑:

我更新了问题代码的几个部分，以确保它完整运行。

首先我们需要导入我们将要使用的所有库，注意添加了 numpy

import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import train_test_split
import numpy as np

其次我们需要从源获取数据

train = pd.read_csv("https://raw.githubusercontent.com/oo92/Boston-Kaggle/master/train.csv")
test = pd.read_csv("https://raw.githubusercontent.com/oo92/Boston-Kaggle/master/test.csv")

现在我们将从数据集中删除所有 NaN

# Create a series of how many NaN's are in each column
nanCounts = train.isna().sum()
# Find the total number of NaN's and print it (used to check that this bits doing somethin)
nanTotal = train.isna().sum().sum()
print('NaN\'s found: ', nanTotal)

# Create a template list
nanCols = []
# Iterate over the series and if the value is more than 0 (i.e there are some NaN's present)
for i in range(0,len(nanCounts)):
    if nanCounts[i] > 0:
        # If it is append the current column to the list of columns that contain NaN's
        nanCols.append(train.columns[i])

# Iterate through all the columns which are known to have NaN's
for i in nanCols:
    if train[nanCols][i].dtypes == 'float64':
        # If the column is of the data type float64 (a floating point number), replace it with the mean of the column
        train[i] = train[i].fillna(train[i].mean())
    elif train[nanCols][i].dtypes == 'object':
        # If it's of the data type object (a text string) replace it with XX
        train[i] = train[i].fillna('XX')

# Reprint the total number of NaN's
nanTotal = train.isna().sum().sum()
print('NaN\'s after removal: ', nanTotal)

既然数据集中没有 NaN，就可以组装分类数据列表

# Create a template list
categorical = []
# Iterate across all the columns checking if they're of the object datatype and if they are appending them to the categorical list
for i in range(0, len(train.dtypes)):
    if train.dtypes[i] == 'object':
        categorical.append(train.columns[i])
# Print out the list of categorical features
print('Categorical columns are: \n', categorical)

现在代码与原始代码非常相似，只是由于变量更改而发生了一些细微的变化

# take one-hot encoding
OHE_sdf = pd.get_dummies(train[categorical])

# drop the old categorical column from original df
train.drop(columns = categorical, axis = 1, inplace = True)

# attach one-hot encoded columns to original data frame
train = pd.concat([train, OHE_sdf], axis = 1, ignore_index = False)

print('splitting dataset')
x_train, x_test, y_train, y_test = train_test_split(train, train['SalePrice'], test_size = 0.3, random_state = 42)

print('Selecting features')
# Note that here i changed the threshold so that it would actually show some features to use
sel = SelectFromModel(RandomForestClassifier(n_estimators = 100), threshold = '1.25*mean')
sel.fit(x_train, y_train)
# Also just straight up save the boolean array it will be quicker and i prefer the formatting this way
selected = sel.get_support()

# Print the boolean array of selected features
print(selected)
# Print the finally selected features
print(train.columns[selected])

综合起来看起来就像

import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import train_test_split
import numpy as np

train = pd.read_csv("https://raw.githubusercontent.com/oo92/Boston-Kaggle/master/train.csv")
test = pd.read_csv("https://raw.githubusercontent.com/oo92/Boston-Kaggle/master/test.csv")

nanCounts = train.isna().sum()
nanTotal = train.isna().sum().sum()
print('NaN\'s found: ', nanTotal)

nanCols = []
for i in range(0,len(nanCounts)):
    if nanCounts[i] > 0:
        nanCols.append(train.columns[i])

for i in nanCols:
    if train[nanCols][i].dtypes == 'float64':
        train[i] = train[i].fillna(train[i].mean())
    elif train[nanCols][i].dtypes == 'object':
        train[i] = train[i].fillna('XX')

nanTotal = train.isna().sum().sum()

print('NaN\'s after removal: ', nanTotal)

categorical = []
for i in range(0, len(train.dtypes)):
    if train.dtypes[i] == 'object':
        categorical.append(train.columns[i])

print('Categorical columns are: \n', categorical)

# take one-hot encoding
OHE_sdf = pd.get_dummies(train[categorical])

# drop the old categorical column from original df
train.drop(columns = categorical, axis = 1, inplace = True)

# attach one-hot encoded columns to original data frame
train = pd.concat([train, OHE_sdf], axis = 1, ignore_index = False)

print('splitting dataset')
x_train, x_test, y_train, y_test = train_test_split(train, train['SalePrice'], test_size = 0.3, random_state = 42)

print('Selecting features')
sel = SelectFromModel(RandomForestClassifier(n_estimators = 100), threshold = '1.25*mean')
sel.fit(x_train, y_train)
selected = sel.get_support()

print(selected)
print(train.columns[selected])