python - 为什么带有 Scikit-learn 的 Sagemaker 无法工作，并给出 ParamValidationError？

Question

我正在尝试在 sagemaker 中构建 sklearn 模型，如下所示:

import os
import boto3
import pandas as pd
import numpy as np
from sklearn import datasets

import sagemaker 

# Get IRIS dataset and create CSV file
iris = datasets.load_iris()
joined_iris = np.insert(iris.data, 0, iris.target, axis=1)

np.savetxt('./data/iris.csv', joined_iris, delimiter=',', fmt='%1.1f, %1.3f, %1.3f, %1.3f, %1.3f')

# Boto session init
session = boto3.Session(
    aws_access_key_id=ACCESS_KEY,
    aws_secret_access_key=SECRET_KEY,
    region_name="us-east-2")

my_region = session.region_name

# Upload data to S3 bucket - generic bucket is used here
sg_session = sagemaker.Session(session)
train_input = sg_session.upload_data("data", key_prefix="{}/{}".format("sklearn-iris", "data"))

from sagemaker.sklearn.estimator import SKLearn

role = "arn:aws:iam::XYZXYZ:role/service-role/XYZXYZXYZ"  # this is masked

# Define the model
model = SKLearn(
    entry_point="train.py",
    train_instance_type="ml.m4.xlarge",
    role=role,
    sagemaker_session=sg_session)

# Train the model
model.fit({'train': train_input})

火车.py:

import os
import argparse
import pandas as pd

from sklearn import tree
from sklearn.externals import joblib


if __name__ == '__main__':

    train_dir = os.environ['SM_CHANNEL_TRAIN']
    model_dir = os.environ['SM_MODEL_DIR']
    output_data_dir = os.environ['SM_OUTPUT_DATA_DIR']

    # Take the set of files and read them all into a single pandas dataframe
    input_files = [os.path.join(train_dir, file) for file in os.listdir(train_dir)]
    if len(input_files) == 0:
        raise ValueError(('There are no files in {}.\n' +
                          'This usually indicates that the channel ({}) was incorrectly specified,\n' +
                          'the data specification in S3 was incorrectly specified or the role specified\n' +
                          'does not have permission to access the data.').format(train_dir, "train"))

    raw_data = [pd.read_csv(file, header=None, engine="python") for file in input_files]
    train_data = pd.concat(raw_data)

    # labels are in the first column
    train_y = train_data.ix[:,0]
    train_X = train_data.ix[:,1:]

    # Here we support a single hyperparameter, 'max_leaf_nodes'. Note that you can add as many
    # as your training my require in the ArgumentParser above.
    max_leaf_nodes = 10

    # Now use scikit-learn's decision tree classifier to train the model.
    clf = tree.DecisionTreeClassifier(max_leaf_nodes=max_leaf_nodes)
    clf = clf.fit(train_X, train_y)

    # Print the coefficients of the trained classifier, and save the coefficients
    joblib.dump(clf, os.path.join(model_dir, "model.joblib"))


def model_fn(model_dir):
    """Deserialized and return fitted model

    Note that this should have the same name as the serialized model in the main method
    """
    clf = joblib.load(os.path.join(model_dir, "model.joblib"))
    return clf

此代码几乎与 https://github.com/awslabs/amazon-sagemaker-examples/blob/master/sagemaker-python-sdk/scikit_learn_iris/Scikit-learn%20Estimator%20Example%20With%20Batch%20Transform.ipynb 中描述的代码类似。 .

但是在 sklearn.fit({'train': train_input}) 中，我收到以下错误:

ParamValidationError: Parameter validation failed: Unknown parameter in input: "DebugHookConfig", must be one of: TrainingJobName, HyperParameters, AlgorithmSpecification, RoleArn, InputDataConfig, OutputDataConfig, ResourceConfig, VpcConfig, StoppingCondition, Tags, EnableNetworkIsolation, EnableInterContainerTrafficEncryption, EnableManagedSpotTraining, CheckpointConfig

这是关于什么的？以及如何解决这个问题？我在任何文档中都找不到任何提及此错误的信息，甚至还没有得到 AWS 支持团队的回复。

Answer 1

我认为这里的问题是您的 boto3 版本可能已经过时。您可以尝试更新吗？

pip install boto3 --upgrade