java - 运行 Spark fpgrowth 示例代码时出现错误

Question

import java.util.Arrays;
import java.util.List;

import org.apache.spark.SparkConf;
import org.apache.spark.ml.fpm.FPGrowth;
import org.apache.spark.ml.fpm.FPGrowthModel;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.types.*;
public class BasketAnalysis {
    public static void main(String[] args) {
        SparkConf sparkConf = new SparkConf().setAppName("JavaFPGrowthExample")
                .setMaster("local[*]");
        System.setProperty("hadoop.home.dir", "D:\\BigData\\hadoop-common");
        SparkSession spark = SparkSession
                .builder()
                .config(sparkConf)
                .getOrCreate();
        // $example on$
        List<Row> data = Arrays.asList(
                RowFactory.create(Arrays.asList("1 2 5".split(" "))),
                RowFactory.create(Arrays.asList("1 2 3 5".split(" "))),
                RowFactory.create(Arrays.asList("1 2".split(" ")))
        );
        StructType schema = new StructType(new StructField[]{ new StructField(
                "items", new ArrayType(DataTypes.StringType, true), false, Metadata.empty())
        });
        Dataset<Row> itemsDF = spark.createDataFrame(data, schema);
        itemsDF.show();
        FPGrowthModel model = new FPGrowth()
                .setItemsCol("items")
                .setMinSupport(0.5)
                .setMinConfidence(0.6)
                .fit(itemsDF);

        // Display frequent itemsets.
        model.freqItemsets().show();

        // Display generated association rules.
        model.associationRules().show();

        // transform examines the input items against all the association rules and summarize the
        // consequents as prediction
        model.transform(itemsDF).show();
        // $example off$

        spark.stop();
    }
}

这是spark源代码中的示例代码，但是当我运行它时，出现以下错误:

Exception in thread "main" java.lang.AbstractMethodError
at org.apache.spark.internal.Logging$class.initializeLogIfNecessary(Logging.scala:99)
at org.apache.spark.ml.PipelineStage.initializeLogIfNecessary(Pipeline.scala:42)
at org.apache.spark.internal.Logging$class.log(Logging.scala:46)
at org.apache.spark.ml.PipelineStage.log(Pipeline.scala:42)
at org.apache.spark.internal.Logging$class.logDebug(Logging.scala:58)
at org.apache.spark.ml.PipelineStage.logDebug(Pipeline.scala:42)
at org.apache.spark.ml.PipelineStage.transformSchema(Pipeline.scala:72)
at org.apache.spark.ml.fpm.FPGrowth.fit(FPGrowth.scala:156)
at com.fudan.BasketAnalysis.main(BasketAnalysis.java:38)

如有任何帮助，我们将不胜感激。

Answer 1

spark 2.4.0 和 java8 上没有问题，你能显示一下你的 pom.xml 吗？