java - 无法打开与 Cassandra 的节俭连接

Question

只想使用 spark-cassandra-connector 从键空间player 访问表playerinfo。

import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;

import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import com.datastax.spark.connector.cql.CassandraConnector;
import static com.datastax.spark.connector.japi.CassandraJavaUtil.*;
import com.datastax.spark.connector.japi.CassandraRow;


public class SparkTest implements Serializable {

private transient SparkConf conf;
private SparkTest(SparkConf conf) {
    this.conf = conf;
}

private void run() {
    JavaSparkContext sc = new JavaSparkContext(conf);
    CassandraConnector connector = CassandraConnector.apply(sc.getConf());

    JavaRDD<String> rdd = javaFunctions(sc).cassandraTable("player", "playerinfo")
            .select("id").map(new Function<CassandraRow, String>() {
                @Override
                public String call(CassandraRow cassandraRow) throws Exception {
                    return cassandraRow.toString();
                }
            });
    System.out.println("Data with only 'id' column fetched: \n" + StringUtils.join(rdd.toArray(), "\n"));
    System.out.println("finished!");
}

public static void main(String[] args) {
    SparkConf conf = new SparkConf();
    conf.setAppName("Java API demo");
    conf.setMaster("local[1]");
    System.out.println("---------------------------------");
    conf.set("spark.cassandra.connection.host", "10.6.1.244");


    SparkTest app = new SparkTest(conf);
    app.run();
} 
}

这是错误堆栈:

Exception in thread "main" java.io.IOException: Failed to open thrift connection to Cassandra at 10.6.1.244:9160
at com.datastax.spark.connector.cql.CassandraConnector.createThriftClient(CassandraConnector.scala:132)
at com.datastax.spark.connector.cql.CassandraConnector.createThriftClient(CassandraConnector.scala:138)
at com.datastax.spark.connector.cql.CassandraConnector.withCassandraClientDo(CassandraConnector.scala:144)
at com.datastax.spark.connector.rdd.partitioner.CassandraRDDPartitioner.partitions(CassandraRDDPartitioner.scala:158)
at com.datastax.spark.connector.rdd.CassandraRDD.getPartitions(CassandraRDD.scala:294)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
at org.apache.spark.rdd.MappedRDD.getPartitions(MappedRDD.scala:28)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1135)
at org.apache.spark.rdd.RDD.collect(RDD.scala:774)
at org.apache.spark.api.java.JavaRDDLike$class.collect(JavaRDDLike.scala:305)
at org.apache.spark.api.java.JavaRDD.collect(JavaRDD.scala:32)
at org.apache.spark.api.java.JavaRDDLike$class.toArray(JavaRDDLike.scala:325)
at org.apache.spark.api.java.JavaRDD.toArray(JavaRDD.scala:32)
at sparkDemo.SparkTest.run(SparkTest.java:36)
at sparkDemo.SparkTest.main(SparkTest.java:49)
Caused by: java.lang.NoSuchMethodError: org.apache.cassandra.thrift.TFramedTransportFactory.openTransport(Ljava/lang/String;I)Lorg/apache/thrift/transport/TTransport;
at com.datastax.spark.connector.cql.DefaultConnectionFactory$.createThriftClient(CassandraConnectionFactory.scala:47)
at com.datastax.spark.connector.cql.CassandraConnector.createThriftClient(CassandraConnector.scala:127)
... 21 more

为什么会出现这样的错误:

Failed to open thrift connection to Cassandra at 10.6.1.244:9160

谁知道这是怎么回事？这个错误让我发疯......

Answer 1

“10.6.1.244”上的 Cassandra 似乎没有在端口 9160 上监听 Thrift 客户端。您可以通过 telnet 10.6.1.244 9160 确认这一点。如果您看到“连接被拒绝”消息，那么您的 Cassandra 没有在监听端口号。请检查您的 Cassandra 配置。