apache-spark - 如何在 Spark 日志中隐藏 key 密码？

Question

当我运行 spark 作业时，我可以看到 SSL key 密码，keystorepassword 在事件日志中以纯文本形式显示。你能帮我看看如何从日志中隐藏这些密码吗？

当我看下面的https://issues.apache.org/jira/browse/SPARK-16796似乎他们修复了它以将其隐藏在网络用户界面中。但不确定我能否在日志中修复它

非常感谢您的帮助!!

"{"Event":"SparkListenerLogStart","Spark Version":"2.1.1"} {"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"xx.xxx.xx.xxx","Port":43556},"Maximum Memory":434031820,"Timestamp":1512750709305} {"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.141-1.b16.32.amzn1.x86_64/jre","Java Version":"1.8.0_141 (Oracle Corporation)","Scala Version":"version 2.11.8"},"Spark Properties":{"spark.sql.warehouse.dir":"hdfs:///user/spark/warehouse","spark.yarn.dist.files":"file:/etc/spark/conf/hive-site.xml","spark.executor.extraJavaOptions":"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=70 -XX:MaxHeapFreeRatio=70 -XX:+CMSClassUnloadingEnabled -XX:OnOutOfMemoryError='kill -9 %p'","spark.driver.host":"xx.xxx.xx.xxx","spark.serializer.objectStreamReset":"100","spark.history.fs.logDirectory":"hdfs:///var/log/spark/apps","spark.eventLog.enabled":"true","spark.driver.port":"44832","spark.shuffle.service.enabled":"true","spark.rdd.compress":"True","spark.driver.extraLibraryPath":"/usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native","spark.ssl.keyStore":"/usr/share/aws/emr/security/conf/keystore.jks","spark.executorEnv.PYTHONPATH":"{{PWD}}/pyspark.zip{{PWD}}/py4j-0.10.4-src.zip","spark.ssl.enabled":"true","spark.yarn.historyServer.address":"ip-xx-xxx-xx-xxx.xxx.com:18080","spark.ssl.trustStore":"/usr/share/aws/emr/security/conf/truststore.jks","spark.app.name":"claim_line_fact_main","spark.scheduler.mode":"FIFO","spark.network.sasl.serverAlwaysEncrypt":"true","spark.ssl.keyPassword":"xxxxxx","spark.ssl.keyStorePassword":"xxxxxx","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=70 -XX:MaxHeapFreeRatio=70 -XX:+CMSClassUnloadingEnabled -XX:OnOutOfMemoryError='kill -9 %p'","spark.submit.deployMode":"client","spark.master":"yarn","spark.authenticate.enableSaslEncryption":"true","spark.authenticate":"true","spark.ui.filters":"org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter","spark.executor.extraLibraryPath":"/usr/lib/hadoop/lib/native:/usr/lib/hadoop-lzo/lib/native","spark.sql.hive.metastore.sharedPrefixes":"com.amazonaws.services.dynamodbv2","spark.executor.memory":"5120M","spark.driver.extraClassPath":"/usr/lib/hadoop-lzo/lib/:/usr/lib/hadoop/hadoop-aws.jar:/usr/share/aws/aws-java-sdk/:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/:/usr/share/aws/emr/emrfs/auxlib/:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/","spark.eventLog.dir":"hdfs:///var/log/spark/apps","spark.ssl.protocol":"TLSv1.2","spark.dynamicAllocation.enabled":"true","spark.executor.extraClassPath":"/usr/lib/hadoop-lzo/lib/:/usr/lib/hadoop/hadoop-aws.jar:/usr/share/aws/aws-java-sdk/:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/:/usr/share/aws/emr/emrfs/auxlib/:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/","spark.executor.cores":"4","spark.history.ui.port":"18080","spark.driver.appUIAddress":"http://","spark.yarn.isPython":"true","spark.ssl.trustStorePassword":"xxxxxx","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_HOSTS":"ip-xx-xxx-xx-xxx.xxx.com","spark.ssl.enabledAlgorithms":"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_CBC_SHA","spark.org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter.param.PROXY_URI_BASES":"

Answer 1

log4j.properties 文件可以控制只记录 INFO、WARN、ERROR 的消息。如果你想隐藏密码或任何通过 -D 传递给 spark 的 secret 参数，可以通过删除 spark-submit 中的参数 --verbose 来隐藏。这对我有用