python - 从 RDD 到联合 DataFrame PySpark

Question

我正在寻找一种按键组合两个 DataFrame 的方法。我首先从 rdds 创建数据帧:

给定:

x = sc.parallelize([('_guid_YWKnKkcrg_Ej0icb07bhd-mXPjw-FcPi764RRhVrOxE=', 'FR', '75001'),
                ('_guid_XblBPCaB8qx9SK3D4HuAZwO-1cuBPc1GgfgNUC2PYm4=', 'TN', '8160'),
               ]
              )

y = sc.parallelize([('_guid_oX6Lu2xxHtA_T93sK6igyW5RaHH1tAsWcF0RpNx_kUQ=', 'JmJCFu3N'),
                ('_guid_hG88Yt5EUsqT8a06Cy380ga3XHPwaFylNyuvvqDslCw=', 'KNPQLQth'),
                ('_guid_YWKnKkcrg_Ej0icb07bhd-mXPjw-FcPi764RRhVrOxE=', 'KlGZj08d'),
               ]
              )

我的代码:

df_x = x.toDF(['id', 'countrycode', 'postalcode'])
df_y = y.toDF(['id_gigya', 'krux'])

df = df_x.join(df_y, df_x.id == df_y.id_gigya, 'fullouter')

给出:

[Row(id=u'_guid_XblBPCaB8qx9SK3D4HuAZwO-1cuBPc1GgfgNUC2PYm4=', countrycode=u'TN', postalcode=u'8160', id_gigya=None, krux=None),
 Row(id=None, countrycode=None, postalcode=None, id_gigya=u'_guid_oX6Lu2xxHtA_T93sK6igyW5RaHH1tAsWcF0RpNx_kUQ=', krux=u'JmJCFu3N'),
 Row(id=None, countrycode=None, postalcode=None, id_gigya=u'_guid_hG88Yt5EUsqT8a06Cy380ga3XHPwaFylNyuvvqDslCw=', krux=u'KNPQLQth'),
 Row(id=u'_guid_YWKnKkcrg_Ej0icb07bhd-mXPjw-FcPi764RRhVrOxE=', countrycode=u'FR', postalcode=u'75001', id_gigya=u'_guid_YWKnKkcrg_Ej0icb07bhd-mXPjw-FcPi764RRhVrOxE=', krux=u'KlGZj08d')]

它很完美，但我想要保留一个唯一的 id，“id_gigya”或“id”，因为它是相同的 id!

与:

df_x.join(df_y, df_x.id == df_y.id_gigya, 'fullouter').drop(df_y.id_gigya).collect()

Or

df_x.join(df_y, df_x.id == df_y.id_gigya, 'fullouter').drop(df_x.id).collect()

我得到了这个:

[Row(id=u'_guid_XblBPCaB8qx9SK3D4HuAZwO-1cuBPc1GgfgNUC2PYm4=', countrycode=u'TN', postalcode=u'8160', krux=None),
 Row(id=None, countrycode=None, postalcode=None, krux=u'JmJCFu3N'),
 Row(id=None, countrycode=None, postalcode=None, krux=u'KNPQLQth'),
 Row(id=u'_guid_YWKnKkcrg_Ej0icb07bhd-mXPjw-FcPi764RRhVrOxE=', countrycode=u'FR', postalcode=u'75001', krux=u'KlGZj08d')]

[Row(countrycode=u'TN', postalcode=u'8160', id_gigya=None, krux=None),
 Row(countrycode=None, postalcode=None, id_gigya=u'_guid_oX6Lu2xxHtA_T93sK6igyW5RaHH1tAsWcF0RpNx_kUQ=', krux=u'JmJCFu3N'),
 Row(countrycode=None, postalcode=None, id_gigya=u'_guid_hG88Yt5EUsqT8a06Cy380ga3XHPwaFylNyuvvqDslCw=', krux=u'KNPQLQth'),
 Row(countrycode=u'FR', postalcode=u'75001', id_gigya=u'_guid_YWKnKkcrg_Ej0icb07bhd-mXPjw-FcPi764RRhVrOxE=', krux=u'KlGZj08d')]

无论如何，我的目标是按行拥有一个 id ..想法？谢谢!

Answer 1

一旦您连接了数据集，您就可以运行另一个 select 来输出特定列，然后转换为 rdd，将其映射以仅获取非空 ID:

df.select('id','id_gigya','countrycode','postalcode')\
  .rdd\
  .map(lambda x: Row(id=(x.id if x.id_gigya == None else x.id_gigya), postalcode=x.postalcode, countrycode=x.countrycode))\
  .collect()

输出:

[
  Row(countrycode=u'TN', id=u'_guid_XblBPCaB8qx9SK3D4HuAZwO-1cuBPc1GgfgNUC2PYm4=', postalcode=u'8160'),
  Row(countrycode=None, id=u'_guid_hG88Yt5EUsqT8a06Cy380ga3XHPwaFylNyuvvqDslCw=', postalcode=None), 
  Row(countrycode=u'FR', id=u'_guid_YWKnKkcrg_Ej0icb07bhd-mXPjw-FcPi764RRhVrOxE=', postalcode=u'75001'),
  Row(countrycode=None, id=u'_guid_oX6Lu2xxHtA_T93sK6igyW5RaHH1tAsWcF0RpNx_kUQ=', postalcode=None)
]