Python:计算 Pandas Dataframe 中列表的 PMF

Question

我在 pandas 数据框中有以下数据集。我希望提取每个 user_id 的 trajectory 中每个项目的计数，计算/返回每个项目的概率质量函数 (pmf)在列表中，并通过 pmf 返回前 3 个(或任意数量)的项目。所有这些都应该返回到同一个数据帧。

代码:

这是我用来原型(prototype)化我想要完成的任务的代码:

import math
from collections import Counter

cluster = trajs['cluster_label'][0]
#print(user)

counter = Counter(cluster[0])
print(counter)
print("Top 3 clusters: ", counter.most_common(3))

cluster_norm = sum(counter.values(), 0.0)
for key in counter:
    counter[key] /= user_norm
print("PMF:",counter.most_common(5))

计数器({6: 907, 12: 728, 7: 716, 4: 638, 0: 594, 3: 551, 5: 352, 8: 335, 11: 236, 14: 207, 2 : 206、1: 195、16: 190、13: 150、9: 115、10: 78、15: 55、17: 52})

前 3 个集群:[(6, 907)、(12, 728)、(7, 716)]

PMF: [(6, 0.14385408406026962), (12, 0.1154639175257732), (7, 0.11356066613798572), (4, 0.10118953211736717), (0, 0.094210 94369547978)]

我已经能够使用上面的代码手动完成一些条目，但是在迭代整个数据帧(约 14k 行)并将结果格式化为新的数据帧时遇到了一些麻烦。

数据:

user_id,trajectory
431997632,[[28, 215, 278, 213, 432, 238, 122, 188, 401, 401, 289, 264, 468, 401, 537, 401, 34, 191, 469, 471, 609, 506, 254, 144, 451]]
4725565346,[[259, 585, 585, 513, 432, 662, 527, 527, 527, 527, 513, 513, 662, 527, 527, 662, 527, 662, 662, 662, 662, 513]]
1331881832,[[215, 432, 432, 432, 432, 432, 432, 432, 432, 432, 432, 432, 432, 432, 432, 205, 432, 205, 432]]
58637506,[[432, 156, 109, 401, 401, 327, 205, 372, 536, 206, 633, 538, 384, 572, 401, 125]]
1396149025,[[537, 372, 372, 421, 440, 100, 645, 55, 401, 537, 384, 317, 317, 351, 459, 109]]
26412887,[[344, 335, 335, 286, 314, 335, 330, 283, 286, 307, 314, 432, 335, 335, 335]]
3259595431,[[400, 339, 401, 400, 28, 307, 327, 327, 537, 36, 472, 472, 522]]
45429271,[[473, 360, 314, 314, 314, 314, 314, 330, 330, 360, 330, 330]]
22536391,[[609, 407, 384, 470, 485, 415, 384, 384, 466, 403, 388, 388, 388, 432, 415, 408, 314, 513, 311, 464, 473, 473, 467]]
8.13E+17,[[421, 384, 401, 230, 330, 609, 401, 424, 264, 265, 384, 661, 445, 215, 257, 140, 601, 213, 265, 79, 378]]
7.63E+17,[[238, 238, 238, 457, 267, 474, 338, 401, 512, 401, 486, 278, 384, 133, 304, 537, 407, 304, 384]]
98221214,[[432, 432, 88, 432, 384, 215, 259, 384, 522, 259, 384, 432, 384, 384, 384, 384, 384, 384, 384]]
7.20E+17,[[465, 329, 465, 329, 432, 432, 432, 432, 432, 18, 465, 432, 432, 269, 465, 465, 288, 288, 152]]
323346248,[[407, 401, 603, 641, 521, 327, 0, 432, 262, 453, 628, 289, 634, 125, 63, 385, 395, 432, 327]]
4036152552,[[327, 632, 632, 168, 28, 168, 632, 632, 632, 168, 106, 28, 168, 106, 168, 168, 168, 168, 28]]
17281102,[[225, 225, 225, 225, 225, 225, 225, 225, 384, 384, 628, 628, 628, 628, 628, 628, 708, 708]]
24685146,[[396, 330, 330, 330, 396, 330, 330, 330, 330, 330, 330, 330, 330, 330, 330, 264, 264, 330]]
24822125,[[401, 354, 314, 360, 432, 360, 432, 360, 689, 689, 314, 314, 689, 689, 300, 432, 300, 300]]
28477232,[[432, 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, 385]]
136675517,[[410, 404, 466, 527, 473, 403, 410, 503, 503, 485, 467, 463, 458, 384, 384, 384, 384, 384]]
228534984,[[432, 213, 432, 213, 432, 432, 432, 432, 205, 432, 432, 432, 432, 552, 432, 432, 432, 432]]
237564656,[[327, 254, 213, 254, 254, 254, 432, 213, 213, 213, 254, 167, 254, 228, 240, 509, 254, 213]]
423924903,[[267, 432, 609, 342, 432, 432, 199, 122, 150, 372, 265, 432, 194, 456, 401, 401, 432, 401]]
496539092,[[609, 597, 597, 597, 597, 432, 597, 597, 597, 634, 634, 609, 597, 597, 597, 597, 634, 311]]
18193826,[[299, 299, 299, 427, 299, 28, 595, 401, 401, 432, 299, 299, 197, 401]]

Answer 1

不是最有效的方法，但相当简单:

df['counter'] = df.trajectory.apply(lambda x: Counter(x[0]))
df['top3'] = df.counter.apply(lambda x: x.most_common(3))
def pfm(cntr):
    s = sum(cntr.values())
    for key in cntr:
        cntr[key] /= s
    return cntr.most_common(5)
df['pfm_top5'] = df.counter.apply(pfm)
df = df.drop('counter', 1)

结果:

       user_id                                         trajectory                             top3                                           pfm_top5
0    431997632  [[28, 215, 278, 213, 432, 238, 122, 188, 401, ...   [(401, 4), (451, 1), (609, 1)]  [(401, 0.15999999999999995), (451, 0.039999999...
1   4725565346  [[259, 585, 585, 513, 432, 662, 527, 527, 527,...   [(662, 7), (527, 7), (513, 4)]  [(662, 0.3181818181818182), (527, 0.3181818181...
2   1331881832  [[215, 432, 432, 432, 432, 432, 432, 432, 432,...  [(432, 16), (205, 2), (215, 1)]  [(432, 0.8421052631578947), (205, 0.1052631578...
3     58637506  [[432, 156, 109, 401, 401, 327, 205, 372, 536,...   [(401, 3), (384, 1), (327, 1)]  [(401, 0.1875), (384, 0.0625), (327, 0.0625), ...
4   1396149025  [[537, 372, 372, 421, 440, 100, 645, 55, 401, ...   [(372, 2), (537, 2), (317, 2)]  [(372, 0.125), (537, 0.125), (317, 0.125), (38...
5     26412887  [[344, 335, 335, 286, 314, 335, 330, 283, 286,...   [(335, 6), (314, 2), (286, 2)]  [(335, 0.4), (314, 0.13333333333333333), (286,...
6   3259595431  [[400, 339, 401, 400, 28, 307, 327, 327, 537, ...   [(400, 2), (327, 2), (472, 2)]  [(400, 0.15384615384615385), (327, 0.153846153...
7     45429271  [[473, 360, 314, 314, 314, 314, 314, 330, 330,...   [(314, 5), (330, 4), (360, 2)]  [(314, 0.4166666666666667), (330, 0.3333333333...
8     22536391  [[609, 407, 384, 470, 485, 415, 384, 384, 466,...   [(384, 3), (388, 3), (473, 2)]  [(384, 0.1304347826086957), (388, 0.1304347826...
9     8.13E+17  [[421, 384, 401, 230, 330, 609, 401, 424, 264,...   [(384, 2), (265, 2), (401, 2)]  [(384, 0.09523809523809526), (265, 0.095238095...
10    7.63E+17  [[238, 238, 238, 457, 267, 474, 338, 401, 512,...   [(238, 3), (384, 2), (304, 2)]  [(238, 0.15789473684210525), (384, 0.105263157...
11    98221214  [[432, 432, 88, 432, 384, 215, 259, 384, 522, ...  [(384, 10), (432, 4), (259, 2)]  [(384, 0.5263157894736842), (432, 0.2105263157...
12    7.20E+17  [[465, 329, 465, 329, 432, 432, 432, 432, 432,...   [(432, 7), (465, 5), (329, 2)]  [(432, 0.3684210526315789), (465, 0.2631578947...
13   323346248  [[407, 401, 603, 641, 521, 327, 0, 432, 262, 4...     [(327, 2), (432, 2), (0, 1)]  [(327, 0.10526315789473684), (432, 0.105263157...
14  4036152552  [[327, 632, 632, 168, 28, 168, 632, 632, 632, ...    [(168, 8), (632, 5), (28, 3)]  [(168, 0.42105263157894735), (632, 0.263157894...
15    17281102  [[225, 225, 225, 225, 225, 225, 225, 225, 384,...   [(225, 8), (628, 6), (384, 2)]  [(225, 0.4444444444444444), (628, 0.3333333333...
16    24685146  [[396, 330, 330, 330, 396, 330, 330, 330, 330,...  [(330, 14), (264, 2), (396, 2)]  [(330, 0.7777777777777778), (264, 0.1111111111...
17    24822125  [[401, 354, 314, 360, 432, 360, 432, 360, 689,...   [(689, 4), (432, 3), (360, 3)]  [(689, 0.22222222222222218), (432, 0.166666666...
18    28477232  [[432, 385, 385, 385, 385, 385, 385, 385, 385,...            [(385, 17), (432, 1)]  [(385, 0.9444444444444444), (432, 0.0555555555...
19   136675517  [[410, 404, 466, 527, 473, 403, 410, 503, 503,...   [(384, 5), (503, 2), (410, 2)]  [(384, 0.2777777777777778), (503, 0.1111111111...
20   228534984  [[432, 213, 432, 213, 432, 432, 432, 432, 205,...  [(432, 14), (213, 2), (552, 1)]  [(432, 0.7777777777777778), (213, 0.1111111111...
21   237564656  [[327, 254, 213, 254, 254, 254, 432, 213, 213,...   [(254, 7), (213, 5), (432, 1)]  [(254, 0.3888888888888889), (213, 0.2777777777...
22   423924903  [[267, 432, 609, 342, 432, 432, 199, 122, 150,...   [(432, 5), (401, 3), (609, 1)]  [(432, 0.2777777777777778), (401, 0.1666666666...
23   496539092  [[609, 597, 597, 597, 597, 432, 597, 597, 597,...  [(597, 11), (634, 3), (609, 2)]  [(597, 0.6111111111111112), (634, 0.1666666666...
24    18193826  [[299, 299, 299, 427, 299, 28, 595, 401, 401, ...   [(299, 6), (401, 3), (432, 1)]  [(299, 0.42857142857142855), (401, 0.214285714...