python - 计算 DataFrame 列中的 NaN 窗口(及其大小)

Question

我有巨大的数据帧(数百万、数十)和沿列的大量缺失(NaN)值。我需要以尽可能最快的方式计算每列的 NaN 窗口及其大小(我的代码太慢)。

类似这样的:来自这里

import pandas as pd
import numpy as np
df = pd.DataFrame({'a':[1,2, np.nan, np.nan,3,3,np.nan,4,np.nan,np.nan],\
               'b':[np.nan, 2, 1, 1, 3, 3, np.nan, np.nan,2, np.nan],\
               'c':[np.nan, 2, 1, np.nan, 3, 3, np.nan, np.nan,2, 8]})

df
Out[65]: 
 a    b    c
0  1.0  NaN  NaN
1  2.0  2.0  2.0
2  NaN  1.0  1.0
3  NaN  1.0  NaN
4  3.0  3.0  3.0
5  3.0  3.0  3.0
6  NaN  NaN  NaN
7  4.0  NaN  NaN
8  NaN  2.0  2.0
9  NaN  NaN  8.0

到这里:

result
Out[61]: 
    a  b  c
 0  2  1  1
 1  1  2  1
 2  2  1  2

Answer 1

这是一种方法:

import pandas as pd
import numpy as np

df = pd.DataFrame({'a':[1,2, np.nan, np.nan,3,3,np.nan,4,np.nan,np.nan],\
               'b':[np.nan, 2, 1, 1, 3, 3, np.nan, np.nan,2, np.nan],\
               'c':[np.nan, 2, 1, np.nan, 3, 3, np.nan, np.nan,2, 8]})
df_n = pd.DataFrame({'a':df['a'].isnull().values,
                      'b':df['b'].isnull().values,
                      'c':df['c'].isnull().values})

pr={}
for column_name, _ in df_n.iteritems():

    fst = df_n.index[df_n[column_name] & ~ df_n[column_name].shift(1).fillna(False)]
    lst = df_n.index[df_n[column_name] & ~ df_n[column_name].shift(-1).fillna(False)]

    pr[column_name] = [j-i+1 for i, j in zip(fst, lst)]

df_new=pd.DataFrame(pr)

输出:

    a   b   c
0   2   1   1
1   1   2   1
2   2   1   2