python - 提高for循环效率

Question

我正在尝试将包含事件 Web 数据的 12,000 个 JSON 文件转换为单个 pandas 数据帧。该代码运行时间太长。关于如何提高效率有什么想法吗？

加载的 JSON 文件示例:

{'$schema': 12,                       
 'amplitude_id': None,                
 'app': '',                           
 'city': ' ',                         
 'device_carrier': None,              
 'dma': ' ',                          
 'event_time': '2018-03-12 22:00:01.646000',                                
 'group_properties': {'[Segment] Group': {'': {}}},                         
 'ip_address': ' ',                   
 'os_version': None,                  
 'paying': None,                      
 'platform': 'analytics-ruby',        
 'processed_time': '2018-03-12 22:00:06.004940',                            
 'server_received_time': '2018-03-12 22:00:02.993000',                      
 'user_creation_time': '2018-01-12 18:57:20.212000',                        
 'user_id': ' ',                      
 'user_properties': {'initial_referrer': '',                                
  'last_name': '',                    
  'organization_id': 2},              
 'uuid': ' ',                         
 'version_name': None}                

谢谢!

import os
import pandas as pd


data = pd.DataFrame()

for filename in os.listdir('path'):
    file = open(filename, "r")
    file_read1 = file.read()
    file_read1 = pd.read_json(file_read1, lines = True)
    data = data.append(file_read1, ignore_index = True)

Answer 1

将 JSON 字符串转换为数据帧的最快方法似乎是 pd.io.json.json_normalize。根据 JSON 的数量，它比附加到现有数据帧快大约 15 到 >500 倍。它比 pd.concat 击败了 13 到 170 倍。

副作用是 JSON 的嵌套部分(group_properties 和 user_properties)也会被展平，并且 dtypes 需要手动设置。

12,000 个 JSON 的运行时(不考虑磁盘 I/O)

• 附加:~177 秒
• 连续:约 126 秒
• json_normalize:~0.7 秒

import pandas as pd
import json
import os

data = []
for filename in os.listdir('path'):
    with open(filename, 'r') as f:
        data.append(f)

# read one JSON and use it as a reference dataframe
df_ref = pd.read_json(data[0], lines=True)

# create a temporary dataframe, get its column 0 and flatten it via json_normalize
df_temp = pd.DataFrame(data)[0]
df = pd.io.json.json_normalize(df_temp.apply(json.loads))

# fix the column dtypes
for col, dtype in df_ref.dtypes.to_dict().items():
    if col not in df.columns:
        continue
    df[col] = df[col].astype(dtype, inplace=True)

完整代码

import pandas as pd
import json
import time

j = {'$schema': 12,                       
 'amplitude_id': None,                
 'app': '',                           
 'city': ' ',                         
 'device_carrier': None,              
 'dma': ' ',                          
 'event_time': '2018-03-12 22:00:01.646000',                                
 'group_properties': {'[Segment] Group': {'': {}}},                         
 'ip_address': ' ',                   
 'os_version': None,                  
 'paying': None,                      
 'platform': 'analytics-ruby',        
 'processed_time': '2018-03-12 22:00:06.004940',                            
 'server_received_time': '2018-03-12 22:00:02.993000',                      
 'user_creation_time': '2018-01-12 18:57:20.212000',                        
 'user_id': ' ',                      
 'user_properties': {'initial_referrer': '',                                
  'last_name': '',                    
  'organization_id': 2},              
 'uuid': ' ',                         
 'version_name': None}

json_str = json.dumps(j)

def df_append():
    t0 = time.time()
    df = pd.DataFrame()
    for _ in range(n_lines):
        file_read1 = pd.read_json(json_str, lines=True)
        df = df.append(file_read1, ignore_index=True)
    return df, time.time() - t0

def df_concat():
    t0 = time.time()
    data = []
    for _ in range(n_lines):
        file_read1 = pd.read_json(json_str, lines=True)
        data.append(file_read1)

    df = pd.concat(data)
    df.index = list(range(len(df)))
    return df, time.time() - t0

def df_io_json():
    df_ref = pd.read_json(json_str, lines=True)
    t0 = time.time()
    data = []
    for _ in range(n_lines):
        data.append(json_str)

    df = pd.io.json.json_normalize(pd.DataFrame(data)[0].apply(json.loads))
    for col, dtype in df_ref.dtypes.to_dict().items():
        if col not in df.columns:
            continue
        df[col] = df[col].astype(dtype, inplace=True)
    return df, time.time() - t0


n_datapoints = (10, 10**2, 10**3, 12000, 10**4, 10**5)
times = {}
for n_lines in n_datapoints:
    times[n_lines] = [[], [], []]
    for _ in range(3):
        df1, t1 = df_append()
        df2, t2 = df_concat()
        df3, t3 = df_io_json()
        times[n_lines][0].append(t1)
        times[n_lines][1].append(t2)
        times[n_lines][2].append(t3)
        pd.testing.assert_frame_equal(df1, df2)
        pd.testing.assert_frame_equal(df1[df1.columns[0:7]], df3[df3.columns[0:7]])
        pd.testing.assert_frame_equal(df2[df2.columns[8:16]], df3[df3.columns[7:15]])
        pd.testing.assert_frame_equal(df2[df2.columns[17:]], df3[df3.columns[18:]])
    for i in range(3):
        times[n_lines][i] = sum(times[n_lines][i]) / 3
times  

x = n_datapoints

fig = plt.figure()

plt.plot(x, [t[0] for t in times.values()], 'o-', label='append')
plt.plot(x, [t[1] for t in times.values()], 'o-', label='concat')
plt.plot(x, [t[2] for t in times.values()], 'o-', label='json_normalize')

plt.xlabel('number of JSONs', fontsize=16)
plt.ylabel('time in seconds', fontsize=18)
plt.yscale('log')

plt.legend()
plt.show()