python - 合并计算文件内容并在输出中显示先前计算的数据

Question

我正在处理 2 个文件，oldFile.txt 和 newFile.txt 并计算它们之间的一些变化。 newFile.txt 不断更新，任何更新都将写入 oldFile.txt

我试图通过保存以前的计算值并将其添加到 finalOutput.txt 来改进下面的代码段。任何想法都将非常有助于完成所需的输出。提前谢谢你。

import pandas as pd
from time import sleep
def read_file(fn):
    data = {}
    with open(fn, 'r') as f:
        for lines in f:
            line = lines.rstrip()
            pname, cnt, cat = line.split(maxsplit=2)
            data.update({pname: {'pname': pname, 'cnt': int(cnt), 'cat': cat}})
    return data

def process_data(oldfn, newfn):  
    old = read_file(oldfn)
    new = read_file(newfn)

    u_data = {}
    for ko, vo in old.items():
        if ko in new:
            n = new[ko]
            
            old_cnt = vo['cnt']
            new_cnt = n['cnt']
            u_cnt = old_cnt + new_cnt
            tmp_old_cnt = 1 if old_cnt == 0 else old_cnt
            cnt_change = 100 * (new_cnt - tmp_old_cnt) / tmp_old_cnt
            u_data.update({ko: {'pname': n['pname'], 'cnt': new_cnt, 'cat': n['cat'],
                                'curr_change%': round(cnt_change, 0)}})
    for kn, vn in new.items():
        if kn not in old:        
            old_cnt = 1
            new_cnt = vn['cnt']
            cnt_change = 0
            vn.update({'cnt_change': round(cnt_change, 0)})        
            u_data.update({kn: vn})

    pd.options.display.float_format = "{:,.0f}".format
    mydata = []
    
    for _, v in u_data.items():
        mydata.append(v)
        
    df = pd.DataFrame(mydata)
    df = df.sort_values(by=['cnt'], ascending=False)
    
    # Save to text file.
    with open('finalOutput.txt', 'w') as w:
        w.write(df.to_string(header=None, index=False))
        
    # Overwrite oldFile.txt
    with open('oldFile.txt', 'w') as w:
        w.write(df.to_string(header=None, index=False))

    # Print in console.
    df.insert(0, '#', range(1, 1 + len(df)))
    print(df.to_string(index=False,header=True))
    
while True:
    oldfn = './oldFile.txt'
    newfn = './newFile.txt'
    process_data(oldfn, newfn)
    sleep(60)

旧文件.txt

e6c76e4810a464bc 1                   Hello(HLL)
65b66cc4e81ac81d 2           CryptoCars (CCAR)
c42d0c924df124ce 3            GoldNugget (NGT)
ee70ad06df3d2657 4             BabySwap (BABY)
e5b7ebc589ea9ed8 8            Heroes&E... (HE)
7e7e9d75f5da2377 3                Robox (RBOX)

newfile.txt #-- 第一次阅读时的内容

e6c76e4810a464bc 34                  Hello(HLL)
65b66cc4e81ac81d 43           CryptoCars (CCAR)
c42d0c924df124ce 95            GoldNugget (NGT)
ee70ad06df3d2657 15             BabySwap (BABY)
e5b7ebc589ea9ed8 37            Heroes&E... (HE)
7e7e9d75f5da2377 23                Robox (RBOX)
755507d18913a944 49             CharliesFactory

newfile.txt #--二读时的内容

924dfc924df1242d 35              AeroDie (ADie)
e6c76e4810a464bc 34                  Hello(HLL)
65b66cc4e81ac81d 73           CryptoCars (CCAR)
c42d0c924df124ce 15            GoldNugget (NGT)
ee70ad06df3d2657 5              BabySwap (BABY)
e5b7ebc589ea9ed8 12            Heroes&E... (HE)
7e7e9d75f5da2377 19                Robox (RBOX)
755507d18913a944 169            CharliesFactory

newfile.txt # 三读内容

924dfc924df1242d 45             AeroDie (ADie)
e6c76e4810a464bc 2                  Hello(HLL)
65b66cc4e81ac81d 4           CryptoCars (CCAR)
c42d0c924df124ce 7            GoldNugget (NGT)
ee70ad06df3d2657 5             BabySwap (BABY)
e5b7ebc589ea9ed8 3            Heroes&E... (HE)
7e7e9d75f5da2377 6                Robox (RBOX)
755507d18913a944 9             CharliesFactory

oldFile.txt #-- 需要改进的当前输出

#            pname  cnt               cat   curr_change%
 1 924dfc924df1242d   35    AeroDie (ADie)            29 
 2 755507d18913a944    9   CharliesFactory           -95
 3 c42d0c924df124ce    7  GoldNugget (NGT)           -53
 4 7e7e9d75f5da2377    6      Robox (RBOX)           -68
 5 ee70ad06df3d2657    5   BabySwap (BABY)             0
 6 65b66cc4e81ac81d    4 CryptoCars (CCAR)           -95
 7 e5b7ebc589ea9ed8    3  Heroes&E... (HE)           -75
 8 e6c76e4810a464bc    2        Hello(HLL)           -94

finalOutput.txt #-- 需要改进输出，根据更新读数的数量增加 r1、r2 等列

# curr_change% is the latest 3rd reading
# r2% is based on the 2nd reading
# r1% is based on the 1st reading

 #            pname  cnt               cat  curr_change%    r2%      r1%
 1 924dfc924df1242d   35    AeroDie (ADie)            29      0        0
 2 755507d18913a944    9   CharliesFactory           -95    245        0
 3 c42d0c924df124ce    7  GoldNugget (NGT)           -53    -84    3,067
 4 7e7e9d75f5da2377    6      Robox (RBOX)           -68    -17      667
 5 ee70ad06df3d2657    5   BabySwap (BABY)             0    -67      275
 6 65b66cc4e81ac81d    4 CryptoCars (CCAR)           -95     70    2,050
 7 e5b7ebc589ea9ed8    3  Heroes&E... (HE)           -75    -68      362
 8 e6c76e4810a464bc    2        Hello(HLL)           -94      0    3,300

Answer 1

更新反馈，我进行了调整，以便它可以处理实时提供给它的数据。每当加载新数据时，将文件名加载到 process_new_file() 函数中，它将更新 'finalOutput.txt'。

为简单起见，我将不同的文件命名为 file1、file2、file3 和 file4。

我正在使用 pandas Dataframe 进行大部分操作。我认为使用 Pandas DataFrames 会让您轻松完成任务。

总的来说，我创建了一个函数来读取文件并返回格式正确的 DataFrame。我创建了第二个函数来比较旧文件和新文件并进行您正在寻找的计算。我将这些计算的结果合并在一起。最后，我将所有这些计算与最后一个文件的数据合并以获得您正在寻找的输出。

import pandas as pd

global global_old_df
global results_df
global count

global_old_df = None
results_df = pd.DataFrame()
count = 0


def read_file(file_name):

    rows = []
    with open(file_name) as f:
      for line in f:
         rows.append(line.split(" ", 2))
    df = pd.DataFrame(rows, columns=['pname', 'cnt', 'cat'])
    df['cat'] = df['cat'].str.strip()
    df['cnt'] = df['cnt'].astype(float)

    return df


def compare_dfs(df_old, df_new, count):

    df_ = df_old.merge(df_new, on=['pname', 'cat'], how='outer')
    df_['r%s' % count] = (df_['cnt_y'] / df_['cnt_x'] - 1) * 100
    df_ = df_[['pname', 'r%s' % count]]
    df_ = df_.set_index('pname')

    return df_


def process_new_file(file):

    global global_old_df
    global results_df
    global count

    df_new = read_file(file)

    if global_old_df is None:
        global_old_df = df_new
        return

    else:
        count += 1
        r_df = compare_dfs(global_old_df, df_new, count)
        results_df = pd.concat([r_df, results_df], axis=1)
        global_old_df = df_new

        output_df = df_new.merge(results_df, left_on='pname', right_index=True)
        output_df.to_csv('finalOutput.txt')
        pd.options.display.float_format = "{:,.1f}".format
        print(output_df.to_string())




files = ['file1.txt', 'file2.txt', 'file3.txt', 'file4.txt']

for file in files:
    process_new_file(file)

这给出了输出:

              pname  cnt                cat    r3    r2      r1
0  924dfc924df1242d 45.0     AeroDie (ADie)  28.6   NaN     NaN
1  e6c76e4810a464bc  2.0         Hello(HLL) -94.1   0.0 3,300.0
2  65b66cc4e81ac81d  4.0  CryptoCars (CCAR) -94.5  69.8 2,050.0
3  c42d0c924df124ce  7.0   GoldNugget (NGT) -53.3 -84.2 3,066.7
4  ee70ad06df3d2657  5.0    BabySwap (BABY)   0.0 -66.7   275.0
5  e5b7ebc589ea9ed8  3.0   Heroes&E... (HE) -75.0 -67.6   362.5
6  7e7e9d75f5da2377  6.0       Robox (RBOX) -68.4 -17.4   666.7
7  755507d18913a944  9.0    CharliesFactory -94.7 244.9     NaN

因此，要实时运行它，您只需将最后一部分替换为:

while True:
    newfn = './newFile.txt'
    process_new_file(newfn)
    sleep(60)