Python:将.txt文件读入数据框并写回文件后无法获取原始格式

Question

我有一个 .txt 文件，我需要对其进行异常值删除。该文件如下所示:

{"mille":"802821", "type":"tc", "test":"mod6", "hrow":"C", "pcnt":"1", "pid":"0", "pidx":"0", "act":"DOWN-1ST", "x":"557.00", "y":"1043.00", "size":"0.3333", "press":"0.6000", "vx":"0.0007", "vy":"0.0013"}
{"mille":"802821", "type":"tc", "test":"mod6", "hrow":"C", "pcnt":"2", "pid":"0", "pidx":"0", "act":"DOWN-P2", "x":"557.00", "y":"1043.00", "size":"0.3333", "press":"0.6000", "vx":"NaN", "vy":"NaN"}
{"mille":"802821", "type":"tc", "test":"mod6", "hrow":"C", "pcnt":"2", "pid":"1", "pidx":"1", "act":"DOWN-P2", "x":"641.00", "y":"754.00", "size":"0.2000", "press":"0.5500", "vx":"Infinity", "vy":"-Infinity"}
{"mille":"802837", "type":"th", "test":"mod6", "hrow":"0", "pcnt":"2", "pid":"0", "pidx":"0", "act":"MOVE", "x":"556.00", "y":"1043.00", "size":"0.3333", "press":"0.6000", "vx":"-5.3125", "vy":"18.0625"}
{"mille":"802837", "type":"th", "test":"mod6", "hrow":"0", "pcnt":"2", "pid":"1", "pidx":"1", "act":"MOVE", "x":"641.00", "y":"754.00", "size":"0.2000", "press":"0.5500", "vx":"5.3125", "vy":"-18.0625"}
{"mille":"802846", "type":"th", "test":"mod6", "hrow":"1", "pcnt":"2", "pid":"0", "pidx":"0", "act":"MOVE", "x":"555.00", "y":"1044.00", "size":"0.3333", "press":"0.6000", "vx":"-3.4400", "vy":"11.6000"}
{"mille":"802846", "type":"th", "test":"mod6", "hrow":"1", "pcnt":"2", "pid":"1", "pidx":"1", "act":"MOVE", "x":"641.00", "y":"754.00", "size":"0.2000", "press":"0.5500", "vx":"3.4400", "vy":"-11.6000"}
{"mille":"802854", "type":"th", "test":"mod6", "hrow":"2", "pcnt":"2", "pid":"0", "pidx":"0", "act":"MOVE", "x":"554.00", "y":"1045.00", "size":"0.3333", "press":"0.6000", "vx":"-2.6364", "vy":"8.8182"}
{"mille":"802854", "type":"th", "test":"mod6", "hrow":"2", "pcnt":"2", "pid":"1", "pidx":"1", "act":"MOVE", "x":"641.00", "y":"754.00", "size":"0.2000", "press":"0.5500", "vx":"2.6364", "vy":"-8.8182"}
{"mille":"802863", "type":"th", "test":"mod6", "hrow":"3", "pcnt":"2", "pid":"0", "pidx":"0", "act":"MOVE", "x":"553.00", "y":"1047.00", "size":"0.3333", "press":"0.6125", "vx":"-2.0952", "vy":"6.9762"}

................(每个文件中还有很多这样的行，我有几个文件)

(注意原文文件中每两个{}之间没有空格)

我使用 read_txt() 函数将其读入数据帧并完成异常值删除。现在我需要将它读回与以前格式完全相同的文本文件。

这是我的代码:

path = 'c:/Users/USER/.spyder-py3/machine-learning/data2/test/*.txt'
filelist = glob.glob(path, recursive = True)

for i in range(0,3):
    df = pd.read_json(filelist[i], lines=True)

    outlier_x = df['x'].mean() + df['x'].std() * 3
    outlier_x2 = df['x'].mean() - df['x'].std() * 3
    outlier_y = df['y'].mean() + df['y'].std() * 3
    outlier_y2 = df['y'].mean() - df['y'].std() * 3
    outlier_vx = df['vx'].mean() + df['vx'].std() * 3
    outlier_vx2 = df['vx'].mean() - df['vx'].std() * 3
    outlier_vy = df['vy'].mean() + df['vy'].std() * 3
    outlier_vy2 = df['vy'].mean() - df['vy'].std() * 3
    outlier_pr = df['press'].mean() + df['press'].std() * 3
    outlier_pr2 = df['press'].mean() - df['press'].std() * 3
    outlier_sz = df['size'].mean() + df['size'].std() * 3
    outlier_sz2 = df['size'].mean() - df['size'].std() * 3

    df.drop(['act1','act2','size1','size2','x1','x2','y1','y2'],axis = 1, 
    inplace = True)
    df = df[['mille','type','test','xfocus','yfocus','span','sfact','hrow',
          'pcnt','pid','pidx','act','x','y','size','press','vx','vy']]


    # remove ouliers for column 'x'
    df = df.drop(df[((df['x'] > outlier_x) & (df['act'] == 'MOVE'))].index)
    df = df.drop(df[((df['x'] < outlier_x2) & (df['act'] == 'MOVE'))].index)
    # remove ouliers for column 'y'
    df = df.drop(df[((df['y'] > outlier_y) & (df['act'] == 'MOVE'))].index)
    df = df.drop(df[((df['y'] < outlier_y2) & (df['act'] == 'MOVE'))].index)
    # remove part of the infinite values from column 'vx'
    df = df.drop(df[(((df['vx'] == np.inf) & (df['act'] == 'MOVE')))].index)
    df = df.drop(df[(((df['vx'] == -np.inf) & (df['act'] == 'MOVE')))].index)
    # replace infinit with NAN
    df['vx'] = df['vx'].replace([np.inf,-np.inf],df['vx'].mean())
    # remove ouliers from column 'vx'  
    df = df.drop(df[((df['vx'] > outlier_vx) & (df['act'] == 'MOVE'))].index)
    df = df.drop(df[((df['vx'] < outlier_vx2) & (df['act'] == 'MOVE'))].index)
    # replace infinit with NAN
    df['vy'] = df['vy'].replace([np.inf,-np.inf],df['vy'].mean())
    # fill na with '0' in columns 'vx'
    df['vx'] = df['vx'].fillna(0.0)
     # fill na with '0' in columns 'vy'
    df['vy'] = df['vy'].fillna(0.0)
    # remove outliers from column 'vy'
    df = df.drop(df[((df['vy'] > outlier_vy) & (df['act'] == 'MOVE'))].index)
    df = df.drop(df[((df['vy'] < outlier_vy2) & (df['act'] == 'MOVE'))].index)
    # remove outliers from column 'press'
    df = df.drop(df[((df['press'] > outlier_pr) & (df['act'] == 'MOVE'))].index)
    df = df.drop(df[((df['press'] < outlier_pr2) & (df['act'] == 
    'MOVE'))].index)
    # remove outliers from column 'size'
    df = df.drop(df[((df['size'] > outlier_sz) & (df['act'] == 'MOVE'))].index)
    df = df.drop(df[((df['size'] < outlier_sz2) & (df['act'] == 'MOVE'))].index)

    df.loc[df.xfocus.notnull(), ['vx','vy']] = np.nan,np.nan

    col_select = ['mille','type','test','xfocus','yfocus','span','sfact','hrow',
                 'pcnt','pid','pidx','act','x','y','size','press','vx','vy']

     # modify dataframe to propriate json format
    jsonresult = df.to_json(orient='records')
     # read the json string to get a list of dictionaries
    rows = json.loads(jsonresult)



   # remove some null values
    new_rows = [OrderedDict([(key, row[key]) for key in col_select if (key in 
    row) and pd.notnull(row[key])])for row in rows]

    jsonfile = json.dump(new_rows)


    #save them into destination
    outfile =  "c:/Users/USER/.spyder-py3/machine- 
    learning/data2/testresult/user_" + str(i) + "_mod6.txt"
    thefile = open(outfile, 'w')

    json_output = jsonfile.strip("[]").split('},')

    for i in range(len(json_output)):
       json_output[i] = json_output[i] + '}'

    for item in json_output:    
       thefile.write("%s\n" % item)

我试图得到一个和原始文件一样的 txt 文件，outpur 看起来确实很相似。但是，当我尝试读取清理后的 txt 文件并对其进行其他操作时，出现如下错误:JSONDecodeError: Extra data: line 1 column 201 (char 200)。整个错误信息如下:

---------------------------------------------------------------------------
JSONDecodeError                           Traceback (most recent call last)
<ipython-input-7-a2c25911084b> in <module>()
   2321             print('-----------------------test where am I--------------------------------')
   2322             for line in file_object:
-> 2323                 jrecord = json.loads(line)
   2324                 try:
   2325                     typ = jrecord['type']

~\Anaconda3\lib\json\__init__.py in loads(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
    352             parse_int is None and parse_float is None and
    353             parse_constant is None and object_pairs_hook is None and not kw):
--> 354         return _default_decoder.decode(s)
    355     if cls is None:
    356         cls = JSONDecoder

~\Anaconda3\lib\json\decoder.py in decode(self, s, _w)
    340         end = _w(s, end).end()
    341         if end != len(s):
--> 342             raise JSONDecodeError("Extra data", s, end)
    343         return obj
    344 

JSONDecodeError: Extra data: line 1 column 201 (char 200)

我在处理未清理的.txt文件时没有出现这个错误。所以很明显，当我写回数据时出了点问题。现在我被困在这里，不知道我能做些什么来继续前进。有人可以帮帮我吗？提前致谢!

Answer 1

您应该能够使用 df.to_json(outfile, orient='records', lines=True)

写出您清理过的数据框