python - shutil.rmtree : FileNotFoundError: [Errno 2] No such file or directory: '._xxx'

Question

代码和错误(精简，如果你想看完整代码，请引用上一节):

import shutil
import os

filteredCleaned = 
 '/Volumes/Extreme_SSD/Raymond_Lab/Day_4_Rotarod_Videos_Rotated_if_Necessary_copy/filtered_cleaned_WT_cleaned_YAC128'

if os.path.exists(filteredCleaned):
    shutil.rmtree(filteredCleaned)

^ 显示隐藏文件(按 shift+command+。)和其他目录显示隐藏文件(如果存在)。

runfile('/Users/ksb7640/Documents/UBC_Academic/Raymond_Lab/448/rotarod/svm_all/data_filter.py', wdir='/Users/ksb7640/Documents/UBC_Academic/Raymond_Lab/448/rotarod/svm_all')
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3417, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-b9676f9bf96c>", line 1, in <module>
    runfile('/Users/ksb7640/Documents/UBC_Academic/Raymond_Lab/448/rotarod/svm_all/data_filter.py', wdir='/Users/ksb7640/Documents/UBC_Academic/Raymond_Lab/448/rotarod/svm_all')
  File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/Users/ksb7640/Documents/UBC_Academic/Raymond_Lab/448/rotarod/svm_all/data_filter.py", line 21, in <module>
    shutil.rmtree(filteredCleaned)
  File "/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/shutil.py", line 715, in rmtree
    _rmtree_safe_fd(fd, path, onerror)
  File "/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/shutil.py", line 672, in _rmtree_safe_fd
    onerror(os.unlink, fullname, sys.exc_info())
  File "/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/shutil.py", line 670, in _rmtree_safe_fd
    os.unlink(entry.name, dir_fd=topfd)
FileNotFoundError: [Errno 2] No such file or directory: '._filtered_combined_Experiment2-190630_Day4_145m2_rotarod2_Cam2onRotarodDeepCut_resnet50_rotarod3Jul17shuffle1_1030000.csv'

问题:

'._filtered_combined_Experiment2-190630_Day4_145m2_rotarod2_Cam2onRotarodDeepCut_resnet50_rotarod3Jul17shuffle1_1030000.csv' 去哪儿了？从哪里来？

我确实使用了 df.to_csv(csv, index=False)创建filtered_combined_Experiment2-190630_Day4_145m2_rotarod2_Cam2onRotarodDeepCut_resnet50_rotarod3Jul17shuffle1_1030000.csv在filtered_cleaned_WT_cleaned_YAC128但我从未使用过chmod隐藏它们。

此外，正如您在屏幕截图中所见，甚至没有隐藏文件。错误是什么？

完整的代码可能是冗余的，但以防万一...(有关缩短的代码，请参阅顶部。)

数据过滤器.py

import shutil

from export_df_to_csv import export_df_to_csv
from extract_parent_current import extract_parent_current
from import_df import *

prefix = 'filtered_'

bound = 0.9

# make a directory for output files
cleanedWT = '/Volumes/Extreme_SSD/Raymond_Lab/Day_4_Rotarod_Videos_Rotated_if_Necessary_copy/cleaned_WT'
cleanedYAC128 = '/Volumes/Extreme_SSD/Raymond_Lab/Day_4_Rotarod_Videos_Rotated_if_Necessary_copy/cleaned_YAC128'

cleanedWTParentDir, WTdir = extract_parent_current(cleanedWT)
cleanedYAC128ParentDir, YAC128dir = extract_parent_current(cleanedYAC128)

filteredCleaned = os.path.join(cleanedYAC128ParentDir, prefix + WTdir + '_' + YAC128dir)

if os.path.exists(filteredCleaned):
    shutil.rmtree(filteredCleaned)
os.mkdir(filteredCleaned)

csvs_labels_arr = import_csvs(cleanedWT, cleanedYAC128)
paths_dfs_labels_arr = csvs_to_paths_dfs_labels_arr(csvs_labels_arr)
for path_df_label in paths_dfs_labels_arr:
    _, dfFileName = extract_parent_current(path_df_label[0])
    df = path_df_label[1]

    # filter out rows below bound
    df = df[
        (df['Rightpaw likelihood'] > bound) & (df['Leftpaw likelihood'] > bound) & (df['Tail likelihood'] > bound)]

    export_df_to_csv(df, os.path.join(filteredCleaned, prefix + dfFileName))

export_df_to_csv.py

input_file = '/Volumes/Extreme SSD/Raymond Lab/Day_4_Rotarod_Videos_Rotated_if_Necessary copy/cleaned_WT/cleaned_Experiment2-190630_Day4_145m1_rotarod3_Cam2onRotarodDeepCut_resnet50_rotarod3Jul17shuffle1_1030000.csv'
output_file = '/Volumes/Extreme SSD/Raymond Lab/Day_4_Rotarod_Videos_Rotated_if_Necessary copy/cleaned_WT/cleaned_Experiment2-190630_Day4_145m1_rotarod3_Cam2onRotarodDeepCut_resnet50_rotarod3Jul17shuffle1_1030000.csv'


def export_df_to_csv(df, csv):
    df.to_csv(csv, index=False)

extract_parent_current.py

import os


def extract_parent_current(dir):
    if str.endswith(dir, '/'):
        dir = dir[:-1]
    return os.path.split(dir)

导入_df.py

import os
import random
from copy import deepcopy
import pandas as pd


def import_csvs(WT_file_path, YAC_file_path):
    csv_paths_arr = []
    for root, dirs, files in os.walk(WT_file_path, topdown=False):
        for file in files:
            if not file.startswith('.'):
                csv_paths_arr.append([os.path.join(root, file), 0])

    for root, dirs, files in os.walk(YAC_file_path, topdown=False):
        for file in files:
            if not file.startswith('.'):
                csv_paths_arr.append([os.path.join(root, file), 1])

    return csv_paths_arr


def csvs_to_paths_dfs_labels_arr(csvpaths_labels_arr):
    paths_dfs_labels_arr = deepcopy(csvpaths_labels_arr)
    for i, csvpath_label_arr in enumerate(csvpaths_labels_arr):
        (paths_dfs_labels_arr[i])[0] = pd.read_csv(csvpath_label_arr[0], encoding='unicode_escape')
        paths_dfs_labels_arr[i].insert(0, csvpath_label_arr[0])
    return paths_dfs_labels_arr


def import_df(WT_file_path, YAC_file_path):
    csv_paths_arr = import_csvs(WT_file_path, YAC_file_path)
    dfs_labels = csvs_to_paths_dfs_labels_arr(csv_paths_arr)
    random.shuffle(dfs_labels)
    return dfs_labels

Answer 1

我已经为链接问题 Delete directory and all symlinks recursively 提供了这个答案, 但决定在这里重现它。在那里，它陷入了为潜在问题提供解决方案而不是为实际问题提供答案的谬误，而它在这里直接回答了 ._filtered_combined_Experiment2-190630_Day4_145m2_rotarod2_Cam2onRotarodDeepCut_resnet50_rotarod3Jul17shuffle1_1030000.csv 的来源。

/Volumes/Extreme_SSD/ 路径表明您使用的是 Mac OSX，并且您的目录至少部分位于非 Mac 文件系统(即不是 HFS+)上。在这些设备上，Mac 文件系统驱动程序会自动创建前缀为 ._ 的二进制伴随文件，以记录所谓的扩展属性(在 https://apple.stackexchange.com/questions/14980/why-are-dot-underscore-files-created-and-how-can-i-avoid-them 中解释，但也在下面说明)，在普通操作，例如使用默认 OSX 查看器查看文件。

rmtree 在不支持 os.scandir 中的文件描述符的系统上(例如 Mac OSX)现在会不安全地创建条目列表，然后将它们一一取消链接(创建一个已知的竞争条件:https://github.com/python/cpython/blob/908fd691f96403a3c30d85c17dd74ed1f26a60fd/Lib/shutil.py#L592-L621)。不幸的是，两种不同的行为每次都使这个条件为真:

1. 原始文件总是列在扩展属性之前，并且
2. 当原始文件取消链接 (test.txt) 时，元文件 (._test.txt) 会同时删除。

因此，扩展属性文件将在轮到它时丢失并抛出您遇到的 FileNotFoundError。

我认为这个错误最好由 cpython#14064 解决, 旨在忽略 rmtree 中的 FileNotFoundError。

缓解

与此同时，您可以使用 onerror 忽略那些元文件的取消链接错误:

def ignore_extended_attributes(func, filename, exc_info):
    is_meta_file = os.path.basename(filename).startswith("._")
    if not (func is os.unlink and is_meta_file):
        raise

shutil.rmtree(path_dir, onerror=ignore_extended_attributes)

Mac扩展属性展示

为了说明，您可以创建一个小的 ExFAT 磁盘镜像并使用命令将其装载到 /Volumes/Untitled

hdiutil create -size 5m -fs exfat test.dmg
hdiutil attach test.dmg            # mounts at /Volumes/Untitled
cd /Volumes/Untitled

mkdir test                         # create a directory to remove
cd test
touch test.txt
open test.txt                      # open the test.txt file in the standard editor 

只需在标准文本编辑器中打开文件，就会创建一个扩展属性文件 ._test.txt 并在其中记录上次访问时间:

/Volumes/Untitled/test $ ls -a
.          ..         ._test.txt test.txt
/Volumes/Untitled/test $ xattr test.txt
com.apple.lastuseddate#PS

问题是自动取消链接原始文件也会取消链接伴随文件。

/Volumes/Untitled/test $ rm test.txt
/Volumes/Untitled/test $ ls -a
.          ..