python - 具有多个条件的清除脚本

Question

我正在编写另一个 python 清除脚本。这将用大量的 find -delete 替换非常旧的 bash 脚本，这需要长达 9 小时才能清除我们的视频后端。

我知道有大量的代码要么在堆栈上，要么在谷歌中，但问题是我还有一些限制，这让我不得不编写我发现的糟糕/低效的代码。

考虑以下目录结构:

/data/channel1/video_800/0001/somefile_800_001.ts
/data/channel1/video_800/0001/somefile_800_002.ts
/data/channel1/video_800/0002/somediffile_800_001.ts
/data/channel1/video_800/0002/somediffile_800_002.ts
/data/channel1/video_800.m3u8
/data/channel1/video_900/0001/someotherfile_900_001.ts
/data/channel1/video_900/0002/afile_900_001.ts
/data/channel1/video_900/0003/bfile_900_001.ts
/data/channel1/video_900/0003/cfile_900_001.ts
/data/channel1/video_900.m3u8

/data/channel2/video_800/0001/againsomefile_800_001.ts
/data/channel2/video_800/0001/againsomefile_800_001.ts
/data/channel2/video_800.m3u8

/data/sport_channel/video_1000/0001/somefile.ts/data/sport_channel/video_1000/0001/somefile2.ts

我感兴趣的第一件事是 channel 名称，因为 channel *有一个规则，体育*有一个规则。

第二件事是等于比特率的视频目录的结尾...800、900、1000，因为它们可以有不同的保留天数。

最后，我将检查所有内容并根据比特率和扩展名删除文件。

下面的代码可以工作，但是过于复杂，而且我确信不是很Pythonic。由于在这种情况下我最关心的是性能，我确信有一种更有效的方法来做到这一点。在 for 循环中堆叠 for 循环不仅是糟糕的设计，而且在我的 pymode 中还会导致 'find_files' is too complex [mccabe]。

** 将删除函数排除在代码示例之外，但这只是一个简单的尝试:除了使用 os.rmdir 和 os.remove

我愿意接受所有改进我的代码的建议。

谢谢!

#!/usr/bin/python

import os
import time
import fnmatch

path = '/data'
debits_short = ['200', '700', '1000', '1300', '2500']
debits_long = ['400', '1800']

def find_files(chan_name, debits, duration):

    time_in_secs = time.time() - (duration * 24 * 60 * 60)

    # List channel
    for channel in os.listdir(path):

        # Match category channels
        if fnmatch.fnmatch(channel, chan_name):

            # Go through bitrates
            for debit in debits:

                # Channel path now the default search path
                channel_path = path + channel

                # Walk through channel path to match bitrate files
                for root, dirs, files in os.walk(channel_path, topdown=False):
                    for filename in files:

                        # Remove files that contain _bitrate_ and end with ts
                        if '_' + debit + '_' in filename:
                            if filename.endswith('.ts'):
                                if os.path.isfile(os.path.join(root, filename)):
                                    if os.stat(os.path.join(root, filename)).st_mtime <= time_in_secs:
                                        remove(os.path.join(root, filename))

                        # Remove playlist files that contain bitrate.m3u8
                        if filename.endswith(debit + '.m3u8'):
                            if os.path.isfile(os.path.join(root, filename)):
                                if os.stat(os.path.join(root, filename)).st_mtime <= time_in_secs:
                                    remove(os.path.join(root, filename))

                    # Remove empty dirs
                    for dir in dirs:
                        if not os.listdir(os.path.join(root, dir)):
                            remove(os.path.join(root, dir))


find_files('channel*', debits_long, 3)
find_files('sport*', debits_short, 7)

Answer 1

这是一种可能的方法:

import os
import glob
import time


class Purge(object):

    removable_extensions = ['ts', 'm3u8']

    def __init__(self, basedir, channel_pattern, debits,
                 older_than_days, test_mode=False):
        self.basedir = basedir
        self.channel_pattern = channel_pattern
        self.debits = debits
        self.older_than_secs = time.time() - 24*60*60*older_than_days
        self.test_mode = test_mode  # If `True`, do not delete files.

    def delete_file(self, filepath):
        try:
            os.remove(filepath)
        except OSError:
            pass

    def file_for_deletion(self, filepath):
        # Return `True` if a file meets all conditions for deletion.
        filename, ext = os.path.splitext(os.path.basename(filepath))
        condition_ext = ext[1:] in self.removable_extensions
        condition_old = os.stat(filepath).st_mtime <= self.older_than_secs
        condition_deb = any(
            '_{}_'.format(d) in filename or filename.endswith(d)
            for d in self.debits
            )
        return all((condition_ext, condition_old, condition_deb))

    def purge_channel(self, channel_dir):
        for root, dirs, files in os.walk(channel_dir):
            for name in files:
                filepath = os.path.join(root, name)
                if self.file_for_deletion(filepath):
                    print filepath
                    if not self.test_mode:
                        self.delete_file(filepath)
            #TODO: delete empty directories here.

    def purge(self):
        channels = glob.glob(os.path.join(self.basedir, self.channel_pattern))
        for channel_dir in channels:
            self.purge_channel(channel_dir)


if __name__ == '__main__':

    purge_job_info = dict(
        basedir=r'path/to/data',  # All channel folders live here.
        channel_pattern='channel*',  # `glob` pattern.
        debits=['400', '1800'],
        older_than_days=7,
        )

    p = Purge(**purge_job_info)
    p.test_mode = True
    p.purge()