haskell - 如何使用 Iteratee 读取文件的所有内容

Question

我在 Haskell 中有下一个代码，这对于读取文件的第一行很好，但我需要读取目录中文件的所有内容(许多文件递归)。我正在尝试更改 firstLineE 函数，我不明白如何更改行: EIO.enumFile 1024 filename $ joinI $ ((mapChunks B.pack) >> EC.enumLinesBS) 。你有这方面的一些文档吗？或者你能帮我举一些例子吗？

我正在查看文档，但 Iteratee 对我来说非常新:

http://www.mew.org/~kazu/proj/enumerator/

http://blog-mno2.csie.org/blog/2011/11/19/yet-another-guide-to-understand-iteratee-for-haskell-programmers/

import Control.Monad
import Control.Monad.IO.Class
import Control.Applicative
import System.Environment
import System.Directory
import System.FilePath
import qualified Data.List as L
import qualified Data.ByteString.Char8 as B
import qualified Data.Iteratee as I
import Data.Iteratee.Iteratee
import qualified Data.Iteratee.Char as EC
import qualified Data.Iteratee.IO.Fd as EIO
import qualified Data.Iteratee.ListLike as EL


getValidContents :: FilePath -> IO [String]
getValidContents path =
    filter (`notElem` [".", "..",".git", ".svn",".metadata",".idea",".project",".gitignore",".settings",".hsproject",".dist-scion",".dist-buildwrapper"])
    <$> getDirectoryContents path

isSearchableDir :: FilePath -> IO Bool
isSearchableDir dir = doesDirectoryExist dir
    -- (&&) <$> doesDirectoryExist dir
    --     <*> (searchable <$> getPermissions dir)


doesFileExistAndFilter :: FilePath -> IO Bool
doesFileExistAndFilter dir =
    (&&) <$> doesFileExist dir
         <*> return (snd (splitExtension dir) == ".java" || snd (splitExtension dir) == ".mora")


printI :: Iteratee [B.ByteString] IO ()
printI = do
    mx <- EL.tryHead
    case mx of
         Nothing -> return ()
         Just l -> do
             liftIO . B.putStrLn $ l
             printI


firstLineE :: Enumeratee [FilePath] [B.ByteString] IO ()
firstLineE = mapChunksM $ \filenames -> do
    forM filenames $ \filename -> do
        i <- EIO.enumFile 1024 filename $ joinI $ ((mapChunks B.pack) ><> EC.enumLinesBS) EL.head
        result <- run i
        return result

enumDir :: FilePath -> Enumerator [FilePath] IO b
enumDir dir iter = runIter iter idoneM onCont
    where
        onCont k Nothing = do
            (files, dirs) <- liftIO getFilesDirs
            if null dirs
                then return $ k (Chunk files)
                else walk dirs $ k (Chunk files)
        walk dirs = foldr1 (>>>) $ map enumDir dirs
        getFilesDirs = do
            cnts <- map (dir </>) <$> getValidContents dir
            (,) <$> filterM doesFileExist  cnts
                <*> filterM isSearchableDir cnts

allFirstLines :: FilePath -> IO ()
allFirstLines dir = do
    i' <- enumDir dir $ joinI $ firstLineE printI
    run i'


main = do
    dir:_ <- getArgs
    allFirstLines dir

Answer 1

我不知道如何用迭代器来做，但这是基于管道的解决方案。此版本解决方案的一些优点是:

• 它绝不会一次将多个字节串 block 带入内存

• 它流式传输文件列表，因此它不会在具有大量直接子级的目录上阻塞

• 它递归地遍历目录树，就像你问的那样

其中一些东西很快就会出现在 pipes 实用程序库中:

import Control.Monad (when, unless)
import Control.Proxy
import Control.Proxy.Safe hiding (readFileS)
import qualified Data.ByteString as B
import qualified Data.ByteString.Char8 as B8
import System.Directory (readable, getPermissions, doesDirectoryExist)
import System.FilePath ((</>), takeFileName)
import System.Posix (openDirStream, readDirStream, closeDirStream)
import System.IO (openFile, hClose, IOMode(ReadMode), hIsEOF)

contents
     :: (CheckP p)
     => FilePath -> () -> Producer (ExceptionP p) FilePath SafeIO ()
contents path () = do
     canRead <- tryIO $ fmap readable $ getPermissions path
     when canRead $ bracket id (openDirStream path) closeDirStream $ \dirp -> do
         let loop = do
                 file <- tryIO $ readDirStream dirp
                 case file of
                     [] -> return ()
                     _  -> do
                         respond (path </> file)
                         loop
         loop

contentsRecursive
     :: (CheckP p)
     => FilePath -> () -> Producer (ExceptionP p) FilePath SafeIO ()
contentsRecursive path () = loop path
   where
     loop path = do
         contents path () //> \newPath -> do
             respond newPath
             isDir <- tryIO $ doesDirectoryExist newPath
             let isChild = not $ takeFileName newPath `elem` [".", ".."]
             when (isDir && isChild) $ loop newPath

readFileS
    :: (CheckP p)
    => Int -> FilePath -> () -> Producer (ExceptionP p) B.ByteString SafeIO ()
readFileS chunkSize path () =
    bracket id (openFile path ReadMode) hClose $ \handle -> do
        let loop = do
                eof <- tryIO $ hIsEOF handle
                unless eof $ do
                    bs <- tryIO $ B.hGetSome handle chunkSize
                    respond bs
                    loop
        loop

firstLine :: (Proxy p) => () -> Consumer p B.ByteString IO ()
firstLine () = runIdentityP loop
  where
    loop = do
        bs <- request ()
        let (prefix, suffix) = B8.span (/= '\n') bs
        lift $ B8.putStr prefix
        if (B.null suffix) then loop else lift $ B8.putStr (B8.pack "\n")

handler :: (CheckP p) => FilePath -> Session (ExceptionP p) SafeIO ()
handler path = do
    canRead <- tryIO $ fmap readable $ getPermissions path
    isDir   <- tryIO $ doesDirectoryExist path
    when (not isDir && canRead) $
        (readFileS 1024 path >-> try . firstLine) ()

main = runSafeIO $ runProxy $ runEitherK $
      contentsRecursive "/home" />/ handler

如果你想了解更多关于管道的知识，你可以从the tutorial开始。 .