concurrency - Golang 上传整个目录并发返回到许多打开的文件

Question

我正在尝试将整个目录上传到服务器。它适用于小目录，但如果有 100 多张图片，它会返回“对许多打开的文件”的错误。我在读取文件后立即关闭文件。知道如何解决这个问题吗？

这是我的代码

    func uploadDir(path string) error {
    dir, err := os.Open(path)
    if err != nil {
        return err
    }

    files, err := dir.Readdirnames(-1)
    if err != nil {
        return err
    }
    dir.Close()

    errChan := make(chan error)
    resChan := make(chan *client.PutResult)
    remaining := len(files)
    for _, file := range files {
        file := file
        go func() {
            file, err := os.Open(path + "/" + file)
            if err != nil {
                errChan <- err
            }
            c := client.NewClient(os.Getenv("DROPS_SERVER"))
            res, err := c.Upload(client.NewUploadHandleFromReader(file))
            file.Close()
            if err != nil {
                errChan <- err
            }
            resChan <- res
        }()
    }

    for {
        select {
        case res := <-resChan:
            log.Println(res)
            remaining--
        case err := <-errChan:
            if err != nil {
                return err
            }
        }
        if remaining == 0 {
            break
        }
    }
    return nil
}

Answer 1

原始代码不限制事件的go例程的数量，因此不限制打开的文件描述符的数量。一些操作系统对打开的文件描述符的数量有限制。解决方法是创建固定数量的 worker go 例程。

func uploadDir(path string) error {

    // Read directory and close.

    dir, err := os.Open(path)
    if err != nil {
        return err
    }
    names, err := dir.Readdirnames(-1)
    if err != nil {
        return err
    }
    dir.Close()

    // Copy names to a channel for workers to consume. Close the
    // channel so that workers stop when all work is complete.

    namesChan := make(chan string, len(names))
    for _, name := range names {
        namesChan <- name
    }
    close(namesChan)

    // Create a maximum of 8 workers

    workers := 8
    if len(names) < workers {
        workers = len(names)
    }

    errChan := make(chan error, 1)
    resChan := make(chan *client.PutResult, len(names))

    // Run workers

    for i := 0; i < workers; i++ {
        go func() {
            // Consume work from namesChan. Loop will end when no more work.
            for name := range namesChan {
                file, err := os.Open(filepath.Join(path, name))
                if err != nil {
                    select {
                    case errChan <- err:
                        // will break parent goroutine out of loop
                    default:
                       // don't care, first error wins
                    }
                    return
                }
                c := client.NewClient(os.Getenv("DROPS_SERVER"))
                res, err := c.Upload(client.NewUploadHandleFromReader(file))
                file.Close()
                if err != nil {
                    select {
                    case errChan <- err:
                        // will break parent goroutine out of loop
                    default:
                       // don't care, first error wins
                    }
                    return
                }
                resChan <- res
            }
        }()
    }

    // Collect results from workers 

    for i := 0; i < len(names); i++ {
        select {
        case res := <-resChan:
            log.Println(res)
        case err := <-errChan:
            return err
        }
    }
    return nil
}

作为奖励，我修改了 channel 大小和发送操作，以便 goroutine 在出现错误时不会卡住。