amazon-web-services - 读取一个文件，将其压缩并将压缩后的输出通过管道传输到 S3

Question

我正在尝试编写一个工具来压缩目录并将压缩的输出流式传输到 S3，而无需先将其缓存在磁盘上。

package main

import (
    "compress/gzip"
    "io"
    "log"
    "os"
    "sync"

    "github.com/rlmcpherson/s3gof3r"
)

// log.Fatal() implies os.Exit(1)
func logerror(err error) {
    if err != nil {
        log.Fatalf("%s\n", err)
    }
}

func main() {
    k, err := s3gof3r.EnvKeys()
    logerror(err)

    // Open bucket we want to write a file to
    s3 := s3gof3r.New("", k)
    bucket := s3.Bucket("somebucket")

    // Open file to upload
    files, err := os.Open("somefile")
    logerror(err)
    defer files.Close()

    // open a PutWriter for S3 upload
    s3writer, err := bucket.PutWriter("somezipfile.gz", nil, nil)
    logerror(err)

    // Create io pipe for passing gzip output to putwriter input
    pipereader, pipewriter := io.Pipe()
    defer pipereader.Close()

    var wg sync.WaitGroup
    wg.Add(2)

    // Compress
    go func() {
        defer wg.Done()
        defer pipewriter.Close()

        gw := gzip.NewWriter(pipewriter)
        defer gw.Close()

        _, err := io.Copy(gw, files)
        logerror(err)
    }()

    // Transmit
    go func() {
        defer wg.Done()

        _, err := io.Copy(s3writer, pipereader)
        logerror(err)
    }()

    wg.Wait()

}

当我编译并运行它时，没有错误输出，S3 中也没有文件。如果有帮助的话，添加一堆打印品可以得到以下输出:

files:  &{0xc4200d0a00}
s3writer:  &{{https  <nil> somebucket.s3.amazonaws.com /somezipfile.gz  false  } 0xc4200d0a60 0xc420014540 20971520 [] 0 0xc42010e2a0 0 false <nil> {{} [0 0 0 0 0 0 0 0 0 0 0 0] 0} 0xc42010e300 0xc42010e360  0xc42035a740 0 97wUYO2YZPjLXqOLTma_Y1ASo.0IdeoKkif6pch60s3._J1suo9pUTCFwUj23uT.puzzDEHcV1KJPze.1EnLeoNehhBXeSpsH_.e4gXlNqBZ0HFsvyABJfHNYwUyXASx { []} 0}
pipewriter:  &{0xc42013c180}
gzipwriter:  &{{ [] 0001-01-01 00:00:00 +0000 UTC  255} 0xc420116020 -1 false <nil> 0 0 false [0 0 0 0 0 0 0 0 0 0] <nil>}
archive:  1283
upload:  606

感谢帮助!

Answer 1

我最终通过另一个途径获得了一些帮助，工作代码:|

package s3upload

import (
    "compress/gzip"
    "io"
    "os"

    "github.com/crielly/mongosnap/logger"
    "github.com/rlmcpherson/s3gof3r"
)

// S3upload streams compressed output to S3
func S3upload(toarchive, s3bucket, object string) {
    keys, err := s3gof3r.EnvKeys()
    logger.LogError(err)

    // Open bucket we want to write a file to
    s3 := s3gof3r.New("", keys)
    bucket := s3.Bucket(s3bucket)

    // open a PutWriter for S3 upload
    s3writer, err := bucket.PutWriter(object, nil, nil)
    logger.LogError(err)
    defer s3writer.Close()

    // Open a compressed writer to handle gzip and pass it to S3 writer
    zipwriter := gzip.NewWriter(s3writer)
    defer zipwriter.Close()

    // Open files we want archived
    file, err := os.Open(toarchive)
    logger.LogError(err)
    defer file.Close()

    // Pass opened file to compression writer
    _, err = io.Copy(zipwriter, file)
    logger.LogError(err)

}