memory-leaks - Golang程序内存泄漏？

Question

我的 golang 程序(url 监视器)有内存泄漏，它最终被内核(oom)杀死。环境:

$ go version
go version go1.0.3

$ go env
GOARCH="amd64"
GOBIN=""
GOCHAR="6"
GOEXE=""
GOGCCFLAGS="-g -O2 -fPIC -m64 -pthread"
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GOOS="linux"
GOPATH="/data/apps/go"
GOROOT="/usr/local/go"
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64"
CGO_ENABLED="1"

代码:

package main

import (
    "bytes"
    "database/sql"
    "flag"
    "fmt"
    _ "github.com/Go-SQL-Driver/MySQL"
    "ijinshan.com/cfg"
    "log"
    "net"
    "net/http"
    "net/smtp"
    "os"
    "strconv"
    "strings"
    "sync"
    "time"
)

var (
    Log           *log.Logger
    Conf          cfg.KVConfig
    Debug         bool
    CpuCore       int
    HttpTransport = &http.Transport{
        Dial: func(netw, addr string) (net.Conn, error) {
            deadline := time.Now().Add(30 * time.Second)
            c, err := net.DialTimeout(netw, addr, 20*time.Second)
            if err != nil {
                return nil, err
            }

            c.SetDeadline(deadline)
            return c, nil
        },
        DisableKeepAlives: true,
    }
    HttpClient = &http.Client{
        Transport: HttpTransport,
    }
    WG            sync.WaitGroup
)

const (
    LogFileFlag   = os.O_WRONLY | os.O_CREATE | os.O_APPEND
    LogFileMode   = 0644
    LogFlag       = log.LstdFlags | log.Lshortfile
    GET_VIDEO_SQL = `SELECT B.Name, A.TSID, A.Chapter, A.ChapterNum, 
    IFNULL(A.Website, ''), IFNULL(A.Descr, ''), 
    IFNULL(A.VideoId, ''), IFNULL(AndroidWebURL, ''), IFNULL(IOSWebURL, ''), 
    IFNULL(AndroidURL, ''), IFNULL(AndroidURL2, ''), IFNULL(IOSURL, '')
    FROM Video A INNER JOIN TVS B ON A.TSID = B.ID LIMIT 200`

    HtmlHead = `<table border=1 width=100% height=100%><tr><td>节目名
    </td><td>tsid</td><td>章节</td><td>章节号</td><td>描述
    </td><td>videoid</td><td>网站</td><td>地址</td></tr>`
    HtmlTail = "</table>"
)

type videoInfo struct {
    name          string
    tsid          uint
    chapter       string
    chapterNum    uint
    descr         string
    videoId       string
    website       string
    androidWebUrl string
    iosWebUrl     string
    androidUrl    string
    androidUrl2   string
    iosUrl        string
}

func init() {
    var (
        confFile string
        err      error
    )

    // parse command argument:w
    flag.StringVar(&confFile, "c", "./vsmonitor.conf", " set config file path")
    flag.Parse()
    // read config
    if Conf, err = cfg.Read(confFile); err != nil {
        panic(fmt.Sprintf("Read config file \"%s\" failed (%s)",
            confFile, err.Error()))
    }
    // open log file
    file, err := os.OpenFile(Conf["log.file"], LogFileFlag, LogFileMode)
    if err != nil {
        panic(fmt.Sprintf("OpenFile \"%s\" failed (%s)", Conf["log.file"],
            err.Error()))
    }
    // init LOG
    Log = log.New(file, "", LogFlag)
    Debug = false
    i, err := strconv.ParseInt(Conf["cpucore.num"], 10, 32)
    if err != nil {
        panic(fmt.Sprintf("ParseInt \"%s\" failed (%s)", Conf["cpucore.num"],
            err.Error()))
    }

    CpuCore = int(i)
}

func getHttpStatusCode(url string) int {
    if url == "" {
        return 200
    }

    req, err := http.NewRequest("GET", url, nil)
    if err != nil {
        return 0
    }

    req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17")
    req.Header.Add("Connection", "close")
    resp, err := HttpClient.Do(req)
    if err != nil {
        return 0
    }

    defer resp.Body.Close()
    return resp.StatusCode
}

func sendMail(host, user, pwd, from, to, subject, body, mailType string) error {
    auth := smtp.PlainAuth("", user, pwd, strings.Split(host, ":")[0])
    cntType := fmt.Sprintf("Content-Type: text/%s;charset=UTF-8", mailType)
    msg := fmt.Sprintf("To: %s\r\nFrom: %s<%s>\r\nSubject: %s\r\n%s\r\n\r\n%s",
        to, from, user, subject, cntType, body)

    return smtp.SendMail(host, auth, user, strings.Split(to, ","), []byte(msg))
}

func getVideos(videoChan chan *videoInfo, htmlBuf *bytes.Buffer) error {
    defer HttpTransport.CloseIdleConnections()
    db, err := sql.Open("mysql", Conf["weikan.mysql"])
    if err != nil {
        return err
    }

    rows, err := db.Query(GET_VIDEO_SQL)
    if err != nil {
        db.Close()
        return err
    }

    for rows.Next() {
        video := &videoInfo{}
        err = rows.Scan(&video.name, &video.tsid, &video.chapter,
            &video.chapterNum,
            &video.website, &video.descr, &video.videoId, &video.androidWebUrl,
            &video.iosWebUrl, &video.androidUrl, &video.androidUrl2,
            &video.iosUrl)
        if err != nil {
            db.Close()
            return err
        }

        videoChan <- video
        WG.Add(1)
    }

    db.Close()
    // wait check url finish
    WG.Wait()
    // send mail
    for {
        if htmlBuf.Len() == 0 {
            Log.Print("no error found!!!!!!!!")
            break
        }

        Log.Print("found error !!!!!!!!")
        /*
        err := sendMail("smtp.gmail.com:587", "xxxx",
            "xxx", "xxx <xxx>",
            Conf["mail.to"], "xxxxx",
            HtmlHead+htmlBuf.String()+HtmlTail, "html")
        if err != nil {
            Log.Printf("sendMail failed (%s)", err.Error())
            time.Sleep(10 * time.Second)
            continue
        }
        */

        Log.Print("send mail")
        break
    }

    Log.Print("reset buf")
    htmlBuf.Reset()
    return nil
}

func checkUrl(videoChan chan *videoInfo, errChan chan string) {
    defer func() {
        if err := recover(); err != nil {
            Log.Print("rouintes failed : ", err)
        }
    }()

    for {
        video := <-videoChan
        ok := true
        errUrl := ""

        if code := getHttpStatusCode(video.androidWebUrl); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.androidWebUrl, code)
            ok = false
        }

        if code := getHttpStatusCode(video.iosWebUrl); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.iosWebUrl, code)
            ok = false
        }

        if code := getHttpStatusCode(video.androidUrl); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.androidUrl, code)
            ok = false
        }

        if code := getHttpStatusCode(video.androidUrl2); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.androidUrl2, code)
            ok = false
        }

        if code := getHttpStatusCode(video.iosUrl); code >= 400 {
            errUrl += fmt.Sprintf("%s (%d)<br />",
                video.iosUrl, code)
            ok = false
        }

        if !ok {
            errChan <- fmt.Sprintf(`<tr><td>%s</td><td>%d</td><td>%s</td>
            <td>%d</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>`,
                video.name, video.tsid, video.chapter, video.chapterNum,
                video.descr, video.videoId,
                video.website, errUrl)
            Log.Printf("\"%s\" (%s) —— \"%s\" checkurl err", video.name,
                video.chapter, video.descr)
        } else {
            Log.Printf("\"%s\" (%s) —— \"%s\" checkurl ok", video.name,
                video.chapter, video.descr)
            WG.Done()
        }
    }
}

func mergeErr(errChan chan string, htmlBuf *bytes.Buffer) {
    defer func() {
        if err := recover(); err != nil {
            Log.Print("rouintes failed : ", err)
        }
    }()

    for {
        html := <-errChan
        _, err := htmlBuf.WriteString(html)
        if err != nil {
            Log.Printf("htmlBuf WriteString \"%s\" failed (%s)", html,
                err.Error())
            panic(err)
        }

        WG.Done()
    }
}

func main() {
    videoChan := make(chan *videoInfo, 100000)
    errChan := make(chan string, 100000)
    htmlBuf := &bytes.Buffer{}
    defer func() {
        if err := recover(); err != nil {
            Log.Print("rouintes failed : ", err)
        }
    }()

    // check url
    for i := 0; i < CpuCore; i++ {
        go checkUrl(videoChan, errChan)
    }
    // merge error string then send mail
    go mergeErr(errChan, htmlBuf)

    for {
        // get Video and LiveSrc video source
        if err := getVideos(videoChan, htmlBuf); err != nil {
            Log.Printf("getVideos failed (%s)", err.Error())
            time.Sleep(10 * time.Second)
            continue
        }

        // time.Sleep(1 * time.Hour)
    }

    Log.Print("exit...")
}

代码有四个函数:

getHttpStatusCode

免费资源使用resp.Body.Close()

sendMail

我不需要手动释放资源

mergeErr

使用 htmlBuf(*bytes.Buffer) 连接错误字符串

getVideos

首先它获取视频 url，然后将它们发送到 videoChan，然后它等待所有例程完成它们的检查工作。然后发送邮件并重置 htmlBuf。

我没有找到任何需要免费的资源，但是。

$ top

显示:

PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND                                                                                                                      
6451 root      20   0 3946m 115m 2808 S  0.7  0.2   6:11.20 vsmonitor

VIRT 和 RES 会增长......

内存分析:

(pprof) top
Total: 10.8 MB
2.3  21.2%  21.2%      2.3  21.2% main.main
2.0  18.5%  39.8%      2.0  18.5% bufio.NewWriterSize
1.5  13.9%  53.7%      1.5  13.9% bufio.NewReaderSize
1.5  13.9%  67.6%      1.5  13.9% compress/flate.NewReader
0.5   4.6%  72.2%      0.5   4.6% net.newFD
0.5   4.6%  76.8%      0.5   4.6% net.sockaddrToTCP
0.5   4.6%  81.5%      4.5  41.7% net/http.(*Transport).getConn
0.5   4.6%  86.1%      2.5  23.2% net/http.(*persistConn).readLoop
0.5   4.6%  90.7%      0.5   4.6% net/textproto.(*Reader).ReadMIMEHeader
0.5   4.6%  95.4%      0.5   4.6% net/url.(*URL).ResolveReference

Answer 1

向您的程序添加一个选项非常容易，因此它会记录内存的使用位置。在你的程序中，没有什么让我感到非常错误的地方。您下载的文件很大吗？你可以做一个 HEAD 请求吗？我不知道这是否有帮助；如果您有大量请求，也许会。

在 http://blog.golang.org/2011/06/profiling-go-programs.html 的 Go 博客上有一篇关于内存分析的(旧的)文章和文档在 http://golang.org/pkg/runtime/pprof/和 http://golang.org/pkg/net/http/pprof/