csv - 并发写入csv : “slice bounds out of range”

Question

我正在尝试从网站列表中提取证书信息，并将其写入csv。
我一直遇到相同的错误，但并非总是同时出现在不同的域中。

我在第63行收到错误:err := writer.Write(data)

main.analyzeDomains(0xc0000840c0, 0xc0000126c0)
        /root/BreakCert/SSLCert/src/main.go:95 +0x5f
created by main.main
        /root/BreakCert/SSLCert/src/main.go:113 +0x1bf
panic: runtime error: slice bounds out of range

goroutine 35 [running]:
bufio.(*Writer).Flush(0xc000024140, 0x400002400, 0x0)
        /usr/local/go/src/bufio/bufio.go:590 +0x1c0
bufio.(*Writer).WriteByte(0xc000024140, 0xc0000aa92c, 0xc000452500, 0x4d1)
        /usr/local/go/src/bufio/bufio.go:645 +0x96
bufio.(*Writer).WriteRune(0xc000024140, 0xc00000002c, 0x4d1, 0x4d1, 0x0)
        /usr/local/go/src/bufio/bufio.go:657 +0x1aa
encoding/csv.(*Writer).Write(0xc0000126c0, 0xc00060a000, 0x5, 0x8, 0x2, 0x1a)
        /usr/local/go/src/encoding/csv/writer.go:47 +0x4b8
main.storeCertificate(0xc00018cb00, 0xc0000126c0, 0xc000396380, 0x12)
        /root/BreakCert/SSLCert/src/main.go:63 +0x3e9
main.analyzeDomain(0xc000396380, 0x12, 0xc0000126c0)
        /root/BreakCert/SSLCert/src/main.go:88 +0x19d
main.analyzeDomains(0xc0000840c0, 0xc0000126c0)
        /root/BreakCert/SSLCert/src/main.go:95 +0x5f
created by main.main
        /root/BreakCert/SSLCert/src/main.go:113 +0x1bf
exit status 2

我的代码是

package main

import (
    "bufio"
    "crypto/rsa"
    "crypto/tls"
    "crypto/x509"
    "encoding/csv"
    "fmt"
    "log"
    "net"
    "os"
    "strconv"
    "strings"
    "sync"
    "time"
)

type CsvWriter struct {
    mutex     *sync.Mutex
    csvWriter *csv.Writer
}

func NewCsvWriter(fileName string) (*CsvWriter, error) {
    csvFile, err := os.Create(fileName)
    if err != nil {
        return nil, err
    }
    w := csv.NewWriter(csvFile)
    return &CsvWriter{csvWriter: w, mutex: &sync.Mutex{}}, nil
}

func (w *CsvWriter) Write(row []string) {
    w.mutex.Lock()
    w.csvWriter.Write(row)
    w.mutex.Unlock()
}

func (w *CsvWriter) Flush() {
    w.mutex.Lock()
    w.csvWriter.Flush()
    w.mutex.Unlock()
}

func storeCertificate(cert *x509.Certificate, writer *csv.Writer, domain string) {

    if v := cert.PublicKeyAlgorithm.String(); v == "RSA" {
        if len(cert.Issuer.Organization) != 0 {

            var data []string
            // Get Issuer Organization
            data = append(data, domain[:len(domain)-4])
            data = append(data, cert.Issuer.Organization[0])
            rsaPublicKey := cert.PublicKey.(*rsa.PublicKey)
            if rsaPublicKey != nil {
                data = append(data, rsaPublicKey.N.String())
                data = append(data, strconv.Itoa(rsaPublicKey.E))
                data = append(data, strconv.Itoa(rsaPublicKey.Size()))
                fmt.Println("Done: ", domain)
                if 6 <= len(data) {
                    data = data[:5]
                }
                err := writer.Write(data)
                if err != nil {
                    log.Fatal(err)
                }

            }

        }
    }

}

func analyzeDomain(domain string, writer *csv.Writer) {
    //fmt.Println("analyzing", domain)
    dialer := net.Dialer{}
    dialer.Timeout = 10 * time.Second
    conn, err := tls.DialWithDialer(&dialer, "tcp", domain, &tls.Config{
        InsecureSkipVerify: true,
    })
    if err != nil {
        fmt.Println(fmt.Sprintf("\x1b[31;1mfailed to connect to %s", domain), err, "\x1b[0m")
        return
    }
    defer conn.Close()
    for _, cert := range conn.ConnectionState().PeerCertificates {
        storeCertificate(cert, writer, domain)
    }
}

func analyzeDomains(queue chan string, writer *csv.Writer) {
    for {
        domain := <-queue
        analyzeDomain(domain, writer)

    }
}

func main() {
    // Creates a channel
    cs := make(chan string)

    // Creates result.csv
    file, err := os.Create("result.csv")

    //Verifies that the file has been created
    checkError("Cannot create file", err)
    defer file.Close()
    writer := csv.NewWriter(file)

    for i := 0; i < 80; i++ {
        go analyzeDomains(cs, writer)

    }
    writer.Flush()

    scanner := bufio.NewScanner(os.Stdin)
    for scanner.Scan() {
        line := scanner.Text()
        if !strings.Contains(line, ":") {
            line = line + ":443"
        }
        cs <- line
    }
    time.Sleep(2 * time.Second)
}

func checkError(message string, err error) {
    if err != nil {
        log.Fatal(message, err)
    }
}

像那样使用

cat domains | go run main.go

域每行包含一个网址。

Answer 1

这是OP问题的一种解决方案

// echo -e "google.com\ncnn.com\nstackoverflow.com" | go run main.go
package main

import (
  "bufio"
  "crypto/rsa"
  "crypto/tls"
  "crypto/x509"
  "encoding/csv"
  "io"
  "log"
  "net"
  "os"
  "strconv"
  "strings"
  "sync"
  "time"
)

func certToCSV(cert *x509.Certificate, domain string) []string {
  var data []string
  data = append(data, domain[:len(domain)-4])
  var org string
  if len(cert.Issuer.Organization) > 0 {
    org = cert.Issuer.Organization[0]
  }
  data = append(data, org)
  if cert.PublicKey != nil {
    rsaPublicKey := cert.PublicKey.(*rsa.PublicKey)
    data = append(data, rsaPublicKey.N.String())
    data = append(data, strconv.Itoa(rsaPublicKey.E))
    data = append(data, strconv.Itoa(rsaPublicKey.Size()))
  }
  return data
}

func getCerts(d string) ([]*x509.Certificate, error) {
  out := []*x509.Certificate{}
  dialer := net.Dialer{}
  dialer.Timeout = 10 * time.Second
  conn, err := tls.DialWithDialer(&dialer, "tcp", d, &tls.Config{
    InsecureSkipVerify: true,
  })
  if err != nil {
    return out, err
  }
  defer conn.Close()
  for _, cert := range conn.ConnectionState().PeerCertificates {
    if v := cert.PublicKeyAlgorithm.String(); v != "RSA" {
      log.Printf("%q not using RSA algorithm but %q", d, cert.PublicKeyAlgorithm)
      continue
    }
    if len(cert.Issuer.Organization) < 1 {
      log.Printf("%q does not have organization", d)
      continue
    }
    out = append(out, cert)
  }
  return out, err
}

func analyze(dst chan []string, src chan string, errs chan error) {
  for domain := range src {
    certs, err := getCerts(domain)
    if err != nil {
      errs <- err
      continue
    }
    for _, cert := range certs {
      record := certToCSV(cert, domain)
      dst <- record
    }
  }
}

func readCSVFile(dst chan string, fp string) error {
  file, err := os.Create(fp)
  if err != nil {
    return err
  }
  defer file.Close()

  scanner := bufio.NewScanner(os.Stdin)
  for scanner.Scan() {
    line := scanner.Text()
    if !strings.Contains(line, ":") {
      line = line + ":443"
    }
    dst <- line
  }
  return scanner.Err()
}

func readCSV(dst chan string, src io.Reader) error {
  scanner := bufio.NewScanner(src)
  for scanner.Scan() {
    line := scanner.Text()
    if !strings.Contains(line, ":") {
      line = line + ":443"
    }
    dst <- line
  }
  return scanner.Err()
}

func writeCSV(dst io.Writer, src chan []string, errs chan error) {
  w := csv.NewWriter(dst)
  for record := range src {
    if err := w.Write(record); err != nil {
      errs <- err
    }
    w.Flush()
  }
  if err := w.Error(); err != nil {
    errs <- err
  }
}

func main() {
  var wg sync.WaitGroup
  errs := make(chan error)
  src := make(chan string)
  t1 := make(chan []string)

  // synchronize all routines to close errs once
  go func() {
    wg.Wait()
    close(errs)
  }()

  var wg2 sync.WaitGroup
  // analyze multiple domains in //
  for i := 0; i < 4; i++ {
    wg.Add(1)
    wg2.Add(1)
    go func() {
      defer wg.Done()
      defer wg2.Done()
      analyze(t1, src, errs)
    }()
  }

  // synchronize with analyze routines to close t1
  go func() {
    wg2.Wait()
    close(t1)
  }()

  // write the csv file
  wg.Add(1)
  go func() {
    defer wg.Done()
    writeCSV(os.Stdout, t1, errs)
  }()

  // read the csv, fail if an error occurs reading the source
  wg.Add(1)
  go func() {
      defer wg.Done()
      err := readCSV(src, os.Stdin)
      if err != nil {
          log.Fatal(err)
      }
      close(src)
  }()

  // read and print errors, adjust exit code
  var exitCode int
  for err := range errs {
    log.Println(err)
    exitCode = 1
  }
  os.Exit(exitCode)
}