r - 将文本文件解析为数据框

Question

我需要将 haproxy 日志文件解析为数据帧。日志文件行看起来像这样。

Feb  6 12:14:14 localhost \
      haproxy[14389]: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in \
      static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} \
      {} "GET /index.html HTTP/1.1"

格式定义为(来自 here ):

 Field   Format                                Extract from the example above
      1   process_name '[' pid ']:'                            haproxy[14389]:
      2   client_ip ':' client_port                             10.0.1.2:33317
      3   '[' accept_date ']'                       [06/Feb/2009:12:14:14.655]
      4   frontend_name                                                http-in
      5   backend_name '/' server_name                             static/srv1
      6   Tq '/' Tw '/' Tc '/' Tr '/' Tt*                       10/0/30/69/109
      7   status_code                                                      200
      8   bytes_read*                                                     2750
      9   captured_request_cookie                                            -
     10   captured_response_cookie                                           -
     11   termination_state                                               ----
     12   actconn '/' feconn '/' beconn '/' srv_conn '/' retries*    1/1/1/1/0
     13   srv_queue '/' backend_queue                                      0/0
     14   '{' captured_request_headers* '}'                   {haproxy.1wt.eu}
     15   '{' captured_response_headers* '}'                                {}
     16   '"' http_request '"'                      "GET /index.html HTTP/1.1"

我当前的想法是逐行循环文件，解析每一行并将其附加到数据框:

read.haproxy <- function(filename)
{
  process_name   <- c()
  client_ip      <- c()
  ...
  http_request   <- c()

  con<- file(filename, 'r')
  while (length(input<- readLines(con, n=1000)> 0))  {
    for (i in 1:length(input)){
      # regex to split line into variables
      # append values to vectors
    }
  }
  # append vector to dataframe and return
}

问题:这种方法有效还是效率低下？有更传统的 R 方法可以做到这一点吗？

Answer 1

rex有一个vignette for parsing server logs 。虽然格式与您的日志不完全相同，但您应该能够相当轻松地将其适应您的情况。

就读取日志而言，假设文件适合内存，您最好的选择是首先使用 readLines() 读取整个文件，然后以下内容会将每个字段放入 data.frame 列。

x <- "Feb  6 12:14:14 localhost haproxy[14389]: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} {} \"GET /index.html HTTP/1.1\""
library(rex)
re <- rex(

  capture(name = "process_name", alpha),
  "[",
    capture(name = "pid", digits),
  "]:",
  spaces,
  capture(name = "client_ip", any_of(digit, ".")),
  ":",
  capture(name = "client_port", digits),
  spaces,
  "[",
    capture(name = "accept_date", except_some_of("]")),
  "]",
  spaces,
  capture(name = "frontend_name", non_spaces),
  spaces,
  capture(name = "backend_name", except_some_of("/")),
  "/",
  capture(name = "server_name", non_spaces),
  spaces,
  capture(name = "Tq", some_of("-", digit)),
  "/",
  capture(name = "Tw", some_of("-", digit)),
  "/",
  capture(name = "Tc", some_of("-", digit)),
  "/",
  capture(name = "Tr", some_of("-", digit)),
  "/",
  capture(name = "Tt", some_of("+", digit)),
  spaces,
  capture(name = "status_code", digits),
  spaces,
  capture(name = "bytes_read", some_of("+", digit)),
  spaces,
  capture(name = "captured_request_cookie", non_spaces),
  spaces,
  capture(name = "captured_response_cookie", non_spaces),
  spaces,
  capture(name = "termination_state", non_spaces),
  spaces,
  capture(name = "actconn", digits),
  "/",
  capture(name = "feconn", digits),
  "/",
  capture(name = "beconn", digits),
  "/",
  capture(name = "srv_conn", digits),
  "/",
  capture(name = "retries", some_of("+", digit)),
  spaces,
  capture(name = "srv_queue", digits),
  "/",
  capture(name = "backend_queue", digits),
  spaces,
  "{",
    capture(name = "captured_request_headers", except_any_of("}")),
  "}",
  spaces,
  "{",
    capture(name = "captured_response_headers", except_any_of("}")),
  "}",
  spaces,
  double_quote,
    capture(name = "http_request", non_quotes),
  double_quote)

re_matches(x, re)

#>   process_name   pid client_ip client_port              accept_date
#> 1            y 14389  10.0.1.2       33317 06/Feb/2009:12:14:14.655
#>   frontend_name backend_name server_name Tq Tw Tc Tr  Tt status_code
#> 1       http-in       static        srv1 10  0 30 69 109         200
#>   bytes_read captured_request_cookie captured_response_cookie
#> 1       2750                       -                        -
#>   termination_state actconn feconn beconn srv_conn retries srv_queue
#> 1              ----       1      1      1        1       0         0
#>   backend_queue captured_request_headers captured_response_headers
#> 1             0                   1wt.eu                          
#>               http_request
#> 1 GET /index.html HTTP/1.1