varnish - 使 Varnish 成为代理而不重定向

Question

我想将Varnish用作“智能”代理，并且几乎可以正常工作。这个想法是，某些请求应通过Varnish传递，到达后端并返回，所有其他请求应返回“synt”消息，表明特定响应不包含任何结果。

这与Varnish将301重定向返回到后端而不是仅来自实际后端的响应这一事实有所不同。

后端和缓存不在同一主机上(在这种情况下，甚至不在同一网络上)。

后端还运行一个单独的Varnish实例，并且此请求始终通过该实例传递。

// VCL.SHOW 0 1820 input
#
# This is an example VCL file for Varnish.
#
# It does not do anything by default, delegating control to the
# builtin VCL. The builtin VCL is called when there is no explicit
# return statement.
#
# See the VCL chapters in the Users Guide at https://www.varnish-cache.org/docs/
# and http://varnish-cache.org/trac/wiki/VCLExamples for more examples.

# Marker to tell the VCL compiler that this VCL has been adapted to the
# new 4.0 format.
vcl 4.0;

### Here starts my part of the VCL    

# Default backend definition. Set this to point to your content server.
backend default {
    .host = "myhost.mydomain";
    .port = "80";
}

sub vcl_recv {
    # Happens before we check if we have this in cache already.
    #
    # Typically you clean up the request here, removing cookies you don't need,
    # rewriting the request, etc.

    if (req.url ~ "^\/cgi-bin\/wspd_cgi\.sh/apiFlightSearch.p\?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27")
    {
       return (hash);
    }

    return (synth(750));
}

sub vcl_synth {
    if (resp.status == 750) {
        # Set a status the client will understand
        set resp.status = 200;
        # Create our synthetic response
        set resp.http.content-type = "text/xml";
        synthetic("<flights xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'><status><status>No flights</status></status></flights>");
        return(deliver);
    }
}

sub vcl_backend_response {
    # Happens after we have read the response headers from the backend.
    #
    # Here you clean the response headers, removing silly Set-Cookie headers
    # and other mistakes your backend does.

    set beresp.ttl = 10 s;
}

sub vcl_deliver {
    # Happens when we have all the pieces we need, and are about to send the
    # response to the client.
    #
    # You can do accounting or modifying the final object here.
}

### Here ends my part of the VCL. The rest I guess is built in.

// VCL.SHOW 1 5479 Builtin
/*-
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2014 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *

 *
 * The built-in (previously called default) VCL code.
 *
 * NB! You do NOT need to copy & paste all of these functions into your
 * own vcl code, if you do not provide a definition of one of these
 * functions, the compiler will automatically fall back to the default
 * code from this file.
 *
 * This code will be prefixed with a backend declaration built from the
 * -b argument.
 */

vcl 4.0;

#######################################################################
# Client side

sub vcl_recv {
    if (req.method == "PRI") {
        /* We do not support SPDY or HTTP/2.0 */
        return (synth(405));
    }
    if (req.method != "GET" &&
      req.method != "HEAD" &&
      req.method != "PUT" &&
      req.method != "POST" &&
      req.method != "TRACE" &&
      req.method != "OPTIONS" &&
      req.method != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
        return (pipe);
    }

    if (req.method != "GET" && req.method != "HEAD") {
        /* We only deal with GET and HEAD by default */
        return (pass);
    }
    if (req.http.Authorization || req.http.Cookie) {
        /* Not cacheable by default */
        return (pass);
    }
    return (hash);
}

sub vcl_pipe {
    # By default Connection: close is set on all piped requests, to stop
    # connection reuse from sending future requests directly to the
    # (potentially) wrong backend. If you do want this to happen, you can undo
    # it here.
    # unset bereq.http.connection;
    return (pipe);
}

sub vcl_pass {
    return (fetch);
}

sub vcl_hash {
    hash_data(req.url);
    if (req.http.host) {
        hash_data(req.http.host);
    } else {
        hash_data(server.ip);
    }
    return (lookup);
}

sub vcl_purge {
    return (synth(200, "Purged"));
}

sub vcl_hit {
    if (obj.ttl >= 0s) {
        // A pure unadultered hit, deliver it
        return (deliver);
    }
    if (obj.ttl + obj.grace > 0s) {
        // Object is in grace, deliver it
        // Automatically triggers a background fetch
        return (deliver);
    }
    // fetch & deliver once we get the result
    return (fetch);
}

sub vcl_miss {
    return (fetch);
}

sub vcl_deliver {
    return (deliver);
}

/*
 * We can come here "invisibly" with the following errors: 413, 417 & 503
 */
sub vcl_synth {
    set resp.http.Content-Type = "text/html; charset=utf-8";
    set resp.http.Retry-After = "5";
    synthetic( {"<!DOCTYPE html>
<html>
  <head>
    <title>"} + resp.status + " " + resp.reason + {"</title>
  </head>
  <body>
    <h1>Error "} + resp.status + " " + resp.reason + {"</h1>
    <p>"} + resp.reason + {"</p>
    <h3>Guru Meditation:</h3>
    <p>XID: "} + req.xid + {"</p>
    <hr>
    <p>Varnish cache server</p>
  </body>
</html>
"} );
    return (deliver);
}

#######################################################################
# Backend Fetch

sub vcl_backend_fetch {
    return (fetch);
}

sub vcl_backend_response {
    if (beresp.ttl <= 0s ||
      beresp.http.Set-Cookie ||
      beresp.http.Surrogate-control ~ "no-store" ||
      (!beresp.http.Surrogate-Control &&
        beresp.http.Cache-Control ~ "no-cache|no-store|private") ||
      beresp.http.Vary == "*") {
        /*
        * Mark as "Hit-For-Pass" for the next 2 minutes
        */
        set beresp.ttl = 120s;
        set beresp.uncacheable = true;
    }
    return (deliver);
}

sub vcl_backend_error {
    set beresp.http.Content-Type = "text/html; charset=utf-8";
    set beresp.http.Retry-After = "5";
    synthetic( {"<!DOCTYPE html>
<html>
  <head>
    <title>"} + beresp.status + " " + beresp.reason + {"</title>
  </head>
  <body>
    <h1>Error "} + beresp.status + " " + beresp.reason + {"</h1>
    <p>"} + beresp.reason + {"</p>
    <h3>Guru Meditation:</h3>
    <p>XID: "} + bereq.xid + {"</p>
    <hr>
    <p>Varnish cache server</p>
  </body>
</html>
"} );
    return (deliver);
}

#######################################################################
# Housekeeping

sub vcl_init {
    return (ok);
}

sub vcl_fini {
    return (ok);
}

curl的输出:

$ curl "thisandthatip.compute.amazonaws.com/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0" -i
HTTP/1.1 301 Moved Permanently
Date: Wed, 15 Mar 2017 07:14:11 GMT
Server: Apache/2.2.15 (Red Hat)
Location: http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
Content-Length: 514
Content-Type: text/html; charset=iso-8859-1
X-Varnish: 529144137
Via: 1.1 varnish-v4
X-Varnish: 98309 11
Age: 2
Via: 1.1 varnish-v4
Connection: keep-alive

<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>301 Moved Permanently</title>
</head><body>
<h1>Moved Permanently</h1>
<p>The document has moved <a href="http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&amp;to=AOK&amp;date=2017-05-20&amp;homedate=2017-05-27&amp;adults=2&amp;triptype=return&amp;children=0&amp;infants=0">here</a>.</p>
<hr>
<address>Apache/2.2.15 (Red Hat) Server at thisandthatip.eu-central-1.compute.amazonaws.com Port 80</address>
</body></html>

后端apache访问日志

127.0.0.1 - - [15/Mar/2017:08:09:49 +0100] "GET /cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=arn&to=aok&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0 HTTP/1.1" 200 994 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"

从AWS实例向后端发送请求不会呈现任何301重定向:

$ curl "myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=arn&to=aok&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0" -i
HTTP/1.1 200 OK
Date: Wed, 15 Mar 2017 08:54:14 GMT
Server: Apache/2.2.15 (Red Hat)
Cache-Control: max-age=1
Expires: Wed, 15 Mar 2017 08:54:15 GMT
Content-Type: text/xml
X-Varnish: 527559784
Age: 0
Via: 1.1 varnish-v4
Transfer-Encoding: chunked
Connection: keep-alive
Accept-Ranges: bytes

... Response body here ...

单个请求的完整varnishlog输出

*   << BeReq    >> 98314
-   Begin          bereq 98313 fetch
-   Timestamp      Start: 1489568144.701450 0.000000 0.000000
-   BereqMethod    GET
-   BereqURL       /cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
-   BereqProtocol  HTTP/1.1
-   BereqHeader    Host: thisandthatip.compute.amazonaws.com
-   BereqHeader    Upgrade-Insecure-Requests: 1
-   BereqHeader    User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36
-   BereqHeader    Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
-   BereqHeader    Accept-Language: sv-SE,sv;q=0.8,en-US;q=0.6,en;q=0.4
-   BereqHeader    X-Forwarded-For: ip.ip.ip.ip
-   BereqHeader    Accept-Encoding: gzip
-   BereqHeader    X-Varnish: 98314
-   VCL_call       BACKEND_FETCH
-   VCL_return     fetch
-   BackendClose   17 default(ip.ip.ip.ip,,80) toolate
-   BackendOpen    17 default(ip.ip.ip.ip,,80) 172.31.31.195 42868
-   Backend        17 default default(ip.ip.ip.ip,,80)
-   Timestamp      Bereq: 1489568144.730329 0.028878 0.028878
-   Timestamp      Beresp: 1489568144.759773 0.058322 0.029444
-   BerespProtocol HTTP/1.1
-   BerespStatus   301
-   BerespReason   Moved Permanently
-   BerespHeader   Date: Wed, 15 Mar 2017 08:55:44 GMT
-   BerespHeader   Server: Apache/2.2.15 (Red Hat)
-   BerespHeader   Location: http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
-   BerespHeader   Content-Length: 514
-   BerespHeader   Content-Type: text/html; charset=iso-8859-1
-   BerespHeader   X-Varnish: 526644873
-   BerespHeader   Age: 0
-   BerespHeader   Via: 1.1 varnish-v4
-   BerespHeader   Connection: keep-alive
-   TTL            RFC 120 -1 -1 1489568145 1489568145 1489568144 0 0
-   VCL_call       BACKEND_RESPONSE
-   TTL            VCL 10 10 0 1489568145
-   VCL_return     deliver
-   Storage        malloc s0
-   ObjProtocol    HTTP/1.1
-   ObjStatus      301
-   ObjReason      Moved Permanently
-   ObjHeader      Date: Wed, 15 Mar 2017 08:55:44 GMT
-   ObjHeader      Server: Apache/2.2.15 (Red Hat)
-   ObjHeader      Location: http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
-   ObjHeader      Content-Length: 514
-   ObjHeader      Content-Type: text/html; charset=iso-8859-1
-   ObjHeader      X-Varnish: 526644873
-   ObjHeader      Via: 1.1 varnish-v4
-   Fetch_Body     3 length stream
-   BackendReuse   17 default(ip.ip.ip.ip,,80)
-   Timestamp      BerespBody: 1489568144.759849 0.058398 0.000076
-   Length         514
-   BereqAcct      578 0 578 415 514 929
-   End

*   << Request  >> 98313
-   Begin          req 98312 rxreq
-   Timestamp      Start: 1489568144.701372 0.000000 0.000000
-   Timestamp      Req: 1489568144.701372 0.000000 0.000000
-   ReqStart       ip.ip.ip.ip 63485
-   ReqMethod      GET
-   ReqURL         /cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
-   ReqProtocol    HTTP/1.1
-   ReqHeader      Host: thisandthatip.compute.amazonaws.com
-   ReqHeader      Connection: keep-alive
-   ReqHeader      Upgrade-Insecure-Requests: 1
-   ReqHeader      User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36
-   ReqHeader      Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
-   ReqHeader      Accept-Encoding: gzip, deflate, sdch
-   ReqHeader      Accept-Language: sv-SE,sv;q=0.8,en-US;q=0.6,en;q=0.4
-   ReqHeader      X-Forwarded-For: ip.ip.ip.ip
-   VCL_call       RECV
-   VCL_return     hash
-   ReqUnset       Accept-Encoding: gzip, deflate, sdch
-   ReqHeader      Accept-Encoding: gzip
-   VCL_call       HASH
-   VCL_return     lookup
-   Debug          "XXXX MISS"
-   VCL_call       MISS
-   VCL_return     fetch
-   Link           bereq 98314 fetch
-   Timestamp      Fetch: 1489568144.759883 0.058511 0.058511
-   RespProtocol   HTTP/1.1
-   RespStatus     301
-   RespReason     Moved Permanently
-   RespHeader     Date: Wed, 15 Mar 2017 08:55:44 GMT
-   RespHeader     Server: Apache/2.2.15 (Red Hat)
-   RespHeader     Location: http://myserver.mydomain/cgi-bin/wspd_cgi.sh/apiFlightSearch.p?from=ARN&to=AOK&date=2017-05-20&homedate=2017-05-27&adults=2&triptype=return&children=0&infants=0
-   RespHeader     Content-Length: 514
-   RespHeader     Content-Type: text/html; charset=iso-8859-1
-   RespHeader     X-Varnish: 526644873
-   RespHeader     Via: 1.1 varnish-v4
-   RespHeader     X-Varnish: 98313
-   RespHeader     Age: 0
-   RespHeader     Via: 1.1 varnish-v4
-   VCL_call       DELIVER
-   VCL_return     deliver
-   Timestamp      Process: 1489568144.759907 0.058535 0.000024
-   Debug          "RES_MODE 2"
-   RespHeader     Connection: keep-alive
-   Timestamp      Resp: 1489568144.759933 0.058561 0.000026
-   Debug          "XXX REF 2"
-   ReqAcct        566 0 566 454 514 968
-   End

在AWS Amazon Linux上运行的Varnish 4.0.4。

Answer 1

301重定向操作不是由您的 Varnish 完成的。它由可能是您的后端的apache服务器完成。可以通过curl中的X-Server header 看到。

Varnish 的作用是代理请求并将其转发到声明myhost.mydomain的后端。实际上，Varnish会在启动时解析dns，并将请求转发到它获得的ip。

我在这里看到两件事要检查:

禁止您从 Varnish 缓存中请求(它可能会在测试期间的某个时间导致301并仍被缓存，情况并非如此，但最好从新的缓存开始)

curl 一下后端，看看是否获得301或200。

如果那不起作用，我将重新启动您的 Varnish 服务以刷新dns分辨率。