gpt4 book ai didi

java-8 - 尽管超时,但 CloseableHttpClient.execute 每隔几周就会卡住一次

转载 作者:行者123 更新时间:2023-12-03 23:57:33 29 4
gpt4 key购买 nike

我们有一个 groovy 单例,它使用池大小为 200 的 PoolingHttpClientConnectionManager(httpclient:4.3.6) 来处理与搜索服务的非常高的并发连接并处理 xml 响应。

尽管有指定的超时,它大约每月卡住一次,但在其余时间运行得很好。

下面的 groovy 单例。方法retrieveInputFromURL 似乎在client.execute(get); 上阻塞了。


@Singleton(strict=false)
class StreamManagerUtil {
// Instantiate once and cache for lifetime of Signleton class

private static PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();

private static CloseableHttpClient client;

private static final IdleConnectionMonitorThread staleMonitor = new IdleConnectionMonitorThread(connManager);

private int warningLimit;
private int readTimeout;
private int connectionTimeout;
private int connectionFetchTimeout;

private int poolSize;
private int routeSize;

PropertyManager propertyManager = PropertyManagerFactory.getInstance().getPropertyManager("sebe.properties")

StreamManagerUtil() {
// Initialize all instance variables in singleton from properties file

readTimeout = 6
connectionTimeout = 6
connectionFetchTimeout =6

// Pooling
poolSize = 200
routeSize = 50

// Connection pool size and number of routes to cache
connManager.setMaxTotal(poolSize);
connManager.setDefaultMaxPerRoute(routeSize);

// ConnectTimeout : time to establish connection with GSA
// ConnectionRequestTimeout : time to get connection from pool
// SocketTimeout : waiting for packets form GSA

RequestConfig config = RequestConfig.custom()
.setConnectTimeout(connectionTimeout * 1000)
.setConnectionRequestTimeout(connectionFetchTimeout * 1000)
.setSocketTimeout(readTimeout * 1000).build();

// Keep alive for 5 seconds if server does not have keep alive header
ConnectionKeepAliveStrategy myStrategy = new ConnectionKeepAliveStrategy() {
@Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
HeaderElementIterator it = new BasicHeaderElementIterator
(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String param = he.getName();
String value = he.getValue();
if (value != null && param.equalsIgnoreCase
("timeout")) {
return Long.parseLong(value) * 1000;
}
}
return 5 * 1000;
}
};

// Close all connection older than 5 seconds. Run as separate thread.
staleMonitor.start();
staleMonitor.join(1000);

client = HttpClients.custom().setDefaultRequestConfig(config).setKeepAliveStrategy(myStrategy).setConnectionManager(connManager).build();
}

private retrieveInputFromURL (String categoryUrl, String xForwFor, boolean isXml) throws Exception {

URL url = new URL( categoryUrl );

GPathResult searchResponse = null
InputStream inputStream = null
HttpResponse response;
HttpGet get;
try {
long startTime = System.nanoTime();

get = new HttpGet(categoryUrl);
response = client.execute(get);

int resCode = response.getStatusLine().getStatusCode();

if (xForwFor != null) {
get.setHeader("X-Forwarded-For", xForwFor)
}

if (resCode == HttpStatus.SC_OK) {
if (isXml) {
extractXmlString(response)
} else {
StringBuffer buffer = buildStringFromResponse(response)
return buffer.toString();
}
}

}
catch (Exception e)
{
throw e;
}
finally {
// Release connection back to pool
if (response != null) {
EntityUtils.consume(response.getEntity());
}
}

}

private extractXmlString(HttpResponse response) {
InputStream inputStream = response.getEntity().getContent()

XmlSlurper slurper = new XmlSlurper()
slurper.setFeature("http://xml.org/sax/features/validation", false)
slurper.setFeature("http://apache.org/xml/features/disallow-doctype-decl", false)
slurper.setFeature("http://apache.org/xml/features/nonvalidating/load-dtd-grammar", false)
slurper.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)

return slurper.parse(inputStream)
}

private StringBuffer buildStringFromResponse(HttpResponse response) {
StringBuffer buffer= new StringBuffer();
BufferedReader rd = new BufferedReader(new InputStreamReader(response.getEntity().getContent()));
String line = "";
while ((line = rd.readLine()) != null) {
buffer.append(line);
System.out.println(line);
}
return buffer
}


public class IdleConnectionMonitorThread extends Thread {

private final HttpClientConnectionManager connMgr;
private volatile boolean shutdown;

public IdleConnectionMonitorThread
(PoolingHttpClientConnectionManager connMgr) {
super();
this.connMgr = connMgr;
}

@Override
public void run() {
try {
while (!shutdown) {
synchronized (this) {
wait(5000);
connMgr.closeExpiredConnections();
connMgr.closeIdleConnections(10, TimeUnit.SECONDS);
}
}
} catch (InterruptedException ex) {
// Ignore
}
}
public void shutdown() {
shutdown = true;
synchronized (this) {
notifyAll();
}
}
}

我还在日志中发现了这一点,这让我相信它是在等待响应数据时发生的


java.net.SocketTimeoutException:读取超时在 java.net.SocketInputStream.socketRead0(Native Method) at java.net.SocketInputStream.read(SocketInputStream.java:150) at java.net.SocketInputStream.read(SocketInputStream.java:121) ) 在 sun.security.ssl.InputRecord.readFully(InputRecord.java:465)

迄今为止的发现:
  • 我们使用的是 java 1.8u25。在类似的情况下有一个 Unresolved 问题
    https://bugs.openjdk.java.net/browse/JDK-8075484
  • HttpClient 有类似的报告 https://issues.apache.org/jira/browse/HTTPCLIENT-1589但这是固定的
    我们使用的 4.3.6 版本

  • 问题
  • 这可能是同步问题吗?根据我的理解,即使单例被多个线程访问,唯一的共享数据是缓存的 CloseableHttpClient
  • 此代码是否还有其他根本错误,可能导致此行为的方法?
  • 最佳答案

    我看不出您的代码有任何明显错误。不过,我强烈建议在连接管理器上设置 SO_TIMEOUT 参数,以确保它在创建时适用于所有新套接字,而不是在请求执行时。

    我也会帮助了解“卡住”的确切含义。工作线程是否被阻塞以等待从池中获取连接或等待响应数据?

    另请注意,如果服务器继续发送块编码数据位,则工作线程可能会出现“卡住”。像往常一样,客户端 session 的线路/上下文日志会有很大帮助
    http://hc.apache.org/httpcomponents-client-4.3.x/logging.html

    关于java-8 - 尽管超时,但 CloseableHttpClient.execute 每隔几周就会卡住一次,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/30070745/

    29 4 0
    Copyright 2021 - 2024 cfsdn All Rights Reserved 蜀ICP备2022000587号
    广告合作:1813099741@qq.com 6ren.com