#46 Downloader thread hang up when timeout

Author: code4craft

user-manual.md

@@ -27,12 +27,12 @@ webmagic使用maven管理依赖，在项目中添加对应的依赖即可使用w
 		<dependency>
             <groupId>us.codecraft</groupId>
             <artifactId>webmagic-core</artifactId>
-            <version>0.4.0</version>
+            <version>0.4.1</version>
         </dependency>
 		<dependency>
             <groupId>us.codecraft</groupId>
             <artifactId>webmagic-extension</artifactId>
-            <version>0.4.0</version>
+            <version>0.4.1</version>
         </dependency>
 
 #### 项目结构

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -105,6 +105,7 @@ public Page download(Request request, Task task) {
         }
         RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
                 .setConnectionRequestTimeout(site.getTimeOut())
+                .setSocketTimeout(site.getTimeOut())
                 .setConnectTimeout(site.getTimeOut())
                 .setCookieSpec(CookieSpecs.BEST_MATCH);
         if (site != null && site.getHttpProxy() != null) {

webmagic-scripts/src/main/java/us/codecraft/webmagic/scripts/ScriptConsole.java

@@ -93,6 +93,7 @@ private static void startSpider(Params params) {
                 .language(params.getLanguage()).scriptFromFile(params.getScriptFileName()).thread(params.getThread()).build();
         pageProcessor.getSite().setSleepTime(params.getSleepTime());
         pageProcessor.getSite().setAcceptStatCode(Sets.<Integer>newHashSet(200, 404, 500));
+        pageProcessor.getSite().setUserAgent("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5");
         Spider spider = Spider.create(pageProcessor).thread(params.getThread());
         spider.clearPipeline().addPipeline(new Pipeline() {
             @Override

webmagic-scripts/src/main/java/us/codecraft/webmagic/scripts/ScriptEnginePool.java

@@ -34,6 +34,7 @@ public ScriptEngine getEngine() {
 
     public void release(ScriptEngine scriptEngine){
         scriptEngines.add(scriptEngine);
+        availableCount.incrementAndGet();
     }
 
 }