Merge branch 'release/0.7.6'

Author: sutra

pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>us.codecraft</groupId>
         <artifactId>webmagic-parent</artifactId>
-        <version>0.7.5</version>
+        <version>0.7.6</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -52,8 +52,8 @@
         </dependency>
 
         <dependency>
-            <groupId>commons-collections</groupId>
-            <artifactId>commons-collections</artifactId>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-collections4</artifactId>
         </dependency>
 
         <dependency>

webmagic-core/pom.xml

@@ -1,6 +1,20 @@
 package us.codecraft.webmagic;
 
-import org.apache.commons.collections.CollectionUtils;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -17,16 +31,6 @@
 import us.codecraft.webmagic.utils.UrlUtils;
 import us.codecraft.webmagic.utils.WMCollections;
 
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantLock;
-
 /**
  * Entrance of a crawler.<br>
  * A spider contains four modules: Downloader, Scheduler, PageProcessor and
@@ -106,7 +110,7 @@ public class Spider implements Runnable, Task {
 
     private Date startTime;
 
-    private int emptySleepTime = 30000;
+    private long emptySleepTime = 30000;
 
     /**
      * create a spider with pageProcessor.
@@ -305,32 +309,53 @@ protected void initComponent() {
     public void run() {
         checkRunningStat();
         initComponent();
-        logger.info("Spider {} started!",getUUID());
+        logger.info("Spider {} started!", getUUID());
+        // interrupt won't be necessarily detected
         while (!Thread.currentThread().isInterrupted() && stat.get() == STAT_RUNNING) {
-            final Request request = scheduler.poll(this);
-            if (request == null) {
-                if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
-                    break;
-                }
-                // wait until new url added
-                waitNewUrl();
-            } else {
-                threadPool.execute(new Runnable() {
-                    @Override
-                    public void run() {
-                        try {
-                            processRequest(request);
-                            onSuccess(request);
-                        } catch (Exception e) {
-                            onError(request, e);
-                            logger.error("process request " + request + " error", e);
-                        } finally {
-                            pageCount.incrementAndGet();
-                            signalNewUrl();
+            Request poll = scheduler.poll(this);
+            if (poll == null) {
+                if (threadPool.getThreadAlive() == 0) {
+                    //no alive thread anymore , try again
+                    poll = scheduler.poll(this);
+                    if (poll == null) {
+                        if (exitWhenComplete) {
+                            break;
+                        } else {
+                            // wait
+                            try {
+                                Thread.sleep(emptySleepTime);
+                                continue;
+                            } catch (InterruptedException e) {
+                                Thread.currentThread().interrupt();
+                                break;
+                            }
                         }
                     }
-                });
+                } else {
+                    // wait until new url added，
+                    if (waitNewUrl())
+                        //if interrupted
+                        break;
+                    continue;
+                }
             }
+            final Request request = poll;
+            //this may swallow the interruption
+            threadPool.execute(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        processRequest(request);
+                        onSuccess(request);
+                    } catch (Exception e) {
+                        onError(request, e);
+                        logger.error("process request " + request + " error", e);
+                    } finally {
+                        pageCount.incrementAndGet();
+                        signalNewUrl();
+                    }
+                }
+            });
         }
         stat.set(STAT_STOPPED);
         // release some resources
@@ -469,6 +494,7 @@ protected void sleep(int time) {
             Thread.sleep(time);
         } catch (InterruptedException e) {
             logger.error("Thread interrupted when sleep",e);
+            Thread.currentThread().interrupt();
         }
     }
 
@@ -565,16 +591,24 @@ public Spider addRequest(Request... requests) {
         return this;
     }
 
-    private void waitNewUrl() {
+    /**
+     *
+     * @return isInterrupted
+     */
+    private boolean waitNewUrl() {
+        // now there may not be any thread live
         newUrlLock.lock();
         try {
-            //double check
-            if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
-                return;
+            //double check，unnecessary, unless very fast concurrent
+            if (threadPool.getThreadAlive() == 0) {
+                return false;
             }
+            //wait for amount of time
             newUrlCondition.await(emptySleepTime, TimeUnit.MILLISECONDS);
+            return false;
         } catch (InterruptedException e) {
-            logger.warn("waitNewUrl - interrupted, error {}", e);
+            // logger.warn("waitNewUrl - interrupted, error {}", e);
+            return true;
         } finally {
             newUrlLock.unlock();
         }
@@ -772,7 +806,10 @@ public Scheduler getScheduler() {
      *
      * @param emptySleepTime In MILLISECONDS.
      */
-    public void setEmptySleepTime(int emptySleepTime) {
+    public void setEmptySleepTime(long emptySleepTime) {
+        if(emptySleepTime<=0){
+            throw new IllegalArgumentException("emptySleepTime should be more than zero!");
+        }
         this.emptySleepTime = emptySleepTime;
     }
 }

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -8,13 +8,14 @@
  */
 public interface SpiderListener {
 
-    public void onSuccess(Request request);
+    void onSuccess(Request request);
 
     /**
      * @deprecated Use {@link #onError(Request, Exception)} instead.
      */
     @Deprecated
-    public void onError(Request request);
+    default void onError(Request request) {
+    }
 
     default void onError(Request request, Exception e) {
         this.onError(request);

webmagic-core/src/main/java/us/codecraft/webmagic/SpiderListener.java

@@ -3,6 +3,7 @@
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Request;
 import us.codecraft.webmagic.Site;
+import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.selector.Html;
 
 /**
@@ -26,7 +27,7 @@ public Html download(String url) {
     /**
      * A simple method to download a url.
      *
-     * @param url url
+     * @param url     url
      * @param charset charset
      * @return html
      */
@@ -35,10 +36,26 @@ public Html download(String url, String charset) {
         return (Html) page.getHtml();
     }
 
+    @Deprecated
     protected void onSuccess(Request request) {
     }
 
+    /**
+     * @since 0.7.6
+     */
+    protected void onSuccess(Request request, Task task) {
+        this.onSuccess(request);
+    }
+
+    @Deprecated
     protected void onError(Request request) {
     }
 
+    /**
+     * @since 0.7.6
+     */
+    protected void onError(Request request, Task task, Throwable e) {
+        this.onError(request);
+    }
+
 }

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java

@@ -82,12 +82,12 @@ public Page download(Request request, Task task) {
         try {
             httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
             page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
-            onSuccess(request);
+            onSuccess(request, task);
             logger.info("downloading page success {}", request.getUrl());
             return page;
         } catch (IOException e) {
             logger.warn("download page {} error", request.getUrl(), e);
-            onError(request);
+            onError(request, task, e);
             return page;
         } finally {
             if (httpResponse != null) {
@@ -110,7 +110,7 @@ protected Page handleResponse(Request request, String charset, HttpResponse http
         String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue();
         Page page = new Page();
         page.setBytes(bytes);
-        if (!request.isBinaryContent()){
+        if (!request.isBinaryContent()) {
             if (charset == null) {
                 charset = getHtmlCharset(contentType, bytes);
             }

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -4,13 +4,16 @@
 import us.codecraft.webmagic.Site;
 
 /**
- * Interface to be implemented to customize a crawler.<br>
- * <br>
+ * Interface to be implemented to customize a crawler.
+ *
+ * <p>
  * In PageProcessor, you can customize:
- * <br>
- * start urls and other settings in {@link Site}<br>
- * how the urls to fetch are detected               <br>
- * how the data are extracted and stored             <br>
+ * </p>
+ * <ul>
+ * <li>start URLs and other settings in {@link Site}</li>
+ * <li>how the URLs to fetch are detected</li>
+ * <li>how the data are extracted and stored</li>
+ * </ul>
  *
  * @author [email protected] <br>
  * @see Site
@@ -20,17 +23,20 @@
 public interface PageProcessor {
 
     /**
-     * process the page, extract urls to fetch, extract the data and store
+     * Processes the page, extract URLs to fetch, extract the data and store.
      *
      * @param page page
      */
-    public void process(Page page);
+    void process(Page page);
 
     /**
-     * get the site settings
+     * Returns the site settings.
      *
      * @return site
      * @see Site
      */
-    public Site getSite();
+    default Site getSite() {
+        return Site.me();
+    }
+
 }

webmagic-core/src/main/java/us/codecraft/webmagic/processor/PageProcessor.java

@@ -1,9 +1,9 @@
 package us.codecraft.webmagic.selector;
 
-import org.apache.commons.collections.CollectionUtils;
 
 import java.util.ArrayList;
 import java.util.List;
+import org.apache.commons.collections4.CollectionUtils;
 
 /**
  * @author [email protected]

webmagic-core/src/main/java/us/codecraft/webmagic/selector/AbstractSelectable.java

@@ -1,6 +1,7 @@
 package us.codecraft.webmagic.selector;
 
 import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 
 import java.util.ArrayList;
@@ -11,34 +12,47 @@
  * @since 0.3.0
  */
 public abstract class BaseElementSelector implements Selector, ElementSelector {
+    private Document parse(String text) {
+        if (text == null) {
+            return null;
+        }
+
+        // Jsoup could not parse <tr></tr> or <td></td> tag directly
+        // https://stackoverflow.com/questions/63607740/jsoup-couldnt-parse-tr-tag
+        if ((text.startsWith("<tr>") && text.endsWith("</tr>"))
+                || (text.startsWith("<td>") && text.endsWith("</td>"))) {
+            text = "<table>" + text + "</table>";
+        }
+        return Jsoup.parse(text);
+    }
 
     @Override
     public String select(String text) {
         if (text != null) {
-            return select(Jsoup.parse(text));
+            return select(parse(text));
         }
         return null;
     }
 
     @Override
     public List<String> selectList(String text) {
         if (text != null) {
-            return selectList(Jsoup.parse(text));
+            return selectList(parse(text));
         } else {
             return new ArrayList<String>();
         }
     }
 
     public Element selectElement(String text) {
         if (text != null) {
-            return selectElement(Jsoup.parse(text));
+            return selectElement(parse(text));
         }
         return null;
     }
 
     public List<Element> selectElements(String text) {
         if (text != null) {
-            return selectElements(Jsoup.parse(text));
+            return selectElements(parse(text));
         } else {
             return new ArrayList<Element>();
         }