Webmagic爬虫類の例
13045 ワード
に頼る
<dependency>
<groupId>us.codecraftgroupId>
<artifactId>webmagic-coreartifactId>
<version>0.7.3version>
dependency>
<dependency>
<groupId>us.codecraftgroupId>
<artifactId>webmagic-extensionartifactId>
<version>0.7.3version>
dependency>
/**
*
* @Author zhengyingjun
* @Description :demo
* : webmagic
* @Date 2019/4/16 21:39
**/
/** pageprocessor
*
*/
public class Start implements PageProcessor {
public static void main(String[] args) {
Spider.create(new Start()).addUrl("http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html")
.run();
}
/**
* , 、 、
*/
private Site site = Site.me()
.setDomain("blog.sina.com.cn")
.setSleepTime(3000)
.setUserAgent(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
/**
* d
*/
public static final String URL_LIST = "http://blog\\.sina\\.com\\.cn/s/articlelist_1487828712_0_\\d+\\.html";
/**
* “http://blog.sina.com.cn/s/blog_58ae76e80100g8au.html”, blog_
*/
public static final String URL_POST = "http://blog\\.sina\\.com\\.cn/s/blog_\\w+\\.html";
/**
*
* @param page
*/
@Override
public void process(Page page) {
//
if (page.getUrl().regex(URL_LIST).match()) {
page.addTargetRequests(page.getHtml().xpath("//div[@class=\"articleList\"]").links().regex(URL_POST).all());
//
page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());
//
} else {
//
page.putField("title", page.getHtml().xpath("//div[@class='articalTitle']/h2"));
//
page.putField("content", page.getHtml().xpath("//div[@id='articlebody']//div[@class='articalContent']"));
//
page.putField("date",
page.getHtml().xpath("//div[@id='articlebody']//span[@class='time SG_txtc']").regex("\\((.*)\\)"));
}
}
@Override
public Site getSite() {
return site;
}
}