1.首先来对比一下java和python爬虫中比较常用的爬虫框架，如下图457
2.我们发现WebCollector爬虫工具其实网上使用的还是比较广泛的，因此建议大家使用这个框架去爬虫
1）可以自定义遍历策略，分页、AJAX请求等方式爬取
2）可以深度获取、锚文本获取、引用页面获取、POST参数传递、增量更新等。
4）实时爬取数据
6）集成selenium，可以对javascript生成信息进行抽取，可以使用chrome浏览器实时查看
3.下载链接，下载这个Jar包放在自己的java普通项目中即可
https://raw.githubusercontent.com/CrawlScript/WebCollector/master/webcollector-2.71-bin.zip
4.环境测试，运行下面的代码进行测试
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import org.jsoup.nodes.Document;
/**
* Crawling news from hfut news
*
* @author hu
*/
public class AutoNewsCrawler extends BreadthCrawler {
/**
* @param crawlPath crawlPath is the path of the directory which maintains
* information of this crawler
* @param autoParse if autoParse is true,BreadthCrawler will auto extract
* links which match regex rules from pag
*/
public AutoNewsCrawler(String crawlPath, boolean autoParse) {
super(crawlPath, autoParse);
/*start page*/
this.addSeed("http://news.hfut.edu.cn/list-1-1.html");
/*fetch url like http://news.hfut.edu.cn/show-xxxxxxhtml*/
this.addRegex("http://news.hfut.edu.cn/show-.*html");
/*do not fetch jpg|png|gif*/
this.addRegex("-.*\\.(jpg|png|gif).*");
/*do not fetch url contains #*/
this.addRegex("-.*#.*");
setThreads(50);
getConf().setTopN(100);
// setResumable(true);
}
@Override
public void visit(Page page, CrawlDatums next) {
String url = page.url();
/*if page is news page*/
if (page.matchUrl("http://news.hfut.edu.cn/show-.*html")) {
/*extract title and content of news by css selector*/
String title = page.select("div>h2").first().text();
String content = page.selectText("div#artibody");
System.out.println("URL:\n" + url);
System.out.println("title:\n" + title);
System.out.println("content:\n" + content);
/*If you want to add urls to crawl,add them to nextLink*/
/*WebCollector automatically filters links that have been fetched before*/
/*If autoParse is true and the link you add to nextLinks does not match the
regex rules,the link will also been filtered.*/
//next.add("http://xxxxxx.com");
}
}
public static void main(String args) throws Exception {
AutoNewsCrawler crawler = new AutoNewsCrawler("crawl", true);
/*start crawl with depth of 4*/
crawler.start(4);
}
}
评论
填写昵称与邮箱即可评论，无需登录。