MiseringThread.java 解析頁面線程

http://injavawetrust.iteye.com

package com.iteye.injavawetrust.miner;import java.util.Set;import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory;/*** 解析頁面線程* @author InJavaWeTrust**/ public class MiseringThread extends Thread {private static final Log LOG = LogFactory.getLog(MiseringThread.class);private MinerConfig config = null;public MiseringThread(MinerConfig config) {this.config = config;}@Overridepublic void run() {while (!MinerMonitorThread.done) {misering();}}private synchronized void misering() {Html html = MinerQueue.waitingMiseringPoll(); // 等待提取URL的分析頁面出隊列if (null == html || MinerUtil.isBlank(html.getHtml())) {return;}//當前頁面深度<爬取深度 取出當前頁面全部URLif (html.getDepth() < config.getMaxDepth()) {LOG.info("MiseringThread獲取頁面[" + html.getUrl() + "]下所有URL。。。。。。 當前線程 [" + Thread.currentThread().getName() + "]");Set<String> urls = MinerUtil.getAllUrl(html.getUrl());for(String url : urls){if(null == url || url.equals("")){continue;}if(url.substring(url.length() - 1, url.length()).equals("/")){url = url.substring(0, url.length() - 1);}MinerUrl minerUrl = new MinerUrl();minerUrl.setUrl(url);minerUrl.setDepth(html.getDepth() + 1); // 爬取深度+1// 判斷URL列表是否包含關鍵字if(!MinerUtil.checkKeys(url, config.getKeys())){continue;}// 添加到待訪問隊列，每個URL只訪問一次MinerQueue.addUnVisited(minerUrl);// 將頁面URL 添加到URL隊列 保證每個URL只訪問一次MinerQueue.addUrlSet(minerUrl.getUrl()); }}}}

?

返回列表

?

轉載于:https://www.cnblogs.com/muyuge/p/6152077.html

總結

以上是生活随笔為你收集整理的MiseringThread.java 解析页面线程的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。