經過了幾天的摸索，照貓畫虎的把爬蟲的部分做完了。

但是很多原理性的東西都不是很理解，就是照著抄的，還需要繼續學習。

?

看這個目錄結構，只看.py的文件，.pyc的文件是運行的時候生成的不管它。

items.py:定義想要導出的數據

Pipelines.py:用于將數據導出

settings.py:告訴程序數據傳輸需要的文件

init.py:沒用到過還

（以上是我自己暫時的理解）

?

我最終抓取的是智聯招聘的信息，注意點都在注釋里

獲取職位連接的關鍵代碼，linkspider，其他文件都是生成時默認的

#encoding:utf-8 from scrapy.spider import BaseSpider from scrapy.http import FormRequest, Request from scrapy.selector import HtmlXPathSelector import os import sys import datetime import reclass ZhaoPinSpider(BaseSpider):name = "zhaopin"allowed_domains = ["zhaopin.com"]#url加上pd=1就只顯示今天新添加的職位 城市后面都固定為全國zlzp_urlpatten = "http://sou.zhaopin.com/jobs/searchresult.ashx?pd=1&jl={CITY}&kw={KEYWORD}&p={CURR_PAGE}"def __init__(self):self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0',\'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',\'Connection':'keep-alive'} self.start_urls = self.set_url()#set_url方法動態設定要抓取的鏈接列表def set_url(self):url_list = []#從配置文件中取出所有的關鍵字列表，逐個檢索keys = '大數據,hadoop,hive,hbase,spark,storm,sqoop,pig'for keyword in keys.split(','):url = self.zlzp_urlpattenurl = url.format(CITY='全國', KEYWORD=keyword, CURR_PAGE=1)url_list.append(url)return url_listdef start_requests(self): #該方法必須返回一個可迭代對象(iterable)。該對象包含了spider用于爬取的第一個Request。for url in self.start_urls:yield FormRequest(url,\headers = self.headers,\callback = self.parse)#使用回調函數def parse(self, response):hxs = HtmlXPathSelector(response)keyword = hxs.select('//div[@class="search"]//input[@name="KeyWord"]/@value').extract()[0]keyword = keyword.encode('utf-8')url = self.zlzp_urlpatten#找總頁數pageInfo = hxs.select('//div[@class="pagesDown"]//button/@onclick').extract()if pageInfo: #注意 只有一頁時找不到pageInfopageInfo = pageInfo[0]pattern = re.compile('.*?value,(.*?),.*', re.S)findPageNum = re.search(pattern, pageInfo)pageNum = int(findPageNum.group(1))else:pageNum = 1for curPage in range(1,pageNum + 1):each_url = url.format(CITY='全國', KEYWORD=keyword, CURR_PAGE=curPage)yield Request(each_url,callback=self.get_joburls_bypage)def get_joburls_bypage(self, response):hxs = HtmlXPathSelector(response)links = hxs.select('//td[@class="zwmc"]//a/@href').extract()# 獲得的信息都是當天的 直接入庫for link in links:if(link != 'http://e.zhaopin.com/products/1/detail.do'): #有的地方會冒出這個鏈接 去掉open('../output/link_output/link.txt', 'ab').write(link+'\n')

?

獲取具體職位信息的page相關代碼：

settings

# Scrapy settings for zhaopin_page project # # For simplicity, this file contains only the most important settings by # default. All the other settings are documented here: # # http://doc.scrapy.org/topics/settings.html # BOT_NAME = 'zhaopin_page' BOT_VERSION = '1.0'SPIDER_MODULES = ['zhaopin_page.spiders'] NEWSPIDER_MODULE = 'zhaopin_page.spiders' USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) ITEM_PIPELINES = {'zhaopin_page.FilePipelines.PagePipeline':5}

FilePipelines

# encoding: utf-8 import traceback import datetime import sys reload(sys) sys.setdefaultencoding( "utf-8" ) sys.path.append("../../../")class PagePipeline(object):#把解析后的內容放入文件中def process_item(self, item, spider):fname = '../output/page_output/' + item['file_id'] + '.txt'try:outfile = open(fname, 'wb')outfile.write(item['web_id']+self.getJobFieldSpt()+item['job_url']+self.getJobFieldSpt()+item['job_name']+self.getJobFieldSpt()+item['job_location']+self.getJobFieldSpt()+item['job_desc']+self.getJobFieldSpt()+item['edu']+self.getJobFieldSpt()+item['gender']+self.getJobFieldSpt()+item['language']+self.getJobFieldSpt()+item['major']+self.getJobFieldSpt()+item['work_years']+self.getJobFieldSpt()+item['salary']+self.getJobFieldSpt()+item['company_name']+self.getJobFieldSpt()+item['company_desc']+self.getJobFieldSpt()+item['company_address']+self.getJobFieldSpt()+item['company_worktype']+self.getJobFieldSpt()+item['company_scale']+self.getJobFieldSpt()+item['company_prop']+self.getJobFieldSpt()+item['company_website']+self.getJobFieldSpt()+self.getCurrentTimestamp())except Exception as e:print "ERROR GEN FILE!! >>> " + fnameprint traceback.format_exc()def getCurrentTimestamp(self):# 得到時間戳return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')def getJobFieldSpt(self):#得到生成的職位文件字段間的分隔符。使用ascii碼1，和hive中默認的分隔符相同 return chr(1)

items

# encoding: utf-8from scrapy.item import Item, Field#定義存放帖子內容的類 class PageItem(Item):#網站標識web_id = Field()#生成的文件名file_id = Field()#職位來源網址job_url = Field()#工作名稱job_name = Field()#工作地點 job_location = Field()#職位描述 job_desc = Field()#學歷要求 edu = Field()#性別要求 gender = Field()#語言要求 language = Field()#專業要求 major = Field()#工作年限 work_years = Field()#薪水范圍 salary = Field()#職位發布時間job_datetime = Field()#公司名稱 company_name = Field()#企業介紹company_desc = Field()#公司地址company_address = Field()#行業company_worktype = Field()#規模company_scale = Field()#性質company_prop = Field()#網址company_website = Field()

spider

# encoding: utf-8from scrapy.spider import BaseSpider from scrapy.http import FormRequest, Request from scrapy.selector import HtmlXPathSelector from zhaopin_page import items import traceback import sys import datetime import re#定義要抓取頁面的爬蟲類 class ZhaoPinPageSpider(BaseSpider):name = "page" start_urls = []def __init__(self): self.start_urls = self.set_url()#從jobs_task表中讀出要抓取的鏈接列表，放入數組中def set_url(self):url_list = []link_file = open('../output/link_output/link.txt', 'r')loops = 0for each_link in link_file:each_link = each_link.replace('\r','')each_link = each_link.replace('\n','')url_list.append(each_link)loops+=1if (loops == 100):breaklink_file.close()return url_listdef parse(self, response):try:#url中后面的數字串file_id = response.url.split("/")[-1].split(".")[0]hxs = HtmlXPathSelector(response)#獲取最上面一欄的內容title = '' #職位名companyName = '' #公司名basicInfo = hxs.select('//div[@class="fixed-inner-box"]').extract()[0] #會有兩個 后面有個clone的pattern = re.compile('.*?<h1>(.*?)</h1>.*?<a.*?>(.*?)</a>.*?', re.S)findBasicInfo = re.search(pattern, basicInfo)if findBasicInfo:title = findBasicInfo.group(1).strip() #職位名companyName = findBasicInfo.group(2).strip() #公司名#獲取左側基本公司信息 不能用正則表達式，因為有信息不全的情況 如http://jobs.zhaopin.com/297851037250005.htmcompanySize = '' #公司規模companyType = '' #公司性質 companyLine = '' #公司行業companyHost = '' #公司主頁companyAddress = '' #公司地companyInfo = hxs.select('//div[@class="company-box"]').extract()[0].encode('utf-8') #盡管只有一個，但是是列表形式，還是需要取出來if(companyInfo.find('公司規模：')>-1):companySize = companyInfo.split('公司規模：</span>')[1]companySize = companySize.split('<strong>')[1]companySize = companySize.split('</strong>')[0].strip()if(companyInfo.find('公司性質：')>-1):companyType = companyInfo.split('公司性質：</span>')[1]companyType = companyType.split('<strong>')[1]companyType = companyType.split('</strong>')[0].strip()if(companyInfo.find('公司行業：')>-1):companyLine = companyInfo.split('公司行業：</span>')[1]companyLine = companyLine.split('<strong>')[1]companyLine = companyLine.split('</a>')[0]companyLine = companyLine.split('>')[1].strip()if(companyInfo.find('公司主頁：')>-1):companyHost = companyInfo.split('公司主頁：</span>')[1]companyHost = companyHost.split('<strong>')[1]companyHost = companyHost.split('</a>')[0]companyHost = companyHost.split('>')[1].strip()if(companyInfo.find('公司地址：')>-1):companyAddress = companyInfo.split('公司地址：</span>')[1]companyAddress = companyAddress.split('<strong>')[1]companyAddress = companyAddress.split('</strong>')[0].strip()#獲取中部工作要求信息salary = '' #職位月薪 必須先聲明變量 否則會出錯address = '' #工作地點jobDateTime = '' #發布日期jobCategory = '' #工作性質experience = '' #工作經驗education = '' #最低學歷numberInNeed = '' #招聘人數jobType = '' #職位類別jobRequirementInfo = hxs.select('/html/body/div[4]/div[1]/ul').extract()[0]pattern = re.compile('.*?<strong>(.*?)</strong>\ .*?<strong>.*?<a.*?>(.*?)</a>\ .*?<strong>.*?<span.*?>(.*?)</span>\ .*?<strong>(.*?)</strong>\ .*?<strong>(.*?)</strong>\ .*?<strong>(.*?)</strong>\ .*?<strong>(.*?)</strong>\ .*?<strong>.*?target.*?>(.*?)</a>',re.S) #前面不能有空格或者TAB 否則匹配不上findJobRequirementInfo = re.search(pattern, jobRequirementInfo)if findJobRequirementInfo:salary = findJobRequirementInfo.group(1).strip() #職位月薪address = findJobRequirementInfo.group(2).strip() #工作地點jobDateTime = findJobRequirementInfo.group(3).strip() #發布日期jobCategory = findJobRequirementInfo.group(4).strip() #工作性質experience = findJobRequirementInfo.group(5).strip() #工作經驗education = findJobRequirementInfo.group(6).strip() #最低學歷numberInNeed = findJobRequirementInfo.group(7).strip() #招聘人數jobType = findJobRequirementInfo.group(8).strip() #職位類別#獲取描述信息detailInfo = hxs.select('//div[@class="tab-inner-cont"]').extract()jobDescribe = detailInfo[0]companyDescribe = detailInfo[1]pattern = re.compile('<.*?>|&nbsp',re.S) #刪除無用的信息jobDescribe = re.sub(pattern,'',jobDescribe).strip() #職位描述companyDescribe = re.sub(pattern,'',companyDescribe).strip() #公司介紹companySize = re.sub(pattern,'',companySize).strip()companyType = re.sub(pattern,'',companyType).strip()companyLine = re.sub(pattern,'',companyLine).strip()companyHost = re.sub(pattern,'',companyHost).strip()companyAddress = re.sub(pattern,'',companyAddress).strip() salary = re.sub(pattern,'',salary).strip()address = re.sub(pattern,'',address).strip()jobDateTime = re.sub(pattern,'',jobDateTime).strip()jobCategory = re.sub(pattern,'',jobCategory).strip()experience = re.sub(pattern,'',experience).strip()education = re.sub(pattern,'',education).strip()numberInNeed = re.sub(pattern,'',numberInNeed).strip()jobType = re.sub(pattern,'',jobType).strip()title = re.sub(pattern,'',title).strip()companyName = re.sub(pattern,'',companyName).strip()data = items.PageItem()data['web_id'] = "zhaopin"data['file_id'] = file_iddata['job_url'] = response.urldata['job_name'] = titledata['job_desc'] = jobDescribedata['gender'] = ""data['major'] = ""data['company_name'] = companyNamedata['job_datetime'] = jobDateTimedata['job_location'] = addressdata['work_years'] = experiencedata['edu'] = educationdata['salary'] = salarydata['company_desc'] = companyDescribedata['company_address'] = companyAddressdata['company_website'] = companyHostdata['language'] = ""data['company_worktype'] = companyLinedata['company_prop'] = companyTypedata['company_scale'] = companySize#更新任務表中抓取狀態#self.jobsTool.updateCrulInfo(ConfigPropObj.liepin_webid, response.url, 1, "")return dataexcept Exception as e:print "ERROR PARSE"print response.urlprint traceback.format_exc()

?

轉載于:https://www.cnblogs.com/dplearning/p/4925542.html

總結

以上是生活随笔為你收集整理的【飞谷六期】爬虫项目4的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。