# Obey robots.txt rules
ROBOTSTXT_OBEY = False　　不遵從網(wǎng)站的robots.txt法則

# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3　　每次下載延遲3秒，防止造成網(wǎng)站攻擊

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 設(shè)置默認請求頭
'Accept-Language': 'en',
} # Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'xiaoshuo.pipelines.XiaoshuoPipeline': 300,　　數(shù)字越小，優(yōu)先級越高
}

FEED_EXPORT_ENCODING ='utf-8' 文件亂碼設(shè)置

轉(zhuǎn)載于:https://www.cnblogs.com/ShadowXie/p/9699800.html

總結(jié)

以上是生活随笔為你收集整理的scrapy爬虫-setting.py的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。