Python Bs4 + request 簡單的爬蟲

很早之前的python寫著玩的東西，如果爬一些簡單的東西，還用不到scrapy之類的爬蟲框架

#!/usr/bin/env python # -*- coding: utf-8 -*- from bs4 import BeautifulSoup import requests import threadingcommon_data = threading.Lock() winning_data = threading.Lock()def splite_table2(element, encoding, champions):winning_data.acquire()div_len = len(element)dic = dict()for i in range(div_len):span_element = element[i].find_all('div')span_len = len(span_element)for x in range(span_len):text = span_element[x].text.replace('\t', '').replace('\n', '')if text != '':dic[x] = textdic = sorted(dic.items(), key=lambda d: d[0], reverse=False)for key, values in dic:with open("勝利最高的技能加點.txt", 'a+', encoding=encoding) as f:if key == 0:f.write(champions + '={' + values + ',')elif key == 17:f.write(values + '}\n')else:f.write(values + ',')print(champions + '》》' + '勝利最高的技能加點.txt')winning_data.release()def splite_table(element, encoding, champions):common_data.acquire()div_len = len(element)dic = dict()for i in range(div_len):span_element = element[i].find_all('div')span_len = len(span_element)for x in range(span_len):text = span_element[x].text.replace('\t', '').replace('\n', '')if text != '':dic[x] = textdic = sorted(dic.items(), key=lambda d: d[0], reverse=False)for key, values in dic:with open("最常見的技能加點.txt", 'a+', encoding=encoding) as f:if key == 0:f.write(champions + '={' + values + ',')elif key == 17:f.write(values + '}\n')else:f.write(values + ',')print(champions + '》》' + '最常見的技能加點.txt')common_data.release()def jishukong(text, encoding, champions):document = BeautifulSoup(text, 'lxml')div_element = document.find_all('div', class_='skill-order clearfix')div_selected = div_element[0].find_all('div', class_='skill-selections')[1:5]div_selected2 = div_element[1].find_all('div', class_='skill-selections')[1:5]splite_table(div_selected, encoding, champions)splite_table2(div_selected2, encoding, champions)class Mythread(threading.Thread):def __init__(self, url, champ):self.url = urlself.champ = champsuper(Mythread, self).__init__()def run(self):self.url = self.url + self.champres = requests.get(self.url, verify=False)res.close()if res.ok and res.status_code == 200:res.encoding = res.apparent_encoding# spells_table = split_spells(res.text, res.apparent_encoding ,champions)# return spells_tableprint(self.champ + '》》等待處理')jishukong(res.text, res.apparent_encoding, self.champ)else:passdef main():Champs = "Aatrox", "Ahri", "Akali", "Alistar", "Amumu", "Anivia", "Annie", "Ashe", "AurelionSol", "Azir", "Bard", \"Blitzcrank", "Brand", "Braum", "Caitlyn", "Camille", "Cassiopeia", "Chogath", "Corki", "Darius", "Diana", \"Draven", "DrMundo", "Ekko", "Elise", "Evelynn", "Ezreal", "Fiddlesticks", "Fiora", "Fizz", "Galio", \"Gangplank", "Garen", "Gnar", "Gragas", "Graves", "Hecarim", "Heimerdinger", "Illaoi", "Irelia", "Ivern", \"Janna", "JarvanIV", "Jax", "Jayce", "Jhin", "Jinx", "Kaisa", "Kalista", "Karma", "Karthus", "Kassadin", \"Katarina", "Kayle", "Kayn", "Kennen", "Khazix", "Kindred", "Kled", "KogMaw", "Leblanc", "LeeSin", \"Leona", "Lissandra", "Lucian", "Lulu", "Lux", "Malphite", "Malzahar", "Maokai", "MasterYi", "MissFortune", \"MonkeyKing", "Mordekaiser", "Morgana", "Nami", "Nasus", "Nautilus", "Nidalee", "Nocturne", "Nunu", \"Olaf", "Orianna", "Ornn", "Pantheon", "Poppy", "Pyke", "Quinn", "Rakan", "Rammus", "RekSai", "Renekton", \"Rengar", "Riven", "Rumble", "Ryze", "Sejuani", "Senna", "Sett", "Shaco", "Shen", "Shyvana", "Singed", "Sion", "Sivir", \"Skarner", "Sona", "Soraka", "Swain", "Syndra", "TahmKench", "Taliyah", "Talon", "Taric", "Teemo", \"Thresh", "Tristana", "Trundle", "Tryndamere", "TwistedFate", "Twitch", "Udyr", "Urgot", "Varus", \"Vayne", "Veigar", "Velkoz", "Vi", "Viktor", "Vladimir", "Volibear", "Warwick", "Xayah", "Xerath", \"XinZhao", "Yasuo", "Yorick", "Zac", "Zed", "Ziggs", "Zilean", "Zoe", "Zyra"t_list = list()for i in range(len(Champs)):champs_name = Champs[i]i = Mythread('http://jishukong.com/champion?champKey=', champs_name)i.start()print(champs_name + '》》 開始搜集數據')t_list.append(i)for i in t_list:i.join()if __name__ == '__main__':main()

總結

以上是生活随笔為你收集整理的Python3 bs4 + requests 简单的爬虫 爬取LOL胜率加点的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。