pythonは簡単な爬虫類検索機能を実現します
1756 ワード
from html.parser import HTMLParser
from urllib.request import urlopen
from urllib import parse
class LinkParser(HTMLParser):
def handle_starttag(self, tag, attrs):
if tag == 'a':
for (key, value) in attrs:
if key == 'href':
newUrl = parse.urljoin(self.baseUrl, value)
self.links = self.links + [newUrl]
def getLinks(self, url):
self.links = []
self.baseUrl = url
response = urlopen(url)
if response.getheader('Content-Type')=='text/html; charset=UTF-8':
htmlBytes = response.read()
htmlString = htmlBytes.decode("utf-8")
self.feed(htmlString)
return htmlString, self.links
else:
return "",[]
def spider(url, word, maxPages):
pagesToVisit = [url]
numberVisited = 0
foundWord = 4
while numberVisited < maxPages and pagesToVisit != [] and not foundWord:
numberVisited = numberVisited +1
url = pagesToVisit[0]
pagesToVisit = pagesToVisit[1:]
try:
print(numberVisited, " :", url)
parser = LinkParser()
data, links = parser.getLinks(url)
#print("data:",links)
pagesToVisit = pagesToVisit + links
if data.find(word)>-1:
foundWord = True
pagesToVisit = pagesToVisit + links
print(" ** !**")
except:
print(" ** !**")
if foundWord:
print(" ", word, " ", url)
else:
print(" ")
spider("http://yuedu.fm/"," ",100)