Python 3爬虫類の解読画像防犯チェーン
6943 ワード
import requests, os, os.path, random
from bs4 import BeautifulSoup
def get_soup(url):
"""
soup
"""
req = requests.get(url,)
html = req.text
soup = BeautifulSoup(html, features='html.parser')
return soup
def get_pages(url):
"""
"""
soup = get_soup(url)
nums = soup.find_all('a', class_='page-numbers')
pages = int(nums[-2].text)
return pages
def get_menu(url):
"""
,
"""
soup = get_soup(url)
menu = []
menu_list = soup.find_all('a', target='_blank')
for i in menu_list:
result = i.find_all('img', class_='lazy')
if result:
#
name = result[0]['alt']
address = i['href']
menu.append([name, address])
return menu
def get_links(url):
"""
"""
soup = get_soup(url)
all_ = soup.find_all('a')
nums = []
for i in all_:
span = i.find_all('span')
if span:
nums.append(span[0].text)
return nums[-2]
def get_image(url, filename):
"""
filename
"""
soup = get_soup(url)
image = soup.find('img').attrs.get('src')
img_content = requests.get(image,).content
with open(filename, 'wb') as f:
f.write(img_content)
def main(page):
"""
page
"""
print(' %s ' % page)
page_url = url + '/page/' + str(page)
menu = get_menu(page_url)
print('@@@@@@@@@@@@@@@@ %s %s @@@@@@@@@@@@@@@@' % (page, len(menu)))
for i in menu:
# i[0]
dir_name = os.path.join('MeiZiTu', i[0])
if not os.path.exists(dir_name):
os.makedirs(dir_name)
pic_nums = int(get_links(i[1]))
print('
@@@@@@@@@ %s %d @@@@@@@@@
' % (i[0], pic_nums))
for pic in range(1, pic_nums + 1):
basename = str(pic) + '.jpg'
filename = os.path.join(dir_name, basename)
# MeiZiTu\ \6.jpg
pic_url = i[1] + '/' + str(pic)
# url http://www.mzitu.com/102205/3
if not os.path.exists(filename):
pass
print('......%s' % basename, get_image(pic_url, filename))
else:
print(filename + ' , ')
if __name__ == '__main__':
url = 'http://www.mzitu.com/'
pages = get_pages(url)
print('$$$$$$$$$ %s $$$$$$$$$$$$$$' % pages)
if not os.path.exists('MeiZiTu'):
os.mkdir('MeiZiTu')
page_start = input('Input the first page number:
')
page_end = input('Input the last page number:
')
if int(page_end) > int(page_start):
for page in range(int(page_start), int(page_end)):
main(page)
elif page_end == page_start:
main(page_end)
else:
print(" ,
")