Python爬虫学习笔记(例:猫眼映画ランキング上位100)

#        ，          
import json
import requests
from requests.exceptions import RequestException
import re
import time

#       
def get_one_page(url):
    try:   #   cookies,headers,params                                   
           #   ，        ——          
        cookies = {
            '__mta': '208109101.1580868743910.1580869334708.1580869391871.6',
            'uuid_n_v': 'v1',
            'uuid': 'F1A445C047BC11EAB2946104D9850EAF183D937C41A04091ADBCEF33315FF34C',
            '_csrf': '29dfa3ddd059c3a9db12143e29026730092990239ef75898af0b1751e3f8f4ff',
            '_lx_utm': 'utm_source%3DBaidu%26utm_medium%3Dorganic',
            '_lxsdk_cuid': '170131eb0cac8-06dda5778e29a9-b383f66-100200-170131eb0ca1',
            '_lxsdk': 'F1A445C047BC11EAB2946104D9850EAF183D937C41A04091ADBCEF33315FF34C',
            'mojo-uuid': 'b9015c74d0f312cee2a094515d087a8a',
            'mojo-session-id': '{"id":"71271fe111c7821d0b4446a38b2b7dcc","time":1580868744683}',
            'SL_GWPT_Show_Hide_tmp': '1',
            'SL_wptGlobTipTmp': '1',
            'lt': 'Yty7hi4TdDYZxEZhvpeYtaajBpoAAAAA-QkAAJKPH-4NCbTaSVLzCzf_TNVZrtzxSTBgGfPP4-CJmai8YCLwDdVe-7I0NZ-BGQdd1g',
            'lt.sig': 'WXkP89KWgZ0hwZC1_IufMi11VrA',
            'mojo-trace-id': '11',
            'Hm_lvt_703e94591e87be68cc8da0da7cbd0be2': '1580868743,1580869392',
            'Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2': '1580869392',
            '_lxsdk_s': '170131eb0cc-ef-30a-66b%7C%7C12',
        }

        headers = {
            'Connection': 'keep-alive',
            'Pragma': 'no-cache',
            'Cache-Control': 'no-cache',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
            'Sec-Fetch-User': '?1',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Sec-Fetch-Site': 'cross-site',
            'Sec-Fetch-Mode': 'navigate',
            'Referer': 'https://passport.meituan.com/account/secondverify?response_code=ff1857e9b5eb422eb5da21966a3dd53d&request_code=da67f3c519af4f57a311dab06eb7cced',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
        }

        params = (
            ('offset', '0'),
        )

        response = requests.get(url, headers=headers, params=params, cookies=cookies)
        response.encoding = response.apparent_encoding #           

        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None

#       Network     ，   Element   ，    JS  
#           
def parse_one_page(html):
    pattern = re.compile('
.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">(.*?).*?star">(.*?).*?releasetime">(.*?)'
                         + '.*?integer">(.*?).*?fraction">(.*?).*?
', re.S)
    items = re.findall(pattern, html) #  HTML      pattern
    #        
    for item in items:
        yield {
            'index': item[0],
            'image': item[1],
            'title': item[2],
            'actor': item[3].strip()[3:],#         
            'time': item[4].strip()[5:],
            'score': item[5] + item[6]
        }

#           
def write_to_file(content):
    with open('D:/result.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False) + '
') #JSON  dumps()          ，ensure_ascii=false      


def main(offset):
    url = 'http://maoyan.com/board/4?offset=' + str(offset) #        URL
    html = get_one_page(url) #       HTML
    for item in parse_one_page(html): #                    
        print(item) #  
        write_to_file(item) #    

#main        URL   ，       URL
if __name__ == '__main__':
    for i in range(10):
        main(offset=i * 10)
        time.sleep(1)