��Python��、��、��xpath��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、��、

#  ��� ｆ���  url
#   �15 ������ �� � ��  �������� �url � ｆ��� �� ��  url
#  ���xpath ｆ���   � �
#   �� � � ｆ��� �ョ���  url � �  �, ｆ��� �� ����  �� � �   �  � ��
#  ���   �� � ��� ���� ��

#�   � ��� � ★� main_menu => page => item => source
import requests
from lxml import etree
import os
def get_html(url):
    headers=''
    req=requests.get(url)
    # print(type(req))
    ret=req.status_code
    if ret!=200 :
        print('� ��  ����    � ����� � �'+str(ret))
        return ('')
    req.encoding='gb2312'
    html=req.text
    return(html)

def get_main_menus(url):
    html=get_html(url)
    # print(html)
    selector=etree.HTML(html)
    main_menus=selector.xpath('//div[@class="contain"]/ul/li[position()<=9]/a')
    main_menus_dict={}
    for main_menu in main_menus :
        text=main_menu.xpath('text()')[0]
        if text == '� ���  ' or text == '���  ��' or text =='�� �� �' or text==' х��  ' or text==' х�� �' or text=='�ラ�╃�  ': # � � ��������ц� ��� ��� ��ュ� ��� ������� �
            pass
        else:
            if len(text)>=1 :
                href = url + main_menu.xpath('@href')[0]
                # print(text,href)
                main_menus_dict.update({text:href})
            else:
                pass
    return(main_menus_dict)

def get_pages(menus):
    url_list=[]
    # main_menus_dict={}

    menu_name=menu[0]
    menu_url=menu[1]
    html_one_menu = get_html(menu_url)
    # print(html_one_menu)
    selector_one_menu=etree.HTML(html_one_menu)
    # // *[ @ id = "header"] / div / div[3] / div[3] / div[2] / div[2] / div[2] / div / select / option[2]
    pages=selector_one_menu.xpath('//select/option[starts-with(@value,"list")]')
    print(menu_name,' � �'+str(len(pages))+' �:')
    if len(pages)>=1 :
        for page in pages :
            page_url=menu_url.split('index.html')[0] + page.xpath('@value')[0]
            # print(page_url)
            url_list.append(page_url)
    return(url_list)
def get_items(menu_name,page_url):
    html_one_page=get_html(page_url)
    selector_one_page = etree.HTML(html_one_page)
    #// *[ @ id = "header"] / div / div[3] / div[3] / div[2] / div[2] / div[2] / ul / table[23] / tbody / tr[2] / td[2] / b / a[2]
    items=selector_one_page.xpath('//a[@class="ulink" and not(contains(@href,"index.html")) ]')
    i=0
    for item in items :
        i = i+1
        if len(item.xpath('text()')) >=1 :
            item_name=item.xpath('text()')[0]
            item_url=url+item.xpath('@href')[0]
            # print(menu_name,item_name)
            get_source(menu_name, item_name, item_url)
    return(i)

def get_source(menu_name,item_name,item_url):
    print(item_name,item_url)
    html_one_item=get_html(item_url)
    selector_one_item = etree.HTML(html_one_item)
    sources=selector_one_item.xpath('//td[@style="WORD-WRAP: break-word"]/a')
    if len(sources)>=1 :
        dirname = 'G:\\� ���   �\\' + menu_name
        if not (os.path.exists(dirname)):
            os.mkdir(dirname)
        filename = dirname + '\\' + item_name.replace('/', '-').replace('?',' �').replace(':', ' �').replace('|','_').replace('"', '_') + '.txt'
        fp=open(filename,'w',encoding='utf-8')
        for source in sources:
            print(menu_name,item_name)
            if len(source.xpath('text()'))>=1 :
                source_text=source.xpath('text()')[0]
                source_url=source.xpath('@href')[0]
                print(source_text,source_url)
                fp.write(source_url+'
')
                fp.write(source_text+'
')
        fp.close()




if __name__ == '__main__' :
    url = 'http://www.ygdy8.com'
    main_menu_dict=get_main_menus(url)
    # print(main_menu_dict)
    total_page=0
    item_count = 0
    for menu in main_menu_dict.items():
        print('����',menu)
        menu_name = menu[0]
        page_url_list=[] # ������� ����ㄩ�ㄧ�    �㈢���� �
        page_url_list=get_pages(menu)
        total_page = total_page + len(page_url_list)

        for page_url in page_url_list :
            item_count=item_count+ get_items(menu_name,page_url)
        print(' � �' + str(total_page) + ' �'+str(item_count)+'��')