Azureのサービス一覧を抽出する ※2021/04/09更新

import requests
from bs4 import BeautifulSoup
import re
import copy

def parse_html():
    url = 'https://azure.microsoft.com/ja-jp/services/'
    r = requests.get(url)
    soup = BeautifulSoup(r.content, 'lxml')

    groups = []
    service = []

    current_category = ''

    for div1 in soup.select('div#products-list'):
        for div2 in div1.select('div.row'):
            category = div2.select('h2.product-category')
            if not category:
                for div3 in div2.select('div.column.medium-6'):
                    s = {}
                    span = div3.select('span', limit=1)
                    href = div3.select('a', limit=1)
                    description = div3.select('p.text-body4', limit=1)

                    if len(span) > 0 and len(href) > 0 and len(description) > 0:
                        s['service_name'] = span[0].text.rstrip('\n')
                        s['href'] = href[0].get('href').rstrip('\n')
                        s['description'] = description[0].text.rstrip('\n')
                        service.append(s)

            else:
                if current_category != category[0].text:
                    if current_category == '':
                        current_category = category[0].text
                    elif len(service) > 0:
                        group = {}
                        tmp = copy.deepcopy(service)
                        group['category_name'] = current_category
                        group['service'] = tmp

                        groups.append(group)

                        current_category = category[0].text
                        service.clear()

        if len(service) > 0:
            group = {}
            group['category_name'] = current_category
            group['service'] = service
            groups.append(group)

    return (groups)


def print_markdown(services):
    base_url = 'https://azure.microsoft.com'

    for s in services:
        print('### %s' % (s['category_name']))

        print()
        print('| サービス名 | 説明 |')
        print('| --- | --- |')

        for service in s['service']:
            name = service['service_name']
            description = service['description']
            if (service['href'].startswith('/')):
                href = base_url + service['href']
            else:
                href = service['href']

            print('| [%s](%s) | %s |' % (name, href, description))
        print()


print_markdown(parse_html())