有価証券報告書(XBRL)の「従業員の状況」から従業員数や平均年収の数値を取得し整形する試み(その１)

import requests
from bs4 import BeautifulSoup
import os
import zipfile
import re 
import xml.etree.ElementTree as ET
import pandas as pd

def fn_xbrl(sic,url):
       #XBRLダウンロード
       fn = str(sic) +".zip"
       os.system("wget -O " + str(fn) + " " + str(url))

       # ZIP解凍
       with zipfile.ZipFile( str(fn), 'r' )  as myzip:
          infos = myzip.infolist()
          for info in infos:
              base, ext = os.path.splitext(info.filename)
              if  str(base).find('Public')>0 and ext == '.xbrl':
                  myzip.extract(info.filename)
                  # XBRLパース 
                  print('■' + info.filename)
                  dict = fn_parse(sic , info.filename)

def fn_srch(sic):
        # 有報キャッチャー検索
        url='http://resource.ufocatch.com/atom/edinet/query/'+str(sic)+'0'
        r = requests.get(url)
        soup = BeautifulSoup( r.text,'html.parser')
        enty = soup.find_all("entry")

        for i in enty:
              ttl=i.find_all("title")
              # 有報のXBRL
              if str(ttl).find('有価証券報告書')>-1and str(ttl).find('訂正')==-1:
                  lnks=i.find_all("link")
                  print(ttl)
                  print(lnks[0])
                  print(lnks[1])
                  url =lnks[1].get('href')
                  fn = fn_xbrl(sic, url) if url !=''else 'hoge'

def fn_parse(sic , xbrl_path):

    ### タクソノミーURLの取得(using beautiful Soup)
    ff = open( xbrl_path , "r" ,encoding="utf-8" ).read() 
    soup = BeautifulSoup( ff ,"html.parser")

    x = soup.find_all("xbrli:xbrl" )
    x = str(x)[0:2000] 
    x = x.rsplit("xmlns")

    # 該当attr（crp）の取得
    crp  =['{' + i.replace( i[0:11] ,"").rstrip(' ').replace('"',"") + '}' for i in x[1:10] if i[0:11] == ':jpcrp_cor=']
    dei  =['{' + i.replace( i[0:11] ,"").rstrip(' ').replace('"',"") + '}' for i in x[1:10] if i[0:11] == ':jpdei_cor=']
    crp,dei = crp[0] ,dei[0]

    #df=pd.DataFrame(index=[], columns=[])
    #df.to_csv('test', header=False)

    # XML項目の取得(using elementTree)
    tree = ET.parse( xbrl_path ) 
    root = tree.getroot() 

    for c1 in root:
          if c1.tag==crp+'InformationAboutEmployeesTextBlock':
             tx = str(c1.text)
             df = pd.read_html( tx ,flavor='bs4' )
             for i in df:
                  if i.iloc[0, 0].find('従業員数')>-1 and i.iloc[0, 1].find('平均年齢')>-1 :
                     print( i )

# main
if __name__ == '__main__':

      sic=1301
      fn_srch(sic)