FROM ubuntu:latest
RUN apt update && apt install -y \
    curl \
    file \
    git \
    libmecab-dev \
    make \
    mecab \
    mecab-ipadic-utf8 \
    sudo \
    wget \
    vim \
    xz-utils

# install anaconda3 you can change the version if you want to 
WORKDIR /opt
RUN wget https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh && \
    sh Anaconda3-2021.05-Linux-x86_64.sh -b -p /opt/anaconda3 && \
    rm -f Anaconda3-2021.05-Linux-x86_64.sh

ENV PATH=/opt/anaconda3/bin:$PATH
RUN pip install --upgrade pip && \
    pip install mecab-python3 && \
    pip install mojimoji && \
    pip install unidic-lite
WORKDIR /
# get the ipadic-neologd
RUN git clone --depth 1 https://github.com/neologd/mecab-ipadic-neologd.git && \
    echo yes | mecab-ipadic-neologd/bin/install-mecab-ipadic-neologd -n -a
CMD ["jupyter", "lab", "--ip=0.0.0.0", "--allow-root", "--LabApp.token=''"]

def make_reputation_csv(AREA_URL='https://travel.rakuten.co.jp/yado/aichi/A.html'):
    try:
        soup_area = modules.get_soup(AREA_URL)
        hotels = soup_area.select('section > div.htlHead> div.info > span')


        for hotel in hotels:

            hotel_ID = hotel['id'][-6:]
            hotel_review_url = f'https://travel.rakuten.co.jp/HOTEL/{hotel_ID}/review.html'

            # get data souce
            soup_hotel = modules.get_soup(hotel_review_url)

            # hotel name
            hotel_name = soup_hotel.select("#RthNameArea > h2")[0].text

            _reputations, _comments, _purposes, _companions, _dates = modules.make_object_lists(soup_hotel)

            # make df and to csv
            data = pd.DataFrame(
                        {'HotelName': hotel_name,
                         'Date': _dates, 
                         'Reputations': _reputations, 
                         'Comments': _comments, 
                         'Purposes': _purposes, 
                         'companions': _companions}
                    ).to_csv(f'./sample_csv/{hotel_ID}_reputation.csv', encoding='utf-8', index=False)

            time.sleep(2)
    except:
        print('実行は終了されました')

# データ読み込みを一括で
csv_lists = glob('./sample_csv/*')
data_dict = {}
preffixes = 'data'
for i, csv in enumerate(csv_lists):
    data_dict[preffixes+str(i+1)] = pd.read_csv(csv)


data = pd.concat([data for data in data_dict.values()]).reset_index(drop=True)

data['Date'] = pd.to_datetime(data['Date'], format='%Y年%m月')
data['Comments'] = [word.strip() for word in data['Comments']] # 不要な\n, \tを削除
data['Comment_words'] = data['Comments'].apply(lambda x: text_to_word(x))

tagger = MeCab.Tagger("-d /usr/lib/x86_64-linux-gnu/mecab/dic/mecab-ipadic-neologd")
tagger.parse("")

def text_to_word(text, stopword_pass="./stopwords/Japanese.txt"):
    """
    Tokenize texts with MeCab neologd

    Parameters
    ----------
    text: str
        text to tokenize
    stopword_pass: str
        path of stopwords

    Returns
    -------
    basic_words_list: str
        tokenized and basic text

    """

    #stopwordのリスト作成
    stopword_list =[]
    with open(stopword_pass, "r") as file:
        lines = file.readlines()

        stopword_list = [stopword for stopword in lines if stopword.strip()]

    #mojimojiで全角数字、英字を半角に統一。大文字もすべて小文字に統一する
    text = mojimoji.zen_to_han(text, kana=False).lower()
    #解析
    parse_text = tagger.parse(text)

    #ここから解析したものの原型などを取り出して抽出していく
    basic_words_list = []

    #各単語の解析結果ごとにsplitしていく
    split_parse_text = parse_text.split("\n")

    for parse_word in split_parse_text:
        #\tで区切り、表層と品詞の情報に分ける
        split_parse_word = parse_word.split("\t")
        surface_word = split_parse_word[0]

        #最終行はEOSのため、終了させる
        if surface_word =="EOS":
            break
        else:
            #品詞が動詞、形容詞の場合、原形を格納
            morph_info = split_parse_word[1]
            morphs = morph_info.split(",")
            #品詞情報
            morph = morphs[0]
            #原型
            #原型は品詞情報の後ろから3番目
            basic = morphs[-3]

            if morph == "記号":
                continue
            elif morph in("動詞", "形容詞") and basic not in stopword_list:
                basic_words_list.append(basic)

            elif morph =='名詞' and basic not in stopword_list:
                basic_words_list.append(basic)

    #最終的にまとめたものを半角スペースでjoinし、リストで返す
    basic_words_list = " ".join(basic_words_list)
    return basic_words_list

data_copy = data.copy()
X = data_copy[['Comment_words', 'Purposes', 'companions']]
X = pd.get_dummies(X, columns=['Purposes', 'companions'], drop_first=True)
y = data_copy['Reputations']

tfidf = TfidfVectorizer(min_df=4, max_df=.7)
x = tfidf.fit_transform(X['Comment_words'])
X_sparse = X.drop('Comment_words', axis=1).astype(pd.SparseDtype("int", np.nan))
X = hstack((x, X_sparse))

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
nb = MultinomialNB()
nb.fit(X_train, y_train)

y_pred = nb.predict(X_test)
print(accuracy_score(y_true=y_test, y_pred=y_pred))
print(f1_score(y_true=y_test, y_pred=y_pred, average='macro'))

0.5714285714285714
0.1865531914893617

param_dst = {'C': np.arange(0.01, 10, 0.3), 'kernel': ['rbf', 'poly'], 'gamma': [0.01, 0.1]}

RS = RandomizedSearchCV(estimator=SVC(), param_distributions=param_dst, n_iter=50, cv=10, random_state=123)
RS.fit(X_train, y_train)
y_pred2 = RS.predict(X_test)

print('f1', f1_score(y_true=y_test, y_pred=y_pred, average='macro'))
print('acc', accuracy_score(y_true=y_test, y_pred=y_pred2))

f1 0.17315837339223075
acc 0.6134453781512605