import pandas as pd
# 读取数据
data = pd.read_csv("spotify_songs.csv")
# 数据速览
data.head()

# 数据基本信息
data.info()

# 数据工具库
import pandas as pd
import re

# 歌词处理的nlp工具库
import nltk
from nltk.corpus import stopwords
from collections import Counter
# nltk.download('stopwords')

# 读取数据
data = pd.read_csv("spotify_songs.csv")
# 字段选择
keep_cols = [x for x in data.columns if not x.startswith("track") and not x.startswith("playlist")]
keep_cols.append("playlist_genre")
df = data[keep_cols].copy()
# 只保留英文歌曲
subdf = df[(df.language == "en") & (df.playlist_genre != "latin")].copy().drop(columns = "language")


# 歌词规整化，全部小写
pattern = r"[^a-zA-Z ]"
subdf.lyrics = subdf.lyrics.apply(lambda x: re.sub(pattern, "", x.lower()))

# 移除停用词
subdf.lyrics = subdf.lyrics.apply(lambda x: ' '.join([word for word in x.split() if word not in (stopwords.words("english"))]))

# 查看歌词中的词汇出现的频次

# 连接所有歌词
all_text = " ".join(subdf.lyrics)
# 统计词频
word_count = Counter(all_text.split())
# 如果一个词在200首以上的歌里都出现，则保留，否则视作低频过滤掉
keep_words = [k for k, v in word_count.items() if v > 200]
# 构建一个副本
lyricdf = subdf.copy().reset_index(drop=True)
# 字段名称规范化
lyricdf.columns = ["audio_"+ x if not x in ["lyrics", "playlist_genre"] else x for x in lyricdf.columns]
# 歌词内容
lyricdf.lyrics = lyricdf.lyrics.apply(lambda x: Counter([word for word in x.split() if word in keep_words]))
# 构建词汇词频Dataframe
unpacked_lyrics = pd.DataFrame.from_records(lyricdf.lyrics).add_prefix("lyrics_")
# 缺失填充为0
unpacked_lyrics = unpacked_lyrics.fillna(0) 
# 拼接并删除原始歌词列
lyricdf = pd.concat([lyricdf, unpacked_lyrics], axis = 1).drop(columns = "lyrics")
# 排序
reordered_cols = [col for col in lyricdf.columns if not col.startswith("lyrics_")] + sorted([col for col in lyricdf.columns if col.startswith("lyrics_")])
lyricdf = lyricdf[reordered_cols]

# 存储为新的csv文件
lyricdf.to_csv("music_data.csv", index = False)

# 分组统计
by_genre = data.groupby("playlist_genre")["audio_key"].count().reset_index()
fig, ax = plt.subplots()

# 绘图
ax.bar(by_genre.playlist_genre, by_genre.audio_key)
ax.set_ylabel("Number of Observations")
ax.set_xlabel("Genre")
ax.set_title("Observations per Class")
ax.set_ylim(0, 4000)

# 每个柱子上标注数量
rects = ax.patches
for rect in rects:
    height = rect.get_height()
    ax.text(
        rect.get_x() + rect.get_width() / 2, height + 5, height, ha="center", va="bottom"
    )

# 把所有字段切分为音频和歌词列
audio = data[[x for x in data.columns if x.startswith("audio")]]
lyric = data[[x for x in data.columns if x.startswith("lyric")]]
# 让字段命名更简单一些
audio.columns = audio.columns.str.replace("audio_", "")
lyric.columns = lyric.columns.str.replace("lyric_", "")

# 常规数据工具库
import pandas as pd
import numpy as np
# 绘图
import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
# 数据处理
from sklearn.preprocessing import MinMaxScaler
from sklearn.decomposition import PCA

# 读取数据
data = pd.read_csv("music_data.csv")
# 切分为音频与歌词
audio = data[[x for x in data.columns if x.startswith("audio")]]
lyric = data[[x for x in data.columns if x.startswith("lyric")]]
# 特征字段
y = data.playlist_genre

# 数据幅度缩放 + PCA降维
scaler = MinMaxScaler()
audio_features = scaler.fit_transform(audio)
lyric_features = scaler.fit_transform(lyric)

pca = PCA()
lyric_pca  = pca.fit_transform(lyric_features)
var_explained_ratio = pca.explained_variance_ratio_
   
# Plot graph
fig, ax = plt.subplots()
# Reduce margins
plt.margins(x=0.01)
# Get cumuluative sum of variance explained
cum_var_explained = np.cumsum(var_explained_ratio)
# Plot cumulative sum
ax.fill_between(range(len(cum_var_explained)), cum_var_explained,
                alpha = 0.4, color = "tab:orange",
                label = "Cum. Var.")
ax.set_ylim(0, 1)
# Plot actual proportions
ax2 = ax.twinx()
ax2.plot(range(len(var_explained_ratio)), var_explained_ratio,
         alpha = 1, color = "tab:blue", lw  = 4, ls = "--",
         label = "Var per PC")
ax2.set_ylim(0, 0.005)

# Add lines to indicate where good values of components may be
ax.hlines(0.6, 0, var_explained_ratio.shape[0], color = "tab:green", lw = 3, alpha = 0.6, ls=":")
ax.hlines(0.8, 0, var_explained_ratio.shape[0], color = "tab:green", lw = 3, alpha = 0.6, ls=":")
# Plot both legends together
lines, labels = ax.get_legend_handles_labels()
lines2, labels2 = ax2.get_legend_handles_labels()
ax2.legend(lines + lines2, labels + labels2)
# Format axis as percentages
ax.yaxis.set_major_formatter(mtick.PercentFormatter(1))
ax2.yaxis.set_major_formatter(mtick.PercentFormatter(1)) 

# Add titles and labels
ax.set_ylabel("Cum. Prop. of Variance Explained")
ax2.set_ylabel("Prop. of Variance Explained per PC", rotation = 270, labelpad=30)
ax.set_title("Variance Explained by Number of Principal Components")
ax.set_xlabel("Number of Principal Components")

from sklearn.manifold import TSNE
import seaborn as sns

# Merge numeric labels with normalised audio data and lyric principal components
tsne_processed = pd.concat([
    pd.Series(y, name = "genre"),
    pd.DataFrame(audio_features, columns=audio.columns),
    # Add prefix to make selecting pcs easier later on
    pd.DataFrame(lyric_pca).add_prefix("lyrics_pc_")
          ], axis = 1)

# Get t-SNE values for a range of principal component cutoffs, 1806 is all PCs
all_tsne = pd.DataFrame()
for cutoff in ["1806", "1000", "500", "100"]:
    # Create t-SNE object
    tsne = TSNE(init = "random", learning_rate = "auto")
    # Fit on normalised features (excluding the y/label column)
    tsne_results = tsne.fit_transform(tsne_processed.loc[:, "audio_danceability":f"lyrics_pc_{cutoff}"])
    
    # neater graph
    if cutoff == "1806":
        cutoff = "All 1806"
    # Get results
    tsne_df = pd.DataFrame({"y":y,
                        "tsne-2d-one":tsne_results[:,0],
                       "tsne-2d-two":tsne_results[:,1],
                           "Cutoff":cutoff})
    # Store results
    all_tsne = pd.concat([all_tsne, tsne_df], axis = 0)
    
# Plot gridplot
g = sns.FacetGrid(all_tsne, col="Cutoff", hue = "y",
                col_wrap = 2, height = 6,
                palette=sns.color_palette("hls", 4),
               )
# Add plots
g.map(sns.scatterplot, "tsne-2d-one", "tsne-2d-two", alpha = 0.3)
# Add titles/legends
g.fig.suptitle("t-SNE Plots vs Number of Principal Components Included", y = 1)
g.add_legend()

# 通用库
import pandas as pd
import numpy as np
# 建模库
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
# 神经网络
from keras.layers import Dense, Input, LeakyReLU, BatchNormalization
from keras.callbacks import EarlyStopping
from keras import Model

# 定义自编码器
def autoencode(lyric_tr, n_components):
    """Build, compile and fit an autoencoder for
    lyric data using Keras. Uses a batch normalised,
    undercomplete encoder with leaky ReLU activations.
    It will take a while to train.
    --------------------------------------------------
    lyric_tr = df of lyric training data
    n_components = int, number of output dimensions
    from encoder
    """
    n_inputs = lyric_tr.shape[1]
    # 定义encoder
    visible = Input(shape=(n_inputs,))

    # encoder模块1
    e = Dense(n_inputs*2)(visible)
    e = BatchNormalization()(e)
    e = LeakyReLU()(e)
    # encoder模块2
    e = Dense(n_inputs)(e) 
    e = BatchNormalization()(e)
    e = LeakyReLU()(e)
    bottleneck = Dense(n_components)(e)

    # decoder模块1
    d = Dense(n_inputs)(bottleneck)
    d = BatchNormalization()(d)
    d = LeakyReLU()(d)
    # decoder模块2
    d = Dense(n_inputs*2)(d)
    d = BatchNormalization()(d)
    d = LeakyReLU()(d)
    # 输出层
    output = Dense(n_inputs, activation='linear')(d)
    # 完整的autoencoder模型
    model = Model(inputs=visible, outputs=output)

    # 编译
    model.compile(optimizer='adam', loss='mse')
    # 回调函数
    callbacks = EarlyStopping(patience = 20, restore_best_weights = True)
    # 训练模型
    model.fit(lyric_tr, lyric_tr, epochs=200,
                        batch_size=16, verbose=1, validation_split=0.2,
             callbacks = callbacks)
    
    # 在降维阶段，我们只用encoder部分就可以(对数据进行压缩)
    encoder = Model(inputs=visible, outputs=bottleneck)

    return encoder

# 数据预处理函数，主要是对特征列进行降维，标签列进行编码
def pre_process(train = pd.DataFrame,
                test = pd.DataFrame,
                reduction_method = "pca",
                n_components = 400):
    # 切分X和y
    y_train = train.playlist_genre
    y_test = test.playlist_genre
    X_train = train.drop(columns = "playlist_genre")
    X_test = test.drop(columns = "playlist_genre")
    
    # 标签编码为数字
    label_encoder = LabelEncoder()
    label_train = label_encoder.fit_transform(y_train)
    label_test = label_encoder.transform(y_test)

    # 对数据进行幅度缩放处理
    scaler = MinMaxScaler()
    X_norm_tr = scaler.fit_transform(X_train)
    X_norm_te = scaler.transform(X_test)

    # 重建数据
    X_norm_tr = pd.DataFrame(X_norm_tr, columns = X_train.columns)
    X_norm_te = pd.DataFrame(X_norm_te, columns = X_test.columns)

    # mode和key都设定为类别型
    X_norm_tr["audio_mode"] = X_train["audio_mode"].astype("category").reset_index(drop = True)
    X_norm_tr["audio_key"] = X_train["audio_key"].astype("category").reset_index(drop = True)
    X_norm_te["audio_mode"] = X_test["audio_mode"].astype("category").reset_index(drop = True)
    X_norm_te["audio_key"] = X_test["audio_key"].astype("category").reset_index(drop = True)
    
    # 歌词特征
    lyric_tr = X_norm_tr.loc[:, "lyrics_aah":]
    lyric_te = X_norm_te.loc[:, "lyrics_aah":]

    # 如果使用PCA降维
    if reduction_method == "pca":
        pca = PCA(n_components)
        # 拟合训练集
        reduced_tr = pd.DataFrame(pca.fit_transform(lyric_tr)).add_prefix("lyrics_pca_")
        # 对测试集变换（降维）
        reduced_te = pd.DataFrame(pca.transform(lyric_te)).add_prefix("lyrics_pca_")
    
    # 如果使用SVD降维
    if reduction_method == "svd":
        svd = TruncatedSVD(n_components)
        # 拟合训练集
        reduced_tr = pd.DataFrame(svd.fit_transform(lyric_tr)).add_prefix("lyrics_svd_")
        # 对测试集变换（降维）
        reduced_te = pd.DataFrame(svd.transform(lyric_te)).add_prefix("lyrics_svd_")
    
    # 如果使用自编码器降维（注意，神经网络的训练时间会长一点，要耐心等待）
    if reduction_method == "keras":
        # 构建自编码器
        encoder = autoencode(lyric_tr, n_components)
        
        # 通过编码器部分进行数据降维
        reduced_tr = pd.DataFrame(encoder.predict(lyric_tr)).add_prefix("lyrics_keras_")
        reduced_te = pd.DataFrame(encoder.predict(lyric_te)).add_prefix("lyrics_keras_")

        
        
    # 合并降维后的歌词特征与音频特征
    X_norm_tr = pd.concat([X_norm_tr.loc[:, :"audio_duration_ms"],
                          reduced_tr
                          ], axis = 1)

    X_norm_te = pd.concat([X_norm_te.loc[:, :"audio_duration_ms"],
                           reduced_te
                           ], axis = 1)


    return X_norm_tr, label_train, X_norm_te, label_test, label_encoder


# 分层切分数据
train_raw, test_raw = train_test_split(data, test_size = 0.2,
                                       shuffle = True, random_state = 42, # random, reproducible split
                                       stratify = data.playlist_genre)
# 设定降维最终维度
n_components = 500
# 选择降维方法，候选: "pca", "svd", "keras"
reduction_method = "pca"

# 完整的数据预处理
X_train, y_train, X_test, y_test, label_encoder = pre_process(train_raw, test_raw,
                                                      reduction_method = reduction_method,
                                                     n_components = n_components)

from sklearn.metrics import f1_score

# 定义评估准则(Macro F1)
def lgb_f1_score(preds, data):
    labels = data.get_label()
    preds = preds.reshape(5, -1).T
    preds = preds.argmax(axis = 1)
    f_score = f1_score(labels , preds,  average = 'macro')
    return 'f1_score', f_score, True

# 用于编译的参数
fixed_params = {
        'objective': 'multiclass',
        'metric': "None",   # 我们自定义的f1-score可以应用
        'num_class': 5,
        'verbosity': -1,
}

# 建模
from sklearn.model_selection import StratifiedKFold
import lightgbm as lgb
from optuna.integration import LightGBMPruningCallback

# 定义目标函数
def objective(trial, X, y):    
    # 候选超参数
    param = {**fixed_params,
        'boosting_type': 'gbdt',
        'num_leaves': trial.suggest_int('num_leaves', 2, 3000, step = 20),
        'feature_fraction': trial.suggest_float('feature_fraction', 0.2, 0.99, step = 0.05),
        'bagging_fraction': trial.suggest_float('bagging_fraction', 0.2, 0.99, step = 0.05),
        'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
        'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
        "n_estimators": trial.suggest_int("n_estimators", 200, 5000),
        "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.3),
        "max_depth": trial.suggest_int("max_depth", 3, 12),
        "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 5, 2000, step=5),
        "lambda_l1": trial.suggest_float("lambda_l1", 1e-8, 10.0, log=True),
        "lambda_l2": trial.suggest_float("lambda_l2", 1e-8, 10.0, log=True),
        "min_gain_to_split": trial.suggest_float("min_gain_to_split", 0, 10),
        "max_bin": trial.suggest_int("max_bin", 200, 300),
    }
    
    # 构建分层交叉验证
    cv = StratifiedKFold(n_splits = 5, shuffle = True)
    # 5组得分
    cv_scores = np.empty(5)
    
    # 切分为K个数据组，轮番作为训练集和验证集进行实验
    for idx, (train_idx, test_idx) in enumerate(cv.split(X, y)):
        # 数据切分
        X_train_cv, X_test_cv = X.iloc[train_idx], X.iloc[test_idx]
        y_train_cv, y_test_cv = y[train_idx], y[test_idx]

        # 转为lightgbm的Dataset格式
        train_data = lgb.Dataset(X_train_cv, label = y_train_cv, categorical_feature="auto")
        val_data = lgb.Dataset(X_test_cv, label = y_test_cv,  categorical_feature="auto",
                              reference = train_data)
        
        # 回调函数
        callbacks = [
            LightGBMPruningCallback(trial, metric = "f1_score"),
                     # 间歇输出信息
                    lgb.log_evaluation(period = 100),
                     # 早停止，防止过拟合
                    lgb.early_stopping(50)]

        # 训练模型
        model = lgb.train(params = param,  train_set = train_data,
                          valid_sets = val_data,   
                          callbacks = callbacks,
                          feval = lgb_f1_score # 自定义评估准则
                         )
        
        # 预估
        preds = np.argmax(model.predict(X_test_cv), axis = 1)
        # 计算f1-score
        cv_scores[idx] = f1_score(y_test_cv, preds, average = "macro")

    return np.mean(cv_scores)

# 超参数优化
import optuna

# 定义Optuna的实验次数
n_trials = 200
# 构建Optuna study去进行超参数检索与调优
study = optuna.create_study(direction = "maximize", # 最大化交叉验证的F1得分
                            study_name = "LGBM Classifier",
                           pruner=optuna.pruners.HyperbandPruner())
func = lambda trial: objective(trial, X_train, y_train)
study.optimize(func, n_trials = n_trials)

plot_param_importances(study)

plot_parallel_coordinate(study)

plot_optimization_history(study)

# 最佳模型实验
cv = StratifiedKFold(n_splits = 5, shuffle = True)
# 5组得分
cv_scores = np.empty(5)

# 切分为K个数据组，轮番作为训练集和验证集进行实验
for idx, (train_idx, test_idx) in enumerate(cv.split(X, y)):
    # 数据切分
    X_train_cv, X_test_cv = X.iloc[train_idx], X.iloc[test_idx]
    y_train_cv, y_test_cv = y[train_idx], y[test_idx]

    # 转为lightgbm的Dataset格式
    train_data = lgb.Dataset(X_train_cv, label = y_train_cv, categorical_feature="auto")
    val_data = lgb.Dataset(X_test_cv, label = y_test_cv,  categorical_feature="auto",
                          reference = train_data)
    
    # 回调函数
    callbacks = [
        LightGBMPruningCallback(trial, metric = "f1_score"),
                 # 间歇输出信息
                lgb.log_evaluation(period = 100),
                 # 早停止，防止过拟合
                lgb.early_stopping(50)]

    # 训练模型
    model = lgb.train(params = {**fixed_params, **study.best_params},  train_set = train_data,
                      valid_sets = val_data,   
                      callbacks = callbacks,
                      feval = lgb_f1_score # 自定义评估准则
                     )
    
    # 预估
    preds = np.argmax(model.predict(X_test_cv), axis = 1)
    # 计算f1-score
    cv_scores[idx] = f1_score(y_test_cv, preds, average = "macro")

# 预估与评估训练集
train_preds = model.predict(X_train)
train_predictions = np.argmax(train_preds, axis = 1)
train_error = f1_score(y_train, train_predictions, average = "macro")

# 交叉验证结果
cv_error = np.mean(cv_scores)

# 评估测试集
test_preds = model.predict(X_test)
test_predictions = np.argmax(test_preds, axis = 1)
test_error = f1_score(y_test, test_predictions, average = "macro")

# 存储评估结果
results = pd.DataFrame({"n_components": n_components,
                        "reduction_method": reduction_method,
                        "train_error": train_error,
                        "cv_error": cv_error,
                        "test_error": test_error,
                        "n_trials": n_trials
                       }, index = [0])