clustering_in_progress.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import davies_bouldin_score, silhouette_score\n",
    "from sklearn.preprocessing import StandardScaler, PowerTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "from sympy import Point\n",
    "import os, shutil\n",
    "\n",
    "np.warnings = warnings\n",
    "import import_ipynb\n",
    "%store -r transformed_defence_data\n",
    "%store -r evenStrengthD_names\n",
    "\n",
    "%store -r transformed_center_data\n",
    "%store -r evenStrengthC_names\n",
    "\n",
    "UNIMPORTANT = 0.5\n",
    "JUICE = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "juiced_columns = [\"gameScore\", \"onIce_xGoalsPercentage\", \"onIce_corsiPercentage\", \"I_F_xGoals\", \"I_F_hits\", \"I_F_goals\", \"I_F_points\", \"shotsBlockedByPlayer\", \"OnIce_A_xGoals\", \"OnIce_A_goals\", \"OnIce_A_highDangerxGoals\", \"OnIce_A_goals\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_dimensionality(df):\n",
    "    pca = PCA(n_components=0.95)\n",
    "    pca.set_output(transform='pandas')\n",
    "    new_dataset = pca.fit_transform(df)\n",
    "    return new_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_off_ice_differences(df):\n",
    "    df[\"xGoalsPercentage_Difference\"] = df[\"onIce_xGoalsPercentage\"] - df[\"offIce_xGoalsPercentage\"]\n",
    "    df[\"corsiPercentage_Difference\"] = df[\"onIce_corsiPercentage\"] - df[\"offIce_corsiPercentage\"]\n",
    "    df[\"fenwickPercentage_Difference\"] = df[\"onIce_fenwickPercentage\"] - df[\"offIce_fenwickPercentage\"]\n",
    "    df[\"F_xGoals_Difference\"] = df[\"OnIce_F_xGoals\"] - df[\"OffIce_F_xGoals\"]\n",
    "    df[\"A_xGoals_Difference\"] = df[\"OnIce_A_xGoals\"] - df[\"OffIce_A_xGoals\"]\n",
    "    df[\"F_shotAttempts_Difference\"] = df[\"OnIce_F_shotAttempts\"] - df[\"OffIce_F_shotAttempts\"]\n",
    "    df[\"A_shotAttempts_Difference\"] = df[\"OnIce_A_shotAttempts\"] - df[\"OffIce_A_shotAttempts\"]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transformer(df):\n",
    "    pt = PowerTransformer()\n",
    "    pt.set_output(transform='pandas')\n",
    "    new_df = pt.fit_transform(df)\n",
    "\n",
    "    # if anything we juice I_F so that players who play together arent in the same cluster\n",
    "    new_df[new_df.filter(regex=\"I_F\").columns] *= JUICE\n",
    "    # for column in juiced_columns:\n",
    "    #     new_df[column] *= JUICE\n",
    "    new_df[new_df.filter(regex=\"(o|O)ff(i|I)ce\").columns] *= UNIMPORTANT\n",
    "    new_df[new_df.filter(regex=\"fter(s|S)hift\").columns] *= UNIMPORTANT\n",
    "    new_df[new_df.filter(regex=\"shift\").columns] *= UNIMPORTANT\n",
    "    new_df[new_df.filter(regex=\"I_F_(p|P)lay\").columns] *= UNIMPORTANT\n",
    "    \n",
    "\n",
    "    final_df = reduce_dimensionality(new_df)\n",
    "\n",
    "    return final_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# determine the number of clusters needed with the elbow method\n",
    "\n",
    "def elbow_method(data):\n",
    "\n",
    "    wcss = []\n",
    "    for i in range(2, 12):\n",
    "        method = KMeans(n_clusters=i, init='k-means++', max_iter=300, n_init=10, random_state=0)\n",
    "        method.fit(data)\n",
    "        wcss.append(method.inertia_)\n",
    "\n",
    "    for i in range(1, 11):\n",
    "        point1 = Point()\n",
    "\n",
    "    array = np.array(wcss, dtype=\"float\")\n",
    "    gradiant = np.gradient(array)\n",
    "    print(gradiant)\n",
    "\n",
    "    print(wcss)\n",
    "    plt.plot(range(2, 12), wcss)\n",
    "    plt.title('Elbow Method')\n",
    "    plt.xlabel('Number of clusters')\n",
    "    plt.ylabel('WCSS')\n",
    "    plt.show()    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def silhouette_method(data):\n",
    "    if len(data) == 1:\n",
    "        return 1\n",
    "    wcss = []\n",
    "    max = -2 #silhoutte score ranges from -1 to 1\n",
    "    index = -1\n",
    "\n",
    "    max_clusters = min(len(data)-1, 12)\n",
    "    for i in range(2, max_clusters):\n",
    "        method = KMeans(n_clusters=i, init='k-means++', max_iter=300, n_init=10, random_state=0)\n",
    "        method.fit(data)\n",
    "        labels = method.labels_\n",
    "\n",
    "        score = silhouette_score(data, labels)\n",
    "        wcss.append(score)\n",
    "\n",
    "        if score > max:\n",
    "            max = score\n",
    "            index = i\n",
    "    if index == -1:\n",
    "        return 1\n",
    "    return index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def davies_bouldin_method(data):\n",
    "    if len(data) == 1:\n",
    "        return 1\n",
    "    wcss = []\n",
    "    min = 100000\n",
    "    index = 0\n",
    "    for i in range(2, len(data)-1):\n",
    "        method = KMeans(n_clusters=i, init='k-means++', max_iter=300, n_init=10, random_state=0)\n",
    "        method.fit(data)\n",
    "        labels = method.labels_\n",
    "\n",
    "        score = davies_bouldin_score(data, labels)\n",
    "        wcss.append(score)\n",
    "\n",
    "        if score < min:\n",
    "            min = score\n",
    "            index = i\n",
    "    return index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def k_means_scikit(num_clusters, data):\n",
    "    method = KMeans(n_clusters=num_clusters, init=\"k-means++\", random_state=0)\n",
    "    method.fit(data)\n",
    "    #labels = method.predict(data)\n",
    "    labels = method.labels_\n",
    "\n",
    "    data[\"labels\"] = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hierarchical_clustering(data, names, folderPath, filename, max_iter = 10):\n",
    "\n",
    "    # base case: if we have a small enough cluster end it\n",
    "    if len(data) < 12 or max_iter == 0:\n",
    "        if not os.path.exists(folderPath):\n",
    "            os.mkdir(folderPath)\n",
    "\n",
    "        full_df = pd.concat([names, data], axis=1)\n",
    "        full_df.to_csv(folderPath + '/' + filename + '.csv', index=False)\n",
    "        return\n",
    "\n",
    "    # Step 1 get number of clusters\n",
    "    num_clusters = silhouette_method(data)\n",
    "    #print(num_clusters)\n",
    "\n",
    "    # Step 2 create folders\n",
    "    if not os.path.exists(folderPath):\n",
    "        os.mkdir(folderPath)\n",
    "\n",
    "    # Step 3 calculate the clusters\n",
    "    k_means_scikit(num_clusters, data)\n",
    "\n",
    "    # Step 4 create dataframes for each labels\n",
    "    full_df = pd.concat([names, data], axis=1)\n",
    "    #print(full_df)\n",
    "    for i in range(num_clusters):\n",
    "        temp = full_df.loc[full_df[\"labels\"] == i]\n",
    "        #print(temp)\n",
    "        name_index = temp.columns.get_loc(\"position\")\n",
    "        new_name, new_data = temp.iloc[:,:name_index+1], temp.iloc[:,name_index+1:]\n",
    "        new_data.drop(columns=[\"labels\"], inplace = True)\n",
    "\n",
    "        #newFolderPath = folderPath + \"/cluster-\" + str(i)\n",
    "        new_filename = filename + str(i)\n",
    "        if len(new_data) > 12:\n",
    "            hierarchical_clustering(new_data, new_name, folderPath, new_filename, max_iter-1)\n",
    "        else:\n",
    "            df = pd.concat([new_name, new_data], axis=1)\n",
    "            df.to_csv(folderPath + '/' + new_filename + \".csv\")     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"icetime\" in transformed_defence_data.columns:\n",
    "    transformed_defence_data.drop(columns=[\"icetime\"], inplace=True)\n",
    "elif \"iceTimeRank\" in transformed_defence_data.columns:\n",
    "    transformed_defence_data.drop(columns=[\"iceTimeRank\"], inplace=True)\n",
    "\n",
    "folderPath ='./k_means_scikit_clusters_defence'\n",
    "if os.path.exists(folderPath):\n",
    "    shutil.rmtree(folderPath)\n",
    "\n",
    "on_off_ice_differences(transformed_defence_data)\n",
    "scaled_transformed_defence_data = transformer(transformed_defence_data)\n",
    "hierarchical_clustering(scaled_transformed_defence_data, evenStrengthD_names, \"./k_means_scikit_clusters_defence\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"icetime\" in transformed_center_data.columns:\n",
    "    transformed_center_data.drop(columns=[\"icetime\"], inplace=True)\n",
    "elif \"iceTimeRank\" in transformed_center_data.columns:\n",
    "    transformed_center_data.drop(columns=[\"iceTimeRank\"], inplace=True)\n",
    "\n",
    "folderPath ='./k_means_scikit_clusters_centers'\n",
    "if os.path.exists(folderPath):\n",
    "    shutil.rmtree(folderPath)\n",
    "\n",
    "on_off_ice_differences(transformed_center_data)\n",
    "scaled_transformed_defence_data = transformer(transformed_center_data)\n",
    "hierarchical_clustering(scaled_transformed_defence_data, evenStrengthC_names, \"./k_means_scikit_clusters_centers\", \"\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}