Commit ·
1aac388
1
Parent(s): 8c7ecd7
modify json input
Browse files
.ipynb_checkpoints/2179-proj-checkpoint.ipynb
DELETED
|
@@ -1,383 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"cells": [
|
| 3 |
-
{
|
| 4 |
-
"cell_type": "markdown",
|
| 5 |
-
"id": "5ede69a1",
|
| 6 |
-
"metadata": {},
|
| 7 |
-
"source": [
|
| 8 |
-
"# Classification Challenge using CatBoost\n",
|
| 9 |
-
"\n",
|
| 10 |
-
"## INF2179 Fall 2021\n",
|
| 11 |
-
"### Hamid Yuksel\n",
|
| 12 |
-
"\n",
|
| 13 |
-
"This submission uses [CatBoost](https://catboost.ai/).\n",
|
| 14 |
-
"CatBoost was chosen for its listed benefits, mainly in requiring less hyperparameter tuning and preprocessing of categorical and text features. It is also fast and fairly easy to set up.\n",
|
| 15 |
-
"\n",
|
| 16 |
-
"<img src=\"https://cdn.britannica.com/39/7139-050-A88818BB/Himalayan-chocolate-point.jpg\"\n",
|
| 17 |
-
" alt=\"Markdown Monster icon\"\n",
|
| 18 |
-
" style=\"float: left; margin-right: 10px;\" />\n"
|
| 19 |
-
]
|
| 20 |
-
},
|
| 21 |
-
{
|
| 22 |
-
"cell_type": "code",
|
| 23 |
-
"execution_count": 1,
|
| 24 |
-
"id": "ee82451e",
|
| 25 |
-
"metadata": {},
|
| 26 |
-
"outputs": [
|
| 27 |
-
{
|
| 28 |
-
"name": "stdout",
|
| 29 |
-
"output_type": "stream",
|
| 30 |
-
"text": [
|
| 31 |
-
"Requirement already satisfied: catboost in /Users/yuksel/.local/lib/python3.8/site-packages (1.0.3)\n",
|
| 32 |
-
"Requirement already satisfied: graphviz in /Users/yuksel/.local/lib/python3.8/site-packages (from catboost) (0.18)\n",
|
| 33 |
-
"Requirement already satisfied: matplotlib in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from catboost) (3.4.3)\n",
|
| 34 |
-
"Requirement already satisfied: plotly in /Users/yuksel/.local/lib/python3.8/site-packages (from catboost) (5.3.1)\n",
|
| 35 |
-
"Requirement already satisfied: scipy in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from catboost) (1.8.0)\n",
|
| 36 |
-
"Requirement already satisfied: numpy>=1.16.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from catboost) (1.20.1)\n",
|
| 37 |
-
"Requirement already satisfied: six in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from catboost) (1.15.0)\n",
|
| 38 |
-
"Requirement already satisfied: pandas>=0.24.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from catboost) (1.2.4)\n",
|
| 39 |
-
"Requirement already satisfied: python-dateutil>=2.7.3 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from pandas>=0.24.0->catboost) (2.8.1)\n",
|
| 40 |
-
"Requirement already satisfied: pytz>=2017.3 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from pandas>=0.24.0->catboost) (2021.1)\n",
|
| 41 |
-
"Requirement already satisfied: pyparsing>=2.2.1 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from matplotlib->catboost) (2.4.7)\n",
|
| 42 |
-
"Requirement already satisfied: pillow>=6.2.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from matplotlib->catboost) (8.2.0)\n",
|
| 43 |
-
"Requirement already satisfied: kiwisolver>=1.0.1 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from matplotlib->catboost) (1.3.1)\n",
|
| 44 |
-
"Requirement already satisfied: cycler>=0.10 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from matplotlib->catboost) (0.10.0)\n",
|
| 45 |
-
"Requirement already satisfied: tenacity>=6.2.0 in /Users/yuksel/.local/lib/python3.8/site-packages (from plotly->catboost) (8.0.1)\n",
|
| 46 |
-
"\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.2.1 is available.\n",
|
| 47 |
-
"You should consider upgrading via the '/Users/yuksel/opt/anaconda3/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n",
|
| 48 |
-
"Requirement already satisfied: ipywidgets in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (7.6.3)\n",
|
| 49 |
-
"Requirement already satisfied: widgetsnbextension~=3.5.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets) (3.5.1)\n",
|
| 50 |
-
"Requirement already satisfied: ipykernel>=4.5.1 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets) (5.3.4)\n",
|
| 51 |
-
"Requirement already satisfied: nbformat>=4.2.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets) (5.1.3)\n",
|
| 52 |
-
"Requirement already satisfied: traitlets>=4.3.1 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets) (5.0.5)\n",
|
| 53 |
-
"Requirement already satisfied: ipython>=4.0.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets) (7.22.0)\n",
|
| 54 |
-
"Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipywidgets) (1.0.0)\n",
|
| 55 |
-
"Requirement already satisfied: appnope in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.2)\n",
|
| 56 |
-
"Requirement already satisfied: jupyter-client in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (6.1.12)\n",
|
| 57 |
-
"Requirement already satisfied: tornado>=4.2 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipykernel>=4.5.1->ipywidgets) (6.1)\n",
|
| 58 |
-
"Requirement already satisfied: pygments in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (2.8.1)\n",
|
| 59 |
-
"Requirement already satisfied: jedi>=0.16 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.17.2)\n",
|
| 60 |
-
"Requirement already satisfied: backcall in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.2.0)\n",
|
| 61 |
-
"Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (3.0.17)\n",
|
| 62 |
-
"Requirement already satisfied: decorator in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (5.0.6)\n",
|
| 63 |
-
"Requirement already satisfied: setuptools>=18.5 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (52.0.0.post20210125)\n",
|
| 64 |
-
"Requirement already satisfied: pexpect>4.3 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\n",
|
| 65 |
-
"Requirement already satisfied: pickleshare in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\n",
|
| 66 |
-
"Requirement already satisfied: ipython-genutils in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbformat>=4.2.0->ipywidgets) (0.2.0)\n",
|
| 67 |
-
"Requirement already satisfied: jupyter-core in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbformat>=4.2.0->ipywidgets) (4.7.1)\n",
|
| 68 |
-
"Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbformat>=4.2.0->ipywidgets) (3.2.0)\n",
|
| 69 |
-
"Requirement already satisfied: notebook>=4.4.1 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from widgetsnbextension~=3.5.0->ipywidgets) (6.3.0)\n",
|
| 70 |
-
"Requirement already satisfied: parso<0.8.0,>=0.7.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets) (0.7.0)\n",
|
| 71 |
-
"Requirement already satisfied: attrs>=17.4.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets) (20.3.0)\n",
|
| 72 |
-
"Requirement already satisfied: pyrsistent>=0.14.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets) (0.17.3)\n",
|
| 73 |
-
"Requirement already satisfied: six>=1.11.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.2.0->ipywidgets) (1.15.0)\n",
|
| 74 |
-
"Requirement already satisfied: prometheus-client in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.10.1)\n",
|
| 75 |
-
"Requirement already satisfied: terminado>=0.8.3 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.9.4)\n",
|
| 76 |
-
"Requirement already satisfied: Send2Trash>=1.5.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (1.5.0)\n",
|
| 77 |
-
"Requirement already satisfied: jinja2 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (2.11.3)\n",
|
| 78 |
-
"Requirement already satisfied: nbconvert in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (6.0.7)\n",
|
| 79 |
-
"Requirement already satisfied: pyzmq>=17 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (20.0.0)\n",
|
| 80 |
-
"Requirement already satisfied: argon2-cffi in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (20.1.0)\n",
|
| 81 |
-
"Requirement already satisfied: python-dateutil>=2.1 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from jupyter-client->ipykernel>=4.5.1->ipywidgets) (2.8.1)\n",
|
| 82 |
-
"Requirement already satisfied: ptyprocess>=0.5 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets) (0.7.0)\n",
|
| 83 |
-
"Requirement already satisfied: wcwidth in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython>=4.0.0->ipywidgets) (0.2.5)\n",
|
| 84 |
-
"Requirement already satisfied: cffi>=1.0.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (1.14.5)\n",
|
| 85 |
-
"Requirement already satisfied: MarkupSafe>=0.23 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from jinja2->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (2.0.1)\n",
|
| 86 |
-
"Requirement already satisfied: defusedxml in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.7.1)\n",
|
| 87 |
-
"Requirement already satisfied: jupyterlab-pygments in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.1.2)\n",
|
| 88 |
-
"Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.5.3)\n",
|
| 89 |
-
"Requirement already satisfied: mistune<2,>=0.8.1 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.8.4)\n",
|
| 90 |
-
"Requirement already satisfied: pandocfilters>=1.4.1 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (1.4.3)\n",
|
| 91 |
-
"Requirement already satisfied: entrypoints>=0.2.2 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.3)\n",
|
| 92 |
-
"Requirement already satisfied: testpath in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.4.4)\n",
|
| 93 |
-
"Requirement already satisfied: bleach in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (3.3.0)\n"
|
| 94 |
-
]
|
| 95 |
-
},
|
| 96 |
-
{
|
| 97 |
-
"name": "stdout",
|
| 98 |
-
"output_type": "stream",
|
| 99 |
-
"text": [
|
| 100 |
-
"Requirement already satisfied: pycparser in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from cffi>=1.0.0->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (2.20)\n",
|
| 101 |
-
"Requirement already satisfied: nest-asyncio in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (1.5.1)\n",
|
| 102 |
-
"Requirement already satisfied: async-generator in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (1.10)\n",
|
| 103 |
-
"Requirement already satisfied: packaging in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (21.3)\n",
|
| 104 |
-
"Requirement already satisfied: webencodings in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (0.5.1)\n",
|
| 105 |
-
"Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /Users/yuksel/opt/anaconda3/lib/python3.8/site-packages (from packaging->bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets) (2.4.7)\n",
|
| 106 |
-
"\u001b[33mWARNING: You are using pip version 21.3.1; however, version 22.2.1 is available.\n",
|
| 107 |
-
"You should consider upgrading via the '/Users/yuksel/opt/anaconda3/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n",
|
| 108 |
-
"Enabling notebook extension jupyter-js-widgets/extension...\n",
|
| 109 |
-
" - Validating: \u001b[32mOK\u001b[0m\n"
|
| 110 |
-
]
|
| 111 |
-
}
|
| 112 |
-
],
|
| 113 |
-
"source": [
|
| 114 |
-
"#Installing and Importing required libraries\n",
|
| 115 |
-
"! pip3 install --user catboost\n",
|
| 116 |
-
"! pip3 install --user ipywidgets\n",
|
| 117 |
-
"! jupyter nbextension enable --py widgetsnbextension\n",
|
| 118 |
-
"\n",
|
| 119 |
-
"import pandas as pd \n",
|
| 120 |
-
"import numpy as np\n",
|
| 121 |
-
"from sklearn.neural_network import MLPClassifier\n",
|
| 122 |
-
"from sklearn.metrics import accuracy_score \n",
|
| 123 |
-
"from catboost import Pool, CatBoostClassifier"
|
| 124 |
-
]
|
| 125 |
-
},
|
| 126 |
-
{
|
| 127 |
-
"cell_type": "code",
|
| 128 |
-
"execution_count": 2,
|
| 129 |
-
"id": "1853f85d",
|
| 130 |
-
"metadata": {},
|
| 131 |
-
"outputs": [
|
| 132 |
-
{
|
| 133 |
-
"data": {
|
| 134 |
-
"text/plain": [
|
| 135 |
-
"Rock 24486\n",
|
| 136 |
-
"Pop 16251\n",
|
| 137 |
-
"Hip Hop 9263\n",
|
| 138 |
-
"unknown 5000\n",
|
| 139 |
-
"Name: Genre, dtype: int64"
|
| 140 |
-
]
|
| 141 |
-
},
|
| 142 |
-
"execution_count": 2,
|
| 143 |
-
"metadata": {},
|
| 144 |
-
"output_type": "execute_result"
|
| 145 |
-
}
|
| 146 |
-
],
|
| 147 |
-
"source": [
|
| 148 |
-
"# Reading data\n",
|
| 149 |
-
"df = pd.read_csv('data.csv')\n",
|
| 150 |
-
"\n",
|
| 151 |
-
"# Splitting\n",
|
| 152 |
-
"training = df.head(50000)\n",
|
| 153 |
-
"holdout_set = training.sample(5000, random_state=1) # pick 5000 observations randomly\n",
|
| 154 |
-
"training = training.drop(holdout_set.index) # Remove holdout from training data\n",
|
| 155 |
-
"testing = df.tail(5000)\n",
|
| 156 |
-
"\n",
|
| 157 |
-
"#Looking at counts per genre\n",
|
| 158 |
-
"df['Genre'].value_counts()"
|
| 159 |
-
]
|
| 160 |
-
},
|
| 161 |
-
{
|
| 162 |
-
"cell_type": "code",
|
| 163 |
-
"execution_count": 3,
|
| 164 |
-
"id": "f0a20f7a",
|
| 165 |
-
"metadata": {},
|
| 166 |
-
"outputs": [
|
| 167 |
-
{
|
| 168 |
-
"data": {
|
| 169 |
-
"text/html": [
|
| 170 |
-
"<div>\n",
|
| 171 |
-
"<style scoped>\n",
|
| 172 |
-
" .dataframe tbody tr th:only-of-type {\n",
|
| 173 |
-
" vertical-align: middle;\n",
|
| 174 |
-
" }\n",
|
| 175 |
-
"\n",
|
| 176 |
-
" .dataframe tbody tr th {\n",
|
| 177 |
-
" vertical-align: top;\n",
|
| 178 |
-
" }\n",
|
| 179 |
-
"\n",
|
| 180 |
-
" .dataframe thead th {\n",
|
| 181 |
-
" text-align: right;\n",
|
| 182 |
-
" }\n",
|
| 183 |
-
"</style>\n",
|
| 184 |
-
"<table border=\"1\" class=\"dataframe\">\n",
|
| 185 |
-
" <thead>\n",
|
| 186 |
-
" <tr style=\"text-align: right;\">\n",
|
| 187 |
-
" <th></th>\n",
|
| 188 |
-
" <th>Lyric</th>\n",
|
| 189 |
-
" </tr>\n",
|
| 190 |
-
" </thead>\n",
|
| 191 |
-
" <tbody>\n",
|
| 192 |
-
" <tr>\n",
|
| 193 |
-
" <th>50000</th>\n",
|
| 194 |
-
" <td>Feels so good,. Feels so good,. Feels so good ...</td>\n",
|
| 195 |
-
" </tr>\n",
|
| 196 |
-
" <tr>\n",
|
| 197 |
-
" <th>50001</th>\n",
|
| 198 |
-
" <td>Shadow of a doubt. I heard your heart,. you he...</td>\n",
|
| 199 |
-
" </tr>\n",
|
| 200 |
-
" <tr>\n",
|
| 201 |
-
" <th>50002</th>\n",
|
| 202 |
-
" <td>Slaves. Hebrews born to serve to the pharaoh. ...</td>\n",
|
| 203 |
-
" </tr>\n",
|
| 204 |
-
" <tr>\n",
|
| 205 |
-
" <th>50003</th>\n",
|
| 206 |
-
" <td>You've been picked and it's over. What's the c...</td>\n",
|
| 207 |
-
" </tr>\n",
|
| 208 |
-
" <tr>\n",
|
| 209 |
-
" <th>50004</th>\n",
|
| 210 |
-
" <td>Magic happens. But only if you are open to the...</td>\n",
|
| 211 |
-
" </tr>\n",
|
| 212 |
-
" <tr>\n",
|
| 213 |
-
" <th>...</th>\n",
|
| 214 |
-
" <td>...</td>\n",
|
| 215 |
-
" </tr>\n",
|
| 216 |
-
" <tr>\n",
|
| 217 |
-
" <th>54995</th>\n",
|
| 218 |
-
" <td>I can't believe what you did to me. Down on my...</td>\n",
|
| 219 |
-
" </tr>\n",
|
| 220 |
-
" <tr>\n",
|
| 221 |
-
" <th>54996</th>\n",
|
| 222 |
-
" <td>Have all the songs been written?. Have all the...</td>\n",
|
| 223 |
-
" </tr>\n",
|
| 224 |
-
" <tr>\n",
|
| 225 |
-
" <th>54997</th>\n",
|
| 226 |
-
" <td>Everything you do you do so right. The clothes...</td>\n",
|
| 227 |
-
" </tr>\n",
|
| 228 |
-
" <tr>\n",
|
| 229 |
-
" <th>54998</th>\n",
|
| 230 |
-
" <td>(trecho). (Rule Number Two. Understanding what...</td>\n",
|
| 231 |
-
" </tr>\n",
|
| 232 |
-
" <tr>\n",
|
| 233 |
-
" <th>54999</th>\n",
|
| 234 |
-
" <td>As fall rides off in the Sunset. I sweep the S...</td>\n",
|
| 235 |
-
" </tr>\n",
|
| 236 |
-
" </tbody>\n",
|
| 237 |
-
"</table>\n",
|
| 238 |
-
"<p>5000 rows × 1 columns</p>\n",
|
| 239 |
-
"</div>"
|
| 240 |
-
],
|
| 241 |
-
"text/plain": [
|
| 242 |
-
" Lyric\n",
|
| 243 |
-
"50000 Feels so good,. Feels so good,. Feels so good ...\n",
|
| 244 |
-
"50001 Shadow of a doubt. I heard your heart,. you he...\n",
|
| 245 |
-
"50002 Slaves. Hebrews born to serve to the pharaoh. ...\n",
|
| 246 |
-
"50003 You've been picked and it's over. What's the c...\n",
|
| 247 |
-
"50004 Magic happens. But only if you are open to the...\n",
|
| 248 |
-
"... ...\n",
|
| 249 |
-
"54995 I can't believe what you did to me. Down on my...\n",
|
| 250 |
-
"54996 Have all the songs been written?. Have all the...\n",
|
| 251 |
-
"54997 Everything you do you do so right. The clothes...\n",
|
| 252 |
-
"54998 (trecho). (Rule Number Two. Understanding what...\n",
|
| 253 |
-
"54999 As fall rides off in the Sunset. I sweep the S...\n",
|
| 254 |
-
"\n",
|
| 255 |
-
"[5000 rows x 1 columns]"
|
| 256 |
-
]
|
| 257 |
-
},
|
| 258 |
-
"execution_count": 3,
|
| 259 |
-
"metadata": {},
|
| 260 |
-
"output_type": "execute_result"
|
| 261 |
-
}
|
| 262 |
-
],
|
| 263 |
-
"source": [
|
| 264 |
-
"# Splitting training/testing set to feature (X) and labels (y)\n",
|
| 265 |
-
"train_y = training.Genre\n",
|
| 266 |
-
"train_X = training.drop('Genre', axis=1)\n",
|
| 267 |
-
"\n",
|
| 268 |
-
"test_X = testing.drop('Genre', axis=1)\n",
|
| 269 |
-
"\n",
|
| 270 |
-
"test_X"
|
| 271 |
-
]
|
| 272 |
-
},
|
| 273 |
-
{
|
| 274 |
-
"cell_type": "code",
|
| 275 |
-
"execution_count": null,
|
| 276 |
-
"id": "1c8eb420",
|
| 277 |
-
"metadata": {},
|
| 278 |
-
"outputs": [],
|
| 279 |
-
"source": [
|
| 280 |
-
"# Build a classifier\n",
|
| 281 |
-
"text_features = ['Lyric']\n",
|
| 282 |
-
"\n",
|
| 283 |
-
"\n",
|
| 284 |
-
"train_dataset = Pool(data=train_X,\n",
|
| 285 |
-
" label=train_y,\n",
|
| 286 |
-
" text_features=text_features)\n",
|
| 287 |
-
"\n",
|
| 288 |
-
"model = CatBoostClassifier(iterations=1000,\n",
|
| 289 |
-
" learning_rate=1,\n",
|
| 290 |
-
" depth=5,\n",
|
| 291 |
-
" loss_function='MultiClass')\n",
|
| 292 |
-
"\n",
|
| 293 |
-
"model.fit(train_dataset)"
|
| 294 |
-
]
|
| 295 |
-
},
|
| 296 |
-
{
|
| 297 |
-
"cell_type": "code",
|
| 298 |
-
"execution_count": null,
|
| 299 |
-
"id": "f770a8f2",
|
| 300 |
-
"metadata": {},
|
| 301 |
-
"outputs": [],
|
| 302 |
-
"source": [
|
| 303 |
-
"# Estimate accuracy\n",
|
| 304 |
-
"pred = model.predict(holdout_set.drop('Genre',axis=1))\n",
|
| 305 |
-
"estimated_accuracy = accuracy_score(holdout_set['Genre'], pred)\n",
|
| 306 |
-
"print(estimated_accuracy)\n",
|
| 307 |
-
"pd.Series(estimated_accuracy).to_csv('ea.csv', index=False, header=False)"
|
| 308 |
-
]
|
| 309 |
-
},
|
| 310 |
-
{
|
| 311 |
-
"cell_type": "code",
|
| 312 |
-
"execution_count": 131,
|
| 313 |
-
"id": "035f58c6",
|
| 314 |
-
"metadata": {},
|
| 315 |
-
"outputs": [
|
| 316 |
-
{
|
| 317 |
-
"name": "stdout",
|
| 318 |
-
"output_type": "stream",
|
| 319 |
-
"text": [
|
| 320 |
-
"['Pop' 'Rock' 'Rock' ... 'Rock' 'Pop' 'Rock']\n"
|
| 321 |
-
]
|
| 322 |
-
}
|
| 323 |
-
],
|
| 324 |
-
"source": [
|
| 325 |
-
"# Predict testing set\n",
|
| 326 |
-
"pred = model.predict(test_X)\n",
|
| 327 |
-
"print(pred.flatten())\n",
|
| 328 |
-
"pred = pd.Series(pred.flatten()).to_csv('pred.csv', index=False, header=False)"
|
| 329 |
-
]
|
| 330 |
-
},
|
| 331 |
-
{
|
| 332 |
-
"cell_type": "code",
|
| 333 |
-
"execution_count": 128,
|
| 334 |
-
"id": "2137c958",
|
| 335 |
-
"metadata": {},
|
| 336 |
-
"outputs": [
|
| 337 |
-
{
|
| 338 |
-
"data": {
|
| 339 |
-
"text/plain": [
|
| 340 |
-
"(array(['Hip Hop', 'Pop', 'Rock'], dtype=object), array([ 802, 1307, 2891]))"
|
| 341 |
-
]
|
| 342 |
-
},
|
| 343 |
-
"execution_count": 128,
|
| 344 |
-
"metadata": {},
|
| 345 |
-
"output_type": "execute_result"
|
| 346 |
-
}
|
| 347 |
-
],
|
| 348 |
-
"source": [
|
| 349 |
-
"# to check number of instances of each genre in pred\n",
|
| 350 |
-
"np.unique(model.predict(test_X), return_counts=True)"
|
| 351 |
-
]
|
| 352 |
-
},
|
| 353 |
-
{
|
| 354 |
-
"cell_type": "code",
|
| 355 |
-
"execution_count": null,
|
| 356 |
-
"id": "83aa22c4",
|
| 357 |
-
"metadata": {},
|
| 358 |
-
"outputs": [],
|
| 359 |
-
"source": []
|
| 360 |
-
}
|
| 361 |
-
],
|
| 362 |
-
"metadata": {
|
| 363 |
-
"kernelspec": {
|
| 364 |
-
"display_name": "Python 3",
|
| 365 |
-
"language": "python",
|
| 366 |
-
"name": "python3"
|
| 367 |
-
},
|
| 368 |
-
"language_info": {
|
| 369 |
-
"codemirror_mode": {
|
| 370 |
-
"name": "ipython",
|
| 371 |
-
"version": 3
|
| 372 |
-
},
|
| 373 |
-
"file_extension": ".py",
|
| 374 |
-
"mimetype": "text/x-python",
|
| 375 |
-
"name": "python",
|
| 376 |
-
"nbconvert_exporter": "python",
|
| 377 |
-
"pygments_lexer": "ipython3",
|
| 378 |
-
"version": "3.8.8"
|
| 379 |
-
}
|
| 380 |
-
},
|
| 381 |
-
"nbformat": 4,
|
| 382 |
-
"nbformat_minor": 5
|
| 383 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2179-proj.ipynb → .ipynb_checkpoints/model-checkpoint.ipynb
RENAMED
|
@@ -477,13 +477,13 @@
|
|
| 477 |
},
|
| 478 |
{
|
| 479 |
"cell_type": "code",
|
| 480 |
-
"execution_count":
|
| 481 |
"id": "83aa22c4",
|
| 482 |
"metadata": {},
|
| 483 |
"outputs": [],
|
| 484 |
"source": [
|
| 485 |
"import pickle\n",
|
| 486 |
-
"pickle.dump(model, open('
|
| 487 |
]
|
| 488 |
},
|
| 489 |
{
|
|
|
|
| 477 |
},
|
| 478 |
{
|
| 479 |
"cell_type": "code",
|
| 480 |
+
"execution_count": null,
|
| 481 |
"id": "83aa22c4",
|
| 482 |
"metadata": {},
|
| 483 |
"outputs": [],
|
| 484 |
"source": [
|
| 485 |
"import pickle\n",
|
| 486 |
+
"pickle.dump(model, open('model.pickle', 'wb'))"
|
| 487 |
]
|
| 488 |
},
|
| 489 |
{
|
2179-proj.pickle
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:8f64ecdff3d7935ef3bf5bb870fad3e4c619c20312e70bbb79455855cfda1204
|
| 3 |
-
size 7738041
|
|
|
|
|
|
|
|
|
|
|
|
index.py
DELETED
|
@@ -1,25 +0,0 @@
|
|
| 1 |
-
from flask import Flask, request, redirect, url_for, flash, jsonify
|
| 2 |
-
import numpy as np
|
| 3 |
-
import pickle as p
|
| 4 |
-
import json
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
app = Flask(__name__)
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
@app.route("/")
|
| 11 |
-
def home():
|
| 12 |
-
return "<h1>Lyrics Classifier</h1>"
|
| 13 |
-
|
| 14 |
-
@app.route('/api/', methods=['POST'])
|
| 15 |
-
def makecalc():
|
| 16 |
-
data = request.get_json()
|
| 17 |
-
prediction = np.array2string(model.predict(data))
|
| 18 |
-
|
| 19 |
-
return jsonify(prediction)
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
if __name__ == '__main__':
|
| 23 |
-
modelfile = '2179-proj.pickle'
|
| 24 |
-
model = p.load(open(modelfile, 'rb'))
|
| 25 |
-
app.run(debug=True, host='0.0.0.0')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|