{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "### Equations" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "(1) Hyperplane Centroid:\n", "\n", "$HC_{k} = {\\bf w}^{T}{\\bf x}- \\frac{1}{ n_{k} } \\sum\\limits_{y_{i}=k} {\\bf w}^{T}{\\bf x_{i}}=0$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "(2) Hyperplane Centroid Loss:\n", "\n", "$HCL = \\sum\\limits_{i=1}^{k-1}=max(HC_{i} -HC_{i+1} + \\delta,0)$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "(3) Hyperplane Point Loss:\n", "\n", "(4) $HPL_{i}^{+}= max(f(x)-HC)-(HC_{+1}-HC)+\\gamma (HC_{+1} - HC),0)$\n", "\n", "$=max(f(x_{i})-\\gamma HC - (1-\\gamma)HC_{+1},0)$ \n", "\n", "\n", "\n", "(5)\n", "$HPL_{i}^{-}= max(\\gamma HC - f(x_{i}) + (1-\\gamma)HC_{-1},0)$\n", "\n", "(6)\n", "$HPL = \\sum\\limits_{x_{i}\\in S} HPL_{i}^{+} + HPL_{i}^{-}$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "(7) $OHPL = \\alpha HCL + HPL$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Define Loss Functions" ] }, { "cell_type": "code", "execution_count": 72, "metadata": {}, "outputs": [], "source": [ "\"\"\"Metrics to assess performance on ordinal classification task given class prediction\n", " using hyper plane loss techniques \n", "\"\"\"\n", "\n", "# Authors: Bob Vanderheyden \n", "# Ying Xie \n", "# \n", "# Contributor: Shayan Shamskolahi\n", "\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "import tensorflow as tf\n", "import numpy as np\n", "\n", "def hp_ordering_loss(y_true, y_pred, min_label, max_label):\n", "\n", " \"\"\" Evaluate the ordinal loss of the predictions y_pred.\n", " \n", " Parameters\n", " ----------\n", " y_true : array-like\n", " y_pred : array-like\n", " \n", " Returns\n", " -------\n", " loss: float\n", " A non-negative floating point value (best value is 0.0)\n", " \n", " Usage\n", " -------\n", " loss = hp_ordering_loss([4,1,2,0,4,2,1], [6.0,3.1,5.2,1.0,4.0,2.2,3.7])\n", " print('Loss: ', loss.numpy()) # Loss: 2.8\n", " \n", " Usage with the compile API:\n", " \n", " python\n", " model = tf.keras.Model(inputs, outputs)\n", " model.compile(loss=hp_ordering_loss, optimizer='adam', loss=hp_ordering_loss)\n", " \n", " \n", " \"\"\"\n", "\n", " y_pred = tf.convert_to_tensor(y_pred)\n", " y_true = tf.dtypes.cast(y_true, y_pred.dtype)\n", " y_pred = tf.reshape(tf.transpose(y_pred),[-1,1])\n", "\n", " # one hot vector for y_true\n", " ords, idx = tf.unique(tf.reshape(y_true, [-1])) \n", " num = tf.shape(ords)[0]\n", " y_true_1hot = tf.one_hot(idx, num)\n", "\n", " # mean distance for each class\n", " yO = tf.transpose(y_pred) @ y_true_1hot\n", " yc = tf.reduce_sum(y_true_1hot,0)\n", " class_mean = tf.divide(yO,yc) \n", "\n", " # min. distance\n", " ords = tf.dtypes.cast(ords, tf.float32)\n", " ords0 = tf.reshape(ords, [-1,1])\n", " ords1 = tf.reshape(ords, [1,-1])\n", " \n", " min_distance = tf.subtract(ords0, ords1)\n", " # apply ReLU\n", " min_distance = tf.nn.relu (min_distance)\n", " \n", " # keeps min. distance\n", " keep = tf.minimum(min_distance,1)\n", "\n", " # distance to centroid \n", " class_mean0 = tf.reshape(class_mean, [-1,1])\n", " class_mean1 = tf.reshape(class_mean, [1,-1])\n", " class_mean = tf.subtract(class_mean0, class_mean1) \n", " # apply ReLU \n", " class_mean = tf.nn.relu(class_mean)\n", " centroid_distance = tf.multiply(keep, class_mean)\n", " \n", " order_loss = tf.subtract(min_distance,centroid_distance)\n", " # apply ReLU\n", " order_loss = tf.nn.relu(order_loss)\n", " \n", " return tf.reduce_sum(order_loss)\n", "\n", "\n", "def hp_point_loss(y_true, y_pred, L, U):\n", " \"\"\" Evaluate the point loss of the predictions y_pred.\n", "\n", " Parameters\n", " ----------\n", " y_true : array-like\n", " y_pred : array-like\n", " minlabel : integer\n", " maxlabel : integer\n", " margin : float\n", "\n", " Returns\n", " -------\n", " loss: float\n", " A non-negative floating point value (best value is 0.0)\n", " \n", " Usage\n", " -------\n", " loss = hp_point_loss([4,1,2,0,4,2,1], [6.0,3.1,5.2,1.0,4.0,2.2,3.7],0,4,.3)\n", " print('Loss: ', loss.numpy()) # Loss: 3.1\n", " \n", " \n", " Usage with the compile API:\n", " \n", " python\n", " \n", " Example Keras wrapper for hp_point_loss:\n", " \n", " def get_ohpl_wrapper (min_label, max_label, margin):\n", " def ohpl(y_true, y_pred):\n", " return hp_point_loss(y_true, y_pred, min_label, max_label, margin)\n", " return ohpl\n", "\n", " point_loss = get_ohpl_wrapper(0,4,0.01)\n", " \n", " model = tf.keras.Model(inputs, outputs)\n", " model.compile(loss=ohpl_point_loss, optimizer='adam', loss=ohpl_point_loss)\n", " \n", " \n", " \"\"\"\n", " y_pred = tf.reshape(tf.convert_to_tensor(y_pred), [-1,1])\n", " y_true = tf.cast(y_true, dtype=tf.float32)\n", " \n", " upper = tf.reshape(tf.constant(U, dtype=tf.float32), [-1,1])\n", " lower = tf.reshape(tf.constant(L, dtype=tf.float32), [-1,1])\n", " uthreshold = tf.matmul(y_true, upper, name='upper_MM')\n", " lthreshold = tf.matmul(y_true, lower, name='lower_MM')\n", "\n", " upper_loss = tf.nn.relu(tf.subtract(y_pred, uthreshold))\n", " lower_loss = tf.nn.relu(tf.subtract(lthreshold, y_pred))\n", " \n", " hp_point_loss = tf.add(upper_loss, lower_loss)\n", " \n", " return tf.reduce_sum(hp_point_loss)\n", " \n", " \"\"\" \n", " References\n", " ----------\n", " .. [1] Vanderheyden, Bob and Ying Xie. Ordinal Hyperplane Loss. (2018). \n", " 2018 IEEE International Conference on Big Data (Big Data), \n", " 2018 IEEE International Conference On, 2337. https://doi-org.proxy.kennesaw.edu/10.1109/BigData.2018.8622079\n", " \"\"\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test the result:" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loss: 2.8000002\n" ] } ], "source": [ "loss = hp_ordering_loss([4,1,2,0,4,2,1], [6.0,3.1,5.2,1.0,4.0,2.2,3.7])\n", "print('Loss: ', loss.numpy()) # Loss: 2.8" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loss: 5.2\n" ] } ], "source": [ "y_pred = [6.0,3.1,5.2,1.0,4.0,2.2,3.7]\n", "\n", "y_true = np.array([[0,0,0,1],[0,1,0,0],[0,0,1,0],[1,0,0,0],[0,0,0,1],[0,0,1,0],[0,1,0,0]])\n", "\n", "centers = np.array([1.0, 3.4, 3.7, 5.0])\n", "lo = np.array([-1e9, 3.3, 3.6, 5.9])\n", "up = np.array([1.1, 3.5, 3.7, 1e9])\n", "\n", "loss = hp_point_loss(y_true, y_pred, lo, up)\n", "print('Loss: ', loss.numpy()) # Loss: 5,2\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example wrapper for Keras:" ] }, { "cell_type": "code", "execution_count": 88, "metadata": {}, "outputs": [], "source": [ "# example Keras wrapper for hp_ordering_loss\n", "\n", "def get_ordering_wrapper(min_label, max_label):\n", " def hohpl(y_true, y_pred):\n", " return hp_ordering_loss(y_true, y_pred, min_label, max_label)\n", " return hohpl\n", "\n", "ordering_loss = get_ordering_wrapper(0,4)\n", "\n", "\n", "# example Keras wrapper for hp_point_loss\n", "\n", "def get_point_wrapper (l, u):\n", " def ohpl(y_true, y_pred):\n", " return hp_point_loss(y_true, y_pred, l, u)\n", " return ohpl\n", "\n", "\n", "# centers is training set centers see 'train_cent' above\n", "# margin is the point margin\n", "# num_ords is the number of unique ordinal label values\n", "\n", "def ohpl_margins(centers, num_ords, margin=0.1):\n", " inner_ord_array = np.ones(num_ords - 1).reshape(-1,1)\n", " up = np.concatenate((inner_ord_array*margin, [[1e9]]), axis=0)+centers\n", " lo = centers-np.concatenate(([[1e9]], inner_ord_array*margin), axis=0)\n", "\n", " return lo, up\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Wrapper in action - Keras sequential model:" ] }, { "cell_type": "code", "execution_count": 168, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Train on 670 samples, validate on 330 samples\n", "Epoch 1/10\n", "670/670 [==============================] - 0s 724us/sample - loss: 80.6744 - val_loss: 99.9912\n", "Epoch 2/10\n", "670/670 [==============================] - 0s 60us/sample - loss: 77.6038 - val_loss: 89.2979\n", "Epoch 3/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 72.3876 - val_loss: 75.9204\n", "Epoch 4/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 57.2885 - val_loss: 60.2535\n", "Epoch 5/10\n", "670/670 [==============================] - 0s 60us/sample - loss: 40.4422 - val_loss: 44.1772\n", "Epoch 6/10\n", "670/670 [==============================] - 0s 71us/sample - loss: 28.5711 - val_loss: 28.5624\n", "Epoch 7/10\n", "670/670 [==============================] - 0s 60us/sample - loss: 26.4750 - val_loss: 20.0634\n", "Epoch 8/10\n", "670/670 [==============================] - 0s 60us/sample - loss: 24.0863 - val_loss: 16.8277\n", "Epoch 9/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 17.7293 - val_loss: 14.5462\n", "Epoch 10/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 16.1565 - val_loss: 13.4749\n", "Minumum spacing is: 0.24087745118987458\n", "Train on 670 samples, validate on 330 samples\n", "Epoch 1/10\n", "670/670 [==============================] - 0s 76us/sample - loss: 14.7049 - val_loss: 12.8082\n", "Epoch 2/10\n", "670/670 [==============================] - 0s 67us/sample - loss: 15.0379 - val_loss: 11.9400\n", "Epoch 3/10\n", "670/670 [==============================] - 0s 73us/sample - loss: 11.8772 - val_loss: 11.1420\n", "Epoch 4/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 15.0867 - val_loss: 10.5418\n", "Epoch 5/10\n", "670/670 [==============================] - 0s 60us/sample - loss: 13.3476 - val_loss: 10.2156\n", "Epoch 6/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 14.9345 - val_loss: 9.8355\n", "Epoch 7/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 11.8668 - val_loss: 9.7258\n", "Epoch 8/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 8.8083 - val_loss: 9.6004\n", "Epoch 9/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 17.1439 - val_loss: 9.4359\n", "Epoch 10/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 8.6628 - val_loss: 9.2895\n", "Minumum spacing is: 0.42570958922549007\n", "Train on 670 samples, validate on 330 samples\n", "Epoch 1/10\n", "670/670 [==============================] - 0s 74us/sample - loss: 12.2926 - val_loss: 9.1882\n", "Epoch 2/10\n", "670/670 [==============================] - 0s 68us/sample - loss: 7.0342 - val_loss: 8.9157\n", "Epoch 3/10\n", "670/670 [==============================] - 0s 64us/sample - loss: 7.9756 - val_loss: 8.9089\n", "Epoch 4/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 9.9432 - val_loss: 8.8604\n", "Epoch 5/10\n", "670/670 [==============================] - 0s 60us/sample - loss: 10.5006 - val_loss: 8.8569\n", "Epoch 6/10\n", "670/670 [==============================] - 0s 65us/sample - loss: 9.8542 - val_loss: 8.6167\n", "Epoch 7/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 8.7208 - val_loss: 8.4615\n", "Epoch 8/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 5.9275 - val_loss: 8.4868\n", "Epoch 9/10\n", "670/670 [==============================] - 0s 64us/sample - loss: 8.0351 - val_loss: 8.6984\n", "Epoch 10/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 8.3398 - val_loss: 8.7777\n", "Minumum spacing is: 0.5557570425777243\n", "Train on 670 samples, validate on 330 samples\n", "Epoch 1/10\n", "670/670 [==============================] - 0s 68us/sample - loss: 7.0051 - val_loss: 8.9218\n", "Epoch 2/10\n", "670/670 [==============================] - 0s 73us/sample - loss: 9.6484 - val_loss: 8.7272\n", "Epoch 3/10\n", "670/670 [==============================] - 0s 71us/sample - loss: 8.8673 - val_loss: 8.5788\n", "Epoch 4/10\n", "670/670 [==============================] - 0s 70us/sample - loss: 8.9178 - val_loss: 8.8831\n", "Epoch 5/10\n", "670/670 [==============================] - 0s 60us/sample - loss: 8.4241 - val_loss: 8.5788\n", "Epoch 6/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 5.1767 - val_loss: 8.5136\n", "Epoch 7/10\n", "670/670 [==============================] - 0s 60us/sample - loss: 8.2882 - val_loss: 8.8093\n", "Epoch 8/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 7.1851 - val_loss: 9.0137\n", "Epoch 9/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 6.8966 - val_loss: 9.0894\n", "Epoch 10/10\n", "670/670 [==============================] - 0s 63us/sample - loss: 6.5958 - val_loss: 8.9989\n", "Minumum spacing is: 0.8330272856364322\n", "Train on 670 samples, validate on 330 samples\n", "Epoch 1/10\n", "670/670 [==============================] - 0s 132us/sample - loss: 8.3705 - val_loss: 8.9275\n", "Epoch 2/10\n", "670/670 [==============================] - 0s 107us/sample - loss: 9.3202 - val_loss: 8.9864\n", "Epoch 3/10\n", "670/670 [==============================] - 0s 119us/sample - loss: 7.1511 - val_loss: 9.0435\n", "Epoch 4/10\n", "670/670 [==============================] - 0s 129us/sample - loss: 10.4449 - val_loss: 8.3233\n", "Epoch 5/10\n", "670/670 [==============================] - 0s 137us/sample - loss: 5.1511 - val_loss: 7.6375\n", "Epoch 6/10\n", "670/670 [==============================] - 0s 116us/sample - loss: 9.2908 - val_loss: 7.4775\n", "Epoch 7/10\n", "670/670 [==============================] - 0s 132us/sample - loss: 6.0680 - val_loss: 7.3165\n", "Epoch 8/10\n", "670/670 [==============================] - 0s 121us/sample - loss: 5.9022 - val_loss: 7.1734\n", "Epoch 9/10\n", "670/670 [==============================] - 0s 129us/sample - loss: 7.2901 - val_loss: 7.2078\n", "Epoch 10/10\n", "670/670 [==============================] - 0s 97us/sample - loss: 6.3104 - val_loss: 7.2451\n", "Minumum spacing is: 0.7831922810828136\n", "Train on 670 samples, validate on 330 samples\n", "Epoch 1/10\n", "670/670 [==============================] - 0s 84us/sample - loss: 10.2243 - val_loss: 7.0771\n", "Epoch 2/10\n", "670/670 [==============================] - 0s 85us/sample - loss: 7.2483 - val_loss: 7.2139\n", "Epoch 3/10\n", "670/670 [==============================] - 0s 78us/sample - loss: 5.7181 - val_loss: 7.2216\n", "Epoch 4/10\n", "670/670 [==============================] - 0s 74us/sample - loss: 6.3994 - val_loss: 7.1797\n", "Epoch 5/10\n", "670/670 [==============================] - 0s 91us/sample - loss: 6.8894 - val_loss: 7.2073\n", "Epoch 6/10\n", "670/670 [==============================] - 0s 80us/sample - loss: 10.4982 - val_loss: 7.3759\n", "Epoch 7/10\n", "670/670 [==============================] - 0s 77us/sample - loss: 6.8592 - val_loss: 7.5676\n", "Epoch 8/10\n", "670/670 [==============================] - 0s 73us/sample - loss: 5.8382 - val_loss: 7.6413\n", "Epoch 9/10\n", "670/670 [==============================] - 0s 79us/sample - loss: 7.6254 - val_loss: 7.7458\n", "Epoch 10/10\n", "670/670 [==============================] - 0s 79us/sample - loss: 4.5302 - val_loss: 7.6529\n", "Minumum spacing is: 0.9674323377003642\n", "Train on 670 samples, validate on 330 samples\n", "Epoch 1/10\n", "670/670 [==============================] - 0s 70us/sample - loss: 7.6809 - val_loss: 7.6109\n", "Epoch 2/10\n", "670/670 [==============================] - 0s 58us/sample - loss: 7.5941 - val_loss: 7.4265\n", "Epoch 3/10\n", "670/670 [==============================] - 0s 61us/sample - loss: 7.2981 - val_loss: 6.8968\n", "Epoch 4/10\n", "670/670 [==============================] - 0s 58us/sample - loss: 4.2635 - val_loss: 6.7962\n", "Epoch 5/10\n", "670/670 [==============================] - 0s 55us/sample - loss: 4.9446 - val_loss: 6.8516\n", "Epoch 6/10\n", "670/670 [==============================] - 0s 55us/sample - loss: 5.2650 - val_loss: 7.0854\n", "Epoch 7/10\n", "670/670 [==============================] - 0s 54us/sample - loss: 6.7341 - val_loss: 7.9177\n", "Epoch 8/10\n", "670/670 [==============================] - 0s 57us/sample - loss: 7.8536 - val_loss: 8.8406\n", "Epoch 9/10\n", "670/670 [==============================] - 0s 57us/sample - loss: 7.6863 - val_loss: 7.8821\n", "Epoch 10/10\n", "670/670 [==============================] - 0s 57us/sample - loss: 12.3550 - val_loss: 7.9975\n", "Minumum spacing is: 1.1449549874703102\n" ] } ], "source": [ "import pandas as pd\n", "from sklearn.model_selection import train_test_split\n", "from tensorflow.keras.models import Sequential\n", "from tensorflow.keras.layers import Dense, Dropout\n", "\n", "df = pd.read_csv('datasets-arie_ben_david-era.csv', header=None, sep = ',')\n", "#df = pd.read_csv('fred.csv', header=None, sep = ',')\n", "\n", "X = df.iloc[:,:4]\n", "y = df.iloc[:,4]\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n", "\n", "y_train = np.array(y_train)\n", "y_test = np.array(y_test)\n", "\n", "labels, freq = np.unique(y_train, return_counts=True)\n", "ords = labels.reshape(-1, 1)\n", "l = min(labels)\n", "u = max(labels)\n", "labs = len(labels)\n", "\n", "eyes = np.eye(labs)\n", "cDiff = eyes[1:labs,:]-eyes[:(labs-1),:]\n", "\n", "# Create matrix from on hot encoded training labels to use to calculate class centroids\n", "from sklearn.preprocessing import OneHotEncoder\n", "onehot_encoder = OneHotEncoder(sparse=False)\n", "onehot = onehot_encoder.fit_transform(y_train.reshape((-1, 1)))\n", "testhot = onehot_encoder.fit_transform(y_test.reshape((-1, 1)))\n", "onehot_inverse = 1/np.sum((onehot.T), axis=1)\n", "new_y_train = onehot.T*onehot_inverse.reshape(-1,1)\n", "\n", "model = Sequential()\n", "model.add(Dense(40, activation='relu', input_shape=(4, )))\n", "model.add(Dropout(0.1))\n", "model.add(Dense(28, activation='relu'))\n", "model.add(Dropout(0.1))\n", "model.add(Dense(20, activation='relu'))\n", "model.add(Dropout(0.1))\n", "model.add(Dense(1))\n", "\n", "# # Stage 1: hyperplane ordering loss\n", "opt = Adam(lr=1e-2, beta_1=0.9, beta_2=0.999, decay=1e-4)\n", "model.compile(loss=ordering_loss, optimizer=\"adam\")\n", "\n", "# Run model fit until the proper ordering is achieved and the minimum spacing between centroids is 1.0 or greater\n", "mingap = 0.0\n", "while mingap < 1.0:\n", " model.fit(X_train, y_train, validation_data=(X_test,y_test), epochs=10, batch_size=64, shuffle=True)\n", "\n", " # Calculate hyperplane centroid constant value\n", " pred = model.predict(X_train)\n", " train_cent = np.matmul(new_y_train, pred)\n", " mingap = np.min(np.matmul(cDiff,train_cent))\n", " print('Minumum spacing is: ', mingap)" ] }, { "cell_type": "code", "execution_count": 169, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Train on 670 samples, validate on 330 samples\n", "Epoch 1/100\n", "670/670 [==============================] - 1s 782us/sample - loss: 167.2567 - val_loss: 127.5963\n", "Epoch 2/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 121.0400 - val_loss: 117.3905\n", "Epoch 3/100\n", "670/670 [==============================] - 0s 155us/sample - loss: 116.2964 - val_loss: 115.7787\n", "Epoch 4/100\n", "670/670 [==============================] - 0s 155us/sample - loss: 110.4013 - val_loss: 114.0356\n", "Epoch 5/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 112.2520 - val_loss: 113.8429\n", "Epoch 6/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 114.7526 - val_loss: 114.1633\n", "Epoch 7/100\n", "670/670 [==============================] - 0s 170us/sample - loss: 112.9706 - val_loss: 116.9861\n", "Epoch 8/100\n", "670/670 [==============================] - 0s 161us/sample - loss: 111.4261 - val_loss: 115.8533\n", "Epoch 9/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 110.9044 - val_loss: 114.7312\n", "Epoch 10/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 111.2418 - val_loss: 113.8277\n", "Epoch 11/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 110.9243 - val_loss: 116.4442\n", "Epoch 12/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 113.8810 - val_loss: 116.4488\n", "Epoch 13/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 111.0496 - val_loss: 114.6983\n", "Epoch 14/100\n", "670/670 [==============================] - 0s 162us/sample - loss: 108.8283 - val_loss: 115.1277\n", "Epoch 15/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 108.1028 - val_loss: 115.7120\n", "Epoch 16/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 108.8197 - val_loss: 116.8253\n", "Epoch 17/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 110.1036 - val_loss: 114.5012\n", "Epoch 18/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 108.8452 - val_loss: 116.1498\n", "Epoch 19/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 110.1040 - val_loss: 114.9499\n", "Epoch 20/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 110.2793 - val_loss: 116.2565\n", "Epoch 21/100\n", "670/670 [==============================] - 0s 164us/sample - loss: 109.1190 - val_loss: 116.7019\n", "Epoch 22/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 109.1000 - val_loss: 115.7320\n", "Epoch 23/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 108.6086 - val_loss: 115.8812\n", "Epoch 24/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 112.0533 - val_loss: 116.2427\n", "Epoch 25/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 109.3852 - val_loss: 116.4871\n", "Epoch 26/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 111.0436 - val_loss: 115.7306\n", "Epoch 27/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 109.9564 - val_loss: 113.8913\n", "Epoch 28/100\n", "670/670 [==============================] - 0s 160us/sample - loss: 110.3742 - val_loss: 115.7601\n", "Epoch 29/100\n", "670/670 [==============================] - 0s 157us/sample - loss: 109.9922 - val_loss: 114.5301\n", "Epoch 30/100\n", "670/670 [==============================] - 0s 173us/sample - loss: 111.6758 - val_loss: 117.4590\n", "Epoch 31/100\n", "670/670 [==============================] - 0s 161us/sample - loss: 108.4441 - val_loss: 114.6143\n", "Epoch 32/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 109.9118 - val_loss: 115.6038\n", "Epoch 33/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 108.7277 - val_loss: 114.8794\n", "Epoch 34/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 109.2932 - val_loss: 117.1811\n", "Epoch 35/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 109.6905 - val_loss: 115.6170\n", "Epoch 36/100\n", "670/670 [==============================] - 0s 161us/sample - loss: 108.6115 - val_loss: 115.8114\n", "Epoch 37/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 112.2230 - val_loss: 119.0462\n", "Epoch 38/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 110.2826 - val_loss: 117.0952\n", "Epoch 39/100\n", "670/670 [==============================] - 0s 157us/sample - loss: 108.0821 - val_loss: 116.5656\n", "Epoch 40/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 105.3330 - val_loss: 115.9079\n", "Epoch 41/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 106.4841 - val_loss: 114.7833\n", "Epoch 42/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 109.7752 - val_loss: 114.0051\n", "Epoch 43/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 107.7225 - val_loss: 115.9208\n", "Epoch 44/100\n", "670/670 [==============================] - 0s 160us/sample - loss: 108.7316 - val_loss: 118.6175\n", "Epoch 45/100\n", "670/670 [==============================] - 0s 162us/sample - loss: 107.8814 - val_loss: 114.0660\n", "Epoch 46/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 106.9862 - val_loss: 115.7919\n", "Epoch 47/100\n", "670/670 [==============================] - 0s 162us/sample - loss: 106.7643 - val_loss: 116.9899\n", "Epoch 48/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 106.2131 - val_loss: 113.4357\n", "Epoch 49/100\n", "670/670 [==============================] - 0s 161us/sample - loss: 108.2564 - val_loss: 117.8364\n", "Epoch 50/100\n", "670/670 [==============================] - 0s 161us/sample - loss: 107.4816 - val_loss: 115.8018\n", "Epoch 51/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 107.2132 - val_loss: 115.6249\n", "Epoch 52/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 106.6608 - val_loss: 113.7666\n", "Epoch 53/100\n", "670/670 [==============================] - 0s 159us/sample - loss: 105.8652 - val_loss: 115.0510\n", "Epoch 54/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 109.1945 - val_loss: 115.9417\n", "Epoch 55/100\n", "670/670 [==============================] - 0s 153us/sample - loss: 107.8359 - val_loss: 115.6586\n", "Epoch 56/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 109.7294 - val_loss: 114.5936\n", "Epoch 57/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 108.0911 - val_loss: 116.6708\n", "Epoch 58/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 107.0040 - val_loss: 116.4183\n", "Epoch 59/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 108.4589 - val_loss: 114.7785\n", "Epoch 60/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 106.0424 - val_loss: 115.9743\n", "Epoch 61/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 106.1926 - val_loss: 113.1458\n", "Epoch 62/100\n", "670/670 [==============================] - 0s 153us/sample - loss: 105.4583 - val_loss: 115.6466\n", "Epoch 63/100\n", "670/670 [==============================] - 0s 155us/sample - loss: 106.6337 - val_loss: 115.0320\n", "Epoch 64/100\n", "670/670 [==============================] - 0s 158us/sample - loss: 106.3839 - val_loss: 116.0217\n", "Epoch 65/100\n", "670/670 [==============================] - 0s 156us/sample - loss: 107.9024 - val_loss: 116.8155\n", "Epoch 66/100\n", "670/670 [==============================] - 0s 155us/sample - loss: 108.4517 - val_loss: 118.3348\n", "Epoch 67/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 109.2334 - val_loss: 113.1309\n", "Epoch 68/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 106.0288 - val_loss: 114.1471\n", "Epoch 69/100\n", "670/670 [==============================] - 0s 154us/sample - loss: 104.3934 - val_loss: 115.4455\n", "Epoch 70/100\n", "670/670 [==============================] - 0s 153us/sample - loss: 103.7005 - val_loss: 113.3451\n", "Epoch 71/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 106.8613 - val_loss: 117.6837\n", "Epoch 72/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 107.9370 - val_loss: 115.7556\n", "Epoch 73/100\n", "670/670 [==============================] - 0s 151us/sample - loss: 103.8183 - val_loss: 116.0632\n", "Epoch 74/100\n", "670/670 [==============================] - 0s 151us/sample - loss: 105.3355 - val_loss: 115.5602\n", "Epoch 75/100\n", "670/670 [==============================] - 0s 155us/sample - loss: 107.0530 - val_loss: 113.7497\n", "Epoch 76/100\n", "670/670 [==============================] - 0s 149us/sample - loss: 105.9348 - val_loss: 116.3795\n", "Epoch 77/100\n", "670/670 [==============================] - 0s 149us/sample - loss: 105.8056 - val_loss: 115.6437\n", "Epoch 78/100\n", "670/670 [==============================] - 0s 153us/sample - loss: 104.5948 - val_loss: 115.6150\n", "Epoch 79/100\n", "670/670 [==============================] - 0s 153us/sample - loss: 106.7824 - val_loss: 115.5974\n", "Epoch 80/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 107.2536 - val_loss: 114.4200\n", "Epoch 81/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 105.1371 - val_loss: 116.5220\n", "Epoch 82/100\n", "670/670 [==============================] - 0s 147us/sample - loss: 104.6773 - val_loss: 117.5102\n", "Epoch 83/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 107.7667 - val_loss: 116.5561\n", "Epoch 84/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 106.6779 - val_loss: 114.9916\n", "Epoch 85/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 105.2873 - val_loss: 116.3106\n", "Epoch 86/100\n", "670/670 [==============================] - 0s 153us/sample - loss: 106.0653 - val_loss: 116.9897\n", "Epoch 87/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 106.2050 - val_loss: 113.6906\n", "Epoch 88/100\n", "670/670 [==============================] - 0s 154us/sample - loss: 104.6824 - val_loss: 114.1683\n", "Epoch 89/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 104.7383 - val_loss: 116.2440\n", "Epoch 90/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 105.8883 - val_loss: 114.1893\n", "Epoch 91/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 106.2581 - val_loss: 116.8756\n", "Epoch 92/100\n", "670/670 [==============================] - 0s 147us/sample - loss: 104.5226 - val_loss: 115.4248\n", "Epoch 93/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 105.0512 - val_loss: 114.4435\n", "Epoch 94/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 106.9903 - val_loss: 112.2592\n", "Epoch 95/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 104.3434 - val_loss: 114.0656\n", "Epoch 96/100\n", "670/670 [==============================] - 0s 152us/sample - loss: 106.1934 - val_loss: 115.0479\n", "Epoch 97/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 104.4684 - val_loss: 116.1641\n", "Epoch 98/100\n", "670/670 [==============================] - 0s 150us/sample - loss: 103.9686 - val_loss: 112.2959\n", "Epoch 99/100\n", "670/670 [==============================] - 0s 154us/sample - loss: 108.0042 - val_loss: 114.0467\n", "Epoch 100/100\n", "670/670 [==============================] - 0s 147us/sample - loss: 104.6460 - val_loss: 112.9288\n" ] }, { "data": { "text/plain": [ "" ] }, "execution_count": 169, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from tensorflow.keras.optimizers import Adam\n", "from tensorflow.keras.models import Sequential\n", "\n", "# Create point loss functiona\n", "# train_cent from stage 1\n", "# ords from model fit loop in stage one\n", "\n", "lower_threshold, upper_threshold = ohpl_margins(train_cent, len(ords), 0.1)\n", "point_loss = get_point_wrapper(lower_threshold, upper_threshold)\n", "\n", "# Recompile Model with point loss \n", "opt = tf.keras.optimizers.Adam(lr=1e-3, beta_1=0.9, beta_2=0.999, decay=1e-4)\n", "model.compile(loss=point_loss, optimizer=opt)\n", "\n", "# model fit uses onehot encoded labels\n", "model.fit(X_train, onehot, validation_data=(X_test,testhot), epochs=100, shuffle=True, batch_size=16)\n", "\n" ] }, { "cell_type": "code", "execution_count": 170, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "1.3181818181818181 0.7696969696969697\n" ] } ], "source": [ "# Calculate MZE and MAE\n", "\n", "pred = model.predict(X_train)\n", "new_pred = model.predict(X_test)\n", "\n", "# Identify the closest centroid\n", "rcenter = train_cent.T # create row matrix of centroids from Stage 1\n", "y_pred = np.argmin(abs(new_pred - rcenter), axis=1) + l # l is the min class from Stage 1\n", "\n", "mae = np.mean(abs(y_pred - y_test))\n", "mze = np.mean(abs(y_pred - y_test) > 0) \n", "print(mae, mze)" ] }, { "cell_type": "code", "execution_count": 105, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "4" ] }, "execution_count": 105, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 }