{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 排序模型\n",
    "通过召回的操作， 我们已经进行了问题规模的缩减， 对于每个用户， 选择出了N篇文章作为了候选集，并基于召回的候选集构建了与用户历史相关的特征，以及用户本身的属性特征，文章本省的属性特征，以及用户与文章之间的特征，下面就是使用机器学习模型来对构造好的特征进行学习，然后对测试集进行预测，得到测试集中的每个候选集用户点击的概率，返回点击概率最大的topk个文章，作为最终的结果。\n",
    "\n",
    "排序阶段选择了三个比较有代表性的排序模型，它们分别是：\n",
    "\n",
    "1. LGB的排序模型\n",
    "2. LGB的分类模型\n",
    "3. 深度学习的分类模型DIN\n",
    "\n",
    "得到了最终的排序模型输出的结果之后，还选择了两种比较经典的模型集成的方法：\n",
    "\n",
    "1. 输出结果加权融合\n",
    "2. Staking（将模型的输出结果再使用一个简单模型进行预测）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:20:39.770642Z",
     "start_time": "2020-11-18T04:20:38.500875Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "import gc, os\n",
    "import time\n",
    "from datetime import datetime\n",
    "import lightgbm as lgb\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取排序特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:20:41.843180Z",
     "start_time": "2020-11-18T04:20:41.837287Z"
    }
   },
   "outputs": [],
   "source": [
    "data_path = './data_raw/'\n",
    "save_path = './temp_results/'\n",
    "offline = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:20:53.358138Z",
     "start_time": "2020-11-18T04:20:44.232944Z"
    }
   },
   "outputs": [],
   "source": [
    "# 重新读取数据的时候，发现click_article_id是一个浮点数，所以将其转换成int类型\n",
    "trn_user_item_feats_df = pd.read_csv(save_path + 'trn_user_item_feats_df.csv')\n",
    "trn_user_item_feats_df['click_article_id'] = trn_user_item_feats_df['click_article_id'].astype(int)\n",
    "\n",
    "if offline:\n",
    "    val_user_item_feats_df = pd.read_csv(save_path + 'val_user_item_feats_df.csv')\n",
    "    val_user_item_feats_df['click_article_id'] = val_user_item_feats_df['click_article_id'].astype(int)\n",
    "else:\n",
    "    val_user_item_feats_df = None\n",
    "    \n",
    "tst_user_item_feats_df = pd.read_csv(save_path + 'tst_user_item_feats_df.csv')\n",
    "tst_user_item_feats_df['click_article_id'] = tst_user_item_feats_df['click_article_id'].astype(int)\n",
    "\n",
    "# 做特征的时候为了方便，给测试集也打上了一个无效的标签，这里直接删掉就行\n",
    "del tst_user_item_feats_df['label']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 返回排序后的结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:01.809368Z",
     "start_time": "2020-11-18T04:21:01.799641Z"
    }
   },
   "outputs": [],
   "source": [
    "def submit(recall_df, topk=5, model_name=None):\n",
    "    recall_df = recall_df.sort_values(by=['user_id', 'pred_score'])\n",
    "    recall_df['rank'] = recall_df.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first')\n",
    "    \n",
    "    # 判断是不是每个用户都有5篇文章及以上\n",
    "    tmp = recall_df.groupby('user_id').apply(lambda x: x['rank'].max())\n",
    "    assert tmp.min() >= topk\n",
    "    \n",
    "    del recall_df['pred_score']\n",
    "    submit = recall_df[recall_df['rank'] <= topk].set_index(['user_id', 'rank']).unstack(-1).reset_index()\n",
    "    \n",
    "    submit.columns = [int(col) if isinstance(col, int) else col for col in submit.columns.droplevel(0)]\n",
    "    # 按照提交格式定义列名\n",
    "    submit = submit.rename(columns={'': 'user_id', 1: 'article_1', 2: 'article_2', \n",
    "                                                  3: 'article_3', 4: 'article_4', 5: 'article_5'})\n",
    "    \n",
    "    save_name = save_path + model_name + '_' + datetime.today().strftime('%m-%d') + '.csv'\n",
    "    submit.to_csv(save_name, index=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:04.332198Z",
     "start_time": "2020-11-18T04:21:04.325020Z"
    }
   },
   "outputs": [],
   "source": [
    "# 排序结果归一化\n",
    "def norm_sim(sim_df, weight=0.0):\n",
    "    # print(sim_df.head())\n",
    "    min_sim = sim_df.min()\n",
    "    max_sim = sim_df.max()\n",
    "    if max_sim == min_sim:\n",
    "        sim_df = sim_df.apply(lambda sim: 1.0)\n",
    "    else:\n",
    "        sim_df = sim_df.apply(lambda sim: 1.0 * (sim - min_sim) / (max_sim - min_sim))\n",
    "\n",
    "    sim_df = sim_df.apply(lambda sim: sim + weight)  # plus one\n",
    "    return sim_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LGB排序模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:07.787698Z",
     "start_time": "2020-11-18T04:21:07.536514Z"
    }
   },
   "outputs": [],
   "source": [
    "# 防止中间出错之后重新读取数据\n",
    "trn_user_item_feats_df_rank_model = trn_user_item_feats_df.copy()\n",
    "\n",
    "if offline:\n",
    "    val_user_item_feats_df_rank_model = val_user_item_feats_df.copy()\n",
    "    \n",
    "tst_user_item_feats_df_rank_model = tst_user_item_feats_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:10.839656Z",
     "start_time": "2020-11-18T04:21:10.833109Z"
    }
   },
   "outputs": [],
   "source": [
    "# 定义特征列\n",
    "lgb_cols = ['sim0', 'time_diff0', 'word_diff0','sim_max', 'sim_min', 'sim_sum', \n",
    "            'sim_mean', 'score','click_size', 'time_diff_mean', 'active_level',\n",
    "            'click_environment','click_deviceGroup', 'click_os', 'click_country', \n",
    "            'click_region','click_referrer_type', 'user_time_hob1', 'user_time_hob2',\n",
    "            'words_hbo', 'category_id', 'created_at_ts','words_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:14.126608Z",
     "start_time": "2020-11-18T04:21:13.493653Z"
    }
   },
   "outputs": [],
   "source": [
    "# 排序模型分组\n",
    "trn_user_item_feats_df_rank_model.sort_values(by=['user_id'], inplace=True)\n",
    "g_train = trn_user_item_feats_df_rank_model.groupby(['user_id'], as_index=False).count()[\"label\"].values\n",
    "\n",
    "if offline:\n",
    "    val_user_item_feats_df_rank_model.sort_values(by=['user_id'], inplace=True)\n",
    "    g_val = val_user_item_feats_df_rank_model.groupby(['user_id'], as_index=False).count()[\"label\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:16.136151Z",
     "start_time": "2020-11-18T04:21:16.124444Z"
    }
   },
   "outputs": [],
   "source": [
    "# 排序模型定义\n",
    "lgb_ranker = lgb.LGBMRanker(boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,\n",
    "                            max_depth=-1, n_estimators=100, subsample=0.7, colsample_bytree=0.7, subsample_freq=1,\n",
    "                            learning_rate=0.01, min_child_weight=50, random_state=2018, n_jobs= 16)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:22.965433Z",
     "start_time": "2020-11-18T04:21:17.799127Z"
    }
   },
   "outputs": [],
   "source": [
    "# 排序模型训练\n",
    "if offline:\n",
    "    lgb_ranker.fit(trn_user_item_feats_df_rank_model[lgb_cols], trn_user_item_feats_df_rank_model['label'], group=g_train,\n",
    "                eval_set=[(val_user_item_feats_df_rank_model[lgb_cols], val_user_item_feats_df_rank_model['label'])], \n",
    "                eval_group= [g_val], eval_at=[1, 2, 3, 4, 5], eval_metric=['ndcg', ], early_stopping_rounds=50, )\n",
    "else:\n",
    "    lgb_ranker.fit(trn_user_item_feats_df[lgb_cols], trn_user_item_feats_df['label'], group=g_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:28.616665Z",
     "start_time": "2020-11-18T04:21:24.672280Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型预测\n",
    "tst_user_item_feats_df['pred_score'] = lgb_ranker.predict(tst_user_item_feats_df[lgb_cols], num_iteration=lgb_ranker.best_iteration_)\n",
    "\n",
    "# 将这里的排序结果保存一份，用户后面的模型融合\n",
    "tst_user_item_feats_df[['user_id', 'click_article_id', 'pred_score']].to_csv(save_path + 'lgb_ranker_score.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:21:40.253692Z",
     "start_time": "2020-11-18T04:21:30.546587Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 预测结果重新排序, 及生成提交结果\n",
    "rank_results = tst_user_item_feats_df[['user_id', 'click_article_id', 'pred_score']]\n",
    "rank_results['click_article_id'] = rank_results['click_article_id'].astype(int)\n",
    "submit(rank_results, topk=5, model_name='lgb_ranker')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:22:26.195838Z",
     "start_time": "2020-11-18T04:21:46.115002Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\tvalid_0's ndcg@1: 0.909975\tvalid_0's ndcg@2: 0.963068\tvalid_0's ndcg@3: 0.96533\tvalid_0's ndcg@4: 0.965729\tvalid_0's ndcg@5: 0.965864\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's ndcg@1: 0.9143\tvalid_0's ndcg@2: 0.964711\tvalid_0's ndcg@3: 0.966961\tvalid_0's ndcg@4: 0.967338\tvalid_0's ndcg@5: 0.967483\n",
      "[3]\tvalid_0's ndcg@1: 0.9181\tvalid_0's ndcg@2: 0.966114\tvalid_0's ndcg@3: 0.968289\tvalid_0's ndcg@4: 0.968773\tvalid_0's ndcg@5: 0.96887\n",
      "[4]\tvalid_0's ndcg@1: 0.925575\tvalid_0's ndcg@2: 0.969093\tvalid_0's ndcg@3: 0.971193\tvalid_0's ndcg@4: 0.971603\tvalid_0's ndcg@5: 0.97169\n",
      "[5]\tvalid_0's ndcg@1: 0.9267\tvalid_0's ndcg@2: 0.969635\tvalid_0's ndcg@3: 0.97166\tvalid_0's ndcg@4: 0.972037\tvalid_0's ndcg@5: 0.972133\n",
      "[6]\tvalid_0's ndcg@1: 0.927\tvalid_0's ndcg@2: 0.969682\tvalid_0's ndcg@3: 0.971757\tvalid_0's ndcg@4: 0.972134\tvalid_0's ndcg@5: 0.972231\n",
      "[7]\tvalid_0's ndcg@1: 0.928825\tvalid_0's ndcg@2: 0.970451\tvalid_0's ndcg@3: 0.972476\tvalid_0's ndcg@4: 0.97282\tvalid_0's ndcg@5: 0.972927\n",
      "[8]\tvalid_0's ndcg@1: 0.930025\tvalid_0's ndcg@2: 0.970988\tvalid_0's ndcg@3: 0.972951\tvalid_0's ndcg@4: 0.973295\tvalid_0's ndcg@5: 0.973402\n",
      "[9]\tvalid_0's ndcg@1: 0.931125\tvalid_0's ndcg@2: 0.971347\tvalid_0's ndcg@3: 0.973384\tvalid_0's ndcg@4: 0.973707\tvalid_0's ndcg@5: 0.973794\n",
      "[10]\tvalid_0's ndcg@1: 0.9311\tvalid_0's ndcg@2: 0.971385\tvalid_0's ndcg@3: 0.973372\tvalid_0's ndcg@4: 0.973717\tvalid_0's ndcg@5: 0.973794\n",
      "[11]\tvalid_0's ndcg@1: 0.930975\tvalid_0's ndcg@2: 0.971433\tvalid_0's ndcg@3: 0.973333\tvalid_0's ndcg@4: 0.973699\tvalid_0's ndcg@5: 0.973767\n",
      "[12]\tvalid_0's ndcg@1: 0.93145\tvalid_0's ndcg@2: 0.971656\tvalid_0's ndcg@3: 0.973493\tvalid_0's ndcg@4: 0.973881\tvalid_0's ndcg@5: 0.973949\n",
      "[13]\tvalid_0's ndcg@1: 0.932525\tvalid_0's ndcg@2: 0.971927\tvalid_0's ndcg@3: 0.973839\tvalid_0's ndcg@4: 0.974227\tvalid_0's ndcg@5: 0.974304\n",
      "[14]\tvalid_0's ndcg@1: 0.932575\tvalid_0's ndcg@2: 0.971898\tvalid_0's ndcg@3: 0.973823\tvalid_0's ndcg@4: 0.974243\tvalid_0's ndcg@5: 0.97432\n",
      "[15]\tvalid_0's ndcg@1: 0.9335\tvalid_0's ndcg@2: 0.972239\tvalid_0's ndcg@3: 0.974189\tvalid_0's ndcg@4: 0.974587\tvalid_0's ndcg@5: 0.974665\n",
      "[16]\tvalid_0's ndcg@1: 0.933475\tvalid_0's ndcg@2: 0.972309\tvalid_0's ndcg@3: 0.974209\tvalid_0's ndcg@4: 0.974596\tvalid_0's ndcg@5: 0.974674\n",
      "[17]\tvalid_0's ndcg@1: 0.933725\tvalid_0's ndcg@2: 0.972369\tvalid_0's ndcg@3: 0.974307\tvalid_0's ndcg@4: 0.974684\tvalid_0's ndcg@5: 0.974761\n",
      "[18]\tvalid_0's ndcg@1: 0.9339\tvalid_0's ndcg@2: 0.972497\tvalid_0's ndcg@3: 0.974372\tvalid_0's ndcg@4: 0.974749\tvalid_0's ndcg@5: 0.974836\n",
      "[19]\tvalid_0's ndcg@1: 0.9345\tvalid_0's ndcg@2: 0.972845\tvalid_0's ndcg@3: 0.974645\tvalid_0's ndcg@4: 0.974979\tvalid_0's ndcg@5: 0.975085\n",
      "[20]\tvalid_0's ndcg@1: 0.9349\tvalid_0's ndcg@2: 0.973103\tvalid_0's ndcg@3: 0.97484\tvalid_0's ndcg@4: 0.975174\tvalid_0's ndcg@5: 0.975271\n",
      "[21]\tvalid_0's ndcg@1: 0.935\tvalid_0's ndcg@2: 0.973092\tvalid_0's ndcg@3: 0.97488\tvalid_0's ndcg@4: 0.975192\tvalid_0's ndcg@5: 0.975289\n",
      "[22]\tvalid_0's ndcg@1: 0.93525\tvalid_0's ndcg@2: 0.9732\tvalid_0's ndcg@3: 0.974988\tvalid_0's ndcg@4: 0.975289\tvalid_0's ndcg@5: 0.975386\n",
      "[23]\tvalid_0's ndcg@1: 0.934825\tvalid_0's ndcg@2: 0.972949\tvalid_0's ndcg@3: 0.974824\tvalid_0's ndcg@4: 0.975136\tvalid_0's ndcg@5: 0.975223\n",
      "[24]\tvalid_0's ndcg@1: 0.93545\tvalid_0's ndcg@2: 0.973274\tvalid_0's ndcg@3: 0.975087\tvalid_0's ndcg@4: 0.975388\tvalid_0's ndcg@5: 0.975475\n",
      "[25]\tvalid_0's ndcg@1: 0.9356\tvalid_0's ndcg@2: 0.973345\tvalid_0's ndcg@3: 0.97512\tvalid_0's ndcg@4: 0.975443\tvalid_0's ndcg@5: 0.97553\n",
      "[26]\tvalid_0's ndcg@1: 0.93525\tvalid_0's ndcg@2: 0.9732\tvalid_0's ndcg@3: 0.975\tvalid_0's ndcg@4: 0.975313\tvalid_0's ndcg@5: 0.9754\n",
      "[27]\tvalid_0's ndcg@1: 0.935175\tvalid_0's ndcg@2: 0.97322\tvalid_0's ndcg@3: 0.974983\tvalid_0's ndcg@4: 0.975295\tvalid_0's ndcg@5: 0.975382\n",
      "[28]\tvalid_0's ndcg@1: 0.935425\tvalid_0's ndcg@2: 0.973328\tvalid_0's ndcg@3: 0.975041\tvalid_0's ndcg@4: 0.975374\tvalid_0's ndcg@5: 0.975471\n",
      "[29]\tvalid_0's ndcg@1: 0.935275\tvalid_0's ndcg@2: 0.973225\tvalid_0's ndcg@3: 0.974963\tvalid_0's ndcg@4: 0.975297\tvalid_0's ndcg@5: 0.975403\n",
      "[30]\tvalid_0's ndcg@1: 0.9353\tvalid_0's ndcg@2: 0.973235\tvalid_0's ndcg@3: 0.97501\tvalid_0's ndcg@4: 0.975311\tvalid_0's ndcg@5: 0.975418\n",
      "[31]\tvalid_0's ndcg@1: 0.9356\tvalid_0's ndcg@2: 0.973361\tvalid_0's ndcg@3: 0.975099\tvalid_0's ndcg@4: 0.975422\tvalid_0's ndcg@5: 0.975528\n",
      "[32]\tvalid_0's ndcg@1: 0.9364\tvalid_0's ndcg@2: 0.973641\tvalid_0's ndcg@3: 0.975391\tvalid_0's ndcg@4: 0.975714\tvalid_0's ndcg@5: 0.97582\n",
      "[33]\tvalid_0's ndcg@1: 0.9367\tvalid_0's ndcg@2: 0.973751\tvalid_0's ndcg@3: 0.975501\tvalid_0's ndcg@4: 0.975824\tvalid_0's ndcg@5: 0.975931\n",
      "[34]\tvalid_0's ndcg@1: 0.93715\tvalid_0's ndcg@2: 0.973902\tvalid_0's ndcg@3: 0.975677\tvalid_0's ndcg@4: 0.975989\tvalid_0's ndcg@5: 0.976095\n",
      "[35]\tvalid_0's ndcg@1: 0.9377\tvalid_0's ndcg@2: 0.974105\tvalid_0's ndcg@3: 0.975892\tvalid_0's ndcg@4: 0.976194\tvalid_0's ndcg@5: 0.9763\n",
      "[36]\tvalid_0's ndcg@1: 0.938\tvalid_0's ndcg@2: 0.974184\tvalid_0's ndcg@3: 0.975984\tvalid_0's ndcg@4: 0.976296\tvalid_0's ndcg@5: 0.976402\n",
      "[37]\tvalid_0's ndcg@1: 0.93845\tvalid_0's ndcg@2: 0.974366\tvalid_0's ndcg@3: 0.976166\tvalid_0's ndcg@4: 0.976467\tvalid_0's ndcg@5: 0.976574\n",
      "[38]\tvalid_0's ndcg@1: 0.938925\tvalid_0's ndcg@2: 0.974557\tvalid_0's ndcg@3: 0.976332\tvalid_0's ndcg@4: 0.976655\tvalid_0's ndcg@5: 0.976751\n",
      "[39]\tvalid_0's ndcg@1: 0.93865\tvalid_0's ndcg@2: 0.974471\tvalid_0's ndcg@3: 0.976234\tvalid_0's ndcg@4: 0.976557\tvalid_0's ndcg@5: 0.976653\n",
      "[40]\tvalid_0's ndcg@1: 0.938325\tvalid_0's ndcg@2: 0.974335\tvalid_0's ndcg@3: 0.97611\tvalid_0's ndcg@4: 0.976433\tvalid_0's ndcg@5: 0.97653\n",
      "[41]\tvalid_0's ndcg@1: 0.9391\tvalid_0's ndcg@2: 0.974669\tvalid_0's ndcg@3: 0.976431\tvalid_0's ndcg@4: 0.976743\tvalid_0's ndcg@5: 0.97683\n",
      "[42]\tvalid_0's ndcg@1: 0.939375\tvalid_0's ndcg@2: 0.974833\tvalid_0's ndcg@3: 0.976546\tvalid_0's ndcg@4: 0.976858\tvalid_0's ndcg@5: 0.976945\n",
      "[43]\tvalid_0's ndcg@1: 0.939625\tvalid_0's ndcg@2: 0.974878\tvalid_0's ndcg@3: 0.976628\tvalid_0's ndcg@4: 0.97694\tvalid_0's ndcg@5: 0.977027\n",
      "[44]\tvalid_0's ndcg@1: 0.9395\tvalid_0's ndcg@2: 0.974832\tvalid_0's ndcg@3: 0.97657\tvalid_0's ndcg@4: 0.976893\tvalid_0's ndcg@5: 0.97698\n",
      "[45]\tvalid_0's ndcg@1: 0.939775\tvalid_0's ndcg@2: 0.974949\tvalid_0's ndcg@3: 0.976674\tvalid_0's ndcg@4: 0.976997\tvalid_0's ndcg@5: 0.977084\n",
      "[46]\tvalid_0's ndcg@1: 0.93985\tvalid_0's ndcg@2: 0.974945\tvalid_0's ndcg@3: 0.976708\tvalid_0's ndcg@4: 0.97702\tvalid_0's ndcg@5: 0.977107\n",
      "[47]\tvalid_0's ndcg@1: 0.94005\tvalid_0's ndcg@2: 0.975004\tvalid_0's ndcg@3: 0.976766\tvalid_0's ndcg@4: 0.977078\tvalid_0's ndcg@5: 0.977175\n",
      "[48]\tvalid_0's ndcg@1: 0.940425\tvalid_0's ndcg@2: 0.975189\tvalid_0's ndcg@3: 0.976939\tvalid_0's ndcg@4: 0.97723\tvalid_0's ndcg@5: 0.977327\n",
      "[49]\tvalid_0's ndcg@1: 0.940425\tvalid_0's ndcg@2: 0.975189\tvalid_0's ndcg@3: 0.976939\tvalid_0's ndcg@4: 0.97723\tvalid_0's ndcg@5: 0.977327\n",
      "[50]\tvalid_0's ndcg@1: 0.9405\tvalid_0's ndcg@2: 0.975264\tvalid_0's ndcg@3: 0.976989\tvalid_0's ndcg@4: 0.977291\tvalid_0's ndcg@5: 0.977368\n",
      "[51]\tvalid_0's ndcg@1: 0.941125\tvalid_0's ndcg@2: 0.975526\tvalid_0's ndcg@3: 0.977226\tvalid_0's ndcg@4: 0.977528\tvalid_0's ndcg@5: 0.977605\n",
      "[52]\tvalid_0's ndcg@1: 0.941\tvalid_0's ndcg@2: 0.97548\tvalid_0's ndcg@3: 0.977193\tvalid_0's ndcg@4: 0.977484\tvalid_0's ndcg@5: 0.977561\n",
      "[53]\tvalid_0's ndcg@1: 0.9411\tvalid_0's ndcg@2: 0.975596\tvalid_0's ndcg@3: 0.977259\tvalid_0's ndcg@4: 0.977539\tvalid_0's ndcg@5: 0.977616\n",
      "[54]\tvalid_0's ndcg@1: 0.9412\tvalid_0's ndcg@2: 0.975712\tvalid_0's ndcg@3: 0.977299\tvalid_0's ndcg@4: 0.97759\tvalid_0's ndcg@5: 0.977667\n",
      "[55]\tvalid_0's ndcg@1: 0.94155\tvalid_0's ndcg@2: 0.975841\tvalid_0's ndcg@3: 0.977429\tvalid_0's ndcg@4: 0.977719\tvalid_0's ndcg@5: 0.977797\n",
      "[56]\tvalid_0's ndcg@1: 0.941825\tvalid_0's ndcg@2: 0.975943\tvalid_0's ndcg@3: 0.97753\tvalid_0's ndcg@4: 0.977821\tvalid_0's ndcg@5: 0.977898\n",
      "[57]\tvalid_0's ndcg@1: 0.9416\tvalid_0's ndcg@2: 0.975891\tvalid_0's ndcg@3: 0.977429\tvalid_0's ndcg@4: 0.977741\tvalid_0's ndcg@5: 0.977818\n",
      "[58]\tvalid_0's ndcg@1: 0.941725\tvalid_0's ndcg@2: 0.975969\tvalid_0's ndcg@3: 0.977494\tvalid_0's ndcg@4: 0.977795\tvalid_0's ndcg@5: 0.977873\n",
      "[59]\tvalid_0's ndcg@1: 0.942025\tvalid_0's ndcg@2: 0.975985\tvalid_0's ndcg@3: 0.977547\tvalid_0's ndcg@4: 0.977881\tvalid_0's ndcg@5: 0.977958\n",
      "[60]\tvalid_0's ndcg@1: 0.94205\tvalid_0's ndcg@2: 0.975994\tvalid_0's ndcg@3: 0.977569\tvalid_0's ndcg@4: 0.977892\tvalid_0's ndcg@5: 0.977969\n",
      "[61]\tvalid_0's ndcg@1: 0.94205\tvalid_0's ndcg@2: 0.975947\tvalid_0's ndcg@3: 0.977559\tvalid_0's ndcg@4: 0.977882\tvalid_0's ndcg@5: 0.97796\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[62]\tvalid_0's ndcg@1: 0.942225\tvalid_0's ndcg@2: 0.976027\tvalid_0's ndcg@3: 0.97764\tvalid_0's ndcg@4: 0.977941\tvalid_0's ndcg@5: 0.978028\n",
      "[63]\tvalid_0's ndcg@1: 0.942125\tvalid_0's ndcg@2: 0.976022\tvalid_0's ndcg@3: 0.977622\tvalid_0's ndcg@4: 0.977912\tvalid_0's ndcg@5: 0.977999\n",
      "[64]\tvalid_0's ndcg@1: 0.942675\tvalid_0's ndcg@2: 0.976193\tvalid_0's ndcg@3: 0.977793\tvalid_0's ndcg@4: 0.978105\tvalid_0's ndcg@5: 0.978192\n",
      "[65]\tvalid_0's ndcg@1: 0.942725\tvalid_0's ndcg@2: 0.976227\tvalid_0's ndcg@3: 0.977802\tvalid_0's ndcg@4: 0.978125\tvalid_0's ndcg@5: 0.978212\n",
      "[66]\tvalid_0's ndcg@1: 0.942425\tvalid_0's ndcg@2: 0.976132\tvalid_0's ndcg@3: 0.977695\tvalid_0's ndcg@4: 0.978018\tvalid_0's ndcg@5: 0.978105\n",
      "[67]\tvalid_0's ndcg@1: 0.9424\tvalid_0's ndcg@2: 0.976092\tvalid_0's ndcg@3: 0.977679\tvalid_0's ndcg@4: 0.978002\tvalid_0's ndcg@5: 0.978089\n",
      "[68]\tvalid_0's ndcg@1: 0.942425\tvalid_0's ndcg@2: 0.976148\tvalid_0's ndcg@3: 0.977698\tvalid_0's ndcg@4: 0.978021\tvalid_0's ndcg@5: 0.978108\n",
      "[69]\tvalid_0's ndcg@1: 0.9424\tvalid_0's ndcg@2: 0.976123\tvalid_0's ndcg@3: 0.977686\tvalid_0's ndcg@4: 0.978009\tvalid_0's ndcg@5: 0.978096\n",
      "[70]\tvalid_0's ndcg@1: 0.942625\tvalid_0's ndcg@2: 0.976222\tvalid_0's ndcg@3: 0.977785\tvalid_0's ndcg@4: 0.978097\tvalid_0's ndcg@5: 0.978184\n",
      "[71]\tvalid_0's ndcg@1: 0.942575\tvalid_0's ndcg@2: 0.976188\tvalid_0's ndcg@3: 0.977763\tvalid_0's ndcg@4: 0.978075\tvalid_0's ndcg@5: 0.978162\n",
      "[72]\tvalid_0's ndcg@1: 0.9427\tvalid_0's ndcg@2: 0.976234\tvalid_0's ndcg@3: 0.977809\tvalid_0's ndcg@4: 0.978121\tvalid_0's ndcg@5: 0.978208\n",
      "[73]\tvalid_0's ndcg@1: 0.9428\tvalid_0's ndcg@2: 0.976255\tvalid_0's ndcg@3: 0.977843\tvalid_0's ndcg@4: 0.978155\tvalid_0's ndcg@5: 0.978242\n",
      "[74]\tvalid_0's ndcg@1: 0.94295\tvalid_0's ndcg@2: 0.97631\tvalid_0's ndcg@3: 0.977898\tvalid_0's ndcg@4: 0.97821\tvalid_0's ndcg@5: 0.978297\n",
      "[75]\tvalid_0's ndcg@1: 0.943\tvalid_0's ndcg@2: 0.976329\tvalid_0's ndcg@3: 0.977941\tvalid_0's ndcg@4: 0.978232\tvalid_0's ndcg@5: 0.978319\n",
      "[76]\tvalid_0's ndcg@1: 0.9433\tvalid_0's ndcg@2: 0.976471\tvalid_0's ndcg@3: 0.978059\tvalid_0's ndcg@4: 0.97836\tvalid_0's ndcg@5: 0.978437\n",
      "[77]\tvalid_0's ndcg@1: 0.94315\tvalid_0's ndcg@2: 0.976416\tvalid_0's ndcg@3: 0.977991\tvalid_0's ndcg@4: 0.978314\tvalid_0's ndcg@5: 0.978381\n",
      "[78]\tvalid_0's ndcg@1: 0.943675\tvalid_0's ndcg@2: 0.976657\tvalid_0's ndcg@3: 0.978194\tvalid_0's ndcg@4: 0.978517\tvalid_0's ndcg@5: 0.978585\n",
      "[79]\tvalid_0's ndcg@1: 0.94365\tvalid_0's ndcg@2: 0.976663\tvalid_0's ndcg@3: 0.978188\tvalid_0's ndcg@4: 0.978501\tvalid_0's ndcg@5: 0.978578\n",
      "[80]\tvalid_0's ndcg@1: 0.943725\tvalid_0's ndcg@2: 0.976628\tvalid_0's ndcg@3: 0.978203\tvalid_0's ndcg@4: 0.978515\tvalid_0's ndcg@5: 0.978593\n",
      "[81]\tvalid_0's ndcg@1: 0.943975\tvalid_0's ndcg@2: 0.97672\tvalid_0's ndcg@3: 0.978295\tvalid_0's ndcg@4: 0.978607\tvalid_0's ndcg@5: 0.978685\n",
      "[82]\tvalid_0's ndcg@1: 0.94425\tvalid_0's ndcg@2: 0.976822\tvalid_0's ndcg@3: 0.978397\tvalid_0's ndcg@4: 0.97872\tvalid_0's ndcg@5: 0.978787\n",
      "[83]\tvalid_0's ndcg@1: 0.9442\tvalid_0's ndcg@2: 0.976788\tvalid_0's ndcg@3: 0.978375\tvalid_0's ndcg@4: 0.978698\tvalid_0's ndcg@5: 0.978766\n",
      "[84]\tvalid_0's ndcg@1: 0.94425\tvalid_0's ndcg@2: 0.97679\tvalid_0's ndcg@3: 0.97839\tvalid_0's ndcg@4: 0.978702\tvalid_0's ndcg@5: 0.97878\n",
      "[85]\tvalid_0's ndcg@1: 0.9443\tvalid_0's ndcg@2: 0.976809\tvalid_0's ndcg@3: 0.978421\tvalid_0's ndcg@4: 0.978723\tvalid_0's ndcg@5: 0.9788\n",
      "[86]\tvalid_0's ndcg@1: 0.944525\tvalid_0's ndcg@2: 0.976939\tvalid_0's ndcg@3: 0.978502\tvalid_0's ndcg@4: 0.978814\tvalid_0's ndcg@5: 0.978891\n",
      "[87]\tvalid_0's ndcg@1: 0.944625\tvalid_0's ndcg@2: 0.976976\tvalid_0's ndcg@3: 0.978551\tvalid_0's ndcg@4: 0.978852\tvalid_0's ndcg@5: 0.97893\n",
      "[88]\tvalid_0's ndcg@1: 0.944925\tvalid_0's ndcg@2: 0.977102\tvalid_0's ndcg@3: 0.978677\tvalid_0's ndcg@4: 0.978968\tvalid_0's ndcg@5: 0.979045\n",
      "[89]\tvalid_0's ndcg@1: 0.945125\tvalid_0's ndcg@2: 0.977208\tvalid_0's ndcg@3: 0.978758\tvalid_0's ndcg@4: 0.979048\tvalid_0's ndcg@5: 0.979126\n",
      "[90]\tvalid_0's ndcg@1: 0.9451\tvalid_0's ndcg@2: 0.977135\tvalid_0's ndcg@3: 0.978735\tvalid_0's ndcg@4: 0.979026\tvalid_0's ndcg@5: 0.979104\n",
      "[91]\tvalid_0's ndcg@1: 0.945425\tvalid_0's ndcg@2: 0.977208\tvalid_0's ndcg@3: 0.978858\tvalid_0's ndcg@4: 0.979138\tvalid_0's ndcg@5: 0.979215\n",
      "[92]\tvalid_0's ndcg@1: 0.9455\tvalid_0's ndcg@2: 0.977267\tvalid_0's ndcg@3: 0.978905\tvalid_0's ndcg@4: 0.979174\tvalid_0's ndcg@5: 0.979251\n",
      "[93]\tvalid_0's ndcg@1: 0.9453\tvalid_0's ndcg@2: 0.977193\tvalid_0's ndcg@3: 0.978818\tvalid_0's ndcg@4: 0.979098\tvalid_0's ndcg@5: 0.979176\n",
      "[94]\tvalid_0's ndcg@1: 0.94545\tvalid_0's ndcg@2: 0.97728\tvalid_0's ndcg@3: 0.97888\tvalid_0's ndcg@4: 0.97916\tvalid_0's ndcg@5: 0.979238\n",
      "[95]\tvalid_0's ndcg@1: 0.9458\tvalid_0's ndcg@2: 0.977394\tvalid_0's ndcg@3: 0.979006\tvalid_0's ndcg@4: 0.979286\tvalid_0's ndcg@5: 0.979364\n",
      "[96]\tvalid_0's ndcg@1: 0.946075\tvalid_0's ndcg@2: 0.977527\tvalid_0's ndcg@3: 0.979114\tvalid_0's ndcg@4: 0.979394\tvalid_0's ndcg@5: 0.979472\n",
      "[97]\tvalid_0's ndcg@1: 0.946475\tvalid_0's ndcg@2: 0.977659\tvalid_0's ndcg@3: 0.979259\tvalid_0's ndcg@4: 0.979539\tvalid_0's ndcg@5: 0.979616\n",
      "[98]\tvalid_0's ndcg@1: 0.94675\tvalid_0's ndcg@2: 0.97776\tvalid_0's ndcg@3: 0.97936\tvalid_0's ndcg@4: 0.979651\tvalid_0's ndcg@5: 0.979719\n",
      "[99]\tvalid_0's ndcg@1: 0.9469\tvalid_0's ndcg@2: 0.977831\tvalid_0's ndcg@3: 0.979419\tvalid_0's ndcg@4: 0.97971\tvalid_0's ndcg@5: 0.979777\n",
      "[100]\tvalid_0's ndcg@1: 0.9468\tvalid_0's ndcg@2: 0.977794\tvalid_0's ndcg@3: 0.979369\tvalid_0's ndcg@4: 0.979671\tvalid_0's ndcg@5: 0.979739\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[99]\tvalid_0's ndcg@1: 0.9469\tvalid_0's ndcg@2: 0.977831\tvalid_0's ndcg@3: 0.979419\tvalid_0's ndcg@4: 0.97971\tvalid_0's ndcg@5: 0.979777\n",
      "[1]\tvalid_0's ndcg@1: 0.909075\tvalid_0's ndcg@2: 0.963019\tvalid_0's ndcg@3: 0.965069\tvalid_0's ndcg@4: 0.965543\tvalid_0's ndcg@5: 0.965601\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's ndcg@1: 0.9123\tvalid_0's ndcg@2: 0.964273\tvalid_0's ndcg@3: 0.966248\tvalid_0's ndcg@4: 0.966722\tvalid_0's ndcg@5: 0.966789\n",
      "[3]\tvalid_0's ndcg@1: 0.915075\tvalid_0's ndcg@2: 0.965691\tvalid_0's ndcg@3: 0.967466\tvalid_0's ndcg@4: 0.967854\tvalid_0's ndcg@5: 0.967922\n",
      "[4]\tvalid_0's ndcg@1: 0.91845\tvalid_0's ndcg@2: 0.967047\tvalid_0's ndcg@3: 0.968735\tvalid_0's ndcg@4: 0.969133\tvalid_0's ndcg@5: 0.969201\n",
      "[5]\tvalid_0's ndcg@1: 0.92355\tvalid_0's ndcg@2: 0.968961\tvalid_0's ndcg@3: 0.970674\tvalid_0's ndcg@4: 0.97104\tvalid_0's ndcg@5: 0.971098\n",
      "[6]\tvalid_0's ndcg@1: 0.9253\tvalid_0's ndcg@2: 0.969607\tvalid_0's ndcg@3: 0.971345\tvalid_0's ndcg@4: 0.971689\tvalid_0's ndcg@5: 0.971747\n",
      "[7]\tvalid_0's ndcg@1: 0.926225\tvalid_0's ndcg@2: 0.969933\tvalid_0's ndcg@3: 0.971708\tvalid_0's ndcg@4: 0.972031\tvalid_0's ndcg@5: 0.972079\n",
      "[8]\tvalid_0's ndcg@1: 0.926475\tvalid_0's ndcg@2: 0.970104\tvalid_0's ndcg@3: 0.971804\tvalid_0's ndcg@4: 0.972116\tvalid_0's ndcg@5: 0.972184\n",
      "[9]\tvalid_0's ndcg@1: 0.9277\tvalid_0's ndcg@2: 0.970682\tvalid_0's ndcg@3: 0.972307\tvalid_0's ndcg@4: 0.972598\tvalid_0's ndcg@5: 0.972675\n",
      "[10]\tvalid_0's ndcg@1: 0.92775\tvalid_0's ndcg@2: 0.970653\tvalid_0's ndcg@3: 0.972316\tvalid_0's ndcg@4: 0.972617\tvalid_0's ndcg@5: 0.972685\n",
      "[11]\tvalid_0's ndcg@1: 0.9283\tvalid_0's ndcg@2: 0.97084\tvalid_0's ndcg@3: 0.97254\tvalid_0's ndcg@4: 0.97281\tvalid_0's ndcg@5: 0.972887\n",
      "[12]\tvalid_0's ndcg@1: 0.9287\tvalid_0's ndcg@2: 0.971051\tvalid_0's ndcg@3: 0.972701\tvalid_0's ndcg@4: 0.97297\tvalid_0's ndcg@5: 0.973048\n",
      "[13]\tvalid_0's ndcg@1: 0.9297\tvalid_0's ndcg@2: 0.971389\tvalid_0's ndcg@3: 0.973001\tvalid_0's ndcg@4: 0.973313\tvalid_0's ndcg@5: 0.9734\n",
      "[14]\tvalid_0's ndcg@1: 0.92955\tvalid_0's ndcg@2: 0.971444\tvalid_0's ndcg@3: 0.972994\tvalid_0's ndcg@4: 0.973284\tvalid_0's ndcg@5: 0.973371\n",
      "[15]\tvalid_0's ndcg@1: 0.930225\tvalid_0's ndcg@2: 0.97174\tvalid_0's ndcg@3: 0.973253\tvalid_0's ndcg@4: 0.973543\tvalid_0's ndcg@5: 0.97363\n",
      "[16]\tvalid_0's ndcg@1: 0.930425\tvalid_0's ndcg@2: 0.971798\tvalid_0's ndcg@3: 0.973298\tvalid_0's ndcg@4: 0.97361\tvalid_0's ndcg@5: 0.973698\n",
      "[17]\tvalid_0's ndcg@1: 0.93125\tvalid_0's ndcg@2: 0.971992\tvalid_0's ndcg@3: 0.97358\tvalid_0's ndcg@4: 0.973903\tvalid_0's ndcg@5: 0.97398\n",
      "[18]\tvalid_0's ndcg@1: 0.931925\tvalid_0's ndcg@2: 0.972257\tvalid_0's ndcg@3: 0.973845\tvalid_0's ndcg@4: 0.974146\tvalid_0's ndcg@5: 0.974224\n",
      "[19]\tvalid_0's ndcg@1: 0.932375\tvalid_0's ndcg@2: 0.972376\tvalid_0's ndcg@3: 0.974038\tvalid_0's ndcg@4: 0.974318\tvalid_0's ndcg@5: 0.974376\n",
      "[20]\tvalid_0's ndcg@1: 0.932\tvalid_0's ndcg@2: 0.972269\tvalid_0's ndcg@3: 0.973907\tvalid_0's ndcg@4: 0.974187\tvalid_0's ndcg@5: 0.974245\n",
      "[21]\tvalid_0's ndcg@1: 0.932725\tvalid_0's ndcg@2: 0.972568\tvalid_0's ndcg@3: 0.974181\tvalid_0's ndcg@4: 0.974471\tvalid_0's ndcg@5: 0.974529\n",
      "[22]\tvalid_0's ndcg@1: 0.93305\tvalid_0's ndcg@2: 0.972735\tvalid_0's ndcg@3: 0.974298\tvalid_0's ndcg@4: 0.974599\tvalid_0's ndcg@5: 0.974657\n",
      "[23]\tvalid_0's ndcg@1: 0.932925\tvalid_0's ndcg@2: 0.972642\tvalid_0's ndcg@3: 0.974255\tvalid_0's ndcg@4: 0.974545\tvalid_0's ndcg@5: 0.974594\n",
      "[24]\tvalid_0's ndcg@1: 0.933175\tvalid_0's ndcg@2: 0.972734\tvalid_0's ndcg@3: 0.974347\tvalid_0's ndcg@4: 0.974638\tvalid_0's ndcg@5: 0.974686\n",
      "[25]\tvalid_0's ndcg@1: 0.9331\tvalid_0's ndcg@2: 0.972754\tvalid_0's ndcg@3: 0.974366\tvalid_0's ndcg@4: 0.974636\tvalid_0's ndcg@5: 0.974674\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[26]\tvalid_0's ndcg@1: 0.933275\tvalid_0's ndcg@2: 0.972787\tvalid_0's ndcg@3: 0.974424\tvalid_0's ndcg@4: 0.974694\tvalid_0's ndcg@5: 0.974732\n",
      "[27]\tvalid_0's ndcg@1: 0.93325\tvalid_0's ndcg@2: 0.972809\tvalid_0's ndcg@3: 0.974434\tvalid_0's ndcg@4: 0.974703\tvalid_0's ndcg@5: 0.974732\n",
      "[28]\tvalid_0's ndcg@1: 0.933625\tvalid_0's ndcg@2: 0.972932\tvalid_0's ndcg@3: 0.974557\tvalid_0's ndcg@4: 0.974826\tvalid_0's ndcg@5: 0.974855\n",
      "[29]\tvalid_0's ndcg@1: 0.933725\tvalid_0's ndcg@2: 0.972937\tvalid_0's ndcg@3: 0.974587\tvalid_0's ndcg@4: 0.974856\tvalid_0's ndcg@5: 0.974885\n",
      "[30]\tvalid_0's ndcg@1: 0.93355\tvalid_0's ndcg@2: 0.972873\tvalid_0's ndcg@3: 0.974523\tvalid_0's ndcg@4: 0.974792\tvalid_0's ndcg@5: 0.974821\n",
      "[31]\tvalid_0's ndcg@1: 0.9342\tvalid_0's ndcg@2: 0.973065\tvalid_0's ndcg@3: 0.974753\tvalid_0's ndcg@4: 0.975022\tvalid_0's ndcg@5: 0.975051\n",
      "[32]\tvalid_0's ndcg@1: 0.93435\tvalid_0's ndcg@2: 0.973152\tvalid_0's ndcg@3: 0.974815\tvalid_0's ndcg@4: 0.975084\tvalid_0's ndcg@5: 0.975113\n",
      "[33]\tvalid_0's ndcg@1: 0.934475\tvalid_0's ndcg@2: 0.97323\tvalid_0's ndcg@3: 0.974855\tvalid_0's ndcg@4: 0.975135\tvalid_0's ndcg@5: 0.975164\n",
      "[34]\tvalid_0's ndcg@1: 0.9342\tvalid_0's ndcg@2: 0.973113\tvalid_0's ndcg@3: 0.974738\tvalid_0's ndcg@4: 0.975028\tvalid_0's ndcg@5: 0.975057\n",
      "[35]\tvalid_0's ndcg@1: 0.93455\tvalid_0's ndcg@2: 0.973258\tvalid_0's ndcg@3: 0.97487\tvalid_0's ndcg@4: 0.975172\tvalid_0's ndcg@5: 0.975201\n",
      "[36]\tvalid_0's ndcg@1: 0.9344\tvalid_0's ndcg@2: 0.973265\tvalid_0's ndcg@3: 0.974828\tvalid_0's ndcg@4: 0.975129\tvalid_0's ndcg@5: 0.975158\n",
      "[37]\tvalid_0's ndcg@1: 0.934825\tvalid_0's ndcg@2: 0.973438\tvalid_0's ndcg@3: 0.975013\tvalid_0's ndcg@4: 0.975304\tvalid_0's ndcg@5: 0.975323\n",
      "[38]\tvalid_0's ndcg@1: 0.934975\tvalid_0's ndcg@2: 0.973541\tvalid_0's ndcg@3: 0.975066\tvalid_0's ndcg@4: 0.975367\tvalid_0's ndcg@5: 0.975386\n",
      "[39]\tvalid_0's ndcg@1: 0.935275\tvalid_0's ndcg@2: 0.973667\tvalid_0's ndcg@3: 0.975192\tvalid_0's ndcg@4: 0.975483\tvalid_0's ndcg@5: 0.975502\n",
      "[40]\tvalid_0's ndcg@1: 0.9352\tvalid_0's ndcg@2: 0.973624\tvalid_0's ndcg@3: 0.975174\tvalid_0's ndcg@4: 0.975454\tvalid_0's ndcg@5: 0.975473\n",
      "[41]\tvalid_0's ndcg@1: 0.935325\tvalid_0's ndcg@2: 0.973686\tvalid_0's ndcg@3: 0.975223\tvalid_0's ndcg@4: 0.975503\tvalid_0's ndcg@5: 0.975522\n",
      "[42]\tvalid_0's ndcg@1: 0.93545\tvalid_0's ndcg@2: 0.973716\tvalid_0's ndcg@3: 0.975266\tvalid_0's ndcg@4: 0.975546\tvalid_0's ndcg@5: 0.975565\n",
      "[43]\tvalid_0's ndcg@1: 0.93615\tvalid_0's ndcg@2: 0.974022\tvalid_0's ndcg@3: 0.975534\tvalid_0's ndcg@4: 0.975814\tvalid_0's ndcg@5: 0.975843\n",
      "[44]\tvalid_0's ndcg@1: 0.936225\tvalid_0's ndcg@2: 0.974112\tvalid_0's ndcg@3: 0.975562\tvalid_0's ndcg@4: 0.975853\tvalid_0's ndcg@5: 0.975882\n",
      "[45]\tvalid_0's ndcg@1: 0.9365\tvalid_0's ndcg@2: 0.974167\tvalid_0's ndcg@3: 0.975654\tvalid_0's ndcg@4: 0.975945\tvalid_0's ndcg@5: 0.975974\n",
      "[46]\tvalid_0's ndcg@1: 0.93665\tvalid_0's ndcg@2: 0.974206\tvalid_0's ndcg@3: 0.975694\tvalid_0's ndcg@4: 0.975995\tvalid_0's ndcg@5: 0.976024\n",
      "[47]\tvalid_0's ndcg@1: 0.93685\tvalid_0's ndcg@2: 0.974311\tvalid_0's ndcg@3: 0.975786\tvalid_0's ndcg@4: 0.976077\tvalid_0's ndcg@5: 0.976106\n",
      "[48]\tvalid_0's ndcg@1: 0.937025\tvalid_0's ndcg@2: 0.974408\tvalid_0's ndcg@3: 0.975845\tvalid_0's ndcg@4: 0.976147\tvalid_0's ndcg@5: 0.976185\n",
      "[49]\tvalid_0's ndcg@1: 0.936975\tvalid_0's ndcg@2: 0.974342\tvalid_0's ndcg@3: 0.975829\tvalid_0's ndcg@4: 0.97612\tvalid_0's ndcg@5: 0.976159\n",
      "[50]\tvalid_0's ndcg@1: 0.9371\tvalid_0's ndcg@2: 0.974388\tvalid_0's ndcg@3: 0.97585\tvalid_0's ndcg@4: 0.976152\tvalid_0's ndcg@5: 0.976191\n",
      "[51]\tvalid_0's ndcg@1: 0.937025\tvalid_0's ndcg@2: 0.974329\tvalid_0's ndcg@3: 0.975841\tvalid_0's ndcg@4: 0.976121\tvalid_0's ndcg@5: 0.97616\n",
      "[52]\tvalid_0's ndcg@1: 0.9377\tvalid_0's ndcg@2: 0.974578\tvalid_0's ndcg@3: 0.976078\tvalid_0's ndcg@4: 0.976369\tvalid_0's ndcg@5: 0.976407\n",
      "[53]\tvalid_0's ndcg@1: 0.9378\tvalid_0's ndcg@2: 0.974615\tvalid_0's ndcg@3: 0.976115\tvalid_0's ndcg@4: 0.976405\tvalid_0's ndcg@5: 0.976444\n",
      "[54]\tvalid_0's ndcg@1: 0.938\tvalid_0's ndcg@2: 0.974689\tvalid_0's ndcg@3: 0.976214\tvalid_0's ndcg@4: 0.976483\tvalid_0's ndcg@5: 0.976521\n",
      "[55]\tvalid_0's ndcg@1: 0.938225\tvalid_0's ndcg@2: 0.974803\tvalid_0's ndcg@3: 0.976303\tvalid_0's ndcg@4: 0.976572\tvalid_0's ndcg@5: 0.976611\n",
      "[56]\tvalid_0's ndcg@1: 0.938175\tvalid_0's ndcg@2: 0.9748\tvalid_0's ndcg@3: 0.976275\tvalid_0's ndcg@4: 0.976555\tvalid_0's ndcg@5: 0.976594\n",
      "[57]\tvalid_0's ndcg@1: 0.938525\tvalid_0's ndcg@2: 0.974914\tvalid_0's ndcg@3: 0.976414\tvalid_0's ndcg@4: 0.976683\tvalid_0's ndcg@5: 0.976722\n",
      "[58]\tvalid_0's ndcg@1: 0.93875\tvalid_0's ndcg@2: 0.975028\tvalid_0's ndcg@3: 0.976503\tvalid_0's ndcg@4: 0.976773\tvalid_0's ndcg@5: 0.976811\n",
      "[59]\tvalid_0's ndcg@1: 0.939125\tvalid_0's ndcg@2: 0.975198\tvalid_0's ndcg@3: 0.976648\tvalid_0's ndcg@4: 0.976918\tvalid_0's ndcg@5: 0.976956\n",
      "[60]\tvalid_0's ndcg@1: 0.939025\tvalid_0's ndcg@2: 0.975177\tvalid_0's ndcg@3: 0.976615\tvalid_0's ndcg@4: 0.976884\tvalid_0's ndcg@5: 0.976923\n",
      "[61]\tvalid_0's ndcg@1: 0.9391\tvalid_0's ndcg@2: 0.975205\tvalid_0's ndcg@3: 0.976642\tvalid_0's ndcg@4: 0.976912\tvalid_0's ndcg@5: 0.97695\n",
      "[62]\tvalid_0's ndcg@1: 0.93965\tvalid_0's ndcg@2: 0.975424\tvalid_0's ndcg@3: 0.976836\tvalid_0's ndcg@4: 0.977116\tvalid_0's ndcg@5: 0.977155\n",
      "[63]\tvalid_0's ndcg@1: 0.940075\tvalid_0's ndcg@2: 0.975596\tvalid_0's ndcg@3: 0.976996\tvalid_0's ndcg@4: 0.977276\tvalid_0's ndcg@5: 0.977315\n",
      "[64]\tvalid_0's ndcg@1: 0.940375\tvalid_0's ndcg@2: 0.975723\tvalid_0's ndcg@3: 0.977123\tvalid_0's ndcg@4: 0.977392\tvalid_0's ndcg@5: 0.977431\n",
      "[65]\tvalid_0's ndcg@1: 0.94045\tvalid_0's ndcg@2: 0.975766\tvalid_0's ndcg@3: 0.977154\tvalid_0's ndcg@4: 0.977423\tvalid_0's ndcg@5: 0.977462\n",
      "[66]\tvalid_0's ndcg@1: 0.940475\tvalid_0's ndcg@2: 0.975744\tvalid_0's ndcg@3: 0.977156\tvalid_0's ndcg@4: 0.977426\tvalid_0's ndcg@5: 0.977464\n",
      "[67]\tvalid_0's ndcg@1: 0.940475\tvalid_0's ndcg@2: 0.97576\tvalid_0's ndcg@3: 0.977172\tvalid_0's ndcg@4: 0.977431\tvalid_0's ndcg@5: 0.977469\n",
      "[68]\tvalid_0's ndcg@1: 0.940675\tvalid_0's ndcg@2: 0.975849\tvalid_0's ndcg@3: 0.977249\tvalid_0's ndcg@4: 0.977508\tvalid_0's ndcg@5: 0.977546\n",
      "[69]\tvalid_0's ndcg@1: 0.9413\tvalid_0's ndcg@2: 0.976017\tvalid_0's ndcg@3: 0.977454\tvalid_0's ndcg@4: 0.977724\tvalid_0's ndcg@5: 0.977762\n",
      "[70]\tvalid_0's ndcg@1: 0.94105\tvalid_0's ndcg@2: 0.975925\tvalid_0's ndcg@3: 0.977362\tvalid_0's ndcg@4: 0.977631\tvalid_0's ndcg@5: 0.97767\n",
      "[71]\tvalid_0's ndcg@1: 0.94105\tvalid_0's ndcg@2: 0.975925\tvalid_0's ndcg@3: 0.97735\tvalid_0's ndcg@4: 0.97763\tvalid_0's ndcg@5: 0.977668\n",
      "[72]\tvalid_0's ndcg@1: 0.941325\tvalid_0's ndcg@2: 0.976058\tvalid_0's ndcg@3: 0.97747\tvalid_0's ndcg@4: 0.977739\tvalid_0's ndcg@5: 0.977778\n",
      "[73]\tvalid_0's ndcg@1: 0.941375\tvalid_0's ndcg@2: 0.976076\tvalid_0's ndcg@3: 0.977476\tvalid_0's ndcg@4: 0.977756\tvalid_0's ndcg@5: 0.977795\n",
      "[74]\tvalid_0's ndcg@1: 0.941725\tvalid_0's ndcg@2: 0.97619\tvalid_0's ndcg@3: 0.97759\tvalid_0's ndcg@4: 0.97788\tvalid_0's ndcg@5: 0.977919\n",
      "[75]\tvalid_0's ndcg@1: 0.941725\tvalid_0's ndcg@2: 0.97619\tvalid_0's ndcg@3: 0.977602\tvalid_0's ndcg@4: 0.977882\tvalid_0's ndcg@5: 0.977921\n",
      "[76]\tvalid_0's ndcg@1: 0.94195\tvalid_0's ndcg@2: 0.976273\tvalid_0's ndcg@3: 0.977685\tvalid_0's ndcg@4: 0.977965\tvalid_0's ndcg@5: 0.978004\n",
      "[77]\tvalid_0's ndcg@1: 0.9419\tvalid_0's ndcg@2: 0.97627\tvalid_0's ndcg@3: 0.97767\tvalid_0's ndcg@4: 0.97795\tvalid_0's ndcg@5: 0.977989\n",
      "[78]\tvalid_0's ndcg@1: 0.94235\tvalid_0's ndcg@2: 0.976452\tvalid_0's ndcg@3: 0.977839\tvalid_0's ndcg@4: 0.978119\tvalid_0's ndcg@5: 0.978158\n",
      "[79]\tvalid_0's ndcg@1: 0.94265\tvalid_0's ndcg@2: 0.976562\tvalid_0's ndcg@3: 0.977937\tvalid_0's ndcg@4: 0.978228\tvalid_0's ndcg@5: 0.978267\n",
      "[80]\tvalid_0's ndcg@1: 0.942975\tvalid_0's ndcg@2: 0.976667\tvalid_0's ndcg@3: 0.978067\tvalid_0's ndcg@4: 0.978347\tvalid_0's ndcg@5: 0.978385\n",
      "[81]\tvalid_0's ndcg@1: 0.94305\tvalid_0's ndcg@2: 0.97671\tvalid_0's ndcg@3: 0.978098\tvalid_0's ndcg@4: 0.978378\tvalid_0's ndcg@5: 0.978416\n",
      "[82]\tvalid_0's ndcg@1: 0.943175\tvalid_0's ndcg@2: 0.97674\tvalid_0's ndcg@3: 0.978115\tvalid_0's ndcg@4: 0.978417\tvalid_0's ndcg@5: 0.978456\n",
      "[83]\tvalid_0's ndcg@1: 0.94325\tvalid_0's ndcg@2: 0.976752\tvalid_0's ndcg@3: 0.97814\tvalid_0's ndcg@4: 0.978441\tvalid_0's ndcg@5: 0.97848\n",
      "[84]\tvalid_0's ndcg@1: 0.943375\tvalid_0's ndcg@2: 0.976767\tvalid_0's ndcg@3: 0.978179\tvalid_0's ndcg@4: 0.978481\tvalid_0's ndcg@5: 0.97852\n",
      "[85]\tvalid_0's ndcg@1: 0.94325\tvalid_0's ndcg@2: 0.976721\tvalid_0's ndcg@3: 0.978146\tvalid_0's ndcg@4: 0.978437\tvalid_0's ndcg@5: 0.978475\n",
      "[86]\tvalid_0's ndcg@1: 0.9434\tvalid_0's ndcg@2: 0.976792\tvalid_0's ndcg@3: 0.978204\tvalid_0's ndcg@4: 0.978506\tvalid_0's ndcg@5: 0.978535\n",
      "[87]\tvalid_0's ndcg@1: 0.943475\tvalid_0's ndcg@2: 0.976851\tvalid_0's ndcg@3: 0.978239\tvalid_0's ndcg@4: 0.97854\tvalid_0's ndcg@5: 0.978569\n",
      "[88]\tvalid_0's ndcg@1: 0.9436\tvalid_0's ndcg@2: 0.976882\tvalid_0's ndcg@3: 0.978282\tvalid_0's ndcg@4: 0.978572\tvalid_0's ndcg@5: 0.978611\n",
      "[89]\tvalid_0's ndcg@1: 0.943775\tvalid_0's ndcg@2: 0.976915\tvalid_0's ndcg@3: 0.97834\tvalid_0's ndcg@4: 0.97863\tvalid_0's ndcg@5: 0.978669\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[90]\tvalid_0's ndcg@1: 0.943925\tvalid_0's ndcg@2: 0.976986\tvalid_0's ndcg@3: 0.978398\tvalid_0's ndcg@4: 0.978689\tvalid_0's ndcg@5: 0.978728\n",
      "[91]\tvalid_0's ndcg@1: 0.943875\tvalid_0's ndcg@2: 0.976999\tvalid_0's ndcg@3: 0.978399\tvalid_0's ndcg@4: 0.978679\tvalid_0's ndcg@5: 0.978717\n",
      "[92]\tvalid_0's ndcg@1: 0.94395\tvalid_0's ndcg@2: 0.977058\tvalid_0's ndcg@3: 0.978421\tvalid_0's ndcg@4: 0.978711\tvalid_0's ndcg@5: 0.97876\n",
      "[93]\tvalid_0's ndcg@1: 0.944075\tvalid_0's ndcg@2: 0.977104\tvalid_0's ndcg@3: 0.978479\tvalid_0's ndcg@4: 0.978759\tvalid_0's ndcg@5: 0.978807\n",
      "[94]\tvalid_0's ndcg@1: 0.944175\tvalid_0's ndcg@2: 0.977125\tvalid_0's ndcg@3: 0.978513\tvalid_0's ndcg@4: 0.978793\tvalid_0's ndcg@5: 0.978841\n",
      "[95]\tvalid_0's ndcg@1: 0.94425\tvalid_0's ndcg@2: 0.977153\tvalid_0's ndcg@3: 0.97854\tvalid_0's ndcg@4: 0.97882\tvalid_0's ndcg@5: 0.978869\n",
      "[96]\tvalid_0's ndcg@1: 0.944225\tvalid_0's ndcg@2: 0.977144\tvalid_0's ndcg@3: 0.978531\tvalid_0's ndcg@4: 0.978811\tvalid_0's ndcg@5: 0.97886\n",
      "[97]\tvalid_0's ndcg@1: 0.94435\tvalid_0's ndcg@2: 0.977221\tvalid_0's ndcg@3: 0.978584\tvalid_0's ndcg@4: 0.978864\tvalid_0's ndcg@5: 0.978912\n",
      "[98]\tvalid_0's ndcg@1: 0.944575\tvalid_0's ndcg@2: 0.977289\tvalid_0's ndcg@3: 0.978651\tvalid_0's ndcg@4: 0.978942\tvalid_0's ndcg@5: 0.97899\n",
      "[99]\tvalid_0's ndcg@1: 0.944675\tvalid_0's ndcg@2: 0.977341\tvalid_0's ndcg@3: 0.978691\tvalid_0's ndcg@4: 0.978993\tvalid_0's ndcg@5: 0.979032\n",
      "[100]\tvalid_0's ndcg@1: 0.9451\tvalid_0's ndcg@2: 0.977482\tvalid_0's ndcg@3: 0.978857\tvalid_0's ndcg@4: 0.979148\tvalid_0's ndcg@5: 0.979187\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's ndcg@1: 0.9451\tvalid_0's ndcg@2: 0.977482\tvalid_0's ndcg@3: 0.978857\tvalid_0's ndcg@4: 0.979148\tvalid_0's ndcg@5: 0.979187\n",
      "[1]\tvalid_0's ndcg@1: 0.911575\tvalid_0's ndcg@2: 0.964384\tvalid_0's ndcg@3: 0.966321\tvalid_0's ndcg@4: 0.966623\tvalid_0's ndcg@5: 0.966671\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's ndcg@1: 0.9136\tvalid_0's ndcg@2: 0.965257\tvalid_0's ndcg@3: 0.967107\tvalid_0's ndcg@4: 0.967398\tvalid_0's ndcg@5: 0.967456\n",
      "[3]\tvalid_0's ndcg@1: 0.917425\tvalid_0's ndcg@2: 0.966732\tvalid_0's ndcg@3: 0.968545\tvalid_0's ndcg@4: 0.968814\tvalid_0's ndcg@5: 0.968882\n",
      "[4]\tvalid_0's ndcg@1: 0.9222\tvalid_0's ndcg@2: 0.968558\tvalid_0's ndcg@3: 0.970383\tvalid_0's ndcg@4: 0.970619\tvalid_0's ndcg@5: 0.970668\n",
      "[5]\tvalid_0's ndcg@1: 0.925875\tvalid_0's ndcg@2: 0.969914\tvalid_0's ndcg@3: 0.971714\tvalid_0's ndcg@4: 0.971972\tvalid_0's ndcg@5: 0.972021\n",
      "[6]\tvalid_0's ndcg@1: 0.926875\tvalid_0's ndcg@2: 0.970425\tvalid_0's ndcg@3: 0.972112\tvalid_0's ndcg@4: 0.972371\tvalid_0's ndcg@5: 0.972419\n",
      "[7]\tvalid_0's ndcg@1: 0.927475\tvalid_0's ndcg@2: 0.970631\tvalid_0's ndcg@3: 0.972306\tvalid_0's ndcg@4: 0.972586\tvalid_0's ndcg@5: 0.972634\n",
      "[8]\tvalid_0's ndcg@1: 0.93015\tvalid_0's ndcg@2: 0.971649\tvalid_0's ndcg@3: 0.973287\tvalid_0's ndcg@4: 0.973567\tvalid_0's ndcg@5: 0.973625\n",
      "[9]\tvalid_0's ndcg@1: 0.9312\tvalid_0's ndcg@2: 0.972084\tvalid_0's ndcg@3: 0.973684\tvalid_0's ndcg@4: 0.973964\tvalid_0's ndcg@5: 0.974022\n",
      "[10]\tvalid_0's ndcg@1: 0.93225\tvalid_0's ndcg@2: 0.972456\tvalid_0's ndcg@3: 0.974081\tvalid_0's ndcg@4: 0.974361\tvalid_0's ndcg@5: 0.974409\n",
      "[11]\tvalid_0's ndcg@1: 0.93305\tvalid_0's ndcg@2: 0.972704\tvalid_0's ndcg@3: 0.974379\tvalid_0's ndcg@4: 0.974648\tvalid_0's ndcg@5: 0.974696\n",
      "[12]\tvalid_0's ndcg@1: 0.9335\tvalid_0's ndcg@2: 0.972949\tvalid_0's ndcg@3: 0.974574\tvalid_0's ndcg@4: 0.974832\tvalid_0's ndcg@5: 0.974881\n",
      "[13]\tvalid_0's ndcg@1: 0.93415\tvalid_0's ndcg@2: 0.97322\tvalid_0's ndcg@3: 0.97482\tvalid_0's ndcg@4: 0.975079\tvalid_0's ndcg@5: 0.975127\n",
      "[14]\tvalid_0's ndcg@1: 0.9352\tvalid_0's ndcg@2: 0.973671\tvalid_0's ndcg@3: 0.975246\tvalid_0's ndcg@4: 0.975483\tvalid_0's ndcg@5: 0.975531\n",
      "[15]\tvalid_0's ndcg@1: 0.9358\tvalid_0's ndcg@2: 0.973877\tvalid_0's ndcg@3: 0.975452\tvalid_0's ndcg@4: 0.975699\tvalid_0's ndcg@5: 0.975748\n",
      "[16]\tvalid_0's ndcg@1: 0.935825\tvalid_0's ndcg@2: 0.973917\tvalid_0's ndcg@3: 0.975442\tvalid_0's ndcg@4: 0.975712\tvalid_0's ndcg@5: 0.97576\n",
      "[17]\tvalid_0's ndcg@1: 0.936475\tvalid_0's ndcg@2: 0.97411\tvalid_0's ndcg@3: 0.975697\tvalid_0's ndcg@4: 0.975956\tvalid_0's ndcg@5: 0.975995\n",
      "[18]\tvalid_0's ndcg@1: 0.936925\tvalid_0's ndcg@2: 0.974292\tvalid_0's ndcg@3: 0.975867\tvalid_0's ndcg@4: 0.976114\tvalid_0's ndcg@5: 0.976163\n",
      "[19]\tvalid_0's ndcg@1: 0.937525\tvalid_0's ndcg@2: 0.974545\tvalid_0's ndcg@3: 0.976095\tvalid_0's ndcg@4: 0.976342\tvalid_0's ndcg@5: 0.976391\n",
      "[20]\tvalid_0's ndcg@1: 0.937775\tvalid_0's ndcg@2: 0.974653\tvalid_0's ndcg@3: 0.976203\tvalid_0's ndcg@4: 0.976429\tvalid_0's ndcg@5: 0.976487\n",
      "[21]\tvalid_0's ndcg@1: 0.938825\tvalid_0's ndcg@2: 0.975072\tvalid_0's ndcg@3: 0.976597\tvalid_0's ndcg@4: 0.976823\tvalid_0's ndcg@5: 0.976881\n",
      "[22]\tvalid_0's ndcg@1: 0.93885\tvalid_0's ndcg@2: 0.975097\tvalid_0's ndcg@3: 0.976609\tvalid_0's ndcg@4: 0.976846\tvalid_0's ndcg@5: 0.976895\n",
      "[23]\tvalid_0's ndcg@1: 0.939125\tvalid_0's ndcg@2: 0.975246\tvalid_0's ndcg@3: 0.976733\tvalid_0's ndcg@4: 0.976959\tvalid_0's ndcg@5: 0.977008\n",
      "[24]\tvalid_0's ndcg@1: 0.939125\tvalid_0's ndcg@2: 0.975246\tvalid_0's ndcg@3: 0.976721\tvalid_0's ndcg@4: 0.976947\tvalid_0's ndcg@5: 0.977005\n",
      "[25]\tvalid_0's ndcg@1: 0.9396\tvalid_0's ndcg@2: 0.975421\tvalid_0's ndcg@3: 0.976909\tvalid_0's ndcg@4: 0.977124\tvalid_0's ndcg@5: 0.977182\n",
      "[26]\tvalid_0's ndcg@1: 0.9393\tvalid_0's ndcg@2: 0.975342\tvalid_0's ndcg@3: 0.976804\tvalid_0's ndcg@4: 0.97702\tvalid_0's ndcg@5: 0.977078\n",
      "[27]\tvalid_0's ndcg@1: 0.93925\tvalid_0's ndcg@2: 0.975323\tvalid_0's ndcg@3: 0.976798\tvalid_0's ndcg@4: 0.977014\tvalid_0's ndcg@5: 0.977062\n",
      "[28]\tvalid_0's ndcg@1: 0.93925\tvalid_0's ndcg@2: 0.975308\tvalid_0's ndcg@3: 0.976783\tvalid_0's ndcg@4: 0.977009\tvalid_0's ndcg@5: 0.977057\n",
      "[29]\tvalid_0's ndcg@1: 0.94\tvalid_0's ndcg@2: 0.975569\tvalid_0's ndcg@3: 0.977056\tvalid_0's ndcg@4: 0.977282\tvalid_0's ndcg@5: 0.977331\n",
      "[30]\tvalid_0's ndcg@1: 0.940325\tvalid_0's ndcg@2: 0.975673\tvalid_0's ndcg@3: 0.977173\tvalid_0's ndcg@4: 0.977399\tvalid_0's ndcg@5: 0.977447\n",
      "[31]\tvalid_0's ndcg@1: 0.940525\tvalid_0's ndcg@2: 0.975731\tvalid_0's ndcg@3: 0.977243\tvalid_0's ndcg@4: 0.977469\tvalid_0's ndcg@5: 0.977518\n",
      "[32]\tvalid_0's ndcg@1: 0.940625\tvalid_0's ndcg@2: 0.975831\tvalid_0's ndcg@3: 0.977306\tvalid_0's ndcg@4: 0.977521\tvalid_0's ndcg@5: 0.97757\n",
      "[33]\tvalid_0's ndcg@1: 0.94045\tvalid_0's ndcg@2: 0.975766\tvalid_0's ndcg@3: 0.977241\tvalid_0's ndcg@4: 0.977457\tvalid_0's ndcg@5: 0.977505\n",
      "[34]\tvalid_0's ndcg@1: 0.940625\tvalid_0's ndcg@2: 0.975831\tvalid_0's ndcg@3: 0.977306\tvalid_0's ndcg@4: 0.977521\tvalid_0's ndcg@5: 0.97757\n",
      "[35]\tvalid_0's ndcg@1: 0.940725\tvalid_0's ndcg@2: 0.975868\tvalid_0's ndcg@3: 0.977343\tvalid_0's ndcg@4: 0.977558\tvalid_0's ndcg@5: 0.977606\n",
      "[36]\tvalid_0's ndcg@1: 0.94115\tvalid_0's ndcg@2: 0.976056\tvalid_0's ndcg@3: 0.977506\tvalid_0's ndcg@4: 0.977722\tvalid_0's ndcg@5: 0.97777\n",
      "[37]\tvalid_0's ndcg@1: 0.9414\tvalid_0's ndcg@2: 0.976133\tvalid_0's ndcg@3: 0.977595\tvalid_0's ndcg@4: 0.977811\tvalid_0's ndcg@5: 0.977859\n",
      "[38]\tvalid_0's ndcg@1: 0.94175\tvalid_0's ndcg@2: 0.976278\tvalid_0's ndcg@3: 0.977715\tvalid_0's ndcg@4: 0.977941\tvalid_0's ndcg@5: 0.97799\n",
      "[39]\tvalid_0's ndcg@1: 0.942075\tvalid_0's ndcg@2: 0.976366\tvalid_0's ndcg@3: 0.977841\tvalid_0's ndcg@4: 0.978056\tvalid_0's ndcg@5: 0.978105\n",
      "[40]\tvalid_0's ndcg@1: 0.94215\tvalid_0's ndcg@2: 0.976409\tvalid_0's ndcg@3: 0.977872\tvalid_0's ndcg@4: 0.978087\tvalid_0's ndcg@5: 0.978136\n",
      "[41]\tvalid_0's ndcg@1: 0.94245\tvalid_0's ndcg@2: 0.97652\tvalid_0's ndcg@3: 0.977983\tvalid_0's ndcg@4: 0.978198\tvalid_0's ndcg@5: 0.978246\n",
      "[42]\tvalid_0's ndcg@1: 0.942975\tvalid_0's ndcg@2: 0.976682\tvalid_0's ndcg@3: 0.97817\tvalid_0's ndcg@4: 0.978385\tvalid_0's ndcg@5: 0.978434\n",
      "[43]\tvalid_0's ndcg@1: 0.942975\tvalid_0's ndcg@2: 0.976682\tvalid_0's ndcg@3: 0.97817\tvalid_0's ndcg@4: 0.978385\tvalid_0's ndcg@5: 0.978434\n",
      "[44]\tvalid_0's ndcg@1: 0.94285\tvalid_0's ndcg@2: 0.976636\tvalid_0's ndcg@3: 0.978111\tvalid_0's ndcg@4: 0.978337\tvalid_0's ndcg@5: 0.978386\n",
      "[45]\tvalid_0's ndcg@1: 0.94325\tvalid_0's ndcg@2: 0.9768\tvalid_0's ndcg@3: 0.978262\tvalid_0's ndcg@4: 0.978488\tvalid_0's ndcg@5: 0.978537\n",
      "[46]\tvalid_0's ndcg@1: 0.9436\tvalid_0's ndcg@2: 0.976913\tvalid_0's ndcg@3: 0.978388\tvalid_0's ndcg@4: 0.978614\tvalid_0's ndcg@5: 0.978663\n",
      "[47]\tvalid_0's ndcg@1: 0.943525\tvalid_0's ndcg@2: 0.976885\tvalid_0's ndcg@3: 0.97836\tvalid_0's ndcg@4: 0.978576\tvalid_0's ndcg@5: 0.978634\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[48]\tvalid_0's ndcg@1: 0.943525\tvalid_0's ndcg@2: 0.976885\tvalid_0's ndcg@3: 0.978373\tvalid_0's ndcg@4: 0.978577\tvalid_0's ndcg@5: 0.978636\n",
      "[49]\tvalid_0's ndcg@1: 0.9436\tvalid_0's ndcg@2: 0.976913\tvalid_0's ndcg@3: 0.978388\tvalid_0's ndcg@4: 0.978614\tvalid_0's ndcg@5: 0.978663\n",
      "[50]\tvalid_0's ndcg@1: 0.943975\tvalid_0's ndcg@2: 0.97702\tvalid_0's ndcg@3: 0.97852\tvalid_0's ndcg@4: 0.978746\tvalid_0's ndcg@5: 0.978794\n",
      "[51]\tvalid_0's ndcg@1: 0.9441\tvalid_0's ndcg@2: 0.97705\tvalid_0's ndcg@3: 0.97855\tvalid_0's ndcg@4: 0.978787\tvalid_0's ndcg@5: 0.978836\n",
      "[52]\tvalid_0's ndcg@1: 0.94425\tvalid_0's ndcg@2: 0.977121\tvalid_0's ndcg@3: 0.978609\tvalid_0's ndcg@4: 0.978846\tvalid_0's ndcg@5: 0.978894\n",
      "[53]\tvalid_0's ndcg@1: 0.944225\tvalid_0's ndcg@2: 0.977081\tvalid_0's ndcg@3: 0.978618\tvalid_0's ndcg@4: 0.978834\tvalid_0's ndcg@5: 0.978882\n",
      "[54]\tvalid_0's ndcg@1: 0.9442\tvalid_0's ndcg@2: 0.977071\tvalid_0's ndcg@3: 0.978609\tvalid_0's ndcg@4: 0.978824\tvalid_0's ndcg@5: 0.978873\n",
      "[55]\tvalid_0's ndcg@1: 0.94435\tvalid_0's ndcg@2: 0.977143\tvalid_0's ndcg@3: 0.978668\tvalid_0's ndcg@4: 0.978883\tvalid_0's ndcg@5: 0.978931\n",
      "[56]\tvalid_0's ndcg@1: 0.9444\tvalid_0's ndcg@2: 0.977177\tvalid_0's ndcg@3: 0.978702\tvalid_0's ndcg@4: 0.978906\tvalid_0's ndcg@5: 0.978955\n",
      "[57]\tvalid_0's ndcg@1: 0.944675\tvalid_0's ndcg@2: 0.977263\tvalid_0's ndcg@3: 0.978788\tvalid_0's ndcg@4: 0.979003\tvalid_0's ndcg@5: 0.979051\n",
      "[58]\tvalid_0's ndcg@1: 0.9448\tvalid_0's ndcg@2: 0.977293\tvalid_0's ndcg@3: 0.978843\tvalid_0's ndcg@4: 0.979047\tvalid_0's ndcg@5: 0.979096\n",
      "[59]\tvalid_0's ndcg@1: 0.9452\tvalid_0's ndcg@2: 0.977472\tvalid_0's ndcg@3: 0.978997\tvalid_0's ndcg@4: 0.979202\tvalid_0's ndcg@5: 0.97925\n",
      "[60]\tvalid_0's ndcg@1: 0.9455\tvalid_0's ndcg@2: 0.97763\tvalid_0's ndcg@3: 0.979118\tvalid_0's ndcg@4: 0.979322\tvalid_0's ndcg@5: 0.979371\n",
      "[61]\tvalid_0's ndcg@1: 0.945725\tvalid_0's ndcg@2: 0.977682\tvalid_0's ndcg@3: 0.979194\tvalid_0's ndcg@4: 0.979399\tvalid_0's ndcg@5: 0.979447\n",
      "[62]\tvalid_0's ndcg@1: 0.94595\tvalid_0's ndcg@2: 0.977812\tvalid_0's ndcg@3: 0.979312\tvalid_0's ndcg@4: 0.979495\tvalid_0's ndcg@5: 0.979543\n",
      "[63]\tvalid_0's ndcg@1: 0.946\tvalid_0's ndcg@2: 0.977878\tvalid_0's ndcg@3: 0.97934\tvalid_0's ndcg@4: 0.979523\tvalid_0's ndcg@5: 0.979572\n",
      "[64]\tvalid_0's ndcg@1: 0.946525\tvalid_0's ndcg@2: 0.978056\tvalid_0's ndcg@3: 0.979531\tvalid_0's ndcg@4: 0.979714\tvalid_0's ndcg@5: 0.979762\n",
      "[65]\tvalid_0's ndcg@1: 0.9467\tvalid_0's ndcg@2: 0.978105\tvalid_0's ndcg@3: 0.979592\tvalid_0's ndcg@4: 0.979775\tvalid_0's ndcg@5: 0.979823\n",
      "[66]\tvalid_0's ndcg@1: 0.9465\tvalid_0's ndcg@2: 0.978046\tvalid_0's ndcg@3: 0.979534\tvalid_0's ndcg@4: 0.979706\tvalid_0's ndcg@5: 0.979755\n",
      "[67]\tvalid_0's ndcg@1: 0.946675\tvalid_0's ndcg@2: 0.978127\tvalid_0's ndcg@3: 0.979614\tvalid_0's ndcg@4: 0.979776\tvalid_0's ndcg@5: 0.979824\n",
      "[68]\tvalid_0's ndcg@1: 0.9467\tvalid_0's ndcg@2: 0.97812\tvalid_0's ndcg@3: 0.979608\tvalid_0's ndcg@4: 0.97978\tvalid_0's ndcg@5: 0.979828\n",
      "[69]\tvalid_0's ndcg@1: 0.946875\tvalid_0's ndcg@2: 0.978216\tvalid_0's ndcg@3: 0.979679\tvalid_0's ndcg@4: 0.979851\tvalid_0's ndcg@5: 0.9799\n",
      "[70]\tvalid_0's ndcg@1: 0.9469\tvalid_0's ndcg@2: 0.978194\tvalid_0's ndcg@3: 0.979682\tvalid_0's ndcg@4: 0.979854\tvalid_0's ndcg@5: 0.979902\n",
      "[71]\tvalid_0's ndcg@1: 0.947025\tvalid_0's ndcg@2: 0.978209\tvalid_0's ndcg@3: 0.979721\tvalid_0's ndcg@4: 0.979893\tvalid_0's ndcg@5: 0.979942\n",
      "[72]\tvalid_0's ndcg@1: 0.9472\tvalid_0's ndcg@2: 0.978273\tvalid_0's ndcg@3: 0.979773\tvalid_0's ndcg@4: 0.979956\tvalid_0's ndcg@5: 0.980005\n",
      "[73]\tvalid_0's ndcg@1: 0.947475\tvalid_0's ndcg@2: 0.978391\tvalid_0's ndcg@3: 0.979878\tvalid_0's ndcg@4: 0.980061\tvalid_0's ndcg@5: 0.980109\n",
      "[74]\tvalid_0's ndcg@1: 0.94715\tvalid_0's ndcg@2: 0.978271\tvalid_0's ndcg@3: 0.979758\tvalid_0's ndcg@4: 0.979941\tvalid_0's ndcg@5: 0.97999\n",
      "[75]\tvalid_0's ndcg@1: 0.947275\tvalid_0's ndcg@2: 0.978333\tvalid_0's ndcg@3: 0.979808\tvalid_0's ndcg@4: 0.979991\tvalid_0's ndcg@5: 0.980039\n",
      "[76]\tvalid_0's ndcg@1: 0.9474\tvalid_0's ndcg@2: 0.97841\tvalid_0's ndcg@3: 0.979873\tvalid_0's ndcg@4: 0.980045\tvalid_0's ndcg@5: 0.980093\n",
      "[77]\tvalid_0's ndcg@1: 0.94745\tvalid_0's ndcg@2: 0.97846\tvalid_0's ndcg@3: 0.979898\tvalid_0's ndcg@4: 0.98007\tvalid_0's ndcg@5: 0.980118\n",
      "[78]\tvalid_0's ndcg@1: 0.94775\tvalid_0's ndcg@2: 0.978555\tvalid_0's ndcg@3: 0.980005\tvalid_0's ndcg@4: 0.980177\tvalid_0's ndcg@5: 0.980226\n",
      "[79]\tvalid_0's ndcg@1: 0.947875\tvalid_0's ndcg@2: 0.978617\tvalid_0's ndcg@3: 0.980055\tvalid_0's ndcg@4: 0.980238\tvalid_0's ndcg@5: 0.980276\n",
      "[80]\tvalid_0's ndcg@1: 0.947875\tvalid_0's ndcg@2: 0.978617\tvalid_0's ndcg@3: 0.980055\tvalid_0's ndcg@4: 0.980238\tvalid_0's ndcg@5: 0.980276\n",
      "[81]\tvalid_0's ndcg@1: 0.948175\tvalid_0's ndcg@2: 0.978744\tvalid_0's ndcg@3: 0.980169\tvalid_0's ndcg@4: 0.980352\tvalid_0's ndcg@5: 0.98039\n",
      "[82]\tvalid_0's ndcg@1: 0.948375\tvalid_0's ndcg@2: 0.97888\tvalid_0's ndcg@3: 0.980255\tvalid_0's ndcg@4: 0.980438\tvalid_0's ndcg@5: 0.980477\n",
      "[83]\tvalid_0's ndcg@1: 0.94825\tvalid_0's ndcg@2: 0.978834\tvalid_0's ndcg@3: 0.980209\tvalid_0's ndcg@4: 0.980392\tvalid_0's ndcg@5: 0.980431\n",
      "[84]\tvalid_0's ndcg@1: 0.948275\tvalid_0's ndcg@2: 0.978844\tvalid_0's ndcg@3: 0.980219\tvalid_0's ndcg@4: 0.980402\tvalid_0's ndcg@5: 0.98044\n",
      "[85]\tvalid_0's ndcg@1: 0.948475\tvalid_0's ndcg@2: 0.978917\tvalid_0's ndcg@3: 0.980292\tvalid_0's ndcg@4: 0.980475\tvalid_0's ndcg@5: 0.980514\n",
      "[86]\tvalid_0's ndcg@1: 0.948975\tvalid_0's ndcg@2: 0.979102\tvalid_0's ndcg@3: 0.980477\tvalid_0's ndcg@4: 0.98066\tvalid_0's ndcg@5: 0.980699\n",
      "[87]\tvalid_0's ndcg@1: 0.948975\tvalid_0's ndcg@2: 0.979086\tvalid_0's ndcg@3: 0.980474\tvalid_0's ndcg@4: 0.980657\tvalid_0's ndcg@5: 0.980695\n",
      "[88]\tvalid_0's ndcg@1: 0.949025\tvalid_0's ndcg@2: 0.979136\tvalid_0's ndcg@3: 0.980499\tvalid_0's ndcg@4: 0.980682\tvalid_0's ndcg@5: 0.98072\n",
      "[89]\tvalid_0's ndcg@1: 0.9493\tvalid_0's ndcg@2: 0.979285\tvalid_0's ndcg@3: 0.98061\tvalid_0's ndcg@4: 0.980793\tvalid_0's ndcg@5: 0.980832\n",
      "[90]\tvalid_0's ndcg@1: 0.9493\tvalid_0's ndcg@2: 0.979269\tvalid_0's ndcg@3: 0.980607\tvalid_0's ndcg@4: 0.98079\tvalid_0's ndcg@5: 0.980828\n",
      "[91]\tvalid_0's ndcg@1: 0.9493\tvalid_0's ndcg@2: 0.979269\tvalid_0's ndcg@3: 0.980607\tvalid_0's ndcg@4: 0.98079\tvalid_0's ndcg@5: 0.980828\n",
      "[92]\tvalid_0's ndcg@1: 0.9494\tvalid_0's ndcg@2: 0.97929\tvalid_0's ndcg@3: 0.98064\tvalid_0's ndcg@4: 0.980823\tvalid_0's ndcg@5: 0.980862\n",
      "[93]\tvalid_0's ndcg@1: 0.949375\tvalid_0's ndcg@2: 0.979297\tvalid_0's ndcg@3: 0.980634\tvalid_0's ndcg@4: 0.980817\tvalid_0's ndcg@5: 0.980856\n",
      "[94]\tvalid_0's ndcg@1: 0.949525\tvalid_0's ndcg@2: 0.979336\tvalid_0's ndcg@3: 0.980686\tvalid_0's ndcg@4: 0.980869\tvalid_0's ndcg@5: 0.980908\n",
      "[95]\tvalid_0's ndcg@1: 0.949825\tvalid_0's ndcg@2: 0.979416\tvalid_0's ndcg@3: 0.980791\tvalid_0's ndcg@4: 0.980974\tvalid_0's ndcg@5: 0.981012\n",
      "[96]\tvalid_0's ndcg@1: 0.94975\tvalid_0's ndcg@2: 0.979404\tvalid_0's ndcg@3: 0.980779\tvalid_0's ndcg@4: 0.980951\tvalid_0's ndcg@5: 0.98099\n",
      "[97]\tvalid_0's ndcg@1: 0.950025\tvalid_0's ndcg@2: 0.979537\tvalid_0's ndcg@3: 0.980874\tvalid_0's ndcg@4: 0.981057\tvalid_0's ndcg@5: 0.981096\n",
      "[98]\tvalid_0's ndcg@1: 0.9501\tvalid_0's ndcg@2: 0.979564\tvalid_0's ndcg@3: 0.980889\tvalid_0's ndcg@4: 0.981083\tvalid_0's ndcg@5: 0.981122\n",
      "[99]\tvalid_0's ndcg@1: 0.950275\tvalid_0's ndcg@2: 0.979629\tvalid_0's ndcg@3: 0.980967\tvalid_0's ndcg@4: 0.98115\tvalid_0's ndcg@5: 0.981188\n",
      "[100]\tvalid_0's ndcg@1: 0.950325\tvalid_0's ndcg@2: 0.979647\tvalid_0's ndcg@3: 0.980985\tvalid_0's ndcg@4: 0.981168\tvalid_0's ndcg@5: 0.981207\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's ndcg@1: 0.950325\tvalid_0's ndcg@2: 0.979647\tvalid_0's ndcg@3: 0.980985\tvalid_0's ndcg@4: 0.981168\tvalid_0's ndcg@5: 0.981207\n",
      "[1]\tvalid_0's ndcg@1: 0.910175\tvalid_0's ndcg@2: 0.96382\tvalid_0's ndcg@3: 0.965707\tvalid_0's ndcg@4: 0.966009\tvalid_0's ndcg@5: 0.966086\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's ndcg@1: 0.91415\tvalid_0's ndcg@2: 0.965492\tvalid_0's ndcg@3: 0.967254\tvalid_0's ndcg@4: 0.967556\tvalid_0's ndcg@5: 0.967604\n",
      "[3]\tvalid_0's ndcg@1: 0.916025\tvalid_0's ndcg@2: 0.966389\tvalid_0's ndcg@3: 0.967976\tvalid_0's ndcg@4: 0.968278\tvalid_0's ndcg@5: 0.968355\n",
      "[4]\tvalid_0's ndcg@1: 0.919\tvalid_0's ndcg@2: 0.967392\tvalid_0's ndcg@3: 0.96903\tvalid_0's ndcg@4: 0.969364\tvalid_0's ndcg@5: 0.969431\n",
      "[5]\tvalid_0's ndcg@1: 0.921125\tvalid_0's ndcg@2: 0.968192\tvalid_0's ndcg@3: 0.969855\tvalid_0's ndcg@4: 0.970156\tvalid_0's ndcg@5: 0.970224\n",
      "[6]\tvalid_0's ndcg@1: 0.921675\tvalid_0's ndcg@2: 0.968411\tvalid_0's ndcg@3: 0.970111\tvalid_0's ndcg@4: 0.97037\tvalid_0's ndcg@5: 0.970437\n",
      "[7]\tvalid_0's ndcg@1: 0.9237\tvalid_0's ndcg@2: 0.969332\tvalid_0's ndcg@3: 0.970882\tvalid_0's ndcg@4: 0.97113\tvalid_0's ndcg@5: 0.971217\n",
      "[8]\tvalid_0's ndcg@1: 0.925775\tvalid_0's ndcg@2: 0.970129\tvalid_0's ndcg@3: 0.971642\tvalid_0's ndcg@4: 0.971922\tvalid_0's ndcg@5: 0.97199\n",
      "[9]\tvalid_0's ndcg@1: 0.926775\tvalid_0's ndcg@2: 0.970435\tvalid_0's ndcg@3: 0.971985\tvalid_0's ndcg@4: 0.972276\tvalid_0's ndcg@5: 0.972334\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10]\tvalid_0's ndcg@1: 0.9277\tvalid_0's ndcg@2: 0.970761\tvalid_0's ndcg@3: 0.972311\tvalid_0's ndcg@4: 0.972612\tvalid_0's ndcg@5: 0.97267\n",
      "[11]\tvalid_0's ndcg@1: 0.928975\tvalid_0's ndcg@2: 0.97131\tvalid_0's ndcg@3: 0.972798\tvalid_0's ndcg@4: 0.973089\tvalid_0's ndcg@5: 0.973166\n",
      "[12]\tvalid_0's ndcg@1: 0.929375\tvalid_0's ndcg@2: 0.971505\tvalid_0's ndcg@3: 0.972968\tvalid_0's ndcg@4: 0.973259\tvalid_0's ndcg@5: 0.973326\n",
      "[13]\tvalid_0's ndcg@1: 0.929375\tvalid_0's ndcg@2: 0.971426\tvalid_0's ndcg@3: 0.972939\tvalid_0's ndcg@4: 0.97324\tvalid_0's ndcg@5: 0.973318\n",
      "[14]\tvalid_0's ndcg@1: 0.929775\tvalid_0's ndcg@2: 0.971621\tvalid_0's ndcg@3: 0.973121\tvalid_0's ndcg@4: 0.973412\tvalid_0's ndcg@5: 0.97348\n",
      "[15]\tvalid_0's ndcg@1: 0.9304\tvalid_0's ndcg@2: 0.971868\tvalid_0's ndcg@3: 0.97338\tvalid_0's ndcg@4: 0.97365\tvalid_0's ndcg@5: 0.973717\n",
      "[16]\tvalid_0's ndcg@1: 0.930975\tvalid_0's ndcg@2: 0.972096\tvalid_0's ndcg@3: 0.973558\tvalid_0's ndcg@4: 0.973849\tvalid_0's ndcg@5: 0.973926\n",
      "[17]\tvalid_0's ndcg@1: 0.93105\tvalid_0's ndcg@2: 0.972108\tvalid_0's ndcg@3: 0.973583\tvalid_0's ndcg@4: 0.973884\tvalid_0's ndcg@5: 0.973952\n",
      "[18]\tvalid_0's ndcg@1: 0.931725\tvalid_0's ndcg@2: 0.972373\tvalid_0's ndcg@3: 0.97386\tvalid_0's ndcg@4: 0.974129\tvalid_0's ndcg@5: 0.974207\n",
      "[19]\tvalid_0's ndcg@1: 0.932175\tvalid_0's ndcg@2: 0.972681\tvalid_0's ndcg@3: 0.974068\tvalid_0's ndcg@4: 0.974348\tvalid_0's ndcg@5: 0.974406\n",
      "[20]\tvalid_0's ndcg@1: 0.93305\tvalid_0's ndcg@2: 0.973019\tvalid_0's ndcg@3: 0.974382\tvalid_0's ndcg@4: 0.974673\tvalid_0's ndcg@5: 0.974731\n",
      "[21]\tvalid_0's ndcg@1: 0.933075\tvalid_0's ndcg@2: 0.97306\tvalid_0's ndcg@3: 0.974423\tvalid_0's ndcg@4: 0.974703\tvalid_0's ndcg@5: 0.97477\n",
      "[22]\tvalid_0's ndcg@1: 0.93375\tvalid_0's ndcg@2: 0.973262\tvalid_0's ndcg@3: 0.974649\tvalid_0's ndcg@4: 0.974929\tvalid_0's ndcg@5: 0.975007\n",
      "[23]\tvalid_0's ndcg@1: 0.933675\tvalid_0's ndcg@2: 0.973219\tvalid_0's ndcg@3: 0.974606\tvalid_0's ndcg@4: 0.974886\tvalid_0's ndcg@5: 0.974973\n",
      "[24]\tvalid_0's ndcg@1: 0.934\tvalid_0's ndcg@2: 0.97337\tvalid_0's ndcg@3: 0.974745\tvalid_0's ndcg@4: 0.975014\tvalid_0's ndcg@5: 0.975101\n",
      "[25]\tvalid_0's ndcg@1: 0.934825\tvalid_0's ndcg@2: 0.973674\tvalid_0's ndcg@3: 0.975062\tvalid_0's ndcg@4: 0.975342\tvalid_0's ndcg@5: 0.97541\n",
      "[26]\tvalid_0's ndcg@1: 0.93495\tvalid_0's ndcg@2: 0.973721\tvalid_0's ndcg@3: 0.975096\tvalid_0's ndcg@4: 0.975365\tvalid_0's ndcg@5: 0.975452\n",
      "[27]\tvalid_0's ndcg@1: 0.9358\tvalid_0's ndcg@2: 0.974082\tvalid_0's ndcg@3: 0.975444\tvalid_0's ndcg@4: 0.975713\tvalid_0's ndcg@5: 0.975781\n",
      "[28]\tvalid_0's ndcg@1: 0.935325\tvalid_0's ndcg@2: 0.973875\tvalid_0's ndcg@3: 0.975275\tvalid_0's ndcg@4: 0.975512\tvalid_0's ndcg@5: 0.975599\n",
      "[29]\tvalid_0's ndcg@1: 0.935925\tvalid_0's ndcg@2: 0.974159\tvalid_0's ndcg@3: 0.975522\tvalid_0's ndcg@4: 0.975759\tvalid_0's ndcg@5: 0.975836\n",
      "[30]\tvalid_0's ndcg@1: 0.9362\tvalid_0's ndcg@2: 0.974214\tvalid_0's ndcg@3: 0.975589\tvalid_0's ndcg@4: 0.975847\tvalid_0's ndcg@5: 0.975924\n",
      "[31]\tvalid_0's ndcg@1: 0.93625\tvalid_0's ndcg@2: 0.974216\tvalid_0's ndcg@3: 0.975629\tvalid_0's ndcg@4: 0.975876\tvalid_0's ndcg@5: 0.975944\n",
      "[32]\tvalid_0's ndcg@1: 0.93665\tvalid_0's ndcg@2: 0.974427\tvalid_0's ndcg@3: 0.975814\tvalid_0's ndcg@4: 0.97603\tvalid_0's ndcg@5: 0.976107\n",
      "[33]\tvalid_0's ndcg@1: 0.936775\tvalid_0's ndcg@2: 0.974505\tvalid_0's ndcg@3: 0.975855\tvalid_0's ndcg@4: 0.976081\tvalid_0's ndcg@5: 0.976158\n",
      "[34]\tvalid_0's ndcg@1: 0.93715\tvalid_0's ndcg@2: 0.974643\tvalid_0's ndcg@3: 0.975993\tvalid_0's ndcg@4: 0.976219\tvalid_0's ndcg@5: 0.976296\n",
      "[35]\tvalid_0's ndcg@1: 0.937675\tvalid_0's ndcg@2: 0.974805\tvalid_0's ndcg@3: 0.97618\tvalid_0's ndcg@4: 0.976406\tvalid_0's ndcg@5: 0.976484\n",
      "[36]\tvalid_0's ndcg@1: 0.9382\tvalid_0's ndcg@2: 0.974983\tvalid_0's ndcg@3: 0.976371\tvalid_0's ndcg@4: 0.976597\tvalid_0's ndcg@5: 0.976674\n",
      "[37]\tvalid_0's ndcg@1: 0.938175\tvalid_0's ndcg@2: 0.974974\tvalid_0's ndcg@3: 0.976349\tvalid_0's ndcg@4: 0.976586\tvalid_0's ndcg@5: 0.976663\n",
      "[38]\tvalid_0's ndcg@1: 0.938675\tvalid_0's ndcg@2: 0.975143\tvalid_0's ndcg@3: 0.976518\tvalid_0's ndcg@4: 0.976776\tvalid_0's ndcg@5: 0.976844\n",
      "[39]\tvalid_0's ndcg@1: 0.938575\tvalid_0's ndcg@2: 0.975106\tvalid_0's ndcg@3: 0.976481\tvalid_0's ndcg@4: 0.976739\tvalid_0's ndcg@5: 0.976807\n",
      "[40]\tvalid_0's ndcg@1: 0.938675\tvalid_0's ndcg@2: 0.97519\tvalid_0's ndcg@3: 0.976528\tvalid_0's ndcg@4: 0.976775\tvalid_0's ndcg@5: 0.976853\n",
      "[41]\tvalid_0's ndcg@1: 0.9391\tvalid_0's ndcg@2: 0.975347\tvalid_0's ndcg@3: 0.976697\tvalid_0's ndcg@4: 0.976934\tvalid_0's ndcg@5: 0.977001\n",
      "[42]\tvalid_0's ndcg@1: 0.939825\tvalid_0's ndcg@2: 0.975599\tvalid_0's ndcg@3: 0.976961\tvalid_0's ndcg@4: 0.977198\tvalid_0's ndcg@5: 0.977266\n",
      "[43]\tvalid_0's ndcg@1: 0.93985\tvalid_0's ndcg@2: 0.975639\tvalid_0's ndcg@3: 0.976977\tvalid_0's ndcg@4: 0.977214\tvalid_0's ndcg@5: 0.977282\n",
      "[44]\tvalid_0's ndcg@1: 0.9398\tvalid_0's ndcg@2: 0.975605\tvalid_0's ndcg@3: 0.976955\tvalid_0's ndcg@4: 0.977192\tvalid_0's ndcg@5: 0.97726\n",
      "[45]\tvalid_0's ndcg@1: 0.9401\tvalid_0's ndcg@2: 0.9757\tvalid_0's ndcg@3: 0.977075\tvalid_0's ndcg@4: 0.977291\tvalid_0's ndcg@5: 0.977368\n",
      "[46]\tvalid_0's ndcg@1: 0.94045\tvalid_0's ndcg@2: 0.975845\tvalid_0's ndcg@3: 0.977183\tvalid_0's ndcg@4: 0.97742\tvalid_0's ndcg@5: 0.977497\n",
      "[47]\tvalid_0's ndcg@1: 0.940475\tvalid_0's ndcg@2: 0.975854\tvalid_0's ndcg@3: 0.977204\tvalid_0's ndcg@4: 0.97743\tvalid_0's ndcg@5: 0.977508\n",
      "[48]\tvalid_0's ndcg@1: 0.940575\tvalid_0's ndcg@2: 0.975923\tvalid_0's ndcg@3: 0.977273\tvalid_0's ndcg@4: 0.977488\tvalid_0's ndcg@5: 0.977556\n",
      "[49]\tvalid_0's ndcg@1: 0.9407\tvalid_0's ndcg@2: 0.975922\tvalid_0's ndcg@3: 0.977297\tvalid_0's ndcg@4: 0.977501\tvalid_0's ndcg@5: 0.977588\n",
      "[50]\tvalid_0's ndcg@1: 0.940725\tvalid_0's ndcg@2: 0.975947\tvalid_0's ndcg@3: 0.977322\tvalid_0's ndcg@4: 0.977505\tvalid_0's ndcg@5: 0.977592\n",
      "[51]\tvalid_0's ndcg@1: 0.9406\tvalid_0's ndcg@2: 0.975837\tvalid_0's ndcg@3: 0.97725\tvalid_0's ndcg@4: 0.977422\tvalid_0's ndcg@5: 0.977509\n",
      "[52]\tvalid_0's ndcg@1: 0.941075\tvalid_0's ndcg@2: 0.975997\tvalid_0's ndcg@3: 0.977422\tvalid_0's ndcg@4: 0.977594\tvalid_0's ndcg@5: 0.977691\n",
      "[53]\tvalid_0's ndcg@1: 0.940925\tvalid_0's ndcg@2: 0.975989\tvalid_0's ndcg@3: 0.977376\tvalid_0's ndcg@4: 0.977538\tvalid_0's ndcg@5: 0.977644\n",
      "[54]\tvalid_0's ndcg@1: 0.94125\tvalid_0's ndcg@2: 0.976062\tvalid_0's ndcg@3: 0.977487\tvalid_0's ndcg@4: 0.977659\tvalid_0's ndcg@5: 0.977756\n",
      "[55]\tvalid_0's ndcg@1: 0.94145\tvalid_0's ndcg@2: 0.976183\tvalid_0's ndcg@3: 0.97757\tvalid_0's ndcg@4: 0.977742\tvalid_0's ndcg@5: 0.977839\n",
      "[56]\tvalid_0's ndcg@1: 0.941475\tvalid_0's ndcg@2: 0.976176\tvalid_0's ndcg@3: 0.977576\tvalid_0's ndcg@4: 0.977748\tvalid_0's ndcg@5: 0.977845\n",
      "[57]\tvalid_0's ndcg@1: 0.941375\tvalid_0's ndcg@2: 0.976139\tvalid_0's ndcg@3: 0.977539\tvalid_0's ndcg@4: 0.977712\tvalid_0's ndcg@5: 0.977808\n",
      "[58]\tvalid_0's ndcg@1: 0.941675\tvalid_0's ndcg@2: 0.97625\tvalid_0's ndcg@3: 0.97765\tvalid_0's ndcg@4: 0.977822\tvalid_0's ndcg@5: 0.977919\n",
      "[59]\tvalid_0's ndcg@1: 0.941725\tvalid_0's ndcg@2: 0.976253\tvalid_0's ndcg@3: 0.977653\tvalid_0's ndcg@4: 0.977836\tvalid_0's ndcg@5: 0.977932\n",
      "[60]\tvalid_0's ndcg@1: 0.941675\tvalid_0's ndcg@2: 0.976234\tvalid_0's ndcg@3: 0.977634\tvalid_0's ndcg@4: 0.977817\tvalid_0's ndcg@5: 0.977914\n",
      "[61]\tvalid_0's ndcg@1: 0.9419\tvalid_0's ndcg@2: 0.976333\tvalid_0's ndcg@3: 0.977745\tvalid_0's ndcg@4: 0.977918\tvalid_0's ndcg@5: 0.978005\n",
      "[62]\tvalid_0's ndcg@1: 0.941975\tvalid_0's ndcg@2: 0.976345\tvalid_0's ndcg@3: 0.977757\tvalid_0's ndcg@4: 0.97794\tvalid_0's ndcg@5: 0.978027\n",
      "[63]\tvalid_0's ndcg@1: 0.9423\tvalid_0's ndcg@2: 0.976496\tvalid_0's ndcg@3: 0.977871\tvalid_0's ndcg@4: 0.978065\tvalid_0's ndcg@5: 0.978152\n",
      "[64]\tvalid_0's ndcg@1: 0.942625\tvalid_0's ndcg@2: 0.976632\tvalid_0's ndcg@3: 0.977995\tvalid_0's ndcg@4: 0.978188\tvalid_0's ndcg@5: 0.978275\n",
      "[65]\tvalid_0's ndcg@1: 0.942575\tvalid_0's ndcg@2: 0.976629\tvalid_0's ndcg@3: 0.977979\tvalid_0's ndcg@4: 0.978173\tvalid_0's ndcg@5: 0.97826\n",
      "[66]\tvalid_0's ndcg@1: 0.942725\tvalid_0's ndcg@2: 0.976685\tvalid_0's ndcg@3: 0.978035\tvalid_0's ndcg@4: 0.978229\tvalid_0's ndcg@5: 0.978316\n",
      "[67]\tvalid_0's ndcg@1: 0.94275\tvalid_0's ndcg@2: 0.976678\tvalid_0's ndcg@3: 0.978041\tvalid_0's ndcg@4: 0.978224\tvalid_0's ndcg@5: 0.97832\n",
      "[68]\tvalid_0's ndcg@1: 0.94275\tvalid_0's ndcg@2: 0.976694\tvalid_0's ndcg@3: 0.978044\tvalid_0's ndcg@4: 0.978227\tvalid_0's ndcg@5: 0.978324\n",
      "[69]\tvalid_0's ndcg@1: 0.943\tvalid_0's ndcg@2: 0.976834\tvalid_0's ndcg@3: 0.978146\tvalid_0's ndcg@4: 0.978329\tvalid_0's ndcg@5: 0.978426\n",
      "[70]\tvalid_0's ndcg@1: 0.943025\tvalid_0's ndcg@2: 0.976827\tvalid_0's ndcg@3: 0.978152\tvalid_0's ndcg@4: 0.978324\tvalid_0's ndcg@5: 0.978431\n",
      "[71]\tvalid_0's ndcg@1: 0.9432\tvalid_0's ndcg@2: 0.976923\tvalid_0's ndcg@3: 0.978236\tvalid_0's ndcg@4: 0.978397\tvalid_0's ndcg@5: 0.978504\n",
      "[72]\tvalid_0's ndcg@1: 0.943225\tvalid_0's ndcg@2: 0.976917\tvalid_0's ndcg@3: 0.978254\tvalid_0's ndcg@4: 0.978405\tvalid_0's ndcg@5: 0.978511\n",
      "[73]\tvalid_0's ndcg@1: 0.94315\tvalid_0's ndcg@2: 0.976936\tvalid_0's ndcg@3: 0.978236\tvalid_0's ndcg@4: 0.978409\tvalid_0's ndcg@5: 0.978496\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[74]\tvalid_0's ndcg@1: 0.94325\tvalid_0's ndcg@2: 0.976957\tvalid_0's ndcg@3: 0.97827\tvalid_0's ndcg@4: 0.978431\tvalid_0's ndcg@5: 0.978528\n",
      "[75]\tvalid_0's ndcg@1: 0.943075\tvalid_0's ndcg@2: 0.976861\tvalid_0's ndcg@3: 0.978199\tvalid_0's ndcg@4: 0.97836\tvalid_0's ndcg@5: 0.978457\n",
      "[76]\tvalid_0's ndcg@1: 0.94335\tvalid_0's ndcg@2: 0.976963\tvalid_0's ndcg@3: 0.978288\tvalid_0's ndcg@4: 0.978471\tvalid_0's ndcg@5: 0.978568\n",
      "[77]\tvalid_0's ndcg@1: 0.94345\tvalid_0's ndcg@2: 0.977031\tvalid_0's ndcg@3: 0.978331\tvalid_0's ndcg@4: 0.978514\tvalid_0's ndcg@5: 0.978611\n",
      "[78]\tvalid_0's ndcg@1: 0.943475\tvalid_0's ndcg@2: 0.977088\tvalid_0's ndcg@3: 0.97835\tvalid_0's ndcg@4: 0.978533\tvalid_0's ndcg@5: 0.97863\n",
      "[79]\tvalid_0's ndcg@1: 0.943625\tvalid_0's ndcg@2: 0.977096\tvalid_0's ndcg@3: 0.978396\tvalid_0's ndcg@4: 0.978579\tvalid_0's ndcg@5: 0.978676\n",
      "[80]\tvalid_0's ndcg@1: 0.943825\tvalid_0's ndcg@2: 0.977154\tvalid_0's ndcg@3: 0.978479\tvalid_0's ndcg@4: 0.978651\tvalid_0's ndcg@5: 0.978748\n",
      "[81]\tvalid_0's ndcg@1: 0.943775\tvalid_0's ndcg@2: 0.977135\tvalid_0's ndcg@3: 0.97846\tvalid_0's ndcg@4: 0.978633\tvalid_0's ndcg@5: 0.978729\n",
      "[82]\tvalid_0's ndcg@1: 0.9443\tvalid_0's ndcg@2: 0.977361\tvalid_0's ndcg@3: 0.978673\tvalid_0's ndcg@4: 0.978845\tvalid_0's ndcg@5: 0.978933\n",
      "[83]\tvalid_0's ndcg@1: 0.9442\tvalid_0's ndcg@2: 0.977324\tvalid_0's ndcg@3: 0.978624\tvalid_0's ndcg@4: 0.978796\tvalid_0's ndcg@5: 0.978893\n",
      "[84]\tvalid_0's ndcg@1: 0.94405\tvalid_0's ndcg@2: 0.977253\tvalid_0's ndcg@3: 0.978565\tvalid_0's ndcg@4: 0.978737\tvalid_0's ndcg@5: 0.978834\n",
      "[85]\tvalid_0's ndcg@1: 0.944175\tvalid_0's ndcg@2: 0.977283\tvalid_0's ndcg@3: 0.978633\tvalid_0's ndcg@4: 0.978795\tvalid_0's ndcg@5: 0.978882\n",
      "[86]\tvalid_0's ndcg@1: 0.9445\tvalid_0's ndcg@2: 0.97745\tvalid_0's ndcg@3: 0.978763\tvalid_0's ndcg@4: 0.978924\tvalid_0's ndcg@5: 0.979011\n",
      "[87]\tvalid_0's ndcg@1: 0.9445\tvalid_0's ndcg@2: 0.977419\tvalid_0's ndcg@3: 0.978756\tvalid_0's ndcg@4: 0.978918\tvalid_0's ndcg@5: 0.979005\n",
      "[88]\tvalid_0's ndcg@1: 0.944825\tvalid_0's ndcg@2: 0.977554\tvalid_0's ndcg@3: 0.978867\tvalid_0's ndcg@4: 0.979039\tvalid_0's ndcg@5: 0.979126\n",
      "[89]\tvalid_0's ndcg@1: 0.9454\tvalid_0's ndcg@2: 0.977767\tvalid_0's ndcg@3: 0.979079\tvalid_0's ndcg@4: 0.979262\tvalid_0's ndcg@5: 0.97934\n",
      "[90]\tvalid_0's ndcg@1: 0.945375\tvalid_0's ndcg@2: 0.977773\tvalid_0's ndcg@3: 0.979073\tvalid_0's ndcg@4: 0.979256\tvalid_0's ndcg@5: 0.979334\n",
      "[91]\tvalid_0's ndcg@1: 0.945425\tvalid_0's ndcg@2: 0.977792\tvalid_0's ndcg@3: 0.979092\tvalid_0's ndcg@4: 0.979275\tvalid_0's ndcg@5: 0.979352\n",
      "[92]\tvalid_0's ndcg@1: 0.945425\tvalid_0's ndcg@2: 0.977776\tvalid_0's ndcg@3: 0.979088\tvalid_0's ndcg@4: 0.979261\tvalid_0's ndcg@5: 0.979348\n",
      "[93]\tvalid_0's ndcg@1: 0.945375\tvalid_0's ndcg@2: 0.977757\tvalid_0's ndcg@3: 0.979082\tvalid_0's ndcg@4: 0.979244\tvalid_0's ndcg@5: 0.979331\n",
      "[94]\tvalid_0's ndcg@1: 0.9453\tvalid_0's ndcg@2: 0.977761\tvalid_0's ndcg@3: 0.979061\tvalid_0's ndcg@4: 0.979223\tvalid_0's ndcg@5: 0.97931\n",
      "[95]\tvalid_0's ndcg@1: 0.9454\tvalid_0's ndcg@2: 0.977798\tvalid_0's ndcg@3: 0.979086\tvalid_0's ndcg@4: 0.979258\tvalid_0's ndcg@5: 0.979345\n",
      "[96]\tvalid_0's ndcg@1: 0.945825\tvalid_0's ndcg@2: 0.977955\tvalid_0's ndcg@3: 0.97923\tvalid_0's ndcg@4: 0.979413\tvalid_0's ndcg@5: 0.9795\n",
      "[97]\tvalid_0's ndcg@1: 0.945925\tvalid_0's ndcg@2: 0.97796\tvalid_0's ndcg@3: 0.97926\tvalid_0's ndcg@4: 0.979443\tvalid_0's ndcg@5: 0.979531\n",
      "[98]\tvalid_0's ndcg@1: 0.9464\tvalid_0's ndcg@2: 0.97812\tvalid_0's ndcg@3: 0.97942\tvalid_0's ndcg@4: 0.979625\tvalid_0's ndcg@5: 0.979702\n",
      "[99]\tvalid_0's ndcg@1: 0.94655\tvalid_0's ndcg@2: 0.978191\tvalid_0's ndcg@3: 0.979479\tvalid_0's ndcg@4: 0.979683\tvalid_0's ndcg@5: 0.97977\n",
      "[100]\tvalid_0's ndcg@1: 0.94665\tvalid_0's ndcg@2: 0.978244\tvalid_0's ndcg@3: 0.979531\tvalid_0's ndcg@4: 0.979725\tvalid_0's ndcg@5: 0.979812\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's ndcg@1: 0.94665\tvalid_0's ndcg@2: 0.978244\tvalid_0's ndcg@3: 0.979531\tvalid_0's ndcg@4: 0.979725\tvalid_0's ndcg@5: 0.979812\n",
      "[1]\tvalid_0's ndcg@1: 0.910175\tvalid_0's ndcg@2: 0.963031\tvalid_0's ndcg@3: 0.965281\tvalid_0's ndcg@4: 0.965819\tvalid_0's ndcg@5: 0.965887\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's ndcg@1: 0.9141\tvalid_0's ndcg@2: 0.964748\tvalid_0's ndcg@3: 0.96681\tvalid_0's ndcg@4: 0.967316\tvalid_0's ndcg@5: 0.967394\n",
      "[3]\tvalid_0's ndcg@1: 0.915925\tvalid_0's ndcg@2: 0.9655\tvalid_0's ndcg@3: 0.967575\tvalid_0's ndcg@4: 0.968028\tvalid_0's ndcg@5: 0.968105\n",
      "[4]\tvalid_0's ndcg@1: 0.91915\tvalid_0's ndcg@2: 0.966943\tvalid_0's ndcg@3: 0.968968\tvalid_0's ndcg@4: 0.969334\tvalid_0's ndcg@5: 0.969373\n",
      "[5]\tvalid_0's ndcg@1: 0.920625\tvalid_0's ndcg@2: 0.967598\tvalid_0's ndcg@3: 0.969498\tvalid_0's ndcg@4: 0.969896\tvalid_0's ndcg@5: 0.969944\n",
      "[6]\tvalid_0's ndcg@1: 0.922625\tvalid_0's ndcg@2: 0.968336\tvalid_0's ndcg@3: 0.970261\tvalid_0's ndcg@4: 0.970659\tvalid_0's ndcg@5: 0.970688\n",
      "[7]\tvalid_0's ndcg@1: 0.923625\tvalid_0's ndcg@2: 0.968768\tvalid_0's ndcg@3: 0.970656\tvalid_0's ndcg@4: 0.971043\tvalid_0's ndcg@5: 0.971072\n",
      "[8]\tvalid_0's ndcg@1: 0.925825\tvalid_0's ndcg@2: 0.969612\tvalid_0's ndcg@3: 0.971462\tvalid_0's ndcg@4: 0.97186\tvalid_0's ndcg@5: 0.971879\n",
      "[9]\tvalid_0's ndcg@1: 0.926475\tvalid_0's ndcg@2: 0.969899\tvalid_0's ndcg@3: 0.971711\tvalid_0's ndcg@4: 0.97211\tvalid_0's ndcg@5: 0.972129\n",
      "[10]\tvalid_0's ndcg@1: 0.927775\tvalid_0's ndcg@2: 0.97041\tvalid_0's ndcg@3: 0.972185\tvalid_0's ndcg@4: 0.972594\tvalid_0's ndcg@5: 0.972614\n",
      "[11]\tvalid_0's ndcg@1: 0.92885\tvalid_0's ndcg@2: 0.970838\tvalid_0's ndcg@3: 0.972588\tvalid_0's ndcg@4: 0.973008\tvalid_0's ndcg@5: 0.973028\n",
      "[12]\tvalid_0's ndcg@1: 0.930325\tvalid_0's ndcg@2: 0.971367\tvalid_0's ndcg@3: 0.973129\tvalid_0's ndcg@4: 0.973549\tvalid_0's ndcg@5: 0.973569\n",
      "[13]\tvalid_0's ndcg@1: 0.931125\tvalid_0's ndcg@2: 0.971631\tvalid_0's ndcg@3: 0.973443\tvalid_0's ndcg@4: 0.973842\tvalid_0's ndcg@5: 0.973871\n",
      "[14]\tvalid_0's ndcg@1: 0.931525\tvalid_0's ndcg@2: 0.971778\tvalid_0's ndcg@3: 0.973616\tvalid_0's ndcg@4: 0.973993\tvalid_0's ndcg@5: 0.974022\n",
      "[15]\tvalid_0's ndcg@1: 0.9311\tvalid_0's ndcg@2: 0.9717\tvalid_0's ndcg@3: 0.973475\tvalid_0's ndcg@4: 0.973852\tvalid_0's ndcg@5: 0.973872\n",
      "[16]\tvalid_0's ndcg@1: 0.931775\tvalid_0's ndcg@2: 0.971902\tvalid_0's ndcg@3: 0.973702\tvalid_0's ndcg@4: 0.97409\tvalid_0's ndcg@5: 0.974109\n",
      "[17]\tvalid_0's ndcg@1: 0.931425\tvalid_0's ndcg@2: 0.971805\tvalid_0's ndcg@3: 0.97358\tvalid_0's ndcg@4: 0.973967\tvalid_0's ndcg@5: 0.973986\n",
      "[18]\tvalid_0's ndcg@1: 0.931575\tvalid_0's ndcg@2: 0.971876\tvalid_0's ndcg@3: 0.973651\tvalid_0's ndcg@4: 0.974027\tvalid_0's ndcg@5: 0.974047\n",
      "[19]\tvalid_0's ndcg@1: 0.932\tvalid_0's ndcg@2: 0.97208\tvalid_0's ndcg@3: 0.973805\tvalid_0's ndcg@4: 0.974192\tvalid_0's ndcg@5: 0.974212\n",
      "[20]\tvalid_0's ndcg@1: 0.932075\tvalid_0's ndcg@2: 0.972092\tvalid_0's ndcg@3: 0.973829\tvalid_0's ndcg@4: 0.974217\tvalid_0's ndcg@5: 0.974236\n",
      "[21]\tvalid_0's ndcg@1: 0.932675\tvalid_0's ndcg@2: 0.972282\tvalid_0's ndcg@3: 0.974057\tvalid_0's ndcg@4: 0.974444\tvalid_0's ndcg@5: 0.974454\n",
      "[22]\tvalid_0's ndcg@1: 0.932925\tvalid_0's ndcg@2: 0.972358\tvalid_0's ndcg@3: 0.974146\tvalid_0's ndcg@4: 0.974533\tvalid_0's ndcg@5: 0.974543\n",
      "[23]\tvalid_0's ndcg@1: 0.93325\tvalid_0's ndcg@2: 0.972478\tvalid_0's ndcg@3: 0.974253\tvalid_0's ndcg@4: 0.974651\tvalid_0's ndcg@5: 0.974661\n",
      "[24]\tvalid_0's ndcg@1: 0.9335\tvalid_0's ndcg@2: 0.972539\tvalid_0's ndcg@3: 0.974351\tvalid_0's ndcg@4: 0.974739\tvalid_0's ndcg@5: 0.974749\n",
      "[25]\tvalid_0's ndcg@1: 0.93475\tvalid_0's ndcg@2: 0.973\tvalid_0's ndcg@3: 0.974788\tvalid_0's ndcg@4: 0.975197\tvalid_0's ndcg@5: 0.975206\n",
      "[26]\tvalid_0's ndcg@1: 0.935075\tvalid_0's ndcg@2: 0.97312\tvalid_0's ndcg@3: 0.974895\tvalid_0's ndcg@4: 0.975315\tvalid_0's ndcg@5: 0.975325\n",
      "[27]\tvalid_0's ndcg@1: 0.9349\tvalid_0's ndcg@2: 0.973103\tvalid_0's ndcg@3: 0.974865\tvalid_0's ndcg@4: 0.975264\tvalid_0's ndcg@5: 0.975273\n",
      "[28]\tvalid_0's ndcg@1: 0.935075\tvalid_0's ndcg@2: 0.973152\tvalid_0's ndcg@3: 0.974939\tvalid_0's ndcg@4: 0.975327\tvalid_0's ndcg@5: 0.975336\n",
      "[29]\tvalid_0's ndcg@1: 0.935475\tvalid_0's ndcg@2: 0.973315\tvalid_0's ndcg@3: 0.975128\tvalid_0's ndcg@4: 0.975483\tvalid_0's ndcg@5: 0.975492\n",
      "[30]\tvalid_0's ndcg@1: 0.93595\tvalid_0's ndcg@2: 0.973522\tvalid_0's ndcg@3: 0.975297\tvalid_0's ndcg@4: 0.975663\tvalid_0's ndcg@5: 0.975673\n",
      "[31]\tvalid_0's ndcg@1: 0.93595\tvalid_0's ndcg@2: 0.973506\tvalid_0's ndcg@3: 0.975281\tvalid_0's ndcg@4: 0.975658\tvalid_0's ndcg@5: 0.975668\n",
      "[32]\tvalid_0's ndcg@1: 0.93675\tvalid_0's ndcg@2: 0.973833\tvalid_0's ndcg@3: 0.975595\tvalid_0's ndcg@4: 0.975961\tvalid_0's ndcg@5: 0.975971\n",
      "[33]\tvalid_0's ndcg@1: 0.936475\tvalid_0's ndcg@2: 0.973763\tvalid_0's ndcg@3: 0.975488\tvalid_0's ndcg@4: 0.975865\tvalid_0's ndcg@5: 0.975874\n",
      "[34]\tvalid_0's ndcg@1: 0.9367\tvalid_0's ndcg@2: 0.973893\tvalid_0's ndcg@3: 0.975568\tvalid_0's ndcg@4: 0.975956\tvalid_0's ndcg@5: 0.975966\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[35]\tvalid_0's ndcg@1: 0.93715\tvalid_0's ndcg@2: 0.974059\tvalid_0's ndcg@3: 0.975722\tvalid_0's ndcg@4: 0.97612\tvalid_0's ndcg@5: 0.97613\n",
      "[36]\tvalid_0's ndcg@1: 0.9374\tvalid_0's ndcg@2: 0.974183\tvalid_0's ndcg@3: 0.975846\tvalid_0's ndcg@4: 0.976223\tvalid_0's ndcg@5: 0.976232\n",
      "[37]\tvalid_0's ndcg@1: 0.9374\tvalid_0's ndcg@2: 0.974183\tvalid_0's ndcg@3: 0.975846\tvalid_0's ndcg@4: 0.976223\tvalid_0's ndcg@5: 0.976232\n",
      "[38]\tvalid_0's ndcg@1: 0.938725\tvalid_0's ndcg@2: 0.974672\tvalid_0's ndcg@3: 0.97636\tvalid_0's ndcg@4: 0.976715\tvalid_0's ndcg@5: 0.976725\n",
      "[39]\tvalid_0's ndcg@1: 0.93865\tvalid_0's ndcg@2: 0.974676\tvalid_0's ndcg@3: 0.976364\tvalid_0's ndcg@4: 0.976697\tvalid_0's ndcg@5: 0.976707\n",
      "[40]\tvalid_0's ndcg@1: 0.939125\tvalid_0's ndcg@2: 0.974867\tvalid_0's ndcg@3: 0.97653\tvalid_0's ndcg@4: 0.976874\tvalid_0's ndcg@5: 0.976884\n",
      "[41]\tvalid_0's ndcg@1: 0.9396\tvalid_0's ndcg@2: 0.975042\tvalid_0's ndcg@3: 0.976705\tvalid_0's ndcg@4: 0.97705\tvalid_0's ndcg@5: 0.977059\n",
      "[42]\tvalid_0's ndcg@1: 0.93985\tvalid_0's ndcg@2: 0.975072\tvalid_0's ndcg@3: 0.976784\tvalid_0's ndcg@4: 0.977129\tvalid_0's ndcg@5: 0.977138\n",
      "[43]\tvalid_0's ndcg@1: 0.940075\tvalid_0's ndcg@2: 0.97517\tvalid_0's ndcg@3: 0.97687\tvalid_0's ndcg@4: 0.977215\tvalid_0's ndcg@5: 0.977225\n",
      "[44]\tvalid_0's ndcg@1: 0.94045\tvalid_0's ndcg@2: 0.97534\tvalid_0's ndcg@3: 0.977015\tvalid_0's ndcg@4: 0.97736\tvalid_0's ndcg@5: 0.97737\n",
      "[45]\tvalid_0's ndcg@1: 0.94055\tvalid_0's ndcg@2: 0.975409\tvalid_0's ndcg@3: 0.977059\tvalid_0's ndcg@4: 0.977403\tvalid_0's ndcg@5: 0.977413\n",
      "[46]\tvalid_0's ndcg@1: 0.940525\tvalid_0's ndcg@2: 0.975415\tvalid_0's ndcg@3: 0.97704\tvalid_0's ndcg@4: 0.977396\tvalid_0's ndcg@5: 0.977405\n",
      "[47]\tvalid_0's ndcg@1: 0.940425\tvalid_0's ndcg@2: 0.975363\tvalid_0's ndcg@3: 0.977013\tvalid_0's ndcg@4: 0.977357\tvalid_0's ndcg@5: 0.977367\n",
      "[48]\tvalid_0's ndcg@1: 0.94045\tvalid_0's ndcg@2: 0.975388\tvalid_0's ndcg@3: 0.977025\tvalid_0's ndcg@4: 0.97737\tvalid_0's ndcg@5: 0.977379\n",
      "[49]\tvalid_0's ndcg@1: 0.940525\tvalid_0's ndcg@2: 0.975447\tvalid_0's ndcg@3: 0.977097\tvalid_0's ndcg@4: 0.977409\tvalid_0's ndcg@5: 0.977419\n",
      "[50]\tvalid_0's ndcg@1: 0.941075\tvalid_0's ndcg@2: 0.975666\tvalid_0's ndcg@3: 0.977303\tvalid_0's ndcg@4: 0.977615\tvalid_0's ndcg@5: 0.977625\n",
      "[51]\tvalid_0's ndcg@1: 0.94135\tvalid_0's ndcg@2: 0.975751\tvalid_0's ndcg@3: 0.977376\tvalid_0's ndcg@4: 0.97771\tvalid_0's ndcg@5: 0.97772\n",
      "[52]\tvalid_0's ndcg@1: 0.9413\tvalid_0's ndcg@2: 0.975717\tvalid_0's ndcg@3: 0.977355\tvalid_0's ndcg@4: 0.977688\tvalid_0's ndcg@5: 0.977698\n",
      "[53]\tvalid_0's ndcg@1: 0.941375\tvalid_0's ndcg@2: 0.975713\tvalid_0's ndcg@3: 0.977376\tvalid_0's ndcg@4: 0.977699\tvalid_0's ndcg@5: 0.977718\n",
      "[54]\tvalid_0's ndcg@1: 0.94185\tvalid_0's ndcg@2: 0.975857\tvalid_0's ndcg@3: 0.977557\tvalid_0's ndcg@4: 0.977869\tvalid_0's ndcg@5: 0.977889\n",
      "[55]\tvalid_0's ndcg@1: 0.941925\tvalid_0's ndcg@2: 0.975837\tvalid_0's ndcg@3: 0.9776\tvalid_0's ndcg@4: 0.977891\tvalid_0's ndcg@5: 0.97791\n",
      "[56]\tvalid_0's ndcg@1: 0.942325\tvalid_0's ndcg@2: 0.975969\tvalid_0's ndcg@3: 0.977719\tvalid_0's ndcg@4: 0.978032\tvalid_0's ndcg@5: 0.978051\n",
      "[57]\tvalid_0's ndcg@1: 0.942425\tvalid_0's ndcg@2: 0.976022\tvalid_0's ndcg@3: 0.977772\tvalid_0's ndcg@4: 0.978073\tvalid_0's ndcg@5: 0.978093\n",
      "[58]\tvalid_0's ndcg@1: 0.9425\tvalid_0's ndcg@2: 0.976081\tvalid_0's ndcg@3: 0.977806\tvalid_0's ndcg@4: 0.978108\tvalid_0's ndcg@5: 0.978127\n",
      "[59]\tvalid_0's ndcg@1: 0.9424\tvalid_0's ndcg@2: 0.976076\tvalid_0's ndcg@3: 0.977788\tvalid_0's ndcg@4: 0.978079\tvalid_0's ndcg@5: 0.978098\n",
      "[60]\tvalid_0's ndcg@1: 0.942375\tvalid_0's ndcg@2: 0.976067\tvalid_0's ndcg@3: 0.977779\tvalid_0's ndcg@4: 0.97807\tvalid_0's ndcg@5: 0.978089\n",
      "[61]\tvalid_0's ndcg@1: 0.942225\tvalid_0's ndcg@2: 0.976043\tvalid_0's ndcg@3: 0.97773\tvalid_0's ndcg@4: 0.978021\tvalid_0's ndcg@5: 0.97804\n",
      "[62]\tvalid_0's ndcg@1: 0.942425\tvalid_0's ndcg@2: 0.976117\tvalid_0's ndcg@3: 0.977792\tvalid_0's ndcg@4: 0.978093\tvalid_0's ndcg@5: 0.978112\n",
      "[63]\tvalid_0's ndcg@1: 0.942675\tvalid_0's ndcg@2: 0.976193\tvalid_0's ndcg@3: 0.977881\tvalid_0's ndcg@4: 0.978182\tvalid_0's ndcg@5: 0.978201\n",
      "[64]\tvalid_0's ndcg@1: 0.942925\tvalid_0's ndcg@2: 0.976254\tvalid_0's ndcg@3: 0.977966\tvalid_0's ndcg@4: 0.978268\tvalid_0's ndcg@5: 0.978287\n",
      "[65]\tvalid_0's ndcg@1: 0.9431\tvalid_0's ndcg@2: 0.97635\tvalid_0's ndcg@3: 0.978025\tvalid_0's ndcg@4: 0.978337\tvalid_0's ndcg@5: 0.978357\n",
      "[66]\tvalid_0's ndcg@1: 0.9434\tvalid_0's ndcg@2: 0.976445\tvalid_0's ndcg@3: 0.978132\tvalid_0's ndcg@4: 0.978445\tvalid_0's ndcg@5: 0.978464\n",
      "[67]\tvalid_0's ndcg@1: 0.943275\tvalid_0's ndcg@2: 0.976399\tvalid_0's ndcg@3: 0.978074\tvalid_0's ndcg@4: 0.978397\tvalid_0's ndcg@5: 0.978416\n",
      "[68]\tvalid_0's ndcg@1: 0.943325\tvalid_0's ndcg@2: 0.976401\tvalid_0's ndcg@3: 0.978089\tvalid_0's ndcg@4: 0.978412\tvalid_0's ndcg@5: 0.978431\n",
      "[69]\tvalid_0's ndcg@1: 0.943675\tvalid_0's ndcg@2: 0.976578\tvalid_0's ndcg@3: 0.97819\tvalid_0's ndcg@4: 0.978546\tvalid_0's ndcg@5: 0.978565\n",
      "[70]\tvalid_0's ndcg@1: 0.944025\tvalid_0's ndcg@2: 0.976707\tvalid_0's ndcg@3: 0.97832\tvalid_0's ndcg@4: 0.978675\tvalid_0's ndcg@5: 0.978694\n",
      "[71]\tvalid_0's ndcg@1: 0.9442\tvalid_0's ndcg@2: 0.976772\tvalid_0's ndcg@3: 0.978384\tvalid_0's ndcg@4: 0.97874\tvalid_0's ndcg@5: 0.978759\n",
      "[72]\tvalid_0's ndcg@1: 0.94425\tvalid_0's ndcg@2: 0.976822\tvalid_0's ndcg@3: 0.978409\tvalid_0's ndcg@4: 0.978765\tvalid_0's ndcg@5: 0.978784\n",
      "[73]\tvalid_0's ndcg@1: 0.94445\tvalid_0's ndcg@2: 0.976864\tvalid_0's ndcg@3: 0.978464\tvalid_0's ndcg@4: 0.97883\tvalid_0's ndcg@5: 0.978849\n",
      "[74]\tvalid_0's ndcg@1: 0.9446\tvalid_0's ndcg@2: 0.976919\tvalid_0's ndcg@3: 0.978519\tvalid_0's ndcg@4: 0.978885\tvalid_0's ndcg@5: 0.978905\n",
      "[75]\tvalid_0's ndcg@1: 0.9446\tvalid_0's ndcg@2: 0.976919\tvalid_0's ndcg@3: 0.978519\tvalid_0's ndcg@4: 0.978885\tvalid_0's ndcg@5: 0.978905\n",
      "[76]\tvalid_0's ndcg@1: 0.944625\tvalid_0's ndcg@2: 0.97696\tvalid_0's ndcg@3: 0.978535\tvalid_0's ndcg@4: 0.978901\tvalid_0's ndcg@5: 0.978921\n",
      "[77]\tvalid_0's ndcg@1: 0.944675\tvalid_0's ndcg@2: 0.976979\tvalid_0's ndcg@3: 0.978554\tvalid_0's ndcg@4: 0.97892\tvalid_0's ndcg@5: 0.978939\n",
      "[78]\tvalid_0's ndcg@1: 0.944675\tvalid_0's ndcg@2: 0.976979\tvalid_0's ndcg@3: 0.978554\tvalid_0's ndcg@4: 0.97892\tvalid_0's ndcg@5: 0.978939\n",
      "[79]\tvalid_0's ndcg@1: 0.944525\tvalid_0's ndcg@2: 0.976907\tvalid_0's ndcg@3: 0.978507\tvalid_0's ndcg@4: 0.978863\tvalid_0's ndcg@5: 0.978882\n",
      "[80]\tvalid_0's ndcg@1: 0.94455\tvalid_0's ndcg@2: 0.976885\tvalid_0's ndcg@3: 0.97851\tvalid_0's ndcg@4: 0.978865\tvalid_0's ndcg@5: 0.978885\n",
      "[81]\tvalid_0's ndcg@1: 0.944725\tvalid_0's ndcg@2: 0.97695\tvalid_0's ndcg@3: 0.978575\tvalid_0's ndcg@4: 0.978919\tvalid_0's ndcg@5: 0.978948\n",
      "[82]\tvalid_0's ndcg@1: 0.945225\tvalid_0's ndcg@2: 0.977103\tvalid_0's ndcg@3: 0.978765\tvalid_0's ndcg@4: 0.97911\tvalid_0's ndcg@5: 0.979129\n",
      "[83]\tvalid_0's ndcg@1: 0.945125\tvalid_0's ndcg@2: 0.977066\tvalid_0's ndcg@3: 0.978716\tvalid_0's ndcg@4: 0.979071\tvalid_0's ndcg@5: 0.97909\n",
      "[84]\tvalid_0's ndcg@1: 0.945225\tvalid_0's ndcg@2: 0.97715\tvalid_0's ndcg@3: 0.978775\tvalid_0's ndcg@4: 0.97912\tvalid_0's ndcg@5: 0.979139\n",
      "[85]\tvalid_0's ndcg@1: 0.945025\tvalid_0's ndcg@2: 0.977092\tvalid_0's ndcg@3: 0.978692\tvalid_0's ndcg@4: 0.979047\tvalid_0's ndcg@5: 0.979067\n",
      "[86]\tvalid_0's ndcg@1: 0.9452\tvalid_0's ndcg@2: 0.977172\tvalid_0's ndcg@3: 0.97876\tvalid_0's ndcg@4: 0.979115\tvalid_0's ndcg@5: 0.979135\n",
      "[87]\tvalid_0's ndcg@1: 0.9453\tvalid_0's ndcg@2: 0.977178\tvalid_0's ndcg@3: 0.97879\tvalid_0's ndcg@4: 0.979156\tvalid_0's ndcg@5: 0.979166\n",
      "[88]\tvalid_0's ndcg@1: 0.9453\tvalid_0's ndcg@2: 0.977178\tvalid_0's ndcg@3: 0.978815\tvalid_0's ndcg@4: 0.979149\tvalid_0's ndcg@5: 0.979168\n",
      "[89]\tvalid_0's ndcg@1: 0.94555\tvalid_0's ndcg@2: 0.977333\tvalid_0's ndcg@3: 0.978933\tvalid_0's ndcg@4: 0.979267\tvalid_0's ndcg@5: 0.979277\n",
      "[90]\tvalid_0's ndcg@1: 0.9459\tvalid_0's ndcg@2: 0.977462\tvalid_0's ndcg@3: 0.979062\tvalid_0's ndcg@4: 0.979396\tvalid_0's ndcg@5: 0.979406\n",
      "[91]\tvalid_0's ndcg@1: 0.94595\tvalid_0's ndcg@2: 0.977481\tvalid_0's ndcg@3: 0.979081\tvalid_0's ndcg@4: 0.979414\tvalid_0's ndcg@5: 0.979424\n",
      "[92]\tvalid_0's ndcg@1: 0.945875\tvalid_0's ndcg@2: 0.977437\tvalid_0's ndcg@3: 0.97905\tvalid_0's ndcg@4: 0.979384\tvalid_0's ndcg@5: 0.979393\n",
      "[93]\tvalid_0's ndcg@1: 0.945875\tvalid_0's ndcg@2: 0.977421\tvalid_0's ndcg@3: 0.979046\tvalid_0's ndcg@4: 0.97938\tvalid_0's ndcg@5: 0.97939\n",
      "[94]\tvalid_0's ndcg@1: 0.9459\tvalid_0's ndcg@2: 0.977431\tvalid_0's ndcg@3: 0.979068\tvalid_0's ndcg@4: 0.979391\tvalid_0's ndcg@5: 0.979401\n",
      "[95]\tvalid_0's ndcg@1: 0.94595\tvalid_0's ndcg@2: 0.977449\tvalid_0's ndcg@3: 0.979074\tvalid_0's ndcg@4: 0.979408\tvalid_0's ndcg@5: 0.979418\n",
      "[96]\tvalid_0's ndcg@1: 0.946075\tvalid_0's ndcg@2: 0.977527\tvalid_0's ndcg@3: 0.979127\tvalid_0's ndcg@4: 0.979461\tvalid_0's ndcg@5: 0.97947\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[97]\tvalid_0's ndcg@1: 0.946375\tvalid_0's ndcg@2: 0.977622\tvalid_0's ndcg@3: 0.979222\tvalid_0's ndcg@4: 0.979577\tvalid_0's ndcg@5: 0.979577\n",
      "[98]\tvalid_0's ndcg@1: 0.946625\tvalid_0's ndcg@2: 0.977714\tvalid_0's ndcg@3: 0.979339\tvalid_0's ndcg@4: 0.979673\tvalid_0's ndcg@5: 0.979673\n",
      "[99]\tvalid_0's ndcg@1: 0.94665\tvalid_0's ndcg@2: 0.977739\tvalid_0's ndcg@3: 0.979352\tvalid_0's ndcg@4: 0.979685\tvalid_0's ndcg@5: 0.979685\n",
      "[100]\tvalid_0's ndcg@1: 0.946675\tvalid_0's ndcg@2: 0.97778\tvalid_0's ndcg@3: 0.97938\tvalid_0's ndcg@4: 0.979703\tvalid_0's ndcg@5: 0.979703\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's ndcg@1: 0.946675\tvalid_0's ndcg@2: 0.97778\tvalid_0's ndcg@3: 0.97938\tvalid_0's ndcg@4: 0.979703\tvalid_0's ndcg@5: 0.979703\n"
     ]
    }
   ],
   "source": [
    "# 五折交叉验证，这里的五折交叉是以用户为目标进行五折划分\n",
    "#  这一部分与前面的单独训练和验证是分开的\n",
    "def get_kfold_users(trn_df, n=5):\n",
    "    user_ids = trn_df['user_id'].unique()\n",
    "    user_set = [user_ids[i::n] for i in range(n)]\n",
    "    return user_set\n",
    "\n",
    "k_fold = 5\n",
    "trn_df = trn_user_item_feats_df_rank_model\n",
    "user_set = get_kfold_users(trn_df, n=k_fold)\n",
    "\n",
    "score_list = []\n",
    "score_df = trn_df[['user_id', 'click_article_id','label']]\n",
    "sub_preds = np.zeros(tst_user_item_feats_df_rank_model.shape[0])\n",
    "\n",
    "# 五折交叉验证，并将中间结果保存用于staking\n",
    "for n_fold, valid_user in enumerate(user_set):\n",
    "    train_idx = trn_df[~trn_df['user_id'].isin(valid_user)] # add slide user\n",
    "    valid_idx = trn_df[trn_df['user_id'].isin(valid_user)]\n",
    "    \n",
    "    # 训练集与验证集的用户分组\n",
    "    train_idx.sort_values(by=['user_id'], inplace=True)\n",
    "    g_train = train_idx.groupby(['user_id'], as_index=False).count()[\"label\"].values\n",
    "    \n",
    "    valid_idx.sort_values(by=['user_id'], inplace=True)\n",
    "    g_val = valid_idx.groupby(['user_id'], as_index=False).count()[\"label\"].values\n",
    "    \n",
    "    # 定义模型\n",
    "    lgb_ranker = lgb.LGBMRanker(boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,\n",
    "                            max_depth=-1, n_estimators=100, subsample=0.7, colsample_bytree=0.7, subsample_freq=1,\n",
    "                            learning_rate=0.01, min_child_weight=50, random_state=2018, n_jobs= 16)  \n",
    "    # 训练模型\n",
    "    lgb_ranker.fit(train_idx[lgb_cols], train_idx['label'], group=g_train,\n",
    "                   eval_set=[(valid_idx[lgb_cols], valid_idx['label'])], eval_group= [g_val], \n",
    "                   eval_at=[1, 2, 3, 4, 5], eval_metric=['ndcg', ], early_stopping_rounds=50, )\n",
    "    \n",
    "    # 预测验证集结果\n",
    "    valid_idx['pred_score'] = lgb_ranker.predict(valid_idx[lgb_cols], num_iteration=lgb_ranker.best_iteration_)\n",
    "    \n",
    "    # 对输出结果进行归一化\n",
    "    valid_idx['pred_score'] = valid_idx[['pred_score']].transform(lambda x: norm_sim(x))\n",
    "    \n",
    "    valid_idx.sort_values(by=['user_id', 'pred_score'])\n",
    "    valid_idx['pred_rank'] = valid_idx.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first')\n",
    "    \n",
    "    # 将验证集的预测结果放到一个列表中，后面进行拼接\n",
    "    score_list.append(valid_idx[['user_id', 'click_article_id', 'pred_score', 'pred_rank']])\n",
    "    \n",
    "    # 如果是线上测试，需要计算每次交叉验证的结果相加，最后求平均\n",
    "    if not offline:\n",
    "        sub_preds += lgb_ranker.predict(tst_user_item_feats_df_rank_model[lgb_cols], lgb_ranker.best_iteration_)\n",
    "    \n",
    "score_df_ = pd.concat(score_list, axis=0)\n",
    "score_df = score_df.merge(score_df_, how='left', on=['user_id', 'click_article_id'])\n",
    "# 保存训练集交叉验证产生的新特征\n",
    "score_df[['user_id', 'click_article_id', 'pred_score', 'pred_rank', 'label']].to_csv(save_path + 'trn_lgb_ranker_feats.csv', index=False)\n",
    "    \n",
    "# 测试集的预测结果，多次交叉验证求平均,将预测的score和对应的rank特征保存，可以用于后面的staking，这里还可以构造其他更多的特征\n",
    "tst_user_item_feats_df_rank_model['pred_score'] = sub_preds / k_fold\n",
    "tst_user_item_feats_df_rank_model['pred_score'] = tst_user_item_feats_df_rank_model['pred_score'].transform(lambda x: norm_sim(x))\n",
    "tst_user_item_feats_df_rank_model.sort_values(by=['user_id', 'pred_score'])\n",
    "tst_user_item_feats_df_rank_model['pred_rank'] = tst_user_item_feats_df_rank_model.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first')\n",
    "\n",
    "# 保存测试集交叉验证的新特征\n",
    "tst_user_item_feats_df_rank_model[['user_id', 'click_article_id', 'pred_score', 'pred_rank']].to_csv(save_path + 'tst_lgb_ranker_feats.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:22:52.604397Z",
     "start_time": "2020-11-18T04:22:43.253034Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测结果重新排序, 及生成提交结果\n",
    "# 单模型生成提交结果\n",
    "rank_results = tst_user_item_feats_df_rank_model[['user_id', 'click_article_id', 'pred_score']]\n",
    "rank_results['click_article_id'] = rank_results['click_article_id'].astype(int)\n",
    "submit(rank_results, topk=5, model_name='lgb_ranker')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LGB分类模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:22:58.259730Z",
     "start_time": "2020-11-18T04:22:58.254297Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型及参数的定义\n",
    "lgb_Classfication = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,\n",
    "                            max_depth=-1, n_estimators=500, subsample=0.7, colsample_bytree=0.7, subsample_freq=1,\n",
    "                            learning_rate=0.01, min_child_weight=50, random_state=2018, n_jobs= 16, verbose=10)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:23:11.258774Z",
     "start_time": "2020-11-18T04:23:00.861936Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型训练\n",
    "if offline:\n",
    "    lgb_Classfication.fit(trn_user_item_feats_df_rank_model[lgb_cols], trn_user_item_feats_df_rank_model['label'],\n",
    "                    eval_set=[(val_user_item_feats_df_rank_model[lgb_cols], val_user_item_feats_df_rank_model['label'])], \n",
    "                    eval_metric=['auc', ],early_stopping_rounds=50, )\n",
    "else:\n",
    "    lgb_Classfication.fit(trn_user_item_feats_df_rank_model[lgb_cols], trn_user_item_feats_df_rank_model['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:23:19.591396Z",
     "start_time": "2020-11-18T04:23:13.813850Z"
    }
   },
   "outputs": [],
   "source": [
    "# 模型预测\n",
    "tst_user_item_feats_df['pred_score'] = lgb_Classfication.predict_proba(tst_user_item_feats_df[lgb_cols])[:,1]\n",
    "\n",
    "# 将这里的排序结果保存一份，用户后面的模型融合\n",
    "tst_user_item_feats_df[['user_id', 'click_article_id', 'pred_score']].to_csv(save_path + 'lgb_cls_score.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:23:32.352931Z",
     "start_time": "2020-11-18T04:23:22.346609Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测结果重新排序, 及生成提交结果\n",
    "rank_results = tst_user_item_feats_df[['user_id', 'click_article_id', 'pred_score']]\n",
    "rank_results['click_article_id'] = rank_results['click_article_id'].astype(int)\n",
    "submit(rank_results, topk=5, model_name='lgb_cls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:24:11.241196Z",
     "start_time": "2020-11-18T04:23:41.377394Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\tvalid_0's auc: 0.764896\tvalid_0's binary_logloss: 0.522153\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's auc: 0.767857\tvalid_0's binary_logloss: 0.52057\n",
      "[3]\tvalid_0's auc: 0.783096\tvalid_0's binary_logloss: 0.519584\n",
      "[4]\tvalid_0's auc: 0.784354\tvalid_0's binary_logloss: 0.518485\n",
      "[5]\tvalid_0's auc: 0.790554\tvalid_0's binary_logloss: 0.516886\n",
      "[6]\tvalid_0's auc: 0.791954\tvalid_0's binary_logloss: 0.515334\n",
      "[7]\tvalid_0's auc: 0.794257\tvalid_0's binary_logloss: 0.514032\n",
      "[8]\tvalid_0's auc: 0.795222\tvalid_0's binary_logloss: 0.512516\n",
      "[9]\tvalid_0's auc: 0.795417\tvalid_0's binary_logloss: 0.511671\n",
      "[10]\tvalid_0's auc: 0.795913\tvalid_0's binary_logloss: 0.510226\n",
      "[11]\tvalid_0's auc: 0.798222\tvalid_0's binary_logloss: 0.508858\n",
      "[12]\tvalid_0's auc: 0.79825\tvalid_0's binary_logloss: 0.507928\n",
      "[13]\tvalid_0's auc: 0.798842\tvalid_0's binary_logloss: 0.50708\n",
      "[14]\tvalid_0's auc: 0.798935\tvalid_0's binary_logloss: 0.505752\n",
      "[15]\tvalid_0's auc: 0.799543\tvalid_0's binary_logloss: 0.504388\n",
      "[16]\tvalid_0's auc: 0.800844\tvalid_0's binary_logloss: 0.503126\n",
      "[17]\tvalid_0's auc: 0.800855\tvalid_0's binary_logloss: 0.501809\n",
      "[18]\tvalid_0's auc: 0.801653\tvalid_0's binary_logloss: 0.500676\n",
      "[19]\tvalid_0's auc: 0.801518\tvalid_0's binary_logloss: 0.49987\n",
      "[20]\tvalid_0's auc: 0.801662\tvalid_0's binary_logloss: 0.498625\n",
      "[21]\tvalid_0's auc: 0.802093\tvalid_0's binary_logloss: 0.498113\n",
      "[22]\tvalid_0's auc: 0.803071\tvalid_0's binary_logloss: 0.496933\n",
      "[23]\tvalid_0's auc: 0.803222\tvalid_0's binary_logloss: 0.495864\n",
      "[24]\tvalid_0's auc: 0.802927\tvalid_0's binary_logloss: 0.494691\n",
      "[25]\tvalid_0's auc: 0.802581\tvalid_0's binary_logloss: 0.493543\n",
      "[26]\tvalid_0's auc: 0.802965\tvalid_0's binary_logloss: 0.492444\n",
      "[27]\tvalid_0's auc: 0.80298\tvalid_0's binary_logloss: 0.491336\n",
      "[28]\tvalid_0's auc: 0.803226\tvalid_0's binary_logloss: 0.490275\n",
      "[29]\tvalid_0's auc: 0.803436\tvalid_0's binary_logloss: 0.489126\n",
      "[30]\tvalid_0's auc: 0.803796\tvalid_0's binary_logloss: 0.48802\n",
      "[31]\tvalid_0's auc: 0.803601\tvalid_0's binary_logloss: 0.486988\n",
      "[32]\tvalid_0's auc: 0.804416\tvalid_0's binary_logloss: 0.485972\n",
      "[33]\tvalid_0's auc: 0.804529\tvalid_0's binary_logloss: 0.484939\n",
      "[34]\tvalid_0's auc: 0.804534\tvalid_0's binary_logloss: 0.483927\n",
      "[35]\tvalid_0's auc: 0.804819\tvalid_0's binary_logloss: 0.483271\n",
      "[36]\tvalid_0's auc: 0.804774\tvalid_0's binary_logloss: 0.482273\n",
      "[37]\tvalid_0's auc: 0.805237\tvalid_0's binary_logloss: 0.481639\n",
      "[38]\tvalid_0's auc: 0.805546\tvalid_0's binary_logloss: 0.480959\n",
      "[39]\tvalid_0's auc: 0.805598\tvalid_0's binary_logloss: 0.479955\n",
      "[40]\tvalid_0's auc: 0.806011\tvalid_0's binary_logloss: 0.47903\n",
      "[41]\tvalid_0's auc: 0.806664\tvalid_0's binary_logloss: 0.478439\n",
      "[42]\tvalid_0's auc: 0.807021\tvalid_0's binary_logloss: 0.477798\n",
      "[43]\tvalid_0's auc: 0.80726\tvalid_0's binary_logloss: 0.476829\n",
      "[44]\tvalid_0's auc: 0.807157\tvalid_0's binary_logloss: 0.475976\n",
      "[45]\tvalid_0's auc: 0.807788\tvalid_0's binary_logloss: 0.475056\n",
      "[46]\tvalid_0's auc: 0.80805\tvalid_0's binary_logloss: 0.474446\n",
      "[47]\tvalid_0's auc: 0.808097\tvalid_0's binary_logloss: 0.473576\n",
      "[48]\tvalid_0's auc: 0.80815\tvalid_0's binary_logloss: 0.472676\n",
      "[49]\tvalid_0's auc: 0.808304\tvalid_0's binary_logloss: 0.471918\n",
      "[50]\tvalid_0's auc: 0.808749\tvalid_0's binary_logloss: 0.471481\n",
      "[51]\tvalid_0's auc: 0.808972\tvalid_0's binary_logloss: 0.471104\n",
      "[52]\tvalid_0's auc: 0.809326\tvalid_0's binary_logloss: 0.470289\n",
      "[53]\tvalid_0's auc: 0.809472\tvalid_0's binary_logloss: 0.469508\n",
      "[54]\tvalid_0's auc: 0.809505\tvalid_0's binary_logloss: 0.46869\n",
      "[55]\tvalid_0's auc: 0.809594\tvalid_0's binary_logloss: 0.467885\n",
      "[56]\tvalid_0's auc: 0.809847\tvalid_0's binary_logloss: 0.467356\n",
      "[57]\tvalid_0's auc: 0.810262\tvalid_0's binary_logloss: 0.466531\n",
      "[58]\tvalid_0's auc: 0.810407\tvalid_0's binary_logloss: 0.46573\n",
      "[59]\tvalid_0's auc: 0.810618\tvalid_0's binary_logloss: 0.465205\n",
      "[60]\tvalid_0's auc: 0.81066\tvalid_0's binary_logloss: 0.464435\n",
      "[61]\tvalid_0's auc: 0.810638\tvalid_0's binary_logloss: 0.463721\n",
      "[62]\tvalid_0's auc: 0.810658\tvalid_0's binary_logloss: 0.462982\n",
      "[63]\tvalid_0's auc: 0.811106\tvalid_0's binary_logloss: 0.462246\n",
      "[64]\tvalid_0's auc: 0.811313\tvalid_0's binary_logloss: 0.461748\n",
      "[65]\tvalid_0's auc: 0.811351\tvalid_0's binary_logloss: 0.461038\n",
      "[66]\tvalid_0's auc: 0.811433\tvalid_0's binary_logloss: 0.460323\n",
      "[67]\tvalid_0's auc: 0.81158\tvalid_0's binary_logloss: 0.459662\n",
      "[68]\tvalid_0's auc: 0.811561\tvalid_0's binary_logloss: 0.458988\n",
      "[69]\tvalid_0's auc: 0.811748\tvalid_0's binary_logloss: 0.458592\n",
      "[70]\tvalid_0's auc: 0.811919\tvalid_0's binary_logloss: 0.457934\n",
      "[71]\tvalid_0's auc: 0.812073\tvalid_0's binary_logloss: 0.457508\n",
      "[72]\tvalid_0's auc: 0.812273\tvalid_0's binary_logloss: 0.457038\n",
      "[73]\tvalid_0's auc: 0.812561\tvalid_0's binary_logloss: 0.456439\n",
      "[74]\tvalid_0's auc: 0.812633\tvalid_0's binary_logloss: 0.455789\n",
      "[75]\tvalid_0's auc: 0.812757\tvalid_0's binary_logloss: 0.455173\n",
      "[76]\tvalid_0's auc: 0.812923\tvalid_0's binary_logloss: 0.454533\n",
      "[77]\tvalid_0's auc: 0.81295\tvalid_0's binary_logloss: 0.45392\n",
      "[78]\tvalid_0's auc: 0.813073\tvalid_0's binary_logloss: 0.453517\n",
      "[79]\tvalid_0's auc: 0.813202\tvalid_0's binary_logloss: 0.452932\n",
      "[80]\tvalid_0's auc: 0.813611\tvalid_0's binary_logloss: 0.452285\n",
      "[81]\tvalid_0's auc: 0.813769\tvalid_0's binary_logloss: 0.45191\n",
      "[82]\tvalid_0's auc: 0.814468\tvalid_0's binary_logloss: 0.451455\n",
      "[83]\tvalid_0's auc: 0.814656\tvalid_0's binary_logloss: 0.450885\n",
      "[84]\tvalid_0's auc: 0.814755\tvalid_0's binary_logloss: 0.450308\n",
      "[85]\tvalid_0's auc: 0.814824\tvalid_0's binary_logloss: 0.449739\n",
      "[86]\tvalid_0's auc: 0.81499\tvalid_0's binary_logloss: 0.449348\n",
      "[87]\tvalid_0's auc: 0.815232\tvalid_0's binary_logloss: 0.448759\n",
      "[88]\tvalid_0's auc: 0.815452\tvalid_0's binary_logloss: 0.44823\n",
      "[89]\tvalid_0's auc: 0.815593\tvalid_0's binary_logloss: 0.447861\n",
      "[90]\tvalid_0's auc: 0.815591\tvalid_0's binary_logloss: 0.447323\n",
      "[91]\tvalid_0's auc: 0.815672\tvalid_0's binary_logloss: 0.446796\n",
      "[92]\tvalid_0's auc: 0.815875\tvalid_0's binary_logloss: 0.446472\n",
      "[93]\tvalid_0's auc: 0.815984\tvalid_0's binary_logloss: 0.445961\n",
      "[94]\tvalid_0's auc: 0.816026\tvalid_0's binary_logloss: 0.445439\n",
      "[95]\tvalid_0's auc: 0.816172\tvalid_0's binary_logloss: 0.444909\n",
      "[96]\tvalid_0's auc: 0.816321\tvalid_0's binary_logloss: 0.444413\n",
      "[97]\tvalid_0's auc: 0.816751\tvalid_0's binary_logloss: 0.44405\n",
      "[98]\tvalid_0's auc: 0.817226\tvalid_0's binary_logloss: 0.443626\n",
      "[99]\tvalid_0's auc: 0.817286\tvalid_0's binary_logloss: 0.443136\n",
      "[100]\tvalid_0's auc: 0.817391\tvalid_0's binary_logloss: 0.442854\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's auc: 0.817391\tvalid_0's binary_logloss: 0.442854\n",
      "[1]\tvalid_0's auc: 0.771584\tvalid_0's binary_logloss: 0.527139\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's auc: 0.775446\tvalid_0's binary_logloss: 0.525462\n",
      "[3]\tvalid_0's auc: 0.790092\tvalid_0's binary_logloss: 0.524461\n",
      "[4]\tvalid_0's auc: 0.791432\tvalid_0's binary_logloss: 0.523322\n",
      "[5]\tvalid_0's auc: 0.797482\tvalid_0's binary_logloss: 0.521614\n",
      "[6]\tvalid_0's auc: 0.79893\tvalid_0's binary_logloss: 0.520007\n",
      "[7]\tvalid_0's auc: 0.800753\tvalid_0's binary_logloss: 0.5187\n",
      "[8]\tvalid_0's auc: 0.802197\tvalid_0's binary_logloss: 0.517125\n",
      "[9]\tvalid_0's auc: 0.802828\tvalid_0's binary_logloss: 0.516269\n",
      "[10]\tvalid_0's auc: 0.803496\tvalid_0's binary_logloss: 0.51474\n",
      "[11]\tvalid_0's auc: 0.804972\tvalid_0's binary_logloss: 0.513321\n",
      "[12]\tvalid_0's auc: 0.804995\tvalid_0's binary_logloss: 0.512334\n",
      "[13]\tvalid_0's auc: 0.80525\tvalid_0's binary_logloss: 0.51151\n",
      "[14]\tvalid_0's auc: 0.805026\tvalid_0's binary_logloss: 0.510149\n",
      "[15]\tvalid_0's auc: 0.805622\tvalid_0's binary_logloss: 0.508708\n",
      "[16]\tvalid_0's auc: 0.806974\tvalid_0's binary_logloss: 0.507384\n",
      "[17]\tvalid_0's auc: 0.807045\tvalid_0's binary_logloss: 0.506017\n",
      "[18]\tvalid_0's auc: 0.807265\tvalid_0's binary_logloss: 0.504853\n",
      "[19]\tvalid_0's auc: 0.807126\tvalid_0's binary_logloss: 0.503972\n",
      "[20]\tvalid_0's auc: 0.806948\tvalid_0's binary_logloss: 0.502693\n",
      "[21]\tvalid_0's auc: 0.807315\tvalid_0's binary_logloss: 0.502166\n",
      "[22]\tvalid_0's auc: 0.808067\tvalid_0's binary_logloss: 0.500948\n",
      "[23]\tvalid_0's auc: 0.808226\tvalid_0's binary_logloss: 0.49987\n",
      "[24]\tvalid_0's auc: 0.808268\tvalid_0's binary_logloss: 0.498623\n",
      "[25]\tvalid_0's auc: 0.808569\tvalid_0's binary_logloss: 0.497389\n",
      "[26]\tvalid_0's auc: 0.809069\tvalid_0's binary_logloss: 0.49624\n",
      "[27]\tvalid_0's auc: 0.809312\tvalid_0's binary_logloss: 0.495095\n",
      "[28]\tvalid_0's auc: 0.809549\tvalid_0's binary_logloss: 0.494012\n",
      "[29]\tvalid_0's auc: 0.809944\tvalid_0's binary_logloss: 0.492834\n",
      "[30]\tvalid_0's auc: 0.810047\tvalid_0's binary_logloss: 0.491735\n",
      "[31]\tvalid_0's auc: 0.810086\tvalid_0's binary_logloss: 0.490633\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[32]\tvalid_0's auc: 0.810566\tvalid_0's binary_logloss: 0.489595\n",
      "[33]\tvalid_0's auc: 0.810539\tvalid_0's binary_logloss: 0.488536\n",
      "[34]\tvalid_0's auc: 0.810529\tvalid_0's binary_logloss: 0.487489\n",
      "[35]\tvalid_0's auc: 0.810932\tvalid_0's binary_logloss: 0.486775\n",
      "[36]\tvalid_0's auc: 0.810769\tvalid_0's binary_logloss: 0.48577\n",
      "[37]\tvalid_0's auc: 0.811363\tvalid_0's binary_logloss: 0.485123\n",
      "[38]\tvalid_0's auc: 0.811801\tvalid_0's binary_logloss: 0.484413\n",
      "[39]\tvalid_0's auc: 0.811987\tvalid_0's binary_logloss: 0.483371\n",
      "[40]\tvalid_0's auc: 0.812268\tvalid_0's binary_logloss: 0.482407\n",
      "[41]\tvalid_0's auc: 0.813297\tvalid_0's binary_logloss: 0.481742\n",
      "[42]\tvalid_0's auc: 0.813453\tvalid_0's binary_logloss: 0.481108\n",
      "[43]\tvalid_0's auc: 0.813603\tvalid_0's binary_logloss: 0.480163\n",
      "[44]\tvalid_0's auc: 0.813654\tvalid_0's binary_logloss: 0.479239\n",
      "[45]\tvalid_0's auc: 0.814267\tvalid_0's binary_logloss: 0.478299\n",
      "[46]\tvalid_0's auc: 0.81455\tvalid_0's binary_logloss: 0.477678\n",
      "[47]\tvalid_0's auc: 0.81452\tvalid_0's binary_logloss: 0.476766\n",
      "[48]\tvalid_0's auc: 0.814925\tvalid_0's binary_logloss: 0.475815\n",
      "[49]\tvalid_0's auc: 0.814907\tvalid_0's binary_logloss: 0.47503\n",
      "[50]\tvalid_0's auc: 0.815278\tvalid_0's binary_logloss: 0.474588\n",
      "[51]\tvalid_0's auc: 0.815535\tvalid_0's binary_logloss: 0.474171\n",
      "[52]\tvalid_0's auc: 0.815685\tvalid_0's binary_logloss: 0.473335\n",
      "[53]\tvalid_0's auc: 0.815787\tvalid_0's binary_logloss: 0.472509\n",
      "[54]\tvalid_0's auc: 0.815827\tvalid_0's binary_logloss: 0.471686\n",
      "[55]\tvalid_0's auc: 0.815871\tvalid_0's binary_logloss: 0.470838\n",
      "[56]\tvalid_0's auc: 0.816238\tvalid_0's binary_logloss: 0.470285\n",
      "[57]\tvalid_0's auc: 0.816269\tvalid_0's binary_logloss: 0.469495\n",
      "[58]\tvalid_0's auc: 0.816528\tvalid_0's binary_logloss: 0.468654\n",
      "[59]\tvalid_0's auc: 0.816706\tvalid_0's binary_logloss: 0.468122\n",
      "[60]\tvalid_0's auc: 0.816821\tvalid_0's binary_logloss: 0.467352\n",
      "[61]\tvalid_0's auc: 0.816759\tvalid_0's binary_logloss: 0.466622\n",
      "[62]\tvalid_0's auc: 0.81682\tvalid_0's binary_logloss: 0.465867\n",
      "[63]\tvalid_0's auc: 0.817251\tvalid_0's binary_logloss: 0.465112\n",
      "[64]\tvalid_0's auc: 0.817476\tvalid_0's binary_logloss: 0.464589\n",
      "[65]\tvalid_0's auc: 0.817613\tvalid_0's binary_logloss: 0.463831\n",
      "[66]\tvalid_0's auc: 0.817648\tvalid_0's binary_logloss: 0.463098\n",
      "[67]\tvalid_0's auc: 0.817719\tvalid_0's binary_logloss: 0.462414\n",
      "[68]\tvalid_0's auc: 0.817814\tvalid_0's binary_logloss: 0.461727\n",
      "[69]\tvalid_0's auc: 0.817973\tvalid_0's binary_logloss: 0.461329\n",
      "[70]\tvalid_0's auc: 0.818108\tvalid_0's binary_logloss: 0.460674\n",
      "[71]\tvalid_0's auc: 0.818347\tvalid_0's binary_logloss: 0.460222\n",
      "[72]\tvalid_0's auc: 0.818456\tvalid_0's binary_logloss: 0.45977\n",
      "[73]\tvalid_0's auc: 0.818727\tvalid_0's binary_logloss: 0.459157\n",
      "[74]\tvalid_0's auc: 0.818988\tvalid_0's binary_logloss: 0.458437\n",
      "[75]\tvalid_0's auc: 0.819144\tvalid_0's binary_logloss: 0.457808\n",
      "[76]\tvalid_0's auc: 0.819259\tvalid_0's binary_logloss: 0.457159\n",
      "[77]\tvalid_0's auc: 0.819343\tvalid_0's binary_logloss: 0.456512\n",
      "[78]\tvalid_0's auc: 0.81954\tvalid_0's binary_logloss: 0.456045\n",
      "[79]\tvalid_0's auc: 0.819687\tvalid_0's binary_logloss: 0.455416\n",
      "[80]\tvalid_0's auc: 0.819958\tvalid_0's binary_logloss: 0.454765\n",
      "[81]\tvalid_0's auc: 0.820115\tvalid_0's binary_logloss: 0.45436\n",
      "[82]\tvalid_0's auc: 0.820536\tvalid_0's binary_logloss: 0.453965\n",
      "[83]\tvalid_0's auc: 0.820649\tvalid_0's binary_logloss: 0.453383\n",
      "[84]\tvalid_0's auc: 0.820663\tvalid_0's binary_logloss: 0.452804\n",
      "[85]\tvalid_0's auc: 0.820809\tvalid_0's binary_logloss: 0.452167\n",
      "[86]\tvalid_0's auc: 0.821024\tvalid_0's binary_logloss: 0.451735\n",
      "[87]\tvalid_0's auc: 0.821124\tvalid_0's binary_logloss: 0.451167\n",
      "[88]\tvalid_0's auc: 0.821243\tvalid_0's binary_logloss: 0.45061\n",
      "[89]\tvalid_0's auc: 0.821404\tvalid_0's binary_logloss: 0.450215\n",
      "[90]\tvalid_0's auc: 0.821488\tvalid_0's binary_logloss: 0.449656\n",
      "[91]\tvalid_0's auc: 0.821538\tvalid_0's binary_logloss: 0.449107\n",
      "[92]\tvalid_0's auc: 0.82172\tvalid_0's binary_logloss: 0.448752\n",
      "[93]\tvalid_0's auc: 0.821809\tvalid_0's binary_logloss: 0.448188\n",
      "[94]\tvalid_0's auc: 0.82184\tvalid_0's binary_logloss: 0.447659\n",
      "[95]\tvalid_0's auc: 0.821971\tvalid_0's binary_logloss: 0.447108\n",
      "[96]\tvalid_0's auc: 0.822086\tvalid_0's binary_logloss: 0.446596\n",
      "[97]\tvalid_0's auc: 0.82247\tvalid_0's binary_logloss: 0.446244\n",
      "[98]\tvalid_0's auc: 0.822951\tvalid_0's binary_logloss: 0.445812\n",
      "[99]\tvalid_0's auc: 0.822991\tvalid_0's binary_logloss: 0.445329\n",
      "[100]\tvalid_0's auc: 0.823174\tvalid_0's binary_logloss: 0.445037\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's auc: 0.823174\tvalid_0's binary_logloss: 0.445037\n",
      "[1]\tvalid_0's auc: 0.769525\tvalid_0's binary_logloss: 0.526256\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's auc: 0.775857\tvalid_0's binary_logloss: 0.524594\n",
      "[3]\tvalid_0's auc: 0.785307\tvalid_0's binary_logloss: 0.523606\n",
      "[4]\tvalid_0's auc: 0.786356\tvalid_0's binary_logloss: 0.522495\n",
      "[5]\tvalid_0's auc: 0.793385\tvalid_0's binary_logloss: 0.520812\n",
      "[6]\tvalid_0's auc: 0.794014\tvalid_0's binary_logloss: 0.519253\n",
      "[7]\tvalid_0's auc: 0.795454\tvalid_0's binary_logloss: 0.517961\n",
      "[8]\tvalid_0's auc: 0.79807\tvalid_0's binary_logloss: 0.516363\n",
      "[9]\tvalid_0's auc: 0.798756\tvalid_0's binary_logloss: 0.51548\n",
      "[10]\tvalid_0's auc: 0.798314\tvalid_0's binary_logloss: 0.514021\n",
      "[11]\tvalid_0's auc: 0.799343\tvalid_0's binary_logloss: 0.512678\n",
      "[12]\tvalid_0's auc: 0.799573\tvalid_0's binary_logloss: 0.511708\n",
      "[13]\tvalid_0's auc: 0.799563\tvalid_0's binary_logloss: 0.510892\n",
      "[14]\tvalid_0's auc: 0.800333\tvalid_0's binary_logloss: 0.509532\n",
      "[15]\tvalid_0's auc: 0.800672\tvalid_0's binary_logloss: 0.508117\n",
      "[16]\tvalid_0's auc: 0.801953\tvalid_0's binary_logloss: 0.506866\n",
      "[17]\tvalid_0's auc: 0.802078\tvalid_0's binary_logloss: 0.5055\n",
      "[18]\tvalid_0's auc: 0.802449\tvalid_0's binary_logloss: 0.504358\n",
      "[19]\tvalid_0's auc: 0.802329\tvalid_0's binary_logloss: 0.503503\n",
      "[20]\tvalid_0's auc: 0.802437\tvalid_0's binary_logloss: 0.502233\n",
      "[21]\tvalid_0's auc: 0.802653\tvalid_0's binary_logloss: 0.50174\n",
      "[22]\tvalid_0's auc: 0.803753\tvalid_0's binary_logloss: 0.50056\n",
      "[23]\tvalid_0's auc: 0.803956\tvalid_0's binary_logloss: 0.499496\n",
      "[24]\tvalid_0's auc: 0.804231\tvalid_0's binary_logloss: 0.498283\n",
      "[25]\tvalid_0's auc: 0.804554\tvalid_0's binary_logloss: 0.497059\n",
      "[26]\tvalid_0's auc: 0.805133\tvalid_0's binary_logloss: 0.495963\n",
      "[27]\tvalid_0's auc: 0.805333\tvalid_0's binary_logloss: 0.494842\n",
      "[28]\tvalid_0's auc: 0.805644\tvalid_0's binary_logloss: 0.493771\n",
      "[29]\tvalid_0's auc: 0.806029\tvalid_0's binary_logloss: 0.492598\n",
      "[30]\tvalid_0's auc: 0.806321\tvalid_0's binary_logloss: 0.491474\n",
      "[31]\tvalid_0's auc: 0.806201\tvalid_0's binary_logloss: 0.490419\n",
      "[32]\tvalid_0's auc: 0.806671\tvalid_0's binary_logloss: 0.489393\n",
      "[33]\tvalid_0's auc: 0.806899\tvalid_0's binary_logloss: 0.488331\n",
      "[34]\tvalid_0's auc: 0.807105\tvalid_0's binary_logloss: 0.487277\n",
      "[35]\tvalid_0's auc: 0.807257\tvalid_0's binary_logloss: 0.486592\n",
      "[36]\tvalid_0's auc: 0.80729\tvalid_0's binary_logloss: 0.485607\n",
      "[37]\tvalid_0's auc: 0.807752\tvalid_0's binary_logloss: 0.484951\n",
      "[38]\tvalid_0's auc: 0.808191\tvalid_0's binary_logloss: 0.484269\n",
      "[39]\tvalid_0's auc: 0.808417\tvalid_0's binary_logloss: 0.483242\n",
      "[40]\tvalid_0's auc: 0.808761\tvalid_0's binary_logloss: 0.482291\n",
      "[41]\tvalid_0's auc: 0.80965\tvalid_0's binary_logloss: 0.48164\n",
      "[42]\tvalid_0's auc: 0.810065\tvalid_0's binary_logloss: 0.480962\n",
      "[43]\tvalid_0's auc: 0.810209\tvalid_0's binary_logloss: 0.479995\n",
      "[44]\tvalid_0's auc: 0.810091\tvalid_0's binary_logloss: 0.479077\n",
      "[45]\tvalid_0's auc: 0.810573\tvalid_0's binary_logloss: 0.478185\n",
      "[46]\tvalid_0's auc: 0.810924\tvalid_0's binary_logloss: 0.477558\n",
      "[47]\tvalid_0's auc: 0.810951\tvalid_0's binary_logloss: 0.476662\n",
      "[48]\tvalid_0's auc: 0.811101\tvalid_0's binary_logloss: 0.475745\n",
      "[49]\tvalid_0's auc: 0.811269\tvalid_0's binary_logloss: 0.474951\n",
      "[50]\tvalid_0's auc: 0.81173\tvalid_0's binary_logloss: 0.474514\n",
      "[51]\tvalid_0's auc: 0.811937\tvalid_0's binary_logloss: 0.474114\n",
      "[52]\tvalid_0's auc: 0.812136\tvalid_0's binary_logloss: 0.473297\n",
      "[53]\tvalid_0's auc: 0.812249\tvalid_0's binary_logloss: 0.472497\n",
      "[54]\tvalid_0's auc: 0.812121\tvalid_0's binary_logloss: 0.471696\n",
      "[55]\tvalid_0's auc: 0.812164\tvalid_0's binary_logloss: 0.470905\n",
      "[56]\tvalid_0's auc: 0.812462\tvalid_0's binary_logloss: 0.470384\n",
      "[57]\tvalid_0's auc: 0.812613\tvalid_0's binary_logloss: 0.4696\n",
      "[58]\tvalid_0's auc: 0.812615\tvalid_0's binary_logloss: 0.468778\n",
      "[59]\tvalid_0's auc: 0.812842\tvalid_0's binary_logloss: 0.468211\n",
      "[60]\tvalid_0's auc: 0.81312\tvalid_0's binary_logloss: 0.467385\n",
      "[61]\tvalid_0's auc: 0.813039\tvalid_0's binary_logloss: 0.466632\n",
      "[62]\tvalid_0's auc: 0.812942\tvalid_0's binary_logloss: 0.465933\n",
      "[63]\tvalid_0's auc: 0.813274\tvalid_0's binary_logloss: 0.465214\n",
      "[64]\tvalid_0's auc: 0.813572\tvalid_0's binary_logloss: 0.464692\n",
      "[65]\tvalid_0's auc: 0.813594\tvalid_0's binary_logloss: 0.463925\n",
      "[66]\tvalid_0's auc: 0.813719\tvalid_0's binary_logloss: 0.463177\n",
      "[67]\tvalid_0's auc: 0.814011\tvalid_0's binary_logloss: 0.462513\n",
      "[68]\tvalid_0's auc: 0.813989\tvalid_0's binary_logloss: 0.461843\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[69]\tvalid_0's auc: 0.814218\tvalid_0's binary_logloss: 0.461443\n",
      "[70]\tvalid_0's auc: 0.814334\tvalid_0's binary_logloss: 0.460775\n",
      "[71]\tvalid_0's auc: 0.814493\tvalid_0's binary_logloss: 0.460332\n",
      "[72]\tvalid_0's auc: 0.814663\tvalid_0's binary_logloss: 0.459867\n",
      "[73]\tvalid_0's auc: 0.814856\tvalid_0's binary_logloss: 0.459266\n",
      "[74]\tvalid_0's auc: 0.815017\tvalid_0's binary_logloss: 0.458585\n",
      "[75]\tvalid_0's auc: 0.815186\tvalid_0's binary_logloss: 0.457958\n",
      "[76]\tvalid_0's auc: 0.815374\tvalid_0's binary_logloss: 0.457316\n",
      "[77]\tvalid_0's auc: 0.81554\tvalid_0's binary_logloss: 0.45665\n",
      "[78]\tvalid_0's auc: 0.81569\tvalid_0's binary_logloss: 0.456217\n",
      "[79]\tvalid_0's auc: 0.815861\tvalid_0's binary_logloss: 0.455615\n",
      "[80]\tvalid_0's auc: 0.816443\tvalid_0's binary_logloss: 0.454895\n",
      "[81]\tvalid_0's auc: 0.816659\tvalid_0's binary_logloss: 0.454503\n",
      "[82]\tvalid_0's auc: 0.817017\tvalid_0's binary_logloss: 0.454149\n",
      "[83]\tvalid_0's auc: 0.817162\tvalid_0's binary_logloss: 0.453578\n",
      "[84]\tvalid_0's auc: 0.817274\tvalid_0's binary_logloss: 0.452984\n",
      "[85]\tvalid_0's auc: 0.817283\tvalid_0's binary_logloss: 0.452416\n",
      "[86]\tvalid_0's auc: 0.817339\tvalid_0's binary_logloss: 0.452022\n",
      "[87]\tvalid_0's auc: 0.817494\tvalid_0's binary_logloss: 0.45146\n",
      "[88]\tvalid_0's auc: 0.817594\tvalid_0's binary_logloss: 0.450926\n",
      "[89]\tvalid_0's auc: 0.817771\tvalid_0's binary_logloss: 0.450553\n",
      "[90]\tvalid_0's auc: 0.81789\tvalid_0's binary_logloss: 0.449985\n",
      "[91]\tvalid_0's auc: 0.817931\tvalid_0's binary_logloss: 0.449439\n",
      "[92]\tvalid_0's auc: 0.818138\tvalid_0's binary_logloss: 0.449094\n",
      "[93]\tvalid_0's auc: 0.818334\tvalid_0's binary_logloss: 0.448527\n",
      "[94]\tvalid_0's auc: 0.818426\tvalid_0's binary_logloss: 0.447989\n",
      "[95]\tvalid_0's auc: 0.818676\tvalid_0's binary_logloss: 0.447407\n",
      "[96]\tvalid_0's auc: 0.818852\tvalid_0's binary_logloss: 0.446884\n",
      "[97]\tvalid_0's auc: 0.81945\tvalid_0's binary_logloss: 0.446455\n",
      "[98]\tvalid_0's auc: 0.819861\tvalid_0's binary_logloss: 0.446045\n",
      "[99]\tvalid_0's auc: 0.819943\tvalid_0's binary_logloss: 0.445543\n",
      "[100]\tvalid_0's auc: 0.820076\tvalid_0's binary_logloss: 0.445258\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's auc: 0.820076\tvalid_0's binary_logloss: 0.445258\n",
      "[1]\tvalid_0's auc: 0.770032\tvalid_0's binary_logloss: 0.527241\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's auc: 0.779881\tvalid_0's binary_logloss: 0.525545\n",
      "[3]\tvalid_0's auc: 0.791308\tvalid_0's binary_logloss: 0.524508\n",
      "[4]\tvalid_0's auc: 0.790788\tvalid_0's binary_logloss: 0.52341\n",
      "[5]\tvalid_0's auc: 0.795645\tvalid_0's binary_logloss: 0.521753\n",
      "[6]\tvalid_0's auc: 0.797745\tvalid_0's binary_logloss: 0.520131\n",
      "[7]\tvalid_0's auc: 0.79931\tvalid_0's binary_logloss: 0.518872\n",
      "[8]\tvalid_0's auc: 0.800014\tvalid_0's binary_logloss: 0.517353\n",
      "[9]\tvalid_0's auc: 0.800549\tvalid_0's binary_logloss: 0.516487\n",
      "[10]\tvalid_0's auc: 0.800261\tvalid_0's binary_logloss: 0.515039\n",
      "[11]\tvalid_0's auc: 0.801261\tvalid_0's binary_logloss: 0.513695\n",
      "[12]\tvalid_0's auc: 0.801062\tvalid_0's binary_logloss: 0.512735\n",
      "[13]\tvalid_0's auc: 0.801155\tvalid_0's binary_logloss: 0.51192\n",
      "[14]\tvalid_0's auc: 0.801315\tvalid_0's binary_logloss: 0.510559\n",
      "[15]\tvalid_0's auc: 0.80185\tvalid_0's binary_logloss: 0.509147\n",
      "[16]\tvalid_0's auc: 0.803029\tvalid_0's binary_logloss: 0.507914\n",
      "[17]\tvalid_0's auc: 0.803035\tvalid_0's binary_logloss: 0.506583\n",
      "[18]\tvalid_0's auc: 0.803433\tvalid_0's binary_logloss: 0.505441\n",
      "[19]\tvalid_0's auc: 0.803717\tvalid_0's binary_logloss: 0.504599\n",
      "[20]\tvalid_0's auc: 0.803819\tvalid_0's binary_logloss: 0.503327\n",
      "[21]\tvalid_0's auc: 0.803923\tvalid_0's binary_logloss: 0.502782\n",
      "[22]\tvalid_0's auc: 0.804939\tvalid_0's binary_logloss: 0.501596\n",
      "[23]\tvalid_0's auc: 0.804707\tvalid_0's binary_logloss: 0.500572\n",
      "[24]\tvalid_0's auc: 0.804632\tvalid_0's binary_logloss: 0.499367\n",
      "[25]\tvalid_0's auc: 0.804756\tvalid_0's binary_logloss: 0.498161\n",
      "[26]\tvalid_0's auc: 0.805067\tvalid_0's binary_logloss: 0.497061\n",
      "[27]\tvalid_0's auc: 0.805119\tvalid_0's binary_logloss: 0.495933\n",
      "[28]\tvalid_0's auc: 0.805304\tvalid_0's binary_logloss: 0.494849\n",
      "[29]\tvalid_0's auc: 0.805688\tvalid_0's binary_logloss: 0.493677\n",
      "[30]\tvalid_0's auc: 0.805822\tvalid_0's binary_logloss: 0.492594\n",
      "[31]\tvalid_0's auc: 0.805869\tvalid_0's binary_logloss: 0.49152\n",
      "[32]\tvalid_0's auc: 0.807267\tvalid_0's binary_logloss: 0.490435\n",
      "[33]\tvalid_0's auc: 0.807301\tvalid_0's binary_logloss: 0.489392\n",
      "[34]\tvalid_0's auc: 0.80736\tvalid_0's binary_logloss: 0.488325\n",
      "[35]\tvalid_0's auc: 0.807706\tvalid_0's binary_logloss: 0.487654\n",
      "[36]\tvalid_0's auc: 0.807758\tvalid_0's binary_logloss: 0.486651\n",
      "[37]\tvalid_0's auc: 0.808051\tvalid_0's binary_logloss: 0.486012\n",
      "[38]\tvalid_0's auc: 0.808429\tvalid_0's binary_logloss: 0.485355\n",
      "[39]\tvalid_0's auc: 0.808663\tvalid_0's binary_logloss: 0.484327\n",
      "[40]\tvalid_0's auc: 0.809007\tvalid_0's binary_logloss: 0.483386\n",
      "[41]\tvalid_0's auc: 0.809781\tvalid_0's binary_logloss: 0.482745\n",
      "[42]\tvalid_0's auc: 0.810071\tvalid_0's binary_logloss: 0.482124\n",
      "[43]\tvalid_0's auc: 0.810383\tvalid_0's binary_logloss: 0.481154\n",
      "[44]\tvalid_0's auc: 0.810446\tvalid_0's binary_logloss: 0.480243\n",
      "[45]\tvalid_0's auc: 0.811148\tvalid_0's binary_logloss: 0.479261\n",
      "[46]\tvalid_0's auc: 0.811245\tvalid_0's binary_logloss: 0.478687\n",
      "[47]\tvalid_0's auc: 0.811214\tvalid_0's binary_logloss: 0.477812\n",
      "[48]\tvalid_0's auc: 0.811408\tvalid_0's binary_logloss: 0.47689\n",
      "[49]\tvalid_0's auc: 0.811486\tvalid_0's binary_logloss: 0.476132\n",
      "[50]\tvalid_0's auc: 0.811806\tvalid_0's binary_logloss: 0.475718\n",
      "[51]\tvalid_0's auc: 0.812017\tvalid_0's binary_logloss: 0.475342\n",
      "[52]\tvalid_0's auc: 0.812255\tvalid_0's binary_logloss: 0.474505\n",
      "[53]\tvalid_0's auc: 0.812249\tvalid_0's binary_logloss: 0.473707\n",
      "[54]\tvalid_0's auc: 0.812235\tvalid_0's binary_logloss: 0.47289\n",
      "[55]\tvalid_0's auc: 0.812233\tvalid_0's binary_logloss: 0.472091\n",
      "[56]\tvalid_0's auc: 0.812492\tvalid_0's binary_logloss: 0.471563\n",
      "[57]\tvalid_0's auc: 0.812579\tvalid_0's binary_logloss: 0.47077\n",
      "[58]\tvalid_0's auc: 0.812598\tvalid_0's binary_logloss: 0.469992\n",
      "[59]\tvalid_0's auc: 0.812885\tvalid_0's binary_logloss: 0.469458\n",
      "[60]\tvalid_0's auc: 0.812995\tvalid_0's binary_logloss: 0.468676\n",
      "[61]\tvalid_0's auc: 0.812961\tvalid_0's binary_logloss: 0.467939\n",
      "[62]\tvalid_0's auc: 0.812919\tvalid_0's binary_logloss: 0.467232\n",
      "[63]\tvalid_0's auc: 0.813291\tvalid_0's binary_logloss: 0.466491\n",
      "[64]\tvalid_0's auc: 0.813702\tvalid_0's binary_logloss: 0.465945\n",
      "[65]\tvalid_0's auc: 0.813803\tvalid_0's binary_logloss: 0.465197\n",
      "[66]\tvalid_0's auc: 0.813851\tvalid_0's binary_logloss: 0.4645\n",
      "[67]\tvalid_0's auc: 0.814011\tvalid_0's binary_logloss: 0.463814\n",
      "[68]\tvalid_0's auc: 0.814027\tvalid_0's binary_logloss: 0.463113\n",
      "[69]\tvalid_0's auc: 0.814138\tvalid_0's binary_logloss: 0.462727\n",
      "[70]\tvalid_0's auc: 0.814365\tvalid_0's binary_logloss: 0.462077\n",
      "[71]\tvalid_0's auc: 0.814432\tvalid_0's binary_logloss: 0.461655\n",
      "[72]\tvalid_0's auc: 0.8146\tvalid_0's binary_logloss: 0.461194\n",
      "[73]\tvalid_0's auc: 0.815324\tvalid_0's binary_logloss: 0.460477\n",
      "[74]\tvalid_0's auc: 0.815411\tvalid_0's binary_logloss: 0.459805\n",
      "[75]\tvalid_0's auc: 0.815548\tvalid_0's binary_logloss: 0.459189\n",
      "[76]\tvalid_0's auc: 0.815625\tvalid_0's binary_logloss: 0.458525\n",
      "[77]\tvalid_0's auc: 0.81562\tvalid_0's binary_logloss: 0.457905\n",
      "[78]\tvalid_0's auc: 0.815786\tvalid_0's binary_logloss: 0.45747\n",
      "[79]\tvalid_0's auc: 0.815834\tvalid_0's binary_logloss: 0.456884\n",
      "[80]\tvalid_0's auc: 0.816475\tvalid_0's binary_logloss: 0.45617\n",
      "[81]\tvalid_0's auc: 0.816677\tvalid_0's binary_logloss: 0.455787\n",
      "[82]\tvalid_0's auc: 0.817255\tvalid_0's binary_logloss: 0.455358\n",
      "[83]\tvalid_0's auc: 0.817383\tvalid_0's binary_logloss: 0.454775\n",
      "[84]\tvalid_0's auc: 0.817509\tvalid_0's binary_logloss: 0.454176\n",
      "[85]\tvalid_0's auc: 0.817572\tvalid_0's binary_logloss: 0.453609\n",
      "[86]\tvalid_0's auc: 0.817721\tvalid_0's binary_logloss: 0.453213\n",
      "[87]\tvalid_0's auc: 0.817992\tvalid_0's binary_logloss: 0.452586\n",
      "[88]\tvalid_0's auc: 0.81808\tvalid_0's binary_logloss: 0.45204\n",
      "[89]\tvalid_0's auc: 0.818202\tvalid_0's binary_logloss: 0.451643\n",
      "[90]\tvalid_0's auc: 0.818336\tvalid_0's binary_logloss: 0.451081\n",
      "[91]\tvalid_0's auc: 0.818347\tvalid_0's binary_logloss: 0.450531\n",
      "[92]\tvalid_0's auc: 0.818558\tvalid_0's binary_logloss: 0.450179\n",
      "[93]\tvalid_0's auc: 0.818743\tvalid_0's binary_logloss: 0.449647\n",
      "[94]\tvalid_0's auc: 0.818789\tvalid_0's binary_logloss: 0.449133\n",
      "[95]\tvalid_0's auc: 0.818849\tvalid_0's binary_logloss: 0.44862\n",
      "[96]\tvalid_0's auc: 0.81913\tvalid_0's binary_logloss: 0.448072\n",
      "[97]\tvalid_0's auc: 0.819526\tvalid_0's binary_logloss: 0.447713\n",
      "[98]\tvalid_0's auc: 0.819971\tvalid_0's binary_logloss: 0.447296\n",
      "[99]\tvalid_0's auc: 0.819972\tvalid_0's binary_logloss: 0.446814\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100]\tvalid_0's auc: 0.820086\tvalid_0's binary_logloss: 0.446533\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's auc: 0.820086\tvalid_0's binary_logloss: 0.446533\n",
      "[1]\tvalid_0's auc: 0.768646\tvalid_0's binary_logloss: 0.527167\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[2]\tvalid_0's auc: 0.779902\tvalid_0's binary_logloss: 0.525481\n",
      "[3]\tvalid_0's auc: 0.789868\tvalid_0's binary_logloss: 0.524485\n",
      "[4]\tvalid_0's auc: 0.791895\tvalid_0's binary_logloss: 0.523382\n",
      "[5]\tvalid_0's auc: 0.795453\tvalid_0's binary_logloss: 0.521759\n",
      "[6]\tvalid_0's auc: 0.796672\tvalid_0's binary_logloss: 0.520166\n",
      "[7]\tvalid_0's auc: 0.798023\tvalid_0's binary_logloss: 0.518857\n",
      "[8]\tvalid_0's auc: 0.799331\tvalid_0's binary_logloss: 0.517297\n",
      "[9]\tvalid_0's auc: 0.800181\tvalid_0's binary_logloss: 0.516416\n",
      "[10]\tvalid_0's auc: 0.800373\tvalid_0's binary_logloss: 0.514967\n",
      "[11]\tvalid_0's auc: 0.801087\tvalid_0's binary_logloss: 0.513631\n",
      "[12]\tvalid_0's auc: 0.801122\tvalid_0's binary_logloss: 0.512658\n",
      "[13]\tvalid_0's auc: 0.801043\tvalid_0's binary_logloss: 0.511833\n",
      "[14]\tvalid_0's auc: 0.801238\tvalid_0's binary_logloss: 0.510461\n",
      "[15]\tvalid_0's auc: 0.801847\tvalid_0's binary_logloss: 0.509034\n",
      "[16]\tvalid_0's auc: 0.803139\tvalid_0's binary_logloss: 0.507759\n",
      "[17]\tvalid_0's auc: 0.803577\tvalid_0's binary_logloss: 0.506361\n",
      "[18]\tvalid_0's auc: 0.803834\tvalid_0's binary_logloss: 0.505229\n",
      "[19]\tvalid_0's auc: 0.803943\tvalid_0's binary_logloss: 0.504371\n",
      "[20]\tvalid_0's auc: 0.80415\tvalid_0's binary_logloss: 0.503102\n",
      "[21]\tvalid_0's auc: 0.804446\tvalid_0's binary_logloss: 0.502564\n",
      "[22]\tvalid_0's auc: 0.805163\tvalid_0's binary_logloss: 0.501396\n",
      "[23]\tvalid_0's auc: 0.805323\tvalid_0's binary_logloss: 0.500327\n",
      "[24]\tvalid_0's auc: 0.805314\tvalid_0's binary_logloss: 0.499123\n",
      "[25]\tvalid_0's auc: 0.80535\tvalid_0's binary_logloss: 0.497927\n",
      "[26]\tvalid_0's auc: 0.805864\tvalid_0's binary_logloss: 0.496834\n",
      "[27]\tvalid_0's auc: 0.805919\tvalid_0's binary_logloss: 0.495667\n",
      "[28]\tvalid_0's auc: 0.806272\tvalid_0's binary_logloss: 0.494606\n",
      "[29]\tvalid_0's auc: 0.806599\tvalid_0's binary_logloss: 0.49343\n",
      "[30]\tvalid_0's auc: 0.806932\tvalid_0's binary_logloss: 0.492303\n",
      "[31]\tvalid_0's auc: 0.806656\tvalid_0's binary_logloss: 0.491249\n",
      "[32]\tvalid_0's auc: 0.807436\tvalid_0's binary_logloss: 0.490188\n",
      "[33]\tvalid_0's auc: 0.807629\tvalid_0's binary_logloss: 0.489117\n",
      "[34]\tvalid_0's auc: 0.807501\tvalid_0's binary_logloss: 0.48808\n",
      "[35]\tvalid_0's auc: 0.807885\tvalid_0's binary_logloss: 0.487383\n",
      "[36]\tvalid_0's auc: 0.807921\tvalid_0's binary_logloss: 0.48636\n",
      "[37]\tvalid_0's auc: 0.808267\tvalid_0's binary_logloss: 0.485724\n",
      "[38]\tvalid_0's auc: 0.808563\tvalid_0's binary_logloss: 0.485076\n",
      "[39]\tvalid_0's auc: 0.808813\tvalid_0's binary_logloss: 0.484039\n",
      "[40]\tvalid_0's auc: 0.809023\tvalid_0's binary_logloss: 0.483091\n",
      "[41]\tvalid_0's auc: 0.809782\tvalid_0's binary_logloss: 0.482441\n",
      "[42]\tvalid_0's auc: 0.810135\tvalid_0's binary_logloss: 0.48179\n",
      "[43]\tvalid_0's auc: 0.810219\tvalid_0's binary_logloss: 0.48082\n",
      "[44]\tvalid_0's auc: 0.81031\tvalid_0's binary_logloss: 0.479906\n",
      "[45]\tvalid_0's auc: 0.810514\tvalid_0's binary_logloss: 0.479024\n",
      "[46]\tvalid_0's auc: 0.810566\tvalid_0's binary_logloss: 0.478437\n",
      "[47]\tvalid_0's auc: 0.810611\tvalid_0's binary_logloss: 0.477529\n",
      "[48]\tvalid_0's auc: 0.810781\tvalid_0's binary_logloss: 0.476637\n",
      "[49]\tvalid_0's auc: 0.81089\tvalid_0's binary_logloss: 0.475883\n",
      "[50]\tvalid_0's auc: 0.811266\tvalid_0's binary_logloss: 0.475459\n",
      "[51]\tvalid_0's auc: 0.811402\tvalid_0's binary_logloss: 0.475078\n",
      "[52]\tvalid_0's auc: 0.811765\tvalid_0's binary_logloss: 0.474246\n",
      "[53]\tvalid_0's auc: 0.811891\tvalid_0's binary_logloss: 0.473452\n",
      "[54]\tvalid_0's auc: 0.811868\tvalid_0's binary_logloss: 0.47263\n",
      "[55]\tvalid_0's auc: 0.81192\tvalid_0's binary_logloss: 0.471804\n",
      "[56]\tvalid_0's auc: 0.812272\tvalid_0's binary_logloss: 0.471275\n",
      "[57]\tvalid_0's auc: 0.812639\tvalid_0's binary_logloss: 0.470396\n",
      "[58]\tvalid_0's auc: 0.812764\tvalid_0's binary_logloss: 0.469597\n",
      "[59]\tvalid_0's auc: 0.813084\tvalid_0's binary_logloss: 0.469049\n",
      "[60]\tvalid_0's auc: 0.813342\tvalid_0's binary_logloss: 0.468244\n",
      "[61]\tvalid_0's auc: 0.813302\tvalid_0's binary_logloss: 0.467499\n",
      "[62]\tvalid_0's auc: 0.813221\tvalid_0's binary_logloss: 0.466758\n",
      "[63]\tvalid_0's auc: 0.813697\tvalid_0's binary_logloss: 0.466017\n",
      "[64]\tvalid_0's auc: 0.813985\tvalid_0's binary_logloss: 0.465501\n",
      "[65]\tvalid_0's auc: 0.81416\tvalid_0's binary_logloss: 0.464725\n",
      "[66]\tvalid_0's auc: 0.814227\tvalid_0's binary_logloss: 0.46398\n",
      "[67]\tvalid_0's auc: 0.814397\tvalid_0's binary_logloss: 0.463309\n",
      "[68]\tvalid_0's auc: 0.814426\tvalid_0's binary_logloss: 0.462627\n",
      "[69]\tvalid_0's auc: 0.814593\tvalid_0's binary_logloss: 0.462244\n",
      "[70]\tvalid_0's auc: 0.814789\tvalid_0's binary_logloss: 0.461571\n",
      "[71]\tvalid_0's auc: 0.814889\tvalid_0's binary_logloss: 0.461144\n",
      "[72]\tvalid_0's auc: 0.815078\tvalid_0's binary_logloss: 0.460684\n",
      "[73]\tvalid_0's auc: 0.815439\tvalid_0's binary_logloss: 0.460063\n",
      "[74]\tvalid_0's auc: 0.815511\tvalid_0's binary_logloss: 0.459386\n",
      "[75]\tvalid_0's auc: 0.815574\tvalid_0's binary_logloss: 0.45877\n",
      "[76]\tvalid_0's auc: 0.815634\tvalid_0's binary_logloss: 0.458128\n",
      "[77]\tvalid_0's auc: 0.815618\tvalid_0's binary_logloss: 0.457495\n",
      "[78]\tvalid_0's auc: 0.81582\tvalid_0's binary_logloss: 0.457057\n",
      "[79]\tvalid_0's auc: 0.81594\tvalid_0's binary_logloss: 0.456475\n",
      "[80]\tvalid_0's auc: 0.815961\tvalid_0's binary_logloss: 0.455885\n",
      "[81]\tvalid_0's auc: 0.816153\tvalid_0's binary_logloss: 0.455511\n",
      "[82]\tvalid_0's auc: 0.816433\tvalid_0's binary_logloss: 0.455186\n",
      "[83]\tvalid_0's auc: 0.816546\tvalid_0's binary_logloss: 0.454625\n",
      "[84]\tvalid_0's auc: 0.816586\tvalid_0's binary_logloss: 0.454039\n",
      "[85]\tvalid_0's auc: 0.816584\tvalid_0's binary_logloss: 0.453482\n",
      "[86]\tvalid_0's auc: 0.816881\tvalid_0's binary_logloss: 0.453048\n",
      "[87]\tvalid_0's auc: 0.817029\tvalid_0's binary_logloss: 0.452485\n",
      "[88]\tvalid_0's auc: 0.81707\tvalid_0's binary_logloss: 0.451941\n",
      "[89]\tvalid_0's auc: 0.817298\tvalid_0's binary_logloss: 0.451544\n",
      "[90]\tvalid_0's auc: 0.817343\tvalid_0's binary_logloss: 0.450975\n",
      "[91]\tvalid_0's auc: 0.817357\tvalid_0's binary_logloss: 0.450422\n",
      "[92]\tvalid_0's auc: 0.817592\tvalid_0's binary_logloss: 0.450109\n",
      "[93]\tvalid_0's auc: 0.817729\tvalid_0's binary_logloss: 0.449542\n",
      "[94]\tvalid_0's auc: 0.817834\tvalid_0's binary_logloss: 0.448982\n",
      "[95]\tvalid_0's auc: 0.81809\tvalid_0's binary_logloss: 0.448398\n",
      "[96]\tvalid_0's auc: 0.818269\tvalid_0's binary_logloss: 0.447908\n",
      "[97]\tvalid_0's auc: 0.818682\tvalid_0's binary_logloss: 0.447547\n",
      "[98]\tvalid_0's auc: 0.819015\tvalid_0's binary_logloss: 0.447165\n",
      "[99]\tvalid_0's auc: 0.819016\tvalid_0's binary_logloss: 0.446669\n",
      "[100]\tvalid_0's auc: 0.819127\tvalid_0's binary_logloss: 0.446397\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\tvalid_0's auc: 0.819127\tvalid_0's binary_logloss: 0.446397\n"
     ]
    }
   ],
   "source": [
    "# 五折交叉验证，这里的五折交叉是以用户为目标进行五折划分\n",
    "#  这一部分与前面的单独训练和验证是分开的\n",
    "def get_kfold_users(trn_df, n=5):\n",
    "    user_ids = trn_df['user_id'].unique()\n",
    "    user_set = [user_ids[i::n] for i in range(n)]\n",
    "    return user_set\n",
    "\n",
    "k_fold = 5\n",
    "trn_df = trn_user_item_feats_df_rank_model\n",
    "user_set = get_kfold_users(trn_df, n=k_fold)\n",
    "\n",
    "score_list = []\n",
    "score_df = trn_df[['user_id', 'click_article_id', 'label']]\n",
    "sub_preds = np.zeros(tst_user_item_feats_df_rank_model.shape[0])\n",
    "\n",
    "# 五折交叉验证，并将中间结果保存用于staking\n",
    "for n_fold, valid_user in enumerate(user_set):\n",
    "    train_idx = trn_df[~trn_df['user_id'].isin(valid_user)] # add slide user\n",
    "    valid_idx = trn_df[trn_df['user_id'].isin(valid_user)]\n",
    "    \n",
    "    # 模型及参数的定义\n",
    "    lgb_Classfication = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=31, reg_alpha=0.0, reg_lambda=1,\n",
    "                            max_depth=-1, n_estimators=100, subsample=0.7, colsample_bytree=0.7, subsample_freq=1,\n",
    "                            learning_rate=0.01, min_child_weight=50, random_state=2018, n_jobs= 16, verbose=10)  \n",
    "    # 训练模型\n",
    "    lgb_Classfication.fit(train_idx[lgb_cols], train_idx['label'],eval_set=[(valid_idx[lgb_cols], valid_idx['label'])], \n",
    "                          eval_metric=['auc', ],early_stopping_rounds=50, )\n",
    "    \n",
    "    # 预测验证集结果\n",
    "    valid_idx['pred_score'] = lgb_Classfication.predict_proba(valid_idx[lgb_cols], \n",
    "                                                              num_iteration=lgb_Classfication.best_iteration_)[:,1]\n",
    "    \n",
    "    # 对输出结果进行归一化 分类模型输出的值本身就是一个概率值不需要进行归一化\n",
    "    # valid_idx['pred_score'] = valid_idx[['pred_score']].transform(lambda x: norm_sim(x))\n",
    "    \n",
    "    valid_idx.sort_values(by=['user_id', 'pred_score'])\n",
    "    valid_idx['pred_rank'] = valid_idx.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first')\n",
    "    \n",
    "    # 将验证集的预测结果放到一个列表中，后面进行拼接\n",
    "    score_list.append(valid_idx[['user_id', 'click_article_id', 'pred_score', 'pred_rank']])\n",
    "    \n",
    "    # 如果是线上测试，需要计算每次交叉验证的结果相加，最后求平均\n",
    "    if not offline:\n",
    "        sub_preds += lgb_Classfication.predict_proba(tst_user_item_feats_df_rank_model[lgb_cols], \n",
    "                                                     num_iteration=lgb_Classfication.best_iteration_)[:,1]\n",
    "    \n",
    "score_df_ = pd.concat(score_list, axis=0)\n",
    "score_df = score_df.merge(score_df_, how='left', on=['user_id', 'click_article_id'])\n",
    "# 保存训练集交叉验证产生的新特征\n",
    "score_df[['user_id', 'click_article_id', 'pred_score', 'pred_rank', 'label']].to_csv(save_path + 'trn_lgb_cls_feats.csv', index=False)\n",
    "    \n",
    "# 测试集的预测结果，多次交叉验证求平均,将预测的score和对应的rank特征保存，可以用于后面的staking，这里还可以构造其他更多的特征\n",
    "tst_user_item_feats_df_rank_model['pred_score'] = sub_preds / k_fold\n",
    "tst_user_item_feats_df_rank_model['pred_score'] = tst_user_item_feats_df_rank_model['pred_score'].transform(lambda x: norm_sim(x))\n",
    "tst_user_item_feats_df_rank_model.sort_values(by=['user_id', 'pred_score'])\n",
    "tst_user_item_feats_df_rank_model['pred_rank'] = tst_user_item_feats_df_rank_model.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first')\n",
    "\n",
    "# 保存测试集交叉验证的新特征\n",
    "tst_user_item_feats_df_rank_model[['user_id', 'click_article_id', 'pred_score', 'pred_rank']].to_csv(save_path + 'tst_lgb_cls_feats.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:24:23.074237Z",
     "start_time": "2020-11-18T04:24:13.812284Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测结果重新排序, 及生成提交结果\n",
    "rank_results = tst_user_item_feats_df_rank_model[['user_id', 'click_article_id', 'pred_score']]\n",
    "rank_results['click_article_id'] = rank_results['click_article_id'].astype(int)\n",
    "submit(rank_results, topk=5, model_name='lgb_cls')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DIN模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用户的历史点击行为列表\n",
    "这个是为后面的DIN模型服务的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:24:30.508213Z",
     "start_time": "2020-11-18T04:24:27.426372Z"
    }
   },
   "outputs": [],
   "source": [
    "if offline:\n",
    "    all_data = pd.read_csv('./data_raw/train_click_log.csv')\n",
    "else:\n",
    "    trn_data = pd.read_csv('./data_raw/train_click_log.csv')\n",
    "    tst_data = pd.read_csv('./data_raw/testA_click_log.csv')\n",
    "    all_data = trn_data.append(tst_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:25:28.082071Z",
     "start_time": "2020-11-18T04:24:33.649524Z"
    }
   },
   "outputs": [],
   "source": [
    "hist_click =all_data[['user_id', 'click_article_id']].groupby('user_id').agg({list}).reset_index()\n",
    "his_behavior_df = pd.DataFrame()\n",
    "his_behavior_df['user_id'] = hist_click['user_id']\n",
    "his_behavior_df['hist_click_article_id'] = hist_click['click_article_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:25:52.925866Z",
     "start_time": "2020-11-18T04:25:52.863922Z"
    }
   },
   "outputs": [],
   "source": [
    "trn_user_item_feats_df_din_model = trn_user_item_feats_df.copy()\n",
    "\n",
    "if offline:\n",
    "    val_user_item_feats_df_din_model = val_user_item_feats_df.copy()\n",
    "else: \n",
    "    val_user_item_feats_df_din_model = None\n",
    "    \n",
    "tst_user_item_feats_df_din_model = tst_user_item_feats_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:26:00.070681Z",
     "start_time": "2020-11-18T04:25:56.417197Z"
    }
   },
   "outputs": [],
   "source": [
    "trn_user_item_feats_df_din_model = trn_user_item_feats_df_din_model.merge(his_behavior_df, on='user_id')\n",
    "\n",
    "if offline:\n",
    "    val_user_item_feats_df_din_model = val_user_item_feats_df_din_model.merge(his_behavior_df, on='user_id')\n",
    "else:\n",
    "    val_user_item_feats_df_din_model = None\n",
    "\n",
    "tst_user_item_feats_df_din_model = tst_user_item_feats_df_din_model.merge(his_behavior_df, on='user_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DIN模型简介\n",
    "我们下面尝试使用DIN模型， DIN的全称是Deep Interest Network， 这是阿里2018年基于前面的深度学习模型无法表达用户多样化的兴趣而提出的一个模型， 它可以通过考虑【给定的候选广告】和【用户的历史行为】的相关性，来计算用户兴趣的表示向量。具体来说就是通过引入局部激活单元，通过软搜索历史行为的相关部分来关注相关的用户兴趣，并采用加权和来获得有关候选广告的用户兴趣的表示。与候选广告相关性较高的行为会获得较高的激活权重，并支配着用户兴趣。该表示向量在不同广告上有所不同，大大提高了模型的表达能力。所以该模型对于此次新闻推荐的任务也比较适合， 我们在这里通过当前的候选文章与用户历史点击文章的相关性来计算用户对于文章的兴趣。 该模型的结构如下：\n",
    "\n",
    "![image-20201116201646983](http://ryluo.oss-cn-chengdu.aliyuncs.com/abc/image-20201116201646983.png)\n",
    "\n",
    "\n",
    "我们这里直接调包来使用这个模型， 关于这个模型的详细细节部分我们会在下一期的推荐系统组队学习中给出。下面说一下该模型如何具体使用：deepctr的函数原型如下：\n",
    "> def DIN(dnn_feature_columns, history_feature_list, dnn_use_bn=False,\n",
    ">        dnn_hidden_units=(200, 80), dnn_activation='relu', att_hidden_size=(80, 40), att_activation=\"dice\",\n",
    ">       att_weight_normalization=False, l2_reg_dnn=0, l2_reg_embedding=1e-6, dnn_dropout=0, seed=1024,\n",
    ">        task='binary'):\n",
    "> \n",
    "> * dnn_feature_columns: 特征列， 包含数据所有特征的列表\n",
    "> * history_feature_list: 用户历史行为列， 反应用户历史行为的特征的列表\n",
    "> * dnn_use_bn: 是否使用BatchNormalization\n",
    "> * dnn_hidden_units: 全连接层网络的层数和每一层神经元的个数， 一个列表或者元组\n",
    "> * dnn_activation_relu: 全连接网络的激活单元类型\n",
    "> * att_hidden_size: 注意力层的全连接网络的层数和每一层神经元的个数\n",
    "> * att_activation: 注意力层的激活单元类型\n",
    "> * att_weight_normalization: 是否归一化注意力得分\n",
    "> * l2_reg_dnn: 全连接网络的正则化系数\n",
    "> * l2_reg_embedding: embedding向量的正则化稀疏\n",
    "> * dnn_dropout: 全连接网络的神经元的失活概率\n",
    "> * task: 任务， 可以是分类， 也可是是回归\n",
    "\n",
    "在具体使用的时候， 我们必须要传入特征列和历史行为列， 但是再传入之前， 我们需要进行一下特征列的预处理。具体如下：\n",
    "\n",
    "1. 首先，我们要处理数据集， 得到数据， 由于我们是基于用户过去的行为去预测用户是否点击当前文章， 所以我们需要把数据的特征列划分成数值型特征， 离散型特征和历史行为特征列三部分， 对于每一部分， DIN模型的处理会有不同\n",
    "    1. 对于离散型特征， 在我们的数据集中就是那些类别型的特征， 比如user_id这种， 这种类别型特征， 我们首先要经过embedding处理得到每个特征的低维稠密型表示， 既然要经过embedding， 那么我们就需要为每一列的类别特征的取值建立一个字典，并指明embedding维度， 所以在使用deepctr的DIN模型准备数据的时候， 我们需要通过SparseFeat函数指明这些类别型特征, 这个函数的传入参数就是列名， 列的唯一取值(建立字典用)和embedding维度。\n",
    "    2. 对于用户历史行为特征列， 比如文章id， 文章的类别等这种， 同样的我们需要先经过embedding处理， 只不过和上面不一样的地方是，对于这种特征， 我们在得到每个特征的embedding表示之后， 还需要通过一个Attention_layer计算用户的历史行为和当前候选文章的相关性以此得到当前用户的embedding向量， 这个向量就可以基于当前的候选文章与用户过去点击过得历史文章的相似性的程度来反应用户的兴趣， 并且随着用户的不同的历史点击来变化，去动态的模拟用户兴趣的变化过程。这类特征对于每个用户都是一个历史行为序列， 对于每个用户， 历史行为序列长度会不一样， 可能有的用户点击的历史文章多，有的点击的历史文章少， 所以我们还需要把这个长度统一起来， 在为DIN模型准备数据的时候， 我们首先要通过SparseFeat函数指明这些类别型特征， 然后还需要通过VarLenSparseFeat函数再进行序列填充， 使得每个用户的历史序列一样长， 所以这个函数参数中会有个maxlen，来指明序列的最大长度是多少。\n",
    "    3. 对于连续型特征列， 我们只需要用DenseFeat函数来指明列名和维度即可。\n",
    "2. 处理完特征列之后， 我们把相应的数据与列进行对应，就得到了最后的数据。\n",
    "\n",
    "下面根据具体的代码感受一下， 逻辑是这样， 首先我们需要写一个数据准备函数， 在这里面就是根据上面的具体步骤准备数据， 得到数据和特征列， 然后就是建立DIN模型并训练， 最后基于模型进行测试。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:26:08.405211Z",
     "start_time": "2020-11-18T04:26:04.887013Z"
    }
   },
   "outputs": [],
   "source": [
    "# 导入deepctr\n",
    "from deepctr.models import DIN\n",
    "from deepctr.feature_column import SparseFeat, VarLenSparseFeat, DenseFeat, get_feature_names\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.models import *\n",
    "from tensorflow.keras.callbacks import * \n",
    "import tensorflow as tf\n",
    "\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:26:13.485712Z",
     "start_time": "2020-11-18T04:26:13.476042Z"
    }
   },
   "outputs": [],
   "source": [
    "# 数据准备函数\n",
    "def get_din_feats_columns(df, dense_fea, sparse_fea, behavior_fea, his_behavior_fea, emb_dim=32, max_len=100):\n",
    "    \"\"\"\n",
    "    数据准备函数:\n",
    "    df: 数据集\n",
    "    dense_fea: 数值型特征列\n",
    "    sparse_fea: 离散型特征列\n",
    "    behavior_fea: 用户的候选行为特征列\n",
    "    his_behavior_fea: 用户的历史行为特征列\n",
    "    embedding_dim: embedding的维度， 这里为了简单， 统一把离散型特征列采用一样的隐向量维度\n",
    "    max_len: 用户序列的最大长度\n",
    "    \"\"\"\n",
    "    \n",
    "    sparse_feature_columns = [SparseFeat(feat, vocabulary_size=df[feat].nunique() + 1, embedding_dim=emb_dim) for feat in sparse_fea]\n",
    "    \n",
    "    dense_feature_columns = [DenseFeat(feat, 1, ) for feat in dense_fea]\n",
    "    \n",
    "    var_feature_columns = [VarLenSparseFeat(SparseFeat(feat, vocabulary_size=df['click_article_id'].nunique() + 1,\n",
    "                                    embedding_dim=emb_dim, embedding_name='click_article_id'), maxlen=max_len) for feat in hist_behavior_fea]\n",
    "    \n",
    "    dnn_feature_columns = sparse_feature_columns + dense_feature_columns + var_feature_columns\n",
    "    \n",
    "    # 建立x, x是一个字典的形式\n",
    "    x = {}\n",
    "    for name in get_feature_names(dnn_feature_columns):\n",
    "        if name in his_behavior_fea:\n",
    "            # 这是历史行为序列\n",
    "            his_list = [l for l in df[name]]\n",
    "            x[name] = pad_sequences(his_list, maxlen=max_len, padding='post')      # 二维数组\n",
    "        else:\n",
    "            x[name] = df[name].values\n",
    "    \n",
    "    return x, dnn_feature_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:26:18.783217Z",
     "start_time": "2020-11-18T04:26:18.776795Z"
    }
   },
   "outputs": [],
   "source": [
    "# 把特征分开\n",
    "sparse_fea = ['user_id', 'click_article_id', 'category_id', 'click_environment', 'click_deviceGroup', \n",
    "              'click_os', 'click_country', 'click_region', 'click_referrer_type', 'is_cat_hab']\n",
    "\n",
    "behavior_fea = ['click_article_id']\n",
    "\n",
    "hist_behavior_fea = ['hist_click_article_id']\n",
    "\n",
    "dense_fea = ['sim0', 'time_diff0', 'word_diff0', 'sim_max', 'sim_min', 'sim_sum', 'sim_mean', 'score',\n",
    "             'rank','click_size','time_diff_mean','active_level','user_time_hob1','user_time_hob2',\n",
    "             'words_hbo','words_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:26:25.469810Z",
     "start_time": "2020-11-18T04:26:24.779347Z"
    }
   },
   "outputs": [],
   "source": [
    "# dense特征进行归一化, 神经网络训练都需要将数值进行归一化处理\n",
    "mm = MinMaxScaler()\n",
    "\n",
    "# 下面是做一些特殊处理，当在其他的地方出现无效值的时候，不处理无法进行归一化，刚开始可以先把他注释掉，在运行了下面的代码\n",
    "# 之后如果发现报错，应该先去想办法处理如何不出现inf之类的值\n",
    "# trn_user_item_feats_df_din_model.replace([np.inf, -np.inf], 0, inplace=True)\n",
    "# tst_user_item_feats_df_din_model.replace([np.inf, -np.inf], 0, inplace=True)\n",
    "\n",
    "for feat in dense_fea:\n",
    "    trn_user_item_feats_df_din_model[feat] = mm.fit_transform(trn_user_item_feats_df_din_model[[feat]])\n",
    "    \n",
    "    if val_user_item_feats_df_din_model is not None:\n",
    "        val_user_item_feats_df_din_model[feat] = mm.fit_transform(val_user_item_feats_df_din_model[[feat]])\n",
    "    \n",
    "    tst_user_item_feats_df_din_model[feat] = mm.fit_transform(tst_user_item_feats_df_din_model[[feat]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:26:36.727753Z",
     "start_time": "2020-11-18T04:26:28.854705Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ryluo/anaconda3/lib/python3.6/site-packages/tensorflow/python/keras/initializers.py:143: calling RandomNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    }
   ],
   "source": [
    "# 准备训练数据\n",
    "x_trn, dnn_feature_columns = get_din_feats_columns(trn_user_item_feats_df_din_model, dense_fea, \n",
    "                                               sparse_fea, behavior_fea, hist_behavior_fea, max_len=50)\n",
    "y_trn = trn_user_item_feats_df_din_model['label'].values\n",
    "\n",
    "if offline:\n",
    "    # 准备验证数据\n",
    "    x_val, dnn_feature_columns = get_din_feats_columns(val_user_item_feats_df_din_model, dense_fea, \n",
    "                                                   sparse_fea, behavior_fea, hist_behavior_fea, max_len=50)\n",
    "    y_val = val_user_item_feats_df_din_model['label'].values\n",
    "    \n",
    "dense_fea = [x for x in dense_fea if x != 'label']\n",
    "x_tst, dnn_feature_columns = get_din_feats_columns(tst_user_item_feats_df_din_model, dense_fea, \n",
    "                                               sparse_fea, behavior_fea, hist_behavior_fea, max_len=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:26:45.146318Z",
     "start_time": "2020-11-18T04:26:40.423914Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ryluo/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1288: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From /home/ryluo/anaconda3/lib/python3.6/site-packages/tensorflow/python/autograph/impl/api.py:255: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "user_id (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "click_article_id (InputLayer)   [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "category_id (InputLayer)        [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "click_environment (InputLayer)  [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "click_deviceGroup (InputLayer)  [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "click_os (InputLayer)           [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "click_country (InputLayer)      [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "click_region (InputLayer)       [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "click_referrer_type (InputLayer [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "is_cat_hab (InputLayer)         [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_user_id (Embedding)  (None, 1, 32)        1600032     user_id[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "sparse_seq_emb_hist_click_artic multiple             525664      click_article_id[0][0]           \n",
      "                                                                 hist_click_article_id[0][0]      \n",
      "                                                                 click_article_id[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_category_id (Embeddi (None, 1, 32)        7776        category_id[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_click_environment (E (None, 1, 32)        128         click_environment[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_click_deviceGroup (E (None, 1, 32)        160         click_deviceGroup[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_click_os (Embedding) (None, 1, 32)        288         click_os[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_click_country (Embed (None, 1, 32)        384         click_country[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_click_region (Embedd (None, 1, 32)        928         click_region[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_click_referrer_type  (None, 1, 32)        256         click_referrer_type[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "sparse_emb_is_cat_hab (Embeddin (None, 1, 32)        64          is_cat_hab[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "no_mask (NoMask)                (None, 1, 32)        0           sparse_emb_user_id[0][0]         \n",
      "                                                                 sparse_seq_emb_hist_click_article\n",
      "                                                                 sparse_emb_category_id[0][0]     \n",
      "                                                                 sparse_emb_click_environment[0][0\n",
      "                                                                 sparse_emb_click_deviceGroup[0][0\n",
      "                                                                 sparse_emb_click_os[0][0]        \n",
      "                                                                 sparse_emb_click_country[0][0]   \n",
      "                                                                 sparse_emb_click_region[0][0]    \n",
      "                                                                 sparse_emb_click_referrer_type[0]\n",
      "                                                                 sparse_emb_is_cat_hab[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "hist_click_article_id (InputLay [(None, 50)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 1, 320)       0           no_mask[0][0]                    \n",
      "                                                                 no_mask[1][0]                    \n",
      "                                                                 no_mask[2][0]                    \n",
      "                                                                 no_mask[3][0]                    \n",
      "                                                                 no_mask[4][0]                    \n",
      "                                                                 no_mask[5][0]                    \n",
      "                                                                 no_mask[6][0]                    \n",
      "                                                                 no_mask[7][0]                    \n",
      "                                                                 no_mask[8][0]                    \n",
      "                                                                 no_mask[9][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "no_mask_1 (NoMask)              (None, 1, 320)       0           concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "attention_sequence_pooling_laye (None, 1, 32)        13961       sparse_seq_emb_hist_click_article\n",
      "                                                                 sparse_seq_emb_hist_click_article\n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 1, 352)       0           no_mask_1[0][0]                  \n",
      "                                                                 attention_sequence_pooling_layer[\n",
      "__________________________________________________________________________________________________\n",
      "sim0 (InputLayer)               [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "time_diff0 (InputLayer)         [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "word_diff0 (InputLayer)         [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sim_max (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sim_min (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sim_sum (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sim_mean (InputLayer)           [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "score (InputLayer)              [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "rank (InputLayer)               [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "click_size (InputLayer)         [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "time_diff_mean (InputLayer)     [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "active_level (InputLayer)       [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "user_time_hob1 (InputLayer)     [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "user_time_hob2 (InputLayer)     [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_hbo (InputLayer)          [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "words_count (InputLayer)        [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 352)          0           concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "no_mask_3 (NoMask)              (None, 1)            0           sim0[0][0]                       \n",
      "                                                                 time_diff0[0][0]                 \n",
      "                                                                 word_diff0[0][0]                 \n",
      "                                                                 sim_max[0][0]                    \n",
      "                                                                 sim_min[0][0]                    \n",
      "                                                                 sim_sum[0][0]                    \n",
      "                                                                 sim_mean[0][0]                   \n",
      "                                                                 score[0][0]                      \n",
      "                                                                 rank[0][0]                       \n",
      "                                                                 click_size[0][0]                 \n",
      "                                                                 time_diff_mean[0][0]             \n",
      "                                                                 active_level[0][0]               \n",
      "                                                                 user_time_hob1[0][0]             \n",
      "                                                                 user_time_hob2[0][0]             \n",
      "                                                                 words_hbo[0][0]                  \n",
      "                                                                 words_count[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "no_mask_2 (NoMask)              (None, 352)          0           flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 16)           0           no_mask_3[0][0]                  \n",
      "                                                                 no_mask_3[1][0]                  \n",
      "                                                                 no_mask_3[2][0]                  \n",
      "                                                                 no_mask_3[3][0]                  \n",
      "                                                                 no_mask_3[4][0]                  \n",
      "                                                                 no_mask_3[5][0]                  \n",
      "                                                                 no_mask_3[6][0]                  \n",
      "                                                                 no_mask_3[7][0]                  \n",
      "                                                                 no_mask_3[8][0]                  \n",
      "                                                                 no_mask_3[9][0]                  \n",
      "                                                                 no_mask_3[10][0]                 \n",
      "                                                                 no_mask_3[11][0]                 \n",
      "                                                                 no_mask_3[12][0]                 \n",
      "                                                                 no_mask_3[13][0]                 \n",
      "                                                                 no_mask_3[14][0]                 \n",
      "                                                                 no_mask_3[15][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 352)          0           no_mask_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 16)           0           concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "no_mask_4 (NoMask)              multiple             0           flatten_1[0][0]                  \n",
      "                                                                 flatten_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 368)          0           no_mask_4[0][0]                  \n",
      "                                                                 no_mask_4[1][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dnn_1 (DNN)                     (None, 80)           89880       concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1)            80          dnn_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "prediction_layer (PredictionLay (None, 1)            1           dense[0][0]                      \n",
      "==================================================================================================\n",
      "Total params: 2,239,602\n",
      "Trainable params: 2,239,362\n",
      "Non-trainable params: 240\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 建立模型\n",
    "model = DIN(dnn_feature_columns, behavior_fea)\n",
    "\n",
    "# 查看模型结构\n",
    "model.summary()\n",
    "\n",
    "# 模型编译\n",
    "model.compile('adam', 'binary_crossentropy',metrics=['binary_crossentropy', tf.keras.metrics.AUC()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:28:43.885773Z",
     "start_time": "2020-11-18T04:26:48.746787Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      "290964/290964 [==============================] - 55s 189us/sample - loss: 0.4209 - binary_crossentropy: 0.4206 - auc: 0.7842\n",
      "Epoch 2/2\n",
      "290964/290964 [==============================] - 52s 178us/sample - loss: 0.3630 - binary_crossentropy: 0.3618 - auc: 0.8478\n"
     ]
    }
   ],
   "source": [
    "# 模型训练\n",
    "if offline:\n",
    "    history = model.fit(x_trn, y_trn, verbose=1, epochs=10, validation_data=(x_val, y_val) , batch_size=256)\n",
    "else:\n",
    "    # 也可以使用上面的语句用自己采样出来的验证集\n",
    "    # history = model.fit(x_trn, y_trn, verbose=1, epochs=3, validation_split=0.3, batch_size=256)\n",
    "    history = model.fit(x_trn, y_trn, verbose=1, epochs=2, batch_size=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:29:20.436591Z",
     "start_time": "2020-11-18T04:28:58.102057Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500000/500000 [==============================] - 20s 39us/sample\n"
     ]
    }
   ],
   "source": [
    "# 模型预测\n",
    "tst_user_item_feats_df_din_model['pred_score'] = model.predict(x_tst, verbose=1, batch_size=256)\n",
    "tst_user_item_feats_df_din_model[['user_id', 'click_article_id', 'pred_score']].to_csv(save_path + 'din_rank_score.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:29:34.985535Z",
     "start_time": "2020-11-18T04:29:26.264531Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测结果重新排序, 及生成提交结果\n",
    "rank_results = tst_user_item_feats_df_din_model[['user_id', 'click_article_id', 'pred_score']]\n",
    "submit(rank_results, topk=5, model_name='din')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-15T06:15:49.490705Z",
     "start_time": "2020-11-15T06:15:49.473794Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:38:53.760383Z",
     "start_time": "2020-11-18T04:29:51.737721Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 232681 samples, validate on 58283 samples\n",
      "Epoch 1/2\n",
      "232681/232681 [==============================] - 44s 189us/sample - loss: 0.2864 - binary_crossentropy: 0.2846 - auc: 0.9008 - val_loss: 0.2830 - val_binary_crossentropy: 0.2813 - val_auc: 0.9072\n",
      "Epoch 2/2\n",
      "232681/232681 [==============================] - 44s 187us/sample - loss: 0.2832 - binary_crossentropy: 0.2816 - auc: 0.9034 - val_loss: 0.2846 - val_binary_crossentropy: 0.2830 - val_auc: 0.9053\n",
      "58283/58283 [==============================] - 2s 36us/sample\n",
      "500000/500000 [==============================] - 19s 37us/sample\n",
      "Train on 232798 samples, validate on 58166 samples\n",
      "Epoch 1/2\n",
      "232798/232798 [==============================] - 43s 184us/sample - loss: 0.2818 - binary_crossentropy: 0.2802 - auc: 0.9051 - val_loss: 0.2968 - val_binary_crossentropy: 0.2953 - val_auc: 0.9062\n",
      "Epoch 2/2\n",
      "232798/232798 [==============================] - 44s 187us/sample - loss: 0.2796 - binary_crossentropy: 0.2782 - auc: 0.9069 - val_loss: 0.2820 - val_binary_crossentropy: 0.2806 - val_auc: 0.9071\n",
      "58166/58166 [==============================] - 2s 38us/sample\n",
      "500000/500000 [==============================] - 18s 37us/sample\n",
      "Train on 232847 samples, validate on 58117 samples\n",
      "Epoch 1/2\n",
      "232847/232847 [==============================] - 43s 185us/sample - loss: 0.2786 - binary_crossentropy: 0.2773 - auc: 0.9080 - val_loss: 0.2761 - val_binary_crossentropy: 0.2749 - val_auc: 0.9113\n",
      "Epoch 2/2\n",
      "232847/232847 [==============================] - 39s 166us/sample - loss: 0.2766 - binary_crossentropy: 0.2754 - auc: 0.9097 - val_loss: 0.2872 - val_binary_crossentropy: 0.2862 - val_auc: 0.9090\n",
      "58117/58117 [==============================] - 2s 34us/sample\n",
      "500000/500000 [==============================] - 17s 33us/sample\n",
      "Train on 232716 samples, validate on 58248 samples\n",
      "Epoch 1/2\n",
      "232716/232716 [==============================] - 39s 169us/sample - loss: 0.2763 - binary_crossentropy: 0.2753 - auc: 0.9100 - val_loss: 0.2739 - val_binary_crossentropy: 0.2730 - val_auc: 0.9116\n",
      "Epoch 2/2\n",
      "232716/232716 [==============================] - 39s 168us/sample - loss: 0.2743 - binary_crossentropy: 0.2735 - auc: 0.9119 - val_loss: 0.2859 - val_binary_crossentropy: 0.2851 - val_auc: 0.9090\n",
      "58248/58248 [==============================] - 2s 35us/sample\n",
      "500000/500000 [==============================] - 17s 34us/sample\n",
      "Train on 232814 samples, validate on 58150 samples\n",
      "Epoch 1/2\n",
      "232814/232814 [==============================] - 40s 170us/sample - loss: 0.2747 - binary_crossentropy: 0.2739 - auc: 0.9115 - val_loss: 0.2702 - val_binary_crossentropy: 0.2695 - val_auc: 0.9163\n",
      "Epoch 2/2\n",
      "232814/232814 [==============================] - 40s 170us/sample - loss: 0.2725 - binary_crossentropy: 0.2719 - auc: 0.9132 - val_loss: 0.2751 - val_binary_crossentropy: 0.2745 - val_auc: 0.9151\n",
      "58150/58150 [==============================] - 2s 34us/sample\n",
      "500000/500000 [==============================] - 17s 34us/sample\n"
     ]
    }
   ],
   "source": [
    "# 五折交叉验证，这里的五折交叉是以用户为目标进行五折划分\n",
    "#  这一部分与前面的单独训练和验证是分开的\n",
    "def get_kfold_users(trn_df, n=5):\n",
    "    user_ids = trn_df['user_id'].unique()\n",
    "    user_set = [user_ids[i::n] for i in range(n)]\n",
    "    return user_set\n",
    "\n",
    "k_fold = 5\n",
    "trn_df = trn_user_item_feats_df_din_model\n",
    "user_set = get_kfold_users(trn_df, n=k_fold)\n",
    "\n",
    "score_list = []\n",
    "score_df = trn_df[['user_id', 'click_article_id', 'label']]\n",
    "sub_preds = np.zeros(tst_user_item_feats_df_rank_model.shape[0])\n",
    "\n",
    "dense_fea = [x for x in dense_fea if x != 'label']\n",
    "x_tst, dnn_feature_columns = get_din_feats_columns(tst_user_item_feats_df_din_model, dense_fea, \n",
    "                                                   sparse_fea, behavior_fea, hist_behavior_fea, max_len=50)\n",
    "\n",
    "# 五折交叉验证，并将中间结果保存用于staking\n",
    "for n_fold, valid_user in enumerate(user_set):\n",
    "    train_idx = trn_df[~trn_df['user_id'].isin(valid_user)] # add slide user\n",
    "    valid_idx = trn_df[trn_df['user_id'].isin(valid_user)]\n",
    "    \n",
    "    # 准备训练数据\n",
    "    x_trn, dnn_feature_columns = get_din_feats_columns(train_idx, dense_fea, \n",
    "                                                       sparse_fea, behavior_fea, hist_behavior_fea, max_len=50)\n",
    "    y_trn = train_idx['label'].values\n",
    "\n",
    "    # 准备验证数据\n",
    "    x_val, dnn_feature_columns = get_din_feats_columns(valid_idx, dense_fea, \n",
    "                                                   sparse_fea, behavior_fea, hist_behavior_fea, max_len=50)\n",
    "    y_val = valid_idx['label'].values\n",
    "    \n",
    "    history = model.fit(x_trn, y_trn, verbose=1, epochs=2, validation_data=(x_val, y_val) , batch_size=256)\n",
    "    \n",
    "    # 预测验证集结果\n",
    "    valid_idx['pred_score'] = model.predict(x_val, verbose=1, batch_size=256)   \n",
    "    \n",
    "    valid_idx.sort_values(by=['user_id', 'pred_score'])\n",
    "    valid_idx['pred_rank'] = valid_idx.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first')\n",
    "    \n",
    "    # 将验证集的预测结果放到一个列表中，后面进行拼接\n",
    "    score_list.append(valid_idx[['user_id', 'click_article_id', 'pred_score', 'pred_rank']])\n",
    "    \n",
    "    # 如果是线上测试，需要计算每次交叉验证的结果相加，最后求平均\n",
    "    if not offline:\n",
    "        sub_preds += model.predict(x_tst, verbose=1, batch_size=256)[:, 0]   \n",
    "    \n",
    "score_df_ = pd.concat(score_list, axis=0)\n",
    "score_df = score_df.merge(score_df_, how='left', on=['user_id', 'click_article_id'])\n",
    "# 保存训练集交叉验证产生的新特征\n",
    "score_df[['user_id', 'click_article_id', 'pred_score', 'pred_rank', 'label']].to_csv(save_path + 'trn_din_cls_feats.csv', index=False)\n",
    "    \n",
    "# 测试集的预测结果，多次交叉验证求平均,将预测的score和对应的rank特征保存，可以用于后面的staking，这里还可以构造其他更多的特征\n",
    "tst_user_item_feats_df_din_model['pred_score'] = sub_preds / k_fold\n",
    "tst_user_item_feats_df_din_model['pred_score'] = tst_user_item_feats_df_din_model['pred_score'].transform(lambda x: norm_sim(x))\n",
    "tst_user_item_feats_df_din_model.sort_values(by=['user_id', 'pred_score'])\n",
    "tst_user_item_feats_df_din_model['pred_rank'] = tst_user_item_feats_df_din_model.groupby(['user_id'])['pred_score'].rank(ascending=False, method='first')\n",
    "\n",
    "# 保存测试集交叉验证的新特征\n",
    "tst_user_item_feats_df_din_model[['user_id', 'click_article_id', 'pred_score', 'pred_rank']].to_csv(save_path + 'tst_din_cls_feats.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型融合"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加权融合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:44:27.351996Z",
     "start_time": "2020-11-18T04:44:26.561275Z"
    }
   },
   "outputs": [],
   "source": [
    "# 读取多个模型的排序结果文件\n",
    "lgb_ranker = pd.read_csv(save_path + 'lgb_ranker_score.csv')\n",
    "lgb_cls = pd.read_csv(save_path + 'lgb_cls_score.csv')\n",
    "din_ranker = pd.read_csv(save_path + 'din_rank_score.csv')\n",
    "\n",
    "# 这里也可以换成交叉验证输出的测试结果进行加权融合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:44:31.593981Z",
     "start_time": "2020-11-18T04:44:31.589439Z"
    }
   },
   "outputs": [],
   "source": [
    "rank_model = {'lgb_ranker': lgb_ranker, \n",
    "              'lgb_cls': lgb_cls, \n",
    "              'din_ranker': din_ranker}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:44:36.135860Z",
     "start_time": "2020-11-18T04:44:36.130577Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_ensumble_predict_topk(rank_model, topk=5):\n",
    "    final_recall = rank_model['lgb_cls'].append(rank_model['din_ranker'])\n",
    "    rank_model['lgb_ranker']['pred_score'] = rank_model['lgb_ranker']['pred_score'].transform(lambda x: norm_sim(x))\n",
    "    \n",
    "    final_recall = final_recall.append(rank_model['lgb_ranker'])\n",
    "    final_recall = final_recall.groupby(['user_id', 'click_article_id'])['pred_score'].sum().reset_index()\n",
    "    \n",
    "    submit(final_recall, topk=topk, model_name='ensemble_fuse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:44:51.659270Z",
     "start_time": "2020-11-18T04:44:40.445659Z"
    }
   },
   "outputs": [],
   "source": [
    "get_ensumble_predict_topk(rank_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Staking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:44:58.025992Z",
     "start_time": "2020-11-18T04:44:56.146962Z"
    }
   },
   "outputs": [],
   "source": [
    "# 读取多个模型的交叉验证生成的结果文件\n",
    "# 训练集\n",
    "trn_lgb_ranker_feats = pd.read_csv(save_path + 'trn_lgb_ranker_feats.csv')\n",
    "trn_lgb_cls_feats = pd.read_csv(save_path + 'trn_lgb_cls_feats.csv')\n",
    "trn_din_cls_feats = pd.read_csv(save_path + 'trn_din_cls_feats.csv')\n",
    "\n",
    "# 测试集\n",
    "tst_lgb_ranker_feats = pd.read_csv(save_path + 'tst_lgb_ranker_feats.csv')\n",
    "tst_lgb_cls_feats = pd.read_csv(save_path + 'tst_lgb_cls_feats.csv')\n",
    "tst_din_cls_feats = pd.read_csv(save_path + 'tst_din_cls_feats.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:45:07.701862Z",
     "start_time": "2020-11-18T04:45:07.644335Z"
    }
   },
   "outputs": [],
   "source": [
    "# 将多个模型输出的特征进行拼接\n",
    "\n",
    "finall_trn_ranker_feats = trn_lgb_ranker_feats[['user_id', 'click_article_id', 'label']]\n",
    "finall_tst_ranker_feats = tst_lgb_ranker_feats[['user_id', 'click_article_id']]\n",
    "\n",
    "for idx, trn_model in enumerate([trn_lgb_ranker_feats, trn_lgb_cls_feats, trn_din_cls_feats]):\n",
    "    for feat in [ 'pred_score', 'pred_rank']:\n",
    "        col_name = feat + '_' + str(idx)\n",
    "        finall_trn_ranker_feats[col_name] = trn_model[feat]\n",
    "\n",
    "for idx, tst_model in enumerate([tst_lgb_ranker_feats, tst_lgb_cls_feats, tst_din_cls_feats]):\n",
    "    for feat in [ 'pred_score', 'pred_rank']:\n",
    "        col_name = feat + '_' + str(idx)\n",
    "        finall_tst_ranker_feats[col_name] = tst_model[feat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:45:15.044242Z",
     "start_time": "2020-11-18T04:45:13.138252Z"
    }
   },
   "outputs": [],
   "source": [
    "# 定义一个逻辑回归模型再次拟合交叉验证产生的特征对测试集进行预测\n",
    "# 这里需要注意的是，在做交叉验证的时候可以构造多一些与输出预测值相关的特征，来丰富这里简单模型的特征\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "feat_cols = ['pred_score_0', 'pred_rank_0', 'pred_score_1', 'pred_rank_1', 'pred_score_2', 'pred_rank_2']\n",
    "\n",
    "trn_x = finall_trn_ranker_feats[feat_cols]\n",
    "trn_y = finall_trn_ranker_feats['label']\n",
    "\n",
    "tst_x = finall_tst_ranker_feats[feat_cols]\n",
    "\n",
    "# 定义模型\n",
    "lr = LogisticRegression()\n",
    "\n",
    "# 模型训练\n",
    "lr.fit(trn_x, trn_y)\n",
    "\n",
    "# 模型预测\n",
    "finall_tst_ranker_feats['pred_score'] = lr.predict_proba(tst_x)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-18T04:45:29.018764Z",
     "start_time": "2020-11-18T04:45:19.423130Z"
    }
   },
   "outputs": [],
   "source": [
    "# 预测结果重新排序, 及生成提交结果\n",
    "rank_results = finall_tst_ranker_feats[['user_id', 'click_article_id', 'pred_score']]\n",
    "submit(rank_results, topk=5, model_name='ensumble_staking')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 总结\n",
    "本章主要学习了三个排序模型，包括LGB的Rank， LGB的Classifier还有深度学习的DIN模型， 当然，对于这三个模型的原理部分，我们并没有给出详细的介绍， 请大家课下自己探索原理，也欢迎大家把自己的探索与所学分享出来，我们一块学习和进步。最后，我们进行了简单的模型融合策略，包括简单的加权和Stacking。\n",
    "\n",
    "关于Datawhale： Datawhale是一个专注于数据科学与AI领域的开源组织，汇集了众多领域院校和知名企业的优秀学习者，聚合了一群有开源精神和探索精神的团队成员。Datawhale 以“for the learner，和学习者一起成长”为愿景，鼓励真实地展现自我、开放包容、互信互助、敢于试错和勇于担当。同时 Datawhale 用开源的理念去探索开源内容、开源学习和开源方案，赋能人才培养，助力人才成长，建立起人与人，人与知识，人与企业和人与未来的联结。 本次数据挖掘路径学习，专题知识将在天池分享，详情可关注Datawhale：\n",
    "\n",
    "![image-20201119112159065](http://ryluo.oss-cn-chengdu.aliyuncs.com/abc/image-20201119112159065.png)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "170px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}