new_org/team-learning-data-mining
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

课程内容提交

Browse Source
This commit is contained in:
LSGOMYP
2020-07-30 09:21:29 +08:00
parent f5f1c7be26
commit 0047da493d
10 changed files with 49891 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
RentForecast/Task1 赛题分析.ipynb
+1619
View File
File diff suppressed because one or more lines are too long
RentForecast/Task2 数据清洗.ipynb
+455
View File
File diff suppressed because one or more lines are too long
RentForecast/Task3 特征工程.ipynb
+754
View File
@@ -0,0 +1,754 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 任务3 特征工程&特征选择(3天)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 特征工程"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#核心代码举例\n",
"\n",
"# 统计特征\n",
" #计算均值\n",
" gp = train.groupby(by)[fea].mean()\n",
" #计算中位数\n",
" gp = train.groupby(by)[fea].median()\n",
" #计算方差\n",
" gp = train.groupby(by)[fea].std()\n",
" #计算最大值\n",
" gp = train.groupby(by)[fea].max()\n",
" #计算最小值\n",
" gp = train.groupby(by)[fea].min()\n",
" #计算出现次数\n",
" gp = train.groupby(by)[fea].size()\n",
" \n",
"\n",
"# groupby生成统计特征：mean,std\n",
" # 按照communityName分组计算面积的均值和方差\n",
" temp = data.groupby('communityName')['area'].agg({'com_area_mean': 'mean', 'com_area_std': 'std'})\n",
"\n",
"# 特征拆分\n",
" # 将houseType转为'Room'，'Hall'，'Bath'\n",
" def Room(x):\n",
" Room = int(x.split('室')[0])\n",
" return Room\n",
" def Hall(x):\n",
" Hall = int(x.split(\"室\")[1].split(\"厅\")[0])\n",
" return Hall\n",
" def Bath(x):\n",
" Bath = int(x.split(\"室\")[1].split(\"厅\")[1].split(\"卫\")[0])\n",
" return Bath\n",
"\n",
" data['Room'] = data['houseType'].apply(lambda x: Room(x))\n",
" data['Hall'] = data['houseType'].apply(lambda x: Hall(x))\n",
" data['Bath'] = data['houseType'].apply(lambda x: Bath(x))\n",
" \n",
"#特征合并\n",
" # 合并部分配套设施特征\n",
" data['trainsportNum'] = 5 * data['subwayStationNum'] / data['subwayStationNum'].mean() + data['busStationNum'] / \\\n",
" data[\n",
" 'busStationNum'].mean()\n",
"\n",
"# 交叉生成特征:特征之间交叉+ - * / \n",
"data['Room_Bath'] = (data['Bath']+1) / (data['Room']+1)\n",
"\n",
"\n",
"# 聚类特征\n",
"from sklearn.mixture import GaussianMixture 使用GaussianMixture做聚类特征\n",
"gmm = GaussianMixture(n_components=4, covariance_type='full', random_state=0)\n",
"gmm.fit_predict(data)\n",
" \n",
"# 特征编码\n",
"from sklearn.preprocessing import LabelEncoder\n",
"data['communityName'] = LabelEncoder().fit_transform(data['communityName'])\n",
"from sklearn import preprocessing.OneHotEncoder\n",
"data['communityName'] = OneHotEncoder().fit_transform(data['communityName'])\n",
"\n",
"\n",
"# 过大量级值取log平滑（针对线性模型有效）\n",
"data[feature]=np.log1p(data[feature])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T13:40:19.692972Z",
"start_time": "2019-12-24T13:40:19.126469Z"
}
},
"outputs": [],
"source": [
"import pandas as pd\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"from sklearn.preprocessing import LabelEncoder\n",
"\n",
"train = pd.read_csv('./train_data.csv')\n",
"test = pd.read_csv('./test_a.csv')\n",
"target_train = train.pop('tradeMoney')\n",
"target_test = test.pop('tradeMoney')\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 特征合并"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T13:40:55.454321Z",
"start_time": "2019-12-24T13:40:55.291756Z"
}
},
"outputs": [],
"source": [
"def newfeature(data):\n",
"\n",
"\n",
" # 将houseType转为'Room'，'Hall'，'Bath'\n",
" def Room(x):\n",
" Room = int(x.split('室')[0])\n",
" return Room\n",
" def Hall(x):\n",
" Hall = int(x.split(\"室\")[1].split(\"厅\")[0])\n",
" return Hall\n",
" def Bath(x):\n",
" Bath = int(x.split(\"室\")[1].split(\"厅\")[1].split(\"卫\")[0])\n",
" return Bath\n",
"\n",
" data['Room'] = data['houseType'].apply(lambda x: Room(x))\n",
" data['Hall'] = data['houseType'].apply(lambda x: Hall(x))\n",
" data['Bath'] = data['houseType'].apply(lambda x: Bath(x))\n",
" data['Room_Bath'] = (data['Bath']+1) / (data['Room']+1)\n",
" # 填充租房类型\n",
" data.loc[(data['rentType'] == '未知方式') & (data['Room'] <= 1), 'rentType'] = '整租'\n",
" # print(data.loc[(data['rentType']=='未知方式')&(data['Room_Bath']>1),'rentType'])\n",
" data.loc[(data['rentType'] == '未知方式') & (data['Room_Bath'] > 1), 'rentType'] = '合租'\n",
" data.loc[(data['rentType'] == '未知方式') & (data['Room'] > 1) & (data['area'] < 50), 'rentType'] = '合租'\n",
" data.loc[(data['rentType'] == '未知方式') & (data['area'] / data['Room'] < 20), 'rentType'] = '合租'\n",
" # data.loc[(data['rentType']=='未知方式')&(data['area']>60),'rentType']='合租'\n",
" data.loc[(data['rentType'] == '未知方式') & (data['area'] <= 50) & (data['Room'] == 2), 'rentType'] = '合租'\n",
" data.loc[(data['rentType'] == '未知方式') & (data['area'] > 60) & (data['Room'] == 2), 'rentType'] = '整租'\n",
" data.loc[(data['rentType'] == '未知方式') & (data['area'] <= 60) & (data['Room'] == 3), 'rentType'] = '合租'\n",
" data.loc[(data['rentType'] == '未知方式') & (data['area'] > 60) & (data['Room'] == 3), 'rentType'] = '整租'\n",
" data.loc[(data['rentType'] == '未知方式') & (data['area'] >= 100) & (data['Room'] > 3), 'rentType'] = '整租'\n",
"\n",
" # data.drop('Room_Bath', axis=1, inplace=True)\n",
" # 提升0.0001\n",
" def month(x):\n",
" month = int(x.split('/')[1])\n",
" return month\n",
" # def day(x):\n",
" # day = int(x.split('/')[2])\n",
" # return day\n",
" # 结果变差\n",
"\n",
" # 分割交易时间\n",
" # data['year']=data['tradeTime'].apply(lambda x:year(x))\n",
" data['month'] = data['tradeTime'].apply(lambda x: month(x))\n",
" # data['day'] = data['tradeTime'].apply(lambda x: day(x))# 结果变差\n",
" # data['pv/uv'] = data['pv'] / data['uv']\n",
" # data['房间总数'] = data['室'] + data['厅'] + data['卫']\n",
"\n",
" # 合并部分配套设施特征\n",
" data['trainsportNum'] = 5 * data['subwayStationNum'] / data['subwayStationNum'].mean() + data['busStationNum'] / \\\n",
" data[\n",
" 'busStationNum'].mean()\n",
" data['all_SchoolNum'] = 2 * data['interSchoolNum'] / data['interSchoolNum'].mean() + data['schoolNum'] / data[\n",
" 'schoolNum'].mean() \\\n",
" + data['privateSchoolNum'] / data['privateSchoolNum'].mean()\n",
" data['all_hospitalNum'] = 2 * data['hospitalNum'] / data['hospitalNum'].mean() + \\\n",
" data['drugStoreNum'] / data['drugStoreNum'].mean()\n",
" data['all_mall'] = data['mallNum'] / data['mallNum'].mean() + \\\n",
" data['superMarketNum'] / data['superMarketNum'].mean()\n",
" data['otherNum'] = data['gymNum'] / data['gymNum'].mean() + data['bankNum'] / data['bankNum'].mean() + \\\n",
" data['shopNum'] / data['shopNum'].mean() + 2 * data['parkNum'] / data['parkNum'].mean()\n",
"\n",
" data.drop(['subwayStationNum', 'busStationNum',\n",
" 'interSchoolNum', 'schoolNum', 'privateSchoolNum',\n",
" 'hospitalNum', 'drugStoreNum', 'mallNum', 'superMarketNum', 'gymNum', 'bankNum', 'shopNum', 'parkNum'],\n",
" axis=1, inplace=True)\n",
" # 提升0.0005\n",
" \n",
"# data['houseType_1sumcsu']=data['Bath'].map(lambda x:str(x))+data['month'].map(lambda x:str(x))\n",
"# data['houseType_2sumcsu']=data['Bath'].map(lambda x:str(x))+data['communityName']\n",
"# data['houseType_3sumcsu']=data['Bath'].map(lambda x:str(x))+data['plate']\n",
" \n",
" data.drop('houseType', axis=1, inplace=True)\n",
" data.drop('tradeTime', axis=1, inplace=True)\n",
" \n",
" data[\"area\"] = data[\"area\"].astype(int)\n",
"\n",
"\n",
" # categorical_feats = ['rentType', 'houseFloor', 'houseToward', 'houseDecoration', 'communityName','region', 'plate']\n",
" categorical_feats = ['rentType', 'houseFloor', 'houseToward', 'houseDecoration', 'region', 'plate','cluster']\n",
"\n",
" return data, categorical_feats"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 计算统计特征"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T13:41:02.458588Z",
"start_time": "2019-12-24T13:41:00.981539Z"
}
},
"outputs": [],
"source": [
"#计算统计特征\n",
"def featureCount(train,test):\n",
" train['data_type'] = 0\n",
" test['data_type'] = 1\n",
" data = pd.concat([train, test], axis=0, join='outer')\n",
" def feature_count(data, features=[]):\n",
" new_feature = 'count'\n",
" for i in features:\n",
" new_feature += '_' + i\n",
" temp = data.groupby(features).size().reset_index().rename(columns={0: new_feature})\n",
" data = data.merge(temp, 'left', on=features)\n",
" return data\n",
"\n",
" data = feature_count(data, ['communityName'])\n",
" data = feature_count(data, ['buildYear'])\n",
" data = feature_count(data, ['totalFloor'])\n",
" data = feature_count(data, ['communityName', 'totalFloor'])\n",
" data = feature_count(data, ['communityName', 'newWorkers'])\n",
" data = feature_count(data, ['communityName', 'totalTradeMoney'])\n",
" new_train = data[data['data_type'] == 0]\n",
" new_test = data[data['data_type'] == 1]\n",
" new_train.drop('data_type', axis=1, inplace=True)\n",
" new_test.drop(['data_type'], axis=1, inplace=True)\n",
" return new_train, new_test\n",
" \n",
"train, test = featureCount(train, test)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## groupby方法生成统计特征"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T13:41:05.546332Z",
"start_time": "2019-12-24T13:41:04.242821Z"
}
},
"outputs": [],
"source": [
"#groupby生成统计特征：mean,std等\n",
"\n",
"def gourpby(train,test):\n",
" train['data_type'] = 0\n",
" test['data_type'] = 1\n",
" data = pd.concat([train, test], axis=0, join='outer')\n",
" columns = ['rentType', 'houseFloor', 'houseToward', 'houseDecoration', 'communityName', 'region', 'plate']\n",
" for feature in columns:\n",
" data[feature] = LabelEncoder().fit_transform(data[feature])\n",
"\n",
" temp = data.groupby('communityName')['area'].agg({'com_area_mean': 'mean', 'com_area_std': 'std'})\n",
" temp.fillna(0, inplace=True)\n",
" data = data.merge(temp, on='communityName', how='left')\n",
" \n",
" data['price_per_area'] = data.tradeMeanPrice / data.area * 100\n",
" temp = data.groupby('communityName')['price_per_area'].agg(\n",
" {'comm_price_mean': 'mean', 'comm_price_std': 'std'})\n",
" temp.fillna(0, inplace=True)\n",
" data = data.merge(temp, on='communityName', how='left')\n",
" \n",
" temp = data.groupby('plate')['price_per_area'].agg(\n",
" {'plate_price_mean': 'mean', 'plate_price_std': 'std'})\n",
" temp.fillna(0, inplace=True)\n",
" data = data.merge(temp, on='plate', how='left')\n",
" data.drop('price_per_area', axis=1, inplace=True)\n",
"\n",
" temp = data.groupby('plate')['area'].agg({'plate_area_mean': 'mean', 'plate_area_std': 'std'})\n",
" temp.fillna(0, inplace=True)\n",
" data = data.merge(temp, on='plate', how='left')\n",
" \n",
" temp = data.groupby(['plate'])['buildYear'].agg({'plate_year_mean': 'mean', 'plate_year_std': 'std'})\n",
" data = data.merge(temp, on='plate', how='left')\n",
" data.plate_year_mean = data.plate_year_mean.astype('int')\n",
" data['comm_plate_year_diff'] = data.buildYear - data.plate_year_mean\n",
" data.drop('plate_year_mean', axis=1, inplace=True)\n",
"\n",
" temp = data.groupby('plate')['trainsportNum'].agg('sum').reset_index(name='plate_trainsportNum')\n",
" data = data.merge(temp, on='plate', how='left')\n",
" temp = data.groupby(['communityName', 'plate'])['trainsportNum'].agg('sum').reset_index(name='com_trainsportNum')\n",
" data = data.merge(temp, on=['communityName', 'plate'], how='left')\n",
" data['trainsportNum_ratio'] = list(map(lambda x, y: round(x / y, 3) if y != 0 else -1,\n",
" data['com_trainsportNum'], data['plate_trainsportNum']))\n",
" data = data.drop(['com_trainsportNum', 'plate_trainsportNum'], axis=1)\n",
"\n",
" temp = data.groupby('plate')['all_SchoolNum'].agg('sum').reset_index(name='plate_all_SchoolNum')\n",
" data = data.merge(temp, on='plate', how='left')\n",
" temp = data.groupby(['communityName', 'plate'])['all_SchoolNum'].agg('sum').reset_index(name='com_all_SchoolNum')\n",
" data = data.merge(temp, on=['communityName', 'plate'], how='left')\n",
" data = data.drop(['com_all_SchoolNum', 'plate_all_SchoolNum'], axis=1)\n",
"\n",
" temp = data.groupby(['communityName', 'plate'])['all_mall'].agg('sum').reset_index(name='com_all_mall')\n",
" data = data.merge(temp, on=['communityName', 'plate'], how='left')\n",
"\n",
" temp = data.groupby('plate')['otherNum'].agg('sum').reset_index(name='plate_otherNum')\n",
" data = data.merge(temp, on='plate', how='left')\n",
" temp = data.groupby(['communityName', 'plate'])['otherNum'].agg('sum').reset_index(name='com_otherNum')\n",
" data = data.merge(temp, on=['communityName', 'plate'], how='left')\n",
" data['other_ratio'] = list(map(lambda x, y: round(x / y, 3) if y != 0 else -1,\n",
" data['com_otherNum'], data['plate_otherNum']))\n",
" data = data.drop(['com_otherNum', 'plate_otherNum'], axis=1)\n",
"\n",
" temp = data.groupby(['month', 'communityName']).size().reset_index(name='communityName_saleNum')\n",
" data = data.merge(temp, on=['month', 'communityName'], how='left')\n",
" temp = data.groupby(['month', 'plate']).size().reset_index(name='plate_saleNum')\n",
" data = data.merge(temp, on=['month', 'plate'], how='left')\n",
"\n",
" data['sale_ratio'] = round((data.communityName_saleNum + 1) / (data.plate_saleNum + 1), 3)\n",
" data['sale_newworker_differ'] = 3 * data.plate_saleNum - data.newWorkers\n",
" data.drop(['communityName_saleNum', 'plate_saleNum'], axis=1, inplace=True)\n",
"\n",
" new_train = data[data['data_type'] == 0]\n",
" new_test = data[data['data_type'] == 1]\n",
" new_train.drop('data_type', axis=1, inplace=True)\n",
" new_test.drop(['data_type'], axis=1, inplace=True)\n",
" return new_train, new_test\n",
"\n",
"train, test = gourpby(train, test)"
]
},
{
"cell_type": "markdown",
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T13:38:33.198959Z",
"start_time": "2019-12-24T13:38:33.193970Z"
}
},
"source": [
"## 聚类方法"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T13:41:25.894916Z",
"start_time": "2019-12-24T13:41:25.241666Z"
}
},
"outputs": [],
"source": [
"#聚类\n",
"def cluster(train,test):\n",
" from sklearn.mixture import GaussianMixture\n",
"\n",
" train['data_type'] = 0\n",
" test['data_type'] = 1\n",
" data = pd.concat([train, test], axis=0, join='outer')\n",
" col = ['totalFloor',\n",
" 'houseDecoration', 'communityName', 'region', 'plate', 'buildYear',\n",
"\n",
" 'tradeMeanPrice', 'tradeSecNum', 'totalNewTradeMoney',\n",
" 'totalNewTradeArea', 'tradeNewMeanPrice', 'tradeNewNum', 'remainNewNum',\n",
"\n",
" 'landTotalPrice', 'landMeanPrice', 'totalWorkers',\n",
" 'newWorkers', 'residentPopulation', 'lookNum',\n",
" 'trainsportNum',\n",
" 'all_SchoolNum', 'all_hospitalNum', 'all_mall', 'otherNum']\n",
"\n",
" # EM\n",
" gmm = GaussianMixture(n_components=3, covariance_type='full', random_state=0)\n",
" data['cluster']= pd.DataFrame(gmm.fit_predict(data[col]))\n",
"\n",
"\n",
" col1 = ['totalFloor','houseDecoration', 'communityName', 'region', 'plate', 'buildYear']\n",
" col2 = ['tradeMeanPrice', 'tradeSecNum', 'totalNewTradeMoney',\n",
" 'totalNewTradeArea', 'tradeNewMeanPrice', 'tradeNewNum', 'remainNewNum',\n",
" 'landTotalPrice', 'landMeanPrice', 'totalWorkers',\n",
" 'newWorkers', 'residentPopulation', 'lookNum',\n",
" 'trainsportNum',\n",
" 'all_SchoolNum', 'all_hospitalNum', 'all_mall', 'otherNum']\n",
" for feature1 in col1:\n",
" for feature2 in col2:\n",
" \n",
" temp = data.groupby(['cluster',feature1])[feature2].agg('mean').reset_index(name=feature2+'_'+feature1+'_cluster_mean')\n",
" temp.fillna(0, inplace=True)\n",
" \n",
" data = data.merge(temp, on=['cluster', feature1], how='left')\n",
" \n",
" \n",
" new_train = data[data['data_type'] == 0]\n",
" new_test = data[data['data_type'] == 1]\n",
" new_train.drop('data_type', axis=1, inplace=True)\n",
" new_test.drop(['data_type'], axis=1, inplace=True)\n",
" \n",
" return new_train, new_test\n",
"\n",
"train, test = cluster(train, test) "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## log平滑"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# 过大量级值取log平滑（针对线性模型有效）\n",
"big_num_cols = ['totalTradeMoney','totalTradeArea','tradeMeanPrice','totalNewTradeMoney', 'totalNewTradeArea',\n",
" 'tradeNewMeanPrice','remainNewNum', 'supplyNewNum', 'supplyLandArea',\n",
" 'tradeLandArea','landTotalPrice','landMeanPrice','totalWorkers','newWorkers',\n",
" 'residentPopulation','pv','uv']\n",
"for col in big_num_cols:\n",
" train[col] = train[col].map(lambda x: np.log1p(x))\n",
" test[col] = test[col].map(lambda x: np.log1p(x))\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#对比特征工程前后线性模型结果情况\n",
"test=test.fillna(0)\n",
"# Lasso回归\n",
"from sklearn.linear_model import Lasso\n",
"lasso=Lasso(alpha=0.1)\n",
"lasso.fit(train,target_train)\n",
"#预测测试集和训练集结果\n",
"y_pred_train=lasso.predict(train)\n",
"y_pred_test=lasso.predict(test)\n",
"\n",
"#对比结果\n",
"from sklearn.metrics import r2_score\n",
"score_train=r2_score(y_pred_train,target_train)\n",
"print(\"训练集结果：\",score_train)\n",
"score_test=r2_score(y_pred_test, target_test)\n",
"print(\"测试集结果：\",score_test)"
]
},
{
"cell_type": "markdown",
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T12:31:08.989972Z",
"start_time": "2019-12-24T12:31:08.986978Z"
}
},
"source": [
"# 特征选择"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"from sklearn.preprocessing import LabelEncoder\n",
"#读取数据\n",
"train = pd.read_csv('')\n",
"test = pd.read_csv('')\n",
"\n",
"target_train = train.pop('tradeMoney')\n",
"target_test = test.pop('tradeMoney')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 相关系数法"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#相关系数法特征选择\n",
"from sklearn.feature_selection import SelectKBest\n",
"\n",
"print(train.shape)\n",
"\n",
"sk=SelectKBest(k=150)\n",
"new_train=sk.fit_transform(train,target_train)\n",
"print(new_train.shape)\n",
"\n",
"# 获取对应列索引\n",
"select_columns=sk.get_support(indices = True)\n",
"# print(select_columns)\n",
"\n",
"# 获取对应列名\n",
"# print(test.columns[select_columns])\n",
"select_columns_name=test.columns[select_columns]\n",
"new_test=test[select_columns_name]\n",
"print(new_test.shape)\n",
"# Lasso回归\n",
"from sklearn.linear_model import Lasso\n",
"\n",
"lasso=Lasso(alpha=0.1)\n",
"lasso.fit(new_train,target_train)\n",
"#预测测试集和训练集结果\n",
"y_pred_train=lasso.predict(new_train)\n",
"\n",
"y_pred_test=lasso.predict(new_test)\n",
"\n",
"#对比结果\n",
"from sklearn.metrics import r2_score\n",
"score_train=r2_score(y_pred_train,target_train)\n",
"print(\"训练集结果：\",score_train)\n",
"score_test=r2_score(y_pred_test, target_test)\n",
"print(\"测试集结果：\",score_test)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Wrapper"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Wrapper\n",
"\n",
"from sklearn.feature_selection import RFE\n",
"from sklearn.linear_model import LinearRegression\n",
"lr = LinearRegression()\n",
"rfe = RFE(lr, n_features_to_select=160)\n",
"rfe.fit(train,target_train)\n",
"\n",
"RFE(estimator=LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None,\n",
" normalize=False),\n",
" n_features_to_select=40, step=1, verbose=0)\n",
"\n",
"\n",
"\n",
"\n",
"select_columns = [f for f, s in zip(train.columns, rfe.support_) if s]\n",
"print(select_columns)\n",
"new_train = train[select_columns]\n",
"new_test = test[select_columns]\n",
"\n",
"# Lasso回归\n",
"from sklearn.linear_model import Lasso\n",
"\n",
"lasso=Lasso(alpha=0.1)\n",
"lasso.fit(new_train,target_train)\n",
"#预测测试集和训练集结果\n",
"y_pred_train=lasso.predict(new_train)\n",
"\n",
"y_pred_test=lasso.predict(new_test)\n",
"\n",
"#对比结果\n",
"from sklearn.metrics import r2_score\n",
"score_train=r2_score(y_pred_train,target_train)\n",
"print(\"训练集结果：\",score_train)\n",
"score_test=r2_score(y_pred_test, target_test)\n",
"print(\"测试集结果：\",score_test)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Embedded\n",
"### 基于惩罚项的特征选择法\n",
"### Lasso(l1)和Ridge(l2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Embedded\n",
"# 基于惩罚项的特征选择法\n",
"# Lasso(l1)和Ridge(l2)\n",
"\n",
"from sklearn.linear_model import Ridge\n",
"\n",
"ridge = Ridge(alpha=5)\n",
"ridge.fit(train,target_train)\n",
"\n",
"Ridge(alpha=5, copy_X=True, fit_intercept=True, max_iter=None, normalize=False,\n",
" random_state=None, solver='auto', tol=0.001)\n",
"\n",
"# 特征系数排序\n",
"coefSort = ridge.coef_.argsort()\n",
"print(coefSort)\n",
"\n",
"\n",
"# 特征系数\n",
"featureCoefSore=ridge.coef_[coefSort]\n",
"print(featureCoefSore)\n",
"\n",
"\n",
"select_columns = [f for f, s in zip(train.columns, featureCoefSore) if abs(s)> 0.0000005 ] \n",
"# 选择绝对值大于0.0000005的特征\n",
"\n",
"new_train = train[select_columns]\n",
"new_test = test[select_columns]\n",
"# Lasso回归\n",
"from sklearn.linear_model import Lasso\n",
"\n",
"lasso=Lasso(alpha=0.1)\n",
"lasso.fit(new_train,target_train)\n",
"#预测测试集和训练集结果\n",
"y_pred_train=lasso.predict(new_train)\n",
"\n",
"y_pred_test=lasso.predict(new_test)\n",
"\n",
"#对比结果\n",
"from sklearn.metrics import r2_score\n",
"score_train=r2_score(y_pred_train,target_train)\n",
"print(\"训练集结果：\",score_train)\n",
"score_test=r2_score(y_pred_test, target_test)\n",
"print(\"测试集结果：\",score_test)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 基于树模型的特征选择法\n",
"### 随机森林 平均不纯度减少（mean decrease impurity"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Embedded\n",
"# 基于树模型的特征选择法\n",
"# 随机森林 平均不纯度减少（mean decrease impurity\n",
"\n",
"\n",
"from sklearn.ensemble import RandomForestRegressor\n",
"rf = RandomForestRegressor()\n",
"# 训练随机森林模型，并通过feature_importances_属性获取每个特征的重要性分数。rf = RandomForestRegressor()\n",
"rf.fit(train,target_train)\n",
"print(\"Features sorted by their score:\")\n",
"print(sorted(zip(map(lambda x: round(x, 4), rf.feature_importances_), train.columns),\n",
" reverse=True))\n",
"\n",
"select_columns = [f for f, s in zip(train.columns, rf.feature_importances_) if abs(s)> 0.00005 ] \n",
"# 选择绝对值大于0.00005的特征\n",
"\n",
"new_train = train[select_columns]\n",
"new_test = test[select_columns]\n",
"\n",
"# Lasso回归\n",
"from sklearn.linear_model import Lasso\n",
"\n",
"lasso=Lasso(alpha=0.1)\n",
"lasso.fit(new_train,target_train)\n",
"#预测测试集和训练集结果\n",
"y_pred_train=lasso.predict(new_train)\n",
"\n",
"y_pred_test=lasso.predict(new_test)\n",
"\n",
"#对比结果\n",
"from sklearn.metrics import r2_score\n",
"score_train=r2_score(y_pred_train,target_train)\n",
"print(\"训练集结果：\",score_train)\n",
"score_test=r2_score(y_pred_test, target_test)\n",
"print(\"测试集结果：\",score_test)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {
"height": "calc(100% - 180px)",
"left": "10px",
"top": "150px",
"width": "307.2px"
},
"toc_section_display": true,
"toc_window_display": true
}
},
"nbformat": 4,
"nbformat_minor": 2
}
RentForecast/Task4 模型选择.ipynb
+192
View File
@@ -0,0 +1,192 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 任务4 模型选择(3天)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 以lightGBM为例"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T13:53:49.093200Z",
"start_time": "2019-12-24T13:53:43.498159Z"
}
},
"outputs": [],
"source": [
"from __future__ import print_function\n",
"import lightgbm as lgb\n",
"import sklearn\n",
"import numpy\n",
"import hyperopt\n",
"from hyperopt import hp, fmin, tpe, STATUS_OK, Trials\n",
"import colorama\n",
"import numpy as np\n",
"\n",
"N_HYPEROPT_PROBES = 500\n",
"HYPEROPT_ALGO = tpe.suggest # tpe.suggest OR hyperopt.rand.suggest\n",
"\n",
"# ----------------------------------------------------------\n",
"\n",
"colorama.init()\n",
"\n",
"# ---------------------------------------------------------------------\n",
"\n",
"def get_lgb_params(space):\n",
" lgb_params = dict()\n",
" lgb_params['boosting_type'] = space['boosting_type'] if 'boosting_type' in space else 'gbdt'\n",
" lgb_params['objective'] = 'regression'\n",
" lgb_params['metric'] = 'rmse'\n",
" lgb_params['learning_rate'] = space['learning_rate']\n",
" lgb_params['num_leaves'] = int(space['num_leaves'])\n",
" lgb_params['min_data_in_leaf'] = int(space['min_data_in_leaf'])\n",
" lgb_params['min_sum_hessian_in_leaf'] = space['min_sum_hessian_in_leaf']\n",
" lgb_params['max_depth'] = -1\n",
" lgb_params['lambda_l1'] = space['lambda_l1'] if 'lambda_l1' in space else 0.0\n",
" lgb_params['lambda_l2'] = space['lambda_l2'] if 'lambda_l2' in space else 0.0\n",
" lgb_params['max_bin'] = int(space['max_bin']) if 'max_bin' in space else 256\n",
" lgb_params['feature_fraction'] = space['feature_fraction']\n",
" lgb_params['bagging_fraction'] = space['bagging_fraction']\n",
" lgb_params['bagging_freq'] = int(space['bagging_freq']) if 'bagging_freq' in space else 1\n",
" lgb_params['nthread'] = 4\n",
" return lgb_params\n",
"\n",
"# ---------------------------------------------------------------------\n",
"\n",
"obj_call_count = 0\n",
"cur_best_score = 0 # 0 or np.inf\n",
"log_writer = open( '../log/lgb-hyperopt-log.txt', 'w' )\n",
"\n",
"\n",
"def objective(space):\n",
" global obj_call_count, cur_best_score\n",
"\n",
" obj_call_count += 1\n",
"\n",
" print('\\nLightGBM objective call #{} cur_best_score={:7.5f}'.format(obj_call_count,cur_best_score) )\n",
"\n",
" lgb_params = get_lgb_params(space)\n",
"\n",
" sorted_params = sorted(space.items(), key=lambda z: z[0])\n",
" params_str = str.join(' ', ['{}={}'.format(k, v) for k, v in sorted_params])\n",
" print('Params: {}'.format(params_str) )\n",
" \n",
" kf = KFold(n_splits=3, shuffle=True, random_state=0)\n",
" out_of_fold = np.zeros(len(X_train))\n",
" for fold, (train_idx, val_idx) in enumerate(kf.split(X_train)):\n",
" D_train = lgb.Dataset(X_train.iloc[train_idx], label=Y_train[train_idx])\n",
" D_val = lgb.Dataset(X_train.iloc[val_idx], label=Y_train[val_idx])\n",
" # Train\n",
" num_round = 10000\n",
" clf = lgb.train(lgb_params,\n",
" D_train,\n",
" num_boost_round=num_round,\n",
" # metrics='mlogloss',\n",
" valid_sets=D_val,\n",
" # valid_names='val',\n",
" # fobj=None,\n",
" # feval=None,\n",
" # init_model=None,\n",
" # feature_name='auto',\n",
" # categorical_feature='auto',\n",
" early_stopping_rounds=200,\n",
" # evals_result=None,\n",
" verbose_eval=False,\n",
" # learning_rates=None,\n",
" # keep_training_booster=False,\n",
" # callbacks=None\n",
" )\n",
" # predict\n",
" nb_trees = clf.best_iteration\n",
" val_loss = clf.best_score['valid_0']\n",
" print('nb_trees={} val_loss={}'.format(nb_trees, val_loss))\n",
" out_of_fold[val_idx] = clf.predict(X_train.iloc[val_idx], num_iteration=nb_trees)\n",
" score = r2_score(out_of_fold, Y_train)\n",
"\n",
" print('val_r2_score={}'.format(score))\n",
"\n",
" log_writer.write('score={} Params:{} nb_trees={}\\n'.format(score, params_str, nb_trees ))\n",
" log_writer.flush()\n",
"\n",
" if score>cur_best_score:\n",
" cur_best_score = score\n",
" print(colorama.Fore.GREEN + 'NEW BEST SCORE={}'.format(cur_best_score) + colorama.Fore.RESET)\n",
" return {'loss': -score, 'status': STATUS_OK}\n",
"\n",
"# --------------------------------------------------------------------------------\n",
"\n",
"space ={\n",
" 'num_leaves': hp.quniform ('num_leaves', 10, 100, 1),\n",
" 'min_data_in_leaf': hp.quniform ('min_data_in_leaf', 10, 100, 1),\n",
" 'feature_fraction': hp.uniform('feature_fraction', 0.75, 1.0),\n",
" 'bagging_fraction': hp.uniform('bagging_fraction', 0.75, 1.0),\n",
" 'learning_rate': hp.uniform('learning_rate', 0, 0.01),\n",
"# 'learning_rate': hp.loguniform('learning_rate', -5.0, -2.3),\n",
" 'min_sum_hessian_in_leaf': hp.loguniform('min_sum_hessian_in_leaf', 0, 2.3),\n",
" 'max_bin': hp.quniform ('max_bin', 88, 200, 1),\n",
" 'bagging_freq': hp.quniform ('bagging_freq', 1, 15, 1),\n",
" 'lambda_l1': hp.uniform('lambda_l1', 0, 10 ),\n",
" 'lambda_l2': hp.uniform('lambda_l2', 0, 10 ),\n",
" }\n",
"\n",
"trials = Trials()\n",
"best = hyperopt.fmin(fn=objective,\n",
" space=space,\n",
" algo=HYPEROPT_ALGO,\n",
" max_evals=N_HYPEROPT_PROBES,\n",
" trials=trials,\n",
" verbose=1)\n",
"\n",
"print('-'*50)\n",
"print('The best params:')\n",
"print( best )\n",
"print('\\n\\n')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": true
}
},
"nbformat": 4,
"nbformat_minor": 2
}
RentForecast/Task5 模型融合.ipynb
+338
View File
File diff suppressed because one or more lines are too long
RentForecast/Task6 竞赛整理.ipynb
+98
View File
@@ -0,0 +1,98 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 任务6 比赛整理(2天)\n",
"\n",
"以赛方最后给的答辩模板为主线整理比赛思路，模拟比赛答辩环节，进行比赛整理。\n",
"\n",
"## Part1\n",
"\n",
"**参赛队成员简介**\n",
"\n",
" （这边主要介绍成员情况，如果有竞赛获奖就最好啦）\n",
"\n",
"ps：这个由于是模拟比赛所以这个部分可以不写哦\n",
"\n",
"## Part2\n",
"\n",
"**参赛作品概述**\n",
"\n",
"## Part3\n",
"\n",
"**关键技术阐述（数据清洗、特征工程、模型、模型融合，并强调对比赛提分最有帮助的部分）**\n",
"\n",
"## Part4\n",
"\n",
"**探索与创新（写明做的与众不同的创新点）**\n",
"\n",
"## Part5\n",
"\n",
"**实施与优化过程（在过程中尝试过的方法都可以提及并总结）**\n",
"\n",
"## Part6\n",
"\n",
"**其他（有其他需要补充的可以写在这个部分）**\n",
"\n",
"\n",
"\n",
"**（注：因为比赛是和企业合作，并具有实际意义的比赛，所以强调你的代码模型的实际意义，商业价值都会在答辩环节有帮助哦）**\n",
"\n",
"\n",
"\n",
"\n",
"\n",
"**任务时间 2天**\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"ExecuteTime": {
"end_time": "2019-12-24T12:40:43.526848Z",
"start_time": "2019-12-24T12:40:43.515876Z"
}
},
"source": [
"**参考连接：**\n",
"[https://blog.csdn.net/qq_39756719/article/details/95634744](https://blog.csdn.net/qq_39756719/article/details/95634744)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.3"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": true
}
},
"nbformat": 4,
"nbformat_minor": 2
}
RentForecast/数据集/test_a.csv
+2471
View File
File diff suppressed because it is too large Load Diff
RentForecast/数据集/train_data.csv
+41442
View File
File diff suppressed because it is too large Load Diff
RentForecast/数据集/数据集字段说明.csv
+52
View File
@@ -0,0 +1,52 @@
类别,字段,字段内容,说明
租赁房源,ID,房屋编号,每间房屋编号唯一
,area,房屋面积,租赁房间的面积单位平方米
,ren tType,1出租方式:整租/合租/未知,房间的出租方式
,houseType,房型,房屋的形状
,houseFloor,所在楼层,房间所在的楼层(分为高、中、低三类)
,totalFloor,总楼层数,房间所在楼栋的总楼层数
,houseToward,房屋朝向,房间的朝向
,houseDecorati on,房屋装修,房屋内的装修情况
小区信息,commu nityName,小区名称,房屋所在小区(XQ00001)
,city,城市,城市(SH)
,region,区域,城市行政区域(RG00001)
,plate,板块,区域板块(BK00001)
,buildYear,小区建筑年代,小区建筑年代
,saleSecHouseNum,该板块当月挂牌房源数,板块当月二手房挂牌房源数
配套设施,subwayStati onNum,该板块地铁站数量,板块当前地铁站总数量
,busStati onNum,该板块公交站数量,板块当前公交站总数量
,interSchoolNum,该板块国际学校数量,板块当前国际学校总数量
,schoolNum,该板块公立学校数量,板块当前公立学校总数量
,privateSchoolNum,该板块私立学校数量,板块当前私立学校总数量
,hospitalNum,该板块综合医院数量,板块当前综合医院总数量
,drugStoreNum,该板块药房数量,板块当前药房总数量
,gymNum,该板块健身中心数量,板块当前健身中心总数量
,bankNum,该板块银行数量,板块当前银行总数量
,shopNum,该板块便利店数量,板块当前便利店总数量
,parkNum,该板块公园数量,板块当前公园总数量
,mallNum,该板块购物中心教量,板块当前购物中心总数量
,superMarketNum,该板块超市数量,板块当前超市总数量
二手房,totalTradeMoney ,该板块当月二手房成交总金额（元）,当月板块二手房成交总金额
,totalTradeArea, 该板块当月二手房成交总面积（平米）,当月板块二手房成交总面积
,tradeMea nPrice,该板块当月二手房成交均价（元/平 |米）,当月板块二手房成交均价
,tradeSecNum,该板块当月二手房成交套数,当月板块二手房成交总套数
新房,totalNewTradeMo n ey,该板块当月新房成交总金额,当月板块新房成交总套数
,tota IN ewTra deArea,该板块当月新房成交总面积,当月板块新房成交总面积
,tradeNewMeanPric e,该板块当月新房成交均价,当月板块新房成交均价
,tradeNewNum,该板块当月新房成交套数,当月板块新房成交总套数
,remainNewNum,该板块当月新房剩余未成交套数,当月板块新房剩余未成交套数
,supplyNewNum,该板块当月新房供应套数,当月板块新房供应总套数
土地,supply La ndNum,该板块当月土地供应幅数,当月板块土地供应幅数
,supply La ndArea,该板块当月土地供应面积,当月板块土地供应面积
,tradeLa ndNum,该板块当月土地成交幅数,当月板块土地成交幅数 1 .
,tradeLa ndArea,该板块当月土地成交面积,当月板块土地板楼面积
,Ian dTotalPrice,该板块当月土地成交总价,当月板块土地成交总价
,landMeanPrice,板块土地楼板价（元/* ）,板块楼板均价（板块）
人口,totalworkers,当前板块现有办公人数,当前板块现有办公人数（包括本月新增 招聘）
,newWorkers,该板块当月流入人口 （新招聘人数）,本月新增招聘人数
,reside ntPopulatio n,当前板块常住人口,当前板块常住人口
客户,PV,该板块当月租客浏览网页次数,当月板块租客浏览网页次数
,uv,该板块当月租客浏览网页总人数,当月板块租客浏览网页总人数
,lookNum,带看次数,线下看房次数
真实租金,tradeTime,成交日期 ,租房的日期
, tradeMoney,成交租金,成交房屋每月租金
1 类别 字段 字段内容 说明
2 租赁房源 ID 房屋编号 每间房屋编号唯一
3 area 房屋面积 租赁房间的面积单位平方米
4 ren tType 1出租方式:整租/合租/未知 房间的出租方式
5 houseType 房型 房屋的形状
6 houseFloor 所在楼层 房间所在的楼层(分为高、中、低三类)
7 totalFloor 总楼层数 房间所在楼栋的总楼层数
8 houseToward 房屋朝向 房间的朝向
9 houseDecorati on 房屋装修 房屋内的装修情况
10 小区信息 commu nityName 小区名称 房屋所在小区(XQ00001)
11 city 城市 城市(SH)
12 region 区域 城市行政区域(RG00001)
13 plate 板块 区域板块(BK00001)
14 buildYear 小区建筑年代 小区建筑年代
15 saleSecHouseNum 该板块当月挂牌房源数 板块当月二手房挂牌房源数
16 配套设施 subwayStati onNum 该板块地铁站数量 板块当前地铁站总数量
17 busStati onNum 该板块公交站数量 板块当前公交站总数量
18 interSchoolNum 该板块国际学校数量 板块当前国际学校总数量
19 schoolNum 该板块公立学校数量 板块当前公立学校总数量
20 privateSchoolNum 该板块私立学校数量 板块当前私立学校总数量
21 hospitalNum 该板块综合医院数量 板块当前综合医院总数量
22 drugStoreNum 该板块药房数量 板块当前药房总数量
23 gymNum 该板块健身中心数量 板块当前健身中心总数量
24 bankNum 该板块银行数量 板块当前银行总数量
25 shopNum 该板块便利店数量 板块当前便利店总数量
26 parkNum 该板块公园数量 板块当前公园总数量
27 mallNum 该板块购物中心教量 板块当前购物中心总数量
28 superMarketNum 该板块超市数量 板块当前超市总数量
29 二手房 totalTradeMoney 该板块当月二手房成交总金额（元） 当月板块二手房成交总金额
30 totalTradeArea 该板块当月二手房成交总面积（平米） 当月板块二手房成交总面积
31 tradeMea nPrice 该板块当月二手房成交均价（元/平 |米） 当月板块二手房成交均价
32 tradeSecNum 该板块当月二手房成交套数 当月板块二手房成交总套数
33 新房 totalNewTradeMo n ey 该板块当月新房成交总金额 当月板块新房成交总套数
34 tota IN ewTra deArea 该板块当月新房成交总面积 当月板块新房成交总面积
35 tradeNewMeanPric e 该板块当月新房成交均价 当月板块新房成交均价
36 tradeNewNum 该板块当月新房成交套数 当月板块新房成交总套数
37 remainNewNum 该板块当月新房剩余未成交套数 当月板块新房剩余未成交套数
38 supplyNewNum 该板块当月新房供应套数 当月板块新房供应总套数
39 土地 supply La ndNum 该板块当月土地供应幅数 当月板块土地供应幅数
40 supply La ndArea 该板块当月土地供应面积 当月板块土地供应面积
41 tradeLa ndNum 该板块当月土地成交幅数 当月板块土地成交幅数 1 .
42 tradeLa ndArea 该板块当月土地成交面积 当月板块土地板楼面积
43 Ian dTotalPrice 该板块当月土地成交总价 当月板块土地成交总价
44 landMeanPrice 板块土地楼板价（元/* ） 板块楼板均价（板块）
45 人口 totalworkers 当前板块现有办公人数 当前板块现有办公人数（包括本月新增 招聘）
46 newWorkers 该板块当月流入人口 （新招聘人数） 本月新增招聘人数
47 reside ntPopulatio n 当前板块常住人口 当前板块常住人口
48 客户 PV 该板块当月租客浏览网页次数 当月板块租客浏览网页次数
49 uv 该板块当月租客浏览网页总人数 当月板块租客浏览网页总人数
50 lookNum 带看次数 线下看房次数
51 真实租金 tradeTime 成交日期 租房的日期
52 tradeMoney 成交租金 成交房屋每月租金
RentForecast/数据集/评分文件/sub_a_913.csv
+2470
View File
File diff suppressed because it is too large Load Diff