机器学习 xgboost 笔记

一、数据预处理、特征工程

类别变量 labelencoder就够了，使用onehotencoder反而会降低性能。其他处理方式还有均值编码（对于存在大量分类的特征，通过监督学习，生成数值变量）、转换处理（低频分类合并）、特征构造（结合其他数值变量生成新特征）。

二、模型调参

网格调参、随机调参。

模型参数没必要太过于纠结，调参到合适的地步就好了，太过沉迷会导致过拟合。

三、样本划分

一般是对数据集按7:3、8:2、7.5:2.5等划分为训练集和测试集。

更用心一点，把数据集随机划分为k折，以任意一部分为测试集，其余部分为训练集，建立k个模型。分别调参。最后对预测结果求平均值（加权或单纯求平均）。这种思路原理在于每个模型都会存在一定方差，会学到部分特征，通过对多个模型求平均值，可以起到消除误差的作用。

我在sofasofa上，第一个练习题排名25/257。rmse：14.925

代码如下：

# -*- coding: utf-8 -*-
"""
Created on Sat Sep  1 21:20:29 2018
@author: 蚂蚁不在线
从原理上来说，机器学习最靠谱的调参方法就是对训练集进行n折交叉验证。
单纯划分训练集和测试集调参的过程中，不可避免地在训练集上过拟合。
k折交叉划分训练集、测试集。
"""
import pandas as pd
import numpy as np
from xgboost import XGBRegressor
#from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import KFold
from sklearn import metrics

##网格搜索

def gsearcher(f_train,t_train,param_test):
    gs=GridSearchCV(estimator=XGBRegressor(objective='reg:linear',
                                           eval_metric='rmse'),
                    param_grid=param_test,
                    verbose=1,
                    cv=3)
    gs.fit(f_train,t_train)
    gs.score=np.sqrt(abs(gs.best_score_))
    return gs
##随机搜索
def rsearcher(f_train,t_train,param_test):
    rs=RandomizedSearchCV(estimator=XGBRegressor(objective='reg:linear'),
                          param_distributions=param_test,
                          verbose=1,
                          n_iter=100,
                          cv=3,
                          scoring='neg_mean_squared_error')
    rs.fit(f_train,t_train)
    rs.score=np.sqrt(abs(rs.best_score_))
    return rs
#k折交叉验证
def kmodel(train,y_train,param,test):
    result,col,score=pd.DataFrame(),0,[]
    kf = KFold(n_splits=5,shuffle=True,random_state=0)
    model=XGBRegressor(objective='reg:linear',
                       eval_metric='rmse',
                       n_estimators=param['n_estimators'],
                       max_depth=param['max_depth'],
                       learning_rate=param['learning_rate'],
                       subsample=param['subsample'],
                       colsample_bytree=param['colsample_bytree'],
                       min_child_weight=param['min_child_weight'],
                       reg_lambda=param['reg_lambda'],
                       reg_alpha=param['reg_alpha'],
                       gamma=param['gamma']
                       )
    scores=[]
    for train_index , test_index in kf.split(train):
        col+=1
        ktrain=train.iloc[list(train_index),:]
        ktest=train.iloc[list(test_index),:]
        y_ktrain=y_train[list(train_index)]
        y_ktest=y_train[list(test_index)]
        model.fit(ktrain,y_ktrain)
        yp=model.predict(ktest)
        score=np.sqrt(abs(metrics.mean_squared_error(y_ktest,yp)))
        print(score)
        scores.append(score)
        yp=model.predict(test)
        result['result_'+str(col)]=yp
        result['result_'+str(col)]=result['result_'+str(col)].apply(lambda x:0 if x<0 else x)
    result['score']=result.apply(lambda x:x.mean(),axis=1)
    print(sum(scores)/5)
    return result

# 读取数据
train = pd.read_csv("train.csv")
test = pd.read_csv("test.csv")
submit = pd.read_csv("sample_submit.csv")
# 删除id
train.drop(['id'], axis=1, inplace=True)
test.drop(['id'], axis=1, inplace=True)
# 取出训练集的y
t_train = train.pop('y')
#train,test=data_process(train,test)
param_test={'n_estimators':range(100,120,1),
            'max_depth':range(5,11,1),
            'learning_rate':[0.1+0.01*x for x in range(6)],
            'subsample':[0.7+0.01*x for x in range(11)],
            'colsample_bytree':[0.7+0.01*x for x in range(11)],
            'min_child_weight':range(5,15,1),
            'reg_lambda':range(1,3,2),
            'reg_alpha':range(2,8,1),
            'gamma':range(2,6,1)
            }
rs=rsearcher(train,t_train,param_test)
print('最优参数:',rs.best_params_,rs.score)
with open('log.txt','a+',encoding='utf-8') as f:
    f.write(str(rs.score)+str(rs.best_params_)+'\n')
result=kmodel(train,t_train,rs.best_params_,test)
submit['y']=result['score']
submit.to_csv('submit_0903_1.csv',index=False)