一、基于统计分析库statsmodels

1.数据读取

import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

df = pd.read_csv('C:\house_prices.csv')

2.数据清洗

def outlier_test(data, column, method=None, z=2):
    if method == None:
        print(f'以 {column} 列为依据，使用 上下截断点法(iqr) 检测异常值...')
        print('=' * 70)
        column_iqr = np.quantile(data[column], 0.75) - np.quantile(data[column], 0.25)
        (q1, q3) = np.quantile(data[column], 0.25), np.quantile(data[column], 0.75)
        upper, lower = (q3 + 1.5 * column_iqr), (q1 - 1.5 * column_iqr) 
        outlier = data[(data[column] <= lower) | (data[column] >= upper)]
        print(f'第一分位数: {q1}, 第三分位数：{q3}, 四分位极差：{column_iqr}')
        print(f"上截断点：{upper}, 下截断点：{lower}")
        return outlier, upper, lower
    if method == 'z':
        print(f'以 {column} 列为依据，使用 Z 分数法，z 分位数取 {z} 来检测异常值...')
        print('=' * 70)
        mean, std = np.mean(data[column]), np.std(data[column])
        upper, lower = (mean + z * std), (mean - z * std)
        print(f"取 {z} 个 Z分数：大于 {upper} 或小于 {lower} 的即可被视为异常值。")
        print('=' * 70)
        outlier = data[(data[column] <= lower) | (data[column] >= upper)]
        return outlier, upper, lower

outlier, upper, lower = outlier_test(data=df, column='price', method='z')
outlier.info(); outlier.sample(5)
df.drop(index=outlier.index, inplace=True)

3.数据分析

def heatmap(data, method='pearson', camp='RdYlGn', figsize=(10 ,8)):
    
    plt.figure(figsize=figsize, dpi= 80)
    sns.heatmap(data.corr(method=method), 
                xticklabels=data.corr(method=method).columns, 
                yticklabels=data.corr(method=method).columns, cmap=camp, 
                center=0, annot=True)
heatmap(data=df, figsize=(6,5))

4.拟合及模型优化

import statsmodels.api as sm
from statsmodels.formula.api import ols
from statsmodels.stats.anova import anova_lm
from statsmodels.formula.api import ols
from statsmodels.formula.api import ols

lm = ols('price ~ area + bedrooms + bathrooms', data=df).fit()
lm.summary()

nominal_data = df['neighborhood']

dummies = pd.get_dummies(nominal_data)
dummies.sample() 

dummies.drop(columns=['C'], inplace=True)
dummies.sample()

results = pd.concat(objs=[df, dummies], axis='columns')  
results.sample(3)
lm = ols('price ~ area + bedrooms + bathrooms + A + B', data=results).fit()
lm.summary()

结果

自定义方差膨胀因子的检测公式：

def vif(df, col_i):
    """
    df: 整份数据
    col_i：被检测的列名
    """
    cols = list(df.columns)
    cols.remove(col_i)
    cols_noti = cols
    formula = col_i + '~' + '+'.join(cols_noti)
    r2 = ols(formula, df).fit().rsquared
    return 1. / (1. - r2)
    
test_data = results[['area', 'bedrooms', 'bathrooms', 'A', 'B']]
for i in test_data.columns:
    print(i, '\t', vif(df=test_data, col_i=i))
lm = ols(formula='price ~ area + bathrooms + A + B', data=results).fit()
lm.summary()

结果：

二、Excel重做多元线性回归

选择数据分析功能，选择回归选项
接着选择xy的值域
x的值是area、bedroom和bathroom y值为price
点击确定

最终结果price=10072.1+345.911 area-2925.8bedroom+7345.39*bathroom

三、机器学习库Sklearn库重做多元线性回归

1.直接求解

import pandas as pd
import numpy as np
import math
import matplotlib.pyplot as plt 
from sklearn import linear_model 
data = pd.read_csv('C:\house_prices.csv') 
data.head() 
new_data=data.iloc[:,1:] 
new_data.head()
new_data.corr() 
x_data = new_data.iloc[:, 1:4]
y_data = new_data.iloc[:, -1] 
print(x_data, y_data, len(x_data))
print("回归系数：", model.coef_)
print("截距：", model.intercept_)
print('回归方程: price=',model.coef_[0],'*area +',model.coef_[1],'*bedrooms +',model.coef_[2],'*bathromms +',model.intercept_)

2.数据清洗求解

new_data_Z=new_data.iloc[:,0:]
new_data_IQR=new_data.iloc[:,0:]
def outlier_test(data, column, method=None, z=2):
    
    if method == None:
        print(f'以 {column} 列为依据，使用 上下截断点法(iqr) 检测异常值...')
        print('=' * 70)
        column_iqr = np.quantile(data[column], 0.75) - np.quantile(data[column], 0.25)
        (q1, q3) = np.quantile(data[column], 0.25), np.quantile(data[column], 0.75)
        upper, lower = (q3 + 1.5 * column_iqr), (q1 - 1.5 * column_iqr)
        outlier = data[(data[column] <= lower) | (data[column] >= upper)]
        print(f'第一分位数: {q1}, 第三分位数：{q3}, 四分位极差：{column_iqr}')
        print(f"上截断点：{upper}, 下截断点：{lower}")
        return outlier, upper, lower
    
    if method == 'z':
        
        print(f'以 {column} 列为依据，使用 Z 分数法，z 分位数取 {z} 来检测异常值...')
        print('=' * 70)    
        mean, std = np.mean(data[column]), np.std(data[column])
        upper, lower = (mean + z * std), (mean - z * std)
        print(f"取 {z} 个 Z分数：大于 {upper} 或小于 {lower} 的即可被视为异常值。")
        print('=' * 70)
        outlier = data[(data[column] <= lower) | (data[column] >= upper)]
        return outlier, upper, lower

outlier, upper, lower = outlier_test(data=new_data_Z, column='price', method='z')
outlier.info(); outlier.sample(5)

new_data_Z.drop(index=outlier.index, inplace=True)

print("原数据相关性矩阵")
new_data.corr()

print("Z方法处理的数据相关性矩阵")
new_data_Z.corr()

print("IQR方法处理的数据相关性矩阵")
new_data_IQR.corr()

x_data = new_data_Z.iloc[:, 1:4]
y_data = new_data_Z.iloc[:, -1]
model = linear_model.LinearRegression()
model.fit(x_data, y_data)
print("回归系数：", model.coef_)
print("截距：", model.intercept_)
print('回归方程: price=',model.coef_[0],'*area +',model.coef_[1],'*bedrooms +',model.coef_[2],'*bathromms +',model.intercept_)

x_data = new_data_IQR.iloc[:, 1:4]
y_data = new_data_IQR.iloc[:, -1]
model = linear_model.LinearRegression()
model.fit(x_data, y_data)
print("回归系数：", model.coef_)
print("截距：", model.intercept_)
print('回归方程: price=',model.coef_[0],'*area +',model.coef_[1],'*bedrooms +',model.coef_[2],'*bathromms +',model.intercept_)

3.结果对比

不作处理的数据求解的结果为:
price= 345.911018840024 *area + -2925.806324666705 *bedrooms + 7345.391713693825 *bathromms + 10072.107046726742
采用Z方式清洗数据的求解结果为:
price= 226.4211697383351 *area + 49931.50311720713 *bedrooms + -12224.71724496588 *bathromms + 64356.04135007458
采用IQR放心清洗数据的求解结果为:
price= 242.6111551782956 *area + 41547.43068790577 *bedrooms + -6415.78250090158 *bathromms + 58018.13845504692