import pandas as pd

import matplotlib.pyplot as plt

# matplotlib画图注释中文需要设置

from matplotlib.font_manager import FontProperties

titleYW_font_set = FontProperties(fname=r"c:\windows\fonts\Gabriola.ttf", size=15)

test = pd.read_csv("test.csv")

train = pd.read_csv("train.csv")

gender_submission = pd.read_csv("gender_submission.csv")

# print(test.head())

# print(train.head())

print(train.info())

# ----------------------------数据处理-----------------------------

# 数据可视化

# # --------------对Name的处理----------------

# train_test_data = [train]

# for dataset in train_test_data:

# dataset['Title'] = dataset['Name'].str.extract(' ([A-Za-z]+)\.', expand=False)

# print(train['Title'].value_counts())

# # 统计名字前缀

#

# title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,

# "Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,

# "Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }

# for dataset in train_test_data:

# dataset['Title'] = dataset['Title'].map(title_mapping)

# --------------对Pclass的处理--------------

# 看看哪种乘客等级下的存活率高

train_pclass_0 = train['Pclass'][train['Survived'] == 0].value_counts()

train_pclass_1 = train['Pclass'][train['Survived'] == 1].value_counts()

train_pclass_01 = pd.concat([train_pclass_0, train_pclass_1], axis=1)

train_pclass_01.columns = ['Not_Surived', 'Survived']

train_pclass_01.plot(kind='bar', alpha=0.9)

plt.xticks([0, 1, 2], ['Pclass_1', 'Pclass_2', 'Pclass_3'], rotation=0)

plt.grid(linestyle="--", color="green", alpha=0.5)

plt.title('Survived_Rate in Pclass', size=20)

# --------------对Sex的处理--------------

# 看看那种性别下的乘客存活率高

train_Sex_0 = train['Sex'][train['Survived'] == 0].value_counts()

train_Sex_1 = train['Sex'][train['Survived'] == 1].value_counts()

train_Sex_01 = pd.concat([train_Sex_0, train_Sex_1], axis=1)

train_Sex_01.columns = ['Not_Surived', 'Survived']

train_Sex_01.plot(kind='bar', alpha=0.9)

plt.xticks(rotation=0)

plt.grid(linestyle="--", color="green", alpha=0.5)

plt.title('Survived_Rate in Sex', size=20)

# --------------对Embarked的处理--------------

# 看看那种登船港口下的乘客存活率高

train_Embarked_0 = train['Embarked'][train['Survived'] == 0].value_counts()

train_Embarked_1 = train['Embarked'][train['Survived'] == 1].value_counts()

train_Embarked_01 =pd.concat([train_Embarked_0, train_Embarked_1], axis=1)

train_Embarked_01.columns = ['Not_Surived', 'Survived']

train_Embarked_01.plot(kind='bar', alpha=0.9)

plt.xticks(rotation=0)

plt.grid(linestyle="--", color="green", alpha=0.5)

plt.title('Survived_Rate in Embarked', size=20)

# 查看缺失值

# print(train.isnull().sum())

# 填补空缺值

train['Age'].fillna(train['Age'].median(), inplace=True)

# print(train['Age'].describe()) # max80,min0.42

# --------------对Age的处理--------------

# 对年龄进行离散化，查看每一组的存活率

# 等宽离散化函数

bins = pd.IntervalIndex.from_tuples([(0, 13), (13, 26),(26,39), (39, 52), (52, 65), (65,90)])

train['Age_set'] = pd.cut(train['Age'], bins, labels=['child', 'Teenager', 'universe', 'Adults', 'elder', 'old man'])

# 看看那种年龄段的乘客存活率高

train_Age_set_0 = train['Age_set'][train['Survived'] == 0].value_counts()

train_Age_set_1 = train['Age_set'][train['Survived'] == 1].value_counts()

train_Age_set_01 =pd.concat([train_Age_set_0, train_Age_set_1], axis=1)

train_Age_set_01.columns = ['Not_Surived', 'Survived']

train_Age_set_01.plot(kind='bar', alpha=0.9)

plt.xticks(rotation=0)

plt.grid(linestyle="--", color="green", alpha=0.5)

plt.title('Survived_Rate in Age_Set', size=20)

# --------------对SibSp和Parch的处理--------------

# 把SibSp与Parch相加

train['Family_N'] = train['Parch'] + train['SibSp']+1

# print(train[['Family_N', 'Survived']])

# 分组，按不同的家人数分组

bins = pd.IntervalIndex.from_tuples([(0, 1), (1, 2), (2, 20)])

train['Family_N'] = pd.cut(train['Family_N'], bins)

# 看看那种家庭人数的乘客存活率高

train_Family_N_0 = train['Family_N'][train['Survived'] == 0].value_counts()

train_Family_N_1 = train['Family_N'][train['Survived'] == 1].value_counts()

train_Family_N_01 = pd.concat([train_Family_N_0, train_Family_N_1], axis=1)

train_Family_N_01.columns = ['Not_Surived', 'Survived']

train_Family_N_01.plot(kind='bar', alpha=0.9)

plt.xticks([0, 1, 2], ['one', 'more_than_three', 'two'], rotation=0)

plt.grid(linestyle="--", color="green", alpha=0.5)

plt.title('Survived_Rate in Faminly_N', size=20)

# plt.show()

# train.info()

# train.drop(['SibSp', 'Parch', 'Ticket'], axis=1, inplace=True)

# --------------对Cabin的处理--------------

# 对已知的Cbiin进行分组，聚合时采用众数的方法

# 这里构建数据透视表即可

train_notna = train.dropna()

train_C_F = pd.pivot_table(data=train_notna[['Cabin', 'Fare']], index='Cabin', values='Fare',

aggfunc=lambda x: x.mode())

# print(train_C_F)

# 发现众数可能不止一个,所以进行分离众数的操作

for i in range(train_C_F.shape[0]):

if type(train_C_F['Fare'][i]) != type(train_C_F['Fare'][1]):

train_C_F['Fare'][i] = train_C_F['Fare'][i][0]

# 对众数进行排序

train_C_F_sort = train_C_F.sort_values(by=['Fare'])

# print(train_C_F_sort)

# 对缺失的Cabin进行填补

# 首先找出空白处

train_bool = train['Cabin'].isnull()

# print(train_bool)

na_index = train_bool[train_bool == True].index

# 从上述的index来赋予客舱位置

for i in na_index:

for j in range(train_C_F_sort.shape[0]):

if train['Fare'][i] <= train_C_F_sort['Fare'][j]:

train['Cabin'][i] = train_C_F_sort.index[j]

break

# print(train['Cabin'])

# -----------------------------------------------------------------

# 查看列名

# print(train.columns)

# # 提取出训练集

X_train = train.drop(['Survived', 'PassengerId', 'Name', 'Age','Fare','SibSp', 'Parch', 'Ticket'], axis=1)

# X_train = train.drop(['Survived', 'PassengerId', 'Name', 'Age_set', 'SibSp', 'Parch', 'Ticket'], axis=1)

Y_train = train['Survived']

# print(X_train.columns)

# 哑变量处理

# 把空白值也当作变量处理

X_train = pd.get_dummies(X_train, columns=['Pclass', 'Sex', 'Cabin', 'Embarked', 'Age_set', 'Family_N'],

dummy_na=True)

# X_train = pd.get_dummies(X_train, columns=['Pclass', 'Sex', 'Cabin', 'Embarked', 'Family_N'],

# dummy_na=True)

X = X_train

y = Y_train

# 数据集划分

from sklearn.model_selection import train_test_split

# 标准化

# X_train['Age'].transform(lambda x: (x - x.min())/(x.max()-x.min()))

# X_train['Fare'].transform(lambda x: (x - x.min())/(x.max()-x.min()))

X_train, X_test, y_train, y_test = train_test_split(X_train,Y_train, test_size=0.2, random_state=123)

# # 标准化

# from sklearn.preprocessing import StandardScaler

# Standard = StandardScaler().fit(X_train) # 训练产生标准化的规则,因为数据集分为训练与测试，测试相当于后来的。

#

# Xtrain = Standard.transform(X_train) # 将规则应用于训练集

# Xtest = Standard.transform(X_test) # 将规则应用于测试集

# 进行分类算法

# from sklearn.ensemble import GradientBoostingClassifier

# from sklearn import linear_model

from sklearn.neighbors import KNeighborsClassifier

# clf = GradientBoostingClassifier().fit(X_train, y_train)

# clf = linear_model.SGDClassifier().fit(Xtrain, y_train)

clf = KNeighborsClassifier(n_neighbors=10).fit(X_train,y_train)

y_pred =clf.predict(X_test)

# y_pred = clf.predict(Xtest)

# clf = linear_model.SGDClassifier().fit(X_train, y_train)

# y_pred = clf.predict(X_test)

# 判定分类算法

from sklearn.metrics import classification_report, auc

print(classification_report(y_test, y_pred))

# 绘制roc曲线

from sklearn.metrics import roc_curve

import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif'] = 'SimHei' # 改字体

# 求出ROC曲线的x轴和Y轴

fpr, tpr, thresholds = roc_curve(y_test, y_pred)

print(auc(fpr, tpr))

plt.figure(figsize=(10, 6))

plt.xlim(0, 1) # 设定x轴的范围

plt.ylim(0.0, 1.1) # 设定y轴的范围

plt.xlabel('假正率')

plt.ylabel('真正率')

plt.plot(fpr, tpr, linewidth=2, linestyle="-", color='red')

plt.title('Line Roc of X_train by estimator KNN', size=20)

# plt.show()

# # 交叉验证

# from sklearn.cross_validation import cross_val_score

# k_score = []

# for i in range(1,50):

# knn = KNeighborsClassifier(n_neighbors=i)

# score = cross_val_score(knn,X,y,scoring='accuracy',cv=5)

# k_score.append(score.mean())

# print(k_score)

# ----------------------------------------------------------------------------

# 测试test

# 对测试集做与训练集类似的操作

# 填补空缺值

test['Age'].fillna(test['Age'].median(), inplace=True)

# test.info()

# 寻找Fare空值

# tt = test['Fare'].isnull()

# print(tt.sort_values()) 空值index为152

# print(test[151:153][['Fare','Cabin']]) # 发现此行数据fare 与 cabin均为空，所以授予其Cabin为随便一个即可,或者删除

test.dropna(subset=['Fare'],inplace=True)

# 对age离散化时必须以训练集的规则

# test.info()

# --------------对Age的处理--------------

# 对年龄进行离散化，查看每一组的存活率

# 等宽离散化函数

bins = pd.IntervalIndex.from_tuples([(0, 13), (13, 26),(26,39),(39, 52), (52, 65), (65,90)])

test['Age_set'] = pd.cut(test['Age'], bins, labels=['child', 'Teenager', 'universe', 'Adults', 'elder', 'old man'])

# test.info()

# --------------对SibSp和Parch的处理--------------

# 把SibSp与Parch相加

test['Family_N'] = test['Parch'] + test['SibSp']+1

# print(train[['Family_N', 'Survived']])

# 分组，按不同的家人数分组

bins = pd.IntervalIndex.from_tuples([(0, 1), (1, 2), (2, 20)])

test['Family_N'] = pd.cut(test['Family_N'], bins)

# 对缺失的Cabin进行填补

# 首先找出空白处

test_bool = test['Cabin'].isnull()

# print(train_bool)

na_index = test_bool[test_bool == True].index

# 从上述的index来赋予客舱位置

for i in na_index:

for j in range(train_C_F_sort.shape[0]):

if test['Fare'][i] <= train_C_F_sort['Fare'][j]:

test['Cabin'][i] = train_C_F_sort.index[j]

break

# print(train['Cabin'])

# test.info()

X_test = test.drop(['PassengerId', 'Name', 'Age','Fare','SibSp', 'Parch', 'Ticket'], axis=1)

y_test = gender_submission.drop(index=152)

y_test = y_test['Survived'].values

# 哑变量处理

# 把空白值也当作变量处理

X_test = pd.get_dummies(X_test, columns=['Pclass', 'Sex', 'Cabin', 'Embarked', 'Age_set', 'Family_N'],

dummy_na=True)

X.info()

# 发现维数不一样。所以应该对X_test添加一群0列，并且排号列序，必须与X_train(X)一致。

for i in X_test.columns:

if i not in X.columns:

X[i] = 0

for i in X.columns:

if i not in X_test.columns:

X_test[i] = 0

# X_test.info()

# X_train.info()

X_test = X_test[X.columns]

X_train, XTrain_test, y_train, ytrain_test = train_test_split(X,y, test_size=0.2, random_state=123)

clf = KNeighborsClassifier(n_neighbors=10).fit(X_train,y_train)

y_pred =clf.predict(X_test)

print(y_pred)

print(y_test)

# 判定分类算法

from sklearn.metrics import classification_report, auc

print(classification_report(y_test, y_pred))

# 绘制roc曲线

from sklearn.metrics import roc_curve

import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif'] = 'SimHei' # 改字体

# 求出ROC曲线的x轴和Y轴

fpr, tpr, thresholds = roc_curve(y_test, y_pred)

print(auc(fpr, tpr))

plt.figure(figsize=(10, 6))

plt.xlim(0, 1) # 设定x轴的范围

plt.ylim(0.0, 1.1) # 设定y轴的范围

plt.xlabel('假正率')

plt.ylabel('真正率')

plt.plot(fpr, tpr, linewidth=2, linestyle="-", color='red')

plt.title('Line Roc of X_train by estimator KNN', size=20)

plt.show()

---------------------------------结果-----------------------------------------------

训练模型的roc曲线如下：

训练模型的召回率和精准率和roc曲线积分值如下：

测试模型的roc曲线如下：

训练模型的召回率和精准率和roc曲线积分值如下：

用来测试的survived如下：

训练模型得到的预测结果如下：

计算预测与实际的准确率：

k=0

# 有417个样本待预测

for i in range(417):

if y_test[i] == y_pred[i]:

k=k+1

print(k/417)

得到结果：

准确率有大约84.65%。