2026泰迪杯数据挖掘B题新手入门指南：从数据预处理到模型构建全流程解析

1次阅读

共计 4125 个字符，预计需要花费 11 分钟才能阅读完成。

2026 泰迪杯数据挖掘竞赛 B 题是一个典型的结构化数据挖掘任务。根据往届赛题特点，这类题目通常涉及以下数据特征：

数据量中等规模（约 10 万 -100 万条记录）
包含数值型和类别型混合特征
存在不同程度的缺失值和异常值
目标变量可能是分类或回归问题

以分类问题为例，题目可能要求预测用户行为、产品类别等。理解数据的业务背景对特征工程至关重要，建议首先仔细阅读赛题说明文档。

import pandas as pd
import matplotlib.pyplot as plt

# 加载数据
data = pd.read_csv('contest_data.csv')

# 查看数据概览
print(data.info())
print(data.describe())

# 可视化特征分布
for col in data.select_dtypes(include=['float64', 'int64']):
    plt.figure(figsize=(8,4))
    data[col].hist(bins=50)
    plt.title(col)
    plt.show()

根据特征重要性和缺失比例选择处理策略：

删除缺失率 >50% 的特征
数值特征用中位数填充
类别特征用众数或 ’Missing’ 标识

# 计算各列缺失率
missing_ratio = data.isnull().mean().sort_values(ascending=False)

# 处理高缺失率特征
data = data.loc[:, missing_ratio < 0.5]

# 填充数值特征
num_cols = data.select_dtypes(include=['float64', 'int64']).columns
for col in num_cols:
    data[col] = data[col].fillna(data[col].median())

# 填充类别特征
cat_cols = data.select_dtypes(include=['object']).columns
for col in cat_cols:
    data[col] = data[col].fillna(data[col].mode()[0])

# 使用 IQR 方法检测异常值
for col in num_cols:
    Q1 = data[col].quantile(0.25)
    Q3 = data[col].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5*IQR
    upper_bound = Q3 + 1.5*IQR

    # 将异常值截断到边界值
    data[col] = data[col].clip(lower_bound, upper_bound)

from sklearn.preprocessing import LabelEncoder, OneHotEncoder

# 标签编码有序类别特征
ordinal_cols = ['education_level']  # 示例
for col in ordinal_cols:
    le = LabelEncoder()
    data[col] = le.fit_transform(data[col])

# 独热编码名义类别特征
nominal_cols = ['city', 'gender']  # 示例
data = pd.get_dummies(data, columns=nominal_cols, drop_first=True)

# 对数变换处理偏态分布
skewed_cols = ['income', 'transaction_amount']  # 示例
for col in skewed_cols:
    data[col] = np.log1p(data[col])

# 标准化数值特征
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
data[num_cols] = scaler.fit_transform(data[num_cols])

# 创建交互特征
data['age_income_ratio'] = data['age'] / (data['income'] + 1e-6)

# 时间特征分解
if 'timestamp' in data.columns:
    data['hour'] = pd.to_datetime(data['timestamp']).dt.hour
    data['day_of_week'] = pd.to_datetime(data['timestamp']).dt.dayofweek

from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from sklearn.metrics import accuracy_score, f1_score

# 划分训练测试集
X = data.drop('target', axis=1)
y = data['target']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 随机森林
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)
rf_pred = rf.predict(X_test)
print(f"RF Accuracy: {accuracy_score(y_test, rf_pred):.4f}")
print(f"RF F1 Score: {f1_score(y_test, rf_pred, average='macro'):.4f}")

# XGBoost
xgb = XGBClassifier(n_estimators=100, learning_rate=0.1, random_state=42)
xgb.fit(X_train, y_train)
xgb_pred = xgb.predict(X_test)
print(f"XGB Accuracy: {accuracy_score(y_test, xgb_pred):.4f}")
print(f"XGB F1 Score: {f1_score(y_test, xgb_pred, average='macro'):.4f}")

from sklearn.model_selection import GridSearchCV

# XGBoost 参数网格
param_grid = {'learning_rate': [0.01, 0.1, 0.2],
    'max_depth': [3, 5, 7],
    'subsample': [0.6, 0.8, 1.0],
    'colsample_bytree': [0.6, 0.8, 1.0]
}

# 网格搜索
xgb = XGBClassifier(n_estimators=100, random_state=42)
grid_search = GridSearchCV(estimator=xgb, param_grid=param_grid, 
                          cv=5, scoring='f1_macro', verbose=1)
grid_search.fit(X_train, y_train)

# 输出最佳参数
print("Best parameters:", grid_search.best_params_)
best_xgb = grid_search.best_estimator_

from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns

# 生成分类报告
print("\nRandom Forest Report:")
print(classification_report(y_test, rf_pred))

print("\nXGBoost Report:")
print(classification_report(y_test, xgb_pred))

# 绘制混淆矩阵
plt.figure(figsize=(10,6))
sns.heatmap(confusion_matrix(y_test, xgb_pred), annot=True, fmt='d')
plt.title('Confusion Matrix')
plt.show()

# 获取特征重要性
feature_importance = pd.DataFrame({
    'feature': X.columns,
    'importance': best_xgb.feature_importances_
}).sort_values('importance', ascending=False)

# 可视化
plt.figure(figsize=(12,8))
sns.barplot(x='importance', y='feature', data=feature_importance.head(20))
plt.title('Top 20 Important Features')
plt.show()