第六单元：AI应用开发

# 机器学习的主要类型

# 1. 监督学习
# 从标记数据中学习，预测结果
# 例子：分类、回归

# 2. 无监督学习
# 从未标记数据中学习模式
# 例子：聚类、降维

# 3. 半监督学习
# 结合标记和未标记数据

# 4. 强化学习
# 通过与环境交互学习最优策略

# 5. 深度学习
# 使用神经网络模型

# 常见的机器学习库
# scikit-learn: 传统机器学习算法
# TensorFlow: 深度学习框架
# PyTorch: 深度学习框架
# Keras: 高级神经网络API
# pandas: 数据处理
# NumPy: 数值计算
# matplotlib: 数据可视化

# 安装scikit-learn
# pip install scikit-learn

# 基本导入
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score

# 加载数据集
iris = load_iris()
X, y = iris.data, iris.target

# 数据集信息
print("特征名称:", iris.feature_names)
print("目标类别:", iris.target_names)
print("数据集形状:", X.shape)

# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

print("训练集大小:", X_train.shape)
print("测试集大小:", X_test.shape)

# 创建模型
model = LogisticRegression()

# 训练模型
model.fit(X_train, y_train)

# 预测
y_pred = model.predict(X_test)

# 评估模型
accuracy = accuracy_score(y_test, y_pred)
print(f"模型准确率: {accuracy:.2f}")

# 预测新数据
new_data = [[5.1, 3.5, 1.4, 0.2]]  # 山鸢尾花的特征
prediction = model.predict(new_data)
print(f"预测结果: {iris.target_names[prediction[0]]}")

import pandas as pd
import numpy as np

# 创建示例数据
data = {
    'age': [25, 30, np.nan, 40, 45, 50, 55, np.nan],
    'salary': [50000, 60000, 70000, np.nan, 90000, 100000, 110000, 120000],
    'department': ['IT', 'HR', 'IT', 'Finance', 'HR', 'IT', 'Finance', 'IT']
}

df = pd.DataFrame(data)
print("原始数据:")
print(df)

# 处理缺失值
# 1. 删除缺失值
# df_clean = df.dropna()

# 2. 填充缺失值
df['age'] = df['age'].fillna(df['age'].mean())
df['salary'] = df['salary'].fillna(df['salary'].median())

print("\n处理后的数据:")
print(df)

# 处理重复数据
df_duplicates = pd.concat([df, df.iloc[[0]]])  # 添加重复行
print("\n有重复数据:")
print(df_duplicates)

df_no_duplicates = df_duplicates.drop_duplicates()
print("\n无重复数据:")
print(df_no_duplicates)

# 处理异常值
# 检测异常值（使用IQR方法）
Q1 = df['salary'].quantile(0.25)
Q3 = df['salary'].quantile(0.75)
IQR = Q3 - Q1

lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR

print(f"\n异常值边界: [{lower_bound}, {upper_bound}]")

# 过滤异常值
df_no_outliers = df[(df['salary'] >= lower_bound) & (df['salary'] <= upper_bound)]
print("\n无异常值数据:")
print(df_no_outliers)

import pandas as pd
from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder

# 创建示例数据
data = {
    'age': [25, 30, 35, 40, 45],
    'salary': [50000, 60000, 70000, 80000, 90000],
    'department': ['IT', 'HR', 'IT', 'Finance', 'HR'],
    'gender': ['Male', 'Female', 'Male', 'Female', 'Male']
}

df = pd.DataFrame(data)
print("原始数据:")
print(df)

# 1. 特征缩放（标准化）
scaler = StandardScaler()
df[['age', 'salary']] = scaler.fit_transform(df[['age', 'salary']])
print("\n标准化后的数据:")
print(df)

# 2. 标签编码（用于有序分类变量）
label_encoder = LabelEncoder()
df['gender_encoded'] = label_encoder.fit_transform(df['gender'])
print("\n标签编码后的数据:")
print(df)

# 3. 独热编码（用于无序分类变量）
df_encoded = pd.get_dummies(df, columns=['department'])
print("\n独热编码后的数据:")
print(df_encoded)

# 4. 特征选择
# 相关性分析
correlation = df.corr()
print("\n相关性矩阵:")
print(correlation)

# 选择与目标变量相关性高的特征
# 假设目标变量是salary
target_correlation = correlation['salary'].abs().sort_values(ascending=False)
print("\n与salary的相关性:")
print(target_correlation)

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score

# 加载数据集
cancer = load_breast_cancer()
X, y = cancer.data, cancer.target

# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 定义模型
t models = {
    'Logistic Regression': LogisticRegression(max_iter=10000),
    'Decision Tree': DecisionTreeClassifier(),
    'Random Forest': RandomForestClassifier(),
    'SVM': SVC(),
    'KNN': KNeighborsClassifier()
}

# 训练和评估模型
for name, model in models.items():
    # 训练模型
    model.fit(X_train, y_train)
    # 预测
    y_pred = model.predict(X_test)
    # 计算准确率
    accuracy = accuracy_score(y_test, y_pred)
    print(f"{name} 准确率: {accuracy:.2f}")

# 超参数调优
from sklearn.model_selection import GridSearchCV

# 定义参数网格
param_grid = {
    'n_estimators': [10, 50, 100],
    'max_depth': [3, 5, 7, None]
}

# 创建网格搜索
grid_search = GridSearchCV(
    RandomForestClassifier(),
    param_grid,
    cv=5,
    scoring='accuracy'
)

# 运行网格搜索
grid_search.fit(X_train, y_train)

print("\n最佳参数:")
print(grid_search.best_params_)
print(f"最佳交叉验证准确率: {grid_search.best_score_:.2f}")

# 使用最佳模型
best_model = grid_search.best_estimator_
y_pred = best_model.predict(X_test)
print(f"测试集准确率: {accuracy_score(y_test, y_pred):.2f}")

from sklearn.datasets import load_diabetes
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score
import numpy as np

# 加载数据集
diabetes = load_diabetes()
X, y = diabetes.data, diabetes.target

# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 定义模型
models = {
    'Linear Regression': LinearRegression(),
    'Ridge Regression': Ridge(),
    'Lasso Regression': Lasso(),
    'Decision Tree': DecisionTreeRegressor(),
    'Random Forest': RandomForestRegressor()
}

# 训练和评估模型
for name, model in models.items():
    # 训练模型
    model.fit(X_train, y_train)
    # 预测
    y_pred = model.predict(X_test)
    # 计算评估指标
    mse = mean_squared_error(y_test, y_pred)
    rmse = np.sqrt(mse)
    r2 = r2_score(y_test, y_pred)
    print(f"{name}:")
    print(f"  MSE: {mse:.2f}")
    print(f"  RMSE: {rmse:.2f}")
    print(f"  R²: {r2:.2f}")
    print()

# 特征重要性（以随机森林为例）
rf_model = RandomForestRegressor()
rf_model.fit(X_train, y_train)

feature_importances = rf_model.feature_importances_
feature_names = diabetes.feature_names

print("特征重要性:")
for feature, importance in zip(feature_names, feature_importances):
    print(f"{feature}: {importance:.4f}")

# 可视化特征重要性
import matplotlib.pyplot as plt

plt.figure(figsize=(10, 6))
plt.barh(feature_names, feature_importances)
plt.xlabel('重要性')
plt.ylabel('特征')
plt.title('随机森林特征重要性')
plt.tight_layout()
plt.show()

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, f1_score,
    confusion_matrix, classification_report, roc_curve, auc
)
import matplotlib.pyplot as plt

# 加载数据集
cancer = load_breast_cancer()
X, y = cancer.data, cancer.target

# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 训练模型
model = LogisticRegression(max_iter=10000)
model.fit(X_train, y_train)

# 预测
y_pred = model.predict(X_test)
y_pred_proba = model.predict_proba(X_test)[:, 1]

# 计算评估指标
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)

print(f"准确率: {accuracy:.2f}")
print(f"精确率: {precision:.2f}")
print(f"召回率: {recall:.2f}")
print(f"F1分数: {f1:.2f}")

# 混淆矩阵
print("\n混淆矩阵:")
print(confusion_matrix(y_test, y_pred))

# 分类报告
print("\n分类报告:")
print(classification_report(y_test, y_pred))

# ROC曲线和AUC
fpr, tpr, thresholds = roc_curve(y_test, y_pred_proba)
roc_auc = auc(fpr, tpr)

print(f"\nAUC: {roc_auc:.2f}")

# 可视化ROC曲线
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, color='blue', lw=2, label=f'ROC curve (AUC = {roc_auc:.2f})')
plt.plot([0, 1], [0, 1], color='gray', linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('假阳性率')
plt.ylabel('真阳性率')
plt.title('接收器操作特征曲线 (ROC)')
plt.legend(loc="lower right")
plt.show()

from sklearn.datasets import load_iris
from sklearn.model_selection import cross_val_score, KFold
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

# 加载数据集
iris = load_iris()
X, y = iris.data, iris.target

# 定义模型
models = {
    'Logistic Regression': LogisticRegression(max_iter=1000),
    'Decision Tree': DecisionTreeClassifier(),
    'Random Forest': RandomForestClassifier()
}

# 使用5折交叉验证
cv = KFold(n_splits=5, shuffle=True, random_state=42)

for name, model in models.items():
    # 计算交叉验证分数
    scores = cross_val_score(model, X, y, cv=cv, scoring='accuracy')
    print(f"{name}:")
    print(f"  交叉验证准确率: {scores.mean():.2f} ± {scores.std():.2f}")
    print(f"  各折分数: {scores}")
    print()

# 留一交叉验证
from sklearn.model_selection import LeaveOneOut

loo = LeaveOneOut()
scores = cross_val_score(LogisticRegression(max_iter=1000), X, y, cv=loo, scoring='accuracy')
print(f"留一交叉验证准确率: {scores.mean():.2f}")
print(f"测试次数: {len(scores)}")
print(f"正确预测次数: {scores.sum()}")

# 安装必要的库
# pip install tensorflow keras

import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.utils import to_categorical
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# 加载数据集
iris = load_iris()
X, y = iris.data, iris.target

# 数据预处理
scaler = StandardScaler()
X = scaler.fit_transform(X)

# 转换标签为独热编码
y = to_categorical(y)

# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 创建模型
model = Sequential([
    Dense(16, activation='relu', input_shape=(4,)),
    Dropout(0.2),
    Dense(8, activation='relu'),
    Dropout(0.2),
    Dense(3, activation='softmax')
])

# 编译模型
model.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=['accuracy']
)

# 查看模型架构
model.summary()

# 训练模型
history = model.fit(
    X_train, y_train,
    epochs=50,
    batch_size=8,
    validation_split=0.2,
    verbose=1
)

# 评估模型
loss, accuracy = model.evaluate(X_test, y_test, verbose=0)
print(f"测试集准确率: {accuracy:.2f}")

# 预测
predictions = model.predict(X_test)
predicted_classes = np.argmax(predictions, axis=1)
true_classes = np.argmax(y_test, axis=1)

print("\n预测结果:")
print(f"预测类别: {predicted_classes}")
print(f"真实类别: {true_classes}")

# 保存模型
model.save('iris_classifier.h5')

# 加载模型
from tensorflow.keras.models import load_model
loaded_model = load_model('iris_classifier.h5')

# 使用加载的模型进行预测
loaded_predictions = loaded_model.predict(X_test)
loaded_predicted_classes = np.argmax(loaded_predictions, axis=1)
print(f"\n加载模型后的预测结果: {loaded_predicted_classes}")

import nltk
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer, PorterStemmer
import string

# 下载必要的资源
# nltk.download('punkt')
# nltk.download('stopwords')
# nltk.download('wordnet')

# 示例文本
text = "Hello, world! This is an example of natural language processing. We are learning about text preprocessing."

print("原始文本:")
print(text)
print()

# 1. 转换为小写
text_lower = text.lower()
print("小写文本:")
print(text_lower)
print()

# 2. 分词
tokens = word_tokenize(text_lower)
print("分词结果:")
print(tokens)
print()

# 3. 移除标点符号
 tokens = [token for token in tokens if token not in string.punctuation]
print("移除标点符号后:")
print(tokens)
print()

# 4. 移除停用词
stop_words = set(stopwords.words('english'))
tokens = [token for token in tokens if token not in stop_words]
print("移除停用词后:")
print(tokens)
print()

# 5. 词干提取
stemmer = PorterStemmer()
stemmed_tokens = [stemmer.stem(token) for token in tokens]
print("词干提取后:")
print(stemmed_tokens)
print()

# 6. 词形还原
lemmatizer = WordNetLemmatizer()
lemmatized_tokens = [lemmatizer.lemmatize(token) for token in tokens]
print("词形还原后:")
print(lemmatized_tokens)
print()

# 7. 句子分割
sentences = sent_tokenize(text)
print("句子分割:")
for i, sentence in enumerate(sentences, 1):
    print(f"句子 {i}: {sentence}")

from sklearn.datasets import fetch_20newsgroups
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score, classification_report

# 加载数据集
data = fetch_20newsgroups(subset='all', categories=['alt.atheism', 'soc.religion.christian', 'comp.graphics', 'sci.med'])

print(f"数据集大小: {len(data.data)}")
print(f"类别: {data.target_names}")

# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(
    data.data, data.target, test_size=0.2, random_state=42
)

# 特征提取
vectorizer = TfidfVectorizer(stop_words='english', max_features=5000)
X_train_vec = vectorizer.fit_transform(X_train)
X_test_vec = vectorizer.transform(X_test)

print(f"训练集特征形状: {X_train_vec.shape}")
print(f"测试集特征形状: {X_test_vec.shape}")

# 训练模型
model = MultinomialNB()
model.fit(X_train_vec, y_train)

# 预测
y_pred = model.predict(X_test_vec)

# 评估模型
accuracy = accuracy_score(y_test, y_pred)
print(f"准确率: {accuracy:.2f}")

print("\n分类报告:")
print(classification_report(y_test, y_pred, target_names=data.target_names))

# 测试新文本
new_texts = [
    "God is love and Jesus is the son of God",
    "The GPU in my computer is not working properly",
    "Atheism is a belief system that rejects the existence of deities",
    "I have a headache and fever, what should I do?"
]

new_texts_vec = vectorizer.transform(new_texts)
predictions = model.predict(new_texts_vec)

print("\n新文本预测:")
for text, pred in zip(new_texts, predictions):
    print(f"文本: {text}")
    print(f"预测类别: {data.target_names[pred]}")
    print()

import cv2
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt

# 注意：由于环境限制，这里提供代码示例，但可能无法在当前环境中运行

# 加载图像
# img = cv2.imread('image.jpg')
# 转换为RGB
# img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

# 图像尺寸
# height, width, channels = img.shape
# print(f"图像尺寸: {width}x{height}, 通道数: {channels}")

# 调整图像大小
# resized = cv2.resize(img, (200, 200))

# 灰度转换
# gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

# 边缘检测
# edges = cv2.Canny(gray, 100, 200)

# 显示图像
# plt.figure(figsize=(12, 4))
# plt.subplot(131)
# plt.imshow(img_rgb)
# plt.title('原始图像')
# plt.axis('off')
# 
# plt.subplot(132)
# plt.imshow(gray, cmap='gray')
# plt.title('灰度图像')
# plt.axis('off')
# 
# plt.subplot(133)
# plt.imshow(edges, cmap='gray')
# plt.title('边缘检测')
# plt.axis('off')
# 
# plt.tight_layout()
# plt.show()

# 图像数据标准化
# img_array = np.array(img)
# normalized = img_array / 255.0
# print(f"标准化前范围: {img_array.min()} - {img_array.max()}")
# print(f"标准化后范围: {normalized.min():.2f} - {normalized.max():.2f}")

# 安装必要的库
# pip install tensorflow keras

import numpy as np
from tensorflow.keras.datasets import mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D
from tensorflow.keras.utils import to_categorical
import matplotlib.pyplot as plt

# 加载MNIST数据集
(X_train, y_train), (X_test, y_test) = mnist.load_data()

print(f"训练集大小: {X_train.shape}")
print(f"测试集大小: {X_test.shape}")

# 显示示例图像
plt.figure(figsize=(10, 10))
for i in range(25):
    plt.subplot(5, 5, i+1)
    plt.imshow(X_train[i], cmap='gray')
    plt.title(f"标签: {y_train[i]}")
    plt.axis('off')
plt.tight_layout()
plt.show()

# 数据预处理
# 调整形状和归一化
X_train = X_train.reshape(-1, 28, 28, 1).astype('float32') / 255
X_test = X_test.reshape(-1, 28, 28, 1).astype('float32') / 255

# 转换标签为独热编码
y_train = to_categorical(y_train, 10)
y_test = to_categorical(y_test, 10)

# 创建CNN模型
model = Sequential([
    Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),
    MaxPooling2D((2, 2)),
    Conv2D(64, (3, 3), activation='relu'),
    MaxPooling2D((2, 2)),
    Flatten(),
    Dense(128, activation='relu'),
    Dense(10, activation='softmax')
])

# 编译模型
model.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=['accuracy']
)

# 查看模型架构
model.summary()

# 训练模型
history = model.fit(
    X_train, y_train,
    epochs=5,
    batch_size=32,
    validation_split=0.2,
    verbose=1
)

# 评估模型
loss, accuracy = model.evaluate(X_test, y_test, verbose=0)
print(f"测试集准确率: {accuracy:.2f}")

# 预测
predictions = model.predict(X_test[:10])
predicted_classes = np.argmax(predictions, axis=1)
true_classes = np.argmax(y_test[:10], axis=1)

print("\n预测结果:")
print(f"预测类别: {predicted_classes}")
print(f"真实类别: {true_classes}")

# 显示预测结果
plt.figure(figsize=(10, 5))
for i in range(10):
    plt.subplot(2, 5, i+1)
    plt.imshow(X_test[i].reshape(28, 28), cmap='gray')
    plt.title(f"预测: {predicted_classes[i]}, 真实: {true_classes[i]}")
    plt.axis('off')
plt.tight_layout()
plt.show()