数据集详解：像生活一样理解数据

订单数据 = {
    "日期": "2024-01-20",
    "饮品": "珍珠奶茶",
    "价格": 18,
    "评分": 5
}

import pandas as pd

# 读取CSV数据
df = pd.read_csv('data.csv')

# 基本操作
print(df.head())  # 查看前5行
print(df.info())  # 数据信息
print(df.describe())  # 统计描述

import json

# 读取JSON数据
with open('data.json', 'r') as f:
    data = json.load(f)

# 转换为DataFrame
df = pd.DataFrame(data)

# 读取Parquet文件
df = pd.read_parquet('data.parquet')

# 保存为Parquet
df.to_parquet('output.parquet')

from PIL import Image
import torch
from torchvision import transforms

# 图像预处理
transform = transforms.Compose([
    transforms.Resize((224, 224)),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],
                        std=[0.229, 0.224, 0.225])
])

# 加载图像
image = Image.open('image.jpg')
image_tensor = transform(image)

from datasets import load_dataset

# 加载HuggingFace数据集
dataset = load_dataset('text', data_files={'train': 'train.txt'})

# 文本预处理
def preprocess(text):
    return text.lower().strip()

dataset = dataset.map(lambda x: {'text': preprocess(x['text'])})

import pandas as pd
import numpy as np

def load_and_check_data(file_path):
    # 加载数据
    df = pd.read_csv(file_path)
    
    # 基础检查
    print("数据形状:", df.shape)
    print("
缺失值统计:")
    print(df.isnull().sum())
    print("
数据类型:")
    print(df.dtypes)
    
    return df

def clean_data(df):
    # 删除重复行
    df = df.drop_duplicates()
    
    # 处理缺失值
    df = df.fillna({
        'numeric_col': df['numeric_col'].mean(),
        'categorical_col': 'unknown'
    })
    
    # 处理异常值
    def remove_outliers(x, n_sigmas=3):
        mean = x.mean()
        std = x.std()
        return x[abs(x - mean) <= n_sigmas * std]
    
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    for col in numeric_cols:
        df[col] = remove_outliers(df[col])
    
    return df

from sklearn.preprocessing import StandardScaler, LabelEncoder

def feature_engineering(df):
    # 数值特征标准化
    scaler = StandardScaler()
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    df[numeric_cols] = scaler.fit_transform(df[numeric_cols])
    
    # 类别特征编码
    le = LabelEncoder()
    categorical_cols = df.select_dtypes(include=['object']).columns
    for col in categorical_cols:
        df[col] = le.fit_transform(df[col])
    
    return df

from sklearn.model_selection import train_test_split

def split_dataset(X, y, test_size=0.2, val_size=0.2):
    # 首先分割出测试集
    X_temp, X_test, y_temp, y_test = train_test_split(
        X, y, test_size=test_size, random_state=42
    )
    
    # 从剩余数据中分割出验证集
    val_size_adjusted = val_size / (1 - test_size)
    X_train, X_val, y_train, y_val = train_test_split(
        X_temp, y_temp, test_size=val_size_adjusted, random_state=42
    )
    
    return (X_train, y_train), (X_val, y_val), (X_test, y_test)

def evaluate_dataset_quality(df):
    quality_metrics = {
        "总样本数": len(df),
        "特征数量": df.shape[1],
        "缺失值比例": df.isnull().sum().sum() / (df.shape[0] * df.shape[1]),
        "重复样本比例": len(df[df.duplicated()]) / len(df),
    }
    
    # 类别分布（针对目标变量）
    if 'target' in df.columns:
        quality_metrics["类别分布"] = df['target'].value_counts(normalize=True).to_dict()
    
    return quality_metrics

import seaborn as sns
import matplotlib.pyplot as plt

def visualize_dataset(df):
    # 数值特征分布
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    plt.figure(figsize=(15, 5))
    for i, col in enumerate(numeric_cols, 1):
        plt.subplot(1, len(numeric_cols), i)
        sns.histplot(df[col], kde=True)
        plt.title(f'{col} Distribution')
    plt.tight_layout()
    plt.show()
    
    # 相关性热力图
    plt.figure(figsize=(10, 8))
    sns.heatmap(df[numeric_cols].corr(), annot=True, cmap='coolwarm')
    plt.title('Feature Correlations')
    plt.show()

def process_dataset(file_path):
    # 1. 加载数据
    df = load_and_check_data(file_path)
    
    # 2. 数据清洗
    df_cleaned = clean_data(df)
    
    # 3. 特征工程
    df_processed = feature_engineering(df_cleaned)
    
    # 4. 评估数据质量
    quality_metrics = evaluate_dataset_quality(df_processed)
    print("数据集质量指标:", quality_metrics)
    
    # 5. 可视化分析
    visualize_dataset(df_processed)
    
    # 6. 数据集分割
    X = df_processed.drop('target', axis=1)
    y = df_processed['target']
    return split_dataset(X, y)

# 使用示例
if __name__ == "__main__":
    (X_train, y_train), (X_val, y_val), (X_test, y_test) = process_dataset('data.csv')
    print("训练集大小:", X_train.shape)
    print("验证集大小:", X_val.shape)
    print("测试集大小:", X_test.shape)

# PyTorch数据集
from torch.utils.data import Dataset, DataLoader

class CustomDataset(Dataset):
    def __init__(self, X, y, transform=None):
        self.X = X
        self.y = y
        self.transform = transform
    
    def __len__(self):
        return len(self.X)
    
    def __getitem__(self, idx):
        sample = self.X[idx]
        if self.transform:
            sample = self.transform(sample)
        return sample, self.y[idx]

# 使用示例
dataset = CustomDataset(X_train, y_train)
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)

from sklearn.preprocessing import StandardScaler
from imblearn.over_sampling import SMOTE

def augment_data(X, y):
    # SMOTE过采样
    smote = SMOTE(random_state=42)
    X_augmented, y_augmented = smote.fit_resample(X, y)
    
    return X_augmented, y_augmented