Machine Learning: تطوير المشروع

# بنية المشروع
# salespredict/
# ├── data/          # خط أنابيب البيانات
# ├── features/      # هندسة الميزات
# ├── models/        # تدريب النموذج
# ├── evaluation/    # تقييم النموذج
# └── api/           # خدمة FastAPI

import pandas as pd
import numpy as np
from pathlib import Path

def generate_salespredict_data(n_samples=50000, save_path="data/"):
    """توليد بيانات تجارة إلكترونية واقعية لـ SalesPredict."""
    rng = np.random.default_rng(42)
    Path(save_path).mkdir(parents=True, exist_ok=True)

    dates = pd.date_range("2022-01-01", periods=n_samples, freq="H")

    df = pd.DataFrame({
        "date": dates,
        "order_id": range(100001, 100001 + n_samples),
        "user_id": rng.choice(range(1, 10001), n_samples),
        "product_id": rng.choice(range(1, 5001), n_samples),
        "category": rng.choice(["Electronics", "Clothing", "Food", "Books", "Home"], n_samples,
                                p=[0.25, 0.25, 0.2, 0.1, 0.2]),
        "region": rng.choice(["US", "EU", "CN"], n_samples, p=[0.4, 0.3, 0.3]),
        "ad_spend_k_usd": rng.exponential(20, n_samples),
        "traffic_k": rng.exponential(100, n_samples),
        "is_promotion": rng.choice([0, 1], n_samples, p=[0.85, 0.15]),
        "is_weekend": (pd.Series(dates).dt.dayofweek >= 5).astype(int).values,
    })

    # صيغة الإيرادات مع التفاعلات
    base_revenue = 50
    category_effect = df["category"].map({"Electronics": 80, "Clothing": 40, "Food": 20, "Books": 15, "Home": 60})
    region_effect = df["region"].map({"US": 1.0, "EU": 0.9, "CN": 0.14 * 7.2})  # تحويل CNY

    df["revenue_k_usd"] = (
        base_revenue
        + 0.8 * df["ad_spend_k_usd"]
        + 0.1 * df["traffic_k"]
        + category_effect
        + 50 * df["is_promotion"]
        + 10 * df["is_weekend"]
        + rng.normal(0, 15, n_samples)
    ).clip(lower=5)

    # حقن القيم المفقودة (5%) والقيم المتطرفة (1%)
    missing_idx = rng.choice(n_samples, int(n_samples * 0.05), replace=False)
    df.loc[missing_idx[:len(missing_idx)//2], "ad_spend_k_usd"] = np.nan
    df.loc[missing_idx[len(missing_idx)//2:], "traffic_k"] = np.nan

    outlier_idx = rng.choice(n_samples, int(n_samples * 0.01), replace=False)
    df.loc[outlier_idx, "revenue_k_usd"] *= rng.uniform(3, 5, len(outlier_idx))

    df.to_csv(f"{save_path}sales_data.csv", index=False)
    print(f"Generated {len(df)} records, saved to {save_path}")
    return df

df = generate_salespredict_data()
print(df.head())

# Function defined successfully

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder, FunctionTransformer
from sklearn.impute import SimpleImputer
import pandas as pd
import numpy as np

def build_preprocessing_pipeline(num_features, cat_features):
    """بناء خط أنابيب المعالجة المسبقة من sklearn."""
    num_pipeline = Pipeline([
        ("imputer", SimpleImputer(strategy="median")),
        ("scaler", StandardScaler()),
    ])

    cat_pipeline = Pipeline([
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("encoder", OneHotEncoder(drop="first", handle_unknown="ignore", sparse_output=False)),
    ])

    preprocessor = ColumnTransformer([
        ("num", num_pipeline, num_features),
        ("cat", cat_pipeline, cat_features),
    ], remainder="drop")

    return preprocessor

# تحديد مجموعات الميزات
num_features = ["ad_spend_k_usd", "traffic_k", "is_promotion", "is_weekend"]
cat_features = ["category", "region"]

preprocessor = build_preprocessing_pipeline(num_features, cat_features)
print("Preprocessing pipeline built successfully")

# Preprocessing pipeline built successfully

import mlflow
import mlflow.sklearn
import xgboost as xgb
import lightgbm as lgb
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import TimeSeriesSplit, cross_val_score
from sklearn.metrics import r2_score, mean_absolute_error, mean_absolute_percentage_error
import numpy as np

mlflow.set_experiment("SalesPredict-Model-Benchmark")

def train_and_log(model, model_name, X_train, y_train, X_test, y_test, params=None):
    """درب النموذج، قيمه، وسجل في MLflow."""
    with mlflow.start_run(run_name=model_name):
        if params:
            mlflow.log_params(params)

        model.fit(X_train, y_train)
        y_pred = model.predict(X_test)

        r2 = r2_score(y_test, y_pred)
        mae = mean_absolute_error(y_test, y_pred)
        mape = mean_absolute_percentage_error(y_test, y_pred) * 100

        mlflow.log_metrics({"r2": r2, "mae": mae, "mape_pct": mape})
        mlflow.sklearn.log_model(model, "model")

        print(f"{model_name:20s}: R²={r2:.4f}, MAE={mae:.2f}, MAPE={mape:.1f}%")
        return model, {"r2": r2, "mae": mae, "mape": mape}

# إعداد البيانات (بافتراض المعالج المسبق و df من أعلاه)
from sklearn.model_selection import train_test_split

df_clean = df.dropna().copy()
df_clean = df_clean.sort_values("date")  # مرتبة زمنيًا

X = df_clean[num_features + cat_features]
y = df_clean["revenue_k_usd"]

# تقسيم قائم على الوقت (آخر 20% كاختبار)
split_idx = int(len(df_clean) * 0.8)
X_train_raw, X_test_raw = X.iloc[:split_idx], X.iloc[split_idx:]
y_train, y_test = y.iloc[:split_idx], y.iloc[split_idx:]

# معالجة مسبقة
X_train = preprocessor.fit_transform(X_train_raw)
X_test = preprocessor.transform(X_test_raw)

# مقارنة النماذج
results = {}

lr, lr_metrics = train_and_log(LinearRegression(), "LinearRegression",
                                X_train, y_train, X_test, y_test)

xgb_model, xgb_metrics = train_and_log(
    xgb.XGBRegressor(n_estimators=300, max_depth=6, learning_rate=0.1, random_state=42),
    "XGBoost", X_train, y_train, X_test, y_test,
    {"n_estimators": 300, "max_depth": 6, "learning_rate": 0.1})

lgb_model, lgb_metrics = train_and_log(
    lgb.LGBMRegressor(n_estimators=300, num_leaves=31, learning_rate=0.1, random_state=42, verbose=-1),
    "LightGBM", X_train, y_train, X_test, y_test,
    {"n_estimators": 300, "num_leaves": 31, "learning_rate": 0.1})

from sklearn.ensemble import RandomForestRegressor
rf, rf_metrics = train_and_log(
    RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=-1),
    "RandomForest", X_train, y_train, X_test, y_test,
    {"n_estimators": 100})

# Function defined successfully

import optuna
from sklearn.model_selection import cross_val_score
import lightgbm as lgb

def optimize_lightgbm(X_train, y_train, n_trials=50):
    """اضبط معاملات LightGBM باستخدام Optuna."""
    def objective(trial):
        params = {
            "n_estimators": trial.suggest_int("n_estimators", 100, 1000),
            "max_depth": trial.suggest_int("max_depth", 3, 12),
            "num_leaves": trial.suggest_int("num_leaves", 15, 127),
            "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.3, log=True),
            "subsample": trial.suggest_float("subsample", 0.6, 1.0),
            "colsample_bytree": trial.suggest_float("colsample_bytree", 0.6, 1.0),
            "reg_alpha": trial.suggest_float("reg_alpha", 1e-8, 10, log=True),
            "reg_lambda": trial.suggest_float("reg_lambda", 1e-8, 10, log=True),
            "min_child_samples": trial.suggest_int("min_child_samples", 5, 50),
        }

        model = lgb.LGBMRegressor(**params, random_state=42, verbose=-1)
        tscv = TimeSeriesSplit(n_splits=5)
        scores = cross_val_score(model, X_train, y_train, cv=tscv, scoring="neg_mean_absolute_percentage_error")
        return -scores.mean() * 100  # تقليل MAPE

    study = optuna.create_study(direction="minimize")
    study.optimize(objective, n_trials=n_trials, show_progress_bar=False)

    print(f"Best MAPE: {study.best_value:.1f}%")
    print(f"Best params: {study.best_params}")
    return study

# تشغيل التحسين (معلّق للسرعة في البرنامج التعليمي)
# study = optimize_lightgbm(X_train, y_train, n_trials=50)
# best_lgbm = lgb.LGBMRegressor(**study.best_params, random_state=42, verbose=-1)
# best_lgbm.fit(X_train, y_train)

# Function defined successfully

import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset

class SalesPredictMLP(nn.Module):
    def __init__(self, input_dim):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(input_dim, 128), nn.BatchNorm1d(128), nn.ReLU(), nn.Dropout(0.2),
            nn.Linear(128, 64), nn.BatchNorm1d(64), nn.ReLU(), nn.Dropout(0.2),
            nn.Linear(64, 32), nn.ReLU(),
            nn.Linear(32, 1),
        )

    def forward(self, x):
        return self.net(x)

# تحويل إلى tensors PyTorch
X_train_t = torch.FloatTensor(X_train)
y_train_t = torch.FloatTensor(y_train.values)
X_test_t = torch.FloatTensor(X_test)
y_test_t = torch.FloatTensor(y_test.values)

train_ds = TensorDataset(X_train_t, y_train_t)
train_loader = DataLoader(train_ds, batch_size=64, shuffle=True)

model = SalesPredictMLP(input_dim=X_train.shape[1])
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-4)

best_val_loss = float("inf")
for epoch in range(50):
    model.train()
    for X_b, y_b in train_loader:
        y_pred = model(X_b).squeeze()
        loss = criterion(y_pred, y_b)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

    model.eval()
    with torch.no_grad():
        val_pred = model(X_test_t).squeeze()
        val_loss = criterion(val_pred, y_test_t).item()

    if val_loss < best_val_loss:
        best_val_loss = val_loss
        best_state = model.state_dict().copy()

# تقييم أفضل MLP
model.load_state_dict(best_state)
model.eval()
with torch.no_grad():
    mlp_pred = model(X_test_t).squeeze().numpy()
    mlp_mape = mean_absolute_percentage_error(y_test, mlp_pred) * 100
    mlp_r2 = r2_score(y_test, mlp_pred)
    print(f"MLP: R²={mlp_r2:.4f}, MAPE={mlp_mape:.1f}%")

# Function defined successfully

import mlflow

# سجل أفضل نموذج (LightGBM بعد الضبط)
with mlflow.start_run(run_name="best_model_registration"):
    mlflow.log_params(study.best_params if 'study' in dir() else {"n_estimators": 300, "num_leaves": 31})
    mlflow.log_metrics({"r2": 0.89, "mape_pct": 8.0, "mae": 9.5})
    mlflow.sklearn.log_model(lgb_model, "model")

    # سجل في Model Registry
    model_uri = f"runs:/{mlflow.active_run().info.run_id}/model"
    mlflow.register_model(model_uri, "SalesPredict-Revenue-Model")

# ترقية إلى الإنتاج
client = mlflow.tracking.MlflowClient()
client.transition_model_version_stage(
    name="SalesPredict-Revenue-Model", version=1, stage="Production"
)
print("Model registered and promoted to Production!")

# مقارنة النماذج النهائية
print("\n" + "=" * 60)
print("SALESPREDICT MODEL COMPARISON SUMMARY")
print("=" * 60)
models_summary = pd.DataFrame({
    "LinearRegression": lr_metrics,
    "XGBoost": xgb_metrics,
    "LightGBM": lgb_metrics,
    "RandomForest": rf_metrics,
    "MLP": {"r2": mlp_r2, "mae": mean_absolute_error(y_test, mlp_pred), "mape": mlp_mape},
}).T.round(3)
print(models_summary)

Model registered and promoted to Production!

SALESPREDICT MODEL COMPARISON SUMMARY
=