AI: أساسيات البيانات

# Check data distribution — first step before any AI project
import pandas as pd

df = pd.DataFrame({
    'weather': ['sunny'] * 70 + ['rainy'] * 15 + ['foggy'] * 10 + ['night'] * 5,
    'label': ['car'] * 50 + ['car'] * 5 + ['car'] * 3 + ['car'] * 2 +
             ['truck'] * 20 + ['truck'] * 10 + ['truck'] * 7 + ['truck'] * 3
})

print("Weather distribution:")
print(df['weather'].value_counts())
print(f"\nSunny images: {70/100*100:.0f}% — That's a problem!")

Weather distribution:
sunny    70
rainy    15
foggy    10
night     5
Name: weather, dtype: int64

Sunny images: 70% — That's a problem!

# مثال: هندسة ميزات يدوية
import pandas as pd

df = pd.DataFrame({
    'text': ['Free iPhone winner!', 'Meeting at 3pm', 'URGENT: Claim now'],
    'label': ['spam', 'ham', 'spam']
})

# استخراج ميزات يدوي: عد كلمات محددة
df['has_free'] = df['text'].str.lower().str.contains('free').astype(int)
df['has_urgent'] = df['text'].str.lower().str.contains('urgent').astype(int)
df['exclamation_count'] = df['text'].str.count('!')
df['text_length'] = df['text'].str.len()

print(df)

                  text label  has_free  has_urgent  exclamation_count  text_length
0  Free iPhone winner!  spam         1           0                  1           19
1     Meeting at 3pm    ham         0           0                  0           14
2  URGENT: Claim now   spam         0           1                  0           17

# Split dataset into train/validation/test sets
from sklearn.model_selection import train_test_split
import pandas as pd

# Sample dataset
df = pd.DataFrame({
    'area_sqm': [50, 80, 120, 65, 200, 90, 55, 110, 75, 150,
                 60, 85, 130, 95, 180, 70, 100, 45, 140, 105],
    'price_usd': [150000, 280000, 350000, 220000, 500000, 260000,
                  165000, 320000, 240000, 420000, 175000, 270000,
                  380000, 290000, 460000, 210000, 300000, 135000,
                  400000, 310000]
})

# First split: 80% train+val, 20% test
train_val, test = train_test_split(df, test_size=0.2, random_state=42)

# Second split: 75% of train_val = 60% total train, 25% = 20% total val
train, val = train_test_split(train_val, test_size=0.25, random_state=42)

print(f"Full dataset:  {len(df)} samples")
print(f"Training set:  {len(train)} samples ({len(train)/len(df)*100:.0f}%)")
print(f"Validation set:{len(val)} samples ({len(val)/len(df)*100:.0f}%)")
print(f"Test set:      {len(test)} samples ({len(test)/len(df)*100:.0f}%)")

مجموعة البيانات الكاملة:  20 عينة
مجموعة التدريب:  12 عينة (60%)
مجموعة التحقق: 4 عينات (20%)
مجموعة الاختبار: 4 عينات (20%)

# قائمة التحقق من جودة البيانات
import pandas as pd
import numpy as np

df = pd.DataFrame({
    'age': [25, 30, np.nan, 45, 200, 28, 30, 30, 35, np.nan],
    'income': [50000, 60000, 45000, np.nan, 80000, 55000, 60000, 62000, 70000, 48000],
    'label': ['buy', 'no', 'buy', 'no', 'buy', 'no', 'no', 'no', 'buy', 'no']
})

print("=== تقرير جودة البيانات ===")
print(f"إجمالي الصفوف: {len(df)}")
print(f"\nالقيم المفقودة لكل عمود:")
print(df.isnull().sum())
print(f"\nالصفوف المكررة: {df.duplicated().sum()}")
print(f"\nتوزيع التصنيفات:")
print(df['label'].value_counts())
print(f"\nالقيم المشبوهة (العمر > 120):")
print(df[df['age'] > 120])

=== تقرير جودة البيانات ===
إجمالي الصفوف: 10

القيم المفقودة لكل عمود:
age       2
income    1
label     0
dtype: int64

الصفوف المكررة: 0

توزيع التصنيفات:
no     6
buy    4
Name: label, dtype: int64

القيم المشبوهة (العمر > 120):
   age  income label
4  200   80000   buy

# ============================================
# Complete Data Preparation Pipeline
# Dataset: Titanic (simulated)
# ============================================
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split

# Simulated Titanic-like dataset
np.random.seed(42)
df = pd.DataFrame({
    'pclass': np.random.choice([1, 2, 3], 100, p=[0.2, 0.3, 0.5]),
    'sex': np.random.choice(['male', 'female'], 100, p=[0.6, 0.4]),
    'age': np.where(np.random.rand(100) > 0.15,
                    np.random.randint(1, 80, 100), np.nan),
    'fare': np.round(np.random.uniform(10, 500, 100), 2),
    'survived': np.random.choice([0, 1], 100, p=[0.6, 0.4])
})

# Step 1: Overview
print("=== Step 1: Data Overview ===")
print(f"Shape: {df.shape}")
print(df.head(5))

# Step 2: Missing values
print("\n=== Step 2: Missing Values ===")
missing = df.isnull().sum()
missing_pct = (df.isnull().sum() / len(df) * 100).round(1)
print(pd.DataFrame({'count': missing, 'percent': missing_pct}))

# Step 3: Label distribution
print("\n=== Step 3: Label Distribution ===")
print(df['survived'].value_counts())
print(f"Survival rate: {df['survived'].mean()*100:.1f}%")

# Step 4: Train/test split
train, test = train_test_split(df, test_size=0.2, random_state=42, 
                                stratify=df['survived'])
print(f"\n=== Step 4: Data Split ===")
print(f"Train: {len(train)} samples, survival rate: {train['survived'].mean()*100:.1f}%")
print(f"Test:  {len(test)} samples, survival rate: {test['survived'].mean()*100:.1f}%")
print(f"\nStratified split ensures same survival rate in both sets!")

=== Step 1: Data Overview ===
Shape: (100, 5)
   pclass     sex   age    fare  survived
0       3  female  47.0  339.89         1
1       3  female  63.0  361.38         0
2       1  female  44.0  309.18         0
3       3    male  28.0  385.59         0
4       1    male   8.0  477.95         0

=== Step 2: Missing Values ===
         count  percent
pclass       0      0.0
sex          0      0.0
age         14     14.0
fare         0      0.0
survived     0      0.0

=== Step 3: Label Distribution ===
0    58
1    42
Name: survived, dtype: int64
Survival rate: 42.0%

=== Step 4: Data Split ===
Train: 80 samples, survival rate: 42.5%
Test:  20 samples, survival rate: 40.0%

Stratified split ensures same survival rate in both sets!