Machine Learning: NLPの基礎 — テキスト前処理と感情分析ガイド

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline

pipe = Pipeline([
    ("tfidf", TfidfVectorizer(max_features=5000, ngram_range=(1, 2))),
    ("clf", LogisticRegression(max_iter=500)),
])
pipe.fit(X_train_reviews, y_train_sentiment)
print(f"Sentiment accuracy: {pipe.score(X_test, y_test):.3f}")

import re
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer, WordNetLemmatizer

import nltk
nltk.download("punkt_tab", quiet=True)
nltk.download("stopwords", quiet=True)
nltk.download("wordnet", quiet=True)

def preprocess_text(text):
    # Lowercase
    text = text.lower()
    # Remove special characters and numbers
    text = re.sub(r"[^a-zA-Z\s]", "", text)
    # Tokenize
    tokens = word_tokenize(text)
    # Remove stopwords
    stop_words = set(stopwords.words("english"))
    tokens = [t for t in tokens if t not in stop_words and len(t) > 2]
    # Stemming
    stemmer = PorterStemmer()
    tokens_stemmed = [stemmer.stem(t) for t in tokens]
    return " ".join(tokens_stemmed)

# Test
reviews = [
    "This product is absolutely amazing! Best purchase ever.",
    "Terrible quality, broke after 2 days. Very disappointed.",
    "Decent product for the price, but shipping was slow.",
]

for review in reviews:
    cleaned = preprocess_text(review)
    print(f"Original: {review}")
    print(f"Cleaned:  {cleaned}\n")

# Function defined successfully

# Word tokenization with NLTK
# import nltk
# nltk.download('punkt')
# from nltk.tokenize import word_tokenize
# text = "This product has great quality and fast shipping"
# tokens = word_tokenize(text)
# print(tokens)  # ['This', 'product', 'has', 'great', 'quality', 'and', 'fast', 'shipping']

# For English reviews in SalesPredict (international scenario)
from sklearn.feature_extraction.text import CountVectorizer

reviews = [
    "Great laptop fast performance",
    "Terrible screen quality",
    "Good price fast delivery",
]

vectorizer = CountVectorizer()
X_bow = vectorizer.fit_transform(reviews)
print(f"Vocabulary: {vectorizer.vocabulary_}")
print(f"BoW matrix shape: {X_bow.shape}")

# Executed successfully

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import numpy as np

reviews = [
    "great product fast delivery happy customer",
    "terrible quality broke disappointed waste money",
    "great quality decent price fast shipping",
    "awful product terrible experience never buy again",
    "excellent quality great value fast delivery",
]

labels = [1, 0, 1, 0, 1]  # 1=positive, 0=negative

# Method 1: Bag-of-Words (count)
bow = CountVectorizer()
X_bow = bow.fit_transform(reviews)
print(f"BoW vocabulary size: {len(bow.vocabulary_)}")

# Method 2: TF-IDF
tfidf = TfidfVectorizer(max_features=1000)
X_tfidf = tfidf.fit_transform(reviews)
print(f"TF-IDF shape: {X_tfidf.shape}")

# Method 3: N-gram (capture phrases)
tfidf_ngram = TfidfVectorizer(ngram_range=(1, 2), max_features=1000)
X_ngram = tfidf_ngram.fit_transform(reviews)
print(f"TF-IDF with bigrams shape: {X_ngram.shape}")
print(f"Top bigrams: {[w for w in tfidf_ngram.get_feature_names_out() if ' ' in w][:10]}")

# Executed successfully

from gensim.models import Word2Vec
from gensim.test.utils import common_texts
import numpy as np

# Train Word2Vec on sample corpus
# In practice, use real review data corpus
sentences = [
    ["great", "product", "fast", "delivery", "happy"],
    ["terrible", "quality", "broke", "disappointed"],
    ["great", "quality", "decent", "price"],
    ["awful", "product", "terrible", "experience"],
    ["excellent", "quality", "great", "value"],
    ["fast", "shipping", "good", "price", "recommend"],
    ["poor", "quality", "slow", "delivery", "angry"],
    ["amazing", "product", "best", "purchase", "happy"],
]

model = Word2Vec(sentences, vector_size=50, window=3, min_count=1, workers=4, epochs=50)

# Word similarity
print("Similar to 'great':")
for word, sim in model.wv.most_similar("great", topn=5):
    print(f"  {word}: {sim:.3f}")

# Word vector
print(f"\nVector for 'quality': {model.wv['quality'][:5]}... (50-dim)")

# Analogy: great - good + terrible = ?
# result = model.wv.most_similar(positive=["terrible"], negative=["good"], topn=3)

Similar to 

from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

words = ["great", "terrible", "happy", "angry", "fast", "slow",
         "quality", "price", "product", "delivery"]

vectors = [model.wv[w] for w in words]
vectors_2d = PCA(n_components=2).fit_transform(vectors)

fig, ax = plt.subplots(figsize=(10, 8))
ax.scatter(vectors_2d[:, 0], vectors_2d[:, 1], s=100)
for i, word in enumerate(words):
    ax.annotate(word, (vectors_2d[i, 0], vectors_2d[i, 1]), fontsize=12)
ax.set_title("Word2Vec Embedding Visualization")
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig("word2vec_vis.png", dpi=150)

# Executed successfully

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report, confusion_matrix
import numpy as np

# Simulate product reviews
rng = np.random.default_rng(42)

positive_templates = [
    "great product love it highly recommend",
    "excellent quality fast delivery satisfied",
    "amazing value best purchase ever happy",
    "good product decent price works well",
    "love this item perfect condition great",
]

negative_templates = [
    "terrible quality broke after one day",
    "awful product waste money very disappointed",
    "poor quality slow delivery never again",
    "worst purchase defective product angry",
    "bad product cheap material not recommend",
]

# Generate 2000 reviews
reviews = []
labels = []
for _ in range(1000):
    reviews.append(rng.choice(positive_templates))
    labels.append(1)
    reviews.append(rng.choice(negative_templates))
    labels.append(0)

X_train, X_test, y_train, y_test = train_test_split(reviews, labels, test_size=0.2, random_state=42)

# TF-IDF + Logistic Regression pipeline
pipe = Pipeline([
    ("tfidf", TfidfVectorizer(max_features=5000, ngram_range=(1, 2), min_df=2)),
    ("clf", LogisticRegression(max_iter=500, C=1.0)),
])

pipe.fit(X_train, y_train)
y_pred = pipe.predict(X_test)

print(f"Accuracy: {pipe.score(X_test, y_test):.4f}")
print(f"\nClassification Report:\n{classification_report(y_test, y_pred, target_names=['Negative', 'Positive'])}")

# Extract most important words for each sentiment
feature_names = pipe.named_steps["tfidf"].get_feature_names_out()
coefs = pipe.named_steps["clf"].coef_[0]
top_positive = np.argsort(coefs)[-10:][::-1]
top_negative = np.argsort(coefs)[:10]

print("\nTop positive words:")
for idx in top_positive:
    print(f"  {feature_names[idx]:20s}: {coefs[idx]:.3f}")

print("\nTop negative words:")
for idx in top_negative:
    print(f"  {feature_names[idx]:20s}: {coefs[idx]:.3f}")

\nTop positive words:
\nTop negative words: