Linux: テキスト処理三銃士 — awk / sed / cut

cat data.csv | cut -d',' -f2,5 | sed '/^$/d' | tr ',' '\t' > output.tsv

# Split by delimiter
cut -d',' -f1,3 file.csv    # Delimiter comma, take columns 1 and 3
cut -d: -f1 /etc/passwd     # Delimiter colon, take usernames

> ℹ️ 注意：`cut`は単純な列抽出（固定区切り文字、固定列番号）に適しています。条件フィルタリング、計算、フォーマットが必要な場合は`awk`を使用してください。例えば`awk -F: '$3 >= 1000 {print $1}' /etc/passwd`はUID ≥ 1000のユーザーをフィルタリングできます — `cut`では不可能です。

# Split by character position
cut -c1-5 file.txt          # Take characters 1-5
cut -c1,3,5 file.txt        # Take characters 1, 3, and 5

awk 'pattern {action}' file

# Basic usage
awk '{print $1}' file.txt           # Print column 1
awk '{print $1, $3}' file.txt       # Print columns 1 and 3
awk '{print NR, $0}' file.txt       # Print line number and entire line
awk '{print $NF}' file.txt           # Print last column
awk -F: '{print $1}' /etc/passwd    # Specify colon as delimiter

# Pattern matching
awk '/error/ {print}' log.txt       # Only process lines containing error
awk '$3 > 100 {print $1, $3}' data.txt  # Output only when column 3 > 100

# Aggregation
awk '{sum += $1} END {print sum}' numbers.txt  # Sum
awk '{count++} END {print count}' file.txt     # Count

# Replace
sed 's/old/new/' file.txt          # Replace first occurrence per line
sed 's/old/new/g' file.txt         # Global replace
sed 's/old/new/2' file.txt         # Replace only 2nd occurrence per line
sed 's/old/new/g' file.txt > new.txt  # Output to new file
sed -i 's/old/new/g' file.txt     # In-place edit (modifies file directly)
sed -i.bak 's/old/new/g' file.txt # Backup original as file.txt.bak

> 💡 ヒント：`sed -i`は元ファイルをその場で変更し、元に戻せません。まず`-i`なしでプレビュー：`sed 's/old/new/g' file.txt`、確認後に`-i`を追加。または`sed -i.bak`で自動バックアップを作成。

# Delete
sed '/pattern/d' file.txt          # Delete lines matching pattern
sed '3d' file.txt                  # Delete line 3
sed '5,10d' file.txt               # Delete lines 5-10
sed '/^$/d' file.txt               # Delete blank lines

# Print
sed -n '5,10p' file.txt            # Print only lines 5-10
sed -n '/pattern/p' file.txt       # Print only matching lines

# Case conversion
echo "hello" | tr 'a-z' 'A-Z'      # HELLO
cat file.txt | tr '[:lower:]' '[:upper:]'

# Replace and delete
echo "a,b,c" | tr ',' ' '          # a b c
echo "hello   world" | tr -s ' '   # hello world (squeeze repeated spaces)
echo "hello123" | tr -d '0-9'      # hello (delete digits)
echo "abc123" | tr -d '[:digit:]'  # abc (delete digits)

# Line ending conversion (Windows ↔ Unix)
tr -d '\r' < win.txt > unix.txt    # Remove Windows CR
tr '\n' ',' < list.txt             # Replace newlines with commas

diff file1.txt file2.txt           # Basic comparison
diff -u file1.txt file2.txt        # Unified format (more readable)
diff -i file1.txt file2.txt        # Ignore case
diff -w file1.txt file2.txt        # Ignore whitespace differences
diff -r dir1/ dir2/                # Recursive directory comparison

# Extract username and shell from /etc/passwd
cut -d: -f1,7 /etc/passwd | head -5
# root:/bin/bash
# daemon:/usr/sbin/nologin
# bin:/usr/sbin/nologin

# Take column 5 (file size) from ls -l output
ls -la | tr -s ' ' | cut -d' ' -f5 | head -5

# Print username and home directory
awk -F: '{print $1, "Home:", $6}' /etc/passwd | head -5

# Find regular users with UID greater than 1000
awk -F: '$3 >= 1000 {print $1, "(UID:" $3 ")"}' /etc/passwd

# Calculate total file size
ls -la | grep "^-" | awk '{sum += $5} END {print "Total:", sum/1024/1024, "MB"}'

# Formatted output (printf)
ls -la | awk '{printf "%-20s %8s\n", $9, $5}'

# Replace localhost with 127.0.0.1 in the file
sed -i 's/localhost/127.0.0.1/g' config.txt

# Comment out lines starting with listen in the config file
sed -i '/^listen/s/^/#/' nginx.conf

# Insert a new line after line 2
sed '2a\New line inserted' file.txt

# Extract timestamps from log file
sed -n 's/.*\[\(.*\)\].*/\1/p' access.log | head -5

# Count words in text
cat file.txt | tr -s ' ' '\n' | sort | uniq -c | sort -rn | head -10

# Convert a list to a single comma-separated line
cat list.txt | tr '\n' ',' | sed 's/,$//'

# Remove all comments and blank lines from a config file
grep -v "^#" config.txt | grep -v "^$" | tr -s '\n'

# Create two versions
echo -e "apple\nbanana\norange" > v1.txt
echo -e "apple\nblueberry\norange" > v2.txt

# Compare
diff -u v1.txt v2.txt
# --- v1.txt
# +++ v2.txt
# @@ -1,3 +1,3 @@
#  apple
# -banana
# +blueberry
#  orange

#!/bin/bash
# sys-report.sh - Generate system monitoring report

echo "========================================="
echo "        System Monitoring Report"
echo "        Generated at: $(date)"
echo "========================================="
echo ""

echo "1. Top 5 Processes by CPU Usage"
ps aux --sort=-%cpu | head -6 | awk '{printf "%-10s %-5s %-5s %s\n", $1, $2, $3, $11}'

echo ""
echo "2. Disk Usage"
df -h | grep "^/" | awk '{printf "%-20s %5s %5s %5s\n", $6, $2, $3, $5}'

echo ""
echo "3. Top 5 by Memory Usage"
ps aux --sort=-%mem | head -6 | awk '{printf "%-10s %-5s %-5s %s\n", $1, $2, $4, $11}'

echo ""
echo "4. Error Count in /var/log"
grep -ric "error" /var/log/*.log 2>/dev/null | grep -v ":0" | head -10

echo ""
echo "5. Listening Ports"
ss -tlnp | tail -n +2 | awk '{print $4, $1}' | sed 's/.*://'

echo ""
echo "========================================="