R: R والتعبيرات النمطية

192.168.1.1 - - [15/Jan/2024:10:30:45 +0800] "GET /api/users/123 HTTP/1.1" 404 512
192.168.1.2 - - [15/Jan/2024:10:30:46 +0800] "POST /api/login HTTP/1.1" 200 1024
192.168.1.3 - - [15/Jan/2024:10:30:47 +0800] "GET /api/products/456 HTTP/1.1" 404 256
...

library(stringr)

# 1. Read the log
log <- readLines("nginx.log")

# 2. Match using regular expressions 404 Incorrect URL(One line)
pattern <- '"GET (\\S+) HTTP.*" 404'
urls_404 <- str_match(log, pattern)[, 2]

# 3. Track Traffic
url_table <- table(urls_404)
sort(url_table, decreasing = TRUE)[1:10]

# Exact match of literal characters
"cat"  # Matches "cat"
"123"  # Matches "123"

# Extract the year, month, and day of the date
str_match("2024-01-15", "(\\d{4})-(\\d{2})-(\\d{2})")
#      [,1]         [,2]   [,3]   [,4]
# [1,] "2024-01-15" "2024" "01"   "15"

# The first column is an exact match,Below are the capture groups

# Match Duplicate Words(e.g. "the the")
str_detect("the the cat", "\\b(\\w+)\\s+\\1\\b")
# [1] TRUE

# Match XML/HTML Tags
str_detect("<div>content</div>", "<(\\w+)>.*</\\1>")
# [1] TRUE

# Uncaptured groups(Use ?:)
str_match("John, Smith", "(\\w+)(?:,\\s+)(\\w+)")
# [,1]         [,2]   [,3]
# "John, Smith" "John" "Smith"  ← Commas and spaces are not grouped separately

# Default Greedy:.* Match as many as possible
str_match("<b>text1</b><b>text2</b>", "<b>(.*)</b>")
# [,1]                          [,2]
# "<b>text1</b><b>text2</b>"    "text1</b><b>text2"
#                                 ↑ Greedy matching until the last one </b>

# Add ? to change to "non-greedy":.*? Match as few as possible
str_match("<b>text1</b><b>text2</b>", "<b>(.*?)</b>")
# [,1]            [,2]
# "<b>text1</b>"  "text1"  ← Matched the first one </b>

# Which strings start with "a" Introduction
str_detect(c("apple", "banana", "cherry"), "^a")
# [1]  TRUE FALSE FALSE

# Contains numbers
str_detect(c("abc", "a1c", "123"), "\\d")
# [1] FALSE  TRUE  TRUE

# Extract the number
str_extract(c("abc 123", "def 456", "no digits"), "\\d+")
# [1] "123"  "456"  NA

# Extract all numbers(List)
str_extract_all(c("a1b2c3", "no digits"), "\\d")
# [[1]] "1" "2" "3"
# [[2]] character(0)

# Extraction Date: Year/Month/Day
str_match("2024-01-15", "(\\d{4})-(\\d{2})-(\\d{2})")
#      [,1]         [,2]   [,3]   [,4]
# [1,] "2024-01-15" "2024" "01"   "15"

# Vectorization
dates <- c("2024-01-15", "2024-02-20", "2024-03-25")
str_match(dates, "(\\d{4})-(\\d{2})-(\\d{2})")
#      [,1]         [,2]   [,3]   [,4]
# [1,] "2024-01-15" "2024" "01"   "15"
# [2,] "2024-02-20" "2024" "02"   "20"
# [3,] "2024-03-25" "2024" "03"   "25"

# Replace all numbers with X
str_replace_all("abc 123 def 456", "\\d+", "X")
# [1] "abc X def X"

# Complex Replacements:Using capture groups
str_replace_all("John Smith", "(\\w+) (\\w+)", "\\2, \\1")
# [1] "Smith, John"  ← Last Name First

# Cell phone number
phone_pat <- "^1[3-9]\\d{9}$"

# Email
email_pat <- "^[\\w.+-]+@[\\w.-]+\\.[a-zA-Z]{2,}$"

# ID Number(18 bit)
id_pat <- "^[1-9]\\d{5}(18|19|20)\\d{2}(0[1-9]|1[0-2])(0[1-9]|[12]\\d|3[01])\\d{3}[\\dXx]$"

# URL
url_pat <- "^https?://[\\w.-]+(:\\d+)?(/\\S*)?$"

# IPv4
ipv4_pat <- "^((25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)$"

# Date
date_pat <- "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$"

# Test
str_detect("13800001234", phone_pat)    # TRUE
str_detect("user@example.com", email_pat)  # TRUE
str_detect("110101199003078888", id_pat)  # TRUE
str_detect("https://example.com", url_pat)  # TRUE

url <- "https://www.example.com:8080/path/to/page?query=1#section"

# Extraction Agreement,Domain Name,Port,Path,Parameters
pattern <- "^(https?)://([^:/]+)(?::(\\d+))?(/[^?#]*)?(?:\\?([^#]*))?(?:#(.*))?$"
matches <- str_match(url, pattern)
# [,1]                            [,2]      [,3]              [,4]   [,5]            [,6]      [,7]
# "https://..."                    "https"   "www.example.com" "8080" "/path/to/page" "query=1" "section"

# ============================================
# Nginx Access Log Analysis
# Features:Parse using regular expressions @,000 rows ofLog,Key Statistical Indicators
# ============================================

library(stringr)
library(dplyr)

# 1. Simulation 1000 row Nginx Log(For use in actual projects readLines Read File)
set.seed(42)
n <- 1000
ips <- paste0("192.168.", sample(1:255, n, replace = TRUE), ".",
              sample(1:255, n, replace = TRUE))
methods <- sample(c("GET", "POST", "PUT", "DELETE"), n, replace = TRUE,
                  prob = c(0.7, 0.2, 0.05, 0.05))
paths <- paste0("/api/", sample(c("users", "products", "orders", "login"),
                                n, replace = TRUE), "/",
                sample(1:1000, n, replace = TRUE))
status_codes <- sample(c(200, 201, 301, 400, 404, 500), n, replace = TRUE,
                       prob = c(0.7, 0.05, 0.05, 0.05, 0.1, 0.05))
sizes <- sample(100:5000, n)

# Concatenate into log lines
log_lines <- sprintf(
  '%s - - [15/Jan/2024:10:30:%02d +0800] "%s %s HTTP/1.1" %d %d',
  ips, sample(0:59, n, replace = TRUE), methods, paths, status_codes, sizes
)

# 2. Write to the log file
writeLines(log_lines, "nginx_demo.log")
cat("Generated 1000 Transaction Log\n")

# 3. Parsing Logs with Regular Expressions
# Format:IP - - [Time] "Methods URL Agreement" Status Code Size
log_pattern <- '^(\\S+) - - \\[([^\\]]+)\\] "(\\S+) (\\S+) (\\S+)" (\\d+) (\\d+)$'

# Extract all capture groups
matches <- str_match(log_lines, log_pattern)
colnames(matches) <- c("full", "ip", "time", "method", "path", "protocol",
                       "status", "size")

# Rotate DataFrame
logs <- as_tibble(matches) |> select(-full) |>
  mutate(
    status = as.integer(status),
    size = as.integer(size)
  )

cat("\n=== First 6 Analysis results (entries) ===\n")
print(head(logs, 6))

# 4. Analysis 1:Status Code Distribution
cat("\n=== Status Code Distribution ===\n")
status_summary <- logs |>
  group_by(status) |>
  summarise(Number of times = n(), Percentage = round(n() / nrow(logs) * 100, 2)) |>
  arrange(desc(Number of times))
print(status_summary)

# 5. Analysis 2:Find 404 All Errors URL
cat("\n=== 404 Error Top 5 ===\n")
errors_404 <- logs |>
  filter(status == 404) |>
  group_by(path) |>
  summarise(Number of times = n()) |>
  arrange(desc(Number of times)) |>
  head(5)
print(errors_404)

# 6. Analysis 3:Each IP traffic
cat("\n=== Top 5 Active IP ===\n")
top_ips <- logs |>
  group_by(ip) |>
  summarise(
    Number of visits = n(),
    Number of errors = sum(status >= 400),
    Total Traffic = sum(size)
  ) |>
  arrange(desc(Number of visits)) |>
  head(5)
print(top_ips)

# 7. Analysis 4:Hourly traffic
cat("\n=== Visits per minute(First 10 minutes)===\n")
logs <- logs |>
  mutate(minute = str_sub(time, 15, 16))

per_minute <- logs |>
  group_by(minute) |>
  summarise(Number of visits = n(), Number of errors = sum(status >= 400)) |>
  head(10)
print(per_minute)

# 8. Retrieve Email Address(If the log contains an email address)
text_with_emails <- "Contact alice@example.com or bob@test.org for support."
email_pat <- "[\\w.+-]+@[\\w.-]+\\.[a-zA-Z]{2,}"
emails <- str_extract_all(text_with_emails, email_pat)
cat("\n=== Extracted email addresses ===\n")
print(emails)

# 9. Cleanup
file.remove("nginx_demo.log")

=== Status Code Distribution ===
# A tibble: 6 × 3
  status  Number of times Percentage
   <int> <int> <dbl>
1    200   698    70
2    404   105    10
3    201    52     5
...

=== Top 5 Active IP ===
# A tibble: 5 × 4
  ip           Number of visits Number of errors Total Traffic
  <chr>            <int>  <int>  <int>
1 192.168.97.1        8      1   18499
2 192.168.52.244      7      2   18438
...