Splunk

# 下载 Splunk (去 splunk.com 注册免费账号)
# macOS 用 .dmg, Linux 用 .tgz

# 解压后启动
cd /Applications/Splunk/bin   # macOS 默认路径
./splunk start --accept-license

# 首次启动会提示设置 admin 密码
# 访问 http://localhost:8000 ，用 admin + 你的密码登录

# 配置开机自启
./splunk enable boot-start

# 状态检查 / 停止
./splunk status
./splunk stop

# 方法1：通过 Web UI
Settings → Add Data → Upload Files → 选择文件 → 设置 sourcetype → Review → Submit

# 方法2：编辑 inputs.conf 监控目录
$SPLUNK_HOME/etc/system/local/inputs.conf
[monitor:///var/log/myapp/]
index = main
sourcetype = myapp_log
disabled = false

# 方法3：通过 REST API 用 HEC 发送
curl -k https://localhost:8088/services/collector/event \
  -H "Authorization: Splunk <token>" \
  -d '{"event": "login failed user=alice", "sourcetype": "myapp"}'

# inputs.conf — 定义采集什么
[monitor:///var/log/nginx/access.log]
index = web
sourcetype = nginx_access

# outputs.conf — 定义发给谁
[tcpout]
defaultGroup = primary
[tcpout:primary]
server = 10.0.1.20:9997,10.0.1.21:9997
sslVerifyServerCert = false

# 关键词搜索（最宽的结果）
error

# 字段过滤（key=value 是自动字段匹配）
sourcetype=nginx_access status=500

# 布尔逻辑（AND/OR/NOT 大写）
error AND server NOT localhost

# 比较运算符
bytes>10000  status>=400

# 通配符
host=web-*  uri_path="/api/*"

# top — 统计最常见的值
sourcetype=nginx_access | top uri_path limit=10

# stats — 聚合统计（最有用的命令之一）
sourcetype=nginx_access | stats count avg(bytes) max(bytes) min(bytes)

# stats + by — 分组统计
sourcetype=nginx_access | stats count avg(bytes) max(bytes) by status

# timechart — 时间序列聚合
sourcetype=nginx_access | timechart span=1h count by status

# sort — 排序
sourcetype=nginx_access | stats count by uri_path | sort -count | head 20

# dedup — 去重
sourcetype=nginx_access | dedup client_ip

# 把 bytes 转为 KB
... | eval size_kb = round(bytes / 1024, 2)

# 条件判断 — 给请求分类
... | eval latency_level = case(
    response_time < 200, "fast",
    response_time < 1000, "normal",
    response_time >= 1000, "slow"
)

# 时间运算 — 计算耗时
... | eval duration_ms = (end_time - start_time) * 1000

# 字符串操作
... | eval domain = replace(client_ip, "(\d+\.\d+)\.\d+\.\d+", "\1.0.0")
... | eval upper_path = upper(uri_path)

# 从日志中提取自定义字段
... | rex "User (?<username>\S+) logged in from (?<ip>\d+\.\d+\.\d+\.\d+)"

# 从 URL 路径提取版本号
... | rex field=uri_path "/api/v(?<api_version>\d+)/"

# 从 URL query string 提取参数
... | rex field=uri_query "page=(?<page>\d+)"

# 查找"访问了 /login 且报错"的 IP 们
sourcetype=nginx_access status=500
    [ search sourcetype=nginx_access uri_path="/login" | fields client_ip ]

# 查找最近 1 小时没有心跳的服务器
sourcetype=server_heartbeat NOT
    [ search sourcetype=server_heartbeat earliest=-1h | fields host ]

# 按 session_id 和 client_ip 拼接用户的一次完整访问链路
sourcetype=nginx_access | transaction session_id client_ip maxspan=5m \
    startswith="uri_path=/login" endswith="uri_path=/checkout"

# 1. 准备 CSV 文件 (ip_location.csv)
# ip_start,ip_end,country,city
# 1.0.0.0,1.0.0.255,US,Los Angeles

# 2. Settings → Lookups → Add new → 上传 CSV

# 3. 在搜索中使用
sourcetype=nginx_access
| lookup ip_location ip_start AS client_ip OUTPUT country city
| stats count by country city

# 定义 macro: "web_errors(1)"
sourcetype=$arg1$ status>=500

# 使用（带参数）
`web_errors(nginx_access)` | stats count by status, uri_path

# 面板1：请求量趋势（时间序列图）
sourcetype=nginx_access
| timechart span=1m count

# 面板2：HTTP 状态码分布（饼图）
sourcetype=nginx_access
| stats count by status

# 面板3：Top 10 慢请求（表格）
sourcetype=nginx_access response_time>1000
| stats count avg(response_time) as avg_ms by uri_path
| sort -avg_ms | head 10

# 面板4：错误率趋势（面积图）
sourcetype=nginx_access
| eval is_error = if(status>=500, 1, 0)
| timechart span=1h eval(round(sum(is_error)/count*100,2)) as error_rate_pct

# 面板5：单值指标（Single Value）
sourcetype=nginx_access
| stats avg(response_time) as avg_latency_ms
| eval avg_latency_ms = round(avg_latency_ms, 1)

# Step 1：写好搜索查询（验证结果符合预期）
sourcetype=nginx_access status>=500 earliest=-5m
  | stats count by status, uri_path

# Step 2：Save As → Alert
#   - Alert type: Scheduled
#   - Schedule: every 5 minutes
#   - Trigger: Number of Results > 0

# Step 3：配置 Trigger Actions
#   - Send Email (含结果表格)
#   - Webhook → Slack / PagerDuty / 企业微信
#   - Run a script (自定义脚本)

# Step 4：设置 Throttling（防止告警风暴）
#   - Suppress for 30 minutes after first alert

# 每日错误汇总，早上 9:00 发送 PDF 到老板
sourcetype=nginx_access status>=500 earliest=-1d@d latest=@d
| stats count by status, uri_path
| sort -count

# Save As → Report → Schedule: daily at 9:00 → Email PDF

# 名称：Web Analytics
# 添加 Root Event：
#   - Constraint: sourcetype=nginx_access
#   - Auto-extracted fields → 勾选 status, uri_path, bytes, client_ip

# 添加 Child Object "Errors"：
#   - Additional Constraint: status >= 400

# 添加 Calculated Field "Size MB"：
#   - Formula: bytes / 1048576

# 然后用户就能去 Pivot 里拖拽 status, uri_path 做交叉分析了

# 传统 stats（慢，需要扫描全部 event）
index=web | stats count by host

# tstats（快，直接读 tsidx metadata）
| tstats count WHERE index=web BY host

# tstats + prestats 用于 Dashboard
| tstats prestats=t count avg(bytes) WHERE index=web BY sourcetype
| timechart span=1h count avg(bytes) by sourcetype

# 查看 Splunk 自身日志
index=_internal | stats count by component, log_level

# 查看索引和 bucket 状态
| dbinspect index=* | stats sum(sizeOnDiskMB) as total_mb by index

# 查看慢搜索
index=_audit action=search search=* total_run_time>10
| table user search total_run_time

# License 用量
index=_internal source=*license_usage.log type=Usage
| timechart span=1d sum(b) as daily_bytes