Files
notes/resource/脚本/数据清洗-日志筛选.md
T
2026-03-01 01:43:46 +08:00

416 lines
14 KiB
Markdown
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
一、创建 `find_bpm_logs.sh` 脚本,内容,**注意修改**LOGS_BASE_DIR 值为线上路径。
```shell
#!/bin/bash
# 定义包含SQL查询结果的文件
INPUT_FILE="data.txt"
# 定义日志根目录 (请根据你的实际情况修改)
LOGS_BASE_DIR="/opt/cdnportal/logs"
# 检查数据文件是否存在
if [ ! -f "$INPUT_FILE" ]; then
echo "错误: 数据文件 '$INPUT_FILE' 不存在。"
exit 1
fi
# 逐行读取数据文件
while IFS=',' read -r updated_time code
do
# 去除 code 字段可能存在的前后空格或换行符
code=$(echo "$code" | tr -d '[:space:]')
# 如果行是空的,则跳过
if [ -z "$updated_time" ] || [ -z "$code" ]; then
continue
fi
# --- 根据 updated_time 构建日志路径 ---
year_month=$(date -d "$updated_time" "+%Y-%m")
full_date=$(date -d "$updated_time" "+%Y-%m-%d")
hour=$(date -d "$updated_time" "+%H")
# 拼接出目标日志所在的目录
log_dir_path="$LOGS_BASE_DIR/$year_month/$full_date/$full_date-$hour"
# --- 开始搜索日志 ---
if [ -d "$log_dir_path" ]; then
# 使用 zgrep 搜索,然后用 grep 过滤,最后用 sed 提取 JSON
# 【【【 这是修改过的核心行 】】】
found_json=$(zgrep "$code" "$log_dir_path"/app-*.log.gz | grep 'callback from bpm' | grep 'approve' | sed 's/.*callback from bpm: //')
if [ -n "$found_json" ]; then
echo "$found_json"
else
echo "未找到相关的 callback 日志。"
fi
else
echo "警告: 日志目录不存在: $log_dir_path"
fi
done < "$INPUT_FILE"
echo "======================================================================"
echo "所有数据处理完毕。"
```
二、给权限:
```
chmod +x find_bpm_logs.sh
```
三、同目录下放data.txt
```
2024-12-09 09:11:28,001-20241127001
2024-12-11 09:05:24,001-20241128005
2024-12-12 09:08:51,001-20241128006
2024-12-13 13:57:43,001-20241128008
2024-12-06 09:20:20,001-20241204001
2025-01-22 09:13:02,001-20241205001
2024-12-11 11:18:45,001-20241206001
2024-12-11 11:18:33,001-20241209001
2024-12-11 11:18:20,001-20241209002
2024-12-09 17:54:21,001-20241209003
2024-12-10 16:45:36,001-20241210001
2024-12-12 11:35:20,001-20241211001
2024-12-12 11:35:32,001-20241211002
2024-12-13 17:01:29,001-20241211003
2024-12-15 09:53:29,001-20241212001
2024-12-16 15:05:56,001-20241212002
2024-12-18 09:57:59,001-20241212003
2024-12-16 11:07:09,001-20241215001
2024-12-17 10:40:36,001-20241216001
2024-12-19 14:05:55,001-20241217001
2024-12-18 15:51:48,001-20241218001
2024-12-18 15:41:35,001-20241218002
2024-12-18 19:41:56,001-20241218003
2024-12-18 15:41:53,001-20241218004
2024-12-18 17:21:49,001-20241218005
2024-12-30 15:59:48,001-20241219001
2024-12-30 15:59:23,001-20241220001
2024-12-30 16:00:00,001-20241220002
2024-12-30 15:59:39,001-20241223001
2024-12-26 11:19:01,001-20241224001
2025-01-09 16:03:44,001-20241226001
2024-12-30 09:23:33,001-20241227001
2024-12-27 16:03:06,001-20241227002
2025-01-02 11:12:56,001-20241227003
2024-12-30 15:59:19,001-20241230001
2024-12-30 15:59:35,001-20241230002
2025-01-01 00:33:38,001-20250101001
2025-01-10 09:50:34,001-20250102001
2025-01-17 19:36:51,001-20250106001
2025-01-09 14:23:50,001-20250107001
2025-01-09 15:25:37,001-20250108001
2025-01-16 16:38:27,001-20250109001
2025-01-10 15:12:13,001-20250109002
2025-01-23 14:22:23,001-20250110001
2025-01-13 16:54:28,001-20250113001
2025-01-13 16:54:09,001-20250113002
2025-01-22 10:16:48,001-20250114001
2025-01-20 16:24:38,001-20250120001
2025-01-20 16:24:50,001-20250120002
2025-01-23 09:55:58,001-20250120003
2025-02-18 11:16:00,001-20250121001
2025-01-24 09:15:23,001-20250122001
2025-01-24 09:15:42,001-20250122002
2025-01-24 14:50:31,001-20250123001
2025-02-20 09:20:58,001-20250123002
2025-02-08 09:31:49,001-20250207001
2025-02-14 10:00:33,001-20250208001
2025-02-12 11:11:17,001-20250208002
2025-02-20 14:18:08,001-20250211001
2025-02-11 22:51:26,001-20250211002
2025-02-20 09:20:43,001-20250212001
2025-02-13 10:00:22,001-20250212002
2025-02-14 08:58:25,001-20250213001
2025-02-17 10:21:01,001-20250217001
2025-02-19 09:50:56,001-20250218001
2025-02-18 16:23:45,001-20250218002
2025-03-27 09:53:17,001-20250223001
2025-02-26 16:57:50,001-20250225001
2025-02-26 16:57:35,001-20250225002
2025-02-26 16:56:37,001-20250225003
2025-02-26 16:57:02,001-20250226001
2025-03-26 09:05:43,001-20250228001
2025-03-03 15:41:53,001-20250228002
2025-03-07 09:05:39,001-20250306001
2025-03-17 14:11:15,001-20250310001
2025-03-17 14:10:11,001-20250310002
2025-03-17 14:10:47,001-20250310003
2025-03-14 09:54:21,001-20250310004
2025-03-19 15:17:03,001-20250314001
2025-03-17 16:35:12,001-20250317001
2025-03-17 16:53:25,001-20250317002
2025-03-17 16:35:31,001-20250317003
2025-03-17 16:53:57,001-20250317004
2025-04-07 09:44:51,001-20250317005
2025-04-07 09:46:31,001-20250317006
2025-04-07 09:40:32,001-20250318001
2025-04-07 09:42:20,001-20250318002
2025-04-07 09:47:08,001-20250318003
2025-04-07 09:45:56,001-20250318004
2025-04-07 09:43:12,001-20250318005
2025-04-07 09:43:36,001-20250318006
2025-04-07 09:40:34,001-20250318007
2025-04-07 09:41:48,001-20250318008
2025-03-19 09:02:49,001-20250318009
2025-03-19 16:03:54,001-20250319001
2025-03-24 09:43:05,001-20250319002
2025-03-26 17:13:18,001-20250320002
2025-03-26 17:13:04,001-20250320004
2025-03-20 22:45:26,001-20250320005
2025-03-20 16:54:57,001-20250320006
2025-03-26 10:41:02,001-20250320007
2025-03-26 10:44:58,001-20250321001
2025-03-31 18:44:02,001-20250324002
2025-03-25 10:12:15,001-20250324003
2025-03-25 11:35:56,001-20250324004
2025-03-25 11:35:21,001-20250324005
2025-04-07 09:39:35,001-20250325002
2025-04-07 09:39:18,001-20250325003
2025-03-26 10:03:49,001-20250325004
2025-03-27 17:22:05,001-20250326001
2025-03-27 17:21:51,001-20250327001
2025-03-28 10:06:29,001-20250327002
2025-03-28 10:04:42,001-20250327003
2025-03-28 10:05:42,001-20250327004
2025-03-28 16:19:10,001-20250328001
2025-04-01 09:34:41,001-20250328002
2025-03-31 16:16:11,001-20250331001
2025-04-01 16:11:21,001-20250331002
2025-04-01 10:24:04,001-20250331004
2025-04-01 16:12:24,001-20250401001
2025-04-10 11:29:34,001-20250401003
2025-04-02 09:10:30,001-20250401004
2025-04-10 11:29:49,001-20250401005
2025-04-02 16:41:58,001-20250401006
2025-04-03 14:29:20,001-20250402001
2025-04-08 16:02:27,001-20250403001
2025-04-22 14:43:38,001-20250407001
2025-04-08 15:04:40,001-20250407002
2025-04-09 11:03:54,001-20250407003
2025-04-23 11:06:21,001-20250408001
2025-04-09 14:18:36,001-20250408002
2025-04-23 16:42:17,001-20250410001
2025-04-11 09:01:58,001-20250410003
2025-04-18 10:00:26,001-20250410004
2025-04-15 11:25:19,001-20250414001
2025-04-18 16:45:15,001-20250414002
2025-04-18 16:19:08,001-20250414003
2025-04-16 15:51:05,001-20250415001
2025-04-16 15:50:47,001-20250415002
2025-04-16 15:50:32,001-20250415003
2025-04-16 18:50:38,001-20250416002
2025-04-30 12:03:49,001-20250421001
2025-04-22 15:07:39,001-20250421002
2025-04-24 17:37:20,001-20250422001
2025-04-23 15:49:28,001-20250422004
2025-04-24 17:36:58,001-20250422005
2025-04-23 10:07:00,001-20250422006
2025-04-23 10:07:40,001-20250422007
2025-05-15 09:01:49,001-20250422008
2025-04-24 16:20:44,001-20250423001
2025-05-30 17:42:38,001-20250423002
2025-04-23 20:16:50,001-20250423003
2025-04-30 15:12:13,001-20250424001
2025-04-28 16:23:20,001-20250425001
2025-04-30 09:41:32,001-20250425002
2025-05-06 11:16:49,001-20250425003
2025-04-27 12:14:38,001-20250425004
2025-04-27 11:25:26,001-20250425005
2025-04-27 11:25:03,001-20250425006
2025-04-30 10:32:21,001-20250427001
2025-04-28 12:44:44,001-20250427002
2025-04-28 14:30:42,001-20250427003
2025-04-28 11:14:17,001-20250427004
2025-04-29 09:50:03,001-20250428001
2025-04-30 10:31:51,001-20250428002
2025-05-19 09:10:09,001-20250429001
2025-04-30 10:02:59,001-20250429002
2025-04-30 10:08:35,001-20250429003
2025-05-15 09:00:59,001-20250507002
2025-05-19 15:57:38,001-20250507003
2025-05-08 11:36:56,001-20250507004
2025-05-08 16:13:28,001-20250508001
2025-05-08 16:44:51,001-20250508002
2025-05-08 17:21:48,001-20250508003
2025-05-09 09:15:34,001-20250508004
2025-05-08 16:44:36,001-20250508005
2025-05-08 16:45:04,001-20250508006
2025-05-09 10:49:08,001-20250508007
2025-05-20 09:05:14,001-20250508008
2025-05-13 16:29:50,001-20250509001
2025-07-11 08:54:27,001-20250509002
2025-05-16 09:16:50,001-20250512001
2025-05-12 19:33:51,001-20250512002
2025-05-15 11:01:48,001-20250513001
2025-05-19 16:17:21,001-20250515001
2025-05-15 15:01:49,001-20250515002
2025-05-16 14:25:02,001-20250515003
2025-05-19 17:51:24,001-20250516001
2025-05-19 16:27:50,001-20250516002
2025-05-19 10:48:16,001-20250516003
2025-05-19 10:48:01,001-20250516005
2025-05-27 09:47:46,001-20250516006
2025-05-19 11:31:30,001-20250516007
2025-05-19 14:35:48,001-20250516008
2025-05-19 17:52:52,001-20250516009
2025-05-27 09:31:27,001-20250519001
2025-05-20 16:05:47,001-20250519002
2025-05-22 11:30:42,001-20250519003
2025-05-23 16:27:12,001-20250521001
2025-05-26 16:14:43,001-20250521002
2025-05-30 15:07:01,001-20250523001
2025-06-18 17:58:20,001-20250523002
2025-05-26 16:39:39,001-20250526001
2025-05-30 09:27:16,001-20250526002
2025-08-01 10:30:09,001-20250526003
2025-06-09 08:51:39,001-20250526004
2025-05-28 14:49:53,001-20250526005
2025-05-27 14:53:14,001-20250526006
2025-05-27 15:12:22,001-20250527001
2025-05-27 15:11:59,001-20250527002
2025-05-28 11:42:57,001-20250527003
2025-06-09 17:05:38,001-20250529001
2025-05-30 14:00:04,001-20250530001
2025-05-30 17:43:24,001-20250530002
2025-06-03 16:40:22,001-20250603001
2025-06-04 09:17:19,001-20250603002
2025-06-04 09:16:55,001-20250603003
2025-06-03 21:01:44,001-20250603004
2025-06-12 09:22:20,001-20250605001
2025-06-11 09:29:51,001-20250606001
2025-06-09 09:49:46,001-20250606002
2025-06-10 11:02:11,001-20250609001
2025-06-10 11:01:43,001-20250609002
2025-06-16 15:31:14,001-20250610001
2025-06-16 15:31:50,001-20250610003
2025-06-16 15:32:23,001-20250610004
2025-06-16 15:32:54,001-20250610005
2025-06-20 09:27:19,001-20250611001
2025-06-12 13:09:58,001-20250611002
2025-06-20 09:22:57,001-20250611003
2025-06-16 15:42:29,001-20250612001
2025-06-13 15:14:04,001-20250613001
2025-06-13 15:13:41,001-20250613002
2025-06-16 14:48:52,001-20250613003
2025-06-15 10:07:34,001-20250613004
2025-06-24 11:13:30,001-20250616001
2025-06-20 17:15:30,001-20250617001
2025-06-20 15:50:54,001-20250617002
2025-06-17 18:10:18,001-20250617003
2025-06-17 18:10:41,001-20250617004
2025-06-17 14:41:23,001-20250617005
2025-06-18 16:19:33,001-20250617006
2025-06-18 14:54:31,001-20250617007
2025-06-18 14:56:56,001-20250618001
2025-06-18 16:14:04,001-20250618002
2025-06-19 17:02:27,001-20250619001
2025-06-25 16:00:46,001-20250619002
2025-06-20 14:54:34,001-20250620001
2025-06-23 14:28:18,001-20250620002
2025-06-23 16:04:31,001-20250623001
2025-07-11 14:31:31,001-20250623002
2025-06-30 10:08:16,001-20250623004
2025-06-24 17:25:56,001-20250623005
2025-06-23 23:12:00,001-20250623006
2025-06-30 10:08:33,001-20250624001
2025-07-11 15:46:48,001-20250624003
2025-07-11 15:47:22,001-20250625001
2025-06-26 17:11:25,001-20250625002
2025-06-26 11:25:22,001-20250626001
2025-06-27 10:28:59,001-20250626002
2025-06-27 15:45:07,001-20250626003
2025-06-27 15:54:58,001-20250626004
2025-06-27 15:54:36,001-20250626005
2025-07-11 15:47:36,001-20250626006
2025-06-27 18:10:44,001-20250627002
2025-07-09 09:36:11,001-20250630001
2025-07-04 10:13:08,001-20250630002
2025-07-11 15:47:01,001-20250630003
2025-07-04 16:57:31,001-20250702001
2025-07-07 15:20:22,001-20250703001
2025-07-04 10:50:18,001-20250703002
2025-07-16 15:17:00,001-20250703003
2025-07-08 10:12:48,001-20250707002
2025-07-10 09:32:57,001-20250708001
2025-07-08 17:15:52,001-20250708003
2025-07-17 11:23:04,001-20250709001
2025-07-10 15:37:47,001-20250709002
2025-07-10 17:04:17,001-20250709004
2025-07-28 17:20:31,001-20250710001
2025-07-16 14:58:33,001-20250711001
2025-07-14 10:48:20,001-20250711002
2025-07-14 17:01:05,001-20250711003
2025-07-17 16:17:24,001-20250714001
2025-07-14 15:13:37,001-20250714002
2025-07-28 17:20:43,001-20250714003
2025-07-28 17:22:09,001-20250714004
2025-07-28 17:20:59,001-20250714005
2025-08-04 11:30:01,001-20250714006
2025-07-28 17:22:21,001-20250714007
2025-07-23 14:42:36,001-20250715002
2025-07-17 10:42:55,001-20250716001
2025-08-01 22:43:00,001-20250717002
2025-07-18 10:16:42,001-20250717003
2025-07-23 10:25:20,001-20250721001
2025-07-22 11:33:37,001-20250721002
2025-07-22 11:33:51,001-20250721003
2025-07-22 09:55:09,001-20250721004
2025-07-29 17:17:50,001-20250722001
2025-07-29 17:38:09,001-20250722002
2025-07-29 17:21:13,001-20250723001
2025-07-25 11:20:44,001-20250724001
2025-08-01 10:23:40,001-20250724002
2025-08-06 16:06:47,001-20250728001
2025-08-07 08:43:15,001-20250728002
2025-08-06 16:02:31,001-20250728003
2025-07-31 17:24:50,001-20250728004
2025-07-29 13:56:35,001-20250728005
2025-07-29 13:57:06,001-20250728006
2025-07-29 13:56:49,001-20250728007
2025-07-31 17:23:43,001-20250729001
2025-07-31 17:25:09,001-20250729002
2025-07-29 19:37:40,001-20250729004
2025-07-30 09:57:56,001-20250729005
2025-07-31 17:24:28,001-20250729006
2025-08-18 09:40:08,001-20250730001
2025-08-11 10:14:00,001-20250731002
2025-08-01 08:50:47,001-20250731003
2025-08-01 08:51:17,001-20250731004
2025-08-05 21:39:10,001-20250801002
2025-08-03 17:47:16,001-20250801003
2025-08-11 10:13:36,001-20250804001
2025-08-05 21:38:36,001-20250805002
2025-08-06 15:51:15,001-20250806001
2025-08-06 15:51:29,001-20250806002
2025-08-18 15:23:53,001-20250808002
2025-08-13 15:31:12,001-20250808003
2025-08-18 09:29:23,001-20250811001
2025-08-14 10:09:22,001-20250812001
2025-08-14 14:47:45,001-20250813001
2025-08-26 11:04:05,001-20250814001
2025-08-14 16:19:31,001-20250814003
2025-09-01 18:47:11,001-20250814004
2025-08-14 15:33:23,001-20250814005
2025-08-15 23:28:36,001-20250814007
2025-08-21 14:06:15,001-20250814009
2025-08-27 10:09:53,001-20250815002
2025-08-20 10:30:19,001-20250818001
2025-08-19 08:58:50,001-20250818002
2025-08-19 10:06:51,001-20250818004
2025-08-19 17:51:23,001-20250819003
2025-08-21 14:58:52,001-20250819004
2025-08-20 09:45:32,001-20250820001
2025-08-25 17:34:49,001-20250821001
2025-08-24 10:45:44,001-20250822001
2025-08-27 10:10:37,001-20250822002
2025-08-26 11:03:26,001-20250825001
2025-08-27 09:21:46,001-20250826001
2025-08-27 14:41:22,001-20250827001
2025-08-28 15:14:37,001-20250827003
```