Caddy监控与数据采集优化内存版
Caddy 指标监控系统优化方案 (1C1G / SD卡环境)
一、 架构概述
针对资源受限的嵌入式开发板（1核1G、SD卡存储环境），通过将 SQLite 数据库整体迁移至内存盘 (/dev/shm) 运行，结合长周期持久化同步策略，彻底消除高频随机写入对 SD 卡的物理磨损，同时大幅提升数据采集与 Web 查询性能。
二、 核心优化策略
- 内存盘全内存运行：将数据库实时读写路径指向
/dev/shm/vqq2.db（Linux 内存文件系统），实现零 SD 卡 I/O 损耗。 - 长周期持久化同步：保持计划任务每 5 分钟高频采集，但通过时间戳控制每 2 小时（7200秒）将内存数据库整库覆盖备份回 SD 卡一次，兼顾硬件保护与数据安全。
- 单事务批量提交：采用全局单事务
executemany写入，减少磁盘碎片与写放大。 - 正则预编译：在全局预编译正则表达式，降低单核 CPU 负载。
- Datasette 内存直读：Web 查询服务直接挂载内存数据库，实现零延迟实时响应。
三、 完整代码实现 (/data/jishu/2jishu.py)
import sqlite3
import time
import re
import urllib.request
import ssl
import base64
import os
import shutil
NODES = [
{
"name": "zcaddyjishu1",
"url": "https://zcaddyjishu1.zhaopeng.site/metrics",
"user": "zadmin",
"pass": "P4mG"
},
{
"name": "zcaddyjishu2",
"url": "https://zcaddyjishu2.zhaopeng.site/metrics",
"user": "zadmin",
"pass": "P4mG"
}
]
DB_PATH = "/dev/shm/vqq2.db"
SD_BACKUP_PATH = "/data/jishu/vqq2.db"
RETENTION_DAYS = 30
LAST_CLEANUP_FILE = "/data/jishu/.last_cleanup"
LAST_SYNC_FILE = "/data/jishu/.last_sync"
SYNC_INTERVAL = 7200 # 同步间隔：7200秒 = 2小时
HOST_PATTERN = re.compile(
r'^caddy_http_requests_total\{[^}]*host="([^"]+)"[^}]*\}\s+(\d+(?:\.\d+)?)',
re.MULTILINE
)
CODE_PATTERN = re.compile(r'code="([^"]+)"')
def init_db(conn):
try:
conn.execute("PRAGMA journal_mode=WAL;")
except sqlite3.OperationalError:
pass
conn.execute("PRAGMA synchronous=NORMAL;")
conn.execute("PRAGMA busy_timeout=5000;")
conn.execute("PRAGMA wal_autocheckpoint=5000;")
conn.execute("PRAGMA temp_store=MEMORY;")
with conn:
conn.execute("""
CREATE TABLE IF NOT EXISTS http_requests (
timestamp INTEGER,
node TEXT,
host TEXT,
code INTEGER,
count INTEGER,
delta_count INTEGER DEFAULT 0,
PRIMARY KEY (timestamp, node, host, code)
) WITHOUT ROWID;
""")
def should_cleanup(now):
today = now // 86400
try:
with open(LAST_CLEANUP_FILE) as f:
last_day = int(f.read().strip())
except (FileNotFoundError, ValueError):
last_day = 0
return today > last_day, today
def mark_cleaned(today):
try:
with open(LAST_CLEANUP_FILE, "w") as f:
f.write(str(today))
except OSError as e:
print(f"写清理标记失败: {e}")
def should_sync_to_sd(now):
try:
with open(LAST_SYNC_FILE, "r") as f:
last_sync_time = int(f.read().strip())
except (FileNotFoundError, ValueError):
last_sync_time = 0
return (now - last_sync_time) >= SYNC_INTERVAL
def mark_synced(now):
try:
with open(LAST_SYNC_FILE, "w") as f:
f.write(str(now))
except OSError as e:
print(f"写同步标记失败: {e}")
def fetch_metrics(node_cfg):
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
req = urllib.request.Request(node_cfg["url"])
auth_str = f"{node_cfg['user']}:{node_cfg['pass']}"
b64_auth = base64.b64encode(auth_str.encode('utf-8')).decode('utf-8')
req.add_header("Authorization", f"Basic {b64_auth}")
with urllib.request.urlopen(req, context=ctx, timeout=10) as response:
return response.read().decode('utf-8')
def parse_metrics(node_name, metrics_text, conn, now):
rows = []
cursor = conn.cursor()
cursor.execute("""
SELECT host, code, count
FROM http_requests
WHERE node = ? AND timestamp = (
SELECT MAX(timestamp) FROM http_requests WHERE node = ?
)
""", (node_name, node_name))
last_records = {(row[0], row[1]): row[2] for row in cursor.fetchall()}
for match in HOST_PATTERN.finditer(metrics_text):
labels_str = match.group(0)
host = match.group(1)
current_count = int(float(match.group(2)))
code_m = CODE_PATTERN.search(labels_str)
try:
code = int(code_m.group(1)) if code_m else 200
except (ValueError, AttributeError):
code = 0
last_count = last_records.get((host, code))
if last_count is None:
delta = 0
elif current_count >= last_count:
delta = current_count - last_count
else:
delta = current_count
rows.append((now, node_name, host, code, current_count, delta))
return rows
def query_db(conn):
cursor = conn.cursor()
print("-" * 80)
print("多节点内存增量查询 (最近 10 条)")
print("-" * 80)
cursor.execute("""
SELECT datetime(timestamp, 'unixepoch', 'localtime'),
node, host, code, delta_count, count
FROM http_requests
ORDER BY timestamp DESC
LIMIT 10;
""")
for row in cursor.fetchall():
print(f"时间: {row[0]} | 节点: {row[1]} | 域名: {row[2]} | 状态: {row[3]} | 增量: +{row[4]} | 累计: {row[5]}")
print("-" * 80)
def main():
now = int(time.time())
if not os.path.exists(DB_PATH) and os.path.exists(SD_BACKUP_PATH):
try:
shutil.copy2(SD_BACKUP_PATH, DB_PATH)
print("【初始化】已从 SD 卡恢复历史数据库到内存盘 (/dev/shm)")
except Exception as e:
print(f"恢复数据库失败: {e}")
all_rows = []
try:
conn = sqlite3.connect(DB_PATH, timeout=30)
init_db(conn)
for node in NODES:
try:
metrics_text = fetch_metrics(node)
rows = parse_metrics(node["name"], metrics_text, conn, now)
all_rows.extend(rows)
except Exception as e:
print(f"拉取节点 [{node['name']}] 失败: {e}")
if all_rows:
need_clean, today = should_cleanup(now)
with conn:
conn.executemany("""
INSERT OR IGNORE INTO http_requests
(timestamp, node, host, code, count, delta_count)
VALUES (?, ?, ?, ?, ?, ?);
""", all_rows)
if need_clean:
expire_time = now - (RETENTION_DAYS * 86400)
conn.execute("DELETE FROM http_requests WHERE timestamp < ?;", (expire_time,))
if need_clean:
mark_cleaned(today)
print(f"成功更新 {len(all_rows)} 条数据项\n")
query_db(conn)
conn.execute("PRAGMA wal_checkpoint(TRUNCATE);")
conn.close()
if should_sync_to_sd(now):
shutil.copy2(DB_PATH, SD_BACKUP_PATH)
mark_synced(now)
print("【持久化】内存数据库已按周期备份回 SD 卡")
else:
print("【跳过同步】未到 2 小时同步周期，当前仅在内存运行")
except Exception as e:
print(f"数据库异常: {e}")
if __name__ == "__main__":
main()
四、 部署与维护命令
更新 Web 服务指向内存数据库：
pkill -f datasette nohup datasette /dev/shm/vqq2.db -h 0.0.0.0 -p 8001 --cors > /dev/null 2>&1 &配置计划任务 (Crontab)：
*/5 * * * * /usr/bin/python3 /data/jishu/2jishu.py >> /data/jishu/cron.log 2>&1
AdGuard Home 运行在内存盘与定时持久化方案 (1C1G / SD卡环境)
一、 架构概述
针对资源受限的嵌入式开发板（1核1G、SD卡存储环境），通过将 AdGuard Home 的运行时工作目录 (workdir) 迁移至内存盘 (/dev/shm)，彻底消除 stats.db 等高频写入对 SD 卡的物理磨损。同时采用定时同步脚本，实现每 2 小时将内存数据持久化备份回 SD 卡。
二、 核心改造步骤
1. 迁移现有数据到内存并修改挂载
在重建容器前，需将 SD 卡现有的工作目录复制到内存盘，并调整 podman-compose.yml 的卷挂载路径。
执行初始化迁移与停止容器：
podman-compose down
cp -r /data/adguard/workdir /dev/shm/adguard_work
修改 podman-compose.yml：
podman run -d --name adguardhome \
-p 192.168.1.9:53:53/tcp \
-p 192.168.1.9:53:53/udp \
-p 3000:3000/tcp \
-m 300m --memory-swap 300m \
-p 8081:80/tcp \
-v /dev/shm/adguard_work:/opt/adguardhome/work:Z \
-v /data/adguard/confdir:/opt/adguardhome/conf:Z \
--log-driver none \
-e TZ=Asia/Shanghai \
docker.1ms.run/adguard/adguardhome
2. 编写定时持久化同步脚本 (/data/adguard/sync_adguard.sh)
创建同步脚本：
cat << 'EOF' > /data/adguard/sync_adguard.sh
#!/bin/bash
SRC="/dev/shm/adguard_work/"
DST="/data/adguard/workdir/"
if [ -d "$SRC" ]; then
cp -rp "$SRC"/* "$DST"
echo "$(date '+%Y-%m-%d %H:%M:%S') - AdGuard workdir synced to SD card"
else
echo "Error: /dev/shm/adguard_work does not exist!"
fi
EOF
赋予执行权限：
chmod +x /data/adguard/sync_adguard.sh
3. 配置计划任务 (Crontab)
通过 Cron 实现每 2 小时自动将内存中的运行数据备份回 SD 卡：
crontab -e
添加以下任务：
0 */2 * * * /bin/bash /data/adguard/sync_adguard.sh >> /data/adguard/sync.log 2>&1
root@pbsbc01h3:/dev/shm# du -sh *
21M adguard_work
52K libpod_lock
6.6M vqq2.db
32K vqq2.db-shm
0 vqq2.db-wal
运行状态非常完美。
从 du 输出可以确认：
- AdGuard Home 工作目录 (
adguard_work, 21M) 与 Caddy 监控数据库 (vqq2.db, 6.6M) 均已成功托管至内存盘/dev/shm。 - SQLite 的
vqq2.db-wal大小为0，说明脚本中的PRAGMA wal_checkpoint(TRUNCATE);已成功执行，日志已完全归档并清空，内存占用处于极佳状态。
当前硬件的 SD 卡写入瓶颈已被彻底消除。