diff --git a/deploy/pre_deploy_check.sh b/deploy/pre_deploy_check.sh index 02ea6a37..b9ebd6f1 100644 --- a/deploy/pre_deploy_check.sh +++ b/deploy/pre_deploy_check.sh @@ -12,20 +12,32 @@ # # 用法: bash deploy/pre_deploy_check.sh # 退出码: 0=全部通过, 1=至少一道门未过 +# 日志: deploy/gate_log.jsonl(每次检查自动追加记录) set -euo pipefail DEPLOY_DIR="${NEXUS_DEPLOY_DIR:-/opt/nexus}" TODAY=$(date +%Y-%m-%d) +TIMESTAMP=$(date -Iseconds) GATES_PASSED=0 GATES_TOTAL=7 BLOCKED=0 +GATE_RESULTS="" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[0;33m' NC='\033[0m' # No Color +# ── 日志函数 ── +gate_log() { + local gate="$1" result="$2" detail="${3:-}" + local entry="{\"ts\":\"${TIMESTAMP}\",\"date\":\"${TODAY}\",\"gate\":\"${gate}\",\"result\":\"${result}\",\"detail\":\"${detail}\"}" + GATE_RESULTS="${GATE_RESULTS}${entry}," + # 追加到 JSONL 日志文件 + echo "${entry}" >> "${DEPLOY_DIR}/deploy/gate_log.jsonl" +} + echo "========================================" echo " Nexus Pre-deploy Gate Check (v2)" echo " Date: ${TODAY}" @@ -43,17 +55,20 @@ if ls "${CHANGELOG_DIR}/${TODAY}-"*.md 1>/dev/null 2>&1; then echo -e "${GREEN}PASS${NC}" echo " └─ Found: $(basename "${FILE}") (${LINES} lines)" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "changelog" "PASS" "$(basename "${FILE}") ${LINES}lines" else echo -e "${RED}BLOCK${NC}" echo " └─ File exists but only ${LINES} lines (need ≥10)" echo " └─ Changelog must have substance, not just a placeholder" BLOCKED=1 + gate_log "changelog" "BLOCK" "only ${LINES} lines" fi else echo -e "${RED}BLOCK${NC}" echo " └─ No changelog file found: ${CHANGELOG_DIR}/${TODAY}-*.md" echo " └─ Create changelog before deploying" BLOCKED=1 + gate_log "changelog" "BLOCK" "file not found" fi # ── Gate 2: Audit (存在 + 关键段落) ── @@ -69,6 +84,7 @@ if ls "${AUDIT_DIR}/${TODAY}-"*.md 1>/dev/null 2>&1; then echo -e "${GREEN}PASS${NC}" echo " └─ Found: $(basename "${FILE}") (Step3✓ Closure✓ DoD✓)" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "audit" "PASS" "$(basename "${FILE}")" else echo -e "${RED}BLOCK${NC}" echo " └─ Audit file exists but missing required sections:" @@ -76,12 +92,14 @@ if ls "${AUDIT_DIR}/${TODAY}-"*.md 1>/dev/null 2>&1; then [ "${HAS_CLOSURE}" -eq 0 ] && echo " └─ Missing: Closure表" [ "${HAS_DOD}" -eq 0 ] && echo " └─ Missing: DoD (Definition of Done)" BLOCKED=1 + gate_log "audit" "BLOCK" "missing sections" fi else echo -e "${RED}BLOCK${NC}" echo " └─ No audit file found: ${AUDIT_DIR}/${TODAY}-*.md" echo " └─ Run line-walk audit (8 steps) and save report before deploying" BLOCKED=1 + gate_log "audit" "BLOCK" "file not found" fi # ── Gate 3: Test (必须存在 + 必须通过) ── @@ -92,17 +110,22 @@ if [ ! -f "${TEST_SCRIPT}" ]; then echo " └─ Test script not found: ${TEST_SCRIPT}" echo " └─ Tests are MANDATORY — cannot deploy without test coverage" BLOCKED=1 + gate_log "test" "BLOCK" "script not found" else TEST_OUTPUT=$(cd "${DEPLOY_DIR}" && python3 "${TEST_SCRIPT}" 2>&1) || true - if echo "${TEST_OUTPUT}" | grep -qiE "FAIL|ERROR|failed|error"; then + FAIL_COUNT=$(echo "${TEST_OUTPUT}" | grep -cE "^\s+\[FAIL\]" 2>/dev/null || true) + FAIL_COUNT=${FAIL_COUNT:-0} + if [ "${FAIL_COUNT}" -gt 0 ] || echo "${TEST_OUTPUT}" | grep -qE "[0-9]+ test(s) failed"; then echo -e "${RED}BLOCK${NC}" echo " └─ Tests failed. Output:" echo "${TEST_OUTPUT}" | head -20 | sed 's/^/ │ /' BLOCKED=1 + gate_log "test" "BLOCK" "tests failed" else echo -e "${GREEN}PASS${NC}" echo " └─ All tests passed" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "test" "PASS" "all passed" fi fi @@ -119,16 +142,19 @@ if command -v ruff &>/dev/null; then echo -e "${GREEN}PASS${NC}" echo " └─ ruff check: 0 violations" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "lint" "PASS" "0 violations" else echo -e "${RED}BLOCK${NC}" echo " └─ ruff found ${LINT_COUNT} violations:" echo "${LINT_OUTPUT}" | head -15 | sed 's/^/ │ /' BLOCKED=1 + gate_log "lint" "BLOCK" "${LINT_COUNT} violations" fi else echo -e "${YELLOW}SKIP${NC}" echo " └─ ruff not installed (pip install ruff)" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "lint" "SKIP" "ruff not installed" fi # ── Gate 5: Import (python -c "import server.main") ── @@ -145,6 +171,7 @@ if [ "${IMPORT_OK}" -eq 1 ]; then echo -e "${GREEN}PASS${NC}" echo " └─ server.main imports successfully" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "import" "PASS" "ok" else # 检查是否有真正的导入错误(不是缺少运行时依赖如数据库) if echo "${IMPORT_OUTPUT}" | grep -qiE "ImportError|ModuleNotFoundError|SyntaxError"; then @@ -152,11 +179,13 @@ else echo " └─ Import failed:" echo "${IMPORT_OUTPUT}" | head -5 | sed 's/^/ │ /' BLOCKED=1 + gate_log "import" "BLOCK" "import/syntax error" else # 运行时依赖缺失(如数据库连接)不算BLOCK,但警告 echo -e "${YELLOW}WARN${NC}" echo " └─ Import has runtime dependency issues (not code errors)" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "import" "WARN" "runtime deps only" fi fi @@ -172,16 +201,19 @@ if command -v bandit &>/dev/null; then echo -e "${GREEN}PASS${NC}" echo " └─ bandit: 0 HIGH findings" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "security" "PASS" "0 HIGH" else echo -e "${RED}BLOCK${NC}" echo " └─ bandit found ${HIGH_COUNT} HIGH severity issues:" echo "${SEC_OUTPUT}" | grep -A3 "Severity: High" | head -15 | sed 's/^/ │ /' BLOCKED=1 + gate_log "security" "BLOCK" "${HIGH_COUNT} HIGH findings" fi else echo -e "${YELLOW}SKIP${NC}" echo " └─ bandit not installed (pip install bandit)" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "security" "SKIP" "bandit not installed" fi # ── Gate 7: Review (审计文件交叉验证) ── @@ -212,13 +244,16 @@ if ls "${AUDIT_DIR}/${TODAY}-"*.md 1>/dev/null 2>&1; then echo -e "${GREEN}PASS${NC}" echo " └─ Audit covers all changed files" GATES_PASSED=$((GATES_PASSED + 1)) + gate_log "review" "PASS" "audit covers changes" else BLOCKED=1 + gate_log "review" "BLOCK" "audit missing changed files" fi else echo -e "${RED}BLOCK${NC}" echo " └─ No audit file to validate against" BLOCKED=1 + gate_log "review" "BLOCK" "no audit file" fi # ── Summary ── @@ -227,9 +262,11 @@ echo "========================================" if [ "${BLOCKED}" -eq 0 ]; then echo -e " Result: ${GREEN}${GATES_PASSED}/${GATES_TOTAL} gates passed${NC} — Deploy allowed" echo "========================================" + gate_log "summary" "PASS" "${GATES_PASSED}/${GATES_TOTAL}" exit 0 else echo -e " Result: ${RED}${GATES_PASSED}/${GATES_TOTAL} gates passed${NC} — Deploy BLOCKED" echo "========================================" + gate_log "summary" "BLOCK" "${GATES_PASSED}/${GATES_TOTAL}" exit 1 fi diff --git a/docs/changelog/2026-05-26-gate3-test-fix.md b/docs/changelog/2026-05-26-gate3-test-fix.md new file mode 100644 index 00000000..7b1f10ac --- /dev/null +++ b/docs/changelog/2026-05-26-gate3-test-fix.md @@ -0,0 +1,43 @@ +# Gate3 误判修复 + test_api.py 生产适配 + +**日期**: 2026-05-26 +**变更摘要**: 修复 Gate3 测试门控误判 BLOCK 问题;完善 test_api.py 适配生产环境 + +## 动机 + +1. Gate3 使用 `grep -qiE "FAIL|ERROR|failed|error"` 匹配测试输出,但测试摘要行 "0/25 failed" 也会被匹配,导致全部通过时仍被 BLOCK +2. test_api.py 在生产环境登录失败(默认密码 admin 不对),需从 .env 加载测试凭据 +3. test_api.py 使用错误的状态码期望(POST 创建期望 200 应为 201,DELETE 期望 200 应为 204) + +## 变更内容 + +### deploy/pre_deploy_check.sh — Gate3 修复 +- 旧逻辑: `grep -qiE "FAIL|ERROR|failed|error"` — 匹配摘要行 "0/25 failed" 导致误判 +- 新逻辑: 精确计数 `[FAIL]` 标记 + 检查 "N test(s) failed" 模式 +- 修复 sed 替换丢失 `\s` 转义的问题(`^s+` → `^\s+\[FAIL\]`) + +### tests/test_api.py — 生产适配 +- 新增 `_load_credentials_from_env_file()` 从 .env 加载 NEXUS_TEST_ADMIN_USER / NEXUS_TEST_ADMIN_PASSWORD +- POST 创建操作 expect_code 从 200 改为 201(REST 标准) +- DELETE 操作 expect_code 从 200 改为 204(REST 标准) +- 修复 server_id 回退逻辑:`else 1` → `else None`(避免操作不存在的服务器) +- 新增测试前清理残留的 test-server-e2e(防止重复创建报错) +- 新增 204 No Content 空响应体处理(`json.loads("")` 会崩溃) +- Schedules 创建改用 `run_mode: "cron"` + `cron_expr`(避免 once 模式需要 fire_at) +- Schedules 的 `server_ids` 从数组改为字符串 `"all"` + +## 涉及文件 + +- `deploy/pre_deploy_check.sh` — Gate3 检测逻辑修复 +- `tests/test_api.py` — 生产适配 + REST 状态码修复 + +## 是否需迁移/重启 + +- 无需重启后端(test_api.py 是独立脚本) +- 需部署 pre_deploy_check.sh 到远程服务器 + +## 验证方式 + +1. 远程运行 `bash deploy/pre_deploy_check.sh` → Gate3 PASS(之前 BLOCK) +2. 远程运行 `python3 tests/test_api.py` → 25/25 passed +3. 全部 7/7 门控通过 diff --git a/docs/changelog/2026-05-26-knowledge-graph-restructure.md b/docs/changelog/2026-05-26-knowledge-graph-restructure.md new file mode 100644 index 00000000..d2f280cf --- /dev/null +++ b/docs/changelog/2026-05-26-knowledge-graph-restructure.md @@ -0,0 +1,70 @@ +# 知识图谱重组 + MegaMemory 嵌入模型部署 + +**日期**: 2026-05-26 +**变更摘要**: 重组两个知识库,去重规则知识,补全项目架构知识;部署 Xenova 嵌入模型使 MegaMemory 语义搜索可用 + +## 动机 + +1. `完美实现原则`(全局CLAUDE.md)、`Nexus实现原则`、`Nexus强制标准纪律` 三者 80% 内容重叠,知识图谱里再存一遍纯属冗余——规则已在 CLAUDE.md 每次session自动加载 +2. 知识图谱完全缺少项目架构知识(模块、技术栈、数据流、部署拓扑),无法通过语义搜索快速定位模块 +3. MegaMemory 的 `understand` 语义搜索一直报 "fetch failed"——嵌入模型未下载 + +## 变更内容 + +### server-memory(全局跨项目) +- **删除 9 个重复规则实体**: 完美实现原则、Nexus实现原则、Nexus强制标准纪律、ThreeIronRules、FourDimensionChecklist、ProblemReportFormat、AuditEnforcementMechanism、MandatoryFileModificationFlow、LineWalkAudit8Steps +- **保留 2 个非规则实体**: uzuma(用户偏好)、Nexus-Production-Server(服务器信息) + +### MegaMemory(项目级架构知识) +- **创建 19 个概念节点**: 项目顶层 + 9个架构模块 + 4个基础设施子模块 + 3个认证子模块 + 2个独立配置 +- **创建 30 条关系边**: 项目→模块(connects_to/depends_on)、模块→子模块(implements)、数据流→基础设施(connects_to/depends_on) +- **生成 384 维嵌入向量**: 所有节点通过 Xenova/all-MiniLM-L6-v2 本地模型生成嵌入 + +### 嵌入模型部署 +- 从 hf-mirror.com 下载 Xenova/all-MiniLM-L6-v2 量化 ONNX 模型(23MB) +- 先通过远程 Linux 服务器下载,再 SCP 传回本地 +- 模型放置到 `node_modules/@xenova/transformers/models/` 本地路径 + +## 知识图谱结构 + +``` +nexus-project (feature) +├── nexus-backend (module) — FastAPI Clean Architecture 4层 +├── nexus-frontend (module) — Tailwind+Alpine.js 12页面 +├── nexus-infra (module) — SSH池+Redis+Telegram+WebSocket +│ ├── nexus-ssh-pool (component) +│ ├── nexus-redis (component) +│ ├── nexus-telegram (component) +│ └── nexus-websocket (component) +├── nexus-auth (feature) — JWT+TOTP+Agent API Key +│ ├── nexus-jwt (component) +│ ├── nexus-totp (component) +│ └── nexus-agent-auth (component) +├── nexus-dataflow (pattern) — 心跳→Redis→MySQL+WS+TG +├── nexus-gates (config) — 7门控部署 +├── nexus-sync (feature) — 4种同步模式 +├── nexus-database (module) — 14表+加密+脱敏 +└── nexus-deploy (config) — 3层守护+SCP部署 + +nexus-production (config) — 服务器连接信息 +uzuma (config) — 用户偏好 +``` + +## 涉及文件 + +- `.megamemory/knowledge.db` — MegaMemory SQLite 数据库(19节点+30边+19嵌入) +- `.megamemory/populate.py` — 数据导入脚本(一次性使用) +- `~/.cache/xenova/transformers/` — Xenova 模型缓存 +- `node_modules/@xenova/transformers/models/Xenova/all-MiniLM-L6-v2/` — 本地模型文件 + +## 是否需迁移/重启 + +- 重启 Claude Code 后 MegaMemory MCP 会自动连接并使用已下载的模型 + +## 验证方式 + +1. `megamemory stats` 显示 19 nodes / 30 edges +2. `megamemory list_roots` 返回 3 个根节点 +3. `megamemory understand "认证系统"` 返回 JWT/TOTP/Agent认证(相似度 >0.15) +4. `megamemory understand "心跳数据流转"` 返回数据流概念(相似度 >0.50) +5. server-memory `read_graph` 只剩 2 个实体(uzuma + Nexus-Production-Server) diff --git a/tests/test_api.py b/tests/test_api.py index 874d6736..ca243da4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -16,6 +16,41 @@ import urllib.error BASE = os.environ.get("NEXUS_TEST_BASE", "http://127.0.0.1:8600") ADMIN_USER = os.environ.get("NEXUS_TEST_ADMIN_USER", "admin") ADMIN_PASSWORD = os.environ.get("NEXUS_TEST_ADMIN_PASSWORD", "admin") + + +def _load_credentials_from_env_file(): + """Try to load test credentials from .env file (production deploy scenario). + + The gate check runs test_api.py automatically on the deploy server. + Instead of hardcoding production passwords, read from .env variables: + NEXUS_TEST_ADMIN_USER / NEXUS_TEST_ADMIN_PASSWORD + These can be set in .env by the admin during installation. + """ + env_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env") + if not os.path.exists(env_path): + return + try: + with open(env_path) as f: + for line in f: + line = line.strip() + if line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + value = value.strip().strip("\"'") + if key == "NEXUS_TEST_ADMIN_USER" and not os.environ.get("NEXUS_TEST_ADMIN_USER"): + os.environ["NEXUS_TEST_ADMIN_USER"] = value + elif key == "NEXUS_TEST_ADMIN_PASSWORD" and not os.environ.get("NEXUS_TEST_ADMIN_PASSWORD"): + os.environ["NEXUS_TEST_ADMIN_PASSWORD"] = value + except Exception: + pass + + +# Load from .env before reading env vars +_load_credentials_from_env_file() +BASE = os.environ.get("NEXUS_TEST_BASE", "http://127.0.0.1:8600") +ADMIN_USER = os.environ.get("NEXUS_TEST_ADMIN_USER", "admin") +ADMIN_PASSWORD = os.environ.get("NEXUS_TEST_ADMIN_PASSWORD", "admin") PASS = 0 FAIL = 0 @@ -45,7 +80,18 @@ def test(name: str, method: str, path: str, body=None, expect_code=200, headers= try: resp = urllib.request.urlopen(req, timeout=10) code = resp.getcode() - result = json.loads(resp.read()) + resp_body = resp.read() + # Handle 204 No Content and empty responses + if not resp_body or resp_body.strip() == b"": + if code == expect_code: + PASS += 1 + print(f" [PASS] {name}") + return {} + else: + FAIL += 1 + print(f" [FAIL] {name}: expected {expect_code}, got {code}") + return None + result = json.loads(resp_body) if code == expect_code: PASS += 1 print(f" [PASS] {name}") @@ -58,6 +104,9 @@ def test(name: str, method: str, path: str, body=None, expect_code=200, headers= if code == expect_code: PASS += 1 result_text = e.read().decode()[:200] + if not result_text or result_text.strip() == "": + print(f" [PASS] {name}") + return {} try: return json.loads(result_text) except Exception: @@ -98,6 +147,14 @@ test("GET /api/auth/me", "GET", "/api/auth/me") # --- Server CRUD --- print("\n[3] Server CRUD") +# Clean up any leftover test server from previous runs +existing_servers = test("GET /api/servers/ (pre-cleanup)", "GET", "/api/servers/?search=test-server-e2e", expect_code=200) +if existing_servers and isinstance(existing_servers, dict): + for srv in existing_servers.get("items", existing_servers.get("servers", [])): + if isinstance(srv, dict) and srv.get("name") == "test-server-e2e": + test("DELETE leftover test server", "DELETE", f"/api/servers/{srv['id']}", expect_code=200) + break + created = test("POST /api/servers/ (create)", "POST", "/api/servers/", body={ "name": "test-server-e2e", "domain": "192.168.1.100", @@ -107,16 +164,19 @@ created = test("POST /api/servers/ (create)", "POST", "/api/servers/", body={ "password": "test123", "target_path": "/tmp/nexus-test", "category": "test", -}) -server_id = created.get("id") if created and isinstance(created, dict) else 1 +}, expect_code=201) +server_id = created.get("id") if created and isinstance(created, dict) else None test("GET /api/servers/ (list)", "GET", "/api/servers/") test("GET /api/servers/stats", "GET", "/api/servers/stats") -test(f"GET /api/servers/{server_id}", "GET", f"/api/servers/{server_id}") -test(f"PUT /api/servers/{server_id} (update)", "PUT", f"/api/servers/{server_id}", body={ - "description": "E2E test server", -}) -test(f"DELETE /api/servers/{server_id}", "DELETE", f"/api/servers/{server_id}") +if server_id: + test(f"GET /api/servers/{server_id}", "GET", f"/api/servers/{server_id}") + test(f"PUT /api/servers/{server_id} (update)", "PUT", f"/api/servers/{server_id}", body={ + "description": "E2E test server", + }) + test(f"DELETE /api/servers/{server_id}", "DELETE", f"/api/servers/{server_id}", expect_code=204) +else: + print(" → SKIP: Server CRUD detail tests (no server_id from create)") # --- Scripts --- print("\n[4] Scripts") @@ -124,36 +184,40 @@ script = test("POST /api/scripts/ (create)", "POST", "/api/scripts/", body={ "name": "test-script", "category": "ops", "content": "echo hello", -}) -script_id = script.get("id") if script and isinstance(script, dict) else 1 +}, expect_code=201) +script_id = script.get("id") if script and isinstance(script, dict) else None test("GET /api/scripts/ (list)", "GET", "/api/scripts/") -test(f"DELETE /api/scripts/{script_id}", "DELETE", f"/api/scripts/{script_id}") +if script_id: + test(f"DELETE /api/scripts/{script_id}", "DELETE", f"/api/scripts/{script_id}", expect_code=204) # --- Schedules --- print("\n[5] Schedules") sched = test("POST /api/schedules/ (create)", "POST", "/api/schedules/", body={ "name": "test-schedule", "source_path": "/tmp/nexus-test", + "run_mode": "cron", "cron_expr": "0 2 * * *", - "server_ids": "[]", + "server_ids": "all", "enabled": True, -}) -sched_id = sched.get("id") if sched and isinstance(sched, dict) else 1 +}, expect_code=201) +sched_id = sched.get("id") if sched and isinstance(sched, dict) else None test("GET /api/schedules/ (list)", "GET", "/api/schedules/") -test(f"PUT /api/schedules/{sched_id} (disable)", "PUT", f"/api/schedules/{sched_id}", body={ - "enabled": False, -}) -test(f"DELETE /api/schedules/{sched_id}", "DELETE", f"/api/schedules/{sched_id}") +if sched_id: + test(f"PUT /api/schedules/{sched_id} (disable)", "PUT", f"/api/schedules/{sched_id}", body={ + "enabled": False, + }) + test(f"DELETE /api/schedules/{sched_id}", "DELETE", f"/api/schedules/{sched_id}", expect_code=204) # --- Presets --- print("\n[6] Password Presets") preset = test("POST /api/presets/ (create)", "POST", "/api/presets/", body={ "name": "test-preset", "encrypted_pw": "test-password-123", -}) -preset_id = preset.get("id") if preset and isinstance(preset, dict) else 1 +}, expect_code=201) +preset_id = preset.get("id") if preset and isinstance(preset, dict) else None test("GET /api/presets/ (list)", "GET", "/api/presets/") -test(f"DELETE /api/presets/{preset_id}", "DELETE", f"/api/presets/{preset_id}") +if preset_id: + test(f"DELETE /api/presets/{preset_id}", "DELETE", f"/api/presets/{preset_id}", expect_code=204) # --- Settings --- print("\n[7] Settings")