feat(server): modularize backend and admin console

This commit is contained in:
2026-07-14 01:38:41 +00:00
parent 9a6ac1e4ed
commit 95dd32c75a
261 changed files with 39289 additions and 2156 deletions
+1
View File
@@ -0,0 +1 @@
"""Service package placeholder for the template migration."""
+63
View File
@@ -0,0 +1,63 @@
import os
from pathlib import Path
from fastapi import Request
from fastapi import HTTPException
PUBLIC_API_BASE_URL = os.getenv("PUBLIC_API_BASE_URL", "").strip().rstrip("/")
VALID_CLIENT_TOKEN = os.getenv("CLIENT_API_TOKEN")
RELEASE_MAIN_EXECUTABLE = os.getenv("RELEASE_MAIN_EXECUTABLE", "MainApp.exe").strip().replace(chr(92), "/").strip("/")
TARGET_PLATFORM = os.getenv("TARGET_PLATFORM", "windows")
TARGET_ARCH = os.getenv("TARGET_ARCH", "x64")
PUBLISH_MAX_REQUEST_BYTES = int(os.getenv("PUBLISH_MAX_REQUEST_MB", "4096")) * 1024 * 1024
PUBLISH_MAX_FILES = int(os.getenv("PUBLISH_MAX_FILES", "20000"))
PUBLISH_MAX_FIELDS = int(os.getenv("PUBLISH_MAX_FIELDS", str(PUBLISH_MAX_FILES + 100)))
def public_api_base_url(request: Request) -> str:
if PUBLIC_API_BASE_URL:
return PUBLIC_API_BASE_URL
forwarded_proto = request.headers.get("x-forwarded-proto", "").split(",")[0].strip()
forwarded_host = request.headers.get("x-forwarded-host", "").split(",")[0].strip()
scheme = forwarded_proto or request.url.scheme
host = forwarded_host or request.headers.get("host", "")
if host:
return f"{scheme}://{host}".rstrip("/")
return str(request.base_url).rstrip("/")
def default_client_main_executable() -> str:
path = RELEASE_MAIN_EXECUTABLE.strip().replace(chr(92), "/").strip("/")
if path.casefold().startswith("bin/"):
path = path[4:]
return path or "SimCAE.exe"
def default_client_config_values(request: Request) -> dict:
return {
"api_base_url": public_api_base_url(request),
"client_token": VALID_CLIENT_TOKEN or "",
"launch_token": os.getenv("CLIENT_LAUNCH_TOKEN", "SimCAE_Launch_Token_2026_ChangeMe_32Bytes"),
"request_timeout_ms": os.getenv("CLIENT_REQUEST_TIMEOUT_MS", "5000"),
"temp_folder": os.getenv("CLIENT_TEMP_FOLDER", "update_temp"),
"install_root": os.getenv("CLIENT_INSTALL_ROOT", ".."),
"main_executable": os.getenv("CLIENT_MAIN_EXECUTABLE", default_client_main_executable()),
"launcher_executable": os.getenv("CLIENT_LAUNCHER_EXECUTABLE", "Launcher.exe"),
"updater_executable": os.getenv("CLIENT_UPDATER_EXECUTABLE", "Updater.exe"),
"bootstrap_executable": os.getenv("CLIENT_BOOTSTRAP_EXECUTABLE", "Bootstrap.exe"),
"health_check_timeout_ms": os.getenv("CLIENT_HEALTH_CHECK_TIMEOUT_MS", "15000"),
"platform": TARGET_PLATFORM,
"arch": TARGET_ARCH,
}
def normalize_executable_name(value: str) -> str:
normalized = str(value or "").strip().replace(chr(92), "/").strip("/")
if not normalized:
return ""
if normalized.startswith("/") or ":" in normalized or any(part in ("", ".", "..") for part in Path(normalized).parts):
raise HTTPException(status_code=400, detail="主程序文件名必须是安全的相对路径")
if not normalized.casefold().endswith(".exe"):
normalized += ".exe"
return normalized
+56
View File
@@ -0,0 +1,56 @@
import json
from fastapi import HTTPException
def policy_row_to_dict(row) -> dict:
if not row:
return {
"policy_seq": 1,
"force_update": False,
"allow_rollback": False,
"offline_allowed": True,
"valid_until": "2099-12-31T23:59:59Z",
"min_supported_version": "",
"disabled_versions": [],
"message": "",
}
try:
disabled = json.loads(row["disabled_versions"] or "[]")
except (TypeError, json.JSONDecodeError):
disabled = []
return {
"policy_seq": int(row["policy_seq"]),
"force_update": bool(row["force_update"]),
"allow_rollback": bool(row["allow_rollback"]),
"offline_allowed": bool(row["offline_allowed"]),
"valid_until": row["valid_until"],
"min_supported_version": row["min_supported_version"] or "",
"disabled_versions": disabled if isinstance(disabled, list) else [],
"message": row["message"] or "",
}
def validate_channel_code(code: str) -> bool:
return bool(code) and len(code) <= 32 and code[0].isalnum() and all(c.isalnum() or c in "_-" for c in code)
def require_channel(conn, app_id: str, channel: str, require_enabled: bool = True):
row = conn.execute("SELECT * FROM channels WHERE app_id=? AND channel_code=?", (app_id, channel)).fetchone()
if not row:
raise HTTPException(status_code=400, detail={"error": "channel_not_found", "msg": f"渠道 {channel} 不存在"})
if require_enabled and not row["enabled"]:
raise HTTPException(status_code=403, detail={"error": "channel_disabled", "msg": f"渠道 {channel} 已停用"})
return row
def is_executable_path(path: str) -> bool:
return path.lower().endswith((".exe", ".dll"))
def version_key(version: str):
parts = []
for part in version.split("."):
digits = "".join(ch for ch in part if ch.isdigit())
parts.append(int(digits) if digits else 0)
return tuple(parts)
+598
View File
@@ -0,0 +1,598 @@
import hashlib
import json
import os
import secrets
import shutil
import sqlite3
import uuid
from datetime import datetime, timezone
from pathlib import Path
from fastapi import Request
from fastapi.responses import FileResponse, JSONResponse
import db
from app.core import security
from app.core.config import configured_path
from app.repositories import crash_report_repository
CRASH_STORAGE_ROOT = configured_path("CRASH_STORAGE_ROOT", "crash_storage")
CRASH_METADATA_MAX_BYTES = int(os.getenv("CRASH_METADATA_MAX_KB", "256")) * 1024
CRASH_MINIDUMP_MAX_BYTES = int(os.getenv("CRASH_MINIDUMP_MAX_MB", "128")) * 1024 * 1024
CRASH_ATTACHMENTS_MAX_BYTES = int(os.getenv("CRASH_ATTACHMENTS_MAX_MB", "64")) * 1024 * 1024
CRASH_REQUEST_MAX_BYTES = int(os.getenv("CRASH_REQUEST_MAX_MB", "200")) * 1024 * 1024
CRASH_SYMBOLS_MAX_BYTES = int(os.getenv("CRASH_SYMBOLS_MAX_MB", "512")) * 1024 * 1024
CRASH_SERVICE_VERSION = os.getenv("CRASH_SERVICE_VERSION", "1.0.0")
CRASH_REPORT_TOKEN = os.getenv("CRASH_REPORT_TOKEN") or os.getenv("SIMCAE_CRASH_CLIENT_TOKEN") or os.getenv("CLIENT_API_TOKEN")
CRASH_SYMBOL_TOKEN = os.getenv("CRASH_SYMBOL_TOKEN") or os.getenv("SIMCAE_SYMBOL_TOKEN")
CRASH_ADMIN_TOKEN = os.getenv("CRASH_ADMIN_TOKEN")
class CrashApiException(Exception):
def __init__(self, status_code: int, code: str, message: str, retryable: bool = False):
self.status_code = status_code
self.code = code
self.message = message
self.retryable = retryable
async def crash_api_exception_handler(request: Request, exc: CrashApiException):
return JSONResponse(
status_code=exc.status_code,
content={"error": {"code": exc.code, "message": exc.message, "retryable": exc.retryable}},
)
def ensure_storage_dirs():
CRASH_STORAGE_ROOT.mkdir(parents=True, exist_ok=True)
(CRASH_STORAGE_ROOT / "reports").mkdir(parents=True, exist_ok=True)
(CRASH_STORAGE_ROOT / "symbols").mkdir(parents=True, exist_ok=True)
def utc_now_text() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def crash_api_error(status_code: int, code: str, message: str, retryable: bool = False):
raise CrashApiException(status_code, code, message, retryable)
def bearer_token_from_request(request: Request) -> str:
value = request.headers.get("authorization", "")
if not value.lower().startswith("bearer "):
crash_api_error(401, "unauthorized", "missing bearer token", False)
token = value[7:].strip()
if not token:
crash_api_error(401, "unauthorized", "missing bearer token", False)
return token
def require_crash_bearer(request: Request, allowed_roles: set[str]) -> str:
token = bearer_token_from_request(request)
roles: set[str] = set()
if CRASH_REPORT_TOKEN and secrets.compare_digest(token, CRASH_REPORT_TOKEN):
roles.add("crash_report")
if CRASH_SYMBOL_TOKEN:
if secrets.compare_digest(token, CRASH_SYMBOL_TOKEN):
roles.add("symbols")
elif secrets.compare_digest(token, security.ADMIN_TOKEN):
roles.add("symbols")
if CRASH_ADMIN_TOKEN:
if secrets.compare_digest(token, CRASH_ADMIN_TOKEN):
roles.add("admin")
elif secrets.compare_digest(token, security.ADMIN_TOKEN):
roles.add("admin")
if not roles:
crash_api_error(401, "unauthorized", "invalid bearer token", False)
if roles.isdisjoint(allowed_roles):
crash_api_error(403, "forbidden", "token is not allowed to access this endpoint", False)
return token
def ensure_multipart_request(request: Request, max_bytes: int = CRASH_REQUEST_MAX_BYTES):
content_type = request.headers.get("content-type", "")
if "multipart/form-data" not in content_type.lower():
crash_api_error(415, "unsupported_media_type", "Content-Type must be multipart/form-data", False)
raw_length = request.headers.get("content-length")
if raw_length:
try:
content_length = int(raw_length)
except ValueError:
crash_api_error(400, "metadata_invalid", "Content-Length is invalid", False)
if content_length > max_bytes:
crash_api_error(413, "payload_too_large", "request body exceeds limit", False)
def is_upload_file(value) -> bool:
return hasattr(value, "filename") and hasattr(value, "read")
async def read_metadata_field(value, max_bytes: int) -> bytes:
if value is None:
crash_api_error(400, "metadata_invalid", "metadata is required", False)
if is_upload_file(value):
data = bytearray()
while True:
chunk = await value.read(64 * 1024)
if not chunk:
break
data.extend(chunk)
if len(data) > max_bytes:
crash_api_error(413, "payload_too_large", "metadata exceeds size limit", False)
return bytes(data)
data = str(value).encode("utf-8")
if len(data) > max_bytes:
crash_api_error(413, "payload_too_large", "metadata exceeds size limit", False)
return data
async def save_upload_limited(upload, target: Path, max_bytes: int, field_name: str) -> dict:
if not is_upload_file(upload):
crash_api_error(400, "metadata_invalid", f"{field_name} file is required", False)
target.parent.mkdir(parents=True, exist_ok=True)
total = 0
digest = hashlib.sha256()
try:
with open(target, "wb") as out_file:
while True:
chunk = await upload.read(1024 * 1024)
if not chunk:
break
total += len(chunk)
if total > max_bytes:
crash_api_error(413, "payload_too_large", f"{field_name} exceeds size limit", False)
digest.update(chunk)
out_file.write(chunk)
except Exception:
if target.exists():
target.unlink()
raise
if total <= 0:
crash_api_error(400, "metadata_invalid", f"{field_name} file is empty", False)
return {"sha256": digest.hexdigest(), "size": total, "filename": upload.filename or field_name}
def load_json_metadata(raw: bytes, error_code: str = "metadata_invalid") -> dict:
try:
obj = json.loads(raw.decode("utf-8"))
except Exception:
crash_api_error(400, error_code, "metadata must be valid UTF-8 JSON", False)
if not isinstance(obj, dict):
crash_api_error(400, error_code, "metadata must be a JSON object", False)
return obj
def require_metadata_fields(metadata: dict, fields: list[str], error_code: str = "metadata_invalid"):
for field in fields:
if metadata.get(field) in (None, ""):
crash_api_error(400, error_code, f"metadata.{field} is required", False)
if metadata.get("schemaVersion") != 1:
crash_api_error(400, error_code, "metadata.schemaVersion must be 1", False)
def normalize_declared_sha256(value, field_path: str) -> str:
if value in (None, ""):
return ""
text = str(value).strip().lower()
if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text):
crash_api_error(400, "metadata_invalid", f"{field_path} must be a SHA-256 hex string", False)
return text
def declared_file_sha256(metadata: dict, kind: str, filename: str) -> str:
files = metadata.get("files")
if not isinstance(files, list):
crash_api_error(400, "metadata_invalid", "metadata.files must be an array", False)
for index, item in enumerate(files):
if not isinstance(item, dict):
crash_api_error(400, "metadata_invalid", f"metadata.files[{index}] must be an object", False)
if item.get("kind") == kind or item.get("name") == filename:
return normalize_declared_sha256(item.get("sha256"), f"metadata.files[{index}].sha256")
return ""
def validate_client_report_id(value: str):
try:
uuid.UUID(value)
except Exception:
crash_api_error(400, "metadata_invalid", "metadata.clientReportId must be a UUID", False)
def safe_storage_component(value, fallback: str = "unknown") -> str:
text = str(value or fallback).strip()
cleaned = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text)
return cleaned[:96] or fallback
def crash_content_hash(metadata_sha256: str, minidump_sha256: str, attachments_sha256: str) -> str:
payload = json.dumps(
{"metadata": metadata_sha256, "minidump": minidump_sha256, "attachments": attachments_sha256 or ""},
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(payload).hexdigest()
def generate_prefixed_id(conn, table: str, column: str, prefix: str) -> str:
value = crash_report_repository.generate_prefixed_id(conn, table, column, prefix)
if not value:
crash_api_error(503, "service_unavailable", "could not allocate report id", True)
return value
def cleanup_path(path: Path | None):
if not path:
return
try:
if path.is_dir():
shutil.rmtree(path)
elif path.exists():
path.unlink()
except Exception as err:
print(f"清理临时文件失败: {path}: {err}")
def crash_report_response(row, duplicate: bool, status_code: int):
return JSONResponse(
status_code=status_code,
content={
"reportId": row["report_id"],
"duplicate": duplicate,
"status": row["status"],
"acceptedAtUtc": row["received_at_utc"],
},
)
def symbol_upload_response(row, duplicate: bool, status_code: int):
return JSONResponse(
status_code=status_code,
content={
"symbolUploadId": row["symbol_upload_id"],
"status": row["status"],
"duplicate": duplicate,
"acceptedAtUtc": row["received_at_utc"],
},
)
def health_payload():
return {
"status": "ok",
"service": "simcae-crash-server",
"version": CRASH_SERVICE_VERSION,
"timeUtc": utc_now_text(),
}
async def create_crash_report(request: Request):
tmp_dir: Path | None = None
final_dir: Path | None = None
stored = False
conn = None
metadata_raw = b""
package_hash = ""
try:
require_crash_bearer(request, {"crash_report"})
ensure_multipart_request(request)
try:
form = await request.form()
except Exception:
crash_api_error(400, "metadata_invalid", "multipart form is invalid", False)
metadata_raw = await read_metadata_field(form.get("metadata"), CRASH_METADATA_MAX_BYTES)
metadata_sha = hashlib.sha256(metadata_raw).hexdigest()
metadata = load_json_metadata(metadata_raw)
require_metadata_fields(
metadata,
[
"schemaVersion",
"clientReportId",
"crashTimeUtc",
"product",
"appVersion",
"gitCommit",
"buildType",
"channel",
"platform",
"crash",
"userConsent",
"files",
],
)
if metadata.get("product") != "SIMCAE":
crash_api_error(400, "metadata_invalid", "metadata.product must be SIMCAE", False)
if not isinstance(metadata.get("platform"), dict):
crash_api_error(400, "metadata_invalid", "metadata.platform must be an object", False)
if not isinstance(metadata.get("crash"), dict):
crash_api_error(400, "metadata_invalid", "metadata.crash must be an object", False)
if not isinstance(metadata.get("userConsent"), dict):
crash_api_error(400, "metadata_invalid", "metadata.userConsent must be an object", False)
client_report_id = str(metadata.get("clientReportId") or "").strip()
validate_client_report_id(client_report_id)
idempotency_key = request.headers.get("idempotency-key", "").strip()
if not idempotency_key:
crash_api_error(400, "metadata_invalid", "Idempotency-Key header is required", False)
if idempotency_key != client_report_id:
crash_api_error(400, "metadata_invalid", "Idempotency-Key must equal metadata.clientReportId", False)
tmp_dir = CRASH_STORAGE_ROOT / "_incoming" / f"crash_{secrets.token_hex(12)}"
tmp_dir.mkdir(parents=True, exist_ok=False)
(tmp_dir / "metadata.json").write_bytes(metadata_raw)
minidump_info = await save_upload_limited(form.get("minidump"), tmp_dir / "crash.dmp", CRASH_MINIDUMP_MAX_BYTES, "minidump")
declared_dump_sha = declared_file_sha256(metadata, "minidump", "crash.dmp")
if declared_dump_sha and declared_dump_sha != minidump_info["sha256"]:
crash_api_error(400, "file_hash_mismatch", "minidump SHA-256 does not match metadata.files", False)
attachments_info = None
if form.get("attachments") is not None:
attachments_info = await save_upload_limited(form.get("attachments"), tmp_dir / "attachments.zip", CRASH_ATTACHMENTS_MAX_BYTES, "attachments")
package_hash = crash_content_hash(metadata_sha, minidump_info["sha256"], attachments_info["sha256"] if attachments_info else "")
received_at = utc_now_text()
content_length = int(request.headers.get("content-length") or 0)
remote_addr = request.client.host if request.client else ""
server_info = {
"storageVersion": 1,
"clientReportId": client_report_id,
"receivedAtUtc": received_at,
"remoteAddress": remote_addr,
"contentLength": content_length,
}
conn = db.get_conn()
existing = crash_report_repository.get_report_by_client_report_id(conn, client_report_id)
if existing:
if existing["content_hash"] != package_hash:
crash_api_error(409, "idempotency_conflict", "same clientReportId was uploaded with different content", False)
return crash_report_response(existing, True, 200)
report_id = generate_prefixed_id(conn, "crash_reports", "report_id", "srv")
server_info["reportId"] = report_id
(tmp_dir / "server.json").write_text(json.dumps(server_info, ensure_ascii=False, indent=2), encoding="utf-8")
final_dir = CRASH_STORAGE_ROOT / "reports" / report_id
final_dir.parent.mkdir(parents=True, exist_ok=True)
tmp_dir.rename(final_dir)
tmp_dir = None
crash_obj = metadata.get("crash") or {}
crash_report_repository.insert_crash_report(
conn,
{
"report_id": report_id,
"client_report_id": client_report_id,
"content_hash": package_hash,
"status": "stored",
"symbolication_status": "not_started",
"product": metadata.get("product"),
"app_version": metadata.get("appVersion"),
"git_commit": metadata.get("gitCommit"),
"build_type": metadata.get("buildType"),
"channel": metadata.get("channel"),
"exception_code": str(crash_obj.get("exceptionCode") or ""),
"crash_time_utc": metadata.get("crashTimeUtc"),
"received_at_utc": received_at,
"remote_address": remote_addr,
"content_length": content_length,
"metadata_sha256": metadata_sha,
"metadata_size": len(metadata_raw),
"minidump_sha256": minidump_info["sha256"],
"minidump_size": minidump_info["size"],
"attachments_sha256": attachments_info["sha256"] if attachments_info else "",
"attachments_size": attachments_info["size"] if attachments_info else 0,
"storage_path": str(final_dir),
"metadata_json": metadata_raw.decode("utf-8", errors="replace"),
"server_json": json.dumps(server_info, ensure_ascii=False, separators=(",", ":")),
},
)
conn.commit()
stored = True
row = crash_report_repository.get_report_by_id(conn, report_id)
return crash_report_response(row, False, 201)
except sqlite3.IntegrityError:
if conn:
conn.rollback()
client_report_id = ""
try:
metadata = load_json_metadata(metadata_raw)
client_report_id = str(metadata.get("clientReportId") or "").strip()
except Exception:
pass
if client_report_id:
existing = crash_report_repository.get_report_by_client_report_id(conn, client_report_id)
if existing and existing["content_hash"] == package_hash:
return crash_report_response(existing, True, 200)
crash_api_error(409, "idempotency_conflict", "same clientReportId was uploaded with different content", False)
except CrashApiException:
if conn:
conn.rollback()
raise
except Exception as err:
if conn:
conn.rollback()
print("崩溃报告上传失败:", err)
crash_api_error(500, "server_error", "server failed to store crash report", True)
finally:
if conn:
conn.close()
cleanup_path(tmp_dir)
if final_dir and not stored:
cleanup_path(final_dir)
def get_crash_report_detail(report_id: str, request: Request):
require_crash_bearer(request, {"admin"})
row = crash_report_repository.get_crash_report(report_id)
if not row:
crash_api_error(404, "metadata_invalid", "crash report not found", False)
return {
"reportId": row["report_id"],
"clientReportId": row["client_report_id"],
"status": row["status"],
"receivedAtUtc": row["received_at_utc"],
"symbolicationStatus": row["symbolication_status"],
"appVersion": row["app_version"],
"gitCommit": row["git_commit"],
"product": row["product"],
"buildType": row["build_type"],
"channel": row["channel"],
"exceptionCode": row["exception_code"],
"files": {
"metadata": {"name": "metadata.json", "sha256": row["metadata_sha256"], "size": row["metadata_size"]},
"minidump": {"name": "crash.dmp", "sha256": row["minidump_sha256"], "size": row["minidump_size"]},
"attachments": {"name": "attachments.zip", "sha256": row["attachments_sha256"], "size": row["attachments_size"]} if row["attachments_size"] else None,
"server": {"name": "server.json"},
},
}
def download_crash_report_file(report_id: str, file_name: str, request: Request):
token = require_crash_bearer(request, {"admin"})
allowed = {
"metadata": "metadata.json",
"metadata.json": "metadata.json",
"minidump": "crash.dmp",
"crash.dmp": "crash.dmp",
"attachments": "attachments.zip",
"attachments.zip": "attachments.zip",
"server": "server.json",
"server.json": "server.json",
}
stored_name = allowed.get(file_name)
if not stored_name:
crash_api_error(404, "metadata_invalid", "file not found", False)
row = crash_report_repository.get_crash_report(report_id)
if not row:
crash_api_error(404, "metadata_invalid", "crash report not found", False)
target = Path(row["storage_path"]) / stored_name
crash_report_repository.log_file_access(
report_id,
stored_name,
security.token_digest(token)[:16],
request.client.host if request.client else "",
request.headers.get("user-agent", "")[:300],
)
if not target.is_file():
crash_api_error(404, "metadata_invalid", "file not found", False)
return FileResponse(target, media_type="application/octet-stream", filename=stored_name)
async def upload_symbols(request: Request):
tmp_dir: Path | None = None
final_dir: Path | None = None
stored = False
conn = None
try:
require_crash_bearer(request, {"symbols"})
ensure_multipart_request(request, CRASH_SYMBOLS_MAX_BYTES + CRASH_METADATA_MAX_BYTES)
try:
form = await request.form()
except Exception:
crash_api_error(400, "metadata_invalid", "multipart form is invalid", False)
metadata_raw = await read_metadata_field(form.get("metadata"), CRASH_METADATA_MAX_BYTES)
metadata_sha = hashlib.sha256(metadata_raw).hexdigest()
metadata = load_json_metadata(metadata_raw)
require_metadata_fields(
metadata,
["schemaVersion", "product", "appVersion", "gitCommit", "buildType", "platform", "toolchain", "createdAtUtc", "files"],
)
if metadata.get("product") != "SIMCAE":
crash_api_error(400, "metadata_invalid", "metadata.product must be SIMCAE", False)
if not isinstance(metadata.get("files"), list):
crash_api_error(400, "metadata_invalid", "metadata.files must be an array", False)
tmp_dir = CRASH_STORAGE_ROOT / "_incoming" / f"symbols_{secrets.token_hex(12)}"
tmp_dir.mkdir(parents=True, exist_ok=False)
(tmp_dir / "metadata.json").write_bytes(metadata_raw)
symbols_info = await save_upload_limited(form.get("symbols"), tmp_dir / "symbols.zip", CRASH_SYMBOLS_MAX_BYTES, "symbols")
identity_payload = json.dumps(
{
"product": metadata.get("product"),
"appVersion": metadata.get("appVersion"),
"gitCommit": metadata.get("gitCommit"),
"buildType": metadata.get("buildType"),
"platform": metadata.get("platform"),
},
sort_keys=True,
separators=(",", ":"),
)
identity_hash = hashlib.sha256(identity_payload.encode("utf-8")).hexdigest()
received_at = utc_now_text()
conn = db.get_conn()
existing = crash_report_repository.get_symbol_upload_by_identity_hash(conn, identity_hash)
if existing:
if existing["symbols_sha256"] != symbols_info["sha256"]:
crash_api_error(409, "idempotency_conflict", "same build identity already has different symbols", False)
return symbol_upload_response(existing, True, 200)
upload_id = generate_prefixed_id(conn, "crash_symbol_uploads", "symbol_upload_id", "sym")
final_dir = (
CRASH_STORAGE_ROOT
/ "symbols"
/ safe_storage_component(metadata.get("product"))
/ safe_storage_component(metadata.get("appVersion"))
/ safe_storage_component(metadata.get("gitCommit"))
/ safe_storage_component(metadata.get("buildType"))
/ safe_storage_component(metadata.get("platform"))
)
final_dir.parent.mkdir(parents=True, exist_ok=True)
if final_dir.exists():
shutil.rmtree(final_dir)
server_info = {
"storageVersion": 1,
"symbolUploadId": upload_id,
"receivedAtUtc": received_at,
"remoteAddress": request.client.host if request.client else "",
"contentLength": int(request.headers.get("content-length") or 0),
}
(tmp_dir / "server.json").write_text(json.dumps(server_info, ensure_ascii=False, indent=2), encoding="utf-8")
tmp_dir.rename(final_dir)
tmp_dir = None
crash_report_repository.insert_symbol_upload(
conn,
{
"symbol_upload_id": upload_id,
"identity_hash": identity_hash,
"status": "stored",
"product": metadata.get("product"),
"app_version": metadata.get("appVersion"),
"git_commit": metadata.get("gitCommit"),
"build_type": metadata.get("buildType"),
"platform": metadata.get("platform"),
"toolchain": metadata.get("toolchain"),
"created_at_utc": metadata.get("createdAtUtc"),
"received_at_utc": received_at,
"metadata_sha256": metadata_sha,
"metadata_size": len(metadata_raw),
"symbols_sha256": symbols_info["sha256"],
"symbols_size": symbols_info["size"],
"storage_path": str(final_dir),
"metadata_json": metadata_raw.decode("utf-8", errors="replace"),
},
)
conn.commit()
stored = True
row = crash_report_repository.get_symbol_upload_by_id(conn, upload_id)
return symbol_upload_response(row, False, 201)
except sqlite3.IntegrityError:
if conn:
conn.rollback()
crash_api_error(409, "idempotency_conflict", "same build identity already exists", False)
except CrashApiException:
if conn:
conn.rollback()
raise
except Exception as err:
if conn:
conn.rollback()
print("符号包上传失败:", err)
crash_api_error(500, "server_error", "server failed to store symbols", True)
finally:
if conn:
conn.close()
cleanup_path(tmp_dir)
if final_dir and not stored:
cleanup_path(final_dir)
+45
View File
@@ -0,0 +1,45 @@
import base64
import json
from datetime import datetime, timezone, timedelta
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding
from fastapi import HTTPException
from app.repositories import client_update_repository
from app.services.signing_service import load_manifest_private_key
def verify_device_credential(encoded: str) -> dict:
try:
wrapper = json.loads(base64.b64decode(encoded, validate=True).decode("utf-8"))
identity_text = wrapper["identity_text"]
signature = base64.b64decode(wrapper["signature"], validate=True)
load_manifest_private_key().public_key().verify(signature, identity_text.encode(), padding.PKCS1v15(), hashes.SHA256())
identity = json.loads(identity_text)
if any(not identity.get(k) for k in ("device_id", "license_id", "app_id", "channel", "installation_id", "credential_seq", "valid_until")):
raise ValueError("missing field")
except Exception:
raise HTTPException(status_code=401, detail={"error": "invalid_device_credential", "msg": "设备凭证格式或签名无效"})
row = client_update_repository.get_device(identity["device_id"])
if not row:
raise HTTPException(status_code=401, detail={"error": "device_not_registered", "msg": "设备未登记"})
if row["disabled"]:
reason = row["disabled_reason"] or "设备已被管理员禁用"
raise HTTPException(status_code=403, detail={"error": "device_disabled", "msg": reason})
if row["app_id"] != identity["app_id"] or row["installation_id"] != identity["installation_id"] or int(row["credential_seq"]) != int(identity["credential_seq"]):
raise HTTPException(status_code=401, detail={"error": "device_credential_revoked", "msg": "设备凭证已失效"})
license_row = client_update_repository.get_license(identity.get("license_id", ""))
now = datetime.now(timezone.utc)
try:
license_expiry = datetime.fromisoformat((license_row["valid_until"] if license_row else "").replace("Z", "+00:00"))
except ValueError:
license_expiry = now - timedelta(seconds=1)
if not license_row or license_row["status"] != "active" or license_expiry <= now:
raise HTTPException(status_code=403, detail={"error": "license_invalid", "msg": "授权不存在、已禁用或已过期"})
if license_row["app_id"] != identity["app_id"] or license_row["channel_code"] != identity.get("channel"):
raise HTTPException(status_code=403, detail={"error": "license_scope_mismatch", "msg": "授权应用或渠道不匹配"})
client_update_repository.touch_device(identity["device_id"])
return identity
+37
View File
@@ -0,0 +1,37 @@
import base64
import hashlib
import os
from pathlib import Path
from cryptography.fernet import Fernet, InvalidToken
from app.core.security import ADMIN_TOKEN
from app.services.signing_service import MANIFEST_PRIVATE_KEY_PATH
def license_key_encryption_material() -> bytes:
configured_secret = os.getenv("LICENSE_KEY_ENCRYPTION_SECRET", "").strip()
if configured_secret:
return configured_secret.encode("utf-8")
key_path = Path(MANIFEST_PRIVATE_KEY_PATH)
if key_path.exists():
return key_path.read_bytes()
return ADMIN_TOKEN.encode("utf-8")
def license_key_cipher() -> Fernet:
key = base64.urlsafe_b64encode(hashlib.sha256(license_key_encryption_material()).digest())
return Fernet(key)
def encrypt_license_key(license_key: str) -> str:
return license_key_cipher().encrypt(license_key.encode("utf-8")).decode("ascii")
def decrypt_license_key(cipher_text: str) -> str:
if not cipher_text:
return ""
try:
return license_key_cipher().decrypt(cipher_text.encode("ascii")).decode("utf-8")
except (InvalidToken, UnicodeDecodeError, ValueError):
return ""
+643
View File
@@ -0,0 +1,643 @@
import hashlib
import os
import secrets
import shutil
import sqlite3
import subprocess
import tarfile
import tempfile
import zipfile
from pathlib import Path, PurePosixPath
from fastapi import HTTPException, Request
import db
import minio_tool
from app.core.config import configured_path
from app.repositories import publish_repository
from app.services.common_service import require_channel
RELEASE_MAIN_EXECUTABLE = os.getenv("RELEASE_MAIN_EXECUTABLE", "MainApp.exe").strip().replace(chr(92), "/").strip("/")
UPLOAD_SPOOL_DIR = configured_path("UPLOAD_SPOOL_DIR", "upload_spool")
UPLOAD_SPOOL_DIR.mkdir(parents=True, exist_ok=True)
tempfile.tempdir = str(UPLOAD_SPOOL_DIR)
MINIO_DATA_DIR = configured_path("MINIO_DATA_DIR", "minio_data")
UPLOAD_SPACE_RESERVE = int(os.getenv("UPLOAD_SPACE_RESERVE_MB", "256")) * 1024 * 1024
PUBLISH_MAX_REQUEST_BYTES = int(os.getenv("PUBLISH_MAX_REQUEST_MB", "4096")) * 1024 * 1024
PUBLISH_MAX_FILES = int(os.getenv("PUBLISH_MAX_FILES", "20000"))
PUBLISH_MAX_FIELDS = int(os.getenv("PUBLISH_MAX_FIELDS", str(PUBLISH_MAX_FILES + 100)))
def normalize_relative_path(raw_path: str) -> str:
if not isinstance(raw_path, str):
raise HTTPException(status_code=400, detail="文件相对路径无效")
normalized = raw_path.replace(chr(92), "/")
if normalized.startswith("/"):
raise HTTPException(status_code=400, detail=f"文件路径必须是相对路径: {raw_path}")
normalized = normalized.strip("/")
if not normalized or chr(0) in normalized:
raise HTTPException(status_code=400, detail="文件相对路径为空或包含非法字符")
path = PurePosixPath(normalized)
if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
raise HTTPException(status_code=400, detail=f"不安全的文件路径: {raw_path}")
if path.parts and ":" in path.parts[0]:
raise HTTPException(status_code=400, detail=f"文件路径不能包含盘符: {raw_path}")
return path.as_posix()
def validate_release_path(relative_path: str):
folded = relative_path.casefold()
parts = PurePosixPath(folded).parts
for part in parts[:-1]:
if should_skip_release_directory(part):
raise HTTPException(
status_code=400,
detail=f"发布目录包含构建或运行时目录,必须选择干净的 Release 输出目录: {relative_path}",
)
def should_skip_release_directory(part: str) -> bool:
folded = part.casefold()
blocked_directories = {".git", ".vs", "cmakefiles", "debug", "update", "update_temp"}
return folded in blocked_directories or folded.startswith("build") or folded.endswith("_autogen")
def should_skip_release_file(relative_path: str) -> bool:
folded = relative_path.casefold()
runtime_protected_files = {
"client.ini",
"bootstrap.exe",
"config/app_config.json",
"config/local_state.json",
"config/client_identity.dat",
"config/version_policy.dat",
}
if folded in runtime_protected_files:
return True
if folded.startswith("bin/") and folded[4:] in runtime_protected_files:
return True
if any(folded.endswith("/" + protected) for protected in runtime_protected_files):
return True
if any(folded.endswith("/bin/" + protected) for protected in runtime_protected_files):
return True
return folded.endswith((".pdb", ".ilk", ".obj"))
def should_skip_release_path(relative_path: str) -> bool:
folded = relative_path.casefold()
parts = PurePosixPath(folded).parts
if any(should_skip_release_directory(part) for part in parts[:-1]):
return True
return should_skip_release_file(relative_path)
def calc_local_file_sha256(path: Path, chunk_size: int = 1024 * 1024):
sha = hashlib.sha256()
total = 0
try:
with path.open("rb") as stream:
while True:
chunk = stream.read(chunk_size)
if not chunk:
break
sha.update(chunk)
total += len(chunk)
except OSError as err:
raise HTTPException(status_code=500, detail={"error": "local_file_read_failed", "msg": str(err), "path": str(path)})
return sha.hexdigest(), total
def supported_archive_kind(filename: str) -> str:
name = (filename or "").casefold()
if name.endswith(".zip"):
return "zip"
if name.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tbz2")):
return "tar"
if name.endswith(".rar"):
return "rar"
return ""
def ensure_inside_directory(root: Path, relative_path: str) -> Path:
target = (root / relative_path).resolve()
root_resolved = root.resolve()
if target != root_resolved and root_resolved not in target.parents:
raise HTTPException(status_code=400, detail=f"压缩包包含不安全路径: {relative_path}")
return target
def publish_space_detail(error: str, msg: str, required_bytes: int, available_bytes: int):
return {
"error": error,
"msg": msg,
"required_bytes": required_bytes,
"available_bytes": available_bytes,
"required_mb": round(required_bytes / 1024 / 1024, 2),
"available_mb": round(available_bytes / 1024 / 1024, 2),
}
def check_extracted_publish_limits(total_size: int, file_count: int):
if PUBLISH_MAX_FILES > 0 and file_count > PUBLISH_MAX_FILES:
raise HTTPException(status_code=413, detail=f"压缩包解压后文件数量过多:{file_count},当前上限为 {PUBLISH_MAX_FILES}")
if PUBLISH_MAX_REQUEST_BYTES > 0 and total_size > PUBLISH_MAX_REQUEST_BYTES:
raise HTTPException(
status_code=413,
detail=publish_space_detail(
"publish_extracted_too_large",
f"压缩包解压后总大小过大,当前上限为 {PUBLISH_MAX_REQUEST_BYTES // 1024 // 1024} MB",
total_size,
PUBLISH_MAX_REQUEST_BYTES,
),
)
def copy_member_stream_to_path(source, target: Path, size: int | None, limits: dict):
target.parent.mkdir(parents=True, exist_ok=True)
total = 0
with target.open("wb") as output:
while True:
chunk = source.read(1024 * 1024)
if not chunk:
break
output.write(chunk)
total += len(chunk)
limits["total_size"] += len(chunk)
check_extracted_publish_limits(limits["total_size"], limits["file_count"])
if size is not None and size >= 0 and total != size:
raise HTTPException(status_code=400, detail=f"压缩包文件大小异常: {target.name}")
def extract_zip_archive(archive_path: Path, extract_dir: Path):
limits = {"total_size": 0, "file_count": 0}
try:
with zipfile.ZipFile(archive_path) as archive:
for info in archive.infolist():
if info.is_dir():
continue
rel_path = normalize_relative_path(info.filename)
if should_skip_release_path(rel_path):
continue
target = ensure_inside_directory(extract_dir, rel_path)
limits["file_count"] += 1
check_extracted_publish_limits(limits["total_size"] + max(info.file_size, 0), limits["file_count"])
with archive.open(info, "r") as source:
copy_member_stream_to_path(source, target, info.file_size, limits)
except zipfile.BadZipFile:
raise HTTPException(status_code=400, detail="zip 发布包无效或已损坏")
def extract_tar_archive(archive_path: Path, extract_dir: Path):
limits = {"total_size": 0, "file_count": 0}
try:
with tarfile.open(archive_path, mode="r:*") as archive:
for member in archive.getmembers():
if not member.isfile():
continue
rel_path = normalize_relative_path(member.name)
if should_skip_release_path(rel_path):
continue
target = ensure_inside_directory(extract_dir, rel_path)
limits["file_count"] += 1
check_extracted_publish_limits(limits["total_size"] + max(member.size, 0), limits["file_count"])
source = archive.extractfile(member)
if source is None:
raise HTTPException(status_code=400, detail=f"无法读取压缩包文件: {member.name}")
with source:
copy_member_stream_to_path(source, target, member.size, limits)
except tarfile.TarError:
raise HTTPException(status_code=400, detail="tar 发布包无效或已损坏")
def extract_rar_archive(archive_path: Path, extract_dir: Path):
commands = []
if shutil.which("bsdtar"):
commands.append(["bsdtar", "-xf", str(archive_path), "-C", str(extract_dir)])
if shutil.which("unrar"):
commands.append(["unrar", "x", "-o+", "-idq", str(archive_path), str(extract_dir) + os.sep])
if shutil.which("7z"):
commands.append(["7z", "x", "-y", f"-o{extract_dir}", str(archive_path)])
if not commands:
raise HTTPException(
status_code=400,
detail="服务器未安装 rar 解压工具,无法解压 .rar。请安装 bsdtar/unrar/7z,或上传 zip/tar.gz/tar.bz2 发布包",
)
last_error = ""
for command in commands:
try:
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=3600)
except Exception as err:
last_error = str(err)
continue
if result.returncode == 0:
return
last_error = (result.stderr or result.stdout or "").strip()
raise HTTPException(status_code=400, detail=f"rar 发布包解压失败: {last_error or 'unknown error'}")
def extract_release_archive(archive_path: Path, filename: str, extract_dir: Path):
kind = supported_archive_kind(filename)
if kind == "zip":
extract_zip_archive(archive_path, extract_dir)
elif kind == "tar":
extract_tar_archive(archive_path, extract_dir)
elif kind == "rar":
extract_rar_archive(archive_path, extract_dir)
else:
raise HTTPException(status_code=400, detail="压缩发布包仅支持 zip、tar.gz、tgz、tar.bz2、tbz2、rar")
def release_items_from_extracted_dir(extract_dir: Path):
items = []
for path in extract_dir.rglob("*"):
if path.is_symlink() or not path.is_file():
continue
rel_path = normalize_relative_path(path.relative_to(extract_dir).as_posix())
if should_skip_release_path(rel_path):
continue
items.append({"path": rel_path, "local_path": path})
return normalize_release_item_roots(items)
def normalize_release_item_roots(items: list[dict]):
if not items:
return items
required_main_key = RELEASE_MAIN_EXECUTABLE.casefold()
lower_paths = [item["path"].casefold() for item in items]
if required_main_key in lower_paths:
return items
nested_main = next((path for path in lower_paths if path.endswith("/" + required_main_key)), None)
if not nested_main:
return items
top_levels = set()
for item in items:
parts = PurePosixPath(item["path"]).parts
if parts:
top_levels.add(parts[0])
if len(top_levels) != 1:
return items
prefix = next(iter(top_levels)) + "/"
normalized = []
for item in items:
if not item["path"].startswith(prefix):
return items
stripped = item["path"][len(prefix):]
if stripped:
clone = dict(item)
clone["path"] = stripped
normalized.append(clone)
return normalized
def validate_release_items(items: list[dict]):
if not items:
raise HTTPException(status_code=400, detail="未选择任何文件,请选择程序文件夹或压缩发布包")
if len(items) > PUBLISH_MAX_FILES:
raise HTTPException(status_code=413, detail=f"文件数量过多:{len(items)},当前上限为 {PUBLISH_MAX_FILES}")
seen_paths = set()
filtered = []
for item in items:
rel_path = normalize_relative_path(item["path"])
if should_skip_release_path(rel_path):
continue
validate_release_path(rel_path)
path_key = rel_path.casefold()
if path_key in seen_paths:
raise HTTPException(status_code=400, detail=f"存在重复文件路径: {rel_path}")
seen_paths.add(path_key)
clone = dict(item)
clone["path"] = rel_path
filtered.append(clone)
if not filtered:
raise HTTPException(status_code=400, detail="发布内容为空;请确认发布目录或压缩包中包含程序文件")
required_main_key = RELEASE_MAIN_EXECUTABLE.casefold()
if required_main_key not in seen_paths:
nested_main = next((path for path in seen_paths if path.endswith("/" + required_main_key)), None)
if nested_main:
raise HTTPException(
status_code=400,
detail=f"{RELEASE_MAIN_EXECUTABLE} 位于更深层目录,请选择包含该相对路径的正确发布根目录。当前发现: {nested_main}",
)
raise HTTPException(status_code=400, detail=f"发布目录中缺少 {RELEASE_MAIN_EXECUTABLE}")
nested_main = next((path for path in seen_paths if path.endswith("/" + required_main_key)), None)
if nested_main:
raise HTTPException(
status_code=400,
detail=f"发布目录中存在嵌套的重复主程序 {nested_main},请使用干净的 Release 输出目录",
)
return filtered
def calc_upload_file_sha256(upload, chunk_size: int = 1024 * 1024):
stream = getattr(upload, "file", None)
if stream is None:
raise HTTPException(status_code=400, detail="上传文件对象无效")
sha = hashlib.sha256()
total = 0
try:
stream.seek(0)
while True:
chunk = stream.read(chunk_size)
if not chunk:
break
if isinstance(chunk, str):
chunk = chunk.encode("utf-8")
sha.update(chunk)
total += len(chunk)
stream.seek(0)
except OSError as err:
raise HTTPException(status_code=500, detail={"error": "upload_file_read_failed", "msg": str(err)})
return sha.hexdigest(), total
async def close_upload_safely(upload):
try:
close = getattr(upload, "close", None)
if close:
result = close()
if hasattr(result, "__await__"):
await result
return
except Exception as err:
print(f"close upload temp file failed: {err}")
stream = getattr(upload, "file", None)
if stream is not None:
try:
stream.close()
except Exception as err:
print(f"close upload stream failed: {err}")
def ensure_publish_storage_space(next_file_size: int):
required_bytes = max(next_file_size, 0) + UPLOAD_SPACE_RESERVE
available_bytes = shutil.disk_usage(MINIO_DATA_DIR).free
if available_bytes < required_bytes:
raise HTTPException(
status_code=507,
detail=publish_space_detail(
"storage_space_insufficient",
"版本存储空间不足,已停止发布。请删除旧版本、清理磁盘或扩容后重试",
required_bytes,
available_bytes,
),
)
async def collect_publish_items(request: Request):
content_length_header = request.headers.get("content-length")
try:
content_length = int(content_length_header or 0)
except ValueError:
raise HTTPException(status_code=400, detail="无效的 Content-Length")
if content_length <= 0:
raise HTTPException(status_code=411, detail="发布请求必须提供 Content-Length")
if PUBLISH_MAX_REQUEST_BYTES > 0 and content_length > PUBLISH_MAX_REQUEST_BYTES:
raise HTTPException(
status_code=413,
detail=publish_space_detail(
"publish_request_too_large",
f"发布请求过大,当前上限为 {PUBLISH_MAX_REQUEST_BYTES // 1024 // 1024} MB。请减小发布目录、清理无关文件,或调整 PUBLISH_MAX_REQUEST_MB",
content_length,
PUBLISH_MAX_REQUEST_BYTES,
),
)
spool_free = shutil.disk_usage(UPLOAD_SPOOL_DIR).free
storage_free = shutil.disk_usage(MINIO_DATA_DIR).free
same_storage_device = os.stat(UPLOAD_SPOOL_DIR).st_dev == os.stat(MINIO_DATA_DIR).st_dev
if same_storage_device:
required_bytes = content_length * 2 + UPLOAD_SPACE_RESERVE
if spool_free < required_bytes:
raise HTTPException(
status_code=507,
detail=publish_space_detail(
"publish_space_insufficient",
"发布目录上传和版本存储位于同一磁盘,空间不足。请删除旧版本、扩容磁盘,或把 UPLOAD_SPOOL_DIR 放到其他磁盘",
required_bytes,
spool_free,
),
)
else:
if spool_free < content_length + 64 * 1024 * 1024:
raise HTTPException(
status_code=507,
detail=publish_space_detail(
"upload_temp_space_insufficient",
"上传临时空间不足,请减小发布包或扩容上传临时目录",
content_length + 64 * 1024 * 1024,
spool_free,
),
)
if storage_free < content_length + UPLOAD_SPACE_RESERVE:
raise HTTPException(
status_code=507,
detail=publish_space_detail(
"storage_space_insufficient",
"版本存储空间不足,请扩容服务器根卷或删除不再需要的历史版本",
content_length + UPLOAD_SPACE_RESERVE,
storage_free,
),
)
try:
form = await request.form(max_files=PUBLISH_MAX_FILES, max_fields=PUBLISH_MAX_FIELDS)
except OSError as err:
if getattr(err, "errno", None) == 28:
raise HTTPException(status_code=507, detail="上传临时空间已满,请扩容后重试")
raise
app_id = form.get("app_id")
channel = form.get("channel") or ""
version = form.get("version")
try:
client_protocol = int(form.get("client_protocol") or 2)
except (TypeError, ValueError):
raise HTTPException(status_code=400, detail="client_protocol 必须是正整数")
if client_protocol < 1:
raise HTTPException(status_code=400, detail="client_protocol 必须是正整数")
files = []
archives = []
if hasattr(form, "getlist"):
files = form.getlist("files")
archives = form.getlist("archive")
else:
for key, value in form.multi_items():
if key == "files":
files.append(value)
elif key == "archive":
archives.append(value)
files = [item for item in files if hasattr(item, "filename") and item.filename]
archives = [item for item in archives if hasattr(item, "filename") and item.filename]
if not app_id or not version:
headers = {key: value for key, value in request.headers.items()}
raise HTTPException(
status_code=422,
detail={
"error": "missing form fields",
"have_app_id": bool(app_id),
"have_version": bool(version),
"content_type": request.headers.get("content-type"),
"headers": headers,
"form_keys": list(form.keys()),
"app_id_value": app_id,
"version_value": version,
"app_id_len": len(app_id or ""),
"version_len": len(version or ""),
},
)
if files and archives:
raise HTTPException(status_code=400, detail="请只选择一种发布方式:软件根目录或压缩发布包")
if len(archives) > 1:
raise HTTPException(status_code=400, detail="一次只能上传一个压缩发布包")
if not files and not archives:
raise HTTPException(status_code=400, detail="未选择任何文件,请选择程序文件夹或压缩发布包")
archive_upload = None
archive_temp_dir = None
try:
if archives:
archive_upload = archives[0]
archive_name = str(archive_upload.filename or "")
if not supported_archive_kind(archive_name):
raise HTTPException(status_code=400, detail="压缩发布包仅支持 zip、tar.gz、tgz、tar.bz2、tbz2、rar")
archive_temp_dir = Path(tempfile.mkdtemp(prefix="publish_archive_", dir=UPLOAD_SPOOL_DIR))
archive_path = archive_temp_dir / ("upload_" + secrets.token_hex(8))
try:
archive_upload.file.seek(0)
with archive_path.open("wb") as target:
shutil.copyfileobj(archive_upload.file, target, length=1024 * 1024)
except OSError as err:
raise HTTPException(status_code=500, detail={"error": "archive_spool_failed", "msg": str(err)})
extract_dir = archive_temp_dir / "extracted"
extract_dir.mkdir(parents=True, exist_ok=True)
extract_release_archive(archive_path, archive_name, extract_dir)
raw_items = release_items_from_extracted_dir(extract_dir)
upload_items = validate_release_items(raw_items)
extracted_total = sum(item["local_path"].stat().st_size for item in upload_items)
check_extracted_publish_limits(extracted_total, len(upload_items))
else:
if len(files) > PUBLISH_MAX_FILES:
raise HTTPException(status_code=413, detail=f"文件数量过多:{len(files)},当前上限为 {PUBLISH_MAX_FILES}")
relative_paths = form.getlist("relative_paths") if hasattr(form, "getlist") else []
if relative_paths and len(relative_paths) != len(files):
raise HTTPException(status_code=400, detail="文件数量与相对路径数量不一致")
raw_items = []
for index, file in enumerate(files):
raw_path = relative_paths[index] if relative_paths else file.filename
raw_items.append({"upload": file, "path": str(raw_path)})
upload_items = validate_release_items(raw_items)
except Exception:
if archive_upload is not None:
await close_upload_safely(archive_upload)
cleanup_path(archive_temp_dir)
raise
return {
"app_id": str(app_id),
"channel": str(channel),
"version": str(version),
"client_protocol": client_protocol,
"upload_items": upload_items,
"archive_upload": archive_upload,
"archive_temp_dir": archive_temp_dir,
}
def cleanup_path(path: Path | None):
if not path:
return
try:
if path.is_dir():
shutil.rmtree(path)
elif path.exists():
path.unlink()
except Exception as err:
print(f"清理临时文件失败: {path}: {err}")
async def publish_version(request: Request):
payload = await collect_publish_items(request)
upload_items = payload["upload_items"]
archive_upload = payload["archive_upload"]
archive_temp_dir = payload["archive_temp_dir"]
app_id = payload["app_id"]
channel = payload["channel"]
version = payload["version"]
client_protocol = payload["client_protocol"]
try:
conn = db.get_conn()
require_channel(conn, app_id, channel)
publish_started = False
publish_prefix = f"{app_id}/{channel}/{version}/"
try:
if publish_repository.version_exists(conn, app_id, channel, version):
raise HTTPException(status_code=400, detail="该版本已存在,请更换版本号或先删除旧版本")
new_vid = publish_repository.create_version(conn, app_id, channel, version, client_protocol)
publish_started = True
for item in upload_items:
rel_path = item["path"]
if "upload" in item:
file = item["upload"]
f_sha, f_size = calc_upload_file_sha256(file)
stream = file.file
else:
local_path = item["local_path"]
f_sha, f_size = calc_local_file_sha256(local_path)
stream = local_path.open("rb")
try:
ensure_publish_storage_space(f_size)
obj_path = f"{app_id}/{channel}/{version}/files/{rel_path}"
put_result = minio_tool.put_file_stream(obj_path, stream, f_size)
if put_result.get("storage") == "local":
print(f"MinIO 不可用,已保存到本地回退:{put_result.get('path')}")
publish_repository.insert_version_file(conn, new_vid, rel_path, f_sha, f_size)
finally:
if "local_path" in item:
stream.close()
conn.commit()
except sqlite3.IntegrityError as err:
conn.rollback()
if publish_started:
minio_tool.remove_prefix(publish_prefix)
if "UNIQUE constraint failed: versions.app_id, versions.channel, versions.version" in str(err):
raise HTTPException(status_code=400, detail="该版本已存在,请更换版本号或先删除旧版本")
raise HTTPException(status_code=500, detail=str(err))
except sqlite3.OperationalError as err:
conn.rollback()
if publish_started:
minio_tool.remove_prefix(publish_prefix)
raise HTTPException(status_code=500, detail={"error": "database locked", "msg": str(err)})
except HTTPException:
conn.rollback()
if publish_started:
minio_tool.remove_prefix(publish_prefix)
raise
except Exception as err:
conn.rollback()
if publish_started:
minio_tool.remove_prefix(publish_prefix)
if isinstance(err, OSError) and getattr(err, "errno", None) == 28:
raise HTTPException(status_code=507, detail="发布过程中存储空间耗尽,已清理未完成版本")
raise HTTPException(status_code=500, detail={"error": "publish_failed", "msg": str(err)})
finally:
conn.close()
return {"msg": f"版本 {version} 发布完成,共上传 {len(upload_items)} 个文件", "file_count": len(upload_items)}
finally:
for item in upload_items:
upload = item.get("upload") if isinstance(item, dict) else None
if upload is not None:
await close_upload_safely(upload)
if archive_upload is not None:
await close_upload_safely(archive_upload)
cleanup_path(archive_temp_dir)
+49
View File
@@ -0,0 +1,49 @@
import base64
import json
from pathlib import Path
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from app.core.config import configured_path
MANIFEST_PRIVATE_KEY_PATH = str(configured_path("MANIFEST_PRIVATE_KEY_PATH", "keys/manifest_private_key.pem"))
def load_manifest_private_key():
key_path = Path(MANIFEST_PRIVATE_KEY_PATH)
if not key_path.exists():
raise FileNotFoundError(f"Manifest private key not found: {key_path}")
key_data = key_path.read_bytes()
return load_pem_private_key(key_data, password=None)
def canonical_manifest_bytes(manifest_obj: dict) -> bytes:
copy = {k: manifest_obj[k] for k in manifest_obj if k != "signature"}
return json.dumps(copy, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
def sign_manifest(manifest_obj: dict) -> str:
json_text = canonical_manifest_bytes(manifest_obj)
private_key = load_manifest_private_key()
signature = private_key.sign(json_text, padding.PKCS1v15(), hashes.SHA256())
return base64.b64encode(signature).decode("ascii")
def canonical_signed_bytes(obj: dict) -> bytes:
copy = {k: obj[k] for k in obj if k != "signature"}
return json.dumps(copy, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
def sign_policy(policy_obj: dict) -> tuple[str, str]:
payload = canonical_signed_bytes(policy_obj)
signature = load_manifest_private_key().sign(payload, padding.PKCS1v15(), hashes.SHA256())
return payload.decode("utf-8"), base64.b64encode(signature).decode("ascii")
def sign_device_identity(identity: dict) -> tuple[str, str]:
payload = canonical_signed_bytes(identity)
signature = load_manifest_private_key().sign(payload, padding.PKCS1v15(), hashes.SHA256())
return payload.decode("utf-8"), base64.b64encode(signature).decode("ascii")