import hashlib import json import os import secrets import shutil import sqlite3 import uuid from datetime import datetime, timezone from pathlib import Path from fastapi import Request from fastapi.responses import FileResponse, JSONResponse import db from app.core import security from app.core.config import configured_path from app.repositories import crash_report_repository CRASH_STORAGE_ROOT = configured_path("CRASH_STORAGE_ROOT", "crash_storage") CRASH_METADATA_MAX_BYTES = int(os.getenv("CRASH_METADATA_MAX_KB", "256")) * 1024 CRASH_MINIDUMP_MAX_BYTES = int(os.getenv("CRASH_MINIDUMP_MAX_MB", "128")) * 1024 * 1024 CRASH_ATTACHMENTS_MAX_BYTES = int(os.getenv("CRASH_ATTACHMENTS_MAX_MB", "64")) * 1024 * 1024 CRASH_REQUEST_MAX_BYTES = int(os.getenv("CRASH_REQUEST_MAX_MB", "200")) * 1024 * 1024 CRASH_SYMBOLS_MAX_BYTES = int(os.getenv("CRASH_SYMBOLS_MAX_MB", "512")) * 1024 * 1024 CRASH_SERVICE_VERSION = os.getenv("CRASH_SERVICE_VERSION", "1.0.0") CRASH_REPORT_TOKEN = os.getenv("CRASH_REPORT_TOKEN") or os.getenv("SIMCAE_CRASH_CLIENT_TOKEN") or os.getenv("CLIENT_API_TOKEN") CRASH_SYMBOL_TOKEN = os.getenv("CRASH_SYMBOL_TOKEN") or os.getenv("SIMCAE_SYMBOL_TOKEN") CRASH_ADMIN_TOKEN = os.getenv("CRASH_ADMIN_TOKEN") class CrashApiException(Exception): def __init__(self, status_code: int, code: str, message: str, retryable: bool = False): self.status_code = status_code self.code = code self.message = message self.retryable = retryable async def crash_api_exception_handler(request: Request, exc: CrashApiException): return JSONResponse( status_code=exc.status_code, content={"error": {"code": exc.code, "message": exc.message, "retryable": exc.retryable}}, ) def ensure_storage_dirs(): CRASH_STORAGE_ROOT.mkdir(parents=True, exist_ok=True) (CRASH_STORAGE_ROOT / "reports").mkdir(parents=True, exist_ok=True) (CRASH_STORAGE_ROOT / "symbols").mkdir(parents=True, exist_ok=True) def utc_now_text() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") def crash_api_error(status_code: int, code: str, message: str, retryable: bool = False): raise CrashApiException(status_code, code, message, retryable) def bearer_token_from_request(request: Request) -> str: value = request.headers.get("authorization", "") if not value.lower().startswith("bearer "): crash_api_error(401, "unauthorized", "missing bearer token", False) token = value[7:].strip() if not token: crash_api_error(401, "unauthorized", "missing bearer token", False) return token def require_crash_bearer(request: Request, allowed_roles: set[str]) -> str: token = bearer_token_from_request(request) roles: set[str] = set() if CRASH_REPORT_TOKEN and secrets.compare_digest(token, CRASH_REPORT_TOKEN): roles.add("crash_report") if CRASH_SYMBOL_TOKEN: if secrets.compare_digest(token, CRASH_SYMBOL_TOKEN): roles.add("symbols") elif secrets.compare_digest(token, security.ADMIN_TOKEN): roles.add("symbols") if CRASH_ADMIN_TOKEN: if secrets.compare_digest(token, CRASH_ADMIN_TOKEN): roles.add("admin") elif secrets.compare_digest(token, security.ADMIN_TOKEN): roles.add("admin") if not roles: crash_api_error(401, "unauthorized", "invalid bearer token", False) if roles.isdisjoint(allowed_roles): crash_api_error(403, "forbidden", "token is not allowed to access this endpoint", False) return token def ensure_multipart_request(request: Request, max_bytes: int = CRASH_REQUEST_MAX_BYTES): content_type = request.headers.get("content-type", "") if "multipart/form-data" not in content_type.lower(): crash_api_error(415, "unsupported_media_type", "Content-Type must be multipart/form-data", False) raw_length = request.headers.get("content-length") if raw_length: try: content_length = int(raw_length) except ValueError: crash_api_error(400, "metadata_invalid", "Content-Length is invalid", False) if content_length > max_bytes: crash_api_error(413, "payload_too_large", "request body exceeds limit", False) def is_upload_file(value) -> bool: return hasattr(value, "filename") and hasattr(value, "read") async def read_metadata_field(value, max_bytes: int) -> bytes: if value is None: crash_api_error(400, "metadata_invalid", "metadata is required", False) if is_upload_file(value): data = bytearray() while True: chunk = await value.read(64 * 1024) if not chunk: break data.extend(chunk) if len(data) > max_bytes: crash_api_error(413, "payload_too_large", "metadata exceeds size limit", False) return bytes(data) data = str(value).encode("utf-8") if len(data) > max_bytes: crash_api_error(413, "payload_too_large", "metadata exceeds size limit", False) return data async def save_upload_limited(upload, target: Path, max_bytes: int, field_name: str) -> dict: if not is_upload_file(upload): crash_api_error(400, "metadata_invalid", f"{field_name} file is required", False) target.parent.mkdir(parents=True, exist_ok=True) total = 0 digest = hashlib.sha256() try: with open(target, "wb") as out_file: while True: chunk = await upload.read(1024 * 1024) if not chunk: break total += len(chunk) if total > max_bytes: crash_api_error(413, "payload_too_large", f"{field_name} exceeds size limit", False) digest.update(chunk) out_file.write(chunk) except Exception: if target.exists(): target.unlink() raise if total <= 0: crash_api_error(400, "metadata_invalid", f"{field_name} file is empty", False) return {"sha256": digest.hexdigest(), "size": total, "filename": upload.filename or field_name} def load_json_metadata(raw: bytes, error_code: str = "metadata_invalid") -> dict: try: obj = json.loads(raw.decode("utf-8")) except Exception: crash_api_error(400, error_code, "metadata must be valid UTF-8 JSON", False) if not isinstance(obj, dict): crash_api_error(400, error_code, "metadata must be a JSON object", False) return obj def require_metadata_fields(metadata: dict, fields: list[str], error_code: str = "metadata_invalid"): for field in fields: if metadata.get(field) in (None, ""): crash_api_error(400, error_code, f"metadata.{field} is required", False) if metadata.get("schemaVersion") != 1: crash_api_error(400, error_code, "metadata.schemaVersion must be 1", False) def normalize_declared_sha256(value, field_path: str) -> str: if value in (None, ""): return "" text = str(value).strip().lower() if len(text) != 64 or any(ch not in "0123456789abcdef" for ch in text): crash_api_error(400, "metadata_invalid", f"{field_path} must be a SHA-256 hex string", False) return text def declared_file_sha256(metadata: dict, kind: str, filename: str) -> str: files = metadata.get("files") if not isinstance(files, list): crash_api_error(400, "metadata_invalid", "metadata.files must be an array", False) for index, item in enumerate(files): if not isinstance(item, dict): crash_api_error(400, "metadata_invalid", f"metadata.files[{index}] must be an object", False) if item.get("kind") == kind or item.get("name") == filename: return normalize_declared_sha256(item.get("sha256"), f"metadata.files[{index}].sha256") return "" def validate_client_report_id(value: str): try: uuid.UUID(value) except Exception: crash_api_error(400, "metadata_invalid", "metadata.clientReportId must be a UUID", False) def safe_storage_component(value, fallback: str = "unknown") -> str: text = str(value or fallback).strip() cleaned = "".join(ch if ch.isalnum() or ch in "._-" else "_" for ch in text) return cleaned[:96] or fallback def crash_content_hash(metadata_sha256: str, minidump_sha256: str, attachments_sha256: str) -> str: payload = json.dumps( {"metadata": metadata_sha256, "minidump": minidump_sha256, "attachments": attachments_sha256 or ""}, sort_keys=True, separators=(",", ":"), ).encode("utf-8") return hashlib.sha256(payload).hexdigest() def generate_prefixed_id(conn, table: str, column: str, prefix: str) -> str: value = crash_report_repository.generate_prefixed_id(conn, table, column, prefix) if not value: crash_api_error(503, "service_unavailable", "could not allocate report id", True) return value def cleanup_path(path: Path | None): if not path: return try: if path.is_dir(): shutil.rmtree(path) elif path.exists(): path.unlink() except Exception as err: print(f"清理临时文件失败: {path}: {err}") def crash_report_response(row, duplicate: bool, status_code: int): return JSONResponse( status_code=status_code, content={ "reportId": row["report_id"], "duplicate": duplicate, "status": row["status"], "acceptedAtUtc": row["received_at_utc"], }, ) def symbol_upload_response(row, duplicate: bool, status_code: int): return JSONResponse( status_code=status_code, content={ "symbolUploadId": row["symbol_upload_id"], "status": row["status"], "duplicate": duplicate, "acceptedAtUtc": row["received_at_utc"], }, ) def health_payload(): return { "status": "ok", "service": "simcae-crash-server", "version": CRASH_SERVICE_VERSION, "timeUtc": utc_now_text(), } async def create_crash_report(request: Request): tmp_dir: Path | None = None final_dir: Path | None = None stored = False conn = None metadata_raw = b"" package_hash = "" try: require_crash_bearer(request, {"crash_report"}) ensure_multipart_request(request) try: form = await request.form() except Exception: crash_api_error(400, "metadata_invalid", "multipart form is invalid", False) metadata_raw = await read_metadata_field(form.get("metadata"), CRASH_METADATA_MAX_BYTES) metadata_sha = hashlib.sha256(metadata_raw).hexdigest() metadata = load_json_metadata(metadata_raw) require_metadata_fields( metadata, [ "schemaVersion", "clientReportId", "crashTimeUtc", "product", "appVersion", "gitCommit", "buildType", "channel", "platform", "crash", "userConsent", "files", ], ) if metadata.get("product") != "SIMCAE": crash_api_error(400, "metadata_invalid", "metadata.product must be SIMCAE", False) if not isinstance(metadata.get("platform"), dict): crash_api_error(400, "metadata_invalid", "metadata.platform must be an object", False) if not isinstance(metadata.get("crash"), dict): crash_api_error(400, "metadata_invalid", "metadata.crash must be an object", False) if not isinstance(metadata.get("userConsent"), dict): crash_api_error(400, "metadata_invalid", "metadata.userConsent must be an object", False) client_report_id = str(metadata.get("clientReportId") or "").strip() validate_client_report_id(client_report_id) idempotency_key = request.headers.get("idempotency-key", "").strip() if not idempotency_key: crash_api_error(400, "metadata_invalid", "Idempotency-Key header is required", False) if idempotency_key != client_report_id: crash_api_error(400, "metadata_invalid", "Idempotency-Key must equal metadata.clientReportId", False) tmp_dir = CRASH_STORAGE_ROOT / "_incoming" / f"crash_{secrets.token_hex(12)}" tmp_dir.mkdir(parents=True, exist_ok=False) (tmp_dir / "metadata.json").write_bytes(metadata_raw) minidump_info = await save_upload_limited(form.get("minidump"), tmp_dir / "crash.dmp", CRASH_MINIDUMP_MAX_BYTES, "minidump") declared_dump_sha = declared_file_sha256(metadata, "minidump", "crash.dmp") if declared_dump_sha and declared_dump_sha != minidump_info["sha256"]: crash_api_error(400, "file_hash_mismatch", "minidump SHA-256 does not match metadata.files", False) attachments_info = None if form.get("attachments") is not None: attachments_info = await save_upload_limited(form.get("attachments"), tmp_dir / "attachments.zip", CRASH_ATTACHMENTS_MAX_BYTES, "attachments") package_hash = crash_content_hash(metadata_sha, minidump_info["sha256"], attachments_info["sha256"] if attachments_info else "") received_at = utc_now_text() content_length = int(request.headers.get("content-length") or 0) remote_addr = request.client.host if request.client else "" server_info = { "storageVersion": 1, "clientReportId": client_report_id, "receivedAtUtc": received_at, "remoteAddress": remote_addr, "contentLength": content_length, } conn = db.get_conn() existing = crash_report_repository.get_report_by_client_report_id(conn, client_report_id) if existing: if existing["content_hash"] != package_hash: crash_api_error(409, "idempotency_conflict", "same clientReportId was uploaded with different content", False) return crash_report_response(existing, True, 200) report_id = generate_prefixed_id(conn, "crash_reports", "report_id", "srv") server_info["reportId"] = report_id (tmp_dir / "server.json").write_text(json.dumps(server_info, ensure_ascii=False, indent=2), encoding="utf-8") final_dir = CRASH_STORAGE_ROOT / "reports" / report_id final_dir.parent.mkdir(parents=True, exist_ok=True) tmp_dir.rename(final_dir) tmp_dir = None crash_obj = metadata.get("crash") or {} crash_report_repository.insert_crash_report( conn, { "report_id": report_id, "client_report_id": client_report_id, "content_hash": package_hash, "status": "stored", "symbolication_status": "not_started", "product": metadata.get("product"), "app_version": metadata.get("appVersion"), "git_commit": metadata.get("gitCommit"), "build_type": metadata.get("buildType"), "channel": metadata.get("channel"), "exception_code": str(crash_obj.get("exceptionCode") or ""), "crash_time_utc": metadata.get("crashTimeUtc"), "received_at_utc": received_at, "remote_address": remote_addr, "content_length": content_length, "metadata_sha256": metadata_sha, "metadata_size": len(metadata_raw), "minidump_sha256": minidump_info["sha256"], "minidump_size": minidump_info["size"], "attachments_sha256": attachments_info["sha256"] if attachments_info else "", "attachments_size": attachments_info["size"] if attachments_info else 0, "storage_path": str(final_dir), "metadata_json": metadata_raw.decode("utf-8", errors="replace"), "server_json": json.dumps(server_info, ensure_ascii=False, separators=(",", ":")), }, ) conn.commit() stored = True row = crash_report_repository.get_report_by_id(conn, report_id) return crash_report_response(row, False, 201) except sqlite3.IntegrityError: if conn: conn.rollback() client_report_id = "" try: metadata = load_json_metadata(metadata_raw) client_report_id = str(metadata.get("clientReportId") or "").strip() except Exception: pass if client_report_id: existing = crash_report_repository.get_report_by_client_report_id(conn, client_report_id) if existing and existing["content_hash"] == package_hash: return crash_report_response(existing, True, 200) crash_api_error(409, "idempotency_conflict", "same clientReportId was uploaded with different content", False) except CrashApiException: if conn: conn.rollback() raise except Exception as err: if conn: conn.rollback() print("崩溃报告上传失败:", err) crash_api_error(500, "server_error", "server failed to store crash report", True) finally: if conn: conn.close() cleanup_path(tmp_dir) if final_dir and not stored: cleanup_path(final_dir) def get_crash_report_detail(report_id: str, request: Request): require_crash_bearer(request, {"admin"}) row = crash_report_repository.get_crash_report(report_id) if not row: crash_api_error(404, "metadata_invalid", "crash report not found", False) return { "reportId": row["report_id"], "clientReportId": row["client_report_id"], "status": row["status"], "receivedAtUtc": row["received_at_utc"], "symbolicationStatus": row["symbolication_status"], "appVersion": row["app_version"], "gitCommit": row["git_commit"], "product": row["product"], "buildType": row["build_type"], "channel": row["channel"], "exceptionCode": row["exception_code"], "files": { "metadata": {"name": "metadata.json", "sha256": row["metadata_sha256"], "size": row["metadata_size"]}, "minidump": {"name": "crash.dmp", "sha256": row["minidump_sha256"], "size": row["minidump_size"]}, "attachments": {"name": "attachments.zip", "sha256": row["attachments_sha256"], "size": row["attachments_size"]} if row["attachments_size"] else None, "server": {"name": "server.json"}, }, } def download_crash_report_file(report_id: str, file_name: str, request: Request): token = require_crash_bearer(request, {"admin"}) allowed = { "metadata": "metadata.json", "metadata.json": "metadata.json", "minidump": "crash.dmp", "crash.dmp": "crash.dmp", "attachments": "attachments.zip", "attachments.zip": "attachments.zip", "server": "server.json", "server.json": "server.json", } stored_name = allowed.get(file_name) if not stored_name: crash_api_error(404, "metadata_invalid", "file not found", False) row = crash_report_repository.get_crash_report(report_id) if not row: crash_api_error(404, "metadata_invalid", "crash report not found", False) target = Path(row["storage_path"]) / stored_name crash_report_repository.log_file_access( report_id, stored_name, security.token_digest(token)[:16], request.client.host if request.client else "", request.headers.get("user-agent", "")[:300], ) if not target.is_file(): crash_api_error(404, "metadata_invalid", "file not found", False) return FileResponse(target, media_type="application/octet-stream", filename=stored_name) async def upload_symbols(request: Request): tmp_dir: Path | None = None final_dir: Path | None = None stored = False conn = None try: require_crash_bearer(request, {"symbols"}) ensure_multipart_request(request, CRASH_SYMBOLS_MAX_BYTES + CRASH_METADATA_MAX_BYTES) try: form = await request.form() except Exception: crash_api_error(400, "metadata_invalid", "multipart form is invalid", False) metadata_raw = await read_metadata_field(form.get("metadata"), CRASH_METADATA_MAX_BYTES) metadata_sha = hashlib.sha256(metadata_raw).hexdigest() metadata = load_json_metadata(metadata_raw) require_metadata_fields( metadata, ["schemaVersion", "product", "appVersion", "gitCommit", "buildType", "platform", "toolchain", "createdAtUtc", "files"], ) if metadata.get("product") != "SIMCAE": crash_api_error(400, "metadata_invalid", "metadata.product must be SIMCAE", False) if not isinstance(metadata.get("files"), list): crash_api_error(400, "metadata_invalid", "metadata.files must be an array", False) tmp_dir = CRASH_STORAGE_ROOT / "_incoming" / f"symbols_{secrets.token_hex(12)}" tmp_dir.mkdir(parents=True, exist_ok=False) (tmp_dir / "metadata.json").write_bytes(metadata_raw) symbols_info = await save_upload_limited(form.get("symbols"), tmp_dir / "symbols.zip", CRASH_SYMBOLS_MAX_BYTES, "symbols") identity_payload = json.dumps( { "product": metadata.get("product"), "appVersion": metadata.get("appVersion"), "gitCommit": metadata.get("gitCommit"), "buildType": metadata.get("buildType"), "platform": metadata.get("platform"), }, sort_keys=True, separators=(",", ":"), ) identity_hash = hashlib.sha256(identity_payload.encode("utf-8")).hexdigest() received_at = utc_now_text() conn = db.get_conn() existing = crash_report_repository.get_symbol_upload_by_identity_hash(conn, identity_hash) if existing: if existing["symbols_sha256"] != symbols_info["sha256"]: crash_api_error(409, "idempotency_conflict", "same build identity already has different symbols", False) return symbol_upload_response(existing, True, 200) upload_id = generate_prefixed_id(conn, "crash_symbol_uploads", "symbol_upload_id", "sym") final_dir = ( CRASH_STORAGE_ROOT / "symbols" / safe_storage_component(metadata.get("product")) / safe_storage_component(metadata.get("appVersion")) / safe_storage_component(metadata.get("gitCommit")) / safe_storage_component(metadata.get("buildType")) / safe_storage_component(metadata.get("platform")) ) final_dir.parent.mkdir(parents=True, exist_ok=True) if final_dir.exists(): shutil.rmtree(final_dir) server_info = { "storageVersion": 1, "symbolUploadId": upload_id, "receivedAtUtc": received_at, "remoteAddress": request.client.host if request.client else "", "contentLength": int(request.headers.get("content-length") or 0), } (tmp_dir / "server.json").write_text(json.dumps(server_info, ensure_ascii=False, indent=2), encoding="utf-8") tmp_dir.rename(final_dir) tmp_dir = None crash_report_repository.insert_symbol_upload( conn, { "symbol_upload_id": upload_id, "identity_hash": identity_hash, "status": "stored", "product": metadata.get("product"), "app_version": metadata.get("appVersion"), "git_commit": metadata.get("gitCommit"), "build_type": metadata.get("buildType"), "platform": metadata.get("platform"), "toolchain": metadata.get("toolchain"), "created_at_utc": metadata.get("createdAtUtc"), "received_at_utc": received_at, "metadata_sha256": metadata_sha, "metadata_size": len(metadata_raw), "symbols_sha256": symbols_info["sha256"], "symbols_size": symbols_info["size"], "storage_path": str(final_dir), "metadata_json": metadata_raw.decode("utf-8", errors="replace"), }, ) conn.commit() stored = True row = crash_report_repository.get_symbol_upload_by_id(conn, upload_id) return symbol_upload_response(row, False, 201) except sqlite3.IntegrityError: if conn: conn.rollback() crash_api_error(409, "idempotency_conflict", "same build identity already exists", False) except CrashApiException: if conn: conn.rollback() raise except Exception as err: if conn: conn.rollback() print("符号包上传失败:", err) crash_api_error(500, "server_error", "server failed to store symbols", True) finally: if conn: conn.close() cleanup_path(tmp_dir) if final_dir and not stored: cleanup_path(final_dir)