752 lines
32 KiB
Python
752 lines
32 KiB
Python
import hashlib
|
||
import os
|
||
import secrets
|
||
import shutil
|
||
import sqlite3
|
||
import subprocess
|
||
import tarfile
|
||
import tempfile
|
||
import zipfile
|
||
from pathlib import Path, PurePosixPath
|
||
|
||
from fastapi import HTTPException, Request
|
||
|
||
import db
|
||
import minio_tool
|
||
from app.core.config import configured_path
|
||
from app.repositories import publish_repository
|
||
from app.services.common_service import require_channel
|
||
|
||
|
||
TARGET_PLATFORM = os.getenv("TARGET_PLATFORM", "windows").strip().lower() or "windows"
|
||
DEFAULT_RELEASE_MAIN_EXECUTABLE = "bin/SimCAE" if TARGET_PLATFORM == "linux" else "bin/SimCAE.exe"
|
||
RELEASE_MAIN_EXECUTABLE = os.getenv("RELEASE_MAIN_EXECUTABLE", DEFAULT_RELEASE_MAIN_EXECUTABLE).strip().replace(chr(92), "/").strip("/")
|
||
if TARGET_PLATFORM == "linux" and RELEASE_MAIN_EXECUTABLE.casefold().endswith(".exe"):
|
||
RELEASE_MAIN_EXECUTABLE = RELEASE_MAIN_EXECUTABLE[:-4]
|
||
elif TARGET_PLATFORM == "windows":
|
||
leaf = RELEASE_MAIN_EXECUTABLE.rsplit("/", 1)[-1]
|
||
if "." not in leaf:
|
||
RELEASE_MAIN_EXECUTABLE += ".exe"
|
||
UPLOAD_SPOOL_DIR = configured_path("UPLOAD_SPOOL_DIR", "upload_spool")
|
||
UPLOAD_SPOOL_DIR.mkdir(parents=True, exist_ok=True)
|
||
tempfile.tempdir = str(UPLOAD_SPOOL_DIR)
|
||
MINIO_DATA_DIR = configured_path("MINIO_DATA_DIR", "minio_data")
|
||
UPLOAD_SPACE_RESERVE = int(os.getenv("UPLOAD_SPACE_RESERVE_MB", "256")) * 1024 * 1024
|
||
PUBLISH_MAX_REQUEST_BYTES = int(os.getenv("PUBLISH_MAX_REQUEST_MB", "4096")) * 1024 * 1024
|
||
PUBLISH_MAX_FILES = int(os.getenv("PUBLISH_MAX_FILES", "20000"))
|
||
PUBLISH_MAX_FIELDS = int(os.getenv("PUBLISH_MAX_FIELDS", str(PUBLISH_MAX_FILES + 100)))
|
||
|
||
|
||
def normalize_install_root(raw_value: str) -> str:
|
||
value = str(raw_value or os.getenv("CLIENT_INSTALL_ROOT", "..")).strip().replace(chr(92), "/")
|
||
if value in ("", "."):
|
||
return "."
|
||
if value == "..":
|
||
return ".."
|
||
raise HTTPException(status_code=400, detail="install_root 目前仅支持 . 或 ..")
|
||
|
||
|
||
def normalize_publish_executable(raw_value: str) -> str:
|
||
value = str(raw_value or "").strip().replace(chr(92), "/").strip("/")
|
||
if not value:
|
||
value = RELEASE_MAIN_EXECUTABLE.rsplit("/", 1)[-1]
|
||
if TARGET_PLATFORM == "linux" and value.casefold().endswith(".exe"):
|
||
value = value[:-4]
|
||
elif TARGET_PLATFORM == "windows":
|
||
parts = value.split("/")
|
||
leaf = parts[-1]
|
||
if "." not in leaf:
|
||
leaf += ".exe"
|
||
parts[-1] = leaf
|
||
value = "/".join(parts)
|
||
return normalize_relative_path(value)
|
||
|
||
|
||
def release_main_executable_for_install(install_root: str, main_executable: str) -> str:
|
||
normalized_install_root = normalize_install_root(install_root)
|
||
normalized_main = normalize_publish_executable(main_executable)
|
||
if normalized_install_root == ".":
|
||
return normalized_main
|
||
if normalized_main.casefold().startswith("bin/"):
|
||
return normalized_main
|
||
return normalize_relative_path(f"bin/{normalized_main}")
|
||
|
||
|
||
def normalize_relative_path(raw_path: str) -> str:
|
||
if not isinstance(raw_path, str):
|
||
raise HTTPException(status_code=400, detail="文件相对路径无效")
|
||
normalized = raw_path.replace(chr(92), "/")
|
||
if normalized.startswith("/"):
|
||
raise HTTPException(status_code=400, detail=f"文件路径必须是相对路径: {raw_path}")
|
||
normalized = normalized.strip("/")
|
||
if not normalized or chr(0) in normalized:
|
||
raise HTTPException(status_code=400, detail="文件相对路径为空或包含非法字符")
|
||
path = PurePosixPath(normalized)
|
||
if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
|
||
raise HTTPException(status_code=400, detail=f"不安全的文件路径: {raw_path}")
|
||
if path.parts and ":" in path.parts[0]:
|
||
raise HTTPException(status_code=400, detail=f"文件路径不能包含盘符: {raw_path}")
|
||
return path.as_posix()
|
||
|
||
|
||
def validate_release_path(relative_path: str):
|
||
folded = relative_path.casefold()
|
||
parts = PurePosixPath(folded).parts
|
||
for part in parts[:-1]:
|
||
if should_skip_release_directory(part):
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail=f"发布目录包含构建或运行时目录，必须选择干净的 Release 输出目录: {relative_path}",
|
||
)
|
||
|
||
|
||
def should_skip_release_directory(part: str) -> bool:
|
||
folded = part.casefold()
|
||
blocked_directories = {".git", ".vs", "cmakefiles", "debug", "update", "update_temp"}
|
||
return folded in blocked_directories or folded.startswith("build") or folded.endswith("_autogen")
|
||
|
||
|
||
def should_skip_release_file(relative_path: str) -> bool:
|
||
folded = relative_path.casefold()
|
||
# 发布包里不能带客户端本机运行态文件。
|
||
# 这些文件包含 License、设备身份、策略缓存等机器相关数据，应由客户端运行时生成或服务端重新签发。
|
||
runtime_protected_files = {
|
||
"client.ini",
|
||
"bootstrap.exe",
|
||
"config/app_config.json",
|
||
"config/local_state.json",
|
||
"config/client_identity.dat",
|
||
"config/version_policy.dat",
|
||
}
|
||
if folded in runtime_protected_files:
|
||
return True
|
||
if folded.startswith("bin/") and folded[4:] in runtime_protected_files:
|
||
return True
|
||
if any(folded.endswith("/" + protected) for protected in runtime_protected_files):
|
||
return True
|
||
if any(folded.endswith("/bin/" + protected) for protected in runtime_protected_files):
|
||
return True
|
||
return folded.endswith((".pdb", ".ilk", ".obj"))
|
||
|
||
|
||
def should_skip_release_path(relative_path: str) -> bool:
|
||
folded = relative_path.casefold()
|
||
parts = PurePosixPath(folded).parts
|
||
if any(should_skip_release_directory(part) for part in parts[:-1]):
|
||
return True
|
||
return should_skip_release_file(relative_path)
|
||
|
||
|
||
def calc_local_file_sha256(path: Path, chunk_size: int = 1024 * 1024):
|
||
sha = hashlib.sha256()
|
||
total = 0
|
||
try:
|
||
with path.open("rb") as stream:
|
||
while True:
|
||
chunk = stream.read(chunk_size)
|
||
if not chunk:
|
||
break
|
||
sha.update(chunk)
|
||
total += len(chunk)
|
||
except OSError as err:
|
||
raise HTTPException(status_code=500, detail={"error": "local_file_read_failed", "msg": str(err), "path": str(path)})
|
||
return sha.hexdigest(), total
|
||
|
||
|
||
def supported_archive_kind(filename: str) -> str:
|
||
name = (filename or "").casefold()
|
||
if name.endswith(".zip"):
|
||
return "zip"
|
||
if name.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tbz2")):
|
||
return "tar"
|
||
if name.endswith(".rar"):
|
||
return "rar"
|
||
return ""
|
||
|
||
|
||
def ensure_inside_directory(root: Path, relative_path: str) -> Path:
|
||
# 解压压缩包时必须防止 Zip Slip / 路径穿越。
|
||
# 任何 ../ 或绝对路径都不能写出临时解压目录。
|
||
target = (root / relative_path).resolve()
|
||
root_resolved = root.resolve()
|
||
if target != root_resolved and root_resolved not in target.parents:
|
||
raise HTTPException(status_code=400, detail=f"压缩包包含不安全路径: {relative_path}")
|
||
return target
|
||
|
||
|
||
def publish_space_detail(error: str, msg: str, required_bytes: int, available_bytes: int):
|
||
return {
|
||
"error": error,
|
||
"msg": msg,
|
||
"required_bytes": required_bytes,
|
||
"available_bytes": available_bytes,
|
||
"required_mb": round(required_bytes / 1024 / 1024, 2),
|
||
"available_mb": round(available_bytes / 1024 / 1024, 2),
|
||
}
|
||
|
||
|
||
def check_extracted_publish_limits(total_size: int, file_count: int):
|
||
if PUBLISH_MAX_FILES > 0 and file_count > PUBLISH_MAX_FILES:
|
||
raise HTTPException(status_code=413, detail=f"压缩包解压后文件数量过多：{file_count}，当前上限为 {PUBLISH_MAX_FILES}")
|
||
if PUBLISH_MAX_REQUEST_BYTES > 0 and total_size > PUBLISH_MAX_REQUEST_BYTES:
|
||
raise HTTPException(
|
||
status_code=413,
|
||
detail=publish_space_detail(
|
||
"publish_extracted_too_large",
|
||
f"压缩包解压后总大小过大，当前上限为 {PUBLISH_MAX_REQUEST_BYTES // 1024 // 1024} MB",
|
||
total_size,
|
||
PUBLISH_MAX_REQUEST_BYTES,
|
||
),
|
||
)
|
||
|
||
|
||
def copy_member_stream_to_path(source, target: Path, size: int | None, limits: dict):
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
total = 0
|
||
with target.open("wb") as output:
|
||
while True:
|
||
chunk = source.read(1024 * 1024)
|
||
if not chunk:
|
||
break
|
||
output.write(chunk)
|
||
total += len(chunk)
|
||
limits["total_size"] += len(chunk)
|
||
check_extracted_publish_limits(limits["total_size"], limits["file_count"])
|
||
if size is not None and size >= 0 and total != size:
|
||
raise HTTPException(status_code=400, detail=f"压缩包文件大小异常: {target.name}")
|
||
|
||
|
||
def extract_zip_archive(archive_path: Path, extract_dir: Path):
|
||
limits = {"total_size": 0, "file_count": 0}
|
||
try:
|
||
with zipfile.ZipFile(archive_path) as archive:
|
||
for info in archive.infolist():
|
||
if info.is_dir():
|
||
continue
|
||
rel_path = normalize_relative_path(info.filename)
|
||
if should_skip_release_path(rel_path):
|
||
continue
|
||
target = ensure_inside_directory(extract_dir, rel_path)
|
||
limits["file_count"] += 1
|
||
check_extracted_publish_limits(limits["total_size"] + max(info.file_size, 0), limits["file_count"])
|
||
with archive.open(info, "r") as source:
|
||
copy_member_stream_to_path(source, target, info.file_size, limits)
|
||
except zipfile.BadZipFile:
|
||
raise HTTPException(status_code=400, detail="zip 发布包无效或已损坏")
|
||
|
||
|
||
def extract_tar_archive(archive_path: Path, extract_dir: Path):
|
||
limits = {"total_size": 0, "file_count": 0}
|
||
try:
|
||
with tarfile.open(archive_path, mode="r:*") as archive:
|
||
for member in archive.getmembers():
|
||
if not member.isfile():
|
||
continue
|
||
rel_path = normalize_relative_path(member.name)
|
||
if should_skip_release_path(rel_path):
|
||
continue
|
||
target = ensure_inside_directory(extract_dir, rel_path)
|
||
limits["file_count"] += 1
|
||
check_extracted_publish_limits(limits["total_size"] + max(member.size, 0), limits["file_count"])
|
||
source = archive.extractfile(member)
|
||
if source is None:
|
||
raise HTTPException(status_code=400, detail=f"无法读取压缩包文件: {member.name}")
|
||
with source:
|
||
copy_member_stream_to_path(source, target, member.size, limits)
|
||
except tarfile.TarError:
|
||
raise HTTPException(status_code=400, detail="tar 发布包无效或已损坏")
|
||
|
||
|
||
def extract_rar_archive(archive_path: Path, extract_dir: Path):
|
||
commands = []
|
||
if shutil.which("bsdtar"):
|
||
commands.append(["bsdtar", "-xf", str(archive_path), "-C", str(extract_dir)])
|
||
if shutil.which("unrar"):
|
||
commands.append(["unrar", "x", "-o+", "-idq", str(archive_path), str(extract_dir) + os.sep])
|
||
if shutil.which("7z"):
|
||
commands.append(["7z", "x", "-y", f"-o{extract_dir}", str(archive_path)])
|
||
if not commands:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail="服务器未安装 rar 解压工具，无法解压 .rar。请安装 bsdtar/unrar/7z，或上传 zip/tar.gz/tar.bz2 发布包",
|
||
)
|
||
last_error = ""
|
||
for command in commands:
|
||
try:
|
||
result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=3600)
|
||
except Exception as err:
|
||
last_error = str(err)
|
||
continue
|
||
if result.returncode == 0:
|
||
return
|
||
last_error = (result.stderr or result.stdout or "").strip()
|
||
raise HTTPException(status_code=400, detail=f"rar 发布包解压失败: {last_error or 'unknown error'}")
|
||
|
||
|
||
def extract_release_archive(archive_path: Path, filename: str, extract_dir: Path):
|
||
kind = supported_archive_kind(filename)
|
||
if kind == "zip":
|
||
extract_zip_archive(archive_path, extract_dir)
|
||
elif kind == "tar":
|
||
extract_tar_archive(archive_path, extract_dir)
|
||
elif kind == "rar":
|
||
extract_rar_archive(archive_path, extract_dir)
|
||
else:
|
||
raise HTTPException(status_code=400, detail="压缩发布包仅支持 zip、tar.gz、tgz、tar.bz2、tbz2、rar")
|
||
|
||
|
||
def release_items_from_extracted_dir(extract_dir: Path, required_main_executable: str):
|
||
items = []
|
||
for path in extract_dir.rglob("*"):
|
||
if path.is_symlink() or not path.is_file():
|
||
continue
|
||
rel_path = normalize_relative_path(path.relative_to(extract_dir).as_posix())
|
||
if should_skip_release_path(rel_path):
|
||
continue
|
||
items.append({"path": rel_path, "local_path": path})
|
||
return normalize_release_item_roots(items, required_main_executable)
|
||
|
||
|
||
def normalize_release_item_roots(items: list[dict], required_main_executable: str):
|
||
# 有些压缩包会多包一层顶级目录，例如 SimCAE-1.0.0/bin/SimCAE.exe。
|
||
# 如果所有文件都在同一个顶级目录下，并且能找到主程序，就自动剥掉这一层，降低发布者操作成本。
|
||
if not items:
|
||
return items
|
||
required_main_key = required_main_executable.casefold()
|
||
lower_paths = [item["path"].casefold() for item in items]
|
||
if required_main_key in lower_paths:
|
||
return items
|
||
nested_main = next((path for path in lower_paths if path.endswith("/" + required_main_key)), None)
|
||
if not nested_main:
|
||
return items
|
||
top_levels = set()
|
||
for item in items:
|
||
parts = PurePosixPath(item["path"]).parts
|
||
if parts:
|
||
top_levels.add(parts[0])
|
||
if len(top_levels) != 1:
|
||
return items
|
||
prefix = next(iter(top_levels)) + "/"
|
||
normalized = []
|
||
for item in items:
|
||
if not item["path"].startswith(prefix):
|
||
return items
|
||
stripped = item["path"][len(prefix):]
|
||
if stripped:
|
||
clone = dict(item)
|
||
clone["path"] = stripped
|
||
normalized.append(clone)
|
||
return normalized
|
||
|
||
|
||
def validate_release_items(items: list[dict], required_main_executable: str):
|
||
if not items:
|
||
raise HTTPException(status_code=400, detail="未选择任何文件，请选择程序文件夹或压缩发布包")
|
||
if len(items) > PUBLISH_MAX_FILES:
|
||
raise HTTPException(status_code=413, detail=f"文件数量过多：{len(items)}，当前上限为 {PUBLISH_MAX_FILES}")
|
||
|
||
seen_paths = set()
|
||
filtered = []
|
||
for item in items:
|
||
rel_path = normalize_relative_path(item["path"])
|
||
if should_skip_release_path(rel_path):
|
||
continue
|
||
validate_release_path(rel_path)
|
||
path_key = rel_path.casefold()
|
||
if path_key in seen_paths:
|
||
raise HTTPException(status_code=400, detail=f"存在重复文件路径: {rel_path}")
|
||
seen_paths.add(path_key)
|
||
clone = dict(item)
|
||
clone["path"] = rel_path
|
||
filtered.append(clone)
|
||
|
||
if not filtered:
|
||
raise HTTPException(status_code=400, detail="发布内容为空；请确认发布目录或压缩包中包含程序文件")
|
||
|
||
required_main_key = required_main_executable.casefold()
|
||
if required_main_key not in seen_paths:
|
||
nested_main = next((path for path in seen_paths if path.endswith("/" + required_main_key)), None)
|
||
if nested_main:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail=f"{required_main_executable} 位于更深层目录，请选择包含该相对路径的正确发布根目录。当前发现: {nested_main}",
|
||
)
|
||
raise HTTPException(status_code=400, detail=f"发布目录中缺少 {required_main_executable}")
|
||
nested_main = next((path for path in seen_paths if path.endswith("/" + required_main_key)), None)
|
||
if nested_main:
|
||
raise HTTPException(
|
||
status_code=400,
|
||
detail=f"发布目录中存在嵌套的重复主程序 {nested_main}，请使用干净的 Release 输出目录",
|
||
)
|
||
return filtered
|
||
|
||
|
||
def calc_upload_file_sha256(upload, chunk_size: int = 1024 * 1024):
|
||
stream = getattr(upload, "file", None)
|
||
if stream is None:
|
||
raise HTTPException(status_code=400, detail="上传文件对象无效")
|
||
sha = hashlib.sha256()
|
||
total = 0
|
||
try:
|
||
stream.seek(0)
|
||
while True:
|
||
chunk = stream.read(chunk_size)
|
||
if not chunk:
|
||
break
|
||
if isinstance(chunk, str):
|
||
chunk = chunk.encode("utf-8")
|
||
sha.update(chunk)
|
||
total += len(chunk)
|
||
stream.seek(0)
|
||
except OSError as err:
|
||
raise HTTPException(status_code=500, detail={"error": "upload_file_read_failed", "msg": str(err)})
|
||
return sha.hexdigest(), total
|
||
|
||
|
||
async def close_upload_safely(upload):
|
||
try:
|
||
close = getattr(upload, "close", None)
|
||
if close:
|
||
result = close()
|
||
if hasattr(result, "__await__"):
|
||
await result
|
||
return
|
||
except Exception as err:
|
||
print(f"close upload temp file failed: {err}")
|
||
stream = getattr(upload, "file", None)
|
||
if stream is not None:
|
||
try:
|
||
stream.close()
|
||
except Exception as err:
|
||
print(f"close upload stream failed: {err}")
|
||
|
||
|
||
def ensure_publish_storage_space(next_file_size: int):
|
||
required_bytes = max(next_file_size, 0) + UPLOAD_SPACE_RESERVE
|
||
MINIO_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||
available_bytes = shutil.disk_usage(MINIO_DATA_DIR).free
|
||
if available_bytes < required_bytes:
|
||
raise HTTPException(
|
||
status_code=507,
|
||
detail=publish_space_detail(
|
||
"storage_space_insufficient",
|
||
"版本存储空间不足，已停止发布。请删除旧版本、清理磁盘或扩容后重试",
|
||
required_bytes,
|
||
available_bytes,
|
||
),
|
||
)
|
||
|
||
|
||
async def collect_publish_items(request: Request):
|
||
# 发布入口支持两种方式：浏览器选择 Release 目录，或上传一个压缩发布包。
|
||
# 这里先做请求大小、临时目录空间和 MinIO 存储空间检查，避免大包把服务器拖死。
|
||
content_length_header = request.headers.get("content-length")
|
||
try:
|
||
content_length = int(content_length_header or 0)
|
||
except ValueError:
|
||
raise HTTPException(status_code=400, detail="无效的 Content-Length")
|
||
if content_length <= 0:
|
||
raise HTTPException(status_code=411, detail="发布请求必须提供 Content-Length")
|
||
if PUBLISH_MAX_REQUEST_BYTES > 0 and content_length > PUBLISH_MAX_REQUEST_BYTES:
|
||
raise HTTPException(
|
||
status_code=413,
|
||
detail=publish_space_detail(
|
||
"publish_request_too_large",
|
||
f"发布请求过大，当前上限为 {PUBLISH_MAX_REQUEST_BYTES // 1024 // 1024} MB。请减小发布目录、清理无关文件，或调整 PUBLISH_MAX_REQUEST_MB",
|
||
content_length,
|
||
PUBLISH_MAX_REQUEST_BYTES,
|
||
),
|
||
)
|
||
|
||
spool_free = shutil.disk_usage(UPLOAD_SPOOL_DIR).free
|
||
storage_free = shutil.disk_usage(MINIO_DATA_DIR).free
|
||
same_storage_device = os.stat(UPLOAD_SPOOL_DIR).st_dev == os.stat(MINIO_DATA_DIR).st_dev
|
||
if same_storage_device:
|
||
required_bytes = content_length * 2 + UPLOAD_SPACE_RESERVE
|
||
if spool_free < required_bytes:
|
||
raise HTTPException(
|
||
status_code=507,
|
||
detail=publish_space_detail(
|
||
"publish_space_insufficient",
|
||
"发布目录上传和版本存储位于同一磁盘，空间不足。请删除旧版本、扩容磁盘，或把 UPLOAD_SPOOL_DIR 放到其他磁盘",
|
||
required_bytes,
|
||
spool_free,
|
||
),
|
||
)
|
||
else:
|
||
if spool_free < content_length + 64 * 1024 * 1024:
|
||
raise HTTPException(
|
||
status_code=507,
|
||
detail=publish_space_detail(
|
||
"upload_temp_space_insufficient",
|
||
"上传临时空间不足，请减小发布包或扩容上传临时目录",
|
||
content_length + 64 * 1024 * 1024,
|
||
spool_free,
|
||
),
|
||
)
|
||
if storage_free < content_length + UPLOAD_SPACE_RESERVE:
|
||
raise HTTPException(
|
||
status_code=507,
|
||
detail=publish_space_detail(
|
||
"storage_space_insufficient",
|
||
"版本存储空间不足，请扩容服务器根卷或删除不再需要的历史版本",
|
||
content_length + UPLOAD_SPACE_RESERVE,
|
||
storage_free,
|
||
),
|
||
)
|
||
|
||
try:
|
||
form = await request.form(max_files=PUBLISH_MAX_FILES, max_fields=PUBLISH_MAX_FIELDS)
|
||
except OSError as err:
|
||
if getattr(err, "errno", None) == 28:
|
||
raise HTTPException(status_code=507, detail="上传临时空间已满，请扩容后重试")
|
||
raise
|
||
|
||
app_id = form.get("app_id")
|
||
channel = form.get("channel") or ""
|
||
version = form.get("version")
|
||
install_root = normalize_install_root(str(form.get("install_root") or ""))
|
||
main_executable = str(form.get("main_executable") or "")
|
||
required_main_executable = release_main_executable_for_install(install_root, main_executable)
|
||
try:
|
||
client_protocol = int(form.get("client_protocol") or 2)
|
||
except (TypeError, ValueError):
|
||
raise HTTPException(status_code=400, detail="client_protocol 必须是正整数")
|
||
if client_protocol < 1:
|
||
raise HTTPException(status_code=400, detail="client_protocol 必须是正整数")
|
||
|
||
files = []
|
||
archives = []
|
||
if hasattr(form, "getlist"):
|
||
files = form.getlist("files")
|
||
archives = form.getlist("archive")
|
||
else:
|
||
for key, value in form.multi_items():
|
||
if key == "files":
|
||
files.append(value)
|
||
elif key == "archive":
|
||
archives.append(value)
|
||
files = [item for item in files if hasattr(item, "filename") and item.filename]
|
||
archives = [item for item in archives if hasattr(item, "filename") and item.filename]
|
||
|
||
if not app_id or not version:
|
||
headers = {key: value for key, value in request.headers.items()}
|
||
raise HTTPException(
|
||
status_code=422,
|
||
detail={
|
||
"error": "missing form fields",
|
||
"have_app_id": bool(app_id),
|
||
"have_version": bool(version),
|
||
"content_type": request.headers.get("content-type"),
|
||
"headers": headers,
|
||
"form_keys": list(form.keys()),
|
||
"app_id_value": app_id,
|
||
"version_value": version,
|
||
"app_id_len": len(app_id or ""),
|
||
"version_len": len(version or ""),
|
||
},
|
||
)
|
||
|
||
if files and archives:
|
||
raise HTTPException(status_code=400, detail="请只选择一种发布方式：软件根目录或压缩发布包")
|
||
if len(archives) > 1:
|
||
raise HTTPException(status_code=400, detail="一次只能上传一个压缩发布包")
|
||
if not files and not archives:
|
||
raise HTTPException(status_code=400, detail="未选择任何文件，请选择程序文件夹或压缩发布包")
|
||
|
||
archive_upload = None
|
||
archive_temp_dir = None
|
||
try:
|
||
if archives:
|
||
archive_upload = archives[0]
|
||
archive_name = str(archive_upload.filename or "")
|
||
if not supported_archive_kind(archive_name):
|
||
raise HTTPException(status_code=400, detail="压缩发布包仅支持 zip、tar.gz、tgz、tar.bz2、tbz2、rar")
|
||
archive_temp_dir = Path(tempfile.mkdtemp(prefix="publish_archive_", dir=UPLOAD_SPOOL_DIR))
|
||
archive_path = archive_temp_dir / ("upload_" + secrets.token_hex(8))
|
||
try:
|
||
archive_upload.file.seek(0)
|
||
with archive_path.open("wb") as target:
|
||
shutil.copyfileobj(archive_upload.file, target, length=1024 * 1024)
|
||
except OSError as err:
|
||
raise HTTPException(status_code=500, detail={"error": "archive_spool_failed", "msg": str(err)})
|
||
extract_dir = archive_temp_dir / "extracted"
|
||
extract_dir.mkdir(parents=True, exist_ok=True)
|
||
extract_release_archive(archive_path, archive_name, extract_dir)
|
||
raw_items = release_items_from_extracted_dir(extract_dir, required_main_executable)
|
||
upload_items = validate_release_items(raw_items, required_main_executable)
|
||
extracted_total = sum(item["local_path"].stat().st_size for item in upload_items)
|
||
check_extracted_publish_limits(extracted_total, len(upload_items))
|
||
else:
|
||
if len(files) > PUBLISH_MAX_FILES:
|
||
raise HTTPException(status_code=413, detail=f"文件数量过多：{len(files)}，当前上限为 {PUBLISH_MAX_FILES}")
|
||
relative_paths = form.getlist("relative_paths") if hasattr(form, "getlist") else []
|
||
if relative_paths and len(relative_paths) != len(files):
|
||
raise HTTPException(status_code=400, detail="文件数量与相对路径数量不一致")
|
||
raw_items = []
|
||
for index, file in enumerate(files):
|
||
raw_path = relative_paths[index] if relative_paths else file.filename
|
||
raw_items.append({"upload": file, "path": str(raw_path)})
|
||
upload_items = validate_release_items(raw_items, required_main_executable)
|
||
except Exception:
|
||
if archive_upload is not None:
|
||
await close_upload_safely(archive_upload)
|
||
cleanup_path(archive_temp_dir)
|
||
raise
|
||
|
||
return {
|
||
"app_id": str(app_id),
|
||
"channel": str(channel),
|
||
"version": str(version),
|
||
"client_protocol": client_protocol,
|
||
"install_root": install_root,
|
||
"required_main_executable": required_main_executable,
|
||
"upload_items": upload_items,
|
||
"archive_upload": archive_upload,
|
||
"archive_temp_dir": archive_temp_dir,
|
||
"job_temp_dir": None,
|
||
}
|
||
|
||
|
||
def cleanup_path(path: Path | None):
|
||
if not path:
|
||
return
|
||
try:
|
||
if path.is_dir():
|
||
shutil.rmtree(path)
|
||
elif path.exists():
|
||
path.unlink()
|
||
except Exception as err:
|
||
print(f"清理临时文件失败: {path}: {err}")
|
||
|
||
|
||
async def make_background_publish_payload(request: Request):
|
||
# 后台发布任务不能继续持有浏览器上传流，所以目录模式要先复制到服务端临时目录。
|
||
# 压缩包模式在 collect_publish_items 中已经解压为本地文件，只需要保留解压目录到任务结束。
|
||
payload = await collect_publish_items(request)
|
||
job_temp_dir = Path(tempfile.mkdtemp(prefix="publish_job_", dir=UPLOAD_SPOOL_DIR))
|
||
converted_items = []
|
||
|
||
try:
|
||
for item in payload["upload_items"]:
|
||
if "upload" not in item:
|
||
converted_items.append(item)
|
||
continue
|
||
upload = item["upload"]
|
||
target = ensure_inside_directory(job_temp_dir, item["path"])
|
||
target.parent.mkdir(parents=True, exist_ok=True)
|
||
try:
|
||
upload.file.seek(0)
|
||
with target.open("wb") as output:
|
||
shutil.copyfileobj(upload.file, output, length=1024 * 1024)
|
||
except OSError as err:
|
||
raise HTTPException(status_code=500, detail={"error": "publish_job_spool_failed", "msg": str(err)})
|
||
finally:
|
||
await close_upload_safely(upload)
|
||
converted_items.append({"path": item["path"], "local_path": target})
|
||
payload["upload_items"] = converted_items
|
||
payload["job_temp_dir"] = job_temp_dir
|
||
payload["archive_upload"] = None
|
||
return payload
|
||
except Exception:
|
||
for item in payload.get("upload_items", []):
|
||
upload = item.get("upload") if isinstance(item, dict) else None
|
||
if upload is not None:
|
||
await close_upload_safely(upload)
|
||
archive_upload = payload.get("archive_upload")
|
||
if archive_upload is not None:
|
||
await close_upload_safely(archive_upload)
|
||
cleanup_path(payload.get("archive_temp_dir"))
|
||
cleanup_path(job_temp_dir)
|
||
raise
|
||
|
||
|
||
async def publish_payload(payload: dict):
|
||
# 完整发布事务：
|
||
# 1. 使用已收集的发布文件；2. 创建版本记录；3. 上传文件到 MinIO；4. 写入文件 hash/size。
|
||
# 任何一步失败都会回滚数据库，并删除已经上传的对象前缀，避免出现“半个版本”。
|
||
upload_items = payload["upload_items"]
|
||
archive_upload = payload["archive_upload"]
|
||
archive_temp_dir = payload["archive_temp_dir"]
|
||
job_temp_dir = payload.get("job_temp_dir")
|
||
app_id = payload["app_id"]
|
||
channel = payload["channel"]
|
||
version = payload["version"]
|
||
client_protocol = payload["client_protocol"]
|
||
|
||
try:
|
||
conn = db.get_conn()
|
||
require_channel(conn, app_id, channel)
|
||
publish_started = False
|
||
publish_prefix = f"{app_id}/{channel}/{version}/"
|
||
try:
|
||
if publish_repository.version_exists(conn, app_id, channel, version):
|
||
raise HTTPException(status_code=400, detail="该版本已存在，请更换版本号或先删除旧版本")
|
||
new_vid = publish_repository.create_version(conn, app_id, channel, version, client_protocol)
|
||
publish_started = True
|
||
|
||
for item in upload_items:
|
||
rel_path = item["path"]
|
||
if "upload" in item:
|
||
file = item["upload"]
|
||
f_sha, f_size = calc_upload_file_sha256(file)
|
||
stream = file.file
|
||
else:
|
||
local_path = item["local_path"]
|
||
f_sha, f_size = calc_local_file_sha256(local_path)
|
||
stream = local_path.open("rb")
|
||
try:
|
||
ensure_publish_storage_space(f_size)
|
||
obj_path = f"{app_id}/{channel}/{version}/files/{rel_path}"
|
||
put_result = minio_tool.put_file_stream(obj_path, stream, f_size)
|
||
if put_result.get("storage") == "local":
|
||
print(f"MinIO 不可用，已保存到本地回退：{put_result.get('path')}")
|
||
publish_repository.insert_version_file(conn, new_vid, rel_path, f_sha, f_size)
|
||
finally:
|
||
if "local_path" in item:
|
||
stream.close()
|
||
|
||
conn.commit()
|
||
except sqlite3.IntegrityError as err:
|
||
conn.rollback()
|
||
if publish_started:
|
||
minio_tool.remove_prefix(publish_prefix)
|
||
if "UNIQUE constraint failed: versions.app_id, versions.channel, versions.version" in str(err):
|
||
raise HTTPException(status_code=400, detail="该版本已存在，请更换版本号或先删除旧版本")
|
||
raise HTTPException(status_code=500, detail=str(err))
|
||
except sqlite3.OperationalError as err:
|
||
conn.rollback()
|
||
if publish_started:
|
||
minio_tool.remove_prefix(publish_prefix)
|
||
raise HTTPException(status_code=500, detail={"error": "database locked", "msg": str(err)})
|
||
except HTTPException:
|
||
conn.rollback()
|
||
if publish_started:
|
||
minio_tool.remove_prefix(publish_prefix)
|
||
raise
|
||
except Exception as err:
|
||
conn.rollback()
|
||
if publish_started:
|
||
minio_tool.remove_prefix(publish_prefix)
|
||
if isinstance(err, OSError) and getattr(err, "errno", None) == 28:
|
||
raise HTTPException(status_code=507, detail="发布过程中存储空间耗尽，已清理未完成版本")
|
||
raise HTTPException(status_code=500, detail={"error": "publish_failed", "msg": str(err)})
|
||
finally:
|
||
conn.close()
|
||
|
||
return {"msg": f"版本 {version} 发布完成，共上传 {len(upload_items)} 个文件", "file_count": len(upload_items)}
|
||
finally:
|
||
for item in upload_items:
|
||
upload = item.get("upload") if isinstance(item, dict) else None
|
||
if upload is not None:
|
||
await close_upload_safely(upload)
|
||
if archive_upload is not None:
|
||
await close_upload_safely(archive_upload)
|
||
cleanup_path(archive_temp_dir)
|
||
cleanup_path(job_temp_dir)
|
||
|
||
|
||
async def publish_version(request: Request):
|
||
payload = await collect_publish_items(request)
|
||
return await publish_payload(payload)
|