新增了客户端打包成sdk的功能和服务端docker镜像打包的功能,并修复了一些问题
This commit is contained in:
+63
-6
@@ -7,39 +7,69 @@ import socket
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
import urllib3
|
||||
from urllib.parse import urlparse
|
||||
from urllib3.util import Retry, Timeout
|
||||
|
||||
load_dotenv()
|
||||
BASE_DIR = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def env_bool(name: str, default: bool = False) -> bool:
|
||||
value = os.getenv(name)
|
||||
return default if value is None else value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT")
|
||||
MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY")
|
||||
MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY")
|
||||
MINIO_BUCKET = os.getenv("MINIO_BUCKET")
|
||||
SIGN_EXPIRE_MIN = int(os.getenv("SIGN_EXPIRE_MIN") or 60)
|
||||
LOCAL_UPLOAD_ROOT = Path(os.getenv("LOCAL_UPLOAD_ROOT", "local_uploads"))
|
||||
LOCAL_UPLOAD_ROOT = Path(os.getenv("LOCAL_UPLOAD_ROOT", "local_uploads")).expanduser()
|
||||
if not LOCAL_UPLOAD_ROOT.is_absolute():
|
||||
LOCAL_UPLOAD_ROOT = BASE_DIR / LOCAL_UPLOAD_ROOT
|
||||
LOCAL_FILE_URL_BASE = os.getenv("LOCAL_FILE_URL_BASE")
|
||||
MINIO_SECURE = env_bool("MINIO_SECURE")
|
||||
MINIO_CONNECT_TIMEOUT_SEC = float(os.getenv("MINIO_CONNECT_TIMEOUT_SEC", "2"))
|
||||
MINIO_READ_TIMEOUT_SEC = float(os.getenv("MINIO_READ_TIMEOUT_SEC", "5"))
|
||||
MINIO_RETRY_TOTAL = int(os.getenv("MINIO_RETRY_TOTAL", "1"))
|
||||
LOCAL_UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
http_client = urllib3.PoolManager(
|
||||
timeout=Timeout(connect=2.0, read=5.0),
|
||||
retries=Retry(total=1, connect=1, read=1, status=1, backoff_factor=0.2),
|
||||
timeout=Timeout(connect=MINIO_CONNECT_TIMEOUT_SEC, read=MINIO_READ_TIMEOUT_SEC),
|
||||
retries=Retry(total=MINIO_RETRY_TOTAL, connect=MINIO_RETRY_TOTAL,
|
||||
read=MINIO_RETRY_TOTAL, status=MINIO_RETRY_TOTAL, backoff_factor=0.2),
|
||||
)
|
||||
|
||||
mc = Minio(
|
||||
MINIO_ENDPOINT,
|
||||
access_key=MINIO_ACCESS_KEY,
|
||||
secret_key=MINIO_SECRET_KEY,
|
||||
secure=False,
|
||||
secure=MINIO_SECURE,
|
||||
http_client=http_client,
|
||||
)
|
||||
|
||||
public_endpoint_raw = os.getenv("MINIO_PUBLIC_ENDPOINT", "").strip()
|
||||
if public_endpoint_raw:
|
||||
parsed_public_endpoint = urlparse(
|
||||
public_endpoint_raw if "://" in public_endpoint_raw else f"http://{public_endpoint_raw}"
|
||||
)
|
||||
public_mc = Minio(
|
||||
parsed_public_endpoint.netloc,
|
||||
access_key=MINIO_ACCESS_KEY,
|
||||
secret_key=MINIO_SECRET_KEY,
|
||||
secure=parsed_public_endpoint.scheme == "https",
|
||||
)
|
||||
else:
|
||||
public_mc = mc
|
||||
|
||||
|
||||
def get_url(file_path: str) -> str:
|
||||
bucket = MINIO_BUCKET
|
||||
full_object_path = file_path
|
||||
try:
|
||||
mc.stat_object(bucket, full_object_path)
|
||||
url = mc.presigned_get_object(bucket, full_object_path, expires=timedelta(minutes=SIGN_EXPIRE_MIN))
|
||||
url = public_mc.presigned_get_object(
|
||||
bucket, full_object_path, expires=timedelta(minutes=SIGN_EXPIRE_MIN)
|
||||
)
|
||||
return url
|
||||
except Exception as err:
|
||||
print(f"generate url error: {err}")
|
||||
@@ -55,7 +85,8 @@ def get_url(file_path: str) -> str:
|
||||
host = '127.0.0.1'
|
||||
port = os.getenv('SERVER_PORT') or '8000'
|
||||
return f"http://{host}:{port}/static/{file_path}"
|
||||
return f"http://{MINIO_ENDPOINT}/{bucket}/{full_object_path}"
|
||||
scheme = "https" if MINIO_SECURE else "http"
|
||||
return f"{scheme}://{MINIO_ENDPOINT}/{bucket}/{full_object_path}"
|
||||
|
||||
|
||||
def put_file(object_path: str, data: bytes, size: int):
|
||||
@@ -91,3 +122,29 @@ def remove_prefix(prefix: str):
|
||||
shutil.rmtree(local_prefix)
|
||||
except Exception as e:
|
||||
print(f"local remove_prefix error: {e}")
|
||||
|
||||
def iter_file(object_path: str, chunk_size: int = 1024 * 1024):
|
||||
"""Stream an object without creating a second full local copy."""
|
||||
response = None
|
||||
emitted = 0
|
||||
try:
|
||||
response = mc.get_object(MINIO_BUCKET, object_path)
|
||||
while True:
|
||||
chunk = response.read(chunk_size)
|
||||
if not chunk: break
|
||||
emitted += len(chunk)
|
||||
yield chunk
|
||||
return
|
||||
except Exception as err:
|
||||
if emitted:
|
||||
raise RuntimeError(f"MinIO stream interrupted after {emitted} bytes: {object_path}") from err
|
||||
print(f"MinIO stream fallback for {object_path}: {err}")
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close(); response.release_conn()
|
||||
local_path = LOCAL_UPLOAD_ROOT / object_path
|
||||
with open(local_path, "rb") as source:
|
||||
while True:
|
||||
chunk = source.read(chunk_size)
|
||||
if not chunk: break
|
||||
yield chunk
|
||||
|
||||
Reference in New Issue
Block a user