import os import aiohttp import asyncio from typing import Optional, Tuple, Dict, Any from urllib.parse import urlparse from fastapi import UploadFile from app.core.config import settings from app.core.biz_exception import BizLogicError from app.core.logger import logger class SeaweedFSService: """ SeaweedFS 分布式存储服务 具备环境自适应、Volume 地址重写规整、动态下载网关及重试容错能力 """ @classmethod def get_master_url(cls) -> str: """获取并规整 Master URL(优先读取 settings,兼容环境变量)""" url = settings.seaweedfs_master_url or os.getenv("SEAWEEDFS_MASTER_URL", "http://seaweedfs-master:9333") url = url.strip().rstrip("/") if not url.startswith("http://") and not url.startswith("https://"): url = f"http://{url}" return url @classmethod def get_download_url(cls, fid: str) -> str: """根据当前配置的 API Base URL 动态生成公网下载地址""" base = (settings.api_base_url or os.getenv("API_BASE_URL", "https://api.text.skin")).strip().rstrip("/") return f"{base}/api/resource/download_file?fid={fid}" @classmethod def resolve_volume_url(cls, raw_url: str, fid: str) -> str: """ 智能解析并规整 Volume Server 访问地址: 1. 自动处理协议前缀 (http:// / https://) 2. 如果配置了 seaweedfs_volume_override_host,则将 Master 返回的主机/IP 替换为覆盖主机,保留端口 3. 拼接目标 fid """ if not raw_url: return "" raw = raw_url.strip() has_scheme = raw.startswith("http://") or raw.startswith("https://") if not has_scheme: raw = f"http://{raw}" parsed = urlparse(raw) scheme = parsed.scheme or "http" port = parsed.port override_host = (settings.seaweedfs_volume_override_host or os.getenv("SEAWEEDFS_VOLUME_OVERRIDE_HOST", "")).strip() if override_host: # 如果 override_host 自带端口,直接使用;否则保留原始端口 if ":" in override_host: netloc = override_host elif port: netloc = f"{override_host}:{port}" else: netloc = override_host else: netloc = parsed.netloc base_volume = f"{scheme}://{netloc}".rstrip("/") return f"{base_volume}/{fid}" # 防刷安全配置:单文件最大 1MB,拒绝超大文件刷爆存储 MAX_FILE_SIZE: int = 1 * 1024 * 1024 # 允许上传的安全扩展名白名单(包含插件所需的 .py) ALLOWED_EXTENSIONS = { ".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg", ".bmp", ".ico", ".pdf", ".txt", ".csv", ".json", ".zip", ".tar", ".gz", ".py" } @classmethod async def upload(cls, file: UploadFile, max_retries: int = 2) -> Dict[str, Any]: """ 上传文件到 SeaweedFS(带安全校验:大小与扩展名拦截,带自动重试与 Volume 地址自适应) """ file_content = await file.read() filename = file.filename or "file.bin" content_type = file.content_type or "application/octet-stream" # 1. 拦截超大文件 if len(file_content) > cls.MAX_FILE_SIZE: max_mb = cls.MAX_FILE_SIZE // (1024 * 1024) raise BizLogicError(f"File size exceeds the allowed limit (max {max_mb}MB)") # 2. 拦截危险文件类型 ext = os.path.splitext(filename)[1].lower() if ext and ext not in cls.ALLOWED_EXTENSIONS: raise BizLogicError(f"File extension '{ext}' is not supported. Only safe media and document formats are allowed.") master_url = cls.get_master_url() timeout = aiohttp.ClientTimeout(total=15.0) for attempt in range(max_retries + 1): try: async with aiohttp.ClientSession(timeout=timeout) as client: # 1. 向 Master 申请 fid 和 volume 地址 assign_url = f"{master_url}/dir/assign" async with client.get(assign_url) as assign_resp: if assign_resp.status >= 300: err_text = await assign_resp.text() raise BizLogicError(f"Storage master assign failed: {assign_resp.status} - {err_text}") assign_data = await assign_resp.json() fid = assign_data.get("fid") public_url = assign_data.get("publicUrl") or assign_data.get("url") if not fid or not public_url: raise BizLogicError("Invalid response from storage master: missing fid or volume url") # 2. 解析可达的 Volume 上传地址 upload_url = cls.resolve_volume_url(public_url, fid) # 3. 构造表单并上传文件 form = aiohttp.FormData() form.add_field("file", file_content, filename=filename, content_type=content_type) async with client.post(upload_url, data=form) as upload_resp: status = upload_resp.status resp_text = await upload_resp.text() if status in (200, 201): return { "fid": fid, "url": cls.get_download_url(fid), "size": len(file_content), } raise BizLogicError(f"Volume upload failed (status {status}): {resp_text}") except BizLogicError as e: if attempt == max_retries: logger.error(f"[SeaweedFS] Upload failed after {attempt + 1} attempts: {e}") raise e logger.warning(f"[SeaweedFS] Upload attempt {attempt + 1} failed: {e}. Retrying...") await asyncio.sleep(0.5 * (attempt + 1)) except Exception as e: if attempt == max_retries: logger.exception(f"[SeaweedFS] Upload unexpected error on attempt {attempt + 1}") raise BizLogicError(f"Storage service unavailable: {str(e)}") logger.warning(f"[SeaweedFS] Network error on attempt {attempt + 1}: {e}. Retrying...") await asyncio.sleep(0.5 * (attempt + 1)) raise BizLogicError("Storage upload failed") @classmethod async def get(cls, fid: str, max_retries: int = 2) -> Optional[Tuple[bytes, str]]: """ 根据 fid 读取文件流及 Content-Type(带重试与自适应) """ if not fid: return None volume_id = fid.split(",")[0] master_url = cls.get_master_url() timeout = aiohttp.ClientTimeout(total=10.0) for attempt in range(max_retries + 1): try: async with aiohttp.ClientSession(timeout=timeout) as client: # 1. 向 Master 查询 Volume 位置 lookup_url = f"{master_url}/dir/lookup" async with client.get(lookup_url, params={"volumeId": volume_id}) as lookup_resp: if lookup_resp.status != 200: return None data = await lookup_resp.json() locations = data.get("locations") or [] if not locations: return None # 2. 依次尝试可用的 Volume 节点 for loc in locations: raw_pub = loc.get("publicUrl") or loc.get("url") if not raw_pub: continue file_url = cls.resolve_volume_url(raw_pub, fid) try: async with client.get(file_url) as file_resp: if file_resp.status == 200: content = await file_resp.read() mime = file_resp.headers.get("Content-Type", "application/octet-stream") return (content, mime) except Exception as node_err: logger.warning(f"[SeaweedFS] Failed to fetch from node {file_url}: {node_err}") continue return None except Exception as e: if attempt == max_retries: logger.warning(f"[SeaweedFS] Get failed for fid {fid}: {e}") return None await asyncio.sleep(0.3) return None @classmethod async def delete(cls, fid: str) -> bool: """ 删除文件 """ if not fid: return False volume_id = fid.split(",")[0] master_url = cls.get_master_url() timeout = aiohttp.ClientTimeout(total=8.0) try: async with aiohttp.ClientSession(timeout=timeout) as client: lookup_url = f"{master_url}/dir/lookup" async with client.get(lookup_url, params={"volumeId": volume_id}) as lookup_resp: if lookup_resp.status != 200: return False data = await lookup_resp.json() locations = data.get("locations") or [] if not locations: return False public_url = locations[0].get("publicUrl") or locations[0].get("url") if not public_url: return False delete_url = cls.resolve_volume_url(public_url, fid) async with client.delete(delete_url) as del_resp: return del_resp.status in (200, 202) except Exception as e: logger.error(f"[SeaweedFS] Delete failed for fid {fid}: {e}") return False