| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238 |
- import os
- import aiohttp
- import asyncio
- from typing import Optional, Tuple, Dict, Any
- from urllib.parse import urlparse
- from fastapi import UploadFile
- from app.core.config import settings
- from app.core.biz_exception import BizLogicError
- from app.core.logger import logger
- class SeaweedFSService:
- """
- SeaweedFS 分布式存储服务
- 具备环境自适应、Volume 地址重写规整、动态下载网关及重试容错能力
- """
- @classmethod
- def get_master_url(cls) -> str:
- """获取并规整 Master URL(优先读取 settings,兼容环境变量)"""
- url = settings.seaweedfs_master_url or os.getenv("SEAWEEDFS_MASTER_URL", "http://seaweedfs-master:9333")
- url = url.strip().rstrip("/")
- if not url.startswith("http://") and not url.startswith("https://"):
- url = f"http://{url}"
- return url
- @classmethod
- def get_download_url(cls, fid: str) -> str:
- """根据当前配置的 API Base URL 动态生成公网下载地址"""
- base = (settings.api_base_url or os.getenv("API_BASE_URL", "https://api.text.skin")).strip().rstrip("/")
- return f"{base}/api/resource/download_file?fid={fid}"
- @classmethod
- def resolve_volume_url(cls, raw_url: str, fid: str) -> str:
- """
- 智能解析并规整 Volume Server 访问地址:
- 1. 自动处理协议前缀 (http:// / https://)
- 2. 如果配置了 seaweedfs_volume_override_host,则将 Master 返回的主机/IP 替换为覆盖主机,保留端口
- 3. 拼接目标 fid
- """
- if not raw_url:
- return ""
- raw = raw_url.strip()
- has_scheme = raw.startswith("http://") or raw.startswith("https://")
- if not has_scheme:
- raw = f"http://{raw}"
- parsed = urlparse(raw)
- scheme = parsed.scheme or "http"
- port = parsed.port
- override_host = (settings.seaweedfs_volume_override_host or os.getenv("SEAWEEDFS_VOLUME_OVERRIDE_HOST", "")).strip()
- if override_host:
- # 如果 override_host 自带端口,直接使用;否则保留原始端口
- if ":" in override_host:
- netloc = override_host
- elif port:
- netloc = f"{override_host}:{port}"
- else:
- netloc = override_host
- else:
- netloc = parsed.netloc
- base_volume = f"{scheme}://{netloc}".rstrip("/")
- return f"{base_volume}/{fid}"
- # 防刷安全配置:单文件最大 1MB,拒绝超大文件刷爆存储
- MAX_FILE_SIZE: int = 1 * 1024 * 1024
- # 允许上传的安全扩展名白名单(包含插件所需的 .py)
- ALLOWED_EXTENSIONS = {
- ".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg", ".bmp", ".ico",
- ".pdf", ".txt", ".csv", ".json", ".zip", ".tar", ".gz", ".py"
- }
- @classmethod
- async def upload(cls, file: UploadFile, max_retries: int = 2) -> Dict[str, Any]:
- """
- 上传文件到 SeaweedFS(带安全校验:大小与扩展名拦截,带自动重试与 Volume 地址自适应)
- """
- file_content = await file.read()
- filename = file.filename or "file.bin"
- content_type = file.content_type or "application/octet-stream"
- # 1. 拦截超大文件
- if len(file_content) > cls.MAX_FILE_SIZE:
- max_mb = cls.MAX_FILE_SIZE // (1024 * 1024)
- raise BizLogicError(f"File size exceeds the allowed limit (max {max_mb}MB)")
- # 2. 拦截危险文件类型
- ext = os.path.splitext(filename)[1].lower()
- if ext and ext not in cls.ALLOWED_EXTENSIONS:
- raise BizLogicError(f"File extension '{ext}' is not supported. Only safe media and document formats are allowed.")
- master_url = cls.get_master_url()
- timeout = aiohttp.ClientTimeout(total=15.0)
- for attempt in range(max_retries + 1):
- try:
- async with aiohttp.ClientSession(timeout=timeout) as client:
- # 1. 向 Master 申请 fid 和 volume 地址
- assign_url = f"{master_url}/dir/assign"
- async with client.get(assign_url) as assign_resp:
- if assign_resp.status >= 300:
- err_text = await assign_resp.text()
- raise BizLogicError(f"Storage master assign failed: {assign_resp.status} - {err_text}")
- assign_data = await assign_resp.json()
- fid = assign_data.get("fid")
- public_url = assign_data.get("publicUrl") or assign_data.get("url")
- if not fid or not public_url:
- raise BizLogicError("Invalid response from storage master: missing fid or volume url")
- # 2. 解析可达的 Volume 上传地址
- upload_url = cls.resolve_volume_url(public_url, fid)
- # 3. 构造表单并上传文件
- form = aiohttp.FormData()
- form.add_field("file", file_content, filename=filename, content_type=content_type)
- async with client.post(upload_url, data=form) as upload_resp:
- status = upload_resp.status
- resp_text = await upload_resp.text()
- if status in (200, 201):
- return {
- "fid": fid,
- "url": cls.get_download_url(fid),
- "size": len(file_content),
- }
- raise BizLogicError(f"Volume upload failed (status {status}): {resp_text}")
- except BizLogicError as e:
- if attempt == max_retries:
- logger.error(f"[SeaweedFS] Upload failed after {attempt + 1} attempts: {e}")
- raise e
- logger.warning(f"[SeaweedFS] Upload attempt {attempt + 1} failed: {e}. Retrying...")
- await asyncio.sleep(0.5 * (attempt + 1))
- except Exception as e:
- if attempt == max_retries:
- logger.exception(f"[SeaweedFS] Upload unexpected error on attempt {attempt + 1}")
- raise BizLogicError(f"Storage service unavailable: {str(e)}")
- logger.warning(f"[SeaweedFS] Network error on attempt {attempt + 1}: {e}. Retrying...")
- await asyncio.sleep(0.5 * (attempt + 1))
- raise BizLogicError("Storage upload failed")
- @classmethod
- async def get(cls, fid: str, max_retries: int = 2) -> Optional[Tuple[bytes, str]]:
- """
- 根据 fid 读取文件流及 Content-Type(带重试与自适应)
- """
- if not fid:
- return None
- volume_id = fid.split(",")[0]
- master_url = cls.get_master_url()
- timeout = aiohttp.ClientTimeout(total=10.0)
- for attempt in range(max_retries + 1):
- try:
- async with aiohttp.ClientSession(timeout=timeout) as client:
- # 1. 向 Master 查询 Volume 位置
- lookup_url = f"{master_url}/dir/lookup"
- async with client.get(lookup_url, params={"volumeId": volume_id}) as lookup_resp:
- if lookup_resp.status != 200:
- return None
- data = await lookup_resp.json()
- locations = data.get("locations") or []
- if not locations:
- return None
- # 2. 依次尝试可用的 Volume 节点
- for loc in locations:
- raw_pub = loc.get("publicUrl") or loc.get("url")
- if not raw_pub:
- continue
- file_url = cls.resolve_volume_url(raw_pub, fid)
- try:
- async with client.get(file_url) as file_resp:
- if file_resp.status == 200:
- content = await file_resp.read()
- mime = file_resp.headers.get("Content-Type", "application/octet-stream")
- return (content, mime)
- except Exception as node_err:
- logger.warning(f"[SeaweedFS] Failed to fetch from node {file_url}: {node_err}")
- continue
- return None
- except Exception as e:
- if attempt == max_retries:
- logger.warning(f"[SeaweedFS] Get failed for fid {fid}: {e}")
- return None
- await asyncio.sleep(0.3)
- return None
- @classmethod
- async def delete(cls, fid: str) -> bool:
- """
- 删除文件
- """
- if not fid:
- return False
- volume_id = fid.split(",")[0]
- master_url = cls.get_master_url()
- timeout = aiohttp.ClientTimeout(total=8.0)
- try:
- async with aiohttp.ClientSession(timeout=timeout) as client:
- lookup_url = f"{master_url}/dir/lookup"
- async with client.get(lookup_url, params={"volumeId": volume_id}) as lookup_resp:
- if lookup_resp.status != 200:
- return False
- data = await lookup_resp.json()
- locations = data.get("locations") or []
- if not locations:
- return False
- public_url = locations[0].get("publicUrl") or locations[0].get("url")
- if not public_url:
- return False
- delete_url = cls.resolve_volume_url(public_url, fid)
- async with client.delete(delete_url) as del_resp:
- return del_resp.status in (200, 202)
- except Exception as e:
- logger.error(f"[SeaweedFS] Delete failed for fid {fid}: {e}")
- return False
|