seaweedfs_service.py 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
  1. import os
  2. import aiohttp
  3. import asyncio
  4. from typing import Optional, Tuple, Dict, Any
  5. from urllib.parse import urlparse
  6. from fastapi import UploadFile
  7. from app.core.config import settings
  8. from app.core.biz_exception import BizLogicError
  9. from app.core.logger import logger
  10. class SeaweedFSService:
  11. """
  12. SeaweedFS 分布式存储服务
  13. 具备环境自适应、Volume 地址重写规整、动态下载网关及重试容错能力
  14. """
  15. @classmethod
  16. def get_master_url(cls) -> str:
  17. """获取并规整 Master URL(优先读取 settings,兼容环境变量)"""
  18. url = settings.seaweedfs_master_url or os.getenv("SEAWEEDFS_MASTER_URL", "http://seaweedfs-master:9333")
  19. url = url.strip().rstrip("/")
  20. if not url.startswith("http://") and not url.startswith("https://"):
  21. url = f"http://{url}"
  22. return url
  23. @classmethod
  24. def get_download_url(cls, fid: str) -> str:
  25. """根据当前配置的 API Base URL 动态生成公网下载地址"""
  26. base = (settings.api_base_url or os.getenv("API_BASE_URL", "https://api.text.skin")).strip().rstrip("/")
  27. return f"{base}/api/resource/download_file?fid={fid}"
  28. @classmethod
  29. def resolve_volume_url(cls, raw_url: str, fid: str) -> str:
  30. """
  31. 智能解析并规整 Volume Server 访问地址:
  32. 1. 自动处理协议前缀 (http:// / https://)
  33. 2. 如果配置了 seaweedfs_volume_override_host,则将 Master 返回的主机/IP 替换为覆盖主机,保留端口
  34. 3. 拼接目标 fid
  35. """
  36. if not raw_url:
  37. return ""
  38. raw = raw_url.strip()
  39. has_scheme = raw.startswith("http://") or raw.startswith("https://")
  40. if not has_scheme:
  41. raw = f"http://{raw}"
  42. parsed = urlparse(raw)
  43. scheme = parsed.scheme or "http"
  44. port = parsed.port
  45. override_host = (settings.seaweedfs_volume_override_host or os.getenv("SEAWEEDFS_VOLUME_OVERRIDE_HOST", "")).strip()
  46. if override_host:
  47. # 如果 override_host 自带端口,直接使用;否则保留原始端口
  48. if ":" in override_host:
  49. netloc = override_host
  50. elif port:
  51. netloc = f"{override_host}:{port}"
  52. else:
  53. netloc = override_host
  54. else:
  55. netloc = parsed.netloc
  56. base_volume = f"{scheme}://{netloc}".rstrip("/")
  57. return f"{base_volume}/{fid}"
  58. # 防刷安全配置:单文件最大 1MB,拒绝超大文件刷爆存储
  59. MAX_FILE_SIZE: int = 1 * 1024 * 1024
  60. # 允许上传的安全扩展名白名单(包含插件所需的 .py)
  61. ALLOWED_EXTENSIONS = {
  62. ".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg", ".bmp", ".ico",
  63. ".pdf", ".txt", ".csv", ".json", ".zip", ".tar", ".gz", ".py"
  64. }
  65. @classmethod
  66. async def upload(cls, file: UploadFile, max_retries: int = 2) -> Dict[str, Any]:
  67. """
  68. 上传文件到 SeaweedFS(带安全校验:大小与扩展名拦截,带自动重试与 Volume 地址自适应)
  69. """
  70. file_content = await file.read()
  71. filename = file.filename or "file.bin"
  72. content_type = file.content_type or "application/octet-stream"
  73. # 1. 拦截超大文件
  74. if len(file_content) > cls.MAX_FILE_SIZE:
  75. max_mb = cls.MAX_FILE_SIZE // (1024 * 1024)
  76. raise BizLogicError(f"File size exceeds the allowed limit (max {max_mb}MB)")
  77. # 2. 拦截危险文件类型
  78. ext = os.path.splitext(filename)[1].lower()
  79. if ext and ext not in cls.ALLOWED_EXTENSIONS:
  80. raise BizLogicError(f"File extension '{ext}' is not supported. Only safe media and document formats are allowed.")
  81. master_url = cls.get_master_url()
  82. timeout = aiohttp.ClientTimeout(total=15.0)
  83. for attempt in range(max_retries + 1):
  84. try:
  85. async with aiohttp.ClientSession(timeout=timeout) as client:
  86. # 1. 向 Master 申请 fid 和 volume 地址
  87. assign_url = f"{master_url}/dir/assign"
  88. async with client.get(assign_url) as assign_resp:
  89. if assign_resp.status >= 300:
  90. err_text = await assign_resp.text()
  91. raise BizLogicError(f"Storage master assign failed: {assign_resp.status} - {err_text}")
  92. assign_data = await assign_resp.json()
  93. fid = assign_data.get("fid")
  94. public_url = assign_data.get("publicUrl") or assign_data.get("url")
  95. if not fid or not public_url:
  96. raise BizLogicError("Invalid response from storage master: missing fid or volume url")
  97. # 2. 解析可达的 Volume 上传地址
  98. upload_url = cls.resolve_volume_url(public_url, fid)
  99. # 3. 构造表单并上传文件
  100. form = aiohttp.FormData()
  101. form.add_field("file", file_content, filename=filename, content_type=content_type)
  102. async with client.post(upload_url, data=form) as upload_resp:
  103. status = upload_resp.status
  104. resp_text = await upload_resp.text()
  105. if status in (200, 201):
  106. return {
  107. "fid": fid,
  108. "url": cls.get_download_url(fid),
  109. "size": len(file_content),
  110. }
  111. raise BizLogicError(f"Volume upload failed (status {status}): {resp_text}")
  112. except BizLogicError as e:
  113. if attempt == max_retries:
  114. logger.error(f"[SeaweedFS] Upload failed after {attempt + 1} attempts: {e}")
  115. raise e
  116. logger.warning(f"[SeaweedFS] Upload attempt {attempt + 1} failed: {e}. Retrying...")
  117. await asyncio.sleep(0.5 * (attempt + 1))
  118. except Exception as e:
  119. if attempt == max_retries:
  120. logger.exception(f"[SeaweedFS] Upload unexpected error on attempt {attempt + 1}")
  121. raise BizLogicError(f"Storage service unavailable: {str(e)}")
  122. logger.warning(f"[SeaweedFS] Network error on attempt {attempt + 1}: {e}. Retrying...")
  123. await asyncio.sleep(0.5 * (attempt + 1))
  124. raise BizLogicError("Storage upload failed")
  125. @classmethod
  126. async def get(cls, fid: str, max_retries: int = 2) -> Optional[Tuple[bytes, str]]:
  127. """
  128. 根据 fid 读取文件流及 Content-Type(带重试与自适应)
  129. """
  130. if not fid:
  131. return None
  132. volume_id = fid.split(",")[0]
  133. master_url = cls.get_master_url()
  134. timeout = aiohttp.ClientTimeout(total=10.0)
  135. for attempt in range(max_retries + 1):
  136. try:
  137. async with aiohttp.ClientSession(timeout=timeout) as client:
  138. # 1. 向 Master 查询 Volume 位置
  139. lookup_url = f"{master_url}/dir/lookup"
  140. async with client.get(lookup_url, params={"volumeId": volume_id}) as lookup_resp:
  141. if lookup_resp.status != 200:
  142. return None
  143. data = await lookup_resp.json()
  144. locations = data.get("locations") or []
  145. if not locations:
  146. return None
  147. # 2. 依次尝试可用的 Volume 节点
  148. for loc in locations:
  149. raw_pub = loc.get("publicUrl") or loc.get("url")
  150. if not raw_pub:
  151. continue
  152. file_url = cls.resolve_volume_url(raw_pub, fid)
  153. try:
  154. async with client.get(file_url) as file_resp:
  155. if file_resp.status == 200:
  156. content = await file_resp.read()
  157. mime = file_resp.headers.get("Content-Type", "application/octet-stream")
  158. return (content, mime)
  159. except Exception as node_err:
  160. logger.warning(f"[SeaweedFS] Failed to fetch from node {file_url}: {node_err}")
  161. continue
  162. return None
  163. except Exception as e:
  164. if attempt == max_retries:
  165. logger.warning(f"[SeaweedFS] Get failed for fid {fid}: {e}")
  166. return None
  167. await asyncio.sleep(0.3)
  168. return None
  169. @classmethod
  170. async def delete(cls, fid: str) -> bool:
  171. """
  172. 删除文件
  173. """
  174. if not fid:
  175. return False
  176. volume_id = fid.split(",")[0]
  177. master_url = cls.get_master_url()
  178. timeout = aiohttp.ClientTimeout(total=8.0)
  179. try:
  180. async with aiohttp.ClientSession(timeout=timeout) as client:
  181. lookup_url = f"{master_url}/dir/lookup"
  182. async with client.get(lookup_url, params={"volumeId": volume_id}) as lookup_resp:
  183. if lookup_resp.status != 200:
  184. return False
  185. data = await lookup_resp.json()
  186. locations = data.get("locations") or []
  187. if not locations:
  188. return False
  189. public_url = locations[0].get("publicUrl") or locations[0].get("url")
  190. if not public_url:
  191. return False
  192. delete_url = cls.resolve_volume_url(public_url, fid)
  193. async with client.delete(delete_url) as del_resp:
  194. return del_resp.status in (200, 202)
  195. except Exception as e:
  196. logger.error(f"[SeaweedFS] Delete failed for fid {fid}: {e}")
  197. return False