seaweedfs_service.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218
  1. import os
  2. import aiohttp
  3. import asyncio
  4. from typing import Optional, Tuple, Dict, Any
  5. from urllib.parse import urlparse
  6. from fastapi import UploadFile
  7. from app.core.config import settings
  8. from app.core.biz_exception import BizLogicError
  9. from app.core.logger import logger
  10. class SeaweedFSService:
  11. """
  12. SeaweedFS 分布式存储服务
  13. 具备环境自适应、Volume 地址重写规整、动态下载网关及重试容错能力
  14. """
  15. @classmethod
  16. def get_master_url(cls) -> str:
  17. """获取并规整 Master URL(优先读取 settings,兼容环境变量)"""
  18. url = settings.seaweedfs_master_url or os.getenv("SEAWEEDFS_MASTER_URL", "http://seaweedfs-master:9333")
  19. url = url.strip().rstrip("/")
  20. if not url.startswith("http://") and not url.startswith("https://"):
  21. url = f"http://{url}"
  22. return url
  23. @classmethod
  24. def get_download_url(cls, fid: str) -> str:
  25. """根据当前配置的 API Base URL 动态生成公网下载地址"""
  26. base = (settings.api_base_url or os.getenv("API_BASE_URL", "https://api.text.skin")).strip().rstrip("/")
  27. return f"{base}/api/resource/download_file?fid={fid}"
  28. @classmethod
  29. def resolve_volume_url(cls, raw_url: str, fid: str) -> str:
  30. """
  31. 智能解析并规整 Volume Server 访问地址:
  32. 1. 自动处理协议前缀 (http:// / https://)
  33. 2. 如果配置了 seaweedfs_volume_override_host,则将 Master 返回的主机/IP 替换为覆盖主机,保留端口
  34. 3. 拼接目标 fid
  35. """
  36. if not raw_url:
  37. return ""
  38. raw = raw_url.strip()
  39. has_scheme = raw.startswith("http://") or raw.startswith("https://")
  40. if not has_scheme:
  41. raw = f"http://{raw}"
  42. parsed = urlparse(raw)
  43. scheme = parsed.scheme or "http"
  44. port = parsed.port
  45. override_host = (settings.seaweedfs_volume_override_host or os.getenv("SEAWEEDFS_VOLUME_OVERRIDE_HOST", "")).strip()
  46. if override_host:
  47. # 如果 override_host 自带端口,直接使用;否则保留原始端口
  48. if ":" in override_host:
  49. netloc = override_host
  50. elif port:
  51. netloc = f"{override_host}:{port}"
  52. else:
  53. netloc = override_host
  54. else:
  55. netloc = parsed.netloc
  56. base_volume = f"{scheme}://{netloc}".rstrip("/")
  57. return f"{base_volume}/{fid}"
  58. @classmethod
  59. async def upload(cls, file: UploadFile, max_retries: int = 2) -> Dict[str, Any]:
  60. """
  61. 上传文件到 SeaweedFS(带自动重试与 Volume 地址自适应)
  62. """
  63. file_content = await file.read()
  64. filename = file.filename or "file.bin"
  65. content_type = file.content_type or "application/octet-stream"
  66. master_url = cls.get_master_url()
  67. timeout = aiohttp.ClientTimeout(total=15.0)
  68. for attempt in range(max_retries + 1):
  69. try:
  70. async with aiohttp.ClientSession(timeout=timeout) as client:
  71. # 1. 向 Master 申请 fid 和 volume 地址
  72. assign_url = f"{master_url}/dir/assign"
  73. async with client.get(assign_url) as assign_resp:
  74. if assign_resp.status >= 300:
  75. err_text = await assign_resp.text()
  76. raise BizLogicError(f"Storage master assign failed: {assign_resp.status} - {err_text}")
  77. assign_data = await assign_resp.json()
  78. fid = assign_data.get("fid")
  79. public_url = assign_data.get("publicUrl") or assign_data.get("url")
  80. if not fid or not public_url:
  81. raise BizLogicError("Invalid response from storage master: missing fid or volume url")
  82. # 2. 解析可达的 Volume 上传地址
  83. upload_url = cls.resolve_volume_url(public_url, fid)
  84. # 3. 构造表单并上传文件
  85. form = aiohttp.FormData()
  86. form.add_field("file", file_content, filename=filename, content_type=content_type)
  87. async with client.post(upload_url, data=form) as upload_resp:
  88. status = upload_resp.status
  89. resp_text = await upload_resp.text()
  90. if status in (200, 201):
  91. return {
  92. "fid": fid,
  93. "url": cls.get_download_url(fid),
  94. "size": len(file_content),
  95. }
  96. raise BizLogicError(f"Volume upload failed (status {status}): {resp_text}")
  97. except BizLogicError as e:
  98. if attempt == max_retries:
  99. logger.error(f"[SeaweedFS] Upload failed after {attempt + 1} attempts: {e}")
  100. raise e
  101. logger.warning(f"[SeaweedFS] Upload attempt {attempt + 1} failed: {e}. Retrying...")
  102. await asyncio.sleep(0.5 * (attempt + 1))
  103. except Exception as e:
  104. if attempt == max_retries:
  105. logger.exception(f"[SeaweedFS] Upload unexpected error on attempt {attempt + 1}")
  106. raise BizLogicError(f"Storage service unavailable: {str(e)}")
  107. logger.warning(f"[SeaweedFS] Network error on attempt {attempt + 1}: {e}. Retrying...")
  108. await asyncio.sleep(0.5 * (attempt + 1))
  109. raise BizLogicError("Storage upload failed")
  110. @classmethod
  111. async def get(cls, fid: str, max_retries: int = 2) -> Optional[Tuple[bytes, str]]:
  112. """
  113. 根据 fid 读取文件流及 Content-Type(带重试与自适应)
  114. """
  115. if not fid:
  116. return None
  117. volume_id = fid.split(",")[0]
  118. master_url = cls.get_master_url()
  119. timeout = aiohttp.ClientTimeout(total=10.0)
  120. for attempt in range(max_retries + 1):
  121. try:
  122. async with aiohttp.ClientSession(timeout=timeout) as client:
  123. # 1. 向 Master 查询 Volume 位置
  124. lookup_url = f"{master_url}/dir/lookup"
  125. async with client.get(lookup_url, params={"volumeId": volume_id}) as lookup_resp:
  126. if lookup_resp.status != 200:
  127. return None
  128. data = await lookup_resp.json()
  129. locations = data.get("locations") or []
  130. if not locations:
  131. return None
  132. # 2. 依次尝试可用的 Volume 节点
  133. for loc in locations:
  134. raw_pub = loc.get("publicUrl") or loc.get("url")
  135. if not raw_pub:
  136. continue
  137. file_url = cls.resolve_volume_url(raw_pub, fid)
  138. try:
  139. async with client.get(file_url) as file_resp:
  140. if file_resp.status == 200:
  141. content = await file_resp.read()
  142. mime = file_resp.headers.get("Content-Type", "application/octet-stream")
  143. return (content, mime)
  144. except Exception as node_err:
  145. logger.warning(f"[SeaweedFS] Failed to fetch from node {file_url}: {node_err}")
  146. continue
  147. return None
  148. except Exception as e:
  149. if attempt == max_retries:
  150. logger.warning(f"[SeaweedFS] Get failed for fid {fid}: {e}")
  151. return None
  152. await asyncio.sleep(0.3)
  153. return None
  154. @classmethod
  155. async def delete(cls, fid: str) -> bool:
  156. """
  157. 删除文件
  158. """
  159. if not fid:
  160. return False
  161. volume_id = fid.split(",")[0]
  162. master_url = cls.get_master_url()
  163. timeout = aiohttp.ClientTimeout(total=8.0)
  164. try:
  165. async with aiohttp.ClientSession(timeout=timeout) as client:
  166. lookup_url = f"{master_url}/dir/lookup"
  167. async with client.get(lookup_url, params={"volumeId": volume_id}) as lookup_resp:
  168. if lookup_resp.status != 200:
  169. return False
  170. data = await lookup_resp.json()
  171. locations = data.get("locations") or []
  172. if not locations:
  173. return False
  174. public_url = locations[0].get("publicUrl") or locations[0].get("url")
  175. if not public_url:
  176. return False
  177. delete_url = cls.resolve_volume_url(public_url, fid)
  178. async with client.delete(delete_url) as del_resp:
  179. return del_resp.status in (200, 202)
  180. except Exception as e:
  181. logger.error(f"[SeaweedFS] Delete failed for fid {fid}: {e}")
  182. return False