\u67d0\u89c6\u9891 SaaS \u5e73\u53f0\u5141\u8bb8\u7528\u6237\u901a\u8fc7 REST API \u4e0a\u4f20\u89c6\u9891\u7d20\u6750（\u6700\u5927 10GB）\u3002\u521d\u671f\u5b9e\u73b0\u662f\u7528 files \u53c2\u6570\u76f4\u63a5\u4e0a\u4f20\u2014\u2014requests.post(url, files={"video": open("movie.mp4", "rb")})\u3002\u5bf9\u4e8e 50MB \u4ee5\u5185\u7684\u5c0f\u89c6\u9891\u6b63\u5e38\u8fd0\u884c，\u4f46\u7528\u6237\u5f00\u59cb\u4e0a\u4f20 5GB+ \u7684\u7535\u5f71\u7d20\u6750\u65f6，\u670d\u52a1\u5668\u5185\u5b58\u76f4\u63a5\u7206\u4e86\u2014\u2014files \u53c2\u6570\u4f1a\u628a\u6574\u4e2a\u6587\u4ef6\u8bfb\u5165\u5185\u5b58\u6784\u9020 multipart \u8bf7\u6c42\u4f53\u3002<\/p>\n
\u56e2\u961f\u5c1d\u8bd5\u7528 data=open("movie.mp4", "rb") \u52a0 Content-Type: video\/mp4 \u76f4\u63a5\u4f20\u539f\u59cb\u5b57\u8282，\u7ed5\u8fc7\u4e86 multipart \u7684\u5185\u5b58\u95ee\u9898\u3002\u4f46\u65b0\u7684\u95ee\u9898\u51fa\u73b0\u4e86：\u670d\u52a1\u5668\u4e0d\u77e5\u9053\u6587\u4ef6\u4ec0\u4e48\u65f6\u5019\u4f20\u5b8c（\u6ca1\u6709 Content-Length \u5934，\u56e0\u4e3a\u6587\u4ef6\u5927\u5c0f\u672a\u77e5\u6216\u592a\u5927\u65e0\u6cd5\u9884\u5148\u8ba1\u7b97），\u5bfc\u81f4\u8fde\u63a5\u5728\u4f20\u5b8c\u4e4b\u524d\u88ab\u8d85\u65f6\u65ad\u5f00\u3002\u800c\u4e14\u4e0a\u4f20\u6ca1\u6709\u8fdb\u5ea6\u53cd\u9988\u2014\u2014\u7528\u6237\u4e0d\u77e5\u9053\u5df2\u7ecf\u4f20\u4e86\u591a\u5c11\u4e86\u3002<\/p>\n
\u95ee\u9898\u4e00：multipart \u4e0a\u4f20 vs \u6d41\u5f0f\u4e0a\u4f20\u2014\u2014\u5185\u5b58\u5360\u7528\u7684\u5de8\u5927\u5dee\u5f02\u3002 files \u53c2\u6570\u4f1a\u628a\u6574\u4e2a multipart \u8bf7\u6c42\u4f53（\u6240\u6709\u6587\u4ef6 + \u6240\u6709 form \u5b57\u6bb5）\u5728\u5185\u5b58\u4e2d\u7ec4\u88c5\u5b8c\u6bd5\u540e\u518d\u53d1\u9001\u3002\u4e00\u4e2a 5GB \u7684\u89c6\u9891\u610f\u5473\u7740\u81f3\u5c11 5GB \u7684\u5185\u5b58\u5cf0\u503c\u3002\u800c\u6d41\u5f0f\u4e0a\u4f20\u4f7f\u7528\u751f\u6210\u5668\u4f5c\u4e3a data \u53c2\u6570，\u6bcf\u6b21\u53ea\u8bfb\u53d6\u4e00\u4e2a\u6587\u4ef6\u5757\u5230\u5185\u5b58\u3001\u53d1\u9001\u3001\u91ca\u653e\u2014\u20145GB \u6587\u4ef6\u5185\u5b58\u5cf0\u503c\u53ef\u4ee5\u63a7\u5236\u5728 8KB（chunk_size）\u3002<\/p>\n
\u95ee\u9898\u4e8c：Transfer-Encoding: chunked\u2014\u2014\u4e0d\u77e5\u9053\u6587\u4ef6\u5927\u5c0f\u65f6\u7684\u6700\u4f73\u9009\u62e9\u3002 HTTP \u534f\u8bae\u89c4\u5b9a：\u5982\u679c\u8bf7\u6c42\u6709 body，\u5fc5\u987b\u901a\u8fc7 Content-Length \u5934\u544a\u77e5\u670d\u52a1\u7aef body \u7684\u5927\u5c0f，\u6216\u8005\u4f7f\u7528 Transfer-Encoding: chunked \u5206\u5757\u4f20\u8f93\u3002\u5f53\u6587\u4ef6\u5927\u5c0f\u672a\u77e5\u6216\u592a\u5927\u65e0\u6cd5\u9884\u5148\u8ba1\u7b97\u65f6，chunked \u7f16\u7801\u662f\u552f\u4e00\u9009\u62e9\u3002requests \u5728\u4f7f\u7528\u751f\u6210\u5668\u4f5c\u4e3a data \u53c2\u6570\u65f6\u4f1a\u81ea\u52a8\u542f\u7528 chunked \u7f16\u7801\u3002<\/p>\n
\u95ee\u9898\u4e09：\u5927\u6587\u4ef6\u4e0a\u4f20\u7f3a\u4e4f\u53ef\u89c2\u6d4b\u6027\u3002 \u6ca1\u6709\u8fdb\u5ea6\u6761\u3001\u6ca1\u6709\u4f20\u8f93\u901f\u5ea6\u3001\u6ca1\u6709\u9884\u8ba1\u5269\u4f59\u65f6\u95f4\u2014\u2014\u8fd0\u7ef4\u65e0\u6cd5\u5224\u65ad\u4e0a\u4f20\u662f\u5728\u6b63\u5e38\u8fdb\u884c\u8fd8\u662f\u5361\u6b7b\u4e86\u3002\u201c\u5df2\u7ecf\u4f20\u4e86 2 \u5c0f\u65f6\u4e86，\u662f\u670d\u52a1\u5668\u5904\u7406\u6162\u8fd8\u662f\u65ad\u7f51\u4e86？\u201d\u2014\u2014\u8fd9\u4e2a\u95ee\u9898\u6bcf\u5468\u90fd\u4f1a\u5728\u8fd0\u7ef4\u7fa4\u51fa\u73b0\u3002<\/p>\n
\u6d41\u5f0f\u4e0a\u4f20 vs \u4f20\u7edf\u4e0a\u4f20\u5bf9\u6bd4\u56fe：<\/p>\n
\u4f20\u7edf\u4e0a\u4f20（files \u53c2\u6570）:
\n \u5185\u5b58: [\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588] 5GB \u5168\u90e8\u52a0\u8f7d
\n \u65f6\u95f4: 0s \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u4e0a\u4f20\u5f00\u59cb \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500> 300s
\n \u2192 5GB \u5360\u7528\u76f4\u5230\u4e0a\u4f20\u5b8c\u6210<\/p>\n
\u6d41\u5f0f\u4e0a\u4f20（\u751f\u6210\u5668 + chunked）:
\n \u5185\u5b58: [\u2588] 8KB chunk
\n \u65f6\u95f4: 0s \u2500\u2500> chunk1 \u2500\u2500> chunk2 \u2500\u2500> … \u2500\u2500> 300s
\n \u2191 \u6bcf\u5757\u5373\u65f6\u91ca\u653e<\/p>\n
\u5c0f\u80d6（\u770b\u7740\u670d\u52a1\u5668\u76d1\u63a7，CPU \u548c\u5185\u5b58\u5168\u7ebf\u98d9\u7ea2）： \u201c\u5927\u5e08！\u89c6\u9891\u4e0a\u4f20\u670d\u52a1\u53c8\u6302\u4e86\u2014\u2014\u7528\u6237\u4e0a\u4f20\u4e00\u4e2a 8GB \u7684\u7535\u5f71，\u670d\u52a1\u5668\u5185\u5b58 16GB \u76f4\u63a5\u98d9\u5230 95%，\u7136\u540e OOM Killer \u5c31\u6765\u4e86\u3002\u600e\u4e48\u624d\u80fd\u4e0a\u4f20\u5927\u6587\u4ef6\u4e0d\u7206\u5185\u5b58？\u201d<\/p>\n
\u5927\u5e08： \u201c\u4f60\u7684\u4e0a\u4f20\u4ee3\u7801\u662f\u600e\u4e48\u5199\u7684？requests.post(url, files={"video": f}) ？\u8fd9\u79cd\u5199\u6cd5\u4f1a\u628a\u6574\u4e2a\u6587\u4ef6\u52a0\u8f7d\u5230\u5185\u5b58\u6765\u6784\u9020 multipart \u8bf7\u6c42\u4f53\u30028GB \u6587\u4ef6 = 8GB+ \u5185\u5b58\u5cf0\u503c\u3002\u4f60\u9700\u8981\u6d41\u5f0f\u4e0a\u4f20\u3002\u201d<\/p>\n
\u5c0f\u80d6（\u6320\u5934）： \u201c\u6d41\u5f0f\u4e0a\u4f20？\u6211\u5b66\u8fc7\u6d41\u5f0f\u4e0b\u8f7d\u2014\u2014\u7528 stream=True + iter_content()\u3002\u4f46\u4e0a\u4f20\u600e\u4e48\u6d41\u5f0f？\u201d<\/p>\n
\u5927\u5e08： \u201c\u7528\u751f\u6210\u5668\u4f5c\u4e3a data \u53c2\u6570\u3002\u5f53\u4e00\u4e2a\u751f\u6210\u5668\u88ab\u7528\u4f5c data \u65f6，requests \u4f1a\u81ea\u52a8\u5207\u6362\u5230 Transfer-Encoding: chunked \u6a21\u5f0f\u2014\u2014\u5b83\u4e0d\u4f1a\u4e00\u6b21\u6027\u8bfb\u53d6\u6574\u4e2a\u751f\u6210\u5668，\u800c\u662f yield \u4e00\u5757\u3001\u53d1\u4e00\u5757\u3001yield \u4e0b\u4e00\u5757\u3001\u53d1\u4e0b\u4e00\u5757\u3002\u5185\u5b58\u5360\u7528\u5c31\u964d\u5230\u4e86 chunk \u5927\u5c0f\u3002\u201d<\/p>\n
def<\/span> file_chunk_generator<\/span>(<\/span>filepath,<\/span> chunk_size=<\/span>8192<\/span>)<\/span>:<\/span> # \u6d41\u5f0f\u4e0a\u4f20\u2014\u2014\u5185\u5b58\u5cf0\u503c = chunk_size (8KB)<\/span> \u5c0f\u767d（\u8ffd\u95ee）： \u201c\u90a3 Transfer-Encoding: chunked \u548c Content-Length \u6709\u4ec0\u4e48\u533a\u522b？\u4ec0\u4e48\u60c5\u51b5\u4e0b\u7528\u54ea\u4e2a？\u201d<\/p>\n \u5927\u5e08： "\u8fd9\u662f HTTP \u534f\u8bae\u4e2d\u4e24\u4e2a\u4e92\u65a5\u7684 body \u957f\u5ea6\u6307\u5b9a\u65b9\u5f0f：<\/p>\n chunked \u7f16\u7801\u9002\u5408：\u6587\u4ef6\u5927\u5c0f\u672a\u77e5（\u5b9e\u65f6\u751f\u6210\u7684\u6570\u636e）\u3001\u6587\u4ef6\u592a\u5927\u65e0\u6cd5\u4e00\u6b21\u6027\u8ba1\u7b97\u5927\u5c0f\u3001\u6d41\u5f0f\u6570\u636e（\u65e5\u5fd7\u6d41\u3001\u76f4\u64ad\u6d41）\u3002\u4f46\u4ee3\u4ef7\u662f\u670d\u52a1\u5668\u4e0d\u80fd\u63d0\u524d\u77e5\u9053\u603b\u5927\u5c0f（\u6bd4\u5982\u65e0\u6cd5\u663e\u793a\u8fdb\u5ea6\u6761\u7ed9\u7ec8\u7aef\u7528\u6237）\u3002"<\/p>\n \u5c0f\u80d6： \u201c\u90a3\u5982\u679c\u6211\u60f3\u540c\u65f6\u6709\u8fdb\u5ea6\u6761 + \u6d41\u5f0f\u4e0a\u4f20\u5462？\u7528\u6237\u5f97\u77e5\u9053\u4e0a\u4f20\u4e86\u591a\u5c11\u3002\u201d<\/p>\n \u5927\u5e08： \u201c\u4f60\u53ef\u4ee5\u5728\u751f\u6210\u5668\u4e2d\u52a0\u5165\u56de\u8c03\u2014\u2014\u6bcf\u8bfb\u53d6\u4e00\u4e2a chunk \u5c31\u8c03\u7528\u56de\u8c03\u51fd\u6570\u62a5\u544a\u8fdb\u5ea6\u3002tqdm \u53ef\u4ee5\u5305\u88c5\u751f\u6210\u5668，\u5b9e\u65f6\u663e\u793a\u8fdb\u5ea6\u6761\u3002\u201d<\/p>\n from<\/span> tqdm import<\/span> tqdm<\/p>\n def<\/span> progress_generator<\/span>(<\/span>filepath,<\/span> chunk_size=<\/span>8192<\/span>)<\/span>:<\/span> \u5c0f\u767d（\u82e5\u6709\u6240\u601d）： \u201c\u90a3\u5982\u679c\u4e0a\u4f20\u4e2d\u65ad\u4e86\u5462？\u7528\u6237\u5f97\u4ece 50% \u91cd\u65b0\u5f00\u59cb\u5417？\u201d<\/p>\n \u5927\u5e08： \u201c\u8fd9\u5c31\u662f\u65ad\u70b9\u7eed\u4f20\u4e0a\u4f20\u2014\u2014\u4f60\u9700\u8981\u670d\u52a1\u7aef\u652f\u6301\u3002\u6d41\u7a0b\u662f：\u5ba2\u6237\u7aef\u5148\u8be2\u95ee\u670d\u52a1\u7aef\u2019\u4f60\u5df2\u7ecf\u6536\u5230\u4e86\u591a\u5c11\u2019（GET \/upload\/status?file_id=xxx），\u670d\u52a1\u7aef\u8fd4\u56de\u5df2\u63a5\u6536\u7684\u5b57\u8282\u6570\u3002\u5ba2\u6237\u7aef\u4ece\u90a3\u4e2a\u4f4d\u7f6e\u5f00\u59cb\u7ee7\u7eed\u4e0a\u4f20\u2014\u2014\u7c7b\u4f3c\u4e8e\u4e0b\u8f7d\u7684 Range \u5934，\u4f46\u4e0a\u4f20\u6ca1\u6709\u6807\u51c6\u7684 Range \u5934，\u901a\u5e38\u7528\u81ea\u5b9a\u4e49\u5934\u6216 Upload-Offset\u3002\u201d<\/p>\n pip install<\/span> requests requests-toolbelt tqdm<\/p>\n \u76ee\u6807：\u5b9e\u73b0\u6700\u7b80\u5355\u7684\u6d41\u5f0f\u4e0a\u4f20，\u7406\u89e3 chunked \u7f16\u7801\u3002<\/p>\n import<\/span> requests def<\/span> read_file_chunks<\/span>(<\/span>filepath:<\/span> str<\/span>,<\/span> chunk_size:<\/span> int<\/span> =<\/span> 65536<\/span>)<\/span>:<\/span> def<\/span> stream_upload_raw<\/span>(<\/span>url:<\/span> str<\/span>,<\/span> filepath:<\/span> str<\/span>,<\/span> \u6ce8\u610f：\u8fd9\u79cd\u65b9\u5f0f\u53d1\u9001\u7684\u662f\u539f\u59cb\u4e8c\u8fdb\u5236 body，\u4e0d\u8d70 multipart \u5c01\u88c5\u3002 # \u4f7f\u7528\u751f\u6210\u5668\u2014\u2014requests \u81ea\u52a8\u542f\u7528 chunked \u7f16\u7801<\/span> print<\/span>(<\/span>f"\u4e0a\u4f20\u5b8c\u6210: <\/span>{<\/span>resp.<\/span>status_code}<\/span><\/span>"<\/span><\/span>)<\/span> # ===== \u6d4b\u8bd5（\u7528 httpbin） =====<\/span> resp =<\/span> stream_upload_raw(<\/span>"https:\/\/httpbin.org\/put"<\/span>
\n """\u751f\u6210\u5668：\u9010\u5757\u8bfb\u53d6\u6587\u4ef6"""<\/span>
\n with<\/span> open<\/span>(<\/span>filepath,<\/span> "rb"<\/span>)<\/span> as<\/span> f:<\/span>
\n while<\/span> chunk :=<\/span> f.<\/span>read(<\/span>chunk_size)<\/span>:<\/span>
\n yield<\/span> chunk<\/p>\n
\nrequests.<\/span>post(<\/span>
\n "https:\/\/upload.example.com\/videos"<\/span>,<\/span>
\n data=<\/span>file_chunk_generator(<\/span>"movie.mp4"<\/span>,<\/span> 8192<\/span>)<\/span>,<\/span>
\n headers=<\/span>{<\/span>"Content-Type"<\/span>:<\/span> "video\/mp4"<\/span>}<\/span>,<\/span>
\n)<\/span><\/p>\n\n
\n total =<\/span> os.<\/span>path.<\/span>getsize(<\/span>filepath)<\/span>
\n with<\/span> tqdm(<\/span>total=<\/span>total,<\/span> unit=<\/span>"B"<\/span>,<\/span> unit_scale=<\/span>True<\/span>)<\/span> as<\/span> pbar:<\/span>
\n with<\/span> open<\/span>(<\/span>filepath,<\/span> "rb"<\/span>)<\/span> as<\/span> f:<\/span>
\n while<\/span> chunk :=<\/span> f.<\/span>read(<\/span>chunk_size)<\/span>:<\/span>
\n yield<\/span> chunk
\n pbar.<\/span>update(<\/span>len<\/span>(<\/span>chunk)<\/span>)<\/span><\/p>\n\n
\u751f\u6d3b\u6bd4\u55bb\u6280\u672f\u6620\u5c04<\/tr>\n \n\n \u6c34\u7ba1\u9001\u6c34（\u4e00\u70b9\u4e00\u70b9\u6d41）<\/td>\n \u6d41\u5f0f\u4e0a\u4f20（\u751f\u6210\u5668 + chunked）<\/td>\n<\/tr>\n \n \u7528\u5361\u8f66\u4e00\u6b21\u6027\u8fd0\u5168\u90e8\u8d27\u7269<\/td>\n Content-Length \u4e00\u6b21\u6027\u4e0a\u4f20<\/td>\n<\/tr>\n \n \u5206\u6279\u5feb\u9012（\u6bcf\u6b21\u9001\u4e00\u7bb1）<\/td>\n chunked \u5206\u5757\u4f20\u8f93<\/td>\n<\/tr>\n \n \u5feb\u9012\u5355\u53f7\u5b9e\u65f6\u8ddf\u8e2a<\/td>\n \u751f\u6210\u5668\u4e2d\u52a0\u8fdb\u5ea6\u56de\u8c03<\/td>\n<\/tr>\n \n \u65ad\u6389\u7684\u5feb\u9012\u4ece\u4e2d\u8f6c\u7ad9\u7ee7\u7eed\u8fd0<\/td>\n \u65ad\u70b9\u7eed\u4f20\u4e0a\u4f20（\u670d\u52a1\u7aef\u534f\u8c03）<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
\n3. \u9879\u76ee\u5b9e\u6218<\/h3>\n
\u73af\u5883\u51c6\u5907<\/h4>\n
\u5206\u6b65\u5b9e\u73b0<\/h4>\n
\u6b65\u9aa4\u4e00：\u751f\u6210\u5668\u6d41\u5f0f\u4e0a\u4f20\u2014\u2014\u57fa\u7840\u7248<\/h5>\n
\nimport<\/span> os
\nimport<\/span> tempfile<\/p>\n
\n """\u751f\u6210\u5668：\u9010\u5757\u8bfb\u53d6\u6587\u4ef6（\u5185\u5b58\u53cb\u597d）"""<\/span>
\n with<\/span> open<\/span>(<\/span>filepath,<\/span> "rb"<\/span>)<\/span> as<\/span> f:<\/span>
\n while<\/span> True<\/span>:<\/span>
\n chunk =<\/span> f.<\/span>read(<\/span>chunk_size)<\/span>
\n if<\/span> not<\/span> chunk:<\/span>
\n break<\/span>
\n yield<\/span> chunk<\/p>\n
\n content_type:<\/span> str<\/span> =<\/span> "application\/octet-stream"<\/span>)<\/span>:<\/span>
\n """\u6d41\u5f0f\u4e0a\u4f20\u2014\u2014\u7528\u751f\u6210\u5668\u4f5c\u4e3a data \u53c2\u6570<\/p>\n
\n \u9002\u5408\u670d\u52a1\u7aef\u63a5\u53d7\u539f\u59cb\u4e8c\u8fdb\u5236\u4e0a\u4f20\u7684\u573a\u666f（\u5982 S3 presigned URL）\u3002
\n """<\/span>
\n file_size =<\/span> os.<\/span>path.<\/span>getsize(<\/span>filepath)<\/span>
\n print<\/span>(<\/span>f"\u6587\u4ef6: <\/span>{<\/span>os.<\/span>path.<\/span>basename(<\/span>filepath)<\/span>}<\/span><\/span> (<\/span>{<\/span>file_size:<\/span>,<\/span>}<\/span><\/span> bytes)"<\/span><\/span>)<\/span><\/p>\n
\n resp =<\/span> requests.<\/span>put(<\/span>
\n url,<\/span>
\n data=<\/span>read_file_chunks(<\/span>filepath)<\/span>,<\/span>
\n headers=<\/span>{<\/span>
\n "Content-Type"<\/span>:<\/span> content_type,<\/span>
\n "Content-Length"<\/span>:<\/span> str<\/span>(<\/span>file_size)<\/span>,<\/span> # \u5982\u679c\u77e5\u9053\u5927\u5c0f\u5c31\u4f20<\/span>
\n }<\/span>,<\/span>
\n timeout=<\/span>(<\/span>10<\/span>,<\/span> 300<\/span>)<\/span>,<\/span>
\n )<\/span><\/p>\n
\n return<\/span> resp<\/p>\n
\n# httpbin.org\/put \u4f1a\u56de\u663e\u8bf7\u6c42\u4f53\u5927\u5c0f<\/span>
\ntmp =<\/span> tempfile.<\/span>NamedTemporaryFile(<\/span>delete=<\/span>False<\/span>,<\/span> suffix=<\/span>".bin"<\/span>)<\/span>
\ntmp.<\/span>write(<\/span>b"A"<\/span> *<\/span> 100000<\/span>)<\/span> # 100KB \u6d4b\u8bd5\u6587\u4ef6<\/span>
\ntmp.<\/span>close(<\/span>)<\/span><\/p>\n