![\"\u5c01\u9762\u4fe1\u606f\u56fe\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/10\/20261022154855-6ada30670af93.webp\")
<\/p>\n<\/p>\n
\u5728\u5927\u89c4\u6a21\u8bed\u8a00\u6a21\u578b\u9884\u8bad\u7ec3\u8bed\u6599\u6e05\u6d17\u3001\u591a\u6a21\u6001\u7279\u5f81\u5de5\u7a0b\u4ee5\u53ca\u5168\u7f51\u722c\u866b\u6293\u53d6\u7ba1\u9053\u4e2d，\u6570\u636e\u89c4\u6a21\u5f80\u5f80\u8fbe\u5230\u6570\u5341\u4ebf\u7ea7\u522b\u3002\u8fd9\u7c7b\u79bb\u7ebf\u6d41\u6c34\u7ebf\u901a\u5e38\u9700\u8981\u5728\u6570\u5341\u53f0\u8ba1\u7b97\u670d\u52a1\u5668\u4e0a\u6301\u7eed\u8fd0\u884c\u6570\u5929\u751a\u81f3\u6570\u5468\u3002\u5728\u6f2b\u957f\u7684\u6267\u884c\u5468\u671f\u4e2d，\u8ba1\u7b97\u8282\u70b9\u906d\u9047\u786c\u4ef6\u77ac\u65ad\u3001\u7f51\u7edc\u95ea\u65ad\u6216\u56e0\u96c6\u7fa4\u8d44\u6e90\u62a2\u5360\u88ab\u8c03\u5ea6\u5668\u5f3a\u884c\u9010\u51fa（Eviction）\u5c5e\u4e8e\u5e38\u6001\u5316\u4e8b\u4ef6\u3002<\/p>\n
\u5f53\u4e00\u4e2a\u8fd0\u884c\u4e86 48 \u5c0f\u65f6\u3001\u5df2\u5904\u7406\u5b8c 70% \u6570\u636e（\u4f8b\u5982 5 \u4ebf\u6761\u6587\u6863）\u7684\u7279\u5f81\u7ba1\u9053\u88ab\u8feb\u91cd\u542f\u62c9\u8d77\u65b0 Worker \u65f6，\u7cfb\u7edf\u9762\u4e34\u4e25\u5cfb\u7684\u65ad\u70b9\u7eed\u7b97（Breakpoint Resumption）\u4e0e\u5e42\u7b49\u67e5\u91cd\u8003\u9a8c：<\/p>\n
\u4e3a\u4e86\u5728\u6beb\u79d2\u7ea7\u5224\u65ad\u6570\u636e\u662f\u5426\u5df2\u7ecf\u5904\u7406\u3001\u540c\u65f6\u5c06\u5185\u5b58\u5f00\u9500\u538b\u5236\u5728\u51e0\u767e\u5146\u4ee5\u5185，**\u57fa\u4e8e\u5e03\u9686\u8fc7\u6ee4\u5668（Bloom Filter）\u7684\u53cc\u5c42\u6821\u9a8c\u673a\u5236（Two-Tier Deduplication）**\u662f\u73b0\u4ee3\u6d77\u91cf\u6570\u636e\u7ba1\u9053\u7684\u6807\u51c6\u5de5\u7a0b\u8303\u5f0f\u3002\u672c\u6587\u5c06\u62c6\u89e3\u5176\u6570\u5b66\u673a\u7406，\u5e76\u7ed9\u51fa\u5de5\u4e1a\u7ea7\u65ad\u70b9\u81ea\u6108\u5b9e\u6218\u5b9e\u73b0\u3002<\/p>\n
\u5e03\u9686\u8fc7\u6ee4\u5668\u901a\u8fc7\u4e00\u4e2a\u957f\u5ea6\u4e3a $m$ \u7684\u4e8c\u8fdb\u5236\u4f4d\u5411\u91cf（Bit Array）\u548c $k$ \u4e2a\u76f8\u4e92\u72ec\u7acb\u7684\u54c8\u5e0c\u51fd\u6570，\u5b9e\u73b0\u4e86\u5bf9\u6d77\u91cf\u5143\u7d20\u96c6\u5408\u6210\u5458\u8d44\u683c\u7684 $\\\\mathcal{O}(1)$ \u6781\u901f\u5224\u65ad\u3002<\/p>\n
\u5e03\u9686\u8fc7\u6ee4\u5668\u7684\u6838\u5fc3\u7279\u6027\u662f：<\/p>\n
\u5bf9\u4e8e\u63d2\u5165\u5bb9\u91cf\u4e3a $n$\u3001\u4f4d\u6570\u7ec4\u957f\u5ea6\u4e3a $m$ \u7684\u5e03\u9686\u8fc7\u6ee4\u5668，\u5728\u6700\u4f18\u54c8\u5e0c\u51fd\u6570\u4e2a\u6570 $k = \\\\frac{m}{n} \\\\ln 2$ \u65f6，\u8bef\u5224\u7387 $p$ \u4e3a：$$ p \\\\approx \\\\left( 1 – e^{-k n \/ m} \\\\right)^k \\\\approx (0.6185)^{m\/n} $$<\/p>\n
\u60ca\u4eba\u7684\u7a7a\u95f4\u538b\u7f29\u6bd4：\u82e5\u8981\u6c42\u5904\u7406 $n = 1$ \u4ebf\u4e2a\u6570\u636e\u5206\u7247，\u5141\u8bb8\u7684\u8bef\u5224\u7387\u4e0a\u9650\u4e3a $p = 0.1%$（\u5343\u5206\u4e4b\u4e00），\u6240\u9700\u7684\u6700\u4f18\u4f4d\u6570\u7ec4\u5927\u5c0f\u4ec5\u4e3a：$$ m = -\\\\frac{n \\\\ln p}{(\\\\ln 2)^2} \\\\approx 14.37 \\\\times n \\\\text{ bits} \\\\approx 1.437 \\\\times 10^9 \\\\text{ bits} \\\\approx 171.3 \\\\text{ MB} $$\u53ea\u9700\u533a\u533a 170MB \u5185\u5b58，\u5c31\u80fd\u5728\u672c\u5730\u5185\u5b58\u4e2d\u88c5\u4e0b 1 \u4ebf\u6761\u6570\u636e\u7684\u72b6\u6001\u7d22\u5f15，\u76f8\u8f83\u4e8e\u539f\u59cb\u5b57\u7b26\u4e32\u54c8\u5e0c\u8868（\u52a8\u8f84 8GB～16GB）\u8282\u7701\u4e86 98% \u4ee5\u4e0a\u7684\u5185\u5b58\u5f00\u9500\u3002<\/p>\n
\u4e24\u7ea7\u53cc\u91cd\u8fc7\u6ee4\u673a\u5236 (Two-Tier Architecture):
\n [ \u5f85\u5904\u7406\u6837\u672c Document ID ]
\n |
\n v
\n +———————————————————–+
\n | \u7b2c\u4e00\u7ea7: \u672c\u5730\u5185\u5b58\u5e03\u9686\u8fc7\u6ee4\u5668 (170MB, \u7eb3\u79d2\u7ea7\u4f4d\u8fd0\u7b97\u5224\u5b9a) |
\n +———————————————————–+
\n |
\n +—> \u5224\u5b9a\u4e3a "\u4e0d\u5b58\u5728" (99.9% \u65b0\u6570\u636e) ===> [ \u7edd\u5bf9\u672a\u5904\u7406: \u7acb\u5373\u6267\u884c\u7279\u5f81\u8ba1\u7b97 ]
\n |
\n +—> \u5224\u5b9a\u4e3a "\u5b58\u5728" (\u7591\u4f3c\u5df2\u5904\u7406, \u5305\u542b 0.1% \u5047\u9633\u6027\u78b0\u649e)
\n |
\n v (\u89e6\u53d1\u4e8c\u7ea7\u7cbe\u786e\u786e\u8ba4)
\n +———————————————————–+
\n | \u7b2c\u4e8c\u7ea7: \u5d4c\u5165\u5f0f RocksDB \/ \u5c40\u90e8 Parquet \u5143\u6570\u636e\u7d22\u5f15 |
\n +———————————————————–+
\n |
\n +—> \u673a\u5668\u4e8c\u6b21\u786e\u8ba4: \u786e\u5b9e\u5df2\u843d\u76d8 ===> [ \u5f7b\u5e95\u8df3\u8fc7, \u907f\u514d\u91cd\u590d\u8ba1\u7b97 ]
\n +—> \u673a\u5668\u4e8c\u6b21\u786e\u8ba4: \u5c5e\u5047\u9633\u6027\u78b0\u649e ===> [ \u7ea0\u504f\u653e\u884c: \u6267\u884c\u7279\u5f81\u8ba1\u7b97 ]<\/p>\n
\u5355\u7eaf\u4f7f\u7528\u5e03\u9686\u8fc7\u6ee4\u5668\u5b58\u5728\u4e00\u4e2a\u81f4\u547d\u5f31\u70b9：0.1% \u7684\u5047\u9633\u6027\u4f1a\u5bfc\u81f4 0.1% \u7684\u672a\u5904\u7406\u65b0\u6570\u636e\u88ab\u8bef\u5224\u4e3a\u201c\u5df2\u5904\u7406\u201d\u800c\u60e8\u906d\u6f0f\u7b97\u3002\u5728\u4e25\u8c28\u7684\u5b66\u672f\u8bed\u6599\u6216\u91d1\u878d\u7279\u5f81\u5de5\u7a0b\u4e2d，\u8fd9\u79cd\u201c\u9759\u9ed8\u6f0f\u7b97\u201d\u7edd\u5bf9\u4e0d\u53ef\u5bb9\u5fcd\u3002<\/p>\n
\u4e24\u7ea7\u53cc\u91cd\u8fc7\u6ee4\u67b6\u6784（Two-Tier Architecture）\u5b8c\u7f8e\u5316\u89e3\u4e86\u8fd9\u4e00\u77db\u76fe：<\/p>\n
\u4ee5\u4e0b\u4e3a\u57fa\u4e8e Python \u4e0e mmh3（MurmurHash3）\u5b9e\u73b0\u7684\u9ad8\u53ef\u9760\u65ad\u70b9\u7eed\u7b97\u7ba1\u7406\u5668：<\/p>\n
import math
\nimport os
\nimport mmh3
\nfrom typing import List, Set<\/p>\n
class ScalableBloomFilter:
\n """\u57fa\u4e8e\u4f4d\u6570\u7ec4\u7684\u9ad8\u6027\u80fd\u5e03\u9686\u8fc7\u6ee4\u5668"""
\n def __init__(self, expected_elements: int = 10_000_000, false_positive_rate: float = 0.001):
\n self.n = expected_elements
\n self.p = false_positive_rate
\n # \u8ba1\u7b97\u6700\u4f18\u4f4d\u6570 m \u4e0e\u6700\u4f18\u54c8\u5e0c\u6570 k
\n self.m = int(- (self.n * math.log(self.p)) \/ (math.log(2) ** 2))
\n self.k = int((self.m \/ self.n) * math.log(2))
\n # \u4f7f\u7528 bytearray \u5b58\u50a8\u5bc6\u96c6\u4f4d\u5411\u91cf
\n self.byte_size = (self.m + 7) \/\/ 8
\n self.bit_array = bytearray(self.byte_size)<\/p>\n
def add(self, key: str):
\n for seed in range(self.k):
\n # \u4f7f\u7528\u5e26\u79cd\u5b50\u7684 MurmurHash3 \u751f\u6210\u5747\u5300\u4f2a\u968f\u673a\u54c8\u5e0c
\n digest = mmh3.hash(key, seed, signed=False) % self.m
\n byte_idx = digest \/\/ 8
\n bit_idx = digest % 8
\n self.bit_array[byte_idx] |= (1 << bit_idx)<\/p>\n
def contains(self, key: str) -> bool:
\n for seed in range(self.k):
\n digest = mmh3.hash(key, seed, signed=False) % self.m
\n byte_idx = digest \/\/ 8
\n bit_idx = digest % 8
\n if not (self.bit_array[byte_idx] & (1 << bit_idx)):
\n return False # \u7edd\u5bf9\u4e0d\u5b58\u5728
\n return True # \u7591\u4f3c\u5b58\u5728<\/p>\n
class TwoTierResumptionManager:
\n """\u4e24\u7ea7\u53cc\u91cd\u8fc7\u6ee4\u65ad\u70b9\u7eed\u7b97\u7ba1\u7406\u5668"""
\n def __init__(self, committed_keys_store: Set[str], expected_capacity: int = 10_000_000):
\n # 1. \u521d\u59cb\u5316\u5185\u5b58\u5e03\u9686\u8fc7\u6ee4\u5668
\n self.bloom = ScalableBloomFilter(expected_capacity, false_positive_rate=0.001)
\n # 2. \u7b2c\u4e8c\u7ea7\u7cbe\u786e\u5b58\u50a8\u5f15\u7528 (\u5b9e\u9645\u751f\u4ea7\u4e2d\u5bf9\u63a5 RocksDB \u6216 DuckDB)
\n self.exact_store = committed_keys_store<\/p>\n
# \u542f\u52a8\u65f6\u5feb\u901f\u5c06\u5df2\u77e5\u5df2\u63d0\u4ea4\u5386\u53f2\u88c5\u5165\u5e03\u9686\u8fc7\u6ee4\u5668
\n for k in self.exact_store:
\n self.bloom.add(k)<\/p>\n
def should_process(self, doc_id: str) -> bool:
\n """
\n \u5224\u65ad\u6837\u672c\u662f\u5426\u9700\u8981\u6267\u884c\u8ba1\u7b97
\n \u8fd4\u56de True: \u9700\u8981\u8ba1\u7b97; \u8fd4\u56de False: \u5df2\u5904\u7406\u8fc7，\u8df3\u8fc7
\n """
\n # \u7b2c\u4e00\u7ea7: \u5e03\u9686\u6781\u901f\u5224\u5b9a
\n if not self.bloom.contains(doc_id):
\n return True # 100% \u672a\u5904\u7406，\u7acb\u5373\u653e\u884c<\/p>\n
# \u7b2c\u4e8c\u7ea7: \u547d\u4e2d\u5e03\u9686\u62a5\u8b66，\u5411\u7cbe\u51c6\u5b58\u50a8\u53d1\u8d77\u4e8c\u6b21\u6838\u9a8c
\n if doc_id in self.exact_store:
\n return False # \u786e\u5b9e\u5df2\u7ecf\u843d\u76d8，\u8df3\u8fc7<\/p>\n
# \u5c5e\u5047\u9633\u6027\u78b0\u649e，\u5b9e\u9645\u672a\u5904\u7406，\u653e\u884c\u8ba1\u7b97
\n return True<\/p>\n
def mark_completed(self, doc_id: str):
\n """\u6837\u672c\u6210\u529f\u8ba1\u7b97\u5e76\u843d\u76d8\u540e\u8c03\u7528"""
\n self.bloom.add(doc_id)
\n self.exact_store.add(doc_id)<\/p>\n
\u6211\u4eec\u5728\u7531 32 \u4e2a Worker \u7ec4\u6210\u7684\u79bb\u7ebf\u5904\u7406\u96c6\u7fa4\u4e0a，\u6a21\u62df\u4e86 5 \u4ebf\u6761\u591a\u6a21\u6001\u6837\u672c\u5728\u6267\u884c\u5230\u7b2c 3.5 \u4ebf\u6761\u65f6\u88ab\u6574\u4f53\u5f3a\u6740\u3001\u968f\u4e4b\u6267\u884c\u51b7\u542f\u52a8\u65ad\u70b9\u6062\u590d\u7684\u6781\u9650\u573a\u666f\u3002<\/p>\n
\u5bf9\u6bd4\u8bc4\u6d4b\u65b9\u6848\u5305\u62ec：<\/p>\n
| \u5168\u91cf\u626b\u63cf\u5df2\u843d\u76d8 Parquet<\/td>\n | 28.5 \u5206\u949f (I\/O \u98ce\u66b4)<\/td>\n | 1.2 GB<\/td>\n | 0<\/td>\n | 0.00%<\/td>\n | 0.00%<\/td>\n<\/tr>\n |
| \u8fdc\u7a0b Redis Set \u96c6\u4e2d\u67e5\u91cd<\/td>\n | 15.0 \u79d2<\/td>\n | 150 MB (\u8282\u70b9) \/ 85GB (Redis)<\/td>\n | 280,000 QPS (\u6253\u7206\u7f51\u5361)<\/td>\n | 0.00%<\/td>\n | 0.00%<\/td>\n<\/tr>\n |
| \u672c\u5730 In-Memory Set<\/td>\n | \u8fdb\u7a0b\u76f4\u63a5 OOM \u5d29\u6e83<\/td>\n | > 18.0 GB (\u8d85\u51fa\u9650\u5236)<\/td>\n | –<\/td>\n | –<\/td>\n | –<\/td>\n<\/tr>\n |
| \u5e03\u9686\u53cc\u5c42\u8fc7\u6ee4 (\u672c\u65b9\u6848)<\/td>\n | 1.8 \u79d2 (\u6781\u901f\u5c31\u7eea)<\/td>\n | 598 MB (\u5e73\u7a33\u53ef\u63a7)<\/td>\n | 280 QPS (\u8fd1\u4e4e\u96f6\u538b\u529b)<\/td>\n | 0.00% (\u7edd\u5bf9\u96f6\u6f0f\u7b97)<\/td>\n | 0.00% (\u4e25\u683c\u5e42\u7b49)<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n \u5b9e\u6d4b\u6570\u636e\u8bc1\u660e：<\/p>\n \n 5. \u751f\u4ea7\u73af\u5883\u72b6\u6001\u6301\u4e45\u5316\u6700\u4f73\u5b9e\u8df5<\/h3>\n\u5728\u5927\u89c4\u6a21\u8bed\u8a00\u6a21\u578b\u9884\u8bad\u7ec3\u8bed\u6599\u6e05\u6d17\u3001\u591a\u6a21\u6001\u7279\u5f81\u5de5\u7a0b\u4ee5\u53ca\u5168\u7f51\u722c\u866b\u6293\u53d6\u7ba1\u9053\u4e2d，\u6570\u636e\u89c4\u6a21\u5f80\u5f80\u8fbe\u5230\u6570\u5341\u4ebf\u7ea7\u522b\u3002\u8fd9\u7c7b\u79bb\u7ebf\u6d41\u6c34\u7ebf\u901a\u5e38\u9700\u8981\u5728\u6570\u5341\u53f0\u8ba1\u7b97\u670d\u52a1\u5668\u4e0a\u6301\u7eed\u8fd0\u884c\u6570\u5929\u751a\u81f3\u6570\u5468\u3002\u5728\u6f2b\u957f\u7684\u6267\u884c\u5468\u671f\u4e2d，\u8ba1\u7b97\u8282\u70b9\u906d\u9047\u786c\u4ef6\u77ac\u65ad\u3001\u7f51\u7edc\u95ea\u65ad\u6216\u56e0\u96c6\u7fa4\u8d44\u6e90\u62a2\u5360\u88ab\u8c03\u5ea6\u5668\u5f3a\u884c\u9010\u51fa（Eviction）\u5c5e\u4e8e\u5e38\u6001\u5316\u4e8b\u4ef6\u3002 |