\u5728\u6570\u5b57\u65f6\u4ee3，\u827a\u672f\u4f5c\u54c1\u7684\u6570\u5b57\u5316\u7ba1\u7406\u548c\u5206\u6790\u5df2\u6210\u4e3a\u827a\u672f\u754c\u3001\u5b66\u672f\u754c\u548c\u5546\u4e1a\u9886\u57df\u7684\u91cd\u8981\u9700\u6c42\u3002\u65e0\u8bba\u662f\u535a\u7269\u9986\u7684\u85cf\u54c1\u7ba1\u7406\u3001\u827a\u672f\u5e02\u573a\u7684\u4ef7\u683c\u5206\u6790，\u8fd8\u662fAI\u827a\u672f\u521b\u4f5c\u7684\u8bad\u7ec3\u6570\u636e，\u4e00\u4e2a\u7ed3\u6784\u5b8c\u5584\u7684\u827a\u672f\u4f5c\u54c1\u6570\u636e\u5e93\u90fd\u626e\u6f14\u7740\u81f3\u5173\u91cd\u8981\u7684\u89d2\u8272\u3002<\/p>\n
\u827a\u672f\u4f5c\u54c1\u6570\u636e\u5305\u542b\u4e86\u4e30\u5bcc\u7684\u4fe1\u606f\u7ef4\u5ea6：\u521b\u4f5c\u80cc\u666f\u3001\u827a\u672f\u5bb6\u751f\u5e73\u3001\u6280\u6cd5\u6750\u6599\u3001\u5c3a\u5bf8\u89c4\u683c\u3001\u6536\u85cf\u5386\u53f2\u3001\u5e02\u573a\u4ef7\u503c\u7b49\u3002\u8fd9\u4e9b\u6570\u636e\u4e0d\u4ec5\u5bf9\u4e8e\u827a\u672f\u53f2\u7814\u7a76\u5177\u6709\u5b66\u672f\u4ef7\u503c，\u8fd8\u5728\u4ee5\u4e0b\u9886\u57df\u53d1\u6325\u7740\u91cd\u8981\u4f5c\u7528：<\/p>\n
\u827a\u672f\u5e02\u573a\u5206\u6790：\u901a\u8fc7\u5206\u6790\u4f5c\u54c1\u7684\u6210\u4ea4\u4ef7\u683c\u3001\u6d41\u901a\u9891\u7387\u7b49\u6570\u636e，\u53ef\u4ee5\u6d1e\u5bdf\u827a\u672f\u5e02\u573a\u7684\u8d70\u52bf\u548c\u89c4\u5f8b<\/p>\n<\/li>\n
\u827a\u672f\u54c1\u9274\u5b9a：\u57fa\u4e8e\u5df2\u77e5\u4f5c\u54c1\u7684\u6280\u6cd5\u7279\u5f81\u3001\u6750\u6599\u4f7f\u7528\u7b49\u6570\u636e，\u4e3a\u9274\u5b9a\u63d0\u4f9b\u53c2\u8003\u4f9d\u636e<\/p>\n<\/li>\n
\u6570\u5b57\u5316\u5c55\u89c8：\u6784\u5efa\u865a\u62df\u535a\u7269\u9986\u548c\u5728\u7ebf\u5c55\u89c8\u7684\u57fa\u7840\u6570\u636e\u652f\u6491<\/p>\n<\/li>\n
\u827a\u672f\u6559\u80b2：\u4e3a\u5b66\u751f\u548c\u7231\u597d\u8005\u63d0\u4f9b\u7cfb\u7edf\u5316\u7684\u827a\u672f\u77e5\u8bc6\u5b66\u4e60\u8d44\u6e90<\/p>\n<\/li>\n
AI\u827a\u672f\u521b\u4f5c：\u4e3a\u751f\u6210\u5bf9\u6297\u7f51\u7edc\u7b49\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u63d0\u4f9b\u8bad\u7ec3\u6570\u636e<\/p>\n<\/li>\n<\/ul>\n
\u8fc7\u53bb，\u827a\u672f\u4f5c\u54c1\u4fe1\u606f\u7684\u6536\u96c6\u4e3b\u8981\u4f9d\u8d56\u4eba\u5de5\u6574\u7406，\u8fd9\u79cd\u65b9\u5f0f\u5b58\u5728\u660e\u663e\u4e0d\u8db3：<\/p>\n
\u6548\u7387\u4f4e\u4e0b：\u624b\u52a8\u5f55\u5165\u4e00\u4ef6\u4f5c\u54c1\u53ef\u80fd\u9700\u8981\u6570\u5206\u949f，\u9762\u5bf9\u767e\u4e07\u7ea7\u4f5c\u54c1\u6570\u636e\u65f6\u51e0\u4e4e\u4e0d\u53ef\u884c<\/p>\n<\/li>\n
\u66f4\u65b0\u6ede\u540e：\u65b0\u5c55\u89c8\u3001\u65b0\u4ea4\u6613\u7684\u4fe1\u606f\u65e0\u6cd5\u53ca\u65f6\u6536\u5f55<\/p>\n<\/li>\n
\u8986\u76d6\u4e0d\u5168：\u53d7\u9650\u4e8e\u4eba\u529b，\u96be\u4ee5\u540c\u65f6\u5173\u6ce8\u591a\u4e2a\u827a\u672f\u673a\u6784\u548c\u6e20\u9053<\/p>\n<\/li>\n
\u6807\u51c6\u5316\u56f0\u96be：\u4e0d\u540c\u6765\u6e90\u7684\u6570\u636e\u683c\u5f0f\u5404\u5f02，\u6574\u5408\u6210\u672c\u9ad8<\/p>\n<\/li>\n<\/ul>\n
Python\u722c\u866b\u6280\u672f\u4e3a\u6211\u4eec\u63d0\u4f9b\u4e86\u4e00\u4e2a\u9ad8\u6548\u3001\u53ef\u6269\u5c55\u7684\u6570\u636e\u91c7\u96c6\u65b9\u6848\u3002\u901a\u8fc7\u7f16\u5199\u81ea\u52a8\u5316\u7a0b\u5e8f，\u6211\u4eec\u53ef\u4ee5：<\/p>\n
\u5927\u89c4\u6a21\u91c7\u96c6：\u540c\u65f6\u6293\u53d6\u591a\u4e2a\u827a\u672f\u7f51\u7ad9\u7684\u6570\u767e\u4e07\u6761\u4f5c\u54c1\u4fe1\u606f<\/p>\n<\/li>\n
\u5b9e\u65f6\u66f4\u65b0：\u8bbe\u7f6e\u5b9a\u65f6\u4efb\u52a1，\u786e\u4fdd\u6570\u636e\u7684\u65b0\u9c9c\u5ea6<\/p>\n<\/li>\n
\u667a\u80fd\u89e3\u6790：\u4f7f\u7528\u81ea\u7136\u8bed\u8a00\u5904\u7406\u548c\u56fe\u50cf\u8bc6\u522b\u6280\u672f\u63d0\u53d6\u7ed3\u6784\u5316\u4fe1\u606f<\/p>\n<\/li>\n
\u5f02\u5e38\u5904\u7406：\u5e94\u5bf9\u7f51\u7ad9\u6539\u7248\u3001\u7f51\u7edc\u6ce2\u52a8\u7b49\u590d\u6742\u60c5\u51b5<\/p>\n<\/li>\n
\u6570\u636e\u6e05\u6d17：\u81ea\u52a8\u53bb\u91cd\u3001\u683c\u5f0f\u8f6c\u6362\u3001\u7f3a\u5931\u503c\u5904\u7406<\/p>\n<\/li>\n
\u5728\u5f00\u59cb\u7f16\u5199\u4ee3\u7801\u4e4b\u524d，\u6211\u4eec\u9700\u8981\u5bf9\u6574\u4e2a\u9879\u76ee\u8fdb\u884c\u7cfb\u7edf\u6027\u7684\u89c4\u5212，\u660e\u786e\u6280\u672f\u8def\u7ebf\u548c\u67b6\u6784\u8bbe\u8ba1\u3002<\/p>\n
\u6211\u4eec\u5c06\u4ece\u4ee5\u4e0b\u51e0\u4e2a\u4e3b\u8981\u6e20\u9053\u91c7\u96c6\u827a\u672f\u4f5c\u54c1\u6570\u636e：<\/p>\n
2.1.1 \u4e3b\u6d41\u827a\u672f\u5e73\u53f0<\/p>\n
WikiArt：\u5305\u542b\u8d85\u8fc725\u4e07\u4ef6\u827a\u672f\u4f5c\u54c1，\u6db5\u76d6\u6570\u5343\u4f4d\u827a\u672f\u5bb6<\/p>\n<\/li>\n
Artsy：\u5f53\u4ee3\u827a\u672f\u5e73\u53f0，\u63d0\u4f9b\u8be6\u7ec6\u7684\u827a\u672f\u5bb6\u548c\u4f5c\u54c1\u4fe1\u606f<\/p>\n<\/li>\n
Art Institute of Chicago：\u5f00\u653eAPI，\u63d0\u4f9b\u9ad8\u8d28\u91cf\u7684\u9986\u85cf\u6570\u636e<\/p>\n<\/li>\n
\u5927\u90fd\u4f1a\u827a\u672f\u535a\u7269\u9986：\u5b8c\u6574\u7684\u516c\u5f00\u6570\u636e\u96c6<\/p>\n<\/li>\n
\u6b27\u6d32\u6570\u5b57\u56fe\u4e66\u9986：\u805a\u5408\u6b27\u6d32\u591a\u5bb6\u535a\u7269\u9986\u7684\u85cf\u54c1\u6570\u636e<\/p>\n<\/li>\n<\/ul>\n
2.1.2 \u76ee\u6807\u6570\u636e\u5b57\u6bb5<\/p>\n
\u6211\u4eec\u9700\u8981\u91c7\u96c6\u7684\u7ed3\u6784\u5316\u4fe1\u606f\u5305\u62ec：<\/p>\n
python<\/p>\n
\u827a\u672f\u4f5c\u54c1\u4fe1\u606f\u7ed3\u6784 = {
\n "basic_info": {
\n "title": "\u4f5c\u54c1\u540d\u79f0",
\n "artist": "\u827a\u672f\u5bb6",
\n "date": "\u521b\u4f5c\u5e74\u4efd",
\n "medium": "\u6750\u8d28\/\u6280\u6cd5",
\n "dimensions": "\u5c3a\u5bf8（\u9ad8\u00d7\u5bbd）",
\n "collection": "\u6536\u85cf\u673a\u6784"
\n },
\n "detailed_info": {
\n "description": "\u4f5c\u54c1\u63cf\u8ff0",
\n "provenance": "\u6765\u6e90\u5386\u53f2",
\n "exhibition_history": "\u5c55\u89c8\u5386\u53f2",
\n "bibliography": "\u53c2\u8003\u6587\u732e",
\n "signature": "\u7b7e\u540d\u4f4d\u7f6e"
\n },
\n "market_info": {
\n "estimated_price": "\u4f30\u4ef7",
\n "sale_price": "\u6210\u4ea4\u4ef7",
\n "auction_house": "\u62cd\u5356\u884c",
\n "sale_date": "\u62cd\u5356\u65e5\u671f"
\n },
\n "media_info": {
\n "image_url": "\u56fe\u7247\u94fe\u63a5",
\n "thumbnail_url": "\u7f29\u7565\u56fe\u94fe\u63a5",
\n "image_metadata": "\u56fe\u7247\u5143\u6570\u636e"
\n }
\n} <\/p>\n
\u57fa\u4e8e2024-2025\u5e74\u7684\u6700\u65b0\u6280\u672f\u53d1\u5c55，\u6211\u4eec\u9009\u62e9\u4ee5\u4e0b\u6280\u672f\u6808：<\/p>\n
2.2.1 \u6838\u5fc3\u722c\u866b\u6846\u67b6<\/p>\n
python<\/p>\n
# \u73af\u5883\u8981\u6c42
\nPython >= 3.11
\naiohttp >= 3.9.0 # \u5f02\u6b65HTTP\u5ba2\u6237\u7aef
\nhttpx >= 0.27.0 # \u652f\u6301HTTP\/2\u7684\u5ba2\u6237\u7aef
\nScrapy >= 2.11 # \u5206\u5e03\u5f0f\u722c\u866b\u6846\u67b6
\nPlaywright >= 1.40 # \u65e0\u5934\u6d4f\u89c8\u5668\u81ea\u52a8\u5316
\nSelenium >= 4.15 # Web\u81ea\u52a8\u5316\u6d4b\u8bd5\u5de5\u5177 <\/p>\n
2.2.2 \u6570\u636e\u89e3\u6790\u4e0e\u5904\u7406<\/p>\n
python<\/p>\n
# \u6570\u636e\u89e3\u6790\u5e93
\nBeautifulSoup4 >= 4.12 # HTML\u89e3\u6790
\nlxml >= 4.9 # \u9ad8\u6027\u80fdXML\u89e3\u6790
\nparsel >= 1.8 # \u57fa\u4e8elxml\u7684\u9009\u62e9\u5668
\njsonpath-ng >= 1.6 # JSON\u8def\u5f84\u67e5\u8be2
\njmespath >= 1.0 # \u53e6\u4e00\u4e2aJSON\u67e5\u8be2\u5e93 <\/p>\n
2.2.3 \u6570\u636e\u5b58\u50a8\u65b9\u6848<\/p>\n
python<\/p>\n
# \u6570\u636e\u5e93\u9009\u62e9
\nMongoDB >= 7.0 # \u6587\u6863\u6570\u636e\u5e93，\u9002\u5408\u5b58\u50a8JSON\u683c\u5f0f\u7684\u827a\u672f\u54c1\u6570\u636e
\nPostgreSQL >= 16 # \u5173\u7cfb\u578b\u6570\u636e\u5e93，\u9002\u5408\u7ed3\u6784\u5316\u6570\u636e\u5b58\u50a8
\nRedis >= 7.2 # \u7f13\u5b58\u548c\u4efb\u52a1\u961f\u5217
\nElasticsearch >= 8.11 # \u5168\u6587\u641c\u7d22\u548c\u5206\u6790
\nMinIO >= RELEASE.2024 # \u5bf9\u8c61\u5b58\u50a8，\u5b58\u50a8\u56fe\u7247\u7b49\u5a92\u4f53\u6587\u4ef6 <\/p>\n
2.2.4 \u6570\u636e\u6e05\u6d17\u4e0e\u589e\u5f3a<\/p>\n
python<\/p>\n
# \u6570\u636e\u5904\u7406\u5de5\u5177
\npandas >= 2.1 # \u6570\u636e\u5206\u6790
\nnumpy >= 1.26 # \u6570\u503c\u8ba1\u7b97
\npillow >= 10.1 # \u56fe\u50cf\u5904\u7406
\nopencv-python >= 4.9 # \u8ba1\u7b97\u673a\u89c6\u89c9
\npytesseract >= 0.3.10 # OCR\u6587\u5b57\u8bc6\u522b <\/p>\n
\u6211\u4eec\u5c06\u91c7\u7528\u5206\u5e03\u5f0f\u5fae\u670d\u52a1\u67b6\u6784，\u786e\u4fdd\u7cfb\u7edf\u7684\u53ef\u6269\u5c55\u6027\u548c\u7a33\u5b9a\u6027：<\/p>\n
text<\/p>\n
\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u2502 \u8c03\u5ea6\u4e2d\u5fc3 \u2502
\n\u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502
\n\u2502 \u2502\u5b9a\u65f6\u8c03\u5ea6\u5668 \u2502 \u2502\u4f18\u5148\u7ea7\u961f\u5217 \u2502 \u2502\u5931\u8d25\u91cd\u8bd5\u5668 \u2502 \u2502
\n\u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502
\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n \u2502
\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n \u25bc \u25bc \u25bc
\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u2502 \u722c\u866b\u8282\u70b91 \u2502 \u2502 \u722c\u866b\u8282\u70b92 \u2502 \u2502 \u722c\u866b\u8282\u70b9N \u2502
\n\u2502 – WikiArt \u2502 \u2502 – Artsy \u2502 \u2502 – \u5176\u4ed6\u5e73\u53f0 \u2502
\n\u2502 – \u901f\u7387\u9650\u5236 \u2502 \u2502 – \u4ee3\u7406IP\u6c60 \u2502 \u2502 – \u5206\u5e03\u5f0f \u2502
\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n \u2502 \u2502 \u2502
\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n \u25bc
\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n \u2502 \u6d88\u606f\u961f\u5217（Kafka） \u2502
\n \u2502 – \u6570\u636e\u5206\u53d1 \u2502
\n \u2502 – \u6d41\u91cf\u524a\u5cf0 \u2502
\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n \u2502
\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n \u25bc \u25bc \u25bc
\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u2502 \u6e05\u6d17\u7ba1\u90531 \u2502 \u2502 \u6e05\u6d17\u7ba1\u90532 \u2502 \u2502 \u6e05\u6d17\u7ba1\u9053N \u2502
\n\u2502 – \u6570\u636e\u9a8c\u8bc1 \u2502 \u2502 – \u683c\u5f0f\u8f6c\u6362 \u2502 \u2502 – \u6570\u636e\u589e\u5f3a \u2502
\n\u2502 – \u53bb\u91cd\u5904\u7406 \u2502 \u2502 – \u5b9e\u4f53\u8bc6\u522b \u2502 \u2502 – \u56fe\u50cf\u5904\u7406 \u2502
\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n \u2502 \u2502 \u2502
\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n \u25bc
\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n \u2502 \u6570\u636e\u5b58\u50a8\u5c42 \u2502
\n \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502
\n \u2502 \u2502 MongoDB \u2502 \u2502
\n \u2502 \u2502 PostgreSQL \u2502 \u2502
\n \u2502 \u2502 Elasticsearch\u2502 \u2502
\n \u2502 \u2502 MinIO \u2502 \u2502
\n \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502
\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 <\/p>\n
\u9996\u5148，\u6211\u4eec\u9700\u8981\u521b\u5efa\u4e00\u4e2a\u72ec\u7acb\u7684Python\u73af\u5883，\u786e\u4fdd\u4f9d\u8d56\u5305\u7248\u672c\u7684\u4e00\u81f4\u6027：<\/p>\n
bash<\/p>\n
# \u521b\u5efa\u865a\u62df\u73af\u5883
\npython3.11 -m venv art_crawler_env<\/p>\n
# \u6fc0\u6d3b\u865a\u62df\u73af\u5883（Linux\/Mac）
\nsource art_crawler_env\/bin\/activate<\/p>\n
# \u6fc0\u6d3b\u865a\u62df\u73af\u5883（Windows）
\nart_crawler_env\\\\Scripts\\\\activate<\/p>\n
# \u5347\u7ea7pip\u548csetuptools
\npython -m pip install –upgrade pip setuptools wheel <\/p>\n
\u4e00\u4e2a\u6e05\u6670\u7684\u9879\u76ee\u7ed3\u6784\u5bf9\u4e8e\u5927\u578b\u722c\u866b\u9879\u76ee\u81f3\u5173\u91cd\u8981：<\/p>\n
bash<\/p>\n
art_crawler_project\/
\n\u251c\u2500\u2500 docker\/ # Docker\u914d\u7f6e
\n\u2502 \u251c\u2500\u2500 docker-compose.yml # \u670d\u52a1\u7f16\u6392
\n\u2502 \u251c\u2500\u2500 mongodb\/ # MongoDB\u914d\u7f6e
\n\u2502 \u251c\u2500\u2500 postgres\/ # PostgreSQL\u914d\u7f6e
\n\u2502 \u2514\u2500\u2500 redis\/ # Redis\u914d\u7f6e
\n\u251c\u2500\u2500 src\/ # \u6e90\u4ee3\u7801
\n\u2502 \u251c\u2500\u2500 crawlers\/ # \u722c\u866b\u6a21\u5757
\n\u2502 \u2502 \u251c\u2500\u2500 __init__.py
\n\u2502 \u2502 \u251c\u2500\u2500 base.py # \u57fa\u7840\u722c\u866b\u7c7b
\n\u2502 \u2502 \u251c\u2500\u2500 wikiart.py # WikiArt\u722c\u866b
\n\u2502 \u2502 \u251c\u2500\u2500 artsy.py # Artsy\u722c\u866b
\n\u2502 \u2502 \u251c\u2500\u2500 museum_api.py # \u535a\u7269\u9986API\u722c\u866b
\n\u2502 \u2502 \u2514\u2500\u2500 middlewares\/ # \u4e2d\u95f4\u4ef6
\n\u2502 \u2502 \u251c\u2500\u2500 __init__.py
\n\u2502 \u2502 \u251c\u2500\u2500 proxy.py # \u4ee3\u7406\u4e2d\u95f4\u4ef6
\n\u2502 \u2502 \u251c\u2500\u2500 retry.py # \u91cd\u8bd5\u4e2d\u95f4\u4ef6
\n\u2502 \u2502 \u2514\u2500\u2500 rate_limiter.py # \u901f\u7387\u9650\u5236
\n\u2502 \u251c\u2500\u2500 parsers\/ # \u89e3\u6790\u5668
\n\u2502 \u2502 \u251c\u2500\u2500 __init__.py
\n\u2502 \u2502 \u251c\u2500\u2500 html_parser.py # HTML\u89e3\u6790\u5668
\n\u2502 \u2502 \u251c\u2500\u2500 json_parser.py # JSON\u89e3\u6790\u5668
\n\u2502 \u2502 \u2514\u2500\u2500 image_parser.py # \u56fe\u50cf\u5143\u6570\u636e\u89e3\u6790
\n\u2502 \u251c\u2500\u2500 pipelines\/ # \u6570\u636e\u7ba1\u9053
\n\u2502 \u2502 \u251c\u2500\u2500 __init__.py
\n\u2502 \u2502 \u251c\u2500\u2500 validation.py # \u6570\u636e\u9a8c\u8bc1
\n\u2502 \u2502 \u251c\u2500\u2500 deduplication.py # \u6570\u636e\u53bb\u91cd
\n\u2502 \u2502 \u251c\u2500\u2500 enrichment.py # \u6570\u636e\u589e\u5f3a
\n\u2502 \u2502 \u2514\u2500\u2500 storage.py # \u6570\u636e\u5b58\u50a8
\n\u2502 \u251c\u2500\u2500 models\/ # \u6570\u636e\u6a21\u578b
\n\u2502 \u2502 \u251c\u2500\u2500 __init__.py
\n\u2502 \u2502 \u251c\u2500\u2500 artwork.py # \u4f5c\u54c1\u6a21\u578b
\n\u2502 \u2502 \u251c\u2500\u2500 artist.py # \u827a\u672f\u5bb6\u6a21\u578b
\n\u2502 \u2502 \u2514\u2500\u2500 museum.py # \u535a\u7269\u9986\u6a21\u578b
\n\u2502 \u251c\u2500\u2500 utils\/ # \u5de5\u5177\u51fd\u6570
\n\u2502 \u2502 \u251c\u2500\u2500 __init__.py
\n\u2502 \u2502 \u251c\u2500\u2500 logger.py # \u65e5\u5fd7\u5de5\u5177
\n\u2502 \u2502 \u251c\u2500\u2500 validator.py # \u9a8c\u8bc1\u5de5\u5177
\n\u2502 \u2502 \u2514\u2500\u2500 helpers.py # \u8f85\u52a9\u51fd\u6570
\n\u2502 \u2514\u2500\u2500 config\/ # \u914d\u7f6e\u6587\u4ef6
\n\u2502 \u251c\u2500\u2500 __init__.py
\n\u2502 \u251c\u2500\u2500 settings.py # \u722c\u866b\u914d\u7f6e
\n\u2502 \u2514\u2500\u2500 constants.py # \u5e38\u91cf\u5b9a\u4e49
\n\u251c\u2500\u2500 tests\/ # \u6d4b\u8bd5\u4ee3\u7801
\n\u2502 \u251c\u2500\u2500 __init__.py
\n\u2502 \u251c\u2500\u2500 test_crawlers\/
\n\u2502 \u251c\u2500\u2500 test_parsers\/
\n\u2502 \u2514\u2500\u2500 test_pipelines\/
\n\u251c\u2500\u2500 scripts\/ # \u811a\u672c\u5de5\u5177
\n\u2502 \u251c\u2500\u2500 init_db.py # \u6570\u636e\u5e93\u521d\u59cb\u5316
\n\u2502 \u251c\u2500\u2500 run_crawler.py # \u8fd0\u884c\u722c\u866b
\n\u2502 \u2514\u2500\u2500 clean_data.py # \u6570\u636e\u6e05\u7406
\n\u251c\u2500\u2500 requirements\/ # \u4f9d\u8d56\u7ba1\u7406
\n\u2502 \u251c\u2500\u2500 base.txt # \u57fa\u7840\u4f9d\u8d56
\n\u2502 \u251c\u2500\u2500 dev.txt # \u5f00\u53d1\u4f9d\u8d56
\n\u2502 \u2514\u2500\u2500 prod.txt # \u751f\u4ea7\u4f9d\u8d56
\n\u251c\u2500\u2500 .env.example # \u73af\u5883\u53d8\u91cf\u793a\u4f8b
\n\u251c\u2500\u2500 .gitignore # Git\u5ffd\u7565\u6587\u4ef6
\n\u251c\u2500\u2500 README.md # \u9879\u76ee\u8bf4\u660e
\n\u2514\u2500\u2500 setup.py # \u5b89\u88c5\u811a\u672c <\/p>\n
3.3.1 \u57fa\u7840\u914d\u7f6e (src\/config\/settings.py)<\/p>\n
python<\/p>\n
import os
\nfrom pathlib import Path
\nfrom typing import Dict, List, Optional
\nfrom pydantic_settings import BaseSettings
\nfrom pydantic import Field, validator<\/p>\n
class CrawlerSettings(BaseSettings):
\n """\u722c\u866b\u914d\u7f6e\u7c7b，\u4f7f\u7528Pydantic\u8fdb\u884c\u914d\u7f6e\u9a8c\u8bc1"""<\/p>\n
# \u9879\u76ee\u8def\u5f84
\n BASE_DIR: Path = Path(__file__).resolve().parent.parent.parent
\n LOG_DIR: Path = BASE_DIR \/ "logs"
\n DATA_DIR: Path = BASE_DIR \/ "data"<\/p>\n
# \u722c\u866b\u914d\u7f6e
\n USER_AGENTS: List[str] = [
\n "Mozilla\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/120.0.0.0 Safari\/537.36",
\n "Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit\/605.1.15 (KHTML, like Gecko) Version\/17.1 Safari\/605.1.15",
\n "Mozilla\/5.0 (X11; Linux x86_64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/120.0.0.0 Safari\/537.36"
\n ]<\/p>\n
# \u8bf7\u6c42\u914d\u7f6e
\n REQUEST_TIMEOUT: int = 30
\n MAX_RETRIES: int = 3
\n RETRY_DELAY: int = 5
\n CONCURRENT_REQUESTS: int = 10
\n DOWNLOAD_DELAY: float = 1.0<\/p>\n
# \u4ee3\u7406\u914d\u7f6e
\n USE_PROXY: bool = False
\n PROXY_POOL: List[str] = []
\n PROXY_ROTATION_INTERVAL: int = 300 # 5\u5206\u949f\u66f4\u6362\u4e00\u6b21\u4ee3\u7406<\/p>\n
# \u6570\u636e\u5e93\u914d\u7f6e
\n MONGODB_URI: str = Field(default="mongodb:\/\/localhost:27017\/", env="MONGODB_URI")
\n MONGODB_DB: str = "art_database"
\n MONGODB_COLLECTION: str = "artworks"<\/p>\n
POSTGRESQL_CONFIG: Dict = {
\n "host": os.getenv("POSTGRES_HOST", "localhost"),
\n "port": int(os.getenv("POSTGRES_PORT", 5432)),
\n "database": os.getenv("POSTGRES_DB", "art_db"),
\n "user": os.getenv("POSTGRES_USER", "art_user"),
\n "password": os.getenv("POSTGRES_PASSWORD", "art_password")
\n }<\/p>\n
REDIS_CONFIG: Dict = {
\n "host": os.getenv("REDIS_HOST", "localhost"),
\n "port": int(os.getenv("REDIS_PORT", 6379)),
\n "db": int(os.getenv("REDIS_DB", 0)),
\n "password": os.getenv("REDIS_PASSWORD", None)
\n }<\/p>\n
ELASTICSEARCH_CONFIG: Dict = {
\n "hosts": [os.getenv("ELASTICSEARCH_HOST", "http:\/\/localhost:9200")],
\n "index": "artworks"
\n }<\/p>\n
# \u901f\u7387\u9650\u5236\u914d\u7f6e
\n RATE_LIMIT_ENABLED: bool = True
\n DEFAULT_RATE: int = 5 # \u6bcf\u79d2\u8bf7\u6c42\u6570
\n BURST_RATE: int = 10 # \u7a81\u53d1\u8bf7\u6c42\u6570<\/p>\n
# \u722c\u866b\u76ee\u6807\u914d\u7f6e
\n TARGET_SITES: Dict[str, Dict] = {
\n "wikiart": {
\n "base_url": "https:\/\/www.wikiart.org",
\n "enabled": True,
\n "rate_limit": 2, # \u6bcf\u79d22\u4e2a\u8bf7\u6c42
\n "max_pages": 1000,
\n "use_playwright": False
\n },
\n "artsy": {
\n "base_url": "https:\/\/www.artsy.net",
\n "enabled": True,
\n "rate_limit": 1, # API\u9650\u5236\u8f83\u4e25\u683c
\n "max_pages": 500,
\n "use_playwright": True, # \u9700\u8981\u5904\u7406JavaScript
\n "api_key": os.getenv("ARTSY_API_KEY", "")
\n },
\n "met_museum": {
\n "base_url": "https:\/\/collectionapi.metmuseum.org",
\n "enabled": True,
\n "rate_limit": 10, # \u516c\u5171API\u9650\u5236\u8f83\u5bbd\u677e
\n "max_pages": 2000,
\n "use_playwright": False,
\n "api_key": os.getenv("MET_API_KEY", "")
\n }
\n }<\/p>\n
# \u65e5\u5fd7\u914d\u7f6e
\n LOG_LEVEL: str = "INFO"
\n LOG_FORMAT: str = "%(asctime)s – %(name)s – %(levelname)s – %(message)s"
\n LOG_ROTATION: str = "1 day"
\n LOG_RETENTION: str = "30 days"<\/p>\n
# \u6570\u636e\u6e05\u6d17\u914d\u7f6e
\n DEDUPLICATION_ENABLED: bool = True
\n DEDUPLICATION_FIELDS: List[str] = ["title", "artist", "date"]
\n SIMILARITY_THRESHOLD: float = 0.85 # \u76f8\u4f3c\u5ea6\u9608\u503c<\/p>\n
# \u56fe\u50cf\u5904\u7406\u914d\u7f6e
\n DOWNLOAD_IMAGES: bool = True
\n IMAGE_STORAGE_PATH: Path = DATA_DIR \/ "images"
\n THUMBNAIL_SIZE: tuple = (300, 300)
\n ALLOWED_IMAGE_FORMATS: List[str] = ["jpg", "jpeg", "png", "gif", "webp"]<\/p>\n
class Config:
\n env_file = ".env"
\n case_sensitive = True<\/p>\n
@validator("LOG_DIR", "DATA_DIR", "IMAGE_STORAGE_PATH")
\n def create_directories(cls, v):
\n """\u81ea\u52a8\u521b\u5efa\u5fc5\u8981\u7684\u76ee\u5f55"""
\n v.mkdir(parents=True, exist_ok=True)
\n return v<\/p>\n
# \u521b\u5efa\u5168\u5c40\u914d\u7f6e\u5b9e\u4f8b
\nsettings = CrawlerSettings() <\/p>\n
3.3.2 \u5e38\u91cf\u5b9a\u4e49 (src\/config\/constants.py)<\/p>\n
python<\/p>\n
from enum import Enum, auto
\nfrom typing import Dict, Any<\/p>\n
class ArtworkMedium(Enum):
\n """\u827a\u672f\u4f5c\u54c1\u5a92\u4ecb\u679a\u4e3e"""
\n PAINTING = "\u7ed8\u753b"
\n SCULPTURE = "\u96d5\u5851"
\n PHOTOGRAPHY = "\u6444\u5f71"
\n PRINT = "\u7248\u753b"
\n DRAWING = "\u7d20\u63cf"
\n INSTALLATION = "\u88c5\u7f6e"
\n DIGITAL_ART = "\u6570\u5b57\u827a\u672f"
\n MIXED_MEDIA = "\u6df7\u5408\u5a92\u4ecb"
\n CERAMICS = "\u9676\u74f7"
\n TEXTILE = "\u7eba\u7ec7"
\n PERFORMANCE = "\u884c\u4e3a\u827a\u672f"
\n VIDEO_ART = "\u89c6\u9891\u827a\u672f"<\/p>\n
class ArtworkPeriod(Enum):
\n """\u827a\u672f\u65f6\u671f\u679a\u4e3e"""
\n ANCIENT = "\u53e4\u4ee3\u827a\u672f"
\n MEDIEVAL = "\u4e2d\u4e16\u7eaa\u827a\u672f"
\n RENAISSANCE = "\u6587\u827a\u590d\u5174"
\n BAROQUE = "\u5df4\u6d1b\u514b"
\n ROCOCO = "\u6d1b\u53ef\u53ef"
\n NEOCLASSICISM = "\u65b0\u53e4\u5178\u4e3b\u4e49"
\n ROMANTICISM = "\u6d6a\u6f2b\u4e3b\u4e49"
\n REALISM = "\u73b0\u5b9e\u4e3b\u4e49"
\n IMPRESSIONISM = "\u5370\u8c61\u6d3e"
\n POST_IMPRESSIONISM = "\u540e\u5370\u8c61\u6d3e"
\n MODERN = "\u73b0\u4ee3\u827a\u672f"
\n CONTEMPORARY = "\u5f53\u4ee3\u827a\u672f"<\/p>\n
# \u827a\u672f\u6d41\u6d3e\u6620\u5c04\u5b57\u5178
\nART_MOVEMENTS: Dict[str, str] = {
\n "cubism": "\u7acb\u4f53\u4e3b\u4e49",
\n "surrealism": "\u8d85\u73b0\u5b9e\u4e3b\u4e49",
\n "abstract-expressionism": "\u62bd\u8c61\u8868\u73b0\u4e3b\u4e49",
\n "pop-art": "\u6ce2\u666e\u827a\u672f",
\n "minimalism": "\u6781\u7b80\u4e3b\u4e49",
\n "conceptual-art": "\u6982\u5ff5\u827a\u672f",
\n "fauvism": "\u91ce\u517d\u6d3e",
\n "expressionism": "\u8868\u73b0\u4e3b\u4e49",
\n "art-nouveau": "\u65b0\u827a\u672f\u8fd0\u52a8",
\n "bauhaus": "\u5305\u8c6a\u65af",
\n "dadaism": "\u8fbe\u8fbe\u4e3b\u4e49",
\n "futurism": "\u672a\u6765\u4e3b\u4e49"
\n}<\/p>\n
# HTTP\u72b6\u6001\u7801\u542b\u4e49
\nHTTP_STATUS_CODES: Dict[int, str] = {
\n 200: "\u6210\u529f",
\n 201: "\u5df2\u521b\u5efa",
\n 204: "\u65e0\u5185\u5bb9",
\n 301: "\u6c38\u4e45\u91cd\u5b9a\u5411",
\n 302: "\u4e34\u65f6\u91cd\u5b9a\u5411",
\n 400: "\u9519\u8bef\u8bf7\u6c42",
\n 401: "\u672a\u6388\u6743",
\n 403: "\u7981\u6b62\u8bbf\u95ee",
\n 404: "\u672a\u627e\u5230",
\n 429: "\u8bf7\u6c42\u8fc7\u591a",
\n 500: "\u670d\u52a1\u5668\u5185\u90e8\u9519\u8bef",
\n 502: "\u7f51\u5173\u9519\u8bef",
\n 503: "\u670d\u52a1\u4e0d\u53ef\u7528",
\n 504: "\u7f51\u5173\u8d85\u65f6"
\n}<\/p>\n
# \u6570\u636e\u9a8c\u8bc1\u6b63\u5219\u8868\u8fbe\u5f0f
\nREGEX_PATTERNS: Dict[str, str] = {
\n "year": r"^(1[0-9]{3}|2[0-9]{3})$", # 1000-2999\u5e74
\n "dimension": r"^\\\\d+(\\\\.\\\\d+)?\\\\s*[xX]\\\\s*\\\\d+(\\\\.\\\\d+)?\\\\s*(cm|mm|in)$", # \u5c3a\u5bf8\u683c\u5f0f
\n "artist_name": r"^[A-Za-z\\\\s\\\\-'.]+$", # \u827a\u672f\u5bb6\u59d3\u540d（\u7b80\u5316\u7248）
\n "price": r"^\\\\$?\\\\d+(,\\\\d{3})*(\\\\.\\\\d{2})?$" # \u4ef7\u683c\u683c\u5f0f
\n} <\/p>\n
\u4e3a\u4e86\u786e\u4fdd\u5f00\u53d1\u73af\u5883\u7684\u4e00\u81f4\u6027，\u6211\u4eec\u4f7f\u7528Docker\u6765\u7ba1\u7406\u6240\u6709\u4f9d\u8d56\u670d\u52a1：<\/p>\n
yaml<\/p>\n
# docker\/docker-compose.yml
\nversion: '3.8'<\/p>\n
services:
\n mongodb:
\n image: mongo:7.0
\n container_name: art_mongodb
\n restart: always
\n ports:
\n – "27017:27017"
\n environment:
\n MONGO_INITDB_ROOT_USERNAME: admin
\n MONGO_INITDB_ROOT_PASSWORD: admin123
\n MONGO_INITDB_DATABASE: art_database
\n volumes:
\n – .\/mongodb\/data:\/data\/db
\n – .\/mongodb\/init.js:\/docker-entrypoint-initdb.d\/init.js:ro
\n networks:
\n – art_network
\n healthcheck:
\n test: ["CMD", "mongosh", "–eval", "db.adminCommand('ping')"]
\n interval: 30s
\n timeout: 10s
\n retries: 3<\/p>\n
postgres:
\n image: postgres:16
\n container_name: art_postgres
\n restart: always
\n ports:
\n – "5432:5432"
\n environment:
\n POSTGRES_DB: art_db
\n POSTGRES_USER: art_user
\n POSTGRES_PASSWORD: art_password
\n volumes:
\n – .\/postgres\/data:\/var\/lib\/postgresql\/data
\n – .\/postgres\/init.sql:\/docker-entrypoint-initdb.d\/init.sql:ro
\n networks:
\n – art_network
\n healthcheck:
\n test: ["CMD-SHELL", "pg_isready -U art_user"]
\n interval: 30s
\n timeout: 10s
\n retries: 3<\/p>\n
redis:
\n image: redis:7.2
\n container_name: art_redis
\n restart: always
\n ports:
\n – "6379:6379"
\n command: redis-server –requirepass redis_password
\n volumes:
\n – .\/redis\/data:\/data
\n networks:
\n – art_network
\n healthcheck:
\n test: ["CMD", "redis-cli", "ping"]
\n interval: 30s
\n timeout: 10s
\n retries: 3<\/p>\n
elasticsearch:
\n image: elasticsearch:8.11.1
\n container_name: art_elasticsearch
\n restart: always
\n ports:
\n – "9200:9200"
\n environment:
\n – discovery.type=single-node
\n – xpack.security.enabled=false
\n – "ES_JAVA_OPTS=-Xms512m -Xmx512m"
\n volumes:
\n – .\/elasticsearch\/data:\/usr\/share\/elasticsearch\/data
\n networks:
\n – art_network
\n healthcheck:
\n test: ["CMD", "curl", "-f", "http:\/\/localhost:9200"]
\n interval: 30s
\n timeout: 10s
\n retries: 3<\/p>\n
minio:
\n image: minio\/minio:RELEASE.2024-01-13T07-53-03Z
\n container_name: art_minio
\n restart: always
\n ports:
\n – "9000:9000"
\n – "9001:9001"
\n environment:
\n MINIO_ROOT_USER: minioadmin
\n MINIO_ROOT_PASSWORD: minioadmin123
\n volumes:
\n – .\/minio\/data:\/data
\n command: server \/data –console-address ":9001"
\n networks:
\n – art_network
\n healthcheck:
\n test: ["CMD", "curl", "-f", "http:\/\/localhost:9000\/minio\/health\/live"]
\n interval: 30s
\n timeout: 10s
\n retries: 3<\/p>\n
kafka:
\n image: confluentinc\/cp-kafka:latest
\n container_name: art_kafka
\n restart: always
\n ports:
\n – "9092:9092"
\n environment:
\n KAFKA_BROKER_ID: 1
\n KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
\n KAFKA_ADVERTISED_LISTENERS: PLAINTEXT:\/\/localhost:9092
\n KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
\n depends_on:
\n – zookeeper
\n networks:
\n – art_network<\/p>\n
zookeeper:
\n image: confluentinc\/cp-zookeeper:latest
\n container_name: art_zookeeper
\n restart: always
\n ports:
\n – "2181:2181"
\n environment:
\n ZOOKEEPER_CLIENT_PORT: 2181
\n ZOOKEEPER_TICK_TIME: 2000
\n networks:
\n – art_network<\/p>\n
crawler_ui:
\n image: apify\/actor-web-scraper:latest
\n container_name: art_crawler_ui
\n restart: always
\n ports:
\n – "8080:8080"
\n environment:
\n – APIFY_LOCAL_STORAGE_DIR=\/storage
\n volumes:
\n – .\/crawler_ui\/storage:\/storage
\n networks:
\n – art_network<\/p>\n
networks:
\n art_network:
\n driver: bridge <\/p>\n
\u6211\u4eec\u9996\u5148\u8bbe\u8ba1\u4e00\u4e2a\u57fa\u7840\u722c\u866b\u7c7b，\u5c01\u88c5\u901a\u7528\u7684\u722c\u866b\u529f\u80fd：<\/p>\n
python<\/p>\n
# src\/crawlers\/base.py<\/p>\n
import asyncio
\nimport logging
\nimport random
\nfrom abc import ABC, abstractmethod
\nfrom typing import Any, Dict, List, Optional, Union
\nfrom datetime import datetime
\nfrom urllib.parse import urljoin, urlparse<\/p>\n
import aiohttp
\nfrom aiohttp import ClientTimeout, TCPConnector
\nfrom tenacity import (
\n retry,
\n stop_after_attempt,
\n wait_exponential,
\n retry_if_exception_type,
\n before_sleep_log
\n)
\nimport backoff
\nfrom fake_useragent import UserAgent<\/p>\n
from src.config.settings import settings
\nfrom src.utils.logger import setup_logger
\nfrom src.utils.validator import DataValidator
\nfrom src.middlewares.rate_limiter import RateLimiter
\nfrom src.middlewares.proxy import ProxyMiddleware<\/p>\n
class BaseCrawler(ABC):
\n """
\n \u57fa\u7840\u722c\u866b\u7c7b，\u63d0\u4f9b\u901a\u7528\u7684\u722c\u866b\u529f\u80fd
\n \u6240\u6709\u5177\u4f53\u5e73\u53f0\u7684\u722c\u866b\u90fd\u5e94\u8be5\u7ee7\u627f\u6b64\u7c7b
\n """<\/p>\n
def __init__(self, name: str, config: Dict[str, Any]):
\n """
\n \u521d\u59cb\u5316\u57fa\u7840\u722c\u866b<\/p>\n
Args:
\n name: \u722c\u866b\u540d\u79f0
\n config: \u722c\u866b\u914d\u7f6e\u5b57\u5178
\n """
\n self.name = name
\n self.config = config
\n self.logger = setup_logger(f"crawler.{name}")<\/p>\n
# \u521d\u59cb\u5316\u4e2d\u95f4\u4ef6
\n self.rate_limiter = RateLimiter(
\n rate=config.get("rate_limit", settings.DEFAULT_RATE),
\n burst=settings.BURST_RATE
\n )
\n self.proxy_middleware = ProxyMiddleware() if settings.USE_PROXY else None<\/p>\n
# \u521d\u59cb\u5316\u4f1a\u8bdd\u76f8\u5173\u5c5e\u6027
\n self.session: Optional[aiohttp.ClientSession] = None
\n self.ua = UserAgent()<\/p>\n
# \u7edf\u8ba1\u4fe1\u606f
\n self.stats = {
\n "requests_made": 0,
\n "requests_success": 0,
\n "requests_failed": 0,
\n "items_extracted": 0,
\n "start_time": None,
\n "end_time": None
\n }<\/p>\n
# \u521b\u5efa\u8fde\u63a5\u5668
\n self.connector = TCPConnector(
\n limit=settings.CONCURRENT_REQUESTS,
\n ttl_dns_cache=300,
\n ssl=False, # \u67d0\u4e9b\u827a\u672f\u7f51\u7ad9SSL\u8bc1\u4e66\u53ef\u80fd\u6709\u95ee\u9898
\n force_close=True
\n )<\/p>\n
async def __aenter__(self):
\n """\u5f02\u6b65\u4e0a\u4e0b\u6587\u7ba1\u7406\u5668\u5165\u53e3"""
\n await self.start()
\n return self<\/p>\n
async def __aexit__(self, exc_type, exc_val, exc_tb):
\n """\u5f02\u6b65\u4e0a\u4e0b\u6587\u7ba1\u7406\u5668\u51fa\u53e3"""
\n await self.close()<\/p>\n
async def start(self):
\n """\u542f\u52a8\u722c\u866b"""
\n self.logger.info(f"Starting crawler: {self.name}")
\n self.stats["start_time"] = datetime.now()<\/p>\n
# \u521b\u5efaHTTP\u4f1a\u8bdd
\n self.session = aiohttp.ClientSession(
\n connector=self.connector,
\n timeout=ClientTimeout(total=settings.REQUEST_TIMEOUT),
\n headers=self._get_default_headers()
\n )<\/p>\n
async def close(self):
\n """\u5173\u95ed\u722c\u866b"""
\n self.logger.info(f"Closing crawler: {self.name}")
\n self.stats["end_time"] = datetime.now()<\/p>\n
if self.session and not self.session.closed:
\n await self.session.close()<\/p>\n
# \u8bb0\u5f55\u6700\u7ec8\u7edf\u8ba1\u4fe1\u606f
\n duration = (self.stats["end_time"] – self.stats["start_time"]).total_seconds()
\n self.logger.info(
\n f"Crawler stats – "
\n f"Requests: {self.stats['requests_made']}, "
\n f"Success: {self.stats['requests_success']}, "
\n f"Failed: {self.stats['requests_failed']}, "
\n f"Items: {self.stats['items_extracted']}, "
\n f"Duration: {duration:.2f}s"
\n )<\/p>\n
def _get_default_headers(self) -> Dict[str, str]:
\n """\u83b7\u53d6\u9ed8\u8ba4\u8bf7\u6c42\u5934"""
\n return {
\n "User-Agent": self.ua.random,
\n "Accept": "text\/html,application\/xhtml+xml,application\/xml;q=0.9,image\/webp,*\/*;q=0.8",
\n "Accept-Language": "en-US,en;q=0.5",
\n "Accept-Encoding": "gzip, deflate, br",
\n "DNT": "1",
\n "Connection": "keep-alive",
\n "Upgrade-Insecure-Requests": "1",
\n "Sec-Fetch-Dest": "document",
\n "Sec-Fetch-Mode": "navigate",
\n "Sec-Fetch-Site": "none",
\n "Sec-Fetch-User": "?1",
\n "Cache-Control": "max-age=0"
\n }<\/p>\n
async def _get_proxy(self) -> Optional[str]:
\n """\u83b7\u53d6\u4ee3\u7406"""
\n if self.proxy_middleware:
\n return await self.proxy_middleware.get_proxy()
\n return None<\/p>\n
@retry(
\n stop=stop_after_attempt(settings.MAX_RETRIES),
\n wait=wait_exponential(multiplier=1, min=4, max=10),
\n retry=retry_if_exception_type(
\n (aiohttp.ClientError, asyncio.TimeoutError, ConnectionError)
\n ),
\n before_sleep=before_sleep_log(logging.getLogger(), logging.WARNING)
\n )
\n async def fetch(
\n self,
\n url: str,
\n method: str = "GET",
\n params: Optional[Dict] = None,
\n data: Optional[Dict] = None,
\n headers: Optional[Dict] = None,
\n cookies: Optional[Dict] = None,
\n proxy: Optional[str] = None,
\n timeout: Optional[int] = None
\n ) -> Optional[aiohttp.ClientResponse]:
\n """
\n \u53d1\u9001HTTP\u8bf7\u6c42<\/p>\n
Args:
\n url: \u8bf7\u6c42URL
\n method: \u8bf7\u6c42\u65b9\u6cd5
\n params: URL\u53c2\u6570
\n data: \u8bf7\u6c42\u6570\u636e
\n headers: \u8bf7\u6c42\u5934
\n cookies: Cookie
\n proxy: \u4ee3\u7406
\n timeout: \u8d85\u65f6\u65f6\u95f4<\/p>\n
Returns:
\n \u54cd\u5e94\u5bf9\u8c61
\n """
\n # \u901f\u7387\u9650\u5236
\n await self.rate_limiter.wait_if_needed()<\/p>\n
# \u5408\u5e76\u8bf7\u6c42\u5934
\n request_headers = self._get_default_headers()
\n if headers:
\n request_headers.update(headers)<\/p>\n
# \u83b7\u53d6\u4ee3\u7406
\n if not proxy and settings.USE_PROXY:
\n proxy = await self._get_proxy()<\/p>\n
self.stats["requests_made"] += 1<\/p>\n
try:
\n async with self.session.request(
\n method=method,
\n url=url,
\n params=params,
\n json=data if method.upper() == "POST" else None,
\n data=data if method.upper() != "POST" else None,
\n headers=request_headers,
\n cookies=cookies,
\n proxy=proxy,
\n timeout=timeout or settings.REQUEST_TIMEOUT,
\n ssl=False
\n ) as response:<\/p>\n
# \u68c0\u67e5\u54cd\u5e94\u72b6\u6001
\n if response.status == 200:
\n self.stats["requests_success"] += 1
\n return response
\n elif response.status == 429: # \u8bf7\u6c42\u8fc7\u591a
\n retry_after = response.headers.get("Retry-After", "60")
\n self.logger.warning(f"Rate limited. Retry after {retry_after}s")
\n await asyncio.sleep(int(retry_after))
\n raise aiohttp.ClientError("Rate limited")
\n elif response.status >= 500: # \u670d\u52a1\u5668\u9519\u8bef
\n self.logger.error(f"Server error: {response.status}")
\n raise aiohttp.ClientError(f"Server error: {response.status}")
\n else:
\n self.logger.error(f"HTTP error: {response.status} – {url}")
\n self.stats["requests_failed"] += 1
\n return None<\/p>\n
except asyncio.TimeoutError:
\n self.logger.error(f"Timeout error: {url}")
\n self.stats["requests_failed"] += 1
\n raise
\n except aiohttp.ClientError as e:
\n self.logger.error(f"Client error: {str(e)} – {url}")
\n self.stats["requests_failed"] += 1
\n raise
\n except Exception as e:
\n self.logger.error(f"Unexpected error: {str(e)} – {url}")
\n self.stats["requests_failed"] += 1
\n raise<\/p>\n
@abstractmethod
\n async def parse(self, response: aiohttp.ClientResponse) -> List[Dict[str, Any]]:
\n """
\n \u89e3\u6790\u54cd\u5e94（\u62bd\u8c61\u65b9\u6cd5，\u5b50\u7c7b\u5fc5\u987b\u5b9e\u73b0）<\/p>\n
Args:
\n response: \u54cd\u5e94\u5bf9\u8c61<\/p>\n
Returns:
\n \u89e3\u6790\u540e\u7684\u6570\u636e\u5217\u8868
\n """
\n pass<\/p>\n
@abstractmethod
\n async def run(self, *args, **kwargs):
\n """
\n \u8fd0\u884c\u722c\u866b（\u62bd\u8c61\u65b9\u6cd5，\u5b50\u7c7b\u5fc5\u987b\u5b9e\u73b0）
\n """
\n pass<\/p>\n
def validate_data(self, data: Dict[str, Any]) -> bool:
\n """
\n \u9a8c\u8bc1\u6570\u636e\u683c\u5f0f<\/p>\n
Args:
\n data: \u5f85\u9a8c\u8bc1\u7684\u6570\u636e<\/p>\n
Returns:
\n \u662f\u5426\u6709\u6548
\n """
\n return DataValidator.validate_artwork(data)<\/p>\n
async def save_data(self, data: Union[Dict, List[Dict]]):
\n """
\n \u4fdd\u5b58\u6570\u636e\u5230\u7ba1\u9053<\/p>\n
Args:
\n data: \u5f85\u4fdd\u5b58\u7684\u6570\u636e
\n """
\n # \u8fd9\u91cc\u53ef\u4ee5\u53d1\u9001\u5230\u6d88\u606f\u961f\u5217\u6216\u76f4\u63a5\u5b58\u50a8
\n # \u540e\u7eed\u4f1a\u5728\u7ba1\u9053\u4e2d\u5b9e\u73b0
\n pass<\/p>\n
def normalize_url(self, base_url: str, relative_url: str) -> str:
\n """\u89c4\u8303\u5316URL"""
\n return urljoin(base_url, relative_url)<\/p>\n
def extract_domain(self, url: str) -> str:
\n """\u63d0\u53d6URL\u57df\u540d"""
\n parsed = urlparse(url)
\n return parsed.netloc <\/p>\n
4.2.1 \u901f\u7387\u9650\u5236\u5668<\/p>\n
python<\/p>\n
# src\/middlewares\/rate_limiter.py<\/p>\n
import asyncio
\nimport time
\nfrom collections import deque
\nfrom typing import Optional
\nimport threading<\/p>\n
class RateLimiter:
\n """
\n \u901f\u7387\u9650\u5236\u5668，\u4f7f\u7528\u4ee4\u724c\u6876\u7b97\u6cd5
\n \u652f\u6301\u7a81\u53d1\u6d41\u91cf\u548c\u5e73\u5747\u901f\u7387\u63a7\u5236
\n """<\/p>\n
def __init__(self, rate: float, burst: Optional[int] = None):
\n """
\n \u521d\u59cb\u5316\u901f\u7387\u9650\u5236\u5668<\/p>\n
Args:
\n rate: \u5e73\u5747\u901f\u7387（\u8bf7\u6c42\/\u79d2）
\n burst: \u7a81\u53d1\u5bb9\u91cf，\u9ed8\u8ba4\u7b49\u4e8erate
\n """
\n self.rate = rate
\n self.burst = burst or int(rate)
\n self.tokens = self.burst
\n self.last_update = time.monotonic()
\n self.lock = asyncio.Lock()<\/p>\n
async def wait_if_needed(self):
\n """\u5982\u679c\u9700\u8981\u5219\u7b49\u5f85"""
\n async with self.lock:
\n await self._update_tokens()<\/p>\n
if self.tokens < 1:
\n # \u8ba1\u7b97\u9700\u8981\u7b49\u5f85\u7684\u65f6\u95f4
\n wait_time = (1 – self.tokens) \/ self.rate
\n await asyncio.sleep(wait_time)
\n await self._update_tokens()<\/p>\n
self.tokens -= 1<\/p>\n
async def _update_tokens(self):
\n """\u66f4\u65b0\u4ee4\u724c\u6570"""
\n now = time.monotonic()
\n elapsed = now – self.last_update
\n self.tokens = min(
\n self.burst,
\n self.tokens + elapsed * self.rate
\n )
\n self.last_update = now<\/p>\n
class SlidingWindowRateLimiter:
\n """
\n \u6ed1\u52a8\u7a97\u53e3\u901f\u7387\u9650\u5236\u5668
\n \u66f4\u7cbe\u786e\u7684\u901f\u7387\u63a7\u5236，\u9002\u5408\u4e25\u683c\u7684API\u9650\u5236
\n """<\/p>\n
def __init__(self, max_requests: int, window_size: int = 60):
\n """
\n \u521d\u59cb\u5316\u6ed1\u52a8\u7a97\u53e3\u9650\u5236\u5668<\/p>\n
Args:
\n max_requests: \u7a97\u53e3\u5185\u6700\u5927\u8bf7\u6c42\u6570
\n window_size: \u7a97\u53e3\u5927\u5c0f（\u79d2）
\n """
\n self.max_requests = max_requests
\n self.window_size = window_size
\n self.requests = deque()
\n self.lock = asyncio.Lock()<\/p>\n
async def wait_if_needed(self):
\n """\u68c0\u67e5\u5e76\u7b49\u5f85\u76f4\u5230\u53ef\u4ee5\u53d1\u9001\u8bf7\u6c42"""
\n async with self.lock:
\n now = time.time()<\/p>\n
# \u79fb\u9664\u7a97\u53e3\u5916\u7684\u8bf7\u6c42\u8bb0\u5f55
\n while self.requests and self.requests[0] < now – self.window_size:
\n self.requests.popleft()<\/p>\n
# \u5982\u679c\u8fbe\u5230\u9650\u5236，\u7b49\u5f85
\n if len(self.requests) >= self.max_requests:
\n wait_time = self.requests[0] + self.window_size – now
\n if wait_time > 0:
\n await asyncio.sleep(wait_time)
\n # \u7b49\u5f85\u540e\u91cd\u65b0\u6e05\u7406
\n while self.requests and self.requests[0] < now – self.window_size:
\n self.requests.popleft()<\/p>\n
# \u8bb0\u5f55\u5f53\u524d\u8bf7\u6c42
\n self.requests.append(now) <\/p>\n
4.2.2 \u4ee3\u7406\u4e2d\u95f4\u4ef6<\/p>\n
python<\/p>\n
# src\/middlewares\/proxy.py<\/p>\n
import random
\nimport asyncio
\nfrom typing import List, Optional
\nfrom datetime import datetime, timedelta
\nimport aiohttp
\nfrom src.config.settings import settings<\/p>\n
class ProxyMiddleware:
\n """
\n \u4ee3\u7406\u4e2d\u95f4\u4ef6，\u7ba1\u7406\u4ee3\u7406IP\u6c60
\n \u652f\u6301\u4ee3\u7406\u9a8c\u8bc1\u3001\u8f6e\u6362\u548c\u5065\u5eb7\u68c0\u67e5
\n """<\/p>\n
def __init__(self):
\n self.proxies: List[dict] = []
\n self.current_index = 0
\n self.last_rotation = datetime.now()
\n self.lock = asyncio.Lock()<\/p>\n
async def get_proxy(self) -> Optional[str]:
\n """
\n \u83b7\u53d6\u4e00\u4e2a\u53ef\u7528\u7684\u4ee3\u7406<\/p>\n
Returns:
\n \u4ee3\u7406URL\u6216None
\n """
\n async with self.lock:
\n # \u68c0\u67e5\u662f\u5426\u9700\u8981\u8f6e\u6362
\n if self._should_rotate():
\n await self._rotate_proxy()<\/p>\n
if not self.proxies:
\n # \u4ece\u914d\u7f6e\u6587\u4ef6\u6216\u4ee3\u7406\u670d\u52a1\u83b7\u53d6\u4ee3\u7406
\n await self._fetch_proxies()<\/p>\n
if self.proxies:
\n proxy = self.proxies[self.current_index % len(self.proxies)]
\n self.current_index += 1
\n return proxy.get("url")<\/p>\n
return None<\/p>\n
async def report_failure(self, proxy_url: str):
\n """\u62a5\u544a\u4ee3\u7406\u5931\u8d25"""
\n async with self.lock:
\n for proxy in self.proxies:
\n if proxy["url"] == proxy_url:
\n proxy["failures"] = proxy.get("failures", 0) + 1
\n proxy["last_failure"] = datetime.now()<\/p>\n
# \u5982\u679c\u5931\u8d25\u6b21\u6570\u8fc7\u591a，\u6682\u65f6\u7981\u7528
\n if proxy["failures"] >= 3:
\n proxy["disabled_until"] = datetime.now() + timedelta(minutes=5)
\n break<\/p>\n
def _should_rotate(self) -> bool:
\n """\u68c0\u67e5\u662f\u5426\u9700\u8981\u8f6e\u6362\u4ee3\u7406"""
\n return (
\n datetime.now() – self.last_rotation
\n ).seconds > settings.PROXY_ROTATION_INTERVAL<\/p>\n
async def _rotate_proxy(self):
\n """\u8f6e\u6362\u4ee3\u7406"""
\n self.current_index = random.randint(0, max(0, len(self.proxies) – 1))
\n self.last_rotation = datetime.now()<\/p>\n
async def _fetch_proxies(self):
\n """\u4ece\u4ee3\u7406\u670d\u52a1\u83b7\u53d6\u4ee3\u7406\u5217\u8868"""
\n # \u8fd9\u91cc\u53ef\u4ee5\u5b9e\u73b0\u4ece\u514d\u8d39\u4ee3\u7406\u7f51\u7ad9\u6216\u4ed8\u8d39\u4ee3\u7406API\u83b7\u53d6\u4ee3\u7406
\n # \u4f8b\u5982：ProxyBroker\u3001Scrapy-Proxy-Pool\u7b49<\/p>\n
# \u793a\u4f8b：\u4ece\u914d\u7f6e\u6587\u4ef6\u8bfb\u53d6\u4ee3\u7406
\n if settings.PROXY_POOL:
\n for proxy_url in settings.PROXY_POOL:
\n self.proxies.append({
\n "url": proxy_url,
\n "failures": 0,
\n "last_check": None,
\n "disabled_until": None
\n })<\/p>\n
# \u5982\u679c\u914d\u7f6e\u4e2d\u6ca1\u6709\u4ee3\u7406，\u53ef\u4ee5\u4f7f\u7528\u514d\u8d39\u7684\u4ee3\u7406API
\n try:
\n async with aiohttp.ClientSession() as session:
\n # \u793a\u4f8b：\u4ece\u514d\u8d39\u4ee3\u7406API\u83b7\u53d6
\n async with session.get(
\n "https:\/\/api.proxyscrape.com\/v2\/?request=getproxies&protocol=http&timeout=10000&country=all&ssl=all&anonymity=all"
\n ) as response:
\n if response.status == 200:
\n text = await response.text()
\n proxy_list = text.strip().split('\\\\r\\\\n')
\n for proxy in proxy_list[:10]: # \u9650\u5236\u6570\u91cf
\n self.proxies.append({
\n "url": f"http:\/\/{proxy}",
\n "failures": 0,
\n "last_check": datetime.now(),
\n "disabled_until": None
\n })
\n except Exception as e:
\n print(f"Error fetching proxies: {e}")<\/p>\n
async def validate_proxy(self, proxy_url: str) -> bool:
\n """\u9a8c\u8bc1\u4ee3\u7406\u662f\u5426\u53ef\u7528"""
\n try:
\n async with aiohttp.ClientSession() as session:
\n async with session.get(
\n "http:\/\/httpbin.org\/ip",
\n proxy=proxy_url,
\n timeout=10
\n ) as response:
\n if response.status == 200:
\n return True
\n except:
\n pass
\n return False <\/p>\n
4.3.1 WikiArt\u722c\u866b<\/p>\n
python<\/p>\n
# src\/crawlers\/wikiart.py<\/p>\n
import json
\nimport re
\nfrom typing import List, Dict, Any, Optional
\nfrom urllib.parse import urlencode
\nfrom bs4 import BeautifulSoup
\nimport asyncio<\/p>\n
from src.crawlers.base import BaseCrawler
\nfrom src.config.settings import settings<\/p>\n
class WikiArtCrawler(BaseCrawler):
\n """
\n WikiArt\u7f51\u7ad9\u722c\u866b
\n \u91c7\u96c6\u827a\u672f\u5bb6\u548c\u827a\u672f\u4f5c\u54c1\u4fe1\u606f
\n """<\/p>\n
def __init__(self):
\n super().__init__(
\n name="wikiart",
\n config=settings.TARGET_SITES["wikiart"]
\n )
\n self.base_url = self.config["base_url"]
\n self.api_base = "https:\/\/www.wikiart.org\/en\/api\/2"<\/p>\n
async def run(self, max_pages: int = None):
\n """
\n \u8fd0\u884c\u722c\u866b<\/p>\n
Args:
\n max_pages: \u6700\u5927\u722c\u53d6\u9875\u6570
\n """
\n self.logger.info("Starting WikiArt crawler")<\/p>\n
# \u5148\u83b7\u53d6\u827a\u672f\u5bb6\u5217\u8868
\n artists = await self._get_artists(max_pages)<\/p>\n
# \u5e76\u53d1\u83b7\u53d6\u6bcf\u4e2a\u827a\u672f\u5bb6\u7684\u4f5c\u54c1
\n tasks = []
\n for artist in artists[:10]: # \u5148\u9650\u5236\u6570\u91cf\u6d4b\u8bd5
\n task = self._get_artist_artworks(artist)
\n tasks.append(task)<\/p>\n
results = await asyncio.gather(*tasks, return_exceptions=True)<\/p>\n
# \u5904\u7406\u7ed3\u679c
\n all_artworks = []
\n for artist_artworks in results:
\n if isinstance(artist_artworks, Exception):
\n self.logger.error(f"Error getting artworks: {artist_artworks}")
\n else:
\n all_artworks.extend(artist_artworks)<\/p>\n
self.logger.info(f"Collected {len(all_artworks)} artworks")<\/p>\n
# \u4fdd\u5b58\u6570\u636e
\n await self.save_data(all_artworks)<\/p>\n
return all_artworks<\/p>\n
async def _get_artists(self, max_pages: int = None) -> List[Dict]:
\n """
\n \u83b7\u53d6\u827a\u672f\u5bb6\u5217\u8868<\/p>\n
Args:
\n max_pages: \u6700\u5927\u9875\u6570<\/p>\n
Returns:
\n \u827a\u672f\u5bb6\u5217\u8868
\n """
\n artists = []
\n page = 1
\n max_pages = max_pages or self.config.get("max_pages", 100)<\/p>\n
while page <= max_pages:
\n url = f"{self.api_base}\/Artists"
\n params = {
\n "paginationPage": page,
\n "paginationLimit": 100,
\n "imageLimit": 1,
\n "auth": "" # \u53ef\u80fd\u9700\u8981\u8ba4\u8bc1
\n }<\/p>\n
self.logger.info(f"Fetching artists page {page}")<\/p>\n
response = await self.fetch(url, params=params)
\n if not response:
\n break<\/p>\n
try:
\n data = await response.json()
\n items = data.get("data", [])<\/p>\n
if not items:
\n break<\/p>\n
for item in items:
\n artist = {
\n "name": item.get("artistName"),
\n "url": item.get("url"),
\n "birth_day": item.get("birthDayAsString"),
\n "death_day": item.get("deathDayAsString"),
\n "nationality": item.get("nationality"),
\n "art_movement": item.get("artMovement"),
\n "image": item.get("image"),
\n "wikiart_url": f"{self.base_url}{item.get('url')}"
\n }
\n artists.append(artist)<\/p>\n
self.logger.info(f"Found {len(items)} artists on page {page}")<\/p>\n
# \u68c0\u67e5\u662f\u5426\u8fd8\u6709\u66f4\u591a\u9875
\n pagination = data.get("pagination", {})
\n if page >= pagination.get("pages", 0):
\n break<\/p>\n
page += 1<\/p>\n
except json.JSONDecodeError as e:
\n self.logger.error(f"JSON decode error: {e}")
\n break
\n except Exception as e:
\n self.logger.error(f"Error parsing artists: {e}")
\n break<\/p>\n
# \u907f\u514d\u8bf7\u6c42\u8fc7\u5feb
\n await asyncio.sleep(1)<\/p>\n
self.logger.info(f"Total artists collected: {len(artists)}")
\n return artists<\/p>\n
async def _get_artist_artworks(self, artist: Dict) -> List[Dict]:
\n """
\n \u83b7\u53d6\u827a\u672f\u5bb6\u7684\u6240\u6709\u4f5c\u54c1<\/p>\n
Args:
\n artist: \u827a\u672f\u5bb6\u4fe1\u606f<\/p>\n
Returns:
\n \u4f5c\u54c1\u5217\u8868
\n """
\n artworks = []
\n page = 1<\/p>\n
while True:
\n url = f"{self.api_base}\/Artists\/{artist['url']}\/Paintings"
\n params = {
\n "paginationPage": page,
\n "paginationLimit": 100,
\n "json": 2
\n }<\/p>\n
self.logger.info(f"Fetching artworks for {artist['name']} – page {page}")<\/p>\n
response = await self.fetch(url, params=params)
\n if not response:
\n break<\/p>\n
try:
\n data = await response.json()
\n items = data.get("data", [])<\/p>\n
if not items:
\n break<\/p>\n
for item in items:
\n artwork = {
\n "title": item.get("title"),
\n "artist": artist["name"],
\n "artist_url": artist["url"],
\n "date": item.get("completitionYear"),
\n "year": item.get("yearAsString"),
\n "medium": item.get("technique"),
\n "dimensions": f"{item.get('width', '')} x {item.get('height', '')}".strip(),
\n "style": item.get("artStyle"),
\n "genre": item.get("genre"),
\n "series": item.get("series"),
\n "image_url": item.get("image"),
\n "wikiart_url": f"{self.base_url}{item.get('url')}",
\n "description": item.get("description"),
\n "museum": item.get("museum", "Unknown")
\n }<\/p>\n
# \u6570\u636e\u9a8c\u8bc1
\n if self.validate_data(artwork):
\n artworks.append(artwork)
\n self.stats["items_extracted"] += 1<\/p>\n
self.logger.info(f"Found {len(items)} artworks for {artist['name']} on page {page}")<\/p>\n
# \u68c0\u67e5\u662f\u5426\u8fd8\u6709\u66f4\u591a\u9875
\n pagination = data.get("pagination", {})
\n if page >= pagination.get("pages", 0):
\n break<\/p>\n
page += 1<\/p>\n
except json.JSONDecodeError as e:
\n self.logger.error(f"JSON decode error: {e}")
\n break
\n except Exception as e:
\n self.logger.error(f"Error parsing artworks: {e}")
\n break<\/p>\n
await asyncio.sleep(1)<\/p>\n
return artworks<\/p>\n
async def parse(self, response):
\n """
\n \u89e3\u6790HTML\u9875\u9762（\u7528\u4e8e\u76f4\u63a5\u9875\u9762\u89e3\u6790\u6a21\u5f0f）
\n """
\n html = await response.text()
\n soup = BeautifulSoup(html, 'lxml')<\/p>\n
# \u793a\u4f8b：\u89e3\u6790\u4f5c\u54c1\u8be6\u60c5\u9875
\n artwork = {}<\/p>\n
# \u6807\u9898
\n title_elem = soup.find('h1', class_='painting-page-title')
\n if title_elem:
\n artwork['title'] = title_elem.text.strip()<\/p>\n
# \u827a\u672f\u5bb6
\n artist_elem = soup.find('a', class_='artist-name')
\n if artist_elem:
\n artwork['artist'] = artist_elem.text.strip()<\/p>\n
# \u5e74\u4efd
\n year_elem = soup.find('span', itemprop='dateCreated')
\n if year_elem:
\n artwork['year'] = year_elem.text.strip()<\/p>\n
# \u63cf\u8ff0
\n desc_elem = soup.find('div', class_='description')
\n if desc_elem:
\n artwork['description'] = desc_elem.text.strip()<\/p>\n
# \u56fe\u7247
\n img_elem = soup.find('img', class_='painting-image')
\n if img_elem and img_elem.get('src'):
\n artwork['image_url'] = self.normalize_url(
\n response.url,
\n img_elem['src']
\n )<\/p>\n
return [artwork] if artwork else [] <\/p>\n
4.3.2 Artsy\u722c\u866b（\u4f7f\u7528Playwright）<\/p>\n
python<\/p>\n
# src\/crawlers\/artsy.py<\/p>\n
import asyncio
\nimport json
\nfrom typing import List, Dict, Any, Optional
\nfrom playwright.async_api import async_playwright, Browser, Page
\nfrom src.crawlers.base import BaseCrawler
\nfrom src.config.settings import settings<\/p>\n
class ArtsyCrawler(BaseCrawler):
\n """
\n Artsy\u7f51\u7ad9\u722c\u866b
\n \u4f7f\u7528Playwright\u5904\u7406JavaScript\u6e32\u67d3\u7684\u9875\u9762
\n """<\/p>\n
def __init__(self):
\n super().__init__(
\n name="artsy",
\n config=settings.TARGET_SITES["artsy"]
\n )
\n self.base_url = self.config["base_url"]
\n self.browser: Optional[Browser] = None<\/p>\n
async def start(self):
\n """\u542f\u52a8\u722c\u866b，\u521d\u59cb\u5316\u6d4f\u89c8\u5668"""
\n await super().start()<\/p>\n
# \u542f\u52a8Playwright
\n self.playwright = await async_playwright().start()<\/p>\n
# \u542f\u52a8\u6d4f\u89c8\u5668
\n self.browser = await self.playwright.chromium.launch(
\n headless=True, # \u65e0\u5934\u6a21\u5f0f
\n args=[
\n '–disable-blink-features=AutomationControlled',
\n '–disable-dev-shm-usage',
\n '–no-sandbox',
\n '–disable-setuid-sandbox',
\n '–disable-web-security',
\n '–disable-features=IsolateOrigins,site-per-process'
\n ]
\n )<\/p>\n
async def close(self):
\n """\u5173\u95ed\u722c\u866b，\u5173\u95ed\u6d4f\u89c8\u5668"""
\n if self.browser:
\n await self.browser.close()
\n if hasattr(self, 'playwright'):
\n await self.playwright.stop()
\n await super().close()<\/p>\n
async def run(self, max_items: int = 100):
\n """
\n \u8fd0\u884c\u722c\u866b<\/p>\n
Args:
\n max_items: \u6700\u5927\u91c7\u96c6\u6570\u91cf
\n """
\n self.logger.info("Starting Artsy crawler")<\/p>\n
# \u521b\u5efa\u65b0\u9875\u9762
\n page = await self.browser.new_page()<\/p>\n
try:
\n # \u8bbe\u7f6e\u9875\u9762\u89c6\u56fe
\n await page.set_viewport_size({"width": 1920, "height": 1080})<\/p>\n
# \u8bbe\u7f6e\u7528\u6237\u4ee3\u7406\u548c\u8bf7\u6c42\u5934
\n await page.set_extra_http_headers({
\n "Accept-Language": "en-US,en;q=0.9",
\n "Accept-Encoding": "gzip, deflate, br",
\n "Accept": "text\/html,application\/xhtml+xml,application\/xml;q=0.9,image\/webp,*\/*;q=0.8",
\n "Connection": "keep-alive",
\n "Upgrade-Insecure-Requests": "1"
\n })<\/p>\n
# \u8bbf\u95ee\u827a\u672f\u54c1\u5217\u8868\u9875
\n url = f"{self.base_url}\/collect"
\n self.logger.info(f"Navigating to {url}")<\/p>\n
await page.goto(url, wait_until="networkidle")<\/p>\n
# \u7b49\u5f85\u5185\u5bb9\u52a0\u8f7d
\n await page.wait_for_selector(".artwork-item", timeout=10000)<\/p>\n
# \u6eda\u52a8\u52a0\u8f7d\u66f4\u591a\u5185\u5bb9
\n artworks = []
\n previous_height = 0<\/p>\n
while len(artworks) < max_items:
\n # \u83b7\u53d6\u5f53\u524d\u9875\u9762\u7684\u827a\u672f\u54c1
\n page_artworks = await self._extract_artworks_from_page(page)
\n artworks.extend(page_artworks)<\/p>\n
self.logger.info(f"Collected {len(artworks)} artworks so far")<\/p>\n
# \u68c0\u67e5\u662f\u5426\u8fbe\u5230\u76ee\u6807
\n if len(artworks) >= max_items:
\n break<\/p>\n
# \u6eda\u52a8\u5230\u5e95\u90e8\u52a0\u8f7d\u66f4\u591a
\n previous_height = await page.evaluate("document.body.scrollHeight")
\n await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")<\/p>\n
# \u7b49\u5f85\u65b0\u5185\u5bb9\u52a0\u8f7d
\n await page.wait_for_timeout(2000)<\/p>\n
# \u68c0\u67e5\u662f\u5426\u8fd8\u6709\u66f4\u591a\u5185\u5bb9
\n new_height = await page.evaluate("document.body.scrollHeight")
\n if new_height == previous_height:
\n self.logger.info("Reached end of page")
\n break<\/p>\n
# \u83b7\u53d6\u8be6\u60c5\u9875\u4fe1\u606f
\n detailed_artworks = []
\n for artwork in artworks[:max_items]:
\n if artwork.get("href"):
\n detail = await self._get_artwork_detail(artwork["href"])
\n if detail:
\n artwork.update(detail)
\n detailed_artworks.append(artwork)<\/p>\n
self.logger.info(f"Final collected: {len(detailed_artworks)} artworks")<\/p>\n
# \u4fdd\u5b58\u6570\u636e
\n await self.save_data(detailed_artworks)<\/p>\n
return detailed_artworks<\/p>\n
finally:
\n await page.close()<\/p>\n
async def _extract_artworks_from_page(self, page: Page) -> List[Dict]:
\n """
\n \u4ece\u9875\u9762\u63d0\u53d6\u827a\u672f\u54c1\u4fe1\u606f<\/p>\n
Args:
\n page: Playwright\u9875\u9762\u5bf9\u8c61<\/p>\n
Returns:
\n \u827a\u672f\u54c1\u5217\u8868
\n """
\n artworks = []<\/p>\n
# \u4f7f\u7528JavaScript\u5728\u9875\u9762\u4e0a\u4e0b\u6587\u4e2d\u63d0\u53d6\u6570\u636e
\n data = await page.evaluate("""
\n () => {
\n const items = document.querySelectorAll('.artwork-item');
\n const results = [];<\/p>\n
items.forEach(item => {
\n const link = item.querySelector('a');
\n const img = item.querySelector('img');
\n const title = item.querySelector('.artwork-title');
\n const artist = item.querySelector('.artwork-artist');<\/p>\n
results.push({
\n href: link ? link.href : null,
\n image: img ? img.src : null,
\n title: title ? title.textContent.trim() : null,
\n artist: artist ? artist.textContent.trim() : null
\n });
\n });<\/p>\n
return results;
\n }
\n """)<\/p>\n
for item in data:
\n if item.get('href'):
\n artwork = {
\n 'url': item['href'],
\n 'title': item.get('title', ''),
\n 'artist': item.get('artist', ''),
\n 'image_url': item.get('image', ''),
\n 'source': 'artsy'
\n }
\n artworks.append(artwork)<\/p>\n
return artworks<\/p>\n
async def _get_artwork_detail(self, url: str) -> Dict[str, Any]:
\n """
\n \u83b7\u53d6\u827a\u672f\u54c1\u8be6\u60c5<\/p>\n
Args:
\n url: \u8be6\u60c5\u9875URL<\/p>\n
Returns:
\n \u8be6\u60c5\u4fe1\u606f
\n """
\n page = await self.browser.new_page()<\/p>\n
try:
\n await page.goto(url, wait_until="networkidle")<\/p>\n
# \u7b49\u5f85\u4e3b\u8981\u5185\u5bb9\u52a0\u8f7d
\n await page.wait_for_selector(".artwork-detail", timeout=10000)<\/p>\n
# \u63d0\u53d6\u8be6\u7ec6\u4fe1\u606f
\n detail = await page.evaluate("""
\n () => {
\n const detail = {};<\/p>\n
\/\/ \u6807\u9898
\n const title = document.querySelector('.artwork-title');
\n if (title) detail.title = title.textContent.trim();<\/p>\n
\/\/ \u827a\u672f\u5bb6
\n const artist = document.querySelector('.artwork-artist');
\n if (artist) detail.artist = artist.textContent.trim();<\/p>\n
\/\/ \u5e74\u4efd
\n const year = document.querySelector('.artwork-year');
\n if (year) detail.year = year.textContent.trim();<\/p>\n
\/\/ \u6750\u8d28
\n const medium = document.querySelector('.artwork-medium');
\n if (medium) detail.medium = medium.textContent.trim();<\/p>\n
\/\/ \u5c3a\u5bf8
\n const dimensions = document.querySelector('.artwork-dimensions');
\n if (dimensions) detail.dimensions = dimensions.textContent.trim();<\/p>\n
\/\/ \u63cf\u8ff0
\n const description = document.querySelector('.artwork-description');
\n if (description) detail.description = description.textContent.trim();<\/p>\n
\/\/ \u4ef7\u683c
\n const price = document.querySelector('.artwork-price');
\n if (price) detail.price = price.textContent.trim();<\/p>\n
return detail;
\n }
\n """)<\/p>\n
return detail<\/p>\n
except Exception as e:
\n self.logger.error(f"Error getting artwork detail from {url}: {e}")
\n return {}
\n finally:
\n await page.close()<\/p>\n
async def parse(self, response):
\n """\u5b9e\u73b0\u62bd\u8c61\u65b9\u6cd5，\u4f46Artsy\u4f7f\u7528Playwright，\u6b64\u65b9\u6cd5\u4e0d\u4f1a\u76f4\u63a5\u4f7f\u7528"""
\n return [] <\/p>\n
python<\/p>\n
# src\/pipelines\/validation.py<\/p>\n
import re
\nfrom typing import Dict, Any, List, Optional
\nfrom datetime import datetime
\nfrom pydantic import BaseModel, Field, validator, ValidationError
\nfrom src.config.constants import REGEX_PATTERNS<\/p>\n
class ArtworkModel(BaseModel):
\n """
\n \u827a\u672f\u4f5c\u54c1\u6570\u636e\u6a21\u578b（\u4f7f\u7528Pydantic v2）
\n """
\n # \u57fa\u672c\u4fe1\u606f
\n title: str = Field(…, min_length=1, max_length=500)
\n artist: str = Field(…, min_length=1, max_length=200)
\n date: Optional[str] = Field(None, max_length=50)
\n year: Optional[int] = Field(None, ge=1000, le=2025)
\n medium: Optional[str] = Field(None, max_length=200)
\n dimensions: Optional[str] = Field(None, max_length=100)<\/p>\n
# \u5206\u7c7b\u4fe1\u606f
\n style: Optional[str] = Field(None, max_length=100)
\n genre: Optional[str] = Field(None, max_length=100)
\n series: Optional[str] = Field(None, max_length=200)<\/p>\n
# \u6765\u6e90\u4fe1\u606f
\n source: str = Field(…, max_length=50)
\n url: Optional[str] = Field(None, max_length=500)
\n image_url: Optional[str] = Field(None, max_length=500)<\/p>\n
# \u63cf\u8ff0\u4fe1\u606f
\n description: Optional[str] = Field(None, max_length=5000)<\/p>\n
# \u5143\u6570\u636e
\n created_at: datetime = Field(default_factory=datetime.now)
\n updated_at: datetime = Field(default_factory=datetime.now)<\/p>\n
@validator('title')
\n def validate_title(cls, v):
\n """\u9a8c\u8bc1\u6807\u9898"""
\n if not v or not v.strip():
\n raise ValueError('Title cannot be empty')
\n return v.strip()<\/p>\n
@validator('artist')
\n def validate_artist(cls, v):
\n """\u9a8c\u8bc1\u827a\u672f\u5bb6"""
\n if not v or not v.strip():
\n raise ValueError('Artist cannot be empty')
\n return v.strip()<\/p>\n
@validator('year')
\n def validate_year(cls, v):
\n """\u9a8c\u8bc1\u5e74\u4efd"""
\n if v is not None:
\n current_year = datetime.now().year
\n if v < 1000 or v > current_year + 5:
\n raise ValueError(f'Year must be between 1000 and {current_year + 5}')
\n return v<\/p>\n
@validator('dimensions')
\n def validate_dimensions(cls, v):
\n """\u9a8c\u8bc1\u5c3a\u5bf8\u683c\u5f0f"""
\n if v:
\n # \u68c0\u67e5\u5c3a\u5bf8\u683c\u5f0f：\u6570\u5b57 x \u6570\u5b57 \u5355\u4f4d
\n pattern = r'^\\\\d+(\\\\.\\\\d+)?\\\\s*[xX]\\\\s*\\\\d+(\\\\.\\\\d+)?\\\\s*(cm|mm|in|inches)?$'
\n if not re.match(pattern, v.strip()):
\n # \u5982\u679c\u4e0d\u662f\u6807\u51c6\u683c\u5f0f，\u8bb0\u5f55\u4f46\u4e0d\u629b\u51fa\u5f02\u5e38
\n pass
\n return v<\/p>\n
@validator('image_url')
\n def validate_image_url(cls, v):
\n """\u9a8c\u8bc1\u56fe\u7247URL"""
\n if v:
\n url_pattern = r'^https?:\/\/.+\\\\.(jpg|jpeg|png|gif|webp)(\\\\?.*)?$'
\n if not re.match(url_pattern, v.lower()):
\n # \u5982\u679c\u4e0d\u662f\u56fe\u7247URL，\u8bb0\u5f55\u4f46\u4e0d\u629b\u51fa\u5f02\u5e38
\n pass
\n return v<\/p>\n
class DataValidator:
\n """\u6570\u636e\u9a8c\u8bc1\u5668"""<\/p>\n
@staticmethod
\n def validate_artwork(data: Dict[str, Any]) -> bool:
\n """
\n \u9a8c\u8bc1\u827a\u672f\u4f5c\u54c1\u6570\u636e<\/p>\n
Args:
\n data: \u5f85\u9a8c\u8bc1\u7684\u6570\u636e<\/p>\n
Returns:
\n \u662f\u5426\u6709\u6548
\n """
\n try:
\n # \u4f7f\u7528Pydantic\u6a21\u578b\u9a8c\u8bc1
\n artwork = ArtworkModel(**data)
\n return True
\n except ValidationError as e:
\n print(f"Validation error: {e}")
\n return False
\n except Exception as e:
\n print(f"Unexpected error: {e}")
\n return False<\/p>\n
@staticmethod
\n def validate_batch(artworks: List[Dict]) -> tuple[List[Dict], List[Dict]]:
\n """
\n \u6279\u91cf\u9a8c\u8bc1<\/p>\n
Args:
\n artworks: \u827a\u672f\u54c1\u6570\u636e\u5217\u8868<\/p>\n
Returns:
\n (\u6709\u6548\u6570\u636e, \u65e0\u6548\u6570\u636e)
\n """
\n valid = []
\n invalid = []<\/p>\n
for artwork in artworks:
\n if DataValidator.validate_artwork(artwork):
\n valid.append(artwork)
\n else:
\n invalid.append(artwork)<\/p>\n
return valid, invalid <\/p>\n
python<\/p>\n
# src\/pipelines\/deduplication.py<\/p>\n
import hashlib
\nimport json
\nfrom typing import Dict, List, Set, Optional
\nfrom datetime import datetime, timedelta
\nfrom difflib import SequenceMatcher
\nimport redis.asyncio as redis
\nfrom src.config.settings import settings<\/p>\n
class DeduplicationPipeline:
\n """
\n \u6570\u636e\u53bb\u91cd\u7ba1\u9053
\n \u4f7f\u7528Redis\u5b58\u50a8\u6307\u7eb9，\u652f\u6301\u591a\u79cd\u53bb\u91cd\u7b56\u7565
\n """<\/p>\n
def __init__(self):
\n self.redis_client = None
\n self.seen_keys = set()
\n self.fingerprint_cache = {}<\/p>\n
async def initialize(self):
\n """\u521d\u59cb\u5316Redis\u8fde\u63a5"""
\n self.redis_client = redis.Redis(
\n host=settings.REDIS_CONFIG['host'],
\n port=settings.REDIS_CONFIG['port'],
\n db=settings.REDIS_CONFIG['db'],
\n password=settings.REDIS_CONFIG.get('password'),
\n decode_responses=True
\n )<\/p>\n
async def close(self):
\n """\u5173\u95edRedis\u8fde\u63a5"""
\n if self.redis_client:
\n await self.redis_client.close()<\/p>\n
def generate_fingerprint(self, artwork: Dict, fields: List[str]) -> str:
\n """
\n \u751f\u6210\u6570\u636e\u6307\u7eb9<\/p>\n
Args:
\n artwork: \u827a\u672f\u54c1\u6570\u636e
\n fields: \u7528\u4e8e\u751f\u6210\u6307\u7eb9\u7684\u5b57\u6bb5<\/p>\n
Returns:
\n \u6307\u7eb9\u5b57\u7b26\u4e32
\n """
\n # \u63d0\u53d6\u6307\u5b9a\u5b57\u6bb5\u7684\u503c
\n values = []
\n for field in fields:
\n value = artwork.get(field, '')
\n if value:
\n # \u6e05\u7406\u548c\u89c4\u8303\u5316
\n value = str(value).lower().strip()
\n values.append(value)<\/p>\n
# \u7ec4\u5408\u5e76\u751f\u6210\u54c8\u5e0c
\n combined = '|'.join(values)
\n return hashlib.sha256(combined.encode()).hexdigest()<\/p>\n
async def is_duplicate(self, artwork: Dict) -> bool:
\n """
\n \u68c0\u67e5\u662f\u5426\u91cd\u590d<\/p>\n
Args:
\n artwork: \u827a\u672f\u54c1\u6570\u636e<\/p>\n
Returns:
\n \u662f\u5426\u91cd\u590d
\n """
\n if not settings.DEDUPLICATION_ENABLED:
\n return False<\/p>\n
# \u4f7f\u7528\u914d\u7f6e\u7684\u5b57\u6bb5\u751f\u6210\u6307\u7eb9
\n fingerprint = self.generate_fingerprint(
\n artwork,
\n settings.DEDUPLICATION_FIELDS
\n )<\/p>\n
# \u68c0\u67e5Redis\u4e2d\u662f\u5426\u5b58\u5728
\n if self.redis_client:
\n exists = await self.redis_client.exists(f"artwork:{fingerprint}")
\n if exists:
\n return True<\/p>\n
# \u68c0\u67e5\u5185\u5b58\u7f13\u5b58
\n if fingerprint in self.fingerprint_cache:
\n cache_time = self.fingerprint_cache[fingerprint]
\n if datetime.now() – cache_time < timedelta(hours=24):
\n return True<\/p>\n
return False<\/p>\n
async def mark_as_seen(self, artwork: Dict):
\n """
\n \u6807\u8bb0\u6570\u636e\u4e3a\u5df2\u5904\u7406<\/p>\n
Args:
\n artwork: \u827a\u672f\u54c1\u6570\u636e
\n """
\n fingerprint = self.generate_fingerprint(
\n artwork,
\n settings.DEDUPLICATION_FIELDS
\n )<\/p>\n
# \u5b58\u5165Redis
\n if self.redis_client:
\n await self.redis_client.setex(
\n f"artwork:{fingerprint}",
\n 86400 * 30, # \u4fdd\u755930\u5929
\n json.dumps(artwork, default=str)
\n )<\/p>\n
# \u66f4\u65b0\u5185\u5b58\u7f13\u5b58
\n self.fingerprint_cache[fingerprint] = datetime.now()<\/p>\n
def calculate_similarity(self, artwork1: Dict, artwork2: Dict) -> float:
\n """
\n \u8ba1\u7b97\u4e24\u4e2a\u827a\u672f\u54c1\u7684\u76f8\u4f3c\u5ea6<\/p>\n
Args:
\n artwork1: \u827a\u672f\u54c11
\n artwork2: \u827a\u672f\u54c12<\/p>\n
Returns:
\n \u76f8\u4f3c\u5ea6\u5206\u6570（0-1）
\n """
\n scores = []
\n weights = {
\n 'title': 0.3,
\n 'artist': 0.3,
\n 'date': 0.2,
\n 'medium': 0.1,
\n 'dimensions': 0.1
\n }<\/p>\n
for field, weight in weights.items():
\n value1 = str(artwork1.get(field, '')).lower()
\n value2 = str(artwork2.get(field, '')).lower()<\/p>\n
if value1 and value2:
\n similarity = SequenceMatcher(None, value1, value2).ratio()
\n scores.append(similarity * weight)<\/p>\n
return sum(scores) if scores else 0<\/p>\n
async def find_similar(self, artwork: Dict, threshold: float = None) -> List[Dict]:
\n """
\n \u67e5\u627e\u76f8\u4f3c\u7684\u827a\u672f\u54c1<\/p>\n
Args:
\n artwork: \u76ee\u6807\u827a\u672f\u54c1
\n threshold: \u76f8\u4f3c\u5ea6\u9608\u503c<\/p>\n
Returns:
\n \u76f8\u4f3c\u827a\u672f\u54c1\u5217\u8868
\n """
\n if threshold is None:
\n threshold = settings.SIMILARITY_THRESHOLD<\/p>\n
similar = []<\/p>\n
# \u8fd9\u91cc\u53ef\u4ee5\u4ece\u6570\u636e\u5e93\u67e5\u8be2\u76f8\u4f3c\u6570\u636e
\n # \u793a\u4f8b\u4ee3\u7801\u7701\u7565\u5b9e\u9645\u6570\u636e\u5e93\u67e5\u8be2<\/p>\n
return similar<\/p>\n
async def process_batch(self, artworks: List[Dict]) -> tuple[List[Dict], List[Dict]]:
\n """
\n \u6279\u91cf\u5904\u7406\u53bb\u91cd<\/p>\n
Args:
\n artworks: \u827a\u672f\u54c1\u5217\u8868<\/p>\n
Returns:
\n (\u65b0\u6570\u636e, \u91cd\u590d\u6570\u636e)
\n """
\n new_artworks = []
\n duplicate_artworks = []<\/p>\n
for artwork in artworks:
\n if await self.is_duplicate(artwork):
\n duplicate_artworks.append(artwork)
\n else:
\n await self.mark_as_seen(artwork)
\n new_artworks.append(artwork)<\/p>\n
self.logger.info(
\n f"Deduplication: {len(new_artworks)} new, "
\n f"{len(duplicate_artworks)} duplicates"
\n )<\/p>\n
return new_artworks, duplicate_artworks <\/p>\n
python<\/p>\n
# src\/pipelines\/enrichment.py<\/p>\n
import re
\nfrom typing import Dict, Any, Optional
\nfrom datetime import datetime
\nimport spacy
\nfrom PIL import Image
\nimport requests
\nfrom io import BytesIO
\nimport exifread
\nfrom transformers import pipeline
\nfrom src.config.constants import ART_MOVEMENTS<\/p>\n
class DataEnrichmentPipeline:
\n """
\n \u6570\u636e\u589e\u5f3a\u7ba1\u9053
\n \u4f7f\u7528NLP\u548c\u56fe\u50cf\u5904\u7406\u6280\u672f\u4e30\u5bcc\u6570\u636e
\n """<\/p>\n
def __init__(self):
\n # \u52a0\u8f7dNLP\u6a21\u578b
\n try:
\n self.nlp = spacy.load("en_core_web_sm")
\n except:
\n # \u5982\u679c\u6a21\u578b\u4e0d\u5b58\u5728，\u4e0b\u8f7d
\n import subprocess
\n subprocess.run(["python", "-m", "spacy", "download", "en_core_web_sm"])
\n self.nlp = spacy.load("en_core_web_sm")<\/p>\n
# \u521d\u59cb\u5316\u56fe\u50cf\u63cf\u8ff0\u6a21\u578b
\n self.image_captioner = pipeline(
\n "image-to-text",
\n model="nlpconnect\/vit-gpt2-image-captioning"
\n )<\/p>\n
# \u521d\u59cb\u5316\u5b9e\u4f53\u8bc6\u522b\u6a21\u578b
\n self.ner = pipeline(
\n "ner",
\n model="dbmdz\/bert-large-cased-finetuned-conll03-english"
\n )<\/p>\n
def enrich_artwork(self, artwork: Dict) -> Dict:
\n """
\n \u589e\u5f3a\u827a\u672f\u54c1\u6570\u636e<\/p>\n
Args:
\n artwork: \u539f\u59cb\u827a\u672f\u54c1\u6570\u636e<\/p>\n
Returns:
\n \u589e\u5f3a\u540e\u7684\u6570\u636e
\n """
\n enriched = artwork.copy()<\/p>\n
# 1. \u4ece\u63cf\u8ff0\u4e2d\u63d0\u53d6\u5b9e\u4f53
\n if artwork.get('description'):
\n entities = self._extract_entities(artwork['description'])
\n enriched['extracted_entities'] = entities<\/p>\n
# 2. \u89e3\u6790\u548c\u89c4\u8303\u5316\u65e5\u671f
\n if artwork.get('date'):
\n normalized_date = self._normalize_date(artwork['date'])
\n if normalized_date:
\n enriched['normalized_date'] = normalized_date<\/p>\n
# 3. \u5206\u7c7b\u827a\u672f\u6d41\u6d3e
\n if artwork.get('style'):
\n movement = self._classify_movement(artwork['style'])
\n if movement:
\n enriched['art_movement'] = movement<\/p>\n
# 4. \u4ece\u56fe\u50cf\u4e2d\u63d0\u53d6\u4fe1\u606f
\n if artwork.get('image_url'):
\n image_info = self._extract_image_info(artwork['image_url'])
\n if image_info:
\n enriched['image_metadata'] = image_info<\/p>\n
# 5. \u751f\u6210\u6807\u7b7e
\n tags = self._generate_tags(artwork)
\n if tags:
\n enriched['tags'] = tags<\/p>\n
# 6. \u6dfb\u52a0\u65f6\u95f4\u6233
\n enriched['enriched_at'] = datetime.now().isoformat()<\/p>\n
return enriched<\/p>\n
def _extract_entities(self, text: str) -> Dict[str, list]:
\n """
\n \u4ece\u6587\u672c\u4e2d\u63d0\u53d6\u5b9e\u4f53<\/p>\n
Args:
\n text: \u6587\u672c\u5185\u5bb9<\/p>\n
Returns:
\n \u5b9e\u4f53\u5b57\u5178
\n """
\n doc = self.nlp(text)<\/p>\n
entities = {
\n 'persons': [],
\n 'organizations': [],
\n 'locations': [],
\n 'dates': [],
\n 'artworks': []
\n }<\/p>\n
for ent in doc.ents:
\n if ent.label_ == 'PERSON':
\n entities['persons'].append(ent.text)
\n elif ent.label_ == 'ORG':
\n entities['organizations'].append(ent.text)
\n elif ent.label_ == 'GPE' or ent.label_ == 'LOC':
\n entities['locations'].append(ent.text)
\n elif ent.label_ == 'DATE':
\n entities['dates'].append(ent.text)
\n elif ent.label_ == 'WORK_OF_ART':
\n entities['artworks'].append(ent.text)<\/p>\n
return entities<\/p>\n
def _normalize_date(self, date_str: str) -> Optional[Dict]:
\n """
\n \u89c4\u8303\u5316\u65e5\u671f\u683c\u5f0f<\/p>\n
Args:
\n date_str: \u539f\u59cb\u65e5\u671f\u5b57\u7b26\u4e32<\/p>\n
Returns:
\n \u89c4\u8303\u5316\u540e\u7684\u65e5\u671f\u4fe1\u606f
\n """
\n date_info = {}<\/p>\n
# \u5339\u914d\u5e74\u4efd
\n year_pattern = r'\\\\b(1[0-9]{3}|2[0-9]{3})\\\\b'
\n years = re.findall(year_pattern, date_str)<\/p>\n
if years:
\n date_info['years'] = [int(y) for y in years]<\/p>\n
# \u5339\u914d\u4e16\u7eaa
\n century_pattern = r'(\\\\d+)(st|nd|rd|th)\\\\s+century'
\n century_match = re.search(century_pattern, date_str.lower())
\n if century_match:
\n century = int(century_match.group(1))
\n date_info['century'] = century
\n date_info['century_range'] = [
\n (century – 1) * 100,
\n century * 100 – 1
\n ]<\/p>\n
# \u5339\u914d\u8303\u56f4
\n range_pattern = r'(\\\\d{4})\\\\s*[-\u2013\u2014]\\\\s*(\\\\d{4})'
\n range_match = re.search(range_pattern, date_str)
\n if range_match:
\n date_info['date_range'] = [
\n int(range_match.group(1)),
\n int(range_match.group(2))
\n ]<\/p>\n
# \u5c1d\u8bd5\u89e3\u6790\u5177\u4f53\u65e5\u671f
\n try:
\n # \u5c1d\u8bd5\u591a\u79cd\u65e5\u671f\u683c\u5f0f
\n for fmt in ['%Y', '%Y-%m-%d', '%d %B %Y', '%B %d, %Y']:
\n try:
\n parsed_date = datetime.strptime(date_str, fmt)
\n date_info['iso_date'] = parsed_date.isoformat()
\n break
\n except:
\n continue
\n except:
\n pass<\/p>\n
return date_info if date_info else None<\/p>\n
def _classify_movement(self, style: str) -> Optional[str]:
\n """
\n \u5206\u7c7b\u827a\u672f\u6d41\u6d3e<\/p>\n
Args:
\n style: \u827a\u672f\u98ce\u683c<\/p>\n
Returns:
\n \u89c4\u8303\u5316\u540e\u7684\u6d41\u6d3e
\n """
\n style_lower = style.lower()<\/p>\n
# \u76f4\u63a5\u5339\u914d
\n for eng, chn in ART_MOVEMENTS.items():
\n if eng in style_lower or chn in style_lower:
\n return chn<\/p>\n
# \u6a21\u7cca\u5339\u914d
\n for eng, chn in ART_MOVEMENTS.items():
\n if any(word in style_lower for word in eng.split('-')):
\n return chn<\/p>\n
return None<\/p>\n
def _extract_image_info(self, image_url: str) -> Optional[Dict]:
\n """
\n \u4ece\u56fe\u50cf\u4e2d\u63d0\u53d6\u4fe1\u606f<\/p>\n
Args:
\n image_url: \u56fe\u50cfURL<\/p>\n
Returns:
\n \u56fe\u50cf\u4fe1\u606f
\n """
\n try:
\n # \u4e0b\u8f7d\u56fe\u50cf
\n response = requests.get(image_url, timeout=10)
\n img = Image.open(BytesIO(response.content))<\/p>\n
image_info = {
\n 'format': img.format,
\n 'mode': img.mode,
\n 'size': img.size,
\n 'width': img.width,
\n 'height': img.height,
\n 'aspect_ratio': round(img.width \/ img.height, 2)
\n }<\/p>\n
# \u63d0\u53d6EXIF\u6570\u636e
\n if hasattr(img, '_getexif') and img._getexif():
\n exif_data = {}
\n tags = exifread.process_file(BytesIO(response.content))
\n for tag, value in tags.items():
\n exif_data[tag] = str(value)
\n image_info['exif'] = exif_data<\/p>\n
# \u751f\u6210\u56fe\u50cf\u63cf\u8ff0
\n if img.size[0] * img.size[1] < 1000000: # \u9650\u5236\u5927\u5c0f\u907f\u514d\u5185\u5b58\u95ee\u9898
\n try:
\n caption = self.image_captioner(img)
\n if caption:
\n image_info['ai_description'] = caption[0]['generated_text']
\n except:
\n pass<\/p>\n
return image_info<\/p>\n
except Exception as e:
\n print(f"Error extracting image info: {e}")
\n return None<\/p>\n
def _generate_tags(self, artwork: Dict) -> List[str]:
\n """
\n \u751f\u6210\u6807\u7b7e<\/p>\n
Args:
\n artwork: \u827a\u672f\u54c1\u6570\u636e<\/p>\n
Returns:
\n \u6807\u7b7e\u5217\u8868
\n """
\n tags = set()<\/p>\n
# \u57fa\u4e8e\u6807\u9898\u751f\u6210\u6807\u7b7e
\n if artwork.get('title'):
\n title_words = re.findall(r'\\\\w+', artwork['title'].lower())
\n tags.update(title_words[:5]) # \u53d6\u524d5\u4e2a\u8bcd<\/p>\n
# \u57fa\u4e8e\u827a\u672f\u5bb6
\n if artwork.get('artist'):
\n artist_parts = artwork['artist'].lower().split()
\n tags.update(artist_parts[:3])<\/p>\n
# \u57fa\u4e8e\u98ce\u683c
\n if artwork.get('style'):
\n style_words = artwork['style'].lower().split()
\n tags.update(style_words[:3])<\/p>\n
# \u57fa\u4e8e\u6d41\u6d3e
\n if artwork.get('genre'):
\n tags.add(artwork['genre'].lower())<\/p>\n
# \u8fc7\u6ee4\u6389\u592a\u77ed\u7684\u8bcd
\n tags = {tag for tag in tags if len(tag) > 2}<\/p>\n
return list(tags)[:10] # \u6700\u591a\u8fd4\u56de10\u4e2a\u6807\u7b7e <\/p>\n
python<\/p>\n
# src\/pipelines\/storage.py<\/p>\n
import asyncio
\nimport json
\nfrom typing import Dict, List, Any, Optional
\nfrom datetime import datetime
\nfrom motor.motor_asyncio import AsyncIOMotorClient
\nfrom elasticsearch import AsyncElasticsearch
\nimport asyncpg
\nfrom minio import Minio
\nimport io<\/p>\n
from src.config.settings import settings<\/p>\n
class StoragePipeline:
\n """
\n \u6570\u636e\u5b58\u50a8\u7ba1\u9053
\n \u652f\u6301\u591a\u6570\u636e\u5e93\u5b58\u50a8
\n """<\/p>\n
def __init__(self):
\n self.mongo_client = None
\n self.mongo_db = None
\n self.pg_pool = None
\n self.es_client = None
\n self.minio_client = None<\/p>\n
async def initialize(self):
\n """\u521d\u59cb\u5316\u6240\u6709\u6570\u636e\u5e93\u8fde\u63a5"""
\n # MongoDB\u8fde\u63a5
\n self.mongo_client = AsyncIOMotorClient(settings.MONGODB_URI)
\n self.mongo_db = self.mongo_client[settings.MONGODB_DB]<\/p>\n
# PostgreSQL\u8fde\u63a5\u6c60
\n self.pg_pool = await asyncpg.create_pool(**settings.POSTGRESQL_CONFIG)<\/p>\n
# Elasticsearch\u8fde\u63a5
\n self.es_client = AsyncElasticsearch(hosts=settings.ELASTICSEARCH_CONFIG['hosts'])<\/p>\n
# MinIO\u8fde\u63a5
\n self.minio_client = Minio(
\n 'localhost:9000',
\n access_key='minioadmin',
\n secret_key='minioadmin123',
\n secure=False
\n )<\/p>\n
# \u786e\u4fddbucket\u5b58\u5728
\n if not self.minio_client.bucket_exists('art-images'):
\n self.minio_client.make_bucket('art-images')<\/p>\n
# \u521b\u5efa\u6570\u636e\u5e93\u8868
\n await self._init_postgres_tables()<\/p>\n
# \u521b\u5efaElasticsearch\u7d22\u5f15
\n await self._init_elasticsearch_index()<\/p>\n
async def close(self):
\n """\u5173\u95ed\u6240\u6709\u6570\u636e\u5e93\u8fde\u63a5"""
\n if self.mongo_client:
\n self.mongo_client.close()<\/p>\n
if self.pg_pool:
\n await self.pg_pool.close()<\/p>\n
if self.es_client:
\n await self.es_client.close()<\/p>\n
async def _init_postgres_tables(self):
\n """\u521d\u59cb\u5316PostgreSQL\u8868"""
\n async with self.pg_pool.acquire() as conn:
\n # \u521b\u5efa\u827a\u672f\u5bb6\u8868
\n await conn.execute('''
\n CREATE TABLE IF NOT EXISTS artists (
\n id SERIAL PRIMARY KEY,
\n name VARCHAR(200) NOT NULL,
\n birth_year INTEGER,
\n death_year INTEGER,
\n nationality VARCHAR(100),
\n biography TEXT,
\n created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
\n updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
\n UNIQUE(name)
\n )
\n ''')<\/p>\n
# \u521b\u5efa\u827a\u672f\u54c1\u8868
\n await conn.execute('''
\n CREATE TABLE IF NOT EXISTS artworks (
\n id SERIAL PRIMARY KEY,
\n title VARCHAR(500) NOT NULL,
\n artist_id INTEGER REFERENCES artists(id),
\n year INTEGER,
\n medium VARCHAR(200),
\n dimensions VARCHAR(100),
\n style VARCHAR(100),
\n genre VARCHAR(100),
\n description TEXT,
\n image_url TEXT,
\n source VARCHAR(50),
\n source_url TEXT,
\n created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
\n updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
\n )
\n ''')<\/p>\n
# \u521b\u5efa\u7d22\u5f15
\n await conn.execute('CREATE INDEX IF NOT EXISTS idx_artworks_artist ON artworks(artist_id)')
\n await conn.execute('CREATE INDEX IF NOT EXISTS idx_artworks_year ON artworks(year)')
\n await conn.execute('CREATE INDEX IF NOT EXISTS idx_artworks_style ON artworks(style)')<\/p>\n
async def _init_elasticsearch_index(self):
\n """\u521d\u59cb\u5316Elasticsearch\u7d22\u5f15"""
\n index_name = settings.ELASTICSEARCH_CONFIG['index']<\/p>\n
# \u68c0\u67e5\u7d22\u5f15\u662f\u5426\u5b58\u5728
\n exists = await self.es_client.indices.exists(index=index_name)<\/p>\n
if not exists:
\n # \u521b\u5efa\u7d22\u5f15\u6620\u5c04
\n mappings = {
\n "mappings": {
\n "properties": {
\n "title": {"type": "text", "analyzer": "standard"},
\n "artist": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
\n "description": {"type": "text", "analyzer": "standard"},
\n "year": {"type": "integer"},
\n "style": {"type": "keyword"},
\n "genre": {"type": "keyword"},
\n "medium": {"type": "keyword"},
\n "tags": {"type": "keyword"},
\n "created_at": {"type": "date"}
\n }
\n },
\n "settings": {
\n "number_of_shards": 2,
\n "number_of_replicas": 1,
\n "analysis": {
\n "analyzer": {
\n "art_analyzer": {
\n "type": "custom",
\n "tokenizer": "standard",
\n "filter": ["lowercase", "stop", "snowball"]
\n }
\n }
\n }
\n }
\n }<\/p>\n
await self.es_client.indices.create(
\n index=index_name,
\n body=mappings
\n )<\/p>\n
async def store_artwork(self, artwork: Dict) -> str:
\n """
\n \u5b58\u50a8\u5355\u4e2a\u827a\u672f\u54c1<\/p>\n
Args:
\n artwork: \u827a\u672f\u54c1\u6570\u636e<\/p>\n
Returns:
\n \u5b58\u50a8ID
\n """
\n artwork_id = None<\/p>\n
# 1. \u5b58\u50a8\u5230MongoDB
\n mongo_result = await self.mongo_db.artworks.insert_one(artwork)
\n artwork_id = str(mongo_result.inserted_id)<\/p>\n
# 2. \u5b58\u50a8\u5230PostgreSQL
\n await self._store_to_postgres(artwork)<\/p>\n
# 3. \u7d22\u5f15\u5230Elasticsearch
\n await self._index_to_elasticsearch(artwork, artwork_id)<\/p>\n
# 4. \u5b58\u50a8\u56fe\u7247
\n if artwork.get('image_url'):
\n await self._store_image(artwork['image_url'], artwork_id)<\/p>\n
return artwork_id<\/p>\n
async def store_batch(self, artworks: List[Dict]) -> List[str]:
\n """
\n \u6279\u91cf\u5b58\u50a8\u827a\u672f\u54c1<\/p>\n
Args:
\n artworks: \u827a\u672f\u54c1\u5217\u8868<\/p>\n
Returns:
\n \u5b58\u50a8ID\u5217\u8868
\n """
\n if not artworks:
\n return []<\/p>\n
# \u6279\u91cf\u63d2\u5165MongoDB
\n result = await self.mongo_db.artworks.insert_many(artworks)
\n ids = [str(id) for id in result.inserted_ids]<\/p>\n
# \u6279\u91cf\u63d2\u5165PostgreSQL
\n await self._store_batch_to_postgres(artworks)<\/p>\n
# \u6279\u91cf\u7d22\u5f15Elasticsearch
\n await self._index_batch_to_elasticsearch(artworks, ids)<\/p>\n
return ids<\/p>\n
async def _store_to_postgres(self, artwork: Dict):
\n """\u5b58\u50a8\u5230PostgreSQL"""
\n async with self.pg_pool.acquire() as conn:
\n async with conn.transaction():
\n # \u5148\u5904\u7406\u827a\u672f\u5bb6
\n artist_id = await self._get_or_create_artist(conn, artwork.get('artist'))<\/p>\n
# \u63d2\u5165\u827a\u672f\u54c1
\n await conn.execute('''
\n INSERT INTO artworks
\n (title, artist_id, year, medium, dimensions, style, genre,
\n description, image_url, source, source_url)
\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
\n ON CONFLICT (title, artist_id) DO NOTHING
\n ''',
\n artwork.get('title'),
\n artist_id,
\n artwork.get('year'),
\n artwork.get('medium'),
\n artwork.get('dimensions'),
\n artwork.get('style'),
\n artwork.get('genre'),
\n artwork.get('description'),
\n artwork.get('image_url'),
\n artwork.get('source'),
\n artwork.get('url')
\n )<\/p>\n
async def _get_or_create_artist(self, conn, artist_name: str) -> Optional[int]:
\n """\u83b7\u53d6\u6216\u521b\u5efa\u827a\u672f\u5bb6"""
\n if not artist_name:
\n return None<\/p>\n
# \u67e5\u627e\u73b0\u6709\u827a\u672f\u5bb6
\n result = await conn.fetchrow(
\n 'SELECT id FROM artists WHERE name = $1',
\n artist_name
\n )<\/p>\n
if result:
\n return result['id']<\/p>\n
# \u521b\u5efa\u65b0\u827a\u672f\u5bb6
\n result = await conn.fetchrow(
\n 'INSERT INTO artists (name) VALUES ($1) RETURNING id',
\n artist_name
\n )<\/p>\n
return result['id']<\/p>\n
async def _store_batch_to_postgres(self, artworks: List[Dict]):
\n """\u6279\u91cf\u5b58\u50a8\u5230PostgreSQL"""
\n async with self.pg_pool.acquire() as conn:
\n async with conn.transaction():
\n for artwork in artworks:
\n artist_id = await self._get_or_create_artist(conn, artwork.get('artist'))<\/p>\n
await conn.execute('''
\n INSERT INTO artworks
\n (title, artist_id, year, medium, dimensions, style, genre,
\n description, image_url, source, source_url)
\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
\n ON CONFLICT (title, artist_id) DO NOTHING
\n ''',
\n artwork.get('title'),
\n artist_id,
\n artwork.get('year'),
\n artwork.get('medium'),
\n artwork.get('dimensions'),
\n artwork.get('style'),
\n artwork.get('genre'),
\n artwork.get('description'),
\n artwork.get('image_url'),
\n artwork.get('source'),
\n artwork.get('url')
\n )<\/p>\n
async def _index_to_elasticsearch(self, artwork: Dict, artwork_id: str):
\n """\u7d22\u5f15\u5230Elasticsearch"""
\n doc = {
\n 'title': artwork.get('title'),
\n 'artist': artwork.get('artist'),
\n 'description': artwork.get('description'),
\n 'year': artwork.get('year'),
\n 'style': artwork.get('style'),
\n 'genre': artwork.get('genre'),
\n 'medium': artwork.get('medium'),
\n 'tags': artwork.get('tags', []),
\n 'created_at': datetime.now().isoformat()
\n }<\/p>\n
await self.es_client.index(
\n index=settings.ELASTICSEARCH_CONFIG['index'],
\n id=artwork_id,
\n body=doc,
\n refresh=True
\n )<\/p>\n
async def _index_batch_to_elasticsearch(self, artworks: List[Dict], ids: List[str]):
\n """\u6279\u91cf\u7d22\u5f15\u5230Elasticsearch"""
\n actions = []<\/p>\n
for i, (artwork, artwork_id) in enumerate(zip(artworks, ids)):
\n doc = {
\n 'title': artwork.get('title'),
\n 'artist': artwork.get('artist'),
\n 'description': artwork.get('description'),
\n 'year': artwork.get('year'),
\n 'style': artwork.get('style'),
\n 'genre': artwork.get('genre'),
\n 'medium': artwork.get('medium'),
\n 'tags': artwork.get('tags', []),
\n 'created_at': datetime.now().isoformat()
\n }<\/p>\n
action = {
\n '_index': settings.ELASTICSEARCH_CONFIG['index'],
\n '_id': artwork_id,
\n '_source': doc
\n }
\n actions.append(action)<\/p>\n
if actions:
\n await self.es_client.bulk(body=actions, refresh=True)<\/p>\n
async def _store_image(self, image_url: str, artwork_id: str):
\n """\u5b58\u50a8\u56fe\u7247\u5230MinIO"""
\n try:
\n # \u4e0b\u8f7d\u56fe\u7247
\n import aiohttp
\n async with aiohttp.ClientSession() as session:
\n async with session.get(image_url) as response:
\n if response.status == 200:
\n data = await response.read()<\/p>\n
# \u4e0a\u4f20\u5230MinIO
\n self.minio_client.put_object(
\n 'art-images',
\n f'{artwork_id}.jpg',
\n io.BytesIO(data),
\n length=len(data),
\n content_type='image\/jpeg'
\n )
\n except Exception as e:
\n print(f"Error storing image: {e}")<\/p>\n
async def search_artworks(self, query: str, size: int = 10) -> List[Dict]:
\n """
\n \u641c\u7d22\u827a\u672f\u54c1<\/p>\n
Args:
\n query: \u641c\u7d22\u5173\u952e\u8bcd
\n size: \u8fd4\u56de\u6570\u91cf<\/p>\n
Returns:
\n \u641c\u7d22\u7ed3\u679c
\n """
\n search_body = {
\n "query": {
\n "multi_match": {
\n "query": query,
\n "fields": ["title^3", "artist^2", "description", "tags"]
\n }
\n },
\n "size": size
\n }<\/p>\n
result = await self.es_client.search(
\n index=settings.ELASTICSEARCH_CONFIG['index'],
\n body=search_body
\n )<\/p>\n
return [hit['_source'] for hit in result['hits']['hits']] <\/p>\n
python<\/p>\n
# src\/scheduler.py<\/p>\n
import asyncio
\nimport signal
\nfrom typing import Dict, List, Optional
\nfrom datetime import datetime, timedelta
\nimport aioredis
\nfrom apscheduler.schedulers.asyncio import AsyncIOScheduler
\nfrom apscheduler.triggers.interval import IntervalTrigger
\nfrom apscheduler.triggers.cron import CronTrigger
\nimport json<\/p>\n
from src.crawlers.wikiart import WikiArtCrawler
\nfrom src.crawlers.artsy import ArtsyCrawler
\nfrom src.pipelines.storage import StoragePipeline
\nfrom src.pipelines.deduplication import DeduplicationPipeline
\nfrom src.pipelines.enrichment import DataEnrichmentPipeline
\nfrom src.utils.logger import setup_logger<\/p>\n
class CrawlerScheduler:
\n """
\n \u722c\u866b\u4efb\u52a1\u8c03\u5ea6\u5668
\n \u7ba1\u7406\u6240\u6709\u722c\u866b\u7684\u6267\u884c\u3001\u76d1\u63a7\u548c\u9519\u8bef\u5904\u7406
\n """<\/p>\n
def __init__(self):
\n self.logger = setup_logger("scheduler")
\n self.scheduler = AsyncIOScheduler()
\n self.running = False
\n self.redis = None<\/p>\n
# \u4efb\u52a1\u72b6\u6001
\n self.task_status = {}<\/p>\n
# \u722c\u866b\u5b9e\u4f8b
\n self.crawlers = {
\n 'wikiart': WikiArtCrawler,
\n 'artsy': ArtsyCrawler
\n }<\/p>\n
async def initialize(self):
\n """\u521d\u59cb\u5316\u8c03\u5ea6\u5668"""
\n # \u8fde\u63a5Redis
\n self.redis = await aioredis.from_url(
\n f"redis:\/\/{settings.REDIS_CONFIG['host']}:{settings.REDIS_CONFIG['port']}",
\n password=settings.REDIS_CONFIG.get('password'),
\n decode_responses=True
\n )<\/p>\n
# \u6062\u590d\u4efb\u52a1\u72b6\u6001
\n await self._load_task_status()<\/p>\n
# \u8bbe\u7f6e\u4fe1\u53f7\u5904\u7406
\n loop = asyncio.get_event_loop()
\n for sig in (signal.SIGTERM, signal.SIGINT):
\n loop.add_signal_handler(sig, lambda: asyncio.create_task(self.shutdown()))<\/p>\n
async def _load_task_status(self):
\n """\u4eceRedis\u52a0\u8f7d\u4efb\u52a1\u72b6\u6001"""
\n keys = await self.redis.keys("task:*")
\n for key in keys:
\n status = await self.redis.get(key)
\n if status:
\n self.task_status[key[5:]] = json.loads(status)<\/p>\n
async def save_task_status(self, task_name: str):
\n """\u4fdd\u5b58\u4efb\u52a1\u72b6\u6001\u5230Redis"""
\n if task_name in self.task_status:
\n await self.redis.setex(
\n f"task:{task_name}",
\n 86400 * 7, # \u4fdd\u75597\u5929
\n json.dumps(self.task_status[task_name], default=str)
\n )<\/p>\n
def add_crawler_task(
\n self,
\n crawler_name: str,
\n schedule_type: str = "interval",
\n **kwargs
\n ):
\n """
\n \u6dfb\u52a0\u722c\u866b\u4efb\u52a1<\/p>\n
Args:
\n crawler_name: \u722c\u866b\u540d\u79f0
\n schedule_type: \u8c03\u5ea6\u7c7b\u578b（interval\/cron）
\n **kwargs: \u8c03\u5ea6\u53c2\u6570
\n """
\n if crawler_name not in self.crawlers:
\n self.logger.error(f"Unknown crawler: {crawler_name}")
\n return<\/p>\n
task_id = f"{crawler_name}_task"<\/p>\n
# \u6839\u636e\u8c03\u5ea6\u7c7b\u578b\u521b\u5efa\u89e6\u53d1\u5668
\n if schedule_type == "interval":
\n trigger = IntervalTrigger(
\n minutes=kwargs.get('minutes', 60),
\n hours=kwargs.get('hours', 0),
\n days=kwargs.get('days', 0)
\n )
\n elif schedule_type == "cron":
\n trigger = CronTrigger(
\n hour=kwargs.get('hour', '*'),
\n minute=kwargs.get('minute', '0'),
\n day=kwargs.get('day', '*'),
\n month=kwargs.get('month', '*'),
\n day_of_week=kwargs.get('day_of_week', '*')
\n )
\n else:
\n self.logger.error(f"Unknown schedule type: {schedule_type}")
\n return<\/p>\n
# \u6dfb\u52a0\u4efb\u52a1
\n self.scheduler.add_job(
\n func=self.run_crawler,
\n trigger=trigger,
\n args=[crawler_name],
\n id=task_id,
\n name=f"Run {crawler_name} crawler",
\n replace_existing=True,
\n misfire_grace_time=300
\n )<\/p>\n
self.logger.info(f"Added crawler task: {crawler_name} ({schedule_type})")<\/p>\n
# \u521d\u59cb\u5316\u4efb\u52a1\u72b6\u6001
\n self.task_status[crawler_name] = {
\n 'last_run': None,
\n 'last_success': None,
\n 'last_error': None,
\n 'total_runs': 0,
\n 'successful_runs': 0,
\n 'failed_runs': 0,
\n 'total_items': 0
\n }<\/p>\n
async def run_crawler(self, crawler_name: str):
\n """
\n \u8fd0\u884c\u722c\u866b<\/p>\n
Args:
\n crawler_name: \u722c\u866b\u540d\u79f0
\n """
\n self.logger.info(f"Starting crawler: {crawler_name}")<\/p>\n
# \u66f4\u65b0\u72b6\u6001
\n self.task_status[crawler_name]['last_run'] = datetime.now()
\n self.task_status[crawler_name]['total_runs'] += 1<\/p>\n
crawler_class = self.crawlers[crawler_name]<\/p>\n
try:
\n # \u521b\u5efa\u722c\u866b\u5b9e\u4f8b
\n async with crawler_class() as crawler:
\n # \u8fd0\u884c\u722c\u866b
\n artworks = await crawler.run()<\/p>\n
if artworks:
\n # \u53bb\u91cd
\n dedup_pipeline = DeduplicationPipeline()
\n await dedup_pipeline.initialize()<\/p>\n
new_artworks, duplicates = await dedup_pipeline.process_batch(artworks)<\/p>\n
# \u6570\u636e\u589e\u5f3a
\n enrich_pipeline = DataEnrichmentPipeline()
\n enriched_artworks = [
\n enrich_pipeline.enrich_artwork(artwork)
\n for artwork in new_artworks
\n ]<\/p>\n
# \u5b58\u50a8
\n storage_pipeline = StoragePipeline()
\n await storage_pipeline.initialize()<\/p>\n
ids = await storage_pipeline.store_batch(enriched_artworks)<\/p>\n
# \u5173\u95ed\u7ba1\u9053
\n await dedup_pipeline.close()
\n await storage_pipeline.close()<\/p>\n
# \u66f4\u65b0\u7edf\u8ba1
\n self.task_status[crawler_name]['total_items'] += len(ids)<\/p>\n
self.logger.info(
\n f"Crawler {crawler_name} completed. "
\n f"Collected: {len(artworks)}, "
\n f"New: {len(new_artworks)}, "
\n f"Duplicates: {len(duplicates)}"
\n )<\/p>\n
# \u66f4\u65b0\u6210\u529f\u72b6\u6001
\n self.task_status[crawler_name]['last_success'] = datetime.now()
\n self.task_status[crawler_name]['successful_runs'] += 1<\/p>\n
except Exception as e:
\n self.logger.error(f"Crawler {crawler_name} failed: {e}", exc_info=True)<\/p>\n
# \u66f4\u65b0\u5931\u8d25\u72b6\u6001
\n self.task_status[crawler_name]['last_error'] = {
\n 'time': datetime.now(),
\n 'error': str(e)
\n }
\n self.task_status[crawler_name]['failed_runs'] += 1<\/p>\n
finally:
\n # \u4fdd\u5b58\u72b6\u6001
\n await self.save_task_status(crawler_name)<\/p>\n
async def run_all_crawlers(self):
\n """\u8fd0\u884c\u6240\u6709\u722c\u866b"""
\n tasks = []
\n for crawler_name in self.crawlers:
\n task = self.run_crawler(crawler_name)
\n tasks.append(task)<\/p>\n
await asyncio.gather(*tasks, return_exceptions=True)<\/p>\n
async def get_status(self) -> Dict:
\n """\u83b7\u53d6\u6240\u6709\u4efb\u52a1\u72b6\u6001"""
\n return {
\n 'running': self.running,
\n 'tasks': self.task_status,
\n 'scheduler_jobs': [
\n {
\n 'id': job.id,
\n 'name': job.name,
\n 'next_run': job.next_run_time.isoformat() if job.next_run_time else None
\n }
\n for job in self.scheduler.get_jobs()
\n ]
\n }<\/p>\n
def start(self):
\n """\u542f\u52a8\u8c03\u5ea6\u5668"""
\n self.scheduler.start()
\n self.running = True
\n self.logger.info("Scheduler started")<\/p>\n
async def shutdown(self):
\n """\u5173\u95ed\u8c03\u5ea6\u5668"""
\n self.logger.info("Shutting down scheduler…")<\/p>\n
self.scheduler.shutdown()
\n self.running = False<\/p>\n
if self.redis:
\n await self.redis.close()<\/p>\n
self.logger.info("Scheduler stopped")<\/p>\n
async def run_once(self, crawler_name: str):
\n """
\n \u7acb\u5373\u8fd0\u884c\u4e00\u6b21\u722c\u866b<\/p>\n
Args:
\n crawler_name: \u722c\u866b\u540d\u79f0
\n """
\n await self.run_crawler(crawler_name) <\/p>\n
python<\/p>\n
# src\/web\/dashboard.py<\/p>\n
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
\nfrom fastapi.responses import HTMLResponse
\nfrom fastapi.staticfiles import StaticFiles
\nfrom fastapi.templating import Jinja2Templates
\nfrom starlette.requests import Request
\nimport plotly.graph_objects as go
\nimport plotly.express as px
\nfrom datetime import datetime, timedelta
\nimport pandas as pd
\nfrom typing import List, Dict
\nimport json<\/p>\n
from src.scheduler import CrawlerScheduler
\nfrom src.pipelines.storage import StoragePipeline<\/p>\n
app = FastAPI(title="Art Crawler Dashboard")<\/p>\n
# \u6a21\u677f\u548c\u9759\u6001\u6587\u4ef6
\ntemplates = Jinja2Templates(directory="src\/web\/templates")
\napp.mount("\/static", StaticFiles(directory="src\/web\/static"), name="static")<\/p>\n
# \u5168\u5c40\u5b9e\u4f8b
\nscheduler = CrawlerScheduler()
\nstorage = StoragePipeline()<\/p>\n
class ConnectionManager:
\n """WebSocket\u8fde\u63a5\u7ba1\u7406\u5668"""
\n def __init__(self):
\n self.active_connections: List[WebSocket] = []<\/p>\n
async def connect(self, websocket: WebSocket):
\n await websocket.accept()
\n self.active_connections.append(websocket)<\/p>\n
def disconnect(self, websocket: WebSocket):
\n self.active_connections.remove(websocket)<\/p>\n
async def broadcast(self, message: dict):
\n for connection in self.active_connections:
\n try:
\n await connection.send_json(message)
\n except:
\n pass<\/p>\n
manager = ConnectionManager()<\/p>\n
@app.on_event("startup")
\nasync def startup_event():
\n """\u542f\u52a8\u4e8b\u4ef6"""
\n await scheduler.initialize()
\n await storage.initialize()<\/p>\n
@app.on_event("shutdown")
\nasync def shutdown_event():
\n """\u5173\u95ed\u4e8b\u4ef6"""
\n await storage.close()<\/p>\n
@app.get("\/", response_class=HTMLResponse)
\nasync def dashboard(request: Request):
\n """\u4eea\u8868\u76d8\u4e3b\u9875"""
\n return templates.TemplateResponse(
\n "dashboard.html",
\n {"request": request}
\n )<\/p>\n
@app.get("\/api\/status")
\nasync def get_status():
\n """\u83b7\u53d6\u7cfb\u7edf\u72b6\u6001"""
\n return await scheduler.get_status()<\/p>\n
@app.get("\/api\/stats")
\nasync def get_stats(days: int = 7):
\n """\u83b7\u53d6\u7edf\u8ba1\u6570\u636e"""
\n end_date = datetime.now()
\n start_date = end_date – timedelta(days=days)<\/p>\n
# \u4ece\u6570\u636e\u5e93\u83b7\u53d6\u7edf\u8ba1
\n pipeline = [
\n {
\n "$match": {
\n "created_at": {
\n "$gte": start_date,
\n "$lte": end_date
\n }
\n }
\n },
\n {
\n "$group": {
\n "_id": {
\n "year": {"$year": "$created_at"},
\n "month": {"$month": "$created_at"},
\n "day": {"$dayOfMonth": "$created_at"}
\n },
\n "count": {"$sum": 1},
\n "source": {"$first": "$source"}
\n }
\n },
\n {"$sort": {"_id": 1}}
\n ]<\/p>\n
cursor = storage.mongo_db.artworks.aggregate(pipeline)
\n results = await cursor.to_list(length=None)<\/p>\n
# \u8f6c\u6362\u4e3aDataFrame
\n df = pd.DataFrame(results)<\/p>\n
# \u751f\u6210\u56fe\u8868
\n charts = {}<\/p>\n
if not df.empty:
\n # \u6bcf\u65e5\u91c7\u96c6\u6570\u91cf\u6298\u7ebf\u56fe
\n fig_line = go.Figure()
\n for source in df['source'].unique():
\n source_data = df[df['source'] == source]
\n fig_line.add_trace(go.Scatter(
\n x=pd.to_datetime(source_data['_id'].apply(
\n lambda x: f"{x['year']}-{x['month']}-{x['day']}"
\n )),
\n y=source_data['count'],
\n mode='lines+markers',
\n name=source
\n ))
\n fig_line.update_layout(
\n title="\u6bcf\u65e5\u91c7\u96c6\u6570\u91cf\u8d8b\u52bf",
\n xaxis_title="\u65e5\u671f",
\n yaxis_title="\u6570\u91cf"
\n )
\n charts['daily_trend'] = fig_line.to_json()<\/p>\n
# \u6765\u6e90\u5206\u5e03\u997c\u56fe
\n source_counts = df.groupby('source')['count'].sum().reset_index()
\n fig_pie = px.pie(
\n source_counts,
\n values='count',
\n names='source',
\n title="\u6570\u636e\u6765\u6e90\u5206\u5e03"
\n )
\n charts['source_distribution'] = fig_pie.to_json()<\/p>\n
# \u827a\u672f\u5bb6Top 10
\n pipeline_artists = [
\n {"$group": {"_id": "$artist", "count": {"$sum": 1}}},
\n {"$sort": {"count": -1}},
\n {"$limit": 10}
\n ]
\n top_artists = await storage.mongo_db.artworks.aggregate(
\n pipeline_artists
\n ).to_list(length=None)<\/p>\n
if top_artists:
\n fig_bar = go.Figure(data=[
\n go.Bar(
\n x=[a['_id'] for a in top_artists],
\n y=[a['count'] for a in top_artists]
\n )
\n ])
\n fig_bar.update_layout(
\n title="\u4f5c\u54c1\u6700\u591a\u7684\u827a\u672f\u5bb6Top 10",
\n xaxis_title="\u827a\u672f\u5bb6",
\n yaxis_title="\u4f5c\u54c1\u6570\u91cf"
\n )
\n charts['top_artists'] = fig_bar.to_json()<\/p>\n
# \u83b7\u53d6\u603b\u6570
\n total_count = await storage.mongo_db.artworks.count_documents({})<\/p>\n
# \u83b7\u53d6\u6700\u65b0\u4f5c\u54c1
\n latest = await storage.mongo_db.artworks.find().sort(
\n "created_at", -1
\n ).limit(10).to_list(length=None)<\/p>\n
return {
\n "total": total_count,
\n "charts": charts,
\n "latest": [
\n {
\n "title": item.get("title"),
\n "artist": item.get("artist"),
\n "date": item.get("created_at").isoformat()
\n }
\n for item in latest
\n ]
\n }<\/p>\n
@app.post("\/api\/crawlers\/{crawler_name}\/run")
\nasync def run_crawler(crawler_name: str):
\n """\u624b\u52a8\u8fd0\u884c\u722c\u866b"""
\n asyncio.create_task(scheduler.run_once(crawler_name))
\n return {"status": "started", "crawler": crawler_name}<\/p>\n
@app.post("\/api\/crawlers\/schedule")
\nasync def update_schedule(config: dict):
\n """\u66f4\u65b0\u8c03\u5ea6\u914d\u7f6e"""
\n # \u66f4\u65b0\u8c03\u5ea6\u5668\u914d\u7f6e
\n pass<\/p>\n
@app.websocket("\/ws")
\nasync def websocket_endpoint(websocket: WebSocket):
\n """WebSocket\u5b9e\u65f6\u66f4\u65b0"""
\n await manager.connect(websocket)
\n try:
\n while True:
\n # \u6bcf\u79d2\u53d1\u9001\u4e00\u6b21\u72b6\u6001\u66f4\u65b0
\n status = await scheduler.get_status()
\n await websocket.send_json(status)
\n await asyncio.sleep(1)
\n except WebSocketDisconnect:
\n manager.disconnect(websocket)<\/p>\n","protected":false},"excerpt":{"rendered":"
\u4e00\u3001\u5f15\u8a00：\u4e3a\u4ec0\u4e48\u9700\u8981\u827a\u672f\u4f5c\u54c1\u6570\u636e\u5e93？
\n\u5728\u6570\u5b57\u65f6\u4ee3，\u827a\u672f\u4f5c\u54c1\u7684\u6570\u5b57\u5316\u7ba1\u7406\u548c\u5206\u6790\u5df2\u6210\u4e3a\u827a\u672f\u754c\u3001\u5b66\u672f\u754c\u548c\u5546\u4e1a\u9886\u57df\u7684\u91cd\u8981\u9700\u6c42\u3002\u65e0\u8bba\u662f\u535a\u7269\u9986\u7684\u85cf\u54c1\u7ba1\u7406\u3001\u827a\u672f\u5e02\u573a\u7684\u4ef7\u683c\u5206\u6790，\u8fd8\u662fAI\u827a\u672f\u521b\u4f5c\u7684\u8bad\u7ec3\u6570\u636e，\u4e00\u4e2a\u7ed3\u6784\u5b8c\u5584\u7684\u827a\u672f\u4f5c\u54c1\u6570\u636e\u5e93\u90fd\u626e\u6f14\u7740\u81f3\u5173\u91cd\u8981\u7684\u89d2\u8272\u3002
\n1.1 \u827a\u672f\u4f5c\u54c1\u6570\u636e\u7684\u4ef7\u503c
\n\u827a\u672f\u4f5c\u54c1\u6570\u636e\u5305\u542b\u4e86\u4e30\u5bcc\u7684\u4fe1\u606f\u7ef4\u5ea6：\u521b\u4f5c\u80cc\u666f\u3001\u827a\u672f\u5bb6\u751f\u5e73\u3001\u6280\u6cd5\u6750\u6599\u3001\u5c3a\u5bf8\u89c4\u683c\u3001\u6536\u85cf\u5386\u53f2\u3001\u5e02\u573a\u4ef7\u503c\u7b49\u3002\u8fd9\u4e9b\u6570\u636e\u4e0d\u4ec5\u5bf9\u4e8e\u827a\u672f<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[81,3677,190,677,2653],"topic":[],"class_list":["post-76997","post","type-post","status-publish","format-standard","hentry","category-server","tag-python","tag-tensorflow","tag-190","tag-677","tag-2653"],"yoast_head":"\n