{"id":69072,"date":"2026-01-31T03:57:37","date_gmt":"2026-01-30T19:57:37","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/69072.html"},"modified":"2026-01-31T03:57:37","modified_gmt":"2026-01-30T19:57:37","slug":"gemini-%e7%9a%84%e5%a4%9a%e6%a8%a1%e6%80%81%e6%9e%b6%e6%9e%84%e8%ae%be%e8%ae%a1%e5%a6%82%e4%bd%95%e7%bb%9f%e4%b8%80%e6%96%87%e6%9c%ac%e3%80%81%e5%9b%be%e5%83%8f%e3%80%81%e8%a7%86%e9%a2%91%e7%9a%84","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/69072.html","title":{"rendered":"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f"},"content":{"rendered":"

Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a？<\/h2>\n

\u6587\u6863\u6982\u8ff0<\/h3>\n

\u672c\u6587\u6838\u5fc3\u4ef7\u503c<\/h4>\n
  • \u6df1\u5ea6\u62c6\u89e3 Gemini \u591a\u6a21\u6001\u67b6\u6784\u7684\u6838\u5fc3\u8bbe\u8ba1\u903b\u8f91，\u89e3\u91ca\u5176\u5982\u4f55\u5b9e\u73b0\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u7edf\u4e00\u8868\u793a<\/li>\n
  • \u638c\u63e1 Gemini \u591a\u6a21\u6001\u6a21\u578b\u7684\u672c\u5730\u90e8\u7f72\u4e0e\u73af\u5883\u914d\u7f6e\u65b9\u6cd5<\/li>\n
  • \u901a\u8fc7\u53ef\u8fd0\u884c\u7684\u4ee3\u7801\u5b9e\u6218，\u5b9e\u73b0\u6587\u672c \/ \u56fe\u50cf \/ \u89c6\u9891\u7684\u7edf\u4e00\u5411\u91cf\u8868\u793a\u4e0e\u8de8\u6a21\u6001\u68c0\u7d22<\/li>\n
  • \u7406\u89e3\u591a\u6a21\u6001\u8868\u793a\u7edf\u4e00\u7684\u8c03\u8bd5\u3001\u4f18\u5316\u65b9\u6cd5\u4e0e\u751f\u4ea7\u7ea7\u90e8\u7f72\u6280\u5de7<\/li>\n

    \u5b66\u4e60\u76ee\u6807<\/h4>\n
  • \u7406\u89e3 Gemini \u591a\u6a21\u6001\u67b6\u6784\u7684\u6838\u5fc3\u7ec4\u4ef6\u4e0e\u8de8\u6a21\u6001\u5bf9\u9f50\u539f\u7406<\/li>\n
  • \u638c\u63e1 Gemini \u6a21\u578b\u672c\u5730\u90e8\u7f72\u7684\u73af\u5883\u914d\u7f6e\u4e0e\u4f9d\u8d56\u5b89\u88c5<\/li>\n
  • \u7cbe\u901a\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u6a21\u6001\u8f6c\u6362\u4e0e\u7edf\u4e00\u8868\u793a\u65b9\u6cd5<\/li>\n
  • \u80fd\u591f\u72ec\u7acb\u5b9e\u73b0\u591a\u6a21\u6001\u6570\u636e\u7684\u5411\u91cf\u7f16\u7801\u4e0e\u8de8\u6a21\u6001\u68c0\u7d22\u7cfb\u7edf<\/li>\n
  • \u5b66\u4f1a\u591a\u6a21\u6001\u6a21\u578b\u7684\u6027\u80fd\u4f18\u5316\u4e0e\u90e8\u7f72\u6700\u4f73\u5b9e\u8df5<\/li>\n

    \u4e00\u3001Gemini \u591a\u6a21\u6001\u6280\u672f\u6982\u8ff0<\/h3>\n

    1.1 \u4ec0\u4e48\u662f Gemini \u591a\u6a21\u6001\u6a21\u578b？<\/h4>\n

    Gemini \u662f Google DeepMind \u63a8\u51fa\u7684\u65b0\u4e00\u4ee3\u591a\u6a21\u6001\u5927\u6a21\u578b，\u6838\u5fc3\u5b9a\u4f4d\u662f\u539f\u751f\u652f\u6301\u6587\u672c\u3001\u56fe\u50cf\u3001\u97f3\u9891\u3001\u89c6\u9891\u7b49\u591a\u6a21\u6001\u8f93\u5165，\u5e76\u5728\u7edf\u4e00\u7684\u5f20\u91cf\u7a7a\u95f4\u4e2d\u5b8c\u6210\u8868\u793a\u4e0e\u63a8\u7406\u3002\u4e0e\u4f20\u7edf \u201c\u5355\u6a21\u6001\u6a21\u578b + \u6a21\u6001\u8f6c\u6362\u6a21\u5757\u201d \u7684\u62fc\u63a5\u5f0f\u67b6\u6784\u4e0d\u540c，Gemini \u4ece\u5e95\u5c42\u8bbe\u8ba1\u4e0a\u5b9e\u73b0\u4e86\u591a\u6a21\u6001\u7684\u6df1\u5ea6\u878d\u5408，\u771f\u6b63\u505a\u5230 \u201c\u4e00\u4e2a\u6a21\u578b\u5904\u7406\u6240\u6709\u6a21\u6001\u201d\u3002<\/p>\n

    1.2 \u4f20\u7edf\u591a\u6a21\u6001\u65b9\u6848\u7684\u5c40\u9650\u6027（Gemini \u89e3\u51b3\u7684\u6838\u5fc3\u75db\u70b9）<\/h4>\n\n\u75db\u70b9\u7ef4\u5ea6\u4f20\u7edf\u65b9\u6848\u95ee\u9898Gemini \u89e3\u51b3\u65b9\u6848<\/tr>\n\n\n\n\n\n\n
    \u6a21\u6001\u5272\u88c2<\/td>\n\u6587\u672c \/ \u56fe\u50cf \/ \u89c6\u9891\u5404\u81ea\u4f7f\u7528\u72ec\u7acb\u6a21\u578b\u7f16\u7801，\u4ec5\u5728\u8f93\u51fa\u5c42\u878d\u5408<\/td>\n\u5171\u4eab\u7edf\u4e00\u7684\u7f16\u7801\u5668\u4e0e\u5f20\u91cf\u7a7a\u95f4，\u6240\u6709\u6a21\u6001\u4ece\u8f93\u5165\u5373\u5bf9\u9f50<\/td>\n<\/tr>\n
    \u65f6\u5e8f\u7f3a\u5931<\/td>\n\u89c6\u9891\u5904\u7406\u7b80\u5316\u4e3a \u201c\u56fe\u50cf\u5e27\u5806\u53e0\u201d，\u4e22\u5931\u65f6\u5e8f\u4fe1\u606f<\/td>\n\u5f15\u5165\u65f6\u7a7a\u6ce8\u610f\u529b\u673a\u5236，\u5efa\u6a21\u89c6\u9891\u7684\u5e27\u95f4\u65f6\u5e8f\u4f9d\u8d56<\/td>\n<\/tr>\n
    \u8868\u793a\u4e0d\u4e00\u81f4<\/td>\n\u4e0d\u540c\u6a21\u6001\u7684\u5411\u91cf\u7a7a\u95f4\u65e0\u7edf\u4e00\u5ea6\u91cf\u6807\u51c6，\u8de8\u6a21\u6001\u68c0\u7d22\u7cbe\u5ea6\u4f4e<\/td>\n\u7edf\u4e00\u7684\u5f52\u4e00\u5316\u8868\u793a\u7a7a\u95f4，\u652f\u6301\u8de8\u6a21\u6001\u76f8\u4f3c\u5ea6\u8ba1\u7b97<\/td>\n<\/tr>\n
    \u8ba1\u7b97\u6548\u7387\u4f4e<\/td>\n\u591a\u6a21\u6001\u5904\u7406\u9700\u591a\u6b21\u6a21\u578b\u8c03\u7528，\u5ef6\u8fdf\u9ad8<\/td>\n\u7aef\u5230\u7aef\u7684\u591a\u6a21\u6001\u7f16\u7801，\u5355\u6b21\u524d\u5411\u4f20\u64ad\u5b8c\u6210\u591a\u6a21\u6001\u8868\u793a<\/td>\n<\/tr>\n
    \u4e0a\u4e0b\u6587\u5272\u88c2<\/td>\n\u957f\u89c6\u9891 \/ \u957f\u6587\u672c\u5904\u7406\u65f6\u4e0a\u4e0b\u6587\u4fe1\u606f\u4e22\u5931<\/td>\n\u652f\u6301\u8d85\u957f\u4e0a\u4e0b\u6587\u7a97\u53e3（Gemini 1.5 Pro \u652f\u6301 1M tokens）<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

    1.3 Gemini \u591a\u6a21\u6001\u8868\u793a\u7edf\u4e00\u7684\u6838\u5fc3\u8bbe\u8ba1\u539f\u5219<\/h4>\n

    1.3 Gemini \u591a\u6a21\u6001\u8868\u793a\u7edf\u4e00\u7684\u6838\u5fc3\u8bbe\u8ba1\u539f\u5219<\/h4>\n

    Gemini \u7684\u6838\u5fc3\u7a81\u7834\u5728\u4e8e\u6452\u5f03\u4e86\u4f20\u7edf\u591a\u6a21\u6001\u6a21\u578b\u201c\u62fc\u63a5\u5f0f\u201d\u7684\u5f02\u6784\u8bbe\u8ba1，\u8f6c\u800c\u91c7\u7528\u539f\u751f（Native）\u591a\u6a21\u6001\u67b6\u6784\u3002\u8fd9\u610f\u5473\u7740\u6a21\u578b\u4e0d\u518d\u9700\u8981\u9488\u5bf9\u4e0d\u540c\u6a21\u6001\u8bad\u7ec3\u72ec\u7acb\u7684\u7f16\u7801\u5668（\u5982 CLIP \u4e2d\u7684 Image Encoder \u548c Text Encoder），\u800c\u662f\u901a\u8fc7\u4ee5\u4e0b\u4e94\u5927\u539f\u5219，\u5728\u5e95\u5c42\u7269\u7406\u7a7a\u95f4\u548c\u9ad8\u5c42\u8bed\u4e49\u7a7a\u95f4\u4e0a\u5b9e\u73b0\u4e86\u5f7b\u5e95\u7684\u7edf\u4e00\u3002<\/p>\n

    1.3.1 \u539f\u7406\u67b6\u6784\u6811\u5f62\u56fe (Conceptual Tree)<\/h5>\n

    Gemini_Unified_Principles\/ # \u6838\u5fc3\u8bbe\u8ba1\u539f\u5219\u5168\u666f<\/span>
    \n\u2502
    \n\u251c\u2500\u2500 1<\/span>. Universal_Input (<\/span>\u4e07\u7269\u7686\u5e8f\u5217：\u7269\u7406\u5c42\u7684\u7edf\u4e00)<\/span>
    \n\u2502 \u251c\u2500\u2500 Text_Stream \u2500\u2500><\/span> [<\/span>Tokenization]<\/span> \u2500\u2500><\/span> 1D ID\u5e8f\u5217
    \n\u2502 \u251c\u2500\u2500 Image_Stream \u2500\u2500><\/span> [<\/span>Patch Partition]<\/span> \u2500\u2500><\/span> \u7ebf\u6027\u6295\u5f71 \u2500\u2500><\/span> \u4f2a1D\u5e8f\u5217
    \n\u2502 \u2514\u2500\u2500 Video_Stream \u2500\u2500><\/span> [<\/span>3D Tubelet]<\/span> \u2500\u2500><\/span> \u65f6\u7a7a\u538b\u5e73 \u2500\u2500><\/span> \u5e26\u6709\u65f6\u95f4\u6233\u7684\u5e8f\u5217
    \n\u2502
    \n\u251c\u2500\u2500 2<\/span>. Shared_Backbone (<\/span>\u5171\u4eab\u5927\u8111：\u8ba1\u7b97\u5c42\u7684\u7edf\u4e00)<\/span>
    \n\u2502 \u251c\u2500\u2500 Single_Transformer # \u62d2\u7edd\u53cc\u5854\u7ed3\u6784，\u6240\u6709\u6a21\u6001\u8fdb\u5165\u540c\u4e00\u4e2a\u7f51\u7edc<\/span>
    \n\u2502 \u251c\u2500\u2500 Shared_Parameters # \u6587\u672c\u548c\u56fe\u50cf\u5171\u7528\u540c\u4e00\u7ec4\u6743\u91cd\u8fdb\u884c\u63a8\u7406<\/span>
    \n\u2502 \u2514\u2500\u2500 Early_Fusion # \u6a21\u6001\u878d\u5408\u53d1\u751f\u5728\u7b2c\u4e00\u5c42，\u800c\u975e\u6700\u540e\u8f93\u51fa\u5c42<\/span>
    \n\u2502
    \n\u251c\u2500\u2500 3<\/span>. Attention_Mechanism (<\/span>\u52a8\u6001\u5bf9\u9f50：\u4ea4\u4e92\u5c42\u7684\u7edf\u4e00)<\/span>
    \n\u2502 \u251c\u2500\u2500 Self_Attention # \u6a21\u6001\u5185\u90e8\u5efa\u6a21 (\u5982：\u7406\u89e3\u53e5\u5b50\u8bed\u6cd5 \/ \u56fe\u50cf\u7eb9\u7406)<\/span>
    \n\u2502 \u251c\u2500\u2500 Cross_Attention # \u8de8\u6a21\u6001\u6865\u63a5 (\u5982：\u5355\u8bcd"\u732b" <-> \u56fe\u50cf\u4e2d\u732b\u7684\u50cf\u7d20)<\/span>
    \n\u2502 \u2514\u2500\u2500 Temporal_Attention # (\u89c6\u9891\u4e13\u7528) \u6cbf\u65f6\u95f4\u8f74\u6355\u6349\u52a8\u4f5c\u53d8\u5316<\/span>
    \n\u2502
    \n\u2514\u2500\u2500 4<\/span>. Unified_Metric (<\/span>\u5ea6\u91cf\u4e00\u81f4：\u8bed\u4e49\u5c42\u7684\u7edf\u4e00)<\/span>
    \n \u251c\u2500\u2500 Projection_Layer # \u5f3a\u5236\u7ef4\u5ea6\u5bf9\u9f50 (\u5982\u7edf\u4e00\u6620\u5c04\u5230 2048 \u7ef4)<\/span>
    \n \u2514\u2500\u2500 L2_Normalization # \u5f52\u4e00\u5316\u5230\u8d85\u7403\u9762\u4e0a，\u786e\u4fdd Cosine \u8ddd\u79bb\u53ef\u6bd4<\/span><\/p>\n

    1.3.2 \u6838\u5fc3\u539f\u5219\u6df1\u5ea6\u89e3\u6790<\/h5>\n

    1. \u6a21\u6001\u65e0\u5173\u7684\u8f93\u5165\u7f16\u7801 (Modality-Agnostic Input Encoding)<\/p>\n

      \n
    • \u539f\u7406：Gemini \u5efa\u7acb\u4e86\u4e00\u4e2a\u201c\u901a\u7528\u8bcd\u6c47\u8868\u201d\u3002\u5b83\u4e0d\u76f4\u63a5\u5904\u7406\u539f\u59cb\u7684\u50cf\u7d20\u6216\u97f3\u9891\u6ce2\u5f62，\u800c\u662f\u5c06\u5b83\u4eec\u89c6\u4e3a\u4e00\u79cd\u201c\u5916\u8bed\u201d，\u7ffb\u8bd1\u6210\u6a21\u578b\u80fd\u7406\u89e3\u7684 Token\u3002<\/li>\n
    • \u6269\u5145\u7ec6\u8282：\n
        \n
      • \u6587\u672c：\u4f7f\u7528 SentencePiece \u6a21\u578b\u5c06\u81ea\u7136\u8bed\u8a00\u5207\u5206\u4e3a\u5b50\u8bcd Token\u3002<\/li>\n
      • \u56fe\u50cf：\u91c7\u7528 Patch Embedding \u6280\u672f，\u5c06 \n

        H<\/p>\n

        \u00d7<\/p>\n

        W<\/p>\n

        H \\\\times W<\/p>\n

        <\/span><\/span>H<\/span><\/span>\u00d7<\/span><\/span><\/span><\/span>W<\/span><\/span><\/span><\/span><\/span> \u7684\u56fe\u50cf\u5207\u5206\u4e3a <\/p>\n

        16<\/p>\n

        \u00d7<\/p>\n

        16<\/p>\n

        16 \\\\times 16<\/p>\n

        <\/span><\/span>16<\/span><\/span>\u00d7<\/span><\/span><\/span><\/span>16<\/span><\/span><\/span><\/span><\/span> \u7684\u5c0f\u65b9\u5757，\u901a\u8fc7\u7ebf\u6027\u5c42\u6295\u5f71\u4e3a\u5411\u91cf，\u5e76\u52a0\u4e0a 2D \u4f4d\u7f6e\u7f16\u7801\u3002\u5728\u6a21\u578b\u770b\u6765，\u8fd9\u4e0d\u518d\u662f\u56fe\u7247，\u800c\u662f\u4e00\u53e5\u201c\u7531\u50cf\u7d20\u5757\u7ec4\u6210\u7684\u53e5\u5b50\u201d\u3002<\/li>\n

      • \u89c6\u9891：\u4e0d\u53ea\u662f\u56fe\u50cf\u7684\u5806\u53e0，\u800c\u662f\u91c7\u7528 3D Tubelet \u6216\u65f6\u7a7a\u91c7\u6837，\u5c06\u65f6\u95f4\u7ef4\u5ea6 \n

        T<\/p>\n

        T<\/p>\n

        <\/span><\/span>T<\/span><\/span><\/span><\/span><\/span> \u4e5f\u7eb3\u5165\u5207\u5206\u8303\u56f4，\u5f62\u6210 <\/p>\n

        (<\/p>\n

        t<\/p>\n

        ,<\/p>\n

        h<\/p>\n

        ,<\/p>\n

        w<\/p>\n

        )<\/p>\n

        (t, h, w)<\/p>\n

        <\/span><\/span>(<\/span>t<\/span>,<\/span><\/span>h<\/span>,<\/span><\/span>w<\/span>)<\/span><\/span><\/span><\/span><\/span> \u7684\u4e09\u7ef4 Token，\u4fdd\u7559\u4e86\u65f6\u5e8f\u6d41\u52a8\u7684\u7269\u7406\u7279\u5f81\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

        2. \u5171\u4eab\u7684 Transformer \u9aa8\u5e72\u7f51\u7edc (Shared Transformer Backbone)<\/p>\n

          \n
        • \u539f\u7406：\u8fd9\u662f\u201c\u539f\u751f\u591a\u6a21\u6001\u201d\u7684\u6807\u5fd7\u3002\u6240\u6709\u6a21\u6001\u7684\u6570\u636e\u6d41\u7ecf\u540c\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc，\u4f7f\u7528\u540c\u4e00\u5957\u53c2\u6570（\u6743\u91cd\/\u504f\u7f6e）\u8fdb\u884c\u8ba1\u7b97\u3002<\/li>\n
        • \u6269\u5145\u7ec6\u8282：\n
            \n
          • \u53c2\u6570\u6548\u7387：\u4e0e\u201c\u53cc\u5854\u7ed3\u6784\u201d（\u72ec\u7acb\u89c6\u89c9\u5854 + \u72ec\u7acb\u6587\u672c\u5854）\u76f8\u6bd4，Gemini \u7684\u53c2\u6570\u5229\u7528\u7387\u6781\u9ad8\u3002\u6a21\u578b\u5b66\u4f1a\u7684\u201c\u903b\u8f91\u63a8\u7406\u80fd\u529b\u201d\u53ef\u4ee5\u540c\u65f6\u4f5c\u7528\u4e8e\u6587\u672c\u5206\u6790\u548c\u56fe\u50cf\u7406\u89e3\u3002<\/li>\n
          • \u65e9\u671f\u878d\u5408 (Early Fusion)：\u6a21\u6001\u95f4\u7684\u4ea4\u4e92\u4ece\u7f51\u7edc\u7684\u7b2c\u4e00\u5c42\u5c31\u5f00\u59cb\u4e86，\u800c\u4e0d\u662f\u50cf\u4f20\u7edf\u6a21\u578b\u90a3\u6837\u7b49\u5230\u6700\u540e\u624d\u8fdb\u884c\u7279\u5f81\u62fc\u63a5\u3002\u8fd9\u4f7f\u5f97\u6a21\u578b\u80fd\u6355\u6349\u5230\u66f4\u6df1\u5c42\u6b21\u7684\u56fe\u6587\u9690\u55bb\u5173\u7cfb\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

            3. \u8de8\u6a21\u6001\u6ce8\u610f\u529b\u5bf9\u9f50 (Cross-Modal Attention Alignment)<\/p>\n

              \n
            • \u539f\u7406：\u5229\u7528 Attention \u673a\u5236\u52a8\u6001\u5efa\u7acb\u4e0d\u540c\u6a21\u6001 Token \u4e4b\u95f4\u7684\u8bed\u4e49\u94fe\u63a5\u3002<\/li>\n
            • \u6269\u5145\u7ec6\u8282：\n
                \n
              • \u7ec6\u7c92\u5ea6\u5bf9\u9f50：\u4f20\u7edf\u6a21\u578b\u53ea\u80fd\u5224\u65ad\u201c\u6574\u5f20\u56fe\u201d\u548c\u201c\u6574\u53e5\u8bdd\u201d\u662f\u5426\u5339\u914d\u3002Gemini \u901a\u8fc7\u4ea4\u53c9\u6ce8\u610f\u529b（Cross-Attention），\u53ef\u4ee5\u8ba9\u6587\u672c\u5e8f\u5217\u4e2d\u7684 Token（Query）\u53bb\u67e5\u8be2\u89c6\u89c9\u5e8f\u5217\u4e2d\u7684 Patch（Key\/Value）\u3002\u4f8b\u5982，\u5f53\u5904\u7406\u6587\u672c\u201c\u6234\u7ea2\u5e3d\u5b50\u7684\u7537\u5b69\u201d\u65f6，\u6a21\u578b\u7684\u6ce8\u610f\u529b\u5934\u4f1a\u9ad8\u4eae\u56fe\u50cf\u4e2d\u201c\u7ea2\u8272\u201d\u548c\u201c\u5e3d\u5b50\u201d\u5bf9\u5e94\u7684 Patch \u533a\u57df\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                4. \u65f6\u7a7a\u8054\u5408\u5efa\u6a21 (Joint Spatiotemporal Modeling)<\/p>\n

                  \n
                • \u539f\u7406：\u9488\u5bf9\u89c6\u9891\u6a21\u6001，\u6a21\u578b\u5fc5\u987b\u7406\u89e3\u201c\u53d8\u5316\u201d\u3002<\/li>\n
                • \u6269\u5145\u7ec6\u8282：\n
                    \n
                  • 3D \u4f4d\u7f6e\u7f16\u7801：\u9664\u4e86 x, y \u7a7a\u95f4\u5750\u6807，\u6bcf\u4e2a\u89c6\u9891 Token \u8fd8\u88ab\u8d4b\u4e88\u4e86 \n

                    t<\/p>\n

                    t<\/p>\n

                    <\/span><\/span>t<\/span><\/span><\/span><\/span><\/span> \u65f6\u95f4\u5750\u6807\u3002<\/li>\n

                  • \u56e0\u679c\u6ce8\u610f\u529b：\u5728\u7406\u89e3\u89c6\u9891\u65f6，\u6a21\u578b\u4e0d\u4ec5\u5173\u6ce8\u5f53\u524d\u5e27，\u8fd8\u4f1a\u901a\u8fc7\u65f6\u5e8f\u6ce8\u610f\u529b\u673a\u5236（Temporal Attention）\u56de\u987e\u5386\u53f2\u5e27，\u4ece\u800c\u7406\u89e3\u52a8\u4f5c\u7684\u8fde\u7eed\u6027（\u5982\u5206\u8fa8\u201c\u4eba\u5750\u4e0b\u201d\u548c\u201c\u4eba\u7ad9\u8d77\u201d\u7684\u533a\u522b）\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                    5. \u7edf\u4e00\u7684\u5411\u91cf\u5f52\u4e00\u5316 (Unified Vector Normalization)<\/p>\n

                      \n
                    • \u539f\u7406：\u6d88\u9664\u4e0d\u540c\u6a21\u6001\u5728\u7279\u5f81\u5206\u5e03\u4e0a\u7684\u201c\u91cf\u7eb2\u201d\u5dee\u5f02，\u786e\u4fdd\u5b83\u4eec\u5728\u540c\u4e00\u4e2a\u51e0\u4f55\u7a7a\u95f4\u4e2d\u53ef\u6bd4\u8f83\u3002<\/li>\n
                    • \u6269\u5145\u7ec6\u8282：\n
                        \n
                      • \u8d85\u7403\u9762\u6295\u5f71：\u6240\u6709\u8f93\u51fa\u5411\u91cf\u7ecf\u8fc7 LayerNorm \u548c L2 \u5f52\u4e00\u5316\u540e，\u90fd\u88ab\u6295\u5c04\u5230\u540c\u4e00\u4e2a\u5355\u4f4d\u8d85\u7403\u9762\u4e0a\u3002<\/li>\n
                      • \u6570\u5b66\u610f\u4e49：\u8fd9\u4fdd\u8bc1\u4e86\u6211\u4eec\u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u4f59\u5f26\u76f8\u4f3c\u5ea6 (Cosine Similarity) \u6765\u8861\u91cf\u4efb\u4f55\u4e24\u4e2a\u5bf9\u8c61\u7684\u8ddd\u79bb\u3002\u5982\u679c\u6587\u672c\u201c\u72d7\u201d\u548c\u56fe\u50cf\u201c\u72d7\u201d\u7684\u5411\u91cf\u5939\u89d2\u4e3a 0 \u5ea6，\u8bf4\u660e\u6a21\u578b\u771f\u6b63\u5b9e\u73b0\u4e86\u8bed\u4e49\u4e0a\u7684\u7edf\u4e00，\u800c\u4e0d\u4ec5\u4ec5\u662f\u6570\u503c\u4e0a\u7684\u63a5\u8fd1\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                        \u4e8c\u3001Gemini \u672c\u5730\u90e8\u7f72\u4e0e\u73af\u5883\u914d\u7f6e<\/h3>\n

                        2.1 \u90e8\u7f72\u73af\u5883\u8981\u6c42<\/h4>\n\n\u73af\u5883\u7c7b\u578b\u6700\u4f4e\u914d\u7f6e\u63a8\u8350\u914d\u7f6e<\/tr>\n\n\n\n\n\n\n\n
                        \u64cd\u4f5c\u7cfb\u7edf<\/td>\nWindows 10\/11\u3001macOS 13+\u3001Linux (Ubuntu 20.04+)<\/td>\nLinux (Ubuntu 22.04)<\/td>\n<\/tr>\n
                        CPU<\/td>\n8 \u6838 16 \u7ebf\u7a0b<\/td>\n16 \u6838 32 \u7ebf\u7a0b<\/td>\n<\/tr>\n
                        \u5185\u5b58<\/td>\n32GB<\/td>\n64GB+<\/td>\n<\/tr>\n
                        GPU<\/td>\nNVIDIA RTX 3090 (24GB)<\/td>\nNVIDIA A100 (80GB) \/ RTX 4090 (24GB)<\/td>\n<\/tr>\n
                        CUDA<\/td>\n12.1+<\/td>\n12.2+<\/td>\n<\/tr>\n
                        Python<\/td>\n3.9-3.11<\/td>\n3.10<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

                        2.2 \u73af\u5883\u914d\u7f6e\u6b65\u9aa4<\/h4>\n

                        2.2.1 \u521b\u5efa\u72ec\u7acb Conda \u73af\u5883<\/h5>\n

                        \u8fd0\u884c<\/p>\n

                        # \u521b\u5efa Gemini \u4e13\u5c5e\u73af\u5883<\/span>
                        \nconda create -n<\/span> gemini-multimodal python<\/span>=<\/span>3.10<\/span>
                        \n# \u6fc0\u6d3b\u73af\u5883<\/span>
                        \nconda activate gemini-multimodal<\/p>\n

                        2.2.2 \u5b89\u88c5\u6838\u5fc3\u4f9d\u8d56\u5e93<\/h5>\n

                        \u8fd0\u884c<\/p>\n

                        # \u57fa\u7840\u4f9d\u8d56<\/span>
                        \npip install<\/span> numpy<\/span>==<\/span>1.26<\/span>.4 pandas<\/span>==<\/span>2.2<\/span>.2 pillow<\/span>==<\/span>10.3<\/span>.0 opencv-python==<\/span>4.9<\/span>.0.80
                        \n# PyTorch (GPU\u7248\u672c，\u5339\u914dCUDA 12.1)<\/span>
                        \npip install<\/span> torch<\/span>==<\/span>2.3<\/span>.1 torchvision<\/span>==<\/span>0.18<\/span>.1 torchaudio<\/span>==<\/span>2.3<\/span>.1 –index-url https:\/\/download.pytorch.org\/whl\/cu121
                        \n# Hugging Face \u751f\u6001（\u6a21\u578b\u52a0\u8f7d\/\u5904\u7406）<\/span>
                        \npip install<\/span> transformers<\/span>==<\/span>4.41<\/span>.2 datasets<\/span>==<\/span>2.20<\/span>.0 accelerate<\/span>==<\/span>0.31<\/span>.0
                        \n# Google Gemini API\/\u672c\u5730\u90e8\u7f72\u4f9d\u8d56<\/span>
                        \npip install<\/span> google-generativeai==<\/span>0.7<\/span>.2 sentence-transformers==<\/span>2.7<\/span>.0
                        \n# \u89c6\u9891\u5904\u7406\u4f9d\u8d56<\/span>
                        \npip install<\/span> decord<\/span>==<\/span>0.6<\/span>.0 ffmpeg-python==<\/span>0.2<\/span>.0
                        \n# \u5411\u91cf\u5b58\u50a8\u4e0e\u68c0\u7d22<\/span>
                        \npip install<\/span> chromadb<\/span>==<\/span>0.5<\/span>.17 faiss-gpu==<\/span>1.7<\/span>.4
                        \n# \u65e5\u5fd7\u4e0e\u914d\u7f6e<\/span>
                        \npip install<\/span> python-dotenv==<\/span>1.0<\/span>.1 loguru<\/span>==<\/span>0.7<\/span>.2<\/p>\n

                        2.2.3 \u6a21\u578b\u4e0b\u8f7d（\u672c\u5730\u90e8\u7f72）<\/h5>\n

                        Gemini \u63d0\u4f9b\u4e0d\u540c\u91cf\u7ea7\u7684\u6a21\u578b\u7248\u672c，\u53ef\u6839\u636e\u786c\u4ef6\u6761\u4ef6\u9009\u62e9：<\/p>\n\n\u6a21\u578b\u7248\u672c\u53c2\u6570\u91cf\u786c\u4ef6\u8981\u6c42\u9002\u7528\u573a\u666f<\/tr>\n\n\n\n\n
                        Gemini 1.5 Flash<\/td>\n11B<\/td>\nGPU 16GB+<\/td>\n\u8f7b\u91cf\u7ea7\u591a\u6a21\u6001\u5904\u7406<\/td>\n<\/tr>\n
                        Gemini 1.5 Pro<\/td>\n90B<\/td>\nGPU 40GB+<\/td>\n\u9ad8\u7cbe\u5ea6\u591a\u6a21\u6001\u8868\u793a<\/td>\n<\/tr>\n
                        Gemini Nano<\/td>\n1.8B<\/td>\nCPU 16GB+<\/td>\n\u8fb9\u7f18\u8bbe\u5907\u90e8\u7f72<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

                        \u901a\u8fc7 Hugging Face \u4e0b\u8f7d\u5f00\u6e90\u9002\u914d\u7248：<\/p>\n

                        \u8fd0\u884c<\/p>\n

                        from<\/span> huggingface_hub import<\/span> snapshot_download<\/p>\n

                        # \u4e0b\u8f7d Gemini 1.5 Flash \u591a\u6a21\u6001\u6a21\u578b（\u5f00\u6e90\u9002\u914d\u7248）<\/span>
                        \nmodel_dir =<\/span> snapshot_download(<\/span>
                        \n repo_id=<\/span>"google\/gemma-2-9b-it"<\/span>,<\/span> # Gemini \u5f00\u6e90\u9002\u914d\u7248\u672c<\/span>
                        \n cache_dir=<\/span>"\/data\/models\/gemini"<\/span>,<\/span>
                        \n ignore_patterns=<\/span>[<\/span>"*.bin.index.json"<\/span>]<\/span> # \u8df3\u8fc7\u4e0d\u5fc5\u8981\u6587\u4ef6<\/span>
                        \n)<\/span>
                        \nprint<\/span>(<\/span>f"\u6a21\u578b\u4e0b\u8f7d\u5b8c\u6210，\u8def\u5f84：<\/span>{<\/span>model_dir}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                        \u4e09\u3001Gemini \u591a\u6a21\u6001\u8868\u793a\u7edf\u4e00\u7684\u6838\u5fc3\u539f\u7406<\/h3>\n

                        3.1 \u8f93\u5165\u5c42：\u6a21\u6001\u7684\u7edf\u4e00\u5e8f\u5217\u5316<\/h4>\n

                        Gemini \u9996\u5148\u5c06\u4e0d\u540c\u6a21\u6001\u8f6c\u5316\u4e3a\u7edf\u4e00\u7684\u5e8f\u5217 Token，\u4e3a\u540e\u7eed\u7684\u5171\u4eab\u7f16\u7801\u5960\u5b9a\u57fa\u7840：<\/p>\n\n\u6a21\u6001\u7c7b\u578b\u5904\u7406\u65b9\u5f0fToken \u751f\u6210\u89c4\u5219<\/tr>\n\n\n\n\n
                        \u6587\u672c<\/td>\n\u5b57\u8282\u5bf9\u7f16\u7801（BPE）<\/td>\n\u6309\u8bed\u4e49\u5207\u5206\u4e3a\u5b50\u8bcd，\u751f\u6210 1D Token \u5e8f\u5217<\/td>\n<\/tr>\n
                        \u56fe\u50cf<\/td>\n\u89c6\u89c9\u5206\u5757（Patch）<\/td>\n\u5c06\u56fe\u50cf\u5207\u5206\u4e3a 16\u00d716 \u50cf\u7d20\u5757，\u751f\u6210 2D Token \u5e8f\u5217<\/td>\n<\/tr>\n
                        \u89c6\u9891<\/td>\n\u65f6\u7a7a\u5206\u5757<\/td>\n\u5148\u6309\u65f6\u95f4\u7ef4\u5ea6\u62c6\u5e27（\u5982 1fps），\u518d\u5bf9\u6bcf\u5e27\u5207\u5206\u4e3a 16\u00d716 Patch，\u751f\u6210 3D Token \u5e8f\u5217（\u65f6\u95f4 + \u7a7a\u95f4）<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

                        \u5728\u8fdb\u5165\u795e\u7ecf\u7f51\u7edc\u4e4b\u524d，Gemini \u5fc5\u987b\u6d88\u9664\u4e0d\u540c\u6570\u636e\u5728\u7269\u7406\u5f62\u6001\u4e0a\u7684\u5dee\u5f02（\u50cf\u7d20\u77e9\u9635 vs \u5b57\u7b26\u7f16\u7801）\u3002\u5b83\u901a\u8fc7 \u901a\u7528\u5e8f\u5217\u5316 (Universal Serialization) \u5c06\u6240\u6709\u6a21\u6001\u8f6c\u5316\u4e3a\u7edf\u4e00\u7684 Token \u5e8f\u5217\u3002<\/p>\n

                          \n
                        • \u6587\u672c (1D \u5e8f\u5217)：\n
                            \n
                          • \u673a\u5236：\u4f7f\u7528 SentencePiece\/BPE \u5206\u8bcd\u3002<\/li>\n
                          • \u8f6c\u5316："\u4e00\u53ea\u732b" \n

                            \u2192<\/p>\n

                            \\\\rightarrow<\/p>\n

                            <\/span><\/span>\u2192<\/span><\/span><\/span><\/span><\/span> [ID: 882, ID: 331]\u3002<\/li>\n<\/ul>\n<\/li>\n

                          • \u56fe\u50cf (2D –> 1D \u5e8f\u5217)：\n
                              \n
                            • \u673a\u5236：Patch Partition\u3002\u5c06\u9ad8\u6e05\u56fe\u50cf\u5207\u5206\u4e3a\u56fa\u5b9a\u5927\u5c0f（\u5982 \n

                              16<\/p>\n

                              \u00d7<\/p>\n

                              16<\/p>\n

                              16 \\\\times 16<\/p>\n

                              <\/span><\/span>16<\/span><\/span>\u00d7<\/span><\/span><\/span><\/span>16<\/span><\/span><\/span><\/span><\/span>）\u7684\u5c0f\u65b9\u5757 (Patch)\u3002<\/li>\n

                            • \u8f6c\u5316：\u6bcf\u4e2a Patch \u88ab\u7ebf\u6027\u6295\u5f71 (Linear Projection) \u5c55\u5e73\u4e3a\u4e00\u4e2a\u5411\u91cf，\u8fd9\u5c31\u76f8\u5f53\u4e8e\u56fe\u50cf\u7684\u4e00\u4e2a\u201c\u5355\u8bcd\u201d\u3002<\/li>\n<\/ul>\n<\/li>\n
                            • \u89c6\u9891 (3D –>1D \u5e8f\u5217)：\n
                                \n
                              • \u673a\u5236：Spatiotemporal Patching (\u65f6\u7a7a\u5206\u5757)\u3002\u4e0d\u4ec5\u5728\u7a7a\u95f4\u4e0a\u5207\u5206 Patch，\u8fd8\u5728\u65f6\u95f4\u8f74\u4e0a\u91c7\u6837\u3002<\/li>\n
                              • \u8f6c\u5316：\u89c6\u9891\u88ab\u89c6\u4e3a\u4e00\u7cfb\u5217\u968f\u65f6\u95f4\u53d8\u5316\u7684\u56fe\u50cf Patch，\u6700\u7ec8\u4e5f\u88ab\u62c9\u5e73\u4e3a\u4e00\u6761\u957f\u957f\u7684 Token \u5e8f\u5217\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                                \u7edf\u4e00\u7ed3\u679c：\u65e0\u8bba\u8f93\u5165\u662f\u6587\u5b57\u3001JPG \u8fd8\u662f MP4，\u5728\u6a21\u578b\u5165\u53e3\u5904，\u5b83\u4eec\u90fd\u53d8\u6210\u4e86\u5f62\u72b6\u76f8\u540c\u7684\u5f20\u91cf [Batch_Size, Sequence_Length, Dimension]\u3002<\/p>\n

                                \u793a\u4f8b：\u89c6\u9891\u5e8f\u5217\u5316\u8fc7\u7a0b<\/p>\n

                                \u8fd0\u884c<\/p>\n

                                import<\/span> cv2
                                \nimport<\/span> numpy as<\/span> np<\/p>\n

                                def<\/span> video_to_patches<\/span>(<\/span>video_path,<\/span> frame_rate=<\/span>1<\/span>,<\/span> patch_size=<\/span>16<\/span>)<\/span>:<\/span>
                                \n """\u5c06\u89c6\u9891\u8f6c\u5316\u4e3a\u65f6\u7a7aPatch\u5e8f\u5217"""<\/span>
                                \n # 1. \u8bfb\u53d6\u89c6\u9891\u5e76\u6309\u5e27\u7387\u62bd\u5e27<\/span>
                                \n cap =<\/span> cv2.<\/span>VideoCapture(<\/span>video_path)<\/span>
                                \n frames =<\/span> [<\/span>]<\/span>
                                \n frame_idx =<\/span> 0<\/span>
                                \n while<\/span> cap.<\/span>isOpened(<\/span>)<\/span>:<\/span>
                                \n ret,<\/span> frame =<\/span> cap.<\/span>read(<\/span>)<\/span>
                                \n if<\/span> not<\/span> ret:<\/span>
                                \n break<\/span>
                                \n # \u6309\u5e27\u7387\u91c7\u6837<\/span>
                                \n if<\/span> frame_idx %<\/span> int<\/span>(<\/span>cap.<\/span>get(<\/span>cv2.<\/span>CAP_PROP_FPS)<\/span> \/<\/span> frame_rate)<\/span> ==<\/span> 0<\/span>:<\/span>
                                \n # \u8f6c\u4e3aRGB\u5e76Resize\u5230\u56fa\u5b9a\u5c3a\u5bf8<\/span>
                                \n frame =<\/span> cv2.<\/span>cvtColor(<\/span>frame,<\/span> cv2.<\/span>COLOR_BGR2RGB)<\/span>
                                \n frame =<\/span> cv2.<\/span>resize(<\/span>frame,<\/span> (<\/span>256<\/span>,<\/span> 256<\/span>)<\/span>)<\/span> # 16\u00d716 Patch \u00d7 16<\/span>
                                \n frames.<\/span>append(<\/span>frame)<\/span>
                                \n frame_idx +=<\/span> 1<\/span>
                                \n cap.<\/span>release(<\/span>)<\/span><\/p>\n

                                # 2. \u751f\u6210\u65f6\u7a7aPatch<\/span>
                                \n patches =<\/span> [<\/span>]<\/span>
                                \n for<\/span> t,<\/span> frame in<\/span> enumerate<\/span>(<\/span>frames)<\/span>:<\/span>
                                \n # \u7a7a\u95f4\u7ef4\u5ea6\u5207\u5206Patch<\/span>
                                \n for<\/span> y in<\/span> range<\/span>(<\/span>0<\/span>,<\/span> frame.<\/span>shape[<\/span>0<\/span>]<\/span>,<\/span> patch_size)<\/span>:<\/span>
                                \n for<\/span> x in<\/span> range<\/span>(<\/span>0<\/span>,<\/span> frame.<\/span>shape[<\/span>1<\/span>]<\/span>,<\/span> patch_size)<\/span>:<\/span>
                                \n patch =<\/span> frame[<\/span>y:<\/span>y+<\/span>patch_size,<\/span> x:<\/span>x+<\/span>patch_size]<\/span>
                                \n # \u8bb0\u5f55\u65f6\u7a7a\u5750\u6807 (t, y, x)<\/span>
                                \n patches.<\/span>append(<\/span>{<\/span>
                                \n "patch"<\/span>:<\/span> patch.<\/span>flatten(<\/span>)<\/span>,<\/span>
                                \n "coords"<\/span>:<\/span> (<\/span>t,<\/span> y\/\/<\/span>patch_size,<\/span> x\/\/<\/span>patch_size)<\/span>
                                \n }<\/span>)<\/span><\/p>\n

                                # 3. \u8f6c\u5316\u4e3a\u7edf\u4e00\u5e8f\u5217（\u65f6\u95f4\u2192\u7a7a\u95f4\u5c55\u5f00）<\/span>
                                \n patch_sequence =<\/span> np.<\/span>array(<\/span>[<\/span>p[<\/span>"patch"<\/span>]<\/span> for<\/span> p in<\/span> patches]<\/span>)<\/span>
                                \n return<\/span> patch_sequence,<\/span> frames<\/p>\n

                                # \u6d4b\u8bd5\u89c6\u9891\u5e8f\u5217\u5316<\/span>
                                \nvideo_path =<\/span> "sample_video.mp4"<\/span>
                                \npatch_seq,<\/span> frames =<\/span> video_to_patches(<\/span>video_path)<\/span>
                                \nprint<\/span>(<\/span>f"\u89c6\u9891\u62bd\u5e27\u6570\u91cf：<\/span>{<\/span>len<\/span>(<\/span>frames)<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
                                \nprint<\/span>(<\/span>f"\u751f\u6210Patch\u5e8f\u5217\u957f\u5ea6：<\/span>{<\/span>len<\/span>(<\/span>patch_seq)<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
                                \nprint<\/span>(<\/span>f"\u5355\u4e2aPatch\u7ef4\u5ea6：<\/span>{<\/span>patch_seq[<\/span>0<\/span>]<\/span>.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                3.2 \u7f16\u7801\u5c42：\u5171\u4eab Transformer \u4e0e\u8de8\u6a21\u6001\u6ce8\u610f\u529b<\/h4>\n

                                Gemini \u7684\u6838\u5fc3\u662f\u5171\u4eab\u7684 Transformer \u7f16\u7801\u5668，\u6240\u6709\u6a21\u6001\u7684 Token \u5e8f\u5217\u8f93\u5165\u540c\u4e00\u5957 Transformer \u7f51\u7edc，\u5e76\u901a\u8fc7\u8de8\u6a21\u6001\u6ce8\u610f\u529b\u673a\u5236\u5b9e\u73b0\u8bed\u4e49\u5bf9\u9f50：<\/p>\n

                              • \u57fa\u7840\u6ce8\u610f\u529b\u5c42：\u5bf9\u5355\u4e00\u6a21\u6001 Token \u8fdb\u884c\u81ea\u6ce8\u610f\u529b\u8ba1\u7b97（\u5982\u6587\u672c\u5185\u90e8\u7684\u8bed\u4e49\u5173\u8054\u3001\u56fe\u50cf Patch \u7684\u7a7a\u95f4\u5173\u8054）<\/li>\n
                              • \u4ea4\u53c9\u6ce8\u610f\u529b\u5c42：\u5efa\u6a21\u4e0d\u540c\u6a21\u6001 Token \u95f4\u7684\u5173\u8054（\u5982 \u201c\u7ea2\u8272\u6c7d\u8f66\u201d \u6587\u672c Token \u4e0e\u56fe\u50cf\u4e2d\u7ea2\u8272\u6c7d\u8f66\u533a\u57df Patch \u7684\u5173\u8054）<\/li>\n
                              • \u65f6\u5e8f\u6ce8\u610f\u529b\u5c42（\u89c6\u9891\u4e13\u7528）：\u5bf9\u89c6\u9891\u7684\u65f6\u5e8f Token \u6dfb\u52a0\u65f6\u95f4\u7ef4\u5ea6\u7684\u6ce8\u610f\u529b\u6743\u91cd，\u6355\u6349\u5e27\u95f4\u52a8\u6001<\/li>\n

                                graph TD
                                \n A[<\/span>\u6587\u672c Token]<\/span> —><\/span> D{<\/span>\u5171\u4eab Transformer}<\/span>
                                \n B[<\/span>\u56fe\u50cf Patch Token]<\/span> —><\/span> D
                                \n C[<\/span>\u89c6\u9891\u65f6\u7a7a Token]<\/span> —><\/span> D<\/p>\n

                                subgraph "Gemini \u7edf\u4e00\u7f16\u7801\u5668"<\/span>
                                \n D –\u81ea\u6ce8\u610f\u529b\u673a\u5236 (<\/span>Self-Attention)<\/span>—><\/span> E[<\/span>\u4ea4\u4e92\u4e0e\u878d\u5408]<\/span>
                                \n E –\u524d\u9988\u7f51\u7edc (<\/span>FFN)<\/span>—><\/span> F[<\/span>\u8bed\u4e49\u7279\u5f81\u63d0\u53d6]<\/span>
                                \n end<\/p>\n

                                F —><\/span> G[<\/span>\u7edf\u4e00\u5411\u91cf\u7a7a\u95f4]<\/span><\/p>\n

                                \u6838\u5fc3\u4ee3\u7801：\u8de8\u6a21\u6001\u6ce8\u610f\u529b\u5b9e\u73b0<\/p>\n

                                \u8fd0\u884c<\/p>\n

                                import<\/span> torch
                                \nimport<\/span> torch.<\/span>nn as<\/span> nn
                                \nimport<\/span> torch.<\/span>nn.<\/span>functional as<\/span> F<\/p>\n

                                class<\/span> CrossModalAttention<\/span>(<\/span>nn.<\/span>Module)<\/span>:<\/span>
                                \n def<\/span> __init__<\/span>(<\/span>self,<\/span> d_model=<\/span>2048<\/span>,<\/span> n_heads=<\/span>16<\/span>)<\/span>:<\/span>
                                \n super<\/span>(<\/span>)<\/span>.<\/span>__init__(<\/span>)<\/span>
                                \n self.<\/span>d_model =<\/span> d_model
                                \n self.<\/span>n_heads =<\/span> n_heads
                                \n self.<\/span>head_dim =<\/span> d_model \/\/<\/span> n_heads<\/p>\n

                                # \u5171\u4eab\u7684\u6ce8\u610f\u529b\u6295\u5f71\u5c42<\/span>
                                \n self.<\/span>q_proj =<\/span> nn.<\/span>Linear(<\/span>d_model,<\/span> d_model)<\/span>
                                \n self.<\/span>k_proj =<\/span> nn.<\/span>Linear(<\/span>d_model,<\/span> d_model)<\/span>
                                \n self.<\/span>v_proj =<\/span> nn.<\/span>Linear(<\/span>d_model,<\/span> d_model)<\/span>
                                \n self.<\/span>out_proj =<\/span> nn.<\/span>Linear(<\/span>d_model,<\/span> d_model)<\/span><\/p>\n

                                def<\/span> forward<\/span>(<\/span>self,<\/span> text_embeds,<\/span> visual_embeds)<\/span>:<\/span>
                                \n """
                                \n \u8de8\u6a21\u6001\u6ce8\u610f\u529b\u8ba1\u7b97：\u4ee5\u6587\u672c\u4e3aQuery，\u89c6\u89c9（\u56fe\u50cf\/\u89c6\u9891）\u4e3aKey\/Value
                                \n Args:
                                \n text_embeds: [batch, text_len, d_model] \u6587\u672c\u5d4c\u5165
                                \n visual_embeds: [batch, visual_len, d_model] \u89c6\u89c9\u5d4c\u5165
                                \n Returns:
                                \n cross_embeds: [batch, text_len + visual_len, d_model] \u878d\u5408\u540e\u7684\u5d4c\u5165
                                \n """<\/span>
                                \n batch_size =<\/span> text_embeds.<\/span>shape[<\/span>0<\/span>]<\/span><\/p>\n

                                # 1. \u6295\u5f71\u4e3aQuery\/Key\/Value<\/span>
                                \n q =<\/span> self.<\/span>q_proj(<\/span>text_embeds)<\/span> # \u6587\u672c\u4f5c\u4e3aQuery<\/span>
                                \n k =<\/span> self.<\/span>k_proj(<\/span>visual_embeds)<\/span> # \u89c6\u89c9\u4f5c\u4e3aKey<\/span>
                                \n v =<\/span> self.<\/span>v_proj(<\/span>visual_embeds)<\/span> # \u89c6\u89c9\u4f5c\u4e3aValue<\/span><\/p>\n

                                # 2. \u5206\u62c6\u6ce8\u610f\u529b\u5934<\/span>
                                \n q =<\/span> q.<\/span>view(<\/span>batch_size,<\/span> –<\/span>1<\/span>,<\/span> self.<\/span>n_heads,<\/span> self.<\/span>head_dim)<\/span>.<\/span>transpose(<\/span>1<\/span>,<\/span> 2<\/span>)<\/span>
                                \n k =<\/span> k.<\/span>view(<\/span>batch_size,<\/span> –<\/span>1<\/span>,<\/span> self.<\/span>n_heads,<\/span> self.<\/span>head_dim)<\/span>.<\/span>transpose(<\/span>1<\/span>,<\/span> 2<\/span>)<\/span>
                                \n v =<\/span> v.<\/span>view(<\/span>batch_size,<\/span> –<\/span>1<\/span>,<\/span> self.<\/span>n_heads,<\/span> self.<\/span>head_dim)<\/span>.<\/span>transpose(<\/span>1<\/span>,<\/span> 2<\/span>)<\/span><\/p>\n

                                # 3. \u8ba1\u7b97\u6ce8\u610f\u529b\u5206\u6570（\u7f29\u653e\u70b9\u79ef）<\/span>
                                \n scores =<\/span> torch.<\/span>matmul(<\/span>q,<\/span> k.<\/span>transpose(<\/span>–<\/span>2<\/span>,<\/span> –<\/span>1<\/span>)<\/span>)<\/span> \/<\/span> torch.<\/span>sqrt(<\/span>torch.<\/span>tensor(<\/span>self.<\/span>head_dim,<\/span> dtype=<\/span>torch.<\/span>float32)<\/span>)<\/span>
                                \n attn_weights =<\/span> F.<\/span>softmax(<\/span>scores,<\/span> dim=<\/span>–<\/span>1<\/span>)<\/span><\/p>\n

                                # 4. \u6ce8\u610f\u529b\u52a0\u6743\u6c42\u548c<\/span>
                                \n cross_output =<\/span> torch.<\/span>matmul(<\/span>attn_weights,<\/span> v)<\/span>
                                \n cross_output =<\/span> cross_output.<\/span>transpose(<\/span>1<\/span>,<\/span> 2<\/span>)<\/span>.<\/span>contiguous(<\/span>)<\/span>.<\/span>view(<\/span>batch_size,<\/span> –<\/span>1<\/span>,<\/span> self.<\/span>d_model)<\/span>
                                \n cross_output =<\/span> self.<\/span>out_proj(<\/span>cross_output)<\/span><\/p>\n

                                # 5. \u62fc\u63a5\u6587\u672c\u4e0e\u89c6\u89c9\u5d4c\u5165（\u4fdd\u6301\u5e8f\u5217\u5b8c\u6574\u6027）<\/span>
                                \n cross_embeds =<\/span> torch.<\/span>cat(<\/span>[<\/span>text_embeds +<\/span> cross_output,<\/span> visual_embeds]<\/span>,<\/span> dim=<\/span>1<\/span>)<\/span>
                                \n return<\/span> cross_embeds<\/p>\n

                                # \u6d4b\u8bd5\u8de8\u6a21\u6001\u6ce8\u610f\u529b<\/span>
                                \nif<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
                                \n # \u6a21\u62df\u6587\u672c\u548c\u89c6\u89c9\u5d4c\u5165<\/span>
                                \n batch_size =<\/span> 2<\/span>
                                \n text_len =<\/span> 32<\/span>
                                \n visual_len =<\/span> 256<\/span>
                                \n d_model =<\/span> 2048<\/span><\/p>\n

                                text_embeds =<\/span> torch.<\/span>randn(<\/span>batch_size,<\/span> text_len,<\/span> d_model)<\/span>
                                \n visual_embeds =<\/span> torch.<\/span>randn(<\/span>batch_size,<\/span> visual_len,<\/span> d_model)<\/span><\/p>\n

                                # \u521d\u59cb\u5316\u8de8\u6a21\u6001\u6ce8\u610f\u529b\u5c42<\/span>
                                \n cross_attn =<\/span> CrossModalAttention(<\/span>d_model=<\/span>d_model,<\/span> n_heads=<\/span>16<\/span>)<\/span>
                                \n cross_embeds =<\/span> cross_attn(<\/span>text_embeds,<\/span> visual_embeds)<\/span><\/p>\n

                                print<\/span>(<\/span>f"\u878d\u5408\u540e\u5d4c\u5165\u7ef4\u5ea6：<\/span>{<\/span>cross_embeds.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span> # [2, 32+256=288, 2048]<\/span><\/p>\n

                                3.3 \u8f93\u51fa\u5c42：\u7edf\u4e00\u7684\u5411\u91cf\u8868\u793a\u7a7a\u95f4<\/h4>\n

                                Gemini \u901a\u8fc7\u4ee5\u4e0b\u65b9\u5f0f\u4fdd\u8bc1\u6240\u6709\u6a21\u6001\u8f93\u51fa\u5411\u91cf\u7684\u7edf\u4e00\u53ef\u6bd4\u8f83\u6027：<\/p>\n

                              • \u5411\u91cf\u5f52\u4e00\u5316：\u6240\u6709\u6a21\u6001\u7684\u8f93\u51fa\u5411\u91cf\u7ecf\u8fc7 L2 \u5f52\u4e00\u5316，\u786e\u4fdd\u5411\u91cf\u957f\u5ea6\u4e00\u81f4<\/li>\n
                              • \u7ef4\u5ea6\u5bf9\u9f50：\u5f3a\u5236\u6240\u6709\u6a21\u6001\u7684\u8f93\u51fa\u5411\u91cf\u7ef4\u5ea6\u76f8\u540c（\u5982 2048 \u7ef4）<\/li>\n
                              • \u8bed\u4e49\u6821\u51c6：\u901a\u8fc7\u5927\u89c4\u6a21\u8de8\u6a21\u6001\u5bf9\u6bd4\u5b66\u4e60，\u4f7f\u4e0d\u540c\u6a21\u6001\u7684\u8bed\u4e49\u76f8\u4f3c\u5185\u5bb9\u5728\u5411\u91cf\u7a7a\u95f4\u4e2d\u8ddd\u79bb\u66f4\u8fd1<\/li>\n

                                \u793a\u4f8b：\u591a\u6a21\u6001\u5411\u91cf\u5f52\u4e00\u5316<\/p>\n

                                \u8fd0\u884c<\/p>\n

                                def<\/span> normalize_embeddings<\/span>(<\/span>embeddings)<\/span>:<\/span>
                                \n """L2\u5f52\u4e00\u5316，\u4fdd\u8bc1\u5411\u91cf\u957f\u5ea6\u4e3a1"""<\/span>
                                \n norm =<\/span> torch.<\/span>norm(<\/span>embeddings,<\/span> p=<\/span>2<\/span>,<\/span> dim=<\/span>–<\/span>1<\/span>,<\/span> keepdim=<\/span>True<\/span>)<\/span>
                                \n return<\/span> embeddings \/<\/span> (<\/span>norm +<\/span> 1e-8<\/span>)<\/span><\/p>\n

                                # \u6a21\u62df\u4e0d\u540c\u6a21\u6001\u7684\u539f\u59cb\u5d4c\u5165<\/span>
                                \ntext_emb =<\/span> torch.<\/span>randn(<\/span>1<\/span>,<\/span> 2048<\/span>)<\/span> # \u6587\u672c\u5d4c\u5165<\/span>
                                \nimage_emb =<\/span> torch.<\/span>randn(<\/span>1<\/span>,<\/span> 2048<\/span>)<\/span> # \u56fe\u50cf\u5d4c\u5165<\/span>
                                \nvideo_emb =<\/span> torch.<\/span>randn(<\/span>1<\/span>,<\/span> 2048<\/span>)<\/span> # \u89c6\u9891\u5d4c\u5165<\/span><\/p>\n

                                # \u5f52\u4e00\u5316<\/span>
                                \ntext_emb_norm =<\/span> normalize_embeddings(<\/span>text_emb)<\/span>
                                \nimage_emb_norm =<\/span> normalize_embeddings(<\/span>image_emb)<\/span>
                                \nvideo_emb_norm =<\/span> normalize_embeddings(<\/span>video_emb)<\/span><\/p>\n

                                # \u9a8c\u8bc1\u5f52\u4e00\u5316\u7ed3\u679c<\/span>
                                \nprint<\/span>(<\/span>f"\u6587\u672c\u5411\u91cf\u957f\u5ea6：<\/span>{<\/span>torch.<\/span>norm(<\/span>text_emb_norm)<\/span>.<\/span>item(<\/span>)<\/span>:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span> # \u5e94\u4e3a 1.0<\/span>
                                \nprint<\/span>(<\/span>f"\u56fe\u50cf\u5411\u91cf\u957f\u5ea6：<\/span>{<\/span>torch.<\/span>norm(<\/span>image_emb_norm)<\/span>.<\/span>item(<\/span>)<\/span>:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span> # \u5e94\u4e3a 1.0<\/span>
                                \nprint<\/span>(<\/span>f"\u89c6\u9891\u5411\u91cf\u957f\u5ea6：<\/span>{<\/span>torch.<\/span>norm(<\/span>video_emb_norm)<\/span>.<\/span>item(<\/span>)<\/span>:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span> # \u5e94\u4e3a 1.0<\/span><\/p>\n

                                # \u8ba1\u7b97\u8de8\u6a21\u6001\u76f8\u4f3c\u5ea6<\/span>
                                \ntext_image_sim =<\/span> torch.<\/span>cosine_similarity(<\/span>text_emb_norm,<\/span> image_emb_norm)<\/span>
                                \ntext_video_sim =<\/span> torch.<\/span>cosine_similarity(<\/span>text_emb_norm,<\/span> video_emb_norm)<\/span>
                                \nprint<\/span>(<\/span>f"\u6587\u672c-\u56fe\u50cf\u76f8\u4f3c\u5ea6：<\/span>{<\/span>text_image_sim.<\/span>item(<\/span>)<\/span>:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
                                \nprint<\/span>(<\/span>f"\u6587\u672c-\u89c6\u9891\u76f8\u4f3c\u5ea6：<\/span>{<\/span>text_video_sim.<\/span>item(<\/span>)<\/span>:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                3.4 \u603b\u7ed3：\u4f20\u7edf\u67b6\u6784 vs Gemini \u67b6\u6784<\/h5>\n

                                \u4e3a\u4e86\u66f4\u76f4\u89c2\u5730\u7406\u89e3，\u6211\u4eec\u53ef\u4ee5\u5bf9\u6bd4\u4e24\u79cd\u67b6\u6784：<\/p>\n\n\u7279\u6027\u4f20\u7edf\u591a\u6a21\u6001 (\u5982 CLIP + LLM)Gemini (\u539f\u751f\u591a\u6a21\u6001)<\/tr>\n\n\n\n\n\n\n
                                \u7f16\u7801\u5668<\/td>\n\u5272\u88c2：\u6709\u4e00\u4e2a\u89c6\u89c9\u7f16\u7801\u5668 (ViT) \u548c\u4e00\u4e2a\u6587\u672c\u7f16\u7801\u5668 (BERT\/GPT)<\/td>\n\u7edf\u4e00：\u53ea\u6709\u4e00\u4e2a\u5171\u4eab\u7684 Transformer<\/td>\n<\/tr>\n
                                \u4fe1\u606f\u4ea4\u4e92<\/td>\n\u665a\u671f\u878d\u5408：\u4ec5\u5728\u6700\u540e\u8f93\u51fa\u5c42\u6216\u901a\u8fc7 Adapter \u7b80\u5355\u7684\u62fc\u63a5<\/td>\n\u5168\u7a0b\u878d\u5408：\u4ece\u7b2c\u4e00\u5c42\u5f00\u59cb，\u6587\u672c\u548c\u56fe\u50cf Token \u5c31\u6df7\u5408\u8ba1\u7b97\u6ce8\u610f\u529b<\/td>\n<\/tr>\n
                                \u89c6\u9891\u5904\u7406<\/td>\n\u901a\u5e38\u89c6\u4e3a\u201c\u56fe\u50cf\u7684\u5e73\u5747\u503c\u201d，\u4e22\u5931\u65f6\u5e8f\u4fe1\u606f<\/td>\n\u65f6\u7a7a\u5efa\u6a21：\u5c06\u65f6\u95f4\u4f5c\u4e3a Token \u7684\u4e00\u4e2a\u7ef4\u5ea6，\u7406\u89e3\u52a8\u4f5c\u548c\u53d8\u5316<\/td>\n<\/tr>\n
                                \u4f18\u52bf<\/td>\n\u8bad\u7ec3\u6210\u672c\u8f83\u4f4e，\u6a21\u5757\u53ef\u66ff\u6362<\/td>\n\u7406\u89e3\u529b\u66f4\u5f3a，\u80fd\u5904\u7406\u590d\u6742\u7684\u56fe\u6587\u63a8\u7406\u548c\u957f\u89c6\u9891\u7406\u89e3<\/td>\n<\/tr>\n
                                \u4ee3\u7801\u4f53\u73b0<\/td>\n\u9700\u8981\u52a0\u8f7d\u4e24\u4e2a\u6a21\u578b\u6743\u91cd\u6587\u4ef6<\/td>\n\u53ea\u52a0\u8f7d\u4e00\u4e2a\u6a21\u578b\u6743\u91cd (\u5982 google\/gemma-2-9b-it)<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

                                \u56db\u3001\u5b9e\u6218：Gemini \u7edf\u4e00\u6587\u672c \/ \u56fe\u50cf \/ \u89c6\u9891\u8868\u793a\u7684\u5b8c\u6574\u5b9e\u73b0<\/h3>\n

                                4.1 \u9879\u76ee\u7ed3\u6784\u8bbe\u8ba1<\/h4>\n

                                gemini-multimodal\/
                                \n\u251c\u2500\u2500 .env # [\u5bc6\u94a5\u91d1\u5e93] \u5b58\u653e API Key\u3001\u672c\u5730\u6a21\u578b\u7edd\u5bf9\u8def\u5f84，\u9694\u79bb\u654f\u611f\u73af\u5883\u914d\u7f6e<\/span>
                                \n\u251c\u2500\u2500 requirements.txt # [\u4f9d\u8d56\u6e05\u5355] \u5b9a\u4e49 PyTorch\u3001Transformers\u3001ChromaDB \u7b49\u6838\u5fc3\u8ba1\u7b97\u5e93\u7248\u672c<\/span>
                                \n\u251c\u2500\u2500 config.py # [\u7edf\u4e00\u6807\u51c6] \u5b9a\u4e49\u8de8\u6a21\u6001\u5171\u4eab\u7684\u7ef4\u5ea6(D_MODEL)\u3001\u6700\u5927\u5e8f\u5217\u957f\u5ea6\u53ca\u786c\u4ef6\u53c2\u6570<\/span>
                                \n\u251c\u2500\u2500 main.py # [\u4e2d\u63a7\u53f0] \u7cfb\u7edf\u5165\u53e3，\u8d1f\u8d23\u8c03\u5ea6\u9884\u5904\u7406\u3001\u7f16\u7801\u4e0e\u68c0\u7d22\u6a21\u5757\u7684\u6d41\u6c34\u7ebf\u4ea4\u4e92<\/span>
                                \n\u2502
                                \n\u251c\u2500\u2500 core\/ # [\u591a\u6a21\u6001\u5f15\u64ce] \u6838\u5fc3\u7b97\u6cd5\u5b9e\u73b0\u5e93（\u672c\u7cfb\u7edf\u7684\u201c\u5fc3\u810f\u201d）<\/span>
                                \n\u2502 \u251c\u2500\u2500 __init__.py # \u5305\u521d\u59cb\u5316\u6587\u4ef6<\/span>
                                \n\u2502 \u251c\u2500\u2500 model_setup.py # [\u5171\u4eab\u9aa8\u67b6] \u52a0\u8f7d Gemini \u7edf\u4e00 Transformer \u7f16\u7801\u5668\u4e0e\u8de8\u6a21\u6001\u9002\u914d\u5c42<\/span>
                                \n\u2502 \u251c\u2500\u2500 modal_processor.py# [\u5e8f\u5217\u5316\u5de5\u5382] \u5c06\u6587\u672c\/\u56fe\u50cf\/\u89c6\u9891\u8f6c\u5316\u4e3a\u7edf\u4e00\u683c\u5f0f\u7684 Token \u5e8f\u5217<\/span>
                                \n\u2502 \u251c\u2500\u2500 embedding.py # [\u7edf\u4e00\u6620\u5c04] \u6267\u884c\u524d\u5411\u4f20\u64ad，\u8f93\u51fa\u5e76\u5f52\u4e00\u5316\u591a\u6a21\u6001\u5411\u91cf（\u8bed\u4e49\u5bf9\u9f50\u6838\u5fc3）<\/span>
                                \n\u2502 \u2514\u2500\u2500 retriever.py # [\u6df7\u5408\u8bb0\u5fc6] \u7ba1\u7406\u5411\u91cf\u6570\u636e\u5e93，\u6267\u884c\u8de8\u6a21\u6001（\u5982\u6587\u641c\u56fe\u3001\u56fe\u641c\u89c6\u9891）\u68c0\u7d22<\/span>
                                \n\u2502
                                \n\u251c\u2500\u2500 data\/ # [\u5f02\u6784\u6570\u636e\u6e90] \u5b58\u653e\u5f85\u5904\u7406\u7684\u591a\u6a21\u6001\u539f\u59cb\u6587\u4ef6<\/span>
                                \n\u2502 \u251c\u2500\u2500 text\/ # [\u6587\u672c\u8bed\u6599] .txt \/ .json \u683c\u5f0f\u7684\u6587\u672c\u63cf\u8ff0<\/span>
                                \n\u2502 \u251c\u2500\u2500 images\/ # [\u9759\u6001\u89c6\u89c9] .jpg \/ .png \u683c\u5f0f\u7684\u56fe\u50cf\u6587\u4ef6<\/span>
                                \n\u2502 \u2514\u2500\u2500 videos\/ # [\u52a8\u6001\u89c6\u89c9] .mp4 \/ .avi \u683c\u5f0f\u7684\u89c6\u9891\u7247\u6bb5<\/span>
                                \n\u2502
                                \n\u251c\u2500\u2500 embeddings_db\/ # [\u8bed\u4e49\u7a7a\u95f4] \u6301\u4e45\u5316\u7684\u5411\u91cf\u5b58\u50a8<\/span>
                                \n\u2502 \u2514\u2500\u2500 chroma\/ # [ChromaDB] \u5b58\u50a8\u5df2\u5bf9\u9f50\u7684\u9ad8\u7ef4\u5411\u91cf\u53ca\u5176\u5143\u6570\u636e（Metadata）<\/span>
                                \n\u2502
                                \n\u2514\u2500\u2500 logs\/ # [\u8fd0\u884c\u8bb0\u5f55] <\/span>
                                \n \u2514\u2500\u2500 multimodal.log # \u8bb0\u5f55\u7f16\u7801\u5ef6\u8fdf\u3001\u6a21\u6001\u8f6c\u6362\u9519\u8bef\u53ca\u68c0\u7d22\u5339\u914d\u5ea6\u65e5\u5fd7<\/span><\/p>\n

                                4.1.2 \u6838\u5fc3\u6a21\u5757\u6df1\u5ea6\u89e3\u6790 (The Components)<\/h5>\n

                                \u8fd9\u4e9b\u6587\u4ef6\u5171\u540c\u6784\u6210\u4e86\u4e00\u4e2a\u80fd\u591f\u7406\u89e3\u591a\u79cd\u6a21\u6001\u7684\u201c\u7edf\u4e00\u5927\u8111\u201d\u3002\u4e0e\u4f20\u7edf\u591a\u6a21\u6001\u9879\u76ee\u4e0d\u540c，\u8fd9\u91cc\u7684\u6838\u5fc3\u903b\u8f91\u5728\u4e8e\u5982\u4f55\u62b9\u5e73\u4e0d\u540c\u6a21\u6001\u4e4b\u95f4\u7684\u5dee\u5f02\u3002<\/p>\n

                                1. \u57fa\u7840\u8bbe\u65bd\u4e0e\u6807\u51c6\u5b9a\u4e49 (The Infrastructure)<\/p>\n

                                  \n
                                • config.py (\u7edf\u4e00\u6807\u51c6)\n
                                    \n
                                  • \u7528\u9014：\u8fd9\u662f\u7cfb\u7edf\u7684\u201c\u5ea6\u91cf\u8861\u201d\u3002<\/li>\n
                                  • \u6df1\u5ea6\u89e3\u6790：\u5b83\u5b9a\u4e49\u4e86 Gemini \u7edf\u4e00\u8868\u793a\u7a7a\u95f4 \u7684\u7269\u7406\u6cd5\u5219\u3002\u6700\u5173\u952e\u7684\u53c2\u6570\u662f D_MODEL = 2048（\u6216\u6839\u636e\u6a21\u578b\u53d8\u52a8），\u8fd9\u5f3a\u5236\u8981\u6c42\u65e0\u8bba\u662f\u6587\u672c\u3001\u56fe\u50cf\u8fd8\u662f\u89c6\u9891，\u6700\u7ec8\u8f93\u51fa\u7684\u5411\u91cf\u5fc5\u987b\u662f\u8fd9\u4e2a\u7ef4\u5ea6\u3002\u540c\u65f6，\u5b83\u5b9a\u4e49\u4e86 MAX_SEQ_LEN，\u786e\u4fdd\u4e0d\u540c\u6a21\u6001\u5728\u5e8f\u5217\u5316\u540e\u80fd\u9002\u914d\u540c\u4e00\u4e2a Transformer \u7684\u4e0a\u4e0b\u6587\u7a97\u53e3\u3002<\/li>\n<\/ul>\n<\/li>\n
                                  • core\/model_setup.py (\u5171\u4eab\u9aa8\u67b6)\n
                                      \n
                                    • \u7528\u9014：\u52a0\u8f7d Gemini \u7684\u201c\u5355\u4e00\u6a21\u578b\u6743\u91cd\u201d\u3002<\/li>\n
                                    • \u6df1\u5ea6\u89e3\u6790：\u4f20\u7edf\u67b6\u6784\u53ef\u80fd\u9700\u8981\u5206\u522b\u52a0\u8f7d BERT（\u5904\u7406\u6587\u672c）\u548c ResNet（\u5904\u7406\u56fe\u50cf）\u3002\u800c\u6b64\u6587\u4ef6\u901a\u8fc7 AutoModel.from_pretrained \u52a0\u8f7d \u540c\u4e00\u4e2a Transformer \u9aa8\u5e72\u7f51\u7edc\u3002\u5b83\u786e\u4fdd\u6587\u672c\u548c\u89c6\u89c9\u4fe1\u53f7\u6d41\u5411\u540c\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc，\u8fd9\u662f\u5b9e\u73b0\u201c\u539f\u751f\u591a\u6a21\u6001\u201d\u7684\u7269\u7406\u57fa\u7840\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                                      2. \u7edf\u4e00\u8868\u793a\u5f15\u64ce (The Unification Engine)<\/p>\n

                                      \u8fd9\u662f\u9879\u76ee\u4e2d\u6700\u6838\u5fc3\u7684\u90e8\u5206，\u8d1f\u8d23\u6267\u884c\u201c\u8f93\u5165\u5bf9\u9f50\u201d\u548c\u201c\u8f93\u51fa\u5bf9\u9f50\u201d\u3002<\/p>\n

                                        \n
                                      • core\/modal_processor.py (\u5e8f\u5217\u5316\u5de5\u5382 \/ Input Layer)\n
                                          \n
                                        • \u7528\u9014：\u6a21\u6001\u7684\u201c\u7c89\u788e\u673a\u201d\u4e0e\u201c\u5305\u88c5\u673a\u201d\u3002<\/li>\n
                                        • \u534f\u4f5c\u903b\u8f91：\n
                                            \n
                                          • \u6587\u672c：\u8c03\u7528 Tokenizer \u8fdb\u884c BPE \u7f16\u7801，\u751f\u6210 1D ID \u5e8f\u5217\u3002<\/li>\n
                                          • \u56fe\u50cf：\u8c03\u7528 ImageProcessor \u5c06\u56fe\u7247\u5207\u5206\u4e3a 16×16 \u7684 Patch，\u62c9\u5e73\u6210\u5e8f\u5217\u3002<\/li>\n
                                          • \u89c6\u9891：\u8fd9\u662f\u5173\u952e\u3002\u5b83\u6267\u884c\u201c\u65f6\u7a7a\u91c7\u6837\u201d，\u5148\u6309\u65f6\u95f4\u62bd\u5e27，\u518d\u6309\u7a7a\u95f4\u5207\u5757，\u6700\u7ec8\u751f\u6210\u5e26\u6709\u65f6\u95f4\u6233\u4fe1\u606f\u7684 3D Token \u5e8f\u5217\u3002<\/li>\n<\/ul>\n<\/li>\n
                                          • \u6838\u5fc3\u4ef7\u503c：\u5b83\u5411\u6a21\u578b\u5c4f\u853d\u4e86\u539f\u59cb\u6570\u636e\u7684\u683c\u5f0f\u5dee\u5f02，\u8ba9\u6a21\u578b\u53ea\u770b\u5230\u201cToken \u5e8f\u5217\u201d\u3002<\/li>\n<\/ul>\n<\/li>\n
                                          • core\/embedding.py (\u7edf\u4e00\u6620\u5c04 \/ Output Layer)\n
                                              \n
                                            • \u7528\u9014：\u5411\u91cf\u751f\u6210\u5668\u4e0e\u5f52\u4e00\u5316\u5668\u3002<\/li>\n
                                            • \u6df1\u5ea6\u89e3\u6790：\n
                                                \n
                                              • \u6295\u5f71 (Projection)：\u5982\u679c\u539f\u59cb\u6a21\u578b\u7684\u89c6\u89c9\u5934\u8f93\u51fa\u7ef4\u5ea6（\u5982 1024）\u4e0e\u6587\u672c\u5934（\u5982 4096）\u4e0d\u4e00\u81f4，\u6b64\u6a21\u5757\u4f1a\u901a\u8fc7\u7ebf\u6027\u5c42 (nn.Linear) \u5c06\u5b83\u4eec\u5f3a\u5236\u6295\u5f71\u5230 config.py \u5b9a\u4e49\u7684 D_MODEL \u7ef4\u5ea6\u3002<\/li>\n
                                              • \u5f52\u4e00\u5316 (Normalization)：\u6267\u884c L2 \u5f52\u4e00\u5316\u3002\u8fd9\u662f\u8de8\u6a21\u6001\u68c0\u7d22\u7cbe\u5ea6\u7684\u4fdd\u8bc1，\u5b83\u786e\u4fdd\u201c\u4e00\u53ea\u732b\u201d\u7684\u56fe\u7247\u5411\u91cf\u548c\u201ccat\u201d\u7684\u6587\u672c\u5411\u91cf\u843d\u5728\u540c\u4e00\u4e2a\u8d85\u7403\u9762\u4e0a，\u4f7f\u5176\u4f59\u5f26\u76f8\u4f3c\u5ea6\u53ef\u8ba1\u7b97\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                                                3. \u8bb0\u5fc6\u4e0e\u68c0\u7d22 (The Memory & Retrieval)<\/p>\n

                                                  \n
                                                • core\/retriever.py (\u6df7\u5408\u8bb0\u5fc6)\n
                                                    \n
                                                  • \u7528\u9014：\u8de8\u6a21\u6001\u68c0\u7d22\u5f15\u64ce\u3002<\/li>\n
                                                  • \u6df1\u5ea6\u89e3\u6790：\u5b83\u4e0d\u533a\u5206\u6570\u636e\u6765\u6e90\u3002\u5728 ChromaDB \u4e2d，\u5b83\u5c06\u6587\u672c\u5411\u91cf\u3001\u56fe\u50cf\u5411\u91cf\u548c\u89c6\u9891\u5411\u91cf\u5b58\u50a8\u5728\u540c\u4e00\u4e2a\u96c6\u5408 (Collection) \u4e2d\u3002<\/li>\n
                                                  • \u5de5\u4f5c\u6d41：\u5f53\u4f60\u8f93\u5165\u4e00\u6bb5\u6587\u5b57\u201c\u6d77\u8fb9\u65e5\u843d\u201d\u65f6，\u6b64\u6a21\u5757\u5c06\u5176\u8f6c\u5316\u4e3a\u5411\u91cf，\u5e76\u5728\u540c\u4e00\u4e2a\u6570\u5b66\u7a7a\u95f4\u4e2d\u540c\u65f6\u5bfb\u627e\u8ddd\u79bb\u6700\u8fd1\u7684\u56fe\u7247\u548c\u89c6\u9891\u3002\u8fd9\u8bc1\u660e\u4e86\u4e0d\u540c\u6a21\u6001\u7684\u6570\u636e\u5df2\u7ecf\u5728\u6570\u5b66\u4e0a\u5b9e\u73b0\u4e86\u201c\u7edf\u4e00\u201d\u3002<\/li>\n<\/ul>\n<\/li>\n
                                                  • embeddings_db\/ (\u8bed\u4e49\u7a7a\u95f4)\n
                                                      \n
                                                    • \u7528\u9014：\u6301\u4e45\u5316\u5b58\u50a8\u3002<\/li>\n
                                                    • \u6838\u5fc3\u7279\u70b9：\u8fd9\u91cc\u7684\u4e8c\u8fdb\u5236\u6587\u4ef6\u4e0d\u4ec5\u5b58\u50a8\u4e86\u5411\u91cf，\u8fd8\u5b58\u50a8\u4e86 modal_type (\u6a21\u6001\u7c7b\u578b) \u7b49\u5143\u6570\u636e，\u5141\u8bb8\u6211\u4eec\u5728\u7edf\u4e00\u641c\u7d22\u7684\u57fa\u7840\u4e0a\u8fdb\u884c\u7279\u5b9a\u6a21\u6001\u7684\u8fc7\u6ee4（\u4f8b\u5982：\u53ea\u641c\u89c6\u9891）\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                                                      4.2 \u6838\u5fc3\u914d\u7f6e\u6587\u4ef6（config.py）<\/h4>\n

                                                      \u8fd0\u884c<\/p>\n

                                                      import<\/span> os
                                                      \nfrom<\/span> pathlib import<\/span> Path<\/p>\n

                                                      # \u9879\u76ee\u8def\u5f84\u914d\u7f6e<\/span>
                                                      \nPROJECT_ROOT =<\/span> Path(<\/span>__file__)<\/span>.<\/span>parent
                                                      \nDATA_DIR =<\/span> PROJECT_ROOT \/<\/span> "data"<\/span>
                                                      \nEMBEDDINGS_DB_DIR =<\/span> PROJECT_ROOT \/<\/span> "embeddings_db"<\/span> \/<\/span> "chroma"<\/span>
                                                      \nLOGS_DIR =<\/span> PROJECT_ROOT \/<\/span> "logs"<\/span><\/p>\n

                                                      # \u521b\u5efa\u5fc5\u8981\u76ee\u5f55<\/span>
                                                      \nfor<\/span> dir_path in<\/span> [<\/span>DATA_DIR,<\/span> EMBEDDINGS_DB_DIR,<\/span> LOGS_DIR]<\/span>:<\/span>
                                                      \n dir_path.<\/span>mkdir(<\/span>parents=<\/span>True<\/span>,<\/span> exist_ok=<\/span>True<\/span>)<\/span><\/p>\n

                                                      # \u6a21\u578b\u914d\u7f6e<\/span>
                                                      \nMODEL_DIR =<\/span> Path(<\/span>"\/data\/models\/gemini"<\/span>)<\/span>
                                                      \nMODEL_NAME =<\/span> "google\/gemma-2-9b-it"<\/span>
                                                      \nDEVICE =<\/span> "cuda"<\/span> if<\/span> torch.<\/span>cuda.<\/span>is_available(<\/span>)<\/span> else<\/span> "cpu"<\/span>
                                                      \nD_MODEL =<\/span> 2048<\/span> # \u7edf\u4e00\u5d4c\u5165\u7ef4\u5ea6<\/span>
                                                      \nMAX_SEQ_LEN =<\/span> 4096<\/span> # \u6700\u5927\u5e8f\u5217\u957f\u5ea6<\/span><\/p>\n

                                                      # \u591a\u6a21\u6001\u5904\u7406\u914d\u7f6e<\/span>
                                                      \nVIDEO_FRAME_RATE =<\/span> 1<\/span> # \u89c6\u9891\u62bd\u5e27\u5e27\u7387<\/span>
                                                      \nIMAGE_SIZE =<\/span> (<\/span>256<\/span>,<\/span> 256<\/span>)<\/span> # \u56fe\u50cf\u5c3a\u5bf8<\/span>
                                                      \nPATCH_SIZE =<\/span> 16<\/span> # \u89c6\u89c9Patch\u5c3a\u5bf8<\/span>
                                                      \nNORMALIZE_EMBEDDINGS =<\/span> True<\/span> # \u662f\u5426\u5f52\u4e00\u5316\u5d4c\u5165\u5411\u91cf<\/span><\/p>\n

                                                      # \u5411\u91cf\u6570\u636e\u5e93\u914d\u7f6e<\/span>
                                                      \nCHROMA_PERSIST_DIRECTORY =<\/span> str<\/span>(<\/span>EMBEDDINGS_DB_DIR)<\/span>
                                                      \nCHROMA_COLLECTION_NAME =<\/span> "gemini_multimodal"<\/span>
                                                      \nRETRIEVE_TOP_K =<\/span> 5<\/span> # \u8de8\u6a21\u6001\u68c0\u7d22\u8fd4\u56de\u6570\u91cf<\/span><\/p>\n

                                                      # Gemini API\u914d\u7f6e（\u5907\u7528，\u672c\u5730\u90e8\u7f72\u4f18\u5148）<\/span>
                                                      \nGEMINI_API_KEY =<\/span> os.<\/span>getenv(<\/span>"GEMINI_API_KEY"<\/span>,<\/span> ""<\/span>)<\/span>
                                                      \nGEMINI_API_ENDPOINT =<\/span> "https:\/\/generativelanguage.googleapis.com\/v1\/models\/gemini-1.5-flash:generateContent"<\/span><\/p>\n

                                                      4.3 \u6a21\u578b\u521d\u59cb\u5316（core\/model_setup.py）<\/h4>\n

                                                      \u8fd0\u884c<\/p>\n

                                                      import<\/span> torch
                                                      \nfrom<\/span> transformers import<\/span> AutoModel,<\/span> AutoTokenizer,<\/span> AutoImageProcessor
                                                      \nfrom<\/span> config import<\/span> MODEL_DIR,<\/span> MODEL_NAME,<\/span> DEVICE,<\/span> D_MODEL,<\/span> MAX_SEQ_LEN<\/p>\n

                                                      def<\/span> setup_gemini_model<\/span>(<\/span>)<\/span>:<\/span>
                                                      \n """\u521d\u59cb\u5316Gemini\u591a\u6a21\u6001\u6a21\u578b"""<\/span>
                                                      \n # 1. \u52a0\u8f7d\u6587\u672cTokenizer<\/span>
                                                      \n tokenizer =<\/span> AutoTokenizer.<\/span>from_pretrained(<\/span>
                                                      \n MODEL_NAME,<\/span>
                                                      \n cache_dir=<\/span>str<\/span>(<\/span>MODEL_DIR)<\/span>,<\/span>
                                                      \n padding_side=<\/span>"right"<\/span>,<\/span>
                                                      \n truncation_side=<\/span>"right"<\/span>
                                                      \n )<\/span><\/p>\n

                                                      # 2. \u52a0\u8f7d\u89c6\u89c9\u5904\u7406\u5668（\u56fe\u50cf\/\u89c6\u9891）<\/span>
                                                      \n image_processor =<\/span> AutoImageProcessor.<\/span>from_pretrained(<\/span>
                                                      \n MODEL_NAME,<\/span>
                                                      \n cache_dir=<\/span>str<\/span>(<\/span>MODEL_DIR)<\/span>,<\/span>
                                                      \n image_size=<\/span>IMAGE_SIZE # \u5339\u914dconfig\u4e2d\u7684\u56fe\u50cf\u5c3a\u5bf8<\/span>
                                                      \n )<\/span><\/p>\n

                                                      # 3. \u52a0\u8f7d\u591a\u6a21\u6001\u6a21\u578b<\/span>
                                                      \n model =<\/span> AutoModel.<\/span>from_pretrained(<\/span>
                                                      \n MODEL_NAME,<\/span>
                                                      \n cache_dir=<\/span>str<\/span>(<\/span>MODEL_DIR)<\/span>,<\/span>
                                                      \n torch_dtype=<\/span>torch.<\/span>float16 if<\/span> DEVICE ==<\/span> "cuda"<\/span> else<\/span> torch.<\/span>float32,<\/span>
                                                      \n device_map=<\/span>DEVICE,<\/span>
                                                      \n trust_remote_code=<\/span>True<\/span> # \u52a0\u8f7d\u81ea\u5b9a\u4e49\u6a21\u578b\u4ee3\u7801<\/span>
                                                      \n )<\/span><\/p>\n

                                                      # 4. \u9002\u914d\u7edf\u4e00\u5d4c\u5165\u7ef4\u5ea6（\u82e5\u6a21\u578b\u8f93\u51fa\u7ef4\u5ea6\u4e0d\u7b26）<\/span>
                                                      \n if<\/span> model.<\/span>config.<\/span>hidden_size !=<\/span> D_MODEL:<\/span>
                                                      \n model.<\/span>embed_proj =<\/span> torch.<\/span>nn.<\/span>Linear(<\/span>model.<\/span>config.<\/span>hidden_size,<\/span> D_MODEL)<\/span>.<\/span>to(<\/span>DEVICE)<\/span><\/p>\n

                                                      model.<\/span>eval<\/span>(<\/span>)<\/span> # \u63a8\u7406\u6a21\u5f0f<\/span>
                                                      \n print<\/span>(<\/span>f"Gemini\u6a21\u578b\u521d\u59cb\u5316\u5b8c\u6210，\u8bbe\u5907：<\/span>{<\/span>DEVICE}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                      \n return<\/span> model,<\/span> tokenizer,<\/span> image_processor<\/p>\n

                                                      # \u5355\u4f8b\u6a21\u5f0f\u52a0\u8f7d\u6a21\u578b（\u907f\u514d\u91cd\u590d\u521d\u59cb\u5316）<\/span>
                                                      \n_model,<\/span> _tokenizer,<\/span> _image_processor =<\/span> None<\/span>,<\/span> None<\/span>,<\/span> None<\/span>
                                                      \ndef<\/span> get_gemini_model<\/span>(<\/span>)<\/span>:<\/span>
                                                      \n global<\/span> _model,<\/span> _tokenizer,<\/span> _image_processor
                                                      \n if<\/span> _model is<\/span> None<\/span>:<\/span>
                                                      \n _model,<\/span> _tokenizer,<\/span> _image_processor =<\/span> setup_gemini_model(<\/span>)<\/span>
                                                      \n return<\/span> _model,<\/span> _tokenizer,<\/span> _image_processor<\/p>\n

                                                      4.4 \u591a\u6a21\u6001\u9884\u5904\u7406（core\/modal_processor.py）<\/h4>\n

                                                      \u8fd0\u884c<\/p>\n

                                                      import<\/span> cv2
                                                      \nimport<\/span> numpy as<\/span> np
                                                      \nimport<\/span> torch
                                                      \nfrom<\/span> PIL import<\/span> Image
                                                      \nfrom<\/span> config import<\/span> VIDEO_FRAME_RATE,<\/span> IMAGE_SIZE,<\/span> PATCH_SIZE,<\/span> MAX_SEQ_LEN<\/p>\n

                                                      class<\/span> ModalProcessor<\/span>:<\/span>
                                                      \n def<\/span> __init__<\/span>(<\/span>self,<\/span> tokenizer,<\/span> image_processor)<\/span>:<\/span>
                                                      \n self.<\/span>tokenizer =<\/span> tokenizer
                                                      \n self.<\/span>image_processor =<\/span> image_processor<\/p>\n

                                                      def<\/span> process_text<\/span>(<\/span>self,<\/span> text)<\/span>:<\/span>
                                                      \n """\u5904\u7406\u6587\u672c：Tokenize\u5e76\u622a\u65ad"""<\/span>
                                                      \n inputs =<\/span> self.<\/span>tokenizer(<\/span>
                                                      \n text,<\/span>
                                                      \n max_length=<\/span>MAX_SEQ_LEN,<\/span>
                                                      \n padding=<\/span>"max_length"<\/span>,<\/span>
                                                      \n truncation=<\/span>True<\/span>,<\/span>
                                                      \n return_tensors=<\/span>"pt"<\/span>
                                                      \n )<\/span>
                                                      \n return<\/span> inputs<\/p>\n

                                                      def<\/span> process_image<\/span>(<\/span>self,<\/span> image_path)<\/span>:<\/span>
                                                      \n """\u5904\u7406\u56fe\u50cf：\u52a0\u8f7d\u2192Resize\u2192Patch\u5316"""<\/span>
                                                      \n # \u52a0\u8f7d\u56fe\u50cf<\/span>
                                                      \n image =<\/span> Image.<\/span>open<\/span>(<\/span>image_path)<\/span>.<\/span>convert(<\/span>"RGB"<\/span>)<\/span>
                                                      \n # \u56fe\u50cf\u9884\u5904\u7406<\/span>
                                                      \n inputs =<\/span> self.<\/span>image_processor(<\/span>
                                                      \n images=<\/span>image,<\/span>
                                                      \n return_tensors=<\/span>"pt"<\/span>,<\/span>
                                                      \n do_resize=<\/span>True<\/span>,<\/span>
                                                      \n size=<\/span>IMAGE_SIZE
                                                      \n )<\/span>
                                                      \n return<\/span> inputs<\/p>\n

                                                      def<\/span> process_video<\/span>(<\/span>self,<\/span> video_path)<\/span>:<\/span>
                                                      \n """\u5904\u7406\u89c6\u9891：\u62bd\u5e27\u2192\u9010\u5e27\u5904\u7406\u2192\u62fc\u63a5"""<\/span>
                                                      \n # 1. \u62bd\u5e27<\/span>
                                                      \n cap =<\/span> cv2.<\/span>VideoCapture(<\/span>video_path)<\/span>
                                                      \n frames =<\/span> [<\/span>]<\/span>
                                                      \n frame_idx =<\/span> 0<\/span>
                                                      \n fps =<\/span> cap.<\/span>get(<\/span>cv2.<\/span>CAP_PROP_FPS)<\/span>
                                                      \n sample_interval =<\/span> int<\/span>(<\/span>fps \/<\/span> VIDEO_FRAME_RATE)<\/span><\/p>\n

                                                      while<\/span> cap.<\/span>isOpened(<\/span>)<\/span>:<\/span>
                                                      \n ret,<\/span> frame =<\/span> cap.<\/span>read(<\/span>)<\/span>
                                                      \n if<\/span> not<\/span> ret:<\/span>
                                                      \n break<\/span>
                                                      \n if<\/span> frame_idx %<\/span> sample_interval ==<\/span> 0<\/span>:<\/span>
                                                      \n # \u8f6c\u6362\u4e3aRGB\u5e76Resize<\/span>
                                                      \n frame =<\/span> cv2.<\/span>cvtColor(<\/span>frame,<\/span> cv2.<\/span>COLOR_BGR2RGB)<\/span>
                                                      \n frame =<\/span> cv2.<\/span>resize(<\/span>frame,<\/span> IMAGE_SIZE)<\/span>
                                                      \n frames.<\/span>append(<\/span>Image.<\/span>fromarray(<\/span>frame)<\/span>)<\/span>
                                                      \n frame_idx +=<\/span> 1<\/span>
                                                      \n cap.<\/span>release(<\/span>)<\/span><\/p>\n

                                                      # 2. \u9884\u5904\u7406\u5e27（\u9650\u5236\u6700\u5927\u5e27\u6570）<\/span>
                                                      \n max_frames =<\/span> MAX_SEQ_LEN \/\/<\/span> (<\/span>IMAGE_SIZE[<\/span>0<\/span>]<\/span> \/\/<\/span> PATCH_SIZE)<\/span> **<\/span>2<\/span>
                                                      \n frames =<\/span> frames[<\/span>:<\/span>max_frames]<\/span><\/p>\n

                                                      # 3. \u89c6\u9891\u5e27\u9884\u5904\u7406<\/span>
                                                      \n inputs =<\/span> self.<\/span>image_processor(<\/span>
                                                      \n images=<\/span>frames,<\/span>
                                                      \n return_tensors=<\/span>"pt"<\/span>,<\/span>
                                                      \n do_resize=<\/span>True<\/span>,<\/span>
                                                      \n size=<\/span>IMAGE_SIZE
                                                      \n )<\/span>
                                                      \n # \u6dfb\u52a0\u65f6\u95f4\u7ef4\u5ea6\u6807\u8bb0<\/span>
                                                      \n inputs[<\/span>"time_ids"<\/span>]<\/span> =<\/span> torch.<\/span>arange(<\/span>len<\/span>(<\/span>frames)<\/span>)<\/span>.<\/span>unsqueeze(<\/span>0<\/span>)<\/span>
                                                      \n return<\/span> inputs<\/p>\n

                                                      # \u6d4b\u8bd5\u5904\u7406\u5668<\/span>
                                                      \nif<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
                                                      \n from<\/span> core.<\/span>model_setup import<\/span> get_gemini_model
                                                      \n _,<\/span> tokenizer,<\/span> image_processor =<\/span> get_gemini_model(<\/span>)<\/span>
                                                      \n processor =<\/span> ModalProcessor(<\/span>tokenizer,<\/span> image_processor)<\/span><\/p>\n

                                                      # \u6d4b\u8bd5\u6587\u672c\u5904\u7406<\/span>
                                                      \n text_inputs =<\/span> processor.<\/span>process_text(<\/span>"\u7ea2\u8272\u8dd1\u8f66\u5728\u516c\u8def\u4e0a\u884c\u9a76"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u6587\u672cToken\u5f62\u72b6：<\/span>{<\/span>text_inputs[<\/span>'input_ids'<\/span>]<\/span>.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      # \u6d4b\u8bd5\u56fe\u50cf\u5904\u7406<\/span>
                                                      \n image_inputs =<\/span> processor.<\/span>process_image(<\/span>"data\/images\/car.jpg"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u56fe\u50cfPixel\u503c\u5f62\u72b6：<\/span>{<\/span>image_inputs[<\/span>'pixel_values'<\/span>]<\/span>.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      # \u6d4b\u8bd5\u89c6\u9891\u5904\u7406<\/span>
                                                      \n video_inputs =<\/span> processor.<\/span>process_video(<\/span>"data\/videos\/car_driving.mp4"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u89c6\u9891Pixel\u503c\u5f62\u72b6：<\/span>{<\/span>video_inputs[<\/span>'pixel_values'<\/span>]<\/span>.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u89c6\u9891\u65f6\u95f4ID\u5f62\u72b6：<\/span>{<\/span>video_inputs[<\/span>'time_ids'<\/span>]<\/span>.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      4.5 \u751f\u6210\u7edf\u4e00\u5d4c\u5165（core\/embedding.py）<\/h4>\n

                                                      \u8fd0\u884c<\/p>\n

                                                      import<\/span> torch
                                                      \nfrom<\/span> config import<\/span> DEVICE,<\/span> D_MODEL,<\/span> NORMALIZE_EMBEDDINGS
                                                      \nfrom<\/span> core.<\/span>modal_processor import<\/span> ModalProcessor<\/p>\n

                                                      class<\/span> MultimodalEmbedding<\/span>:<\/span>
                                                      \n def<\/span> __init__<\/span>(<\/span>self)<\/span>:<\/span>
                                                      \n from<\/span> core.<\/span>model_setup import<\/span> get_gemini_model
                                                      \n self.<\/span>model,<\/span> self.<\/span>tokenizer,<\/span> self.<\/span>image_processor =<\/span> get_gemini_model(<\/span>)<\/span>
                                                      \n self.<\/span>processor =<\/span> ModalProcessor(<\/span>self.<\/span>tokenizer,<\/span> self.<\/span>image_processor)<\/span><\/p>\n

                                                      @torch.<\/span>no_grad<\/span>(<\/span>)<\/span>
                                                      \n def<\/span> get_text_embedding<\/span>(<\/span>self,<\/span> text)<\/span>:<\/span>
                                                      \n """\u751f\u6210\u6587\u672c\u7684\u7edf\u4e00\u5d4c\u5165"""<\/span>
                                                      \n inputs =<\/span> self.<\/span>processor.<\/span>process_text(<\/span>text)<\/span>.<\/span>to(<\/span>DEVICE)<\/span>
                                                      \n # \u524d\u5411\u4f20\u64ad\u83b7\u53d6\u9690\u85cf\u72b6\u6001<\/span>
                                                      \n outputs =<\/span> self.<\/span>model(<\/span>**<\/span>inputs)<\/span>
                                                      \n # \u53d6<CLS> token\u7684\u5d4c\u5165（\u6216\u5747\u503c\u6c60\u5316）<\/span>
                                                      \n embedding =<\/span> outputs.<\/span>last_hidden_state[<\/span>:<\/span>,<\/span> 0<\/span>,<\/span> :<\/span>]<\/span>
                                                      \n # \u9002\u914d\u7edf\u4e00\u7ef4\u5ea6<\/span>
                                                      \n if<\/span> hasattr<\/span>(<\/span>self.<\/span>model,<\/span> "embed_proj"<\/span>)<\/span>:<\/span>
                                                      \n embedding =<\/span> self.<\/span>model.<\/span>embed_proj(<\/span>embedding)<\/span>
                                                      \n # \u5f52\u4e00\u5316<\/span>
                                                      \n if<\/span> NORMALIZE_EMBEDDINGS:<\/span>
                                                      \n embedding =<\/span> torch.<\/span>nn.<\/span>functional.<\/span>normalize(<\/span>embedding,<\/span> p=<\/span>2<\/span>,<\/span> dim=<\/span>–<\/span>1<\/span>)<\/span>
                                                      \n return<\/span> embedding.<\/span>cpu(<\/span>)<\/span>.<\/span>numpy(<\/span>)<\/span>[<\/span>0<\/span>]<\/span><\/p>\n

                                                      @torch.<\/span>no_grad<\/span>(<\/span>)<\/span>
                                                      \n def<\/span> get_image_embedding<\/span>(<\/span>self,<\/span> image_path)<\/span>:<\/span>
                                                      \n """\u751f\u6210\u56fe\u50cf\u7684\u7edf\u4e00\u5d4c\u5165"""<\/span>
                                                      \n inputs =<\/span> self.<\/span>processor.<\/span>process_image(<\/span>image_path)<\/span>.<\/span>to(<\/span>DEVICE)<\/span>
                                                      \n # \u89c6\u89c9\u524d\u5411\u4f20\u64ad<\/span>
                                                      \n outputs =<\/span> self.<\/span>model.<\/span>visual_model(<\/span>**<\/span>inputs)<\/span>
                                                      \n # \u5747\u503c\u6c60\u5316\u83b7\u53d6\u56fe\u50cf\u5d4c\u5165<\/span>
                                                      \n embedding =<\/span> outputs.<\/span>last_hidden_state.<\/span>mean(<\/span>dim=<\/span>1<\/span>)<\/span>
                                                      \n # \u9002\u914d\u7edf\u4e00\u7ef4\u5ea6<\/span>
                                                      \n if<\/span> hasattr<\/span>(<\/span>self.<\/span>model,<\/span> "embed_proj"<\/span>)<\/span>:<\/span>
                                                      \n embedding =<\/span> self.<\/span>model.<\/span>embed_proj(<\/span>embedding)<\/span>
                                                      \n # \u5f52\u4e00\u5316<\/span>
                                                      \n if<\/span> NORMALIZE_EMBEDDINGS:<\/span>
                                                      \n embedding =<\/span> torch.<\/span>nn.<\/span>functional.<\/span>normalize(<\/span>embedding,<\/span> p=<\/span>2<\/span>,<\/span> dim=<\/span>–<\/span>1<\/span>)<\/span>
                                                      \n return<\/span> embedding.<\/span>cpu(<\/span>)<\/span>.<\/span>numpy(<\/span>)<\/span>[<\/span>0<\/span>]<\/span><\/p>\n

                                                      @torch.<\/span>no_grad<\/span>(<\/span>)<\/span>
                                                      \n def<\/span> get_video_embedding<\/span>(<\/span>self,<\/span> video_path)<\/span>:<\/span>
                                                      \n """\u751f\u6210\u89c6\u9891\u7684\u7edf\u4e00\u5d4c\u5165"""<\/span>
                                                      \n inputs =<\/span> self.<\/span>processor.<\/span>process_video(<\/span>video_path)<\/span>.<\/span>to(<\/span>DEVICE)<\/span>
                                                      \n # \u89c6\u9891\u524d\u5411\u4f20\u64ad（\u65f6\u7a7a\u7f16\u7801）<\/span>
                                                      \n visual_outputs =<\/span> self.<\/span>model.<\/span>visual_model(<\/span>**<\/span>inputs)<\/span>
                                                      \n # \u65f6\u7a7a\u5747\u503c\u6c60\u5316（\u65f6\u95f4+\u7a7a\u95f4\u7ef4\u5ea6）<\/span>
                                                      \n embedding =<\/span> visual_outputs.<\/span>last_hidden_state.<\/span>mean(<\/span>dim=<\/span>[<\/span>1<\/span>,<\/span> 2<\/span>]<\/span>)<\/span>
                                                      \n # \u9002\u914d\u7edf\u4e00\u7ef4\u5ea6<\/span>
                                                      \n if<\/span> hasattr<\/span>(<\/span>self.<\/span>model,<\/span> "embed_proj"<\/span>)<\/span>:<\/span>
                                                      \n embedding =<\/span> self.<\/span>model.<\/span>embed_proj(<\/span>embedding)<\/span>
                                                      \n # \u5f52\u4e00\u5316<\/span>
                                                      \n if<\/span> NORMALIZE_EMBEDDINGS:<\/span>
                                                      \n embedding =<\/span> torch.<\/span>nn.<\/span>functional.<\/span>normalize(<\/span>embedding,<\/span> p=<\/span>2<\/span>,<\/span> dim=<\/span>–<\/span>1<\/span>)<\/span>
                                                      \n return<\/span> embedding.<\/span>cpu(<\/span>)<\/span>.<\/span>numpy(<\/span>)<\/span>[<\/span>0<\/span>]<\/span><\/p>\n

                                                      # \u6d4b\u8bd5\u5d4c\u5165\u751f\u6210<\/span>
                                                      \nif<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
                                                      \n embedder =<\/span> MultimodalEmbedding(<\/span>)<\/span><\/p>\n

                                                      # \u751f\u6210\u6587\u672c\u5d4c\u5165<\/span>
                                                      \n text_emb =<\/span> embedder.<\/span>get_text_embedding(<\/span>"\u7ea2\u8272\u8dd1\u8f66\u5728\u516c\u8def\u4e0a\u884c\u9a76"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u6587\u672c\u5d4c\u5165\u7ef4\u5ea6：<\/span>{<\/span>text_emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span> # \u5e94\u4e3a (2048,)<\/span><\/p>\n

                                                      # \u751f\u6210\u56fe\u50cf\u5d4c\u5165<\/span>
                                                      \n image_emb =<\/span> embedder.<\/span>get_image_embedding(<\/span>"data\/images\/car.jpg"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u56fe\u50cf\u5d4c\u5165\u7ef4\u5ea6：<\/span>{<\/span>image_emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span> # \u5e94\u4e3a (2048,)<\/span><\/p>\n

                                                      # \u751f\u6210\u89c6\u9891\u5d4c\u5165<\/span>
                                                      \n video_emb =<\/span> embedder.<\/span>get_video_embedding(<\/span>"data\/videos\/car_driving.mp4"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u89c6\u9891\u5d4c\u5165\u7ef4\u5ea6：<\/span>{<\/span>video_emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span> # \u5e94\u4e3a (2048,)<\/span><\/p>\n

                                                      # \u8ba1\u7b97\u8de8\u6a21\u6001\u76f8\u4f3c\u5ea6<\/span>
                                                      \n text_image_sim =<\/span> np.<\/span>dot(<\/span>text_emb,<\/span> image_emb)<\/span>
                                                      \n text_video_sim =<\/span> np.<\/span>dot(<\/span>text_emb,<\/span> video_emb)<\/span>
                                                      \n print<\/span>(<\/span>f"\u6587\u672c-\u56fe\u50cf\u76f8\u4f3c\u5ea6：<\/span>{<\/span>text_image_sim:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u6587\u672c-\u89c6\u9891\u76f8\u4f3c\u5ea6：<\/span>{<\/span>text_video_sim:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      4.6 \u8de8\u6a21\u6001\u68c0\u7d22（core\/retriever.py）<\/h4>\n

                                                      \u8fd0\u884c<\/p>\n

                                                      import<\/span> chromadb
                                                      \nimport<\/span> numpy as<\/span> np
                                                      \nfrom<\/span> chromadb.<\/span>config import<\/span> Settings
                                                      \nfrom<\/span> config import<\/span> CHROMA_PERSIST_DIRECTORY,<\/span> CHROMA_COLLECTION_NAME,<\/span> RETRIEVE_TOP_K<\/p>\n

                                                      class<\/span> MultimodalRetriever<\/span>:<\/span>
                                                      \n def<\/span> __init__<\/span>(<\/span>self)<\/span>:<\/span>
                                                      \n # \u521d\u59cb\u5316Chroma\u5411\u91cf\u6570\u636e\u5e93<\/span>
                                                      \n self.<\/span>client =<\/span> chromadb.<\/span>Client(<\/span>Settings(<\/span>
                                                      \n persist_directory=<\/span>CHROMA_PERSIST_DIRECTORY,<\/span>
                                                      \n anonymized_telemetry=<\/span>False<\/span>
                                                      \n )<\/span>)<\/span>
                                                      \n # \u83b7\u53d6\u6216\u521b\u5efa\u591a\u6a21\u6001\u96c6\u5408<\/span>
                                                      \n self.<\/span>collection =<\/span> self.<\/span>client.<\/span>get_or_create_collection(<\/span>
                                                      \n name=<\/span>CHROMA_COLLECTION_NAME,<\/span>
                                                      \n metadata=<\/span>{<\/span>"description"<\/span>:<\/span> "Gemini\u591a\u6a21\u6001\u7edf\u4e00\u5d4c\u5165\u96c6\u5408"<\/span>}<\/span>
                                                      \n )<\/span><\/p>\n

                                                      def<\/span> add_embedding<\/span>(<\/span>self,<\/span> id<\/span>,<\/span> embedding,<\/span> modal_type,<\/span> metadata=<\/span>None<\/span>)<\/span>:<\/span>
                                                      \n """\u6dfb\u52a0\u5d4c\u5165\u5230\u5411\u91cf\u5e93"""<\/span>
                                                      \n self.<\/span>collection.<\/span>add(<\/span>
                                                      \n ids=<\/span>[<\/span>id<\/span>]<\/span>,<\/span>
                                                      \n embeddings=<\/span>[<\/span>embedding]<\/span>,<\/span>
                                                      \n metadatas=<\/span>[<\/span>{<\/span>"modal_type"<\/span>:<\/span> modal_type,<\/span> **<\/span>(<\/span>metadata or<\/span> {<\/span>}<\/span>)<\/span>}<\/span>]<\/span>
                                                      \n )<\/span>
                                                      \n self.<\/span>client.<\/span>persist(<\/span>)<\/span><\/p>\n

                                                      def<\/span> retrieve<\/span>(<\/span>self,<\/span> query_embedding,<\/span> filter_modal_type=<\/span>None<\/span>)<\/span>:<\/span>
                                                      \n """\u8de8\u6a21\u6001\u68c0\u7d22\u76f8\u4f3c\u5185\u5bb9"""<\/span>
                                                      \n # \u6784\u5efa\u8fc7\u6ee4\u6761\u4ef6<\/span>
                                                      \n where_clause =<\/span> None<\/span>
                                                      \n if<\/span> filter_modal_type:<\/span>
                                                      \n where_clause =<\/span> {<\/span>"modal_type"<\/span>:<\/span> filter_modal_type}<\/span><\/p>\n

                                                      # \u68c0\u7d22<\/span>
                                                      \n results =<\/span> self.<\/span>collection.<\/span>query(<\/span>
                                                      \n query_embeddings=<\/span>[<\/span>query_embedding]<\/span>,<\/span>
                                                      \n n_results=<\/span>RETRIEVE_TOP_K,<\/span>
                                                      \n where=<\/span>where_clause
                                                      \n )<\/span>
                                                      \n # \u683c\u5f0f\u5316\u7ed3\u679c<\/span>
                                                      \n formatted_results =<\/span> [<\/span>]<\/span>
                                                      \n for<\/span> i in<\/span> range<\/span>(<\/span>len<\/span>(<\/span>results[<\/span>"ids"<\/span>]<\/span>[<\/span>0<\/span>]<\/span>)<\/span>)<\/span>:<\/span>
                                                      \n formatted_results.<\/span>append(<\/span>{<\/span>
                                                      \n "id"<\/span>:<\/span> results[<\/span>"ids"<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>i]<\/span>,<\/span>
                                                      \n "score"<\/span>:<\/span> results[<\/span>"distances"<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>i]<\/span>,<\/span>
                                                      \n "modal_type"<\/span>:<\/span> results[<\/span>"metadatas"<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>i]<\/span>[<\/span>"modal_type"<\/span>]<\/span>,<\/span>
                                                      \n "metadata"<\/span>:<\/span> results[<\/span>"metadatas"<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>i]<\/span>
                                                      \n }<\/span>)<\/span>
                                                      \n return<\/span> formatted_results<\/p>\n

                                                      # \u6d4b\u8bd5\u8de8\u6a21\u6001\u68c0\u7d22<\/span>
                                                      \nif<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
                                                      \n from<\/span> core.<\/span>embedding import<\/span> MultimodalEmbedding<\/p>\n

                                                      # \u521d\u59cb\u5316\u5d4c\u5165\u5668\u548c\u68c0\u7d22\u5668<\/span>
                                                      \n embedder =<\/span> MultimodalEmbedding(<\/span>)<\/span>
                                                      \n retriever =<\/span> MultimodalRetriever(<\/span>)<\/span><\/p>\n

                                                      # 1. \u6dfb\u52a0\u6d4b\u8bd5\u6570\u636e\u5230\u5411\u91cf\u5e93<\/span>
                                                      \n # \u6587\u672c\u5d4c\u5165<\/span>
                                                      \n text_id =<\/span> "text_1"<\/span>
                                                      \n text =<\/span> "\u7ea2\u8272\u8dd1\u8f66\u5728\u516c\u8def\u4e0a\u884c\u9a76"<\/span>
                                                      \n text_emb =<\/span> embedder.<\/span>get_text_embedding(<\/span>text)<\/span>
                                                      \n retriever.<\/span>add_embedding(<\/span>
                                                      \n id<\/span>=<\/span>text_id,<\/span>
                                                      \n embedding=<\/span>text_emb,<\/span>
                                                      \n modal_type=<\/span>"text"<\/span>,<\/span>
                                                      \n metadata=<\/span>{<\/span>"content"<\/span>:<\/span> text}<\/span>
                                                      \n )<\/span><\/p>\n

                                                      # \u56fe\u50cf\u5d4c\u5165<\/span>
                                                      \n image_id =<\/span> "image_1"<\/span>
                                                      \n image_path =<\/span> "data\/images\/car.jpg"<\/span>
                                                      \n image_emb =<\/span> embedder.<\/span>get_image_embedding(<\/span>image_path)<\/span>
                                                      \n retriever.<\/span>add_embedding(<\/span>
                                                      \n id<\/span>=<\/span>image_id,<\/span>
                                                      \n embedding=<\/span>image_emb,<\/span>
                                                      \n modal_type=<\/span>"image"<\/span>,<\/span>
                                                      \n metadata=<\/span>{<\/span>"path"<\/span>:<\/span> image_path}<\/span>
                                                      \n )<\/span><\/p>\n

                                                      # \u89c6\u9891\u5d4c\u5165<\/span>
                                                      \n video_id =<\/span> "video_1"<\/span>
                                                      \n video_path =<\/span> "data\/videos\/car_driving.mp4"<\/span>
                                                      \n video_emb =<\/span> embedder.<\/span>get_video_embedding(<\/span>video_path)<\/span>
                                                      \n retriever.<\/span>add_embedding(<\/span>
                                                      \n id<\/span>=<\/span>video_id,<\/span>
                                                      \n embedding=<\/span>video_emb,<\/span>
                                                      \n modal_type=<\/span>"video"<\/span>,<\/span>
                                                      \n metadata=<\/span>{<\/span>"path"<\/span>:<\/span> video_path}<\/span>
                                                      \n )<\/span><\/p>\n

                                                      # 2. \u8de8\u6a21\u6001\u68c0\u7d22（\u6587\u672c\u67e5\u8be2\u2192\u627e\u76f8\u4f3c\u56fe\u50cf\/\u89c6\u9891）<\/span>
                                                      \n query_text =<\/span> "\u7ea2\u8272\u8dd1\u8f66\u5728\u8def\u4e0a\u5f00"<\/span>
                                                      \n query_emb =<\/span> embedder.<\/span>get_text_embedding(<\/span>query_text)<\/span>
                                                      \n results =<\/span> retriever.<\/span>retrieve(<\/span>query_emb)<\/span><\/p>\n

                                                      # 3. \u6253\u5370\u68c0\u7d22\u7ed3\u679c<\/span>
                                                      \n print<\/span>(<\/span>f"\u67e5\u8be2\u6587\u672c：<\/span>{<\/span>query_text}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                      \n print<\/span>(<\/span>"\u8de8\u6a21\u6001\u68c0\u7d22\u7ed3\u679c："<\/span>)<\/span>
                                                      \n for<\/span> res in<\/span> results:<\/span>
                                                      \n print<\/span>(<\/span>f"- ID: <\/span>{<\/span>res[<\/span>'id'<\/span>]<\/span>}<\/span><\/span>, \u7c7b\u578b: <\/span>{<\/span>res[<\/span>'modal_type'<\/span>]<\/span>}<\/span><\/span>, \u76f8\u4f3c\u5ea6: <\/span>{<\/span>1<\/span>–<\/span>res[<\/span>'score'<\/span>]<\/span>:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      4.7 \u4e3b\u7a0b\u5e8f（main.py）<\/h4>\n

                                                      \u8fd0\u884c<\/p>\n

                                                      import<\/span> logging
                                                      \nfrom<\/span> loguru import<\/span> logger
                                                      \nfrom<\/span> config import<\/span> LOGS_DIR
                                                      \nfrom<\/span> core.<\/span>embedding import<\/span> MultimodalEmbedding
                                                      \nfrom<\/span> core.<\/span>retriever import<\/span> MultimodalRetriever<\/p>\n

                                                      # \u914d\u7f6e\u65e5\u5fd7<\/span>
                                                      \nlogger.<\/span>add(<\/span>
                                                      \n LOGS_DIR \/<\/span> "multimodal.log"<\/span>,<\/span>
                                                      \n rotation=<\/span>"100MB"<\/span>,<\/span>
                                                      \n retention=<\/span>"7 days"<\/span>,<\/span>
                                                      \n encoding=<\/span>"utf-8"<\/span>,<\/span>
                                                      \n level=<\/span>"INFO"<\/span>
                                                      \n)<\/span><\/p>\n

                                                      def<\/span> main<\/span>(<\/span>)<\/span>:<\/span>
                                                      \n logger.<\/span>info(<\/span>"\u542f\u52a8Gemini\u591a\u6a21\u6001\u7edf\u4e00\u8868\u793a\u7cfb\u7edf"<\/span>)<\/span><\/p>\n

                                                      # \u521d\u59cb\u5316\u7ec4\u4ef6<\/span>
                                                      \n embedder =<\/span> MultimodalEmbedding(<\/span>)<\/span>
                                                      \n retriever =<\/span> MultimodalRetriever(<\/span>)<\/span><\/p>\n

                                                      # \u4ea4\u4e92\u83dc\u5355<\/span>
                                                      \n print<\/span>(<\/span>"=== Gemini\u591a\u6a21\u6001\u7edf\u4e00\u8868\u793a\u7cfb\u7edf ==="<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>"1. \u751f\u6210\u6587\u672c\u5d4c\u5165"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>"2. \u751f\u6210\u56fe\u50cf\u5d4c\u5165"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>"3. \u751f\u6210\u89c6\u9891\u5d4c\u5165"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>"4. \u8de8\u6a21\u6001\u68c0\u7d22"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>"5. \u9000\u51fa"<\/span>)<\/span><\/p>\n

                                                      while<\/span> True<\/span>:<\/span>
                                                      \n choice =<\/span> input<\/span>(<\/span>"\\\\n\u8bf7\u9009\u62e9\u529f\u80fd（1-5）："<\/span>)<\/span>
                                                      \n try<\/span>:<\/span>
                                                      \n if<\/span> choice ==<\/span> "1"<\/span>:<\/span>
                                                      \n text =<\/span> input<\/span>(<\/span>"\u8bf7\u8f93\u5165\u6587\u672c："<\/span>)<\/span>
                                                      \n emb =<\/span> embedder.<\/span>get_text_embedding(<\/span>text)<\/span>
                                                      \n logger.<\/span>info(<\/span>f"\u751f\u6210\u6587\u672c\u5d4c\u5165：<\/span>{<\/span>text[<\/span>:<\/span>20]<\/span>}<\/span><\/span>… \u7ef4\u5ea6：<\/span>{<\/span>emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u6587\u672c\u5d4c\u5165\u751f\u6210\u6210\u529f，\u7ef4\u5ea6：<\/span>{<\/span>emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      elif<\/span> choice ==<\/span> "2"<\/span>:<\/span>
                                                      \n image_path =<\/span> input<\/span>(<\/span>"\u8bf7\u8f93\u5165\u56fe\u50cf\u8def\u5f84："<\/span>)<\/span>
                                                      \n emb =<\/span> embedder.<\/span>get_image_embedding(<\/span>image_path)<\/span>
                                                      \n logger.<\/span>info(<\/span>f"\u751f\u6210\u56fe\u50cf\u5d4c\u5165：<\/span>{<\/span>image_path}<\/span><\/span> \u7ef4\u5ea6：<\/span>{<\/span>emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u56fe\u50cf\u5d4c\u5165\u751f\u6210\u6210\u529f，\u7ef4\u5ea6：<\/span>{<\/span>emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      elif<\/span> choice ==<\/span> "3"<\/span>:<\/span>
                                                      \n video_path =<\/span> input<\/span>(<\/span>"\u8bf7\u8f93\u5165\u89c6\u9891\u8def\u5f84："<\/span>)<\/span>
                                                      \n emb =<\/span> embedder.<\/span>get_video_embedding(<\/span>video_path)<\/span>
                                                      \n logger.<\/span>info(<\/span>f"\u751f\u6210\u89c6\u9891\u5d4c\u5165：<\/span>{<\/span>video_path}<\/span><\/span> \u7ef4\u5ea6：<\/span>{<\/span>emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u89c6\u9891\u5d4c\u5165\u751f\u6210\u6210\u529f，\u7ef4\u5ea6：<\/span>{<\/span>emb.<\/span>shape}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      elif<\/span> choice ==<\/span> "4"<\/span>:<\/span>
                                                      \n query_type =<\/span> input<\/span>(<\/span>"\u8bf7\u9009\u62e9\u67e5\u8be2\u7c7b\u578b（text\/image\/video）："<\/span>)<\/span>
                                                      \n if<\/span> query_type ==<\/span> "text"<\/span>:<\/span>
                                                      \n query =<\/span> input<\/span>(<\/span>"\u8bf7\u8f93\u5165\u67e5\u8be2\u6587\u672c："<\/span>)<\/span>
                                                      \n query_emb =<\/span> embedder.<\/span>get_text_embedding(<\/span>query)<\/span>
                                                      \n elif<\/span> query_type ==<\/span> "image"<\/span>:<\/span>
                                                      \n query =<\/span> input<\/span>(<\/span>"\u8bf7\u8f93\u5165\u67e5\u8be2\u56fe\u50cf\u8def\u5f84："<\/span>)<\/span>
                                                      \n query_emb =<\/span> embedder.<\/span>get_image_embedding(<\/span>query)<\/span>
                                                      \n elif<\/span> query_type ==<\/span> "video"<\/span>:<\/span>
                                                      \n query =<\/span> input<\/span>(<\/span>"\u8bf7\u8f93\u5165\u67e5\u8be2\u89c6\u9891\u8def\u5f84："<\/span>)<\/span>
                                                      \n query_emb =<\/span> embedder.<\/span>get_video_embedding(<\/span>query)<\/span>
                                                      \n else<\/span>:<\/span>
                                                      \n print<\/span>(<\/span>"\u65e0\u6548\u7684\u67e5\u8be2\u7c7b\u578b"<\/span>)<\/span>
                                                      \n continue<\/span><\/p>\n

                                                      # \u68c0\u7d22<\/span>
                                                      \n results =<\/span> retriever.<\/span>retrieve(<\/span>query_emb)<\/span>
                                                      \n print<\/span>(<\/span>"\\\\n\u68c0\u7d22\u7ed3\u679c（\u6309\u76f8\u4f3c\u5ea6\u6392\u5e8f）："<\/span>)<\/span>
                                                      \n for<\/span> i,<\/span> res in<\/span> enumerate<\/span>(<\/span>results,<\/span> 1<\/span>)<\/span>:<\/span>
                                                      \n similarity =<\/span> 1<\/span> –<\/span> res[<\/span>"score"<\/span>]<\/span> # Chroma\u8fd4\u56de\u7684\u662f\u8ddd\u79bb，\u8f6c\u6362\u4e3a\u76f8\u4f3c\u5ea6<\/span>
                                                      \n print<\/span>(<\/span>f"<\/span>{<\/span>i}<\/span><\/span>. ID: <\/span>{<\/span>res[<\/span>'id'<\/span>]<\/span>}<\/span><\/span>, \u7c7b\u578b: <\/span>{<\/span>res[<\/span>'modal_type'<\/span>]<\/span>}<\/span><\/span>, \u76f8\u4f3c\u5ea6: <\/span>{<\/span>similarity:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      elif<\/span> choice ==<\/span> "5"<\/span>:<\/span>
                                                      \n logger.<\/span>info(<\/span>"\u9000\u51fa\u7cfb\u7edf"<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>"\u518d\u89c1！"<\/span>)<\/span>
                                                      \n break<\/span><\/p>\n

                                                      else<\/span>:<\/span>
                                                      \n print<\/span>(<\/span>"\u65e0\u6548\u7684\u9009\u62e9，\u8bf7\u8f93\u51651-5"<\/span>)<\/span><\/p>\n

                                                      except<\/span> Exception as<\/span> e:<\/span>
                                                      \n logger.<\/span>error(<\/span>f"\u64cd\u4f5c\u5931\u8d25：<\/span>{<\/span>str<\/span>(<\/span>e)<\/span>}<\/span><\/span>"<\/span><\/span>,<\/span> exc_info=<\/span>True<\/span>)<\/span>
                                                      \n print<\/span>(<\/span>f"\u9519\u8bef：<\/span>{<\/span>str<\/span>(<\/span>e)<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                      if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
                                                      \n main(<\/span>)<\/span><\/p>\n

                                                      4.8 \u8fd9\u4e9b\u6587\u4ef6\u662f\u5982\u4f55\u534f\u4f5c\u7684？<\/h4>\n

                                                      Gemini \u591a\u6a21\u6001\u7cfb\u7edf\u7684\u8fd0\u884c\u903b\u8f91\u662f：\u201c\u8f93\u5165\u591a\u6a21\u6001，\u8f6c\u5316\u7686\u5e8f\u5217，\u8f93\u51fa\u7686\u5411\u91cf\u201d\u3002<\/p>\n

                                                      \u2502
                                                      \n\u251c\u2500\u2500 \u3010\u7528\u6237\u8f93\u5165\u3011
                                                      \n\u2502 \u251c\u2500\u2500 \u5f85\u5904\u7406\u6570\u636e: \u6587\u672c "\u5915\u9633\u4e0b\u7684\u6d77\u6ee9"<\/span> \/ \u56fe\u7247 [<\/span>img.jpg]<\/span> \/ \u89c6\u9891 [<\/span>vid.mp4]<\/span>
                                                      \n\u2502 \u2514\u2500\u2500 \u76ee\u7684: \u751f\u6210\u5411\u91cf (<\/span>Embedding)<\/span> \u6216 \u6267\u884c\u68c0\u7d22 (<\/span>Retrieval)<\/span>
                                                      \n\u2502
                                                      \n\u25bc
                                                      \n[<\/span>1<\/span>. \u611f\u77e5\u4e0e\u5e8f\u5217\u5316\u9636\u6bb5 (<\/span>Perception &<\/span> Serialization)<\/span>]<\/span> \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
                                                      \n\u2502 \u2502
                                                      \n\u251c\u2500\u2500 <<\/span>\u8c03\u7528\u6a21\u5757><\/span>: core\/modal_processor.py (<\/span>\u5e8f\u5217\u5316\u5de5\u5382)<\/span> \u2502
                                                      \n\u2502 \u251c\u2500\u2500 \u4f9d\u8d56\u914d\u7f6e: config.py (<\/span>\u5b9a\u4e49 PATCH_SIZE, MAX_SEQ_LEN)<\/span> \u2502
                                                      \n\u2502 \u2502 \u2502
                                                      \n\u2502 \u251c\u2500\u2500 A. \u6587\u672c\u6d41 (<\/span>Text Stream)<\/span> \u2502
                                                      \n\u2502 \u2502 \u251c\u2500\u2500 <<\/span>\u5de5\u5177><\/span>: Tokenizer (<\/span>\u6765\u81ea model_setup.py)<\/span> \u2502
                                                      \n\u2502 \u2502 \u2514\u2500\u2500 ><\/span> \u8f93\u51fa: 1D Token IDs [<\/span>101<\/span>, 2345<\/span>, 889<\/span>..<\/span>.]<\/span> \u2502
                                                      \n\u2502 \u2502 \u2502
                                                      \n\u2502 \u251c\u2500\u2500 B. \u89c6\u89c9\u6d41 (<\/span>Visual Stream – \u56fe\u50cf\/\u89c6\u9891)<\/span> \u2502
                                                      \n\u2502 \u2502 \u251c\u2500\u2500 <<\/span>\u5de5\u5177><\/span>: ImageProcessor (<\/span>\u6765\u81ea model_setup.py)<\/span> \u2502
                                                      \n\u2502 \u2502 \u251c\u2500\u2500 <<\/span>\u52a8\u4f5c><\/span>: Resize –><\/span> Crop –><\/span> Patch\u5316 \u2502
                                                      \n\u2502 \u2502 \u2502 \u251c\u2500\u2500 \u56fe\u50cf: 16×16 Patch \u5c55\u5f00 \u2502
                                                      \n\u2502 \u2502 \u2502 \u2514\u2500\u2500 \u89c6\u9891: \u65f6\u7a7a\u91c7\u6837 (<\/span>Time-Space Sampling)<\/span> \u2502
                                                      \n\u2502 \u2502 \u2514\u2500\u2500 ><\/span> \u8f93\u51fa: Visual Tokens (<\/span>Pixel Values + Time IDs)<\/span> \u2502
                                                      \n\u2502 \u2502
                                                      \n\u2514\u2500\u2500 ><\/span> \u5408\u5e76\u72b6\u6001: \u7edf\u4e00\u7684 PyTorch Tensor (<\/span>\u9002\u914d Transformer \u8f93\u5165)<\/span> \u2518
                                                      \n \u2502
                                                      \n \u25bc
                                                      \n[<\/span>2<\/span>. \u7edf\u4e00\u7f16\u7801\u9636\u6bb5 (<\/span>Unified Encoding)<\/span>]<\/span> <<\/span>\u2605 \u6838\u5fc3\/The Brain><\/span> \u2500\u2500\u2500\u2500\u2500\u2500\u2510
                                                      \n\u2502 \u2502
                                                      \n\u251c\u2500\u2500 <<\/span>\u8c03\u7528\u6a21\u5757><\/span>: core\/embedding.py (<\/span>\u5411\u91cf\u751f\u6210\u5668)<\/span> \u2502
                                                      \n\u2502 \u251c\u2500\u2500 \u4f9d\u8d56\u6a21\u578b: core\/model_setup.py (<\/span>\u52a0\u8f7d\u5171\u4eab Transformer)<\/span> \u2502
                                                      \n\u2502 \u2502 \u2514\u2500\u2500 \u6743\u91cd: google\/gemma-2-9b-it \u2502
                                                      \n\u2502 \u2502 \u2502
                                                      \n\u2502 \u251c\u2500\u2500 \u2699 \u524d\u5411\u4f20\u64ad (<\/span>Forward Pass)<\/span> \u2502
                                                      \n\u2502 \u2502 \u251c\u2500\u2500 \u8f93\u5165: \u5f02\u6784\u7684 Token \u5e8f\u5217 \u2502
                                                      \n\u2502 \u2502 \u251c\u2500\u2500 \u673a\u5236: \u5171\u4eab\u81ea\u6ce8\u610f\u529b (<\/span>Self-Attention)<\/span> \u2502
                                                      \n\u2502 \u2502 \u2502 (<\/span>\u6587\u672cToken\u4e0e\u89c6\u89c9Token\u5728\u540c\u4e00\u5c42\u8fdb\u884c\u4ea4\u4e92)<\/span> \u2502
                                                      \n\u2502 \u2502 \u2514\u2500\u2500 ><\/span> \u539f\u59cb\u8f93\u51fa: [<\/span>Batch, Seq_Len, Hidden_Dim]<\/span> \u2502
                                                      \n\u2502 \u2502 \u2502
                                                      \n\u2502 \u2514\u2500\u2500 📏 \u7ef4\u5ea6\u5bf9\u9f50 (<\/span>Projection &<\/span> Normalization)<\/span> \u2502
                                                      \n\u2502 \u251c\u2500\u2500 \u6295\u5f71: Linear\u5c42\u5c06\u7ef4\u5ea6\u5f3a\u5236\u8f6c\u4e3a config.D_MODEL \u2502
                                                      \n\u2502 \u251c\u2500\u2500 \u5f52\u4e00\u5316: L2 Normalize (<\/span>\u786e\u4fdd\u6a21\u6001\u95f4\u8ddd\u79bb\u53ef\u6bd4)<\/span> \u2502
                                                      \n\u2502 \u2514\u2500\u2500 ><\/span> \u6700\u7ec8\u4ea7\u7269: \u7edf\u4e00\u5411\u91cf (<\/span>Unified Vector, 2048<\/span>\u7ef4)<\/span> \u2502
                                                      \n\u2502 \u2502
                                                      \n\u2514\u2500\u2500 ><\/span> \u8f93\u51fa\u7ed3\u679c: Numpy Array [<\/span>0.12<\/span>, -0.56, 0.99<\/span>, ..<\/span>.]<\/span> \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
                                                      \n \u2502
                                                      \n \u25bc
                                                      \n[<\/span>3<\/span>. \u5b58\u50a8\u4e0e\u68c0\u7d22\u9636\u6bb5 (<\/span>Storage &<\/span> Retrieval)<\/span>]<\/span> \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
                                                      \n\u2502 \u2502
                                                      \n\u251c\u2500\u2500 <<\/span>\u8c03\u7528\u6a21\u5757><\/span>: core\/retriever.py (<\/span>\u6df7\u5408\u8bb0\u5fc6\u7ba1\u5bb6)<\/span> \u2502
                                                      \n\u2502 \u251c\u2500\u2500 \u4f9d\u8d56\u5b58\u50a8: embeddings_db\/chroma (<\/span>\u5411\u91cf\u6570\u636e\u5e93)<\/span> \u2502
                                                      \n\u2502 \u2502 \u2502
                                                      \n\u2502 \u251c\u2500\u2500 A. \u5199\u5165\u6a21\u5f0f (<\/span>Indexing)<\/span> \u2502
                                                      \n\u2502 \u2502 \u251c\u2500\u2500 \u52a8\u4f5c: \u5c06\u5411\u91cf + \u5143\u6570\u636e (<\/span>Metadata)<\/span> \u5b58\u5165\u96c6\u5408 \u2502
                                                      \n\u2502 \u2502 \u2514\u2500\u2500 \u7ed3\u679c: \u6301\u4e45\u5316\u5230\u78c1\u76d8 (<\/span>Chroma Persist)<\/span> \u2502
                                                      \n\u2502 \u2502 \u2502
                                                      \n\u2502 \u2514\u2500\u2500 B. \u67e5\u8be2\u6a21\u5f0f (<\/span>Searching)<\/span> \u2502
                                                      \n\u2502 \u251c\u2500\u2500 \u8f93\u5165: \u67e5\u8be2\u5411\u91cf (<\/span>Query Vector)<\/span> \u2502
                                                      \n\u2502 \u251c\u2500\u2500 \u8ba1\u7b97: \u4f59\u5f26\u76f8\u4f3c\u5ea6 (<\/span>Cosine Similarity)<\/span> \u2502
                                                      \n\u2502 \u2502 (<\/span>\u5728\u540c\u4e00\u7a7a\u95f4\u5185\u8ba1\u7b97 \u6587\u672c-\u56fe \/ \u56fe-\u89c6\u9891 \u7684\u8ddd\u79bb)<\/span> \u2502
                                                      \n\u2502 \u2514\u2500\u2500 ><\/span> \u6700\u7ec8\u7ed3\u679c: Top-K \u5339\u914d\u9879 (<\/span>\u5982: \u7528\u6587\u5b57\u641c\u5230\u7684\u89c6\u9891)<\/span> \u2502
                                                      \n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<\/p>\n

                                                      4.2.2 \u534f\u4f5c\u7ec6\u8282\u6df1\u5ea6\u89e3\u6790<\/h5>\n

                                                      \u4ee5\u4e0b\u662f\u8fd9\u4e9b\u4ee3\u7801\u6587\u4ef6\u5728\u4e00\u6b21\u201c\u8de8\u6a21\u6001\u68c0\u7d22\u201d\u4efb\u52a1（\u4f8b\u5982：\u7528\u6587\u5b57\u641c\u89c6\u9891）\u4e2d\u7684\u5177\u4f53\u63e1\u624b\u8fc7\u7a0b：<\/p>\n

                                                      1. \u9884\u5904\u7406\u534f\u4f5c (Processor & Config)<\/p>\n

                                                        \n
                                                      • \u8f93\u5165：\u7528\u6237\u8f93\u5165\u6587\u5b57 query = \u201c\u7ea2\u8272\u8dd1\u8f66\u5728\u516c\u8def\u4e0a\u201d\u3002<\/li>\n
                                                      • \u534f\u4f5c\u903b\u8f91：\n
                                                          \n
                                                        • main.py \u63a5\u6536\u6307\u4ee4，\u8c03\u7528 core\/modal_processor.py\u3002<\/li>\n
                                                        • modal_processor.py \u8bfb\u53d6 config.py \u4e2d\u7684 MAX_SEQ_LEN=4096，\u786e\u4fdd\u751f\u6210\u7684\u5e8f\u5217\u4e0d\u4f1a\u7206\u663e\u5b58\u3002<\/li>\n
                                                        • \u5b83\u5411 core\/model_setup.py \u501f\u7528\u521d\u59cb\u5316\u597d\u7684 Tokenizer，\u5c06\u6587\u5b57\u5207\u5206\u4e3a\u6570\u5b57 ID\u3002<\/li>\n<\/ul>\n<\/li>\n
                                                        • \u4ea7\u7269：\u4e00\u4e2a\u6807\u51c6\u7684 PyTorch Tensor，\u51c6\u5907\u9001\u5165\u663e\u5361\u3002<\/li>\n<\/ul>\n

                                                          2. \u6838\u5fc3\u7f16\u7801\u534f\u4f5c (Embedding & Model) \u8fd9\u662f Gemini \u67b6\u6784\u7684\u7075\u9b42，\u4e5f\u662f**\u201c\u591a\u6a21\u6001\u7edf\u4e00\u201d**\u53d1\u751f\u7684\u7269\u7406\u573a\u6240\u3002<\/p>\n

                                                            \n
                                                          • \u534f\u4f5c\u903b\u8f91：\n
                                                              \n
                                                            • core\/embedding.py \u83b7\u53d6\u5904\u7406\u597d\u7684 Tensor，\u5c06\u5176\u5582\u7ed9 core\/model_setup.py \u52a0\u8f7d\u7684 \u5171\u4eab Transformer \u6a21\u578b\u3002<\/li>\n
                                                            • \u5173\u952e\u70b9：\u6b64\u65f6\u6a21\u578b\u5e76\u4e0d\u533a\u5206\u8fd9\u662f\u201c\u6587\u5b57\u201d\u8fd8\u662f\u201c\u89c6\u9891\u201d\u3002\u5728\u6a21\u578b\u773c\u4e2d，\u5b83\u4eec\u90fd\u662f\u9700\u8981\u8fdb\u884c\u6ce8\u610f\u529b\u8ba1\u7b97\u7684 Token\u3002<\/li>\n
                                                            • \u5982\u679c\u8f93\u5165\u662f\u89c6\u9891，embedding.py \u4f1a\u7279\u522b\u5904\u7406\u8f93\u51fa\u7684\u7ef4\u5ea6（\u5bf9\u65f6\u95f4\u8f74\u548c\u7a7a\u95f4\u8f74\u53d6\u5e73\u5747\u6c60\u5316），\u5e76\u6839\u636e config.py \u4e2d\u7684 D_MODEL \u5f3a\u884c\u5c06\u5411\u91cf\u957f\u5ea6\u538b\u5236\u4e3a 2048 \u7ef4\u3002<\/li>\n
                                                            • \u6700\u540e，\u6267\u884c L2 \u5f52\u4e00\u5316\u3002\u8fd9\u4e00\u6b65\u81f3\u5173\u91cd\u8981，\u5b83\u628a\u6240\u6709\u5411\u91cf\u62c9\u4f38\u5230\u5355\u4f4d\u957f\u5ea6，\u4f7f\u5f97\u8ba1\u7b97\u76f8\u4f3c\u5ea6（\u5939\u89d2\u4f59\u5f26）\u6210\u4e3a\u53ef\u80fd\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                                                              3. \u8bb0\u5fc6\u68c0\u7d22\u534f\u4f5c (Retriever & DB)<\/p>\n

                                                                \n
                                                              • \u534f\u4f5c\u903b\u8f91：\n
                                                                  \n
                                                                • core\/retriever.py \u5524\u9192 embeddings_db\/chroma \u76ee\u5f55\u4e0b\u7684\u5411\u91cf\u6570\u636e\u5e93\u3002<\/li>\n
                                                                • \u5b83\u63a5\u6536 embedding.py \u5410\u51fa\u7684\u90a3\u4e2a 2048 \u7ef4\u5411\u91cf\u3002<\/li>\n
                                                                • \u5b83\u5728\u6570\u636e\u5e93\u4e2d\u5feb\u901f\u626b\u63cf\u51e0\u767e\u4e07\u4e2a\u5df2\u5b58\u50a8\u7684\u5411\u91cf（\u53ef\u80fd\u662f\u56fe\u7247\u751f\u6210\u7684，\u4e5f\u53ef\u80fd\u662f\u89c6\u9891\u751f\u6210\u7684）\u3002<\/li>\n
                                                                • \u9b54\u6cd5\u65f6\u523b：\u56e0\u4e3a\u5b83\u5728\u6570\u5b66\u4e0a\u627e\u5230\u4e86\u8ddd\u79bb\u6700\u8fd1\u7684\u5411\u91cf，\u800c\u8be5\u5411\u91cf\u7684\u5143\u6570\u636e\u6307\u5411\u4e00\u4e2a .mp4 \u6587\u4ef6，\u7cfb\u7edf\u6210\u529f\u5b9e\u73b0\u4e86\u201c\u7528\u6587\u5b57\u641c\u7d22\u89c6\u9891\u201d\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n

                                                                  \u603b\u7ed3<\/p>\n

                                                                    \n
                                                                  • config.py \u662f\u5baa\u6cd5：\u89c4\u5b9a\u4e86\u6240\u6709\u6a21\u6001\u5fc5\u987b\u9075\u5b88\u7684\u5c3a\u5bf8\u548c\u6807\u51c6\u3002<\/li>\n
                                                                  • model_setup.py \u662f\u5927\u8111：\u63d0\u4f9b\u4e86\u7406\u89e3\u6240\u6709\u6570\u636e\u7684\u5171\u4eab\u795e\u7ecf\u7f51\u7edc\u6743\u91cd\u3002<\/li>\n
                                                                  • modal_processor.py \u662f\u7ffb\u8bd1\u5b98：\u628a\u4eba\u7c7b\u80fd\u770b\u61c2\u7684\u56fe\/\u6587\/\u89c6，\u7ffb\u8bd1\u6210\u5927\u8111\u80fd\u61c2\u7684\u5e8f\u5217\u3002<\/li>\n
                                                                  • embedding.py \u662f\u538b\u7f29\u673a：\u628a\u5e9e\u5927\u7684\u5e8f\u5217\u538b\u7f29\u6210\u4e00\u4e2a\u7cbe\u7b80\u7684\u6570\u5b66\u5411\u91cf\u3002<\/li>\n
                                                                  • retriever.py \u662f\u56fe\u4e66\u7ba1\u7406\u5458：\u8d1f\u8d23\u628a\u8fd9\u4e9b\u5411\u91cf\u5206\u95e8\u522b\u7c7b\u5730\u5b58\u597d，\u5e76\u5728\u9700\u8981\u65f6\u5feb\u901f\u627e\u5230\u3002<\/li>\n<\/ul>\n

                                                                    \u4e94\u3001\u591a\u6a21\u6001\u8868\u793a\u7684\u9a8c\u8bc1\u4e0e\u4f18\u5316<\/h3>\n

                                                                    5.1 \u8868\u793a\u7edf\u4e00\u7684\u9a8c\u8bc1\u65b9\u6cd5<\/h4>\n

                                                                    5.1.1 \u5b9a\u6027\u9a8c\u8bc1：\u8de8\u6a21\u6001\u8bed\u4e49\u4e00\u81f4\u6027<\/h5>\n

                                                                    \u901a\u8fc7\u4eba\u5de5\u6807\u6ce8\u7684 \u201c\u6587\u672c – \u56fe\u50cf – \u89c6\u9891\u201d \u4e09\u5143\u7ec4，\u9a8c\u8bc1\u540c\u4e00\u8bed\u4e49\u5185\u5bb9\u7684\u5411\u91cf\u76f8\u4f3c\u5ea6\u662f\u5426\u9ad8\u4e8e\u4e0d\u540c\u8bed\u4e49：<\/p>\n

                                                                    \u8fd0\u884c<\/p>\n

                                                                    def<\/span> validate_semantic_consistency<\/span>(<\/span>embedder)<\/span>:<\/span>
                                                                    \n """\u9a8c\u8bc1\u8de8\u6a21\u6001\u8bed\u4e49\u4e00\u81f4\u6027"""<\/span>
                                                                    \n # \u6b63\u4f8b：\u540c\u4e00\u8bed\u4e49\u7684\u4e0d\u540c\u6a21\u6001<\/span>
                                                                    \n positive_samples =<\/span> {<\/span>
                                                                    \n "text"<\/span>:<\/span> "\u5c0f\u732b\u8ffd\u7740\u6bdb\u7ebf\u7403\u8dd1"<\/span>,<\/span>
                                                                    \n "image"<\/span>:<\/span> "data\/images\/cat_ball.jpg"<\/span>,<\/span>
                                                                    \n "video"<\/span>:<\/span> "data\/videos\/cat_ball.mp4"<\/span>
                                                                    \n }<\/span><\/p>\n

                                                                    # \u8d1f\u4f8b：\u4e0d\u540c\u8bed\u4e49\u7684\u6a21\u6001<\/span>
                                                                    \n negative_samples =<\/span> {<\/span>
                                                                    \n "text"<\/span>:<\/span> "\u98de\u673a\u5728\u5929\u7a7a\u4e2d\u98de\u884c"<\/span>,<\/span>
                                                                    \n "image"<\/span>:<\/span> "data\/images\/plane.jpg"<\/span>,<\/span>
                                                                    \n "video"<\/span>:<\/span> "data\/videos\/plane_fly.mp4"<\/span>
                                                                    \n }<\/span><\/p>\n

                                                                    # \u751f\u6210\u6b63\u4f8b\u5d4c\u5165<\/span>
                                                                    \n pos_text_emb =<\/span> embedder.<\/span>get_text_embedding(<\/span>positive_samples[<\/span>"text"<\/span>]<\/span>)<\/span>
                                                                    \n pos_image_emb =<\/span> embedder.<\/span>get_image_embedding(<\/span>positive_samples[<\/span>"image"<\/span>]<\/span>)<\/span>
                                                                    \n pos_video_emb =<\/span> embedder.<\/span>get_video_embedding(<\/span>positive_samples[<\/span>"video"<\/span>]<\/span>)<\/span><\/p>\n

                                                                    # \u751f\u6210\u8d1f\u4f8b\u5d4c\u5165<\/span>
                                                                    \n neg_text_emb =<\/span> embedder.<\/span>get_text_embedding(<\/span>negative_samples[<\/span>"text"<\/span>]<\/span>)<\/span><\/p>\n

                                                                    # \u8ba1\u7b97\u76f8\u4f3c\u5ea6<\/span>
                                                                    \n pos_text_image =<\/span> np.<\/span>dot(<\/span>pos_text_emb,<\/span> pos_image_emb)<\/span>
                                                                    \n pos_text_video =<\/span> np.<\/span>dot(<\/span>pos_text_emb,<\/span> pos_video_emb)<\/span>
                                                                    \n neg_text_image =<\/span> np.<\/span>dot(<\/span>neg_text_emb,<\/span> pos_image_emb)<\/span><\/p>\n

                                                                    print<\/span>(<\/span>"=== \u8bed\u4e49\u4e00\u81f4\u6027\u9a8c\u8bc1 ==="<\/span>)<\/span>
                                                                    \n print<\/span>(<\/span>f"\u6b63\u4f8b：\u6587\u672c-\u56fe\u50cf\u76f8\u4f3c\u5ea6 = <\/span>{<\/span>pos_text_image:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                                    \n print<\/span>(<\/span>f"\u6b63\u4f8b：\u6587\u672c-\u89c6\u9891\u76f8\u4f3c\u5ea6 = <\/span>{<\/span>pos_text_video:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                                    \n print<\/span>(<\/span>f"\u8d1f\u4f8b：\u6587\u672c-\u56fe\u50cf\u76f8\u4f3c\u5ea6 = <\/span>{<\/span>neg_text_image:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

                                                                    # \u9a8c\u8bc1\u9608\u503c（\u7406\u60f3\u60c5\u51b5\u4e0b\u6b63\u4f8b>0.7，\u8d1f\u4f8b<0.3）<\/span>
                                                                    \n assert<\/span> pos_text_image ><\/span> 0.7<\/span>,<\/span> "\u6b63\u4f8b\u6587\u672c-\u56fe\u50cf\u76f8\u4f3c\u5ea6\u8fc7\u4f4e"<\/span>
                                                                    \n assert<\/span> pos_text_video ><\/span> 0.7<\/span>,<\/span> "\u6b63\u4f8b\u6587\u672c-\u89c6\u9891\u76f8\u4f3c\u5ea6\u8fc7\u4f4e"<\/span>
                                                                    \n assert<\/span> neg_text_image <<\/span> 0.3<\/span>,<\/span> "\u8d1f\u4f8b\u76f8\u4f3c\u5ea6\u8fc7\u9ad8"<\/span>
                                                                    \n print<\/span>(<\/span>"\u9a8c\u8bc1\u901a\u8fc7！"<\/span>)<\/span><\/p>\n

                                                                    5.1.2 \u5b9a\u91cf\u9a8c\u8bc1：\u68c0\u7d22\u51c6\u786e\u7387<\/h5>\n

                                                                    \u4f7f\u7528\u6807\u51c6\u591a\u6a21\u6001\u6570\u636e\u96c6（\u5982 MSCOCO\u3001Flickr30k）\u8ba1\u7b97\u8de8\u6a21\u6001\u68c0\u7d22\u7684 Top-K \u51c6\u786e\u7387：<\/p>\n

                                                                    \u8fd0\u884c<\/p>\n

                                                                    def<\/span> calculate_retrieval_accuracy<\/span>(<\/span>retriever,<\/span> test_queries,<\/span> test_ground_truth)<\/span>:<\/span>
                                                                    \n """\u8ba1\u7b97\u8de8\u6a21\u6001\u68c0\u7d22\u51c6\u786e\u7387"""<\/span>
                                                                    \n top1_correct =<\/span> 0<\/span>
                                                                    \n top5_correct =<\/span> 0<\/span>
                                                                    \n total =<\/span> len<\/span>(<\/span>test_queries)<\/span><\/p>\n

                                                                    for<\/span> query_id,<\/span> query_emb in<\/span> test_queries.<\/span>items(<\/span>)<\/span>:<\/span>
                                                                    \n # \u68c0\u7d22<\/span>
                                                                    \n results =<\/span> retriever.<\/span>retrieve(<\/span>query_emb)<\/span>
                                                                    \n retrieved_ids =<\/span> [<\/span>res[<\/span>"id"<\/span>]<\/span> for<\/span> res in<\/span> results]<\/span><\/p>\n

                                                                    # \u68c0\u67e5Top-1<\/span>
                                                                    \n if<\/span> retrieved_ids[<\/span>0<\/span>]<\/span> in<\/span> test_ground_truth[<\/span>query_id]<\/span>:<\/span>
                                                                    \n top1_correct +=<\/span> 1<\/span><\/p>\n

                                                                    # \u68c0\u67e5Top-5<\/span>
                                                                    \n if<\/span> any<\/span>(<\/span>id<\/span> in<\/span> test_ground_truth[<\/span>query_id]<\/span> for<\/span> id<\/span> in<\/span> retrieved_ids[<\/span>:<\/span>5<\/span>]<\/span>)<\/span>:<\/span>
                                                                    \n top5_correct +=<\/span> 1<\/span><\/p>\n

                                                                    # \u8ba1\u7b97\u51c6\u786e\u7387<\/span>
                                                                    \n top1_acc =<\/span> top1_correct \/<\/span> total
                                                                    \n top5_acc =<\/span> top5_correct \/<\/span> total<\/p>\n

                                                                    print<\/span>(<\/span>f"Top-1 \u51c6\u786e\u7387：<\/span>{<\/span>top1_acc:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                                    \n print<\/span>(<\/span>f"Top-5 \u51c6\u786e\u7387：<\/span>{<\/span>top5_acc:<\/span>.4f<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
                                                                    \n return<\/span> top1_acc,<\/span> top5_acc<\/p>\n

                                                                    5.2 \u6027\u80fd\u4f18\u5316\u6280\u5de7<\/h4>\n

                                                                    5.2.1 \u6a21\u578b\u4f18\u5316<\/h5>\n
                                                                  • \n

                                                                    \u91cf\u5316：\u4f7f\u7528 INT8\/INT4 \u91cf\u5316\u51cf\u5c11\u663e\u5b58\u5360\u7528（\u901f\u5ea6\u63d0\u5347 2-4 \u500d）<\/p>\n

                                                                    \u8fd0\u884c<\/p>\n

                                                                    from<\/span> transformers import<\/span> BitsAndBytesConfig<\/p>\n

                                                                    # 4-bit\u91cf\u5316\u914d\u7f6e<\/span>
                                                                    \nbnb_config =<\/span> BitsAndBytesConfig(<\/span>
                                                                    \n load_in_4bit=<\/span>True<\/span>,<\/span>
                                                                    \n bnb_4bit_use_double_quant=<\/span>True<\/span>,<\/span>
                                                                    \n bnb_4bit_quant_type=<\/span>"nf4"<\/span>,<\/span>
                                                                    \n bnb_4bit_compute_dtype=<\/span>torch.<\/span>float16
                                                                    \n)<\/span><\/p>\n

                                                                    # \u52a0\u8f7d\u91cf\u5316\u6a21\u578b<\/span>
                                                                    \nmodel =<\/span> AutoModel.<\/span>from_pretrained(<\/span>
                                                                    \n MODEL_NAME,<\/span>
                                                                    \n quantization_config=<\/span>bnb_config,<\/span>
                                                                    \n device_map=<\/span>DEVICE
                                                                    \n)<\/span>\n <\/li>\n

                                                                  • \n

                                                                    \u6a21\u578b\u84b8\u998f：\u4f7f\u7528\u5927\u6a21\u578b\u84b8\u998f\u51fa\u5c0f\u6a21\u578b，\u4fdd\u6301\u8868\u793a\u80fd\u529b\u7684\u540c\u65f6\u63d0\u5347\u901f\u5ea6<\/p>\n<\/li>\n

                                                                    5.2.2 \u63a8\u7406\u4f18\u5316<\/h5>\n
                                                                  • \u6279\u5904\u7406：\u6279\u91cf\u5904\u7406\u591a\u6a21\u6001\u6570\u636e，\u63d0\u5347 GPU \u5229\u7528\u7387<\/li>\n
                                                                  • \u7f13\u5b58\u673a\u5236：\u7f13\u5b58\u9ad8\u9891\u8bbf\u95ee\u7684\u5d4c\u5165\u5411\u91cf，\u907f\u514d\u91cd\u590d\u8ba1\u7b97<\/li>\n
                                                                  • \u5f02\u6b65\u5904\u7406：\u89c6\u9891\u62bd\u5e27\u4e0e\u6a21\u578b\u63a8\u7406\u5f02\u6b65\u6267\u884c，\u51cf\u5c11\u7b49\u5f85\u65f6\u95f4<\/li>\n
                                                                    5.2.3 \u5b58\u50a8\u4f18\u5316<\/h5>\n
                                                                  • \n

                                                                    \u5411\u91cf\u538b\u7f29：\u4f7f\u7528 PCA \/ \u91cf\u5316\u5c06 2048 \u7ef4\u5411\u91cf\u538b\u7f29\u81f3 512 \u7ef4（\u7cbe\u5ea6\u635f\u5931 < 5%）<\/p>\n

                                                                    \u8fd0\u884c<\/p>\n

                                                                    from<\/span> sklearn.<\/span>decomposition import<\/span> PCA<\/p>\n

                                                                    # \u8bad\u7ec3PCA\u6a21\u578b（\u4f7f\u7528\u6837\u672c\u5d4c\u5165）<\/span>
                                                                    \npca =<\/span> PCA(<\/span>n_components=<\/span>512<\/span>)<\/span>
                                                                    \npca.<\/span>fit(<\/span>all_sample_embeddings)<\/span> # all_sample_embeddings\u4e3a\u6837\u672c\u5d4c\u5165\u77e9\u9635<\/span><\/p>\n

                                                                    # \u538b\u7f29\u5d4c\u5165<\/span>
                                                                    \ncompressed_emb =<\/span> pca.<\/span>transform(<\/span>original_emb)<\/span>\n <\/li>\n

                                                                  • \n

                                                                    \u5206\u5c42\u5b58\u50a8：\u9ad8\u9891\u8bbf\u95ee\u7684\u5d4c\u5165\u5b58\u5165\u5185\u5b58，\u4f4e\u9891\u5b58\u5165\u78c1\u76d8<\/p>\n<\/li>\n

                                                                    \u516d\u3001\u751f\u4ea7\u7ea7\u90e8\u7f72\u6700\u4f73\u5b9e\u8df5<\/h3>\n

                                                                    6.1 \u670d\u52a1\u5316\u5c01\u88c5（FastAPI）<\/h4>\n

                                                                    \u8fd0\u884c<\/p>\n

                                                                    from<\/span> fastapi import<\/span> FastAPI,<\/span> UploadFile,<\/span> File
                                                                    \nfrom<\/span> fastapi.<\/span>middleware.<\/span>cors import<\/span> CORSMiddleware
                                                                    \nimport<\/span> uvicorn
                                                                    \nimport<\/span> numpy as<\/span> np
                                                                    \nfrom<\/span> core.<\/span>embedding import<\/span> MultimodalEmbedding
                                                                    \nfrom<\/span> core.<\/span>retriever import<\/span> MultimodalRetriever<\/p>\n

                                                                    # \u521d\u59cb\u5316FastAPI<\/span>
                                                                    \napp =<\/span> FastAPI(<\/span>title=<\/span>"Gemini Multimodal API"<\/span>)<\/span>
                                                                    \napp.<\/span>add_middleware(<\/span>
                                                                    \n CORSMiddleware,<\/span>
                                                                    \n allow_origins=<\/span>[<\/span>"*"<\/span>]<\/span>,<\/span>
                                                                    \n allow_credentials=<\/span>True<\/span>,<\/span>
                                                                    \n allow_methods=<\/span>[<\/span>"*"<\/span>]<\/span>,<\/span>
                                                                    \n allow_headers=<\/span>[<\/span>"*"<\/span>]<\/span>,<\/span>
                                                                    \n)<\/span><\/p>\n

                                                                    # \u9884\u52a0\u8f7d\u7ec4\u4ef6<\/span>
                                                                    \nembedder =<\/span> MultimodalEmbedding(<\/span>)<\/span>
                                                                    \nretriever =<\/span> MultimodalRetriever(<\/span>)<\/span><\/p>\n

                                                                    # \u5065\u5eb7\u68c0\u67e5\u63a5\u53e3<\/span>
                                                                    \n@app.<\/span>get<\/span>(<\/span>"\/health"<\/span>)<\/span>
                                                                    \nasync<\/span> def<\/span> health_check<\/span>(<\/span>)<\/span>:<\/span>
                                                                    \n return<\/span> {<\/span>"status"<\/span>:<\/span> "healthy"<\/span>}<\/span><\/p>\n

                                                                    # \u6587\u672c\u5d4c\u5165\u63a5\u53e3<\/span>
                                                                    \n@app.<\/span>post<\/span>(<\/span>"\/embed\/text"<\/span>)<\/span>
                                                                    \nasync<\/span> def<\/span> embed_text<\/span>(<\/span>text:<\/span> str<\/span>)<\/span>:<\/span>
                                                                    \n emb =<\/span> embedder.<\/span>get_text_embedding(<\/span>text)<\/span>
                                                                    \n return<\/span> {<\/span>
                                                                    \n "embedding"<\/span>:<\/span> emb.<\/span>tolist(<\/span>)<\/span>,<\/span>
                                                                    \n "dim"<\/span>:<\/span> emb.<\/span>shape[<\/span>0<\/span>]<\/span>,<\/span>
                                                                    \n "modal_type"<\/span>:<\/span> "text"<\/span>
                                                                    \n }<\/span><\/p>\n

                                                                    # \u56fe\u50cf\u5d4c\u5165\u63a5\u53e3（\u652f\u6301\u6587\u4ef6\u4e0a\u4f20）<\/span>
                                                                    \n@app.<\/span>post<\/span>(<\/span>"\/embed\/image"<\/span>)<\/span>
                                                                    \nasync<\/span> def<\/span> embed_image<\/span>(<\/span>file<\/span>:<\/span> UploadFile =<\/span> File(<\/span>.<\/span>.<\/span>.<\/span>)<\/span>)<\/span>:<\/span>
                                                                    \n # \u4fdd\u5b58\u4e0a\u4f20\u6587\u4ef6<\/span>
                                                                    \n file_path =<\/span> f"temp\/<\/span>{<\/span>file<\/span>.<\/span>filename}<\/span><\/span>"<\/span><\/span>
                                                                    \n with<\/span> open<\/span>(<\/span>file_path,<\/span> "wb"<\/span>)<\/span> as<\/span> f:<\/span>
                                                                    \n f.<\/span>write(<\/span>await<\/span> file<\/span>.<\/span>read(<\/span>)<\/span>)<\/span>
                                                                    \n # \u751f\u6210\u5d4c\u5165<\/span>
                                                                    \n emb =<\/span> embedder.<\/span>get_image_embedding(<\/span>file_path)<\/span>
                                                                    \n return<\/span> {<\/span>
                                                                    \n "embedding"<\/span>:<\/span> emb.<\/span>tolist(<\/span>)<\/span>,<\/span>
                                                                    \n "dim"<\/span>:<\/span> emb.<\/span>shape[<\/span>0<\/span>]<\/span>,<\/span>
                                                                    \n "modal_type"<\/span>:<\/span> "image"<\/span>,<\/span>
                                                                    \n "filename"<\/span>:<\/span> file<\/span>.<\/span>filename
                                                                    \n }<\/span><\/p>\n

                                                                    # \u8de8\u6a21\u6001\u68c0\u7d22\u63a5\u53e3<\/span>
                                                                    \n@app.<\/span>post<\/span>(<\/span>"\/retrieve"<\/span>)<\/span>
                                                                    \nasync<\/span> def<\/span> retrieve<\/span>(<\/span>embedding:<\/span> list<\/span>[<\/span>float<\/span>]<\/span>,<\/span> modal_type:<\/span> str<\/span> =<\/span> None<\/span>)<\/span>:<\/span>
                                                                    \n query_emb =<\/span> np.<\/span>array(<\/span>embedding)<\/span>
                                                                    \n results =<\/span> retriever.<\/span>retrieve(<\/span>query_emb,<\/span> filter_modal_type=<\/span>modal_type)<\/span>
                                                                    \n return<\/span> {<\/span>"results"<\/span>:<\/span> results}<\/span><\/p>\n

                                                                    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
                                                                    \n uvicorn.<\/span>run(<\/span>app,<\/span> host=<\/span>"0.0.0.0"<\/span>,<\/span> port=<\/span>8000<\/span>)<\/span><\/p>\n

                                                                    6.2 \u76d1\u63a7\u4e0e\u65e5\u5fd7<\/h4>\n
                                                                  • \u6027\u80fd\u76d1\u63a7：\u4f7f\u7528 Prometheus + Grafana \u76d1\u63a7\u63a8\u7406\u5ef6\u8fdf\u3001\u663e\u5b58\u5360\u7528\u3001QPS<\/li>\n
                                                                  • \u65e5\u5fd7\u7ba1\u7406：\u7ed3\u6784\u5316\u65e5\u5fd7\u8bb0\u5f55\u6240\u6709\u8bf7\u6c42\u3001\u54cd\u5e94\u3001\u9519\u8bef\u4fe1\u606f<\/li>\n
                                                                  • \u6a21\u578b\u76d1\u63a7：\u5b9a\u671f\u9a8c\u8bc1\u5d4c\u5165\u8d28\u91cf，\u9632\u6b62\u6a21\u578b\u6f02\u79fb<\/li>\n

                                                                    6.3 \u6269\u5c55\u6027\u8bbe\u8ba1<\/h4>\n
                                                                  • \u5206\u5e03\u5f0f\u90e8\u7f72：\u591a\u8282\u70b9\u8d1f\u8f7d\u5747\u8861，\u652f\u6301\u6c34\u5e73\u6269\u5c55<\/li>\n
                                                                  • \u6a21\u578b\u7248\u672c\u7ba1\u7406：\u652f\u6301\u591a\u7248\u672c\u6a21\u578b\u5e76\u884c\u90e8\u7f72，\u7070\u5ea6\u53d1\u5e03<\/li>\n
                                                                  • \u5bb9\u9519\u673a\u5236：\u6a21\u578b\u63a8\u7406\u5931\u8d25\u65f6\u81ea\u52a8\u964d\u7ea7\u5230\u5907\u7528\u6a21\u578b \/ API<\/li>\n

                                                                    \u4e03\u3001\u603b\u7ed3<\/h3>\n

                                                                    Gemini \u901a\u8fc7\u539f\u751f\u591a\u6a21\u6001\u8bbe\u8ba1\u3001\u7edf\u4e00\u7684\u5e8f\u5217\u5316\u8f93\u5165\u3001\u5171\u4eab\u7684 Transformer \u7f16\u7801\u5668\u3001\u8de8\u6a21\u6001\u6ce8\u610f\u529b\u673a\u5236，\u771f\u6b63\u5b9e\u73b0\u4e86\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u7edf\u4e00\u8868\u793a\u3002\u5176\u6838\u5fc3\u4ef7\u503c\u5728\u4e8e：<\/p>\n

                                                                  • \u6253\u7834\u4e86\u6a21\u6001\u95f4\u7684\u8bed\u4e49\u58c1\u5792，\u4f7f\u8de8\u6a21\u6001\u7406\u89e3\u548c\u68c0\u7d22\u6210\u4e3a\u53ef\u80fd<\/li>\n
                                                                  • \u7edf\u4e00\u7684\u5411\u91cf\u7a7a\u95f4\u7b80\u5316\u4e86\u591a\u6a21\u6001\u5e94\u7528\u7684\u5f00\u53d1\u6d41\u7a0b<\/li>\n
                                                                  • \u7aef\u5230\u7aef\u7684\u67b6\u6784\u63d0\u5347\u4e86\u591a\u6a21\u6001\u5904\u7406\u7684\u6548\u7387\u548c\u7cbe\u5ea6<\/li>\n

                                                                    \u901a\u8fc7\u672c\u6587\u7684\u5b9e\u6218\u4ee3\u7801，\u5f00\u53d1\u8005\u53ef\u4ee5\u5feb\u901f\u642d\u5efa\u57fa\u4e8e Gemini \u7684\u591a\u6a21\u6001\u8868\u793a\u7cfb\u7edf，\u5e76\u901a\u8fc7\u4f18\u5316\u548c\u90e8\u7f72\u6280\u5de7，\u5c06\u5176\u843d\u5730\u4e3a\u751f\u4ea7\u7ea7\u5e94\u7528\u3002\u672a\u6765，\u968f\u7740 Gemini \u6a21\u578b\u7684\u6301\u7eed\u8fed\u4ee3，\u591a\u6a21\u6001\u8868\u793a\u7684\u7edf\u4e00\u7a0b\u5ea6\u548c\u5e94\u7528\u573a\u666f\u8fd8\u5c06\u8fdb\u4e00\u6b65\u6269\u5c55\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"

                                                                    Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a？
                                                                    \n\u6587\u6863\u6982\u8ff0
                                                                    \n\u672c\u6587\u6838\u5fc3\u4ef7\u503c
                                                                    \n\u6df1\u5ea6\u62c6\u89e3 Gemini \u591a\u6a21\u6001\u67b6\u6784\u7684\u6838\u5fc3\u8bbe\u8ba1\u903b\u8f91，\u89e3\u91ca\u5176\u5982\u4f55\u5b9e\u73b0\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u7edf\u4e00\u8868\u793a\u638c\u63e1 Gemini \u591a\u6a21\u6001\u6a21\u578b\u7684\u672c\u5730\u90e8\u7f72\u4e0e\u73af\u5883\u914d\u7f6e\u65b9\u6cd5\u901a\u8fc7\u53ef\u8fd0\u884c\u7684\u4ee3\u7801\u5b9e\u6218，\u5b9e\u73b0\u6587\u672c \/ \u56fe\u50cf \/ \u89c6\u9891\u7684\u7edf\u4e00\u5411\u91cf\u8868\u793a\u4e0e\u8de8\u6a21\u6001\u68c0\u7d22\u7406\u89e3\u591a\u6a21\u6001\u8868\u793a\u7edf\u4e00\u7684\u8c03\u8bd5\u3001\u4f18\u5316\u65b9\u6cd5\u4e0e\u751f\u4ea7\u7ea7\u90e8\u7f72\u6280\u5de7
                                                                    \n\u5b66\u4e60\u76ee\u6807
                                                                    \n\u7406\u89e3 Gemini \u591a\u6a21\u6001\u67b6\u6784\u7684\u6838\u5fc3\u7ec4\u4ef6\u4e0e\u8de8\u6a21\u6001\u5bf9\u9f50\u539f\u7406\u638c<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[50,51,214],"topic":[],"class_list":["post-69072","post","type-post","status-publish","format-standard","hentry","category-server","tag-50","tag-51","tag-214"],"yoast_head":"\nGemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/69072.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a？ \u6587\u6863\u6982\u8ff0 \u672c\u6587\u6838\u5fc3\u4ef7\u503c \u6df1\u5ea6\u62c6\u89e3 Gemini \u591a\u6a21\u6001\u67b6\u6784\u7684\u6838\u5fc3\u8bbe\u8ba1\u903b\u8f91，\u89e3\u91ca\u5176\u5982\u4f55\u5b9e\u73b0\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u7edf\u4e00\u8868\u793a\u638c\u63e1 Gemini \u591a\u6a21\u6001\u6a21\u578b\u7684\u672c\u5730\u90e8\u7f72\u4e0e\u73af\u5883\u914d\u7f6e\u65b9\u6cd5\u901a\u8fc7\u53ef\u8fd0\u884c\u7684\u4ee3\u7801\u5b9e\u6218，\u5b9e\u73b0\u6587\u672c \/ \u56fe\u50cf \/ \u89c6\u9891\u7684\u7edf\u4e00\u5411\u91cf\u8868\u793a\u4e0e\u8de8\u6a21\u6001\u68c0\u7d22\u7406\u89e3\u591a\u6a21\u6001\u8868\u793a\u7edf\u4e00\u7684\u8c03\u8bd5\u3001\u4f18\u5316\u65b9\u6cd5\u4e0e\u751f\u4ea7\u7ea7\u90e8\u7f72\u6280\u5de7 \u5b66\u4e60\u76ee\u6807 \u7406\u89e3 Gemini \u591a\u6a21\u6001\u67b6\u6784\u7684\u6838\u5fc3\u7ec4\u4ef6\u4e0e\u8de8\u6a21\u6001\u5bf9\u9f50\u539f\u7406\u638c\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/69072.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-01-30T19:57:37+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"20 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/69072.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/69072.html\",\"name\":\"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-01-30T19:57:37+00:00\",\"dateModified\":\"2026-01-30T19:57:37+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/69072.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/69072.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/69072.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/69072.html","og_locale":"zh_CN","og_type":"article","og_title":"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a？ \u6587\u6863\u6982\u8ff0 \u672c\u6587\u6838\u5fc3\u4ef7\u503c \u6df1\u5ea6\u62c6\u89e3 Gemini \u591a\u6a21\u6001\u67b6\u6784\u7684\u6838\u5fc3\u8bbe\u8ba1\u903b\u8f91，\u89e3\u91ca\u5176\u5982\u4f55\u5b9e\u73b0\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u7edf\u4e00\u8868\u793a\u638c\u63e1 Gemini \u591a\u6a21\u6001\u6a21\u578b\u7684\u672c\u5730\u90e8\u7f72\u4e0e\u73af\u5883\u914d\u7f6e\u65b9\u6cd5\u901a\u8fc7\u53ef\u8fd0\u884c\u7684\u4ee3\u7801\u5b9e\u6218，\u5b9e\u73b0\u6587\u672c \/ \u56fe\u50cf \/ \u89c6\u9891\u7684\u7edf\u4e00\u5411\u91cf\u8868\u793a\u4e0e\u8de8\u6a21\u6001\u68c0\u7d22\u7406\u89e3\u591a\u6a21\u6001\u8868\u793a\u7edf\u4e00\u7684\u8c03\u8bd5\u3001\u4f18\u5316\u65b9\u6cd5\u4e0e\u751f\u4ea7\u7ea7\u90e8\u7f72\u6280\u5de7 \u5b66\u4e60\u76ee\u6807 \u7406\u89e3 Gemini \u591a\u6a21\u6001\u67b6\u6784\u7684\u6838\u5fc3\u7ec4\u4ef6\u4e0e\u8de8\u6a21\u6001\u5bf9\u9f50\u539f\u7406\u638c","og_url":"https:\/\/www.wsisp.com\/helps\/69072.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-01-30T19:57:37+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"20 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/69072.html","url":"https:\/\/www.wsisp.com\/helps\/69072.html","name":"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-01-30T19:57:37+00:00","dateModified":"2026-01-30T19:57:37+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/69072.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/69072.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/69072.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"Gemini \u7684\u591a\u6a21\u6001\u67b6\u6784\u8bbe\u8ba1\u5982\u4f55\u7edf\u4e00\u6587\u672c\u3001\u56fe\u50cf\u3001\u89c6\u9891\u7684\u8868\u793a\uff1f"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/69072","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=69072"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/69072\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=69072"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=69072"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=69072"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=69072"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}