{"id":119366,"date":"2026-10-29T13:40:46","date_gmt":"2026-10-29T05:40:46","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/119366.html"},"modified":"2026-10-29T13:40:46","modified_gmt":"2026-10-29T05:40:46","slug":"%e7%ac%ac%e4%b8%89%e7%ab%a0-%e5%bc%ba%e5%8c%96%e5%ad%a6%e4%b9%a0-%e5%a4%9a%e8%87%82%e8%80%81%e8%99%8e%e6%9c%ba","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/119366.html","title":{"rendered":"\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 -- \u591a\u81c2\u8001\u864e\u673a"},"content":{"rendered":"<h3>\u4e00\u3001\u591a\u81c2\u8001\u864e\u673a\u95ee\u9898<\/h3>\n<h4>1.1 \u95ee\u9898\u5f62\u5f0f\u5316<\/h4>\n<p>\u591a\u81c2\u8001\u864e\u673a&#xff08;Multi-Armed Bandit, MAB&#xff09; \u662f\u5f3a\u5316\u5b66\u4e60\u6700\u7b80\u5355\u7684\u8bbe\u5b9a&#xff1a;\u667a\u80fd\u4f53\u6bcf\u4e00\u6b65\u4ece <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">KK<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><\/span><\/span><\/span><\/span> \u4e2a\u52a8\u4f5c&#xff08;\u201c\u6447\u81c2\u201d&#xff09;\u4e2d\u9009\u4e00\u4e2a&#xff0c;\u73af\u5883\u8fd4\u56de\u4e00\u4e2a\u5956\u52b1&#xff1b;\u4e0d\u5b58\u5728\u72b6\u6001\u8f6c\u79fb&#xff0c;\u6bcf\u4e00\u6b65\u72ec\u7acb\u3002<\/p>\n<p>\u5f62\u5f0f\u5316\u5730&#xff0c;\u6a21\u578b\u7531\u4ee5\u4e0b\u8981\u7d20\u7ec4\u6210&#xff1a;<\/p>\n<ul>\n<li>\u52a8\u4f5c\u96c6\u5408 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">A&#061;{1,2,\u2026,K}\\\\mathcal{A} &#061; \\\\{1, 2, \\\\ldots, K\\\\}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathcal\">A<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">{<\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">2<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mclose\">}<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u6bcf\u4e2a\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aa<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u5bf9\u5e94\u4e00\u4e2a\u672a\u77e5\u7684\u5956\u52b1\u5206\u5e03 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bda\\\\nu_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0637em\">\u03bd<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0637em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<li>\u5728\u65f6\u523b <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">tt<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6151em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u667a\u80fd\u4f53\u9009\u62e9\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">At\u2208AA_t \\\\in \\\\mathcal{A}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathcal\">A<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u89c2\u6d4b\u5230\u5956\u52b1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Rt\u223c\u03bdAtR_t \\\\sim \\\\nu_{A_t}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0077em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u223c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6807em;vertical-align: -0.2501em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0637em\">\u03bd<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0637em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2963em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2501em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<li>\u667a\u80fd\u4f53\u7684\u7b56\u7565 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03c0\\\\pi<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c0<\/span><\/span><\/span><\/span><\/span> \u53ea\u4f9d\u8d56\u4e8e\u5386\u53f2 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">(A1,R1,\u2026,At\u22121,Rt\u22121)(A_1, R_1, \\\\ldots, A_{t-1}, R_{t-1})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0077em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0077em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u4e0d\u4f9d\u8d56\u4e8e&#034;\u72b6\u6001&#034;\u2014\u2014\u56e0\u4e3a\u8001\u864e\u673a\u6ca1\u6709\u72b6\u6001\u3002<\/li>\n<\/ul>\n<p>\u6bcf\u4e2a\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aa<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u7684\u671f\u671b\u5956\u52b1\u5b9a\u4e49\u4e3a&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bca&#061;E[R\u2223A&#061;a]&#061;\u222br\u2009d\u03bda(r)<br \/>\n\\\\mu_a &#061; \\\\mathbb{E}[R \\\\mid A &#061; a] &#061; \\\\int r \\\\, d\\\\nu_a(r)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathbb\">E<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.2222em;vertical-align: -0.8622em\"><\/span><span class=\"mop op-symbol large-op\" style=\"margin-right: 0.4445em;position: relative;top: -0.0011em\">\u222b<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">d<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0637em\">\u03bd<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0637em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u6700\u4f18\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">a\u2217&#061;arg\u2061max\u2061a\u03bcaa^* &#061; \\\\arg\\\\max_a \\\\mu_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6887em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6887em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mbin mtight\">\u2217<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mop\">ar<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\"><span class=\"mop\">max<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u5bf9\u5e94\u7684\u6700\u5927\u671f\u671b\u5956\u52b1\u4e3a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc\u2217&#061;max\u2061a\u03bca\\\\mu^* &#061; \\\\max_a \\\\mu_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8831em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6887em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mbin mtight\">\u2217<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mop\"><span class=\"mop\">max<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/p>\n<p>MAB \u7684\u6838\u5fc3\u96be\u9898\u662f\u5956\u52b1\u5206\u5e03\u672a\u77e5\u3002\u667a\u80fd\u4f53\u5fc5\u987b\u5728&#034;\u5229\u7528\u5df2\u77e5\u597d\u7684\u52a8\u4f5c&#034;\u548c&#034;\u63a2\u7d22\u672a\u77e5\u53ef\u80fd\u66f4\u597d\u7684\u52a8\u4f5c&#034;\u4e4b\u95f4\u6743\u8861\u3002\u8868 3-1 \u5c55\u793a\u4e86 MAB \u4e0e\u4e00\u822c RL \u7684\u5173\u952e\u533a\u522b&#xff1a;<\/p>\n<p>\u8868 3-1 MAB \u4e0e\u4e00\u822c RL \u7684\u5173\u952e\u533a\u522b<\/p>\n<table>\n<tr>\u7ef4\u5ea6MAB\u4e00\u822c RL<\/tr>\n<tbody>\n<tr>\n<td>\u72b6\u6001<\/td>\n<td>\u65e0&#xff08;\u6bcf\u6b65\u72ec\u7acb&#xff09;<\/td>\n<td>\u6709&#xff08;\u6b65\u95f4\u5173\u8054&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u8f6c\u79fb<\/td>\n<td>\u65e0<\/td>\n<td><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">P(s\u2032\u2223s,a)P(s&#039; \\\\mid s, a)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0019em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">s<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7519em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">s<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/td>\n<\/tr>\n<tr>\n<td>\u76ee\u6807<\/td>\n<td>\u6700\u5927\u5316\u7d2f\u79ef\u5956\u52b1<\/td>\n<td>\u540c\u5de6<\/td>\n<\/tr>\n<tr>\n<td>\u8bc4\u4f30<\/td>\n<td>\u7d2f\u79ef\u9057\u61be&#xff08;regret&#xff09;<\/td>\n<td>\u7d2f\u79ef\u5956\u52b1\u6216\u7b56\u7565\u56de\u62a5<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>1.2 \u63a2\u7d22\u4e0e\u5229\u7528\u6743\u8861<\/h4>\n<p>\u5229\u7528&#xff08;exploitation&#xff09; \u6307\u9009\u62e9\u5f53\u524d\u4f30\u8ba1\u6700\u4f18\u7684\u52a8\u4f5c\u4ee5\u83b7\u53d6\u6700\u9ad8\u5373\u65f6\u5956\u52b1\u3002<\/p>\n<p>\u63a2\u7d22&#xff08;exploration&#xff09; \u6307\u9009\u62e9\u4f30\u8ba1\u4e0d\u786e\u5b9a\u6216\u6b21\u4f18\u7684\u52a8\u4f5c\u4ee5\u83b7\u53d6\u65b0\u7684\u4fe1\u606f&#xff0c;\u53ef\u80fd\u5728\u672a\u6765\u83b7\u5f97\u66f4\u9ad8\u5956\u52b1\u3002<\/p>\n<p>\u4e8c\u8005\u6784\u6210\u4e00\u5bf9\u6839\u672c\u77db\u76fe&#xff1a;\u53ea\u5229\u7528\u5219\u6c38\u8fdc\u4e0d\u77e5\u9053\u662f\u5426\u6709\u66f4\u597d\u7684\u52a8\u4f5c&#xff1b;\u53ea\u63a2\u7d22\u5219\u77ed\u671f\u56de\u62a5\u5f88\u4f4e\u3002MAB \u7b97\u6cd5\u8bbe\u8ba1\u7684\u6838\u5fc3\u5c31\u662f\u5728\u4e8c\u8005\u4e4b\u95f4\u53d6\u5f97\u5e73\u8861\u3002<\/p>\n<p>\u76f4\u89c9\u4f8b\u5b50&#xff1a;\u4e00\u4e2a 10 \u81c2\u8001\u864e\u673a\u4e2d&#xff0c;\u6700\u4f18\u81c2\u7684\u671f\u671b\u5956\u52b1\u662f 1.0&#xff0c;\u5176\u4ed6\u81c2\u7684\u671f\u671b\u5956\u52b1\u5728 0 \u5230 0.9 \u4e4b\u95f4\u3002\u521d\u59cb\u65f6\u4e0d\u77e5\u9053\u54ea\u4e2a\u6700\u4f18\u3002\u5982\u679c\u4e00\u76f4\u8d2a\u5fc3\u9009\u62e9\u7b2c\u4e00\u6b21\u770b\u8d77\u6765\u6700\u597d\u7684\u81c2&#xff0c;\u53ef\u80fd\u6c38\u8fdc\u9519\u8fc7\u4e86\u771f\u6b63\u7684\u6700\u4f18\u81c2&#xff1b;\u5982\u679c\u4e00\u76f4\u968f\u673a\u63a2\u7d22&#xff0c;\u7d2f\u79ef\u5956\u52b1\u8fdc\u4f4e\u4e8e&#034;\u65e9\u65e9\u8bc6\u522b\u6700\u4f18\u81c2&#034;\u7684\u7b56\u7565\u3002<\/p>\n<p>\u5f62\u5f0f\u5316\u5730&#xff0c;\u53ef\u4ee5\u7528&#034;\u76ee\u6807\u51b2\u7a81&#034;\u523b\u753b\u63a2\u7d22-\u5229\u7528\u5f20\u529b&#xff1a;<\/p>\n<ul>\n<li>\u77ed\u671f\u76ee\u6807&#xff1a;\u6bcf\u6b65\u9009 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ata_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u6700\u5927\u5316 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">RtR_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0077em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<li>\u957f\u671f\u76ee\u6807&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u2211t&#061;1TRt\\\\sum_{t&#061;1}^{T} R_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.2809em;vertical-align: -0.2997em\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9812em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2029em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2997em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0077em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u6700\u5927\u5316&#xff0c;\u8981\u6c42\u603b\u6536\u76ca&#xff08;\u4e0d\u53ea\u662f\u5355\u6b65&#xff09;\u6700\u5927\u3002<\/li>\n<li>\u4efb\u4f55 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ata_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u7684&#034;\u662f\u5426\u6700\u4f18&#034;\u53d6\u51b3\u4e8e&#034;\u4fe1\u606f\u662f\u5426\u8db3\u591f&#034;\u2014\u2014\u800c\u4fe1\u606f\u6765\u81ea\u88ab\u9009\u4e2d\u7684\u52a8\u4f5c\u3002<\/li>\n<\/ul>\n<p>MAB \u7b97\u6cd5\u901a\u8fc7\u4e0d\u540c\u65b9\u5f0f\u5904\u7406\u8fd9\u5bf9\u51b2\u7a81&#xff1a;\u6734\u7d20\u968f\u673a\/\u8d2a\u5fc3\u7b97\u6cd5\u53ea\u5229\u7528\u6216\u53ea\u63a2\u7d22&#xff0c;\u6ca1\u6709\u5e73\u8861&#xff1b;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span>-\u8d2a\u5fc3\u4ee5\u56fa\u5b9a\u6bd4\u4f8b\u6df7\u5408&#xff1b;UCB \u7cfb\u5217&#034;\u4e50\u89c2\u9762\u5bf9\u4e0d\u786e\u5b9a\u6027&#034;\u4e3b\u52a8\u9009\u4e0d\u786e\u5b9a\u7684\u52a8\u4f5c&#xff1b;Thompson \u91c7\u6837&#034;\u6982\u7387\u5339\u914d&#034;&#xff0c;\u6309\u540e\u9a8c\u6982\u7387\u9009\u52a8\u4f5c\u3002<\/p>\n<table>\n<tr>\u7b97\u6cd5\u5904\u7406\u65b9\u5f0f\u7d2f\u79ef\u9057\u61be<\/tr>\n<tbody>\n<tr>\n<td>\u968f\u673a<\/td>\n<td>\u7eaf\u63a2\u7d22<\/td>\n<td><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u0398(T)\\\\Theta(T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">\u0398<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/td>\n<\/tr>\n<tr>\n<td>\u8d2a\u5fc3<\/td>\n<td>\u7eaf\u5229\u7528<\/td>\n<td><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u0398(T)\\\\Theta(T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">\u0398<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff08;\u6b21\u4f18\u81c2\u88ab\u9501\u65f6&#xff09;<\/td>\n<\/tr>\n<tr>\n<td><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span>-\u8d2a\u5fc3<\/td>\n<td>\u56fa\u5b9a\u6982\u7387\u6df7\u5408<\/td>\n<td><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(T)O(T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/td>\n<\/tr>\n<tr>\n<td>UCB1<\/td>\n<td>\u4e50\u89c2\u9762\u5bf9\u4e0d\u786e\u5b9a\u6027<\/td>\n<td><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(log\u2061T)O(\\\\log T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff08;gap-dependent&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>Thompson \u91c7\u6837<\/td>\n<td>\u6982\u7387\u5339\u914d\u540e\u9a8c<\/td>\n<td><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O~(KT)\\\\tilde O(\\\\sqrt{KT})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1767em;vertical-align: -0.25em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9202em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><\/span><span class=\"\" style=\"top: -3.6023em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">~<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9267em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8867em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1133em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff08;gap-free&#xff09;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>1.3 \u7d2f\u79ef\u9057\u61be\u4e0e\u5e73\u5747\u9057\u61be<\/h4>\n<p>\u7d2f\u79ef\u9057\u61be&#xff08;cumulative regret&#xff09; \u8861\u91cf\u7b97\u6cd5\u5728 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">TT<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span> \u6b65\u5185\u76f8\u5bf9\u4e8e&#034;\u59cb\u7ec8\u9009\u6700\u4f18\u52a8\u4f5c&#034;\u7684\u603b\u635f\u5931&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">R(T)\u225c\u2211t&#061;1T(\u03bc\u2217\u2212\u03bcAt)&#061;\u2211t&#061;1T\u0394At<br \/>\nR(T) \\\\triangleq \\\\sum_{t&#061;1}^{T} \\\\left( \\\\mu^* &#8211; \\\\mu_{A_t} \\\\right) &#061; \\\\sum_{t&#061;1}^{T} \\\\Delta_{A_t}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1667em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel amsrm\">\u225c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 3.0954em;vertical-align: -1.2671em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8283em\"><span class=\"\" style=\"top: -1.8829em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.2671em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7387em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mbin mtight\">\u2217<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2963em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2501em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\">)<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 3.0954em;vertical-align: -1.2671em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8283em\"><span class=\"\" style=\"top: -1.8829em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.2671em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord\">\u0394<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2963em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2501em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bcAt\\\\mu_{A_t}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6807em;vertical-align: -0.2501em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2963em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2501em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">AtA_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u7684\u771f\u5b9e\u671f\u671b\u5956\u52b1&#xff1b;\u8fd9\u4e00\u5f62\u5f0f\u662f\u4f2a\u9057\u61be&#xff08;pseudo-regret&#xff09;&#xff0c;\u4e0e\u6587\u732e\u6807\u51c6\u4e00\u81f4\u3002<\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u0394a&#061;\u03bc\u2217\u2212\u03bca\\\\Delta_a &#061; \\\\mu^* &#8211; \\\\mu_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord\">\u0394<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8831em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6887em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mbin mtight\">\u2217<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aa<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u7684&#034;\u6b21\u4f18\u5dee\u8ddd&#034;\u3002\u6700\u4f18\u7b97\u6cd5\u76ee\u6807\u662f\u8ba9 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Regret(T)&#061;o(T)\\\\mathrm{Regret}(T) &#061; o(T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">Regret<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">o<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff08;\u6b21\u7ebf\u6027\u589e\u957f&#xff09;&#xff0c;\u5373\u5e73\u5747\u9057\u61be <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Regret(T)\/T\u21920\\\\mathrm{Regret}(T)\/T \\\\to 0<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">Regret<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2192<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0<\/span><\/span><\/span><\/span><\/span>\u3002<\/p>\n<p>\u6b21\u7ebf\u6027\u9057\u61be\u6709\u4e24\u79cd\u5178\u578b\u589e\u957f\u7387&#xff1a;<\/p>\n<ul>\n<li>Gap-dependent&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(\u2211alog\u2061T\/\u0394a)O(\\\\sum_a \\\\log T \/ \\\\Delta_a)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0497em;vertical-align: -0.2997em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.0017em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2997em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mord\">\/<\/span><span class=\"mord\"><span class=\"mord\">\u0394<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;UCB1\u3001Auer et al. 2002 Theorem 1 \u7b49\u3002<\/li>\n<li>Gap-free \/ min-max&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O~(KT)\\\\tilde O(\\\\sqrt{KT})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1767em;vertical-align: -0.25em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9202em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><\/span><span class=\"\" style=\"top: -3.6023em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">~<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9267em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8867em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1133em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;Auer et al. 2002 \/ 2003\u3002<\/li>\n<\/ul>\n<p>\u6734\u7d20\u7b56\u7565\u5982 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span>-\u8d2a\u5fc3\u7684\u9057\u61be\u662f <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(T)O(T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u56e0\u4e3a\u5b83\u6301\u7eed\u4ee5\u5e38\u6570\u6982\u7387\u505a\u6b21\u4f18\u63a2\u7d22&#xff0c;\u4ece\u4e0d\u505c\u4e0b\u3002<\/p>\n<h3>\u4e8c\u3001\u6734\u7d20\u7b56\u7565<\/h3>\n<h4>2.1 \u968f\u673a\u7b56\u7565\u4e0e\u8d2a\u5fc3\u7b56\u7565<\/h4>\n<p>\u968f\u673a\u7b56\u7565&#xff1a;\u6bcf\u4e00\u6b65\u4ece <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">KK<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><\/span><\/span><\/span><\/span> \u4e2a\u52a8\u4f5c\u4e2d\u7b49\u6982\u7387\u91c7\u6837\u3002\u8bbe\u5404\u81c2\u771f\u5b9e\u671f\u671b\u4e3a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bca\\\\mu_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3001\u6700\u4f18 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc\u2217&#061;max\u2061a\u03bca\\\\mu^* &#061; \\\\max_a \\\\mu_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8831em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6887em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mbin mtight\">\u2217<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mop\"><span class=\"mop\">max<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002\u5219\u5728 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">TT<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span> \u6b65\u7684\u7d2f\u79ef\u671f\u671b\u5956\u52b1\u4e3a&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">E\u2009\u2063[\u2211t&#061;1TRt]&#061;T\u22c51K\u2211a\u03bca<br \/>\n\\\\mathbb{E}\\\\!\\\\left[\\\\sum_{t&#061;1}^T R_t\\\\right] &#061; T \\\\cdot \\\\frac{1}{K} \\\\sum_a \\\\mu_a<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 3.0954em;vertical-align: -1.2671em\"><\/span><span class=\"mord mathbb\">E<\/span><span class=\"mspace\" style=\"margin-right: -0.1667em\"><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size4\">[<\/span><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8283em\"><span class=\"\" style=\"top: -1.8829em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.2671em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0077em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size4\">]<\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.5714em;vertical-align: -1.25em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.05em\"><span class=\"\" style=\"top: -1.9em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.25em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u7d2f\u79ef\u9057\u61be\u4e3a&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Regret(T)&#061;T\u22c5(\u03bc\u2217\u22121K\u2211a\u03bca)&#061;\u0398(T)<br \/>\n\\\\mathrm{Regret}(T) &#061; T \\\\cdot \\\\left( \\\\mu^* &#8211; \\\\frac{1}{K} \\\\sum_a \\\\mu_a \\\\right) &#061; \\\\Theta(T)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">Regret<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 3em;vertical-align: -1.25em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size4\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7387em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mbin mtight\">\u2217<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.05em\"><span class=\"\" style=\"top: -1.9em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.25em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size4\">)<\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">\u0398<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u8d2a\u5fc3\u7b56\u7565&#xff1a;\u9009\u5386\u53f2\u5e73\u5747\u5956\u52b1\u6700\u5927\u7684\u52a8\u4f5c\u3002\u5f62\u5f0f\u5316&#xff1a;\u6bcf\u6b65 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">at&#061;arg\u2061max\u2061a\u03bc^aa_t &#061; \\\\arg\\\\max_a \\\\hat{\\\\mu}_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mop\">ar<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\"><span class=\"mop\">max<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002\u95ee\u9898\u5728\u4e8e\u521d\u59cb\u51e0\u6b65\u91c7\u6837\u4e0d\u8db3\u4ee5\u4f30\u8ba1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bca\\\\mu_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff1a;\u5f53 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bca\\\\mu_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u5dee\u5f02\u5c0f\u3001\u6837\u672c\u566a\u58f0\u76f8\u5bf9\u8f83\u5927\u65f6&#xff0c;\u8d2a\u5fc3\u5f88\u5bb9\u6613&#034;\u9501\u5b9a&#034;\u5728\u7b2c\u4e00\u4e2a\u88ab\u968f\u673a\u9009\u5230\u7684\u6b21\u4f18\u52a8\u4f5c\u4e0a\u3002<\/p>\n<p>\u53cd\u4f8b&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">K&#061;2K&#061;2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">2<\/span><\/span><\/span><\/span><\/span>&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc1&#061;0.9,\u03bc2&#061;0.6\\\\mu_1 &#061; 0.9, \\\\mu_2 &#061; 0.6<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8389em;vertical-align: -0.1944em\"><\/span><span class=\"mord\">0.9<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.6<\/span><\/span><\/span><\/span><\/span>\u3002\u82e5\u7b2c\u4e00\u6b65\u968f\u673a\u9009\u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">a&#061;2a&#061;2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">2<\/span><\/span><\/span><\/span><\/span>&#xff08;\u6b21\u4f18&#xff09;&#xff0c;\u6837\u672c\u5e73\u5747 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc^2&#061;0.6\\\\hat\\\\mu_2 &#061; 0.6<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.6<\/span><\/span><\/span><\/span><\/span>\u3002\u5373\u4f7f <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">a&#061;1a&#061;1<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">1<\/span><\/span><\/span><\/span><\/span> \u4f30\u8ba1\u4e3a 0&#xff08;\u4e00\u6b21\u90fd\u6ca1\u9009&#xff09;&#xff0c;\u8d2a\u5fc3\u4ecd\u4f1a\u6301\u7eed\u9009 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">a&#061;2a&#061;2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">2<\/span><\/span><\/span><\/span><\/span>\u3002\u6b64\u65f6 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Regret(T)&#061;0.3T&#061;\u0398(T)\\\\mathrm{Regret}(T) &#061; 0.3 T &#061; \\\\Theta(T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">Regret<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord\">0.3<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">\u0398<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u4e0e\u968f\u673a\u7b56\u7565\u540c\u91cf\u7ea7\u3002<\/p>\n<p><span class=\"token comment\"># \u8d2a\u5fc3\u9501\u5b9a\u53cd\u4f8b<\/span><br \/>\nQ <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">0.0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.6<\/span><span class=\"token punctuation\">]<\/span>   <span class=\"token comment\"># \u5386\u53f2\u5e73\u5747&#xff1a;a&#061;1 \u4f30\u8ba1 0&#xff08;\u672a\u8bbf\u95ee&#xff09;&#xff0c;a&#061;2 \u4f30\u8ba1 0.6<\/span><br \/>\naction <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">.<\/span>index<span class=\"token punctuation\">(<\/span><span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>   <span class=\"token comment\"># \u9009 a&#061;2 \u9501\u5b9a<\/span><\/p>\n<p>\u53cd\u76f4\u89c9\u4e4b\u5904&#xff1a;\u8d2a\u5fc3\u7b56\u7565\u5728\u6837\u672c\u91cf\u6781\u5c0f\u65f6\u7b49\u4ef7\u4e8e&#034;\u5148\u5165\u4e3a\u4e3b&#034;\u3002 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span>-\u8d2a\u5fc3\u4e0e UCB \u662f\u5e38\u89c1\u7684\u4fee\u590d\u65b9\u5f0f\u3002<\/p>\n<h4>2.2 \u03b5-\u8d2a\u5fc3\u4e0e\u8870\u51cf \u03b5-\u8d2a\u5fc3<\/h4>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span>-\u8d2a\u5fc3 \u4ee5\u6982\u7387 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">1\u2212\u03b51 &#8211; \\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">1<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span> \u9009\u5f53\u524d\u6700\u4f18\u52a8\u4f5c&#xff08;\u5229\u7528&#xff09;&#xff0c;\u4ee5\u6982\u7387 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span> \u5728\u6240\u6709\u52a8\u4f5c\u4e2d\u5747\u5300\u968f\u673a&#xff08;\u63a2\u7d22&#xff09;&#xff1a;<\/p>\n<p><span class=\"token keyword\">import<\/span> random<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">epsilon_greedy<\/span><span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">,<\/span> epsilon<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">if<\/span> random<span class=\"token punctuation\">.<\/span>random<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&lt;<\/span> epsilon<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> random<span class=\"token punctuation\">.<\/span>randint<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> K <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>   <span class=\"token comment\"># \u63a2\u7d22&#xff1a;\u968f\u673a\u9009<\/span><br \/>\n    <span class=\"token keyword\">else<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> argmax<span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">)<\/span>                  <span class=\"token comment\"># \u5229\u7528&#xff1a;\u9009\u5386\u53f2\u6700\u4f18<\/span><\/p>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span> \u56fa\u5b9a\u65f6&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span>-\u8d2a\u5fc3\u7684\u9057\u61be\u662f <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(T)O(T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>\u2014\u2014\u56e0\u4e3a\u65e0\u8bba <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">tt<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6151em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><\/span> \u591a\u5927&#xff0c;\u6bcf\u6b65\u90fd\u8fd8\u6709 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span> \u7684\u6982\u7387\u9009\u6b21\u4f18\u52a8\u4f5c\u3002<\/p>\n<p>\u8870\u51cf <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span>-\u8d2a\u5fc3 \u4ee4 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span> \u968f\u65f6\u95f4\u8870\u51cf&#xff0c;\u5982 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5t&#061;min\u2061(1,c\/t)\\\\varepsilon_t &#061; \\\\min(1, c \/ t)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03b5<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">min<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">c<\/span><span class=\"mord\">\/<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>\u3002\u76f4\u89c9\u662f\u65e9\u671f\u9700\u8981\u66f4\u591a\u63a2\u7d22&#xff0c;\u540e\u671f\u4fe1\u606f\u79ef\u7d2f\u540e\u9010\u6b65\u8f6c\u5229\u7528\u3002\u8fd9\u80fd\u8ba9\u9057\u61be\u964d\u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(log\u2061T)O(\\\\log T)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u4f46\u9700\u8981\u9884\u8bbe\u8870\u51cf\u7387&#xff0c;\u5de5\u7a0b\u4e0a\u4e0d\u5982 UCB \u9c81\u68d2\u3002<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">decaying_epsilon_greedy<\/span><span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">,<\/span> t<span class=\"token punctuation\">,<\/span> c<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1.0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    epsilon <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1.0<\/span><span class=\"token punctuation\">,<\/span> c <span class=\"token operator\">\/<\/span> t<span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> epsilon_greedy<span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">,<\/span> epsilon<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b5\\\\varepsilon<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">\u03b5<\/span><\/span><\/span><\/span><\/span>-\u8d2a\u5fc3\u7c7b\u65b9\u6cd5\u7684\u5de5\u7a0b\u4f18\u70b9\u662f\u5b9e\u73b0\u6781\u7b80&#xff0c;\u56e0\u6b64\u5728\u65b0\u7b97\u6cd5\u7814\u7a76\u7684\u65e9\u671f baseline \u4e2d\u7ecf\u5e38\u51fa\u73b0\u3002\u4f46\u5176\u5728\u6700\u4f18\u81c2\u4e0e\u6b21\u4f18\u81c2\u5dee\u8ddd\u5f88\u5c0f\u65f6\u6548\u7387\u4f4e&#xff08;\u9700\u8981\u6301\u7eed\u968f\u673a\u63a2\u7d22\u4ee5\u62c9\u5e73\u5dee\u8ddd&#xff09;&#xff0c;\u8fd9\u4e00\u70b9\u50ac\u751f\u4e86\u66f4\u806a\u660e\u7684 UCB \u4e0e Thompson \u91c7\u6837\u3002<\/p>\n<h3>\u4e09\u3001\u7f6e\u4fe1\u4e0a\u754c\u7b97\u6cd5<\/h3>\n<h4>3.1 Hoeffding \u56de\u987e<\/h4>\n<p>Hoeffding \u4e0d\u7b49\u5f0f \u662f UCB \u7b97\u6cd5\u7684\u7406\u8bba\u57fa\u77f3\u3002\u8bbe <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">X1,\u2026,XnX_1, \\\\ldots, X_n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8778em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0785em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0785em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f\u72ec\u7acb\u968f\u673a\u53d8\u91cf&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Xi\u2208[a,b]X_i \\\\in [a, b]<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0785em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">b<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span>&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">X\u02c9&#061;1n\u2211Xi\\\\bar{X} &#061; \\\\frac{1}{n}\\\\sum X_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8201em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8201em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1901em;vertical-align: -0.345em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8451em\"><span class=\"\" style=\"top: -2.655em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.394em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.345em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0785em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u4e3a\u6837\u672c\u5747\u503c&#xff0c;\u5219\u5bf9\u4efb\u610f <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">u&gt;0u &gt; 0<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5782em;vertical-align: -0.0391em\"><\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&gt;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0<\/span><\/span><\/span><\/span><\/span>&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">P(E[X]\u2212X\u02c9\u2265u)\u2264exp\u2061\u2009\u2063(\u22122nu2(b\u2212a)2)<br \/>\nP(\\\\mathbb{E}[X] &#8211; \\\\bar{X} \\\\ge u) \\\\le \\\\exp\\\\!\\\\left(-\\\\frac{2n u^2}{(b-a)^2}\\\\right)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathbb\">E<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9561em;vertical-align: -0.136em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8201em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2265<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2264<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4411em;vertical-align: -0.95em\"><\/span><span class=\"mop\">exp<\/span><span class=\"mspace\" style=\"margin-right: -0.1667em\"><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4911em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">b<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">2<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord\"><span class=\"mord mathnormal\">u<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.936em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u76f4\u89c2&#xff1a;\u6837\u672c\u5747\u503c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">X\u02c9\\\\bar{X}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8201em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8201em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u4e0e\u771f\u5b9e\u671f\u671b <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">E[X]\\\\mathbb{E}[X]<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathbb\">E<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span> \u7684\u504f\u5dee\u8d85\u8fc7 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">uu<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">u<\/span><\/span><\/span><\/span><\/span> \u7684\u6982\u7387\u968f <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">nn<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">n<\/span><\/span><\/span><\/span><\/span> \u6307\u6570\u8870\u51cf\u3002<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">nn<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">n<\/span><\/span><\/span><\/span><\/span> \u8d8a\u5927&#xff0c;\u4f30\u8ba1\u8d8a\u53ef\u9760\u3002<\/p>\n<p>UCB \u7b97\u6cd5\u628a\u8fd9\u4e00\u4e0d\u7b49\u5f0f&#034;\u53cd\u8f6c&#034;\u2014\u2014\u7ed9\u5b9a\u6837\u672c\u5747\u503c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">X\u02c9\\\\bar{X}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8201em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8201em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u4e0e\u6b65\u6570 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">nn<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">n<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u4f30\u8ba1\u771f\u503c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">E[X]\\\\mathbb{E}[X]<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathbb\">E<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span> \u7684\u4e0a\u754c&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">E[X]\u2264X\u02c9&#043;(b\u2212a)log\u2061(1\/\u03b4)2n<br \/>\n\\\\mathbb{E}[X] \\\\le \\\\bar{X} &#043; (b &#8211; a) \\\\sqrt{\\\\frac{\\\\log(1\/\\\\delta)}{2n}}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathbb\">E<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2264<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9034em;vertical-align: -0.0833em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8201em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">b<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.44em;vertical-align: -0.7356em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">)<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.7044em\"><span class=\"svg-align\" style=\"top: -4.4em\"><span class=\"pstrut\" style=\"height: 4.4em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.427em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">2<\/span><span class=\"mord mathnormal\">n<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">1\/<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0379em\">\u03b4<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.6644em\"><span class=\"pstrut\" style=\"height: 4.4em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 2.48em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7356em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b4\\\\delta<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0379em\">\u03b4<\/span><\/span><\/span><\/span><\/span> \u662f\u7f6e\u4fe1\u6c34\u5e73&#xff08;\u5982 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b4&#061;1\/t2\\\\delta &#061; 1\/t^2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0379em\">\u03b4<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0641em;vertical-align: -0.25em\"><\/span><span class=\"mord\">1\/<\/span><span class=\"mord\"><span class=\"mord mathnormal\">t<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff09;\u3002\u8fd9\u4e2a\u4e0a\u754c\u8d8a\u7a84&#xff0c;\u4f30\u8ba1\u8d8a\u7cbe\u786e\u3002<\/p>\n<h4>3.2 UCB1 \u7b97\u6cd5<\/h4>\n<p>UCB1&#xff08;Auer, Cesa-Bianchi, Fischer 2002&#xff09;\u662f UCB \u7c7b\u7684\u7ecf\u5178\u7b97\u6cd5&#xff0c;\u5bf9\u6bcf\u4e2a\u52a8\u4f5c\u7ef4\u62a4&#xff1a;<\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">NaN_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff1a;\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aa<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u88ab\u9009\u7684\u603b\u6b21\u6570<\/li>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc^a\\\\hat{\\\\mu}_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff1a;\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aa<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u7684\u5e73\u5747\u5956\u52b1<\/li>\n<\/ul>\n<p>\u6bcf\u4e00\u6b65\u9009&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">At&#061;arg\u2061max\u2061a[\u03bc^a&#043;2log\u2061tNa]<br \/>\nA_t &#061; \\\\arg\\\\max_a \\\\left[ \\\\hat{\\\\mu}_a &#043; \\\\sqrt{\\\\frac{2 \\\\log t}{N_a}} \\\\right]<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 3em;vertical-align: -1.25em\"><\/span><span class=\"mop\">ar<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4306em\"><span class=\"\" style=\"top: -2.4em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"\"><span class=\"mop\">max<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size4\">[<\/span><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.6016em\"><span class=\"svg-align\" style=\"top: -4.4em\"><span class=\"pstrut\" style=\"height: 4.4em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3714em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">2<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.5616em\"><span class=\"pstrut\" style=\"height: 4.4em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 2.48em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8384em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size4\">]<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u76f4\u89c9&#xff1a;<\/p>\n<ul>\n<li>\u9009\u5f97\u591a\u7684\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">NaN_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u5927&#xff0c;\u7f6e\u4fe1\u534a\u5f84\u5c0f&#xff0c;\u51b3\u7b56\u4e3b\u8981\u7531 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc^a\\\\hat{\\\\mu}_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u51b3\u5b9a&#xff08;\u5229\u7528&#xff09;\u3002<\/li>\n<li>\u9009\u5f97\u5c11\u7684\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">NaN_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u5c0f&#xff0c;\u7f6e\u4fe1\u534a\u5f84\u5927&#xff0c;\u53ef\u80fd\u88ab\u9009\u4e2d&#xff08;\u63a2\u7d22&#xff09;\u3002<\/li>\n<\/ul>\n<p>\u5b9e\u73b0\u4e2d\u9700\u8981\u5904\u7406&#034;\u67d0\u52a8\u4f5c\u4ece\u672a\u88ab\u9009&#034;\u7684\u60c5\u51b5&#xff1a;\u901a\u5e38\u7528 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">&#043;\u221e&#043;\\\\infty<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6667em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">&#043;<\/span><span class=\"mord\">\u221e<\/span><\/span><\/span><\/span><\/span> \u8868\u793a\u5176\u4e0a\u754c&#xff0c;\u4fdd\u8bc1\u6bcf\u4e2a\u52a8\u4f5c\u81f3\u5c11\u88ab\u8bd5\u4e00\u6b21\u3002<\/p>\n<p><span class=\"token keyword\">import<\/span> math<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">ucb1<\/span><span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">,<\/span> N<span class=\"token punctuation\">,<\/span> t<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">any<\/span><span class=\"token punctuation\">(<\/span>N<span class=\"token punctuation\">[<\/span>a<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token number\">0<\/span> <span class=\"token keyword\">for<\/span> a <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>K<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> argmin<span class=\"token punctuation\">(<\/span>N<span class=\"token punctuation\">)<\/span>         <span class=\"token comment\"># \u4f18\u5148\u9009\u672a\u8bbf\u95ee\u8fc7\u7684<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> argmax<span class=\"token punctuation\">(<\/span><br \/>\n        Q<span class=\"token punctuation\">[<\/span>a<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#043;<\/span> math<span class=\"token punctuation\">.<\/span>sqrt<span class=\"token punctuation\">(<\/span><span class=\"token number\">2<\/span> <span class=\"token operator\">*<\/span> math<span class=\"token punctuation\">.<\/span>log<span class=\"token punctuation\">(<\/span>t<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> N<span class=\"token punctuation\">[<\/span>a<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> a <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>K<span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token punctuation\">)<\/span><\/p>\n<p>UCB1 \u7684\u7406\u8bba\u4fdd\u8bc1&#xff1a;\u7d2f\u79ef\u9057\u61be <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Regret(T)\u22648\u2211a:\u0394a&gt;0log\u2061T\u0394a&#043;O(K)\\\\mathrm{Regret}(T) \\\\le 8 \\\\sum_{a: \\\\Delta_a &gt; 0} \\\\frac{\\\\log T}{\\\\Delta_a} &#043; O(K)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">Regret<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2264<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.3773em;vertical-align: -0.4451em\"><\/span><span class=\"mord\">8<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1786em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">a<\/span><span class=\"mrel mtight\">:<\/span><span class=\"mord mtight\"><span class=\"mord mtight\">\u0394<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mrel mtight\">&gt;<\/span><span class=\"mord mtight\">0<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3998em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9322em\"><span class=\"\" style=\"top: -2.655em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u0394<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.4461em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mop mtight\"><span class=\"mtight\">l<\/span><span class=\"mtight\">o<\/span><span class=\"mtight\" style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace mtight\" style=\"margin-right: 0.1952em\"><\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4451em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u5373\u5bf9\u6570\u7ea7\u9057\u61be\u3002<\/p>\n<h4>3.3 UCB \u53d8\u4f53<\/h4>\n<p>UCB-V&#xff08;Audibert, Munos, Szepesv\u00e1ri 2009&#xff09;\u5728\u7f6e\u4fe1\u4e0a\u754c\u4e2d\u5f15\u5165\u5956\u52b1\u65b9\u5dee\u9879&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">At&#061;arg\u2061max\u2061a[\u03bc^a&#043;2V^a\u03b6log\u2061tNa&#043;3b\u03b6log\u2061tNa]<br \/>\nA_t &#061; \\\\arg\\\\max_a \\\\left[ \\\\hat{\\\\mu}_a &#043; \\\\sqrt{\\\\frac{2 \\\\hat V_a \\\\zeta \\\\log t}{N_a}} &#043; \\\\frac{3 b \\\\zeta \\\\log t}{N_a} \\\\right]<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 3.6em;vertical-align: -1.55em\"><\/span><span class=\"mop\">ar<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4306em\"><span class=\"\" style=\"top: -2.4em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"\"><span class=\"mop\">max<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen\"><span class=\"delimsizing mult\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.05em\"><span class=\"\" style=\"top: -4.05em\"><span class=\"pstrut\" style=\"height: 5.6em\"><\/span><span class=\"\" style=\"width: 0.667em;height: 3.6em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.55em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.0278em\"><span class=\"svg-align\" style=\"top: -5em\"><span class=\"pstrut\" style=\"height: 5em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.6238em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">2<\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em\">V<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.25em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.2222em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0738em\">\u03b6<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.9878em\"><span class=\"pstrut\" style=\"height: 5em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 3.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0122em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3714em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">3<\/span><span class=\"mord mathnormal\">b<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0738em\">\u03b6<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose\"><span class=\"delimsizing mult\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.05em\"><span class=\"\" style=\"top: -4.05em\"><span class=\"pstrut\" style=\"height: 5.6em\"><\/span><span class=\"\" style=\"width: 0.667em;height: 3.6em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.55em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">V^a\\\\hat V_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0968em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em\">V<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.25em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.2222em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aa<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u5956\u52b1\u7684\u6837\u672c\u65b9\u5dee&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b6\\\\zeta<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0738em\">\u03b6<\/span><\/span><\/span><\/span><\/span> \u4e0e <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">bb<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">b<\/span><\/span><\/span><\/span><\/span> \u662f\u8bba\u6587\u4e2d\u7684\u53c2\u6570&#xff08;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">bb<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">b<\/span><\/span><\/span><\/span><\/span> \u662f\u5956\u52b1\u8303\u56f4\u4e0a\u754c&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b6\\\\zeta<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0738em\">\u03b6<\/span><\/span><\/span><\/span><\/span> \u662f\u63a7\u5236\u9879&#xff09;\u3002\u4f4e\u65b9\u5dee\u52a8\u4f5c\u7684\u7f6e\u4fe1\u534a\u5f84\u66f4\u5c0f&#xff0c;\u66f4\u503e\u5411\u4e8e\u88ab\u5229\u7528\u3002<\/p>\n<p>UCB-Tuned&#xff08;Auer, Cesa-Bianchi, Fischer 2002 \u7b2c 4 \u8282&#xff09;\u5728 UCB1 \u7684\u57fa\u7840\u4e0a\u4e58\u4ee5\u4e00\u4e2a\u4f9d\u8d56\u7ecf\u9a8c\u65b9\u5dee\u7684\u4fee\u6b63\u9879&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">At&#061;arg\u2061max\u2061a[\u03bc^a&#043;log\u2061tNa\u22c5min\u2061\u2009\u2063{14,\u03c3^a2&#043;2log\u2061tNa}]<br \/>\nA_t &#061; \\\\arg\\\\max_a \\\\left[ \\\\hat{\\\\mu}_a &#043; \\\\sqrt{\\\\frac{\\\\log t}{N_a} \\\\cdot \\\\min\\\\!\\\\left\\\\{\\\\frac{1}{4}, \\\\hat\\\\sigma_a^2 &#043; \\\\sqrt{\\\\frac{2 \\\\log t}{N_a}}\\\\right\\\\}} \\\\right]<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 3.6em;vertical-align: -1.55em\"><\/span><span class=\"mop\">ar<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4306em\"><span class=\"\" style=\"top: -2.4em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"\"><span class=\"mop\">max<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen\"><span class=\"delimsizing mult\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.05em\"><span class=\"\" style=\"top: -4.05em\"><span class=\"pstrut\" style=\"height: 5.6em\"><\/span><span class=\"\" style=\"width: 0.667em;height: 3.6em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.55em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.9777em\"><span class=\"svg-align\" style=\"top: -5.1878em\"><span class=\"pstrut\" style=\"height: 5.1878em\"><\/span><span class=\"mord\" style=\"padding-left: 1.056em\"><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3714em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mop\">min<\/span><span class=\"mspace\" style=\"margin-right: -0.1667em\"><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size4\">{<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">4<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.25em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.453em;margin-left: -0.0359em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.6016em\"><span class=\"svg-align\" style=\"top: -4.4em\"><span class=\"pstrut\" style=\"height: 4.4em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3714em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">2<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.5616em\"><span class=\"pstrut\" style=\"height: 4.4em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 2.48em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8384em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size4\">}<\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.9378em\"><span class=\"pstrut\" style=\"height: 5.1878em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.742em;height: 3.2678em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.25em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\"><span class=\"delimsizing mult\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.05em\"><span class=\"\" style=\"top: -4.05em\"><span class=\"pstrut\" style=\"height: 5.6em\"><\/span><span class=\"\" style=\"width: 0.667em;height: 3.6em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.55em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03c3^a2\\\\hat\\\\sigma_a^2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0611em;vertical-align: -0.247em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.25em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -2.453em;margin-left: -0.0359em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f\u7ecf\u9a8c\u65b9\u5dee\u3002UCB-Tuned \u5728\u7ecf\u9a8c\u4e0a\u5e38\u5e38\u4f18\u4e8e UCB1\u3002<\/p>\n<p>KL-UCB&#xff08;Garivier, Capp\u00e9 2011&#xff09;\u7528 Kullback-Leibler \u6563\u5ea6\u4ee3\u66ff Hoeffding \u8fb9\u754c&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">At&#061;arg\u2061max\u2061asup\u2061\u2009\u2063{q:d(\u03bc^a,q)\u2264log\u2061t&#043;clog\u2061log\u2061tNa}<br \/>\nA_t &#061; \\\\arg\\\\max_a \\\\sup\\\\!\\\\left\\\\{ q : d(\\\\hat{\\\\mu}_a, q) \\\\le \\\\frac{\\\\log t &#043; c \\\\log \\\\log t}{N_a} \\\\right\\\\}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4em;vertical-align: -0.95em\"><\/span><span class=\"mop\">ar<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4306em\"><span class=\"\" style=\"top: -2.4em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"\"><span class=\"mop\">max<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">sup<\/span><span class=\"mspace\" style=\"margin-right: -0.1667em\"><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">{<\/span><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">q<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord mathnormal\">d<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6944em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">q<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2264<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3714em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.109em\">N<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.109em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\">c<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">}<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">d(p,q)d(p, q)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">d<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">q<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u662f Bernoulli \u5206\u5e03\u53c2\u6570 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">p,qp, q<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">q<\/span><\/span><\/span><\/span><\/span> \u95f4\u7684 KL \u6563\u5ea6\u3002\u7406\u8bba\u4e0a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">c\u22653c \\\\ge 3<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7719em;vertical-align: -0.136em\"><\/span><span class=\"mord mathnormal\">c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2265<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">3<\/span><\/span><\/span><\/span><\/span> \u7ed9\u51fa\u7d27\u4e0a\u754c&#xff0c;\u5b9e\u8df5\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">c&#061;0c &#061; 0<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0<\/span><\/span><\/span><\/span><\/span> \u4e5f\u5e38\u5e38\u8db3\u591f&#xff08;Pinsker \u4e0d\u7b49\u5f0f\u4e0b\u4e0d\u5dee\u4e8e UCB1&#xff09;\u3002KL-UCB \u5229\u7528\u4e86 KL \u6563\u5ea6\u66f4\u7d27\u7684 Chernoff \u754c&#xff0c;\u5728\u6709\u754c\u5956\u52b1\u4e0b\u4e25\u683c\u4e0d\u5f31\u4e8e UCB1\u3002<\/p>\n<h3>\u56db\u3001\u8d1d\u53f6\u65af\u8001\u864e\u673a<\/h3>\n<h4>4.1 Thompson \u91c7\u6837<\/h4>\n<p>Thompson \u91c7\u6837 \u662f\u8d1d\u53f6\u65af\u6846\u67b6\u4e0b\u7684&#034;\u6982\u7387\u5339\u914d&#034;\u7b97\u6cd5&#xff1a;\u6bcf\u4e00\u6b65\u4ece\u52a8\u4f5c\u4ef7\u503c\u7684\u540e\u9a8c\u5206\u5e03\u4e2d\u91c7\u6837\u4e00\u6b21&#xff0c;\u6309\u6837\u672c\u503c\u9009\u52a8\u4f5c\u3002<\/p>\n<p><span class=\"token keyword\">for<\/span> t <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>T<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> a <span class=\"token keyword\">in<\/span> actions<span class=\"token punctuation\">:<\/span><br \/>\n        theta_a <span class=\"token operator\">&#061;<\/span> sample_from_posterior<span class=\"token punctuation\">(<\/span>history<span class=\"token punctuation\">,<\/span> a<span class=\"token punctuation\">)<\/span>   <span class=\"token comment\"># \u4ece\u540e\u9a8c\u91c7\u6837<\/span><br \/>\n    a_t <span class=\"token operator\">&#061;<\/span> argmax<span class=\"token punctuation\">(<\/span>theta_a<span class=\"token punctuation\">)<\/span>                              <span class=\"token comment\"># \u9009\u6837\u672c\u6700\u5927\u7684\u52a8\u4f5c<\/span><br \/>\n    r_t <span class=\"token operator\">&#061;<\/span> env<span class=\"token punctuation\">.<\/span>pull<span class=\"token punctuation\">(<\/span>a_t<span class=\"token punctuation\">)<\/span>                                <span class=\"token comment\"># \u89c2\u6d4b\u5956\u52b1<\/span><br \/>\n    update_posterior<span class=\"token punctuation\">(<\/span>a_t<span class=\"token punctuation\">,<\/span> r_t<span class=\"token punctuation\">)<\/span>                         <span class=\"token comment\"># \u66f4\u65b0\u540e\u9a8c<\/span><\/p>\n<p>\u76f4\u89c9&#xff1a;\u540e\u9a8c\u8d8a\u5206\u6563&#xff08;\u4e0d\u786e\u5b9a\u6027\u9ad8&#xff09;\u7684\u52a8\u4f5c&#xff0c;\u88ab\u91c7\u6837\u5230\u9ad8\u503c\u7684\u6982\u7387\u8d8a\u5927&#xff0c;\u4ece\u800c\u88ab\u9009\u4e2d\u7684\u6982\u7387\u8d8a\u5927\u2014\u2014\u8fd9\u662f&#034;\u6982\u7387\u5f62\u5f0f&#034;\u7684\u63a2\u7d22\u3002Thompson \u91c7\u6837\u540c\u65f6\u5b9e\u73b0\u4e86\u5229\u7528&#xff08;\u9ad8\u5747\u503c\u52a8\u4f5c\u88ab\u91c7\u6837\u5230\u9ad8\u503c\u7684\u6982\u7387\u66f4\u9ad8&#xff09;\u548c\u63a2\u7d22&#xff08;\u4e0d\u786e\u5b9a\u6027\u9ad8\u7684\u52a8\u4f5c\u5076\u5c14\u4f1a\u88ab\u91c7\u6837\u5230\u9ad8\u503c&#xff09;\u3002<\/p>\n<p>Thompson \u91c7\u6837\u7684\u7406\u8bba\u4fdd\u8bc1\u5206\u4e24\u7c7b&#xff1a;<\/p>\n<ul>\n<li>Instance-dependent&#xff08;\u4f9d\u8d56 gap <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u0394a\\\\Delta_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord\">\u0394<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff09;&#xff1a;UCB1 \u8fbe\u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(\u2211alog\u2061T\/\u0394a)O(\\\\sum_a \\\\log T \/ \\\\Delta_a)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0497em;vertical-align: -0.2997em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.0017em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2997em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mord\">\/<\/span><span class=\"mord\"><span class=\"mord\">\u0394<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u7684\u5bf9\u6570\u7ea7\u9057\u61be&#xff1b;Thompson \u91c7\u6837\u5bf9 Bernoulli \u8001\u864e\u673a\u8fbe <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(KTlog\u2061T)O(\\\\sqrt{KT \\\\log T})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.085em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.835em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><span class=\"\" style=\"top: -2.795em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.205em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff08;Kaufmann et al. 2012 \/ Agrawal-Goyal 2013&#xff09;\u3002<\/li>\n<li>Gap-free \/ min-max&#xff08;\u6700\u574f\u60c5\u51b5&#xff09;&#xff1a;UCB1 \u4e0e Thompson \u91c7\u6837\u5747\u8fbe\u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O~(KT)\\\\tilde O(\\\\sqrt{KT})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1767em;vertical-align: -0.25em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9202em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><\/span><span class=\"\" style=\"top: -3.6023em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">~<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9267em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8867em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1133em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<\/ul>\n<p>\u5de5\u7a0b\u4e0a Thompson \u91c7\u6837\u7ecf\u9a8c\u4e0a\u5e38\u5e38\u6bd4 UCB \u66f4\u4f18&#xff0c;\u662f\u5de5\u4e1a\u754c\u6700\u5e38\u7528\u7684\u8001\u864e\u673a\u7b97\u6cd5\u2014\u2014\u5b83\u7684&#034;\u6309\u540e\u9a8c\u91c7\u6837&#034;\u5728\u5206\u5e03\u5f0f\u7cfb\u7edf\u4e2d\u5bb9\u6613\u5e76\u884c\u5316&#xff0c;\u4e14\u8d85\u53c2\u6570\u5c11\u3002<\/p>\n<h4>4.2 \u4f2f\u52aa\u5229 Beta \u5148\u9a8c<\/h4>\n<p>\u4f2f\u52aa\u5229\u8001\u864e\u673a \u6bcf\u4e2a\u52a8\u4f5c\u7684\u5956\u52b1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">R\u2208{0,1}R \\\\in \\\\{0, 1\\\\}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em;vertical-align: -0.0391em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">{<\/span><span class=\"mord\">0<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1<\/span><span class=\"mclose\">}<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u53c2\u6570 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b8a&#061;P(R&#061;1\u2223a)\\\\theta_a &#061; P(R &#061; 1 \\\\mid a)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">1<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>\u3002Beta \u5206\u5e03 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Beta(\u03b1,\u03b2)\\\\mathrm{Beta}(\\\\alpha, \\\\beta)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">Beta<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0037em\">\u03b1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0528em\">\u03b2<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u662f\u4f2f\u52aa\u5229\u53c2\u6570\u7684\u5171\u8f6d\u5148\u9a8c&#xff1a;<\/p>\n<ul>\n<li>\u5148\u9a8c&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b8a\u223cBeta(1,1)\\\\theta_a \\\\sim \\\\mathrm{Beta}(1, 1)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u223c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">Beta<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff08;\u5747\u5300\u5206\u5e03&#xff09;<\/li>\n<li>\u89c2\u6d4b <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">sas_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">s<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u6b21\u6210\u529f\u3001<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">faf_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1076em\">f<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.1076em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u6b21\u5931\u8d25\u540e&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b8a\u2223data\u223cBeta(1&#043;sa,1&#043;fa)\\\\theta_a \\\\mid \\\\text{data} \\\\sim \\\\mathrm{Beta}(1 &#043; s_a, 1 &#043; f_a)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord text\"><span class=\"mord\">data<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u223c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">Beta<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">1<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8389em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">s<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1076em\">f<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.1076em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/li>\n<\/ul>\n<p><span class=\"token keyword\">import<\/span> numpy <span class=\"token keyword\">as<\/span> np<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">thompson_bernoulli<\/span><span class=\"token punctuation\">(<\/span>success<span class=\"token punctuation\">,<\/span> failure<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    samples <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><br \/>\n        np<span class=\"token punctuation\">.<\/span>random<span class=\"token punctuation\">.<\/span>beta<span class=\"token punctuation\">(<\/span>success<span class=\"token punctuation\">[<\/span>a<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> failure<span class=\"token punctuation\">[<\/span>a<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> a <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>K<span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token punctuation\">]<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>argmax<span class=\"token punctuation\">(<\/span>samples<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>Beta \u5206\u5e03\u7684\u4f18\u826f\u6027\u8d28&#xff1a;<\/p>\n<ul>\n<li>\u5747\u503c &#061; <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b1\u03b1&#043;\u03b2\\\\frac{\\\\alpha}{\\\\alpha &#043; \\\\beta}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1765em;vertical-align: -0.4811em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6954em\"><span class=\"\" style=\"top: -2.655em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0037em\">\u03b1<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0528em\">\u03b2<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.394em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0037em\">\u03b1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4811em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u968f\u89c2\u6d4b\u66f4\u65b0\u3002<\/li>\n<li>\u65b9\u5dee &#061; <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b1\u03b2(\u03b1&#043;\u03b2)2(\u03b1&#043;\u03b2&#043;1)\\\\frac{\\\\alpha \\\\beta}{(\\\\alpha &#043; \\\\beta)^2 (\\\\alpha &#043; \\\\beta &#043; 1)}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.4522em;vertical-align: -0.52em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9322em\"><span class=\"\" style=\"top: -2.655em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0037em\">\u03b1<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0528em\">\u03b2<\/span><span class=\"mclose mtight\"><span class=\"mclose mtight\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7463em\"><span class=\"\" style=\"top: -2.786em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0037em\">\u03b1<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0528em\">\u03b2<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.4461em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0037em\">\u03b1<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0528em\">\u03b2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.52em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u89c2\u6d4b\u589e\u591a\u540e\u51cf\u5c0f\u2014\u2014\u4e0d\u786e\u5b9a\u6027\u81ea\u7136\u8870\u51cf\u3002<\/li>\n<li>\u540e\u9a8c\u66f4\u65b0\u53ea\u9700&#034;\u52a0\u4e00&#034;&#xff0c;\u5b9e\u73b0\u6781\u7b80\u3002<\/li>\n<\/ul>\n<h4>4.3 \u9ad8\u65af\u8001\u864e\u673a\u7684\u5171\u8f6d\u5148\u9a8c<\/h4>\n<p>\u9ad8\u65af\u8001\u864e\u673a \u7684\u5956\u52b1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">R\u223cN(\u03bc,\u03c32)R \\\\sim \\\\mathcal{N}(\\\\mu, \\\\sigma^2)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em\">R<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u223c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0641em;vertical-align: -0.25em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u65b9\u5dee\u5df2\u77e5\u6216\u672a\u77e5\u65f6\u4f7f\u7528\u4e0d\u540c\u7684\u5171\u8f6d\u5148\u9a8c&#xff1a;<\/p>\n<ul>\n<li>\u5df2\u77e5\u65b9\u5dee <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03c32\\\\sigma^2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8141em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc\\\\mu<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><\/span><\/span><\/span> \u7684\u5171\u8f6d\u5148\u9a8c\u662f\u6b63\u6001\u5206\u5e03 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">N(\u03bc0,\u03c402)\\\\mathcal{N}(\\\\mu_0, \\\\tau_0^2)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0641em;vertical-align: -0.25em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">0<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1132em\">\u03c4<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -2.4519em;margin-left: -0.1132em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">0<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2481em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u89c2\u6d4b <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">nn<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">n<\/span><\/span><\/span><\/span><\/span> \u4e2a\u6837\u672c\u540e\u5747\u503c\u4e3a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">x\u02c9\\\\bar{x}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5678em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5678em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">x<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff1a;<\/li>\n<\/ul>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc\u2223data\u223cN\u2009\u2063(\u03c40\u22122\u03bc0&#043;n\u03c3\u22122x\u02c9\u03c40\u22122&#043;n\u03c3\u22122,1\u03c40\u22122&#043;n\u03c3\u22122)<br \/>\n\\\\mu \\\\mid \\\\text{data} \\\\sim \\\\mathcal{N}\\\\!\\\\left( \\\\frac{\\\\tau_0^{-2}\\\\mu_0 &#043; n\\\\sigma^{-2}\\\\bar{x}}{\\\\tau_0^{-2} &#043; n\\\\sigma^{-2}}, \\\\frac{1}{\\\\tau_0^{-2} &#043; n\\\\sigma^{-2}} \\\\right)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord text\"><span class=\"mord\">data<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u223c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.5418em;vertical-align: -1.0105em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mspace\" style=\"margin-right: -0.1667em\"><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5312em\"><span class=\"\" style=\"top: -2.2558em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1132em\">\u03c4<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8542em\"><span class=\"\" style=\"top: -2.4337em;margin-left: -0.1132em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">0<\/span><\/span><\/span><span class=\"\" style=\"top: -3.1031em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2663em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1132em\">\u03c4<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8542em\"><span class=\"\" style=\"top: -2.4337em;margin-left: -0.1132em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">0<\/span><\/span><\/span><span class=\"\" style=\"top: -3.1031em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2663em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">0<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5678em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\">x<\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.2222em\"><span class=\"mord\">\u02c9<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0105em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.2558em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1132em\">\u03c4<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8542em\"><span class=\"\" style=\"top: -2.4337em;margin-left: -0.1132em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">0<\/span><\/span><\/span><span class=\"\" style=\"top: -3.1031em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2663em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0105em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<ul>\n<li>\u672a\u77e5\u65b9\u5dee&#xff1a;\u4f7f\u7528 Normal-Inverse-Gamma \u5171\u8f6d\u5148\u9a8c\u3002\u7ed9\u5b9a\u89c2\u6d4b\u6570\u636e&#xff0c;\u5148\u9a8c\u53c2\u6570\u6309\u4ee5\u4e0b\u5f62\u5f0f\u66f4\u65b0&#xff08;\u6807\u51c6 NIG \u5f62\u5f0f&#xff09;&#xff1a;<\/li>\n<\/ul>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc\u2223data,\u03c32\u223cN(mN,\u03c32\/\u03baN)<br \/>\n\\\\mu \\\\mid \\\\text{data}, \\\\sigma^2 \\\\sim \\\\mathcal{N}(m_N, \\\\sigma^2 \/ \\\\kappa_N)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0585em;vertical-align: -0.1944em\"><\/span><span class=\"mord text\"><span class=\"mord\">data<\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u223c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1141em;vertical-align: -0.25em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.109em\">N<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\">\/<\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03ba<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.109em\">N<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">\u03c32\u2223data\u223cInv-Gamma(\u03b1N,\u03b2N)<br \/>\n\\\\sigma^2 \\\\mid \\\\text{data} \\\\sim \\\\mathrm{Inv\\\\text{-}Gamma}(\\\\alpha_N, \\\\beta_N)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1141em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord text\"><span class=\"mord\">data<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u223c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\"><span class=\"mord mathrm\" style=\"margin-right: 0.0139em\">Inv<\/span><span class=\"mord text\"><span class=\"mord\">&#8211;<\/span><\/span><span class=\"mord mathrm\">Gamma<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0037em\">\u03b1<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0037em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.109em\">N<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0528em\">\u03b2<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0528em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.109em\">N<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mN,\u03baN,\u03b1N,\u03b2Nm_N, \\\\kappa_N, \\\\alpha_N, \\\\beta_N<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.109em\">N<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03ba<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.109em\">N<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0037em\">\u03b1<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0037em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.109em\">N<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0528em\">\u03b2<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0528em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.109em\">N<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u7531\u5148\u9a8c\u4e0e\u89c2\u6d4b\u5171\u540c\u51b3\u5b9a\u3002\u5de5\u7a0b\u5b9e\u73b0\u4e2d\u5e38\u76f4\u63a5\u91c7\u6837 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03c32\\\\sigma^2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8141em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c3<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3001\u518d\u91c7\u6837 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bc\\\\mu<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">\u03bc<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u4e8c\u8005\u7ed3\u5408\u7ed9\u51fa <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">rr<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><\/span><\/span><\/span><\/span> \u7684\u540e\u9a8c\u9884\u6d4b\u3002<\/p>\n<p>\u5de5\u7a0b\u4e0a\u9ad8\u65af\u8001\u864e\u673a\u7684 Thompson \u91c7\u6837\u53ea\u9700&#xff1a;<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">thompson_gaussian<\/span><span class=\"token punctuation\">(<\/span>sums<span class=\"token punctuation\">,<\/span> counts<span class=\"token punctuation\">,<\/span> prior_mean<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.0<\/span><span class=\"token punctuation\">,<\/span> prior_var<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1.0<\/span><span class=\"token punctuation\">,<\/span> noise_var<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1.0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    post_vars <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1.0<\/span> <span class=\"token operator\">\/<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token number\">1.0<\/span> <span class=\"token operator\">\/<\/span> prior_var <span class=\"token operator\">&#043;<\/span> counts <span class=\"token operator\">\/<\/span> noise_var<span class=\"token punctuation\">)<\/span><br \/>\n    post_means <span class=\"token operator\">&#061;<\/span> post_vars <span class=\"token operator\">*<\/span> <span class=\"token punctuation\">(<\/span>prior_mean <span class=\"token operator\">\/<\/span> prior_var <span class=\"token operator\">&#043;<\/span> sums <span class=\"token operator\">\/<\/span> noise_var<span class=\"token punctuation\">)<\/span><br \/>\n    samples <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>random<span class=\"token punctuation\">.<\/span>normal<span class=\"token punctuation\">(<\/span>post_means<span class=\"token punctuation\">,<\/span> np<span class=\"token punctuation\">.<\/span>sqrt<span class=\"token punctuation\">(<\/span>post_vars<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>argmax<span class=\"token punctuation\">(<\/span>samples<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u5b9e\u9645\u63a8\u8350\u7cfb\u7edf\u4e2d\u7ecf\u5e38\u4f7f\u7528\u9ad8\u65af\u8001\u864e\u673a&#xff1a;CTR \u573a\u666f\u4e0b\u7528 Beta-Bernoulli&#xff08;\u70b9\u51fb\u662f Bernoulli \u4e8b\u4ef6&#xff09;&#xff1b;\u82e5\u5956\u52b1\u662f\u8fde\u7eed\u8fd1\u4f3c&#xff08;\u5982\u70b9\u51fb\u6df1\u5ea6\u3001\u505c\u7559\u79d2\u6570&#xff09;&#xff0c;\u53ef\u7528\u9ad8\u65af-Inverse-Gamma\u3002\u6bcf\u79cd\u89c2\u6d4b\u7c7b\u578b\u5bf9\u5e94\u4e0d\u540c\u5171\u8f6d\u65cf&#xff0c;\u4e0d\u8981\u6df7\u7528\u3002<\/p>\n<h3>\u4e94\u3001\u4e0a\u4e0b\u6587\u8001\u864e\u673a<\/h3>\n<h4>5.1 \u95ee\u9898\u5b9a\u4e49<\/h4>\n<p>\u4e0a\u4e0b\u6587\u8001\u864e\u673a&#xff08;Contextual Bandit, CB&#xff09; \u5728 MAB \u57fa\u7840\u4e0a\u52a0\u5165&#034;\u4e0a\u4e0b\u6587&#034;\u4fe1\u606f <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">xt\u2208Rdx_t \\\\in \\\\mathbb{R}^d<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6891em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff08;\u7528\u6237\u7279\u5f81\u3001\u5185\u5bb9\u7279\u5f81\u3001\u65f6\u6bb5\u7b49&#xff09;&#xff0c;\u6bcf\u6b65\u667a\u80fd\u4f53\u89c2\u5bdf\u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">xtx_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u7136\u540e\u9009\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ata_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u6536\u5230\u5956\u52b1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">rtr_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002\u5176\u4e0e\u4e00\u822c RL \u7684\u533a\u522b\u4ecd\u7136\u662f\u6ca1\u6709\u72b6\u6001\u8f6c\u79fb\u2014\u2014\u6bcf\u6b65\u72ec\u7acb\u3002<\/p>\n<p>\u5f62\u5f0f\u5316&#xff1a;per-round \u534f\u8bae&#xff1a;<\/p>\n<ul>\n<li>\u5728\u65f6\u523b <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">tt<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6151em\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u73af\u5883\u7ed9\u51fa\u4e0a\u4e0b\u6587 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">xt\u2208X\u2286Rdx_t \\\\in \\\\mathcal{X} \\\\subseteq \\\\mathbb{R}^d<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6891em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8193em;vertical-align: -0.136em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1464em\">X<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2286<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<li>\u667a\u80fd\u4f53\u6839\u636e\u5386\u53f2 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">(x1,a1,r1,\u2026,xt\u22121,at\u22121,rt\u22121)(x_1, a_1, r_1, \\\\ldots, x_{t-1}, a_{t-1}, r_{t-1})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u9009\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ata_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<li>\u73af\u5883\u53cd\u9988\u5373\u65f6\u5956\u52b1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">rt\u2208Rr_t \\\\in \\\\mathbb{R}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6891em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6889em\"><\/span><span class=\"mord mathbb\">R<\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<li>\u8fdb\u5165\u4e0b\u4e00\u8f6e&#xff0c;\u65e0\u72b6\u6001\u8f6c\u79fb\u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">xt&#043;1x_{t&#043;1}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6389em;vertical-align: -0.2083em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<\/ul>\n<p>\u672a\u77e5\u7684\u5956\u52b1\u51fd\u6570 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">f(x,a)&#061;E[r\u2223x,a]f(x, a) &#061; \\\\mathbb{E}[r \\\\mid x, a]<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1076em\">f<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathbb\">E<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u76ee\u6807\u662f\u5b66\u4e60\u4e00\u4e2a\u7b56\u7565 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03c0(a\u2223x)\\\\pi(a \\\\mid x)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em\">\u03c0<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u6700\u5927\u5316\u7d2f\u79ef\u5956\u52b1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u2211t&#061;1Trt\\\\sum_{t&#061;1}^T r_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.2809em;vertical-align: -0.2997em\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9812em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2029em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2997em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/p>\n<p>CB \u4e0e MAB\u3001\u4e00\u822c RL \u7684\u5173\u952e\u533a\u522b&#xff1a;<\/p>\n<table>\n<tr>\u8bbe\u5b9a\u53cd\u9988\u5ef6\u8fdf\u72b6\u6001\u8f6c\u79fb\u4e0a\u4e0b\u6587 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">xtx_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/tr>\n<tbody>\n<tr>\n<td>MAB<\/td>\n<td>1 \u6b65<\/td>\n<td>\u65e0<\/td>\n<td>\u65e0<\/td>\n<\/tr>\n<tr>\n<td>CB<\/td>\n<td>1 \u6b65<\/td>\n<td>\u65e0<\/td>\n<td>\u6709<\/td>\n<\/tr>\n<tr>\n<td>\u4e00\u822c RL<\/td>\n<td>\u591a\u6b65<\/td>\n<td>\u6709<\/td>\n<td>\u901a\u5e38\u5305\u542b\u5728 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">sts_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">s<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>CB \u5728\u5de5\u4e1a\u63a8\u8350\u4e2d\u975e\u5e38\u666e\u904d&#xff1a;\u6bcf\u6761\u5185\u5bb9\u66dd\u5149\u662f\u72ec\u7acb\u4e8b\u4ef6&#xff0c;\u6ca1\u6709\u8de8\u6761\u76ee\u7684\u72b6\u6001\u8f6c\u79fb&#xff0c;\u4f46\u9700\u8981\u6839\u636e\u7528\u6237\u548c\u5185\u5bb9\u7279\u5f81&#xff08;\u4e0a\u4e0b\u6587&#xff09;\u505a\u9009\u62e9\u3002<\/p>\n<p>\u8fb9\u754c\u4e0e\u5751&#xff1a;CB \u4e0d\u5904\u7406\u72b6\u6001\u8f6c\u79fb\u4e0d\u7b49\u4e8e&#034;\u53cd\u9988\u5ef6\u8fdf\u53ef\u5ffd\u7565&#034;\u3002\u5982\u679c\u51b3\u7b56\u5f71\u54cd\u540e\u7eed\u4e0a\u4e0b\u6587\u5206\u5e03&#xff08;\u5982\u9009\u4e86\u67d0\u7c7b\u5185\u5bb9\u540e\u7528\u6237\u504f\u597d\u6539\u53d8&#xff09;&#xff0c;\u4ecd\u7136\u9700\u8981\u8fdb\u5165\u5b8c\u6574\u7684 MDP\/POMDP \u6846\u67b6\u3002<\/p>\n<h4>5.2 LinUCB \u7b97\u6cd5<\/h4>\n<p>LinUCB&#xff08;Li et al. 2010&#xff09;\u5047\u8bbe\u5956\u52b1\u5bf9\u4e0a\u4e0b\u6587\u662f\u7ebf\u6027\u7684&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">E[r\u2223x,a]&#061;x\u22a4\u03b8a\u2217<br \/>\n\\\\mathbb{E}[r \\\\mid x, a] &#061; x^\\\\top \\\\theta_a^*<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathbb\">E<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1461em;vertical-align: -0.247em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8991em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7387em\"><span class=\"\" style=\"top: -2.453em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mbin mtight\">\u2217<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b8a\u2217\\\\theta_a^*<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9414em;vertical-align: -0.247em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6887em\"><span class=\"\" style=\"top: -2.453em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mbin mtight\">\u2217<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aa<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u7684\u672a\u77e5\u53c2\u6570\u5411\u91cf\u3002\u7b97\u6cd5\u5bf9\u6bcf\u4e2a\u52a8\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aa<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u7ef4\u62a4&#xff1a;<\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Aa&#061;\u03bbI&#043;\u2211t:At&#061;axtxt\u22a4A_a &#061; \\\\lambda I &#043; \\\\sum_{t: A_t &#061; a} x_t x_t^\\\\top<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7778em;vertical-align: -0.0833em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">I<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.2489em;vertical-align: -0.3998em\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1786em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mrel mtight\">:<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2963em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3998em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -2.453em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff08;\u8bbe\u8ba1\u77e9\u9635&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bb\\\\lambda<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><\/span><\/span><\/span><\/span> \u662f\u5cad\u56de\u5f52\u6b63\u5219&#xff09;<\/li>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ba&#061;\u2211t:At&#061;artxtb_a &#061; \\\\sum_{t: A_t &#061; a} r_t x_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">b<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1498em;vertical-align: -0.3998em\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1786em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mrel mtight\">:<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2963em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3998em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff08;\u5956\u52b1\u52a0\u6743\u548c&#xff09;<\/li>\n<\/ul>\n<p>\u53c2\u6570\u4f30\u8ba1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b8^a&#061;Aa\u22121ba\\\\hat{\\\\theta}_a &#061; A_a^{-1} b_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1079em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9579em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><\/span><span class=\"\" style=\"top: -3.2634em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0611em;vertical-align: -0.247em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -2.453em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">b<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002\u6bcf\u4e00\u6b65\u7684\u51b3\u7b56&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">At&#061;arg\u2061max\u2061a(xt\u22a4\u03b8^a&#043;\u03b1xt\u22a4Aa\u22121xt)<br \/>\nA_t &#061; \\\\arg\\\\max_a \\\\left( x_t^\\\\top \\\\hat{\\\\theta}_a &#043; \\\\alpha \\\\sqrt{x_t^\\\\top A_a^{-1} x_t} \\\\right)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4em;vertical-align: -0.95em\"><\/span><span class=\"mop\">ar<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4306em\"><span class=\"\" style=\"top: -2.4em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"\"><span class=\"mop\">max<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8991em\"><span class=\"\" style=\"top: -2.453em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9579em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><\/span><span class=\"\" style=\"top: -3.2634em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0037em\">\u03b1<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3258em\"><span class=\"svg-align\" style=\"top: -3.8em\"><span class=\"pstrut\" style=\"height: 3.8em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8309em\"><span class=\"\" style=\"top: -2.4542em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><span class=\"\" style=\"top: -3.0448em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2458em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.453em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2858em\"><span class=\"pstrut\" style=\"height: 3.8em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 1.88em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5142em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d\u7b2c\u4e8c\u9879\u662f\u7f6e\u4fe1\u4e0a\u754c&#xff08;\u7c7b\u4f3c UCB1&#xff09;&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03b1\\\\alpha<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0037em\">\u03b1<\/span><\/span><\/span><\/span><\/span> \u63a7\u5236\u63a2\u7d22\u5f3a\u5ea6\u3002<\/p>\n<p>LinUCB \u7684\u4f18\u70b9&#xff1a;<\/p>\n<ul>\n<li>\u5b9e\u73b0\u76f8\u5bf9\u7b80\u5355&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">AaA_a<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u53ef\u7528 Sherman-Morrison \u516c\u5f0f\u589e\u91cf\u66f4\u65b0&#xff08;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">dd<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><\/span><\/span> \u7ef4\u4e0b\u590d\u6742\u5ea6 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O(d2)O(d^2)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0641em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff09;&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Aa\u22121A_a^{-1}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0611em;vertical-align: -0.247em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -2.453em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u5b9a\u671f\u7f13\u5b58\u6216\u7528 Cholesky \u5206\u89e3\u3002<\/li>\n<li>\u7406\u8bba\u9057\u61be\u4e0a\u754c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O~(KdT)\\\\tilde O(\\\\sqrt{KdT})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1822em;vertical-align: -0.25em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9202em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">O<\/span><\/span><span class=\"\" style=\"top: -3.6023em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">~<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9322em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mord mathnormal\">d<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8922em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1078em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff08;Li et al. 2010 \/ Abbasi-Yadkori et al. 2011&#xff09;&#xff0c;\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">KK<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><\/span><\/span><\/span><\/span> \u4e3a\u81c2\u6570\u3001<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">dd<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><\/span><\/span> \u4e3a\u4e0a\u4e0b\u6587\u7ef4\u5ea6\u3002<\/li>\n<\/ul>\n<p>\u5de5\u7a0b\u4e0a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bb\\\\lambda<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><\/span><\/span><\/span><\/span>&#xff08;\u5cad\u6b63\u5219\u9879&#xff09;\u539f\u8bba\u6587\u53d6 1&#xff1b;\u8c03\u53c2\u65f6\u53ef\u4f5c\u4e3a&#034;\u63a2\u7d22-\u6570\u503c\u7a33\u5b9a&#034;\u6743\u8861\u7684\u65cb\u94ae\u2014\u2014<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bb\\\\lambda<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><\/span><\/span><\/span><\/span> \u8d8a\u5927&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Aa\u22121A_a^{-1}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0611em;vertical-align: -0.247em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -2.453em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u8d8a\u63a5\u8fd1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bb\u22121I\\\\lambda^{-1} I<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8141em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">\u03bb<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">I<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u7f6e\u4fe1\u4e0a\u754c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">xt\u22a4Aa\u22121xtx_t^\\\\top A_a^{-1} x_t<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0961em;vertical-align: -0.247em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -2.453em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">A<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -2.453em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2806em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u8d8a\u5927\u3001\u63a2\u7d22\u8d8a\u6fc0\u8fdb\u3002<\/p>\n<p>LinUCB \u7684\u5c40\u9650\u662f\u7ebf\u6027\u5047\u8bbe\u3002\u5f53\u771f\u5b9e\u7684 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">f(x,a)f(x, a)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1076em\">f<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u662f\u975e\u7ebf\u6027\u65f6&#xff0c;\u9700\u8981&#xff1a;<\/p>\n<ul>\n<li>\u6838\u5316 UCB&#xff1a;\u628a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">xx<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">x<\/span><\/span><\/span><\/span><\/span> \u6620\u5c04\u5230\u9ad8\u7ef4\u6838\u7a7a\u95f4&#xff08;\u5982 RBF \u6838&#xff09;\u3002<\/li>\n<li>\u795e\u7ecf\u7f51\u7edc UCB&#xff1a;\u7528\u795e\u7ecf\u7f51\u7edc\u62df\u5408 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">f^(x,a)\\\\hat{f}(x, a)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.2079em;vertical-align: -0.25em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9579em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1076em\">f<\/span><\/span><span class=\"\" style=\"top: -3.2634em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.0833em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1944em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u5e76\u7528 ensemble \u6216 MC Dropout \u4f30\u8ba1\u4e0d\u786e\u5b9a\u6027&#xff08;\u8be6\u89c1\u7b2c\u5341\u4e94\u7ae0&#xff09;\u3002<\/li>\n<\/ul>\n<p>Thompson \u91c7\u6837\u540c\u6837\u53ef\u4ee5\u6269\u5c55\u5230\u4e0a\u4e0b\u6587\u573a\u666f&#xff1a;\u5bf9\u6bcf\u4e2a\u52a8\u4f5c&#xff0c;\u4ece <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">P(\u03b8a\u2223data)P(\\\\theta_a \\\\mid \\\\text{data})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord text\"><span class=\"mord\">data<\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u91c7\u6837\u518d\u9009\u52a8\u4f5c\u3002<\/p>\n<p>Thompson \u91c7\u6837\u540c\u6837\u53ef\u4ee5\u6269\u5c55\u5230\u4e0a\u4e0b\u6587\u573a\u666f&#xff1a;\u5bf9\u6bcf\u4e2a\u52a8\u4f5c&#xff0c;\u4ece <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">P(\u03b8a\u2223data)P(\\\\theta_a \\\\mid \\\\text{data})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">\u03b8<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0278em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord text\"><span class=\"mord\">data<\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u91c7\u6837\u518d\u9009\u52a8\u4f5c\u3002<\/p>\n<p>\u4f55\u65f6\u7528 CB\u3001\u4f55\u65f6\u5347\u7ea7\u5230\u5b8c\u6574 RL&#xff1a;\u5bf9\u4e8e\u65e0\u72b6\u6001\u8f6c\u79fb\u3001\u5355\u6b65\u72ec\u7acb\u53cd\u9988\u7684\u66dd\u5149\/\u70b9\u51fb\u51b3\u7b56&#xff08;\u5982\u5e7f\u544a CTR \u4f18\u5316\u3001\u5185\u5bb9\u63a8\u8350&#xff09;&#xff0c;CB \u5df2\u8db3\u591f&#xff1b;\u5f53\u51b3\u7b56\u5f71\u54cd\u540e\u7eed\u4e0a\u4e0b\u6587\u5206\u5e03&#xff08;\u5982\u9009\u4e86\u67d0\u7c7b\u5185\u5bb9\u540e\u7528\u6237\u504f\u597d\u6539\u53d8&#xff09;&#xff0c;\u5219\u5fc5\u987b\u8fdb\u5165 MDP\/POMDP \u6846\u67b6\u3002<\/p>\n<p>MAB \u5b50\u96c6&#xff08;contextual bandit\u3001\u8d1d\u53f6\u65af\u8001\u864e\u673a&#xff09;\u5df2\u7ecf\u80fd\u6ee1\u8db3\u5927\u91cf\u4e1a\u52a1\u9700\u6c42&#xff0c;\u672a\u5fc5\u9700\u8981\u8fdb\u5165\u5b8c\u6574\u7684 MDP \u6846\u67b6\u2014\u2014\u4f46\u4f55\u65f6\u8be5\u5347\u7ea7&#xff0c;\u9700\u8981\u7ed3\u5408\u4efb\u52a1\u7684&#034;\u51b3\u7b56\u5bf9\u672a\u6765\u7684\u5f71\u54cd&#034;\u505a\u5224\u65ad\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u4e00\u3001\u591a\u81c2\u8001\u864e\u673a\u95ee\u9898<br \/>\n1.1 \u95ee\u9898\u5f62\u5f0f\u5316<br \/>\n\u591a\u81c2\u8001\u864e\u673a&#xff08;Multi-Armed Bandit, MAB&#xff09; \u662f\u5f3a\u5316\u5b66\u4e60\u6700\u7b80\u5355\u7684\u8bbe\u5b9a&#xff1a;\u667a\u80fd\u4f53\u6bcf\u4e00\u6b65\u4ece KKK \u4e2a\u52a8\u4f5c&#xff08;\u201c\u6447\u81c2\u201d&#xff09;\u4e2d\u9009\u4e00\u4e2a&#xff0c;\u73af\u5883\u8fd4\u56de\u4e00\u4e2a\u5956\u52b1&#xff1b;\u4e0d\u5b58\u5728\u72b6\u6001\u8f6c\u79fb&#xff0c;\u6bcf\u4e00\u6b65\u72ec\u7acb\u3002<br \/>\n\u5f62\u5f0f\u5316\u5730&#xff0c;\u6a21\u578b\u7531\u4ee5\u4e0b\u8981\u7d20\u7ec4\u6210&#xff1a;<br \/>\n\u52a8\u4f5c\u96c6\u5408 A{1,2,\u2026,K}\\\\mathcal{A}  \\\\{1, 2, \\\\ldot<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[13601,13602,13603,50,752],"topic":[],"class_list":["post-119366","post","type-post","status-publish","format-standard","hentry","category-server","tag-13601","tag-13602","tag-13603","tag-50","tag-752"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 - \u591a\u81c2\u8001\u864e\u673a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/119366.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 - \u591a\u81c2\u8001\u864e\u673a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u4e00\u3001\u591a\u81c2\u8001\u864e\u673a\u95ee\u9898 1.1 \u95ee\u9898\u5f62\u5f0f\u5316 \u591a\u81c2\u8001\u864e\u673a&#xff08;Multi-Armed Bandit, MAB&#xff09; \u662f\u5f3a\u5316\u5b66\u4e60\u6700\u7b80\u5355\u7684\u8bbe\u5b9a&#xff1a;\u667a\u80fd\u4f53\u6bcf\u4e00\u6b65\u4ece KKK \u4e2a\u52a8\u4f5c&#xff08;\u201c\u6447\u81c2\u201d&#xff09;\u4e2d\u9009\u4e00\u4e2a&#xff0c;\u73af\u5883\u8fd4\u56de\u4e00\u4e2a\u5956\u52b1&#xff1b;\u4e0d\u5b58\u5728\u72b6\u6001\u8f6c\u79fb&#xff0c;\u6bcf\u4e00\u6b65\u72ec\u7acb\u3002 \u5f62\u5f0f\u5316\u5730&#xff0c;\u6a21\u578b\u7531\u4ee5\u4e0b\u8981\u7d20\u7ec4\u6210&#xff1a; \u52a8\u4f5c\u96c6\u5408 A{1,2,\u2026,K}\\mathcal{A} \\{1, 2, \\ldot\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/119366.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-10-29T05:40:46+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"11 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/119366.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/119366.html\",\"name\":\"\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 - \u591a\u81c2\u8001\u864e\u673a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-10-29T05:40:46+00:00\",\"dateModified\":\"2026-10-29T05:40:46+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/119366.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/119366.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/119366.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 -- \u591a\u81c2\u8001\u864e\u673a\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 - \u591a\u81c2\u8001\u864e\u673a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/119366.html","og_locale":"zh_CN","og_type":"article","og_title":"\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 - \u591a\u81c2\u8001\u864e\u673a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u4e00\u3001\u591a\u81c2\u8001\u864e\u673a\u95ee\u9898 1.1 \u95ee\u9898\u5f62\u5f0f\u5316 \u591a\u81c2\u8001\u864e\u673a&#xff08;Multi-Armed Bandit, MAB&#xff09; \u662f\u5f3a\u5316\u5b66\u4e60\u6700\u7b80\u5355\u7684\u8bbe\u5b9a&#xff1a;\u667a\u80fd\u4f53\u6bcf\u4e00\u6b65\u4ece KKK \u4e2a\u52a8\u4f5c&#xff08;\u201c\u6447\u81c2\u201d&#xff09;\u4e2d\u9009\u4e00\u4e2a&#xff0c;\u73af\u5883\u8fd4\u56de\u4e00\u4e2a\u5956\u52b1&#xff1b;\u4e0d\u5b58\u5728\u72b6\u6001\u8f6c\u79fb&#xff0c;\u6bcf\u4e00\u6b65\u72ec\u7acb\u3002 \u5f62\u5f0f\u5316\u5730&#xff0c;\u6a21\u578b\u7531\u4ee5\u4e0b\u8981\u7d20\u7ec4\u6210&#xff1a; \u52a8\u4f5c\u96c6\u5408 A{1,2,\u2026,K}\\mathcal{A} \\{1, 2, \\ldot","og_url":"https:\/\/www.wsisp.com\/helps\/119366.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-10-29T05:40:46+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"11 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/119366.html","url":"https:\/\/www.wsisp.com\/helps\/119366.html","name":"\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 - \u591a\u81c2\u8001\u864e\u673a - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-10-29T05:40:46+00:00","dateModified":"2026-10-29T05:40:46+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/119366.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/119366.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/119366.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u7b2c\u4e09\u7ae0 \u5f3a\u5316\u5b66\u4e60 -- \u591a\u81c2\u8001\u864e\u673a"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/119366","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=119366"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/119366\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=119366"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=119366"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=119366"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=119366"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}