{"id":8644,"date":"2024-06-01T14:01:01","date_gmt":"2024-06-01T06:01:01","guid":{"rendered":""},"modified":"2024-06-01T14:01:01","modified_gmt":"2024-06-01T06:01:01","slug":"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d","status":"publish","type":"post","link":"https:\/\/mushiming.com\/8644.html","title":{"rendered":"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d"},"content":{"rendered":"<p><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"display: none;\"> \n <path stroke-linecap=\"round\" d=\"M5,0 0,2.5 5,5z\" id=\"raphael-marker-block\" style=\"-webkit-tap-highlight-color: rgba(0, 0, 0, 0);\"><\/path> \n<\/svg> <\/p>\n<h2>note<\/h2>\n<ul>\n<li>Comirec-SA\u57fa\u4e8eattention\u7684\u591a\u5174\u8da3\u5efa\u6a21\uff0c\u8bba\u6587\u4e2d\u5148\u901a\u8fc7attention\u63d0\u53d6\u5355\u4e00\u5174\u8da3\uff0c\u518d\u63a8\u5e7f\u5230\u591a\u5174\u8da3\u5efa\u6a21\u3002\u53e6\u5916\u4f7f\u7528\u8d2a\u5fc3\u7b97\u6cd5\u4f18\u5316\u5e26\u6709\u51c6\u786e\u5ea6+\u591a\u6837\u6027\u7684\u76ee\u6807\u51fd\u6570\u3002<\/li>\n<li>DR\u628aMIND\u7684attention\u6362\u6210argmax\uff08\u8fd8\u6709\u521d\u59cb\u5316\u65b9\u5f0f\u4e0d\u540c\u3001\u5e8f\u5217\u80f6\u56ca\u5230\u5174\u8da3\u80f6\u56ca\u7528\u53ef\u5b66\u4e60\u6743\u91cd\u77e9\u9635\u66ff\u4ee3\u7684\u4e0d\u540c\uff09\uff0cSR\u5219\u672c\u8d28\u662f\u591a\u5934\u6ce8\u610f\u529b\u8fdb\u884c\u591a\u5174\u8da3\u5efa\u6a21\u3002<\/li>\n<li><code>torch.einsum<\/code>\uff08Einstein summation convention\uff0c\u5373\u7231\u56e0\u65af\u5766\u6c42\u548c\u7ea6\u5b9a\uff09\u53ef\u4ee5\u5199tensor\u8fd0\u7b97\u548c\u66f4\u9ad8\u7ef4\u5ea6tensor\u5199\u6cd5\u66f4\u52a0\u7b80\u6d01\uff0c\u5982\u7528<code>torch.einsum(\"bij, bjk -&gt; bik\", batch_tensor_1, batch_tensor_2)<\/code>\u8fdb\u884cbatch\u5185\u77e9\u9635\u4e58\u6cd5\u8fd0\u7b97\u3002<\/li>\n<li>\u6ce8\u610f\u529b\u673a\u5236\u4e2d\u6700\u540e\u5bf9\u7b2c\u4e00\u4e2a\u7ef4\u5ea6<code>seq<\/code>\u8fdb\u884c<code>sofjtmax<\/code>\u5f52\u4e00\u5316\uff0c\u5373\u4f7f\u5f97<code>seq<\/code>\u7ef4\u5ea6\u7684\u5143\u7d20\u4e4b\u548c\u4e3a1\u3002\u56e0\u4e3a\u5bf9\u4e8e\u56fa\u5b9a\u5e8f\u5217\u4e2d\u4e0d\u5b58\u5728\u7684\u4f4d\u7f6e\u8d1f\u65e0\u7a77mask\u6389\u4e86\uff0c\u6240\u4ee5\u7f3a\u5931\u7684\u4f4d\u7f6e\u7ecf\u8fc7<code>softmax<\/code>\u5f52\u4e00\u5316\u540e\u7684\u6570\u503c\u4e3a0\u3002<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> Softmax \u2061 ( x i ) = exp \u2061 ( x i ) \u2211 j exp \u2061 ( x j ) \\operatorname{Softmax}\\left(x_i\\right)=\\frac{\\exp \\left(x_i\\right)}{\\sum_j \\exp \\left(x_j\\right)} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">Softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.5488em; vertical-align: -1.1218em;\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.427em;\"><span class=\"\" style=\"top: -2.314em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative; top: 0em;\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em;\"><span class=\"\" style=\"top: -2.4003em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em;\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\">exp<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em;\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em;\"><\/span><\/span><span class=\"\" style=\"top: -3.677em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mop\">exp<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1218em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/li>\n<\/ul>\n<div class=\"toc\">\n<h4>\u6587\u7ae0\u76ee\u5f55<\/h4>\n<ul>\n<li>note<\/li>\n<li>\u4e00\u3001Comirec\u6a21\u578b\u6982\u89c8<\/li>\n<li>\n<ul>\n<li>1.1 Notation<\/li>\n<li>1.2 \u6a21\u578b\u6846\u67b6<\/li>\n<li>1.3 model training<\/li>\n<\/ul>\n<\/li>\n<li>\u4e8c\u3001Comirec-SA\u6a21\u578b<\/li>\n<li>\n<ul>\n<li>2.0 \u6570\u636e\u96c6\u5212\u5206<\/li>\n<li>2.1 \u57fa\u4e8eAttention\u7684\u5355\u4e00\u5174\u8da3\u5efa\u6a21<\/li>\n<li>2.2 \u57fa\u4e8eAttention\u7684\u591a\u5174\u8da3\u5efa\u6a21<\/li>\n<li>\n<ul>\n<li>\uff081\uff09K-Q-V\u6ce8\u610f\u529b<\/li>\n<li>\n<ul>\n<li>1\uff09\u81ea\u6ce8\u610f\u529b\u673a\u5236<\/li>\n<li>2\uff09\u8ba1\u7b97 K-Q-V \u6ce8\u610f\u529b\uff08\u77e9\u9635\u5f62\u5f0f\uff09\uff1a<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<li>2.3 \u591a\u6837\u6027\u63a7\u5236<\/li>\n<\/ul>\n<\/li>\n<li>\u4e09\u3001\u4ee3\u7801\u5b9e\u8df5<\/li>\n<li>\n<ul>\n<li>3.1 MultiInterest_SA\u7c7b\u5b9a\u4e49<\/li>\n<li>3.2 einsum\u51fd\u6570\u7684\u7528\u6cd5<\/li>\n<li>3.3 Comirec-SA\u7c7b\u5b9a\u4e49<\/li>\n<\/ul>\n<\/li>\n<li>\u65f6\u95f4\u5b89\u6392<\/li>\n<li>Reference<\/li>\n<\/ul>\n<\/div>\n<h2>\u4e00\u3001Comirec\u6a21\u578b\u6982\u89c8<\/h2>\n<p>\u7b2c\u4e00\u90e8\u5206\u548ctask5\u5dee\u4e0d\u591a\uff0c\u662f\u540c\u4e00\u7bc7\u8bba\u6587Comirec-SA\u7528self-attentive method\u4ee3\u66ff\u4e4b\u524dDR\u591a\u5174\u8da3\u62bd\u53d6\u5c42\uff0c<br \/> <img decoding=\"async\" src=\"https:\/\/img.mushiming.top\/app\/mushiming_com\/4bf30c6103cd4aa88f3335ae85672ddd.png\" =\"\" =\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" alt=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d (https:\/\/mushiming.com\/)  \u7b2c1\u5f20\" title=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d  \u7b2c1\u5f20-\u7a46\u4e16\u660e\u535a\u5ba2\" ><br \/> Comirec\uff1aControllable Multi-Interest Framework for Recommendation<br \/> \u8bba\u6587\u94fe\u63a5\uff1ahttps:\/\/arxiv.org\/abs\/2005.09347<br \/> <img decoding=\"async\" src=\"https:\/\/img.mushiming.top\/app\/mushiming_com\/6a74be449f4242889c10809cc5b416cb.png\" =\"\" =\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" alt=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d (https:\/\/mushiming.com\/)  \u7b2c2\u5f20\" title=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d  \u7b2c2\u5f20-\u7a46\u4e16\u660e\u535a\u5ba2\" ><br \/> Comirec\u662f\u963f\u91cc\u53d1\u8868\u5728KDD 2020\u4e0a\u7684\u4e00\u7bc7\u5de5\u4f5c\uff0c\u8fd9\u7bc7\u8bba\u6587\u5bf9MIND\u591a\u884c\u4e3a\u53ec\u56de\u8fdb\u884c\u4e86\u6269\u5c55\uff1a<\/p>\n<ul>\n<li>\u4e00\u65b9\u9762\u6539\u8fdb\u4e86MIND\u4e2d\u7684\u52a8\u6001\u8def\u7531\u7b97\u6cd5\uff0c<\/li>\n<li>\u53e6\u4e00\u65b9\u9762\u63d0\u51fa\u4e86\u4e00\u79cd\u65b0\u7684\u591a\u5174\u8da3\u53ec\u56de\u65b9\u6cd5\uff0c\u540c\u65f6\u5bf9\u63a8\u8350\u7684\u591a\u6837\u6027\u5c42\u9762\u4e5f\u505a\u51fa\u4e86\u4e00\u5b9a\u7684\u8d21\u732e\uff0c\u901a\u8fc7\u4e00\u79cd\u8d2a\u5fc3\u7684\u505a\u6cd5\uff0c\u5728\u635f\u5931\u8f83\u5c0f\u7684\u63a8\u8350\u6548\u679c\u7684\u60c5\u51b5\u4e0b\u53ef\u4ee5\u663e\u8457\u7684\u63d0\u9ad8\u63a8\u8350\u7684\u591a\u6837\u6027(\u611f\u89c9\u975e\u5e38\u975e\u5e38\u7275\u5f3a\uff0c\u6211\u4eec\u8fd9\u91cc\u5c31\u4e0d\u4ecb\u7ecd\u8fd9\u4e2a\u4e86)\uff0c\u4ece\u800c\u6781\u5927\u7684\u63d0\u9ad8\u7528\u6237\u7684\u4f7f\u7528\u4f53\u9a8c\uff0c\u53ef\u4ee5\u66f4\u8fdb\u4e00\u6b65\u7684\u63a2\u7d22\u7528\u6237\u7684\u6f5c\u5728\u5174\u8da3\u3002<\/li>\n<\/ul>\n<h3>1.1 Notation<\/h3>\n<p>\u5047\u8bbe\u4e00\u4e2a\u7528\u6237\u96c6\u5408 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> u \u2208 U u \\in \\mathcal{U} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5782em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.0993em;\">U<\/span><\/span><\/span><\/span><\/span> \u548c\u4e00\u4e2a\u7269\u54c1\u96c6\u5408 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> i \u2208 I i \\in \\mathcal{I} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6986em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.0738em;\">I<\/span><\/span><\/span><\/span><\/span>, \u5bf9\u4e8e\u6bcf\u4e00\u4e2a\u7528\u6237, \u5b9a\u4e49\u7528\u6237\u5e8f\u5217 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> ( e 1 ( u ) , e 2 ( u ) , \u2026 , e n ( u ) ) \\left(e_1^{(u)}, e_2^{(u)}, \\ldots, e_n^{(u)}\\right) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.8em; vertical-align: -0.65em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size2\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0448em;\"><span class=\"\" style=\"top: -2.4337em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><span class=\"\" style=\"top: -3.2198em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2663em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0448em;\"><span class=\"\" style=\"top: -2.4337em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><span class=\"\" style=\"top: -3.2198em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2663em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0448em;\"><span class=\"\" style=\"top: -2.5834em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><span class=\"\" style=\"top: -3.2198em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1166em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size2\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span>, <mark>\u6839\u636e\u65f6\u95f4\u5148\u540e\u987a\u5e8f\u6392\u5e8f<\/mark>\uff0c\u5176\u4e2d<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> e t ( u ) e_t^{(u)} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.2906em; vertical-align: -0.2458em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0448em;\"><span class=\"\" style=\"top: -2.4542em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">t<\/span><\/span><\/span><span class=\"\" style=\"top: -3.2198em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2458em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u8bb0\u5f55\u4e86\u7b2c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> t t <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6151em;\"><\/span><span class=\"mord mathnormal\">t<\/span><\/span><\/span><\/span><\/span> \u4e2a\u7269\u54c1\u4e0e\u7528\u6237\u4ea4\u4e92\u3002<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/img.mushiming.top\/app\/mushiming_com\/e431cdcb673142f59d459f421c9883ad.png\" =\"\" =\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" alt=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d (https:\/\/mushiming.com\/)  \u7b2c3\u5f20\" title=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d  \u7b2c3\u5f20-\u7a46\u4e16\u660e\u535a\u5ba2\" ><\/p>\n<h3>1.2 \u6a21\u578b\u6846\u67b6<\/h3>\n<p><img decoding=\"async\" src=\"https:\/\/img.mushiming.top\/app\/mushiming_com\/eaf24304e84c44588a2126f2a260d3ad.png\" =\"\" =\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" alt=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d (https:\/\/mushiming.com\/)  \u7b2c4\u5f20\" title=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d  \u7b2c4\u5f20-\u7a46\u4e16\u660e\u535a\u5ba2\" ><br \/> Comirec\u6d41\u7a0b\uff1a<\/p>\n<ul>\n<li>\u7528\u6237\u5386\u53f2\u4ea4\u4e92item ids\u7ecf\u8fc7embedding layer\u8f6c\u4e3aitem embeddings\uff1b<\/li>\n<li>\u628a\u521a\u624d\u7684item embedding\u7ecf\u8fc7\u591a\u5174\u8da3\u62bd\u53d6\u5c42\uff08\u8fd9\u91cc\u662f<code>self-attentive<\/code>\u6a21\u5757\uff09\uff0c\u5f97\u5230K\u4e2ainterest embeddings\uff1b<\/li>\n<li>training\uff1a\u548c\u4e4b\u524dMIND\u8bf4\u7684\u505a\u6cd5\u4e00\u6837\uff08\u53d6\u5185\u79ef\u6700\u5927\u503c\uff09\uff0c\u5373\u9009\u51fa\u4e0etarget item embedding\u6700\u63a5\u8fd1\u7684interest embeddings\uff0c\u57fa\u4e8e\u591a\u5206\u7c7b\u4efb\u52a1\uff0c\u4f7f\u7528\u8d1f\u91c7\u6837sampled softmax loss\uff1b<\/li>\n<li>serving\uff08predict\uff09\uff1a\u5bf9\u4e8e\u6bcf\u4e2a\u7528\u6237\u7684K\u4e2ainterest embedding\u8fdb\u884ctop-N\u68c0\u7d22\uff0c\u5373\u53ec\u56de<code>K X N<\/code>\u4e2aitem\u3002<\/li>\n<li>aggregation module\uff1a\u5c06<code>K X N<\/code>\u4e2aitem\u9001\u5165\u8be5\u6a21\u5757\u5f97\u5230N\u4e2aitem\uff08\u9009\u5185\u79ef\u5206\u6570\u6700\u9ad8\u7684N\u4e2a\uff0c\u548cMIND\u505a\u6cd5\u4e00\u81f4\uff09\uff1a<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> f ( u , i ) = max \u2061 1 \u2264 k \u2264 K ( e i \u22a4 v u ( k ) ) f(u, i)=\\max _{1 \\leq k \\leq K}\\left(\\mathbf{e}_i^{\\top} \\mathbf{v}_u^{(k)}\\right) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1076em;\">f<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.9973em; vertical-align: -0.8473em;\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4306em;\"><span class=\"\" style=\"top: -2.3479em; margin-left: 0em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><span class=\"mrel mtight\">\u2264<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em;\">k<\/span><span class=\"mrel mtight\">\u2264<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em;\">K<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"\"><span class=\"mop\">max<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8473em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size2\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathbf\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8991em;\"><span class=\"\" style=\"top: -2.453em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathbf\" style=\"margin-right: 0.016em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.016em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em;\">k<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size2\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/li>\n<\/ul>\n<h3>1.3 model training<\/h3>\n<p>\u548c1.1\u8bf4\u7684\u4e00\u6837\uff0c\u67d0\u7528\u6237\u7ecf\u8fc7\u591a\u5174\u8da3\u63d0\u53d6\u5c42\u540e\u5f97\u5230K\u4e2ainterest embeddings\u540e\uff0c\u4ece\u4e2d\u627e\u51fa\u548ctarget item i\uff08\u5373\u4e0b\u9762embedding\u4e3a<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> e i e_i <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\uff09\u6700\u63a5\u8fd1\uff0c\u5373\u5185\u79ef\u6700\u5927\u7684\u4e00\u4e2ainterest embeeding\uff0c\u65b9\u6cd5\u5982\u4e0b<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> v u = V u [ : , argmax \u2061 ( V u \u22a4 e i ) ] , \\mathbf{v}_u=\\mathbf{V}_u\\left[:, \\operatorname{argmax}\\left(\\mathbf{V}_u^{\\top} \\mathbf{e}_i\\right)\\right], <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5944em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathbf\" style=\"margin-right: 0.016em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.016em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.2491em; vertical-align: -0.35em;\"><\/span><span class=\"mord\"><span class=\"mord mathbf\" style=\"margin-right: 0.016em;\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.016em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">[<\/span><\/span><span class=\"mrel\">:<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">argmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathbf\" style=\"margin-right: 0.016em;\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8991em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.016em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathbf\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">]<\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mpunct\">,<\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u548c\u4e4b\u524d\u4e00\u6837\u662f\u7ed3\u5408\u8d1f\u91c7\u6837\u7684\u6700\u5927\u4f3c\u7136\u6cd5\uff08\u540e\u9762\u4ee3\u7801\u7528\u5c11\u91cf\u6570\u636e\uff0c\u76f4\u63a5\u4ea4\u53c9\u71b5\u635f\u5931\u51fd\u6570\uff09\u3002\u7ed9\u51fa\u4e00\u4e2a\u8bad\u7ec3\u6837\u672c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> ( u , i ) (u, i) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>, \u7528\u6237embedding <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> v u \\mathbf{v}_u <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5944em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathbf\" style=\"margin-right: 0.016em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.016em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, \u548c\u7269\u54c1embedding <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> e i \\mathbf{e}_i <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5944em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathbf\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\uff0c\u5219\u7528\u6237\u4e0e\u7269\u54c1\u4ea4\u4e92\u7684\u4f3c\u7136\u51fd\u6570\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> P \u03b8 ( i \u2223 u ) = exp \u2061 ( v u \u22a4 e i ) \u2211 k \u2208 I exp \u2061 ( v u \u22a4 e k ) P_\\theta(i \\mid u)=\\frac{\\exp \\left(\\mathbf{v}_u^{\\top} \\mathbf{e}_i\\right)}{\\sum_{k \\in I} \\exp \\left(\\mathbf{v}_u^{\\top} \\mathbf{e}_k\\right)} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">P<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0278em;\">\u03b8<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.6031em; vertical-align: -1.0131em;\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.59em;\"><span class=\"\" style=\"top: -2.314em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative; top: 0em;\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1864em;\"><span class=\"\" style=\"top: -2.4003em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em;\">k<\/span><span class=\"mrel mtight\">\u2208<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0785em;\">I<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3271em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\">exp<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathbf\" style=\"margin-right: 0.016em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7751em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.016em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><span class=\"\" style=\"top: -2.989em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathbf\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em;\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em;\"><\/span><\/span><span class=\"\" style=\"top: -3.74em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mop\">exp<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathbf\" style=\"margin-right: 0.016em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.016em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u22a4<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathbf\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0131em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u76ee\u6807\u4e3a\u6700\u5c0f\u5316\u8be5\u4f3c\u7136\u51fd\u6570\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> &nbsp;loss&nbsp; = \u2211 u \u2208 U \u2211 i \u2208 I u \u2212 log \u2061 P \u03b8 ( i \u2223 u ) \\text { loss }=\\sum_{u \\in \\mathcal{U}} \\sum_{i \\in I_u}-\\log P_\\theta(i \\mid u) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord text\"><span class=\"mord\">&nbsp;loss&nbsp;<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4444em; vertical-align: -1.3944em;\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.05em;\"><span class=\"\" style=\"top: -1.8557em; margin-left: 0em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mrel mtight\">\u2208<\/span><span class=\"mord mathcal mtight\" style=\"margin-right: 0.0993em;\">U<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3217em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.05em;\"><span class=\"\" style=\"top: -1.8557em; margin-left: 0em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mrel mtight\">\u2208<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0785em;\">I<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em;\"><span class=\"\" style=\"top: -2.357em; margin-left: -0.0785em; margin-right: 0.0714em;\"><span class=\"pstrut\" style=\"height: 2.5em;\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3944em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em;\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">P<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0278em;\">\u03b8<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2223<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<h2>\u4e8c\u3001Comirec-SA\u6a21\u578b<\/h2>\n<h3>2.0 \u6570\u636e\u96c6\u5212\u5206<\/h3>\n<p>\u539f\u8bba\u6587\u662f\u6309\u71678:1:1\u5212\u5206\u8bad\u7ec3\u96c6\u3001\u9a8c\u8bc1\u96c6\u3001\u6d4b\u8bd5\u96c6\uff08<mark>\u6309\u7167\u7528\u6237\u5212\u5206\uff0c\u66f4\u5177\u6cdb\u5316\u80fd\u529b<\/mark>\uff09\u3002<\/p>\n<ul>\n<li>\u8bad\u7ec3\u6837\u672c\uff1a\u5168\u90e8\u70b9\u51fb\u5e8f\u5217<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> ( e 1 ( u ) , e 2 ( u ) , \u2026 , e k ( u ) , \u2026 , e n ( u ) ) \\left(e_1^{(u)}, e_2^{(u)}, \\ldots, e_k^{(u)}, \\ldots, e_n^{(u)}\\right) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.8em; vertical-align: -0.65em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size2\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0448em;\"><span class=\"\" style=\"top: -2.4337em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><span class=\"\" style=\"top: -3.2198em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2663em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0448em;\"><span class=\"\" style=\"top: -2.4337em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><span class=\"\" style=\"top: -3.2198em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2663em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0448em;\"><span class=\"\" style=\"top: -2.3987em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em;\">k<\/span><\/span><\/span><span class=\"\" style=\"top: -3.2198em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3013em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0448em;\"><span class=\"\" style=\"top: -2.5834em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><span class=\"\" style=\"top: -3.2198em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">u<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1166em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size2\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span>\uff0c\u7528\u524d\uff08n-1\uff09\u4e2aitem\u9884\u6d4b\u7b2cn\u4e2a\uff1b<\/li>\n<li>\u9a8c\u8bc1\u96c6\u5408\u6d4b\u8bd5\u96c6\u5219\u662f\u4f7f\u7528\u7528\u6237\u524d80%\u7684\u70b9\u51fb\u5e8f\u5217\u4f5c\u4e3a\u6a21\u578b\u8f93\u5165\uff0c\u7136\u540e\u9884\u6d4b\u540e\u9762\u7684\u70b9\u51fb\u5e8f\u5217\u3002<\/li>\n<li>\u5728\u9a8c\u8bc1\u96c6\u5408\u6d4b\u8bd5\u96c6\u4e2d\u7684\u7528\u6237\u662f\u4ece\u672a\u5728\u8bad\u7ec3\u96c6\u4e2d\u51fa\u73b0\u8fc7\u7684\u3002<\/li>\n<\/ul>\n<h3>2.1 \u57fa\u4e8eAttention\u7684\u5355\u4e00\u5174\u8da3\u5efa\u6a21<\/h3>\n<p><img decoding=\"async\" src=\"https:\/\/img.mushiming.top\/app\/mushiming_com\/693e5a1fcff849d8964c715ead79e7cd.png\" =\"\" =\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" alt=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d (https:\/\/mushiming.com\/)  \u7b2c5\u5f20\" title=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d  \u7b2c5\u5f20-\u7a46\u4e16\u660e\u535a\u5ba2\" ><\/p>\n<ul>\n<li>\u5c06\u7528\u6237\u7684\u884c\u4e3a\u5e8f\u5217\u8ba1\u4f5c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> H \u2208 R d \u00d7 n H \\in \\mathbb{R}^{d \\times n} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0813em;\">H<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>,\n<ul>\n<li>\u8fd9\u91cc\u7684 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> n n <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em;\"><\/span><span class=\"mord mathnormal\">n<\/span><\/span><\/span><\/span><\/span> \u8868\u793a\u7528\u6237\u884c\u4e3a\u5e8f\u5217\u7684\u957f\u5ea6,<\/li>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> d d <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><\/span><\/span> \u8868\u793aEmbedding\u5411\u91cf\u7684\u7ef4\u5ea6,<\/li>\n<li>\u8fd9\u91cc\u7684 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> H H <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0813em;\">H<\/span><\/span><\/span><\/span><\/span> \u5c31\u4ee3\u8868\u7740\u7528\u6237\u884c\u4e3a\u5e8f\u5217\u4e2d\u6240\u6709Item\u7684Embedding\u5411\u91cf,<\/li>\n<\/ul>\n<\/li>\n<li>\u8fd9\u91cc\u5f15\u5165\u4e24\u4e2a\u53ef\u5b66\u4e60\u53c2\u6570 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> W 1 \u2208 R d a \u00d7 d , w 2 \u2208 R d a W_1 \\in \\mathbb{R}^{d_a \\times d}, w_2 \\in \\mathbb{R}^{d_a} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0435em; vertical-align: -0.1944em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em;\"><span class=\"\" style=\"top: -2.357em; margin-left: 0em; margin-right: 0.0714em;\"><span class=\"pstrut\" style=\"height: 2.5em;\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0269em;\">w<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.0269em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em;\"><span class=\"\" style=\"top: -2.357em; margin-left: 0em; margin-right: 0.0714em;\"><span class=\"pstrut\" style=\"height: 2.5em;\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, \u901a\u8fc7\u5f15\u5165Attention\u673a\u5236\u8fdb\u884c\u52a0\u6743\u6c42\u548c\uff0c\u51f8\u663e\u91cd\u8981\u6027\u9ad8\u7684\u7279\u5f81\u3002<\/li>\n<\/ul>\n<p>\u5728\u6211\u4eec\u7684Comirec-SA\u4e2d, \u6211\u4eec\u7684<mark>\u7279\u5f81\u91cd\u8981\u6027 (\u4e5f\u5c31\u662f\u6211\u4eec\u5b66\u4e60\u51fa\u6765\u7684Attention Score) \u662f\u9488\u5bf9\u5e8f\u5217\u4e2d\u6bcf\u4e2aItem\u7684Attention Score<\/mark>, \u5728\u6709\u4e86Attention Score\u4e4b\u540e\u5c31\u53ef\u4ee5\u5bf9\u5e8f\u5217\u4e2d\u7684|tem\u8fdb\u884c\u52a0\u6743\u6c42\u548c\u5f97\u5230\u5e8f\u5217\u7684\u5355\u4e00\u5174\u8da3\u8868\u5f81\u4e86\u3002\u5355\u4e00\u5174\u8da3\u5efa\u6a21\u65f6\u7684Attention Score\u8ba1\u7b97:<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> a = softmax \u2061 ( w 2 T tanh \u2061 ( W 1 H ) ) T a=\\operatorname{softmax}\\left(w_2^T \\tanh \\left(W_1 H\\right)\\right)^T <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em;\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.4726em; vertical-align: -0.35em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0269em;\">w<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.0269em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\">tanh<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0813em;\">H<\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1226em;\"><span class=\"\" style=\"top: -3.3442em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> a \u2208 R n a \\in \\mathbb{R}^n <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5782em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6889em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, \u8fd9\u91cc\u7684 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> a a <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em;\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u5c31\u662f\u6211\u4eec\u5bf9\u5e8f\u5217\u7684Attention Score\uff08\u6743\u91cd\u5411\u91cf\uff09, \u518d\u5c06 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> a a <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em;\"><\/span><span class=\"mord mathnormal\">a<\/span><\/span><\/span><\/span><\/span> \u4e0e\u5e8f\u5217\u7684\u6240\u6709Item\u7684Embedding\u8fdb\u884c\u52a0\u6743\u6c42\u548c\u5c31\u53ef\u4ee5\u5f97\u5230\u5355\u4e00\u5174\u8da3\u7684\u5efa\u6a21\u3002<\/li>\n<\/ul>\n<h3>2.2 \u57fa\u4e8eAttention\u7684\u591a\u5174\u8da3\u5efa\u6a21<\/h3>\n<p>\u53ef\u4ee5\u628a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> 1.1 1.1 <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em;\"><\/span><span class=\"mord\">1.1<\/span><\/span><\/span><\/span><\/span>\u5355\u4e00\u5174\u8da3\u5efa\u6a21\u4e2d\u7684 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> w 2 \u2208 R d a w_2 \\in \\mathbb{R}^{d_a} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6891em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0269em;\">w<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.0269em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em;\"><span class=\"\" style=\"top: -2.357em; margin-left: 0em; margin-right: 0.0714em;\"><span class=\"pstrut\" style=\"height: 2.5em;\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u6269\u5145\u81f3 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> W 2 \u2208 R d a \u00d7 K W_2 \\in \\mathbb{R}^{d_a \\times K} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em;\"><span class=\"\" style=\"top: -2.357em; margin-left: 0em; margin-right: 0.0714em;\"><span class=\"pstrut\" style=\"height: 2.5em;\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em;\">K<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\uff08\u53ef\u8bad\u7ec3\u5316\u53c2\u6570\uff09, \u8fd9\u91cc\u662f\u56e0\u4e3a<mark>\u5728\u8f93\u5165embedding\u4e2d\u52a0\u5165\u4e86\u53ef\u8bad\u7ec3\u7684position embedding\uff0c\u5176\u7ef4\u5ea6\u548citem embedding\u7684\u7ef4\u5ea6\u4e00\u6837\u90fd\u662f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> d d <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><\/span><\/span><\/mark>\u3002<\/p>\n<p>Attention Score\u7684\u8ba1\u7b97\u516c\u5f0f\u5c31\u53d8\u6210:<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> A = softmax \u2061 ( W 2 T tanh \u2061 ( W 1 H ) ) T A=\\operatorname{softmax}\\left(W_2^T \\tanh \\left(W_1 H\\right)\\right)^T <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.4726em; vertical-align: -0.35em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\">tanh<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0813em;\">H<\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1226em;\"><span class=\"\" style=\"top: -3.3442em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u5176\u4e2d\uff1a<\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> A \u2208 R n \u00d7 K A \\in \\mathbb{R}^{n \\times K} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8413em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">n<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em;\">K<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>,<\/li>\n<li>\u6743\u91cd<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> W 2 \u2208 R d a \u00d7 K W_2 \\in \\mathbb{R}^{d_a \\times K} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em;\"><span class=\"\" style=\"top: -2.357em; margin-left: 0em; margin-right: 0.0714em;\"><span class=\"pstrut\" style=\"height: 2.5em;\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">a<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em;\">K<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/li>\n<li>K\u662f\u5174\u8da3\u5411\u91cf\u4e2a\u6570<\/li>\n<\/ul>\n<p>\u8fd9\u65f6\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u5f0f\u5b50\u5f97\u5230\u7528\u6237\u7684\u591a\u5174\u8da3\u8868\u5f81\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> V u = H A V_u=H A <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.2222em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0813em;\">H<\/span><span class=\"mord mathnormal\">A<\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u5176\u4e2d\uff1a<\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> H d \u00d7 n \\mathbf{H}^{d \\times n} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathbf\">H<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\uff1a\u7528\u6237\u884c\u4e3a\u5e8f\u5217\u7684embedding\u77e9\u9635\uff0c\u5176\u4e2d<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> n n <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em;\"><\/span><span class=\"mord mathnormal\">n<\/span><\/span><\/span><\/span><\/span>\u4e3a\u7528\u6237\u884c\u4e3a\u5e8f\u5217\u7684\u957f\u5ea6\uff0c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> d d <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><\/span><\/span>\u4e3aitem embedding\u7ef4\u5ea6\uff1b<\/li>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> V u \u2208 R d \u00d7 K V_u \\in \\mathbb{R}^{d \\times K} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.2222em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em;\">K<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, \u5373\u4e3a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> K \\mathrm{K} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathrm\">K<\/span><\/span><\/span><\/span><\/span> \u4e2a\u5174\u8da3\u8868\u5f81, \u5176\u6838\u5fc3\u4ee3\u7801\u5982\u4e0b:<\/li>\n<\/ul>\n<pre class=\"language-java prettyprint linenums\"><code class=\"prism language-python\"><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">MultiInterest_SA<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Layer<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> embedding_dim<span class=\"token punctuation\">,<\/span> interest_num<span class=\"token punctuation\">,<\/span> d<span class=\"token operator\">=<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span>MultiInterest_SA<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>\n        self<span class=\"token punctuation\">.<\/span>embedding_dim <span class=\"token operator\">=<\/span> embedding_dim\n        self<span class=\"token punctuation\">.<\/span>interest_num <span class=\"token operator\">=<\/span> interest_num\n        <span class=\"token keyword\">if<\/span> d <span class=\"token operator\">==<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">:<\/span>\n            self<span class=\"token punctuation\">.<\/span>d <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>embedding_dim<span class=\"token operator\">*<\/span><span class=\"token number\">4<\/span>\n\n        self<span class=\"token punctuation\">.<\/span>W1 <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>create_parameter<span class=\"token punctuation\">(<\/span>shape<span class=\"token operator\">=<\/span><span class=\"token punctuation\">[<\/span>self<span class=\"token punctuation\">.<\/span>embedding_dim<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>d<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span>\n        self<span class=\"token punctuation\">.<\/span>W2 <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>create_parameter<span class=\"token punctuation\">(<\/span>shape<span class=\"token operator\">=<\/span><span class=\"token punctuation\">[<\/span>self<span class=\"token punctuation\">.<\/span>d<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>interest_num<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> \n\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> seq_emb<span class=\"token punctuation\">,<\/span> mask <span class=\"token operator\">=<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n        <span class=\"token triple-quoted-string string\">''' seq_emb : batch,seq,emb mask : batch,seq,1 '''<\/span>\n        H <span class=\"token operator\">=<\/span> paddle<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">'bse, ed -&gt; bsd'<\/span><span class=\"token punctuation\">,<\/span> seq_emb<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>W1<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>tanh<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>\n        mask <span class=\"token operator\">=<\/span> mask<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>\n        A <span class=\"token operator\">=<\/span> paddle<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">'bsd, dk -&gt; bsk'<\/span><span class=\"token punctuation\">,<\/span> H<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>W2<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">+<\/span> <span class=\"token operator\">-<\/span><span class=\"token number\">1.e9<\/span> <span class=\"token operator\">*<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span> <span class=\"token operator\">-<\/span> mask<span class=\"token punctuation\">)<\/span>\n        A <span class=\"token operator\">=<\/span> F<span class=\"token punctuation\">.<\/span>softmax<span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">,<\/span> axis<span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>\n        A <span class=\"token operator\">=<\/span> paddle<span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">,<\/span>perm<span class=\"token operator\">=<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span>\n        multi_interest_emb <span class=\"token operator\">=<\/span> paddle<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">,<\/span> seq_emb<span class=\"token punctuation\">)<\/span>\n        <span class=\"token keyword\">return<\/span> multi_interest_emb\n<\/code><\/pre>\n<p>\u9996\u5148\u6765\u56de\u987e\u5355\u5934\u548c\u591a\u5934\u7684\u6ce8\u610f\u529b\uff1a<\/p>\n<h4>\uff081\uff09K-Q-V\u6ce8\u610f\u529b<\/h4>\n<h5>1\uff09\u81ea\u6ce8\u610f\u529b\u673a\u5236<\/h5>\n<p>\u81ea\u6ce8\u610f\u529b\u662f\u6307\u952e(K)\u3001\u67e5\u8be2(Q)\u548c\u503c(V)\u6765\u81ea\u540c\u4e00\u6570\u636e\u6765\u6e90\uff0c\u5373 K=Q=V\u3002<\/p>\n<p>Transformer \u4ee5\u7279\u5b9a\u65b9\u5f0f\u6267\u884c\u6b64\u64cd\u4f5c:<br \/> \u4ee4 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> x 1 , x 2 , \u22ef \u2009 , x T x_{1}, x_{2}, \\cdots, x_{T} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em; vertical-align: -0.1944em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\">\u22ef<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f Transformer \u7f16\u7801\u5668\u7684\u8f93\u5165\u5411\u91cf, \u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> x i \u2208 R d x_{i} \\in R^{d} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6891em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, \u5219\u952e <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> ( K ) (K) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u3001\u67e5\u8be2 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> ( Q ) (Q) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u3001\u503c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> ( V ) (V) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u662f:<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> k i = K x i q i = Q x i v i = V x i \\begin{aligned} k_{i} &amp;=K x_{i} \\\\ q_{i} &amp;=Q x_{i} \\\\ v_{i} &amp;=V x_{i} \\end{aligned} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 4.5em; vertical-align: -2em;\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.5em;\"><span class=\"\" style=\"top: -4.66em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0315em;\">k<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.0315em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.16em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em;\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.0359em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -1.66em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.0359em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.5em;\"><span class=\"\" style=\"top: -4.66em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.16em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -1.66em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<ul>\n<li>\u5176\u4e2d: \u77e9\u9635 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> K \u2208 R d \u00d7 d ; Q \u2208 R d \u00d7 d ; V \u2208 R d \u00d7 d K \\in R^{d \\times d} ; Q \\in R^{d \\times d} ; V \\in R^{d \\times d} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0435em; vertical-align: -0.1944em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">;<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0435em; vertical-align: -0.1944em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">;<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u3002<\/li>\n<li>\u8fd9\u4e9b\u77e9\u9635\u5141\u8bb8\u5c06x\u5411\u91cf\u7684\u4e0d\u540c\u65b9\u9762\u7528\u4e8e\u4e09\u4e2a\u89d2\u8272\u7684\u6bcf\u4e00\u4e2a\u3002<\/li>\n<\/ul>\n<h5>2\uff09\u8ba1\u7b97 K-Q-V \u6ce8\u610f\u529b\uff08\u77e9\u9635\u5f62\u5f0f\uff09\uff1a<\/h5>\n<p>\u4ee4 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> X = [ x 1 , x 2 , \u22ef \u2009 , x T ] \u2208 R T \u00d7 d \\mathrm{X}=\\left[x_{1}, x_{2}, \\cdots, x_{T}\\right] \\in \\mathrm{R}^{\\mathrm{T} \\times \\mathrm{d}} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathrm\">X<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">[<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\">\u22ef<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">]<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">T<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathrm mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f\u8f93\u5165\u5411\u91cf\u7684\u4e32\u8054(\u62fc\u63a5), \u5176\u4e2d:<\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> X K \u2208 R T \u00d7 d , X Q \u2208 R T \u00d7 d , X V \u2208 R T \u00d7 d X K \\in R^{T \\times d}, X Q \\in R^{T \\times d}, X V \\in R^{T \\times d} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0435em; vertical-align: -0.1944em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\">XQ<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0435em; vertical-align: -0.1944em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u3002<\/li>\n<li>\u8f93\u51fa\u5b9a\u4e49\u4e3a: output = <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> softmax \u2061 ( X Q ( X K ) T \u00d7 X V ) \\operatorname{softmax}\\left(X Q(X K)^{\\mathrm{T}} \\times X V\\right) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.2em; vertical-align: -0.35em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord mathnormal\">XQ<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/li>\n<li>\u5373\u8ba1\u7b97\u8fc7\u7a0b\uff1a\u9996\u5148, \u5728\u4e00\u4e2a\u77e9\u9635\u4e58\u6cd5\u4e2d\u53d6\u67e5\u8be2\u548c\u952e\u7684\u70b9\u79ef, \u5f97\u5230: <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> X Q ( X K ) T X Q(X K)^{\\mathrm{T}} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0913em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\">XQ<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\uff1b\u5176\u6b21, \u5bf9\u4e0a\u8ff0\u7ed3\u679c\u8fdb\u884c softmax \u8ba1\u7b97, \u518d\u4e58\u4ee5\u77e9\u9635 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> X V X V <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><\/span><\/span><\/span><\/span>, \u5f97\u5230: <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> softmax \u2061 ( X Q ( X K ) T ) \u00d7 X V \\operatorname{softmax}\\left(X Q(X K)^{\\mathrm{T}}\\right) \\times X V <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.2em; vertical-align: -0.35em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord mathnormal\">XQ<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><\/span><\/span><\/span><\/span> ,\u5982\u4e0b\u56fe\u7684\uff08a\uff09\u3002<\/li>\n<\/ul>\n<p><img decoding=\"async\" src=\"https:\/\/img.mushiming.top\/app\/mushiming_com\/e616eca304004d50b159d2593b0ea50e.png\" =\"\" =\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" alt=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d (https:\/\/mushiming.com\/)  \u7b2c6\u5f20\" title=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d  \u7b2c6\u5f20-\u7a46\u4e16\u660e\u535a\u5ba2\" ><\/p>\n<p>\u5982\u679c\u60f3\u4e00\u6b21\u67e5\u8be2\uff08Q\uff09\u53e5\u5b50\u4e2d\u7684\u591a\u4e2a\u4f4d\u7f6e\uff1a<br \/> \u5bf9\u4e8e\u5355\u8bcd <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> i \\mathrm{i} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6679em;\"><\/span><span class=\"mord mathrm\">i<\/span><\/span><\/span><\/span><\/span>\uff0c self-attention \u53ea \u201c\u770b\u201d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> x i T Q T K x j x_{i}^{T} Q^{T} K x_{j} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1274em; vertical-align: -0.2861em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em;\"><span class=\"\" style=\"top: -2.4413em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2587em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em;\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u5206\u6570\u9ad8\u7684\u5730\u65b9, \u5373\u5355\u5934\u6ce8\u610f\u529b, \u89c1\u4e0a\u56fe\u7684\uff08b\uff09 \u3002\u4f46\u662f, <mark>\u82e5\u51fa\u4e8e\u4e0d\u540c\u7684\u539f\u56e0\u60f3\u8981\u5173\u6ce8\u4e0d\u540c\u7684 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> j \\mathrm{j} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8623em; vertical-align: -0.1944em;\"><\/span><span class=\"mord mathrm\">j<\/span><\/span><\/span><\/span><\/span>\uff0c\u5219\u53ef\u901a\u8fc7\u591a\u4e2a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> Q \u3001 &nbsp; K \u3001 &nbsp; V \\mathrm{Q} \u3001 \\mathrm{~K} \u3001 \\mathrm{~V} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8778em; vertical-align: -0.1944em;\"><\/span><span class=\"mord mathrm\">Q<\/span><span class=\"mord cjk_fallback\">\u3001<\/span><span class=\"mord\"><span class=\"mspace nobreak\">&nbsp;<\/span><span class=\"mord mathrm\">K<\/span><\/span><span class=\"mord cjk_fallback\">\u3001<\/span><span class=\"mord\"><span class=\"mspace nobreak\">&nbsp;<\/span><span class=\"mord mathrm\" style=\"margin-right: 0.0139em;\">V<\/span><\/span><\/span><\/span><\/span><\/span> \u77e9\u9635\u6765\u5b9a\u4e49\u591a\u4e2a\u6ce8\u610f\u529b \u201c\u5934\u201d<\/mark>\uff1a<\/p>\n<p>\u4ee4: <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> Q l , K l , V l \u2208 R d \u00d7 d \/ h Q_{l}, K_{l}, V_{l} \\in R^{d \\times d \/ h} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8778em; vertical-align: -0.1944em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.0715em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.2222em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.888em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mord mtight\">\/<\/span><span class=\"mord mathnormal mtight\">h<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, \u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> h h <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord mathnormal\">h<\/span><\/span><\/span><\/span><\/span> \u662f\u6ce8\u610f\u529b\u5934\u7684\u6570\u91cf, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> l l <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><\/span> \u7684\u8303\u56f4\u4ece 1 \u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> h h <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord mathnormal\">h<\/span><\/span><\/span><\/span><\/span>\uff0c\u5219: \u6bcf\u4e2a\u6ce8\u610f\u529b\u5934\u72ec\u7acb\u5730\u8ba1\u7b97\u6ce8\u610f\u529b:<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> &nbsp;output&nbsp; l = softmax \u2061 ( X Q l K l T X T ) \u2217 X V l \\text { output }_{l}=\\operatorname{softmax}\\left(X Q_{l} K_{l}^{T} X^{T}\\right) * X V_{l} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8592em; vertical-align: -0.2441em;\"><\/span><span class=\"mord\"><span class=\"mord text\"><span class=\"mord\">&nbsp;output&nbsp;<\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.242em;\"><span class=\"\" style=\"top: -2.4559em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2441em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.2413em; vertical-align: -0.35em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.0715em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em;\"><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u2217<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em; vertical-align: -0.15em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em;\">X<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.2222em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d:output <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> l \u2208 R d \/ h _{l} \\in R^{d \/ h} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6891em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"\"><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em;\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.888em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mord mtight\">\/<\/span><span class=\"mord mathnormal mtight\">h<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u7136\u540e\u5408\u5e76\u6240\u6709\u5934\u7684\u8f93\u51fa:<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> &nbsp;output&nbsp; = Y &nbsp;[output&nbsp; 1 , \u2026 , &nbsp;output&nbsp; h &nbsp;]&nbsp; \\text { output }=Y \\text { [output }_{1}, \\ldots, \\text { output }_{h} \\text { ] } <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8095em; vertical-align: -0.1944em;\"><\/span><span class=\"mord text\"><span class=\"mord\">&nbsp;output&nbsp;<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0497em; vertical-align: -0.2997em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">Y<\/span><span class=\"mord\"><span class=\"mord text\"><span class=\"mord\">&nbsp;[output&nbsp;<\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em;\"><span class=\"\" style=\"top: -2.4003em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2997em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord text\"><span class=\"mord\">&nbsp;output&nbsp;<\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.242em;\"><span class=\"\" style=\"top: -2.4559em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">h<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2441em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord text\"><span class=\"mord\">&nbsp;]&nbsp;<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d: <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> Y \u2208 R d \u00d7 d Y \\in R^{d \\times d} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">Y<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0077em;\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">d<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u56e0\u6b64, \u6bcf\u4e2a\u5934\u90fd\u53ef\u4ee5 \u201c\u770b\u201d \u4e0d\u540c\u7684\u4e8b\u7269, \u5e76\u4ee5\u4e0d\u540c\u7684\u65b9\u5f0f\u6784\u5efa\u503c ( <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> V \\mathrm{V} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathrm\" style=\"margin-right: 0.0139em;\">V<\/span><\/span><\/span><\/span><\/span> ) \u5411\u91cf\u3002<mark>\u591a\u5934\u6ce8\u610f\u529b\u548c\u5355\u5934\u81ea\u6ce8\u610f\u529b\u6709\u76f8\u540c\u7684\u8ba1\u7b97\u91cf<\/mark>\u3002\u6700\u7b80\u5355\u7684\u591a\u5934\u6ce8\u610f\u529b\uff0c\u5373\u4e24\u5934\u6ce8\u610f\u529b\uff08\u5982\u4e0a\u56fe\u7684c\uff09\u3002<\/p>\n<h3>2.3 \u591a\u6837\u6027\u63a7\u5236<\/h3>\n<p>\u4e3a\u4e86\u5f3a\u5316\u63a8\u8350item\u7684\u591a\u6837\u6027\uff0c\u4f5c\u8005\u5c06item\u7684\u7c7b\u522b\u4f5c\u4e3a\u8861\u91cf\u591a\u6837\u6027\u7684\u57fa\u7840\u3002<\/p>\n<p>\u95ee\u9898\u5b9a\u4e49\uff1a\u7ed9\u5b9a\u4ece\u7528\u6237 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> u u <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em;\"><\/span><span class=\"mord mathnormal\">u<\/span><\/span><\/span><\/span><\/span> \u7684 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> K K <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><\/span><\/span><\/span><\/span> \u4e2a\u5174\u8da3\u4e2d\u68c0\u7d22\u5230\u4e00\u4e2a\u96c6\u5408 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> M \\mathcal{M} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathcal\">M<\/span><\/span><\/span><\/span><\/span>, \u6709 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> K \u22c5 N K \\cdot N <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.109em;\">N<\/span><\/span><\/span><\/span><\/span> \u4e2a\u7269\u54c1, \u627e\u5230\u4e00\u4e2a\u8f93\u51fa\u96c6\u5408 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> S \\mathcal{S} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.075em;\">S<\/span><\/span><\/span><\/span><\/span>, \u6709 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> N N <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.109em;\">N<\/span><\/span><\/span><\/span><\/span> \u4e2a \u7269\u54c1 (Top-N) , \u4f7f\u9884\u5b9a\u4e49\u7684\u503c\u51fd\u6570<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> Q ( u , S ) Q(u, \\mathcal{S}) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.075em;\">S<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>\u6700\u5927\u5316\u3002<\/p>\n<p>\uff081\uff09\u591a\u6837\u6027\uff1a\u4f5c\u8005\u4f7f\u7528<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> g ( i , j ) g(i, j) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em;\">g<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0572em;\">j<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>\u7528\u6765\u8861\u91cf\u4e24\u4e2aItem i\uff0c j\u4e4b\u95f4\u7684\u591a\u6837\u6027\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> g ( i , j ) = \u03c3 ( C A T E ( i ) \u2260 C A T E ( j ) ) g(i, j)=\\sigma(C A T E(i) \\neq C A T E(j)) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em;\">g<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0572em;\">j<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em;\">\u03c3<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">C<\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em;\">TE<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\"><span class=\"mrel\"><span class=\"mord vbox\"><span class=\"thinbox\"><span class=\"rlap\"><span class=\"strut\" style=\"height: 0.8889em; vertical-align: -0.1944em;\"><\/span><span class=\"inner\"><span class=\"mord\"><span class=\"mrel\">\ue020<\/span><\/span><\/span><span class=\"fix\"><\/span><\/span><\/span><\/span><\/span><span class=\"mrel\">=<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">C<\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em;\">TE<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0572em;\">j<\/span><span class=\"mclose\">))<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> C A T E ( i ) C A T E(i) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">C<\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em;\">TE<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u8868\u793a\u7269\u54c1 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> i i <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em;\"><\/span><span class=\"mord mathnormal\">i<\/span><\/span><\/span><\/span><\/span> \u7684\u7c7b\u522b<\/li>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> \u03c3 \\sigma <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em;\">\u03c3<\/span><\/span><\/span><\/span><\/span> \u4e3a\u6307\u793a\u51fd\u6570\uff0c\u8fd9\u91cc\u5c31\u662f\u5982\u679c\u4e24\u4e2aitem\u7684\u7c7b\u522b\u4e0d\u76f8\u540c\uff0c\u90a3\u4e48\u5176\u7ed3\u679c\u5c31\u662f 1 , \u53cd\u4e4b\u5c31\u662f 0<\/li>\n<li>\u53ef\u4ee5\u770b\u51fa\u5982\u679c\u4e00\u4e2a\u63a8\u8350\u96c6\u5408\u4e2d\u4e24\u4e24Item\u7684\u591a\u6837\u6027\u5f97\u5206\u5927\u7684\u8bdd, \u90a3\u4e48\u53ef\u4ee5\u8ba4\u4e3a\u8fd9\u4e2a\u63a8\u8350\u7ed3\u679c\u4e2d\u7684Item\u7684\u7c7b\u522b\u5206\u5e03\u8f83\u4e3a\u5206\u6563, \u4e5f\u5c31\u53ef\u4ee5\u8ba4\u4e3a\u63a8\u8350\u7ed3\u679c\u7684\u591a\u6837\u6027\u8f83\u9ad8\u4e86,<\/li>\n<\/ul>\n<p>\uff082\uff09\u63a8\u8350\u7cbe\u5ea6\uff1a\u5f53\u63a8\u8350\u7ed3\u679c\u7684\u591a\u6837\u6027\u8f83\u9ad8\u7684\u65f6\u5019, \u5f80\u5f80\u63a8\u8350\u7684\u7cbe\u5ea6\u5c31\u4f1a\u4e0b\u964d, \u8fd9\u662f\u4e00\u4e2aTrade Off\uff0c\u53ef\u4ee5\u6784\u9020\u5982\u4e0b\u7684\u76ee\u6807\u51fd\u6570\uff08\u5305\u542b\u591a\u6837\u6027\u6307\u6807\u548c\u63a8\u8350\u7cbe\u5ea6\u6307\u6807\uff09\uff1a<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> Q ( u , S ) = \u2211 i \u2208 S f ( u , i ) + \u03bb \u2211 i \u2208 S \u2211 j \u2208 S g ( i , j ) Q(u, \\mathcal{S})=\\sum_{i \\in \\mathcal{S}} f(u, i)+\\lambda \\sum_{i \\in \\mathcal{S}} \\sum_{j \\in \\mathcal{S}} g(i, j) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.075em;\">S<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.3717em; vertical-align: -1.3217em;\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.05em;\"><span class=\"\" style=\"top: -1.8557em; margin-left: 0em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mrel mtight\">\u2208<\/span><span class=\"mord mathcal mtight\" style=\"margin-right: 0.075em;\">S<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3217em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1076em;\">f<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">+<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4804em; vertical-align: -1.4304em;\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.05em;\"><span class=\"\" style=\"top: -1.8557em; margin-left: 0em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mrel mtight\">\u2208<\/span><span class=\"mord mathcal mtight\" style=\"margin-right: 0.075em;\">S<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3217em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.05em;\"><span class=\"\" style=\"top: -1.8557em; margin-left: 0em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em;\">j<\/span><span class=\"mrel mtight\">\u2208<\/span><span class=\"mord mathcal mtight\" style=\"margin-right: 0.075em;\">S<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em;\"><span class=\"pstrut\" style=\"height: 3.05em;\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4304em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0359em;\">g<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0572em;\">j<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<ul>\n<li>\u901a\u8fc7 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> \u03bb \\lambda <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><\/span><\/span><\/span><\/span> \u6765\u63a7\u5236\u8fd9\u4e24\u90e8\u5206\u7684\u5360\u6bd4, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> \u03bb \\lambda <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em;\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><\/span><\/span><\/span><\/span> \u8d8a\u5927\u5219\u591a\u6837\u6027\u66f4\u5f3a\uff0c\u540c\u4e8b\u51c6\u786e\u5ea6\u4f1a\u964d\u4f4e\uff1b<\/li>\n<li>\u4f5c\u8005\u8fd9\u91cc\u63d0\u51fa\u4f7f\u7528\u4e00\u79cd\u8d2a\u5fc3\u7684\u505a\u6cd5\u6765\u8fdb\u884c\u76ee\u6807\u51fd\u6570\u7684\u4f18\u5316, \u4f18\u5316\u6d41\u7a0b\u5982\u4e0b:<\/li>\n<\/ul>\n<p><img decoding=\"async\" src=\"https:\/\/img.mushiming.top\/app\/mushiming_com\/b2b51f7b8bbe4c56a5058d8c702cb39f.png\" =\"\" =\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" alt=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d (https:\/\/mushiming.com\/)  \u7b2c7\u5f20\" title=\"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d  \u7b2c7\u5f20-\u7a46\u4e16\u660e\u535a\u5ba2\" ><\/p>\n<h2>\u4e09\u3001\u4ee3\u7801\u5b9e\u8df5<\/h2>\n<h3>3.1 MultiInterest_SA\u7c7b\u5b9a\u4e49<\/h3>\n<p>\u5982\u4e0a\u9762\u4ecb\u7ecd\u7684SA\u8fd0\u7b97\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> A = softmax \u2061 ( W 2 T tanh \u2061 ( W 1 H ) ) T A=\\operatorname{softmax}\\left(W_2^T \\tanh \\left(W_1 H\\right)\\right)^T <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.4726em; vertical-align: -0.35em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\">tanh<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0813em;\">H<\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1226em;\"><span class=\"\" style=\"top: -3.3442em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u53ef\u4ee5\u5f97\u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> A \u2208 R n \u00d7 K A \\in \\mathbb{R}^{n \\times K} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em; vertical-align: -0.0391em;\"><\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8413em;\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">n<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em;\">K<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, \u8fd9\u65f6\u6211\u4eec\u53ef\u4ee5\u901a\u8fc7\u5982\u4e0b\u5f0f\u5b50\u5f97\u5230\u7528\u6237\u7684\u591a\u5174\u8da3\u8868\u5f81\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> V u = H A V_u=H A <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.2222em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">u<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0813em;\">H<\/span><span class=\"mord mathnormal\">A<\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u6ce8\u610f\uff1a<\/p>\n<ul>\n<li>\u4e24\u4e2a\u7ef4\u5ea6\u4e0d\u540c\u7684tensor\u4e5f\u53ef\u4ee5\u201c\u505a\u52a0\u51cf\u6cd5\u201d\uff08\u6307\u7684\u662f\u4ee3\u7801\u4e2d\u7684\u5199\u6cd5\uff09, \u6bd4\u5982<code>[256, 20, 4]<\/code>\u7684tensor-<code>[256, 20, 1]<\/code>\u7684tensor\uff0c\u540e\u8005\u4f1a\u51cf\u5728\u5168\u90e8\uff084\u4e2a\uff09\u7b2c2\u7ef4\u5ea6\u4e4b\u4e0a\u3002<\/li>\n<li>\u4e0b\u9762\u8fd9\u91cc\u7684<code>paddle.einsum<\/code>\u51fd\u6570\u5728tf\u548ctorch\u4e2d\u90fd\u6709\uff0c<code>einsum<\/code>\uff08Einstein summation convention\uff0c\u5373\u7231\u56e0\u65af\u5766\u6c42\u548c\u7ea6\u5b9a\uff09\u53ef\u4ee5\u5199tensor\u8fd0\u7b97\u548c\u66f4\u9ad8\u7ef4\u5ea6tensor\u5199\u6cd5\u66f4\u52a0\u7b80\u6d01\u3002torch\u7248\u672c\u7684\u591a\u5174\u8da3\u62bd\u53d6\u5c42SA\u4ee3\u7801\u5982\u4e0b\u3002<\/li>\n<li>\u6700\u540e\u5bf9\u7b2c\u4e00\u4e2a\u7ef4\u5ea6<code>seq<\/code>\u8fdb\u884c<code>sofjtmax<\/code>\u5f52\u4e00\u5316\uff0c\u5373\u4f7f\u5f97<code>seq<\/code>\u7ef4\u5ea6\u7684\u5143\u7d20\u4e4b\u548c\u4e3a1\u3002\u56e0\u4e3a\u5bf9\u4e8e\u56fa\u5b9a\u5e8f\u5217\u4e2d\u4e0d\u5b58\u5728\u7684\u4f4d\u7f6e\u8d1f\u65e0\u7a77mask\u6389\u4e86\uff0c\u6240\u4ee5\u7f3a\u5931\u7684\u4f4d\u7f6e\u7ecf\u8fc7<code>softmax<\/code>\u5f52\u4e00\u5316\u540e\u7684\u6570\u503c\u4e3a0\u3002<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> Softmax \u2061 ( x i ) = exp \u2061 ( x i ) \u2211 j exp \u2061 ( x j ) \\operatorname{Softmax}\\left(x_i\\right)=\\frac{\\exp \\left(x_i\\right)}{\\sum_j \\exp \\left(x_j\\right)} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">Softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.5488em; vertical-align: -1.1218em;\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.427em;\"><span class=\"\" style=\"top: -2.314em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative; top: 0em;\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em;\"><span class=\"\" style=\"top: -2.4003em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em;\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\">exp<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em;\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em;\"><\/span><\/span><span class=\"\" style=\"top: -3.677em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mop\">exp<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em;\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em;\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1218em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/li>\n<\/ul>\n<pre class=\"language-java prettyprint linenums\"><code class=\"prism language-python\"><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">MultiInterest_SA<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> embedding_dim<span class=\"token punctuation\">,<\/span> interest_num<span class=\"token punctuation\">,<\/span> d<span class=\"token operator\">=<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span>MultiInterest_SA<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>\n        self<span class=\"token punctuation\">.<\/span>embedding_dim <span class=\"token operator\">=<\/span> embedding_dim\n        self<span class=\"token punctuation\">.<\/span>interest_num <span class=\"token operator\">=<\/span> interest_num\n        <span class=\"token keyword\">if<\/span> d <span class=\"token operator\">==<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">:<\/span>\n            self<span class=\"token punctuation\">.<\/span>d <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>embedding_dim<span class=\"token operator\">*<\/span><span class=\"token number\">4<\/span>\n\n        <span class=\"token comment\"># self.W1 = self.create_parameter(shape=[self.embedding_dim, self.d])<\/span>\n        <span class=\"token comment\"># self.W2 = self.create_parameter(shape=[self.d, self.interest_num])<\/span>\n        self<span class=\"token punctuation\">.<\/span>W1 <span class=\"token operator\">=<\/span> Parameter<span class=\"token punctuation\">(<\/span>torch<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>embedding_dim<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>d<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>\n        self<span class=\"token punctuation\">.<\/span>W2 <span class=\"token operator\">=<\/span> Parameter<span class=\"token punctuation\">(<\/span>torch<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>d<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>interest_num<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>\n\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> seq_emb<span class=\"token punctuation\">,<\/span> mask <span class=\"token operator\">=<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n        <span class=\"token triple-quoted-string string\">''' seq_emb : batch,seq,emb mask : batch,seq,1 '''<\/span>\n        H <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">'bse, ed -&gt; bsd'<\/span><span class=\"token punctuation\">,<\/span> seq_emb<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>W1<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>tanh<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>\n        mask <span class=\"token operator\">=<\/span> mask<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># (batch, seq) -&gt; (batch, seq, 1)<\/span>\n        <span class=\"token comment\"># einsum \u7231\u56e0\u65af\u5766\u6c42\u548c\u7ea6\u5b9a<\/span>\n        <span class=\"token comment\"># (batch_size, seq_len, interest_num) - (batch_size, seq_len, 1)\u8d1f\u65e0\u7a77,<\/span>\n        <span class=\"token comment\"># \u4e24\u4e2a\u7ef4\u5ea6\u4e0d\u540c\u53ef\u4ee5\u505a\u51cf\u6cd5, \u6bd4\u5982[256, 20, 4]-[256, 20, 1]\uff0c\u540e\u8005\u4f1a\u51cf\u57284\u4e2a \u5168\u90e8\u4e4b\u4e0a<\/span>\n        A <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">'bsd, dk -&gt; bsk'<\/span><span class=\"token punctuation\">,<\/span> H<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>W2<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">+<\/span> <span class=\"token operator\">-<\/span><span class=\"token number\">1.e9<\/span> <span class=\"token operator\">*<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span> <span class=\"token operator\">-<\/span> mask<span class=\"token punctuation\">)<\/span>\n        A <span class=\"token operator\">=<\/span> F<span class=\"token punctuation\">.<\/span>softmax<span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">,<\/span> dim<span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>\n        A <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>\n        multi_interest_emb <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">,<\/span> seq_emb<span class=\"token punctuation\">)<\/span>\n        <span class=\"token keyword\">return<\/span> multi_interest_emb\n<\/code><\/pre>\n<h3>3.2 einsum\u51fd\u6570\u7684\u7528\u6cd5<\/h3>\n<p><code>einsum<\/code>\uff08Einstein summation convention\uff0c\u5373\u7231\u56e0\u65af\u5766\u6c42\u548c\u7ea6\u5b9a\uff09\u7684\u7528\u6cd5\uff1a<\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> c i k = \u2211 j a i j b j k c_{i k}=\\sum_j a_{i j} b_{j k} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">c<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em;\">ik<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1858em; vertical-align: -0.4358em;\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative; top: 0em;\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em;\"><span class=\"\" style=\"top: -2.4003em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em;\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em;\">ij<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">b<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em;\">jk<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u7684\u5199\u6cd5\u5982\u4e0b\uff1a<\/li>\n<\/ul>\n<pre class=\"language-java prettyprint linenums\"><code class=\"prism language-python\">c <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>dot<span class=\"token punctuation\">(<\/span>a<span class=\"token punctuation\">,<\/span> b<span class=\"token punctuation\">)<\/span>                 <span class=\"token comment\"># \u5e38\u89c4<\/span>\nc <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">'ij,jk-&gt;ik'<\/span><span class=\"token punctuation\">,<\/span> a<span class=\"token punctuation\">,<\/span> b<span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># einsum<\/span>\n<\/code><\/pre>\n<ul>\n<li>\u518d\u6bd4\u5982 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> c k l = \u2211 i \u2211 j a i j k b i j l c_{\\mathrm{kl}}=\\sum_{\\mathrm{i}} \\sum_{\\mathrm{j}} \\mathrm{a}_{\\mathrm{ijk}} \\mathrm{b}_{\\mathrm{ijl}} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">c<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">kl<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1858em; vertical-align: -0.4358em;\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative; top: 0em;\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1678em;\"><span class=\"\" style=\"top: -2.4003em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2997em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative; top: 0em;\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1678em;\"><span class=\"\" style=\"top: -2.4003em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\"><span class=\"mord mathrm\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">ijk<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathrm\">b<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathrm mtight\">ijl<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \uff1a <code>c = np.einsum('ijk,jkl-&gt;kl', a, b)<\/code><\/li>\n<\/ul>\n<pre class=\"language-java prettyprint linenums\"><code class=\"prism language-python\"><span class=\"token keyword\">import<\/span> torch\n\n<span class=\"token comment\"># 1. \u5f20\u91cf\u8f6c\u7f6e<\/span>\nA <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span>\nB <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"ijk-&gt;ikj\"<\/span><span class=\"token punctuation\">,<\/span> A<span class=\"token punctuation\">)<\/span>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> B<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"======\"<\/span><span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># (3, 4, 5) ; (3, 5, 4)<\/span>\n\n<span class=\"token comment\"># 2. \u53d6\u5bf9\u89d2\u5143\u7d20<\/span>\nA <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span>\nB <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"ii-&gt;i\"<\/span><span class=\"token punctuation\">,<\/span> A<span class=\"token punctuation\">)<\/span>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> B<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"======\"<\/span><span class=\"token punctuation\">)<\/span>\n\n<span class=\"token comment\"># 3. \u6c42\u548c\u964d\u7ef4<\/span>\nA <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span>\nB <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"ij-&gt;i\"<\/span><span class=\"token punctuation\">,<\/span> A<span class=\"token punctuation\">)<\/span>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> B<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"======\"<\/span><span class=\"token punctuation\">)<\/span>\n\n<span class=\"token comment\"># 4. \u54c8\u8fbe\u739b\u79ef(\u4e24\u4e2a\u77e9\u9635\u7ef4\u5ea6\u76f8\u540c)<\/span>\nA <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">)<\/span>\nB <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">)<\/span>\nC <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"ij, ij-&gt;ij\"<\/span><span class=\"token punctuation\">,<\/span> A<span class=\"token punctuation\">,<\/span> B<span class=\"token punctuation\">)<\/span>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>A<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> B<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> C<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"\\n\"<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">\"======\"<\/span><span class=\"token punctuation\">)<\/span>\n\n<span class=\"token comment\"># 5. \u5411\u91cf\u5185\u79ef<\/span>\nA <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span>\nB <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span>\n<span class=\"token comment\">#C=torch.dot(A,B)<\/span>\nC <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"i,i-&gt;\"<\/span><span class=\"token punctuation\">,<\/span>A<span class=\"token punctuation\">,<\/span>B<span class=\"token punctuation\">)<\/span>\n\n<span class=\"token comment\"># 6. \u5411\u91cf\u5916\u79ef<\/span>\nA <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span>\nB <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span>\n<span class=\"token comment\">#C = torch.outer(A,B)<\/span>\nC <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"i,j-&gt;ij\"<\/span><span class=\"token punctuation\">,<\/span>A<span class=\"token punctuation\">,<\/span>B<span class=\"token punctuation\">)<\/span>\n\n<span class=\"token comment\"># 7. \u77e9\u9635\u4e58\u6cd5<\/span>\nA <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">)<\/span>\nB <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">6<\/span><span class=\"token punctuation\">)<\/span>\n<span class=\"token comment\">#C = torch.matmul(A,B)<\/span>\nC <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"ik,kj-&gt;ij\"<\/span><span class=\"token punctuation\">,<\/span>A<span class=\"token punctuation\">,<\/span>B<span class=\"token punctuation\">)<\/span>\n\n<span class=\"token comment\"># 8. \u5f20\u91cf\u7f29\u5e76<\/span>\nA <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span>\nB <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">6<\/span><span class=\"token punctuation\">)<\/span>\n<span class=\"token comment\">#C = torch.tensordot(A,B,dims=[(0,1),(1,0)])<\/span>\nC <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"ijk,jih-&gt;kh\"<\/span><span class=\"token punctuation\">,<\/span>A<span class=\"token punctuation\">,<\/span>B<span class=\"token punctuation\">)<\/span>\n\n<span class=\"token comment\"># 9. batch\u77e9\u9635\u4e58\u6cd5<\/span>\nbatch_tensor_1 <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>arange<span class=\"token punctuation\">(<\/span><span class=\"token number\">2<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">4<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">)<\/span>\nbatch_tensor_2 <span class=\"token operator\">=<\/span> torch<span class=\"token punctuation\">.<\/span>arange<span class=\"token punctuation\">(<\/span><span class=\"token number\">2<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">3<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">)<\/span> \ntorch<span class=\"token punctuation\">.<\/span>bmm<span class=\"token punctuation\">(<\/span>batch_tensor_1<span class=\"token punctuation\">,<\/span> batch_tensor_2<span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># [2, 4, 4]<\/span>\ntorch<span class=\"token punctuation\">.<\/span>einsum<span class=\"token punctuation\">(<\/span><span class=\"token string\">\"bij, bjk -&gt; bik\"<\/span><span class=\"token punctuation\">,<\/span> batch_tensor_1<span class=\"token punctuation\">,<\/span> batch_tensor_2<span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># [2, 4, 4]<\/span>\n<\/code><\/pre>\n<h3>3.3 Comirec-SA\u7c7b\u5b9a\u4e49<\/h3>\n<p>\u8fd9\u90e8\u5206\u548c\u4e4b\u524dDR\u7684\u7c7b\u65f6\u4e00\u6837\u7684\uff0c\u6539\u52a8\u7684\u90e8\u5206\u90fd\u4f53\u73b0\u5728<code>MultiInterest_SA<\/code>\u4e0a\u4e86\u3002<\/p>\n<pre><code class=\"prism language-python\"><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">ComirecSA<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Layer<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> config<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span>ComirecSA<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>\n\n        self<span class=\"token punctuation\">.<\/span>config <span class=\"token operator\">=<\/span> config\n        self<span class=\"token punctuation\">.<\/span>embedding_dim <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>config<span class=\"token punctuation\">[<\/span><span class=\"token string\">'embedding_dim'<\/span><span class=\"token punctuation\">]<\/span>\n        self<span class=\"token punctuation\">.<\/span>max_length <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>config<span class=\"token punctuation\">[<\/span><span class=\"token string\">'max_length'<\/span><span class=\"token punctuation\">]<\/span>\n        self<span class=\"token punctuation\">.<\/span>n_items <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>config<span class=\"token punctuation\">[<\/span><span class=\"token string\">'n_items'<\/span><span class=\"token punctuation\">]<\/span>\n\n        self<span class=\"token punctuation\">.<\/span>item_emb <span class=\"token operator\">=<\/span> nn<span class=\"token punctuation\">.<\/span>Embedding<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>n_items<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>embedding_dim<span class=\"token punctuation\">,<\/span> padding_idx<span class=\"token operator\">=<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span>\n        self<span class=\"token punctuation\">.<\/span>multi_interest_layer <span class=\"token operator\">=<\/span> MultiInterest_SA<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>embedding_dim<span class=\"token punctuation\">,<\/span>interest_num<span class=\"token operator\">=<\/span>self<span class=\"token punctuation\">.<\/span>config<span class=\"token punctuation\">[<\/span><span class=\"token string\">'K'<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span>\n        self<span class=\"token punctuation\">.<\/span>loss_fun <span class=\"token operator\">=<\/span> nn<span class=\"token punctuation\">.<\/span>CrossEntropyLoss<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>\n        self<span class=\"token punctuation\">.<\/span>reset_parameters<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>\n\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">calculate_loss<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span>user_emb<span class=\"token punctuation\">,<\/span>pos_item<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n        all_items <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>item_emb<span class=\"token punctuation\">.<\/span>weight\n        scores <span class=\"token operator\">=<\/span> paddle<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>user_emb<span class=\"token punctuation\">,<\/span> all_items<span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>loss_fun<span class=\"token punctuation\">(<\/span>scores<span class=\"token punctuation\">,<\/span>pos_item<span class=\"token punctuation\">)<\/span>\n\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">output_items<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>item_emb<span class=\"token punctuation\">.<\/span>weight\n\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">reset_parameters<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> initializer<span class=\"token operator\">=<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n        <span class=\"token keyword\">for<\/span> weight <span class=\"token keyword\">in<\/span> self<span class=\"token punctuation\">.<\/span>parameters<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n            paddle<span class=\"token punctuation\">.<\/span>nn<span class=\"token punctuation\">.<\/span>initializer<span class=\"token punctuation\">.<\/span>KaimingNormal<span class=\"token punctuation\">(<\/span>weight<span class=\"token punctuation\">)<\/span>\n\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> item_seq<span class=\"token punctuation\">,<\/span> mask<span class=\"token punctuation\">,<\/span> item<span class=\"token punctuation\">,<\/span> train<span class=\"token operator\">=<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n\n        <span class=\"token keyword\">if<\/span> train<span class=\"token punctuation\">:<\/span>\n            seq_emb <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>item_emb<span class=\"token punctuation\">(<\/span>item_seq<span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># Batch,Seq,Emb<\/span>\n            item_e <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>item_emb<span class=\"token punctuation\">(<\/span>item<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>squeeze<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>\n\n            multi_interest_emb <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>multi_interest_layer<span class=\"token punctuation\">(<\/span>seq_emb<span class=\"token punctuation\">,<\/span> mask<span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># Batch,K,Emb<\/span>\n\n            cos_res <span class=\"token operator\">=<\/span> paddle<span class=\"token punctuation\">.<\/span>bmm<span class=\"token punctuation\">(<\/span>multi_interest_emb<span class=\"token punctuation\">,<\/span> item_e<span class=\"token punctuation\">.<\/span>squeeze<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">-<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>\n            k_index <span class=\"token operator\">=<\/span> paddle<span class=\"token punctuation\">.<\/span>argmax<span class=\"token punctuation\">(<\/span>cos_res<span class=\"token punctuation\">,<\/span> axis<span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>\n\n            best_interest_emb <span class=\"token operator\">=<\/span> paddle<span class=\"token punctuation\">.<\/span>rand<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>multi_interest_emb<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> multi_interest_emb<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>\n            <span class=\"token keyword\">for<\/span> k <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>multi_interest_emb<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>\n                best_interest_emb<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">=<\/span> multi_interest_emb<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">,<\/span> k_index<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span>\n\n            loss <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>calculate_loss<span class=\"token punctuation\">(<\/span>best_interest_emb<span class=\"token punctuation\">,<\/span>item<span class=\"token punctuation\">)<\/span>\n            output_dict <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">{ \n   <!-- --><\/span>\n                <span class=\"token string\">'user_emb'<\/span><span class=\"token punctuation\">:<\/span> multi_interest_emb<span class=\"token punctuation\">,<\/span>\n                <span class=\"token string\">'loss'<\/span><span class=\"token punctuation\">:<\/span> loss<span class=\"token punctuation\">,<\/span>\n            <span class=\"token punctuation\">}<\/span>\n        <span class=\"token keyword\">else<\/span><span class=\"token punctuation\">:<\/span>\n            seq_emb <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>item_emb<span class=\"token punctuation\">(<\/span>item_seq<span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># Batch,Seq,Emb<\/span>\n            multi_interest_emb <span class=\"token operator\">=<\/span> self<span class=\"token punctuation\">.<\/span>multi_interest_layer<span class=\"token punctuation\">(<\/span>seq_emb<span class=\"token punctuation\">,<\/span> mask<span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># Batch,K,Emb<\/span>\n            output_dict <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">{ \n   <!-- --><\/span>\n                <span class=\"token string\">'user_emb'<\/span><span class=\"token punctuation\">:<\/span> multi_interest_emb<span class=\"token punctuation\">,<\/span>\n            <span class=\"token punctuation\">}<\/span>\n        <span class=\"token keyword\">return<\/span> output_dict\n<\/code><\/pre>\n<h2>\u65f6\u95f4\u5b89\u6392<\/h2>\n<table>\n<thead>\n<tr>\n<th>\u4efb\u52a1\u4fe1\u606f<\/th>\n<th>\u622a\u6b62\u65f6\u95f4<\/th>\n<th>\u5b8c\u6210\u60c5\u51b5<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>11\u670814\u65e5\u5468\u4e00\u6b63\u5f0f\u5f00\u59cb<\/td>\n<td><\/td>\n<td><\/td>\n<\/tr>\n<tr>\n<td>Task01\uff1aPaddle\u5f00\u53d1\u6df1\u5ea6\u5b66\u4e60\u6a21\u578b\u5feb\u901f\u5165\u95e8<\/td>\n<td>11\u670814\u300115\u300116\u65e5\u5468\u4e09<\/td>\n<td>\u5b8c\u6210<\/td>\n<\/tr>\n<tr>\n<td>Task02\uff1a\u4f20\u7edf\u5e8f\u5217\u53ec\u56de\u5b9e\u8df5\uff1aGRU4Rec<\/td>\n<td>11\u670817\u300118\u300119\u65e5\u5468\u516d<\/td>\n<td>\u5b8c\u6210<\/td>\n<\/tr>\n<tr>\n<td>Task03\uff1aGNN\u5728\u53ec\u56de\u4e2d\u7684\u5e94\u7528\uff1aSR-GNN<\/td>\n<td>11\u670820\u300121\u300122\u65e5\u5468\u4e8c<\/td>\n<td>\u5b8c\u6210<\/td>\n<\/tr>\n<tr>\n<td>Task04\uff1a\u591a\u5174\u8da3\u53ec\u56de\u5b9e\u8df5\uff1aMIND<\/td>\n<td>11\u670823\u300124\u300125\u300126\u65e5\u5468\u516d<\/td>\n<td>\u5b8c\u6210<\/td>\n<\/tr>\n<tr>\n<td>Task05\uff1a\u591a\u5174\u8da3\u53ec\u56de\u5b9e\u8df5\uff1aComirec-DR<\/td>\n<td>11\u670827\u300128\u65e5\u5468\u4e00<\/td>\n<td>\u5b8c\u6210<\/td>\n<\/tr>\n<tr>\n<td>Task06\uff1a\u591a\u5174\u8da3\u53ec\u56de\u5b9e\u8df5\uff1aComirec-SA<\/td>\n<td>11\u670829\u65e5\u5468\u4e8c<\/td>\n<td>\u5b8c\u6210<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h2>Reference<\/h2>\n<p>[1] \u591a\u5174\u8da3\u53ec\u56de\u5b9e\u8df5\uff1aComirec-SA<br \/> \u8bba\u6587\uff1aControllable Multi-Interest Framework for Recommendation <br \/> \u94fe\u63a5\uff1ahttps:\/\/arxiv.org\/abs\/2005.09347<br \/> [2] \u63a8\u8350\u573a\u666f\u4e2d\u53ec\u56de\u6a21\u578b\u7684\u6f14\u5316\u8fc7\u7a0b. \u4eac\u4e1c\u5927\u4f6c<br \/> [3] \u539f\u8bba\u6587\u4f5c\u8005\u4ee3\u7801\uff1ahttps:\/\/github.com\/THUDM\/ComiRec\/blob\/a576eed8b605a531f2971136ce6ae87739d47693\/src\/train.py<br \/> [4] https:\/\/github.com\/ShiningCosmos\/pytorch_ComiRec\/blob\/main\/ComiRec.py<br \/> [5] https:\/\/wangxiaocs.github.io\/<br \/> [6] \u63a8\u8350\u7b97\u6cd5\u70bc\u4e39\u7b14\u8bb0\uff1a\u963f\u91cc\u5e8f\u5217\u5316\u63a8\u8350\u7b97\u6cd5ComiRec<br \/> [7] einsum is all you needed<br \/> [8] Understanding PyTorch einsum<br \/> [9] \u77e9\u9635\u64cd\u4f5c\u4e07\u80fd\u51fd\u6570 einsum \u8be6\u7ec6\u89e3\u6790\uff08\u901a\u6cd5\u6559\u4f60\u5982\u4f55\u770b\u61c2\u5e76\u5199\u51faeinsum\u8868\u8fbe\u5f0f\uff09<br \/> [10] Sparse-Interest Network for Sequential Recommendation .AAAI 2021<br \/> [11] torch_rechub\u4ee3\u7801\u590d\u73b0\uff1ahttps:\/\/github.com\/datawhalechina\/torch-rechub\/blob\/main\/torch_rechub\/models\/matching\/comirec.py<\/p>\n<p>\u5355\u5934\u6ce8\u610f\u529b\u673a\u5236\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> A t t e n t i o n ( Q , K , V ) = Q K T d k \u2217 V Attention(Q,K,V)=\\frac{QK^T}{\\sqrt{d_k}}*V <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mord mathnormal\">tt<\/span><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mord mathnormal\">o<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4483em; vertical-align: -0.93em;\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em;\"><span class=\"\" style=\"top: -2.2528em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em;\"><span class=\"svg-align\" style=\"top: -3em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em;\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em;\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em; height: 1.08em;\"><br \/>\n                   <svg width=\"400em\" height=\"1.08em\" viewbox=\"0 0 400000 1080\" preserveaspectratio=\"xMinYMin slice\"> \n                    <path d=\"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z\"><\/path> \n                   <\/svg><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em;\"><\/span><\/span><span class=\"\" style=\"top: -3.677em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em;\"><span class=\"\" style=\"top: -3.063em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em;\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u2217<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u591a\u5934\u6ce8\u610f\u529b\u673a\u5236\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> M u l t i H e a d ( Q , K , V ) = C o n c a t ( h e a d 1 , . . . , h e a d h ) \u2217 W O MultiHead(Q,K,V)=Concat(head_1,...,head_h)*W^O <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.109em;\">M<\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mord mathnormal\">lt<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mord mathnormal\">He<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mord mathnormal\">d<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">C<\/span><span class=\"mord mathnormal\">o<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord mathnormal\">c<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">h<\/span><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord\">...<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\">h<\/span><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">h<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u2217<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8913em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em;\"><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0278em;\">O<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u5176\u4e2d\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> h e a d i = A t t e n t i o n ( Q \u2217 W i Q , K \u2217 W i K , V \u2217 W i V ) head_i=Attention(Q*W_i^Q,K*W_i^K,V*W_i^V) <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em; vertical-align: -0.15em;\"><\/span><span class=\"mord mathnormal\">h<\/span><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em;\"><span class=\"\" style=\"top: -2.55em; margin-left: 0em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mord mathnormal\">A<\/span><span class=\"mord mathnormal\">tt<\/span><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mord mathnormal\">o<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u2217<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.2361em; vertical-align: -0.2769em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9592em;\"><span class=\"\" style=\"top: -2.4231em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><span class=\"\" style=\"top: -3.1809em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">Q<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2769em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em;\">K<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u2217<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1383em; vertical-align: -0.247em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em;\">K<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em;\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em;\">V<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><span class=\"mbin\">\u2217<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1413em; vertical-align: -0.25em;\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em;\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em;\"><span class=\"\" style=\"top: -2.453em; margin-left: -0.1389em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.2222em;\">V<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.247em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n","protected":false},"excerpt":{"rendered":"\u5e8f\u5217\u63a8\u8350_\u7b2c\u4e00\u5e8f\u5217\u6392\u540d-Comirec-SA\u57fa\u4e8eattention\u7684\u591a\u5174\u8da3\u5efa\u6a21\uff0c\u8bba\u6587\u4e2d\u5148\u901a\u8fc7attention\u63d0\u53d6\u5355\u4e00\u5174\u8da3\uff0c\u518d\u63a8\u5e7f\u5230\u591a\u5174\u8da3\u5efa\u6a21","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[],"tags":[],"_links":{"self":[{"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/posts\/8644"}],"collection":[{"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/comments?post=8644"}],"version-history":[{"count":0,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/posts\/8644\/revisions"}],"wp:attachment":[{"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/media?parent=8644"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/categories?post=8644"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/tags?post=8644"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}