{"id":5094,"date":"2024-09-29T15:01:03","date_gmt":"2024-09-29T07:01:03","guid":{"rendered":""},"modified":"2024-09-29T15:01:03","modified_gmt":"2024-09-29T07:01:03","slug":"\u52a8\u91cf\u6a21\u578b\u6838\u5fc3\u6559\u7a0b_\u52a8\u91cf\u5b88\u6052\u5341\u5927\u6a21\u578b","status":"publish","type":"post","link":"https:\/\/mushiming.com\/5094.html","title":{"rendered":"\u52a8\u91cf\u6a21\u578b\u6838\u5fc3\u6559\u7a0b_\u52a8\u91cf\u5b88\u6052\u5341\u5927\u6a21\u578b"},"content":{"rendered":"<p><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" style=\"display: none;\"> \n <path stroke-linecap=\"round\" d=\"M5,0 0,2.5 5,5z\" id=\"raphael-marker-block\" style=\"-webkit-tap-highlight-color: rgba(0, 0, 0, 0);\"><\/path> \n<\/svg> <\/p>\n<p>\u52a8\u91cf\u68af\u5ea6\u4e0b\u964d\uff08Gradient Descent With Momentum\uff09\uff0c\u7b80\u79f0\u4e3a\u52a8\u91cf\u65b9\u6cd5\uff08Momentum\uff09\uff0c\u8fd0\u884c\u901f\u5ea6\u51e0\u4e4e\u603b\u662f\u5feb\u4e8e\u6807\u51c6\u7684\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\uff0c\u5e76\u4e14\u80fd\u591f\u89e3\u51b3\u968f\u673a\u68af\u5ea6\u4e0b\u964d\u6240\u9047\u5230\u7684\u5c71\u8c37\u9707\u8361\u4ee5\u53ca\u978d\u90e8\u505c\u6ede\u95ee\u9898\uff0c\u8fd9\u90e8\u5206\u5185\u5bb9\u8bf7\u9605\u8bfb\u4e0a\u4e00\u7bc7\u535a\u5ba2\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u3002<\/p>\n<p>\u6839\u636e\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u7684\u53c2\u6570\u66f4\u65b0\u516c\u5f0f\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> w = w \u2212 \u03b7 \u2202 L ( w ) \u2202 w w = w - \\eta\\frac{\\partial L(w)}{\\partial w} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.43056em; vertical-align: 0em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.66666em; vertical-align: -0.08333em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.113em; vertical-align: -0.686em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">\u03b7<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.427em;\"><span class=\"\" style=\"top: -2.314em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\" style=\"margin-right: 0.05556em;\">\u2202<\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em;\"><\/span><\/span><span class=\"\" style=\"top: -3.677em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\" style=\"margin-right: 0.05556em;\">\u2202<\/span><span class=\"mord mathdefault\">L<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u53c2\u6570\u7684\u66f4\u65b0\u4ec5\u4ec5\u53d6\u51b3\u4e8e\u5f53\u524d\u4f4d\u7f6e\u7684\u68af\u5ea6\u4ee5\u53ca\u6b65\u957f\uff0c\u8bd5\u60f3\u4e00\u4e0b\u8fd9\u6837\u4e00\u4e2a\u60c5\u5883\uff1a\u6211\u4eec\u5c06\u67d0\u4e00\u4e2a\u7269\u54c1\u5f80\u5c71\u8c37\u91cc\u4e22\uff0c\u5728\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u7684\u89c4\u5219\u4e0b\uff0c\u8be5\u7269\u54c1\u4ec5\u4ec5\u6536\u5230\u5f53\u524d\u89e6\u78b0\u5728\u5b83\u8eab\u4e0a\u7684\u529b\uff0c\u800c\u5ffd\u7565\u5176\u4ed6\u7684\u529b\uff0c\u4f8b\u5982\u91cd\u529b\u3001\u7a7a\u6c14\u963b\u529b\u7b49\u7b49\u3002\u6211\u4eec\u53ef\u4ee5\u628a\u5b83\u60f3\u8c61\u6210\u4e00\u5f20\u7eb8\u56e2\u3002<\/p>\n<ul>\n<li>\u5c71\u8c37\uff1a\u5728\u5c71\u8c37\u4e2d\u4e0d\u53ef\u907f\u514d\u5730\u649e\u5728\u5c71\u58c1\uff0c\u7531\u4e8e\u8d28\u91cf\u5c0f\u53d7\u5c71\u58c1\u5f39\u529b\u7684\u5e72\u6270\u5927\uff0c\u4ece\u4e00\u4fa7\u5c71\u58c1\u53cd\u5f39\u56de\u6765\u649e\u5411\u53e6\u4e00\u4fa7\u5c71\u58c1\uff0c\u7ed3\u679c\u6765\u56de\u9707\u8361\u5730\u6eda\u4e0b\u3002<\/li>\n<li>\u978d\u90e8\uff1a\u7eb8\u56e2\u6765\u5230\u978d\u90e8\u65f6\uff0c\u7531\u4e8e\u8d28\u91cf\u5c0f\uff0c\u901f\u5ea6\u5f88\u5feb\u51cf\u4e3a\u96f6\uff0c\u65e0\u6cd5\u51b2\u51fa\u978d\u90e8\u533a\u57df\u3002<\/li>\n<\/ul>\n<p>\u5982\u679c\u6b64\u65f6\uff0c\u8be5\u7269\u54c1\u62e5\u6709\u4e86\u5927\u8d28\u91cf\uff0c\u4f8b\u5982\u662f\u4e00\u4e2a\u94c1\u7403\uff0c<\/p>\n<ul>\n<li>\u5c71\u8c37\uff1a\u90a3\u4e48\u5b83\u6cbf\u7740\u5c71\u8c37\u6eda\u4e0b\u65f6\uff0c\u5c31\u4e0d\u5bb9\u6613\u53d7\u5230\u9014\u4e2d\u65c1\u529b\u7684\u5e72\u6270\uff0c\u8f68\u8ff9\u4f1a\u66f4\u7a33\u66f4\u76f4\u3002<\/li>\n<li>\u978d\u90e8\uff1a\u5728\u60ef\u6027\u7684\u4f5c\u7528\u4e0b\u7ee7\u7eed\u6eda\u52a8\uff0c\u4ece\u800c\u6709\u673a\u4f1a\u51b2\u51fa\u978d\u90e8\u3002<\/li>\n<\/ul>\n<p>\u5728\u4e2d\u5b66\u7269\u7406\u4e2d\uff0c\u523b\u753b\u60ef\u6027\u7684\u7269\u7406\u91cf\u662f\u52a8\u91cf\uff0c\u8fd9\u4e5f\u662f\u8be5\u7b97\u6cd5\u540d\u5b57\u7684\u7531\u6765\u3002\u6cbf\u5c71\u8c37\u6eda\u4e0b\u7684\u94c1\u7403\u4f1a\u6536\u5230\u6cbf\u5761\u9053\u5411\u4e0b\u7684\u529b\u548c\u4e0e\u5de6\u53f3\u5c71\u58c1\u78b0\u649e\u7684\u5f39\u529b\u3002\u5411\u4e0b\u7684\u529b\uff08\u91cd\u529b\uff09\u7a33\u5b9a\u4e0d\u53d8\uff0c\u4ea7\u751f\u7684\u52a8\u91cf\u4e0d\u65ad\u7d2f\u79ef\uff0c\u901f\u5ea6\u8d8a\u6765\u8d8a\u5feb\uff1b\u5de6\u53f3\u7684\u5f39\u529b\u603b\u662f\u5728\u4e0d\u505c\u5207\u6362\uff0c\u52a8\u91cf\u7d2f\u79ef\u7684\u7ed3\u679c\u662f\u76f8\u4e92\u62b5\u6d88\uff0c\u51cf\u5f31\u4e86\u7403\u7684\u6765\u56de\u9707\u8361\u3002\u56e0\u6b64\uff0c\u4e0e\u968f\u673a\u68af\u5ea6\u4e0b\u964d\u76f8\u6bd4\uff0c\u52a8\u91cf\u65b9\u6cd5\u7684\u6536\u655b\u901f\u5ea6\u66f4\u5feb\uff0c\u6536\u655b\u66f2\u7ebf\u4e5f\u66f4\u7a33\u5b9a\uff0c\u89c1\u4e0b\u56fe\u3002<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/img-blog.csdnimg.cn\/img_convert\/5c332f0a8cc8b612385119b3aa29123e.png\" =\"\" =\"\u968f\u673a\u68af\u5ea6\u4e0b\u964d\u4e0e\u52a8\u91cf\u65b9\u6cd5\u6bd4\u8f83.png\" alt=\"\u52a8\u91cf\u6a21\u578b\u6838\u5fc3\u6559\u7a0b_\u52a8\u91cf\u5b88\u6052\u5341\u5927\u6a21\u578b (https:\/\/mushiming.com\/)  \u7b2c1\u5f20\" title=\"\u52a8\u91cf\u6a21\u578b\u6838\u5fc3\u6559\u7a0b_\u52a8\u91cf\u5b88\u6052\u5341\u5927\u6a21\u578b  \u7b2c1\u5f20-\u7a46\u4e16\u660e\u535a\u5ba2\" ><\/p>\n<p>\u76f8\u6bd4\u6807\u51c6\u7684\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\uff0c\u52a8\u91cf\u68af\u5ea6\u4e0b\u964d\u662f\u7b97\u6cd5\u5c06\u52a8\u91cf\u7eb3\u5165\u4e86\u53c2\u6570\u66f4\u65b0\u516c\u5f0f\u4e2d\u3002<\/p>\n<p>\u3010\u8ba1\u7b97\u516c\u5f0f\u3011\uff1a<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> v t = \u03b3 v t \u2212 1 + \u03b7 \u2202 L ( w ) \u2202 w w = w \u2212 v t v_t = \\gamma v_{t-1} + \\eta \\frac{\\partial L(w)}{\\partial w} \\quad w = w - v_t <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.58056em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.280556em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.03588em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathdefault mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.791661em; vertical-align: -0.208331em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.05556em;\">\u03b3<\/span><span class=\"mord\"><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.301108em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.03588em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathdefault mtight\">t<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.208331em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><span class=\"mbin\">+<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.113em; vertical-align: -0.686em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">\u03b7<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.427em;\"><span class=\"\" style=\"top: -2.314em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\" style=\"margin-right: 0.05556em;\">\u2202<\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em;\"><\/span><\/span><span class=\"\" style=\"top: -3.677em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\" style=\"margin-right: 0.05556em;\">\u2202<\/span><span class=\"mord mathdefault\">L<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 1em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.66666em; vertical-align: -0.08333em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.58056em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.280556em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.03588em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathdefault mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u5176\u4e2d\uff0c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> \u03b3 \\gamma <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em; vertical-align: -0.19444em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.05556em;\">\u03b3<\/span><\/span><\/span><\/span><\/span> \u662f\u8870\u51cf\u7cfb\u6570\uff0c\u626e\u6f14\u963b\u529b\u7684\u4f5c\u7528\u3002\u524d\u8fdb\u6b65\u4f10 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> v t v_t <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.58056em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.280556em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.03588em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathdefault mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u7531\u4e24\u90e8\u5206\u7ec4\u6210\uff1a<\/p>\n<ul>\n<li>\u6807\u51c6\u7684\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u90e8\u5206\uff0c\u4e5f\u5c31\u662f\u6b65\u957f\u4e58\u4e0a\u5f53\u524d\u4f4d\u7f6e\u7684\u68af\u5ea6\uff1a<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> \u03b7 \u2202 L ( w ) \u2202 w \\eta \\frac{\\partial L(w)}{\\partial w} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.355em; vertical-align: -0.345em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">\u03b7<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.01em;\"><span class=\"\" style=\"top: -2.655em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\" style=\"margin-right: 0.05556em;\">\u2202<\/span><span class=\"mord mathdefault mtight\" style=\"margin-right: 0.02691em;\">w<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em;\"><\/span><\/span><span class=\"\" style=\"top: -3.485em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\" style=\"margin-right: 0.05556em;\">\u2202<\/span><span class=\"mord mathdefault mtight\">L<\/span><span class=\"mopen mtight\">(<\/span><span class=\"mord mathdefault mtight\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.345em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<li>\u5e26\u8870\u51cf\u7684\u524d\u4e00\u6b21\u6b65\u4f10 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"> v t \u2212 1 v_{t-1} <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.638891em; vertical-align: -0.208331em;\"><\/span><span class=\"mord\"><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.301108em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.03588em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathdefault mtight\">t<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.208331em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/li>\n<\/ul>\n<p>\u5728\u8be5\u516c\u5f0f\u4e2d\uff0c\u60ef\u6027\u5c31\u4f53\u73b0\u5728\u5bf9\u524d\u4e00\u6b21\u6b65\u4f10\u4fe1\u606f\u7684\u5229\u7528\u3002\u7c7b\u6bd4\u4e2d\u5b66\u7269\u7406\u77e5\u8bc6\uff0c\u5f53\u524d\u68af\u5ea6\u5c31\u597d\u6bd4\u5f53\u524d\u65f6\u523b\u53d7\u529b\u4ea7\u751f\u7684\u52a0\u901f\u5ea6\uff0c\u800c\u6b65\u957f\u5219\u662f\u65f6\u95f4\uff0c\u524d\u4e00\u6b21\u6b65\u4f10\u597d\u6bd4\u524d\u4e00\u65f6\u523b\u7684\u901f\u5ea6\u3002\u6807\u51c6\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u5728\u6bcf\u6b21\u884c\u52a8\u65f6\uff0c\u90fd\u5ffd\u7565\u524d\u4e00\u65f6\u523b\u7684\u901f\u5ea6\uff0c\u800c\u91cd\u65b0\u6839\u636e\u5f53\u524d\u65f6\u523b\u7684\u52a0\u901f\u5ea6\u548c\u65f6\u95f4\u6765\u884c\u8d70\uff0c\u56e0\u6b64\u5f53\u52a0\u901f\u5ea6\u8d8b\u4e8e\u96f6\u65f6\u5c31\u5f88\u96be\u7ee7\u7eed\u79fb\u52a8\u3002\u800c\u52a8\u91cf\u65b9\u6cd5\u5219\u8003\u8651\u524d\u4e00\u65f6\u523b\u901f\u5ea6\u548c\u5f53\u524d\u52a0\u901f\u5ea6\u7684\u5171\u540c\u4f5c\u7528\u3002<\/p>\n<p>\u4e0d\u540c\u7684\u6587\u732e\u5bf9\u4e8e\u52a8\u91cf\u65b9\u6cd5\u7684\u8ba1\u7b97\u516c\u5f0f\u4e5f\u7565\u6709\u4e0d\u540c\uff0c\u5434\u6069\u8fbe\u8001\u5e08\u63a8\u8350\u4f7f\u7528\u4e0b\u8ff0\u7684\u8ba1\u7b97\u516c\u5f0f\u3002<br \/> <span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"> v t = \u03b2 v t \u2212 1 + ( 1 \u2212 \u03b2 ) \u2202 L ( w ) \u2202 w w = w \u2212 \u03b7 v t v_t = \\beta v_{t-1} + (1 - \\beta)\\frac{\\partial L(w)}{\\partial w} \\\\ w = w - \\eta v_t <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.58056em; vertical-align: -0.15em;\"><\/span><span class=\"mord\"><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.280556em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.03588em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathdefault mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.902771em; vertical-align: -0.208331em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.05278em;\">\u03b2<\/span><span class=\"mord\"><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.301108em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.03588em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathdefault mtight\">t<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.208331em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><span class=\"mbin\">+<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em; vertical-align: -0.25em;\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">1<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.113em; vertical-align: -0.686em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.05278em;\">\u03b2<\/span><span class=\"mclose\">)<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.427em;\"><span class=\"\" style=\"top: -2.314em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\" style=\"margin-right: 0.05556em;\">\u2202<\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em;\"><\/span><\/span><span class=\"\" style=\"top: -3.677em;\"><span class=\"pstrut\" style=\"height: 3em;\"><\/span><span class=\"mord\"><span class=\"mord\" style=\"margin-right: 0.05556em;\">\u2202<\/span><span class=\"mord mathdefault\">L<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><span class=\"mspace newline\"><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.43056em; vertical-align: 0em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><span class=\"mrel\">=<\/span><span class=\"mspace\" style=\"margin-right: 0.277778em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.66666em; vertical-align: -0.08333em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.02691em;\">w<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.222222em;\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em; vertical-align: -0.19444em;\"><\/span><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">\u03b7<\/span><span class=\"mord\"><span class=\"mord mathdefault\" style=\"margin-right: 0.03588em;\">v<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.280556em;\"><span class=\"\" style=\"top: -2.55em; margin-left: -0.03588em; margin-right: 0.05em;\"><span class=\"pstrut\" style=\"height: 2.7em;\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathdefault mtight\">t<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em;\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/> \u81f3\u4e8e\u5b70\u4f18\u5b70\u52a3\u6682\u65e0\u5b9a\u8bba\uff0c\u5927\u5bb6\u53ef\u5404\u6309\u559c\u597d\u8fdb\u884c\u9009\u62e9\u3002<\/p>\n<p>\u52a8\u91cf\u65b9\u6cd5\u53ef\u4ee5\u5d4c\u5165\u5230\u6807\u51c6\u7684\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u4e2d\uff0c\u4f8b\u5982\u4f7f\u7528\u52a8\u91cf\u7684\u968f\u673a\u68af\u5ea6\u4e0b\u964d\u3001\u4f7f\u7528\u52a8\u91cf\u7684\u6279\u91cf\u68af\u5ea6\u4e0b\u964d\u7b49\u7b49\u3002\u65e0\u8bba\u5bf9\u54ea\u4e2a\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\uff0c\u52a0\u5165\u52a8\u91cf\u540e\u90fd\u53ef\u4ee5\u52a0\u5feb\u6536\u655b\u901f\u5ea6\u3002\u5bf9\u4e8e\u968f\u673a\u68af\u5ea6\u4e0b\u964d\u800c\u8a00\uff0c\u8fd8\u53ef\u4ee5\u5728\u4e00\u5b9a\u7a0b\u5ea6\u4e0a\u89e3\u51b3\u978d\u90e8\u505c\u6ede\u95ee\u9898\u3002<\/p>\n<p>\u3010\u4ee3\u7801\u5b9e\u73b0\u3011\uff1a<\/p>\n<pre class=\"language-java prettyprint linenums\"><code class=\"prism language-python\"><span class=\"token keyword\">def<\/span> <span class=\"token function\">BatchGradientDescentM<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> y<span class=\"token punctuation\">,<\/span> step<span class=\"token operator\">=<\/span><span class=\"token number\">0.001<\/span><span class=\"token punctuation\">,<\/span> iter_count<span class=\"token operator\">=<\/span><span class=\"token number\">500<\/span><span class=\"token punctuation\">,<\/span> beta<span class=\"token operator\">=<\/span><span class=\"token number\">0.9<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> length<span class=\"token punctuation\">,<\/span> features <span class=\"token operator\">=<\/span> x<span class=\"token punctuation\">.<\/span>shape <span class=\"token comment\"># \u521d\u59cb\u5316\u53c2\u6570\u548c\u52a8\u91cf\u4ee5\u53ca\u6574\u5408 x'<\/span> data <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>column_stack<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> np<span class=\"token punctuation\">.<\/span>ones<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>length<span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> w <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> v <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># \u5f00\u59cb\u8fed\u4ee3<\/span> <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>iter_count<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token comment\"># \u8ba1\u7b97\u52a8\u91cf<\/span> v <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">(<\/span>beta <span class=\"token operator\">*<\/span> v <span class=\"token operator\">+<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span> <span class=\"token operator\">-<\/span> beta<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> np<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>dot<span class=\"token punctuation\">(<\/span>data<span class=\"token punctuation\">,<\/span> w<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">-<\/span> y<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> data<span class=\"token punctuation\">,<\/span> axis<span class=\"token operator\">=<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> length <span class=\"token comment\"># \u66f4\u65b0\u53c2\u6570<\/span> w <span class=\"token operator\">-=<\/span> step <span class=\"token operator\">*<\/span> v <span class=\"token keyword\">return<\/span> w <\/code><\/pre>\n<p>\u540c\u6837\uff0c\u589e\u91cf\u65b9\u6cd5\u4e5f\u53ef\u4ee5\u5d4c\u5165\u5230\u5c0f\u6279\u91cf\u68af\u5ea6\u4e0b\u964d\u4ee5\u53ca\u968f\u673a\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u4e2d\uff0c\u5177\u4f53\u4ee3\u7801\u8bf7\u53c2\u8003 \u4f20\u9001\u95e8<\/p>\n<p>\u6211\u4eec\u4e5f\u53ef\u4ee5\u5c06\u8fd9\u4e9b\u7b97\u6cd5\u90fd\u6574\u5408\u5230\u4e00\u5757\uff0c\u901a\u8fc7 batch_size \u7684\u5927\u5c0f\u6765\u5224\u65ad\u662f\u6279\u91cf\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\uff0c\u8fd8\u662f\u968f\u673a\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u3002<\/p>\n<p>\u3010\u4ee3\u7801\u5b9e\u73b0\u3011\uff1a<\/p>\n<pre class=\"language-java prettyprint linenums\"><code class=\"prism language-python\"><span class=\"token keyword\">def<\/span> <span class=\"token function\">Momentum<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> y<span class=\"token punctuation\">,<\/span> step<span class=\"token operator\">=<\/span><span class=\"token number\">0.01<\/span><span class=\"token punctuation\">,<\/span> iter_count<span class=\"token operator\">=<\/span><span class=\"token number\">1000<\/span><span class=\"token punctuation\">,<\/span> batch_size<span class=\"token operator\">=<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> beta<span class=\"token operator\">=<\/span><span class=\"token number\">0.9<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> length<span class=\"token punctuation\">,<\/span> features <span class=\"token operator\">=<\/span> x<span class=\"token punctuation\">.<\/span>shape <span class=\"token comment\"># \u521d\u59cb\u5316\u53c2\u6570\u548c\u52a8\u91cf\u4ee5\u53ca\u6574\u5408 x'<\/span> data <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>column_stack<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> np<span class=\"token punctuation\">.<\/span>ones<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>length<span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> w <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> v <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> start<span class=\"token punctuation\">,<\/span> end <span class=\"token operator\">=<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> batch_size <span class=\"token comment\"># \u5f00\u59cb\u8fed\u4ee3<\/span> <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>iter_count<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> v <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">(<\/span>beta <span class=\"token operator\">*<\/span> v <span class=\"token operator\">+<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span> <span class=\"token operator\">-<\/span> beta<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> np<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>dot<span class=\"token punctuation\">(<\/span>data<span class=\"token punctuation\">[<\/span>start<span class=\"token punctuation\">:<\/span>end<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> w<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">-<\/span> y<span class=\"token punctuation\">[<\/span>start<span class=\"token punctuation\">:<\/span>end<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> data<span class=\"token punctuation\">[<\/span>start<span class=\"token punctuation\">:<\/span>end<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> axis<span class=\"token operator\">=<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> length w <span class=\"token operator\">-=<\/span> step <span class=\"token operator\">*<\/span> v start <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">(<\/span>start <span class=\"token operator\">+<\/span> batch_size<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">%<\/span> length <span class=\"token keyword\">if<\/span> start <span class=\"token operator\">&gt;<\/span> length<span class=\"token punctuation\">:<\/span> start <span class=\"token operator\">-=<\/span> length end <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">(<\/span>end <span class=\"token operator\">+<\/span> batch_size<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">%<\/span> length <span class=\"token keyword\">if<\/span> end <span class=\"token operator\">&gt;<\/span> length<span class=\"token punctuation\">:<\/span> end <span class=\"token operator\">-=<\/span> length <span class=\"token keyword\">return<\/span> w <span class=\"token comment\"># \u6279\u91cf\u68af\u5ea6\u4e0b\u964d<\/span> <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>Momentum<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> y<span class=\"token punctuation\">,<\/span> batch_size<span class=\"token operator\">=<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">-<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># \u8f93\u51fa\uff1a<\/span> array<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">5.00<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">0.<\/span> <span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># \u5c0f\u6279\u91cf\u68af\u5ea6\u4e0b\u964d<\/span> Momentum<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> y<span class=\"token punctuation\">,<\/span> batch_size<span class=\"token operator\">=<\/span><span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># \u8f93\u51fa\uff1a<\/span> array<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">4.<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">1.<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># \u968f\u673a\u68af\u5ea6\u4e0b\u964d<\/span> Momentum<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> y<span class=\"token punctuation\">,<\/span> batch_size<span class=\"token operator\">=<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># \u8f93\u51fa\uff1a<\/span> array<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">4.<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">0.<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <\/code><\/pre>\n<h3>\u725b\u987f\u52a8\u91cf<\/h3>\n<p>\u53d7 Nesterov \u52a0\u901f\u68af\u5ea6\u7b97\u6cd5\u542f\u53d1\uff0cSutskever \u63d0\u51fa\u52a8\u91cf\u65b9\u6cd5\u7684\u4e00\u4e2a\u53d8\u79cd\u3002\u4e0e Momentum \u4e0d\u540c\u7684\u662f\uff0cNesterov \u5148\u7528\u5f53\u524d\u7684\u901f\u5ea6\u66f4\u65b0\u53c2\u6570\uff0c\u518d\u7528\u66f4\u65b0\u540e\u7684\u4e34\u65f6\u53c2\u6570\u8ba1\u7b97\u68af\u5ea6\u3002<\/p>\n<p>\u3010\u4ee3\u7801\u5b9e\u73b0\u3011\uff1a<\/p>\n<pre class=\"language-java prettyprint linenums\"><code class=\"prism language-python\"><span class=\"token keyword\">def<\/span> <span class=\"token function\">Nesterov<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> y<span class=\"token punctuation\">,<\/span> step<span class=\"token operator\">=<\/span><span class=\"token number\">0.01<\/span><span class=\"token punctuation\">,<\/span> iter_count<span class=\"token operator\">=<\/span><span class=\"token number\">1000<\/span><span class=\"token punctuation\">,<\/span> batch_size<span class=\"token operator\">=<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> beta<span class=\"token operator\">=<\/span><span class=\"token number\">0.9<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> length<span class=\"token punctuation\">,<\/span> features <span class=\"token operator\">=<\/span> x<span class=\"token punctuation\">.<\/span>shape data <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>column_stack<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> np<span class=\"token punctuation\">.<\/span>ones<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>length<span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> w <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> v <span class=\"token operator\">=<\/span> np<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> start<span class=\"token punctuation\">,<\/span> end <span class=\"token operator\">=<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> batch_size <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>iter_count<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token comment\"># \u5148\u66f4\u65b0\u53c2\u6570<\/span> w_temp <span class=\"token operator\">=<\/span> w <span class=\"token operator\">-<\/span> step <span class=\"token operator\">*<\/span> v <span class=\"token comment\"># \u518d\u8ba1\u7b97\u68af\u5ea6\u4e0e\u901f\u5ea6<\/span> v <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">(<\/span>beta <span class=\"token operator\">*<\/span> v <span class=\"token operator\">+<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span> <span class=\"token operator\">-<\/span> beta<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> np<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>dot<span class=\"token punctuation\">(<\/span>data<span class=\"token punctuation\">[<\/span>start<span class=\"token punctuation\">:<\/span>end<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> w_temp<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">-<\/span> y<span class=\"token punctuation\">[<\/span>start<span class=\"token punctuation\">:<\/span>end<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> data<span class=\"token punctuation\">[<\/span>start<span class=\"token punctuation\">:<\/span>end<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> axis<span class=\"token operator\">=<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>features <span class=\"token operator\">+<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> length w <span class=\"token operator\">-=<\/span> step <span class=\"token operator\">*<\/span> v start <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">(<\/span>start <span class=\"token operator\">+<\/span> batch_size<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">%<\/span> length <span class=\"token keyword\">if<\/span> start <span class=\"token operator\">&gt;<\/span> length<span class=\"token punctuation\">:<\/span> start <span class=\"token operator\">-=<\/span> length end <span class=\"token operator\">=<\/span> <span class=\"token punctuation\">(<\/span>end <span class=\"token operator\">+<\/span> batch_size<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">%<\/span> length <span class=\"token keyword\">if<\/span> end <span class=\"token operator\">&gt;<\/span> length<span class=\"token punctuation\">:<\/span> end <span class=\"token operator\">-=<\/span> length <span class=\"token keyword\">return<\/span> w <\/code><\/pre>\n<p>\u725b\u987f\u589e\u91cf\u76f8\u5f53\u4e8e\u6dfb\u52a0\u4e86\u77eb\u6b63\u56e0\u5b50\u7684 Momentum \u65b9\u6cd5\uff0c\u5728\u6279\u91cf\u68af\u5ea6\u4e0b\u964d\u7b97\u6cd5\u4e2d\u80fd\u8fdb\u4e00\u6b65\u7f29\u5c0f\u8bef\u5dee\uff0c\u4f46\u5bf9\u4e8e\u968f\u673a\u68af\u5ea6\u4e0b\u964d\u800c\u8a00\uff0c\u725b\u987f\u589e\u91cf\u6ca1\u6709\u4efb\u4f55\u6539\u8fdb\u3002<\/p>\n<h3>\u53c2\u8003<\/h3>\n<ul>\n<li>\u5434\u6069\u8fbe\u8001\u5e08\u7684\u6df1\u5ea6\u5b66\u4e60\u8bfe\u7a0b<\/li>\n<li>\u300a\u767e\u9762\u673a\u5668\u5b66\u4e60\u300b- \u6df1\u5ea6\u5b66\u4e60\u4e2d\u7684\u52a8\u91cf<\/li>\n<li>\u6df1\u5ea6\u5b66\u4e60\u4e2d\u7684\u52a8\u91cf\uff1ahttps:\/\/blog.csdn.net\/_\/article\/details\/<\/li>\n<li>\n<ul>\n<li>Deep Learning \u6700\u4f18\u5316\u65b9\u6cd5\u4e4bNesterov(\u725b\u987f\u52a8\u91cf)\uff1ahttps:\/\/blog.csdn.net\/bvl\/article\/details\/<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"\u52a8\u91cf\u6a21\u578b\u6838\u5fc3\u6559\u7a0b_\u52a8\u91cf\u5b88\u6052\u5341\u5927\u6a21\u578b\u52a8\u91cf\u68af\u5ea6\u4e0b\u964d\uff08GradientDescentWithMomentum\uff09\uff0c\u7b80\u79f0\u4e3a\u52a8\u91cf\u65b9\u6cd5\uff08Momentum\uff09\uff0c\u8fd0\u884c\u901f\u5ea6\u51e0...","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[],"tags":[],"_links":{"self":[{"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/posts\/5094"}],"collection":[{"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/comments?post=5094"}],"version-history":[{"count":0,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/posts\/5094\/revisions"}],"wp:attachment":[{"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/media?parent=5094"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/categories?post=5094"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/mushiming.com\/wp-json\/wp\/v2\/tags?post=5094"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}