{"id":1339,"date":"2022-05-18T07:00:00","date_gmt":"2022-05-17T22:00:00","guid":{"rendered":"https:\/\/maru.nagoya\/?p=1339"},"modified":"2023-01-28T15:29:15","modified_gmt":"2023-01-28T06:29:15","slug":"what-is-vision-transformer","status":"publish","type":"post","link":"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/","title":{"rendered":"\u3010ViT\u3011\u753b\u50cf\u8a8d\u8b58\u306e\u5927\u9769\u547d\uff01Vision Transformer\u3068\u306f\uff1f"},"content":{"rendered":"\n<div class=\"wp-block-sgb-say\"><div class=\"sgb-block-say sgb-block-say--left\"><div class=\"sgb-block-say-avatar\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/11233119\/iOS-%E3%81%AE%E7%94%BB%E5%83%8F-14.jpg\" alt=\"\u9ad8\u6a4b\" width=\"80\" height=\"80\" style=\"border-color:#eaedf2\"><div class=\"sgb-block-say-avatar__name\">\u9ad8\u6a4b<\/div><\/div><div class=\"sgb-block-say-text\"><div class=\"sgb-block-say-text__content\" style=\"color:#333;border-color:#d5d5d5;background-color:#FFF\">\n<p>\u521d\u3081\u307e\u3057\u3066\u3002AI-Tech-Lab\u611b\u77e5\u652f\u90e8\u306e\u5b66\u751f\u30a8\u30f3\u30b8\u30cb\u30a2\u306e\u9ad8\u6a4b\u3067\u3059\u3002<\/p>\n\n\n\n<p>\u521d\u3081\u3066\u306e\u30d6\u30ed\u30b0\u3068\u3044\u3046\u3053\u3068\u3067\u3001Vision Transformer\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u306b\u3064\u3044\u3066\u7c21\u5358\u306b\u307e\u3068\u3081\u307e\u3057\u305f\u3002<\/p>\n<span class=\"sgb-block-say-text__before\" style=\"border-right-color:#d5d5d5\"><\/span><span class=\"sgb-block-say-text__after\" style=\"border-right-color:#FFF\"><\/span><\/div><\/div><\/div><\/div>\n\n\n\n<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_82_2 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">\u3082\u304f\u3058<\/p>\n<label for=\"ez-toc-cssicon-toggle-item-69e617134c016\" class=\"ez-toc-cssicon-toggle-label\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/label><input type=\"checkbox\" style='display:none' id=\"ez-toc-cssicon-toggle-item-69e617134c016\"  aria-label=\"Toggle\" \/><nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#Vision_Transformer%E3%81%AE%E6%A6%82%E8%A6%81\" >Vision Transformer\u306e\u6982\u8981<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#Vision_Transformer%E3%81%AE%E3%82%A2%E3%83%BC%E3%82%AD%E3%83%86%E3%82%AF%E3%83%81%E3%83%A3%E3%83%BB%E5%87%A6%E7%90%86%E3%81%AE%E8%A7%A3%E8%AA%AC\" >Vision Transformer\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u30fb\u51e6\u7406\u306e\u89e3\u8aac<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#%E2%91%A0%E5%85%A5%E5%8A%9B%E7%94%BB%E5%83%8F%E3%82%92%E3%83%99%E3%82%AF%E3%83%88%E3%83%AB%E3%81%B8%E5%A4%89%E6%8F%9B\" >\u2460\u5165\u529b\u753b\u50cf\u3092\u30d9\u30af\u30c8\u30eb\u3078\u5909\u63db<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#%E2%91%A1Transformer_Encoder%E3%81%AB%E3%82%88%E3%82%8A%E5%87%A6%E7%90%86\" >\u2461Transformer Encoder\u306b\u3088\u308a\u51e6\u7406<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#%E2%91%A2MLP_Head%E3%81%AB%E3%82%88%E3%82%8A%E5%87%A6%E7%90%86\" >\u2462MLP Head\u306b\u3088\u308a\u51e6\u7406<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#Multi-Head_Attention%E3%81%A8%E3%81%AF\" >Multi-Head Attention\u3068\u306f<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#Scaled_Dot-Product_Attention\" >Scaled Dot-Product Attention<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#Multi-Head_Attention\" >Multi-Head Attention<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#%E3%81%BE%E3%81%A8%E3%82%81\" >\u307e\u3068\u3081<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/05\/18\/what-is-vision-transformer\/#%E5%8E%9F%E6%96%87\" >\u539f\u6587<\/a><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n<h2 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-0\"><span class=\"ez-toc-section\" id=\"Vision_Transformer%E3%81%AE%E6%A6%82%E8%A6%81\"><\/span><span class=\"sgb-heading__inner sgb-heading--type4\" style=\"background-color:#303440;border-color:#303440;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#FFF\">Vision Transformer\u306e\u6982\u8981<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>Vision Transformer\u3068\u306f\u3001\u8ad6\u6587\u300c<a href=\"https:\/\/openreview.net\/forum?id=YicbFdNTTy\" target=\"_blank\" rel=\"noreferrer noopener\" title=\"\">An Image is Worth 16&#215;16 Words: Transformers for Image Recognition at Scale<\/a>\u300d\u3067\u63d0\u6848\u3055\u308c\u305f<span class=\"keiko_yellow\"><strong>\u753b\u50cf\u8a8d\u8b58\u306e\u70ba\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3<\/strong><\/span>\u3067\u3059\u3002<\/p>\n\n\n\n<p>\u753b\u50cf\u51e6\u7406\u306eDeep Learning\u3068\u8a00\u3048\u3070CNN\u3092\u30d9\u30fc\u30b9\u3068\u3057\u305f\u7573\u307f\u8fbc\u307f\u3092\u7528\u3044\u305f\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u304c\u9ad8\u3044\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u3092\u51fa\u3057\u3066\u3044\u307e\u3057\u305f\u3002\u3057\u304b\u3057\u306a\u304c\u3089\u3001Vision Transformer\u306f\u7573\u307f\u8fbc\u307f\u3092\u305b\u305a\u306b\u305d\u308c\u3092\u4e0a\u56de\u308b\u307b\u3069\u306e\u9ad8\u3044\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u3092\u767a\u63ee\u3057\u305f\u3053\u3068\u306b\u3088\u308a\u6ce8\u76ee\u3092\u96c6\u3081\u307e\u3057\u305f\u3002<\/p>\n\n\n\n<div class=\"wp-block-sgb-block-simple sgb-box-simple sgb-box-simple--title-normal sgb-box-simple--with-border\"><div style=\"background-color:#228a93;color:#FFF\" class=\"sgb-box-simple__title\">Vision Transformer\u306e\u7279\u5fb4<\/div><div class=\"sgb-box-simple__body\" style=\"border-color:#228a93;background-color:#FFF\">\n<p>\u30fb\u7573\u307f\u8fbc\u307f\u3092\u884c\u308f\u306a\u3044<\/p>\n\n\n\n<p>\u30fbSOTA\u3092\u66f4\u65b0<\/p>\n\n\n\n<p>\u30fb\u8a08\u7b97\u91cf\u3092\u5927\u5e45\u306b\u6e1b\u5c11<\/p>\n<\/div><\/div>\n\n\n\n<p>Vision Transformer\u306f\u7573\u307f\u8fbc\u307f\u3092\u305b\u305a\u306bSOTA\u30e2\u30c7\u30eb\u4ee5\u4e0a\u306e\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u3092\u767a\u63ee\u3057\u307e\u3057\u305f\u3002\u66f4\u306bViT-H\/14\u306fCNN\u30d9\u30fc\u30b9\u306eBig Transfer (BiT)\u3068\u6bd4\u8f03\u3057\u3066Vision Transformer\u306f\u8a08\u7b97\u91cf\u3092\u7d044\u5206\u306e1\u3068\u5927\u304d\u304f\u6e1b\u5c11\u3055\u305b\u307e\u3057\u305f\u3002\u3053\u306e\u3053\u3068\u306b\u3088\u308a\u4eca\u65e5(2022\u5e74)\u3067\u306fVision Transformer\u3092\u5143\u3068\u3057\u305f\u30e2\u30c7\u30eb\u306e\u7814\u7a76\u304c\u6570\u591a\u304f\u884c\u308f\u308c\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><a href=\"https:\/\/paperswithcode.com\/method\/vision-transformer\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/10184736\/highchart-1024x248.png\" alt=\"\" class=\"wp-image-1643\" width=\"840\" height=\"203\" srcset=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/10184736\/highchart-1024x248.png 1024w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/10184736\/highchart-300x73.png 300w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/10184736\/highchart-768x186.png 768w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/10184736\/highchart-1536x372.png 1536w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/10184736\/highchart-940x228.png 940w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/10184736\/highchart.png 1665w\" sizes=\"auto, (max-width: 840px) 100vw, 840px\" \/><\/a><figcaption><a href=\"https:\/\/paperswithcode.com\/method\/vision-transformer\" target=\"_blank\" rel=\"noreferrer noopener\">Papers With Code<\/a>\u3088\u308a\u5f15\u7528<\/figcaption><\/figure>\n<\/div>\n\n\n<p>\u4ee5\u4e0b\u3067\u306fViT\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u3092\u30e1\u30a4\u30f3\u306b\u89e3\u8aac\u3057\u3066\u3044\u304d\u307e\u3059\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-1\"><span class=\"ez-toc-section\" id=\"Vision_Transformer%E3%81%AE%E3%82%A2%E3%83%BC%E3%82%AD%E3%83%86%E3%82%AF%E3%83%81%E3%83%A3%E3%83%BB%E5%87%A6%E7%90%86%E3%81%AE%E8%A7%A3%E8%AA%AC\"><\/span><span class=\"sgb-heading__inner sgb-heading--type4\" style=\"background-color:#303440;border-color:#303440;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#FFF\">Vision Transformer\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u30fb\u51e6\u7406\u306e\u89e3\u8aac<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>Vision Transformer\u306f\u4e3b\u306b<span class=\"keiko_yellow\"><strong>Transformer\u306e\u30a8\u30f3\u30b3\u30fc\u30c0\u90e8\u5206\u3067\u69cb\u6210<\/strong><\/span>\u3055\u308c\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n<div class=\"wp-block-sgb-block-simple sgb-box-simple sgb-box-simple--title-normal sgb-box-simple--with-border\"><div style=\"background-color:#228a93;color:#FFF\" class=\"sgb-box-simple__title\">Vision Transformer\u306e\u51e6\u7406\u306e\u6d41\u308c<\/div><div class=\"sgb-box-simple__body\" style=\"border-color:#228a93;background-color:#FFF\">\n<ol start=\"1\" class=\"is-style-sango-list-simple wp-block-list\"><li>\u5165\u529b\u753b\u50cf\u3092\u30d9\u30af\u30c8\u30eb\u306b\u5909\u63db<\/li><li>Transformer Encoder\u306b\u3088\u308a\u51e6\u7406<\/li><li>MLP Head\u306b\u3088\u308a\u51e6\u7406<\/li><\/ol>\n<\/div><\/div>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"548\" src=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/04\/16205558\/ViT_model-1024x548.png\" alt=\"\" class=\"wp-image-1344\" srcset=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/04\/16205558\/ViT_model-1024x548.png 1024w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/04\/16205558\/ViT_model-300x161.png 300w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/04\/16205558\/ViT_model-768x411.png 768w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/04\/16205558\/ViT_model-940x503.png 940w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/04\/16205558\/ViT_model.png 1168w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><figcaption>Vision Transformer\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3<br><a href=\"https:\/\/openreview.net\/forum?id=YicbFdNTTy\" target=\"_blank\" rel=\"noreferrer noopener\" title=\"\u8ad6\u6587\">\u8ad6\u6587<\/a>\u304b\u3089\u5f15\u7528<br><\/figcaption><\/figure>\n<\/div>\n\n\n<h3 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-2\"><span class=\"ez-toc-section\" id=\"%E2%91%A0%E5%85%A5%E5%8A%9B%E7%94%BB%E5%83%8F%E3%82%92%E3%83%99%E3%82%AF%E3%83%88%E3%83%AB%E3%81%B8%E5%A4%89%E6%8F%9B\"><\/span><span class=\"sgb-heading__inner sgb-heading--type3\" style=\"background-color:#ccd9f4;border-color:#1b479c;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#333\"><strong>\u2460\u5165\u529b\u753b\u50cf\u3092\u30d9\u30af\u30c8\u30eb\u3078\u5909\u63db<\/strong><\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<div class=\"wp-block-group\"><div class=\"wp-block-group__inner-container is-layout-flow wp-block-group-is-layout-flow\">\n<p>\u307e\u305a\u5165\u529b\u753b\u50cf<img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-3417d00efb0138447528c91356e0cf6e_l3.png\" class=\"ql-img-inline-formula quicklatex-auto-format\" alt=\"&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#120;&#125;&#32;&#92;&#105;&#110;&#32;&#92;&#109;&#97;&#116;&#104;&#98;&#98;&#123;&#82;&#125;&#94;&#123;&#72;&#32;&#92;&#116;&#105;&#109;&#101;&#115;&#32;&#87;&#32;&#92;&#116;&#105;&#109;&#101;&#115;&#32;&#67;&#125;\" title=\"Rendered by QuickLaTeX.com\" height=\"16\" width=\"106\" style=\"vertical-align: -1px;\"\/>\u3092<strong>\u30d1\u30c3\u30c1\u306b\u5206\u5272<\/strong>\u3057\u307e\u3059\u3002<br>\u3053\u3053\u3067(H, W)\u306f\u89e3\u50cf\u5ea6\u3001C\u306f\u30c1\u30e3\u30f3\u30cd\u30eb\u6570\u3092\u8868\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"217\" height=\"212\" src=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/02191655\/patch.png\" alt=\"\" class=\"wp-image-1520\"><figcaption>\u30d1\u30c3\u30c1\u306b\u5206\u5272<br><a href=\"https:\/\/openreview.net\/forum?id=YicbFdNTTy\" target=\"_blank\" rel=\"noreferrer noopener\" title=\"\u8ad6\u6587\">\u8ad6\u6587<\/a>\u304b\u3089\u5f15\u7528<\/figcaption><\/figure>\n<\/div>\n\n\n<p><br>\u6b21\u306b<img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-147d0fb4c698029e225becc168967f1a_l3.png\" class=\"ql-img-inline-formula quicklatex-auto-format\" alt=\"&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#120;&#125;&#95;&#123;&#112;&#125;&#32;&#92;&#105;&#110;&#32;&#92;&#109;&#97;&#116;&#104;&#98;&#98;&#123;&#82;&#125;&#94;&#123;&#78;&#32;&#92;&#116;&#105;&#109;&#101;&#115;&#32;&#40;&#80;&#94;&#50;&#32;&#92;&#99;&#100;&#111;&#116;&#32;&#67;&#41;&#125;\" title=\"Rendered by QuickLaTeX.com\" height=\"24\" width=\"118\" style=\"vertical-align: -6px;\"\/>\u3068\u3044\u3046<strong>\u30d9\u30af\u30c8\u30eb\u306b\u5909\u63db<\/strong>\u3057\u307e\u3059\u3002<br>\u3053\u3053\u3067P\u306f\u30d1\u30c3\u30c1\u306e\u5927\u304d\u3055\u3001N\u306f\u30d1\u30c3\u30c1\u6570<img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-81f08eb8a10181d076db80b5f16beaf5_l3.png\" class=\"ql-img-inline-formula quicklatex-auto-format\" alt=\"&#78;&#61;&#72;&#87;&#47;&#80;&#94;&#123;&#50;&#125;\" title=\"Rendered by QuickLaTeX.com\" height=\"20\" width=\"103\" style=\"vertical-align: -5px;\"\/>\u3092\u8868\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n<p>\u66f4\u306b\u5b66\u7fd2\u53ef\u80fd\u306a<img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-cc7f05b88931ed02eb3f2bafd0aa8397_l3.png\" class=\"ql-img-inline-formula quicklatex-auto-format\" alt=\"&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#69;&#125;&#32;&#92;&#105;&#110;&#32;&#92;&#109;&#97;&#116;&#104;&#98;&#98;&#123;&#82;&#125;&#94;&#123;&#40;&#80;&#94;&#50;&#32;&#92;&#99;&#100;&#111;&#116;&#32;&#67;&#41;&#32;&#92;&#116;&#105;&#109;&#101;&#115;&#32;&#68;&#125;\" title=\"Rendered by QuickLaTeX.com\" height=\"19\" width=\"114\" style=\"vertical-align: -1px;\"\/>\u3068\u4f4d\u7f6e\u30a8\u30f3\u30b3\u30fc\u30c7\u30a3\u30f3\u30b0<img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-23e724595de48606c825da7b8f9fd9bd_l3.png\" class=\"ql-img-inline-formula quicklatex-auto-format\" alt=\"&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#69;&#125;&#95;&#123;&#112;&#111;&#115;&#125;&#32;&#92;&#105;&#110;&#32;&#92;&#109;&#97;&#116;&#104;&#98;&#98;&#123;&#82;&#125;&#94;&#123;&#40;&#78;&#43;&#49;&#41;&#32;&#92;&#116;&#105;&#109;&#101;&#115;&#32;&#68;&#32;\" title=\"Rendered by QuickLaTeX.com\" height=\"22\" width=\"133\" style=\"vertical-align: -6px;\"\/>\u3092\u7528\u3044\u3066\u5f0f(1)\u306e\u69d8\u306b<strong>\u30de\u30c3\u30d4\u30f3\u30b0<\/strong>\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 24px;\"><span class=\"ql-right-eqno\"> (1) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-e582cb10280451e8608d0793d4c89aad_l3.png\" height=\"24\" width=\"331\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#122;&#95;&#123;&#48;&#125;&#125;&#32;&#61;&#32;&#91;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#120;&#125;&#95;&#123;&#99;&#108;&#97;&#115;&#115;&#125;&#59;&#92;&#58;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#120;&#125;&#95;&#123;&#112;&#125;&#94;&#49;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#69;&#125;&#59;&#92;&#58;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#120;&#125;&#95;&#123;&#112;&#125;&#94;&#50;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#69;&#125;&#59;&#92;&#58;&#32;&#92;&#99;&#100;&#111;&#116;&#115;&#59;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#120;&#125;&#95;&#123;&#112;&#125;&#94;&#78;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#69;&#125;&#93;&#32;&#43;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#69;&#125;&#95;&#123;&#112;&#111;&#115;&#125;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"361\" src=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03140449\/image-1024x361.png\" alt=\"\" class=\"wp-image-1529\" srcset=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03140449\/image-1024x361.png 1024w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03140449\/image-300x106.png 300w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03140449\/image-768x271.png 768w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03140449\/image-940x332.png 940w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03140449\/image.png 1091w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><figcaption><a href=\"https:\/\/openreview.net\/forum?id=YicbFdNTTy\" target=\"_blank\" rel=\"noreferrer noopener\" title=\"\u8ad6\u6587\">\u8ad6\u6587<\/a>\u304b\u3089\u5f15\u7528<\/figcaption><\/figure>\n<\/div>\n\n\n<p>\u4ee5\u4e0a\u306e\u51e6\u7406\u304cTransformer\u306eEncoder\u524d\u307e\u3067\u306e\u51e6\u7406\u3068\u306a\u308a\u307e\u3059\u3002<\/p>\n<\/div><\/div>\n\n\n\n<h3 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-3\"><span class=\"ez-toc-section\" id=\"%E2%91%A1Transformer_Encoder%E3%81%AB%E3%82%88%E3%82%8A%E5%87%A6%E7%90%86\"><\/span><span class=\"sgb-heading__inner sgb-heading--type3\" style=\"background-color:#ccd9f4;border-color:#1b479c;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#333\">\u2461Transformer Encoder\u306b\u3088\u308a\u51e6\u7406<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>MSA\u3092Multihead Self-Attention\u3001MLP\u3092Multi Layer Perceptron\u3001LN\u3092Linear Normlization\u3068\u3059\u308b\u3068Transformer\u306eEncoder\u306e\u51e6\u7406\u306f\u5f0f(2)\u3068\u5f0f(3)\u3067\u8868\u3059\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<br>MLP\u306e\u6d3b\u6027\u5316\u95a2\u6570\u306fGELU\u3092\u7528\u3044\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 21px;\"><span class=\"ql-right-eqno\"> (2) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-e4f108c80756bc045651765f981574b8_l3.png\" height=\"21\" width=\"317\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#122;&#39;&#95;&#108;&#32;&#61;&#32;&#77;&#83;&#65;&#40;&#76;&#78;&#40;&#122;&#95;&#123;&#108;&#45;&#49;&#125;&#41;&#41;&#32;&#43;&#32;&#122;&#95;&#123;&#108;&#45;&#49;&#125;&#32;&#92;&#104;&#115;&#112;&#97;&#99;&#101;&#123;&#49;&#112;&#99;&#125;&#32;&#108;&#61;&#49;&#32;&#92;&#99;&#100;&#111;&#116;&#115;&#32;&#76;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><br><p class=\"ql-center-displayed-equation\" style=\"line-height: 21px;\"><span class=\"ql-right-eqno\"> (3) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-3d2666ae63ee8bbad4baf82ece1f624f_l3.png\" height=\"21\" width=\"283\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#122;&#95;&#108;&#32;&#61;&#32;&#77;&#76;&#80;&#40;&#76;&#78;&#40;&#122;&#39;&#95;&#108;&#41;&#41;&#32;&#43;&#32;&#122;&#39;&#95;&#108;&#32;&#92;&#104;&#115;&#112;&#97;&#99;&#101;&#123;&#49;&#112;&#99;&#125;&#32;&#108;&#61;&#49;&#32;&#92;&#99;&#100;&#111;&#116;&#115;&#32;&#76;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n<div class=\"wp-block-image\">\n<figure class=\"aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03144457\/image-1.png\" alt=\"\" class=\"wp-image-1536\" width=\"166\" height=\"352\" srcset=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03144457\/image-1.png 400w, https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/05\/03144457\/image-1-141x300.png 141w\" sizes=\"auto, (max-width: 166px) 100vw, 166px\" \/><figcaption><a href=\"https:\/\/openreview.net\/forum?id=YicbFdNTTy\" target=\"_blank\" rel=\"noreferrer noopener\" title=\"\u8ad6\u6587\">\u8ad6\u6587<\/a>\u304b\u3089\u5f15\u7528<\/figcaption><\/figure>\n<\/div>\n\n\n<h3 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-4\"><span class=\"ez-toc-section\" id=\"%E2%91%A2MLP_Head%E3%81%AB%E3%82%88%E3%82%8A%E5%87%A6%E7%90%86\"><\/span><span class=\"sgb-heading__inner sgb-heading--type3\" style=\"background-color:#ccd9f4;border-color:#1b479c;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#333\">\u2462MLP Head\u306b\u3088\u308a\u51e6\u7406<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-af4230d9439673d4d5c3e65e425be802_l3.png\" class=\"ql-img-inline-formula quicklatex-auto-format\" alt=\"&#90;&#94;&#48;&#95;&#76;\" title=\"Rendered by QuickLaTeX.com\" height=\"20\" width=\"20\" style=\"vertical-align: -5px;\"\/>\u306f\u6700\u7d42\u5c64\u306e\u51fa\u529b\u306b\u304a\u3051\u308b\u524d\u304b\u30890\u756a\u76ee\u306e\u30d9\u30af\u30c8\u30eb\u3092\u8868\u3057\u3066\u3044\u307e\u3059\u3002\u3064\u307e\u308a\u3001[cls]\u30c8\u30fc\u30af\u30f3\u306e\u6700\u7d42\u51fa\u529b\u3067\u3059\u3002\u305d\u308c\u3092NL\u306b\u5165\u308c\u3066\u5f97\u305fy\u3092MLP Head\u306b\u5165\u308c\u308b\u3053\u3068\u3067\u6700\u7d42\u4e88\u6e2c\u3092\u5f97\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 22px;\"><span class=\"ql-right-eqno\"> (4) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-de92672acb48f5839b947a05c8042aa1_l3.png\" height=\"22\" width=\"93\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#121;&#61;&#76;&#78;&#40;&#122;&#94;&#48;&#95;&#76;&#41;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<h2 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-5\"><span class=\"ez-toc-section\" id=\"Multi-Head_Attention%E3%81%A8%E3%81%AF\"><\/span><span class=\"sgb-heading__inner sgb-heading--type4\" style=\"background-color:#303440;border-color:#303440;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#FFF\">Multi-Head Attention\u3068\u306f<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>Multi-Head Attention\u306fTransformer\u306eencoder\u3067\u4f7f\u308f\u308c\u3066\u3044\u308b\u6a5f\u69cb\u306b\u306a\u308a\u307e\u3059\u3002<span class=\"keiko_yellow\">\u3053\u306e\u6a5f\u69cb\u306e\u512a\u308c\u3066\u3044\u308b\u70b9\u306f<strong>\u3069\u306e\u60c5\u5831\u306b\u6ce8\u76ee\u3059\u3079\u304d\u304b\u81ea\u52d5\u3067\u5224\u65ad\u3092\u3057\u3066\u60c5\u5831\u3092\u51e6\u7406\u3059\u308b<\/strong>\u70b9\u3067\u3059\u3002<\/span><\/p>\n\n\n\n<h3 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-6\"><span class=\"ez-toc-section\" id=\"Scaled_Dot-Product_Attention\"><\/span><span class=\"sgb-heading__inner sgb-heading--type3\" style=\"background-color:#ccd9f4;border-color:#1b479c;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#333\">Scaled Dot-Product Attention<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>Multi-Head Attention\u306e\u8aac\u660e\u306e\u524d\u306b\u305d\u308c\u306e\u5143\u3068\u306a\u308bScaled Dot-Product Attention\u306b\u3064\u3044\u3066\u89e3\u8aac\u3092\u3057\u307e\u3059\u3002<br>Scaled Dot-Product Attention\u306f\u5f0f(5)\u3067\u8868\u3059\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 44px;\"><span class=\"ql-right-eqno\"> (5) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-195d4301d053cc26d9a4cf59fac498a1_l3.png\" height=\"44\" width=\"310\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#65;&#116;&#116;&#101;&#110;&#116;&#105;&#111;&#110;&#40;&#81;&#44;&#75;&#44;&#86;&#41;&#61;&#115;&#111;&#102;&#116;&#109;&#97;&#120;&#40;&#92;&#102;&#114;&#97;&#99;&#123;&#81;&#75;&#94;&#116;&#125;&#123;&#92;&#115;&#113;&#114;&#116;&#123;&#100;&#125;&#125;&#41;&#86;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<p>\u3053\u306e\u5f0f\u306f\u5f15\u6570\u3068\u3057\u3066Q(query)\u3001K(key)\u3001V(value)\u3092\u53d6\u308a\u307e\u3059\u3002\u3053\u306e\u5f0f\u306f\u30d0\u30c3\u30c1\u51e6\u7406\u3092\u884c\u3046\u5f0f\u3067\u3042\u308b\u306e\u3067\u3001\u307e\u305a\u306f\u5165\u529b\u304c\u4e00\u3064\u306e\u5f0f\u306b\u3064\u3044\u3066\u89e3\u8aac\u3092\u884c\u3044\u307e\u3059\u3002\u5165\u529b\u304c\u4e00\u3064\u306e\u5834\u5408\u306e\u5f0f(6)\u306f\u3068\u306a\u308a\u307e\u3059\u3002<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 44px;\"><span class=\"ql-right-eqno\"> (6) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-36664c745ae757989ddfbb61012a6dd0_l3.png\" height=\"44\" width=\"303\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#65;&#116;&#116;&#101;&#110;&#116;&#105;&#111;&#110;&#40;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#44;&#75;&#44;&#86;&#41;&#61;&#115;&#111;&#102;&#116;&#109;&#97;&#120;&#40;&#92;&#102;&#114;&#97;&#99;&#123;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#75;&#94;&#116;&#125;&#123;&#92;&#115;&#113;&#114;&#116;&#123;&#100;&#125;&#125;&#41;&#86;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<p>\u307e\u305a<img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-6fed164871a2fcccc5b5f77657b197f9_l3.png\" class=\"ql-img-inline-formula quicklatex-auto-format\" alt=\"&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#75;&#94;&#116;\" title=\"Rendered by QuickLaTeX.com\" height=\"19\" width=\"32\" style=\"vertical-align: -4px;\"\/>\u3092\u8a08\u7b97\u3059\u308b\u3068\u5f0f(7)\u3068\u306a\u308a\u3001\u5185\u7a4d\u304c\u4e26\u3076\u3053\u3068\u306b\u306a\u308a\u307e\u3059\u3002<br><span class=\"keiko_yellow\"><strong>\u5185\u7a4d\u306f\u985e\u4f3c\u5ea6\u3092\u8868\u73fe\u3057\u3066\u3044\u308b\u306e\u3067<\/strong>\u3001\u7d50\u679c\u306fquery\u3068key\u306e\u985e\u4f3c\u5ea6\u3092\u4e26\u3079\u305f\u30d9\u30af\u30c8\u30eb\u3068\u306a\u308a\u307e\u3059\u3002<\/span><\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 22px;\"><span class=\"ql-right-eqno\"> (7) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-cf9e92b3896ea64add445248310aac60_l3.png\" height=\"22\" width=\"413\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#75;&#94;&#116;&#32;&#38;&#61;&#38;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#40;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#107;&#125;&#95;&#49;&#94;&#116;&#44;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#107;&#125;&#95;&#50;&#94;&#116;&#44;&#32;&#92;&#99;&#100;&#111;&#116;&#115;&#32;&#44;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#107;&#125;&#95;&#109;&#94;&#116;&#41;&#32;&#92;&#92;&#38;&#61;&#38;&#32;&#40;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#32;&#92;&#99;&#100;&#111;&#116;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#107;&#95;&#49;&#125;&#44;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#32;&#92;&#99;&#100;&#111;&#116;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#107;&#95;&#50;&#125;&#44;&#32;&#92;&#99;&#100;&#111;&#116;&#115;&#44;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#32;&#92;&#99;&#100;&#111;&#116;&#32;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#107;&#95;&#109;&#125;&#41;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<p>\u6b21\u306b\u52fe\u914d\u6d88\u5931\u3092\u6291\u5236\u3059\u308b\u70ba<img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-450f4e5ea9f6c5dc697e01b79ab7718c_l3.png\" class=\"ql-img-inline-formula quicklatex-auto-format\" alt=\"&#92;&#115;&#113;&#114;&#116;&#123;&#100;&#125;\" title=\"Rendered by QuickLaTeX.com\" height=\"18\" width=\"24\" style=\"vertical-align: -2px;\"\/>\u3067\u5272\u3063\u3066\u88dc\u6b63\u3092\u884c\u3044\u307e\u3059\u3002<\/p>\n\n\n\n<p>\u66f4\u306b\u305d\u308c\u3092\u30bd\u30d5\u30c8\u30de\u30c3\u30af\u30b9\u95a2\u6570\u306b\u5165\u308c\u308b\u3053\u3068\u306b\u3088\u308a\u985e\u4f3c\u5ea6\u3092\u5408\u8a08\u306e\u91cd\u307f\u306b\u5909\u63db\u3057\u3001\u305d\u308c\u3092P\u3068\u3057\u307e\u3059\u3002<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 44px;\"><span class=\"ql-right-eqno\"> (8) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-46ab24fd7c316814d454f22245e5cb71_l3.png\" height=\"44\" width=\"301\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#115;&#111;&#102;&#116;&#109;&#97;&#120;&#40;&#92;&#102;&#114;&#97;&#99;&#123;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#75;&#94;&#116;&#125;&#123;&#92;&#115;&#113;&#114;&#116;&#123;&#100;&#125;&#125;&#41;&#61;&#40;&#112;&#95;&#49;&#44;&#112;&#95;&#50;&#44;&#92;&#99;&#100;&#111;&#116;&#115;&#44;&#112;&#95;&#109;&#41;&#61;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#80;&#125;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<p>\u6700\u5f8c\u306bV\u3092\u304b\u3051\u308b\u3068<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 44px;\"><span class=\"ql-right-eqno\"> (9) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-72a1e09937bd4b05bec3f3b5d87519f6_l3.png\" height=\"44\" width=\"269\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#115;&#111;&#102;&#116;&#109;&#97;&#120;&#40;&#92;&#102;&#114;&#97;&#99;&#123;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#75;&#94;&#116;&#125;&#123;&#92;&#115;&#113;&#114;&#116;&#123;&#100;&#125;&#125;&#41;&#86;&#61;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#80;&#125;&#86;&#61;&#92;&#115;&#117;&#109;&#32;&#112;&#95;&#105;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#118;&#95;&#105;&#125;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<p>\u5f93\u3063\u3066\u3001Attention(q,K,V)\u3067\u306fquery\u3068key\u306e\u985e\u4f3c\u5ea6\u3092\u8a08\u7b97\u3057\u3001\u305d\u308c\u306b\u5fdc\u3058\u305f\u91cd\u307f\u306evalue\u3092\u51fa\u529b\u3059\u308b\u3053\u3068\u306b\u306a\u308a\u307e\u3059\u3002<br>\u4ee5\u4e0a\u306e\u904e\u7a0b\u3092\u5f0f\u306b\u3059\u308b\u3068\u5f0f(10)\u3001\u30d0\u30c3\u30c1\u51e6\u7406\u3092\u884c\u3046\u5f0f\u306f\u5f0f(11)\u3068\u306a\u308a\u307e\u3059\u3002<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 44px;\"><span class=\"ql-right-eqno\"> (10) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-a089ec1a942072d20b75199f46048e77_l3.png\" height=\"44\" width=\"439\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#65;&#116;&#116;&#101;&#110;&#116;&#105;&#111;&#110;&#40;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#44;&#32;&#75;&#44;&#32;&#86;&#41;&#61;&#115;&#111;&#102;&#116;&#109;&#97;&#120;&#40;&#92;&#102;&#114;&#97;&#99;&#123;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#113;&#125;&#75;&#94;&#116;&#125;&#123;&#92;&#115;&#113;&#114;&#116;&#123;&#100;&#125;&#125;&#41;&#86;&#61;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#80;&#125;&#86;&#61;&#92;&#115;&#117;&#109;&#32;&#112;&#95;&#105;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#118;&#95;&#105;&#125;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 75px;\"><span class=\"ql-right-eqno\"> (11) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-8a8d1d1e4708da24852c769f1d6ac8f8_l3.png\" height=\"75\" width=\"539\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;&#65;&#116;&#116;&#101;&#110;&#116;&#105;&#111;&#110;&#40;&#81;&#44;&#32;&#75;&#44;&#32;&#86;&#41;&#61;&#115;&#111;&#102;&#116;&#109;&#97;&#120;&#40;&#92;&#102;&#114;&#97;&#99;&#123;&#81;&#75;&#94;&#116;&#125;&#123;&#92;&#115;&#113;&#114;&#116;&#123;&#100;&#125;&#125;&#41;&#86;&#61;&#32;&#92;&#108;&#101;&#102;&#116;&#40;&#32;&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#97;&#114;&#114;&#97;&#121;&#125;&#123;&#99;&#125;&#112;&#95;&#123;&#49;&#49;&#125;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#118;&#125;&#95;&#123;&#49;&#125;&#43;&#32;&#92;&#99;&#100;&#111;&#116;&#115;&#32;&#43;&#32;&#112;&#95;&#123;&#49;&#109;&#125;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#118;&#125;&#95;&#123;&#110;&#125;&#32;&#92;&#92;&#92;&#118;&#100;&#111;&#116;&#115;&#32;&#92;&#92;&#112;&#95;&#123;&#109;&#49;&#125;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#118;&#125;&#95;&#123;&#49;&#125;&#43;&#32;&#92;&#99;&#100;&#111;&#116;&#115;&#32;&#43;&#32;&#112;&#95;&#123;&#110;&#109;&#125;&#92;&#98;&#111;&#108;&#100;&#115;&#121;&#109;&#98;&#111;&#108;&#123;&#118;&#125;&#95;&#123;&#110;&#125;&#32;&#92;&#101;&#110;&#100;&#123;&#97;&#114;&#114;&#97;&#121;&#125;&#32;&#92;&#114;&#105;&#103;&#104;&#116;&#41;&#92;&#101;&#110;&#100;&#123;&#101;&#113;&#117;&#97;&#116;&#105;&#111;&#110;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<h3 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-7\"><span class=\"ez-toc-section\" id=\"Multi-Head_Attention\"><\/span><span class=\"sgb-heading__inner sgb-heading--type3\" style=\"background-color:#ccd9f4;border-color:#1b479c;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#333\">Multi-Head Attention<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<p>\u6b21\u306bVision Transformer\u3067\u4f7f\u308f\u308c\u3066\u3044\u308bMulti-Head Attention\u306b\u3064\u3044\u3066\u89e3\u8aac\u3092\u3057\u307e\u3059\u3002<br>Multi-Head Attention\u306f\u5f0f(12)\u3068\u306a\u308a\u307e\u3059\u3002<\/p>\n\n\n\n<p><p class=\"ql-center-displayed-equation\" style=\"line-height: 49px;\"><span class=\"ql-right-eqno\"> (12) <\/span><span class=\"ql-left-eqno\"> &nbsp; <\/span><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/maru.nagoya\/wp-content\/ql-cache\/quicklatex.com-d1b0d8ff42c053f98dbb454cce261068_l3.png\" height=\"49\" width=\"342\" class=\"ql-img-displayed-equation quicklatex-auto-format\" alt=\"&#92;&#98;&#101;&#103;&#105;&#110;&#123;&#103;&#97;&#116;&#104;&#101;&#114;&#42;&#125;&#77;&#117;&#108;&#116;&#105;&#45;&#72;&#101;&#97;&#100;&#40;&#81;&#44;&#32;&#75;&#44;&#32;&#86;&#41;&#61;&#99;&#111;&#110;&#99;&#97;&#116;&#40;&#104;&#101;&#97;&#100;&#95;&#123;&#105;&#125;&#41;&#87;&#94;&#123;&#111;&#125;&#32;&#92;&#92;&#104;&#101;&#97;&#100;&#95;&#123;&#105;&#125;&#32;&#61;&#32;&#65;&#116;&#116;&#101;&#110;&#116;&#105;&#111;&#110;&#40;&#81;&#87;&#94;&#123;&#81;&#125;&#95;&#123;&#105;&#125;&#44;&#32;&#75;&#87;&#94;&#123;&#75;&#125;&#95;&#123;&#105;&#125;&#44;&#32;&#86;&#87;&#94;&#123;&#86;&#125;&#95;&#123;&#105;&#125;&#41;&#92;&#101;&#110;&#100;&#123;&#103;&#97;&#116;&#104;&#101;&#114;&#42;&#125;\" title=\"Rendered by QuickLaTeX.com\"\/><\/p><\/p>\n\n\n\n<p>Multi-Head Attention\u306fScaled Dot-Product Attention\u3092\u4e3b\u3068\u3057\u3066\u69cb\u6210\u3055\u308c\u3066\u3044\u307e\u3059\u3002<br>Q\u3001K\u3001V\u305d\u308c\u305e\u308c\u306b\u5b66\u7fd2\u53ef\u80fd\u306a\u884c\u5217\u3092\u304b\u3051\u3066Attention\u306b\u5165\u529b\u3057\u3001\u305d\u306e\u7d50\u679c\u3092\u9023\u7d50\u3057\u3001\u66f4\u306b\u884c\u5217\u3092\u304b\u3051\u305f\u3082\u306e\u304cMulti-Head Attention\u3068\u306a\u308a\u307e\u3059\u3002<\/p>\n\n\n\n<p><span class=\"keiko_yellow\"><strong>\u69d8\u3005\u306a\u89d2\u5ea6\u304b\u3089\u5165\u529b\u30d9\u30af\u30c8\u30eb\u3092\u6bd4\u8f03\u3057\u3001\u3069\u3053\u306b\u6ce8\u76ee\u3059\u3079\u304d\u304b\u5224\u65ad\u3092\u3057\u3066\u3001\u51fa\u529b\u3092\u6c7a\u5b9a\u3057\u307e\u3059\u3002<\/strong><\/span><\/p>\n\n\n\n<p>Attention\u306b\u3064\u3044\u3066\u8a73\u3057\u304f\u77e5\u308a\u305f\u3044\u65b9\u306f<a href=\"https:\/\/arxiv.org\/abs\/1706.03762\" target=\"_blank\" rel=\"noreferrer noopener\" title=\"Attention Is All You Need\">Attention Is All You Need<\/a>\u3092\u3054\u89a7\u4e0b\u3055\u3044\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-8\"><span class=\"ez-toc-section\" id=\"%E3%81%BE%E3%81%A8%E3%82%81\"><\/span><span class=\"sgb-heading__inner sgb-heading--type4\" style=\"background-color:#303440;border-color:#303440;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#FFF\">\u307e\u3068\u3081<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p>\u4eca\u56de\u306fVision Transformer\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u3092\u30e1\u30a4\u30f3\u306b\u307e\u3068\u3081\u3066\u307f\u307e\u3057\u305f\u3002<span class=\"keiko_yellow\"><strong>Vision Transformer\u306fTransformer\u306e\u30a8\u30f3\u30b3\u30fc\u30c0\u3092\u7528\u3044\u3066\u9ad8\u3044\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u3092\u767a\u63ee<\/strong>\u3057\u3066\u3044\u307e\u3059<\/span>\u3002\u3088\u308a\u8a73\u7d30\u306bVision Transformer\u306b\u3064\u3044\u3066\u77e5\u308a\u305f\u3044\u65b9\u306f\u539f\u6587\u3092\u53c2\u8003\u306b\u3057\u3066\u4e0b\u3055\u3044\u3002<\/p>\n\n\n\n<p>\u4eca\u65e5(2022\u5e74)\u3067\u306fSwin Transformer\u3084Data-efficient Image Transformer\u306a\u3069Vision Transformer\u3092\u5143\u3068\u3057\u305f\u30e2\u30c7\u30eb\u304c\u591a\u304f\u7814\u7a76\u3055\u308c\u3066\u304a\u308a\u3001\u65e5\u3005\u306e\u52c9\u5f37\u304c\u6b20\u304b\u305b\u307e\u305b\u3093\u3002<\/p>\n\n\n\n<p>\u79c1\u305f\u3061\u5b66\u751f\u30a8\u30f3\u30b8\u30cb\u30a2\u96c6\u56e3\u300cAI-Tech-Lab\u300d\u3067\u306f\u3001\u4eca\u56de\u7d39\u4ecb\u3057\u305fVision Transformer\u306a\u3069\u306e\u753b\u50cf\u51e6\u7406\u3060\u3051\u3067\u306a\u304f\u6a5f\u68b0\u5b66\u7fd2\u306b\u95a2\u308f\u308b\u3082\u306e\u3092\u5e45\u5e83\u304f\u5b66\u3093\u3060\u308a\u3001PoC\u958b\u767a\u3092\u884c\u3063\u305f\u308a\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n<p><span class=\"keiko_red\">\u25bcAI-Tech-Lab\u306b\u3064\u3044\u3066\u306e\u8a73\u7d30\u306f\u3053\u3061\u3089<\/span><\/p>\n\n\n<a class=\"c_linkto longc_linkto\" href=\"https:\/\/maru.nagoya\/index.php\/2022\/04\/20\/student-ai-development-case\/\">\n  <span class=\"longc_img\"><img loading=\"lazy\" decoding=\"async\" width=\"520\" height=\"300\" src=\"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/04\/19192009\/AI-Tech-Lab%E3%81%AE%E8%87%AA%E7%A4%BE%E9%96%8B%E7%99%BA%E7%B4%B9%E4%BB%8B-2-520x300.png\" class=\"attachment-thumb-520 size-thumb-520 wp-post-image\" alt=\"\" \/><\/span>\n  <span class=\"longc_content c_linkto_text\"><time class=\"pubdate sng-link-time dfont\" itemprop=\"datePublished\" datetime=\"2022-04-20\">2022\u5e744\u670820\u65e5<\/time><span class=\"longc_title\">AI-Tech-Lab\u306e2\u3064\u306e\u81ea\u793e\u958b\u767a\u7d39\u4ecb\u3010\u5b66\u751f\u30a8\u30f3\u30b8\u30cb\u30a2\u306b\u3088\u308bAI\u958b\u767a\u3011<\/span><\/span>\n<\/a>\n\n\n<h3 class=\"wp-block-sgb-headings sgb-heading\" id=\"i-9\"><span class=\"ez-toc-section\" id=\"%E5%8E%9F%E6%96%87\"><\/span><span class=\"sgb-heading__inner sgb-heading--type3\" style=\"background-color:#ccd9f4;border-color:#1b479c;font-size:1.2em\"><span class=\"sgb-heading__text\" style=\"color:#333\">\u539f\u6587<\/span><\/span><span class=\"ez-toc-section-end\"><\/span><\/h3>\n\n\n\n<div class=\"wp-block-sgb-sanko\"><a href=\"https:\/\/openreview.net\/forum?id=YicbFdNTTy\" class=\"reference table\" target=\"_blank\" rel=\"noopener noreferrer\"><span class=\"tbcell refttl\"><i class=\"fas fa-bookmark\" style=\"color: #ffb36b\"><\/i>\u53c2\u8003<\/span><span class=\"tbcell refcite\">An Image is Worth 16&#215;16 Words: Transformers for Image Recognition&#8230;<span>OpenReview<\/span><\/span><\/a><\/div>\n","protected":false},"excerpt":{"rendered":"<p>Vision Transformer\u3068\u306f\u3001\u8ad6\u6587\u300cAn Image is Worth 16&#215;16 Words: Transformers for Image Recognition at Scale\u300d\u3067\u63d0\u6848\u3055\u308c\u305f &#8230; <\/p>\n","protected":false},"author":5,"featured_media":1388,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[11],"tags":[6,33],"class_list":{"0":"post-1339","1":"post","2":"type-post","3":"status-publish","4":"format-standard","5":"has-post-thumbnail","7":"category-artificial-intelligence","8":"tag-dl","9":"tag-33"},"aioseo_notices":[],"jetpack_featured_media_url":"https:\/\/static.maru.nagoya\/wp-content\/uploads\/2022\/04\/22145905\/Blue-and-Slate-Gray-Eye-Donation-Poster-1.jpg","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/posts\/1339","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/users\/5"}],"replies":[{"embeddable":true,"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/comments?post=1339"}],"version-history":[{"count":120,"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/posts\/1339\/revisions"}],"predecessor-version":[{"id":1814,"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/posts\/1339\/revisions\/1814"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/media\/1388"}],"wp:attachment":[{"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/media?parent=1339"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/categories?post=1339"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/maru.nagoya\/index.php\/wp-json\/wp\/v2\/tags?post=1339"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}