diff --git a/packages/vite/src/node/plugins/asset.ts b/packages/vite/src/node/plugins/asset.ts
index 4cdd846e4..4484978a6 100644
--- a/packages/vite/src/node/plugins/asset.ts
+++ b/packages/vite/src/node/plugins/asset.ts
@@ -329,6 +329,7 @@ async function fileToBuiltUrl(
   config: ResolvedConfig,
   pluginContext: PluginContext,
   skipPublicCheck = false,
+  shouldInline?: boolean,
 ): Promise<string> {
   if (!skipPublicCheck && checkPublicFile(id, config)) {
     return publicFileToBuiltUrl(id, config)
@@ -343,15 +344,18 @@ async function fileToBuiltUrl(
   const file = cleanUrl(id)
   const content = await fsp.readFile(file)
 
+  if (shouldInline == null) {
+    shouldInline =
+      !!config.build.lib ||
+      // Don't inline SVG with fragments, as they are meant to be reused
+      (!(file.endsWith('.svg') && id.includes('#')) &&
+        !file.endsWith('.html') &&
+        content.length < Number(config.build.assetsInlineLimit) &&
+        !isGitLfsPlaceholder(content))
+  }
+
   let url: string
-  if (
-    config.build.lib ||
-    // Don't inline SVG with fragments, as they are meant to be reused
-    (!(file.endsWith('.svg') && id.includes('#')) &&
-      !file.endsWith('.html') &&
-      content.length < Number(config.build.assetsInlineLimit) &&
-      !isGitLfsPlaceholder(content))
-  ) {
+  if (shouldInline) {
     if (config.build.lib && isGitLfsPlaceholder(content)) {
       config.logger.warn(
         colors.yellow(`Inlined file ${id} was not downloaded via Git LFS`),
@@ -392,6 +396,7 @@ export async function urlToBuiltUrl(
   importer: string,
   config: ResolvedConfig,
   pluginContext: PluginContext,
+  shouldInline?: boolean,
 ): Promise<string> {
   if (checkPublicFile(url, config)) {
     return publicFileToBuiltUrl(url, config)
@@ -406,6 +411,7 @@ export async function urlToBuiltUrl(
     pluginContext,
     // skip public check since we just did it above
     true,
+    shouldInline,
   )
 }
 
diff --git a/packages/vite/src/node/plugins/html.ts b/packages/vite/src/node/plugins/html.ts
index 8440abe42..b9fe1637d 100644
--- a/packages/vite/src/node/plugins/html.ts
+++ b/packages/vite/src/node/plugins/html.ts
@@ -50,6 +50,7 @@ const inlineCSSRE = /__VITE_INLINE_CSS__([a-z\d]{8}_\d+)__/g
 const inlineImportRE =
   /(?<!(?<!\.\.)\.)\bimport\s*\(("(?:[^"]|(?<=\\)")*"|'(?:[^']|(?<=\\)')*')\)/dg
 const htmlLangRE = /\.(?:html|htm)$/
+const spaceRe = /[\t\n\f\r ]/
 
 const importMapRE =
   /[ \t]*<script[^>]*type\s*=\s*(?:"importmap"|'importmap'|importmap)[^>]*>.*?<\/script>/is
@@ -141,6 +142,17 @@ export const assetAttrsConfig: Record<string, string[]> = {
   use: ['xlink:href', 'href'],
 }
 
+// Some `<link rel>` elements should not be inlined in build. Excluding:
+// - `shortcut`                     : only valid for IE <9, use `icon`
+// - `mask-icon`                    : deprecated since Safari 12 (for pinned tabs)
+// - `apple-touch-icon-precomposed` : only valid for iOS <7 (for avoiding gloss effect)
+const noInlineLinkRels = new Set([
+  'icon',
+  'apple-touch-icon',
+  'apple-touch-startup-image',
+  'manifest',
+])
+
 export const isAsyncScriptMap = new WeakMap<
   ResolvedConfig,
   Map<string, boolean>
@@ -386,14 +398,14 @@ export function buildHtmlPlugin(config: ResolvedConfig): Plugin {
         const namedOutput = Object.keys(
           config?.build?.rollupOptions?.input || {},
         )
-        const processAssetUrl = async (url: string) => {
+        const processAssetUrl = async (url: string, shouldInline?: boolean) => {
           if (
             url !== '' && // Empty attribute
             !namedOutput.includes(url) && // Direct reference to named output
             !namedOutput.includes(removeLeadingSlash(url)) // Allow for absolute references as named output can't be an absolute path
           ) {
             try {
-              return await urlToBuiltUrl(url, id, config, this)
+              return await urlToBuiltUrl(url, id, config, this, shouldInline)
             } catch (e) {
               if (e.code !== 'ENOENT') {
                 throw e
@@ -522,9 +534,26 @@ export function buildHtmlPlugin(config: ResolvedConfig): Plugin {
                       })
                       js += importExpression
                     } else {
+                      // If the node is a link, check if it can be inlined. If not, set `shouldInline`
+                      // to `false` to force no inline. If `undefined`, it leaves to the default heuristics.
+                      const isNoInlineLink =
+                        node.nodeName === 'link' &&
+                        node.attrs.some(
+                          (p) =>
+                            p.name === 'rel' &&
+                            p.value
+                              .split(spaceRe)
+                              .some((v) =>
+                                noInlineLinkRels.has(v.toLowerCase()),
+                              ),
+                        )
+                      const shouldInline = isNoInlineLink ? false : undefined
                       assetUrlsPromises.push(
                         (async () => {
-                          const processedUrl = await processAssetUrl(url)
+                          const processedUrl = await processAssetUrl(
+                            url,
+                            shouldInline,
+                          )
                           if (processedUrl !== url) {
                             overwriteAttrValue(
                               s,
diff --git a/playground/assets/__tests__/assets.spec.ts b/playground/assets/__tests__/assets.spec.ts
index 74aa0f3ff..c4c80e28a 100644
--- a/playground/assets/__tests__/assets.spec.ts
+++ b/playground/assets/__tests__/assets.spec.ts
@@ -223,10 +223,20 @@ describe('css url() references', () => {
     const match = isBuild ? `data:image/png;base64` : `/foo/bar/nested/icon.png`
     expect(await getBg('.css-url-base64-inline')).toMatch(match)
     expect(await getBg('.css-url-quotes-base64-inline')).toMatch(match)
-    const icoMatch = isBuild ? `data:image/x-icon;base64` : `favicon.ico`
-    const el = await page.$(`link.ico`)
-    const href = await el.getAttribute('href')
-    expect(href).toMatch(icoMatch)
+  })
+
+  test('no base64 inline for icon and manifest links', async () => {
+    const iconEl = await page.$(`link.ico`)
+    const href = await iconEl.getAttribute('href')
+    expect(href).toMatch(
+      isBuild ? /\/foo\/bar\/assets\/favicon-[-\w]{8}\.ico/ : 'favicon.ico',
+    )
+
+    const manifestEl = await page.$(`link[rel="manifest"]`)
+    const manifestHref = await manifestEl.getAttribute('href')
+    expect(manifestHref).toMatch(
+      isBuild ? /\/foo\/bar\/assets\/manifest-[-\w]{8}\.json/ : 'manifest.json',
+    )
   })
 
   test('multiple urls on the same line', async () => {
diff --git a/playground/assets/index.html b/playground/assets/index.html
index e0684db90..1765bc6d8 100644
--- a/playground/assets/index.html
+++ b/playground/assets/index.html
@@ -3,6 +3,7 @@
 <head>
   <meta charset="UTF-8" />
   <link class="ico" rel="icon" type="image/svg+xml" href="favicon.ico" />
+  <link rel="manifest" href="manifest.json" />
 </head>
 
 <link class="data-href" rel="icon" href="data:," />
diff --git a/playground/assets/manifest.json b/playground/assets/manifest.json
new file mode 100644
index 000000000..0967ef424
--- /dev/null
+++ b/playground/assets/manifest.json
@@ -0,0 +1 @@
+{}