Add a robots.txt file to disallow indexing our profile links (PR #4753)

firefox-devtools · Sep 19, 2023 · 55f781c · 55f781c
1 parent 2a8c44b
commit 55f781c
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/res/robots.txt b/res/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Disallow: /public/
+Disallow: /from-url/
+Disallow: /compare/
diff --git a/webpack.config.js b/webpack.config.js
@@ -99,6 +99,7 @@ const config = {
         'res/_redirects',
         'res/zee-worker.js',
         'res/contribute.json',
+        'res/robots.txt',
         'res/service-worker-compat.js',
         { from: 'docs-user', to: 'docs' },
         { from: 'locales', to: 'locales' },