diff --git a/site/layouts/robots.txt b/site/layouts/robots.txt
index e9d78f1aed..c6703cd06c 100644
--- a/site/layouts/robots.txt
+++ b/site/layouts/robots.txt
@@ -1,8 +1,12 @@
 # www.robotstxt.org
 
-{{ if (eq (getenv "HUGO_ENV") "production") -}}
+{{- $isProduction := eq (getenv "HUGO_ENV") "production" -}}
+{{- $isNetlify := eq (getenv "NETLIFY") "true" -}}
+{{- $allowCrawling := and (not $isNetlify) $isProduction -}}
+
+{{ if $allowCrawling }}
 # Allow crawling of all content
 {{- end }}
 User-agent: *
-Disallow:{{ if (ne (getenv "HUGO_ENV") "production") }} /{{ end }}
+Disallow:{{ if not $allowCrawling }} /{{ end }}
 Sitemap: {{ .Site.BaseURL }}/sitemap.xml