diff --git a/nginx-reverse-proxy/nginx.conf b/nginx-reverse-proxy/nginx.conf index 27c72d8..037566b 100644 --- a/nginx-reverse-proxy/nginx.conf +++ b/nginx-reverse-proxy/nginx.conf @@ -14,42 +14,95 @@ http { default 0; "~*Prerender" 0; - "~*googlebot" 1; - "~*yahoo!\ slurp" 1; - "~*bingbot" 1; - "~*yandex" 1; - "~*baiduspider" 1; - "~*facebookexternalhit" 1; - "~*twitterbot" 1; - "~*rogerbot" 1; - "~*linkedinbot" 1; - "~*embedly" 1; - "~*quora\ link\ preview" 1; - "~*showyoubot" 1; - "~*outbrain" 1; - "~*pinterest\/0\." 1; - "~*developers.google.com\/\+\/web\/snippet" 1; - "~*slackbot" 1; - "~*vkshare" 1; - "~*w3c_validator" 1; - "~*redditbot" 1; - "~*applebot" 1; - "~*whatsapp" 1; - "~*flipboard" 1; - "~*tumblr" 1; - "~*bitlybot" 1; - "~*skypeuripreview" 1; - "~*nuzzel" 1; - "~*discordbot" 1; - "~*google\ page\ speed" 1; - "~*qwantify" 1; - "~*pinterestbot" 1; - "~*bitrix\ link\ preview" 1; - "~*xing-contenttabreceiver" 1; - "~*chrome-lighthouse" 1; - "~*telegrambot" 1; - "~*google-inspectiontool" 1; - "~*petalbot" 1; + # Search Engines + "~*googlebot" 1; + "~*adsbot-google" 1; + "~*apis-google" 1; + "~*mediapartners-google" 1; + "~*google-safety" 1; + "~*feedfetcher-google" 1; + "~*googleproducer" 1; + "~*google-site-verification" 1; + "~*bingbot" 1; + "~*yandexbot" 1; + "~*yabrowser" 1; + "~*yahoo" 1; + "~*baiduspider" 1; + "~*naver" 1; + "~*seznambot" 1; + "~*sznprohlizec" 1; + "~*qwantbot" 1; + "~*ecosia" 1; + "~*duckduckbot" 1; + "~*duckassistbot" 1; + "~*applebot" 1; + + # Social Media + "~*facebookexternalhit" 1; + "~*facebookcatalog" 1; + "~*facebookbot" 1; + "~*meta-externalagent" 1; + "~*instagram" 1; + "~*twitterbot" 1; + "~*linkedinbot" 1; + "~*whatsapp" 1; + "~*slackbot" 1; + "~*pinterest" 1; + "~*tiktok" 1; + "~*tiktokspider" 1; + "~*bytespider" 1; + "~*discordbot" 1; + + # SEO Tools + "~*semrushbot" 1; + "~*ahrefsbot" 1; + "~*chrome-lighthouse" 1; + "~*screaming-frog" 1; + "~*oncrawlbot" 1; + "~*botifybot" 1; + "~*deepcrawl" 1; + "~*lumar" 1; + "~*rogerbot" 1; + "~*dotbot" 1; + + # AI Bots + "~*gptbot" 1; + "~*chatgpt" 1; + "~*oai-searchbot" 1; + "~*chatgpt-user" 1; + "~*claudebot" 1; + "~*google-extended" 1; + "~*perplexitybot" 1; + "~*perplexity-user" 1; + "~*youbot" 1; + "~*amazonbot" 1; + "~*anthropic-ai" 1; + "~*claude-web" 1; + "~*ccbot" 1; + "~*mistralai-user" 1; + + # Other bots + "~*embedly" 1; + "~*quora\ link\ preview" 1; + "~*showyoubot" 1; + "~*outbrain" 1; + "~*developers.google.com/\+/web/snippet" 1; + "~*vkshare" 1; + "~*w3c_validator" 1; + "~*redditbot" 1; + "~*flipboard" 1; + "~*tumblr" 1; + "~*bitlybot" 1; + "~*skypeuripreview" 1; + "~*nuzzel" 1; + "~*google\ page\ speed" 1; + "~*qwantify" 1; + "~*pinterestbot" 1; + "~*bitrix\ link\ preview" 1; + "~*xing-contenttabreceiver" 1; + "~*telegrambot" 1; + "~*google-inspectiontool" 1; + "~*petalbot" 1; } map $args $prerender_args {