# www.robotstxt.org/ User-agent: Mozilla/5.0 (compatible; Taboolabot/3.7; +http://www.taboola.com) Disallow: # Allow crawling of all content User-agent: * Disallow: /control Disallow: /api/* Disallow: /assets/*