User-agent: * #Standard exclusions Disallow: /assets/ Disallow: /customerrors/ Disallow: /graphics/ Disallow: /images/ #Database related files