User-agent: Scrapy
Allow: /

User-agent: Scrapy
Allow: /

User-agent: *
Disallow: /wp-snapshots/
Disallow: /wp-content/uploads/*.pdf$
Disallow: /wp-content/uploads/*/*.pdf$
Disallow: /wp-content/uploads/*/*/*.pdf$

# START YOAST BLOCK
# ---------------------------
Allow: /

Sitemap: https://www.guidelondon.org.uk/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK