# DigitalSpy Robots.txt file January 2009 # # Added /ZZZZ/ as a disallow in June 2008A - AJ # Added sitemaps during 2008 last ones Nov 08 - AJ # # Jan 2009 parse to find correct server domain name # # www.digitalspy.co.uk # # for Sitemaps. sitemap: http://www.digitalspy.co.uk/ds-www-sitemap_index.xml sitemap: http://www.digitalspy.co.uk/sitemaps_index.xml sitemap: http://www.digitalspy.co.uk/tags_sitemaps.xml sitemap: http://www.digitalspy.co.uk/sitemaps_news.xml # Main config. User-agent: * Disallow: /dynamic/ Disallow: /static/ Disallow: /includes/ Disallow: /test/ Disallow: /ZZZZ/ Disallow: /d/ Disallow: /page_cache/ Disallow: /sitecaching/ Disallow: /search-old/ Disallow: /caching/ #