# robots.txt zu http://www.defi.de/

User-agent: *
Disallow: /cgi-bin/
Disallow: /admin/
Disallow: /tmp/

Disallow: /Impressum/
Disallow: /impressum/
Disallow: /Impressum
Disallow: /impressum

Disallow: /Datenschutz/
Disallow: /datenschutz/
Disallow: /Datenschutz
Disallow: /datenschutz

Disallow: /Widerruf/
Disallow: /widerruf/
Disallow: /Widerruf
Disallow: /widerruf

Disallow: /agb/
Disallow: /Agb/
Disallow: /agb
Disallow: /Agb

Disallow: /404/
Disallow: /404

# ===================================
# Schließe folgende Spider komplett aus:
# ===================================

User-agent: WebReaper
User-agent: WebCopier
User-agent: Offline Explorer
User-agent: HTTrack
User-agent: Microsoft.URL.Control
User-agent: EmailCollector
User-agent: penthesilea
User-agent: ia_archiver
Disallow: /