# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449
# PT

## GENERAL SETTINGS

# Enable robots.txt rules for all crawlers
User-agent: *

# Crawl-delay parameter: number of seconds to wait between successive requests to the same server.
# Set a custom crawl rate if you're experiencing traffic problems with your server.
Crawl-delay: 30

# Do not crawl admin
Disallow: /admin/

# Do not crawl PHP interfaces (fit to your needs)
Disallow: /index.php
Disallow: /api.php
Disallow: /get.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /install.php

# Assets paths
Disallow: /*.js$
Disallow: /*.css$
Disallow: /*.php$
Disallow: /*?limit=all

## COMMON SERVER SETTINGS
# Do not crawl common server technical folders and files
Disallow: /*cgi-bin/
Disallow: /*report/
Disallow: /*stats/
Disallow: /cleanup.php
Disallow: /apc.php
Disallow: /memcache.php
Disallow: /phpinfo.php

## GENERAL MEDIA SETTINGS
# Special for media indexing

User-agent: Googlebot-Image
Disallow: /
Allow: /media/

User-agent: msnbot-media
Disallow: /
Allow: /media/