robots.txt 573 B

123456789101112131415161718192021222324252627
  1. # If you would like to crawl Gitote contact us at support@gitote.in.
  2. User-agent: *
  3. Allow: /humans.txt
  4. Allow: /
  5. Allow: /*/*/src/master
  6. Disallow: /admin
  7. Disallow: /*/*/forks
  8. Disallow: /*/*/stars
  9. Disallow: /*/*/pulls
  10. Disallow: /*/*/archive
  11. Disallow: /*/*/settings
  12. Disallow: /*/*/raw/*
  13. Disallow: /*/*/wiki/_new
  14. Disallow: /*/*/wiki/*/_edit
  15. Disallow: /*/*/commits
  16. Disallow: /*/followers
  17. Disallow: /*/following
  18. Disallow: /*/*/src/*
  19. Disallow: /*/*/_edit
  20. Disallow: /*/*/issues/new
  21. Disallow: /*/*/issues?
  22. Disallow: /*/*/compare
  23. Disallow: /*?
  24. Sitemap: https://gitote.in/sitemap.xml