robots.txt 597 B

12345678910111213141516171819202122232425262728
  1. # If you would like to crawl Gitote contact us at support@gitote.in.
  2. User-agent: *
  3. Allow: /humans.txt
  4. Allow: /
  5. Allow: /*/*/src/master
  6. Disallow: /admin
  7. Disallow: /*/*/forks
  8. Disallow: /*/*/stars
  9. Disallow: /*/*/pulls
  10. Disallow: /*/*/watchers
  11. Disallow: /*/*/archive
  12. Disallow: /*/*/settings
  13. Disallow: /*/*/raw/*
  14. Disallow: /*/*/wiki/_new
  15. Disallow: /*/*/wiki/*/_edit
  16. Disallow: /*/*/commits
  17. Disallow: /*/followers
  18. Disallow: /*/following
  19. Disallow: /*/*/src/*
  20. Disallow: /*/*/_edit
  21. Disallow: /*/*/issues/new
  22. Disallow: /*/*/issues?
  23. Disallow: /*/*/compare
  24. Disallow: /*?
  25. Sitemap: https://gitote.in/sitemap.xml