robots.txt 616 B

123456789101112131415161718192021222324252627282930
  1. # If you would like to crawl Gitote contact us at support@gitote.in.
  2. User-agent: *
  3. Allow: /humans.txt
  4. Allow: /
  5. Allow: /login
  6. Allow: /join
  7. Allow: /*/*/src/master
  8. Disallow: /admin
  9. Disallow: /user
  10. Disallow: /*/*/forks
  11. Disallow: /*/*/stars
  12. Disallow: /*/*/pulls
  13. Disallow: /*/*/archive
  14. Disallow: /*/*/settings
  15. Disallow: /*/*/raw/*
  16. Disallow: /*/*/wiki/_new
  17. Disallow: /*/*/wiki/*/_edit
  18. Disallow: /*/*/commits
  19. Disallow: /*/followers
  20. Disallow: /*/following
  21. Disallow: /*/*/src/*
  22. Disallow: /*/*/_edit
  23. Disallow: /*/*/issues/new
  24. Disallow: /*/*/issues?
  25. Disallow: /*/*/compare
  26. Disallow: /*?
  27. Sitemap: https://gitote.in/sitemap.xml