Removed Unwanted Agents in Robots file

public/robots.txt

@@ -2,179 +2,6 @@
 
 User-agent: *
 Allow: /humans.txt
-Disallow: /
-
-User-agent: Googlebot
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: duckduckbot
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: HTTrack
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: ia_archiver
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: msnbot
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: Bingbot
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: naverbot
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: Slurp
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: Swiftbot
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: Twitterbot
-Allow: /
-Allow: /*/*/src/master
-Disallow: /admin
-Disallow: /*/*/forks
-Disallow: /*/*/stars
-Disallow: /*/*/pulls
-Disallow: /*/*/archive/
-Disallow: /*/*/raw/*
-Disallow: /*/*/commits
-Disallow: /*/followers
-Disallow: /*/following
-Disallow: /*/*/src/*
-Disallow: /*/*/issues/new
-Disallow: /*/*/issues?
-Disallow: /*?
-
-User-agent: Yandex
 Allow: /
 Allow: /*/*/src/master
 Disallow: /admin