Updated Robots File

public/robots.txt

@@ -6,4 +6,88 @@ Disallow: /
 
 User-agent: Googlebot
 Allow: /
-Disallow: /admin/*
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: duckduckbot
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: HTTrack
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: ia_archiver
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: msnbot
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: Bingbot
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: naverbot
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: Slurp
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: Swiftbot
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: Twitterbot
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*
+
+User-agent: Yandex
+Allow: /
+Disallow: /admin/*
+Disallow: /*/*/forks
+Disallow: /*/*/stars
+Disallow: /*/*/archive/
+Disallow: /*/*/raw/*