|
|
@@ -6,7 +6,7 @@ Disallow: /
|
|
|
|
|
|
User-agent: Googlebot
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -14,7 +14,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: duckduckbot
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -22,7 +22,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: HTTrack
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -30,7 +30,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: ia_archiver
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -38,7 +38,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: msnbot
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -46,7 +46,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: Bingbot
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -54,7 +54,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: naverbot
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -62,7 +62,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: Slurp
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -70,7 +70,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: Swiftbot
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -78,7 +78,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: Twitterbot
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|
|
|
@@ -86,7 +86,7 @@ Disallow: /*/*/raw/*
|
|
|
|
|
|
User-agent: Yandex
|
|
|
Allow: /
|
|
|
-Disallow: /admin/*
|
|
|
+Disallow: /admin
|
|
|
Disallow: /*/*/forks
|
|
|
Disallow: /*/*/stars
|
|
|
Disallow: /*/*/archive/
|