pi3c/pyproger
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

robots

Browse Source
This commit is contained in:
Сергей Ванюшкин
2023-10-16 23:22:21 +03:00
parent 54c40985cb
commit 30f2dbde98
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pyproger/templates/robots/robots.txt
View File

@@ -5,6 +5,7 @@ Disallow: /static/
Disallow: /post/$ Disallow: /post/$
Disallow: /tag/ Disallow: /tag/
Disallow: /tags/ Disallow: /tags/
Allow: /post/*
Allow: /static/*.css Allow: /static/*.css
Allow: /static/*.js Allow: /static/*.js
Allow: /static/*.png Allow: /static/*.png
@@ -18,6 +19,7 @@ Disallow: /static/
Disallow: /post/$ Disallow: /post/$
Disallow: /tag/ Disallow: /tag/
Disallow: /tags/ Disallow: /tags/
Allow: /post/*
Allow: /static/*.css Allow: /static/*.css
Allow: /static/*.js Allow: /static/*.js
Allow: /static/*.png Allow: /static/*.png
@@ -31,6 +33,7 @@ Disallow: /static/
Disallow: /post/$ Disallow: /post/$
Disallow: /tag/ Disallow: /tag/
Disallow: /tags/ Disallow: /tags/
Allow: /post/*
Crawl-Delay: 5 Crawl-Delay: 5
Sitemap: https://pyproger.ru/sitemap.xml Sitemap: https://pyproger.ru/sitemap.xml