update crawler list

roipmars · Oct 22, 2024 · 6108357 · 6108357
1 parent 298cfc7
commit 6108357
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/robots.txt b/robots.txt
@@ -1,5 +1,9 @@
 User-agent: *
-Disallow: 
+
 Crawl-delay: 15
+
 Disallow: /assets/
-Sitemap: https://www.roipmars.org.my/sitemap.xml
+Disallow: /media/
+
+Sitemap: https://www.roipmars.org.my/sitemap.xml
+Sitemap: https://www.roipmars.org.my/sitemap_images.xml