added robots.txt and sitemap

astro.config.mjs

@@ -1,18 +1,59 @@
 import { defineConfig } from 'astro/config';
 import dotenv from 'dotenv';
-import node from '@astrojs/node'; // Add this import
+import node from '@astrojs/node';
+import sitemap from '@astrojs/sitemap';
 
 // Environment detection
 const isProd = process.env.NODE_ENV === 'production';
-
 dotenv.config();
 
 export default defineConfig({
+  site: 'https://juchatz.com', // Required for sitemap generation
   output: 'server', 
   
   adapter: node({
     mode: 'standalone'
   }),
+  
+  // Integrations
+  integrations: [
+    sitemap({
+      changefreq: 'monthly',
+      priority: 0.8,
+      serialize(item) {
+        // Homepage gets highest priority
+        if (item.url === 'https://juchatz.com/' || item.url === 'https://juchatz.com') {
+          item.priority = 1.0;
+          item.changefreq = 'monthly';
+        }
+        // Main services page
+        else if (item.url === 'https://juchatz.com/services/' || item.url === 'https://juchatz.com/services') {
+          item.priority = 0.9;
+          item.changefreq = 'monthly';
+        }
+        // Individual service pages
+        else if (item.url.includes('/services/')) {
+          item.priority = 0.8;
+          item.changefreq = 'monthly';
+        }
+        // About page
+        else if (item.url.includes('/about')) {
+          item.priority = 0.8;
+          item.changefreq = 'quarterly';
+        }
+        // Contact page gets lower priority
+        else if (item.url.includes('/contact')) {
+          item.priority = 0.7;
+          item.changefreq = 'quarterly';
+        }
+        
+        return item;
+      },
+      // Filter out API routes and other non-page URLs
+      filter: (page) => !page.includes('/api/')
+    })
+  ],
+  
   // Development server settings
   server: {
     host: '0.0.0.0',

package-lock.json

@@ -9,6 +9,7 @@
       "version": "0.0.1",
       "dependencies": {
         "@astrojs/node": "^9.2.1",
+        "@astrojs/sitemap": "^3.4.1",
         "astro": "^5.7.13",
         "dotenv": "^16.5.0"
       }
@@ -76,6 +77,17 @@
         "node": "^18.17.1 || ^20.3.0 || >=22.0.0"
       }
     },
+    "node_modules/@astrojs/sitemap": {
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/@astrojs/sitemap/-/sitemap-3.4.1.tgz",
+      "integrity": "sha512-VjZvr1e4FH6NHyyHXOiQgLiw94LnCVY4v06wN/D0gZKchTMkg71GrAHJz81/huafcmavtLkIv26HnpfDq6/h/Q==",
+      "license": "MIT",
+      "dependencies": {
+        "sitemap": "^8.0.0",
+        "stream-replace-string": "^2.0.0",
+        "zod": "^3.24.2"
+      }
+    },
     "node_modules/@astrojs/telemetry": {
       "version": "3.2.1",
       "resolved": "https://registry.npmjs.org/@astrojs/telemetry/-/telemetry-3.2.1.tgz",
@@ -1273,6 +1285,15 @@
         "undici-types": "~6.21.0"
       }
     },
+    "node_modules/@types/sax": {
+      "version": "1.2.7",
+      "resolved": "https://registry.npmjs.org/@types/sax/-/sax-1.2.7.tgz",
+      "integrity": "sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/unist": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz",
@@ -1384,6 +1405,12 @@
         "url": "https://github.com/sponsors/jonschlinkert"
       }
     },
+    "node_modules/arg": {
+      "version": "5.0.2",
+      "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
+      "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==",
+      "license": "MIT"
+    },
     "node_modules/argparse": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
@@ -3856,6 +3883,12 @@
         "fsevents": "~2.3.2"
       }
     },
+    "node_modules/sax": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.4.1.tgz",
+      "integrity": "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg==",
+      "license": "ISC"
+    },
     "node_modules/semver": {
       "version": "7.7.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
@@ -3969,6 +4002,31 @@
       "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
       "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="
     },
+    "node_modules/sitemap": {
+      "version": "8.0.0",
+      "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-8.0.0.tgz",
+      "integrity": "sha512-+AbdxhM9kJsHtruUF39bwS/B0Fytw6Fr1o4ZAIAEqA6cke2xcoO2GleBw9Zw7nRzILVEgz7zBM5GiTJjie1G9A==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/node": "^17.0.5",
+        "@types/sax": "^1.2.1",
+        "arg": "^5.0.0",
+        "sax": "^1.2.4"
+      },
+      "bin": {
+        "sitemap": "dist/cli.js"
+      },
+      "engines": {
+        "node": ">=14.0.0",
+        "npm": ">=6.0.0"
+      }
+    },
+    "node_modules/sitemap/node_modules/@types/node": {
+      "version": "17.0.45",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-17.0.45.tgz",
+      "integrity": "sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==",
+      "license": "MIT"
+    },
     "node_modules/smol-toml": {
       "version": "1.3.4",
       "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.3.4.tgz",
@@ -4006,6 +4064,12 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/stream-replace-string": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/stream-replace-string/-/stream-replace-string-2.0.0.tgz",
+      "integrity": "sha512-TlnjJ1C0QrmxRNrON00JvaFFlNh5TTG00APw23j74ET7gkQpTASi6/L2fuiav8pzK715HXtUeClpBTw2NPSn6w==",
+      "license": "MIT"
+    },
     "node_modules/string-width": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",

package.json

@@ -10,6 +10,7 @@
   },
   "dependencies": {
     "@astrojs/node": "^9.2.1",
+    "@astrojs/sitemap": "^3.4.1",
     "astro": "^5.7.13",
     "dotenv": "^16.5.0"
   }

public/robots.txt

@@ -0,0 +1,112 @@
+# Block all bots by default
+User-agent: *
+Disallow: /
+
+# Allow legitimate search engines (for discoverability)
+User-agent: Googlebot
+Allow: /
+Crawl-delay: 10
+
+User-agent: Bingbot
+Allow: /
+Crawl-delay: 15
+
+User-agent: DuckDuckBot
+Allow: /
+Crawl-delay: 10
+
+User-agent: Slurp
+Allow: /
+Crawl-delay: 20
+
+User-agent: Applebot
+Allow: /
+Crawl-delay: 15
+
+User-agent: Brave
+Allow: /
+Crawl-delay: 10
+
+User-agent: StartPageBot
+Allow: /
+Crawl-delay: 10
+
+User-agent: Qwantbot
+Allow: /
+Crawl-delay: 15
+
+# Block Chinese/Russian search engines
+User-agent: Baiduspider
+Disallow: /
+
+User-agent: YandexBot
+Disallow: /
+
+# Block SEO and analysis tools (these provide no value)
+User-agent: AhrefsBot
+Disallow: /
+
+User-agent: SemrushBot
+Disallow: /
+
+User-agent: MJ12bot
+Disallow: /
+
+User-agent: DotBot
+Disallow: /
+
+User-agent: BLEXBot
+Disallow: /
+
+User-agent: SiteAuditBot
+Disallow: /
+
+User-agent: MegaIndex
+Disallow: /
+
+User-agent: SiteAuditBot
+Disallow: /
+
+User-agent: PetalBot
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Claude-Web
+Disallow: /
+
+# Block social media crawlers (optional - they help with link previews)
+User-agent: facebookexternalhit
+Disallow: /
+
+User-agent: Twitterbot
+Disallow: /
+
+User-agent: LinkedInBot
+Disallow: /
+
+User-agent: WhatsApp
+Disallow: /
+
+# Block archiving services
+User-agent: ia_archiver
+Disallow: /
+
+User-agent: archive.org_bot
+Disallow: /
+
+User-agent: WaybackMachine
+Disallow: /
+
+# Sitemap location
+Sitemap: https://juchatz.com/sitemap.xml