Sitemap & Robots.txt

<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url>
    <loc>https://yoursite.com/</loc>
    <lastmod>2024-01-15</lastmod>
    <changefreq>weekly</changefreq>
    <priority>1.0</priority>
  </url>
  <url>
    <loc>https://yoursite.com/about</loc>
    <lastmod>2024-01-10</lastmod>
    <changefreq>monthly</changefreq>
    <priority>0.8</priority>
  </url>
</urlset>

npm install next-sitemap

/** @type {import('next-sitemap').IConfig} */
module.exports = {
  siteUrl: 'https://yoursite.com',
  generateRobotsTxt: true,
  changefreq: 'weekly',
  priority: 0.7,
  sitemapSize: 5000,
  exclude: ['/admin/*', '/api/*', '/private/*'],
  robotsTxtOptions: {
    additionalSitemaps: [
      'https://yoursite.com/server-sitemap.xml',
    ],
    policies: [
      { userAgent: '*', allow: '/' },
      { userAgent: '*', disallow: ['/admin', '/api', '/private'] },
    ],
  },
};

{
  "scripts": {
    "build": "next build",
    "postbuild": "next-sitemap"
  }
}

// app/sitemap.ts
import { MetadataRoute } from 'next';
 
export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
  const baseUrl = 'https://yoursite.com';
 
  // Static pages
  const staticPages = [
    { url: baseUrl, lastModified: new Date(), priority: 1.0 },
    { url: `${baseUrl}/about`, lastModified: new Date(), priority: 0.8 },
    { url: `${baseUrl}/docs`, lastModified: new Date(), priority: 0.9 },
  ];
 
  // Dynamic pages (from database/CMS)
  const posts = await getPosts();
  const postPages = posts.map((post) => ({
    url: `${baseUrl}/blog/${post.slug}`,
    lastModified: new Date(post.updatedAt),
    priority: 0.7,
  }));
 
  return [...staticPages, ...postPages];
}

// pages/server-sitemap.xml.tsx
import { getServerSideSitemapLegacy } from 'next-sitemap';
import { GetServerSideProps } from 'next';
 
export const getServerSideProps: GetServerSideProps = async (ctx) => {
  const posts = await fetch('https://api.yoursite.com/posts').then(r => r.json());
 
  const fields = posts.map((post) => ({
    loc: `https://yoursite.com/blog/${post.slug}`,
    lastmod: post.updatedAt,
    changefreq: 'weekly',
    priority: 0.7,
  }));
 
  return getServerSideSitemapLegacy(ctx, fields);
};
 
export default function Sitemap() {}

# Allow all crawlers
User-agent: *
Allow: /

# Block specific paths
Disallow: /admin/
Disallow: /api/
Disallow: /private/

# Sitemap location
Sitemap: https://yoursite.com/sitemap.xml

// app/robots.ts
import { MetadataRoute } from 'next';
 
export default function robots(): MetadataRoute.Robots {
  return {
    rules: [
      {
        userAgent: '*',
        allow: '/',
        disallow: ['/admin/', '/api/', '/private/'],
      },
    ],
    sitemap: 'https://yoursite.com/sitemap.xml',
  };
}

// app/robots.ts
export default function robots(): MetadataRoute.Robots {
  const isProduction = process.env.VERCEL_ENV === 'production';
 
  if (!isProduction) {
    return {
      rules: { userAgent: '*', disallow: '/' },
    };
  }
 
  return {
    rules: { userAgent: '*', allow: '/' },
    sitemap: 'https://yoursite.com/sitemap.xml',
  };
}

<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <sitemap>
    <loc>https://yoursite.com/sitemap-pages.xml</loc>
    <lastmod>2024-01-15</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://yoursite.com/sitemap-blog.xml</loc>
    <lastmod>2024-01-15</lastmod>
  </sitemap>
  <sitemap>
    <loc>https://yoursite.com/sitemap-products.xml</loc>
    <lastmod>2024-01-15</lastmod>
  </sitemap>
</sitemapindex>