feat: split sitemap

backend/src/app/sitemap-index.xml/route.ts

@@ -0,0 +1,24 @@
+import { prisma } from "@/lib/prisma";
+import { CHUNK_SIZE } from "../sitemap";
+import { NextResponse } from "next/server";
+
+export async function GET() {
+	const baseUrl = process.env.NEXT_PUBLIC_FRONTEND_URL!;
+
+	const total = await prisma.mii.count({
+		where: { in_queue: false, quarantined: false },
+	});
+	const miiChunks = Math.ceil(total / CHUNK_SIZE);
+	const totalSitemaps = miiChunks + 1; // +1 for the static-routes sitemap
+
+	const sitemapEntries = Array.from({ length: totalSitemaps }, (_, i) => `<sitemap><loc>${baseUrl}/sitemap/${i}.xml</loc></sitemap>`).join("");
+
+	const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${sitemapEntries}
+</sitemapindex>`;
+
+	return new NextResponse(xml, {
+		headers: { "Content-Type": "application/xml" },
+	});
+}

backend/src/app/sitemap.ts

@@ -1,54 +1,31 @@
 import { prisma } from "@/lib/prisma";
+import { Prisma } from "@prisma/client";
 import type { MetadataRoute } from "next";
 
 type SitemapRoute = MetadataRoute.Sitemap[0];
 
 export const revalidate = 43200; // update every 12 hours
+export const CHUNK_SIZE = 45000; // stay under 50,000 limit
 
-export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-	const baseUrl = process.env.NEXT_PUBLIC_FRONTEND_URL!;
-	const apiUrl = process.env.NEXT_PUBLIC_BASE_URL!;
-
-	const miis = await prisma.mii.findMany({
-		select: {
-			id: true,
-			createdAt: true,
-		},
+export async function generateSitemaps() {
+	const total = await prisma.mii.count({
 		where: {
 			in_queue: false,
 			quarantined: false,
 		},
 	});
+	const miiChunks = Math.ceil(total / CHUNK_SIZE);
 
-	// const users = await prisma.user.findMany({
-	// 	select: {
-	// 		id: true,
-	// 		updatedAt: true,
-	// 	},
-	// });
+	// id 0 = static routes, ids 1..miiChunks = mii chunks
+	return Array.from({ length: miiChunks + 1 }, (_, id) => ({ id }));
+}
 
-	const dynamicRoutes: MetadataRoute.Sitemap = [
-		...miis.map(
-			(mii) =>
-				({
-					url: `${baseUrl}/mii/${mii.id}`,
-					lastModified: mii.createdAt,
-					changeFrequency: "weekly",
-					priority: 0.7,
-					images: [`${apiUrl}/mii/${mii.id}/image?type=metadata`],
-				}) as SitemapRoute,
-		),
-		// ...users.map(
-		// 	(user) =>
-		// 		({
-		// 			url: `${baseUrl}/profile/${user.id}`,
-		// 			lastModified: user.updatedAt,
-		// 			changeFrequency: "weekly",
-		// 			priority: 0.2,
-		// 		}) as SitemapRoute,
-		// ),
-	];
+export default async function sitemap(props: { id: Promise<string> }): Promise<MetadataRoute.Sitemap> {
+	const baseUrl = process.env.NEXT_PUBLIC_FRONTEND_URL!;
+	const apiUrl = process.env.NEXT_PUBLIC_BASE_URL!;
 
+	// id 0: static routes only
+	if ((await props.id) === "0") {
 		const lastModified = new Date();
 
 		return [
@@ -76,6 +53,34 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
 				changeFrequency: "yearly",
 				priority: 0.4,
 			},
-		...dynamicRoutes,
 		];
 	}
+
+	// id 1+: paginated mii chunks
+	const chunkIndex = Number(await props.id) - 1;
+
+	const miis = await prisma.mii.findMany({
+		select: {
+			id: true,
+			createdAt: true,
+		},
+		where: {
+			in_queue: false,
+			quarantined: false,
+		},
+		orderBy: { id: "asc" },
+		skip: chunkIndex * CHUNK_SIZE,
+		take: CHUNK_SIZE,
+	});
+
+	return miis.map(
+		(mii) =>
+			({
+				url: `${baseUrl}/mii/${mii.id}`,
+				lastModified: mii.createdAt,
+				changeFrequency: "weekly",
+				priority: 0.7,
+				images: [`${apiUrl}/mii/${mii.id}/image?type=metadata`],
+			}) as SitemapRoute,
+	);
+}

frontend/public/robots.txt

@@ -10,4 +10,4 @@ Disallow: /report/mii/*
 Disallow: /report/user/*
 Disallow: /admin
 
-Sitemap: https://api.tomodachishare.com/sitemap.xml
+Sitemap: https://api.tomodachishare.com/sitemap-index.xml