feat: split sitemap

backend/src/app/sitemap-index.xml/route.ts

@@ -0,0 +1,24 @@
+import { prisma } from "@/lib/prisma";
+import { CHUNK_SIZE } from "../sitemap";
+import { NextResponse } from "next/server";
+
+export async function GET() {
+	const baseUrl = process.env.NEXT_PUBLIC_FRONTEND_URL!;
+
+	const total = await prisma.mii.count({
+		where: { in_queue: false, quarantined: false },
+	});
+	const miiChunks = Math.ceil(total / CHUNK_SIZE);
+	const totalSitemaps = miiChunks + 1; // +1 for the static-routes sitemap
+
+	const sitemapEntries = Array.from({ length: totalSitemaps }, (_, i) => `<sitemap><loc>${baseUrl}/sitemap/${i}.xml</loc></sitemap>`).join("");
+
+	const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${sitemapEntries}
+</sitemapindex>`;
+
+	return new NextResponse(xml, {
+		headers: { "Content-Type": "application/xml" },
+	});
+}

backend/src/app/sitemap.ts

@@ -1,14 +1,64 @@
 import { prisma } from "@/lib/prisma";
+import { Prisma } from "@prisma/client";
 import type { MetadataRoute } from "next";
 
 type SitemapRoute = MetadataRoute.Sitemap[0];
 
 export const revalidate = 43200; // update every 12 hours
+export const CHUNK_SIZE = 45000; // stay under 50,000 limit
 
-export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
+export async function generateSitemaps() {
+	const total = await prisma.mii.count({
+		where: {
+			in_queue: false,
+			quarantined: false,
+		},
+	});
+	const miiChunks = Math.ceil(total / CHUNK_SIZE);
+
+	// id 0 = static routes, ids 1..miiChunks = mii chunks
+	return Array.from({ length: miiChunks + 1 }, (_, id) => ({ id }));
+}
+
+export default async function sitemap(props: { id: Promise<string> }): Promise<MetadataRoute.Sitemap> {
 	const baseUrl = process.env.NEXT_PUBLIC_FRONTEND_URL!;
 	const apiUrl = process.env.NEXT_PUBLIC_BASE_URL!;
 
+	// id 0: static routes only
+	if ((await props.id) === "0") {
+		const lastModified = new Date();
+
+		return [
+			{
+				url: baseUrl,
+				lastModified,
+				changeFrequency: "always",
+				priority: 1,
+			},
+			{
+				url: `${baseUrl}/login`,
+				lastModified,
+				changeFrequency: "monthly",
+				priority: 0.6,
+			},
+			{
+				url: `${baseUrl}/privacy`,
+				lastModified,
+				changeFrequency: "yearly",
+				priority: 0.4,
+			},
+			{
+				url: `${baseUrl}/terms-of-service`,
+				lastModified,
+				changeFrequency: "yearly",
+				priority: 0.4,
+			},
+		];
+	}
+
+	// id 1+: paginated mii chunks
+	const chunkIndex = Number(await props.id) - 1;
+
 	const miis = await prisma.mii.findMany({
 		select: {
 			id: true,
@@ -18,64 +68,19 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
 			in_queue: false,
 			quarantined: false,
 		},
+		orderBy: { id: "asc" },
+		skip: chunkIndex * CHUNK_SIZE,
+		take: CHUNK_SIZE,
 	});
 
-	// const users = await prisma.user.findMany({
+	return miis.map(
-	// 	select: {
+		(mii) =>
-	// 		id: true,
+			({
-	// 		updatedAt: true,
+				url: `${baseUrl}/mii/${mii.id}`,
-	// 	},
+				lastModified: mii.createdAt,
-	// });
+				changeFrequency: "weekly",
-
+				priority: 0.7,
-	const dynamicRoutes: MetadataRoute.Sitemap = [
+				images: [`${apiUrl}/mii/${mii.id}/image?type=metadata`],
-		...miis.map(
+			}) as SitemapRoute,
-			(mii) =>
+	);
-				({
-					url: `${baseUrl}/mii/${mii.id}`,
-					lastModified: mii.createdAt,
-					changeFrequency: "weekly",
-					priority: 0.7,
-					images: [`${apiUrl}/mii/${mii.id}/image?type=metadata`],
-				}) as SitemapRoute,
-		),
-		// ...users.map(
-		// 	(user) =>
-		// 		({
-		// 			url: `${baseUrl}/profile/${user.id}`,
-		// 			lastModified: user.updatedAt,
-		// 			changeFrequency: "weekly",
-		// 			priority: 0.2,
-		// 		}) as SitemapRoute,
-		// ),
-	];
-
-	const lastModified = new Date();
-
-	return [
-		{
-			url: baseUrl,
-			lastModified,
-			changeFrequency: "always",
-			priority: 1,
-		},
-		{
-			url: `${baseUrl}/login`,
-			lastModified,
-			changeFrequency: "monthly",
-			priority: 0.6,
-		},
-		{
-			url: `${baseUrl}/privacy`,
-			lastModified,
-			changeFrequency: "yearly",
-			priority: 0.4,
-		},
-		{
-			url: `${baseUrl}/terms-of-service`,
-			lastModified,
-			changeFrequency: "yearly",
-			priority: 0.4,
-		},
-		...dynamicRoutes,
-	];
 }

frontend/public/robots.txt

@@ -10,4 +10,4 @@ Disallow: /report/mii/*
 Disallow: /report/user/*
 Disallow: /admin
 
-Sitemap: https://api.tomodachishare.com/sitemap.xml
+Sitemap: https://api.tomodachishare.com/sitemap-index.xml