Add built-in sitemap generation support for SEO (#701)

playground/site/build.gradle.kts

@@ -19,6 +19,16 @@ kobweb {
                 enableSelfHosting()
             }
         }
+        //testing new feature
+        generateSitemap("http://localhost:8080") {
+            // Define routes to exclude from sitemap
+            val excludedPrefixes = listOf("/fruits", "/markdown")
+            filter.set {
+             // Exclude routes with specified prefixes
+                val hasExcludedPrefix = excludedPrefixes.any { prefix -> route.startsWith(prefix) }
+               !hasExcludedPrefix
+            }
+        }
     }
     markdown {
         defaultLayout.set(".components.layouts.MarkdownLayout")

tools/gradle-plugins/application/src/main/kotlin/com/varabyte/kobweb/gradle/application/KobwebApplicationPlugin.kt

@@ -7,6 +7,7 @@ import com.varabyte.kobweb.gradle.application.extensions.createAppBlock
 import com.varabyte.kobweb.gradle.application.extensions.export
 import com.varabyte.kobweb.gradle.application.extensions.remoteDebugging
 import com.varabyte.kobweb.gradle.application.extensions.server
+import com.varabyte.kobweb.gradle.application.extensions.sitemap
 import com.varabyte.kobweb.gradle.application.ksp.kspBackendFile
 import com.varabyte.kobweb.gradle.application.ksp.kspFrontendFile
 import com.varabyte.kobweb.gradle.application.tasks.KobwebBrowserCacheIdTask
@@ -21,13 +22,15 @@ import com.varabyte.kobweb.gradle.application.tasks.KobwebExportTask
 import com.varabyte.kobweb.gradle.application.tasks.KobwebGenIndexConfInputs
 import com.varabyte.kobweb.gradle.application.tasks.KobwebGenSiteEntryConfInputs
 import com.varabyte.kobweb.gradle.application.tasks.KobwebGenerateApisFactoryTask
+import com.varabyte.kobweb.gradle.application.tasks.KobwebGenerateSitemapTask
 import com.varabyte.kobweb.gradle.application.tasks.KobwebGenerateSiteEntryTask
 import com.varabyte.kobweb.gradle.application.tasks.KobwebGenerateSiteIndexTask
 import com.varabyte.kobweb.gradle.application.tasks.KobwebGenerateTask
 import com.varabyte.kobweb.gradle.application.tasks.KobwebListRoutesTask
 import com.varabyte.kobweb.gradle.application.tasks.KobwebStartTask
 import com.varabyte.kobweb.gradle.application.tasks.KobwebStopTask
 import com.varabyte.kobweb.gradle.application.tasks.KobwebUnpackServerJarTask
+import com.varabyte.kobweb.gradle.application.util.site.kobwebSiteRoutes
 import com.varabyte.kobweb.gradle.core.KobwebCorePlugin
 import com.varabyte.kobweb.gradle.core.extensions.kobwebBlock
 import com.varabyte.kobweb.gradle.core.kmp.JsTarget
@@ -71,6 +74,9 @@ import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpack
 import org.jetbrains.kotlin.gradle.targets.jvm.KotlinJvmTarget
 import javax.inject.Inject
 import kotlin.io.path.exists
+import org.gradle.api.Task
+import org.gradle.api.tasks.Copy
+import org.gradle.api.tasks.Delete
 
 val Project.kobwebFolder: KobwebFolder
     get() = KobwebFolder.fromChildPath(layout.projectDirectory.asFile.toPath())
@@ -173,6 +179,7 @@ class KobwebApplicationPlugin @Inject constructor(
                             "\"${devScript.substringAfterLast('/')}\" to find the right path."
                     )
                 }
+
             }
         }
         project.tasks.register<KobwebStopTask>("kobwebStop")
@@ -211,7 +218,6 @@ class KobwebApplicationPlugin @Inject constructor(
                 KobwebExportConfInputs(kobwebConf),
                 exportLayout,
             )
-
         val kobwebListRoutesTask = project.tasks.register<KobwebListRoutesTask>("kobwebListRoutes")
 
         project.tasks.register<KobwebBrowserCacheIdTask>("kobwebBrowserCacheId") {
@@ -237,6 +243,7 @@ class KobwebApplicationPlugin @Inject constructor(
         project.buildTargets.withType<KotlinJsIrTarget>().configureEach {
             val jsTarget = JsTarget(this)
 
+
             // Beginning with Kotlin 2.1.0, the Kotlin Gradle plugin began using uncompressed klibs for
             // inter-project dependencies. This breaks our code responsible for detecting and extracting metadata from
             // kobweb library dependencies, as the required resources (e.g. `KOBWEB_METADATA_FRONTEND`) are not present
@@ -282,11 +289,12 @@ class KobwebApplicationPlugin @Inject constructor(
                 KobwebGenSiteEntryConfInputs(kobwebConf),
             )
 
-            val kobwebCacheAppFrontendDataTask = project.tasks.register<KobwebCacheAppFrontendDataTask>("kobwebCacheAppFrontendData") {
-                appFrontendMetadataFile.set(project.kspFrontendFile(jsTarget))
-                compileClasspath.from(project.configurations.named(jsTarget.compileClasspath))
-                appDataFile.set(this.kobwebCacheFile("appData.json"))
-            }
+            val kobwebCacheAppFrontendDataTask =
+                project.tasks.register<KobwebCacheAppFrontendDataTask>("kobwebCacheAppFrontendData") {
+                    appFrontendMetadataFile.set(project.kspFrontendFile(jsTarget))
+                    compileClasspath.from(project.configurations.named(jsTarget.compileClasspath))
+                    appDataFile.set(this.kobwebCacheFile("appData.json"))
+                }
 
             kobwebGenSiteEntryTask.configure {
                 appDataFile.set(kobwebCacheAppFrontendDataTask.flatMap { it.appDataFile })
@@ -365,7 +373,106 @@ class KobwebApplicationPlugin @Inject constructor(
             kobwebListRoutesTask.configure {
                 appDataFile.set(kobwebCacheAppFrontendDataTask.flatMap { it.appDataFile })
             }
+
+            // Only register sitemap generation task if sitemap generation has been enabled
+            if (appBlock.sitemap.isEnabled) {
+                val kobwebGenerateSitemapTask = project.tasks.register<KobwebGenerateSitemapTask>(
+                    "kobwebGenerateSitemap"
+                ) {
+                    routes.set(project.kobwebSiteRoutes)
+                    basePath.set(kobwebConf.site.basePath)
+
+                    // Generate to a temporary location first
+                    sitemapFile.set(
+                        project.layout.buildDirectory.file("tmp/sitemap/sitemap.xml")
+                    )
+
+                    sitemapBlock.set(appBlock.sitemap)
+
+                    // Ensure sitemap generation waits for app data to be cached
+                    dependsOn(kobwebCacheAppFrontendDataTask)
+
+                    // Capture the baseUrl presence during configuration time
+                    val hasBaseUrl = appBlock.sitemap.baseUrl.isPresent
+                    onlyIf("baseUrl must be configured for sitemap generation") {
+                        if (!hasBaseUrl) {
+                            logger.info("Skipping sitemap generation: baseUrl not configured")
+                        }
+                        hasBaseUrl
+                    }
+                }
+
+                // Copy the generated sitemap to the source resources directory
+                val copySitemapToResourcesTask = project.tasks.register<Copy>("copySitemapToResources") {
+                    group = "kobweb"
+                    description = "Copy generated sitemap to source resources directory"
+
+                    from(kobwebGenerateSitemapTask.flatMap { it.sitemapFile })
+
+                    val targetDirProvider = project.provider {
+                        project.file("src/${jsTarget.mainSourceSet}/resources/${kobwebBlock.publicPath.get()}")
+                    }
+                    into(targetDirProvider)
+
+                    dependsOn(kobwebGenerateSitemapTask)
+
+                    // Capture the baseUrl presence during configuration time
+                    val hasBaseUrl = appBlock.sitemap.baseUrl.isPresent
+                    onlyIf {
+                        if (!hasBaseUrl) {
+                            logger.info("Skipping sitemap copy: baseUrl not configured")
+                        }
+                        hasBaseUrl
+                    }
+
+                    doFirst {
+                        val targetDir = targetDirProvider.get()
+                        targetDir.mkdirs()
+                        logger.info("Copying sitemap to: ${targetDir.absolutePath}")
+                    }
+
+                    doLast {
+                        logger.info("Successfully copied sitemap.xml to source resources")
+                    }
+                }
+
+                // Make sure sitemap is copied before resources are processed
+                project.tasks.named<ProcessResources>(jsTarget.processResources) {
+                    dependsOn(copySitemapToResourcesTask)
+                }
+
+                // Also ensure sitemap is available during development
+                kobwebStartTask.configure {
+                    dependsOn(copySitemapToResourcesTask)
+                }
+
+                // Make the copy task part of the general generation tasks
+                project.tasks.matching { it.name == "kobwebGenAll" }.configureEach {
+                    dependsOn(copySitemapToResourcesTask)
+                }
+
+                // For production builds, ensure sitemap is copied before webpack
+                project.tasks.matching { it.name == jsTarget.browserProductionWebpack }.configureEach {
+                    dependsOn(copySitemapToResourcesTask)
+                }
+
+                // Add a cleanup task for sitemap files
+                project.tasks.register<Delete>("cleanSitemap") {
+                    group = "kobweb"
+                    description = "Clean generated sitemap files"
+
+                    delete(project.layout.buildDirectory.file("tmp/sitemap"))
+                    delete(project.provider {
+                        project.file("src/${jsTarget.mainSourceSet}/resources/${kobwebBlock.publicPath.get()}/sitemap.xml")
+                    })
+
+                    doLast {
+                        logger.info("Cleaned sitemap files")
+                    }
+                }
+            }
         }
+
         project.buildTargets.withType<KotlinJvmTarget>().configureEach {
             val jvmTarget = JvmTarget(this)
 
@@ -380,11 +487,12 @@ class KobwebApplicationPlugin @Inject constructor(
                 }
             }
 
-            val kobwebCacheAppBackendDataTask = project.tasks.register<KobwebCacheAppBackendDataTask>("kobwebCacheAppBackendData") {
-                appBackendMetadataFile.set(project.kspBackendFile(jvmTarget))
-                compileClasspath.from(project.configurations.named(jvmTarget.compileClasspath))
-                appDataFile.set(this.kobwebCacheFile("appData.json"))
-            }
+            val kobwebCacheAppBackendDataTask =
+                project.tasks.register<KobwebCacheAppBackendDataTask>("kobwebCacheAppBackendData") {
+                    appBackendMetadataFile.set(project.kspBackendFile(jvmTarget))
+                    compileClasspath.from(project.configurations.named(jvmTarget.compileClasspath))
+                    appDataFile.set(this.kobwebCacheFile("appData.json"))
+                }
 
             val kobwebGenApisFactoryTask = project.tasks
                 .register<KobwebGenerateApisFactoryTask>("kobwebGenApisFactory", kobwebBlock.app)
@@ -412,7 +520,6 @@ class KobwebApplicationPlugin @Inject constructor(
                 kotlin.srcDir(kobwebGenApisFactoryTask)
             }
         }
-
         // Convenience task in case you quickly want to run all "kobwebGen..." tasks
         project.tasks.register("kobwebGenAll") {
             group = "kobweb"

tools/gradle-plugins/application/src/main/kotlin/com/varabyte/kobweb/gradle/application/extensions/AppBlock.kt

@@ -12,6 +12,7 @@ import com.varabyte.kobweb.project.conf.KobwebConf
 import kotlinx.html.HEAD
 import kotlinx.html.link
 import kotlinx.html.meta
+import org.gradle.api.GradleException
 import org.gradle.api.plugins.ExtensionAware
 import org.gradle.api.provider.ListProperty
 import org.gradle.api.provider.MapProperty
@@ -328,6 +329,78 @@ abstract class AppBlock @Inject constructor(
         }
     }
 
+
+    /**
+     * A sub-block for defining properties related to sitemap generation for SEO.
+     */
+    abstract class SitemapBlock : ExtensionAware {
+
+
+        /**
+         * Context for sitemap route filtering, providing extensible information about each route.
+         */
+        class SitemapFilterContext(val route: String)
+
+        /**
+         * The base URL for the site (e.g., "https://example.com").
+         * This will be prepended to all routes in the sitemap.
+         *
+         * IMPORTANT: If your site is deployed under a base path (e.g., GitHub Pages project site),
+         * include it in the baseUrl: "https://username.github.io/project-name".
+         * If the base path is configured and not included in the baseUrl, a warning will be printed and the
+         * generated links may be incorrect.
+         */
+        @get:Input
+        abstract val baseUrl: Property<String>
+
+        /**
+         * Additional routes to include that may not be discovered automatically.
+         * Useful for dynamic content routes where you know the specific URLs.
+         *
+         * For markdown-generated pages: these are already discovered automatically.
+         * For dynamic templates like @Page("/blog/{slug}"): add concrete URLs like "/blog/my-post".
+         */
+        @get:Input
+        abstract val extraRoutes: ListProperty<String>
+
+        /**
+         * A filter to determine which routes should be included in the sitemap.
+         * Return true to include the route, false to exclude it.
+         *
+         * Note: This property is marked as `@Internal` to avoid serialization issues with lambdas,
+         * which can affect Gradle's configuration cache.
+         *
+         * IMPORTANT: Dynamic routes (containing '{' and '}') are ALWAYS excluded automatically,
+         * even if you provide a custom filter. This is enforced by the framework.
+         * 
+         * @see [SitemapFilterContext]
+         */
+        @get:Nested // Avoid serialization issues with lambdas
+        abstract val filter: Property<SitemapFilterContext.() -> Boolean>
+
+        /**
+         * Internal flag to track whether sitemap generation has been enabled.
+         * This is used to conditionally register the sitemap generation task.
+         */
+        @get:Internal
+        internal var isEnabled: Boolean = false
+            private set
+
+        /**
+         * Internal method to mark sitemap generation as enabled.
+         * Called by the generateSitemap function.
+         */
+        internal fun markAsEnabled() {
+            isEnabled = true
+        }
+
+        init {
+            extraRoutes.set(emptyList())
+            // Default filter accepts all routes (dynamic route exclusion is handled in the task)
+            filter.convention { true }
+        }
+    }
+
     /**
      * Configuration values for the backend of this Kobweb application.
      */
@@ -574,13 +647,26 @@ abstract class AppBlock @Inject constructor(
     @get:Input
     abstract val cssPrefix: Property<String>
 
+    /**
+     * Enable sitemap generation for this Kobweb application.
+     *
+     * @param baseUrl The base URL for the site (e.g., "https://example.com")
+     * @param config Configuration block for additional sitemap options
+     */
+    fun generateSitemap(baseUrl: String, config: SitemapBlock.() -> Unit = {}) {
+        sitemap.baseUrl.set(baseUrl)
+        config(sitemap)
+        sitemap.markAsEnabled()
+    }
+
     init {
         globals.set(mapOf("title" to conf.site.title))
         cleanUrls.convention(true)
         genDir.convention(baseGenDir.map { "$it/app" })
 
         extensions.create<IndexBlock>("index", BasePath(conf.site.basePath))
         extensions.create<ServerBlock>("server")
+        extensions.create<SitemapBlock>("sitemap")
         extensions.create<ExportBlock>("export", kobwebFolder)
     }
 }
@@ -597,6 +683,9 @@ val AppBlock.export: AppBlock.ExportBlock
 val AppBlock.server: AppBlock.ServerBlock
     get() = extensions.getByType<AppBlock.ServerBlock>()
 
+val AppBlock.sitemap: AppBlock.SitemapBlock
+    get() = extensions.getByType<AppBlock.SitemapBlock>()
+
 val AppBlock.ServerBlock.remoteDebugging: AppBlock.ServerBlock.RemoteDebuggingBlock
     get() = extensions.getByType<AppBlock.ServerBlock.RemoteDebuggingBlock>()