Add robots.txt and sitemap.xml

diff --git a/src/main/kotlin/info/mechyrdia/Configuration.kt b/src/main/kotlin/info/mechyrdia/Configuration.kt
index 06572934b3b5d17e8bbe322a85c278edaf4076a8..73d456143d09d5dd34f7c413902cb57ad99d340b 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/Configuration.kt
+++ b/src/main/kotlin/info/mechyrdia/Configuration.kt
@@ -12,6 +12,7 @@ data class Configuration(
        
        val isDevMode: Boolean = false,
        
+       val rootDir: String = "..",
        val articleDir: String = "../lore",
        val assetDir: String = "../assets",
        val templateDir: String = "../tpl",
@@ -23,27 +24,17 @@ data class Configuration(
        val dbConn: String = "mongodb://localhost:27017",
 ) {
        companion object {
-               private val DEFAULT_CONFIG = Configuration()
-               
-               private var currentConfig: Configuration? = null
-               
-               val CurrentConfiguration: Configuration
-                       get() {
-                               currentConfig?.let { return it }
+               val CurrentConfiguration: Configuration by lazy {
+                       val file = File(System.getProperty("factbooks.configpath", "./config.json"))
+                       if (!file.isFile) {
+                               if (file.exists())
+                                       file.deleteRecursively()
                                
-                               val file = File(System.getProperty("factbooks.configpath", "./config.json"))
-                               if (!file.isFile) {
-                                       if (file.exists())
-                                               file.deleteRecursively()
-                                       
-                                       val json = JsonFileCodec.encodeToString(serializer(), DEFAULT_CONFIG)
-                                       file.writeText(json, Charsets.UTF_8)
-                                       return DEFAULT_CONFIG
-                               }
-                               
-                               val json = file.readText()
-                               return JsonFileCodec.decodeFromString(serializer(), json).also { currentConfig = it }
+                               file.writeText(JsonFileCodec.encodeToString(serializer(), Configuration()), Charsets.UTF_8)
                        }
+                       
+                       JsonFileCodec.decodeFromString(serializer(), file.readText(Charsets.UTF_8))
+               }
        }
 }
 

diff --git a/src/main/kotlin/info/mechyrdia/Factbooks.kt b/src/main/kotlin/info/mechyrdia/Factbooks.kt
index a3d56d3517bf985872aece907809313a894f111b..0f87023ace566b7c5457cb53068abb20f2859426 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/Factbooks.kt
+++ b/src/main/kotlin/info/mechyrdia/Factbooks.kt
@@ -24,6 +24,7 @@ import io.ktor.server.response.*
 import io.ktor.server.routing.*
 import io.ktor.server.sessions.*
 import io.ktor.server.sessions.serialization.*
+import io.ktor.util.*
 import org.slf4j.event.Level
 import java.io.File
 import java.io.IOException
@@ -153,6 +154,16 @@ fun Application.factbooks() {
                        call.respondHtml(HttpStatusCode.OK, call.galaxyMapPage())
                }
                
+               // Routes for robots
+               
+               get("/robots.txt") {
+                       call.respondFile(File(Configuration.CurrentConfiguration.rootDir).combineSafe("robots.txt"))
+               }
+               
+               get("/sitemap.xml") {
+                       call.respondText(buildString { generateSitemap() }, ContentType.Text.Xml)
+               }
+               
                // Client settings
                
                get("/change-theme") {

diff --git a/src/main/kotlin/info/mechyrdia/JSON.kt b/src/main/kotlin/info/mechyrdia/JSON.kt
index d810cca355b941166d5fae5fdbde3865e0b2abc9..41059baa1213d2c877065a3819a7dd3984e45edf 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/JSON.kt
+++ b/src/main/kotlin/info/mechyrdia/JSON.kt
@@ -8,6 +8,8 @@ val JsonFileCodec = Json {
        @OptIn(ExperimentalSerializationApi::class)
        prettyPrintIndent = "\t"
        
+       encodeDefaults = true
+       ignoreUnknownKeys = true
        useAlternativeNames = false
 }
 

diff --git a/src/main/kotlin/info/mechyrdia/data/nations.kt b/src/main/kotlin/info/mechyrdia/data/nations.kt
index 4b6cef1c87750a1f47d0f01f3bf889344b7e8cc7..255e0f0606355314d19abc6538c59c05bdd35aab 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/data/nations.kt
+++ b/src/main/kotlin/info/mechyrdia/data/nations.kt
@@ -83,9 +83,8 @@ suspend fun ApplicationCall.currentNation(): NationData? {
        }
 }
 
-private sealed interface NationSession {
-       object Anonymous : NationSession
+private sealed class NationSession {
+       data object Anonymous : NationSession()
        
-       @JvmInline
-       value class LoggedIn(val nation: NationData) : NationSession
+       data class LoggedIn(val nation: NationData) : NationSession()
 }

diff --git a/src/main/kotlin/info/mechyrdia/lore/article_listing.kt b/src/main/kotlin/info/mechyrdia/lore/article_listing.kt
index 8f98b6c5dcc09710e292e2de0968ca6062ef39ab..416da5307daab8315e727093b607bd489251a312 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/lore/article_listing.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/article_listing.kt
@@ -9,6 +9,10 @@ import java.io.File
 
 data class ArticleNode(val name: String, val subNodes: List<ArticleNode>)
 
+fun rootArticleNodeList(): List<ArticleNode> = File(Configuration.CurrentConfiguration.articleDir)
+       .toArticleNode()
+       .subNodes
+
 fun File.toArticleNode(): ArticleNode = ArticleNode(
        name,
        listFiles()
@@ -19,8 +23,8 @@ fun File.toArticleNode(): ArticleNode = ArticleNode(
 )
 
 fun List<ArticleNode>.renderInto(list: UL, base: String? = null) {
-       val prefix = base?.let { "$it/" } ?: ""
-       forEach { node ->
+       val prefix = lazy(LazyThreadSafetyMode.NONE) { base?.let { "$it/" } ?: "" }
+       for (node in this) {
                if (Configuration.CurrentConfiguration.isDevMode || !(node.name.endsWith(".wip") || node.name.endsWith(".old")))
                        list.li {
                                a(href = "/lore/$prefix${node.name}") { +node.name }

diff --git a/src/main/kotlin/info/mechyrdia/lore/parser_tags.kt b/src/main/kotlin/info/mechyrdia/lore/parser_tags.kt
index 7eeaaa25601cb79ee8ace05f92aaf9b16541a3cd..8b888130b102a2ae24740252c4856bc1c825fbc0 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/lore/parser_tags.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/parser_tags.kt
@@ -2,6 +2,7 @@ package info.mechyrdia.lore
 
 import info.mechyrdia.Configuration
 import info.mechyrdia.JsonStorageCodec
+import io.ktor.util.*
 import kotlinx.serialization.builtins.serializer
 import kotlinx.serialization.json.JsonPrimitive
 import java.io.File
@@ -162,7 +163,8 @@ enum class TextParserFormattingTag(val type: TextParserTagType<Unit>) {
                        val (width, height) = getSizeParam(tagParam)
                        
                        if (imageUrl.endsWith(".svg"))
-                               File(Configuration.CurrentConfiguration.assetDir, "images/$imageUrl")
+                               File(Configuration.CurrentConfiguration.assetDir)
+                                       .combineSafe("images/$imageUrl")
                                        .readText()
                                        .replace("<svg", "<svg${getImageSizeAttributes(width, height)}")
                        else
@@ -188,7 +190,7 @@ enum class TextParserFormattingTag(val type: TextParserTagType<Unit>) {
        ),
        QUIZ(
                TextParserTagType.Indirect { _, content, _ ->
-                       val quizText = File(Configuration.CurrentConfiguration.quizDir, "$content.json").readText()
+                       val quizText = File(Configuration.CurrentConfiguration.quizDir).combineSafe("$content.json").readText()
                        val quizJson = JsonStorageCodec.encodeToString(String.serializer(), quizText)
                        
                        "<script>window.renderQuiz(JSON.parse($quizJson));</script>"

diff --git a/src/main/kotlin/info/mechyrdia/lore/preparser.kt b/src/main/kotlin/info/mechyrdia/lore/preparser.kt
index 34ed415b09b86926086791fbd60a9e8ccaf3055a..6b14cf550ffef914fbdadc0ee05b062b057bccb8 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/lore/preparser.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/preparser.kt
@@ -75,7 +75,7 @@ object PreParser {
        private val compiler = Mustache.compiler()
                .withEscaper(Escapers.NONE)
                .defaultValue("{{ MISSING VALUE \"{{name}}\" }}")
-               .withLoader { File(Configuration.CurrentConfiguration.templateDir, "$it.tpl").bufferedReader() }
+               .withLoader { File(Configuration.CurrentConfiguration.templateDir).combineSafe("$it.tpl").bufferedReader() }
        
        private fun convertJson(json: JsonElement, currentFile: File): Any? = when (json) {
                JsonNull -> null
@@ -91,7 +91,8 @@ object PreParser {
        }
        
        private fun loadJsonContext(name: String): Map<*, *> =
-               File(Configuration.CurrentConfiguration.jsonDocDir).combineSafe("$name.json")
+               File(Configuration.CurrentConfiguration.jsonDocDir)
+                       .combineSafe("$name.json")
                        .takeIf { it.isFile }
                        ?.let { file ->
                                val text = file.readText()

diff --git a/src/main/kotlin/info/mechyrdia/lore/view_nav.kt b/src/main/kotlin/info/mechyrdia/lore/view_nav.kt
index 30c0e04cb6649b7b85075c96af75d61eebb6d068..c2b03d5a115ab983d20dc4b437fa5ba70d80ed1b 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/lore/view_nav.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/view_nav.kt
@@ -21,7 +21,7 @@ private data class ExternalLink(
 )
 
 fun loadExternalLinks(): List<NavItem> {
-       val extraLinksFile = File(Configuration.CurrentConfiguration.articleDir).parentFile.combineSafe("externalLinks.json")
+       val extraLinksFile = File(Configuration.CurrentConfiguration.rootDir).combineSafe("externalLinks.json")
        val extraLinks = JsonFileCodec.decodeFromString(ListSerializer(ExternalLink.serializer()), extraLinksFile.readText())
        return if (extraLinks.isEmpty())
                emptyList()

diff --git a/src/main/kotlin/info/mechyrdia/lore/views_lore.kt b/src/main/kotlin/info/mechyrdia/lore/views_lore.kt
index 092e08778ec704956902e5e03a3468d6c9c772b2..43ccce1de1fe9b6fe1f838d0b37c718fe8a6655d 100644 (file)
--- a/src/main/kotlin/info/mechyrdia/lore/views_lore.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/views_lore.kt
@@ -11,7 +11,6 @@ import kotlinx.coroutines.flow.toList
 import kotlinx.html.*
 import kotlinx.serialization.Serializable
 import java.io.File
-import java.time.Instant
 
 @Serializable
 data class IntroMetaData(
@@ -24,10 +23,12 @@ data class IntroMetaData(
 }
 
 suspend fun ApplicationCall.loreIntroPage(): HTML.() -> Unit {
-       val metaJsonFile = File(Configuration.CurrentConfiguration.articleDir).parentFile.combineSafe("introMeta.json")
+       val rootDirFile = File(Configuration.CurrentConfiguration.rootDir)
+       
+       val metaJsonFile = rootDirFile.combineSafe("introMeta.json")
        val metaData = JsonFileCodec.decodeFromString(IntroMetaData.serializer(), metaJsonFile.readText())
        
-       val htmlFile = File(Configuration.CurrentConfiguration.articleDir).parentFile.combineSafe("intro.html")
+       val htmlFile = rootDirFile.combineSafe("intro.html")
        val fileHtml = htmlFile.readText()
        
        return page(metaData.title, standardNavBar(), null, metaData.ogData) {

diff --git a/src/main/kotlin/info/mechyrdia/lore/views_robots.kt b/src/main/kotlin/info/mechyrdia/lore/views_robots.kt
new file mode 100644 (file)
index 0000000..045bd59
--- /dev/null
+++ b/src/main/kotlin/info/mechyrdia/lore/views_robots.kt
@@ -0,0 +1,65 @@
+package info.mechyrdia.lore
+
+import info.mechyrdia.Configuration
+import io.ktor.util.*
+import java.io.File
+import java.time.Instant
+
+private const val AVERAGE_FACTBOOK_PAGE_CHANGEFREQ = "hourly"
+private const val AVERAGE_FACTBOOK_INDEX_CHANGEFREQ = "daily"
+private const val FACTBOOK_ROOT_PRIORITY = "0.6"
+private const val FACTBOOK_INDEX_PRIORITY = "0.4"
+private const val FACTBOOK_PAGE_PRIORITY = "0.8"
+
+private val File.lastSubFilesModified: Instant?
+       get() = if (isDirectory)
+               (listFiles().orEmpty().mapNotNull {
+                       it.lastSubFilesModified
+               } + Instant.ofEpochMilli(lastModified())).max()
+       else null
+
+private val File.lastContentModified: Instant
+       get() = lastSubFilesModified ?: Instant.ofEpochMilli(lastModified())
+
+private fun List<ArticleNode>.renderIntoSitemap(sitemap: Appendable, base: String? = null) {
+       val prefix by lazy(LazyThreadSafetyMode.NONE) { base?.let { "$it/" } ?: "" }
+       for (node in this) {
+               if (Configuration.CurrentConfiguration.isDevMode || !(node.name.endsWith(".wip") || node.name.endsWith(".old"))) {
+                       val path = "$prefix${node.name}"
+                       
+                       val file = File(Configuration.CurrentConfiguration.articleDir).combineSafe(path)
+                       val lastModified = file.lastContentModified
+                       val changeFreq = if (node.subNodes.isNotEmpty()) AVERAGE_FACTBOOK_INDEX_CHANGEFREQ else AVERAGE_FACTBOOK_PAGE_CHANGEFREQ
+                       val priority = if (node.subNodes.isNotEmpty()) FACTBOOK_INDEX_PRIORITY else FACTBOOK_PAGE_PRIORITY
+                       
+                       sitemap.appendLine("\t<url>")
+                       sitemap.appendLine("\t\t<loc>https://mechyrdia.info/lore/$path</loc>")
+                       sitemap.appendLine("\t\t<lastmod>$lastModified</lastmod>")
+                       sitemap.appendLine("\t\t<changefreq>$changeFreq</changefreq>")
+                       sitemap.appendLine("\t\t<priority>$priority</priority>")
+                       sitemap.appendLine("\t</url>")
+                       node.subNodes.renderIntoSitemap(sitemap, path)
+               }
+       }
+}
+
+private fun renderLoreSitemap(sitemap: Appendable) {
+       val rootFile = File(Configuration.CurrentConfiguration.articleDir)
+       val rootLastModified = rootFile.lastContentModified
+       
+       sitemap.appendLine("\t<url>")
+       sitemap.appendLine("\t\t<loc>https://mechyrdia.info/lore</loc>")
+       sitemap.appendLine("\t\t<lastmod>$rootLastModified</lastmod>")
+       sitemap.appendLine("\t\t<changefreq>$AVERAGE_FACTBOOK_INDEX_CHANGEFREQ</changefreq>")
+       sitemap.appendLine("\t\t<priority>$FACTBOOK_ROOT_PRIORITY</priority>")
+       sitemap.appendLine("\t</url>")
+       
+       rootArticleNodeList().renderIntoSitemap(sitemap)
+}
+
+fun Appendable.generateSitemap() {
+       appendLine("<?xml version='1.0' encoding='UTF-8'?>")
+       appendLine("<urlset xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">")
+       renderLoreSitemap(this)
+       appendLine("</urlset>")
+}