From f33f74a054d34c7b1b4761001df52ff4b57bcc2f Mon Sep 17 00:00:00 2001 From: Lanius Trolling Date: Tue, 12 Dec 2023 10:08:38 -0500 Subject: [PATCH] Add robots.txt and sitemap.xml --- .../kotlin/info/mechyrdia/Configuration.kt | 29 +++------ src/main/kotlin/info/mechyrdia/Factbooks.kt | 11 ++++ src/main/kotlin/info/mechyrdia/JSON.kt | 2 + .../kotlin/info/mechyrdia/data/nations.kt | 7 +- .../info/mechyrdia/lore/article_listing.kt | 8 ++- .../kotlin/info/mechyrdia/lore/parser_tags.kt | 6 +- .../kotlin/info/mechyrdia/lore/preparser.kt | 5 +- .../kotlin/info/mechyrdia/lore/view_nav.kt | 2 +- .../kotlin/info/mechyrdia/lore/views_lore.kt | 7 +- .../info/mechyrdia/lore/views_robots.kt | 65 +++++++++++++++++++ 10 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 src/main/kotlin/info/mechyrdia/lore/views_robots.kt diff --git a/src/main/kotlin/info/mechyrdia/Configuration.kt b/src/main/kotlin/info/mechyrdia/Configuration.kt index 0657293..73d4561 100644 --- a/src/main/kotlin/info/mechyrdia/Configuration.kt +++ b/src/main/kotlin/info/mechyrdia/Configuration.kt @@ -12,6 +12,7 @@ data class Configuration( val isDevMode: Boolean = false, + val rootDir: String = "..", val articleDir: String = "../lore", val assetDir: String = "../assets", val templateDir: String = "../tpl", @@ -23,27 +24,17 @@ data class Configuration( val dbConn: String = "mongodb://localhost:27017", ) { companion object { - private val DEFAULT_CONFIG = Configuration() - - private var currentConfig: Configuration? = null - - val CurrentConfiguration: Configuration - get() { - currentConfig?.let { return it } + val CurrentConfiguration: Configuration by lazy { + val file = File(System.getProperty("factbooks.configpath", "./config.json")) + if (!file.isFile) { + if (file.exists()) + file.deleteRecursively() - val file = File(System.getProperty("factbooks.configpath", "./config.json")) - if (!file.isFile) { - if (file.exists()) - file.deleteRecursively() - - val json = JsonFileCodec.encodeToString(serializer(), DEFAULT_CONFIG) - file.writeText(json, Charsets.UTF_8) - return DEFAULT_CONFIG - } - - val json = file.readText() - return JsonFileCodec.decodeFromString(serializer(), json).also { currentConfig = it } + file.writeText(JsonFileCodec.encodeToString(serializer(), Configuration()), Charsets.UTF_8) } + + JsonFileCodec.decodeFromString(serializer(), file.readText(Charsets.UTF_8)) + } } } diff --git a/src/main/kotlin/info/mechyrdia/Factbooks.kt b/src/main/kotlin/info/mechyrdia/Factbooks.kt index a3d56d3..0f87023 100644 --- a/src/main/kotlin/info/mechyrdia/Factbooks.kt +++ b/src/main/kotlin/info/mechyrdia/Factbooks.kt @@ -24,6 +24,7 @@ import io.ktor.server.response.* import io.ktor.server.routing.* import io.ktor.server.sessions.* import io.ktor.server.sessions.serialization.* +import io.ktor.util.* import org.slf4j.event.Level import java.io.File import java.io.IOException @@ -153,6 +154,16 @@ fun Application.factbooks() { call.respondHtml(HttpStatusCode.OK, call.galaxyMapPage()) } + // Routes for robots + + get("/robots.txt") { + call.respondFile(File(Configuration.CurrentConfiguration.rootDir).combineSafe("robots.txt")) + } + + get("/sitemap.xml") { + call.respondText(buildString { generateSitemap() }, ContentType.Text.Xml) + } + // Client settings get("/change-theme") { diff --git a/src/main/kotlin/info/mechyrdia/JSON.kt b/src/main/kotlin/info/mechyrdia/JSON.kt index d810cca..41059ba 100644 --- a/src/main/kotlin/info/mechyrdia/JSON.kt +++ b/src/main/kotlin/info/mechyrdia/JSON.kt @@ -8,6 +8,8 @@ val JsonFileCodec = Json { @OptIn(ExperimentalSerializationApi::class) prettyPrintIndent = "\t" + encodeDefaults = true + ignoreUnknownKeys = true useAlternativeNames = false } diff --git a/src/main/kotlin/info/mechyrdia/data/nations.kt b/src/main/kotlin/info/mechyrdia/data/nations.kt index 4b6cef1..255e0f0 100644 --- a/src/main/kotlin/info/mechyrdia/data/nations.kt +++ b/src/main/kotlin/info/mechyrdia/data/nations.kt @@ -83,9 +83,8 @@ suspend fun ApplicationCall.currentNation(): NationData? { } } -private sealed interface NationSession { - object Anonymous : NationSession +private sealed class NationSession { + data object Anonymous : NationSession() - @JvmInline - value class LoggedIn(val nation: NationData) : NationSession + data class LoggedIn(val nation: NationData) : NationSession() } diff --git a/src/main/kotlin/info/mechyrdia/lore/article_listing.kt b/src/main/kotlin/info/mechyrdia/lore/article_listing.kt index 8f98b6c..416da53 100644 --- a/src/main/kotlin/info/mechyrdia/lore/article_listing.kt +++ b/src/main/kotlin/info/mechyrdia/lore/article_listing.kt @@ -9,6 +9,10 @@ import java.io.File data class ArticleNode(val name: String, val subNodes: List) +fun rootArticleNodeList(): List = File(Configuration.CurrentConfiguration.articleDir) + .toArticleNode() + .subNodes + fun File.toArticleNode(): ArticleNode = ArticleNode( name, listFiles() @@ -19,8 +23,8 @@ fun File.toArticleNode(): ArticleNode = ArticleNode( ) fun List.renderInto(list: UL, base: String? = null) { - val prefix = base?.let { "$it/" } ?: "" - forEach { node -> + val prefix = lazy(LazyThreadSafetyMode.NONE) { base?.let { "$it/" } ?: "" } + for (node in this) { if (Configuration.CurrentConfiguration.isDevMode || !(node.name.endsWith(".wip") || node.name.endsWith(".old"))) list.li { a(href = "/lore/$prefix${node.name}") { +node.name } diff --git a/src/main/kotlin/info/mechyrdia/lore/parser_tags.kt b/src/main/kotlin/info/mechyrdia/lore/parser_tags.kt index 7eeaaa2..8b88813 100644 --- a/src/main/kotlin/info/mechyrdia/lore/parser_tags.kt +++ b/src/main/kotlin/info/mechyrdia/lore/parser_tags.kt @@ -2,6 +2,7 @@ package info.mechyrdia.lore import info.mechyrdia.Configuration import info.mechyrdia.JsonStorageCodec +import io.ktor.util.* import kotlinx.serialization.builtins.serializer import kotlinx.serialization.json.JsonPrimitive import java.io.File @@ -162,7 +163,8 @@ enum class TextParserFormattingTag(val type: TextParserTagType) { val (width, height) = getSizeParam(tagParam) if (imageUrl.endsWith(".svg")) - File(Configuration.CurrentConfiguration.assetDir, "images/$imageUrl") + File(Configuration.CurrentConfiguration.assetDir) + .combineSafe("images/$imageUrl") .readText() .replace(") { ), QUIZ( TextParserTagType.Indirect { _, content, _ -> - val quizText = File(Configuration.CurrentConfiguration.quizDir, "$content.json").readText() + val quizText = File(Configuration.CurrentConfiguration.quizDir).combineSafe("$content.json").readText() val quizJson = JsonStorageCodec.encodeToString(String.serializer(), quizText) "" diff --git a/src/main/kotlin/info/mechyrdia/lore/preparser.kt b/src/main/kotlin/info/mechyrdia/lore/preparser.kt index 34ed415..6b14cf5 100644 --- a/src/main/kotlin/info/mechyrdia/lore/preparser.kt +++ b/src/main/kotlin/info/mechyrdia/lore/preparser.kt @@ -75,7 +75,7 @@ object PreParser { private val compiler = Mustache.compiler() .withEscaper(Escapers.NONE) .defaultValue("{{ MISSING VALUE \"{{name}}\" }}") - .withLoader { File(Configuration.CurrentConfiguration.templateDir, "$it.tpl").bufferedReader() } + .withLoader { File(Configuration.CurrentConfiguration.templateDir).combineSafe("$it.tpl").bufferedReader() } private fun convertJson(json: JsonElement, currentFile: File): Any? = when (json) { JsonNull -> null @@ -91,7 +91,8 @@ object PreParser { } private fun loadJsonContext(name: String): Map<*, *> = - File(Configuration.CurrentConfiguration.jsonDocDir).combineSafe("$name.json") + File(Configuration.CurrentConfiguration.jsonDocDir) + .combineSafe("$name.json") .takeIf { it.isFile } ?.let { file -> val text = file.readText() diff --git a/src/main/kotlin/info/mechyrdia/lore/view_nav.kt b/src/main/kotlin/info/mechyrdia/lore/view_nav.kt index 30c0e04..c2b03d5 100644 --- a/src/main/kotlin/info/mechyrdia/lore/view_nav.kt +++ b/src/main/kotlin/info/mechyrdia/lore/view_nav.kt @@ -21,7 +21,7 @@ private data class ExternalLink( ) fun loadExternalLinks(): List { - val extraLinksFile = File(Configuration.CurrentConfiguration.articleDir).parentFile.combineSafe("externalLinks.json") + val extraLinksFile = File(Configuration.CurrentConfiguration.rootDir).combineSafe("externalLinks.json") val extraLinks = JsonFileCodec.decodeFromString(ListSerializer(ExternalLink.serializer()), extraLinksFile.readText()) return if (extraLinks.isEmpty()) emptyList() diff --git a/src/main/kotlin/info/mechyrdia/lore/views_lore.kt b/src/main/kotlin/info/mechyrdia/lore/views_lore.kt index 092e087..43ccce1 100644 --- a/src/main/kotlin/info/mechyrdia/lore/views_lore.kt +++ b/src/main/kotlin/info/mechyrdia/lore/views_lore.kt @@ -11,7 +11,6 @@ import kotlinx.coroutines.flow.toList import kotlinx.html.* import kotlinx.serialization.Serializable import java.io.File -import java.time.Instant @Serializable data class IntroMetaData( @@ -24,10 +23,12 @@ data class IntroMetaData( } suspend fun ApplicationCall.loreIntroPage(): HTML.() -> Unit { - val metaJsonFile = File(Configuration.CurrentConfiguration.articleDir).parentFile.combineSafe("introMeta.json") + val rootDirFile = File(Configuration.CurrentConfiguration.rootDir) + + val metaJsonFile = rootDirFile.combineSafe("introMeta.json") val metaData = JsonFileCodec.decodeFromString(IntroMetaData.serializer(), metaJsonFile.readText()) - val htmlFile = File(Configuration.CurrentConfiguration.articleDir).parentFile.combineSafe("intro.html") + val htmlFile = rootDirFile.combineSafe("intro.html") val fileHtml = htmlFile.readText() return page(metaData.title, standardNavBar(), null, metaData.ogData) { diff --git a/src/main/kotlin/info/mechyrdia/lore/views_robots.kt b/src/main/kotlin/info/mechyrdia/lore/views_robots.kt new file mode 100644 index 0000000..045bd59 --- /dev/null +++ b/src/main/kotlin/info/mechyrdia/lore/views_robots.kt @@ -0,0 +1,65 @@ +package info.mechyrdia.lore + +import info.mechyrdia.Configuration +import io.ktor.util.* +import java.io.File +import java.time.Instant + +private const val AVERAGE_FACTBOOK_PAGE_CHANGEFREQ = "hourly" +private const val AVERAGE_FACTBOOK_INDEX_CHANGEFREQ = "daily" +private const val FACTBOOK_ROOT_PRIORITY = "0.6" +private const val FACTBOOK_INDEX_PRIORITY = "0.4" +private const val FACTBOOK_PAGE_PRIORITY = "0.8" + +private val File.lastSubFilesModified: Instant? + get() = if (isDirectory) + (listFiles().orEmpty().mapNotNull { + it.lastSubFilesModified + } + Instant.ofEpochMilli(lastModified())).max() + else null + +private val File.lastContentModified: Instant + get() = lastSubFilesModified ?: Instant.ofEpochMilli(lastModified()) + +private fun List.renderIntoSitemap(sitemap: Appendable, base: String? = null) { + val prefix by lazy(LazyThreadSafetyMode.NONE) { base?.let { "$it/" } ?: "" } + for (node in this) { + if (Configuration.CurrentConfiguration.isDevMode || !(node.name.endsWith(".wip") || node.name.endsWith(".old"))) { + val path = "$prefix${node.name}" + + val file = File(Configuration.CurrentConfiguration.articleDir).combineSafe(path) + val lastModified = file.lastContentModified + val changeFreq = if (node.subNodes.isNotEmpty()) AVERAGE_FACTBOOK_INDEX_CHANGEFREQ else AVERAGE_FACTBOOK_PAGE_CHANGEFREQ + val priority = if (node.subNodes.isNotEmpty()) FACTBOOK_INDEX_PRIORITY else FACTBOOK_PAGE_PRIORITY + + sitemap.appendLine("\t") + sitemap.appendLine("\t\thttps://mechyrdia.info/lore/$path") + sitemap.appendLine("\t\t$lastModified") + sitemap.appendLine("\t\t$changeFreq") + sitemap.appendLine("\t\t$priority") + sitemap.appendLine("\t") + node.subNodes.renderIntoSitemap(sitemap, path) + } + } +} + +private fun renderLoreSitemap(sitemap: Appendable) { + val rootFile = File(Configuration.CurrentConfiguration.articleDir) + val rootLastModified = rootFile.lastContentModified + + sitemap.appendLine("\t") + sitemap.appendLine("\t\thttps://mechyrdia.info/lore") + sitemap.appendLine("\t\t$rootLastModified") + sitemap.appendLine("\t\t$AVERAGE_FACTBOOK_INDEX_CHANGEFREQ") + sitemap.appendLine("\t\t$FACTBOOK_ROOT_PRIORITY") + sitemap.appendLine("\t") + + rootArticleNodeList().renderIntoSitemap(sitemap) +} + +fun Appendable.generateSitemap() { + appendLine("") + appendLine("") + renderLoreSitemap(this) + appendLine("") +} -- 2.25.1