From: Lanius Trolling <lanius@laniustrolling.dev>
Date: Mon, 7 Nov 2022 21:42:57 +0000 (-0500)
Subject: Add data and templating support
X-Git-Url: https://gitweb.starshipfights.net/?a=commitdiff_plain;h=ae4108bf7c2ad55f96fdd4e102a46636b3597512;p=factbooks

Add data and templating support
---

diff --git a/.idea/kotlinc.xml b/.idea/kotlinc.xml
index 7e340a7..e1eea1d 100644
--- a/.idea/kotlinc.xml
+++ b/.idea/kotlinc.xml
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="KotlinJpsPluginSettings">
-    <option name="version" value="1.6.21" />
+    <option name="version" value="1.7.20" />
   </component>
 </project>
\ No newline at end of file
diff --git a/build.gradle.kts b/build.gradle.kts
index 1dd99da..175dd73 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -2,8 +2,8 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
 	java
-	kotlin("jvm") version "1.6.21"
-	kotlin("plugin.serialization") version "1.6.21"
+	kotlin("jvm") version "1.7.20"
+	kotlin("plugin.serialization") version "1.7.20"
 	id("com.github.johnrengelman.shadow") version "7.1.2"
 	application
 }
@@ -19,16 +19,24 @@ dependencies {
 	implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4")
 	implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:1.6.4")
 	implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactive:1.6.4")
-	implementation("org.jetbrains.kotlinx:kotlinx-serialization-core:1.3.3")
-	implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
+	implementation("org.jetbrains.kotlinx:kotlinx-serialization-core:1.4.1")
+	implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1")
 	
-	implementation("io.ktor:ktor-server-netty:1.6.8")
-	implementation("io.ktor:ktor-html-builder:1.6.8")
+	implementation("io.ktor:ktor-server-netty:2.1.3")
+	implementation("io.ktor:ktor-server-html-builder:2.1.3")
 	
-	implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.7.5")
+	implementation("io.ktor:ktor-server-call-id:2.1.3")
+	implementation("io.ktor:ktor-server-call-logging:2.1.3")
+	implementation("io.ktor:ktor-server-forwarded-header:2.1.3")
+	implementation("io.ktor:ktor-server-status-pages:2.1.3")
+	
+	implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.8.0")
+	
+	implementation("com.samskivert:jmustache:1.15")
 	
 	implementation("org.slf4j:slf4j-api:1.7.36")
 	implementation("ch.qos.logback:logback-classic:1.2.11")
+	implementation("io.ktor:ktor-server-forwarded-header-jvm:2.1.3")
 }
 
 tasks.withType<KotlinCompile> {
diff --git a/src/main/kotlin/info/mechyrdia/Configuration.kt b/src/main/kotlin/info/mechyrdia/Configuration.kt
index 8f5d51b..cc0be47 100644
--- a/src/main/kotlin/info/mechyrdia/Configuration.kt
+++ b/src/main/kotlin/info/mechyrdia/Configuration.kt
@@ -1,8 +1,6 @@
 package info.mechyrdia
 
-import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.Serializable
-import kotlinx.serialization.json.Json
 import java.io.File
 
 @Serializable
@@ -11,7 +9,9 @@ data class Configuration(
 	val port: Int = 8080,
 	
 	val articleDir: String = "../lore",
+	val templateDir: String = "../tpl",
 	val assetDir: String = "../assets",
+	val jsonDocDir: String = "../data",
 ) {
 	companion object {
 		private val DEFAULT_CONFIG = Configuration()
@@ -27,21 +27,13 @@ data class Configuration(
 					if (file.exists())
 						file.deleteRecursively()
 					
-					val json = JsonConfigCodec.encodeToString(serializer(), DEFAULT_CONFIG)
+					val json = JSON.encodeToString(serializer(), DEFAULT_CONFIG)
 					file.writeText(json, Charsets.UTF_8)
 					return DEFAULT_CONFIG
 				}
 				
 				val json = file.readText()
-				return JsonConfigCodec.decodeFromString(serializer(), json).also { currentConfig = it }
+				return JSON.decodeFromString(serializer(), json).also { currentConfig = it }
 			}
-		
-		val JsonConfigCodec = Json {
-			prettyPrint = true
-			@OptIn(ExperimentalSerializationApi::class)
-			prettyPrintIndent = "\t"
-			
-			useAlternativeNames = false
-		}
 	}
 }
diff --git a/src/main/kotlin/info/mechyrdia/Factbooks.kt b/src/main/kotlin/info/mechyrdia/Factbooks.kt
index c4268c6..8f6db51 100644
--- a/src/main/kotlin/info/mechyrdia/Factbooks.kt
+++ b/src/main/kotlin/info/mechyrdia/Factbooks.kt
@@ -1,16 +1,20 @@
 package info.mechyrdia
 
 import info.mechyrdia.lore.*
-import io.ktor.application.*
-import io.ktor.features.*
-import io.ktor.html.*
 import io.ktor.http.*
-import io.ktor.http.content.*
-import io.ktor.request.*
-import io.ktor.response.*
-import io.ktor.routing.*
+import io.ktor.server.application.*
 import io.ktor.server.engine.*
+import io.ktor.server.html.*
+import io.ktor.server.http.content.*
 import io.ktor.server.netty.*
+import io.ktor.server.plugins.*
+import io.ktor.server.plugins.callid.*
+import io.ktor.server.plugins.callloging.*
+import io.ktor.server.plugins.forwardedheaders.*
+import io.ktor.server.plugins.statuspages.*
+import io.ktor.server.request.*
+import io.ktor.server.response.*
+import io.ktor.server.routing.*
 import org.slf4j.event.Level
 import java.io.File
 import java.io.IOException
@@ -25,7 +29,7 @@ object Factbooks {
 		
 		embeddedServer(Netty, port = Configuration.CurrentConfiguration.port, host = Configuration.CurrentConfiguration.host) {
 			install(IgnoreTrailingSlash)
-			install(XForwardedHeaderSupport)
+			install(XForwardedHeaders)
 			
 			install(CallId) {
 				val counter = AtomicLong(0)
@@ -45,26 +49,28 @@ object Factbooks {
 			}
 			
 			install(StatusPages) {
-				status(HttpStatusCode.NotFound) {
+				status(HttpStatusCode.NotFound) { call, _ ->
 					call.respondHtml(HttpStatusCode.NotFound, call.error404())
 				}
 				
-				exception<HttpRedirectException> { (url, permanent) ->
+				exception<HttpRedirectException> { call, (url, permanent) ->
 					call.respondRedirect(url, permanent)
 				}
-				exception<MissingRequestParameterException> {
+				exception<MissingRequestParameterException> { call, _ ->
 					call.respondHtml(HttpStatusCode.BadRequest, call.error400())
 				}
-				exception<NullPointerException> {
+				exception<NullPointerException> { call, _ ->
 					call.respondHtml(HttpStatusCode.NotFound, call.error404())
 				}
-				exception<IOException> {
+				exception<IOException> { call, _ ->
 					call.respondHtml(HttpStatusCode.NotFound, call.error404())
 				}
 				
-				exception<Throwable> {
-					call.respondHtml(HttpStatusCode.InternalServerError, call.error503())
-					throw it
+				exception<Throwable> { call, ex ->
+					call.application.log.error("Got uncaught exception from serving call ${call.callId}", ex)
+					
+					call.respondHtml(HttpStatusCode.InternalServerError, call.error500())
+					throw ex
 				}
 			}
 			
diff --git a/src/main/kotlin/info/mechyrdia/JSON.kt b/src/main/kotlin/info/mechyrdia/JSON.kt
new file mode 100644
index 0000000..75754b5
--- /dev/null
+++ b/src/main/kotlin/info/mechyrdia/JSON.kt
@@ -0,0 +1,12 @@
+package info.mechyrdia
+
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.json.Json
+
+val JSON = Json {
+	prettyPrint = true
+	@OptIn(ExperimentalSerializationApi::class)
+	prettyPrintIndent = "\t"
+	
+	useAlternativeNames = false
+}
diff --git a/src/main/kotlin/info/mechyrdia/lore/parser.kt b/src/main/kotlin/info/mechyrdia/lore/parser.kt
index 8737a05..f8bce69 100644
--- a/src/main/kotlin/info/mechyrdia/lore/parser.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/parser.kt
@@ -1,6 +1,7 @@
 package info.mechyrdia.lore
 
 data class TextParserScope<TContext>(
+	var buffer: String = "",
 	val write: Appendable,
 	val tags: TextParserTags<TContext>,
 	val ctx: TContext
@@ -8,24 +9,27 @@ data class TextParserScope<TContext>(
 
 sealed class TextParserState<TContext>(
 	val scope: TextParserScope<TContext>,
-	val insideTags: List<Pair<String, String?>>,
+	val insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>,
 	val insideDirectTags: List<String>
 ) {
 	abstract fun processCharacter(char: Char): TextParserState<TContext>
 	abstract fun processEndOfText()
 	
 	protected fun appendText(text: String) {
-		scope.write.append(
-			insideTags.foldRight(censorText(text)) { (tag, param), t ->
-				(scope.tags[tag] as? TextParserTagType.Indirect<TContext>)
-					?.process(param, t, scope.ctx)
-					?: "[$tag${param?.let { "=$it" } ?: ""}]$t[/$tag]"
-			}
-		)
+		scope.buffer += censorText(text)
 	}
 	
 	protected fun appendTextRaw(text: String) {
-		scope.write.append(text)
+		scope.buffer += text
+	}
+	
+	protected fun flushBuffer() {
+		scope.write.append(
+			insideTags.foldRight(scope.buffer) { (tag, param), t ->
+				tag.process(param, t, scope.ctx)
+			}
+		)
+		scope.buffer = ""
 	}
 	
 	class Initial<TContext>(scope: TextParserScope<TContext>) : TextParserState<TContext>(scope, listOf(), listOf()) {
@@ -41,7 +45,7 @@ sealed class TextParserState<TContext>(
 		}
 	}
 	
-	class PlainText<TContext>(scope: TextParserScope<TContext>, val text: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+	class PlainText<TContext>(scope: TextParserScope<TContext>, val text: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
 		override fun processCharacter(char: Char): TextParserState<TContext> {
 			return if (char == '[') {
 				appendText(text)
@@ -62,10 +66,11 @@ sealed class TextParserState<TContext>(
 		
 		override fun processEndOfText() {
 			appendText(text)
+			flushBuffer()
 		}
 	}
 	
-	class NoFormatText<TContext>(scope: TextParserScope<TContext>, val text: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+	class NoFormatText<TContext>(scope: TextParserScope<TContext>, val text: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
 		override fun processCharacter(char: Char): TextParserState<TContext> {
 			val newText = text + char
 			return if (newText.endsWith("[/$NO_FORMAT_TAG]")) {
@@ -81,10 +86,11 @@ sealed class TextParserState<TContext>(
 		
 		override fun processEndOfText() {
 			appendText(text)
+			flushBuffer()
 		}
 	}
 	
-	class OpenTag<TContext>(scope: TextParserScope<TContext>, val tag: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+	class OpenTag<TContext>(scope: TextParserScope<TContext>, val tag: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
 		override fun processCharacter(char: Char): TextParserState<TContext> {
 			return if (char == ']') {
 				if (tag.equals(NO_FORMAT_TAG, ignoreCase = true))
@@ -93,10 +99,19 @@ sealed class TextParserState<TContext>(
 					(scope.tags[tag] as? TextParserTagType.Direct<TContext>)?.begin(null, scope.ctx)?.let {
 						appendTextRaw(it)
 					}
+					flushBuffer()
 					
 					PlainText(scope, "", insideTags, insideDirectTags + tag)
-				} else
-					PlainText(scope, "", insideTags + (tag to null), insideDirectTags)
+				} else if (scope.tags[tag] is TextParserTagType.Indirect<TContext>) {
+					val tagType = scope.tags[tag] as TextParserTagType.Indirect<TContext>
+					flushBuffer()
+					
+					PlainText(scope, "", insideTags + (tagType to null), insideDirectTags)
+				} else {
+					appendText("[$tag]")
+					
+					PlainText(scope, "", insideTags, insideDirectTags)
+				}
 			} else if (char == '/' && tag == "")
 				CloseTag(scope, tag, insideTags, insideDirectTags)
 			else if (char == '=' && tag != "")
@@ -107,37 +122,51 @@ sealed class TextParserState<TContext>(
 		
 		override fun processEndOfText() {
 			appendText("[$tag")
+			flushBuffer()
 		}
 	}
 	
-	class TagParam<TContext>(scope: TextParserScope<TContext>, val tag: String, val param: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+	class TagParam<TContext>(scope: TextParserScope<TContext>, val tag: String, val param: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
 		override fun processCharacter(char: Char): TextParserState<TContext> {
 			return if (char == ']') {
-				val tagType = scope.tags[tag]
-				if (tagType is TextParserTagType.Direct<TContext>) {
-					appendTextRaw(tagType.begin(param, scope.ctx))
+				when (val tagType = scope.tags[tag]) {
+					is TextParserTagType.Direct<TContext> -> {
+						appendTextRaw(tagType.begin(param, scope.ctx))
+						flushBuffer()
+						
+						PlainText(scope, "", insideTags, insideDirectTags + tag)
+					}
 					
-					PlainText(scope, "", insideTags, insideDirectTags + tag)
-				} else
-					PlainText(scope, "", insideTags + (tag to param), insideDirectTags)
+					is TextParserTagType.Indirect<TContext> -> {
+						flushBuffer()
+						PlainText(scope, "", insideTags + (tagType to param), insideDirectTags)
+					}
+					else -> {
+						appendText("[$tag=$param]")
+						
+						PlainText(scope, "", insideTags, insideDirectTags)
+					}
+				}
 			} else
 				TagParam(scope, tag, param + char, insideTags, insideDirectTags)
 		}
 		
 		override fun processEndOfText() {
 			appendText("[$tag=$param")
+			flushBuffer()
 		}
 	}
 	
-	class CloseTag<TContext>(scope: TextParserScope<TContext>, val tag: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+	class CloseTag<TContext>(scope: TextParserScope<TContext>, val tag: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
 		override fun processCharacter(char: Char): TextParserState<TContext> {
 			return if (char == ']') {
 				val tagType = scope.tags[tag]
 				if (tagType is TextParserTagType.Direct<TContext> && insideDirectTags.last().equals(tag, ignoreCase = true)) {
 					appendTextRaw(tagType.end(scope.ctx))
-					
+					flushBuffer()
 					PlainText(scope, "", insideTags, insideDirectTags.dropLast(1))
-				} else if (insideTags.isNotEmpty() && insideTags.last().first.equals(tag, ignoreCase = true)) {
+				} else if (insideTags.isNotEmpty() && insideTags.last().first == scope.tags[tag]) {
+					flushBuffer()
 					PlainText(scope, "", insideTags.dropLast(1), insideDirectTags)
 				} else {
 					appendText("[/$tag]")
@@ -148,6 +177,7 @@ sealed class TextParserState<TContext>(
 		
 		override fun processEndOfText() {
 			appendText("[/$tag")
+			flushBuffer()
 		}
 	}
 	
@@ -170,7 +200,7 @@ sealed class TextParserState<TContext>(
 			val builder = StringBuilder()
 			try {
 				val fixedText = text.replace("\r\n", "\n").replace('\r', '\n')
-				fixedText.fold<TextParserState<TContext>>(Initial(TextParserScope(builder, tags, context))) { state, char -> state.processCharacter(char) }.processEndOfText()
+				fixedText.fold<TextParserState<TContext>>(Initial(TextParserScope("", builder, tags, context))) { state, char -> state.processCharacter(char) }.processEndOfText()
 			} catch (ex: Exception) {
 				return ParseOutcome("<p>$builder</p><h1>Internal Error!</h1><pre>${ex.stackTraceToString()}</pre>", false)
 			}
diff --git a/src/main/kotlin/info/mechyrdia/lore/preparser.kt b/src/main/kotlin/info/mechyrdia/lore/preparser.kt
new file mode 100644
index 0000000..0a1b17f
--- /dev/null
+++ b/src/main/kotlin/info/mechyrdia/lore/preparser.kt
@@ -0,0 +1,48 @@
+package info.mechyrdia.lore
+
+import com.samskivert.mustache.Escapers
+import com.samskivert.mustache.Mustache
+import com.samskivert.mustache.Template
+import info.mechyrdia.Configuration
+import info.mechyrdia.JSON
+import io.ktor.util.*
+import kotlinx.serialization.json.*
+import java.io.File
+import java.security.MessageDigest
+
+object PreParser {
+	private val compiler = Mustache.compiler()
+		.withEscaper(Escapers.NONE)
+		.defaultValue("{{ MISSING }}")
+		.withLoader { File(Configuration.CurrentConfiguration.templateDir, "$it.tpl").bufferedReader() }
+	
+	private val cache = mutableMapOf<String, Template>()
+	
+	private fun convertJson(json: JsonElement): Any? = when (json) {
+		JsonNull -> null
+		is JsonPrimitive -> if (json.isString)
+			json.content
+		else
+			json.intOrNull ?: json.double
+		
+		is JsonObject -> json.mapValues { (_, it) -> convertJson(it) }
+		is JsonArray -> json.map { convertJson(it) }
+	}
+	
+	private fun loadJson(name: String): JsonElement =
+		File(Configuration.CurrentConfiguration.jsonDocDir).combineSafe("$name.json")
+			.takeIf { it.isFile }
+			?.readText()
+			?.let { JSON.parseToJsonElement(it) }
+			?: JsonNull
+	
+	private val msgDigest = ThreadLocal.withInitial { MessageDigest.getInstance("SHA-256") }
+	
+	fun preparse(name: String, content: String): String {
+		val contentHash = hex(msgDigest.get().digest(content.toByteArray()))
+		val template = cache[contentHash] ?: compiler.compile(content)
+		
+		val context = convertJson(loadJson(name))
+		return template.execute(context)
+	}
+}
diff --git a/src/main/kotlin/info/mechyrdia/lore/view_tpl.kt b/src/main/kotlin/info/mechyrdia/lore/view_tpl.kt
index 22aa21e..ae47caa 100644
--- a/src/main/kotlin/info/mechyrdia/lore/view_tpl.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/view_tpl.kt
@@ -1,6 +1,6 @@
 package info.mechyrdia.lore
 
-import io.ktor.application.*
+import io.ktor.server.application.*
 import kotlinx.html.*
 
 fun ApplicationCall.page(pageTitle: String, navBar: List<NavItem>? = null, sidebar: Sidebar? = null, content: SECTIONS.() -> Unit): HTML.() -> Unit {
diff --git a/src/main/kotlin/info/mechyrdia/lore/views_error.kt b/src/main/kotlin/info/mechyrdia/lore/views_error.kt
index 9d295a6..48f3f3e 100644
--- a/src/main/kotlin/info/mechyrdia/lore/views_error.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/views_error.kt
@@ -19,7 +19,7 @@ fun ApplicationCall.error404(): HTML.() -> Unit = page("Not Found", standardNavB
 	}
 }
 
-fun ApplicationCall.error503(): HTML.() -> Unit = page("Internal Error", standardNavBar()) {
+fun ApplicationCall.error500(): HTML.() -> Unit = page("Internal Error", standardNavBar()) {
 	section {
 		h1 { +"Internal Error" }
 		p { +"The servers made a bit of a mistake. Please be patient while we clean up our mess." }
diff --git a/src/main/kotlin/info/mechyrdia/lore/views_lore.kt b/src/main/kotlin/info/mechyrdia/lore/views_lore.kt
index 7f902ed..9c42923 100644
--- a/src/main/kotlin/info/mechyrdia/lore/views_lore.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/views_lore.kt
@@ -1,7 +1,7 @@
 package info.mechyrdia.lore
 
 import info.mechyrdia.Configuration
-import io.ktor.application.*
+import io.ktor.server.application.*
 import io.ktor.util.*
 import kotlinx.html.*
 import java.io.File
@@ -26,7 +26,8 @@ fun ApplicationCall.loreArticlePage(): HTML.() -> Unit {
 			}
 		}
 	} else {
-		val pageMarkup = pageFile.readText()
+		val pageTemplate = pageFile.readText()
+		val pageMarkup = PreParser.preparse(pagePath, pageTemplate)
 		val pageHtml = TextParserState.parseText(pageMarkup, TextParserFormattingTag.asTags, Unit).html
 		
 		val pageToC = TableOfContentsBuilder()
diff --git a/src/main/kotlin/info/mechyrdia/lore/views_prefs.kt b/src/main/kotlin/info/mechyrdia/lore/views_prefs.kt
index 26f1a42..fc707d6 100644
--- a/src/main/kotlin/info/mechyrdia/lore/views_prefs.kt
+++ b/src/main/kotlin/info/mechyrdia/lore/views_prefs.kt
@@ -1,6 +1,6 @@
 package info.mechyrdia.lore
 
-import io.ktor.application.*
+import io.ktor.server.application.*
 import kotlinx.html.*
 
 fun ApplicationCall.changeThemePage(): HTML.() -> Unit {