Add data and templating support
authorLanius Trolling <lanius@laniustrolling.dev>
Mon, 7 Nov 2022 21:42:57 +0000 (16:42 -0500)
committerLanius Trolling <lanius@laniustrolling.dev>
Mon, 7 Nov 2022 21:42:57 +0000 (16:42 -0500)
.idea/kotlinc.xml
build.gradle.kts
src/main/kotlin/info/mechyrdia/Configuration.kt
src/main/kotlin/info/mechyrdia/Factbooks.kt
src/main/kotlin/info/mechyrdia/JSON.kt [new file with mode: 0644]
src/main/kotlin/info/mechyrdia/lore/parser.kt
src/main/kotlin/info/mechyrdia/lore/preparser.kt [new file with mode: 0644]
src/main/kotlin/info/mechyrdia/lore/view_tpl.kt
src/main/kotlin/info/mechyrdia/lore/views_error.kt
src/main/kotlin/info/mechyrdia/lore/views_lore.kt
src/main/kotlin/info/mechyrdia/lore/views_prefs.kt

index 7e340a776a6a2b978d333a4d2815fa12ccacbd91..e1eea1d6b9d84faa7006e37a676609e51525b125 100644 (file)
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="KotlinJpsPluginSettings">
-    <option name="version" value="1.6.21" />
+    <option name="version" value="1.7.20" />
   </component>
 </project>
\ No newline at end of file
index 1dd99da09cf54e33358b306c923301c113cee4cd..175dd7357b14f23ab1fbadfa76b73ed71371951a 100644 (file)
@@ -2,8 +2,8 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
        java
-       kotlin("jvm") version "1.6.21"
-       kotlin("plugin.serialization") version "1.6.21"
+       kotlin("jvm") version "1.7.20"
+       kotlin("plugin.serialization") version "1.7.20"
        id("com.github.johnrengelman.shadow") version "7.1.2"
        application
 }
@@ -19,16 +19,24 @@ dependencies {
        implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.6.4")
        implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:1.6.4")
        implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactive:1.6.4")
-       implementation("org.jetbrains.kotlinx:kotlinx-serialization-core:1.3.3")
-       implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.3.3")
+       implementation("org.jetbrains.kotlinx:kotlinx-serialization-core:1.4.1")
+       implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.4.1")
        
-       implementation("io.ktor:ktor-server-netty:1.6.8")
-       implementation("io.ktor:ktor-html-builder:1.6.8")
+       implementation("io.ktor:ktor-server-netty:2.1.3")
+       implementation("io.ktor:ktor-server-html-builder:2.1.3")
        
-       implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.7.5")
+       implementation("io.ktor:ktor-server-call-id:2.1.3")
+       implementation("io.ktor:ktor-server-call-logging:2.1.3")
+       implementation("io.ktor:ktor-server-forwarded-header:2.1.3")
+       implementation("io.ktor:ktor-server-status-pages:2.1.3")
+       
+       implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.8.0")
+       
+       implementation("com.samskivert:jmustache:1.15")
        
        implementation("org.slf4j:slf4j-api:1.7.36")
        implementation("ch.qos.logback:logback-classic:1.2.11")
+       implementation("io.ktor:ktor-server-forwarded-header-jvm:2.1.3")
 }
 
 tasks.withType<KotlinCompile> {
index 8f5d51b9872aa567997144821942009111554ec0..cc0be479df7f29a0644f25eb6992dd3c15085c23 100644 (file)
@@ -1,8 +1,6 @@
 package info.mechyrdia
 
-import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.Serializable
-import kotlinx.serialization.json.Json
 import java.io.File
 
 @Serializable
@@ -11,7 +9,9 @@ data class Configuration(
        val port: Int = 8080,
        
        val articleDir: String = "../lore",
+       val templateDir: String = "../tpl",
        val assetDir: String = "../assets",
+       val jsonDocDir: String = "../data",
 ) {
        companion object {
                private val DEFAULT_CONFIG = Configuration()
@@ -27,21 +27,13 @@ data class Configuration(
                                        if (file.exists())
                                                file.deleteRecursively()
                                        
-                                       val json = JsonConfigCodec.encodeToString(serializer(), DEFAULT_CONFIG)
+                                       val json = JSON.encodeToString(serializer(), DEFAULT_CONFIG)
                                        file.writeText(json, Charsets.UTF_8)
                                        return DEFAULT_CONFIG
                                }
                                
                                val json = file.readText()
-                               return JsonConfigCodec.decodeFromString(serializer(), json).also { currentConfig = it }
+                               return JSON.decodeFromString(serializer(), json).also { currentConfig = it }
                        }
-               
-               val JsonConfigCodec = Json {
-                       prettyPrint = true
-                       @OptIn(ExperimentalSerializationApi::class)
-                       prettyPrintIndent = "\t"
-                       
-                       useAlternativeNames = false
-               }
        }
 }
index c4268c60a435a5a3c036d27a63469d462003386f..8f6db51643520e9f3072c831fe5ba18aaec3c89f 100644 (file)
@@ -1,16 +1,20 @@
 package info.mechyrdia
 
 import info.mechyrdia.lore.*
-import io.ktor.application.*
-import io.ktor.features.*
-import io.ktor.html.*
 import io.ktor.http.*
-import io.ktor.http.content.*
-import io.ktor.request.*
-import io.ktor.response.*
-import io.ktor.routing.*
+import io.ktor.server.application.*
 import io.ktor.server.engine.*
+import io.ktor.server.html.*
+import io.ktor.server.http.content.*
 import io.ktor.server.netty.*
+import io.ktor.server.plugins.*
+import io.ktor.server.plugins.callid.*
+import io.ktor.server.plugins.callloging.*
+import io.ktor.server.plugins.forwardedheaders.*
+import io.ktor.server.plugins.statuspages.*
+import io.ktor.server.request.*
+import io.ktor.server.response.*
+import io.ktor.server.routing.*
 import org.slf4j.event.Level
 import java.io.File
 import java.io.IOException
@@ -25,7 +29,7 @@ object Factbooks {
                
                embeddedServer(Netty, port = Configuration.CurrentConfiguration.port, host = Configuration.CurrentConfiguration.host) {
                        install(IgnoreTrailingSlash)
-                       install(XForwardedHeaderSupport)
+                       install(XForwardedHeaders)
                        
                        install(CallId) {
                                val counter = AtomicLong(0)
@@ -45,26 +49,28 @@ object Factbooks {
                        }
                        
                        install(StatusPages) {
-                               status(HttpStatusCode.NotFound) {
+                               status(HttpStatusCode.NotFound) { call, _ ->
                                        call.respondHtml(HttpStatusCode.NotFound, call.error404())
                                }
                                
-                               exception<HttpRedirectException> { (url, permanent) ->
+                               exception<HttpRedirectException> { call, (url, permanent) ->
                                        call.respondRedirect(url, permanent)
                                }
-                               exception<MissingRequestParameterException> {
+                               exception<MissingRequestParameterException> { call, _ ->
                                        call.respondHtml(HttpStatusCode.BadRequest, call.error400())
                                }
-                               exception<NullPointerException> {
+                               exception<NullPointerException> { call, _ ->
                                        call.respondHtml(HttpStatusCode.NotFound, call.error404())
                                }
-                               exception<IOException> {
+                               exception<IOException> { call, _ ->
                                        call.respondHtml(HttpStatusCode.NotFound, call.error404())
                                }
                                
-                               exception<Throwable> {
-                                       call.respondHtml(HttpStatusCode.InternalServerError, call.error503())
-                                       throw it
+                               exception<Throwable> { call, ex ->
+                                       call.application.log.error("Got uncaught exception from serving call ${call.callId}", ex)
+                                       
+                                       call.respondHtml(HttpStatusCode.InternalServerError, call.error500())
+                                       throw ex
                                }
                        }
                        
diff --git a/src/main/kotlin/info/mechyrdia/JSON.kt b/src/main/kotlin/info/mechyrdia/JSON.kt
new file mode 100644 (file)
index 0000000..75754b5
--- /dev/null
@@ -0,0 +1,12 @@
+package info.mechyrdia
+
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.json.Json
+
+val JSON = Json {
+       prettyPrint = true
+       @OptIn(ExperimentalSerializationApi::class)
+       prettyPrintIndent = "\t"
+       
+       useAlternativeNames = false
+}
index 8737a05f6e0e63bc582a9629f7036a2cd178787d..f8bce6928041541457d9449b8661e74824fcc551 100644 (file)
@@ -1,6 +1,7 @@
 package info.mechyrdia.lore
 
 data class TextParserScope<TContext>(
+       var buffer: String = "",
        val write: Appendable,
        val tags: TextParserTags<TContext>,
        val ctx: TContext
@@ -8,24 +9,27 @@ data class TextParserScope<TContext>(
 
 sealed class TextParserState<TContext>(
        val scope: TextParserScope<TContext>,
-       val insideTags: List<Pair<String, String?>>,
+       val insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>,
        val insideDirectTags: List<String>
 ) {
        abstract fun processCharacter(char: Char): TextParserState<TContext>
        abstract fun processEndOfText()
        
        protected fun appendText(text: String) {
-               scope.write.append(
-                       insideTags.foldRight(censorText(text)) { (tag, param), t ->
-                               (scope.tags[tag] as? TextParserTagType.Indirect<TContext>)
-                                       ?.process(param, t, scope.ctx)
-                                       ?: "[$tag${param?.let { "=$it" } ?: ""}]$t[/$tag]"
-                       }
-               )
+               scope.buffer += censorText(text)
        }
        
        protected fun appendTextRaw(text: String) {
-               scope.write.append(text)
+               scope.buffer += text
+       }
+       
+       protected fun flushBuffer() {
+               scope.write.append(
+                       insideTags.foldRight(scope.buffer) { (tag, param), t ->
+                               tag.process(param, t, scope.ctx)
+                       }
+               )
+               scope.buffer = ""
        }
        
        class Initial<TContext>(scope: TextParserScope<TContext>) : TextParserState<TContext>(scope, listOf(), listOf()) {
@@ -41,7 +45,7 @@ sealed class TextParserState<TContext>(
                }
        }
        
-       class PlainText<TContext>(scope: TextParserScope<TContext>, val text: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+       class PlainText<TContext>(scope: TextParserScope<TContext>, val text: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
                override fun processCharacter(char: Char): TextParserState<TContext> {
                        return if (char == '[') {
                                appendText(text)
@@ -62,10 +66,11 @@ sealed class TextParserState<TContext>(
                
                override fun processEndOfText() {
                        appendText(text)
+                       flushBuffer()
                }
        }
        
-       class NoFormatText<TContext>(scope: TextParserScope<TContext>, val text: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+       class NoFormatText<TContext>(scope: TextParserScope<TContext>, val text: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
                override fun processCharacter(char: Char): TextParserState<TContext> {
                        val newText = text + char
                        return if (newText.endsWith("[/$NO_FORMAT_TAG]")) {
@@ -81,10 +86,11 @@ sealed class TextParserState<TContext>(
                
                override fun processEndOfText() {
                        appendText(text)
+                       flushBuffer()
                }
        }
        
-       class OpenTag<TContext>(scope: TextParserScope<TContext>, val tag: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+       class OpenTag<TContext>(scope: TextParserScope<TContext>, val tag: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
                override fun processCharacter(char: Char): TextParserState<TContext> {
                        return if (char == ']') {
                                if (tag.equals(NO_FORMAT_TAG, ignoreCase = true))
@@ -93,10 +99,19 @@ sealed class TextParserState<TContext>(
                                        (scope.tags[tag] as? TextParserTagType.Direct<TContext>)?.begin(null, scope.ctx)?.let {
                                                appendTextRaw(it)
                                        }
+                                       flushBuffer()
                                        
                                        PlainText(scope, "", insideTags, insideDirectTags + tag)
-                               } else
-                                       PlainText(scope, "", insideTags + (tag to null), insideDirectTags)
+                               } else if (scope.tags[tag] is TextParserTagType.Indirect<TContext>) {
+                                       val tagType = scope.tags[tag] as TextParserTagType.Indirect<TContext>
+                                       flushBuffer()
+                                       
+                                       PlainText(scope, "", insideTags + (tagType to null), insideDirectTags)
+                               } else {
+                                       appendText("[$tag]")
+                                       
+                                       PlainText(scope, "", insideTags, insideDirectTags)
+                               }
                        } else if (char == '/' && tag == "")
                                CloseTag(scope, tag, insideTags, insideDirectTags)
                        else if (char == '=' && tag != "")
@@ -107,37 +122,51 @@ sealed class TextParserState<TContext>(
                
                override fun processEndOfText() {
                        appendText("[$tag")
+                       flushBuffer()
                }
        }
        
-       class TagParam<TContext>(scope: TextParserScope<TContext>, val tag: String, val param: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+       class TagParam<TContext>(scope: TextParserScope<TContext>, val tag: String, val param: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
                override fun processCharacter(char: Char): TextParserState<TContext> {
                        return if (char == ']') {
-                               val tagType = scope.tags[tag]
-                               if (tagType is TextParserTagType.Direct<TContext>) {
-                                       appendTextRaw(tagType.begin(param, scope.ctx))
+                               when (val tagType = scope.tags[tag]) {
+                                       is TextParserTagType.Direct<TContext> -> {
+                                               appendTextRaw(tagType.begin(param, scope.ctx))
+                                               flushBuffer()
+                                               
+                                               PlainText(scope, "", insideTags, insideDirectTags + tag)
+                                       }
                                        
-                                       PlainText(scope, "", insideTags, insideDirectTags + tag)
-                               } else
-                                       PlainText(scope, "", insideTags + (tag to param), insideDirectTags)
+                                       is TextParserTagType.Indirect<TContext> -> {
+                                               flushBuffer()
+                                               PlainText(scope, "", insideTags + (tagType to param), insideDirectTags)
+                                       }
+                                       else -> {
+                                               appendText("[$tag=$param]")
+                                               
+                                               PlainText(scope, "", insideTags, insideDirectTags)
+                                       }
+                               }
                        } else
                                TagParam(scope, tag, param + char, insideTags, insideDirectTags)
                }
                
                override fun processEndOfText() {
                        appendText("[$tag=$param")
+                       flushBuffer()
                }
        }
        
-       class CloseTag<TContext>(scope: TextParserScope<TContext>, val tag: String, insideTags: List<Pair<String, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
+       class CloseTag<TContext>(scope: TextParserScope<TContext>, val tag: String, insideTags: List<Pair<TextParserTagType.Indirect<TContext>, String?>>, insideDirectTags: List<String>) : TextParserState<TContext>(scope, insideTags, insideDirectTags) {
                override fun processCharacter(char: Char): TextParserState<TContext> {
                        return if (char == ']') {
                                val tagType = scope.tags[tag]
                                if (tagType is TextParserTagType.Direct<TContext> && insideDirectTags.last().equals(tag, ignoreCase = true)) {
                                        appendTextRaw(tagType.end(scope.ctx))
-                                       
+                                       flushBuffer()
                                        PlainText(scope, "", insideTags, insideDirectTags.dropLast(1))
-                               } else if (insideTags.isNotEmpty() && insideTags.last().first.equals(tag, ignoreCase = true)) {
+                               } else if (insideTags.isNotEmpty() && insideTags.last().first == scope.tags[tag]) {
+                                       flushBuffer()
                                        PlainText(scope, "", insideTags.dropLast(1), insideDirectTags)
                                } else {
                                        appendText("[/$tag]")
@@ -148,6 +177,7 @@ sealed class TextParserState<TContext>(
                
                override fun processEndOfText() {
                        appendText("[/$tag")
+                       flushBuffer()
                }
        }
        
@@ -170,7 +200,7 @@ sealed class TextParserState<TContext>(
                        val builder = StringBuilder()
                        try {
                                val fixedText = text.replace("\r\n", "\n").replace('\r', '\n')
-                               fixedText.fold<TextParserState<TContext>>(Initial(TextParserScope(builder, tags, context))) { state, char -> state.processCharacter(char) }.processEndOfText()
+                               fixedText.fold<TextParserState<TContext>>(Initial(TextParserScope("", builder, tags, context))) { state, char -> state.processCharacter(char) }.processEndOfText()
                        } catch (ex: Exception) {
                                return ParseOutcome("<p>$builder</p><h1>Internal Error!</h1><pre>${ex.stackTraceToString()}</pre>", false)
                        }
diff --git a/src/main/kotlin/info/mechyrdia/lore/preparser.kt b/src/main/kotlin/info/mechyrdia/lore/preparser.kt
new file mode 100644 (file)
index 0000000..0a1b17f
--- /dev/null
@@ -0,0 +1,48 @@
+package info.mechyrdia.lore
+
+import com.samskivert.mustache.Escapers
+import com.samskivert.mustache.Mustache
+import com.samskivert.mustache.Template
+import info.mechyrdia.Configuration
+import info.mechyrdia.JSON
+import io.ktor.util.*
+import kotlinx.serialization.json.*
+import java.io.File
+import java.security.MessageDigest
+
+object PreParser {
+       private val compiler = Mustache.compiler()
+               .withEscaper(Escapers.NONE)
+               .defaultValue("{{ MISSING }}")
+               .withLoader { File(Configuration.CurrentConfiguration.templateDir, "$it.tpl").bufferedReader() }
+       
+       private val cache = mutableMapOf<String, Template>()
+       
+       private fun convertJson(json: JsonElement): Any? = when (json) {
+               JsonNull -> null
+               is JsonPrimitive -> if (json.isString)
+                       json.content
+               else
+                       json.intOrNull ?: json.double
+               
+               is JsonObject -> json.mapValues { (_, it) -> convertJson(it) }
+               is JsonArray -> json.map { convertJson(it) }
+       }
+       
+       private fun loadJson(name: String): JsonElement =
+               File(Configuration.CurrentConfiguration.jsonDocDir).combineSafe("$name.json")
+                       .takeIf { it.isFile }
+                       ?.readText()
+                       ?.let { JSON.parseToJsonElement(it) }
+                       ?: JsonNull
+       
+       private val msgDigest = ThreadLocal.withInitial { MessageDigest.getInstance("SHA-256") }
+       
+       fun preparse(name: String, content: String): String {
+               val contentHash = hex(msgDigest.get().digest(content.toByteArray()))
+               val template = cache[contentHash] ?: compiler.compile(content)
+               
+               val context = convertJson(loadJson(name))
+               return template.execute(context)
+       }
+}
index 22aa21ec5dce2e101e3dd97ad164f291b9b6067b..ae47caae89fddba5d4d8e835195189fba7f1116a 100644 (file)
@@ -1,6 +1,6 @@
 package info.mechyrdia.lore
 
-import io.ktor.application.*
+import io.ktor.server.application.*
 import kotlinx.html.*
 
 fun ApplicationCall.page(pageTitle: String, navBar: List<NavItem>? = null, sidebar: Sidebar? = null, content: SECTIONS.() -> Unit): HTML.() -> Unit {
index 9d295a64486c79fc6976558550f7df7f1333bb97..48f3f3e4be3c4604bf7b102ece39604d24f7a6f1 100644 (file)
@@ -19,7 +19,7 @@ fun ApplicationCall.error404(): HTML.() -> Unit = page("Not Found", standardNavB
        }
 }
 
-fun ApplicationCall.error503(): HTML.() -> Unit = page("Internal Error", standardNavBar()) {
+fun ApplicationCall.error500(): HTML.() -> Unit = page("Internal Error", standardNavBar()) {
        section {
                h1 { +"Internal Error" }
                p { +"The servers made a bit of a mistake. Please be patient while we clean up our mess." }
index 7f902ed41bc25479be74f436541ff818a3755b92..9c4292359525f5dbc44bfe08d60cf7f502426efe 100644 (file)
@@ -1,7 +1,7 @@
 package info.mechyrdia.lore
 
 import info.mechyrdia.Configuration
-import io.ktor.application.*
+import io.ktor.server.application.*
 import io.ktor.util.*
 import kotlinx.html.*
 import java.io.File
@@ -26,7 +26,8 @@ fun ApplicationCall.loreArticlePage(): HTML.() -> Unit {
                        }
                }
        } else {
-               val pageMarkup = pageFile.readText()
+               val pageTemplate = pageFile.readText()
+               val pageMarkup = PreParser.preparse(pagePath, pageTemplate)
                val pageHtml = TextParserState.parseText(pageMarkup, TextParserFormattingTag.asTags, Unit).html
                
                val pageToC = TableOfContentsBuilder()
index 26f1a421384aad4525632487c46b388611e489a7..fc707d67bbace1a0b93e4e4ea3bdaa82f60c30d1 100644 (file)
@@ -1,6 +1,6 @@
 package info.mechyrdia.lore
 
-import io.ktor.application.*
+import io.ktor.server.application.*
 import kotlinx.html.*
 
 fun ApplicationCall.changeThemePage(): HTML.() -> Unit {