From 664b53cba4f4b0f9881e29acbe5c574bc08f8c09 Mon Sep 17 00:00:00 2001 From: Lanius Trolling Date: Sat, 20 Jan 2024 14:38:12 -0500 Subject: [PATCH] Switch from JMustache to Pebble --- build.gradle.kts | 3 +- .../kotlin/info/mechyrdia/lore/preparser.kt | 122 +----- .../info/mechyrdia/lore/preparser_config.kt | 352 ++++++++++++++++++ .../mechyrdia/lore/preparser_functions.kt | 47 --- 4 files changed, 370 insertions(+), 154 deletions(-) create mode 100644 src/jvmMain/kotlin/info/mechyrdia/lore/preparser_config.kt delete mode 100644 src/jvmMain/kotlin/info/mechyrdia/lore/preparser_functions.kt diff --git a/build.gradle.kts b/build.gradle.kts index b8e052c..a0e572d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -130,9 +130,8 @@ kotlin { implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.10.1") - implementation("com.samskivert:jmustache:1.15") + implementation("io.pebbletemplates:pebble:3.2.2") implementation("org.apache.groovy:groovy-jsr223:4.0.10") - implementation("org.apache.groovy:groovy-json:4.0.10") implementation(files("libs/nsapi4j.jar")) diff --git a/src/jvmMain/kotlin/info/mechyrdia/lore/preparser.kt b/src/jvmMain/kotlin/info/mechyrdia/lore/preparser.kt index 6b14cf5..bfd9548 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/lore/preparser.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/lore/preparser.kt @@ -1,119 +1,31 @@ package info.mechyrdia.lore -import com.samskivert.mustache.Escapers -import com.samskivert.mustache.Mustache -import info.mechyrdia.Configuration -import info.mechyrdia.JsonFileCodec import info.mechyrdia.application import io.ktor.server.application.* -import io.ktor.util.* -import kotlinx.serialization.json.* -import java.io.File -import javax.script.ScriptException - -@JvmInline -value class JsonPath private constructor(private val pathElements: List) { - constructor(path: String) : this(path.split('.').filterNot { it.isBlank() }) - - operator fun component1() = pathElements.firstOrNull() - operator fun component2() = JsonPath(pathElements.drop(1)) - - override fun toString(): String { - return pathElements.joinToString(separator = ".") - } -} - -operator fun JsonElement.get(path: JsonPath): JsonElement { - val (pathHead, pathTail) = path - pathHead ?: return this - - return when (this) { - is JsonObject -> this.getValue(pathHead)[pathTail] - is JsonArray -> this[pathHead.toInt()][pathTail] - is JsonPrimitive -> throw NoSuchElementException("Cannot resolve path $path on JSON primitive $this") - } -} - -@JvmInline -value class JsonImport private constructor(private val importFrom: Pair) { - fun resolve(): Pair { - return try { - importFrom.let { (file, path) -> - file to JsonFileCodec.parseToJsonElement(file.readText())[path] - } - } catch (ex: RuntimeException) { - val filePath = importFrom.first.toRelativeString(File(Configuration.CurrentConfiguration.jsonDocDir)) - val jsonPath = importFrom.second - throw IllegalArgumentException("Unable to resolve JSON path $jsonPath on file $filePath", ex) - } - } - - companion object { - operator fun invoke(statement: String, currentFile: File): JsonImport? { - if (!statement.startsWith('@')) return null - val splitterIndex = statement.lastIndexOf('#') - - val (filePath, jsonPath) = if (splitterIndex != -1) - statement.substring(1, splitterIndex) to statement.substring(splitterIndex + 1) - else - statement.substring(1) to "" - - val file = if (filePath.startsWith('/')) - File(Configuration.CurrentConfiguration.jsonDocDir).combineSafe("$filePath.json") - else - currentFile.parentFile.combineSafe("$filePath.json") - - if (!file.isFile) - throw IllegalArgumentException("JSON import path '$filePath' does not point to a file") - - return JsonImport(file to JsonPath(jsonPath)) - } - } -} +import io.pebbletemplates.pebble.PebbleEngine +import io.pebbletemplates.pebble.error.PebbleException +import java.io.StringWriter object PreParser { - private val compiler = Mustache.compiler() - .withEscaper(Escapers.NONE) - .defaultValue("{{ MISSING VALUE \"{{name}}\" }}") - .withLoader { File(Configuration.CurrentConfiguration.templateDir).combineSafe("$it.tpl").bufferedReader() } - - private fun convertJson(json: JsonElement, currentFile: File): Any? = when (json) { - JsonNull -> null - is JsonPrimitive -> if (json.isString) { - JsonImport(json.content, currentFile)?.let { jsonImport -> - val (nextFile, jsonData) = jsonImport.resolve() - convertJson(jsonData, nextFile) - } ?: json.content - } else json.intOrNull ?: json.double - - is JsonObject -> json.mapValues { (_, it) -> convertJson(it, currentFile) } - is JsonArray -> json.map { convertJson(it, currentFile) } - } - - private fun loadJsonContext(name: String): Map<*, *> = - File(Configuration.CurrentConfiguration.jsonDocDir) - .combineSafe("$name.json") - .takeIf { it.isFile } - ?.let { file -> - val text = file.readText() - val data = convertJson(JsonFileCodec.parseToJsonElement(text), file) - if (data !is Map<*, *>) - error("JSON Object expected in file $name, got $text") - - data - } ?: emptyMap() + private val pebble = PebbleEngine.Builder() + .addEscapingStrategy("none", PebbleNoEscaping) + .defaultEscapingStrategy("none") + .autoEscaping(false) + .loader(PebbleTemplateLoader) + .registerExtensionCustomizer(::PebbleExtensionCustomizer) + .extension(PebbleFactbooksExtension) + .build() fun preparse(name: String, content: String): String { return try { - val template = compiler.compile(content) - val context = loadJsonContext(name) + PreParserFunctions.getFunctions() - template.execute(context) - } catch (ex: RuntimeException) { + val template = pebble.getLiteralTemplate(content) + val context = PebbleJsonLoader.loadJsonContextOrEmpty(name).toMutableMap() + val writer = StringWriter() + template.evaluate(writer, context) + return writer.toString() + } catch (ex: PebbleException) { application.log.warn("Runtime error pre-parsing factbook $name", ex) "[h1]Error[/h1]\n\nThere was a runtime error pre-parsing this factbook: ${ex.message}" - } catch (ex: ScriptException) { - application.log.warn("Scripting error pre-parsing factbook $name", ex) - "[h1]Error[/h1]\n\nThere was a scripting error pre-parsing this factbook: ${ex.message}" } } } diff --git a/src/jvmMain/kotlin/info/mechyrdia/lore/preparser_config.kt b/src/jvmMain/kotlin/info/mechyrdia/lore/preparser_config.kt new file mode 100644 index 0000000..d20bdff --- /dev/null +++ b/src/jvmMain/kotlin/info/mechyrdia/lore/preparser_config.kt @@ -0,0 +1,352 @@ +package info.mechyrdia.lore + +import info.mechyrdia.Configuration +import info.mechyrdia.JsonFileCodec +import info.mechyrdia.JsonStorageCodec +import io.ktor.util.* +import io.pebbletemplates.pebble.error.PebbleException +import io.pebbletemplates.pebble.extension.* +import io.pebbletemplates.pebble.extension.Function +import io.pebbletemplates.pebble.extension.escaper.EscapeFilter +import io.pebbletemplates.pebble.extension.escaper.EscapingStrategy +import io.pebbletemplates.pebble.extension.escaper.RawFilter +import io.pebbletemplates.pebble.extension.i18n.i18nFunction +import io.pebbletemplates.pebble.loader.Loader +import io.pebbletemplates.pebble.template.EvaluationContext +import io.pebbletemplates.pebble.template.PebbleTemplate +import io.pebbletemplates.pebble.tokenParser.* +import kotlinx.serialization.json.* +import java.io.File +import java.io.IOException +import java.io.Reader +import java.security.MessageDigest +import java.util.concurrent.ConcurrentHashMap +import javax.script.* +import kotlin.reflect.jvm.jvmName + +object PebbleNoEscaping : EscapingStrategy { + override fun escape(input: String): String { + return input + } +} + +data class PebbleTemplateCacheKey(val fileName: String, val lastModified: Long) { + constructor(name: String) : this(name, File(Configuration.CurrentConfiguration.templateDir).combineSafe("$name.tpl").lastModified()) + + val file: File + get() = File(Configuration.CurrentConfiguration.templateDir).combineSafe("$fileName.tpl") + + companion object { + fun exists(name: String) = File(Configuration.CurrentConfiguration.templateDir).combineSafe("$name.tpl").isFile + } +} + +object PebbleTemplateLoader : Loader { + override fun getReader(cacheKey: PebbleTemplateCacheKey): Reader { + return cacheKey.file.reader() + } + + override fun setCharset(charset: String?) { + // no-op + } + + override fun setPrefix(prefix: String) { + // no-op + } + + override fun setSuffix(suffix: String) { + // no-op + } + + override fun resolveRelativePath(relativePath: String, anchorPath: String): String { + val templateDir = File(Configuration.CurrentConfiguration.templateDir) + if ('\n' in anchorPath) // Probably a raw template string + return relativePath + + return templateDir.combineSafe("$anchorPath/$relativePath").toRelativeString(templateDir) + } + + override fun createCacheKey(templateName: String): PebbleTemplateCacheKey { + return PebbleTemplateCacheKey(templateName) + } + + override fun resourceExists(templateName: String): Boolean { + return PebbleTemplateCacheKey.exists(templateName) + } +} + +class PebbleExtensionCustomizer(delegate: Extension) : ExtensionCustomizer(delegate) { + override fun getFilters(): MutableMap { + return super.getFilters().orEmpty().filterValues { filter -> + filter !is EscapeFilter && filter !is RawFilter + }.toMutableMap() + } + + override fun getFunctions(): MutableMap { + return super.getFunctions().orEmpty().filterValues { function -> + function !is i18nFunction + }.toMutableMap() + } + + override fun getTokenParsers(): MutableList { + return super.getTokenParsers().orEmpty().filter { tokenParser -> + tokenParser !is AutoEscapeTokenParser && tokenParser !is CacheTokenParser && tokenParser !is FlushTokenParser && tokenParser !is ParallelTokenParser + }.toMutableList() + } +} + +object PebbleFactbooksExtension : AbstractExtension() { + override fun getFilters(): MutableMap { + return mutableMapOf( + "toJson" to PebbleToJsonFilter, + "fromJson" to PebbleFromJsonFilter, + "script" to PebbleScriptFilter, + ) + } + + override fun getFunctions(): MutableMap { + return mutableMapOf( + "loadJson" to PebbleLoadJsonFunction, + "script" to PebbleScriptFunction, + ) + } +} + +@JvmInline +value class JsonPath private constructor(private val pathElements: List) { + constructor(path: String) : this(path.split('.').filterNot { it.isBlank() }) + + operator fun component1() = pathElements.firstOrNull() + operator fun component2() = JsonPath(pathElements.drop(1)) + + override fun toString(): String { + return pathElements.joinToString(separator = ".") + } + + companion object { + val Empty = JsonPath(emptyList()) + } +} + +operator fun JsonElement.get(path: JsonPath): JsonElement { + val (pathHead, pathTail) = path + pathHead ?: return this + + return when (this) { + is JsonObject -> this.getValue(pathHead)[pathTail] + is JsonArray -> this[pathHead.toInt()][pathTail] + is JsonPrimitive -> throw NoSuchElementException("Cannot resolve path $path on JSON primitive $this") + } +} + +@JvmInline +value class JsonImport private constructor(private val importFrom: Pair) { + fun resolve(): Pair { + return try { + importFrom.let { (file, path) -> + file to JsonFileCodec.parseToJsonElement(file.readText())[path] + } + } catch (ex: RuntimeException) { + val filePath = importFrom.first.toRelativeString(File(Configuration.CurrentConfiguration.jsonDocDir)) + val jsonPath = importFrom.second + throw IllegalArgumentException("Unable to resolve JSON path $jsonPath on file $filePath", ex) + } + } + + companion object { + operator fun invoke(statement: String, currentFile: File): JsonImport? { + if (!statement.startsWith('@')) return null + val splitterIndex = statement.lastIndexOf('#') + + val (filePath, jsonPath) = if (splitterIndex != -1) + statement.substring(1, splitterIndex) to statement.substring(splitterIndex + 1) + else + statement.substring(1) to null + + val file = if (filePath.startsWith('/')) + File(Configuration.CurrentConfiguration.jsonDocDir).combineSafe("$filePath.json") + else + currentFile.parentFile.combineSafe("$filePath.json") + + if (!file.isFile) + throw IllegalArgumentException("JSON import path '$filePath' does not point to a file") + + return JsonImport(file to (jsonPath?.let { JsonPath(it) } ?: JsonPath.Empty)) + } + } +} + +object PebbleJsonLoader { + private fun resolveImports(json: JsonElement, currentFile: File): JsonElement = when (json) { + JsonNull -> JsonNull + is JsonPrimitive -> if (json.isString) { + JsonImport(json.content, currentFile)?.let { jsonImport -> + val (nextFile, jsonData) = jsonImport.resolve() + resolveImports(jsonData, nextFile) + } ?: json + } else json + + is JsonObject -> JsonObject(json.mapValues { (_, it) -> resolveImports(it, currentFile) }) + is JsonArray -> JsonArray(json.map { resolveImports(it, currentFile) }) + } + + fun convertJson(json: JsonElement): Any? = when (json) { + JsonNull -> null + is JsonPrimitive -> if (json.isString) + json.content + else + json.intOrNull ?: json.double + + is JsonObject -> json.mapValues { (_, it) -> convertJson(it) } + is JsonArray -> json.map { convertJson(it) } + } + + fun deconvertJson(data: Any?): JsonElement = when (data) { + null -> JsonNull + is String -> JsonPrimitive(data) + is Number -> JsonPrimitive(data) + is Boolean -> JsonPrimitive(data) + is List<*> -> JsonArray(data.map { deconvertJson(it) }) + is Set<*> -> JsonArray(data.map { deconvertJson(it) }) + is Map<*, *> -> JsonObject(data.map { (k, v) -> k.toString() to deconvertJson(v) }.toMap()) + else -> throw ClassCastException("Expected null, String, Number, Boolean, List, Set, or Map for converted data, got $data or type ${data::class.jvmName}") + } + + fun loadJson(name: String): JsonElement = + File(Configuration.CurrentConfiguration.jsonDocDir) + .combineSafe("$name.json") + .takeIf { it.isFile } + .let { file -> + file ?: throw IOException("Unable to find $name as a file") + + val text = file.readText() + resolveImports(JsonFileCodec.parseToJsonElement(text), file) + } + + fun loadJsonContext(name: String): Map = loadJson(name).let { json -> + val data = convertJson(json) + if (data !is Map<*, *>) + throw IOException("JSON Object expected in file $name, got $json") + + data.mapKeys { (k, _) -> k.toString() } + } + + fun loadJsonContextOrEmpty(name: String): Map = try { + loadJsonContext(name) + } catch (ex: IOException) { + emptyMap() + } +} + +object PebbleToJsonFilter : Filter { + override fun getArgumentNames(): MutableList { + return mutableListOf() + } + + override fun apply(input: Any?, args: MutableMap?, self: PebbleTemplate?, context: EvaluationContext?, lineNumber: Int): Any { + return PebbleJsonLoader.deconvertJson(input).toString() + } +} + +object PebbleFromJsonFilter : Filter { + override fun getArgumentNames(): MutableList { + return mutableListOf() + } + + override fun apply(input: Any?, args: MutableMap?, self: PebbleTemplate?, context: EvaluationContext?, lineNumber: Int): Any? { + return PebbleJsonLoader.convertJson(JsonStorageCodec.parseToJsonElement(input.toString())) + } +} + +object PebbleLoadJsonFunction : Function { + override fun getArgumentNames(): MutableList { + return mutableListOf("data", "path") + } + + override fun execute(args: MutableMap, self: PebbleTemplate, context: EvaluationContext, lineNumber: Int): Any? { + val dataName = args["data"]?.toString() + ?: throw PebbleException(null, "Missing 'data' argument", lineNumber, self.name) + + val data = try { + PebbleJsonLoader.loadJson(dataName) + } catch (ex: IOException) { + throw PebbleException(ex, ex.message, lineNumber, self.name) + } + + val dataPath = args["path"]?.toString()?.let { JsonPath(it) } ?: JsonPath.Empty + return try { + PebbleJsonLoader.convertJson(data[dataPath]) + } catch (ex: NoSuchElementException) { + throw PebbleException(ex, "Unable to resolve path $dataPath in JSON $data", lineNumber, self.name) + } + } +} + +object PebbleScriptLoader { + private val scriptEngine = ThreadLocal.withInitial { ScriptEngineManager().getEngineByExtension("groovy") } + private val hasher = ThreadLocal.withInitial { MessageDigest.getInstance("SHA-256") } + private val cache = ConcurrentHashMap() + + fun loadFunction(name: String): CompiledScript? { + val scriptFile = File(Configuration.CurrentConfiguration.scriptDir).combineSafe("$name.groovy") + if (!scriptFile.canRead()) + return null + + val script = scriptFile.readText() + val digest = hasher.get().digest(script.toByteArray()).joinToString(separator = "") { it.toUByte().toString(16) } + cache[digest]?.let { return it } + + val compiledScript = (scriptEngine.get() as Compilable).compile(script) + cache[digest] = compiledScript + return compiledScript + } +} + +object PebbleScriptFilter : Filter { + override fun getArgumentNames(): MutableList { + return mutableListOf("script") + } + + override fun apply(input: Any?, args: MutableMap, self: PebbleTemplate, context: EvaluationContext, lineNumber: Int): Any { + val scriptName = args["script"]?.toString() + ?: throw PebbleException(null, "Missing 'script' argument", lineNumber, self.name) + + val script = PebbleScriptLoader.loadFunction(scriptName) + ?: throw PebbleException(null, "Script $scriptName could not be found", lineNumber, self.name) + + val bindings = SimpleBindings() + bindings["text"] = input + bindings["ctx"] = java.util.function.Function(context::getVariable) + bindings["args"] = args.toMutableMap().apply { remove("script") } + + return try { + script.eval(bindings) + } catch (ex: ScriptException) { + throw PebbleException(ex, "Unhandled ScriptException from $scriptName", lineNumber, self.name) + } + } +} + +object PebbleScriptFunction : Function { + override fun getArgumentNames(): MutableList { + return mutableListOf("script") + } + + override fun execute(args: MutableMap, self: PebbleTemplate, context: EvaluationContext, lineNumber: Int): Any { + val scriptName = args["script"]?.toString() + ?: throw PebbleException(null, "Missing 'script' argument", lineNumber, self.name) + + val script = PebbleScriptLoader.loadFunction(scriptName) + ?: throw PebbleException(null, "Script $scriptName could not be found", lineNumber, self.name) + + val bindings = SimpleBindings() + bindings["ctx"] = java.util.function.Function(context::getVariable) + bindings["args"] = args.toMutableMap().apply { remove("script") } + + return try { + script.eval(bindings) + } catch (ex: ScriptException) { + throw PebbleException(ex, "Unhandled ScriptException from $scriptName", lineNumber, self.name) + } + } +} diff --git a/src/jvmMain/kotlin/info/mechyrdia/lore/preparser_functions.kt b/src/jvmMain/kotlin/info/mechyrdia/lore/preparser_functions.kt deleted file mode 100644 index 53c34d0..0000000 --- a/src/jvmMain/kotlin/info/mechyrdia/lore/preparser_functions.kt +++ /dev/null @@ -1,47 +0,0 @@ -package info.mechyrdia.lore - -import com.samskivert.mustache.Mustache -import com.samskivert.mustache.Template -import info.mechyrdia.Configuration -import io.ktor.util.* -import java.io.File -import java.io.Writer -import java.security.MessageDigest -import java.util.concurrent.ConcurrentHashMap -import javax.script.Compilable -import javax.script.CompiledScript -import javax.script.ScriptEngineManager -import javax.script.SimpleBindings - -@JvmInline -value class ScriptedMustacheLambda(private val script: CompiledScript) : Mustache.Lambda { - override fun execute(frag: Template.Fragment, out: Writer) { - val bindings = SimpleBindings() - bindings["frag"] = frag - bindings["out"] = out - script.eval(bindings) - } -} - -object PreParserFunctions { - private val scriptEngine = ThreadLocal.withInitial { ScriptEngineManager().getEngineByExtension("groovy") } - private val hasher = ThreadLocal.withInitial { MessageDigest.getInstance("SHA-256") } - private val cache = ConcurrentHashMap() - - private fun loadFunction(name: String): Mustache.Lambda { - val scriptFile = File(Configuration.CurrentConfiguration.scriptDir).combineSafe("$name.groovy") - val script = scriptFile.readText() - val digest = hasher.get().digest(script.toByteArray()).joinToString(separator = "") { it.toUByte().toString(16) } - cache[digest]?.let { return ScriptedMustacheLambda(it) } - - val compiledScript = (scriptEngine.get() as Compilable).compile(script) - cache[digest] = compiledScript - return ScriptedMustacheLambda(compiledScript) - } - - fun getFunctions(): Map { - val scriptDir = File(Configuration.CurrentConfiguration.scriptDir) - val scripts = scriptDir.listFiles() ?: return emptyMap() - return scripts.associate { it.nameWithoutExtension to loadFunction(it.nameWithoutExtension) } - } -} -- 2.25.1