From 6ecf2e00bd56975e2b9aa9396acf61c3f51f11ee Mon Sep 17 00:00:00 2001 From: Lanius Trolling Date: Tue, 9 Apr 2024 06:28:22 -0400 Subject: [PATCH] Test and fix tag-based templating --- .../kotlin/info/mechyrdia/data/bson.kt | 22 +++ .../kotlin/info/mechyrdia/data/comments.kt | 7 +- .../kotlin/info/mechyrdia/data/data.kt | 9 +- .../kotlin/info/mechyrdia/data/data_files.kt | 7 +- .../kotlin/info/mechyrdia/data/visits.kt | 5 +- .../info/mechyrdia/lore/article_listing.kt | 2 +- .../kotlin/info/mechyrdia/lore/parser_html.kt | 4 +- .../info/mechyrdia/lore/parser_preprocess.kt | 132 ++++++++++++------ .../lore/parser_preprocess_include.kt | 6 +- .../mechyrdia/lore/parser_preprocess_json.kt | 3 +- 10 files changed, 131 insertions(+), 66 deletions(-) diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/bson.kt b/src/jvmMain/kotlin/info/mechyrdia/data/bson.kt index a39c7dc..584eb3c 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/data/bson.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/data/bson.kt @@ -5,6 +5,7 @@ package info.mechyrdia.data import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.SerializationException +import kotlinx.serialization.builtins.nullable import kotlinx.serialization.descriptors.PrimitiveKind import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor import kotlinx.serialization.descriptors.SerialDescriptor @@ -20,6 +21,7 @@ import org.bson.codecs.configuration.CodecProvider import org.bson.codecs.configuration.CodecRegistry import org.bson.codecs.kotlinx.BsonDecoder import org.bson.codecs.kotlinx.BsonEncoder +import org.bson.types.ObjectId import java.time.Instant object IdCodec : Codec> { @@ -45,6 +47,24 @@ object IdCodecProvider : CodecProvider { } } +object ObjectIdSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("ObjectIdSerializer", PrimitiveKind.LONG) + + override fun serialize(encoder: Encoder, value: ObjectId) { + if (encoder !is BsonEncoder) + throw SerializationException("ObjectId is not supported by ${encoder::class}") + + encoder.encodeObjectId(value) + } + + override fun deserialize(decoder: Decoder): ObjectId { + if (decoder !is BsonDecoder) + throw SerializationException("ObjectId is not supported by ${decoder::class}") + + return decoder.decodeObjectId() + } +} + object InstantSerializer : KSerializer { override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("InstantSerializer", PrimitiveKind.LONG) @@ -62,3 +82,5 @@ object InstantSerializer : KSerializer { return Instant.ofEpochMilli(decoder.decodeBsonValue().asDateTime().value) } } + +object InstantNullableSerializer : KSerializer by InstantSerializer.nullable diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/comments.kt b/src/jvmMain/kotlin/info/mechyrdia/data/comments.kt index c7311e4..5497a51 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/data/comments.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/data/comments.kt @@ -4,7 +4,6 @@ import com.mongodb.client.model.Filters import com.mongodb.client.model.Sorts import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.toList -import kotlinx.serialization.Contextual import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import java.time.Instant @@ -16,10 +15,10 @@ data class Comment( val submittedBy: Id, val submittedIn: String, - val submittedAt: @Contextual Instant, + val submittedAt: @Serializable(with = InstantSerializer::class) Instant, val numEdits: Int, - val lastEdit: @Contextual Instant?, + val lastEdit: @Serializable(with = InstantNullableSerializer::class) Instant?, val contents: String ) : DataDocument { @@ -49,7 +48,7 @@ data class CommentReplyLink( val originalPost: Id, val replyingPost: Id, - val repliedAt: @Contextual Instant = Instant.now(), + val repliedAt: @Serializable(with = InstantSerializer::class) Instant = Instant.now(), ) : DataDocument { companion object : TableHolder { override val Table = DocumentTable() diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/data.kt b/src/jvmMain/kotlin/info/mechyrdia/data/data.kt index f085709..0ff91bb 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/data/data.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/data/data.kt @@ -23,12 +23,10 @@ import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.encoding.Decoder import kotlinx.serialization.encoding.Encoder -import kotlinx.serialization.modules.SerializersModule import org.bson.codecs.configuration.CodecRegistries import org.bson.codecs.kotlinx.KotlinSerializerCodecProvider import org.bson.conversions.Bson import java.security.SecureRandom -import java.time.Instant import kotlin.reflect.KClass import kotlin.reflect.KProperty1 import com.mongodb.reactivestreams.client.MongoDatabase as JMongoDatabase @@ -66,10 +64,6 @@ object ConnectionHolder { suspend fun getBucket(): GridFSBucket = GridFSBuckets.create(jDatabaseDeferred.await()) - private val bsonSerializersModule = SerializersModule { - contextual(Instant::class, InstantSerializer) - } - fun initialize(conn: String, db: String) { if (jDatabaseDeferred.isCompleted) error("Cannot initialize database twice") @@ -79,8 +73,9 @@ object ConnectionHolder { MongoClientSettings.builder() .codecRegistry( CodecRegistries.fromProviders( + MongoClientSettings.getDefaultCodecRegistry(), IdCodecProvider, - KotlinSerializerCodecProvider(bsonSerializersModule) + KotlinSerializerCodecProvider() ) ) .applyConnectionString(ConnectionString(conn)) diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/data_files.kt b/src/jvmMain/kotlin/info/mechyrdia/data/data_files.kt index 9f29326..bcffa5e 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/data/data_files.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/data/data_files.kt @@ -17,7 +17,6 @@ import kotlinx.coroutines.flow.* import kotlinx.coroutines.reactive.asFlow import kotlinx.coroutines.reactive.asPublisher import kotlinx.coroutines.reactive.awaitFirst -import kotlinx.serialization.Contextual import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import org.bson.types.ObjectId @@ -272,9 +271,9 @@ private data class GridFsEntry( @SerialName(MONGODB_ID_KEY) override val id: Id, val path: String, - val file: @Contextual ObjectId, - val created: @Contextual Instant, - val updated: @Contextual Instant, + val file: @Serializable(with = ObjectIdSerializer::class) ObjectId, + val created: @Serializable(with = InstantSerializer::class) Instant, + val updated: @Serializable(with = InstantSerializer::class) Instant, ) : DataDocument private class GridFsStorage(val table: DocumentTable, val bucket: GridFSBucket) : FileStorage { diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/visits.kt b/src/jvmMain/kotlin/info/mechyrdia/data/visits.kt index a02f6f7..2f1fa10 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/data/visits.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/data/visits.kt @@ -14,7 +14,6 @@ import kotlinx.coroutines.flow.firstOrNull import kotlinx.html.FlowContent import kotlinx.html.p import kotlinx.html.style -import kotlinx.serialization.Contextual import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import org.intellij.lang.annotations.Language @@ -25,7 +24,7 @@ import java.time.Instant data class PageVisitTotals( val total: Int, val totalUnique: Int, - val mostRecent: @Contextual Instant? + val mostRecent: @Serializable(with = InstantNullableSerializer::class) Instant? ) @Serializable @@ -36,7 +35,7 @@ data class PageVisitData( val path: String, val visitor: String, val visits: Int = 0, - val lastVisit: @Contextual Instant = Instant.now() + val lastVisit: @Serializable(with = InstantSerializer::class) Instant = Instant.now() ) : DataDocument { companion object : TableHolder { override val Table = DocumentTable() diff --git a/src/jvmMain/kotlin/info/mechyrdia/lore/article_listing.kt b/src/jvmMain/kotlin/info/mechyrdia/lore/article_listing.kt index 9a73e18..5dc7ab2 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/lore/article_listing.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/lore/article_listing.kt @@ -28,7 +28,7 @@ suspend fun StoragePath.toArticleNode(): ArticleNode = ArticleNode( val subPath = path / it.name async { subPath.toArticleNode() } }.toList().awaitAll() - } + }.sortedBy { it.name }.sortedBy { it.subNodes.isEmpty() } ) private val String.isViewable: Boolean diff --git a/src/jvmMain/kotlin/info/mechyrdia/lore/parser_html.kt b/src/jvmMain/kotlin/info/mechyrdia/lore/parser_html.kt index 1a4a54e..6c6252a 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/lore/parser_html.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/lore/parser_html.kt @@ -415,8 +415,8 @@ enum class FactbookFormattingTag(val type: HtmlLexerTag) { H5(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h5.toTagCreator(), ParserTree::treeToAnchorText)), H6(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h6.toTagCreator(), ParserTree::treeToAnchorText)), - ALIGN(HtmlTagLexerTag(attributes = ::processAlign, tagCreator = TagConsumer<*>::div.toTagCreator())), - ASIDE(HtmlTagLexerTag(attributes = ::processFloat, tagCreator = TagConsumer<*>::div.toTagCreator())), + ALIGN(HtmlTagLexerTag(attributes = ::processAlign, tagMode = HtmlTagMode.ITEM, tagCreator = TagConsumer<*>::div.toTagCreator())), + ASIDE(HtmlTagLexerTag(attributes = ::processFloat, tagMode = HtmlTagMode.ITEM, tagCreator = TagConsumer<*>::div.toTagCreator())), DESC(HtmlMetadataLexerTag(absorb = false)), THUMB(HtmlMetadataLexerTag(absorb = true)), diff --git a/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess.kt b/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess.kt index b11ec61..1b35be9 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess.kt @@ -181,6 +181,21 @@ object PreProcessorVariableInvoker : PreProcessorFunctionProvider { } } +@JvmInline +value class PreProcessorScopeFilter(private val variable: String) : PreProcessorFilter { + override suspend fun execute(input: ParserTree, env: AsyncLexerTagEnvironment): ParserTree { + return env.copy(context = env.context + env.context[variable].asPreProcessorMap()).processTree(input) + } +} + +object PreProcessorScopeInvoker : PreProcessorFilterProvider { + override val tagName: String = "scope" + + override suspend fun provideFilter(param: String?): PreProcessorFilter? { + return param?.let { PreProcessorScopeFilter(it) } + } +} + fun interface PreProcessorFilter { suspend fun execute(input: ParserTree, env: AsyncLexerTagEnvironment): ParserTree } @@ -242,13 +257,13 @@ suspend fun Map.mapValuesSuspend(processor: suspend (K, V) -> R) enum class PreProcessorTags(val type: PreProcessorLexerTag) { EVAL(PreProcessorLexerTag { env, param, subNodes -> - param?.toDoubleOrNull()?.roundToInt().requireParam("eval") { times -> - var tree = subNodes - repeat(times) { - tree = env.processTree(tree) - } - tree + val times = param?.toDoubleOrNull()?.roundToInt() ?: 1 + + var tree = subNodes + repeat(times) { + tree = env.processTree(tree) } + tree }), LAZY(PreProcessorLexerTag { _, param, subNodes -> param.forbidParam("lazy") { subNodes } @@ -263,7 +278,26 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) { env.context[env.processTree(subNodes).treeToText()] } }), + DEFAULT(PreProcessorLexerTag { env, param, subNodes -> + param.requireParam("var") { varName -> + if (varName in env.context) + env.context[varName] + else env.processTree(subNodes) + } + }), + SET_PARAM(PreProcessorLexerTag { env, param, subNodes -> + param.requireParam("var") { varName -> + val paramValue = env.context[varName].treeToText() + val withParams = subNodes.map { node -> + if (node is ParserTreeNode.Tag && node.param == null) + node.copy(param = paramValue) + else node + } + env.processTree(withParams) + } + }), ENV(PreProcessorVariableInvoker), + SCOPE(PreProcessorScopeInvoker), SET(PreProcessorLexerTag { env, param, subNodes -> param.requireParam("set") { varName -> env.context[varName] = env.processTree(subNodes) @@ -288,9 +322,7 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) { (param?.toDoubleOrNull() ?: param?.let { env.processTree(env.context[param]).treeToNumberOrNull(String::toDoubleOrNull) })?.roundToInt().requireParam("index") { index -> - inputList.getOrNull(index) - ?.let { env.processTree(it) } - .formatError("Index $index is not present in input list") + inputList.getOrNull(index).formatError("Index $index is not present in input list") } }), MEMBER(PreProcessorLexerTag { env, param, subNodes -> @@ -298,35 +330,39 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) { PreProcessorUtils.indexTree(env.processTree(subNodes), index) } }), - FOREACH(PreProcessorLexerTag { env, param, subNodes -> - param.requireParam("foreach") { itemVar -> - val subTags = subNodes.filterIsInstance() - val list = subTags.singleOrNull { it isTag "in" }?.subNodes - ?.let { env.processTree(it) } - ?.asPreProcessorList() - - val body = subTags.singleOrNull { it isTag "do" }?.subNodes - if (list != null && body != null) - list.mapSuspend { item -> - PreProcessorUtils.processWithContext(env, env.context + mapOf(itemVar to item), body) - }.flatten() - else formatErrorToParserTree("Expected child tag [in] to take list input and child tag [do] to take loop body") - } + FOR_EACH(PreProcessorLexerTag { env, param, subNodes -> + val itemToContext: (ParserTree) -> Map = if (param == null) + ParserTree::asPreProcessorMap + else ({ mapOf(param to it) }) + + val subTags = subNodes.filterIsInstance() + val list = subTags.singleOrNull { it isTag "in" }?.subNodes + ?.let { env.processTree(it) } + ?.asPreProcessorList() + + val body = subTags.singleOrNull { it isTag "do" }?.subNodes + if (list != null && body != null) + list.mapSuspend { item -> + PreProcessorUtils.processWithContext(env, env.context + itemToContext(item), body) + }.flatten() + else formatErrorToParserTree("Expected child tag [in] to take list input and child tag [do] to take loop body") }), MAP(PreProcessorLexerTag { env, param, subNodes -> - param.requireParam("map") { itemVar -> - val subTags = subNodes.filterIsInstance() - val list = subTags.singleOrNull { it isTag "in" }?.subNodes - ?.let { env.processTree(it) } - ?.asPreProcessorList() - - val body = subTags.singleOrNull { it isTag "do" }?.subNodes - if (list != null && body != null) - list.mapSuspend { item -> - ParserTreeNode.Tag("item", null, PreProcessorUtils.processWithContext(env, env.context + mapOf(itemVar to item), body)) - } - else formatErrorToParserTree("Expected child tag [in] to take list input and child tag [do] to take loop body") - } + val itemToContext: (ParserTree) -> Map = if (param == null) + ParserTree::asPreProcessorMap + else ({ mapOf(param to it) }) + + val subTags = subNodes.filterIsInstance() + val list = subTags.singleOrNull { it isTag "in" }?.subNodes + ?.let { env.processTree(it) } + ?.asPreProcessorList() + + val body = subTags.singleOrNull { it isTag "do" }?.subNodes + if (list != null && body != null) + list.mapSuspend { item -> + ParserTreeNode.Tag("item", null, PreProcessorUtils.processWithContext(env, env.context + itemToContext(item), body)) + } + else formatErrorToParserTree("Expected child tag [in] to take list input and child tag [do] to take loop body") }), IF(PreProcessorLexerTag { env, param, subNodes -> param.requireParam("if") { boolVar -> @@ -342,6 +378,20 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) { }.formatError("Expected variable $boolVar to contain boolean value") } }), + LET(PreProcessorLexerTag { env, param, subNodes -> + param.requireParam("let") { varName -> + if (varName in env.context && !env.context[varName].isNull()) + env.processTree(subNodes) + else emptyList() + } + }), + FALLBACK(PreProcessorLexerTag { env, param, subNodes -> + param.requireParam("fallback") { varName -> + if (varName !in env.context || env.context[varName].isNull()) + env.processTree(subNodes) + else emptyList() + } + }), MATH(PreProcessorMathOperators), LOGIC(PreProcessorLogicOperator), TEST(PreProcessorInputTest), @@ -357,14 +407,14 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) { }), FUNCTION(PreProcessorLexerTag { env, param, subNodes -> param.requireParam("function") { scriptName -> - PreProcessorScriptLoader.runScriptSafe(scriptName, subNodes.asPreProcessorMap(), env) { + PreProcessorScriptLoader.runScriptSafe(scriptName, env.processTree(subNodes).asPreProcessorMap(), env) { it.renderInBBCode() } } }), FILTER(PreProcessorLexerTag { env, param, subNodes -> param.requireParam("filter") { scriptName -> - PreProcessorScriptLoader.runScriptSafe(scriptName, subNodes, env) { + PreProcessorScriptLoader.runScriptSafe(scriptName, env.processTree(subNodes), env) { it.renderInBBCode() } } @@ -379,9 +429,9 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) { } } }), - IMPORT(PreProcessorLexerTag { _, param, subNodes -> + IMPORT(PreProcessorLexerTag { env, param, subNodes -> param.requireParam("import") { templateName -> - PreProcessorTemplateLoader.runTemplateWith(templateName, subNodes.asPreProcessorMap()) + PreProcessorTemplateLoader.runTemplateWith(templateName, env.processTree(subNodes).asPreProcessorMap()) } }), INCLUDE(PreProcessorLexerTag { env, param, subNodes -> @@ -390,7 +440,7 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) { } }), TEMPLATE(PreProcessorLexerTag { env, param, subNodes -> - param.forbidParam("include") { + param.forbidParam("template") { PreProcessorTemplateLoader.loadTemplate(env.processTree(subNodes).treeToText()) } }), diff --git a/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_include.kt b/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_include.kt index 2c36691..e745e69 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_include.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_include.kt @@ -88,7 +88,7 @@ object PreProcessorScriptLoader { } } - private suspend fun runScriptSafe(scriptName: String, bind: Map, env: AsyncLexerTagEnvironment, errorHandler: (Exception) -> ParserTree): ParserTree { + private suspend fun runScriptWithBindings(scriptName: String, bind: Map, env: AsyncLexerTagEnvironment, errorHandler: (Exception) -> ParserTree): ParserTree { return try { val script = loadFunction(scriptName)!! val result = runScriptInternal(script, bind, env) @@ -103,11 +103,11 @@ object PreProcessorScriptLoader { suspend fun runScriptSafe(scriptName: String, args: Map, env: AsyncLexerTagEnvironment, errorHandler: (Exception) -> ParserTree): ParserTree { val groovyArgs = args.mapValuesTo(mutableMapOf()) { (_, it) -> jsonToGroovy(it.toPreProcessJson()) } - return runScriptSafe(scriptName, mapOf("args" to groovyArgs), env, errorHandler) + return runScriptWithBindings(scriptName, mapOf("args" to groovyArgs), env, errorHandler) } suspend fun runScriptSafe(scriptName: String, input: ParserTree, env: AsyncLexerTagEnvironment, errorHandler: (Exception) -> ParserTree): ParserTree { - return runScriptSafe(scriptName, mapOf("text" to input.unparse()), env, errorHandler) + return runScriptWithBindings(scriptName, mapOf("text" to input.unparse()), env, errorHandler) } } diff --git a/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_json.kt b/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_json.kt index 9fa0fff..ea0890a 100644 --- a/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_json.kt +++ b/src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_json.kt @@ -65,7 +65,8 @@ fun ParserTree.toPreProcessJson(): JsonElement { object FactbookLoader { suspend fun loadJsonData(lorePath: List): JsonObject { - val bytes = FileStorage.instance.readFile(StoragePath.jsonDocDir / lorePath) ?: return JsonObject(emptyMap()) + val jsonPath = lorePath.dropLast(1) + listOf("${lorePath.last()}.json") + val bytes = FileStorage.instance.readFile(StoragePath.jsonDocDir / jsonPath) ?: return JsonObject(emptyMap()) return JsonStorageCodec.parseToJsonElement(String(bytes)) as JsonObject } -- 2.25.1