Test and fix tag-based templating
authorLanius Trolling <lanius@laniustrolling.dev>
Tue, 9 Apr 2024 10:28:22 +0000 (06:28 -0400)
committerLanius Trolling <lanius@laniustrolling.dev>
Tue, 9 Apr 2024 10:28:22 +0000 (06:28 -0400)
src/jvmMain/kotlin/info/mechyrdia/data/bson.kt
src/jvmMain/kotlin/info/mechyrdia/data/comments.kt
src/jvmMain/kotlin/info/mechyrdia/data/data.kt
src/jvmMain/kotlin/info/mechyrdia/data/data_files.kt
src/jvmMain/kotlin/info/mechyrdia/data/visits.kt
src/jvmMain/kotlin/info/mechyrdia/lore/article_listing.kt
src/jvmMain/kotlin/info/mechyrdia/lore/parser_html.kt
src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess.kt
src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_include.kt
src/jvmMain/kotlin/info/mechyrdia/lore/parser_preprocess_json.kt

index a39c7dcd104fffa8a103cc951b85207a3ee3c6ad..584eb3ccb2403a1cf95c0737b2fa238f4e2d533d 100644 (file)
@@ -5,6 +5,7 @@ package info.mechyrdia.data
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.KSerializer
 import kotlinx.serialization.SerializationException
+import kotlinx.serialization.builtins.nullable
 import kotlinx.serialization.descriptors.PrimitiveKind
 import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
 import kotlinx.serialization.descriptors.SerialDescriptor
@@ -20,6 +21,7 @@ import org.bson.codecs.configuration.CodecProvider
 import org.bson.codecs.configuration.CodecRegistry
 import org.bson.codecs.kotlinx.BsonDecoder
 import org.bson.codecs.kotlinx.BsonEncoder
+import org.bson.types.ObjectId
 import java.time.Instant
 
 object IdCodec : Codec<Id<*>> {
@@ -45,6 +47,24 @@ object IdCodecProvider : CodecProvider {
        }
 }
 
+object ObjectIdSerializer : KSerializer<ObjectId> {
+       override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("ObjectIdSerializer", PrimitiveKind.LONG)
+       
+       override fun serialize(encoder: Encoder, value: ObjectId) {
+               if (encoder !is BsonEncoder)
+                       throw SerializationException("ObjectId is not supported by ${encoder::class}")
+               
+               encoder.encodeObjectId(value)
+       }
+       
+       override fun deserialize(decoder: Decoder): ObjectId {
+               if (decoder !is BsonDecoder)
+                       throw SerializationException("ObjectId is not supported by ${decoder::class}")
+               
+               return decoder.decodeObjectId()
+       }
+}
+
 object InstantSerializer : KSerializer<Instant> {
        override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("InstantSerializer", PrimitiveKind.LONG)
        
@@ -62,3 +82,5 @@ object InstantSerializer : KSerializer<Instant> {
                return Instant.ofEpochMilli(decoder.decodeBsonValue().asDateTime().value)
        }
 }
+
+object InstantNullableSerializer : KSerializer<Instant?> by InstantSerializer.nullable
index c7311e41133c7f5c002fd6768e91844fc983f4a6..5497a51500f04b49631aa8268227df991b6b4854 100644 (file)
@@ -4,7 +4,6 @@ import com.mongodb.client.model.Filters
 import com.mongodb.client.model.Sorts
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.toList
-import kotlinx.serialization.Contextual
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import java.time.Instant
@@ -16,10 +15,10 @@ data class Comment(
        
        val submittedBy: Id<NationData>,
        val submittedIn: String,
-       val submittedAt: @Contextual Instant,
+       val submittedAt: @Serializable(with = InstantSerializer::class) Instant,
        
        val numEdits: Int,
-       val lastEdit: @Contextual Instant?,
+       val lastEdit: @Serializable(with = InstantNullableSerializer::class) Instant?,
        
        val contents: String
 ) : DataDocument<Comment> {
@@ -49,7 +48,7 @@ data class CommentReplyLink(
        val originalPost: Id<Comment>,
        val replyingPost: Id<Comment>,
        
-       val repliedAt: @Contextual Instant = Instant.now(),
+       val repliedAt: @Serializable(with = InstantSerializer::class) Instant = Instant.now(),
 ) : DataDocument<CommentReplyLink> {
        companion object : TableHolder<CommentReplyLink> {
                override val Table = DocumentTable<CommentReplyLink>()
index f085709b36413b4be8cb2fe04f11fb5011aa7f07..0ff91bb4c0c466d889317baba0abc18636999b8e 100644 (file)
@@ -23,12 +23,10 @@ import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
 import kotlinx.serialization.descriptors.SerialDescriptor
 import kotlinx.serialization.encoding.Decoder
 import kotlinx.serialization.encoding.Encoder
-import kotlinx.serialization.modules.SerializersModule
 import org.bson.codecs.configuration.CodecRegistries
 import org.bson.codecs.kotlinx.KotlinSerializerCodecProvider
 import org.bson.conversions.Bson
 import java.security.SecureRandom
-import java.time.Instant
 import kotlin.reflect.KClass
 import kotlin.reflect.KProperty1
 import com.mongodb.reactivestreams.client.MongoDatabase as JMongoDatabase
@@ -66,10 +64,6 @@ object ConnectionHolder {
        
        suspend fun getBucket(): GridFSBucket = GridFSBuckets.create(jDatabaseDeferred.await())
        
-       private val bsonSerializersModule = SerializersModule {
-               contextual(Instant::class, InstantSerializer)
-       }
-       
        fun initialize(conn: String, db: String) {
                if (jDatabaseDeferred.isCompleted)
                        error("Cannot initialize database twice")
@@ -79,8 +73,9 @@ object ConnectionHolder {
                                MongoClientSettings.builder()
                                        .codecRegistry(
                                                CodecRegistries.fromProviders(
+                                                       MongoClientSettings.getDefaultCodecRegistry(),
                                                        IdCodecProvider,
-                                                       KotlinSerializerCodecProvider(bsonSerializersModule)
+                                                       KotlinSerializerCodecProvider()
                                                )
                                        )
                                        .applyConnectionString(ConnectionString(conn))
index 9f29326560a9a5aa9f0eeef68e78685e5f244dc2..bcffa5e13bfa4bde0bd9456fc4d7fe6ecde6290a 100644 (file)
@@ -17,7 +17,6 @@ import kotlinx.coroutines.flow.*
 import kotlinx.coroutines.reactive.asFlow
 import kotlinx.coroutines.reactive.asPublisher
 import kotlinx.coroutines.reactive.awaitFirst
-import kotlinx.serialization.Contextual
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import org.bson.types.ObjectId
@@ -272,9 +271,9 @@ private data class GridFsEntry(
        @SerialName(MONGODB_ID_KEY)
        override val id: Id<GridFsEntry>,
        val path: String,
-       val file: @Contextual ObjectId,
-       val created: @Contextual Instant,
-       val updated: @Contextual Instant,
+       val file: @Serializable(with = ObjectIdSerializer::class) ObjectId,
+       val created: @Serializable(with = InstantSerializer::class) Instant,
+       val updated: @Serializable(with = InstantSerializer::class) Instant,
 ) : DataDocument<GridFsEntry>
 
 private class GridFsStorage(val table: DocumentTable<GridFsEntry>, val bucket: GridFSBucket) : FileStorage {
index a02f6f7f4f6276ddbf3e6c5fed3c801c243eaa4d..2f1fa1024e9514dc5748fd086aa48b1b5a97e321 100644 (file)
@@ -14,7 +14,6 @@ import kotlinx.coroutines.flow.firstOrNull
 import kotlinx.html.FlowContent
 import kotlinx.html.p
 import kotlinx.html.style
-import kotlinx.serialization.Contextual
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 import org.intellij.lang.annotations.Language
@@ -25,7 +24,7 @@ import java.time.Instant
 data class PageVisitTotals(
        val total: Int,
        val totalUnique: Int,
-       val mostRecent: @Contextual Instant?
+       val mostRecent: @Serializable(with = InstantNullableSerializer::class) Instant?
 )
 
 @Serializable
@@ -36,7 +35,7 @@ data class PageVisitData(
        val path: String,
        val visitor: String,
        val visits: Int = 0,
-       val lastVisit: @Contextual Instant = Instant.now()
+       val lastVisit: @Serializable(with = InstantSerializer::class) Instant = Instant.now()
 ) : DataDocument<PageVisitData> {
        companion object : TableHolder<PageVisitData> {
                override val Table = DocumentTable<PageVisitData>()
index 9a73e18495a6b25fb9bbec0fe9551dc743c24845..5dc7ab21b2f7ed5f75e6862991d8963f4eda5db9 100644 (file)
@@ -28,7 +28,7 @@ suspend fun StoragePath.toArticleNode(): ArticleNode = ArticleNode(
                        val subPath = path / it.name
                        async { subPath.toArticleNode() }
                }.toList().awaitAll()
-       }
+       }.sortedBy { it.name }.sortedBy { it.subNodes.isEmpty() }
 )
 
 private val String.isViewable: Boolean
index 1a4a54e64d97f99ca0db5c9b0b44ed65d333b3f5..6c6252af267683b9351df2026a913c4d356e498e 100644 (file)
@@ -415,8 +415,8 @@ enum class FactbookFormattingTag(val type: HtmlLexerTag) {
        H5(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h5.toTagCreator(), ParserTree::treeToAnchorText)),
        H6(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h6.toTagCreator(), ParserTree::treeToAnchorText)),
        
-       ALIGN(HtmlTagLexerTag(attributes = ::processAlign, tagCreator = TagConsumer<*>::div.toTagCreator())),
-       ASIDE(HtmlTagLexerTag(attributes = ::processFloat, tagCreator = TagConsumer<*>::div.toTagCreator())),
+       ALIGN(HtmlTagLexerTag(attributes = ::processAlign, tagMode = HtmlTagMode.ITEM, tagCreator = TagConsumer<*>::div.toTagCreator())),
+       ASIDE(HtmlTagLexerTag(attributes = ::processFloat, tagMode = HtmlTagMode.ITEM, tagCreator = TagConsumer<*>::div.toTagCreator())),
        
        DESC(HtmlMetadataLexerTag(absorb = false)),
        THUMB(HtmlMetadataLexerTag(absorb = true)),
index b11ec6120cfc19b2e259d9c9ed69fce9da466d9a..1b35be9213cd4f92069f795aa83e0dbde80a5900 100644 (file)
@@ -181,6 +181,21 @@ object PreProcessorVariableInvoker : PreProcessorFunctionProvider {
        }
 }
 
+@JvmInline
+value class PreProcessorScopeFilter(private val variable: String) : PreProcessorFilter {
+       override suspend fun execute(input: ParserTree, env: AsyncLexerTagEnvironment<PreProcessingContext, PreProcessingSubject>): ParserTree {
+               return env.copy(context = env.context + env.context[variable].asPreProcessorMap()).processTree(input)
+       }
+}
+
+object PreProcessorScopeInvoker : PreProcessorFilterProvider {
+       override val tagName: String = "scope"
+       
+       override suspend fun provideFilter(param: String?): PreProcessorFilter? {
+               return param?.let { PreProcessorScopeFilter(it) }
+       }
+}
+
 fun interface PreProcessorFilter {
        suspend fun execute(input: ParserTree, env: AsyncLexerTagEnvironment<PreProcessingContext, PreProcessingSubject>): ParserTree
 }
@@ -242,13 +257,13 @@ suspend fun <K, V, R> Map<K, V>.mapValuesSuspend(processor: suspend (K, V) -> R)
 
 enum class PreProcessorTags(val type: PreProcessorLexerTag) {
        EVAL(PreProcessorLexerTag { env, param, subNodes ->
-               param?.toDoubleOrNull()?.roundToInt().requireParam("eval") { times ->
-                       var tree = subNodes
-                       repeat(times) {
-                               tree = env.processTree(tree)
-                       }
-                       tree
+               val times = param?.toDoubleOrNull()?.roundToInt() ?: 1
+               
+               var tree = subNodes
+               repeat(times) {
+                       tree = env.processTree(tree)
                }
+               tree
        }),
        LAZY(PreProcessorLexerTag { _, param, subNodes ->
                param.forbidParam("lazy") { subNodes }
@@ -263,7 +278,26 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) {
                        env.context[env.processTree(subNodes).treeToText()]
                }
        }),
+       DEFAULT(PreProcessorLexerTag { env, param, subNodes ->
+               param.requireParam("var") { varName ->
+                       if (varName in env.context)
+                               env.context[varName]
+                       else env.processTree(subNodes)
+               }
+       }),
+       SET_PARAM(PreProcessorLexerTag { env, param, subNodes ->
+               param.requireParam("var") { varName ->
+                       val paramValue = env.context[varName].treeToText()
+                       val withParams = subNodes.map { node ->
+                               if (node is ParserTreeNode.Tag && node.param == null)
+                                       node.copy(param = paramValue)
+                               else node
+                       }
+                       env.processTree(withParams)
+               }
+       }),
        ENV(PreProcessorVariableInvoker),
+       SCOPE(PreProcessorScopeInvoker),
        SET(PreProcessorLexerTag { env, param, subNodes ->
                param.requireParam("set") { varName ->
                        env.context[varName] = env.processTree(subNodes)
@@ -288,9 +322,7 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) {
                (param?.toDoubleOrNull() ?: param?.let {
                        env.processTree(env.context[param]).treeToNumberOrNull(String::toDoubleOrNull)
                })?.roundToInt().requireParam("index") { index ->
-                       inputList.getOrNull(index)
-                               ?.let { env.processTree(it) }
-                               .formatError("Index $index is not present in input list")
+                       inputList.getOrNull(index).formatError("Index $index is not present in input list")
                }
        }),
        MEMBER(PreProcessorLexerTag { env, param, subNodes ->
@@ -298,35 +330,39 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) {
                        PreProcessorUtils.indexTree(env.processTree(subNodes), index)
                }
        }),
-       FOREACH(PreProcessorLexerTag { env, param, subNodes ->
-               param.requireParam("foreach") { itemVar ->
-                       val subTags = subNodes.filterIsInstance<ParserTreeNode.Tag>()
-                       val list = subTags.singleOrNull { it isTag "in" }?.subNodes
-                               ?.let { env.processTree(it) }
-                               ?.asPreProcessorList()
-                       
-                       val body = subTags.singleOrNull { it isTag "do" }?.subNodes
-                       if (list != null && body != null)
-                               list.mapSuspend { item ->
-                                       PreProcessorUtils.processWithContext(env, env.context + mapOf(itemVar to item), body)
-                               }.flatten()
-                       else formatErrorToParserTree("Expected child tag [in] to take list input and child tag [do] to take loop body")
-               }
+       FOR_EACH(PreProcessorLexerTag { env, param, subNodes ->
+               val itemToContext: (ParserTree) -> Map<String, ParserTree> = if (param == null)
+                       ParserTree::asPreProcessorMap
+               else ({ mapOf(param to it) })
+               
+               val subTags = subNodes.filterIsInstance<ParserTreeNode.Tag>()
+               val list = subTags.singleOrNull { it isTag "in" }?.subNodes
+                       ?.let { env.processTree(it) }
+                       ?.asPreProcessorList()
+               
+               val body = subTags.singleOrNull { it isTag "do" }?.subNodes
+               if (list != null && body != null)
+                       list.mapSuspend { item ->
+                               PreProcessorUtils.processWithContext(env, env.context + itemToContext(item), body)
+                       }.flatten()
+               else formatErrorToParserTree("Expected child tag [in] to take list input and child tag [do] to take loop body")
        }),
        MAP(PreProcessorLexerTag { env, param, subNodes ->
-               param.requireParam("map") { itemVar ->
-                       val subTags = subNodes.filterIsInstance<ParserTreeNode.Tag>()
-                       val list = subTags.singleOrNull { it isTag "in" }?.subNodes
-                               ?.let { env.processTree(it) }
-                               ?.asPreProcessorList()
-                       
-                       val body = subTags.singleOrNull { it isTag "do" }?.subNodes
-                       if (list != null && body != null)
-                               list.mapSuspend { item ->
-                                       ParserTreeNode.Tag("item", null, PreProcessorUtils.processWithContext(env, env.context + mapOf(itemVar to item), body))
-                               }
-                       else formatErrorToParserTree("Expected child tag [in] to take list input and child tag [do] to take loop body")
-               }
+               val itemToContext: (ParserTree) -> Map<String, ParserTree> = if (param == null)
+                       ParserTree::asPreProcessorMap
+               else ({ mapOf(param to it) })
+               
+               val subTags = subNodes.filterIsInstance<ParserTreeNode.Tag>()
+               val list = subTags.singleOrNull { it isTag "in" }?.subNodes
+                       ?.let { env.processTree(it) }
+                       ?.asPreProcessorList()
+               
+               val body = subTags.singleOrNull { it isTag "do" }?.subNodes
+               if (list != null && body != null)
+                       list.mapSuspend { item ->
+                               ParserTreeNode.Tag("item", null, PreProcessorUtils.processWithContext(env, env.context + itemToContext(item), body))
+                       }
+               else formatErrorToParserTree("Expected child tag [in] to take list input and child tag [do] to take loop body")
        }),
        IF(PreProcessorLexerTag { env, param, subNodes ->
                param.requireParam("if") { boolVar ->
@@ -342,6 +378,20 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) {
                        }.formatError("Expected variable $boolVar to contain boolean value")
                }
        }),
+       LET(PreProcessorLexerTag { env, param, subNodes ->
+               param.requireParam("let") { varName ->
+                       if (varName in env.context && !env.context[varName].isNull())
+                               env.processTree(subNodes)
+                       else emptyList()
+               }
+       }),
+       FALLBACK(PreProcessorLexerTag { env, param, subNodes ->
+               param.requireParam("fallback") { varName ->
+                       if (varName !in env.context || env.context[varName].isNull())
+                               env.processTree(subNodes)
+                       else emptyList()
+               }
+       }),
        MATH(PreProcessorMathOperators),
        LOGIC(PreProcessorLogicOperator),
        TEST(PreProcessorInputTest),
@@ -357,14 +407,14 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) {
        }),
        FUNCTION(PreProcessorLexerTag { env, param, subNodes ->
                param.requireParam("function") { scriptName ->
-                       PreProcessorScriptLoader.runScriptSafe(scriptName, subNodes.asPreProcessorMap(), env) {
+                       PreProcessorScriptLoader.runScriptSafe(scriptName, env.processTree(subNodes).asPreProcessorMap(), env) {
                                it.renderInBBCode()
                        }
                }
        }),
        FILTER(PreProcessorLexerTag { env, param, subNodes ->
                param.requireParam("filter") { scriptName ->
-                       PreProcessorScriptLoader.runScriptSafe(scriptName, subNodes, env) {
+                       PreProcessorScriptLoader.runScriptSafe(scriptName, env.processTree(subNodes), env) {
                                it.renderInBBCode()
                        }
                }
@@ -379,9 +429,9 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) {
                        }
                }
        }),
-       IMPORT(PreProcessorLexerTag { _, param, subNodes ->
+       IMPORT(PreProcessorLexerTag { env, param, subNodes ->
                param.requireParam("import") { templateName ->
-                       PreProcessorTemplateLoader.runTemplateWith(templateName, subNodes.asPreProcessorMap())
+                       PreProcessorTemplateLoader.runTemplateWith(templateName, env.processTree(subNodes).asPreProcessorMap())
                }
        }),
        INCLUDE(PreProcessorLexerTag { env, param, subNodes ->
@@ -390,7 +440,7 @@ enum class PreProcessorTags(val type: PreProcessorLexerTag) {
                }
        }),
        TEMPLATE(PreProcessorLexerTag { env, param, subNodes ->
-               param.forbidParam("include") {
+               param.forbidParam("template") {
                        PreProcessorTemplateLoader.loadTemplate(env.processTree(subNodes).treeToText())
                }
        }),
index 2c366919ccabd490384e6263c7b064b80a869ff9..e745e69f06d28d36298aa1a0bdaa5c5b75c12b04 100644 (file)
@@ -88,7 +88,7 @@ object PreProcessorScriptLoader {
                }
        }
        
-       private suspend fun runScriptSafe(scriptName: String, bind: Map<String, Any?>, env: AsyncLexerTagEnvironment<PreProcessingContext, PreProcessingSubject>, errorHandler: (Exception) -> ParserTree): ParserTree {
+       private suspend fun runScriptWithBindings(scriptName: String, bind: Map<String, Any?>, env: AsyncLexerTagEnvironment<PreProcessingContext, PreProcessingSubject>, errorHandler: (Exception) -> ParserTree): ParserTree {
                return try {
                        val script = loadFunction(scriptName)!!
                        val result = runScriptInternal(script, bind, env)
@@ -103,11 +103,11 @@ object PreProcessorScriptLoader {
        
        suspend fun runScriptSafe(scriptName: String, args: Map<String, ParserTree>, env: AsyncLexerTagEnvironment<PreProcessingContext, PreProcessingSubject>, errorHandler: (Exception) -> ParserTree): ParserTree {
                val groovyArgs = args.mapValuesTo(mutableMapOf()) { (_, it) -> jsonToGroovy(it.toPreProcessJson()) }
-               return runScriptSafe(scriptName, mapOf("args" to groovyArgs), env, errorHandler)
+               return runScriptWithBindings(scriptName, mapOf("args" to groovyArgs), env, errorHandler)
        }
        
        suspend fun runScriptSafe(scriptName: String, input: ParserTree, env: AsyncLexerTagEnvironment<PreProcessingContext, PreProcessingSubject>, errorHandler: (Exception) -> ParserTree): ParserTree {
-               return runScriptSafe(scriptName, mapOf("text" to input.unparse()), env, errorHandler)
+               return runScriptWithBindings(scriptName, mapOf("text" to input.unparse()), env, errorHandler)
        }
 }
 
index 9fa0fff4576dd84f06133976757b9373f9c18f85..ea0890ac2009627658f2398de3775a7c1c05103d 100644 (file)
@@ -65,7 +65,8 @@ fun ParserTree.toPreProcessJson(): JsonElement {
 
 object FactbookLoader {
        suspend fun loadJsonData(lorePath: List<String>): JsonObject {
-               val bytes = FileStorage.instance.readFile(StoragePath.jsonDocDir / lorePath) ?: return JsonObject(emptyMap())
+               val jsonPath = lorePath.dropLast(1) + listOf("${lorePath.last()}.json")
+               val bytes = FileStorage.instance.readFile(StoragePath.jsonDocDir / jsonPath) ?: return JsonObject(emptyMap())
                return JsonStorageCodec.parseToJsonElement(String(bytes)) as JsonObject
        }