From: Lanius Trolling Date: Sun, 15 Oct 2023 12:44:50 +0000 (-0400) Subject: Update to official Kotlin MongoDB driver AND prevent bots from cluttering page view... X-Git-Url: https://gitweb.starshipfights.net/?a=commitdiff_plain;h=ba4e8f497f09e6b7dbf8b88768181e14954fb400;p=factbooks Update to official Kotlin MongoDB driver AND prevent bots from cluttering page view counting --- diff --git a/.idea/kotlinc.xml b/.idea/kotlinc.xml index fdf8d99..f8467b4 100644 --- a/.idea/kotlinc.xml +++ b/.idea/kotlinc.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 071745c..dd2ac1a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -24,8 +24,8 @@ buildscript { plugins { java - kotlin("jvm") version "1.9.0" - kotlin("plugin.serialization") version "1.9.0" + kotlin("jvm") version "1.9.10" + kotlin("plugin.serialization") version "1.9.10" id("com.github.johnrengelman.shadow") version "7.1.2" application } @@ -44,19 +44,19 @@ dependencies { implementation("org.jetbrains.kotlinx:kotlinx-serialization-core-jvm:1.6.0") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json-jvm:1.6.0") - implementation("io.ktor:ktor-server-core-jvm:2.3.4") - implementation("io.ktor:ktor-server-netty-jvm:2.3.4") + implementation("io.ktor:ktor-server-core-jvm:2.3.5") + implementation("io.ktor:ktor-server-cio-jvm:2.3.5") - implementation("io.ktor:ktor-server-caching-headers:2.3.4") - implementation("io.ktor:ktor-server-call-id:2.3.4") - implementation("io.ktor:ktor-server-call-logging:2.3.4") - implementation("io.ktor:ktor-server-conditional-headers:2.3.4") - implementation("io.ktor:ktor-server-forwarded-header:2.3.4") - implementation("io.ktor:ktor-server-html-builder:2.3.4") - implementation("io.ktor:ktor-server-sessions-jvm:2.3.4") - implementation("io.ktor:ktor-server-status-pages:2.3.4") + implementation("io.ktor:ktor-server-caching-headers:2.3.5") + implementation("io.ktor:ktor-server-call-id:2.3.5") + implementation("io.ktor:ktor-server-call-logging:2.3.5") + implementation("io.ktor:ktor-server-conditional-headers:2.3.5") + implementation("io.ktor:ktor-server-forwarded-header:2.3.5") + implementation("io.ktor:ktor-server-html-builder:2.3.5") + implementation("io.ktor:ktor-server-sessions-jvm:2.3.5") + implementation("io.ktor:ktor-server-status-pages:2.3.5") - implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.9.0") + implementation("org.jetbrains.kotlinx:kotlinx-html-jvm:0.9.1") implementation("com.samskivert:jmustache:1.15") implementation("org.apache.groovy:groovy-jsr223:4.0.10") @@ -64,13 +64,8 @@ dependencies { implementation(files("libs/nsapi4j.jar")) implementation("com.aventrix.jnanoid:jnanoid:2.0.0") - implementation("org.litote.kmongo:kmongo-coroutine-serialization:4.8.0") { - exclude("org.jetbrains.kotlin", "kotlin-reflect") - - exclude("org.jetbrains.kotlinx", "kotlinx-coroutines-jdk8") - exclude("org.jetbrains.kotlinx", "kotlinx-coroutines-reactive") - exclude("org.jetbrains.kotlinx", "kotlinx-serialization-core-jvm") - } + implementation("org.mongodb:mongodb-driver-kotlin-coroutine:4.11.0") + implementation("org.mongodb:bson-kotlinx:4.11.0") implementation("org.slf4j:slf4j-api:2.0.7") implementation("ch.qos.logback:logback-classic:1.4.7") diff --git a/src/main/kotlin/info/mechyrdia/Factbooks.kt b/src/main/kotlin/info/mechyrdia/Factbooks.kt index 6d43060..93e5626 100644 --- a/src/main/kotlin/info/mechyrdia/Factbooks.kt +++ b/src/main/kotlin/info/mechyrdia/Factbooks.kt @@ -8,10 +8,10 @@ import info.mechyrdia.lore.* import io.ktor.http.* import io.ktor.http.content.* import io.ktor.server.application.* +import io.ktor.server.cio.* import io.ktor.server.engine.* import io.ktor.server.html.* import io.ktor.server.http.content.* -import io.ktor.server.netty.* import io.ktor.server.plugins.* import io.ktor.server.plugins.cachingheaders.* import io.ktor.server.plugins.callid.* @@ -39,7 +39,7 @@ fun main() { ConnectionHolder.initialize(Configuration.CurrentConfiguration.dbConn, Configuration.CurrentConfiguration.dbName) - embeddedServer(Netty, port = Configuration.CurrentConfiguration.port, host = Configuration.CurrentConfiguration.host, module = Application::factbooks).start(wait = true) + embeddedServer(CIO, port = Configuration.CurrentConfiguration.port, host = Configuration.CurrentConfiguration.host, module = Application::factbooks).start(wait = true) } fun Application.factbooks() { diff --git a/src/main/kotlin/info/mechyrdia/data/bson.kt b/src/main/kotlin/info/mechyrdia/data/bson.kt new file mode 100644 index 0000000..eb9881c --- /dev/null +++ b/src/main/kotlin/info/mechyrdia/data/bson.kt @@ -0,0 +1,87 @@ +package info.mechyrdia.data + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerializationException +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.modules.SerializersModule +import org.bson.BsonDateTime +import org.bson.BsonNull +import org.bson.BsonReader +import org.bson.BsonWriter +import org.bson.codecs.Codec +import org.bson.codecs.DecoderContext +import org.bson.codecs.EncoderContext +import org.bson.codecs.configuration.CodecProvider +import org.bson.codecs.configuration.CodecRegistry +import org.bson.codecs.kotlinx.BsonDecoder +import org.bson.codecs.kotlinx.BsonEncoder +import java.time.Instant + +object IdCodec : Codec> { + override fun getEncoderClass(): Class> { + return Id::class.java + } + + override fun encode(writer: BsonWriter, value: Id<*>, encoderContext: EncoderContext?) { + writer.writeString(value.id) + } + + override fun decode(reader: BsonReader, decoderContext: DecoderContext?): Id<*> { + return Id(reader.readString()) + } +} + +object IdCodecProvider : CodecProvider { + override fun get(clazz: Class?, registry: CodecRegistry?): Codec? { + @Suppress("UNCHECKED_CAST") + return if (clazz == Id::class.java) + IdCodec as Codec + else null + } +} + +object InstantSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("InstantSerializer", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: Instant) { + if (encoder !is BsonEncoder) + throw SerializationException("Instant is not supported by ${encoder::class}") + + encoder.encodeBsonValue(BsonDateTime(value.toEpochMilli())) + } + + override fun deserialize(decoder: Decoder): Instant { + if (decoder !is BsonDecoder) + throw SerializationException("Instant is not supported by ${decoder::class}") + + return Instant.ofEpochMilli(decoder.decodeBsonValue().asDateTime().value) + } +} + +object InstantNullableSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("InstantSerializer", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: Instant?) { + if (encoder !is BsonEncoder) + throw SerializationException("Instant is not supported by ${encoder::class}") + + if (value == null) + encoder.encodeBsonValue(BsonNull.VALUE) + else + encoder.encodeBsonValue(BsonDateTime(value.toEpochMilli())) + } + + override fun deserialize(decoder: Decoder): Instant? { + if (decoder !is BsonDecoder) + throw SerializationException("Instant is not supported by ${decoder::class}") + + val value = decoder.decodeBsonValue() + if (value.isNull) + return null + return Instant.ofEpochMilli(value.asDateTime().value) + } +} diff --git a/src/main/kotlin/info/mechyrdia/data/comments.kt b/src/main/kotlin/info/mechyrdia/data/comments.kt index 42b9e72..8261498 100644 --- a/src/main/kotlin/info/mechyrdia/data/comments.kt +++ b/src/main/kotlin/info/mechyrdia/data/comments.kt @@ -1,14 +1,12 @@ package info.mechyrdia.data +import com.mongodb.client.model.Filters +import com.mongodb.client.model.Sorts import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.toList import kotlinx.serialization.Contextual import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import org.litote.kmongo.and -import org.litote.kmongo.descending -import org.litote.kmongo.eq -import org.litote.kmongo.nin import java.time.Instant @Serializable @@ -18,10 +16,10 @@ data class Comment( val submittedBy: Id, val submittedIn: String, - val submittedAt: @Contextual Instant, + val submittedAt: @Serializable(with = InstantSerializer::class) Instant, val numEdits: Int, - val lastEdit: @Contextual Instant?, + val lastEdit: @Serializable(with = InstantNullableSerializer::class) Instant?, val contents: String ) : DataDocument { @@ -34,11 +32,11 @@ data class Comment( } suspend fun getCommentsIn(page: String): Flow { - return Table.select(Comment::submittedIn eq page, descending(Comment::submittedAt)) + return Table.select(Filters.eq(Comment::submittedIn.serialName, page), Sorts.descending(Comment::submittedAt.serialName)) } suspend fun getCommentsBy(user: Id): Flow { - return Table.select(Comment::submittedBy eq user, descending(Comment::submittedAt)) + return Table.select(Filters.eq(Comment::submittedBy.serialName, user), Sorts.descending(Comment::submittedAt.serialName)) } } } @@ -51,7 +49,7 @@ data class CommentReplyLink( val originalPost: Id, val replyingPost: Id, - val repliedAt: @Contextual Instant = Instant.now(), + val repliedAt: @Serializable(with = InstantSerializer::class) Instant = Instant.now(), ) : DataDocument { companion object : TableHolder { override val Table = DocumentTable() @@ -64,9 +62,9 @@ data class CommentReplyLink( suspend fun updateComment(updatedReply: Id, repliesTo: Set>) { Table.remove( - and( - CommentReplyLink::originalPost nin repliesTo, - CommentReplyLink::replyingPost eq updatedReply + Filters.and( + Filters.nin(CommentReplyLink::originalPost.serialName, repliesTo), + Filters.eq(CommentReplyLink::replyingPost.serialName, updatedReply) ) ) Table.put( @@ -80,11 +78,11 @@ data class CommentReplyLink( } suspend fun deleteComment(deletedReply: Id) { - Table.remove(CommentReplyLink::replyingPost eq deletedReply) + Table.remove(Filters.eq(CommentReplyLink::replyingPost.serialName, deletedReply)) } suspend fun getReplies(original: Id): List> { - return Table.filter(CommentReplyLink::originalPost eq original) + return Table.filter(Filters.eq(CommentReplyLink::originalPost.serialName, original)) .toList() .sortedBy { it.repliedAt } .map { it.replyingPost } diff --git a/src/main/kotlin/info/mechyrdia/data/data.kt b/src/main/kotlin/info/mechyrdia/data/data.kt index ee9ad5e..2584ee1 100644 --- a/src/main/kotlin/info/mechyrdia/data/data.kt +++ b/src/main/kotlin/info/mechyrdia/data/data.kt @@ -1,12 +1,15 @@ package info.mechyrdia.data import com.aventrix.jnanoid.jnanoid.NanoIdUtils -import com.mongodb.client.model.BulkWriteOptions -import com.mongodb.client.model.IndexOptions -import com.mongodb.client.model.ReplaceOptions +import com.mongodb.ConnectionString +import com.mongodb.MongoClientSettings +import com.mongodb.client.model.* +import com.mongodb.kotlin.client.coroutine.MongoClient import info.mechyrdia.auth.SessionStorageDoc import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.flow.singleOrNull import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.serialization.KSerializer @@ -17,16 +20,17 @@ import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.encoding.Decoder import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.modules.SerializersModule +import org.bson.BsonReader +import org.bson.BsonWriter +import org.bson.codecs.Codec +import org.bson.codecs.DecoderContext +import org.bson.codecs.EncoderContext +import org.bson.codecs.configuration.CodecProvider +import org.bson.codecs.configuration.CodecRegistries +import org.bson.codecs.configuration.CodecRegistry +import org.bson.codecs.kotlinx.KotlinSerializerCodecProvider import org.bson.conversions.Bson -import org.litote.kmongo.coroutine.CoroutineAggregatePublisher -import org.litote.kmongo.coroutine.CoroutineClient -import org.litote.kmongo.coroutine.coroutine -import org.litote.kmongo.reactivestreams.KMongo -import org.litote.kmongo.replaceOne -import org.litote.kmongo.serialization.IdController -import org.litote.kmongo.serialization.changeIdController -import org.litote.kmongo.serialization.registerSerializer -import org.litote.kmongo.util.KMongoUtil import java.security.SecureRandom import kotlin.reflect.KClass import kotlin.reflect.KProperty1 @@ -57,25 +61,10 @@ object IdSerializer : KSerializer> { } } -object DocumentIdController : IdController { - override fun findIdProperty(type: KClass<*>): KProperty1<*, *> { - return DataDocument<*>::id - } - - @Suppress("UNCHECKED_CAST") - override fun getIdValue(idProperty: KProperty1, instance: T): R? { - return (instance as DataDocument<*>).id as R - } - - override fun setIdValue(idProperty: KProperty1, instance: T) { - throw UnsupportedOperationException("Cannot set id property of DataDocument!") - } -} - object ConnectionHolder { private lateinit var databaseName: String - private val clientDeferred = CompletableDeferred() + private val clientDeferred = CompletableDeferred() suspend fun getDatabase() = clientDeferred.await().getDatabase(databaseName) @@ -83,11 +72,20 @@ object ConnectionHolder { if (clientDeferred.isCompleted) error("Cannot initialize database twice!") - changeIdController(DocumentIdController) - registerSerializer(IdSerializer) + MongoClient.create( + MongoClientSettings.builder() + .codecRegistry( + CodecRegistries.fromProviders( + IdCodecProvider, + KotlinSerializerCodecProvider() + ) + ) + .applyConnectionString(ConnectionString(conn)) + .build() + ) databaseName = db - clientDeferred.complete(KMongo.createClient(conn).coroutine) + clientDeferred.complete(MongoClient.create(conn)) runBlocking { for (holder in TableHolder.entries) @@ -103,65 +101,67 @@ interface DataDocument> { val id: Id } +const val MONGODB_ID_KEY = "_id" + class DocumentTable>(private val kClass: KClass) { - private suspend fun collection() = ConnectionHolder.getDatabase().database.getCollection(kClass.simpleName!!, kClass.java).coroutine + private suspend fun collection() = ConnectionHolder.getDatabase().getCollection(kClass.simpleName!!, kClass.java) suspend fun index(vararg properties: KProperty1) { - collection().ensureIndex(*properties) + collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray())) } suspend fun unique(vararg properties: KProperty1) { - collection().ensureUniqueIndex(*properties) + collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().unique(true)) } suspend fun indexIf(condition: Bson, vararg properties: KProperty1) { - collection().ensureIndex(*properties, indexOptions = IndexOptions().partialFilterExpression(condition)) + collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().partialFilterExpression(condition)) } suspend fun uniqueIf(condition: Bson, vararg properties: KProperty1) { - collection().ensureUniqueIndex(*properties, indexOptions = IndexOptions().partialFilterExpression(condition)) + collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().unique(true).partialFilterExpression(condition)) } suspend fun put(doc: T) { - collection().replaceOneById(doc.id, doc, ReplaceOptions().upsert(true)) + collection().replaceOne(Filters.eq(MONGODB_ID_KEY, doc.id), doc, ReplaceOptions().upsert(true)) } suspend fun put(docs: Collection) { if (docs.isNotEmpty()) collection().bulkWrite( docs.map { doc -> - replaceOne(KMongoUtil.idFilterQuery(doc.id), doc, ReplaceOptions().upsert(true)) + ReplaceOneModel(Filters.eq(MONGODB_ID_KEY, doc.id), doc, ReplaceOptions().upsert(true)) }, BulkWriteOptions().ordered(false) ) } suspend fun set(id: Id, set: Bson): Boolean { - return collection().updateOneById(id, set).matchedCount != 0L + return collection().updateOne(Filters.eq(MONGODB_ID_KEY, id), set).matchedCount != 0L } suspend fun get(id: Id): T? { - return collection().findOneById(id) + return collection().find(Filters.eq(MONGODB_ID_KEY, id)).singleOrNull() } suspend fun del(id: Id) { - collection().deleteOneById(id) + collection().deleteOne(Filters.eq(MONGODB_ID_KEY, id)) } suspend fun all(): Flow { - return collection().find().toFlow() + return collection().find() } suspend fun filter(where: Bson): Flow { - return collection().find(where).toFlow() + return collection().find(where) } suspend fun sorted(order: Bson): Flow { - return collection().find().sort(order).toFlow() + return collection().find().sort(order) } suspend fun select(where: Bson, order: Bson): Flow { - return collection().find(where).sort(order).toFlow() + return collection().find(where).sort(order) } suspend fun number(where: Bson): Long { @@ -169,7 +169,7 @@ class DocumentTable>(private val kClass: KClass) { } suspend fun locate(where: Bson): T? { - return collection().findOne(where) + return collection().find(where).singleOrNull() } suspend fun update(where: Bson, set: Bson) { @@ -180,8 +180,8 @@ class DocumentTable>(private val kClass: KClass) { collection().deleteMany(where) } - suspend fun aggregate(pipeline: List, resultClass: KClass): CoroutineAggregatePublisher { - return collection().collection.aggregate(pipeline, resultClass.java).coroutine + suspend fun aggregate(pipeline: List, resultClass: KClass): Flow { + return collection().aggregate(pipeline, resultClass.java) } } diff --git a/src/main/kotlin/info/mechyrdia/data/data_utils.kt b/src/main/kotlin/info/mechyrdia/data/data_utils.kt index 582dd03..274b065 100644 --- a/src/main/kotlin/info/mechyrdia/data/data_utils.kt +++ b/src/main/kotlin/info/mechyrdia/data/data_utils.kt @@ -1,5 +1,9 @@ package info.mechyrdia.data +import kotlinx.serialization.SerialName +import kotlin.reflect.KProperty +import kotlin.reflect.full.findAnnotations + suspend inline fun > DocumentTable.getOrPut(id: Id, defaultValue: () -> T): T { val value = get(id) return if (value == null) { @@ -13,3 +17,6 @@ suspend inline fun > DocumentTable.getOrPut(id: Id, de value } } + +val KProperty.serialName: String + get() = findAnnotations(SerialName::class).singleOrNull()?.value ?: name diff --git a/src/main/kotlin/info/mechyrdia/data/view_comments.kt b/src/main/kotlin/info/mechyrdia/data/view_comments.kt index b069c6e..7328859 100644 --- a/src/main/kotlin/info/mechyrdia/data/view_comments.kt +++ b/src/main/kotlin/info/mechyrdia/data/view_comments.kt @@ -64,7 +64,7 @@ fun FlowContent.commentBox(comment: CommentRenderData, loggedInAs: Id Unit { @@ -33,7 +33,7 @@ suspend fun ApplicationCall.recentCommentsPage(): HTML.() -> Unit { val comments = CommentRenderData( Comment.Table - .sorted(descending(Comment::submittedAt)) + .sorted(Sorts.descending(Comment::submittedAt.serialName)) .filterNot { comment -> NationData.get(comment.submittedBy).isBanned } @@ -165,7 +165,7 @@ suspend fun ApplicationCall.deleteCommentPage(): HTML.() -> Unit { strong { +"It will be gone forever!" } } - commentBox(commentDisplay, null) + commentBox(commentDisplay, currNation.id) form(method = FormMethod.get, action = "/comment/view/$commentId") { submitInput { value = "No, take me back" } @@ -513,6 +513,15 @@ suspend fun ApplicationCall.commentHelpPage(): HTML.() -> Unit = page("Commentin } } } + tr { + td { +"[lang=pokhval]Pokhvalsko Jaargo[/lang]" } + td { + +"Writes text in the Pokhwalish alphabet: " + span(classes = "lang-pokhwal") { + +"poKvalsqo jArgo" + } + } + } tr { td { +"[lang=gothic]\uD800\uDF32\uD800\uDF3F\uD800\uDF44\uD800\uDF39\uD800\uDF43\uD800\uDF3A\uD800\uDF30 \uD800\uDF42\uD800\uDF30\uD800\uDF36\uD800\uDF33\uD800\uDF30[/lang]" } td { diff --git a/src/main/kotlin/info/mechyrdia/data/views_user.kt b/src/main/kotlin/info/mechyrdia/data/views_user.kt index f22b512..3d79c29 100644 --- a/src/main/kotlin/info/mechyrdia/data/views_user.kt +++ b/src/main/kotlin/info/mechyrdia/data/views_user.kt @@ -1,5 +1,6 @@ package info.mechyrdia.data +import com.mongodb.client.model.Updates import info.mechyrdia.OwnerNationId import info.mechyrdia.auth.createCsrfToken import info.mechyrdia.auth.verifyCsrfToken @@ -10,7 +11,6 @@ import info.mechyrdia.lore.standardNavBar import io.ktor.server.application.* import kotlinx.coroutines.flow.toList import kotlinx.html.* -import org.litote.kmongo.setValue suspend fun ApplicationCall.userPage(): HTML.() -> Unit { val currNation = currentNation() @@ -56,7 +56,7 @@ suspend fun ApplicationCall.adminBanUserRoute(): Nothing { val bannedNation = nationCache.getNation(Id(parameters["id"]!!)) if (!bannedNation.isBanned) - NationData.Table.set(bannedNation.id, setValue(NationData::isBanned, true)) + NationData.Table.set(bannedNation.id, Updates.set(NationData::isBanned.serialName, true)) redirect("/user/${bannedNation.id}") } @@ -68,7 +68,7 @@ suspend fun ApplicationCall.adminUnbanUserRoute(): Nothing { val bannedNation = nationCache.getNation(Id(parameters["id"]!!)) if (bannedNation.isBanned) - NationData.Table.set(bannedNation.id, setValue(NationData::isBanned, false)) + NationData.Table.set(bannedNation.id, Updates.set(NationData::isBanned.serialName, false)) redirect("/user/${bannedNation.id}") } diff --git a/src/main/kotlin/info/mechyrdia/data/visits.kt b/src/main/kotlin/info/mechyrdia/data/visits.kt index 72b727b..021cae8 100644 --- a/src/main/kotlin/info/mechyrdia/data/visits.kt +++ b/src/main/kotlin/info/mechyrdia/data/visits.kt @@ -1,18 +1,21 @@ package info.mechyrdia.data +import com.mongodb.client.model.Accumulators +import com.mongodb.client.model.Aggregates +import com.mongodb.client.model.Filters import info.mechyrdia.lore.dateTime import io.ktor.server.application.* import io.ktor.server.plugins.* import io.ktor.server.request.* import io.ktor.server.sessions.* import io.ktor.util.* +import kotlinx.coroutines.flow.firstOrNull import kotlinx.html.FlowContent import kotlinx.html.p import kotlinx.html.style import kotlinx.serialization.Contextual import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import org.litote.kmongo.* import java.security.MessageDigest import java.time.Instant @@ -20,7 +23,7 @@ import java.time.Instant data class PageVisitTotals( val total: Long, val totalUnique: Long, - val mostRecent: @Contextual Instant? + val mostRecent: @Serializable(with = InstantNullableSerializer::class) Instant? ) @Serializable @@ -31,7 +34,7 @@ data class PageVisitData( val path: String, val visitor: String, val visits: Long = 0L, - val lastVisit: @Contextual Instant = Instant.now() + val lastVisit: @Serializable(with = InstantSerializer::class) Instant = Instant.now() ) : DataDocument { companion object : TableHolder { override val Table = DocumentTable() @@ -43,7 +46,12 @@ data class PageVisitData( } suspend fun visit(path: String, visitor: String) { - val data = Table.locate(and(PageVisitData::path eq path, PageVisitData::visitor eq visitor)) + val data = Table.locate( + Filters.and( + Filters.eq(PageVisitData::path.serialName, path), + Filters.eq(PageVisitData::visitor.serialName, visitor) + ) + ) if (data == null) Table.put( PageVisitData( @@ -64,15 +72,15 @@ data class PageVisitData( suspend fun totalVisits(path: String): PageVisitTotals { return Table.aggregate<_, PageVisitTotals>( listOf( - match(PageVisitData::path eq path), - group( + Aggregates.match(Filters.eq(PageVisitData::path.serialName, path)), + Aggregates.group( null, - PageVisitTotals::total sum PageVisitData::visits, - PageVisitTotals::totalUnique sum 1L, - PageVisitTotals::mostRecent max PageVisitData::lastVisit, + Accumulators.sum(PageVisitTotals::total.serialName, "\$${PageVisitData::visits.serialName}"), + Accumulators.sum(PageVisitTotals::totalUnique.serialName, 1L), + Accumulators.max(PageVisitTotals::mostRecent.serialName, "\$${PageVisitData::lastVisit.serialName}"), ) ) - ).first() ?: PageVisitTotals(0L, 0L, null) + ).firstOrNull() ?: PageVisitTotals(0L, 0L, null) } } } @@ -93,7 +101,8 @@ suspend fun ApplicationCall.processGuestbook(): PageVisitTotals { val path = request.path() val totals = PageVisitData.totalVisits(path) - PageVisitData.visit(path, anonymizedClientId()) + if (!RobotDetector.isRobot(request.userAgent())) + PageVisitData.visit(path, anonymizedClientId()) return totals } @@ -113,3 +122,186 @@ fun FlowContent.guestbook(totalsData: PageVisitTotals) { } } } + +object RobotDetector { + private val botRegexes = listOf( + Regex(" daum[ /]"), + Regex(" deusu/"), + Regex(" yadirectfetcher"), + Regex("(?:^| )site"), + Regex("(?:^|[^g])news"), + Regex("@[a-z]"), + Regex("\\(at\\)[a-z]"), + Regex("\\(github\\.com/"), + Regex("\\[at][a-z]"), + Regex("^12345"), + Regex("^<"), + Regex("^[\\w .\\-()]+(/v?\\d+(\\.\\d+)?(\\.\\d{1,10})?)?$"), + Regex("^[^ ]{50,}$"), + Regex("^active"), + Regex("^ad muncher"), + Regex("^amaya"), + Regex("^anglesharp/"), + Regex("^anonymous"), + Regex("^avsdevicesdk/"), + Regex("^axios/"), + Regex("^bidtellect/"), + Regex("^biglotron"), + Regex("^btwebclient/"), + Regex("^castro"), + Regex("^clamav[ /]"), + Regex("^client/"), + Regex("^cobweb/"), + Regex("^coccoc"), + Regex("^custom"), + Regex("^ddg[_-]android"), + Regex("^discourse"), + Regex("^dispatch/\\d"), + Regex("^downcast/"), + Regex("^duckduckgo"), + Regex("^facebook"), + Regex("^fdm[ /]\\d"), + Regex("^getright/"), + Regex("^gozilla/"), + Regex("^hatena"), + Regex("^hobbit"), + Regex("^hotzonu"), + Regex("^hwcdn/"), + Regex("^jeode/"), + Regex("^jetty/"), + Regex("^jigsaw"), + Regex("^linkdex"), + Regex("^lwp[-: ]"), + Regex("^metauri"), + Regex("^microsoft bits"), + Regex("^movabletype"), + Regex("^mozilla/\\d\\.\\d \\(compatible;?\\)$"), + Regex("^mozilla/\\d\\.\\d \\w*$"), + Regex("^navermailapp"), + Regex("^netsurf"), + Regex("^offline explorer"), + Regex("^php"), + Regex("^postman"), + Regex("^postrank"), + Regex("^python"), + Regex("^read"), + Regex("^reed"), + Regex("^restsharp/"), + Regex("^snapchat"), + Regex("^space bison"), + Regex("^svn"), + Regex("^swcd "), + Regex("^taringa"), + Regex("^test certificate info"), + Regex("^thumbor/"), + Regex("^tumblr/"), + Regex("^user-agent:mozilla"), + Regex("^valid"), + Regex("^venus/fedoraplanet"), + Regex("^w3c"), + Regex("^webbandit/"), + Regex("^webcopier"), + Regex("^wget"), + Regex("^whatsapp"), + Regex("^xenu link sleuth"), + Regex("^yahoo"), + Regex("^yandex"), + Regex("^zdm/\\d"), + Regex("^zoom marketplace/"), + Regex("^\\{\\{.*\\}\\}$"), + Regex("adbeat\\.com"), + Regex("appinsights"), + Regex("archive"), + Regex("ask jeeves/teoma"), + Regex("bit\\.ly/"), + Regex("bluecoat drtr"), + Regex("bot"), + Regex("browsex"), + Regex("burpcollaborator"), + Regex("capture"), + Regex("catch"), + Regex("check"), + Regex("chrome-lighthouse"), + Regex("chromeframe"), + Regex("cloud"), + Regex("crawl"), + Regex("cryptoapi"), + Regex("dareboost"), + Regex("datanyze"), + Regex("dataprovider"), + Regex("dejaclick"), + Regex("dmbrowser"), + Regex("download"), + Regex("evc-batch/"), + Regex("feed"), + Regex("firephp"), + Regex("freesafeip"), + Regex("ghost"), + Regex("gomezagent"), + Regex("google"), + Regex("headlesschrome/"), + Regex("http"), + Regex("httrack"), + Regex("hubspot marketing grader"), + Regex("hydra"), + Regex("ibisbrowser"), + Regex("images"), + Regex("iplabel"), + Regex("ips-agent"), + Regex("java"), + Regex("library"), + Regex("mail\\.ru/"), + Regex("manager"), + Regex("monitor"), + Regex("morningscore/"), + Regex("neustar wpm"), + Regex("nutch"), + Regex("offbyone"), + Regex("optimize"), + Regex("pageburst"), + Regex("pagespeed"), + Regex("perl"), + Regex("phantom"), + Regex("pingdom"), + Regex("powermarks"), + Regex("preview"), + Regex("proxy"), + Regex("ptst[ /]\\d"), + Regex("reader"), + Regex("rexx;"), + Regex("rigor"), + Regex("rss"), + Regex("scan"), + Regex("scrape"), + Regex("search"), + Regex("serp ?reputation ?management"), + Regex("server"), + Regex("sogou"), + Regex("sparkler/"), + Regex("speedcurve"), + Regex("spider"), + Regex("splash"), + Regex("statuscake"), + Regex("stumbleupon\\.com"), + Regex("supercleaner"), + Regex("synapse"), + Regex("synthetic"), + Regex("taginspector/"), + Regex("torrent"), + Regex("tracemyfile"), + Regex("transcoder"), + Regex("trendsmapresolver"), + Regex("twingly recon"), + Regex("url"), + Regex("virtuoso"), + Regex("wappalyzer"), + Regex("webglance"), + Regex("webkit2png"), + Regex("websitemetadataretriever"), + Regex("whatcms/"), + Regex("wordpress"), + Regex("zgrab"), + ) + + fun isRobot(userAgent: String?) = userAgent == null || botRegexes.any { it.containsMatchIn(userAgent) } +} diff --git a/src/main/kotlin/info/mechyrdia/lore/parser.kt b/src/main/kotlin/info/mechyrdia/lore/parser.kt index 531513f..a4fda8d 100644 --- a/src/main/kotlin/info/mechyrdia/lore/parser.kt +++ b/src/main/kotlin/info/mechyrdia/lore/parser.kt @@ -6,20 +6,29 @@ data class TextParserScope( val ctx: TContext ) +sealed class InsideTag { + abstract val tag: String + + data class DirectTag(override val tag: String) : InsideTag() + data class IndirectTag(override val tag: String, val param: String?) : InsideTag() +} + sealed class TextParserState( val scope: TextParserScope, - val insideTags: List>, - val insideDirectTags: List + val insideTags: List ) { abstract fun processCharacter(char: Char): TextParserState abstract fun processEndOfText() protected fun appendText(text: String) { scope.write.append( - insideTags.foldRight(censorText(text)) { (tag, param), t -> - (scope.tags[tag] as? TextParserTagType.Indirect) - ?.process(param, t, scope.ctx) - ?: "[$tag${param?.let { "=$it" } ?: ""}]$t[/$tag]" + insideTags.foldRight(censorText(text)) { insideTag, t -> + if (insideTag is InsideTag.IndirectTag) { + val (tag, param) = insideTag + (scope.tags[tag] as? TextParserTagType.Indirect) + ?.process(param, t, scope.ctx) + ?: "[$tag${param?.let { "=$it" } ?: ""}]$t[/$tag]" + } else t } ) } @@ -28,12 +37,12 @@ sealed class TextParserState( scope.write.append(text) } - class Initial(scope: TextParserScope) : TextParserState(scope, listOf(), listOf()) { + class Initial(scope: TextParserScope) : TextParserState(scope, listOf()) { override fun processCharacter(char: Char): TextParserState { return if (char == '[') - OpenTag(scope, "", insideTags, insideDirectTags) + OpenTag(scope, "", insideTags) else - PlainText(scope, "$char", insideTags, insideDirectTags) + PlainText(scope, "$char", insideTags) } override fun processEndOfText() { @@ -41,23 +50,23 @@ sealed class TextParserState( } } - class PlainText(scope: TextParserScope, val text: String, insideTags: List>, insideDirectTags: List) : TextParserState(scope, insideTags, insideDirectTags) { + class PlainText(scope: TextParserScope, private val text: String, insideTags: List) : TextParserState(scope, insideTags) { override fun processCharacter(char: Char): TextParserState { return if (char == '[') { appendText(text) - OpenTag(scope, "", insideTags, insideDirectTags) + OpenTag(scope, "", insideTags) } else if (char == '\n' && text.endsWith('\n')) { appendText(text.removeSuffix("\n")) - val newline = if (insideDirectTags.isEmpty()) + val newline = if (insideTags.none { it is InsideTag.DirectTag }) "

" else "
" appendTextRaw(newline) - PlainText(scope, "", insideTags, insideDirectTags) + PlainText(scope, "", insideTags) } else - PlainText(scope, text + char, insideTags, insideDirectTags) + PlainText(scope, text + char, insideTags) } override fun processEndOfText() { @@ -65,18 +74,18 @@ sealed class TextParserState( } } - class NoFormatText(scope: TextParserScope, val text: String, insideTags: List>, insideDirectTags: List) : TextParserState(scope, insideTags, insideDirectTags) { + class NoFormatText(scope: TextParserScope, private val text: String, insideTags: List) : TextParserState(scope, insideTags) { override fun processCharacter(char: Char): TextParserState { val newText = text + char return if (newText.endsWith("[/$NO_FORMAT_TAG]")) { appendText(newText.removeSuffix("[/$NO_FORMAT_TAG]")) - PlainText(scope, "", insideTags, insideDirectTags) + PlainText(scope, "", insideTags) } else if (newText.endsWith('\n')) { appendText(newText.removeSuffix("\n")) appendTextRaw("
") - NoFormatText(scope, "", insideTags, insideDirectTags) + NoFormatText(scope, "", insideTags) } else - NoFormatText(scope, newText, insideTags, insideDirectTags) + NoFormatText(scope, newText, insideTags) } override fun processEndOfText() { @@ -84,27 +93,27 @@ sealed class TextParserState( } } - class OpenTag(scope: TextParserScope, val tag: String, insideTags: List>, insideDirectTags: List) : TextParserState(scope, insideTags, insideDirectTags) { + class OpenTag(scope: TextParserScope, private val tag: String, insideTags: List) : TextParserState(scope, insideTags) { override fun processCharacter(char: Char): TextParserState { return if (char == ']') { if (tag.equals(NO_FORMAT_TAG, ignoreCase = true)) - NoFormatText(scope, "", insideTags, insideDirectTags) + NoFormatText(scope, "", insideTags) else when (val tagType = scope.tags[tag]) { is TextParserTagType.Direct -> { appendTextRaw(tagType.begin(null, scope.ctx)) - PlainText(scope, "", insideTags, insideDirectTags + tag) + PlainText(scope, "", insideTags + InsideTag.DirectTag(tag)) } - is TextParserTagType.Indirect -> PlainText(scope, "", insideTags + (tag to null), insideDirectTags) + is TextParserTagType.Indirect -> PlainText(scope, "", insideTags + InsideTag.IndirectTag(tag, null)) - else -> PlainText(scope, "[$tag]", insideTags, insideDirectTags) + else -> PlainText(scope, "[$tag]", insideTags) } } else if (char == '/' && tag == "") - CloseTag(scope, tag, insideTags, insideDirectTags) + CloseTag(scope, tag, insideTags) else if (char == '=' && tag != "") - TagParam(scope, tag, "", insideTags, insideDirectTags) + TagParam(scope, tag, "", insideTags) else - OpenTag(scope, tag + char, insideTags, insideDirectTags) + OpenTag(scope, tag + char, insideTags) } override fun processEndOfText() { @@ -112,21 +121,21 @@ sealed class TextParserState( } } - class TagParam(scope: TextParserScope, val tag: String, val param: String, insideTags: List>, insideDirectTags: List) : TextParserState(scope, insideTags, insideDirectTags) { + class TagParam(scope: TextParserScope, private val tag: String, private val param: String, insideTags: List) : TextParserState(scope, insideTags) { override fun processCharacter(char: Char): TextParserState { return if (char == ']') when (val tagType = scope.tags[tag]) { is TextParserTagType.Direct -> { appendTextRaw(tagType.begin(param, scope.ctx)) - PlainText(scope, "", insideTags, insideDirectTags + tag) + PlainText(scope, "", insideTags + InsideTag.DirectTag(tag)) } - is TextParserTagType.Indirect -> PlainText(scope, "", insideTags + (tag to param), insideDirectTags) + is TextParserTagType.Indirect -> PlainText(scope, "", insideTags + InsideTag.IndirectTag(tag, param)) - else -> PlainText(scope, "[$tag=$param]", insideTags, insideDirectTags) + else -> PlainText(scope, "[$tag=$param]", insideTags) } else - TagParam(scope, tag, param + char, insideTags, insideDirectTags) + TagParam(scope, tag, param + char, insideTags) } override fun processEndOfText() { @@ -134,21 +143,21 @@ sealed class TextParserState( } } - class CloseTag(scope: TextParserScope, val tag: String, insideTags: List>, insideDirectTags: List) : TextParserState(scope, insideTags, insideDirectTags) { + class CloseTag(scope: TextParserScope, private val tag: String, insideTags: List) : TextParserState(scope, insideTags) { override fun processCharacter(char: Char): TextParserState { return if (char == ']') { val tagType = scope.tags[tag] - if (tagType is TextParserTagType.Direct && insideDirectTags.lastOrNull().equals(tag, ignoreCase = true)) { + if (tagType is TextParserTagType.Direct && insideTags.lastOrNull() == InsideTag.DirectTag(tag)) { appendTextRaw(tagType.end(scope.ctx)) - PlainText(scope, "", insideTags, insideDirectTags.dropLast(1)) - } else if (insideTags.isNotEmpty() && insideTags.lastOrNull()?.first.equals(tag, ignoreCase = true)) { - PlainText(scope, "", insideTags.dropLast(1), insideDirectTags) + PlainText(scope, "", insideTags.dropLast(1)) + } else if (insideTags.isNotEmpty() && (insideTags.last() as? InsideTag.IndirectTag)?.tag == tag) { + PlainText(scope, "", insideTags.dropLast(1)) } else { appendText("[/$tag]") - PlainText(scope, "", insideTags, insideDirectTags) + PlainText(scope, "", insideTags) } - } else CloseTag(scope, tag + char, insideTags, insideDirectTags) + } else CloseTag(scope, tag + char, insideTags) } override fun processEndOfText() {