From: Lanius Trolling Date: Sat, 13 Apr 2024 22:01:50 +0000 (-0400) Subject: Fix filenames X-Git-Url: https://gitweb.starshipfights.net/?a=commitdiff_plain;h=b31ab93f506e2055971aef15b6e6b40af35c48e9;p=factbooks Fix filenames --- diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/Bson.kt b/src/jvmMain/kotlin/info/mechyrdia/data/Bson.kt new file mode 100644 index 0000000..584eb3c --- /dev/null +++ b/src/jvmMain/kotlin/info/mechyrdia/data/Bson.kt @@ -0,0 +1,86 @@ +@file:OptIn(ExperimentalSerializationApi::class) + +package info.mechyrdia.data + +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerializationException +import kotlinx.serialization.builtins.nullable +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import org.bson.BsonDateTime +import org.bson.BsonReader +import org.bson.BsonWriter +import org.bson.codecs.Codec +import org.bson.codecs.DecoderContext +import org.bson.codecs.EncoderContext +import org.bson.codecs.configuration.CodecProvider +import org.bson.codecs.configuration.CodecRegistry +import org.bson.codecs.kotlinx.BsonDecoder +import org.bson.codecs.kotlinx.BsonEncoder +import org.bson.types.ObjectId +import java.time.Instant + +object IdCodec : Codec> { + override fun getEncoderClass(): Class> { + return Id::class.java + } + + override fun encode(writer: BsonWriter, value: Id<*>, encoderContext: EncoderContext?) { + writer.writeString(value.id) + } + + override fun decode(reader: BsonReader, decoderContext: DecoderContext?): Id<*> { + return Id(reader.readString()) + } +} + +object IdCodecProvider : CodecProvider { + override fun get(clazz: Class?, registry: CodecRegistry?): Codec? { + @Suppress("UNCHECKED_CAST") + return if (clazz == Id::class.java) + IdCodec as Codec + else null + } +} + +object ObjectIdSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("ObjectIdSerializer", PrimitiveKind.LONG) + + override fun serialize(encoder: Encoder, value: ObjectId) { + if (encoder !is BsonEncoder) + throw SerializationException("ObjectId is not supported by ${encoder::class}") + + encoder.encodeObjectId(value) + } + + override fun deserialize(decoder: Decoder): ObjectId { + if (decoder !is BsonDecoder) + throw SerializationException("ObjectId is not supported by ${decoder::class}") + + return decoder.decodeObjectId() + } +} + +object InstantSerializer : KSerializer { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("InstantSerializer", PrimitiveKind.LONG) + + override fun serialize(encoder: Encoder, value: Instant) { + if (encoder !is BsonEncoder) + throw SerializationException("Instant is not supported by ${encoder::class}") + + encoder.encodeBsonValue(BsonDateTime(value.toEpochMilli())) + } + + override fun deserialize(decoder: Decoder): Instant { + if (decoder !is BsonDecoder) + throw SerializationException("Instant is not supported by ${decoder::class}") + + return Instant.ofEpochMilli(decoder.decodeBsonValue().asDateTime().value) + } +} + +object InstantNullableSerializer : KSerializer by InstantSerializer.nullable diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/Comments.kt b/src/jvmMain/kotlin/info/mechyrdia/data/Comments.kt new file mode 100644 index 0000000..4cd6cb1 --- /dev/null +++ b/src/jvmMain/kotlin/info/mechyrdia/data/Comments.kt @@ -0,0 +1,97 @@ +package info.mechyrdia.data + +import com.mongodb.client.model.* +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.toList +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import java.time.Instant + +@Serializable +data class Comment( + @SerialName(MONGODB_ID_KEY) + override val id: Id, + + val submittedBy: Id, + val submittedIn: String, + val submittedAt: @Serializable(with = InstantSerializer::class) Instant, + + val numEdits: Int, + val lastEdit: @Serializable(with = InstantNullableSerializer::class) Instant?, + + val contents: String +) : DataDocument { + companion object : TableHolder { + override val Table = DocumentTable() + + override suspend fun initialize() { + Table.index(Comment::submittedBy, Comment::submittedAt) + Table.index(Comment::submittedIn, Comment::submittedAt) + } + + suspend fun getCommentsIn(page: List): Flow { + return Table.select(Filters.eq(Comment::submittedIn.serialName, page.joinToString(separator = "/")), Sorts.descending(Comment::submittedAt.serialName)) + } + + suspend fun getCommentsBy(user: Id): Flow { + return Table.select(Filters.eq(Comment::submittedBy.serialName, user), Sorts.descending(Comment::submittedAt.serialName)) + } + } +} + +@Serializable +data class CommentReplyLink( + @SerialName(MONGODB_ID_KEY) + override val id: Id = Id(), + + val originalPost: Id, + val replyingPost: Id, + + val repliedAt: @Serializable(with = InstantSerializer::class) Instant = Instant.now(), +) : DataDocument { + companion object : TableHolder { + override val Table = DocumentTable() + + override suspend fun initialize() { + Table.index(CommentReplyLink::originalPost) + Table.index(CommentReplyLink::replyingPost) + Table.unique(CommentReplyLink::replyingPost, CommentReplyLink::originalPost) + } + + suspend fun updateComment(updatedReply: Id, repliesTo: Set>, now: Instant) { + Table.remove( + Filters.and( + Filters.nin(CommentReplyLink::originalPost.serialName, repliesTo), + Filters.eq(CommentReplyLink::replyingPost.serialName, updatedReply) + ) + ) + + Table.insert( + repliesTo.map { original -> + UpdateOneModel( + Filters.and( + Filters.eq(CommentReplyLink::originalPost.serialName, original), + Filters.eq(CommentReplyLink::replyingPost.serialName, updatedReply) + ), + Updates.combine( + Updates.set(CommentReplyLink::repliedAt.serialName, now), + Updates.setOnInsert(MONGODB_ID_KEY, Id()), + ), + UpdateOptions().upsert(true) + ) + } + ) + } + + suspend fun deleteComment(deletedReply: Id) { + Table.remove(Filters.eq(CommentReplyLink::replyingPost.serialName, deletedReply)) + } + + suspend fun getReplies(original: Id): List> { + return Table.filter(Filters.eq(CommentReplyLink::originalPost.serialName, original)) + .toList() + .sortedBy { it.repliedAt } + .map { it.replyingPost } + } + } +} diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/Data.kt b/src/jvmMain/kotlin/info/mechyrdia/data/Data.kt new file mode 100644 index 0000000..02a9671 --- /dev/null +++ b/src/jvmMain/kotlin/info/mechyrdia/data/Data.kt @@ -0,0 +1,237 @@ +package info.mechyrdia.data + +import com.aventrix.jnanoid.jnanoid.NanoIdUtils +import com.mongodb.ConnectionString +import com.mongodb.MongoClientSettings +import com.mongodb.MongoDriverInformation +import com.mongodb.client.model.* +import com.mongodb.kotlin.client.coroutine.MongoDatabase +import com.mongodb.reactivestreams.client.MongoClients +import com.mongodb.reactivestreams.client.gridfs.GridFSBucket +import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets +import info.mechyrdia.auth.SessionStorageDoc +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.singleOrNull +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import org.bson.codecs.configuration.CodecRegistries +import org.bson.codecs.kotlinx.KotlinSerializerCodecProvider +import org.bson.conversions.Bson +import java.security.SecureRandom +import kotlin.reflect.KClass +import kotlin.reflect.KProperty +import kotlin.reflect.KProperty1 +import kotlin.reflect.full.findAnnotations +import com.mongodb.reactivestreams.client.MongoDatabase as JMongoDatabase + +@Serializable(IdSerializer::class) +@JvmInline +value class Id(val id: String) { + override fun toString() = id + + companion object { + fun serializer(): KSerializer> = IdSerializer + } +} + +private val secureRandom = SecureRandom.getInstanceStrong() +private val alphabet = "ABCDEFGHILMNOPQRSTVXYZ0123456789".toCharArray() +fun Id() = Id(NanoIdUtils.randomNanoId(secureRandom, alphabet, 24)) + +object IdSerializer : KSerializer> { + override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("Id", PrimitiveKind.STRING) + + override fun serialize(encoder: Encoder, value: Id<*>) { + encoder.encodeString(value.id) + } + + override fun deserialize(decoder: Decoder): Id<*> { + return Id(decoder.decodeString()) + } +} + +object ConnectionHolder { + private val jDatabaseDeferred = CompletableDeferred() + + suspend fun getDatabase() = MongoDatabase(jDatabaseDeferred.await()) + + suspend fun getBucket(): GridFSBucket = GridFSBuckets.create(jDatabaseDeferred.await()) + + fun initialize(conn: String, db: String) { + if (jDatabaseDeferred.isCompleted) + error("Cannot initialize database twice") + + jDatabaseDeferred.complete( + MongoClients.create( + MongoClientSettings.builder() + .codecRegistry( + CodecRegistries.fromProviders( + MongoClientSettings.getDefaultCodecRegistry(), + IdCodecProvider, + KotlinSerializerCodecProvider() + ) + ) + .applyConnectionString(ConnectionString(conn)) + .build(), + MongoDriverInformation.builder() + .driverName("kotlin") + .build() + ).getDatabase(db) + ) + + runBlocking { + for (holder in TableHolder.entries) + launch { + holder.initialize() + } + } + } +} + +interface DataDocument> { + @SerialName(MONGODB_ID_KEY) + val id: Id +} + +const val MONGODB_ID_KEY = "_id" + +class DocumentTable>(private val kClass: KClass) { + private suspend fun collection() = ConnectionHolder.getDatabase().getCollection(kClass.simpleName!!, kClass.java) + + suspend fun index(vararg properties: KProperty1) { + collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray())) + } + + suspend fun unique(vararg properties: KProperty1) { + collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().unique(true)) + } + + suspend fun indexIf(condition: Bson, vararg properties: KProperty1) { + collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().partialFilterExpression(condition)) + } + + suspend fun uniqueIf(condition: Bson, vararg properties: KProperty1) { + collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().unique(true).partialFilterExpression(condition)) + } + + suspend fun put(doc: T) { + collection().replaceOne(Filters.eq(MONGODB_ID_KEY, doc.id), doc, ReplaceOptions().upsert(true)) + } + + suspend fun put(docs: Collection) { + if (docs.isNotEmpty()) + collection().bulkWrite( + docs.map { doc -> + ReplaceOneModel(Filters.eq(MONGODB_ID_KEY, doc.id), doc, ReplaceOptions().upsert(true)) + }, + BulkWriteOptions().ordered(false) + ) + } + + suspend fun set(id: Id, set: Bson): Boolean { + return collection().updateOne(Filters.eq(MONGODB_ID_KEY, id), set).matchedCount > 0L + } + + suspend fun get(id: Id): T? { + return collection().find(Filters.eq(MONGODB_ID_KEY, id)).singleOrNull() + } + + suspend fun del(id: Id) { + collection().deleteOne(Filters.eq(MONGODB_ID_KEY, id)) + } + + suspend fun all(): Flow { + return collection().find() + } + + suspend fun insert(docs: Collection>) { + if (docs.isNotEmpty()) + collection().bulkWrite( + if (docs is List) docs else docs.toList(), + BulkWriteOptions().ordered(false) + ) + } + + suspend fun filter(where: Bson): Flow { + return collection().find(where) + } + + suspend fun sorted(order: Bson): Flow { + return collection().find().sort(order) + } + + suspend fun select(where: Bson, order: Bson): Flow { + return collection().find(where).sort(order) + } + + suspend fun number(where: Bson): Long { + return collection().countDocuments(where) + } + + suspend fun locate(where: Bson): T? { + return collection().find(where).singleOrNull() + } + + suspend fun update(where: Bson, set: Bson): Long { + return collection().updateMany(where, set).matchedCount + } + + suspend fun change(where: Bson, set: Bson) { + collection().updateOne(where, set, UpdateOptions().upsert(true)) + } + + suspend fun remove(where: Bson): Long { + return collection().deleteMany(where).deletedCount + } + + suspend fun aggregate(pipeline: List, resultClass: KClass): Flow { + return collection().aggregate(pipeline, resultClass.java) + } +} + +suspend inline fun , reified R : Any> DocumentTable.aggregate(pipeline: List) = aggregate(pipeline, R::class) + +suspend inline fun > DocumentTable.getOrPut(id: Id, defaultValue: () -> T): T { + val value = get(id) + return if (value == null) { + val answer = defaultValue() + if (answer.id != id) { + throw IllegalArgumentException("Default value $answer has different Id than provided: $id") + } + put(answer) + answer + } else { + value + } +} + +val KProperty.serialName: String + get() = findAnnotations(SerialName::class).singleOrNull()?.value ?: name + +inline fun > DocumentTable() = DocumentTable(T::class) + +interface TableHolder> { + @Suppress("PropertyName") + val Table: DocumentTable + + suspend fun initialize() + + companion object { + val entries = listOf( + SessionStorageDoc, + NationData, + Comment, + CommentReplyLink, + PageVisitData + ) + } +} diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/Nations.kt b/src/jvmMain/kotlin/info/mechyrdia/data/Nations.kt new file mode 100644 index 0000000..54d9fad --- /dev/null +++ b/src/jvmMain/kotlin/info/mechyrdia/data/Nations.kt @@ -0,0 +1,94 @@ +package info.mechyrdia.data + +import com.github.agadar.nationstates.shard.NationShard +import info.mechyrdia.OwnerNationId +import info.mechyrdia.auth.NSAPI +import info.mechyrdia.auth.UserSession +import info.mechyrdia.auth.executeSuspend +import io.ktor.server.application.* +import io.ktor.server.sessions.* +import io.ktor.util.* +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.concurrent.ConcurrentHashMap + +@Serializable +data class NationData( + @SerialName(MONGODB_ID_KEY) + override val id: Id, + val name: String, + val flag: String, + + val isBanned: Boolean = false +) : DataDocument { + companion object : TableHolder { + private val logger: Logger = LoggerFactory.getLogger(NationData::class.java) + + override val Table = DocumentTable() + + override suspend fun initialize() { + Table.index(NationData::name) + } + + fun unknown(id: Id): NationData { + logger.warn("Unable to find nation with Id $id - did it CTE?") + return NationData(id, "Unknown Nation", "https://www.nationstates.net/images/flags/exnation.png") + } + + suspend fun get(id: Id): NationData = Table.getOrPut(id) { + NSAPI + .getNation(id.id) + .shards(NationShard.NAME, NationShard.FLAG_URL) + .executeSuspend() + ?.let { nation -> + NationData(id = Id(nation.id), name = nation.name, flag = nation.flagUrl) + } ?: unknown(id) + } + } +} + +val CallNationCacheAttribute = AttributeKey, NationData>>("NationCache") + +val ApplicationCall.nationCache: MutableMap, NationData> + get() = attributes.getOrNull(CallNationCacheAttribute) + ?: ConcurrentHashMap, NationData>().also { cache -> + attributes.put(CallNationCacheAttribute, cache) + } + +suspend fun MutableMap, NationData>.getNation(id: Id): NationData { + return getOrPut(id) { + NationData.get(id) + } +} + +private val callCurrentNationAttribute = AttributeKey("CurrentNation") + +fun ApplicationCall.ownerNationOnly() { + if (sessions.get()?.nationId != OwnerNationId) + throw NoSuchElementException("Hidden page") +} + +suspend fun ApplicationCall.currentNation(): NationData? { + attributes.getOrNull(callCurrentNationAttribute)?.let { sess -> + return when (sess) { + NationSession.Anonymous -> null + is NationSession.LoggedIn -> sess.nation + } + } + + val nationId = sessions.get()?.nationId + return if (nationId == null) { + attributes.put(callCurrentNationAttribute, NationSession.Anonymous) + null + } else nationCache.getNation(nationId).also { data -> + attributes.put(callCurrentNationAttribute, NationSession.LoggedIn(data)) + } +} + +private sealed class NationSession { + data object Anonymous : NationSession() + + data class LoggedIn(val nation: NationData) : NationSession() +} diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/Visits.kt b/src/jvmMain/kotlin/info/mechyrdia/data/Visits.kt new file mode 100644 index 0000000..2f1fa10 --- /dev/null +++ b/src/jvmMain/kotlin/info/mechyrdia/data/Visits.kt @@ -0,0 +1,303 @@ +package info.mechyrdia.data + +import com.mongodb.client.model.Accumulators +import com.mongodb.client.model.Aggregates +import com.mongodb.client.model.Filters +import com.mongodb.client.model.Updates +import info.mechyrdia.lore.dateTime +import io.ktor.server.application.* +import io.ktor.server.plugins.* +import io.ktor.server.request.* +import io.ktor.server.sessions.* +import io.ktor.util.* +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.html.FlowContent +import kotlinx.html.p +import kotlinx.html.style +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.intellij.lang.annotations.Language +import java.security.MessageDigest +import java.time.Instant + +@Serializable +data class PageVisitTotals( + val total: Int, + val totalUnique: Int, + val mostRecent: @Serializable(with = InstantNullableSerializer::class) Instant? +) + +@Serializable +data class PageVisitData( + @SerialName(MONGODB_ID_KEY) + override val id: Id = Id(), + + val path: String, + val visitor: String, + val visits: Int = 0, + val lastVisit: @Serializable(with = InstantSerializer::class) Instant = Instant.now() +) : DataDocument { + companion object : TableHolder { + override val Table = DocumentTable() + + override suspend fun initialize() { + Table.index(PageVisitData::path) + Table.unique(PageVisitData::path, PageVisitData::visitor) + Table.index(PageVisitData::lastVisit) + } + + suspend fun visit(path: String, visitor: String) { + Table.change( + Filters.and( + Filters.eq(PageVisitData::path.serialName, path), + Filters.eq(PageVisitData::visitor.serialName, visitor) + ), + Updates.combine( + Updates.inc(PageVisitData::visits.serialName, 1), + Updates.set(PageVisitData::lastVisit.serialName, Instant.now()), + Updates.setOnInsert(MONGODB_ID_KEY, Id()) + ) + ) + } + + suspend fun totalVisits(path: String): PageVisitTotals { + return Table.aggregate<_, PageVisitTotals>( + listOf( + Aggregates.match(Filters.eq(PageVisitData::path.serialName, path)), + Aggregates.group( + null, + Accumulators.sum(PageVisitTotals::total.serialName, "\$${PageVisitData::visits.serialName}"), + Accumulators.sum(PageVisitTotals::totalUnique.serialName, 1), + Accumulators.max(PageVisitTotals::mostRecent.serialName, "\$${PageVisitData::lastVisit.serialName}"), + ) + ) + ).firstOrNull() ?: PageVisitTotals(0, 0, null) + } + } +} + +private val messageDigestProvider = ThreadLocal.withInitial { MessageDigest.getInstance("SHA-256") } + +fun ApplicationCall.anonymizedClientId(): String { + val messageDigest = messageDigestProvider.get() + + messageDigest.reset() + messageDigest.update(request.origin.remoteAddress.encodeToByteArray()) + request.userAgent()?.encodeToByteArray()?.let { messageDigest.update(it) } + + return hex(messageDigest.digest()) +} + +suspend fun ApplicationCall.processGuestbook(): PageVisitTotals { + val path = request.path() + + val totals = PageVisitData.totalVisits(path) + if (!RobotDetector.isRobot(request.userAgent())) + PageVisitData.visit(path, anonymizedClientId()) + + return totals +} + +fun Int.pluralize(singular: String, plural: String = singular + "s") = if (this == 1) singular else plural + +fun FlowContent.guestbook(totalsData: PageVisitTotals) { + p { + style = "font-size:0.8em" + + +"This page has been visited ${totalsData.total} ${totalsData.total.pluralize("time")} by ${totalsData.totalUnique} unique ${totalsData.totalUnique.pluralize("visitor")}, most recently " + + val mostRecent = totalsData.mostRecent + if (mostRecent == null) + +"in the abyss of unwritten history" + else { + +"at " + dateTime(mostRecent) + } + } +} + +object RobotDetector { + private fun botRegex(@Language("RegExp") regex: String) = Regex(regex, RegexOption.IGNORE_CASE) + + private val botRegexes = listOf( + botRegex(" daum[ /]"), + botRegex(" deusu/"), + botRegex(" yadirectfetcher"), + botRegex("(?:^| )site"), + botRegex("(?:^|[^g])news"), + botRegex("@[a-z]"), + botRegex("\\(at\\)[a-z]"), + botRegex("\\(github\\.com/"), + botRegex("\\[at][a-z]"), + botRegex("^12345"), + botRegex("^<"), + botRegex("^[\\w .\\-()]+(/v?\\d+(\\.\\d+)?(\\.\\d{1,10})?)?$"), + botRegex("^[^ ]{50,}$"), + botRegex("^active"), + botRegex("^ad muncher"), + botRegex("^amaya"), + botRegex("^anglesharp/"), + botRegex("^anonymous"), + botRegex("^avsdevicesdk/"), + botRegex("^axios/"), + botRegex("^bidtellect/"), + botRegex("^biglotron"), + botRegex("^btwebclient/"), + botRegex("^castro"), + botRegex("^clamav[ /]"), + botRegex("^client/"), + botRegex("^cobweb/"), + botRegex("^coccoc"), + botRegex("^custom"), + botRegex("^ddg[_-]android"), + botRegex("^discourse"), + botRegex("^dispatch/\\d"), + botRegex("^downcast/"), + botRegex("^duckduckgo"), + botRegex("^facebook"), + botRegex("^fdm[ /]\\d"), + botRegex("^getright/"), + botRegex("^gozilla/"), + botRegex("^hatena"), + botRegex("^hobbit"), + botRegex("^hotzonu"), + botRegex("^hwcdn/"), + botRegex("^jeode/"), + botRegex("^jetty/"), + botRegex("^jigsaw"), + botRegex("^linkdex"), + botRegex("^lwp[-: ]"), + botRegex("^metauri"), + botRegex("^microsoft bits"), + botRegex("^movabletype"), + botRegex("^mozilla/\\d\\.\\d \\(compatible;?\\)$"), + botRegex("^mozilla/\\d\\.\\d \\w*$"), + botRegex("^navermailapp"), + botRegex("^netsurf"), + botRegex("^offline explorer"), + botRegex("^php"), + botRegex("^postman"), + botRegex("^postrank"), + botRegex("^python"), + botRegex("^read"), + botRegex("^reed"), + botRegex("^restsharp/"), + botRegex("^snapchat"), + botRegex("^space bison"), + botRegex("^svn"), + botRegex("^swcd "), + botRegex("^taringa"), + botRegex("^test certificate info"), + botRegex("^thumbor/"), + botRegex("^tumblr/"), + botRegex("^user-agent:mozilla"), + botRegex("^valid"), + botRegex("^venus/fedoraplanet"), + botRegex("^w3c"), + botRegex("^webbandit/"), + botRegex("^webcopier"), + botRegex("^wget"), + botRegex("^whatsapp"), + botRegex("^xenu link sleuth"), + botRegex("^yahoo"), + botRegex("^yandex"), + botRegex("^zdm/\\d"), + botRegex("^zoom marketplace/"), + botRegex("^\\{\\{.*}}$"), + botRegex("adbeat\\.com"), + botRegex("appinsights"), + botRegex("archive"), + botRegex("ask jeeves/teoma"), + botRegex("bit\\.ly/"), + botRegex("bluecoat drtr"), + botRegex("bot"), + botRegex("browsex"), + botRegex("burpcollaborator"), + botRegex("capture"), + botRegex("catch"), + botRegex("check"), + botRegex("chrome-lighthouse"), + botRegex("chromeframe"), + botRegex("cloud"), + botRegex("crawl"), + botRegex("cryptoapi"), + botRegex("dareboost"), + botRegex("datanyze"), + botRegex("dataprovider"), + botRegex("dejaclick"), + botRegex("dmbrowser"), + botRegex("download"), + botRegex("evc-batch/"), + botRegex("feed"), + botRegex("firephp"), + botRegex("freesafeip"), + botRegex("ghost"), + botRegex("gomezagent"), + botRegex("google"), + botRegex("headlesschrome/"), + botRegex("http"), + botRegex("httrack"), + botRegex("hubspot marketing grader"), + botRegex("hydra"), + botRegex("ibisbrowser"), + botRegex("images"), + botRegex("iplabel"), + botRegex("ips-agent"), + botRegex("java"), + botRegex("library"), + botRegex("mail\\.ru/"), + botRegex("manager"), + botRegex("monitor"), + botRegex("morningscore/"), + botRegex("neustar wpm"), + botRegex("nutch"), + botRegex("offbyone"), + botRegex("optimize"), + botRegex("pageburst"), + botRegex("pagespeed"), + botRegex("perl"), + botRegex("phantom"), + botRegex("pingdom"), + botRegex("powermarks"), + botRegex("preview"), + botRegex("proxy"), + botRegex("ptst[ /]\\d"), + botRegex("rainmeter webparser plugin"), + botRegex("reader"), + botRegex("rexx;"), + botRegex("rigor"), + botRegex("rss"), + botRegex("scan"), + botRegex("scrape"), + botRegex("search"), + botRegex("serp ?reputation ?management"), + botRegex("server"), + botRegex("sogou"), + botRegex("sparkler/"), + botRegex("speedcurve"), + botRegex("spider"), + botRegex("splash"), + botRegex("statuscake"), + botRegex("stumbleupon\\.com"), + botRegex("supercleaner"), + botRegex("synapse"), + botRegex("synthetic"), + botRegex("taginspector/"), + botRegex("torrent"), + botRegex("tracemyfile"), + botRegex("transcoder"), + botRegex("trendsmapresolver"), + botRegex("twingly recon"), + botRegex("url"), + botRegex("virtuoso"), + botRegex("wappalyzer"), + botRegex("webglance"), + botRegex("webkit2png"), + botRegex("websitemetadataretriever"), + botRegex("whatcms/"), + botRegex("wordpress"), + botRegex("zgrab"), + ) + + fun isRobot(userAgent: String?) = userAgent == null || botRegexes.any { it.containsMatchIn(userAgent) } +} diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/bson.kt b/src/jvmMain/kotlin/info/mechyrdia/data/bson.kt deleted file mode 100644 index 584eb3c..0000000 --- a/src/jvmMain/kotlin/info/mechyrdia/data/bson.kt +++ /dev/null @@ -1,86 +0,0 @@ -@file:OptIn(ExperimentalSerializationApi::class) - -package info.mechyrdia.data - -import kotlinx.serialization.ExperimentalSerializationApi -import kotlinx.serialization.KSerializer -import kotlinx.serialization.SerializationException -import kotlinx.serialization.builtins.nullable -import kotlinx.serialization.descriptors.PrimitiveKind -import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor -import kotlinx.serialization.descriptors.SerialDescriptor -import kotlinx.serialization.encoding.Decoder -import kotlinx.serialization.encoding.Encoder -import org.bson.BsonDateTime -import org.bson.BsonReader -import org.bson.BsonWriter -import org.bson.codecs.Codec -import org.bson.codecs.DecoderContext -import org.bson.codecs.EncoderContext -import org.bson.codecs.configuration.CodecProvider -import org.bson.codecs.configuration.CodecRegistry -import org.bson.codecs.kotlinx.BsonDecoder -import org.bson.codecs.kotlinx.BsonEncoder -import org.bson.types.ObjectId -import java.time.Instant - -object IdCodec : Codec> { - override fun getEncoderClass(): Class> { - return Id::class.java - } - - override fun encode(writer: BsonWriter, value: Id<*>, encoderContext: EncoderContext?) { - writer.writeString(value.id) - } - - override fun decode(reader: BsonReader, decoderContext: DecoderContext?): Id<*> { - return Id(reader.readString()) - } -} - -object IdCodecProvider : CodecProvider { - override fun get(clazz: Class?, registry: CodecRegistry?): Codec? { - @Suppress("UNCHECKED_CAST") - return if (clazz == Id::class.java) - IdCodec as Codec - else null - } -} - -object ObjectIdSerializer : KSerializer { - override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("ObjectIdSerializer", PrimitiveKind.LONG) - - override fun serialize(encoder: Encoder, value: ObjectId) { - if (encoder !is BsonEncoder) - throw SerializationException("ObjectId is not supported by ${encoder::class}") - - encoder.encodeObjectId(value) - } - - override fun deserialize(decoder: Decoder): ObjectId { - if (decoder !is BsonDecoder) - throw SerializationException("ObjectId is not supported by ${decoder::class}") - - return decoder.decodeObjectId() - } -} - -object InstantSerializer : KSerializer { - override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("InstantSerializer", PrimitiveKind.LONG) - - override fun serialize(encoder: Encoder, value: Instant) { - if (encoder !is BsonEncoder) - throw SerializationException("Instant is not supported by ${encoder::class}") - - encoder.encodeBsonValue(BsonDateTime(value.toEpochMilli())) - } - - override fun deserialize(decoder: Decoder): Instant { - if (decoder !is BsonDecoder) - throw SerializationException("Instant is not supported by ${decoder::class}") - - return Instant.ofEpochMilli(decoder.decodeBsonValue().asDateTime().value) - } -} - -object InstantNullableSerializer : KSerializer by InstantSerializer.nullable diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/comments.kt b/src/jvmMain/kotlin/info/mechyrdia/data/comments.kt deleted file mode 100644 index 4cd6cb1..0000000 --- a/src/jvmMain/kotlin/info/mechyrdia/data/comments.kt +++ /dev/null @@ -1,97 +0,0 @@ -package info.mechyrdia.data - -import com.mongodb.client.model.* -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.toList -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import java.time.Instant - -@Serializable -data class Comment( - @SerialName(MONGODB_ID_KEY) - override val id: Id, - - val submittedBy: Id, - val submittedIn: String, - val submittedAt: @Serializable(with = InstantSerializer::class) Instant, - - val numEdits: Int, - val lastEdit: @Serializable(with = InstantNullableSerializer::class) Instant?, - - val contents: String -) : DataDocument { - companion object : TableHolder { - override val Table = DocumentTable() - - override suspend fun initialize() { - Table.index(Comment::submittedBy, Comment::submittedAt) - Table.index(Comment::submittedIn, Comment::submittedAt) - } - - suspend fun getCommentsIn(page: List): Flow { - return Table.select(Filters.eq(Comment::submittedIn.serialName, page.joinToString(separator = "/")), Sorts.descending(Comment::submittedAt.serialName)) - } - - suspend fun getCommentsBy(user: Id): Flow { - return Table.select(Filters.eq(Comment::submittedBy.serialName, user), Sorts.descending(Comment::submittedAt.serialName)) - } - } -} - -@Serializable -data class CommentReplyLink( - @SerialName(MONGODB_ID_KEY) - override val id: Id = Id(), - - val originalPost: Id, - val replyingPost: Id, - - val repliedAt: @Serializable(with = InstantSerializer::class) Instant = Instant.now(), -) : DataDocument { - companion object : TableHolder { - override val Table = DocumentTable() - - override suspend fun initialize() { - Table.index(CommentReplyLink::originalPost) - Table.index(CommentReplyLink::replyingPost) - Table.unique(CommentReplyLink::replyingPost, CommentReplyLink::originalPost) - } - - suspend fun updateComment(updatedReply: Id, repliesTo: Set>, now: Instant) { - Table.remove( - Filters.and( - Filters.nin(CommentReplyLink::originalPost.serialName, repliesTo), - Filters.eq(CommentReplyLink::replyingPost.serialName, updatedReply) - ) - ) - - Table.insert( - repliesTo.map { original -> - UpdateOneModel( - Filters.and( - Filters.eq(CommentReplyLink::originalPost.serialName, original), - Filters.eq(CommentReplyLink::replyingPost.serialName, updatedReply) - ), - Updates.combine( - Updates.set(CommentReplyLink::repliedAt.serialName, now), - Updates.setOnInsert(MONGODB_ID_KEY, Id()), - ), - UpdateOptions().upsert(true) - ) - } - ) - } - - suspend fun deleteComment(deletedReply: Id) { - Table.remove(Filters.eq(CommentReplyLink::replyingPost.serialName, deletedReply)) - } - - suspend fun getReplies(original: Id): List> { - return Table.filter(Filters.eq(CommentReplyLink::originalPost.serialName, original)) - .toList() - .sortedBy { it.repliedAt } - .map { it.replyingPost } - } - } -} diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/data.kt b/src/jvmMain/kotlin/info/mechyrdia/data/data.kt deleted file mode 100644 index 02a9671..0000000 --- a/src/jvmMain/kotlin/info/mechyrdia/data/data.kt +++ /dev/null @@ -1,237 +0,0 @@ -package info.mechyrdia.data - -import com.aventrix.jnanoid.jnanoid.NanoIdUtils -import com.mongodb.ConnectionString -import com.mongodb.MongoClientSettings -import com.mongodb.MongoDriverInformation -import com.mongodb.client.model.* -import com.mongodb.kotlin.client.coroutine.MongoDatabase -import com.mongodb.reactivestreams.client.MongoClients -import com.mongodb.reactivestreams.client.gridfs.GridFSBucket -import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets -import info.mechyrdia.auth.SessionStorageDoc -import kotlinx.coroutines.CompletableDeferred -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.singleOrNull -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.serialization.KSerializer -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import kotlinx.serialization.descriptors.PrimitiveKind -import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor -import kotlinx.serialization.descriptors.SerialDescriptor -import kotlinx.serialization.encoding.Decoder -import kotlinx.serialization.encoding.Encoder -import org.bson.codecs.configuration.CodecRegistries -import org.bson.codecs.kotlinx.KotlinSerializerCodecProvider -import org.bson.conversions.Bson -import java.security.SecureRandom -import kotlin.reflect.KClass -import kotlin.reflect.KProperty -import kotlin.reflect.KProperty1 -import kotlin.reflect.full.findAnnotations -import com.mongodb.reactivestreams.client.MongoDatabase as JMongoDatabase - -@Serializable(IdSerializer::class) -@JvmInline -value class Id(val id: String) { - override fun toString() = id - - companion object { - fun serializer(): KSerializer> = IdSerializer - } -} - -private val secureRandom = SecureRandom.getInstanceStrong() -private val alphabet = "ABCDEFGHILMNOPQRSTVXYZ0123456789".toCharArray() -fun Id() = Id(NanoIdUtils.randomNanoId(secureRandom, alphabet, 24)) - -object IdSerializer : KSerializer> { - override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("Id", PrimitiveKind.STRING) - - override fun serialize(encoder: Encoder, value: Id<*>) { - encoder.encodeString(value.id) - } - - override fun deserialize(decoder: Decoder): Id<*> { - return Id(decoder.decodeString()) - } -} - -object ConnectionHolder { - private val jDatabaseDeferred = CompletableDeferred() - - suspend fun getDatabase() = MongoDatabase(jDatabaseDeferred.await()) - - suspend fun getBucket(): GridFSBucket = GridFSBuckets.create(jDatabaseDeferred.await()) - - fun initialize(conn: String, db: String) { - if (jDatabaseDeferred.isCompleted) - error("Cannot initialize database twice") - - jDatabaseDeferred.complete( - MongoClients.create( - MongoClientSettings.builder() - .codecRegistry( - CodecRegistries.fromProviders( - MongoClientSettings.getDefaultCodecRegistry(), - IdCodecProvider, - KotlinSerializerCodecProvider() - ) - ) - .applyConnectionString(ConnectionString(conn)) - .build(), - MongoDriverInformation.builder() - .driverName("kotlin") - .build() - ).getDatabase(db) - ) - - runBlocking { - for (holder in TableHolder.entries) - launch { - holder.initialize() - } - } - } -} - -interface DataDocument> { - @SerialName(MONGODB_ID_KEY) - val id: Id -} - -const val MONGODB_ID_KEY = "_id" - -class DocumentTable>(private val kClass: KClass) { - private suspend fun collection() = ConnectionHolder.getDatabase().getCollection(kClass.simpleName!!, kClass.java) - - suspend fun index(vararg properties: KProperty1) { - collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray())) - } - - suspend fun unique(vararg properties: KProperty1) { - collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().unique(true)) - } - - suspend fun indexIf(condition: Bson, vararg properties: KProperty1) { - collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().partialFilterExpression(condition)) - } - - suspend fun uniqueIf(condition: Bson, vararg properties: KProperty1) { - collection().createIndex(Indexes.ascending(*properties.map { it.serialName }.toTypedArray()), IndexOptions().unique(true).partialFilterExpression(condition)) - } - - suspend fun put(doc: T) { - collection().replaceOne(Filters.eq(MONGODB_ID_KEY, doc.id), doc, ReplaceOptions().upsert(true)) - } - - suspend fun put(docs: Collection) { - if (docs.isNotEmpty()) - collection().bulkWrite( - docs.map { doc -> - ReplaceOneModel(Filters.eq(MONGODB_ID_KEY, doc.id), doc, ReplaceOptions().upsert(true)) - }, - BulkWriteOptions().ordered(false) - ) - } - - suspend fun set(id: Id, set: Bson): Boolean { - return collection().updateOne(Filters.eq(MONGODB_ID_KEY, id), set).matchedCount > 0L - } - - suspend fun get(id: Id): T? { - return collection().find(Filters.eq(MONGODB_ID_KEY, id)).singleOrNull() - } - - suspend fun del(id: Id) { - collection().deleteOne(Filters.eq(MONGODB_ID_KEY, id)) - } - - suspend fun all(): Flow { - return collection().find() - } - - suspend fun insert(docs: Collection>) { - if (docs.isNotEmpty()) - collection().bulkWrite( - if (docs is List) docs else docs.toList(), - BulkWriteOptions().ordered(false) - ) - } - - suspend fun filter(where: Bson): Flow { - return collection().find(where) - } - - suspend fun sorted(order: Bson): Flow { - return collection().find().sort(order) - } - - suspend fun select(where: Bson, order: Bson): Flow { - return collection().find(where).sort(order) - } - - suspend fun number(where: Bson): Long { - return collection().countDocuments(where) - } - - suspend fun locate(where: Bson): T? { - return collection().find(where).singleOrNull() - } - - suspend fun update(where: Bson, set: Bson): Long { - return collection().updateMany(where, set).matchedCount - } - - suspend fun change(where: Bson, set: Bson) { - collection().updateOne(where, set, UpdateOptions().upsert(true)) - } - - suspend fun remove(where: Bson): Long { - return collection().deleteMany(where).deletedCount - } - - suspend fun aggregate(pipeline: List, resultClass: KClass): Flow { - return collection().aggregate(pipeline, resultClass.java) - } -} - -suspend inline fun , reified R : Any> DocumentTable.aggregate(pipeline: List) = aggregate(pipeline, R::class) - -suspend inline fun > DocumentTable.getOrPut(id: Id, defaultValue: () -> T): T { - val value = get(id) - return if (value == null) { - val answer = defaultValue() - if (answer.id != id) { - throw IllegalArgumentException("Default value $answer has different Id than provided: $id") - } - put(answer) - answer - } else { - value - } -} - -val KProperty.serialName: String - get() = findAnnotations(SerialName::class).singleOrNull()?.value ?: name - -inline fun > DocumentTable() = DocumentTable(T::class) - -interface TableHolder> { - @Suppress("PropertyName") - val Table: DocumentTable - - suspend fun initialize() - - companion object { - val entries = listOf( - SessionStorageDoc, - NationData, - Comment, - CommentReplyLink, - PageVisitData - ) - } -} diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/nations.kt b/src/jvmMain/kotlin/info/mechyrdia/data/nations.kt deleted file mode 100644 index 54d9fad..0000000 --- a/src/jvmMain/kotlin/info/mechyrdia/data/nations.kt +++ /dev/null @@ -1,94 +0,0 @@ -package info.mechyrdia.data - -import com.github.agadar.nationstates.shard.NationShard -import info.mechyrdia.OwnerNationId -import info.mechyrdia.auth.NSAPI -import info.mechyrdia.auth.UserSession -import info.mechyrdia.auth.executeSuspend -import io.ktor.server.application.* -import io.ktor.server.sessions.* -import io.ktor.util.* -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import java.util.concurrent.ConcurrentHashMap - -@Serializable -data class NationData( - @SerialName(MONGODB_ID_KEY) - override val id: Id, - val name: String, - val flag: String, - - val isBanned: Boolean = false -) : DataDocument { - companion object : TableHolder { - private val logger: Logger = LoggerFactory.getLogger(NationData::class.java) - - override val Table = DocumentTable() - - override suspend fun initialize() { - Table.index(NationData::name) - } - - fun unknown(id: Id): NationData { - logger.warn("Unable to find nation with Id $id - did it CTE?") - return NationData(id, "Unknown Nation", "https://www.nationstates.net/images/flags/exnation.png") - } - - suspend fun get(id: Id): NationData = Table.getOrPut(id) { - NSAPI - .getNation(id.id) - .shards(NationShard.NAME, NationShard.FLAG_URL) - .executeSuspend() - ?.let { nation -> - NationData(id = Id(nation.id), name = nation.name, flag = nation.flagUrl) - } ?: unknown(id) - } - } -} - -val CallNationCacheAttribute = AttributeKey, NationData>>("NationCache") - -val ApplicationCall.nationCache: MutableMap, NationData> - get() = attributes.getOrNull(CallNationCacheAttribute) - ?: ConcurrentHashMap, NationData>().also { cache -> - attributes.put(CallNationCacheAttribute, cache) - } - -suspend fun MutableMap, NationData>.getNation(id: Id): NationData { - return getOrPut(id) { - NationData.get(id) - } -} - -private val callCurrentNationAttribute = AttributeKey("CurrentNation") - -fun ApplicationCall.ownerNationOnly() { - if (sessions.get()?.nationId != OwnerNationId) - throw NoSuchElementException("Hidden page") -} - -suspend fun ApplicationCall.currentNation(): NationData? { - attributes.getOrNull(callCurrentNationAttribute)?.let { sess -> - return when (sess) { - NationSession.Anonymous -> null - is NationSession.LoggedIn -> sess.nation - } - } - - val nationId = sessions.get()?.nationId - return if (nationId == null) { - attributes.put(callCurrentNationAttribute, NationSession.Anonymous) - null - } else nationCache.getNation(nationId).also { data -> - attributes.put(callCurrentNationAttribute, NationSession.LoggedIn(data)) - } -} - -private sealed class NationSession { - data object Anonymous : NationSession() - - data class LoggedIn(val nation: NationData) : NationSession() -} diff --git a/src/jvmMain/kotlin/info/mechyrdia/data/visits.kt b/src/jvmMain/kotlin/info/mechyrdia/data/visits.kt deleted file mode 100644 index 2f1fa10..0000000 --- a/src/jvmMain/kotlin/info/mechyrdia/data/visits.kt +++ /dev/null @@ -1,303 +0,0 @@ -package info.mechyrdia.data - -import com.mongodb.client.model.Accumulators -import com.mongodb.client.model.Aggregates -import com.mongodb.client.model.Filters -import com.mongodb.client.model.Updates -import info.mechyrdia.lore.dateTime -import io.ktor.server.application.* -import io.ktor.server.plugins.* -import io.ktor.server.request.* -import io.ktor.server.sessions.* -import io.ktor.util.* -import kotlinx.coroutines.flow.firstOrNull -import kotlinx.html.FlowContent -import kotlinx.html.p -import kotlinx.html.style -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import org.intellij.lang.annotations.Language -import java.security.MessageDigest -import java.time.Instant - -@Serializable -data class PageVisitTotals( - val total: Int, - val totalUnique: Int, - val mostRecent: @Serializable(with = InstantNullableSerializer::class) Instant? -) - -@Serializable -data class PageVisitData( - @SerialName(MONGODB_ID_KEY) - override val id: Id = Id(), - - val path: String, - val visitor: String, - val visits: Int = 0, - val lastVisit: @Serializable(with = InstantSerializer::class) Instant = Instant.now() -) : DataDocument { - companion object : TableHolder { - override val Table = DocumentTable() - - override suspend fun initialize() { - Table.index(PageVisitData::path) - Table.unique(PageVisitData::path, PageVisitData::visitor) - Table.index(PageVisitData::lastVisit) - } - - suspend fun visit(path: String, visitor: String) { - Table.change( - Filters.and( - Filters.eq(PageVisitData::path.serialName, path), - Filters.eq(PageVisitData::visitor.serialName, visitor) - ), - Updates.combine( - Updates.inc(PageVisitData::visits.serialName, 1), - Updates.set(PageVisitData::lastVisit.serialName, Instant.now()), - Updates.setOnInsert(MONGODB_ID_KEY, Id()) - ) - ) - } - - suspend fun totalVisits(path: String): PageVisitTotals { - return Table.aggregate<_, PageVisitTotals>( - listOf( - Aggregates.match(Filters.eq(PageVisitData::path.serialName, path)), - Aggregates.group( - null, - Accumulators.sum(PageVisitTotals::total.serialName, "\$${PageVisitData::visits.serialName}"), - Accumulators.sum(PageVisitTotals::totalUnique.serialName, 1), - Accumulators.max(PageVisitTotals::mostRecent.serialName, "\$${PageVisitData::lastVisit.serialName}"), - ) - ) - ).firstOrNull() ?: PageVisitTotals(0, 0, null) - } - } -} - -private val messageDigestProvider = ThreadLocal.withInitial { MessageDigest.getInstance("SHA-256") } - -fun ApplicationCall.anonymizedClientId(): String { - val messageDigest = messageDigestProvider.get() - - messageDigest.reset() - messageDigest.update(request.origin.remoteAddress.encodeToByteArray()) - request.userAgent()?.encodeToByteArray()?.let { messageDigest.update(it) } - - return hex(messageDigest.digest()) -} - -suspend fun ApplicationCall.processGuestbook(): PageVisitTotals { - val path = request.path() - - val totals = PageVisitData.totalVisits(path) - if (!RobotDetector.isRobot(request.userAgent())) - PageVisitData.visit(path, anonymizedClientId()) - - return totals -} - -fun Int.pluralize(singular: String, plural: String = singular + "s") = if (this == 1) singular else plural - -fun FlowContent.guestbook(totalsData: PageVisitTotals) { - p { - style = "font-size:0.8em" - - +"This page has been visited ${totalsData.total} ${totalsData.total.pluralize("time")} by ${totalsData.totalUnique} unique ${totalsData.totalUnique.pluralize("visitor")}, most recently " - - val mostRecent = totalsData.mostRecent - if (mostRecent == null) - +"in the abyss of unwritten history" - else { - +"at " - dateTime(mostRecent) - } - } -} - -object RobotDetector { - private fun botRegex(@Language("RegExp") regex: String) = Regex(regex, RegexOption.IGNORE_CASE) - - private val botRegexes = listOf( - botRegex(" daum[ /]"), - botRegex(" deusu/"), - botRegex(" yadirectfetcher"), - botRegex("(?:^| )site"), - botRegex("(?:^|[^g])news"), - botRegex("@[a-z]"), - botRegex("\\(at\\)[a-z]"), - botRegex("\\(github\\.com/"), - botRegex("\\[at][a-z]"), - botRegex("^12345"), - botRegex("^<"), - botRegex("^[\\w .\\-()]+(/v?\\d+(\\.\\d+)?(\\.\\d{1,10})?)?$"), - botRegex("^[^ ]{50,}$"), - botRegex("^active"), - botRegex("^ad muncher"), - botRegex("^amaya"), - botRegex("^anglesharp/"), - botRegex("^anonymous"), - botRegex("^avsdevicesdk/"), - botRegex("^axios/"), - botRegex("^bidtellect/"), - botRegex("^biglotron"), - botRegex("^btwebclient/"), - botRegex("^castro"), - botRegex("^clamav[ /]"), - botRegex("^client/"), - botRegex("^cobweb/"), - botRegex("^coccoc"), - botRegex("^custom"), - botRegex("^ddg[_-]android"), - botRegex("^discourse"), - botRegex("^dispatch/\\d"), - botRegex("^downcast/"), - botRegex("^duckduckgo"), - botRegex("^facebook"), - botRegex("^fdm[ /]\\d"), - botRegex("^getright/"), - botRegex("^gozilla/"), - botRegex("^hatena"), - botRegex("^hobbit"), - botRegex("^hotzonu"), - botRegex("^hwcdn/"), - botRegex("^jeode/"), - botRegex("^jetty/"), - botRegex("^jigsaw"), - botRegex("^linkdex"), - botRegex("^lwp[-: ]"), - botRegex("^metauri"), - botRegex("^microsoft bits"), - botRegex("^movabletype"), - botRegex("^mozilla/\\d\\.\\d \\(compatible;?\\)$"), - botRegex("^mozilla/\\d\\.\\d \\w*$"), - botRegex("^navermailapp"), - botRegex("^netsurf"), - botRegex("^offline explorer"), - botRegex("^php"), - botRegex("^postman"), - botRegex("^postrank"), - botRegex("^python"), - botRegex("^read"), - botRegex("^reed"), - botRegex("^restsharp/"), - botRegex("^snapchat"), - botRegex("^space bison"), - botRegex("^svn"), - botRegex("^swcd "), - botRegex("^taringa"), - botRegex("^test certificate info"), - botRegex("^thumbor/"), - botRegex("^tumblr/"), - botRegex("^user-agent:mozilla"), - botRegex("^valid"), - botRegex("^venus/fedoraplanet"), - botRegex("^w3c"), - botRegex("^webbandit/"), - botRegex("^webcopier"), - botRegex("^wget"), - botRegex("^whatsapp"), - botRegex("^xenu link sleuth"), - botRegex("^yahoo"), - botRegex("^yandex"), - botRegex("^zdm/\\d"), - botRegex("^zoom marketplace/"), - botRegex("^\\{\\{.*}}$"), - botRegex("adbeat\\.com"), - botRegex("appinsights"), - botRegex("archive"), - botRegex("ask jeeves/teoma"), - botRegex("bit\\.ly/"), - botRegex("bluecoat drtr"), - botRegex("bot"), - botRegex("browsex"), - botRegex("burpcollaborator"), - botRegex("capture"), - botRegex("catch"), - botRegex("check"), - botRegex("chrome-lighthouse"), - botRegex("chromeframe"), - botRegex("cloud"), - botRegex("crawl"), - botRegex("cryptoapi"), - botRegex("dareboost"), - botRegex("datanyze"), - botRegex("dataprovider"), - botRegex("dejaclick"), - botRegex("dmbrowser"), - botRegex("download"), - botRegex("evc-batch/"), - botRegex("feed"), - botRegex("firephp"), - botRegex("freesafeip"), - botRegex("ghost"), - botRegex("gomezagent"), - botRegex("google"), - botRegex("headlesschrome/"), - botRegex("http"), - botRegex("httrack"), - botRegex("hubspot marketing grader"), - botRegex("hydra"), - botRegex("ibisbrowser"), - botRegex("images"), - botRegex("iplabel"), - botRegex("ips-agent"), - botRegex("java"), - botRegex("library"), - botRegex("mail\\.ru/"), - botRegex("manager"), - botRegex("monitor"), - botRegex("morningscore/"), - botRegex("neustar wpm"), - botRegex("nutch"), - botRegex("offbyone"), - botRegex("optimize"), - botRegex("pageburst"), - botRegex("pagespeed"), - botRegex("perl"), - botRegex("phantom"), - botRegex("pingdom"), - botRegex("powermarks"), - botRegex("preview"), - botRegex("proxy"), - botRegex("ptst[ /]\\d"), - botRegex("rainmeter webparser plugin"), - botRegex("reader"), - botRegex("rexx;"), - botRegex("rigor"), - botRegex("rss"), - botRegex("scan"), - botRegex("scrape"), - botRegex("search"), - botRegex("serp ?reputation ?management"), - botRegex("server"), - botRegex("sogou"), - botRegex("sparkler/"), - botRegex("speedcurve"), - botRegex("spider"), - botRegex("splash"), - botRegex("statuscake"), - botRegex("stumbleupon\\.com"), - botRegex("supercleaner"), - botRegex("synapse"), - botRegex("synthetic"), - botRegex("taginspector/"), - botRegex("torrent"), - botRegex("tracemyfile"), - botRegex("transcoder"), - botRegex("trendsmapresolver"), - botRegex("twingly recon"), - botRegex("url"), - botRegex("virtuoso"), - botRegex("wappalyzer"), - botRegex("webglance"), - botRegex("webkit2png"), - botRegex("websitemetadataretriever"), - botRegex("whatcms/"), - botRegex("wordpress"), - botRegex("zgrab"), - ) - - fun isRobot(userAgent: String?) = userAgent == null || botRegexes.any { it.containsMatchIn(userAgent) } -}