val jvmMain by getting {
dependencies {
+ implementation(kotlin("stdlib"))
+ implementation(kotlin("stdlib-jdk7"))
+ implementation(kotlin("stdlib-jdk8"))
+ implementation(kotlin("reflect"))
+
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-jvm:1.8.0")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:1.8.0")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-reactive:1.8.0")
implementation("io.ktor:ktor-server-resources:2.3.10")
implementation("io.ktor:ktor-server-sessions-jvm:2.3.10")
implementation("io.ktor:ktor-server-status-pages:2.3.10")
+ implementation("io.ktor:ktor-server-websockets:2.3.10")
implementation("io.ktor:ktor-serialization-kotlinx-json:2.3.10")
implementation("org.slf4j:slf4j-api:2.0.7")
implementation("ch.qos.logback:logback-classic:1.4.14")
+ implementation("com.aallam.ktoken:ktoken:0.3.0")
+
+ implementation("io.ktor:ktor-client-core:2.3.10")
+ implementation("io.ktor:ktor-client-java:2.3.10")
+ implementation("io.ktor:ktor-client-auth:2.3.10")
+ implementation("io.ktor:ktor-client-content-negotiation:2.3.10")
+ implementation("io.ktor:ktor-client-logging:2.3.10")
+
implementation(project(":fontparser"))
//implementation(project(":fightgame"))
}
data object GridFs : FileStorageConfig()
}
+@Serializable
+data class OpenAiConfig(
+ val token: String,
+ val orgId: String,
+ val project: String? = null,
+)
+
@Serializable
data class Configuration(
val host: String = "127.0.0.1",
val dbConn: String = "mongodb://localhost:27017",
val ownerNation: String = "mechyrdia",
+
+ val openAi: OpenAiConfig? = null,
) {
companion object {
val Current: Configuration by lazy {
import info.mechyrdia.auth.*
import info.mechyrdia.data.*
import info.mechyrdia.lore.*
+import info.mechyrdia.robot.JsonRobotCodec
+import info.mechyrdia.robot.RobotService
import info.mechyrdia.route.*
import io.ktor.http.*
import io.ktor.http.content.*
import io.ktor.server.routing.*
import io.ktor.server.sessions.*
import io.ktor.server.sessions.serialization.*
+import io.ktor.server.websocket.*
import org.slf4j.event.Level
import java.io.IOException
import java.util.concurrent.atomic.AtomicLong
FileStorage.initialize()
+ RobotService.initialize()
+
embeddedServer(CIO, port = Configuration.Current.port, host = Configuration.Current.host, module = Application::factbooks).start(wait = true)
}
}
}
+ install(WebSockets) {
+ pingPeriodMillis = 500L
+ timeoutMillis = 5000L
+
+ contentConverter = KotlinxWebsocketSerializationConverter(JsonRobotCodec)
+ }
+
routing {
staticResources("/static", "static", index = null) {
preCompressed(CompressedFileType.BROTLI, CompressedFileType.GZIP)
get<Root.Auth.LoginPage>()
post<Root.Auth.LoginPost, _>()
post<Root.Auth.LogoutPost, _>()
+ get<Root.Nuke>()
+ ws<Root.Nuke.WS>()
get<Root.Comments.HelpPage>()
get<Root.Comments.RecentPage>()
post<Root.Comments.NewPost, _>()
import org.bson.codecs.kotlinx.BsonEncoder
import org.bson.types.ObjectId
import java.time.Instant
+import kotlin.math.absoluteValue
object IdCodec : Codec<Id<*>> {
override fun getEncoderClass(): Class<Id<*>> {
}
}
+fun Instant.toSecondString(): String {
+ val (isNegative, wholeS, fracS) = if (epochSecond < 0 && nano > 0) {
+ Triple(true, epochSecond.absoluteValue - 1, 1_000_000_000 - nano)
+ } else Triple(epochSecond < 0, epochSecond.absoluteValue, nano)
+
+ val sign = if (isNegative) "-" else ""
+
+ val whole = wholeS.toString()
+ val frac = fracS.toString().padStart(9, '0').trimEnd('0')
+
+ return if (frac.isEmpty())
+ "$sign$whole"
+ else
+ "$sign$whole.$frac"
+}
+
+private val instantSecondRegex = Regex("([+-]?)([0-9]+)(?:\\.([0-9]{1,9}))?")
+
+fun String.toSecondInstant() = toSecondInstantOrNull() ?: throw IllegalArgumentException("String given to toSecondInstant must match regex /${instantSecondRegex.pattern}/, got $this")
+
+fun String.toSecondInstantOrNull(): Instant? {
+ val matchResult = instantSecondRegex.matchEntire(this) ?: return null
+ val (signStr, wholeStr, fracStr) = matchResult.destructured
+ val isNegative = signStr == "-"
+
+ val wholeS = wholeStr.toLong()
+ val fracS = if (fracStr.isEmpty()) 0 else fracStr.toInt()
+
+ val (seconds, nanos) = if (isNegative) {
+ if (fracS > 0)
+ (-wholeS - 1) to (1_000_000_000 - fracS)
+ else -wholeS to 0
+ } else wholeS to fracS
+
+ return Instant.ofEpochSecond(seconds, nanos.toLong())
+}
+
object InstantSerializer : KSerializer<Instant> {
- override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("InstantSerializer", PrimitiveKind.LONG)
+ override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("InstantSerializer", PrimitiveKind.STRING)
override fun serialize(encoder: Encoder, value: Instant) {
- if (encoder !is BsonEncoder)
- throw SerializationException("Instant is not supported by ${encoder::class}")
-
- encoder.encodeBsonValue(BsonDateTime(value.toEpochMilli()))
+ if (encoder is BsonEncoder)
+ encoder.encodeBsonValue(BsonDateTime(value.toEpochMilli()))
+ else
+ encoder.encodeString(value.toSecondString())
}
override fun deserialize(decoder: Decoder): Instant {
- if (decoder !is BsonDecoder)
- throw SerializationException("Instant is not supported by ${decoder::class}")
-
- return Instant.ofEpochMilli(decoder.decodeBsonValue().asDateTime().value)
+ return if (decoder is BsonDecoder)
+ Instant.ofEpochMilli(decoder.decodeBsonValue().asDateTime().value)
+ else
+ decoder.decodeString().toSecondInstant()
}
}
import com.mongodb.reactivestreams.client.gridfs.GridFSBuckets
import info.mechyrdia.auth.SessionStorageDoc
import info.mechyrdia.auth.WebDavToken
+import info.mechyrdia.robot.RobotGlobals
import kotlinx.coroutines.CompletableDeferred
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.singleOrNull
WebDavToken,
Comment,
CommentReplyLink,
- PageVisitData
+ PageVisitData,
+ RobotGlobals,
)
}
}
val CallNationCacheAttribute = AttributeKey<MutableMap<Id<NationData>, NationData>>("Mechyrdia.NationCache")
val ApplicationCall.nationCache: MutableMap<Id<NationData>, NationData>
- get() = attributes.getOrNull(CallNationCacheAttribute)
- ?: ConcurrentHashMap<Id<NationData>, NationData>().also { cache ->
+ get() = attributes.computeIfAbsent(CallNationCacheAttribute) {
+ ConcurrentHashMap<Id<NationData>, NationData>().also { cache ->
attributes.put(CallNationCacheAttribute, cache)
}
+ }
suspend fun MutableMap<Id<NationData>, NationData>.getNation(id: Id<NationData>): NationData {
return getOrPut(id) {
suspend fun ApplicationCall.currentNation(): NationData? {
attributes.getOrNull(CallCurrentNationAttribute)?.let { sess ->
- return when (sess) {
- NationSession.Anonymous -> null
- is NationSession.LoggedIn -> sess.nation
- }
+ return sess.nation
}
- val nationId = sessions.get<UserSession>()?.nationId
- return if (nationId == null) {
- attributes.put(CallCurrentNationAttribute, NationSession.Anonymous)
- null
- } else nationCache.getNation(nationId).also { data ->
- attributes.put(CallCurrentNationAttribute, NationSession.LoggedIn(data))
- }
+ return sessions.get<UserSession>()
+ ?.nationId
+ ?.let { nationCache.getNation(it) }
+ ?.also { attributes.put(CallCurrentNationAttribute, NationSession(it)) }
}
+private fun NationSession(nation: NationData?) = if (nation == null)
+ NationSession.Anonymous
+else
+ NationSession.LoggedIn(nation)
+
private sealed class NationSession {
- data object Anonymous : NationSession()
+ abstract val nation: NationData?
+
+ data object Anonymous : NationSession() {
+ override val nation: NationData?
+ get() = null
+ }
- data class LoggedIn(val nation: NationData) : NationSession()
+ data class LoggedIn(override val nation: NationData) : NationSession()
}
import java.time.Month
import java.time.ZoneId
-private val myTimeZone: ZoneId = ZoneId.of("America/New_York")
+val MyTimeZone: ZoneId = ZoneId.of("America/New_York")
fun isApril1st(time: Instant = Instant.now()): Boolean {
- val zonedDateTime = time.atZone(myTimeZone)
+ val zonedDateTime = time.atZone(MyTimeZone)
return zonedDateTime.month == Month.APRIL && zonedDateTime.dayOfMonth == 1
}
private val cache = ConcurrentHashMap<StoragePath, Entry>()
private suspend fun Entry(path: StoragePath) = cacheLock.withLock {
- cache.computeIfAbsent(path) {
+ cache.getOrPut(path) {
Entry(null, null)
}
}
import kotlinx.html.org.w3c.dom.events.Event
import kotlinx.html.stream.createHTML
import kotlinx.serialization.json.JsonPrimitive
+import java.time.Instant
import kotlin.text.toCharArray
typealias HtmlBuilderContext = Unit
)
override fun processInvalidTag(env: LexerTagEnvironment<HtmlBuilderContext, HtmlBuilderSubject>, tag: String, param: String?, subNodes: ParserTree): HtmlBuilderSubject {
+ val content = env.processTree(subNodes)
+
return {
+if (param == null) "[$tag]" else "[$tag=$param]"
- env.processTree(subNodes)()
+ content()
+"[/$tag]"
}
}
val NON_ANCHOR_CHAR = Regex("[^a-zA-Z\\d\\-]+")
fun String.sanitizeAnchor() = replace(NON_ANCHOR_CHAR, "-")
-fun ParserTree.treeToAnchorText(): String = treeToText().sanitizeAnchor()
-
-class HtmlHeaderLexerTag(val tagCreator: TagCreator, val anchor: (ParserTree) -> String?) : HtmlLexerTag {
+class HtmlHeaderLexerTag(val tagCreator: TagCreator, val anchor: (String) -> String?) : HtmlLexerTag {
override fun processTag(env: LexerTagEnvironment<HtmlBuilderContext, HtmlBuilderSubject>, param: String?, subNodes: ParserTree): HtmlBuilderSubject {
+ val content = subNodes.treeToText()
+ val anchorId = anchor(content)
+ val anchorHash = anchorId?.let { "#$it" }.orEmpty()
+
return {
- val anchorId = anchor(subNodes)
-
anchorId?.let { a { id = it } }
+
tagCreator {
- +subNodes.treeToText()
+ +content
}
- val anchorHash = anchorId?.let { "#$it" }.orEmpty()
script { unsafe { +"window.checkRedirectTarget(\"$anchorHash\");" } }
}
}
?.let { mapOf("style" to "color:#$it") }
.orEmpty()
-private val VALID_ALIGNMENTS = mapOf(
+fun <V> uncasedMapOf(vararg pairs: Pair<String, V>): Map<String, V> = buildMap {
+ pairs.associateTo(this) { (k, v) ->
+ k.lowercase() to v
+ }
+}
+
+fun <V> Map<String, V>.getUncased(key: String): V? = get(key.lowercase())
+
+private val VALID_ALIGNMENTS = uncasedMapOf(
"left" to "text-align:left",
"right" to "text-align:right",
"center" to "text-align:center",
fun processAlign(param: String?): Map<String, String> = param
?.lowercase()
- ?.let { VALID_ALIGNMENTS[it] }
+ ?.let { VALID_ALIGNMENTS.getUncased(it) }
?.let { mapOf("style" to it) }
.orEmpty()
-private val VALID_FLOATS = mapOf(
+private val VALID_FLOATS = uncasedMapOf(
"left" to "float:left;max-width:var(--aside-width)",
"right" to "float:right;max-width:var(--aside-width)",
)
fun processFloat(param: String?): Map<String, String> = param
?.lowercase()
- ?.let { VALID_FLOATS[it] }
+ ?.let { VALID_FLOATS.getUncased(it) }
?.let { mapOf("style" to it) }
.orEmpty()
ERROR(HtmlTagLexerTag(attributes = mapOf("style" to "color: #f00"), tagCreator = TagConsumer<*>::div.toTagCreator())),
H1(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h1.toTagCreator()) { null }),
- H2(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h2.toTagCreator(), ParserTree::treeToAnchorText)),
- H3(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h3.toTagCreator(), ParserTree::treeToAnchorText)),
- H4(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h4.toTagCreator(), ParserTree::treeToAnchorText)),
- H5(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h5.toTagCreator(), ParserTree::treeToAnchorText)),
- H6(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h6.toTagCreator(), ParserTree::treeToAnchorText)),
+ H2(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h2.toTagCreator(), String::sanitizeAnchor)),
+ H3(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h3.toTagCreator(), String::sanitizeAnchor)),
+ H4(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h4.toTagCreator(), String::sanitizeAnchor)),
+ H5(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h5.toTagCreator(), String::sanitizeAnchor)),
+ H6(HtmlHeaderLexerTag(tagCreator = TagConsumer<*>::h6.toTagCreator(), String::sanitizeAnchor)),
ALIGN(HtmlTagLexerTag(attributes = ::processAlign, tagMode = HtmlTagMode.ITEM, tagCreator = TagConsumer<*>::div.toTagCreator())),
ASIDE(HtmlTagLexerTag(attributes = ::processFloat, tagMode = HtmlTagMode.ITEM, tagCreator = TagConsumer<*>::div.toTagCreator())),
TH(HtmlTagLexerTag(attributes = ::processTableCell, tagMode = HtmlTagMode.ITEM, tagCreator = TagConsumer<*>::th.toTagCreator())),
MOMENT(HtmlTextBodyLexerTag { _, _, content ->
- val epochMilli = content.toLongOrNull()
- if (epochMilli == null)
+ val instant = content.toLongOrNull()?.let { Instant.ofEpochMilli(it) }
+ if (instant == null)
({ +content })
else
- ({
- span(classes = "moment") {
- style = "display:none"
- +"$epochMilli"
- }
- })
+ ({ dateTime(instant) })
}),
LINK(HtmlTagLexerTag(attributes = ::processInternalLink, tagMode = HtmlTagMode.INLINE, tagCreator = TagConsumer<*>::a.toTagCreator())),
EXTLINK(HtmlTagLexerTag(attributes = ::processExternalLink, tagMode = HtmlTagMode.INLINE, tagCreator = TagConsumer<*>::a.toTagCreator())),
).processTree(this)
}
+class HtmlCommentImageLexerTag(val domain: String) : HtmlLexerTag {
+ override fun processTag(env: LexerTagEnvironment<HtmlBuilderContext, HtmlBuilderSubject>, param: String?, subNodes: ParserTree): HtmlBuilderSubject {
+ val imageUrl = processCommentImage(subNodes.treeToText(), domain)
+ val (width, height) = getSizeParam(param)
+ val sizeStyle = getImageSizeStyleValue(width, height)
+
+ return {
+ img(src = imageUrl) { style = sizeStyle }
+ }
+ }
+}
+
enum class CommentFormattingTag(val type: HtmlLexerTag) {
B(FactbookFormattingTag.B.type),
I(FactbookFormattingTag.I.type),
LANG(FactbookFormattingTag.LANG.type),
- IMGBB(HtmlTextBodyLexerTag { _, tagParam, content ->
- val imageUrl = processCommentImage(content, "i.ibb.co")
- val (width, height) = getSizeParam(tagParam)
- val sizeStyle = getImageSizeStyleValue(width, height)
-
- ({
- img(src = imageUrl) { style = sizeStyle }
- })
- }),
+ IMGBB(HtmlCommentImageLexerTag("i.ibb.co")),
REPLY(HtmlTextBodyLexerTag { _, _, content ->
val id = sanitizeId(content)
+++ /dev/null
-package info.mechyrdia.lore
-
-import info.mechyrdia.route.KeyedEnumSerializer
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.SerialName
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.descriptors.PrimitiveKind
-import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
-import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.encoding.Decoder
-import kotlinx.serialization.encoding.Encoder
-
-@Serializable(with = DocTextColorSerializer::class)
-@JvmInline
-value class DocTextColor(val rgb: Int) {
- constructor(rgb: String) : this(fromStringOrNull(rgb) ?: error("Expected string of 3 or 6 hex digits with optional # prefix, got $rgb"))
-
- override fun toString(): String {
- return "#${rgb.toString(16).padStart(6, '0')}"
- }
-
- companion object {
- fun fromStringOrNull(rgb: String): Int? {
- return repeatColorDigits(rgb.removePrefix("#"))?.toIntOrNull(16)
- }
- }
-}
-
-object DocTextColorSerializer : KSerializer<DocTextColor> {
- override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("DocTextColorSerializer", PrimitiveKind.STRING)
-
- override fun serialize(encoder: Encoder, value: DocTextColor) {
- encoder.encodeString(value.toString())
- }
-
- override fun deserialize(decoder: Decoder): DocTextColor {
- return DocTextColor(decoder.decodeString())
- }
-}
-
-@Serializable(with = DocTextFontSerializer::class)
-enum class DocTextFont {
- NORMAL, CODE, IPA
-}
-
-object DocTextFontSerializer : KeyedEnumSerializer<DocTextFont>(DocTextFont.entries)
-
-@Serializable
-data class DocTextFormat(
- val isBold: Boolean = false,
- val isItalic: Boolean = false,
- val isUnderline: Boolean = false,
- val isStrikeOut: Boolean = false,
- val isSubscript: Boolean = false,
- val isSuperscript: Boolean = false,
- val color: DocTextColor? = null,
- val font: DocTextFont = DocTextFont.NORMAL
-)
-
-@Serializable
-data class DocText(
- val text: String,
- val format: DocTextFormat = DocTextFormat(),
-)
-
-@Serializable
-sealed class DocLayoutItem {
- @Serializable
- @SerialName("textLine")
- data class TextLine(val text: List<DocText>) : DocBlock()
-
- @Serializable
- @SerialName("formatBlock")
- data class FormatBlock(val blocks: List<DocBlock>) : DocBlock()
-}
-
-@Serializable(with = ListingTypeSerializer::class)
-enum class ListingType {
- ORDERED,
- UNORDERED,
-}
-
-object ListingTypeSerializer : KeyedEnumSerializer<ListingType>(ListingType.entries)
-
-@Serializable
-@JvmInline
-value class DocTableRow(
- val cells: List<DocTableCell>
-)
-
-@Serializable
-data class DocTableCell(
- val isHeading: Boolean = false,
- val colSpan: Int = 1,
- val rowSpan: Int = 1,
- val contents: DocLayoutItem
-)
-
-@Serializable
-sealed class DocBlock {
- @Serializable
- @SerialName("paragraph")
- data class Paragraph(val contents: List<DocText>) : DocBlock()
-
- @Serializable
- @SerialName("list")
- data class Listing(val ordering: ListingType, val items: List<DocLayoutItem>) : DocBlock()
-
- @Serializable
- @SerialName("table")
- data class Table(val items: List<DocTableRow>) : DocBlock()
-}
-
-@Serializable
-data class DocSections(
- val headingText: String,
- val headContent: List<DocBlock>,
- val subSections: List<DocSections>,
-)
-
-@Serializable
-data class Document(
- val ogData: OpenGraphData?,
- val sections: DocSections,
-)
fun defaults(lorePath: List<String>) = mapOf(
PAGE_PATH_KEY to "/${lorePath.joinToString(separator = "/")}".textToTree(),
- INSTANT_NOW_KEY to Instant.now().toEpochMilli().toString().textToTree(),
+ INSTANT_NOW_KEY to Instant.now().toEpochMilli().numberToTree(),
)
}
}
it.param to it.subNodes
}.toMap()
-suspend fun <T, R> List<T>.mapSuspend(processor: suspend (T) -> R) = coroutineScope {
+suspend fun <T, R> Iterable<T>.mapSuspend(processor: suspend (T) -> R) = coroutineScope {
map {
async {
processor(it)
val digest = hex(hasher.get().digest(script))
return withContext(Dispatchers.IO) {
- cache.computeIfAbsent(digest) { _ ->
+ cache.getOrPut(digest) {
(scriptEngine.get() as Compilable).compile(String(script))
}
}
}
object FactbookLoader {
- suspend fun loadJsonData(lorePath: List<String>): JsonObject {
+ private suspend fun loadJsonData(lorePath: List<String>): JsonObject {
val jsonPath = lorePath.dropLast(1) + listOf("${lorePath.last()}.json")
val bytes = FileStorage.instance.readFile(StoragePath.jsonDocDir / jsonPath) ?: return JsonObject(emptyMap())
return JsonStorageCodec.parseToJsonElement(String(bytes)) as JsonObject
--- /dev/null
+package info.mechyrdia.lore
+
+import info.mechyrdia.robot.toOpenAiName
+import java.time.Instant
+
+fun String.toRobotUrl(context: RobotTextContext): String {
+ val filePath = if (startsWith("/"))
+ this.removePrefix("/")
+ else
+ context.siblingFile(this).joinToString(separator = "/")
+
+ return filePath.toOpenAiName()
+}
+
+class RobotTextContext(val currentPath: List<String>) {
+ fun siblingFile(file: String) = currentPath.dropLast(1) + file
+}
+
+typealias RobotTextSubject = String
+
+object RobotTextLexerProcessor : LexerTagFallback<RobotTextContext, RobotTextSubject>, LexerTextProcessor<RobotTextContext, RobotTextSubject>, LexerLineBreakProcessor<RobotTextContext, RobotTextSubject>, LexerCombiner<RobotTextContext, RobotTextSubject> {
+ override fun processInvalidTag(env: LexerTagEnvironment<RobotTextContext, RobotTextSubject>, tag: String, param: String?, subNodes: ParserTree): RobotTextSubject {
+ return env.processTree(subNodes)
+ }
+
+ override fun processText(env: LexerTagEnvironment<RobotTextContext, RobotTextSubject>, text: String): RobotTextSubject {
+ return text
+ }
+
+ override fun processLineBreak(env: LexerTagEnvironment<RobotTextContext, RobotTextSubject>): RobotTextSubject {
+ return " "
+ }
+
+ override fun combine(env: LexerTagEnvironment<RobotTextContext, RobotTextSubject>, subjects: List<RobotTextSubject>): RobotTextSubject {
+ return subjects.joinToString(separator = "")
+ }
+}
+
+fun interface RobotTextTag : LexerTagProcessor<RobotTextContext, RobotTextSubject>
+
+object RobotTextEmptyTag : RobotTextTag {
+ override fun processTag(env: LexerTagEnvironment<RobotTextContext, RobotTextSubject>, param: String?, subNodes: ParserTree): RobotTextSubject {
+ return ""
+ }
+}
+
+enum class FactbookRobotFormattingTag(val type: RobotTextTag) {
+ B(RobotTextTag { env, _, subNodes ->
+ "**${env.processTree(subNodes)}**"
+ }),
+ I(RobotTextTag { env, _, subNodes ->
+ "*${env.processTree(subNodes)}*"
+ }),
+ U(RobotTextTag { env, _, subNodes ->
+ "__${env.processTree(subNodes)}__"
+ }),
+ S(RobotTextTag { env, _, subNodes ->
+ "~~${env.processTree(subNodes)}~~"
+ }),
+ SUP(RobotTextTag { env, _, subNodes ->
+ "^(${env.processTree(subNodes)})"
+ }),
+ SUB(RobotTextTag { env, _, subNodes ->
+ "_(${env.processTree(subNodes)})"
+ }),
+ BLOCKQUOTE(RobotTextTag { env, _, subNodes ->
+ ">>${env.processTree(subNodes)}<<"
+ }),
+
+ H1(RobotTextTag { env, _, subNodes -> "=${env.processTree(subNodes)}=" }),
+ H2(RobotTextTag { env, _, subNodes -> "==${env.processTree(subNodes)}==" }),
+ H3(RobotTextTag { env, _, subNodes -> "===${env.processTree(subNodes)}===" }),
+ H4(RobotTextTag { env, _, subNodes -> "====${env.processTree(subNodes)}====" }),
+ H5(RobotTextTag { env, _, subNodes -> "=====${env.processTree(subNodes)}=====" }),
+ H6(RobotTextTag { env, _, subNodes -> "======${env.processTree(subNodes)}======" }),
+
+ THUMB(RobotTextEmptyTag),
+
+ IMAGE(RobotTextTag { _, _, _ ->
+ "(image)"
+ }),
+ MODEL(RobotTextTag { _, _, _ ->
+ "(3D model)"
+ }),
+ AUDIO(RobotTextTag { _, _, _ ->
+ "(audio)"
+ }),
+ QUIZ(RobotTextTag { _, _, _ ->
+ "(quiz)"
+ }),
+
+ UL(RobotTextTag { env, _, subNodes ->
+ subNodes
+ .mapNotNull { subNode ->
+ if (subNode is ParserTreeNode.Tag && subNode isTag "li")
+ " * ${env.processTree(subNode.subNodes)}"
+ else null
+ }.joinToString(separator = "")
+ }),
+ OL(RobotTextTag { env, _, subNodes ->
+ subNodes
+ .mapIndexedNotNull { i, subNode ->
+ if (subNode is ParserTreeNode.Tag && subNode isTag "li")
+ " ${i + 1}. ${env.processTree(subNode.subNodes)}"
+ else null
+ }.joinToString(separator = "")
+ }),
+
+ TABLE(RobotTextTag { env, _, subNodes ->
+ "(table)${env.processTree(subNodes)} ---"
+ }),
+ TR(RobotTextTag { env, _, subNodes ->
+ " ---${env.processTree(subNodes)} |"
+ }),
+ TD(RobotTextTag { env, _, subNodes ->
+ " | ${env.processTree(subNodes)}"
+ }),
+ TH(RobotTextTag { env, _, subNodes ->
+ " | **${env.processTree(subNodes)}**"
+ }),
+
+ MOMENT(RobotTextTag { env, _, subNodes ->
+ val instant = subNodes.treeToNumberOrNull(String::toLongOrNull)?.let { Instant.ofEpochMilli(it) }
+ instant?.toString() ?: env.processTree(subNodes)
+ }),
+ LINK(RobotTextTag { env, param, subNodes ->
+ env.processTree(subNodes) + param
+ ?.sanitizeLink()
+ ?.toRobotUrl(env.context)
+ ?.let { " <$it>" }
+ .orEmpty()
+ }),
+ EXTLINK(RobotTextTag { env, param, subNodes ->
+ env.processTree(subNodes) + param
+ ?.sanitizeLink()
+ ?.toExternalUrl()
+ ?.let { " <$it>" }
+ .orEmpty()
+ }),
+ ANCHOR(RobotTextEmptyTag),
+ REDIRECT(RobotTextTag { env, _, subNodes ->
+ val target = subNodes.treeToText()
+ .sanitizeLink()
+ .toRobotUrl(env.context)
+ "(redirect) <$target>"
+ }),
+ LANG(RobotTextTag { env, param, subNodes ->
+ val langName = if ("tylan".equals(param, ignoreCase = true))
+ "Tylan"
+ else if ("thedish".equals(param, ignoreCase = true))
+ "Thedish"
+ else if ("kishari".equals(param, ignoreCase = true))
+ "Kishari"
+ else if ("pokhval".equals(param, ignoreCase = true) || "pochval".equals(param, ignoreCase = true) || "pokhwal".equals(param, ignoreCase = true))
+ "Pokhwalish"
+ else null
+ val prefix = langName?.let { "(in $it) " }.orEmpty()
+ "$prefix*${env.processTree(subNodes)}*"
+ }),
+ ALPHABET(RobotTextTag { _, param, _ ->
+ if ("mechyrdian".equals(param, ignoreCase = true))
+ "(preview of Mechyrdia Sans font)"
+ else if ("tylan".equals(param, ignoreCase = true))
+ "(preview of Tylan abugida font)"
+ else if ("thedish".equals(param, ignoreCase = true))
+ "(preview of Thedish alphabet font)"
+ else if ("kishari".equals(param, ignoreCase = true))
+ "(preview of Kishari runic alphabet font)"
+ else if ("pokhval".equals(param, ignoreCase = true) || "pochval".equals(param, ignoreCase = true) || "pokhwal".equals(param, ignoreCase = true))
+ "(preview of Pokhwalish alphabet font)"
+ else ""
+ }),
+ VOCAB(RobotTextTag { _, _, _ ->
+ "(searchable dictionary of foreign vocabulary)"
+ }),
+ ;
+
+ companion object {
+ val asTags = LexerTags(entries.associate { it.name to it.type })
+ }
+}
+
+object RobotFactbookLoader {
+ private fun ParserTree.toFactbookRobotText(currentPath: List<String>): String {
+ val context = RobotTextContext(currentPath)
+ val content = LexerTagEnvironment(
+ context,
+ FactbookRobotFormattingTag.asTags,
+ RobotTextLexerProcessor,
+ RobotTextLexerProcessor,
+ RobotTextLexerProcessor,
+ RobotTextLexerProcessor,
+ ).processTree(this)
+
+ return content
+ }
+
+ suspend fun loadAllFactbooks(): Map<String, String> {
+ return allPages().mapSuspend { pathStat ->
+ val lorePath = pathStat.path.elements.drop(1)
+ FactbookLoader.loadFactbook(lorePath)?.toFactbookRobotText(lorePath)?.let { robotText ->
+ lorePath.joinToString(separator = "/") to robotText
+ }
+ }.filterNotNull().toMap()
+ }
+
+ suspend fun loadAllFactbooksSince(lastUpdated: Instant): Map<String, String> {
+ return allPages().mapSuspend { pathStat ->
+ if (pathStat.stat.updated >= lastUpdated) {
+ val lorePath = pathStat.path.elements.drop(1)
+ FactbookLoader.loadFactbook(lorePath)?.toFactbookRobotText(lorePath)?.let { robotText ->
+ lorePath.joinToString(separator = "/") to robotText
+ }
+ } else null
+ }.filterNotNull().toMap()
+ }
+}
private fun endText() {
if (currentString.isEmpty()) return
- nodes.add(ParserTreeNode.Text(currentString.toString().replace('\n', ' ')))
+ nodes.add(ParserTreeNode.Text(currentString.toString()))
currentString.clear()
}
nodes.add(ParserTreeNode.LineBreak)
}
- fun endDoc(): ParserTree {
+ open fun endDoc(): ParserTree {
endText()
return nodes
}
private val tag: String,
private val param: String? = null
) : ParserTreeBuilderState() {
+ override fun endDoc(): ParserTree {
+ return endTag().endDoc()
+ }
+
override fun canEndTag(endTag: String): TreeTag? {
return if (tag.equals(endTag, ignoreCase = true)) this else null
}
fun endTag(): ParserTreeBuilderState {
- return parent.doneTag(ParserTreeNode.Tag(tag, param, endDoc()))
+ return parent.doneTag(ParserTreeNode.Tag(tag, param, super.endDoc()))
}
}
}
-sealed class ParserState(
- protected val builder: ParserTreeBuilderState
+sealed class ParserStreamEvent {
+ data class TagStart(val tag: String, val param: String?) : ParserStreamEvent()
+ data class TagEnd(val tag: String) : ParserStreamEvent()
+ data class CData(val text: String) : ParserStreamEvent()
+ data object ParaBreak : ParserStreamEvent()
+ data object EndOfFile : ParserStreamEvent()
+}
+
+fun interface ParserStreamHandler {
+ fun handleEvent(event: ParserStreamEvent)
+}
+
+class ParserStreamTreeBuilder : ParserStreamHandler {
+ private var builderState: ParserTreeBuilderState? = ParserTreeBuilderState.TreeRoot()
+ private var result: ParserTree? = null
+
+ fun getAndReset(): ParserTree {
+ val done = result ?: error("Attempting to reset ParserStreamTreeBuilder before document has ended")
+ builderState = ParserTreeBuilderState.TreeRoot()
+ return done
+ }
+
+ override fun handleEvent(event: ParserStreamEvent) {
+ val state = builderState ?: error("Attempting to use ParserStreamTreeBuilder after document has ended")
+
+ builderState = when (event) {
+ is ParserStreamEvent.TagStart -> state
+ .beginTag(event.tag, event.param)
+
+ is ParserStreamEvent.TagEnd -> state
+ .canEndTag(event.tag)
+ ?.endTag()
+ ?: state.apply { text("[/${event.tag}]") }
+
+ is ParserStreamEvent.CData -> state
+ .apply { text(event.text) }
+
+ ParserStreamEvent.ParaBreak -> state
+ .apply { newLine() }
+
+ ParserStreamEvent.EndOfFile -> {
+ result = state.endDoc()
+ null
+ }
+ }
+ }
+}
+
+sealed class ParserState<THandler : ParserStreamHandler, TResult>(
+ protected val handler: THandler,
+ protected val resultGetter: THandler.() -> TResult
) {
- abstract fun processCharacter(char: Char): ParserState
- open fun processEndOfText(): ParserTree = builder.unwind()
+ abstract fun processCharacter(char: Char): ParserState<THandler, TResult>
+ open fun processEndOfText(): TResult {
+ handler.handleEvent(ParserStreamEvent.EndOfFile)
+ return handler.resultGetter()
+ }
- class Initial : ParserState(ParserTreeBuilderState.TreeRoot()) {
- override fun processCharacter(char: Char): ParserState {
+ protected fun processedCDataEvent(raw: String) = ParserStreamEvent.CData(raw.replace("\n", ""))
+
+ class Initial<THandler : ParserStreamHandler, TResult>(handler: THandler, resultGetter: THandler.() -> TResult) : ParserState<THandler, TResult>(handler, resultGetter) {
+ override fun processCharacter(char: Char): ParserState<THandler, TResult> {
return if (char == '[')
- OpenTag("", builder)
+ OpenTag("", handler, resultGetter)
else
- PlainText("$char", builder)
- }
-
- override fun processEndOfText(): ParserTree {
- return emptyList()
+ PlainText("$char", handler, resultGetter)
}
}
- class PlainText(private val text: String, builder: ParserTreeBuilderState) : ParserState(builder) {
- override fun processCharacter(char: Char): ParserState {
+ class PlainText<THandler : ParserStreamHandler, TResult>(private val text: String, handler: THandler, resultGetter: THandler.() -> TResult) : ParserState<THandler, TResult>(handler, resultGetter) {
+ override fun processCharacter(char: Char): ParserState<THandler, TResult> {
return if (char == '[') {
- builder.text(text)
- OpenTag("", builder)
+ handler.handleEvent(processedCDataEvent(text))
+ OpenTag("", handler, resultGetter)
} else if (char == '\n' && text.endsWith('\n')) {
- builder.text(text.removeSuffix("\n"))
- builder.newLine()
+ handler.handleEvent(processedCDataEvent(text))
+ handler.handleEvent(ParserStreamEvent.ParaBreak)
- PlainText("", builder)
- } else PlainText("$text$char", builder)
+ PlainText("", handler, resultGetter)
+ } else PlainText("$text$char", handler, resultGetter)
}
- override fun processEndOfText(): ParserTree {
- builder.text(text)
+ override fun processEndOfText(): TResult {
+ handler.handleEvent(processedCDataEvent(text))
return super.processEndOfText()
}
}
- class NoFormatText(private val text: String, builder: ParserTreeBuilderState) : ParserState(builder) {
- override fun processCharacter(char: Char): ParserState {
+ class NoFormatText<THandler : ParserStreamHandler, TResult>(private val text: String, handler: THandler, resultGetter: THandler.() -> TResult) : ParserState<THandler, TResult>(handler, resultGetter) {
+ override fun processCharacter(char: Char): ParserState<THandler, TResult> {
return if (char == '\n' && text.endsWith('\n')) {
- builder.text(text.removeSuffix("\n"))
- builder.newLine()
+ handler.handleEvent(processedCDataEvent(text))
+ handler.handleEvent(ParserStreamEvent.ParaBreak)
- NoFormatText("", builder)
+ NoFormatText("", handler, resultGetter)
} else {
val newText = "$text$char"
val endTag = "[/$NO_FORMAT_TAG]"
if (newText.endsWith(endTag, ignoreCase = true)) {
- builder.text(newText.substring(0, newText.length - endTag.length))
- PlainText("", builder)
- } else NoFormatText(newText, builder)
+ handler.handleEvent(processedCDataEvent(newText.dropLast(endTag.length)))
+ PlainText("", handler, resultGetter)
+ } else NoFormatText(newText, handler, resultGetter)
}
}
- override fun processEndOfText(): ParserTree {
- builder.text(text)
+ override fun processEndOfText(): TResult {
+ handler.handleEvent(processedCDataEvent(text))
return super.processEndOfText()
}
}
- class OpenTag(private val tagName: String, builder: ParserTreeBuilderState) : ParserState(builder) {
- override fun processCharacter(char: Char): ParserState {
+ class OpenTag<THandler : ParserStreamHandler, TResult>(private val tagName: String, handler: THandler, resultGetter: THandler.() -> TResult) : ParserState<THandler, TResult>(handler, resultGetter) {
+ override fun processCharacter(char: Char): ParserState<THandler, TResult> {
return if (char == ']') {
if (tagName.equals(NO_FORMAT_TAG, ignoreCase = true))
- NoFormatText("", builder)
- else
- PlainText("", builder.beginTag(tagName, null))
+ NoFormatText("", handler, resultGetter)
+ else {
+ handler.handleEvent(ParserStreamEvent.TagStart(tagName, null))
+ PlainText("", handler, resultGetter)
+ }
} else if (char == '=')
- TagParam(tagName, "", builder)
+ TagParam(tagName, "", handler, resultGetter)
else if (char == '/' && tagName.isEmpty())
- CloseTag("", builder)
+ CloseTag("", handler, resultGetter)
else
- OpenTag("$tagName$char", builder)
+ OpenTag("$tagName$char", handler, resultGetter)
}
- override fun processEndOfText(): ParserTree {
- builder.text("[$tagName")
+ override fun processEndOfText(): TResult {
+ handler.handleEvent(processedCDataEvent("[$tagName"))
return super.processEndOfText()
}
}
- class TagParam(private val tagName: String, private val tagParam: String, builder: ParserTreeBuilderState) : ParserState(builder) {
- override fun processCharacter(char: Char): ParserState {
- return if (char == ']')
- PlainText("", builder.beginTag(tagName, tagParam))
- else
- TagParam(tagName, "$tagParam$char", builder)
+ class TagParam<THandler : ParserStreamHandler, TResult>(private val tagName: String, private val tagParam: String, handler: THandler, resultGetter: THandler.() -> TResult) : ParserState<THandler, TResult>(handler, resultGetter) {
+ override fun processCharacter(char: Char): ParserState<THandler, TResult> {
+ return if (char == ']') {
+ handler.handleEvent(ParserStreamEvent.TagStart(tagName, tagParam))
+ PlainText("", handler, resultGetter)
+ } else
+ TagParam(tagName, "$tagParam$char", handler, resultGetter)
}
- override fun processEndOfText(): ParserTree {
- builder.text("[$tagName=$tagParam")
+ override fun processEndOfText(): TResult {
+ handler.handleEvent(processedCDataEvent("[$tagName=$tagParam"))
return super.processEndOfText()
}
}
- class CloseTag(private val tagName: String, builder: ParserTreeBuilderState) : ParserState(builder) {
- override fun processCharacter(char: Char): ParserState {
- return if (char == ']')
- builder.canEndTag(tagName)?.endTag()?.let {
- PlainText("", it)
- } ?: PlainText("[/$tagName]", builder)
- else CloseTag("$tagName$char", builder)
+ class CloseTag<THandler : ParserStreamHandler, TResult>(private val tagName: String, handler: THandler, resultGetter: THandler.() -> TResult) : ParserState<THandler, TResult>(handler, resultGetter) {
+ override fun processCharacter(char: Char): ParserState<THandler, TResult> {
+ return if (char == ']') {
+ handler.handleEvent(ParserStreamEvent.TagEnd(tagName))
+ PlainText("", handler, resultGetter)
+ } else CloseTag("$tagName$char", handler, resultGetter)
}
- override fun processEndOfText(): ParserTree {
- builder.text("[/$tagName")
+ override fun processEndOfText(): TResult {
+ handler.handleEvent(processedCDataEvent("[/$tagName"))
return super.processEndOfText()
}
}
companion object {
const val NO_FORMAT_TAG = "noformat"
- private fun ParserTreeBuilderState.unwind(): ParserTree {
- return when (this) {
- is ParserTreeBuilderState.TreeRoot -> endDoc()
- is ParserTreeBuilderState.TreeTag -> endTag().unwind()
- }
- }
-
fun parseText(text: String): ParserTree {
val fixedText = text.replace("\r\n", "\n").replace('\r', '\n')
- return fixedText.fold<ParserState>(Initial()) { state, char ->
+ return fixedText.fold(TreeParserState()) { state, char ->
state.processCharacter(char)
}.processEndOfText()
}
}
}
+
+fun TreeParserState(): TreeParserState = ParserState.Initial(ParserStreamTreeBuilder(), ParserStreamTreeBuilder::getAndReset)
+
+typealias TreeParserState = ParserState<ParserStreamTreeBuilder, ParserTree>
import info.mechyrdia.data.FileStorage
import info.mechyrdia.data.StoragePath
import info.mechyrdia.data.currentNation
+import info.mechyrdia.robot.RobotService
+import info.mechyrdia.robot.RobotServiceStatus
import info.mechyrdia.route.Root
import info.mechyrdia.route.createCsrfToken
import info.mechyrdia.route.href
NavLink(href(Root.LorePage(subPath)), (StoragePath.articleDir / subPath).toFriendlyPageTitle())
}
}.orEmpty() + (currentNation()?.let { data ->
- listOf(
+ (if (RobotService.status == RobotServiceStatus.READY)
+ listOf(NavLink(href(Root.Nuke()), "NUKE"))
+ else emptyList()) + listOf(
NavHead(data.name),
NavLink(href(Root.User()), "Your User Page"),
NavLink("https://www.nationstates.net/nation=${data.id}", "Your NationStates Page"),
}
}
-fun FlowOrPhrasingContent.dateTime(instant: Instant) {
- span(classes = "moment") {
- style = "display:none"
- +"${instant.toEpochMilli()}"
- }
+fun FlowOrPhrasingContent.dateTime(instant: Instant) = span(classes = "moment") {
+ style = "display:none"
+ +instant.toEpochMilli().toString()
+}
+
+fun <T, C : TagConsumer<T>> C.dateTime(instant: Instant) = span(classes = "moment") {
+ style = "display:none"
+ +instant.toEpochMilli().toString()
}
p { +"You are not allowed to do that." }
}
-suspend fun ApplicationCall.error403PageExpired(payload: CsrfProtectedResourcePayload): HTML.() -> Unit = errorPage("Page Expired") {
- with(payload) { displayRetryData() }
+suspend fun ApplicationCall.error403PageExpired(payload: CsrfProtectedResourcePayload?): HTML.() -> Unit = errorPage("Page Expired") {
+ payload?.apply { displayRetryData() }
p {
+"The page you were on has expired."
request.header(HttpHeaders.Referrer)?.let { referrer ->
data class StoragePathWithStat(val path: StoragePath, val stat: StoredFileStats)
+private fun StoragePath.rebase(onto: StoragePath) = onto / elements.drop(onto.elements.size)
+
+private suspend fun statAll(paths: Iterable<StoragePath>): StoredFileStats? {
+ val stats = paths.mapSuspend { path ->
+ FileStorage.instance.statFile(path)
+ }.filterNotNull()
+ if (stats.isEmpty()) return null
+
+ return StoredFileStats(
+ created = stats.minOf { it.created },
+ updated = stats.maxOf { it.updated },
+ size = stats.sumOf { it.size }
+ )
+}
+
private suspend fun ArticleNode.getPages(base: StoragePath): List<StoragePathWithStat> {
if (!this.isViewable)
return emptyList()
val path = base / name
- val stat = FileStorage.instance.statFile(path)
+ val dataPath = path.rebase(StoragePath.jsonDocDir)
+ val stat = statAll(listOf(path, dataPath))
return if (stat != null)
listOf(StoragePathWithStat(path, stat))
else if (subNodes != null) coroutineScope {
--- /dev/null
+package info.mechyrdia.robot
+
+import io.ktor.client.*
+import io.ktor.client.call.*
+import io.ktor.client.request.*
+import io.ktor.client.request.forms.*
+import io.ktor.http.*
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.Flow
+
+@JvmInline
+value class RobotClient(
+ private val client: HttpClient
+) {
+ suspend fun uploadFile(purpose: String, file: FileUpload) = client.submitFormWithBinaryData(
+ "https://api.openai.com/v1/files",
+ formData {
+ append("purpose", purpose)
+ upload("file", file)
+ }
+ ) {
+ attributes.addTokens(file)
+ }.body<RobotFile>()
+
+ suspend fun listFiles(purpose: String? = null) = client.get("https://api.openai.com/v1/files" + parameters {
+ purpose?.let { append("purpose", it) }
+ }.toQueryString()).body<RobotFileList>()
+
+ suspend fun getFile(fileId: RobotFileId) = client.get(
+ "https://api.openai.com/v1/files/${fileId.id}"
+ ).body<RobotFile>()
+
+ suspend fun deleteFile(fileId: RobotFileId) = client.delete(
+ "https://api.openai.com/v1/files/${fileId.id}"
+ ).body<RobotFileDeletionResponse>()
+
+ suspend fun downloadFile(fileId: RobotFileId) = client.delete(
+ "https://api.openai.com/v1/files/${fileId.id}"
+ ).body<ByteArray>()
+
+ suspend fun createVectorStore(request: RobotCreateVectorStoreRequest) = client.post("https://api.openai.com/v1/vector_stores") {
+ setJsonBody(request)
+ }.body<RobotVectorStore>()
+
+ suspend fun addFileToVectorStore(vsId: RobotVectorStoreId, fileId: RobotFileId) = client.post("https://api.openai.com/v1/vector_stores/${vsId.id}") {
+ setJsonBody(RobotAddFileToVectorStoreRequest(fileId))
+ }.body<RobotVectorStoreFile>()
+
+ suspend fun listVectorStores(limit: Int? = null, after: RobotVectorStoreId? = null) = client.get("https://api.openai.com/v1/vector_stores" + parameters {
+ limit?.let { append("limit", it.toString()) }
+ after?.let { append("after", it.id) }
+ }.toQueryString()).body<RobotVectorStoreList>()
+
+ suspend fun getVectorStore(vsId: RobotVectorStoreId) = client.get("https://api.openai.com/v1/vector_stores/${vsId.id}").body<RobotVectorStore>()
+
+ suspend fun modifyVectorStore(vsId: RobotVectorStoreId, request: RobotModifyVectorStoreRequest) = client.post("https://api.openai.com/v1/vector_stores/${vsId.id}") {
+ setJsonBody(request)
+ }.body<RobotVectorStore>()
+
+ suspend fun deleteVectorStore(vsId: RobotVectorStoreId) = client.delete("https://api.openai.com/v1/vector_stores/${vsId.id}").body<RobotVectorStoreDeletionResponse>()
+
+ suspend fun createAssistant(request: RobotCreateAssistantRequest) = client.post("https://api.openai.com/v1/assistants") {
+ setJsonBody(request)
+ }.body<RobotAssistant>()
+
+ suspend fun listAssistants(limit: Int? = null, after: RobotAssistantId? = null) = client.post("https://api.openai.com/v1/assistants" + parameters {
+ limit?.let { append("limit", it.toString()) }
+ after?.let { append("after", it.id) }
+ }.toQueryString()).body<RobotAssistantList>()
+
+ suspend fun getAssistant(assistId: RobotAssistantId) = client.get("https://api.openai.com/v1/assistants/${assistId.id}").body<RobotAssistant>()
+
+ suspend fun deleteAssistant(assistId: RobotAssistantId) = client.delete("https://api.openai.com/v1/assistants/${assistId.id}").body<RobotAssistantDeletionResponse>()
+
+ suspend fun createThread(request: RobotCreateThreadRequest) = client.post("https://api.openai.com/v1/threads") {
+ setJsonBody(request)
+ attributes.addTokens(request)
+ }.body<RobotThread>()
+
+ suspend fun getThread(threadId: RobotThreadId) = client.get("https://api.openai.com/v1/threads/${threadId.id}").body<RobotThread>()
+
+ suspend fun deleteThread(threadId: RobotThreadId) = client.delete("https://api.openai.com/v1/threads/${threadId.id}").body<RobotThreadDeletionResponse>()
+
+ suspend fun createRun(threadId: RobotThreadId, assistId: RobotAssistantId, messages: List<RobotCreateThreadRequestMessage>): Flow<ServerSentEvent> = client.postSse("https://api.openai.com/v1/threads/${threadId.id}/runs") {
+ val request = RobotCreateRunRequest(assistantId = assistId, additionalMessages = messages, stream = true)
+ setJsonBody(request)
+ attributes.addTokens(request)
+ }
+}
+
+inline fun <reified T> HttpRequestBuilder.setJsonBody(body: T) {
+ contentType(ContentType.Application.Json)
+ setBody(body)
+}
+
+suspend inline fun poll(wait: Long = 1_000L, block: () -> Boolean) {
+ while (!block())
+ delay(wait)
+}
+
+suspend inline fun <T : Any> pollValue(wait: Long = 1_000L, block: () -> T?): T {
+ while (true) {
+ block()?.let { return it }
+ delay(wait)
+ }
+}
--- /dev/null
+package info.mechyrdia.robot
+
+import io.ktor.http.*
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.json.Json
+
+@OptIn(ExperimentalSerializationApi::class)
+val JsonRobotCodec = Json {
+ coerceInputValues = true
+ ignoreUnknownKeys = true
+ useAlternativeNames = false
+ decodeEnumsCaseInsensitive = true
+}
+
+fun Parameters.toQueryString(): String {
+ val formEncoded = formUrlEncode()
+ return if (formEncoded.isEmpty())
+ formEncoded
+ else
+ "?$formEncoded"
+}
--- /dev/null
+package info.mechyrdia.robot
+
+import io.ktor.client.request.forms.*
+import io.ktor.http.*
+
+class FileUpload(
+ val content: ByteArray,
+ val contentType: ContentType,
+ val contentName: String,
+) : Tokenizable {
+ override fun getTexts(): List<String> {
+ return if (contentType.match(ContentType.Text.Any))
+ listOf(String(content))
+ else emptyList()
+ }
+}
+
+fun FormBuilder.upload(key: String, file: FileUpload) = append(key, file.content, Headers.build {
+ append(HttpHeaders.ContentType, file.contentType)
+ append(HttpHeaders.ContentDisposition, "filename=\"${file.contentName}\"")
+})
+
+fun String.toOpenAiName() = replace('.', '_') + ".txt"
+fun String.fromOpenAiName() = removeSuffix(".txt").replace('_', '.')
--- /dev/null
+package info.mechyrdia.robot
+
+import com.aallam.ktoken.Encoding
+import com.aallam.ktoken.Tokenizer
+import io.ktor.client.plugins.api.*
+import io.ktor.util.*
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import java.time.Instant
+import java.util.concurrent.atomic.AtomicInteger
+import java.util.concurrent.atomic.AtomicLong
+import kotlin.random.Random
+import kotlin.time.Duration.Companion.seconds
+
+private val DurationRegex = Regex("([0-9]+h)?([0-9]+m)?([0-9]+s)?")
+
+private fun String.parseDurationToSeconds(): Int {
+ val durationMatch = DurationRegex.matchEntire(this) ?: return 0
+ val (hoursStr, minutesStr, secondsStr) = durationMatch.destructured
+
+ val hours = if (hoursStr.endsWith("h")) hoursStr.dropLast(1).toInt() else 0
+ val minutes = if (minutesStr.endsWith("m")) minutesStr.dropLast(1).toInt() else 0
+ val seconds = if (secondsStr.endsWith("s")) secondsStr.dropLast(1).toInt() else 0
+
+ return (hours * 3600) + (minutes * 60) + seconds
+}
+
+private fun Int.secondFromNow() = Instant.now().epochSecond + this
+
+private fun calculateRateLimitDelayDouble(requestsRemaining: Int, requestsResetAt: Long): Double? {
+ val now = Instant.now().epochSecond
+ if (requestsRemaining > 0 && requestsResetAt <= now)
+ return null
+
+ return requestsResetAt - now + 0.5
+}
+
+private fun combineDelays(vararg delays: Double?) = if (delays.all { it == null })
+ null
+else delays.sumOf { it ?: 0.0 } + Random.nextDouble(0.25, 0.75)
+
+val RobotRateLimiter = createClientPlugin("RobotRateLimiter") {
+ val requestsRemaining = AtomicInteger(1)
+ val requestsResetAt = AtomicLong(0)
+
+ val tokensRemaining = AtomicInteger(1)
+ val tokensResetAt = AtomicLong(0)
+
+ onRequest { request, _ ->
+ val requestDelay = calculateRateLimitDelayDouble(requestsRemaining.getAcquire(), requestsResetAt.getAcquire())
+ val tokenDelay = request.attributes.getTokens()?.let { _ ->
+ calculateRateLimitDelayDouble(tokensRemaining.getAcquire(), tokensResetAt.getAcquire())
+ }
+
+ combineDelays(requestDelay, tokenDelay)?.seconds?.let { delay(it) }
+ }
+
+ @Suppress("UastIncorrectHttpHeaderInspection")
+ onResponse { response ->
+ val newRequestsRemaining = response.headers["X-Ratelimit-Remaining-Requests"]?.toIntOrNull() ?: -1
+ val newRequestsResetAt = response.headers["X-Ratelimit-Reset-Requests"]?.parseDurationToSeconds()?.secondFromNow() ?: 0
+ val newTokensRemaining = response.headers["X-Ratelimit-Remaining-Tokens"]?.toIntOrNull() ?: -1
+ val newTokensResetAt = response.headers["X-Ratelimit-Reset-Tokens"]?.parseDurationToSeconds()?.secondFromNow() ?: 0
+
+ requestsRemaining.setRelease(newRequestsRemaining)
+ requestsResetAt.setRelease(newRequestsResetAt)
+ tokensRemaining.setRelease(newTokensRemaining)
+ tokensResetAt.setRelease(newTokensResetAt)
+ }
+}
+
+private val RobotTokenCountKey = AttributeKey<Int>("Mechyrdia.RobotTokenCount")
+
+suspend fun Attributes.addTokens(tokenizable: Tokenizable) {
+ val deltaCount = tokenizable.getTexts().countTokens()
+ put(RobotTokenCountKey, deltaCount + computeIfAbsent(RobotTokenCountKey) { 0 })
+}
+
+fun Attributes.getTokens(): Int? = getOrNull(RobotTokenCountKey)
+
+private var tokenizerStore: Tokenizer? = null
+private val tokenizerMutex = Mutex()
+
+suspend fun getTokenizer(): Tokenizer {
+ return tokenizerStore ?: tokenizerMutex.withLock {
+ Tokenizer.of(Encoding.CL100K_BASE).also {
+ tokenizerStore = it
+ }
+ }
+}
+
+fun interface Tokenizable {
+ fun getTexts(): List<String>
+}
+
+fun List<Tokenizable>.flatten() = Tokenizable {
+ flatMap { it.getTexts() }
+}
+
+suspend fun String.countTokens(): Int {
+ return getTokenizer().encode(this).size
+}
+
+suspend fun List<String>.countTokens(): Int {
+ return sumOf { it.countTokens() }
+}
--- /dev/null
+package info.mechyrdia.robot
+
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+@Serializable
+@JvmInline
+value class RobotFileId(val id: String)
+
+@Serializable
+data class RobotFile(
+ val id: RobotFileId,
+ val bytes: Long,
+ @SerialName("created_at")
+ val createdAt: Long,
+ val filename: String,
+ val purpose: String,
+)
+
+@Serializable
+data class RobotFileList(
+ val data: List<RobotFile>,
+)
+
+@Serializable
+data class RobotFileDeletionResponse(
+ val id: RobotFileId,
+ val deleted: Boolean,
+)
+
+@Serializable
+data class RobotCreateVectorStoreRequest(
+ val name: String? = null,
+ @SerialName("file_ids")
+ val fileIds: List<RobotFileId> = emptyList(),
+)
+
+@Serializable
+data class RobotModifyVectorStoreRequest(
+ val name: String? = null,
+)
+
+@Serializable
+@JvmInline
+value class RobotVectorStoreId(val id: String)
+
+@Serializable
+data class RobotVectorStoreFileCounts(
+ @SerialName("in_progress")
+ val inProgress: Int = 0,
+ val completed: Int = 0,
+ val failed: Int = 0,
+ val cancelled: Int = 0,
+ val total: Int = 0,
+)
+
+@Serializable
+data class RobotVectorStore(
+ val id: RobotVectorStoreId,
+ val name: String,
+ @SerialName("created_at")
+ val createdAt: Long,
+ val bytes: Long = 0L,
+ @SerialName("file_counts")
+ val fileCounts: RobotVectorStoreFileCounts = RobotVectorStoreFileCounts(),
+ val status: String,
+)
+
+@Serializable
+data class RobotVectorStoreList(
+ val data: List<RobotVectorStore>,
+ @SerialName("first_id")
+ val firstId: RobotVectorStoreId? = null,
+ @SerialName("last_id")
+ val lastId: RobotVectorStoreId? = null,
+ @SerialName("has_more")
+ val hasMore: Boolean = false,
+)
+
+@Serializable
+data class RobotVectorStoreDeletionResponse(
+ val id: RobotVectorStoreId,
+ val deleted: Boolean,
+)
+
+@Serializable
+data class RobotAddFileToVectorStoreRequest(
+ val fileId: RobotFileId,
+)
+
+@Serializable
+data class RobotVectorStoreFile(
+ val id: RobotFileId,
+ @SerialName("created_at")
+ val createdAt: Long,
+ @SerialName("vector_store_id")
+ val vectorStoreId: RobotVectorStoreId,
+ val status: String,
+)
+
+@Serializable
+data class RobotCreateAssistantRequestTool(
+ val type: String,
+)
+
+@Serializable
+data class RobotCreateAssistantRequestFileSearchResources(
+ @SerialName("vector_store_ids")
+ val vectorStoreIds: List<RobotVectorStoreId>? = null,
+)
+
+@Serializable
+data class RobotCreateAssistantRequestToolResources(
+ @SerialName("file_search")
+ val fileSearch: RobotCreateAssistantRequestFileSearchResources? = null
+)
+
+@Serializable
+data class RobotCreateAssistantRequest(
+ val model: String,
+ val name: String? = null,
+ val description: String? = null,
+ val instructions: String? = null,
+ val tools: List<RobotCreateAssistantRequestTool>? = null,
+ @SerialName("tool_resources")
+ val toolResources: RobotCreateAssistantRequestToolResources? = null,
+ val temperature: Double? = null,
+)
+
+@Serializable
+@JvmInline
+value class RobotAssistantId(val id: String)
+
+@Serializable
+data class RobotAssistant(
+ val id: RobotAssistantId,
+ @SerialName("created_at")
+ val createdAt: Long,
+ val model: String,
+ val name: String? = null,
+ val description: String? = null,
+ val instructions: String? = null,
+ val tools: List<RobotCreateAssistantRequestTool>? = null,
+ @SerialName("tool_resources")
+ val toolResources: RobotCreateAssistantRequestToolResources,
+ val temperature: Double? = null,
+ @SerialName("top_p")
+ val topP: Double? = null,
+)
+
+@Serializable
+data class RobotAssistantList(
+ val data: List<RobotAssistant>,
+ @SerialName("first_id")
+ val firstId: RobotAssistantId? = null,
+ @SerialName("last_id")
+ val lastId: RobotAssistantId? = null,
+ @SerialName("has_more")
+ val hasMore: Boolean = false,
+)
+
+@Serializable
+data class RobotAssistantDeletionResponse(
+ val id: RobotAssistantId,
+ val deleted: Boolean,
+)
+
+@Serializable
+data class RobotCreateThreadRequestMessage(
+ val role: String,
+ val content: String,
+) : Tokenizable {
+ override fun getTexts(): List<String> {
+ return listOf(content)
+ }
+}
+
+@Serializable
+data class RobotCreateThreadRequest(
+ val messages: List<RobotCreateThreadRequestMessage> = emptyList(),
+) : Tokenizable by messages.flatten()
+
+@Serializable
+@JvmInline
+value class RobotThreadId(val id: String)
+
+@Serializable
+data class RobotThread(
+ val id: RobotThreadId,
+ @SerialName("created_at")
+ val createdAt: Long,
+)
+
+@Serializable
+data class RobotThreadDeletionResponse(
+ val id: RobotThreadId,
+ val deleted: Boolean,
+)
+
+@Serializable
+data class RobotCreateRunRequest(
+ @SerialName("assistant_id")
+ val assistantId: RobotAssistantId,
+ @SerialName("additional_messages")
+ val additionalMessages: List<RobotCreateThreadRequestMessage> = emptyList(),
+ val stream: Boolean,
+) : Tokenizable by additionalMessages.flatten()
+
+@Serializable
+@JvmInline
+value class RobotMessageId(val id: String)
+
+@Serializable
+data class RobotFileCitation(
+ @SerialName("file_id")
+ val fileId: RobotFileId,
+ val quote: String,
+)
+
+@Serializable
+data class RobotMessageTextAnnotation(
+ val index: Int = 0,
+ val text: String,
+ @SerialName("file_citation")
+ val fileCitation: RobotFileCitation,
+ @SerialName("start_index")
+ val startIndex: Int = 0,
+ @SerialName("end_index")
+ val endIndex: Int = 0,
+)
+
+@Serializable
+data class RobotMessageText(
+ val value: String = "",
+ val annotations: List<RobotMessageTextAnnotation> = emptyList(),
+)
+
+@Serializable
+data class RobotMessageDeltaText(
+ val index: Int = 0,
+ val text: RobotMessageText = RobotMessageText(),
+)
+
+@Serializable
+data class RobotMessageDeltaFields(
+ val role: String? = null,
+ val content: List<RobotMessageDeltaText> = emptyList(),
+)
+
+@Serializable
+data class RobotMessageDelta(
+ val id: RobotMessageId,
+ val delta: RobotMessageDeltaFields = RobotMessageDeltaFields()
+)
+
+@Serializable
+data class RobotMessage(
+ val id: RobotMessageId,
+ @SerialName("created_at")
+ val createdAt: Long,
+ @SerialName("thread_id")
+ val threadId: RobotThreadId,
+ val status: String,
+ val role: String,
+ val content: List<RobotMessageText> = emptyList(),
+)
--- /dev/null
+package info.mechyrdia.robot
+
+import info.mechyrdia.Configuration
+import info.mechyrdia.MainDomainName
+import info.mechyrdia.data.*
+import info.mechyrdia.lore.RobotFactbookLoader
+import io.ktor.client.*
+import io.ktor.client.engine.java.*
+import io.ktor.client.plugins.*
+import io.ktor.client.plugins.contentnegotiation.*
+import io.ktor.client.plugins.logging.*
+import io.ktor.client.request.*
+import io.ktor.http.*
+import io.ktor.serialization.kotlinx.json.*
+import kotlinx.coroutines.*
+import kotlinx.coroutines.flow.*
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.time.Instant
+import kotlin.random.Random
+import kotlin.time.Duration.Companion.minutes
+
+val RobotGlobalsId = Id<RobotGlobals>("RobotGlobalsInstance")
+
+@Serializable
+data class RobotGlobals(
+ @SerialName(MONGODB_ID_KEY)
+ override val id: Id<RobotGlobals> = RobotGlobalsId,
+
+ val lastFileUpload: @Serializable(with = InstantNullableSerializer::class) Instant? = null,
+ val fileIdMap: Map<String, RobotFileId> = emptyMap(),
+ val vectorStoreId: RobotVectorStoreId? = null,
+ val assistantId: RobotAssistantId? = null,
+ val ongoingThreadIds: Set<RobotThreadId> = emptySet(),
+) : DataDocument<RobotGlobals> {
+ suspend fun save(): RobotGlobals {
+ set(this)
+ return this
+ }
+
+ companion object : TableHolder<RobotGlobals> {
+ override val Table = DocumentTable<RobotGlobals>()
+
+ suspend fun get() = Table.get(RobotGlobalsId)
+ suspend fun set(instance: RobotGlobals) = Table.put(instance)
+
+ override suspend fun initialize() = Unit
+ }
+}
+
+private fun RobotGlobals.plusThread(threadId: RobotThreadId) = copy(
+ ongoingThreadIds = ongoingThreadIds + threadId
+)
+
+private fun RobotGlobals.minusThread(threadId: RobotThreadId) = copy(
+ ongoingThreadIds = ongoingThreadIds - threadId
+)
+
+enum class RobotServiceStatus {
+ NOT_CONFIGURED,
+ LOADING,
+ FAILED,
+ READY,
+}
+
+class RobotService(
+ token: String,
+ orgId: String,
+ project: String?,
+) {
+ private val robotClient = RobotClient(
+ HttpClient(Java) {
+ defaultRequest {
+ header(HttpHeaders.Authorization, "Bearer $token")
+ header("OpenAI-Organization", orgId)
+ project?.let { header("OpenAI-Project", it) }
+ header("OpenAI-Beta", "assistants=v2")
+ }
+
+ install(ContentNegotiation) {
+ json(JsonRobotCodec)
+ }
+
+ Logging {
+ level = LogLevel.INFO
+ sanitizeHeader("<OPENAI TOKEN>") { it == HttpHeaders.Authorization }
+ }
+
+ install(HttpRequestRetry) {
+ retryOnExceptionOrServerErrors(5)
+ delayMillis { retry ->
+ (1 shl (retry - 1)) * 1000L + Random.nextLong(250L, 750L)
+ }
+ }
+
+ expectSuccess = true
+
+ install(RobotRateLimiter)
+ }
+ )
+
+ private suspend fun createThread(): RobotThreadId {
+ return robotClient.createThread(RobotCreateThreadRequest()).id.also { threadId ->
+ (RobotGlobals.get() ?: RobotGlobals()).plusThread(threadId).save()
+ }
+ }
+
+ private suspend fun deleteThread(threadId: RobotThreadId) {
+ robotClient.deleteThread(threadId)
+ (RobotGlobals.get() ?: RobotGlobals()).minusThread(threadId).save()
+ }
+
+ private suspend fun RobotGlobals.gcOldThreads(): RobotGlobals {
+ for (threadId in ongoingThreadIds)
+ robotClient.deleteThread(threadId)
+ return copy(ongoingThreadIds = emptySet())
+ }
+
+ private suspend fun updateFiles(prevGlobals: RobotGlobals?, onNewFileId: (suspend (RobotFileId) -> Unit)? = null): RobotGlobals {
+ val robotGlobals = prevGlobals ?: RobotGlobals()
+
+ val fileIdMap = buildMap<String, RobotFileId> {
+ putAll(robotGlobals.fileIdMap)
+
+ val factbooks = robotGlobals.lastFileUpload?.let {
+ RobotFactbookLoader.loadAllFactbooksSince(it)
+ } ?: RobotFactbookLoader.loadAllFactbooks()
+
+ for ((name, text) in factbooks) {
+ remove(name)?.let { oldId ->
+ robotClient.deleteFile(oldId)
+ }
+
+ val newId = robotClient.uploadFile(
+ "assistants",
+ FileUpload(
+ text.toByteArray(),
+ ContentType.Text.Plain.withCharset(Charsets.UTF_8),
+ name.toOpenAiName()
+ )
+ ).id
+
+ this[name] = newId
+ onNewFileId?.invoke(newId)
+
+ logger.info("Factbook $name has been uploaded")
+ }
+ }
+
+ return robotGlobals.copy(lastFileUpload = Instant.now(), fileIdMap = fileIdMap).save()
+ }
+
+ suspend fun initialize() {
+ var robotGlobals = updateFiles(RobotGlobals.get()?.gcOldThreads())
+
+ val vectorStoreId = robotGlobals.vectorStoreId ?: robotClient.createVectorStore(
+ RobotCreateVectorStoreRequest(
+ name = "lore_documents",
+ fileIds = robotGlobals.fileIdMap.values.toList(),
+ )
+ ).id.also { vsId ->
+ robotGlobals = robotGlobals.copy(vectorStoreId = vsId).save()
+ }
+
+ logger.info("Vector store has been created")
+
+ poll {
+ robotClient.getVectorStore(vectorStoreId).status == "completed"
+ }
+
+ logger.info("Vector store creation is complete")
+
+ if (robotGlobals.assistantId == null)
+ robotGlobals = robotGlobals.copy(
+ assistantId = robotClient.createAssistant(
+ RobotCreateAssistantRequest(
+ model = "gpt-4-turbo",
+ name = "Natural-language Universal Knowledge Engine",
+ instructions = "You are a helpful interactive encyclopedia, able to answer questions with information from the provided files",
+ tools = listOf(
+ RobotCreateAssistantRequestTool("file_search")
+ ),
+ toolResources = RobotCreateAssistantRequestToolResources(
+ fileSearch = RobotCreateAssistantRequestFileSearchResources(
+ vectorStoreIds = listOf(vectorStoreId)
+ )
+ ),
+ temperature = 1.0
+ )
+ ).id
+ ).save()
+
+ logger.info("Assistant has been created")
+ }
+
+ suspend fun performMaintenance() {
+ var robotGlobals = RobotGlobals.get() ?: RobotGlobals()
+
+ val vectorStoreId = robotGlobals.vectorStoreId ?: robotClient.createVectorStore(
+ RobotCreateVectorStoreRequest(
+ name = "lore_documents",
+ fileIds = robotGlobals.fileIdMap.values.toList(),
+ )
+ ).id.also { vsId ->
+ robotGlobals = robotGlobals.copy(vectorStoreId = vsId).save()
+ }
+
+ updateFiles(robotGlobals) { fileId ->
+ robotClient.addFileToVectorStore(vectorStoreId, fileId)
+ }
+
+ logger.info("Vector store has been updated")
+
+ poll {
+ robotClient.getVectorStore(vectorStoreId).fileCounts.inProgress == 0
+ }
+
+ logger.info("Vector store update is complete")
+ }
+
+ inner class Conversation(val nationId: Id<NationData>) {
+ private var assistantId: RobotAssistantId? = null
+ private var threadId: RobotThreadId? = null
+
+ suspend fun send(userMessage: String): Flow<RobotConversationMessage> {
+ val assistant = assistantId ?: pollValue { RobotGlobals.get()?.assistantId }
+ .also { assistantId = it }
+
+ val thread = threadId ?: createThread().also { threadId = it }
+
+ val messages = listOf(
+ RobotCreateThreadRequestMessage(
+ role = "user",
+ content = userMessage
+ )
+ )
+
+ val tokenTracker = ConversationMessageTokenTracker()
+
+ return flow {
+ emit(RobotConversationMessage.User(userMessage))
+
+ val annotationTargets = mutableListOf<Deferred<String>>()
+ val collectionScope = CoroutineScope(currentCoroutineContext())
+
+ robotClient.createRun(thread, assistant, messages)
+ .filter { it.event == "thread.message.delta" }
+ .mapNotNull { it.data }
+ .map { JsonRobotCodec.decodeFromString(RobotMessageDelta.serializer(), it) }
+ .collect { eventData ->
+ val annotationTexts = eventData.delta.content.flatMap { it.text.annotations }.map { annotation ->
+ val annotationIndex = annotationTargets.size
+ annotationTargets.add(collectionScope.async {
+ val fileName = robotClient.getFile(annotation.fileCitation.fileId).filename.fromOpenAiName()
+ val fileText = annotation.fileCitation.quote.let { if (it.isNotBlank()) ": $it" else it }
+ "$MainDomainName/lore/$fileName$fileText"
+ })
+ annotation.text to " [${annotationIndex + 1}]"
+ }
+
+ val contents = eventData.delta.content.joinToString(separator = "") { textContent ->
+ textContent.text.value
+ }
+
+ val replacedContents = annotationTexts.fold(contents) { text, (replace, replaceWith) ->
+ text.replace(replace, replaceWith)
+ }
+
+ emit(RobotConversationMessage.Robot(replacedContents))
+ }
+
+ emit(RobotConversationMessage.Cite(annotationTargets.awaitAll()))
+
+ emit(RobotConversationMessage.Ready)
+ }.onEach { message ->
+ tokenTracker.addMessage(message)
+ }.onCompletion { _ ->
+ RobotUser.addTokens(nationId, tokenTracker.calculateTokens())
+ }
+ }
+
+ suspend fun isExhausted(): Boolean {
+ val usedTokens = RobotUser.getTokens(nationId)
+ val tokenLimit = RobotUser.getMaxTokens(nationId)
+ return usedTokens >= tokenLimit
+ }
+
+ suspend fun close() {
+ threadId?.let { deleteThread(it) }
+ }
+ }
+
+ companion object {
+ private val logger: Logger = LoggerFactory.getLogger(RobotService::class.java)
+
+ private val maintenanceScope = CoroutineScope(SupervisorJob() + CoroutineName("robot-service-maintenance"))
+
+ private val instanceHolder by lazy {
+ CoroutineScope(CoroutineName("robot-service-initialization")).async {
+ Configuration.Current.openAi?.let { (token, orgId, project) ->
+ RobotService(token, orgId, project).apply {
+ initialize()
+ }
+ }
+ }
+ }
+
+ var status: RobotServiceStatus = if (Configuration.Current.openAi != null) RobotServiceStatus.LOADING else RobotServiceStatus.NOT_CONFIGURED
+ private set
+
+ suspend fun getInstance() = try {
+ instanceHolder.await()
+ } catch (ex: Exception) {
+ null
+ }
+
+ fun initialize() {
+ instanceHolder.invokeOnCompletion { ex ->
+ status = if (ex != null) {
+ logger.error("RobotService failed to initialize", ex)
+ RobotServiceStatus.FAILED
+ } else {
+ logger.info("RobotService successfully initialized")
+ RobotServiceStatus.READY
+ }
+ }
+
+ maintenanceScope.launch {
+ getInstance()?.let { instance ->
+ while (true) {
+ delay(30.minutes)
+
+ launch(SupervisorJob(currentCoroutineContext().job)) {
+ instance.performMaintenance()
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+@Serializable
+sealed class RobotConversationMessage {
+ @Serializable
+ @SerialName("ready")
+ data object Ready : RobotConversationMessage()
+
+ @Serializable
+ @SerialName("user")
+ data class User(val text: String) : RobotConversationMessage()
+
+ @Serializable
+ @SerialName("robot")
+ data class Robot(val text: String) : RobotConversationMessage()
+
+ @Serializable
+ @SerialName("cite")
+ data class Cite(val urls: List<String>) : RobotConversationMessage()
+}
--- /dev/null
+package info.mechyrdia.robot
+
+import io.ktor.client.*
+import io.ktor.client.request.*
+import io.ktor.client.statement.*
+import io.ktor.utils.io.*
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.channels.ProducerScope
+import kotlinx.coroutines.channels.ReceiveChannel
+import kotlinx.coroutines.channels.produce
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.FlowCollector
+import kotlinx.coroutines.flow.flow
+
+data class ServerSentEvent(
+ val data: String?,
+ val event: String?,
+ val id: String?,
+ val retry: Double?,
+)
+
+private class SseBuilder {
+ var data: String? = null
+ var event: String? = null
+ var id: String? = null
+ var retry: Double? = null
+
+ fun build() = ServerSentEvent(data, event, id, retry)
+
+ val isSet: Boolean
+ get() = data != null || event != null || id != null || retry != null
+
+ fun reset() {
+ data = ""
+ event = null
+ id = null
+ retry = null
+ }
+}
+
+private const val SSE_DATA_PREFIX = "data: "
+private const val SSE_EVENT_PREFIX = "event: "
+private const val SSE_ID_PREFIX = "id: "
+private const val SSE_RETRY_PREFIX = "retry: "
+
+private suspend fun FlowCollector<ServerSentEvent>.receiveSse(response: HttpResponse) {
+ val reader = response.bodyAsChannel()
+ val builder = SseBuilder()
+ while (true) {
+ val line = reader.readUTF8Line() ?: break
+
+ if (line.isBlank()) {
+ if (builder.isSet)
+ emit(builder.build())
+ builder.reset()
+ continue
+ }
+
+ if (line.startsWith(":")) continue
+
+ if (line.startsWith(SSE_DATA_PREFIX))
+ builder.data = builder.data?.let { "$it\n" }.orEmpty() + line.substring(SSE_DATA_PREFIX.length)
+ if (line.startsWith(SSE_EVENT_PREFIX))
+ builder.event = line.substring(SSE_EVENT_PREFIX.length)
+ if (line.startsWith(SSE_ID_PREFIX))
+ builder.id = line.substring(SSE_ID_PREFIX.length)
+ if (line.startsWith(SSE_RETRY_PREFIX))
+ builder.retry = line.substring(SSE_RETRY_PREFIX.length).toDoubleOrNull()
+ }
+
+ if (builder.isSet)
+ emit(builder.build())
+}
+
+suspend fun HttpClient.getSse(urlString: String, requestBuilder: suspend HttpRequestBuilder.() -> Unit): Flow<ServerSentEvent> {
+ return flow {
+ prepareGet(urlString) {
+ requestBuilder()
+ }.execute { response ->
+ receiveSse(response)
+ }
+ }
+}
+
+suspend fun HttpClient.postSse(urlString: String, requestBuilder: suspend HttpRequestBuilder.() -> Unit): Flow<ServerSentEvent> {
+ return flow {
+ preparePost(urlString) {
+ requestBuilder()
+ }.execute { response ->
+ receiveSse(response)
+ }
+ }
+}
--- /dev/null
+package info.mechyrdia.robot
+
+import com.mongodb.client.model.Filters
+import com.mongodb.client.model.Updates
+import info.mechyrdia.OwnerNationId
+import info.mechyrdia.data.*
+import info.mechyrdia.lore.MyTimeZone
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import java.time.Instant
+
+@Serializable
+data class RobotUser(
+ @SerialName(MONGODB_ID_KEY)
+ override val id: Id<RobotUser>,
+
+ val usedByUser: Id<NationData>,
+ val usedInMonth: Int,
+
+ val tokensUsed: Int,
+) : DataDocument<RobotUser> {
+ companion object : TableHolder<RobotUser> {
+ override val Table: DocumentTable<RobotUser> = DocumentTable<RobotUser>()
+
+ override suspend fun initialize() {
+ Table.unique()
+ }
+
+ private fun currentMonth(): Int {
+ val now = Instant.now().atZone(MyTimeZone)
+ return (now.year - 2024) * 12 + now.month.ordinal
+ }
+
+ fun getMaxTokens(nationId: Id<NationData>): Int = if (nationId == OwnerNationId)
+ 100_000_000
+ else 100_000
+
+ suspend fun getTokens(nationId: Id<NationData>): Int {
+ return Table.locate(
+ Filters.and(
+ Filters.eq(RobotUser::usedByUser.serialName, nationId),
+ Filters.eq(RobotUser::usedInMonth.serialName, currentMonth()),
+ )
+ )?.tokensUsed ?: 0
+ }
+
+ suspend fun addTokens(nationId: Id<NationData>, tokens: Int) {
+ Table.change(
+ Filters.and(
+ Filters.eq(RobotUser::usedByUser.serialName, nationId),
+ Filters.eq(RobotUser::usedInMonth.serialName, currentMonth()),
+ ),
+ Updates.combine(
+ Updates.inc(RobotUser::tokensUsed.serialName, tokens),
+ Updates.setOnInsert(RobotUser::id.serialName, Id<RobotUser>()),
+ )
+ )
+ }
+ }
+}
+
+private const val REQUEST_TOKEN_WEIGHT = 1
+private const val RESPONSE_TOKEN_WEIGHT = 3
+
+class ConversationMessageTokenTracker {
+ private val request = StringBuffer()
+ private val response = StringBuffer()
+
+ fun addMessage(message: RobotConversationMessage) {
+ when (message) {
+ is RobotConversationMessage.User -> request.append(message.text)
+ is RobotConversationMessage.Robot -> response.append(message.text)
+ else -> {
+ // ignore
+ }
+ }
+ }
+
+ suspend fun calculateTokens(): Int {
+ return (request.toString().countTokens() * REQUEST_TOKEN_WEIGHT) + (response.toString().countTokens() * RESPONSE_TOKEN_WEIGHT)
+ }
+}
--- /dev/null
+package info.mechyrdia.robot
+
+import info.mechyrdia.data.currentNation
+import info.mechyrdia.lore.page
+import info.mechyrdia.lore.redirectHref
+import info.mechyrdia.lore.standardNavBar
+import info.mechyrdia.route.Root
+import info.mechyrdia.route.checkCsrfToken
+import info.mechyrdia.route.createCsrfToken
+import info.mechyrdia.route.href
+import io.ktor.server.application.*
+import io.ktor.server.websocket.*
+import io.ktor.websocket.*
+import io.ktor.websocket.CloseReason.*
+import kotlinx.html.*
+import kotlinx.serialization.json.JsonPrimitive
+
+suspend fun ApplicationCall.robotPage(): HTML.() -> Unit {
+ val nation = currentNation()?.id ?: redirectHref(Root.Auth.LoginPage(Root.Auth(Root(error = "You must be logged in to use the NUKE"))))
+ val exhausted = RobotUser.getTokens(nation) >= RobotUser.getMaxTokens(nation)
+
+ val nukeRoute = href(Root.Nuke.WS())
+ val token = createCsrfToken(nukeRoute)
+
+ val robotServiceStatus = RobotService.status
+
+ return page("NUKE", standardNavBar(), null) {
+ section {
+ h1 { +"NUKE" }
+ p {
+ +"The "
+ b { +"NUKE" }
+ +" (Natural-language Universal Knowledge Engine) is an interactive encyclopedia that answers questions about the galaxy."
+ }
+ if (exhausted)
+ p { +"You have exhausted your monthly limit of NUKE usage." }
+ else
+ when (robotServiceStatus) {
+ RobotServiceStatus.NOT_CONFIGURED -> p { +"Unfortunately, the NUKE is not configured on this website." }
+ RobotServiceStatus.LOADING -> p { +"The NUKE is still in the process of initializing." }
+ RobotServiceStatus.FAILED -> p { +"Tragically, the NUKE has failed to initialize due to an internal error." }
+ RobotServiceStatus.READY -> script {
+ unsafe {
+ val jsToken = JsonPrimitive(token).toString()
+ +"window.createNukeBox($jsToken);"
+ }
+ }
+ }
+ }
+ }
+}
+
+suspend fun WebSocketSession.closeReasonably(reason: String) = close(CloseReason(Codes.NORMAL, reason))
+
+suspend fun DefaultWebSocketServerSession.robotConversation(csrfToken: String? = null) {
+ val nation = call.currentNation()?.id ?: return closeReasonably("Anonymous usage of NUKE is not allowed")
+ if (!call.checkCsrfToken(csrfToken, call.href(Root.Nuke.WS())))
+ return closeReasonably("CSRF token failed verification")
+
+ val robotService = RobotService.getInstance() ?: return closeReasonably("NUKE is not configured on this website")
+
+ val conversation = robotService.Conversation(nation)
+
+ if (conversation.isExhausted()) {
+ conversation.close()
+ return closeReasonably("You have exhausted your monthly limit of NUKE usage")
+ }
+
+ sendSerialized<RobotConversationMessage>(RobotConversationMessage.Ready)
+
+ for (frame in incoming) {
+ if (frame !is Frame.Text) continue
+ val query = frame.readText()
+
+ conversation.send(query).collect { message ->
+ sendSerialized(message)
+ }
+
+ if (conversation.isExhausted()) {
+ conversation.close()
+ return closeReasonably("You have exhausted your monthly limit of NUKE usage")
+ }
+ }
+
+ conversation.close()
+}
private val csrfMap = ConcurrentHashMap<String, CsrfPayload>()
-data class CsrfFailedException(override val message: String, val payload: CsrfProtectedResourcePayload) : RuntimeException(message)
+data class CsrfFailedException(override val message: String, val payload: CsrfProtectedResourcePayload?) : RuntimeException(message)
interface CsrfProtectedResourcePayload {
val csrfToken: String?
- suspend fun ApplicationCall.verifyCsrfToken(route: String = request.uri) {
+ fun ApplicationCall.verifyCsrfToken(route: String = request.uri) {
val token = csrfToken ?: throw CsrfFailedException("The submitted CSRF token is not present", this@CsrfProtectedResourcePayload)
val check = csrfMap.remove(token) ?: throw CsrfFailedException("The submitted CSRF token is not valid", this@CsrfProtectedResourcePayload)
val payload = csrfPayload(route, check.expires)
fun FlowContent.displayRetryData() {}
}
+fun ApplicationCall.checkCsrfToken(csrfToken: String?, route: String = request.uri): Boolean {
+ val token = csrfToken ?: return false
+ val check = csrfMap.remove(token) ?: return false
+ val payload = csrfPayload(route, check.expires)
+ return check == payload && payload.expires >= Instant.now()
+}
+
fun ApplicationCall.createCsrfToken(route: String = request.origin.uri): String {
return token().also { csrfMap[it] = csrfPayload(route) }
}
import io.ktor.http.*
import io.ktor.resources.serialization.*
import io.ktor.server.application.*
+import io.ktor.server.plugins.*
import io.ktor.server.request.*
import io.ktor.server.resources.*
-import io.ktor.server.routing.Route
+import io.ktor.server.resources.post
+import io.ktor.server.routing.*
+import io.ktor.server.websocket.*
+import io.ktor.util.*
import io.ktor.util.pipeline.*
-import kotlinx.serialization.DeserializationStrategy
-import kotlinx.serialization.KSerializer
-import kotlinx.serialization.SerializationStrategy
-import kotlinx.serialization.StringFormat
+import kotlinx.html.P
+import kotlinx.serialization.*
import kotlinx.serialization.builtins.nullable
import kotlinx.serialization.builtins.serializer
import kotlinx.serialization.descriptors.PrimitiveKind
suspend fun PipelineContext<Unit, ApplicationCall>.handleCall()
}
+interface ResourceListener {
+ suspend fun DefaultWebSocketServerSession.handleCall()
+}
+
interface ResourceReceiver<P : Any> {
suspend fun PipelineContext<Unit, ApplicationCall>.handleCall(payload: P)
}
}
}
+val WebSocketResourceInstanceKey: AttributeKey<Any> = AttributeKey("WebSocketResourceInstance")
+
+inline fun <reified T : ResourceListener> Route.ws() {
+ resource<T> {
+ val serializer = serializer<T>()
+ intercept(ApplicationCallPipeline.Plugins) {
+ val resources = application.plugin(Resources)
+ try {
+ val resource = resources.resourcesFormat.decodeFromParameters(serializer, call.parameters)
+ call.attributes.put(WebSocketResourceInstanceKey, resource)
+ } catch (cause: Throwable) {
+ throw BadRequestException("Can't transform call to resource", cause)
+ }
+ }
+
+ webSocket {
+ val resource = call.attributes[WebSocketResourceInstanceKey] as T
+ with(resource) { handleCall() }
+ }
+ }
+}
+
abstract class KeyedEnumSerializer<E : Enum<E>>(val entries: EnumEntries<E>, val getKey: (E) -> String? = { it.name }) : KSerializer<E> {
override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("KeyedEnumSerializer<${entries.first()::class.qualifiedName}>", PrimitiveKind.STRING)
inline fun <reified T : Any> Application.href(resource: T, hash: String? = null): String = URLBuilder().also { href(resource, it) }.build().fullPath + hash?.let { "#$it" }.orEmpty()
inline fun <reified T : Any> ApplicationCall.href(resource: T, hash: String? = null) = application.href(resource, hash)
+inline fun <reified T : Any> WebSocketServerSession.href(resource: T, hash: String? = null) = application.href(resource, hash)
inline fun <reified T : Any> PipelineContext<Unit, ApplicationCall>.href(resource: T, hash: String? = null) = application.href(resource, hash)
import info.mechyrdia.auth.*
import info.mechyrdia.data.*
import info.mechyrdia.lore.*
+import info.mechyrdia.robot.robotConversation
+import info.mechyrdia.robot.robotPage
import io.ktor.http.*
import io.ktor.http.content.*
import io.ktor.resources.*
import io.ktor.server.html.*
import io.ktor.server.plugins.*
import io.ktor.server.response.*
+import io.ktor.server.websocket.*
import io.ktor.util.*
import io.ktor.util.pipeline.*
import kotlinx.coroutines.delay
}
}
+ @Resource("nuke")
+ class Nuke(val root: Root = Root()) : ResourceHandler {
+ override suspend fun PipelineContext<Unit, ApplicationCall>.handleCall() {
+ with(root) { filterCall() }
+
+ call.respondHtml(HttpStatusCode.OK, call.robotPage())
+ }
+
+ @Resource("ws")
+ class WS(val csrfToken: String? = null, val nuke: Nuke = Nuke()) : ResourceListener {
+ override suspend fun DefaultWebSocketServerSession.handleCall() {
+ robotConversation(csrfToken)
+ }
+ }
+ }
+
@Resource("comment")
class Comments(val root: Root = Root()) : ResourceFilter {
override suspend fun PipelineContext<Unit, ApplicationCall>.filterCall() {
});
}
+ function appendWithLineBreaks(element, text) {
+ const lines = text.split("\n");
+ let isFirst = true;
+ for (const line of lines) {
+ if (isFirst)
+ isFirst = false;
+ else
+ element.append(document.createElement("br"));
+ element.append(line);
+ }
+ }
+
window.addEventListener("load", function () {
// Mechyrdian font
async function mechyrdianToFont(input, boldOpt, italicOpt, alignOpt, output, delayLength) {
window.localStorage.removeItem("redirectedFrom");
};
+
+ window.createNukeBox = function (csrfToken) {
+ const chatHistory = document.createElement("blockquote");
+ chatHistory.style.overflowY = "scroll";
+ chatHistory.style.height = "40vh";
+
+ const inputBox = document.createElement("input");
+ inputBox.classList.add("inline");
+ inputBox.style.flexGrow = "1";
+ inputBox.type = "text";
+ inputBox.placeholder = "Enter your message";
+
+ const enterBtn = document.createElement("input");
+ enterBtn.classList.add("inline");
+ enterBtn.style.flexShrink = "0";
+ enterBtn.type = "submit";
+ enterBtn.value = "Send";
+ enterBtn.disabled = true;
+
+ const inputForm = document.createElement("form");
+ inputForm.style.display = "flex";
+ inputForm.append(inputBox, enterBtn);
+
+ const container = document.createElement("div");
+ container.append(chatHistory, inputForm);
+ document.currentScript.after(container);
+
+ const targetUrl = "ws" + window.location.href.substring(4) + "/ws?csrfToken=" + csrfToken;
+ const webSock = new WebSocket(targetUrl);
+
+ inputForm.onsubmit = (ev) => {
+ ev.preventDefault();
+ if (!ev.submitter.disabled) {
+ webSock.send(inputBox.value);
+ inputBox.value = "";
+ enterBtn.disabled = true;
+ }
+ };
+
+ webSock.onmessage = (ev) => {
+ const data = JSON.parse(ev.data);
+ if (data.type === "ready") {
+ enterBtn.disabled = false;
+ } else if (data.type === "user") {
+ const userP = document.createElement("p");
+ userP.style.textAlign = "right";
+ userP.style.paddingLeft = "50%";
+ appendWithLineBreaks(userP, data.text);
+ chatHistory.appendChild(userP);
+
+ const robotP = document.createElement("p");
+ robotP.style.textAlign = "left";
+ robotP.style.paddingRight = "50%";
+ chatHistory.appendChild(robotP);
+ } else if (data.type === "robot") {
+ const robotP = chatHistory.lastElementChild;
+ appendWithLineBreaks(robotP, data.text);
+ } else if (data.type === "cite") {
+ const robotP = chatHistory.lastElementChild;
+ const robotCiteList = robotP.appendChild(document.createElement("ol"));
+ for (const url of data.urls) {
+ const urlLink = robotCiteList.appendChild(document.createElement("li")).appendChild(document.createElement("a"));
+ urlLink.href = url;
+ urlLink.append(url);
+ }
+ }
+ };
+
+ webSock.onclose = (ev) => {
+ const statusP = document.createElement("p");
+ statusP.style.textAlign = "center";
+ statusP.style.paddingLeft = "25%";
+ statusP.style.paddingRight = "25%";
+ appendWithLineBreaks(statusP, "The connection has been closed\n" + ev.reason);
+ chatHistory.appendChild(statusP);
+
+ enterBtn.disabled = true;
+ };
+ };
})();
color: var(--tbl-td-bgr);
}
+input[type=text].inline,
+input[type=password].inline,
+input[type=email].inline {
+ width: unset;
+}
+
input[type=text],
input[type=password],
input[type=email],
border-bottom-color: var(--err-ul);
}
+button.inline, input[type=submit].inline {
+ display: inline;
+ font-size: 1em;
+ margin: 0.25em;
+ padding: 0.45em 0.65em;
+ width: unset;
+}
+
button, input[type=submit] {
background-color: var(--btn-bg);
border: none;
projects / factbooks / commitdiff