cilly/Banksia
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: di, db, and preliminary server-side gtfs parsing

Browse source
This commit is contained in:
LavaDesu 2025-08-08 01:59:32 +10:00
parent ccc748dc1f
commit 6770c01613
Signed by: cilly
GPG key ID: 6500251E087653C9
22 changed files with 555 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

9
server/build.gradle.kts
View file

@ -1,5 +1,6 @@
plugins {
alias(libs.plugins.kotlinJvm)
alias(libs.plugins.kotlinSerialization)
alias(libs.plugins.ktor)
application
}
@ -14,8 +15,16 @@ application {
dependencies {
implementation(projects.shared)
implementation(libs.logback)
implementation(libs.koin.core)
implementation(libs.koin.ktor)
implementation(libs.kotlinx.serialization.csv)
implementation(libs.ktor.client.core)
implementation(libs.ktor.client.contentnegotiation)
implementation(libs.ktor.client.okhttp)
implementation(libs.ktor.server.core)
implementation(libs.ktor.server.netty)
implementation(libs.room.runtime)
implementation(libs.sqlite.bundled)
testImplementation(libs.ktor.server.tests)
testImplementation(libs.kotlin.test.junit)
}

27
server/src/main/kotlin/moe/lava/banksia/server/Application.kt
View file

@ -1,11 +1,22 @@
package moe.lava.banksia.server
import io.ktor.client.HttpClient
import io.ktor.server.application.Application
import io.ktor.server.application.install
import io.ktor.server.application.log
import io.ktor.server.engine.embeddedServer
import io.ktor.server.netty.Netty
import io.ktor.server.response.respondText
import io.ktor.server.routing.get
import io.ktor.server.routing.routing
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import moe.lava.banksia.di.CommonModules
import moe.lava.banksia.server.di.ServerModules
import moe.lava.banksia.server.gtfs.GtfsHandler
import org.koin.dsl.module
import org.koin.ktor.ext.inject
import org.koin.ktor.plugin.Koin
fun main() {
embeddedServer(Netty, port = 8080, host = "0.0.0.0", module = Application::module)
@ -13,9 +24,21 @@ fun main() {
}
fun Application.module() {
install(Koin) {
modules(module { single { log } })
modules(CommonModules, ServerModules)
}
val client = HttpClient()
routing {
get("/") {
call.respondText("Ktor: Hi")
get("/update") {
val datasetUrl = call.parameters["url"] ?: "https://opendata.transport.vic.gov.au/dataset/3f4e292e-7f8a-4ffe-831f-1953be0fe448/resource/e4966d78-dc64-4a1d-a751-2470c9eaf034/download/gtfs.zip"
call.respondText("received")
launch(context = Dispatchers.IO) {
val handler by inject<GtfsHandler>()
handler.update(datasetUrl)
}
}
}
}

11
server/src/main/kotlin/moe/lava/banksia/server/di/ServerModules.kt Normal file
View file

@ -0,0 +1,11 @@
package moe.lava.banksia.server.di
import io.ktor.client.HttpClient
import moe.lava.banksia.server.gtfs.GtfsHandler
import org.koin.core.module.dsl.singleOf
import org.koin.dsl.module
val ServerModules = module {
single { HttpClient() }
singleOf(::GtfsHandler)
}

122
server/src/main/kotlin/moe/lava/banksia/server/gtfs/GtfsHandler.kt Normal file
View file

@ -0,0 +1,122 @@
package moe.lava.banksia.server.gtfs
import com.lightningkite.kotlinx.serialization.csv.CsvFormat
import com.lightningkite.kotlinx.serialization.csv.StringDeferringConfig
import io.ktor.client.HttpClient
import io.ktor.client.request.prepareRequest
import io.ktor.client.request.url
import io.ktor.client.statement.bodyAsChannel
import io.ktor.util.cio.writeChannel
import io.ktor.util.logging.Logger
import io.ktor.utils.io.copyAndClose
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.modules.EmptySerializersModule
import moe.lava.banksia.model.Route
import moe.lava.banksia.model.RouteType
import moe.lava.banksia.model.Shape
import moe.lava.banksia.room.dao.RouteDao
import moe.lava.banksia.room.dao.ShapeDao
import moe.lava.banksia.server.gtfs.structures.GtfsRoute
import moe.lava.banksia.server.gtfs.structures.GtfsShape
import moe.lava.banksia.util.Point
import java.io.File
import java.util.zip.ZipFile
class GtfsHandler(
private val log: Logger,
private val client: HttpClient,
private val routeDao: RouteDao,
private val shapeDao: ShapeDao,
) {
private val csv = CsvFormat(StringDeferringConfig(EmptySerializersModule()))
private val datasetPath = File("/tmp/banksia", "dataset.zip")
suspend fun update(datasetUrl: String) {
val parentDir = datasetPath.parentFile
if (parentDir.exists() && !log.isTraceEnabled) // XXX: hacky check for dev env
parentDir.deleteRecursively()
parentDir.mkdirs()
log.info("fetching..")
client.prepareRequest {
url(datasetUrl)
}.execute { resp ->
if (!datasetPath.exists())
resp.bodyAsChannel().copyAndClose(datasetPath.writeChannel())
log.info("fetched!")
}
log.info("extracting...")
val files = extractAll(datasetPath)
log.info("parsing routes...")
val routes = files
.filter { it.name == "routes.txt" }
.flatMap { fd -> parseRoutes(fd) }
log.info("inserting routes...")
routeDao.deleteAll()
routeDao.insertAll(*routes.toTypedArray())
log.info("parsing shapes...")
val shapes = files
.filter { it.name == "shapes.txt" }
.flatMap { fd -> parseShapes(fd) }
log.info("inserting shapes...")
shapeDao.deleteAll()
shapeDao.insertAll(*shapes.toTypedArray())
log.info("done!")
}
private fun parseRoutes(fd: File) =
fd.parseCsv<GtfsRoute>()
.map { with(it) {
Route(
id = route_id,
type = RouteType.from(fd.parentFile.name.toInt()),
number = route_short_name,
name = route_long_name,
)
} }
private fun parseShapes(fd: File) =
fd.parseCsv<GtfsShape>()
.groupBy { it.shape_id }
.map { (id, group) ->
val points = group
.sortedBy { it.shape_pt_sequence }
.map { Point(it.shape_pt_lat, it.shape_pt_lon) }
Shape(id, points)
}
private fun extract(fd: File): List<File> {
val outputs = mutableListOf<File>()
ZipFile(fd).use { zip ->
for (entry in zip.entries()) {
zip.getInputStream(entry).use { input ->
val out = File(fd.parentFile, entry.name)
out.parentFile.mkdirs()
out.outputStream().use { output ->
input.copyTo(output)
}
outputs.add(out)
}
}
}
return outputs
}
private fun extractAll(fd: File) = extract(fd).flatMap(::extract)
private fun <T> File.parseCsv(): List<T> = this
.readText()
.replace("\uFEFF", "") // remove bom
.replace("\r\n", "\n") // crlf -> lf
.let { csv.decodeFromString(it) }
}

15
server/src/main/kotlin/moe/lava/banksia/server/gtfs/structures/GtfsRoute.kt Normal file
View file

@ -0,0 +1,15 @@
package moe.lava.banksia.server.gtfs.structures
import kotlinx.serialization.Serializable
@Suppress("PropertyName")
@Serializable
data class GtfsRoute(
val route_id: String,
val agency_id: String,
val route_short_name: String,
val route_long_name: String,
val route_type: String,
val route_color: String,
val route_text_color: String,
)

13
server/src/main/kotlin/moe/lava/banksia/server/gtfs/structures/GtfsShape.kt Normal file
View file

@ -0,0 +1,13 @@
package moe.lava.banksia.server.gtfs.structures
import kotlinx.serialization.Serializable
@Suppress("PropertyName")
@Serializable
data class GtfsShape(
val shape_id: String,
val shape_pt_lat: Double,
val shape_pt_lon: Double,
val shape_pt_sequence: Int,
val shape_dist_traveled: String,
)