cilly/Banksia
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(core): switch from room to sqldelight

Browse source 
sqldelight provides far more control over the sql and allows me to make
more optimisations such as removing generated rowid etc. sql also just
looks better than the annotation hell from room.
This commit is contained in:
Cilly Leang 2026-05-02 02:31:18 +10:00
parent ff2af310fb
commit f1770744db
Signed by: cilly
GPG key ID: 6500251E087653C9
74 changed files with 601 additions and 5037 deletions
Download patch file Download diff file Expand all files Collapse all files

4
server/build.gradle.kts
View file

@ -20,7 +20,7 @@ kotlin {
dependencies {
implementation(projects.core)
implementation(projects.core.room)
implementation(projects.core.sqld)
implementation(projects.server.gtfs)
implementation(projects.server.gtfsRt)
@ -36,8 +36,6 @@ dependencies {
implementation(libs.ktor.server.contentnegotiation)
implementation(libs.ktor.server.core)
implementation(libs.ktor.server.netty)
implementation(libs.room.runtime)
implementation(libs.sqlite.bundled)
testImplementation(libs.ktor.server.tests)
testImplementation(libs.kotlin.test.junit)
}

16
server/gtfs/src/main/kotlin/moe/lava/banksia/server/gtfs/GtfsParser.kt
View file

@ -175,8 +175,8 @@ class GtfsParser(
pos = Point(stop_lat, stop_lon),
parent = parent_station.ifEmpty { null },
hasWheelChairBoarding = wheelchair_boarding == "1",
level = level_id,
platformCode = platform_code,
level = level_id.ifEmpty { null },
platformCode = platform_code.ifEmpty { null },
)
} }
@ -210,7 +210,7 @@ class GtfsParser(
if (sunday == 1) add(DayOfWeek.SUNDAY)
}
Service(
id = service_id,
id = "${fd.parentFile.name}_${service_id}",
days = days,
start = LocalDate.parse(start_date, LocalDate.Formats.ISO_BASIC),
end = LocalDate.parse(end_date, LocalDate.Formats.ISO_BASIC),
@ -221,7 +221,7 @@ class GtfsParser(
fd.parseCsv<GtfsServiceException>()
.map { with(it) {
ServiceException(
serviceId = service_id,
serviceId = "${fd.parentFile.name}_${service_id}",
date = LocalDate.parse(date, LocalDate.Formats.ISO_BASIC),
type = exception_type,
)
@ -233,12 +233,12 @@ class GtfsParser(
Trip(
id = trip_id,
routeId = route_id,
service = services[service_id]!!,
shapeId = shape_id.ifEmpty { null },
service = services["${fd.parentFile.name}_${service_id}"]!!,
shapeId = shape_id,
tripHeadsign = trip_headsign,
directionId = direction_id,
blockId = block_id,
wheelchairAccessible = wheelchair_accessible,
blockId = block_id.ifEmpty { null },
wheelchairAccessible = wheelchair_accessible == "1",
)
} }

60
server/src/main/kotlin/moe/lava/banksia/server/Application.kt
View file

@ -20,10 +20,10 @@ import kotlinx.datetime.TimeZone
import kotlinx.datetime.todayIn
import moe.lava.banksia.core.Constants
import moe.lava.banksia.core.model.atDate
import moe.lava.banksia.core.room.dao.RouteDao
import moe.lava.banksia.core.room.dao.StopDao
import moe.lava.banksia.core.room.dao.StopTimeDao
import moe.lava.banksia.core.room.dao.VersionMetadataDao
import moe.lava.banksia.core.sqld.RouteQueries
import moe.lava.banksia.core.sqld.StopQueries
import moe.lava.banksia.core.sqld.StopTimeQueries
import moe.lava.banksia.core.sqld.mappers.asModel
import moe.lava.banksia.core.util.serialise
import moe.lava.banksia.server.di.ServerModules
import moe.lava.banksia.server.gtfsrt.GtfsrtService
@ -88,25 +88,9 @@ fun Application.module() {
}
}
get("/metadata/{type?}") {
val dao = get<VersionMetadataDao>()
val type = call.parameters["type"]
if (type == null) {
call.respond(dao.getAll().map { it.asModel() })
return@get
}
val data = dao.get(type)?.asModel()
if (data == null) {
call.respond(HttpStatusCode.NotFound)
} else {
call.respond(data)
}
}
get("/routes") {
val routes = withContext(context = Dispatchers.IO) {
get<RouteDao>().getAll()
get<RouteQueries>().getAll().executeAsList()
}
val res = routes.map { it.asModel() }
call.respond(res)
@ -114,16 +98,17 @@ fun Application.module() {
get("/routes/{route_id}") {
val routeId = call.parameters["route_id"]!!
val route = withContext(context = Dispatchers.IO) {
get<RouteDao>().get(routeId)
get<RouteQueries>().get(routeId).executeAsOneOrNull()
}
if (route != null)
if (route != null) {
call.respond(route.asModel())
else
} else {
call.respond(HttpStatusCode.NotFound)
}
}
get("/stops") {
val routes = withContext(context = Dispatchers.IO) {
get<StopDao>().getAll()
get<StopQueries>().getAll().executeAsList()
}
val res = routes.map { it.asModel() }
call.respond(res)
@ -131,22 +116,24 @@ fun Application.module() {
get("/stops/{stop_id}") {
val stopId = call.parameters["stop_id"]!!
val stop = withContext(context = Dispatchers.IO) {
get<StopDao>().get(stopId)
get<StopQueries>().get(stopId).executeAsOneOrNull()
}
if (stop != null)
if (stop != null) {
call.respond(stop.asModel())
else
} else {
call.respond(HttpStatusCode.NotFound)
}
}
get("/route_stops/{route_id}") {
val routeId = call.parameters["route_id"]!!
val useParent = call.queryParameters["parent"] !in listOf("false", "0")
val stops = withContext(Dispatchers.IO) {
val routeDao = get<RouteDao>()
if (useParent)
routeDao.stopsParent(routeId)
else
routeDao.stops(routeId)
val queries = get<StopQueries>()
if (useParent) {
queries.getParentsByRoute(routeId).executeAsList()
} else {
queries.getByRoute(routeId).executeAsList()
}
}
call.respond(stops.map { it.asModel() })
}
@ -156,12 +143,13 @@ fun Application.module() {
?.let { LocalDate.parse(it, LocalDate.Formats.ISO) }
?: Clock.System.todayIn(TimeZone.currentSystemDefault())
val times = withContext(context = Dispatchers.IO) {
get<StopTimeDao>()
get<StopTimeQueries>()
.getForStopDated(
listOf(date.dayOfWeek).serialise().toLong(),
date.toEpochDays(),
stopId,
listOf(date.dayOfWeek).serialise(),
date.toEpochDays().toInt(),
)
.executeAsList()
.map { it.asModel().atDate(date) }
.sortedBy { it.departureTime }
}

22
server/src/main/kotlin/moe/lava/banksia/server/GtfsDataFixer.kt
View file

@ -1,16 +1,16 @@
package moe.lava.banksia.server
import moe.lava.banksia.core.room.Database
import moe.lava.banksia.core.room.entity.StopEntity
import moe.lava.banksia.core.sqld.BanksiaDatabase
import moe.lava.banksia.core.util.log
import java.security.MessageDigest
import moe.lava.banksia.core.sqld.Stop as DbStop
class GtfsDataFixer(
private val database: Database,
private val database: BanksiaDatabase,
) {
suspend fun addParentsToStops() {
val dao = database.stopDao
val stops = dao.getAllParentless()
fun addParentsToStops() {
val queries = database.stopQueries
val stops = queries.getAllParentless().executeAsList()
stops
.groupBy { it.name.split("/")[0] }
.filter { (_, stops) -> stops.size > 1 }
@ -19,19 +19,21 @@ class GtfsDataFixer(
val avgLng = stops.map { it.lng }.average()
val hash = name.sha256().substring(0, 7)
val parentId = "bsia:df1:$hash"
val parent = StopEntity(
val parent = DbStop(
id = parentId,
name = name,
lat = avgLat,
lng = avgLng,
parent = null,
hasWheelChairBoarding = stops.all { it.hasWheelChairBoarding },
hasWheelChairBoarding = if (stops.all { it.hasWheelChairBoarding == 1L }) 1L else 0L,
level = "",
platformCode = "",
)
log("datafixer", "inserting ${parentId} for ${stops.size} children")
dao.insertAll(parent)
dao.updateParents(stops.map { it.id }, parentId)
queries.transaction {
queries.insert(parent)
queries.updateParents(parentId, stops.map { it.id })
}
}
}
}

82
server/src/main/kotlin/moe/lava/banksia/server/GtfsImporter.kt
View file

@ -8,12 +8,12 @@ import moe.lava.banksia.core.model.Shape
import moe.lava.banksia.core.model.Stop
import moe.lava.banksia.core.model.StopTime
import moe.lava.banksia.core.model.Trip
import moe.lava.banksia.core.room.Database
import moe.lava.banksia.core.room.DatabaseManager
import moe.lava.banksia.core.room.entity.asEntity
import moe.lava.banksia.core.sqld.DatabaseManager
import moe.lava.banksia.core.sqld.mappers.asDb
import moe.lava.banksia.server.gtfs.GtfsData
import moe.lava.banksia.server.gtfs.GtfsParser
import kotlin.time.Clock
import moe.lava.banksia.core.sqld.BanksiaDatabase as Database
class GtfsImporter(
private val parser: GtfsParser,
@ -21,7 +21,7 @@ class GtfsImporter(
private val log: Logger,
) {
suspend fun import(url: String, date: Long = Clock.System.now().epochSeconds) {
val database = dbm.makeAlt()
val (database, close) = dbm.makeAlt()
parser.update(url).collect { chunk ->
when (chunk) {
@ -35,48 +35,51 @@ class GtfsImporter(
}
}
database.updateMetadata(date)
database.close()
close()
dbm.swap()
}
private suspend fun Database.updateMetadata(date: Long) {
val dao = versionMetadataDao
log.info("updating metadata...")
dao.update(date, listOf("routes", "stops", "shapes", "trips", "stop_times"))
log.info("done")
}
private suspend fun Database.addRoutes(routes: List<Route>) {
val dao = routeDao
private fun Database.addRoutes(routes: List<Route>) {
log.info("inserting routes...")
dao.insertOrReplaceAll(*routes.map { it.asEntity() }.toTypedArray())
routeQueries.transaction {
routes.forEach {
routeQueries.insert(it.asDb())
}
}
log.info("done")
}
private suspend fun Database.addServices(services: List<Service>) {
val dao = serviceDao
private fun Database.addServices(services: List<Service>) {
log.info("inserting services...")
dao.insertOrReplaceAll(*services.map { it.asEntity() }.toTypedArray())
serviceQueries.transaction {
services.forEach {
serviceQueries.insert(it.asDb())
}
}
log.info("done")
}
private suspend fun Database.addServiceExceptions(exceptions: List<ServiceException>) {
val dao = serviceExceptionDao
private fun Database.addServiceExceptions(exceptions: List<ServiceException>) {
log.info("inserting exceptions...")
dao.insertOrReplaceAll(*exceptions.map { it.asEntity() }.toTypedArray())
serviceExceptionQueries.transaction {
exceptions.forEach {
serviceExceptionQueries.insert(it.asDb())
}
}
log.info("done")
}
private suspend fun Database.addShapes(shapes: List<Shape>) {
val dao = shapeDao
private fun Database.addShapes(shapes: List<Shape>) {
log.info("inserting shapes...")
dao.insertOrReplaceAll(*shapes.map { it.asEntity() }.toTypedArray())
shapeQueries.transaction {
shapes.forEach {
shapeQueries.insert(it.asDb())
}
}
log.info("done")
}
private suspend fun Database.addStops(stops: List<Stop>) {
val dao = stopDao
private fun Database.addStops(stops: List<Stop>) {
log.info("inserting stops...")
stops
.groupBy { it.id }
@ -89,21 +92,32 @@ class GtfsImporter(
}
}
}
dao.insertOrReplaceAll(*stops.map { it.asEntity() }.toTypedArray())
stopQueries.transaction {
stops.forEach {
stopQueries.insert(it.asDb())
}
}
log.info("done")
}
private suspend fun Database.addStopTimes(stopTimes: List<StopTime>) {
val dao = stopTimeDao
private fun Database.addStopTimes(stopTimes: List<StopTime>) {
log.info("inserting ${stopTimes.size} stoptimes...")
dao.insertOrReplaceAll(*stopTimes.map { it.asEntity() }.toTypedArray())
stopTimeQueries.transaction {
stopTimes.forEach {
stopTimeQueries.insert(it.asDb())
}
}
log.info("done")
}
private suspend fun Database.addTrips(trips: List<Trip>) {
val dao = tripDao
private fun Database.addTrips(trips: List<Trip>) {
log.info("inserting ${trips.size} trips...")
dao.insertOrReplaceAll(*trips.map { it.asEntity() }.toTypedArray())
tripQueries.transaction {
trips.forEach {
tripQueries.insert(it.asDb())
}
}
log.info("done")
}
}

4
server/src/main/kotlin/moe/lava/banksia/server/di/ServerModules.kt
View file

@ -1,7 +1,7 @@
package moe.lava.banksia.server.di
import io.ktor.client.HttpClient
import moe.lava.banksia.core.room.roomDiModule
import moe.lava.banksia.core.sqld.sqldDiModule
import moe.lava.banksia.server.GtfsDataFixer
import moe.lava.banksia.server.GtfsImporter
import moe.lava.banksia.server.gtfs.GtfsParser
@ -11,7 +11,7 @@ import org.koin.core.module.dsl.singleOf
import org.koin.dsl.module
val ServerModules = module {
includes(roomDiModule)
includes(sqldDiModule)
single { HttpClient() }
singleOf(::GtfsParser)