diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index 2ed2950..5bb422c 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -353,7 +353,7 @@ public String cleanRouteOriginalId(@NotNull String gRouteId) { public long getRouteId(@NotNull GRoute gRoute) { try { //noinspection DiscouragedApi - final String routeIdS = useRouteShortNameForRouteId() ? cleanRouteShortName(getRouteShortName(gRoute)) : cleanRouteOriginalId(gRoute.getRouteId()); + final String routeIdS = useRouteShortNameForRouteId() ? cleanRouteShortName(getRouteShortName(gRoute)) : gRoute.getRouteId(); if (defaultRouteIdEnabled() && !CharUtils.isDigitsOnly(routeIdS)) { return MRouteSNToIDConverter.convert( @@ -601,6 +601,31 @@ public String cleanTripHeadsign(@NotNull String tripHeadsign) { return tripHeadsign; } + @Nullable + @Override + public String getTripIdCleanupRegex() { + return Configs.getRouteConfig().getTripIdCleanupRegex(); + } + + @Nullable + private Pattern tripIdCleanupPattern = null; + + private boolean tripIdCleanupPatternSet = false; + + @Override + public @Nullable Pattern getTripIdCleanupPattern() { + if (this.tripIdCleanupPattern == null && !tripIdCleanupPatternSet) { + this.tripIdCleanupPattern = GTFSCommons.makeIdCleanupPattern(getTripIdCleanupRegex()); + this.tripIdCleanupPatternSet = true; + } + return this.tripIdCleanupPattern; + } + + @Override + public @NotNull String cleanTripOriginalId(@NotNull String gTripId) { + return GTFSCommons.cleanOriginalId(gTripId, getTripIdCleanupPattern()); + } + @Override public boolean directionSplitterEnabled(long routeId) { return false; // OPT-IN feature @@ -926,7 +951,7 @@ public String getStopOriginalId(@NotNull GStop gStop) { public int getStopId(@NotNull GStop gStop) { try { //noinspection DiscouragedApi - final String stopIdS = useStopCodeForStopId() ? getStopCode(gStop) : cleanStopOriginalId(gStop.getStopId()); + final String stopIdS = useStopCodeForStopId() ? getStopCode(gStop) : gStop.getStopId(); //noinspection DiscouragedApi return Integer.parseInt(stopIdS); } catch (Exception e) { diff --git a/src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt b/src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt index 9dbf844..1451ca1 100644 --- a/src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt +++ b/src/main/java/org/mtransit/parser/config/gtfs/data/RouteConfig.kt @@ -35,6 +35,8 @@ data class RouteConfig( @SerialName("trip_headsign_remove_via") val tripHeadsignRemoveVia: Boolean = false, // OPT-IN feature // DIRECTION + @SerialName("trip_id_cleanup_regex") + val tripIdCleanupRegex: String? = null, // optional @SerialName("direction_headsign_cleaners") val directionHeadsignCleaners: List = emptyList(), @SerialName("direction_finder_enabled") diff --git a/src/main/java/org/mtransit/parser/db/GTFSDataBase.kt b/src/main/java/org/mtransit/parser/db/GTFSDataBase.kt index c07cb29..f3a7b01 100644 --- a/src/main/java/org/mtransit/parser/db/GTFSDataBase.kt +++ b/src/main/java/org/mtransit/parser/db/GTFSDataBase.kt @@ -156,9 +156,9 @@ object GTFSDataBase { @JvmOverloads @JvmStatic - fun insertRoute(route: Route, preparedStatement: PreparedStatement? = null) { + fun insertRoute(route: Route, allowUpdate: Boolean = false, preparedStatement: PreparedStatement? = null) { connection.createStatement().use { statement -> - RouteSQL.insertIntoMainTable(route, statement, preparedStatement) + RouteSQL.insertIntoMainTable(route, statement, preparedStatement, allowUpdate) } } diff --git a/src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java b/src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java index e74f2a2..52f9d97 100644 --- a/src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java +++ b/src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java @@ -159,6 +159,14 @@ public interface GAgencyTools { @NotNull String cleanTripHeadsign(@NotNull String tripHeadsign); + @Nullable + String getTripIdCleanupRegex(); + + @Nullable + Pattern getTripIdCleanupPattern(); + + @NotNull String cleanTripOriginalId(@NotNull String gTripId); + boolean directionSplitterEnabled(long routeId); boolean directionOverrideId(long routeId); diff --git a/src/main/java/org/mtransit/parser/gtfs/GReader.java b/src/main/java/org/mtransit/parser/gtfs/GReader.java index fb73750..1b5aeca 100644 --- a/src/main/java/org/mtransit/parser/gtfs/GReader.java +++ b/src/main/java/org/mtransit/parser/gtfs/GReader.java @@ -286,7 +286,7 @@ private static void readCsv(String filename, BufferedReader reader, private static void processStopTime(GAgencyTools agencyTools, GSpec gSpec, HashMap line, @Nullable PreparedStatement insertStopTimePrepared) { try { - final GStopTime gStopTime = GStopTime.fromLine(line); + final GStopTime gStopTime = GStopTime.fromLine(line, agencyTools); if (agencyTools.excludeTripNullable(gSpec.getTrip(gStopTime.getTripIdInt()))) { return; } @@ -316,7 +316,7 @@ private static void processFrequency(GAgencyTools agencyTools, GSpec gSpec, HashMap line) { try { - final GFrequency gFrequency = GFrequency.fromLine(line); + final GFrequency gFrequency = GFrequency.fromLine(line, agencyTools); if (agencyTools.excludeTripNullable(gSpec.getTrip(gFrequency.getTripIdInt()))) { return; } @@ -338,7 +338,7 @@ private static void processAgency(GAgencyTools agencyTools, agencyTools.addSupportedLanguage(gAgency.getAgencyLang()); gSpec.addAgency(gAgency); } catch (Exception e) { - throw new MTLog.Fatal(e, "Error while processing: '%s'!", line); + throw new MTLog.Fatal(e, "Error while processing agency: '%s'!", line); } } @@ -354,7 +354,7 @@ private static void processCalendarDate(GAgencyTools agencyTools, GSpec gSpec, H } gSpec.addCalendarDate(gCalendarDate); } catch (Exception e) { - throw new MTLog.Fatal(e, "Error while processing: '%s'!", line); + throw new MTLog.Fatal(e, "Error while processing calendar date: '%s'!", line); } } @@ -366,13 +366,13 @@ private static void processCalendar(GAgencyTools agencyTools, GSpec gSpec, HashM } gSpec.addCalendar(gCalendar); } catch (Exception e) { - throw new MTLog.Fatal(e, "Error while processing: %s!", line); + throw new MTLog.Fatal(e, "Error while processing calendar: %s!", line); } } private static void processDirection(GAgencyTools agencyTools, GSpec gSpec, HashMap line) { try { - final GDirection gDirection = GDirection.fromLine(line); + final GDirection gDirection = GDirection.fromLine(line, agencyTools); final GRoute gRoute = gSpec.getRoute(gDirection.getRouteIdInt()); if (agencyTools.excludeRouteNullable(gRoute)) { logExclude("Exclude route: %s.", gRoute == null ? null : gRoute.toStringPlus()); @@ -393,7 +393,7 @@ private static void processDirection(GAgencyTools agencyTools, GSpec gSpec, Hash private static void processTrip(GAgencyTools agencyTools, GSpec gSpec, HashMap line, @Nullable PreparedStatement insertStopTimePrepared) { try { - final GTrip gTrip = GTrip.fromLine(line); + final GTrip gTrip = GTrip.fromLine(line, agencyTools); if (agencyTools.excludeTrip(gTrip)) { logExclude("Exclude trip: %s.", gTrip.toStringPlus()); return; @@ -411,16 +411,22 @@ private static void processTrip(GAgencyTools agencyTools, GSpec gSpec, HashMap line) { try { - final GStop gStop = GStop.fromLine(line); + final GStop gStop = GStop.fromLine(line, agencyTools); if (agencyTools.excludeStop(gStop)) { return; } + if (agencyTools.getStopIdCleanupRegex() != null) { // IF stop ID cleanup regex set DO + final GStop previousStop = gSpec.getStop(gStop.getStopIdInt()); + if (previousStop != null && previousStop.equals(gStop)) { + return; // ignore if stop already exists with same values + } + } gSpec.addStop(gStop); } catch (Exception e) { throw new MTLog.Fatal(e, "Error while parsing stop line %s!", line); @@ -429,7 +435,7 @@ private static void processStop(GAgencyTools agencyTools, GSpec gSpec, Map line, @Nullable String defaultAgencyId) { try { - final GRoute gRoute = GRoute.fromLine(line, defaultAgencyId); + final GRoute gRoute = GRoute.fromLine(line, defaultAgencyId, agencyTools); final GAgency routeAgency = gSpec.getAgency(gRoute.getAgencyIdInt()); if (agencyTools.excludeRoute(gRoute)) { logExclude("Exclude route: %s.", gRoute.toStringPlus()); @@ -448,6 +454,22 @@ private static void processRoute(GAgencyTools agencyTools, GSpec gSpec, HashMap< } return; } + if (agencyTools.getRouteIdCleanupRegex() != null) { // IF route ID cleanup regex set DO + final GRoute previousRoute = gSpec.getRoute(gRoute.getRouteIdInt()); + if (previousRoute != null && previousRoute.equals(gRoute)) { + return; // ignore if route already exists with same values + } + if (previousRoute != null && previousRoute.equalsExceptLongNameAndUrl(gRoute)) { + final String mergedRouteLongName = GRoute.mergeRouteLongNames(previousRoute.getRouteLongName(), gRoute.getRouteLongName()); + if (mergedRouteLongName != null) { // merge successful + gSpec.addRoute(previousRoute.clone(mergedRouteLongName), true); + return; + } + } + if (previousRoute != null) { + MTLog.log("Duplicate route ID!\n-%s\n-%s", gRoute.toStringPlus(), previousRoute.toStringPlus()); + } + } gSpec.addRoute(gRoute); } catch (Exception e) { throw new MTLog.Fatal(e, "Error while parsing route line %s!", line); diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt b/src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt index 8fa9a08..167cdda 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GDirection.kt @@ -1,9 +1,11 @@ package org.mtransit.parser.gtfs.data import androidx.annotation.Discouraged +import org.mtransit.commons.GTFSCommons import org.mtransit.commons.gtfs.data.Direction import org.mtransit.commons.gtfs.data.DirectionType import org.mtransit.parser.MTLog +import org.mtransit.parser.gtfs.GAgencyTools data class GDirection( val routeIdInt: Int, @@ -48,7 +50,7 @@ data class GDirection( companion object { const val FILENAME = "directions.txt" - private const val ROUTE_ID = "route_id" + private const val ROUTE_ID = GRoute.ROUTE_ID private const val DIRECTION_ID = "direction_id" private const val DIRECTION = "direction" @@ -57,8 +59,10 @@ data class GDirection( // TODO other alternatives @JvmStatic - fun fromLine(line: Map) = GDirection( - routeId = line[ROUTE_ID] ?: throw MTLog.Fatal("Invalid GDirection from $line!"), + fun fromLine(line: Map, agencyTools: GAgencyTools) = GDirection( + routeId = line[ROUTE_ID]?.trim() + ?.let { agencyTools.cleanRouteOriginalId(it) } + ?: throw MTLog.Fatal("Invalid GDirection from $line!"), directionIdInt = line[DIRECTION_ID]?.toIntOrNull() ?: throw MTLog.Fatal("Invalid GDirection from $line!"), directionTypeValue = line[DIRECTION]?.trim(), destination = line[DESTINATION]?.trim() ?: line[DESTINATION_DIRECTION_NAME]?.trim(), diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt b/src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt index 0e63c68..2edd0af 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GFrequency.kt @@ -3,6 +3,7 @@ package org.mtransit.parser.gtfs.data import androidx.annotation.Discouraged import org.mtransit.commons.gtfs.data.Frequency import org.mtransit.parser.MTLog +import org.mtransit.parser.gtfs.GAgencyTools import java.util.Date import java.util.concurrent.TimeUnit @@ -88,7 +89,7 @@ data class GFrequency( companion object { const val FILENAME = "frequencies.txt" - private const val TRIP_ID = "trip_id" + private const val TRIP_ID = GTrip.TRIP_ID private const val START_TIME = "start_time" private const val END_TIME = "end_time" private const val HEADWAY_SECS = "headway_secs" @@ -101,12 +102,14 @@ data class GFrequency( val DEFAULT_DROP_OFF_TYPE = GDropOffType.REGULAR // Regularly scheduled drop off @JvmStatic - fun fromLine(line: Map) = GFrequency( - line[TRIP_ID] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"), - line[START_TIME] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"), - line[END_TIME] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"), - line[HEADWAY_SECS]?.toInt() ?: throw MTLog.Fatal("Invalid GFrequency from $line!"), - line[EXACT_TIMES]?.toIntOrNull(), + fun fromLine(line: Map, agencyTools: GAgencyTools) = GFrequency( + tripId = line[TRIP_ID]?.trim() + ?.let { agencyTools.cleanTripOriginalId(it) } + ?: throw MTLog.Fatal("Invalid GFrequency from $line!"), + startTime = line[START_TIME] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"), + endTime = line[END_TIME] ?: throw MTLog.Fatal("Invalid GFrequency from $line!"), + headwaySecs = line[HEADWAY_SECS]?.toInt() ?: throw MTLog.Fatal("Invalid GFrequency from $line!"), + exactTimes = line[EXACT_TIMES]?.toIntOrNull(), ) @JvmStatic diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt b/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt index b46edf9..e36f3be 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GRoute.kt @@ -6,6 +6,8 @@ import org.mtransit.commons.gtfs.data.Route import org.mtransit.commons.gtfs.data.RouteId import org.mtransit.parser.Constants.EMPTY import org.mtransit.parser.MTLog +import org.mtransit.parser.gtfs.GAgencyTools +import kotlin.math.max // https://developers.google.com/transit/gtfs/reference#routestxt // https://gtfs.org/reference/static/#routestxt @@ -82,11 +84,7 @@ data class GRoute( } @Suppress("unused") - val shortestRouteName = if (routeShortName.isEmpty()) { - routeLongName - } else { - routeShortName - } + val shortestRouteName = routeShortName.ifEmpty { routeLongName } @Suppress("unused") val longestRouteName = if (routeLongName.isNullOrEmpty()) { @@ -127,11 +125,30 @@ data class GRoute( routeSortOrder = routeSortOrder, ) + fun equalsExceptLongNameAndUrl(obj: Any?): Boolean { + val o = obj as GRoute + return when { + agencyIdInt != o.agencyIdInt -> false // not equal + routeIdInt != o.routeIdInt -> false // not equal + routeShortName != o.routeShortName -> false // not equal + routeDesc != o.routeDesc -> false // not equal + routeType != o.routeType -> false // not equal + routeColor != o.routeColor -> false // not equal + routeTextColor != o.routeTextColor -> false // not equal + routeSortOrder != o.routeSortOrder -> false // not equal + else -> true // mostly equal + } + } + + fun clone(routeLongName: String?) = this.copy( + routeLongName = routeLongName, + ) + companion object { const val FILENAME = "routes.txt" private const val AGENCY_ID = "agency_id" - private const val ROUTE_ID = "route_id" + internal const val ROUTE_ID = "route_id" private const val ROUTE_SHORT_NAME = "route_short_name" private const val ROUTE_LONG_NAME = "route_long_name" private const val ROUTE_DESC = "route_desc" @@ -142,11 +159,13 @@ data class GRoute( private const val ROUTE_SORT_ORDER = "route_sort_order" @JvmStatic - fun fromLine(line: Map, defaultAgencyId: String?) = GRoute( + fun fromLine(line: Map, defaultAgencyId: String?, agencyTools: GAgencyTools) = GRoute( agencyId = line[AGENCY_ID]?.takeIf { it.isNotBlank() } ?: defaultAgencyId ?: throw MTLog.Fatal("Invalid GRoute.$AGENCY_ID from $line!"), - routeId = line[ROUTE_ID] ?: throw MTLog.Fatal("Invalid GRoute.$ROUTE_ID from $line!"), + routeId = line[ROUTE_ID]?.trim() + ?.let { agencyTools.cleanRouteOriginalId(it) } + ?: throw MTLog.Fatal("Invalid GRoute.$ROUTE_ID from $line!"), routeShortName = line[ROUTE_SHORT_NAME]?.trim() ?: EMPTY, routeLongName = line[ROUTE_LONG_NAME], routeDesc = line[ROUTE_DESC], @@ -175,5 +194,40 @@ data class GRoute( routeSortOrder = it.routeSortOrder, ) } + + private const val SLASH_: String = " / " + + @JvmStatic + fun mergeRouteLongNames(routeLongName1: String?, routeLongName2: String?): String? { + if (routeLongName2.isNullOrEmpty()) { + return routeLongName1 + } else if (routeLongName1.isNullOrEmpty()) { + return routeLongName2 + } else if (routeLongName2.contains(routeLongName1)) { + return routeLongName2 + } else if (routeLongName1.contains(routeLongName2)) { + return routeLongName1 + } + val prefix = routeLongName1.commonPrefixWith(routeLongName2) + val maxLength = max(routeLongName1.length, routeLongName2.length) + if (prefix.length > maxLength / 2) { + return prefix + + routeLongName1.substring(prefix.length) + + SLASH_ + + routeLongName2.substring(prefix.length) + } + val suffix = routeLongName1.commonSuffixWith(routeLongName2) + if (suffix.length > maxLength / 2) { + return routeLongName1.substring(0, routeLongName1.length - suffix.length) + + SLASH_ + + routeLongName2.substring(0, routeLongName2.length - suffix.length) + + suffix + } + return if (routeLongName1 > routeLongName2) { + routeLongName2 + SLASH_ + routeLongName1 + } else { + routeLongName1 + SLASH_ + routeLongName2 + } + } } } \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java index ed01f12..023a3d2 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java +++ b/src/main/java/org/mtransit/parser/gtfs/data/GSpec.java @@ -203,7 +203,11 @@ public Collection getOtherRoutes(int agencyIdInt) { } public void addRoute(@NotNull GRoute gRoute) { - GTFSDataBase.insertRoute(gRoute.to()); + addRoute(gRoute, false); + } + + public void addRoute(@NotNull GRoute gRoute, boolean allowUpdate) { + GTFSDataBase.insertRoute(gRoute.to(), allowUpdate); this.routesCache.put(gRoute.getRouteIdInt(), gRoute); } @@ -806,7 +810,7 @@ public void cleanupStopTimesPickupDropOffTypes(@NotNull DefaultAgencyTools agenc boolean forceStopTimeFirstNoDropOffType = agencyTools.forceStopTimeFirstNoDropOffType(); MTLog.logDebug("forceStopTimeFirstNoDropOffType: %s", forceStopTimeFirstNoDropOffType); GTFSDataBase.setAutoCommit(false); // stop times - MTLog.logDebug("Cleanup stop times pickup & drop-off types from (%d trips)...", this.tripIdIntsUIDs.keySet().size()); + MTLog.logDebug("Cleanup stop times pickup & drop-off types from (%d trips)...", this.tripIdIntsUIDs.size()); for (Integer tripIdInt : this.tripIdIntsUIDs.keySet()) { //noinspection ConstantConditions if (forceStopTimeLastNoPickupType) { @@ -832,7 +836,7 @@ public void cleanupStopTimesPickupDropOffTypes(@NotNull DefaultAgencyTools agenc MTLog.logPOINT(); // LOG } // LOG if (stp % 10_000 == 0) { // LOG - MTLog.log("Cleanup stop times pickup & drop-off types from (%d/%d trips) (%d updated objects)...", stp, this.tripIdIntsUIDs.keySet().size(), stu); // LOG + MTLog.log("Cleanup stop times pickup & drop-off types from (%d/%d trips) (%d updated objects)...", stp, this.tripIdIntsUIDs.size(), stu); // LOG } // LOG } GTFSDataBase.setAutoCommit(true); // true => commit() // stop times diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GStop.kt b/src/main/java/org/mtransit/parser/gtfs/data/GStop.kt index 4c07a42..12f18a1 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GStop.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GStop.kt @@ -6,6 +6,7 @@ import org.mtransit.commons.gtfs.data.Stop import org.mtransit.commons.gtfs.data.StopId import org.mtransit.parser.Constants import org.mtransit.parser.MTLog +import org.mtransit.parser.gtfs.GAgencyTools // https://developers.google.com/transit/gtfs/reference#stops_fields // https://gtfs.org/schedule/reference/#stopstxt @@ -91,7 +92,7 @@ data class GStop( companion object { const val FILENAME = "stops.txt" - private const val STOP_ID = "stop_id" + internal const val STOP_ID = "stop_id" private const val STOP_NAME = "stop_name" private const val STOP_LAT = "stop_lat" private const val STOP_LON = "stop_lon" @@ -101,15 +102,17 @@ data class GStop( private const val WHEELCHAIR_BOARDING = "wheelchair_boarding" @JvmStatic - fun fromLine(line: Map) = GStop( - line[STOP_ID] ?: throw MTLog.Fatal("Invalid GStop from $line!"), - line[STOP_NAME] ?: throw MTLog.Fatal("Invalid GStop from $line!"), - line[STOP_LAT]?.toDouble() ?: throw MTLog.Fatal("Invalid GStop from $line!"), - line[STOP_LON]?.toDouble() ?: throw MTLog.Fatal("Invalid GStop from $line!"), - line[STOP_CODE]?.trim() ?: EMPTY, - line[LOCATION_TYPE]?.takeIf { it.isNotBlank() }?.toInt(), - line[PARENT_STATION], - line[WHEELCHAIR_BOARDING]?.takeIf { it.isNotBlank() }?.toInt(), + fun fromLine(line: Map, agencyTools: GAgencyTools) = GStop( + stopId = line[STOP_ID]?.trim() + ?.let { agencyTools.cleanStopOriginalId(it) } + ?: throw MTLog.Fatal("Invalid GStop from $line!"), + stopName = line[STOP_NAME] ?: throw MTLog.Fatal("Invalid GStop from $line!"), + stopLat = line[STOP_LAT]?.toDouble() ?: throw MTLog.Fatal("Invalid GStop from $line!"), + stopLong = line[STOP_LON]?.toDouble() ?: throw MTLog.Fatal("Invalid GStop from $line!"), + stopCode = line[STOP_CODE]?.trim() ?: EMPTY, + locationType = line[LOCATION_TYPE]?.takeIf { it.isNotBlank() }?.toInt(), + parentStationId = line[PARENT_STATION], + wheelchairBoarding = line[WHEELCHAIR_BOARDING]?.takeIf { it.isNotBlank() }?.toInt(), ) @JvmStatic diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt b/src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt index 22f8e24..cd5d2b3 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GStopTime.kt @@ -5,6 +5,7 @@ import org.mtransit.commons.StringUtils import org.mtransit.commons.gtfs.data.StopTime import org.mtransit.parser.Constants import org.mtransit.parser.MTLog +import org.mtransit.parser.gtfs.GAgencyTools import java.util.Date // https://gtfs.org/schedule/reference/#stop_timestxt @@ -194,8 +195,8 @@ data class GStopTime( companion object { const val FILENAME = "stop_times.txt" - const val TRIP_ID = "trip_id" - const val STOP_ID = "stop_id" + const val TRIP_ID = GTrip.TRIP_ID + const val STOP_ID = GStop.STOP_ID const val STOP_SEQUENCE = "stop_sequence" const val ARRIVAL_TIME = "arrival_time" const val DEPARTURE_TIME = "departure_time" @@ -205,11 +206,15 @@ data class GStopTime( const val TIME_POINT = "timepoint" @JvmStatic - fun fromLine(line: Map) = GStopTime( - tripId = line[TRIP_ID] ?: throw MTLog.Fatal("Invalid GStopTime from $line!"), + fun fromLine(line: Map, agencyTools: GAgencyTools) = GStopTime( + tripId = line[TRIP_ID]?.trim() + ?.let { agencyTools.cleanTripOriginalId(it) } + ?: throw MTLog.Fatal("Invalid GStopTime from $line!"), arrivalTime = line[ARRIVAL_TIME]?.trim() ?: throw MTLog.Fatal("Invalid GStopTime from $line!"), departureTime = line[DEPARTURE_TIME]?.trim() ?: throw MTLog.Fatal("Invalid GStopTime from $line!"), - stopId = line[STOP_ID]?.trim() ?: throw MTLog.Fatal("Invalid GStopTime from $line!"), + stopId = line[STOP_ID]?.trim() + ?.let { agencyTools.cleanStopOriginalId(it) } + ?: throw MTLog.Fatal("Invalid GStopTime from $line!"), stopSequence = line[STOP_SEQUENCE]?.trim()?.toInt() ?: throw MTLog.Fatal("Invalid GStopTime from $line!"), stopHeadsign = line[STOP_HEADSIGN]?.takeIf { it.isNotBlank() }, pickupType = GPickupType.parse(line[PICKUP_TYPE]), diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt index 275a42e..edcb5f8 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTrip.kt @@ -142,8 +142,8 @@ data class GTrip( companion object { const val FILENAME = "trips.txt" - private const val TRIP_ID = "trip_id" - private const val ROUTE_ID = "route_id" + internal const val TRIP_ID = "trip_id" + private const val ROUTE_ID = GRoute.ROUTE_ID private const val SERVICE_ID = "service_id" private const val TRIP_HEADSIGN = "trip_headsign" private const val TRIP_SHORT_NAME = "trip_short_name" @@ -154,9 +154,13 @@ data class GTrip( private const val BIKES_ALLOWED = "bikes_allowed" @JvmStatic - fun fromLine(line: Map) = GTrip( - tripId = line[TRIP_ID] ?: throw MTLog.Fatal("Invalid GTrip from $line!"), - routeId = line[ROUTE_ID] ?: throw MTLog.Fatal("Invalid GTrip from $line!"), + fun fromLine(line: Map, agencyTools: GAgencyTools) = GTrip( + tripId = line[TRIP_ID]?.trim() + ?.let { agencyTools.cleanTripOriginalId(it) } + ?: throw MTLog.Fatal("Invalid GTrip from $line!"), + routeId = line[ROUTE_ID]?.trim() + ?.let { agencyTools.cleanRouteOriginalId(it) } + ?: throw MTLog.Fatal("Invalid GTrip from $line!"), originalRouteId = line[ROUTE_ID] ?: throw MTLog.Fatal("Invalid GTrip from $line!"), serviceId = line[SERVICE_ID] ?: throw MTLog.Fatal("Invalid GTrip from $line!"), tripHeadsign = line[TRIP_HEADSIGN]?.takeIf { it.isNotBlank() }, diff --git a/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt b/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt index 490800a..4c2227e 100644 --- a/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt +++ b/src/main/java/org/mtransit/parser/gtfs/data/GTripStop.kt @@ -77,10 +77,10 @@ data class GTripStop( } companion object { - const val ROUTE_ID = "route_id" - const val TRIP_ID = "trip_id" - const val STOP_ID = "stop_id" - const val STOP_SEQUENCE = "stop_sequence" + const val ROUTE_ID = GRoute.ROUTE_ID + const val TRIP_ID = GTrip.TRIP_ID + const val STOP_ID = GStop.STOP_ID + const val STOP_SEQUENCE = GStopTime.STOP_SEQUENCE private const val UID_SEPARATOR = "+" // int IDs can be negative diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 9a3a0a7..afdb598 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -803,6 +803,7 @@ private static String getLastModified(String gtfsDir) { private static final String GTFS_RTS_TIMEZONE = "gtfs_rts_timezone"; private static final String GTFS_RTS_COLOR = "gtfs_rts_color"; private static final String GTFS_RTS_ROUTE_ID_CLEANUP_REGEX = "gtfs_rts_route_id_cleanup_regex"; + private static final String GTFS_RTS_TRIP_ID_CLEANUP_REGEX = "gtfs_rts_trip_id_cleanup_regex"; private static final String GTFS_RTS_STOP_ID_CLEANUP_REGEX = "gtfs_rts_stop_id_cleanup_regex"; private static final String GTFS_RTS_SCHEDULE_AVAILABLE = "gtfs_rts_schedule_available"; @@ -854,6 +855,10 @@ private static void dumpCommonValues(File dumpDirF, GAgencyTools gAgencyTools, M ow.write(getRESOURCES_STRING(GTFS_RTS_ROUTE_ID_CLEANUP_REGEX, escapeResString(gAgencyTools.getRouteIdCleanupRegex()))); ow.write(Constants.NEW_LINE); } + if (gAgencyTools.getTripIdCleanupRegex() != null) { + ow.write(getRESOURCES_STRING(GTFS_RTS_TRIP_ID_CLEANUP_REGEX, escapeResString(gAgencyTools.getTripIdCleanupRegex()))); + ow.write(Constants.NEW_LINE); + } if (gAgencyTools.getStopIdCleanupRegex() != null) { ow.write(getRESOURCES_STRING(GTFS_RTS_STOP_ID_CLEANUP_REGEX, escapeResString(gAgencyTools.getStopIdCleanupRegex()))); ow.write(Constants.NEW_LINE); diff --git a/src/main/java/org/mtransit/parser/mt/data/MRoute.kt b/src/main/java/org/mtransit/parser/mt/data/MRoute.kt index bdf54ed..3a12f0c 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MRoute.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MRoute.kt @@ -30,7 +30,7 @@ data class MRoute( shortName, longName, color, - GTFSCommons.stringIdToHashIfEnabled(originalId, agencyTools?.routeIdCleanupPattern), + GTFSCommons.stringIdToHashIfEnabled(originalId), type, ) diff --git a/src/main/java/org/mtransit/parser/mt/data/MStop.kt b/src/main/java/org/mtransit/parser/mt/data/MStop.kt index 34babf9..263a8e3 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MStop.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MStop.kt @@ -33,7 +33,7 @@ data class MStop( lat, lng, accessible, - GTFSCommons.stringIdToHashIfEnabled(originalId, agencyTools?.stopIdCleanupPattern), + GTFSCommons.stringIdToHashIfEnabled(originalId), ) fun hasLat(): Boolean {